From b20ee8a422b5f53b29e6fd309fc8fb0518129bec Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Wed, 28 Sep 2022 00:35:19 +0200 Subject: [PATCH] Fix bitmap, add memory demotion Signed-off-by: Peter Jung --- 6.0/0001-bbr2.patch | 3672 - 6.0/{testing => }/0001-cachy.patch | 4 +- 6.0/{testing => }/0002-bbr2.patch | 4 +- 6.0/0002-cachy.patch | 6610 - 6.0/0003-futex-winesync.patch | 6 +- ...oducing-OpenVPN-Data-Channel-Offload.patch | 6 +- ...m-demotion-Memory-tiers-and-demotion.patch | 1882 + 6.0/0006-mm-cleanup.patch | 322 - ...ugepaged-add-struct-collapse_control.patch | 2743 + ...-lru.patch => 0007-mm-multi-gen-LRU.patch} | 62 +- 6.0/0007-rtw88.patch | 2488 - ... => 0008-Introducing-the-Maple-Tree.patch} | 8642 +- 6.0/0008-rcu.patch | 74 - 6.0/0009-lrng.patch | 10599 -- ...mm-cleanup.patch => 0009-mm-cleanup.patch} | 8 +- 6.0/0010-per-VMA-locks-proposal.patch | 1442 - .../0007-rtw88.patch => 0010-rtw88.patch} | 4 +- 6.0/0011-folios.patch | 2014 - .../0008-rcu.patch => 0011-rcu.patch} | 4 +- 6.0/0012-fixes.patch | 1692 - .../0009-lrng.patch => 0012-lrng.patch} | 4 +- .../0010-folios.patch => 0013-folios.patch} | 4 +- 6.0/0013-kallsyms.patch | 1141 - .../0011-fixes.patch => 0014-fixes.patch} | 4 +- ...012-kallsyms.patch => 0015-kallsyms.patch} | 4 +- .../0013-bitmap.patch => 0016-bitmap.patch} | 48 +- 6.0/all/0001-cachyos-base-all-exp.patch | 100390 ------------- 6.0/all/0001-cachyos-base-all.patch | 110452 +++++++++++---- 6.0/testing/0003-futex-winesync.patch | 3402 - ...oducing-OpenVPN-Data-Channel-Offload.patch | 6167 - 30 files changed, 89057 insertions(+), 174837 deletions(-) delete mode 100644 6.0/0001-bbr2.patch rename 6.0/{testing => }/0001-cachy.patch (99%) rename 6.0/{testing => }/0002-bbr2.patch (99%) delete mode 100644 6.0/0002-cachy.patch create mode 100644 6.0/0005-mm-demotion-Memory-tiers-and-demotion.patch delete mode 100644 6.0/0006-mm-cleanup.patch create mode 100644 6.0/0006-mm-khugepaged-add-struct-collapse_control.patch rename 6.0/{0005-lru.patch => 0007-mm-multi-gen-LRU.patch} (99%) delete mode 100644 6.0/0007-rtw88.patch rename 6.0/{testing/0005-maple-lru-khugepage.patch => 0008-Introducing-the-Maple-Tree.patch} (88%) delete mode 100644 6.0/0008-rcu.patch delete mode 100644 6.0/0009-lrng.patch rename 6.0/{testing/0006-mm-cleanup.patch => 0009-mm-cleanup.patch} (98%) delete mode 100644 6.0/0010-per-VMA-locks-proposal.patch rename 6.0/{testing/0007-rtw88.patch => 0010-rtw88.patch} (99%) delete mode 100644 6.0/0011-folios.patch rename 6.0/{testing/0008-rcu.patch => 0011-rcu.patch} (96%) delete mode 100644 6.0/0012-fixes.patch rename 6.0/{testing/0009-lrng.patch => 0012-lrng.patch} (99%) rename 6.0/{testing/0010-folios.patch => 0013-folios.patch} (99%) delete mode 100644 6.0/0013-kallsyms.patch rename 6.0/{testing/0011-fixes.patch => 0014-fixes.patch} (99%) rename 6.0/{testing/0012-kallsyms.patch => 0015-kallsyms.patch} (99%) rename 6.0/{testing/0013-bitmap.patch => 0016-bitmap.patch} (98%) delete mode 100644 6.0/all/0001-cachyos-base-all-exp.patch delete mode 100644 6.0/testing/0003-futex-winesync.patch delete mode 100644 6.0/testing/0004-Introducing-OpenVPN-Data-Channel-Offload.patch diff --git a/6.0/0001-bbr2.patch b/6.0/0001-bbr2.patch deleted file mode 100644 index bcb74c1e..00000000 --- a/6.0/0001-bbr2.patch +++ /dev/null @@ -1,3672 +0,0 @@ -From 5162124c2fa507648ad2101787fca971ccc8175b Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 5 Sep 2022 08:34:43 +0200 -Subject: [PATCH 01/13] bbr2 - -Signed-off-by: Peter Jung ---- - Documentation/networking/ip-sysctl.rst | 58 + - include/linux/tcp.h | 6 +- - include/net/inet_connection_sock.h | 3 +- - include/net/netns/ipv4.h | 5 + - include/net/tcp.h | 58 +- - include/uapi/linux/inet_diag.h | 33 + - include/uapi/linux/snmp.h | 1 + - net/ipv4/Kconfig | 22 + - net/ipv4/Makefile | 3 +- - net/ipv4/bpf_tcp_ca.c | 12 + - net/ipv4/proc.c | 1 + - net/ipv4/sysctl_net_ipv4.c | 43 + - net/ipv4/tcp.c | 1 + - net/ipv4/tcp_bbr.c | 38 +- - net/ipv4/tcp_bbr2.c | 2692 ++++++++++++++++++++++++ - net/ipv4/tcp_cong.c | 1 + - net/ipv4/tcp_input.c | 27 +- - net/ipv4/tcp_ipv4.c | 7 + - net/ipv4/tcp_output.c | 26 +- - net/ipv4/tcp_plb.c | 100 + - net/ipv4/tcp_rate.c | 30 +- - net/ipv4/tcp_timer.c | 1 + - 22 files changed, 3133 insertions(+), 35 deletions(-) - create mode 100644 net/ipv4/tcp_bbr2.c - create mode 100644 net/ipv4/tcp_plb.c - -diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst -index a759872a2883..f2372f8f860b 100644 ---- a/Documentation/networking/ip-sysctl.rst -+++ b/Documentation/networking/ip-sysctl.rst -@@ -1040,6 +1040,64 @@ tcp_challenge_ack_limit - INTEGER - TCP stack implements per TCP socket limits anyway. - Default: INT_MAX (unlimited) - -+tcp_plb_enabled - BOOLEAN -+ If set, TCP PLB (Protective Load Balancing) is enabled. PLB is -+ described in the following paper: -+ https://doi.org/10.1145/3544216.3544226. Based on PLB parameters, -+ upon sensing sustained congestion, TCP triggers a change in -+ flow label field for outgoing IPv6 packets. A change in flow label -+ field potentially changes the path of outgoing packets for switches -+ that use ECMP/WCMP for routing. -+ -+ Default: 0 -+ -+tcp_plb_cong_thresh - INTEGER -+ Fraction of packets marked with congestion over a round (RTT) to -+ tag that round as congested. This is referred to as K in the PLB paper: -+ https://doi.org/10.1145/3544216.3544226. -+ -+ The 0-1 fraction range is mapped to 0-256 range to avoid floating -+ point operations. For example, 128 means that if at least 50% of -+ the packets in a round were marked as congested then the round -+ will be tagged as congested. -+ -+ Possible Values: 0 - 256 -+ -+ Default: 128 -+ -+tcp_plb_idle_rehash_rounds - INTEGER -+ Number of consecutive congested rounds (RTT) seen after which -+ a rehash can be performed, given there are no packets in flight. -+ This is referred to as M in PLB paper: -+ https://doi.org/10.1145/3544216.3544226. -+ -+ Possible Values: 0 - 31 -+ -+ Default: 3 -+ -+tcp_plb_rehash_rounds - INTEGER -+ Number of consecutive congested rounds (RTT) seen after which -+ a forced rehash can be performed. Be careful when setting this -+ parameter, as a small value increases the risk of retransmissions. -+ This is referred to as N in PLB paper: -+ https://doi.org/10.1145/3544216.3544226. -+ -+ Possible Values: 0 - 31 -+ -+ Default: 12 -+ -+tcp_plb_suspend_rto_sec - INTEGER -+ Time, in seconds, to suspend PLB in event of an RTO. In order to avoid -+ having PLB repath onto a connectivity "black hole", after an RTO a TCP -+ connection suspends PLB repathing for a random duration between 1x and -+ 2x of this parameter. Randomness is added to avoid concurrent rehashing -+ of multiple TCP connections. This should be set corresponding to the -+ amount of time it takes to repair a failed link. -+ -+ Possible Values: 0 - 255 -+ -+ Default: 60 -+ - UDP variables - ============= - -diff --git a/include/linux/tcp.h b/include/linux/tcp.h -index a9fbe22732c3..4034e46093b4 100644 ---- a/include/linux/tcp.h -+++ b/include/linux/tcp.h -@@ -255,7 +255,8 @@ struct tcp_sock { - u8 compressed_ack; - u8 dup_ack_counter:2, - tlp_retrans:1, /* TLP is a retransmission */ -- unused:5; -+ fast_ack_mode:2, /* which fast ack mode ? */ -+ unused:3; - u32 chrono_start; /* Start time in jiffies of a TCP chrono */ - u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ - u8 chrono_type:2, /* current chronograph type */ -@@ -443,6 +444,9 @@ struct tcp_sock { - */ - struct request_sock __rcu *fastopen_rsk; - struct saved_syn *saved_syn; -+ -+/* Rerouting information */ -+ u16 ecn_rehash; /* PLB triggered rehash attempts */ - }; - - enum tsq_enum { -diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h -index ee88f0f1350f..e3075b3f1ece 100644 ---- a/include/net/inet_connection_sock.h -+++ b/include/net/inet_connection_sock.h -@@ -132,7 +132,8 @@ struct inet_connection_sock { - u32 icsk_probes_tstamp; - u32 icsk_user_timeout; - -- u64 icsk_ca_priv[104 / sizeof(u64)]; -+/* XXX inflated by temporary internal debugging info */ -+ u64 icsk_ca_priv[224 / sizeof(u64)]; - #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) - }; - -diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h -index 6320a76cefdc..2e39e07ed41f 100644 ---- a/include/net/netns/ipv4.h -+++ b/include/net/netns/ipv4.h -@@ -181,6 +181,11 @@ struct netns_ipv4 { - unsigned long tfo_active_disable_stamp; - u32 tcp_challenge_timestamp; - u32 tcp_challenge_count; -+ u8 sysctl_tcp_plb_enabled; -+ int sysctl_tcp_plb_cong_thresh; -+ u8 sysctl_tcp_plb_idle_rehash_rounds; -+ u8 sysctl_tcp_plb_rehash_rounds; -+ u8 sysctl_tcp_plb_suspend_rto_sec; - - int sysctl_udp_wmem_min; - int sysctl_udp_rmem_min; -diff --git a/include/net/tcp.h b/include/net/tcp.h -index d10962b9f0d0..d3947345feaa 100644 ---- a/include/net/tcp.h -+++ b/include/net/tcp.h -@@ -372,6 +372,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk, - #define TCP_ECN_QUEUE_CWR 2 - #define TCP_ECN_DEMAND_CWR 4 - #define TCP_ECN_SEEN 8 -+#define TCP_ECN_ECT_PERMANENT 16 - - enum tcp_tw_status { - TCP_TW_SUCCESS = 0, -@@ -816,6 +817,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) - return max_t(s64, t1 - t0, 0); - } - -+static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) -+{ -+ return max_t(s32, t1 - t0, 0); -+} -+ - static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) - { - return tcp_ns_to_ts(skb->skb_mstamp_ns); -@@ -891,9 +897,14 @@ struct tcp_skb_cb { - /* pkts S/ACKed so far upon tx of skb, incl retrans: */ - __u32 delivered; - /* start of send pipeline phase */ -- u64 first_tx_mstamp; -+ u32 first_tx_mstamp; - /* when we reached the "delivered" count */ -- u64 delivered_mstamp; -+ u32 delivered_mstamp; -+#define TCPCB_IN_FLIGHT_BITS 20 -+#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) -+ u32 in_flight:20, /* packets in flight at transmit */ -+ unused2:12; -+ u32 lost; /* packets lost so far upon tx of skb */ - } tx; /* only used for outgoing skbs */ - union { - struct inet_skb_parm h4; -@@ -1019,7 +1030,11 @@ enum tcp_ca_ack_event_flags { - #define TCP_CONG_NON_RESTRICTED 0x1 - /* Requires ECN/ECT set on all packets */ - #define TCP_CONG_NEEDS_ECN 0x2 --#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) -+/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ -+#define TCP_CONG_WANTS_CE_EVENTS 0x4 -+#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ -+ TCP_CONG_NEEDS_ECN | \ -+ TCP_CONG_WANTS_CE_EVENTS) - - union tcp_cc_info; - -@@ -1039,8 +1054,11 @@ struct ack_sample { - */ - struct rate_sample { - u64 prior_mstamp; /* starting timestamp for interval */ -+ u32 prior_lost; /* tp->lost at "prior_mstamp" */ - u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ - u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ -+ u32 tx_in_flight; /* packets in flight at starting timestamp */ -+ s32 lost; /* number of packets lost over interval */ - s32 delivered; /* number of packets delivered over interval */ - s32 delivered_ce; /* number of packets delivered w/ CE marks*/ - long interval_us; /* time for tp->delivered to incr "delivered" */ -@@ -1054,6 +1072,7 @@ struct rate_sample { - bool is_app_limited; /* is sample from packet with bubble in pipe? */ - bool is_retrans; /* is sample from retransmission? */ - bool is_ack_delayed; /* is this (likely) a delayed ACK? */ -+ bool is_ece; /* did this ACK have ECN marked? */ - }; - - struct tcp_congestion_ops { -@@ -1077,8 +1096,11 @@ struct tcp_congestion_ops { - /* hook for packet ack accounting (optional) */ - void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - -- /* override sysctl_tcp_min_tso_segs */ -- u32 (*min_tso_segs)(struct sock *sk); -+ /* pick target number of segments per TSO/GSO skb (optional): */ -+ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); -+ -+ /* react to a specific lost skb (optional) */ -+ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); - - /* call when packets are delivered to update cwnd and pacing rate, - * after all the ca_state processing. (optional) -@@ -1141,6 +1163,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) - } - #endif - -+static inline bool tcp_ca_wants_ce_events(const struct sock *sk) -+{ -+ const struct inet_connection_sock *icsk = inet_csk(sk); -+ -+ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | -+ TCP_CONG_WANTS_CE_EVENTS); -+} -+ - static inline bool tcp_ca_needs_ecn(const struct sock *sk) - { - const struct inet_connection_sock *icsk = inet_csk(sk); -@@ -1160,6 +1190,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) - void tcp_set_ca_state(struct sock *sk, const u8 ca_state); - - /* From tcp_rate.c */ -+void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); - void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); - void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - struct rate_sample *rs); -@@ -2130,6 +2161,23 @@ extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, - extern void tcp_rack_reo_timeout(struct sock *sk); - extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); - -+/* tcp_plb.c */ -+ -+#define TCP_PLB_SCALE 8 /* scaling factor for fractions in PLB (e.g. ce_ratio) */ -+ -+/* State for PLB (Protective Load Balancing) for a single TCP connection. */ -+struct tcp_plb_state { -+ u8 consec_cong_rounds:5, /* consecutive congested rounds */ -+ enabled:1, /* Check if PLB is enabled */ -+ unused:2; -+ u32 pause_until; /* jiffies32 when PLB can resume repathing */ -+}; -+ -+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, -+ const int cong_ratio); -+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); -+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); -+ - /* At how many usecs into the future should the RTO fire? */ - static inline s64 tcp_rto_delta_us(const struct sock *sk) - { -diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h -index 50655de04c9b..0e24f11627d5 100644 ---- a/include/uapi/linux/inet_diag.h -+++ b/include/uapi/linux/inet_diag.h -@@ -231,9 +231,42 @@ struct tcp_bbr_info { - __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ - }; - -+/* Phase as reported in netlink/ss stats. */ -+enum tcp_bbr2_phase { -+ BBR2_PHASE_INVALID = 0, -+ BBR2_PHASE_STARTUP = 1, -+ BBR2_PHASE_DRAIN = 2, -+ BBR2_PHASE_PROBE_RTT = 3, -+ BBR2_PHASE_PROBE_BW_UP = 4, -+ BBR2_PHASE_PROBE_BW_DOWN = 5, -+ BBR2_PHASE_PROBE_BW_CRUISE = 6, -+ BBR2_PHASE_PROBE_BW_REFILL = 7 -+}; -+ -+struct tcp_bbr2_info { -+ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ -+ __u32 bbr_bw_lsb; /* lower 32 bits of bw */ -+ __u32 bbr_bw_msb; /* upper 32 bits of bw */ -+ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ -+ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ -+ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ -+ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ -+ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ -+ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ -+ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ -+ __u8 bbr_mode; /* current bbr_mode in state machine */ -+ __u8 bbr_phase; /* current state machine phase */ -+ __u8 unused1; /* alignment padding; not used yet */ -+ __u8 bbr_version; /* MUST be at this offset in struct */ -+ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ -+ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ -+ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ -+}; -+ - union tcp_cc_info { - struct tcpvegas_info vegas; - struct tcp_dctcp_info dctcp; - struct tcp_bbr_info bbr; -+ struct tcp_bbr2_info bbr2; - }; - #endif /* _UAPI_INET_DIAG_H_ */ -diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h -index 4d7470036a8b..8ce035f1c874 100644 ---- a/include/uapi/linux/snmp.h -+++ b/include/uapi/linux/snmp.h -@@ -292,6 +292,7 @@ enum - LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */ - LINUX_MIB_TCPMIGRATEREQSUCCESS, /* TCPMigrateReqSuccess */ - LINUX_MIB_TCPMIGRATEREQFAILURE, /* TCPMigrateReqFailure */ -+ LINUX_MIB_TCPECNREHASH, /* TCPECNRehash */ - __LINUX_MIB_MAX - }; - -diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig -index e983bb0c5012..d172ffbdf27f 100644 ---- a/net/ipv4/Kconfig -+++ b/net/ipv4/Kconfig -@@ -668,6 +668,24 @@ config TCP_CONG_BBR - AQM schemes that do not provide a delay signal. It requires the fq - ("Fair Queue") pacing packet scheduler. - -+config TCP_CONG_BBR2 -+ tristate "BBR2 TCP" -+ default n -+ help -+ -+ BBR2 TCP congestion control is a model-based congestion control -+ algorithm that aims to maximize network utilization, keep queues and -+ retransmit rates low, and to be able to coexist with Reno/CUBIC in -+ common scenarios. It builds an explicit model of the network path. It -+ tolerates a targeted degree of random packet loss and delay that are -+ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, -+ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can -+ coexist with flows that use loss-based congestion control, and can -+ operate with shallow buffers, deep buffers, bufferbloat, policers, or -+ AQM schemes that do not provide a delay signal. It requires pacing, -+ using either TCP internal pacing or the fq ("Fair Queue") pacing packet -+ scheduler. -+ - choice - prompt "Default TCP congestion control" - default DEFAULT_CUBIC -@@ -705,6 +723,9 @@ choice - config DEFAULT_BBR - bool "BBR" if TCP_CONG_BBR=y - -+ config DEFAULT_BBR2 -+ bool "BBR2" if TCP_CONG_BBR2=y -+ - config DEFAULT_RENO - bool "Reno" - endchoice -@@ -729,6 +750,7 @@ config DEFAULT_TCP_CONG - default "dctcp" if DEFAULT_DCTCP - default "cdg" if DEFAULT_CDG - default "bbr" if DEFAULT_BBR -+ default "bbr2" if DEFAULT_BBR2 - default "cubic" - - config TCP_MD5SIG -diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile -index bbdd9c44f14e..e7a86a50838a 100644 ---- a/net/ipv4/Makefile -+++ b/net/ipv4/Makefile -@@ -10,7 +10,7 @@ obj-y := route.o inetpeer.o protocol.o \ - tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \ - tcp_minisocks.o tcp_cong.o tcp_metrics.o tcp_fastopen.o \ - tcp_rate.o tcp_recovery.o tcp_ulp.o \ -- tcp_offload.o datagram.o raw.o udp.o udplite.o \ -+ tcp_offload.o tcp_plb.o datagram.o raw.o udp.o udplite.o \ - udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \ - fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \ - inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \ -@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o - obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o - obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o - obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o -+obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o - obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o - obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o - obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o -diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c -index 85a9e500c42d..24fcbac984fe 100644 ---- a/net/ipv4/bpf_tcp_ca.c -+++ b/net/ipv4/bpf_tcp_ca.c -@@ -14,6 +14,18 @@ - /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ - extern struct bpf_struct_ops bpf_tcp_congestion_ops; - -+static u32 optional_ops[] = { -+ offsetof(struct tcp_congestion_ops, init), -+ offsetof(struct tcp_congestion_ops, release), -+ offsetof(struct tcp_congestion_ops, set_state), -+ offsetof(struct tcp_congestion_ops, cwnd_event), -+ offsetof(struct tcp_congestion_ops, in_ack_event), -+ offsetof(struct tcp_congestion_ops, pkts_acked), -+ offsetof(struct tcp_congestion_ops, tso_segs), -+ offsetof(struct tcp_congestion_ops, sndbuf_expand), -+ offsetof(struct tcp_congestion_ops, cong_control), -+}; -+ - static u32 unsupported_ops[] = { - offsetof(struct tcp_congestion_ops, get_info), - }; -diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c -index 0088a4c64d77..e0a664b467e0 100644 ---- a/net/ipv4/proc.c -+++ b/net/ipv4/proc.c -@@ -297,6 +297,7 @@ static const struct snmp_mib snmp4_net_list[] = { - SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS), - SNMP_MIB_ITEM("TCPMigrateReqSuccess", LINUX_MIB_TCPMIGRATEREQSUCCESS), - SNMP_MIB_ITEM("TCPMigrateReqFailure", LINUX_MIB_TCPMIGRATEREQFAILURE), -+ SNMP_MIB_ITEM("TCPECNRehash", LINUX_MIB_TCPECNREHASH), - SNMP_MIB_SENTINEL - }; - -diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c -index 5490c285668b..ed35a9485e71 100644 ---- a/net/ipv4/sysctl_net_ipv4.c -+++ b/net/ipv4/sysctl_net_ipv4.c -@@ -39,6 +39,8 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ; - static int one_day_secs = 24 * 3600; - static u32 fib_multipath_hash_fields_all_mask __maybe_unused = - FIB_MULTIPATH_HASH_FIELD_ALL_MASK; -+static int tcp_plb_max_rounds = 31; -+static int tcp_plb_max_cong_thresh = 256; - - /* obsolete */ - static int sysctl_tcp_low_latency __read_mostly; -@@ -1346,6 +1348,47 @@ static struct ctl_table ipv4_net_table[] = { - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_TWO, - }, -+ { -+ .procname = "tcp_plb_enabled", -+ .data = &init_net.ipv4.sysctl_tcp_plb_enabled, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "tcp_plb_cong_thresh", -+ .data = &init_net.ipv4.sysctl_tcp_plb_cong_thresh, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &tcp_plb_max_cong_thresh, -+ }, -+ { -+ .procname = "tcp_plb_idle_rehash_rounds", -+ .data = &init_net.ipv4.sysctl_tcp_plb_idle_rehash_rounds, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra2 = &tcp_plb_max_rounds, -+ }, -+ { -+ .procname = "tcp_plb_rehash_rounds", -+ .data = &init_net.ipv4.sysctl_tcp_plb_rehash_rounds, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra2 = &tcp_plb_max_rounds, -+ }, -+ { -+ .procname = "tcp_plb_suspend_rto_sec", -+ .data = &init_net.ipv4.sysctl_tcp_plb_suspend_rto_sec, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ }, - { } - }; - -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index e373dde1f46f..b735776c0225 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -3187,6 +3187,7 @@ int tcp_disconnect(struct sock *sk, int flags) - tp->rx_opt.dsack = 0; - tp->rx_opt.num_sacks = 0; - tp->rcv_ooopack = 0; -+ tp->fast_ack_mode = 0; - - - /* Clean up fastopen related fields */ -diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c -index 54eec33c6e1c..bfbf158c71f4 100644 ---- a/net/ipv4/tcp_bbr.c -+++ b/net/ipv4/tcp_bbr.c -@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) - sk->sk_pacing_rate = rate; - } - --/* override sysctl_tcp_min_tso_segs */ - static u32 bbr_min_tso_segs(struct sock *sk) - { - return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; - } - -+/* Return the number of segments BBR would like in a TSO/GSO skb, given -+ * a particular max gso size as a constraint. -+ */ -+static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, -+ u32 gso_max_size) -+{ -+ u32 segs; -+ u64 bytes; -+ -+ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ -+ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; -+ -+ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); -+ return segs; -+} -+ -+/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ -+static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) -+{ -+ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); -+} -+ -+/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ - static u32 bbr_tso_segs_goal(struct sock *sk) - { - struct tcp_sock *tp = tcp_sk(sk); -- u32 segs, bytes; -- -- /* Sort of tcp_tso_autosize() but ignoring -- * driver provided sk_gso_max_size. -- */ -- bytes = min_t(unsigned long, -- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), -- GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); -- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - -- return min(segs, 0x7FU); -+ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); - } - - /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { - .undo_cwnd = bbr_undo_cwnd, - .cwnd_event = bbr_cwnd_event, - .ssthresh = bbr_ssthresh, -- .min_tso_segs = bbr_min_tso_segs, -+ .tso_segs = bbr_tso_segs, - .get_info = bbr_get_info, - .set_state = bbr_set_state, - }; -diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c -new file mode 100644 -index 000000000000..2e39f7a353be ---- /dev/null -+++ b/net/ipv4/tcp_bbr2.c -@@ -0,0 +1,2692 @@ -+/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 -+ * -+ * BBRv2 is a model-based congestion control algorithm that aims for low -+ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model -+ * of the network path, it uses measurements of bandwidth and RTT, as well as -+ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that -+ * although it can use ECN or loss signals explicitly, it does not require -+ * either; it can bound its in-flight data based on its estimate of the BDP. -+ * -+ * The model has both higher and lower bounds for the operating range: -+ * lo: bw_lo, inflight_lo: conservative short-term lower bound -+ * hi: bw_hi, inflight_hi: robust long-term upper bound -+ * The bandwidth-probing time scale is (a) extended dynamically based on -+ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by -+ * an interactive wall-clock time-scale to be more scalable and responsive -+ * than Reno and CUBIC. -+ * -+ * Here is a state transition diagram for BBR: -+ * -+ * | -+ * V -+ * +---> STARTUP ----+ -+ * | | | -+ * | V | -+ * | DRAIN ----+ -+ * | | | -+ * | V | -+ * +---> PROBE_BW ----+ -+ * | ^ | | -+ * | | | | -+ * | +----+ | -+ * | | -+ * +---- PROBE_RTT <--+ -+ * -+ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. -+ * When it estimates the pipe is full, it enters DRAIN to drain the queue. -+ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. -+ * A long-lived BBR flow spends the vast majority of its time remaining -+ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth -+ * in a fair manner, with a small, bounded queue. *If* a flow has been -+ * continuously sending for the entire min_rtt window, and hasn't seen an RTT -+ * sample that matches or decreases its min_rtt estimate for 10 seconds, then -+ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe -+ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if -+ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; -+ * otherwise we enter STARTUP to try to fill the pipe. -+ * -+ * BBR is described in detail in: -+ * "BBR: Congestion-Based Congestion Control", -+ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, -+ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. -+ * -+ * There is a public e-mail list for discussing BBR development and testing: -+ * https://groups.google.com/forum/#!forum/bbr-dev -+ * -+ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, -+ * otherwise TCP stack falls back to an internal pacing using one high -+ * resolution timer per TCP socket and may use more resources. -+ */ -+#include -+#include -+#include -+#include -+#include -+ -+#include "tcp_dctcp.h" -+ -+/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth -+ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. -+ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. -+ * Since the minimum window is >=4 packets, the lower bound isn't -+ * an issue. The upper bound isn't an issue with existing technologies. -+ */ -+#define BW_SCALE 24 -+#define BW_UNIT (1 << BW_SCALE) -+ -+#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ -+#define BBR_UNIT (1 << BBR_SCALE) -+ -+#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ -+#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ -+ -+#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ -+ -+/* BBR has the following modes for deciding how fast to send: */ -+enum bbr_mode { -+ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ -+ BBR_DRAIN, /* drain any queue created during startup */ -+ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ -+ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ -+}; -+ -+/* How does the incoming ACK stream relate to our bandwidth probing? */ -+enum bbr_ack_phase { -+ BBR_ACKS_INIT, /* not probing; not getting probe feedback */ -+ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ -+ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ -+ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ -+ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ -+}; -+ -+/* BBR congestion control block */ -+struct bbr { -+ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ -+ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ -+ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ -+ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ -+ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ -+ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ -+ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ -+ u64 cycle_mstamp; /* time of this cycle phase start */ -+ u32 mode:3, /* current bbr_mode in state machine */ -+ prev_ca_state:3, /* CA state on previous ACK */ -+ packet_conservation:1, /* use packet conservation? */ -+ round_start:1, /* start of packet-timed tx->ack round? */ -+ ce_state:1, /* If most recent data has CE bit set */ -+ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ -+ try_fast_path:1, /* can we take fast path? */ -+ unused2:11, -+ idle_restart:1, /* restarting after idle? */ -+ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ -+ cycle_idx:3, /* current index in pacing_gain cycle array */ -+ has_seen_rtt:1; /* have we seen an RTT sample yet? */ -+ u32 pacing_gain:11, /* current gain for setting pacing rate */ -+ cwnd_gain:11, /* current gain for setting cwnd */ -+ full_bw_reached:1, /* reached full bw in Startup? */ -+ full_bw_cnt:2, /* number of rounds without large bw gains */ -+ init_cwnd:7; /* initial cwnd */ -+ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ -+ u32 full_bw; /* recent bw, to estimate if pipe is full */ -+ -+ /* For tracking ACK aggregation: */ -+ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ -+ u16 extra_acked[2]; /* max excess data ACKed in epoch */ -+ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ -+ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ -+ extra_acked_win_idx:1, /* current index in extra_acked array */ -+ /* BBR v2 state: */ -+ unused1:2, -+ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ -+ loss_in_cycle:1, /* packet loss in this cycle? */ -+ ecn_in_cycle:1; /* ECN in this cycle? */ -+ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ -+ u32 undo_bw_lo; /* bw_lo before latest losses */ -+ u32 undo_inflight_lo; /* inflight_lo before latest losses */ -+ u32 undo_inflight_hi; /* inflight_hi before latest losses */ -+ u32 bw_latest; /* max delivered bw in last round trip */ -+ u32 bw_lo; /* lower bound on sending bandwidth */ -+ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ -+ u32 inflight_latest; /* max delivered data in last round trip */ -+ u32 inflight_lo; /* lower bound of inflight data range */ -+ u32 inflight_hi; /* upper bound of inflight data range */ -+ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ -+ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ -+ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ -+ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ -+ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ -+ bw_probe_samples:1, /* rate samples reflect bw probing? */ -+ prev_probe_too_high:1, /* did last PROBE_UP go too high? */ -+ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ -+ rounds_since_probe:8, /* packet-timed rounds since probed bw */ -+ loss_round_start:1, /* loss_round_delivered round trip? */ -+ loss_in_round:1, /* loss marked in this round trip? */ -+ ecn_in_round:1, /* ECN marked in this round trip? */ -+ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ -+ loss_events_in_round:4,/* losses in STARTUP round */ -+ initialized:1; /* has bbr_init() been called? */ -+ u32 alpha_last_delivered; /* tp->delivered at alpha update */ -+ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ -+ struct tcp_plb_state plb; -+ -+ /* Params configurable using setsockopt. Refer to correspoding -+ * module param for detailed description of params. -+ */ -+ struct bbr_params { -+ u32 high_gain:11, /* max allowed value: 2047 */ -+ drain_gain:10, /* max allowed value: 1023 */ -+ cwnd_gain:11; /* max allowed value: 2047 */ -+ u32 cwnd_min_target:4, /* max allowed value: 15 */ -+ min_rtt_win_sec:5, /* max allowed value: 31 */ -+ probe_rtt_mode_ms:9, /* max allowed value: 511 */ -+ full_bw_cnt:3, /* max allowed value: 7 */ -+ cwnd_tso_budget:1, /* allowed values: {0, 1} */ -+ unused3:6, -+ drain_to_target:1, /* boolean */ -+ precise_ece_ack:1, /* boolean */ -+ extra_acked_in_startup:1, /* allowed values: {0, 1} */ -+ fast_path:1; /* boolean */ -+ u32 full_bw_thresh:10, /* max allowed value: 1023 */ -+ startup_cwnd_gain:11, /* max allowed value: 2047 */ -+ bw_probe_pif_gain:9, /* max allowed value: 511 */ -+ usage_based_cwnd:1, /* boolean */ -+ unused2:1; -+ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ -+ refill_add_inc:2; /* max allowed value: 3 */ -+ u16 extra_acked_gain:11, /* max allowed value: 2047 */ -+ extra_acked_win_rtts:5; /* max allowed value: 31*/ -+ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ -+ /* Mostly BBR v2 parameters below here: */ -+ u32 ecn_alpha_gain:8, /* max allowed value: 255 */ -+ ecn_factor:8, /* max allowed value: 255 */ -+ ecn_thresh:8, /* max allowed value: 255 */ -+ beta:8; /* max allowed value: 255 */ -+ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ -+ bw_probe_reno_gain:9, /* max allowed value: 511 */ -+ full_loss_cnt:4; /* max allowed value: 15 */ -+ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ -+ inflight_headroom:8, /* max allowed value: 255 */ -+ loss_thresh:8, /* max allowed value: 255 */ -+ bw_probe_max_rounds:8; /* max allowed value: 255 */ -+ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ -+ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ -+ full_ecn_cnt:2; /* max allowed value: 3 */ -+ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ -+ undo:1, /* boolean */ -+ tso_rtt_shift:4, /* max allowed value: 15 */ -+ unused5:1; -+ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ -+ unused1:14, -+ ecn_alpha_init:9; /* max allowed value: 256 */ -+ } params; -+ -+ struct { -+ u32 snd_isn; /* Initial sequence number */ -+ u32 rs_bw; /* last valid rate sample bw */ -+ u32 target_cwnd; /* target cwnd, based on BDP */ -+ u8 undo:1, /* Undo even happened but not yet logged */ -+ unused:7; -+ char event; /* single-letter event debug codes */ -+ u16 unused2; -+ } debug; -+}; -+ -+struct bbr_context { -+ u32 sample_bw; -+ u32 target_cwnd; -+ u32 log:1; -+}; -+ -+/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ -+static u32 bbr_min_rtt_win_sec = 10; -+/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. -+ * Max allowed value is 511 (0x1FF). -+ */ -+static u32 bbr_probe_rtt_mode_ms = 200; -+/* Window length of probe_rtt_min_us filter (in ms), and consequently the -+ * typical interval between PROBE_RTT mode entries. -+ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC -+ */ -+static u32 bbr_probe_rtt_win_ms = 5000; -+/* Skip TSO below the following bandwidth (bits/sec): */ -+static int bbr_min_tso_rate = 1200000; -+ -+/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting -+ * in bigger TSO bursts. By default we cut the RTT-based allowance in half -+ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance -+ * is below 1500 bytes after 6 * ~500 usec = 3ms. -+ */ -+static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ -+ -+/* Select cwnd TSO budget approach: -+ * 0: padding -+ * 1: flooring -+ */ -+static uint bbr_cwnd_tso_budget = 1; -+ -+/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. -+ * In order to help drive the network toward lower queues and low latency while -+ * maintaining high utilization, the average pacing rate aims to be slightly -+ * lower than the estimated bandwidth. This is an important aspect of the -+ * design. -+ */ -+static const int bbr_pacing_margin_percent = 1; -+ -+/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain -+ * that will allow a smoothly increasing pacing rate that will double each RTT -+ * and send the same number of packets per RTT that an un-paced, slow-starting -+ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). -+ */ -+static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; -+/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ -+static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; -+/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain -+ * the queue created in BBR_STARTUP in a single round. Max allowed value -+ * is 1023 (0x3FF). -+ */ -+static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; -+/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. -+ * Max allowed value is 2047 (0x7FF). -+ */ -+static int bbr_cwnd_gain = BBR_UNIT * 2; -+/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. -+ * Max allowed value for each element is 1023 (0x3FF). -+ */ -+enum bbr_pacing_gain_phase { -+ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ -+ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ -+ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ -+ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ -+}; -+static int bbr_pacing_gain[] = { -+ BBR_UNIT * 5 / 4, /* probe for more available bw */ -+ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ -+ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ -+ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ -+}; -+ -+/* Try to keep at least this many packets in flight, if things go smoothly. For -+ * smooth functioning, a sliding window protocol ACKing every other packet -+ * needs at least 4 packets in flight. Max allowed value is 15 (0xF). -+ */ -+static u32 bbr_cwnd_min_target = 4; -+ -+/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. -+ * Use 0 to disable. Max allowed value is 255. -+ */ -+static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; -+ -+/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ -+/* If bw has increased significantly (1.25x), there may be more bw available. -+ * Max allowed value is 1023 (0x3FF). -+ */ -+static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; -+/* But after 3 rounds w/o significant bw growth, estimate pipe is full. -+ * Max allowed value is 7 (0x7). -+ */ -+static u32 bbr_full_bw_cnt = 3; -+ -+static u32 bbr_flags; /* Debugging related stuff */ -+ -+/* Whether to debug using printk. -+ */ -+static bool bbr_debug_with_printk; -+ -+/* Whether to debug using ftrace event tcp:tcp_bbr_event. -+ * Ignored when bbr_debug_with_printk is set. -+ */ -+static bool bbr_debug_ftrace; -+ -+/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ -+static bool bbr_drain_to_target = true; /* default: enabled */ -+ -+/* Experiment: Flags to control BBR with ECN behavior. -+ */ -+static bool bbr_precise_ece_ack = true; /* default: enabled */ -+ -+/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is -+ * (2^(16+14) B)/(1024 B/packet) = 1M packets. -+ */ -+static u32 bbr_cwnd_warn_val = 1U << 20; -+ -+static u16 bbr_debug_port_mask; -+ -+/* BBR module parameters. These are module parameters only in Google prod. -+ * Upstream these are intentionally not module parameters. -+ */ -+static int bbr_pacing_gain_size = CYCLE_LEN; -+ -+/* Gain factor for adding extra_acked to target cwnd: */ -+static int bbr_extra_acked_gain = 256; -+ -+/* Window length of extra_acked window. Max allowed val is 31. */ -+static u32 bbr_extra_acked_win_rtts = 5; -+ -+/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ -+static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; -+ -+/* Time period for clamping cwnd increment due to ack aggregation */ -+static u32 bbr_extra_acked_max_us = 100 * 1000; -+ -+/* Use extra acked in startup ? -+ * 0: disabled -+ * 1: use latest extra_acked value from 1-2 rtt in startup -+ */ -+static int bbr_extra_acked_in_startup = 1; /* default: enabled */ -+ -+/* Experiment: don't grow cwnd beyond twice of what we just probed. */ -+static bool bbr_usage_based_cwnd; /* default: disabled */ -+ -+/* For lab testing, researchers can enable BBRv2 ECN support with this flag, -+ * when they know that any ECN marks that the connections experience will be -+ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. -+ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on -+ * negotiation or configuration that is outside the scope of the BBRv2 -+ * alpha release. -+ */ -+static bool bbr_ecn_enable = false; -+ -+module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); -+module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); -+module_param_named(high_gain, bbr_high_gain, int, 0644); -+module_param_named(drain_gain, bbr_drain_gain, int, 0644); -+module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); -+module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); -+module_param_array_named(pacing_gain, bbr_pacing_gain, int, -+ &bbr_pacing_gain_size, 0644); -+module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); -+module_param_named(probe_rtt_cwnd_gain, -+ bbr_probe_rtt_cwnd_gain, uint, 0664); -+module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); -+module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); -+module_param_named(flags, bbr_flags, uint, 0644); -+module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); -+module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); -+module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); -+module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); -+module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); -+module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); -+module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); -+module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); -+module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); -+module_param_named(extra_acked_win_rtts, -+ bbr_extra_acked_win_rtts, uint, 0664); -+module_param_named(extra_acked_max_us, -+ bbr_extra_acked_max_us, uint, 0664); -+module_param_named(ack_epoch_acked_reset_thresh, -+ bbr_ack_epoch_acked_reset_thresh, uint, 0664); -+module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); -+module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); -+module_param_named(extra_acked_in_startup, -+ bbr_extra_acked_in_startup, int, 0664); -+module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); -+module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); -+ -+static void bbr2_exit_probe_rtt(struct sock *sk); -+static void bbr2_reset_congestion_signals(struct sock *sk); -+ -+static void bbr_check_probe_rtt_done(struct sock *sk); -+ -+/* Do we estimate that STARTUP filled the pipe? */ -+static bool bbr_full_bw_reached(const struct sock *sk) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ -+ return bbr->full_bw_reached; -+} -+ -+/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ -+static u32 bbr_max_bw(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return max(bbr->bw_hi[0], bbr->bw_hi[1]); -+} -+ -+/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ -+static u32 bbr_bw(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return min(bbr_max_bw(sk), bbr->bw_lo); -+} -+ -+/* Return maximum extra acked in past k-2k round trips, -+ * where k = bbr_extra_acked_win_rtts. -+ */ -+static u16 bbr_extra_acked(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return max(bbr->extra_acked[0], bbr->extra_acked[1]); -+} -+ -+/* Return rate in bytes per second, optionally with a gain. -+ * The order here is chosen carefully to avoid overflow of u64. This should -+ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. -+ */ -+static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, -+ int margin) -+{ -+ unsigned int mss = tcp_sk(sk)->mss_cache; -+ -+ rate *= mss; -+ rate *= gain; -+ rate >>= BBR_SCALE; -+ rate *= USEC_PER_SEC / 100 * (100 - margin); -+ rate >>= BW_SCALE; -+ rate = max(rate, 1ULL); -+ return rate; -+} -+ -+static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) -+{ -+ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); -+} -+ -+static u64 bbr_rate_kbps(struct sock *sk, u64 rate) -+{ -+ rate = bbr_bw_bytes_per_sec(sk, rate); -+ rate *= 8; -+ do_div(rate, 1000); -+ return rate; -+} -+ -+static u32 bbr_tso_segs_goal(struct sock *sk); -+static void bbr_debug(struct sock *sk, u32 acked, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ static const char ca_states[] = { -+ [TCP_CA_Open] = 'O', -+ [TCP_CA_Disorder] = 'D', -+ [TCP_CA_CWR] = 'C', -+ [TCP_CA_Recovery] = 'R', -+ [TCP_CA_Loss] = 'L', -+ }; -+ static const char mode[] = { -+ 'G', /* Growing - BBR_STARTUP */ -+ 'D', /* Drain - BBR_DRAIN */ -+ 'W', /* Window - BBR_PROBE_BW */ -+ 'M', /* Min RTT - BBR_PROBE_RTT */ -+ }; -+ static const char ack_phase[] = { /* bbr_ack_phase strings */ -+ 'I', /* BBR_ACKS_INIT - 'Init' */ -+ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ -+ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ -+ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ -+ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ -+ }; -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ const u32 una = tp->snd_una - bbr->debug.snd_isn; -+ const u32 fack = tcp_highest_sack_seq(tp); -+ const u16 dport = ntohs(inet_sk(sk)->inet_dport); -+ bool is_port_match = (bbr_debug_port_mask && -+ ((dport & bbr_debug_port_mask) == 0)); -+ char debugmsg[320]; -+ -+ if (sk->sk_state == TCP_SYN_SENT) -+ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ -+ -+ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { -+ char addr[INET6_ADDRSTRLEN + 10] = { 0 }; -+ -+ if (sk->sk_family == AF_INET) -+ snprintf(addr, sizeof(addr), "%pI4:%u", -+ &inet_sk(sk)->inet_daddr, dport); -+ else if (sk->sk_family == AF_INET6) -+ snprintf(addr, sizeof(addr), "%pI6:%u", -+ &sk->sk_v6_daddr, dport); -+ -+ WARN_ONCE(1, -+ "BBR %s cwnd alert: %u " -+ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " -+ "bw: %u rtt: %u min_rtt: %u " -+ "acked: %u tso_segs: %u " -+ "bw: %d %ld %d pif: %u\n", -+ addr, tp->snd_cwnd, -+ una, inet_csk(sk)->icsk_ca_state, -+ bbr->pacing_gain, bbr->cwnd_gain, -+ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, -+ acked, bbr_tso_segs_goal(sk), -+ rs->delivered, rs->interval_us, rs->is_retrans, -+ tcp_packets_in_flight(tp)); -+ } -+ -+ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) -+ return; -+ -+ if (!sock_flag(sk, SOCK_DBG) && !is_port_match) -+ return; -+ -+ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) -+ return; -+ -+ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && -+ !(bbr_flags & FLAG_DEBUG_LOOPBACK)) -+ return; -+ -+ snprintf(debugmsg, sizeof(debugmsg) - 1, -+ "BBR %pI4:%-5u %5u,%03u:%-7u %c " -+ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " -+ "bw %llu lb %llu ib %llu qb %llu " -+ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " -+ "lr %d er %d ea %d bwl %lld il %d ih %d c %d " -+ "v %d %c %u %c %s\n", -+ &inet_sk(sk)->inet_daddr, dport, -+ una / 1000, una % 1000, fack - tp->snd_una, -+ ca_states[inet_csk(sk)->icsk_ca_state], -+ bbr->debug.undo ? '@' : mode[bbr->mode], -+ tp->snd_cwnd, -+ bbr_extra_acked(sk), /* br (legacy): extra_acked */ -+ rs->tx_in_flight, /* cr (legacy): tx_inflight */ -+ rs->rtt_us, -+ rs->delivered, -+ rs->interval_us, -+ bbr->min_rtt_us, -+ rs->is_app_limited ? '_' : 'l', -+ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ -+ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ -+ 0ULL, /* lb: [obsolete] */ -+ 0ULL, /* ib: [obsolete] */ -+ div_u64((u64)sk->sk_pacing_rate * 8, 1000), -+ acked, -+ tcp_packets_in_flight(tp), -+ rs->is_ack_delayed ? 'd' : '.', -+ bbr->round_start ? '*' : '.', -+ tp->delivered, tp->lost, -+ tp->app_limited, -+ 0, /* #: [obsolete] */ -+ ctx->target_cwnd, -+ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ -+ ca_states[bbr->prev_ca_state], -+ (rs->lost + rs->delivered) > 0 ? -+ (1000 * rs->lost / -+ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ -+ (rs->delivered) > 0 ? -+ (1000 * rs->delivered_ce / -+ (rs->delivered)) : 0, /* er: ECN rate x1000 */ -+ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ -+ bbr->bw_lo == ~0U ? -+ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ -+ bbr->inflight_lo, /* il */ -+ bbr->inflight_hi, /* ih */ -+ bbr->bw_probe_up_cnt, /* c */ -+ 2, /* v: version */ -+ bbr->debug.event, -+ bbr->cycle_idx, -+ ack_phase[bbr->ack_phase], -+ bbr->bw_probe_samples ? "Y" : "N"); -+ debugmsg[sizeof(debugmsg) - 1] = 0; -+ -+ /* printk takes a higher precedence. */ -+ if (bbr_debug_with_printk) -+ printk(KERN_DEBUG "%s", debugmsg); -+ -+ if (unlikely(bbr->debug.undo)) -+ bbr->debug.undo = 0; -+} -+ -+/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ -+static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) -+{ -+ u64 rate = bw; -+ -+ rate = bbr_rate_bytes_per_sec(sk, rate, gain, -+ bbr_pacing_margin_percent); -+ rate = min_t(u64, rate, sk->sk_max_pacing_rate); -+ return rate; -+} -+ -+/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ -+static void bbr_init_pacing_rate_from_rtt(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw; -+ u32 rtt_us; -+ -+ if (tp->srtt_us) { /* any RTT sample yet? */ -+ rtt_us = max(tp->srtt_us >> 3, 1U); -+ bbr->has_seen_rtt = 1; -+ } else { /* no RTT sample yet */ -+ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ -+ } -+ bw = (u64)tp->snd_cwnd * BW_UNIT; -+ do_div(bw, rtt_us); -+ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); -+} -+ -+/* Pace using current bw estimate and a gain factor. */ -+static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); -+ -+ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) -+ bbr_init_pacing_rate_from_rtt(sk); -+ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) -+ sk->sk_pacing_rate = rate; -+} -+ -+static u32 bbr_min_tso_segs(struct sock *sk) -+{ -+ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; -+} -+ -+/* Return the number of segments BBR would like in a TSO/GSO skb, given -+ * a particular max gso size as a constraint. -+ */ -+static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, -+ u32 gso_max_size) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 segs, r; -+ u64 bytes; -+ -+ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ -+ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; -+ -+ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every -+ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. -+ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) -+ */ -+ if (bbr->params.tso_rtt_shift) { -+ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; -+ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ -+ bytes += GSO_MAX_SIZE >> r; -+ } -+ -+ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); -+ return segs; -+} -+ -+/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ -+static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) -+{ -+ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); -+} -+ -+/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ -+static u32 bbr_tso_segs_goal(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ -+ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); -+} -+ -+/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -+static void bbr_save_cwnd(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) -+ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ -+ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ -+ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); -+} -+ -+static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (event == CA_EVENT_TX_START) { -+ tcp_plb_check_rehash(sk, &bbr->plb); -+ -+ if (!tp->app_limited) -+ return; -+ bbr->idle_restart = 1; -+ bbr->ack_epoch_mstamp = tp->tcp_mstamp; -+ bbr->ack_epoch_acked = 0; -+ /* Avoid pointless buffer overflows: pace at est. bw if we don't -+ * need more speed (we're restarting from idle and app-limited). -+ */ -+ if (bbr->mode == BBR_PROBE_BW) -+ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); -+ else if (bbr->mode == BBR_PROBE_RTT) -+ bbr_check_probe_rtt_done(sk); -+ } else if ((event == CA_EVENT_ECN_IS_CE || -+ event == CA_EVENT_ECN_NO_CE) && -+ bbr_ecn_enable && -+ bbr->params.precise_ece_ack) { -+ u32 state = bbr->ce_state; -+ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); -+ bbr->ce_state = state; -+ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) -+ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); -+ } -+} -+ -+/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: -+ * -+ * bdp = ceil(bw * min_rtt * gain) -+ * -+ * The key factor, gain, controls the amount of queue. While a small gain -+ * builds a smaller queue, it becomes more vulnerable to noise in RTT -+ * measurements (e.g., delayed ACKs or other ACK compression effects). This -+ * noise may cause BBR to under-estimate the rate. -+ */ -+static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 bdp; -+ u64 w; -+ -+ /* If we've never had a valid RTT sample, cap cwnd at the initial -+ * default. This should only happen when the connection is not using TCP -+ * timestamps and has retransmitted all of the SYN/SYNACK/data packets -+ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which -+ * case we need to slow-start up toward something safe: initial cwnd. -+ */ -+ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ -+ return bbr->init_cwnd; /* be safe: cap at initial cwnd */ -+ -+ w = (u64)bw * bbr->min_rtt_us; -+ -+ /* Apply a gain to the given value, remove the BW_SCALE shift, and -+ * round the value up to avoid a negative feedback loop. -+ */ -+ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; -+ -+ return bdp; -+} -+ -+/* To achieve full performance in high-speed paths, we budget enough cwnd to -+ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: -+ * - one skb in sending host Qdisc, -+ * - one skb in sending host TSO/GSO engine -+ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine -+ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because -+ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, -+ * which allows 2 outstanding 2-packet sequences, to try to keep pipe -+ * full even with ACK-every-other-packet delayed ACKs. -+ */ -+static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 tso_segs_goal; -+ -+ tso_segs_goal = 3 * bbr_tso_segs_goal(sk); -+ -+ /* Allow enough full-sized skbs in flight to utilize end systems. */ -+ if (bbr->params.cwnd_tso_budget == 1) { -+ cwnd = max_t(u32, cwnd, tso_segs_goal); -+ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); -+ } else { -+ cwnd += tso_segs_goal; -+ cwnd = (cwnd + 1) & ~1U; -+ } -+ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) -+ cwnd += 2; -+ -+ return cwnd; -+} -+ -+/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ -+static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) -+{ -+ u32 inflight; -+ -+ inflight = bbr_bdp(sk, bw, gain); -+ inflight = bbr_quantization_budget(sk, inflight); -+ -+ return inflight; -+} -+ -+/* With pacing at lower layers, there's often less data "in the network" than -+ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), -+ * we often have several skbs queued in the pacing layer with a pre-scheduled -+ * earliest departure time (EDT). BBR adapts its pacing rate based on the -+ * inflight level that it estimates has already been "baked in" by previous -+ * departure time decisions. We calculate a rough estimate of the number of our -+ * packets that might be in the network at the earliest departure time for the -+ * next skb scheduled: -+ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw -+ * If we're increasing inflight, then we want to know if the transmit of the -+ * EDT skb will push inflight above the target, so inflight_at_edt includes -+ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, -+ * then estimate if inflight will sink too low just before the EDT transmit. -+ */ -+static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 now_ns, edt_ns, interval_us; -+ u32 interval_delivered, inflight_at_edt; -+ -+ now_ns = tp->tcp_clock_cache; -+ edt_ns = max(tp->tcp_wstamp_ns, now_ns); -+ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); -+ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; -+ inflight_at_edt = inflight_now; -+ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ -+ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ -+ if (interval_delivered >= inflight_at_edt) -+ return 0; -+ return inflight_at_edt - interval_delivered; -+} -+ -+/* Find the cwnd increment based on estimate of ack aggregation */ -+static u32 bbr_ack_aggregation_cwnd(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 max_aggr_cwnd, aggr_cwnd = 0; -+ -+ if (bbr->params.extra_acked_gain && -+ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { -+ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) -+ / BW_UNIT; -+ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) -+ >> BBR_SCALE; -+ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); -+ } -+ -+ return aggr_cwnd; -+} -+ -+/* Returns the cwnd for PROBE_RTT mode. */ -+static u32 bbr_probe_rtt_cwnd(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->params.probe_rtt_cwnd_gain == 0) -+ return bbr->params.cwnd_min_target; -+ return max_t(u32, bbr->params.cwnd_min_target, -+ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); -+} -+ -+/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss -+ * has drawn us down below target), or snap down to target if we're above it. -+ */ -+static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, -+ u32 acked, u32 bw, int gain, u32 cwnd, -+ struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; -+ -+ if (!acked) -+ goto done; /* no packet fully ACKed; just apply caps */ -+ -+ target_cwnd = bbr_bdp(sk, bw, gain); -+ -+ /* Increment the cwnd to account for excess ACKed data that seems -+ * due to aggregation (of data and/or ACKs) visible in the ACK stream. -+ */ -+ target_cwnd += bbr_ack_aggregation_cwnd(sk); -+ target_cwnd = bbr_quantization_budget(sk, target_cwnd); -+ -+ /* If we're below target cwnd, slow start cwnd toward target cwnd. */ -+ bbr->debug.target_cwnd = target_cwnd; -+ -+ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ -+ bbr->try_fast_path = 0; -+ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ -+ cwnd += acked; -+ if (cwnd >= target_cwnd) { -+ cwnd = target_cwnd; -+ bbr->try_fast_path = 1; -+ } -+ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { -+ cwnd += acked; -+ } else { -+ bbr->try_fast_path = 1; -+ } -+ -+ /* When growing cwnd, don't grow beyond twice what we just probed. */ -+ if (bbr->params.usage_based_cwnd) { -+ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); -+ cwnd = min(cwnd, max_probe); -+ } -+ -+ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); -+done: -+ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ -+ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ -+ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); -+ -+ ctx->target_cwnd = target_cwnd; -+ ctx->log = (tp->snd_cwnd != prev_cwnd); -+} -+ -+/* See if we have reached next round trip */ -+static void bbr_update_round_start(struct sock *sk, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->round_start = 0; -+ -+ /* See if we've reached the next RTT */ -+ if (rs->interval_us > 0 && -+ !before(rs->prior_delivered, bbr->next_rtt_delivered)) { -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr->round_start = 1; -+ } -+} -+ -+/* Calculate the bandwidth based on how fast packets are delivered */ -+static void bbr_calculate_bw_sample(struct sock *sk, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw = 0; -+ -+ /* Divide delivered by the interval to find a (lower bound) bottleneck -+ * bandwidth sample. Delivered is in packets and interval_us in uS and -+ * ratio will be <<1 for most connections. So delivered is first scaled. -+ * Round up to allow growth at low rates, even with integer division. -+ */ -+ if (rs->interval_us > 0) { -+ if (WARN_ONCE(rs->delivered < 0, -+ "negative delivered: %d interval_us: %ld\n", -+ rs->delivered, rs->interval_us)) -+ return; -+ -+ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); -+ } -+ -+ ctx->sample_bw = bw; -+ bbr->debug.rs_bw = bw; -+} -+ -+/* Estimates the windowed max degree of ack aggregation. -+ * This is used to provision extra in-flight data to keep sending during -+ * inter-ACK silences. -+ * -+ * Degree of ack aggregation is estimated as extra data acked beyond expected. -+ * -+ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" -+ * cwnd += max_extra_acked -+ * -+ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). -+ * Max filter is an approximate sliding window of 5-10 (packet timed) round -+ * trips for non-startup phase, and 1-2 round trips for startup. -+ */ -+static void bbr_update_ack_aggregation(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ u32 epoch_us, expected_acked, extra_acked; -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct tcp_sock *tp = tcp_sk(sk); -+ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; -+ -+ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || -+ rs->delivered < 0 || rs->interval_us <= 0) -+ return; -+ -+ if (bbr->round_start) { -+ bbr->extra_acked_win_rtts = min(0x1F, -+ bbr->extra_acked_win_rtts + 1); -+ if (bbr->params.extra_acked_in_startup && -+ !bbr_full_bw_reached(sk)) -+ extra_acked_win_rtts_thresh = 1; -+ if (bbr->extra_acked_win_rtts >= -+ extra_acked_win_rtts_thresh) { -+ bbr->extra_acked_win_rtts = 0; -+ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? -+ 0 : 1; -+ bbr->extra_acked[bbr->extra_acked_win_idx] = 0; -+ } -+ } -+ -+ /* Compute how many packets we expected to be delivered over epoch. */ -+ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, -+ bbr->ack_epoch_mstamp); -+ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; -+ -+ /* Reset the aggregation epoch if ACK rate is below expected rate or -+ * significantly large no. of ack received since epoch (potentially -+ * quite old epoch). -+ */ -+ if (bbr->ack_epoch_acked <= expected_acked || -+ (bbr->ack_epoch_acked + rs->acked_sacked >= -+ bbr_ack_epoch_acked_reset_thresh)) { -+ bbr->ack_epoch_acked = 0; -+ bbr->ack_epoch_mstamp = tp->delivered_mstamp; -+ expected_acked = 0; -+ } -+ -+ /* Compute excess data delivered, beyond what was expected. */ -+ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, -+ bbr->ack_epoch_acked + rs->acked_sacked); -+ extra_acked = bbr->ack_epoch_acked - expected_acked; -+ extra_acked = min(extra_acked, tp->snd_cwnd); -+ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) -+ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; -+} -+ -+/* Estimate when the pipe is full, using the change in delivery rate: BBR -+ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by -+ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited -+ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the -+ * higher rwin, 3: we get higher delivery rate samples. Or transient -+ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar -+ * design goal, but uses delay and inter-ACK spacing instead of bandwidth. -+ */ -+static void bbr_check_full_bw_reached(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 bw_thresh; -+ -+ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) -+ return; -+ -+ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; -+ if (bbr_max_bw(sk) >= bw_thresh) { -+ bbr->full_bw = bbr_max_bw(sk); -+ bbr->full_bw_cnt = 0; -+ return; -+ } -+ ++bbr->full_bw_cnt; -+ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; -+} -+ -+/* If pipe is probably full, drain the queue and then enter steady-state. */ -+static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { -+ bbr->mode = BBR_DRAIN; /* drain queue we created */ -+ tcp_sk(sk)->snd_ssthresh = -+ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); -+ bbr2_reset_congestion_signals(sk); -+ } /* fall through to check if in-flight is already small: */ -+ if (bbr->mode == BBR_DRAIN && -+ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= -+ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) -+ return true; /* exiting DRAIN now */ -+ return false; -+} -+ -+static void bbr_check_probe_rtt_done(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (!(bbr->probe_rtt_done_stamp && -+ after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) -+ return; -+ -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr2_exit_probe_rtt(sk); -+} -+ -+/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and -+ * periodically drain the bottleneck queue, to converge to measure the true -+ * min_rtt (unloaded propagation delay). This allows the flows to keep queues -+ * small (reducing queuing delay and packet loss) and achieve fairness among -+ * BBR flows. -+ * -+ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, -+ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. -+ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed -+ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and -+ * re-enter the previous mode. BBR uses 200ms to approximately bound the -+ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). -+ * -+ * Note that flows need only pay 2% if they are busy sending over the last 10 -+ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have -+ * natural silences or low-rate periods within 10 seconds where the rate is low -+ * enough for long enough to drain its queue in the bottleneck. We pick up -+ * these min RTT measurements opportunistically with our min_rtt filter. :-) -+ */ -+static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool probe_rtt_expired, min_rtt_expired; -+ u32 expire; -+ -+ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ -+ expire = bbr->probe_rtt_min_stamp + -+ msecs_to_jiffies(bbr->params.probe_rtt_win_ms); -+ probe_rtt_expired = after(tcp_jiffies32, expire); -+ if (rs->rtt_us >= 0 && -+ (rs->rtt_us <= bbr->probe_rtt_min_us || -+ (probe_rtt_expired && !rs->is_ack_delayed))) { -+ bbr->probe_rtt_min_us = rs->rtt_us; -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; -+ } -+ /* Track min RTT seen in the min_rtt_win_sec filter window: */ -+ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; -+ min_rtt_expired = after(tcp_jiffies32, expire); -+ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || -+ min_rtt_expired) { -+ bbr->min_rtt_us = bbr->probe_rtt_min_us; -+ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; -+ } -+ -+ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && -+ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { -+ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ -+ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ -+ bbr->probe_rtt_done_stamp = 0; -+ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; -+ bbr->next_rtt_delivered = tp->delivered; -+ } -+ -+ if (bbr->mode == BBR_PROBE_RTT) { -+ /* Ignore low rate samples during this mode. */ -+ tp->app_limited = -+ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; -+ /* Maintain min packets in flight for max(200 ms, 1 round). */ -+ if (!bbr->probe_rtt_done_stamp && -+ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { -+ bbr->probe_rtt_done_stamp = tcp_jiffies32 + -+ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); -+ bbr->probe_rtt_round_done = 0; -+ bbr->next_rtt_delivered = tp->delivered; -+ } else if (bbr->probe_rtt_done_stamp) { -+ if (bbr->round_start) -+ bbr->probe_rtt_round_done = 1; -+ if (bbr->probe_rtt_round_done) -+ bbr_check_probe_rtt_done(sk); -+ } -+ } -+ /* Restart after idle ends only once we process a new S/ACK for data */ -+ if (rs->delivered > 0) -+ bbr->idle_restart = 0; -+} -+ -+static void bbr_update_gains(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ bbr->pacing_gain = bbr->params.high_gain; -+ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; -+ break; -+ case BBR_DRAIN: -+ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ -+ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ -+ break; -+ case BBR_PROBE_BW: -+ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; -+ bbr->cwnd_gain = bbr->params.cwnd_gain; -+ break; -+ case BBR_PROBE_RTT: -+ bbr->pacing_gain = BBR_UNIT; -+ bbr->cwnd_gain = BBR_UNIT; -+ break; -+ default: -+ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); -+ break; -+ } -+} -+ -+static void bbr_init(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ int i; -+ -+ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); -+ -+ bbr->initialized = 1; -+ bbr->params.high_gain = min(0x7FF, bbr_high_gain); -+ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); -+ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); -+ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); -+ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); -+ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); -+ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); -+ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); -+ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); -+ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); -+ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); -+ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); -+ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; -+ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; -+ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; -+ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); -+ bbr->params.probe_rtt_win_ms = -+ min(0x3FFFU, -+ min_t(u32, bbr_probe_rtt_win_ms, -+ bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); -+ for (i = 0; i < CYCLE_LEN; i++) -+ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); -+ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; -+ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); -+ -+ bbr->debug.snd_isn = tp->snd_una; -+ bbr->debug.target_cwnd = 0; -+ bbr->debug.undo = 0; -+ -+ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); -+ bbr->prior_cwnd = tp->prior_cwnd; -+ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; -+ bbr->next_rtt_delivered = 0; -+ bbr->prev_ca_state = TCP_CA_Open; -+ bbr->packet_conservation = 0; -+ -+ bbr->probe_rtt_done_stamp = 0; -+ bbr->probe_rtt_round_done = 0; -+ bbr->probe_rtt_min_us = tcp_min_rtt(tp); -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; -+ bbr->min_rtt_us = tcp_min_rtt(tp); -+ bbr->min_rtt_stamp = tcp_jiffies32; -+ -+ bbr->has_seen_rtt = 0; -+ bbr_init_pacing_rate_from_rtt(sk); -+ -+ bbr->round_start = 0; -+ bbr->idle_restart = 0; -+ bbr->full_bw_reached = 0; -+ bbr->full_bw = 0; -+ bbr->full_bw_cnt = 0; -+ bbr->cycle_mstamp = 0; -+ bbr->cycle_idx = 0; -+ bbr->mode = BBR_STARTUP; -+ bbr->debug.rs_bw = 0; -+ -+ bbr->ack_epoch_mstamp = tp->tcp_mstamp; -+ bbr->ack_epoch_acked = 0; -+ bbr->extra_acked_win_rtts = 0; -+ bbr->extra_acked_win_idx = 0; -+ bbr->extra_acked[0] = 0; -+ bbr->extra_acked[1] = 0; -+ -+ bbr->ce_state = 0; -+ bbr->prior_rcv_nxt = tp->rcv_nxt; -+ bbr->try_fast_path = 0; -+ -+ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); -+} -+ -+static u32 bbr_sndbuf_expand(struct sock *sk) -+{ -+ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ -+ return 3; -+} -+ -+/* __________________________________________________________________________ -+ * -+ * Functions new to BBR v2 ("bbr") congestion control are below here. -+ * __________________________________________________________________________ -+ */ -+ -+/* Incorporate a new bw sample into the current window of our max filter. */ -+static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); -+} -+ -+/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ -+static void bbr2_advance_bw_hi_filter(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (!bbr->bw_hi[1]) -+ return; /* no samples in this window; remember old window */ -+ bbr->bw_hi[0] = bbr->bw_hi[1]; -+ bbr->bw_hi[1] = 0; -+} -+ -+/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ -+static u32 bbr2_target_inflight(struct sock *sk) -+{ -+ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); -+ -+ return min(bdp, tcp_sk(sk)->snd_cwnd); -+} -+ -+static bool bbr2_is_probing_bandwidth(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return (bbr->mode == BBR_STARTUP) || -+ (bbr->mode == BBR_PROBE_BW && -+ (bbr->cycle_idx == BBR_BW_PROBE_REFILL || -+ bbr->cycle_idx == BBR_BW_PROBE_UP)); -+} -+ -+/* Has the given amount of time elapsed since we marked the phase start? */ -+static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) -+{ -+ const struct tcp_sock *tp = tcp_sk(sk); -+ const struct bbr *bbr = inet_csk_ca(sk); -+ -+ return tcp_stamp_us_delta(tp->tcp_mstamp, -+ bbr->cycle_mstamp + interval_us) > 0; -+} -+ -+static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->full_bw_reached = 1; -+ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); -+} -+ -+/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ -+static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || -+ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) -+ return; -+ -+ if (ce_ratio >= bbr->params.ecn_thresh) -+ bbr->startup_ecn_rounds++; -+ else -+ bbr->startup_ecn_rounds = 0; -+ -+ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { -+ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ -+ bbr2_handle_queue_too_high_in_startup(sk); -+ return; -+ } -+} -+ -+static int bbr2_update_ecn_alpha(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ s32 delivered, delivered_ce; -+ u64 alpha, ce_ratio; -+ u32 gain; -+ -+ if (bbr->params.ecn_factor == 0) -+ return -1; -+ -+ delivered = tp->delivered - bbr->alpha_last_delivered; -+ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; -+ -+ if (delivered == 0 || /* avoid divide by zero */ -+ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ -+ return -1; -+ -+ /* See if we should use ECN sender logic for this connection. */ -+ if (!bbr->ecn_eligible && bbr_ecn_enable && -+ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || -+ !bbr->params.ecn_max_rtt_us)) -+ bbr->ecn_eligible = 1; -+ -+ ce_ratio = (u64)delivered_ce << BBR_SCALE; -+ do_div(ce_ratio, delivered); -+ gain = bbr->params.ecn_alpha_gain; -+ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; -+ alpha += (gain * ce_ratio) >> BBR_SCALE; -+ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); -+ -+ bbr->alpha_last_delivered = tp->delivered; -+ bbr->alpha_last_delivered_ce = tp->delivered_ce; -+ -+ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); -+ return (int)ce_ratio; -+} -+ -+/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ -+static void bbr2_raise_inflight_hi_slope(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 growth_this_round, cnt; -+ -+ /* Calculate "slope": packets S/Acked per inflight_hi increment. */ -+ growth_this_round = 1 << bbr->bw_probe_up_rounds; -+ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); -+ cnt = tp->snd_cwnd / growth_this_round; -+ cnt = max(cnt, 1U); -+ bbr->bw_probe_up_cnt = cnt; -+ bbr->debug.event = 'G'; /* Grow inflight_hi slope */ -+} -+ -+/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ -+static void bbr2_probe_inflight_hi_upward(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 delta; -+ -+ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { -+ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ -+ return; /* not fully using inflight_hi, so don't grow it */ -+ } -+ -+ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ -+ bbr->bw_probe_up_acks += rs->acked_sacked; -+ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { -+ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; -+ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; -+ bbr->inflight_hi += delta; -+ bbr->debug.event = 'I'; /* Increment inflight_hi */ -+ } -+ -+ if (bbr->round_start) -+ bbr2_raise_inflight_hi_slope(sk); -+} -+ -+/* Does loss/ECN rate for this sample say inflight is "too high"? -+ * This is used by both the bbr_check_loss_too_high_in_startup() function, -+ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which -+ * uses it to notice when loss/ECN rates suggest inflight is too high. -+ */ -+static bool bbr2_is_inflight_too_high(const struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ u32 loss_thresh, ecn_thresh; -+ -+ if (rs->lost > 0 && rs->tx_in_flight) { -+ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> -+ BBR_SCALE; -+ if (rs->lost > loss_thresh) -+ return true; -+ } -+ -+ if (rs->delivered_ce > 0 && rs->delivered > 0 && -+ bbr->ecn_eligible && bbr->params.ecn_thresh) { -+ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> -+ BBR_SCALE; -+ if (rs->delivered_ce >= ecn_thresh) -+ return true; -+ } -+ -+ return false; -+} -+ -+/* Calculate the tx_in_flight level that corresponded to excessive loss. -+ * We find "lost_prefix" segs of the skb where loss rate went too high, -+ * by solving for "lost_prefix" in the following equation: -+ * lost / inflight >= loss_thresh -+ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh -+ * Then we take that equation, convert it to fixed point, and -+ * round up to the nearest packet. -+ */ -+static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, -+ const struct rate_sample *rs, -+ const struct sk_buff *skb) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ u32 loss_thresh = bbr->params.loss_thresh; -+ u32 pcount, divisor, inflight_hi; -+ s32 inflight_prev, lost_prev; -+ u64 loss_budget, lost_prefix; -+ -+ pcount = tcp_skb_pcount(skb); -+ -+ /* How much data was in flight before this skb? */ -+ inflight_prev = rs->tx_in_flight - pcount; -+ if (WARN_ONCE(inflight_prev < 0, -+ "tx_in_flight: %u pcount: %u reneg: %u", -+ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) -+ return ~0U; -+ -+ /* How much inflight data was marked lost before this skb? */ -+ lost_prev = rs->lost - pcount; -+ if (WARN_ON_ONCE(lost_prev < 0)) -+ return ~0U; -+ -+ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ -+ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; -+ loss_budget >>= BBR_SCALE; -+ if (lost_prev >= loss_budget) { -+ lost_prefix = 0; /* previous losses crossed loss_thresh */ -+ } else { -+ lost_prefix = loss_budget - lost_prev; -+ lost_prefix <<= BBR_SCALE; -+ divisor = BBR_UNIT - loss_thresh; -+ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ -+ return ~0U; -+ do_div(lost_prefix, divisor); -+ } -+ -+ inflight_hi = inflight_prev + lost_prefix; -+ return inflight_hi; -+} -+ -+/* If loss/ECN rates during probing indicated we may have overfilled a -+ * buffer, return an operating point that tries to leave unutilized headroom in -+ * the path for other flows, for fairness convergence and lower RTTs and loss. -+ */ -+static u32 bbr2_inflight_with_headroom(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 headroom, headroom_fraction; -+ -+ if (bbr->inflight_hi == ~0U) -+ return ~0U; -+ -+ headroom_fraction = bbr->params.inflight_headroom; -+ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; -+ headroom = max(headroom, 1U); -+ return max_t(s32, bbr->inflight_hi - headroom, -+ bbr->params.cwnd_min_target); -+} -+ -+/* Bound cwnd to a sensible level, based on our current probing state -+ * machine phase and model of a good inflight level (inflight_lo, inflight_hi). -+ */ -+static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 cap; -+ -+ /* tcp_rcv_synsent_state_process() currently calls tcp_ack() -+ * and thus cong_control() without first initializing us(!). -+ */ -+ if (!bbr->initialized) -+ return; -+ -+ cap = ~0U; -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { -+ /* Probe to see if more packets fit in the path. */ -+ cap = bbr->inflight_hi; -+ } else { -+ if (bbr->mode == BBR_PROBE_RTT || -+ (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) -+ cap = bbr2_inflight_with_headroom(sk); -+ } -+ /* Adapt to any loss/ECN since our last bw probe. */ -+ cap = min(cap, bbr->inflight_lo); -+ -+ cap = max_t(u32, cap, bbr->params.cwnd_min_target); -+ tp->snd_cwnd = min(cap, tp->snd_cwnd); -+} -+ -+/* Estimate a short-term lower bound on the capacity available now, based -+ * on measurements of the current delivery process and recent history. When we -+ * are seeing loss/ECN at times when we are not probing bw, then conservatively -+ * move toward flow balance by multiplicatively cutting our short-term -+ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a -+ * multiplicative decrease in order to converge to a lower capacity in time -+ * logarithmic in the magnitude of the decrease. -+ * -+ * However, we do not cut our short-term estimates lower than the current rate -+ * and volume of delivered data from this round trip, since from the current -+ * delivery process we can estimate the measured capacity available now. -+ * -+ * Anything faster than that approach would knowingly risk high loss, which can -+ * cause low bw for Reno/CUBIC and high loss recovery latency for -+ * request/response flows using any congestion control. -+ */ -+static void bbr2_adapt_lower_bounds(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 ecn_cut, ecn_inflight_lo, beta; -+ -+ /* We only use lower-bound estimates when not probing bw. -+ * When probing we need to push inflight higher to probe bw. -+ */ -+ if (bbr2_is_probing_bandwidth(sk)) -+ return; -+ -+ /* ECN response. */ -+ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { -+ /* Reduce inflight to (1 - alpha*ecn_factor). */ -+ ecn_cut = (BBR_UNIT - -+ ((bbr->ecn_alpha * bbr->params.ecn_factor) >> -+ BBR_SCALE)); -+ if (bbr->inflight_lo == ~0U) -+ bbr->inflight_lo = tp->snd_cwnd; -+ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; -+ } else { -+ ecn_inflight_lo = ~0U; -+ } -+ -+ /* Loss response. */ -+ if (bbr->loss_in_round) { -+ /* Reduce bw and inflight to (1 - beta). */ -+ if (bbr->bw_lo == ~0U) -+ bbr->bw_lo = bbr_max_bw(sk); -+ if (bbr->inflight_lo == ~0U) -+ bbr->inflight_lo = tp->snd_cwnd; -+ beta = bbr->params.beta; -+ bbr->bw_lo = -+ max_t(u32, bbr->bw_latest, -+ (u64)bbr->bw_lo * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ bbr->inflight_lo = -+ max_t(u32, bbr->inflight_latest, -+ (u64)bbr->inflight_lo * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ } -+ -+ /* Adjust to the lower of the levels implied by loss or ECN. */ -+ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); -+} -+ -+/* Reset any short-term lower-bound adaptation to congestion, so that we can -+ * push our inflight up. -+ */ -+static void bbr2_reset_lower_bounds(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->bw_lo = ~0U; -+ bbr->inflight_lo = ~0U; -+} -+ -+/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state -+ * machine phase where we adapt our lower bound based on congestion signals. -+ */ -+static void bbr2_reset_congestion_signals(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->loss_in_round = 0; -+ bbr->ecn_in_round = 0; -+ bbr->loss_in_cycle = 0; -+ bbr->ecn_in_cycle = 0; -+ bbr->bw_latest = 0; -+ bbr->inflight_latest = 0; -+} -+ -+/* Update (most of) our congestion signals: track the recent rate and volume of -+ * delivered data, presence of loss, and EWMA degree of ECN marking. -+ */ -+static void bbr2_update_congestion_signals( -+ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw; -+ -+ bbr->loss_round_start = 0; -+ if (rs->interval_us <= 0 || !rs->acked_sacked) -+ return; /* Not a valid observation */ -+ bw = ctx->sample_bw; -+ -+ if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) -+ bbr2_take_bw_hi_sample(sk, bw); -+ -+ bbr->loss_in_round |= (rs->losses > 0); -+ -+ /* Update rate and volume of delivered data from latest round trip: */ -+ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); -+ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); -+ -+ if (before(rs->prior_delivered, bbr->loss_round_delivered)) -+ return; /* skip the per-round-trip updates */ -+ /* Now do per-round-trip updates. */ -+ bbr->loss_round_delivered = tp->delivered; /* mark round trip */ -+ bbr->loss_round_start = 1; -+ bbr2_adapt_lower_bounds(sk); -+ -+ /* Update windowed "latest" (single-round-trip) filters. */ -+ bbr->loss_in_round = 0; -+ bbr->ecn_in_round = 0; -+ bbr->bw_latest = ctx->sample_bw; -+ bbr->inflight_latest = rs->delivered; -+} -+ -+/* Bandwidth probing can cause loss. To help coexistence with loss-based -+ * congestion control we spread out our probing in a Reno-conscious way. Due to -+ * the shape of the Reno sawtooth, the time required between loss epochs for an -+ * idealized Reno flow is a number of round trips that is the BDP of that -+ * flow. We count packet-timed round trips directly, since measured RTT can -+ * vary widely, and Reno is driven by packet-timed round trips. -+ */ -+static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 inflight, rounds, reno_gain, reno_rounds; -+ -+ /* Random loss can shave some small percentage off of our inflight -+ * in each round. To survive this, flows need robust periodic probes. -+ */ -+ rounds = bbr->params.bw_probe_max_rounds; -+ -+ reno_gain = bbr->params.bw_probe_reno_gain; -+ if (reno_gain) { -+ inflight = bbr2_target_inflight(sk); -+ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; -+ rounds = min(rounds, reno_rounds); -+ } -+ return bbr->rounds_since_probe >= rounds; -+} -+ -+/* How long do we want to wait before probing for bandwidth (and risking -+ * loss)? We randomize the wait, for better mixing and fairness convergence. -+ * -+ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. -+ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, -+ * (eg 4K video to a broadband user): -+ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets -+ * -+ * We bound the BBR-native inter-bw-probe wall clock time to be: -+ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time -+ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must -+ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs -+ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable -+ * amount of time to discover unutilized bw on human-scale interactive -+ * time-scales (e.g. perhaps traffic from a web page download that we -+ * were competing with is now complete). -+ */ -+static void bbr2_pick_probe_wait(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ /* Decide the random round-trip bound for wait until probe: */ -+ bbr->rounds_since_probe = -+ prandom_u32_max(bbr->params.bw_probe_rand_rounds); -+ /* Decide the random wall clock bound for wait until probe: */ -+ bbr->probe_wait_us = bbr->params.bw_probe_base_us + -+ prandom_u32_max(bbr->params.bw_probe_rand_us); -+} -+ -+static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->cycle_idx = cycle_idx; -+ /* New phase, so need to update cwnd and pacing rate. */ -+ bbr->try_fast_path = 0; -+} -+ -+/* Send at estimated bw to fill the pipe, but not queue. We need this phase -+ * before PROBE_UP, because as soon as we send faster than the available bw -+ * we will start building a queue, and if the buffer is shallow we can cause -+ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and -+ * inflight_hi estimates will underestimate. -+ */ -+static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr2_reset_lower_bounds(sk); -+ if (bbr->inflight_hi != ~0U) -+ bbr->inflight_hi += bbr->params.refill_add_inc; -+ bbr->bw_probe_up_rounds = bw_probe_up_rounds; -+ bbr->bw_probe_up_acks = 0; -+ bbr->stopped_risky_probe = 0; -+ bbr->ack_phase = BBR_ACKS_REFILLING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); -+} -+ -+/* Now probe max deliverable data rate and volume. */ -+static void bbr2_start_bw_probe_up(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->ack_phase = BBR_ACKS_PROBE_STARTING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr->cycle_mstamp = tp->tcp_mstamp; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); -+ bbr2_raise_inflight_hi_slope(sk); -+} -+ -+/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall -+ * clock time at which to probe beyond an inflight that we think to be -+ * safe. This will knowingly risk packet loss, so we want to do this rarely, to -+ * keep packet loss rates low. Also start a round-trip counter, to probe faster -+ * if we estimate a Reno flow at our BDP would probe faster. -+ */ -+static void bbr2_start_bw_probe_down(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr2_reset_congestion_signals(sk); -+ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ -+ bbr2_pick_probe_wait(sk); -+ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ -+ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); -+} -+ -+/* Cruise: maintain what we estimate to be a neutral, conservative -+ * operating point, without attempting to probe up for bandwidth or down for -+ * RTT, and only reducing inflight in response to loss/ECN signals. -+ */ -+static void bbr2_start_bw_probe_cruise(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->inflight_lo != ~0U) -+ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); -+ -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); -+} -+ -+/* Loss and/or ECN rate is too high while probing. -+ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. -+ */ -+static void bbr2_handle_inflight_too_high(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ const u32 beta = bbr->params.beta; -+ -+ bbr->prev_probe_too_high = 1; -+ bbr->bw_probe_samples = 0; /* only react once per probe */ -+ bbr->debug.event = 'L'; /* Loss/ECN too high */ -+ /* If we are app-limited then we are not robustly -+ * probing the max volume of inflight data we think -+ * might be safe (analogous to how app-limited bw -+ * samples are not known to be robustly probing bw). -+ */ -+ if (!rs->is_app_limited) -+ bbr->inflight_hi = max_t(u32, rs->tx_in_flight, -+ (u64)bbr2_target_inflight(sk) * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) -+ bbr2_start_bw_probe_down(sk); -+} -+ -+/* If we're seeing bw and loss samples reflecting our bw probing, adapt -+ * using the signals we see. If loss or ECN mark rate gets too high, then adapt -+ * inflight_hi downward. If we're able to push inflight higher without such -+ * signals, push higher: adapt inflight_hi upward. -+ */ -+static bool bbr2_adapt_upper_bounds(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ /* Track when we'll see bw/loss samples resulting from our bw probes. */ -+ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) -+ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; -+ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { -+ /* End of samples from bw probing phase. */ -+ bbr->bw_probe_samples = 0; -+ bbr->ack_phase = BBR_ACKS_INIT; -+ /* At this point in the cycle, our current bw sample is also -+ * our best recent chance at finding the highest available bw -+ * for this flow. So now is the best time to forget the bw -+ * samples from the previous cycle, by advancing the window. -+ */ -+ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) -+ bbr2_advance_bw_hi_filter(sk); -+ /* If we had an inflight_hi, then probed and pushed inflight all -+ * the way up to hit that inflight_hi without seeing any -+ * high loss/ECN in all the resulting ACKs from that probing, -+ * then probe up again, this time letting inflight persist at -+ * inflight_hi for a round trip, then accelerating beyond. -+ */ -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { -+ bbr->debug.event = 'R'; /* reprobe */ -+ bbr2_start_bw_probe_refill(sk, 0); -+ return true; /* yes, decided state transition */ -+ } -+ } -+ -+ if (bbr2_is_inflight_too_high(sk, rs)) { -+ if (bbr->bw_probe_samples) /* sample is from bw probing? */ -+ bbr2_handle_inflight_too_high(sk, rs); -+ } else { -+ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ -+ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ -+ return false; -+ -+ /* To be resilient to random loss, we must raise inflight_hi -+ * if we observe in any phase that a higher level is safe. -+ */ -+ if (rs->tx_in_flight > bbr->inflight_hi) { -+ bbr->inflight_hi = rs->tx_in_flight; -+ bbr->debug.event = 'U'; /* raise up inflight_hi */ -+ } -+ -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx == BBR_BW_PROBE_UP) -+ bbr2_probe_inflight_hi_upward(sk, rs); -+ } -+ -+ return false; -+} -+ -+/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ -+static bool bbr2_check_time_to_probe_bw(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 n; -+ -+ /* If we seem to be at an operating point where we are not seeing loss -+ * but we are seeing ECN marks, then when the ECN marks cease we reprobe -+ * quickly (in case a burst of cross-traffic has ceased and freed up bw, -+ * or in case we are sharing with multiplicatively probing traffic). -+ */ -+ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && -+ bbr->ecn_in_cycle && !bbr->loss_in_cycle && -+ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { -+ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ -+ /* Calculate n so that when bbr2_raise_inflight_hi_slope() -+ * computes growth_this_round as 2^n it will be roughly the -+ * desired volume of data (inflight_hi*ecn_reprobe_gain). -+ */ -+ n = ilog2((((u64)bbr->inflight_hi * -+ bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); -+ bbr2_start_bw_probe_refill(sk, n); -+ return true; -+ } -+ -+ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || -+ bbr2_is_reno_coexistence_probe_time(sk)) { -+ bbr2_start_bw_probe_refill(sk, 0); -+ return true; -+ } -+ return false; -+} -+ -+/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ -+static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool is_under_bdp, is_long_enough; -+ -+ /* Always need to pull inflight down to leave headroom in queue. */ -+ if (inflight > bbr2_inflight_with_headroom(sk)) -+ return false; -+ -+ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); -+ if (bbr->params.drain_to_target) -+ return is_under_bdp; -+ -+ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); -+ return is_under_bdp || is_long_enough; -+} -+ -+/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ -+static void bbr2_update_cycle_phase(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool is_risky = false, is_queuing = false; -+ u32 inflight, bw; -+ -+ if (!bbr_full_bw_reached(sk)) -+ return; -+ -+ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ -+ if (bbr2_adapt_upper_bounds(sk, rs)) -+ return; /* already decided state transition */ -+ -+ if (bbr->mode != BBR_PROBE_BW) -+ return; -+ -+ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); -+ bw = bbr_max_bw(sk); -+ -+ switch (bbr->cycle_idx) { -+ /* First we spend most of our time cruising with a pacing_gain of 1.0, -+ * which paces at the estimated bw, to try to fully use the pipe -+ * without building queue. If we encounter loss/ECN marks, we adapt -+ * by slowing down. -+ */ -+ case BBR_BW_PROBE_CRUISE: -+ if (bbr2_check_time_to_probe_bw(sk)) -+ return; /* already decided state transition */ -+ break; -+ -+ /* After cruising, when it's time to probe, we first "refill": we send -+ * at the estimated bw to fill the pipe, before probing higher and -+ * knowingly risking overflowing the bottleneck buffer (causing loss). -+ */ -+ case BBR_BW_PROBE_REFILL: -+ if (bbr->round_start) { -+ /* After one full round trip of sending in REFILL, we -+ * start to see bw samples reflecting our REFILL, which -+ * may be putting too much data in flight. -+ */ -+ bbr->bw_probe_samples = 1; -+ bbr2_start_bw_probe_up(sk); -+ } -+ break; -+ -+ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to -+ * probe for bw. If we have not seen loss/ECN, we try to raise inflight -+ * to at least pacing_gain*BDP; note that this may take more than -+ * min_rtt if min_rtt is small (e.g. on a LAN). -+ * -+ * We terminate PROBE_UP bandwidth probing upon any of the following: -+ * -+ * (1) We've pushed inflight up to hit the inflight_hi target set in the -+ * most recent previous bw probe phase. Thus we want to start -+ * draining the queue immediately because it's very likely the most -+ * recently sent packets will fill the queue and cause drops. -+ * (checked here) -+ * (2) We have probed for at least 1*min_rtt_us, and the -+ * estimated queue is high enough (inflight > 1.25 * estimated_bdp). -+ * (checked here) -+ * (3) Loss filter says loss rate is "too high". -+ * (checked in bbr_is_inflight_too_high()) -+ * (4) ECN filter says ECN mark rate is "too high". -+ * (checked in bbr_is_inflight_too_high()) -+ */ -+ case BBR_BW_PROBE_UP: -+ if (bbr->prev_probe_too_high && -+ inflight >= bbr->inflight_hi) { -+ bbr->stopped_risky_probe = 1; -+ is_risky = true; -+ bbr->debug.event = 'D'; /* D for danger */ -+ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && -+ inflight >= -+ bbr_inflight(sk, bw, -+ bbr->params.bw_probe_pif_gain)) { -+ is_queuing = true; -+ bbr->debug.event = 'Q'; /* building Queue */ -+ } -+ if (is_risky || is_queuing) { -+ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ -+ bbr2_start_bw_probe_down(sk); /* restart w/ down */ -+ } -+ break; -+ -+ /* After probing in PROBE_UP, we have usually accumulated some data in -+ * the bottleneck buffer (if bw probing didn't find more bw). We next -+ * enter PROBE_DOWN to try to drain any excess data from the queue. To -+ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until -+ * our inflight is less then that target cruising point, which is the -+ * minimum of (a) the amount needed to leave headroom, and (b) the -+ * estimated BDP. Once inflight falls to match the target, we estimate -+ * the queue is drained; persisting would underutilize the pipe. -+ */ -+ case BBR_BW_PROBE_DOWN: -+ if (bbr2_check_time_to_probe_bw(sk)) -+ return; /* already decided state transition */ -+ if (bbr2_check_time_to_cruise(sk, inflight, bw)) -+ bbr2_start_bw_probe_cruise(sk); -+ break; -+ -+ default: -+ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); -+ } -+} -+ -+/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ -+static void bbr2_exit_probe_rtt(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr2_reset_lower_bounds(sk); -+ if (bbr_full_bw_reached(sk)) { -+ bbr->mode = BBR_PROBE_BW; -+ /* Raising inflight after PROBE_RTT may cause loss, so reset -+ * the PROBE_BW clock and schedule the next bandwidth probe for -+ * a friendly and randomized future point in time. -+ */ -+ bbr2_start_bw_probe_down(sk); -+ /* Since we are exiting PROBE_RTT, we know inflight is -+ * below our estimated BDP, so it is reasonable to cruise. -+ */ -+ bbr2_start_bw_probe_cruise(sk); -+ } else { -+ bbr->mode = BBR_STARTUP; -+ } -+} -+ -+/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until -+ * the end of the round in recovery to get a good estimate of how many packets -+ * have been lost, and how many we need to drain with a low pacing rate. -+ */ -+static void bbr2_check_loss_too_high_in_startup(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr_full_bw_reached(sk)) -+ return; -+ -+ /* For STARTUP exit, check the loss rate at the end of each round trip -+ * of Recovery episodes in STARTUP. We check the loss rate at the end -+ * of the round trip to filter out noisy/low loss and have a better -+ * sense of inflight (extent of loss), so we can drain more accurately. -+ */ -+ if (rs->losses && bbr->loss_events_in_round < 0xf) -+ bbr->loss_events_in_round++; /* update saturating counter */ -+ if (bbr->params.full_loss_cnt && bbr->loss_round_start && -+ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && -+ bbr->loss_events_in_round >= bbr->params.full_loss_cnt && -+ bbr2_is_inflight_too_high(sk, rs)) { -+ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ -+ bbr2_handle_queue_too_high_in_startup(sk); -+ return; -+ } -+ if (bbr->loss_round_start) -+ bbr->loss_events_in_round = 0; -+} -+ -+/* If we are done draining, advance into steady state operation in PROBE_BW. */ -+static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr_check_drain(sk, rs, ctx)) { -+ bbr->mode = BBR_PROBE_BW; -+ bbr2_start_bw_probe_down(sk); -+ } -+} -+ -+static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ bbr2_update_congestion_signals(sk, rs, ctx); -+ bbr_update_ack_aggregation(sk, rs); -+ bbr2_check_loss_too_high_in_startup(sk, rs); -+ bbr_check_full_bw_reached(sk, rs); -+ bbr2_check_drain(sk, rs, ctx); -+ bbr2_update_cycle_phase(sk, rs); -+ bbr_update_min_rtt(sk, rs); -+} -+ -+/* Fast path for app-limited case. -+ * -+ * On each ack, we execute bbr state machine, which primarily consists of: -+ * 1) update model based on new rate sample, and -+ * 2) update control based on updated model or state change. -+ * -+ * There are certain workload/scenarios, e.g. app-limited case, where -+ * either we can skip updating model or we can skip update of both model -+ * as well as control. This provides signifcant softirq cpu savings for -+ * processing incoming acks. -+ * -+ * In case of app-limited, if there is no congestion (loss/ecn) and -+ * if observed bw sample is less than current estimated bw, then we can -+ * skip some of the computation in bbr state processing: -+ * -+ * - if there is no rtt/mode/phase change: In this case, since all the -+ * parameters of the network model are constant, we can skip model -+ * as well control update. -+ * -+ * - else we can skip rest of the model update. But we still need to -+ * update the control to account for the new rtt/mode/phase. -+ * -+ * Returns whether we can take fast path or not. -+ */ -+static bool bbr2_fast_path(struct sock *sk, bool *update_model, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 prev_min_rtt_us, prev_mode; -+ -+ if (bbr->params.fast_path && bbr->try_fast_path && -+ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && -+ !bbr->loss_in_round && !bbr->ecn_in_round) { -+ prev_mode = bbr->mode; -+ prev_min_rtt_us = bbr->min_rtt_us; -+ bbr2_check_drain(sk, rs, ctx); -+ bbr2_update_cycle_phase(sk, rs); -+ bbr_update_min_rtt(sk, rs); -+ -+ if (bbr->mode == prev_mode && -+ bbr->min_rtt_us == prev_min_rtt_us && -+ bbr->try_fast_path) -+ return true; -+ -+ /* Skip model update, but control still needs to be updated */ -+ *update_model = false; -+ } -+ return false; -+} -+ -+static void bbr2_main(struct sock *sk, const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct bbr_context ctx = { 0 }; -+ bool update_model = true; -+ u32 bw; -+ int ce_ratio = -1; -+ -+ bbr->debug.event = '.'; /* init to default NOP (no event yet) */ -+ -+ bbr_update_round_start(sk, rs, &ctx); -+ if (bbr->round_start) { -+ bbr->rounds_since_probe = -+ min_t(s32, bbr->rounds_since_probe + 1, 0xFF); -+ ce_ratio = bbr2_update_ecn_alpha(sk); -+ tcp_plb_update_state(sk, &bbr->plb, ce_ratio); -+ tcp_plb_check_rehash(sk, &bbr->plb); -+ } -+ -+ bbr->ecn_in_round |= rs->is_ece; -+ bbr_calculate_bw_sample(sk, rs, &ctx); -+ -+ if (bbr2_fast_path(sk, &update_model, rs, &ctx)) -+ goto out; -+ -+ if (update_model) -+ bbr2_update_model(sk, rs, &ctx); -+ -+ bbr_update_gains(sk); -+ bw = bbr_bw(sk); -+ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); -+ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, -+ tp->snd_cwnd, &ctx); -+ bbr2_bound_cwnd_for_inflight_model(sk); -+ -+out: -+ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; -+ bbr->loss_in_cycle |= rs->lost > 0; -+ bbr->ecn_in_cycle |= rs->delivered_ce > 0; -+ -+ bbr_debug(sk, rs->acked_sacked, rs, &ctx); -+} -+ -+/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared -+ * down here, so that the algorithm functions that use the parameters must use -+ * the per-socket parameters; if they accidentally use the global version -+ * then there will be a compile error. -+ * TODO(ncardwell): move all per-socket parameters down to this section. -+ */ -+ -+/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. -+ * No loss response when 0. Max allwed value is 255. -+ */ -+static u32 bbr_beta = BBR_UNIT * 30 / 100; -+ -+/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. -+ * Max allowed value is 255. -+ */ -+static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ -+ -+/* The initial value for the ecn_alpha state variable. Default and max -+ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly -+ * to congestion if the bottleneck is congested when the flow starts up. -+ */ -+static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ -+ -+/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. -+ * No ECN based bounding when 0. Max allwed value is 255. -+ */ -+static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ -+ -+/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. -+ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. -+ */ -+static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ -+ -+/* Max RTT (in usec) at which to use sender-side ECN logic. -+ * Disabled when 0 (ECN allowed at any RTT). -+ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. -+ */ -+static u32 bbr_ecn_max_rtt_us = 5000; -+ -+/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN -+ * clears then use a multiplicative increase to quickly reprobe bw by -+ * starting inflight probing at the given multiple of inflight_hi. -+ * Default for this experimental knob is 0 (disabled). -+ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. -+ */ -+static u32 bbr_ecn_reprobe_gain; -+ -+/* Estimate bw probing has gone too far if loss rate exceeds this level. */ -+static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ -+ -+/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, -+ * and loss rate is higher than bbr_loss_thresh. -+ * Disabled if 0. Max allowed value is 15 (0xF). -+ */ -+static u32 bbr_full_loss_cnt = 8; -+ -+/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh -+ * meets this count. Max allowed value is 3. -+ */ -+static u32 bbr_full_ecn_cnt = 2; -+ -+/* Fraction of unutilized headroom to try to leave in path upon high loss. */ -+static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; -+ -+/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. -+ * Default is 1.25x, as in BBR v1. Max allowed is 511. -+ */ -+static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; -+ -+/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. -+ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. -+ * Max allowed is 511. -+ */ -+static u32 bbr_bw_probe_reno_gain = BBR_UNIT; -+ -+/* Max number of packet-timed rounds to wait before probing for bandwidth. If -+ * we want to tolerate 1% random loss per round, and not have this cut our -+ * inflight too much, we must probe for bw periodically on roughly this scale. -+ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. -+ * We aim to be fair with Reno/CUBIC up to a BDP of at least: -+ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets -+ */ -+static u32 bbr_bw_probe_max_rounds = 63; -+ -+/* Max amount of randomness to inject in round counting for Reno-coexistence. -+ * Max value is 15. -+ */ -+static u32 bbr_bw_probe_rand_rounds = 2; -+ -+/* Use BBR-native probe time scale starting at this many usec. -+ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: -+ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs -+ */ -+static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ -+ -+/* Use BBR-native probes spread over this many usec: */ -+static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ -+ -+/* Undo the model changes made in loss recovery if recovery was spurious? */ -+static bool bbr_undo = true; -+ -+/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ -+static bool bbr_fast_path = true; /* default: enabled */ -+ -+/* Use fast ack mode ? */ -+static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ -+ -+/* How much to additively increase inflight_hi when entering REFILL? */ -+static u32 bbr_refill_add_inc; /* default: disabled */ -+ -+module_param_named(beta, bbr_beta, uint, 0644); -+module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); -+module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); -+module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); -+module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); -+module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); -+module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); -+module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); -+module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664); -+module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664); -+module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664); -+module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664); -+module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664); -+module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664); -+module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664); -+module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664); -+module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664); -+module_param_named(undo, bbr_undo, bool, 0664); -+module_param_named(fast_path, bbr_fast_path, bool, 0664); -+module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664); -+module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664); -+ -+static void bbr2_init(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ const struct net *net = sock_net(sk); -+ -+ bbr_init(sk); /* run shared init code for v1 and v2 */ -+ -+ /* BBR v2 parameters: */ -+ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); -+ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); -+ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); -+ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); -+ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); -+ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); -+ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); -+ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); -+ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); -+ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); -+ bbr->params.inflight_headroom = -+ min_t(u32, 0xFFU, bbr_inflight_headroom); -+ bbr->params.bw_probe_pif_gain = -+ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); -+ bbr->params.bw_probe_reno_gain = -+ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); -+ bbr->params.bw_probe_max_rounds = -+ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); -+ bbr->params.bw_probe_rand_rounds = -+ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); -+ bbr->params.bw_probe_base_us = -+ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); -+ bbr->params.bw_probe_rand_us = -+ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); -+ bbr->params.undo = bbr_undo; -+ bbr->params.fast_path = bbr_fast_path ? 1 : 0; -+ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); -+ -+ /* BBR v2 state: */ -+ bbr->initialized = 1; -+ /* Start sampling ECN mark rate after first full flight is ACKed: */ -+ bbr->loss_round_delivered = tp->delivered + 1; -+ bbr->loss_round_start = 0; -+ bbr->undo_bw_lo = 0; -+ bbr->undo_inflight_lo = 0; -+ bbr->undo_inflight_hi = 0; -+ bbr->loss_events_in_round = 0; -+ bbr->startup_ecn_rounds = 0; -+ bbr2_reset_congestion_signals(sk); -+ bbr->bw_lo = ~0U; -+ bbr->bw_hi[0] = 0; -+ bbr->bw_hi[1] = 0; -+ bbr->inflight_lo = ~0U; -+ bbr->inflight_hi = ~0U; -+ bbr->bw_probe_up_cnt = ~0U; -+ bbr->bw_probe_up_acks = 0; -+ bbr->bw_probe_up_rounds = 0; -+ bbr->probe_wait_us = 0; -+ bbr->stopped_risky_probe = 0; -+ bbr->ack_phase = BBR_ACKS_INIT; -+ bbr->rounds_since_probe = 0; -+ bbr->bw_probe_samples = 0; -+ bbr->prev_probe_too_high = 0; -+ bbr->ecn_eligible = 0; -+ bbr->ecn_alpha = bbr->params.ecn_alpha_init; -+ bbr->alpha_last_delivered = 0; -+ bbr->alpha_last_delivered_ce = 0; -+ -+ bbr->plb.enabled = 0; -+ bbr->plb.consec_cong_rounds = 0; -+ bbr->plb.pause_until = 0; -+ if ((tp->ecn_flags & TCP_ECN_OK) && -+ net->ipv4.sysctl_tcp_plb_enabled) -+ bbr->plb.enabled = 1; -+ -+ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); -+ -+ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable) -+ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; -+} -+ -+/* Core TCP stack informs us that the given skb was just marked lost. */ -+static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct tcp_skb_cb *scb = TCP_SKB_CB(skb); -+ struct rate_sample rs; -+ -+ /* Capture "current" data over the full round trip of loss, -+ * to have a better chance to see the full capacity of the path. -+ */ -+ if (!bbr->loss_in_round) /* first loss in this round trip? */ -+ bbr->loss_round_delivered = tp->delivered; /* set round trip */ -+ bbr->loss_in_round = 1; -+ bbr->loss_in_cycle = 1; -+ -+ if (!bbr->bw_probe_samples) -+ return; /* not an skb sent while probing for bandwidth */ -+ if (unlikely(!scb->tx.delivered_mstamp)) -+ return; /* skb was SACKed, reneged, marked lost; ignore it */ -+ /* We are probing for bandwidth. Construct a rate sample that -+ * estimates what happened in the flight leading up to this lost skb, -+ * then see if the loss rate went too high, and if so at which packet. -+ */ -+ memset(&rs, 0, sizeof(rs)); -+ rs.tx_in_flight = scb->tx.in_flight; -+ rs.lost = tp->lost - scb->tx.lost; -+ rs.is_app_limited = scb->tx.is_app_limited; -+ if (bbr2_is_inflight_too_high(sk, &rs)) { -+ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); -+ bbr2_handle_inflight_too_high(sk, &rs); -+ } -+} -+ -+/* Revert short-term model if current loss recovery event was spurious. */ -+static u32 bbr2_undo_cwnd(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->debug.undo = 1; -+ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ -+ bbr->full_bw_cnt = 0; -+ bbr->loss_in_round = 0; -+ -+ if (!bbr->params.undo) -+ return tp->snd_cwnd; -+ -+ /* Revert to cwnd and other state saved before loss episode. */ -+ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); -+ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); -+ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); -+ return bbr->prior_cwnd; -+} -+ -+/* Entering loss recovery, so save state for when we undo recovery. */ -+static u32 bbr2_ssthresh(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr_save_cwnd(sk); -+ /* For undo, save state that adapts based on loss signal. */ -+ bbr->undo_bw_lo = bbr->bw_lo; -+ bbr->undo_inflight_lo = bbr->inflight_lo; -+ bbr->undo_inflight_hi = bbr->inflight_hi; -+ return tcp_sk(sk)->snd_ssthresh; -+} -+ -+static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) -+{ -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ return BBR2_PHASE_STARTUP; -+ case BBR_DRAIN: -+ return BBR2_PHASE_DRAIN; -+ case BBR_PROBE_BW: -+ break; -+ case BBR_PROBE_RTT: -+ return BBR2_PHASE_PROBE_RTT; -+ default: -+ return BBR2_PHASE_INVALID; -+ } -+ switch (bbr->cycle_idx) { -+ case BBR_BW_PROBE_UP: -+ return BBR2_PHASE_PROBE_BW_UP; -+ case BBR_BW_PROBE_DOWN: -+ return BBR2_PHASE_PROBE_BW_DOWN; -+ case BBR_BW_PROBE_CRUISE: -+ return BBR2_PHASE_PROBE_BW_CRUISE; -+ case BBR_BW_PROBE_REFILL: -+ return BBR2_PHASE_PROBE_BW_REFILL; -+ default: -+ return BBR2_PHASE_INVALID; -+ } -+} -+ -+static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, -+ union tcp_cc_info *info) -+{ -+ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || -+ ext & (1 << (INET_DIAG_VEGASINFO - 1))) { -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); -+ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); -+ u64 bw_lo = bbr->bw_lo == ~0U ? -+ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); -+ -+ memset(&info->bbr2, 0, sizeof(info->bbr2)); -+ info->bbr2.bbr_bw_lsb = (u32)bw; -+ info->bbr2.bbr_bw_msb = (u32)(bw >> 32); -+ info->bbr2.bbr_min_rtt = bbr->min_rtt_us; -+ info->bbr2.bbr_pacing_gain = bbr->pacing_gain; -+ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; -+ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; -+ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); -+ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; -+ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); -+ info->bbr2.bbr_mode = bbr->mode; -+ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); -+ info->bbr2.bbr_version = (__u8)2; -+ info->bbr2.bbr_inflight_lo = bbr->inflight_lo; -+ info->bbr2.bbr_inflight_hi = bbr->inflight_hi; -+ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); -+ *attr = INET_DIAG_BBRINFO; -+ return sizeof(info->bbr2); -+ } -+ return 0; -+} -+ -+static void bbr2_set_state(struct sock *sk, u8 new_state) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (new_state == TCP_CA_Loss) { -+ struct rate_sample rs = { .losses = 1 }; -+ struct bbr_context ctx = { 0 }; -+ -+ tcp_plb_update_state_upon_rto(sk, &bbr->plb); -+ bbr->prev_ca_state = TCP_CA_Loss; -+ bbr->full_bw = 0; -+ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { -+ /* bbr_adapt_lower_bounds() needs cwnd before -+ * we suffered an RTO, to update inflight_lo: -+ */ -+ bbr->inflight_lo = -+ max(tp->snd_cwnd, bbr->prior_cwnd); -+ } -+ bbr_debug(sk, 0, &rs, &ctx); -+ } else if (bbr->prev_ca_state == TCP_CA_Loss && -+ new_state != TCP_CA_Loss) { -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr->try_fast_path = 0; /* bound cwnd using latest model */ -+ } -+} -+ -+static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { -+ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, -+ .name = "bbr2", -+ .owner = THIS_MODULE, -+ .init = bbr2_init, -+ .cong_control = bbr2_main, -+ .sndbuf_expand = bbr_sndbuf_expand, -+ .skb_marked_lost = bbr2_skb_marked_lost, -+ .undo_cwnd = bbr2_undo_cwnd, -+ .cwnd_event = bbr_cwnd_event, -+ .ssthresh = bbr2_ssthresh, -+ .tso_segs = bbr_tso_segs, -+ .get_info = bbr2_get_info, -+ .set_state = bbr2_set_state, -+}; -+ -+static int __init bbr_register(void) -+{ -+ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); -+ return tcp_register_congestion_control(&tcp_bbr2_cong_ops); -+} -+ -+static void __exit bbr_unregister(void) -+{ -+ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); -+} -+ -+module_init(bbr_register); -+module_exit(bbr_unregister); -+ -+MODULE_AUTHOR("Van Jacobson "); -+MODULE_AUTHOR("Neal Cardwell "); -+MODULE_AUTHOR("Yuchung Cheng "); -+MODULE_AUTHOR("Soheil Hassas Yeganeh "); -+MODULE_AUTHOR("Priyaranjan Jha "); -+MODULE_AUTHOR("Yousuk Seung "); -+MODULE_AUTHOR("Kevin Yang "); -+MODULE_AUTHOR("Arjun Roy "); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); -diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c -index d3cae40749e8..0f268f2ff2e9 100644 ---- a/net/ipv4/tcp_cong.c -+++ b/net/ipv4/tcp_cong.c -@@ -189,6 +189,7 @@ void tcp_init_congestion_control(struct sock *sk) - struct inet_connection_sock *icsk = inet_csk(sk); - - tcp_sk(sk)->prior_ssthresh = 0; -+ tcp_sk(sk)->fast_ack_mode = 0; - if (icsk->icsk_ca_ops->init) - icsk->icsk_ca_ops->init(sk); - if (tcp_ca_needs_ecn(sk)) -diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c -index bc2ea12221f9..fa5c3e5d9c85 100644 ---- a/net/ipv4/tcp_input.c -+++ b/net/ipv4/tcp_input.c -@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) - tcp_enter_quickack_mode(sk, 2); - break; - case INET_ECN_CE: -- if (tcp_ca_needs_ecn(sk)) -+ if (tcp_ca_wants_ce_events(sk)) - tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); - - if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { -@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) - tp->ecn_flags |= TCP_ECN_SEEN; - break; - default: -- if (tcp_ca_needs_ecn(sk)) -+ if (tcp_ca_wants_ce_events(sk)) - tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); - tp->ecn_flags |= TCP_ECN_SEEN; - break; -@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) - */ - static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) - { -+ struct sock *sk = (struct sock *)tp; -+ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; -+ - tp->lost += tcp_skb_pcount(skb); -+ if (ca_ops->skb_marked_lost) -+ ca_ops->skb_marked_lost(sk, skb); - } - - void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) -@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, - WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); - tcp_skb_pcount_add(skb, -pcount); - -+ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ -+ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, -+ "prev in_flight: %u skb in_flight: %u pcount: %u", -+ TCP_SKB_CB(prev)->tx.in_flight, -+ TCP_SKB_CB(skb)->tx.in_flight, -+ pcount)) -+ TCP_SKB_CB(skb)->tx.in_flight = 0; -+ else -+ TCP_SKB_CB(skb)->tx.in_flight -= pcount; -+ TCP_SKB_CB(prev)->tx.in_flight += pcount; -+ - /* When we're adding to gso_segs == 1, gso_size will be zero, - * in theory this shouldn't be necessary but as long as DSACK - * code can come after this skb later on it's better to keep -@@ -3811,6 +3827,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) - - prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; - rs.prior_in_flight = tcp_packets_in_flight(tp); -+ tcp_rate_check_app_limited(sk); - - /* ts_recent update must be made after we are sure that the packet - * is in window. -@@ -3909,6 +3926,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) - delivered = tcp_newly_delivered(sk, delivered, flag); - lost = tp->lost - lost; /* freshly marked lost */ - rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); -+ rs.is_ece = !!(flag & FLAG_ECE); - tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); - tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); - tcp_xmit_recovery(sk, rexmit); -@@ -5508,13 +5526,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) - - /* More than one full frame received... */ - if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && -+ (tp->fast_ack_mode == 1 || - /* ... and right edge of window advances far enough. - * (tcp_recvmsg() will send ACK otherwise). - * If application uses SO_RCVLOWAT, we want send ack now if - * we have not received enough bytes to satisfy the condition. - */ -- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || -- __tcp_select_window(sk) >= tp->rcv_wnd)) || -+ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || -+ __tcp_select_window(sk) >= tp->rcv_wnd))) || - /* We ACK each frame or... */ - tcp_in_quickack_mode(sk) || - /* Protocol state mandates a one-time immediate ACK */ -diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c -index 5b019ba2b9d2..5884f4388c61 100644 ---- a/net/ipv4/tcp_ipv4.c -+++ b/net/ipv4/tcp_ipv4.c -@@ -3165,6 +3165,13 @@ static int __net_init tcp_sk_init(struct net *net) - net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0; - atomic_set(&net->ipv4.tfo_active_disable_times, 0); - -+ /* Set default values for PLB */ -+ net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */ -+ net->ipv4.sysctl_tcp_plb_cong_thresh = 128; /* 50% congestion */ -+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3; -+ net->ipv4.sysctl_tcp_plb_rehash_rounds = 12; -+ net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60; -+ - /* Reno is always built in */ - if (!net_eq(net, &init_net) && - bpf_try_module_get(init_net.ipv4.tcp_congestion_control, -diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c -index 290019de766d..15e89addae08 100644 ---- a/net/ipv4/tcp_output.c -+++ b/net/ipv4/tcp_output.c -@@ -375,7 +375,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, - th->cwr = 1; - skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; - } -- } else if (!tcp_ca_needs_ecn(sk)) { -+ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && -+ !tcp_ca_needs_ecn(sk)) { - /* ACK or retransmitted segment: clear ECT|CE */ - INET_ECN_dontxmit(sk); - } -@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, - { - struct tcp_sock *tp = tcp_sk(sk); - struct sk_buff *buff; -- int nsize, old_factor; -+ int nsize, old_factor, inflight_prev; - long limit; - int nlen; - u8 flags; -@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, - - if (diff) - tcp_adjust_pcount(sk, skb, diff); -+ -+ /* Set buff tx.in_flight as if buff were sent by itself. */ -+ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; -+ if (WARN_ONCE(inflight_prev < 0, -+ "inconsistent: tx.in_flight: %u old_factor: %d", -+ TCP_SKB_CB(skb)->tx.in_flight, old_factor)) -+ inflight_prev = 0; -+ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + -+ tcp_skb_pcount(buff); - } - - /* Link BUFF into the send queue. */ -@@ -1988,13 +1998,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, - static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) - { - const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; -- u32 min_tso, tso_segs; -- -- min_tso = ca_ops->min_tso_segs ? -- ca_ops->min_tso_segs(sk) : -- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); -+ u32 tso_segs; - -- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); -+ tso_segs = ca_ops->tso_segs ? -+ ca_ops->tso_segs(sk, mss_now) : -+ tcp_tso_autosize(sk, mss_now, -+ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs)); - return min_t(u32, tso_segs, sk->sk_gso_max_segs); - } - -@@ -2630,6 +2639,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, - skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); - list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); - tcp_init_tso_segs(skb, mss_now); -+ tcp_set_tx_in_flight(sk, skb); - goto repair; /* Skip network transmission */ - } - -diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c -new file mode 100644 -index 000000000000..71b02c0404ce ---- /dev/null -+++ b/net/ipv4/tcp_plb.c -@@ -0,0 +1,100 @@ -+/* Protective Load Balancing (PLB) -+ * -+ * PLB was designed to reduce link load imbalance across datacenter -+ * switches. PLB is a host-based optimization; it leverages congestion -+ * signals from the transport layer to randomly change the path of the -+ * connection experiencing sustained congestion. PLB prefers to repath -+ * after idle periods to minimize packet reordering. It repaths by -+ * changing the IPv6 Flow Label on the packets of a connection, which -+ * datacenter switches include as part of ECMP/WCMP hashing. -+ * -+ * PLB is described in detail in: -+ * -+ * Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu, -+ * Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson, -+ * David Wetherall,Abdul Kabbani: -+ * "PLB: Congestion Signals are Simple and Effective for -+ * Network Load Balancing" -+ * In ACM SIGCOMM 2022, Amsterdam Netherlands. -+ * -+ */ -+ -+#include -+ -+/* Called once per round-trip to update PLB state for a connection. */ -+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, -+ const int cong_ratio) -+{ -+ struct net *net = sock_net(sk); -+ -+ if (!plb->enabled) -+ return; -+ -+ if (cong_ratio >= 0) { -+ if (cong_ratio < net->ipv4.sysctl_tcp_plb_cong_thresh) -+ plb->consec_cong_rounds = 0; -+ else if (plb->consec_cong_rounds < -+ net->ipv4.sysctl_tcp_plb_rehash_rounds) -+ plb->consec_cong_rounds++; -+ } -+} -+EXPORT_SYMBOL_GPL(tcp_plb_update_state); -+ -+/* Check whether recent congestion has been persistent enough to warrant -+ * a load balancing decision that switches the connection to another path. -+ */ -+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) -+{ -+ struct net *net = sock_net(sk); -+ bool can_idle_rehash, can_force_rehash; -+ -+ if (!plb->enabled) -+ return; -+ -+ /* Note that tcp_jiffies32 can wrap, so we clear pause_until -+ * to 0 to indicate there is no recent RTO event that constrains -+ * PLB rehashing. -+ */ -+ if (plb->pause_until && -+ !before(tcp_jiffies32, plb->pause_until)) -+ plb->pause_until = 0; -+ -+ can_idle_rehash = net->ipv4.sysctl_tcp_plb_idle_rehash_rounds && -+ !tcp_sk(sk)->packets_out && -+ plb->consec_cong_rounds >= -+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds; -+ can_force_rehash = plb->consec_cong_rounds >= -+ net->ipv4.sysctl_tcp_plb_rehash_rounds; -+ -+ if (!plb->pause_until && (can_idle_rehash || can_force_rehash)) { -+ sk_rethink_txhash(sk); -+ plb->consec_cong_rounds = 0; -+ tcp_sk(sk)->ecn_rehash++; -+ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPECNREHASH); -+ } -+} -+EXPORT_SYMBOL_GPL(tcp_plb_check_rehash); -+ -+/* Upon RTO, disallow load balancing for a while, to avoid having load -+ * balancing decisions switch traffic to a black-holed path that was -+ * previously avoided with a sk_rethink_txhash() call at RTO time. -+ */ -+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb) -+{ -+ struct net *net = sock_net(sk); -+ u32 pause; -+ -+ if (!plb->enabled) -+ return; -+ -+ pause = net->ipv4.sysctl_tcp_plb_suspend_rto_sec * HZ; -+ pause += prandom_u32_max(pause); -+ plb->pause_until = tcp_jiffies32 + pause; -+ -+ /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call -+ * that may switch this connection to a path with completely different -+ * congestion characteristics. -+ */ -+ plb->consec_cong_rounds = 0; -+} -+EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto); -diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c -index a8f6d9d06f2e..a8b4c9504570 100644 ---- a/net/ipv4/tcp_rate.c -+++ b/net/ipv4/tcp_rate.c -@@ -34,6 +34,24 @@ - * ready to send in the write queue. - */ - -+void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ u32 in_flight; -+ -+ /* Check, sanitize, and record packets in flight after skb was sent. */ -+ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); -+ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, -+ "insane in_flight %u cc %s mss %u " -+ "cwnd %u pif %u %u %u %u\n", -+ in_flight, inet_csk(sk)->icsk_ca_ops->name, -+ tp->mss_cache, tp->snd_cwnd, -+ tp->packets_out, tp->retrans_out, -+ tp->sacked_out, tp->lost_out)) -+ in_flight = TCPCB_IN_FLIGHT_MAX; -+ TCP_SKB_CB(skb)->tx.in_flight = in_flight; -+} -+ - /* Snapshot the current delivery information in the skb, to generate - * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). - */ -@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) - TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; - TCP_SKB_CB(skb)->tx.delivered = tp->delivered; - TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; -+ TCP_SKB_CB(skb)->tx.lost = tp->lost; - TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; -+ tcp_set_tx_in_flight(sk, skb); - } - - /* When an skb is sacked or acked, we fill in the rate sample with the (prior) -@@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - if (!rs->prior_delivered || - tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, - scb->end_seq, rs->last_end_seq)) { -+ rs->prior_lost = scb->tx.lost; - rs->prior_delivered_ce = scb->tx.delivered_ce; - rs->prior_delivered = scb->tx.delivered; - rs->prior_mstamp = scb->tx.delivered_mstamp; - rs->is_app_limited = scb->tx.is_app_limited; - rs->is_retrans = scb->sacked & TCPCB_RETRANS; - rs->last_end_seq = scb->end_seq; -+ rs->tx_in_flight = scb->tx.in_flight; - - /* Record send time of most recently ACKed packet: */ - tp->first_tx_mstamp = tx_tstamp; - /* Find the duration of the "send phase" of this window: */ -- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, -- scb->tx.first_tx_mstamp); -+ rs->interval_us = tcp_stamp32_us_delta( -+ tp->first_tx_mstamp, -+ scb->tx.first_tx_mstamp); - - } - /* Mark off the skb delivered once it's sacked to avoid being -@@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - return; - } - rs->delivered = tp->delivered - rs->prior_delivered; -+ rs->lost = tp->lost - rs->prior_lost; - - rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; - /* delivered_ce occupies less than 32 bits in the skb control block */ -@@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - * longer phase. - */ - snd_us = rs->interval_us; /* send phase */ -- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, -+ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, - rs->prior_mstamp); /* ack phase */ - rs->interval_us = max(snd_us, ack_us); - -diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c -index b4dfb82d6ecb..bb613fc8948a 100644 ---- a/net/ipv4/tcp_timer.c -+++ b/net/ipv4/tcp_timer.c -@@ -605,6 +605,7 @@ void tcp_write_timer_handler(struct sock *sk) - return; - } - -+ tcp_rate_check_app_limited(sk); - tcp_mstamp_refresh(tcp_sk(sk)); - event = icsk->icsk_pending; - --- -2.37.3 - diff --git a/6.0/testing/0001-cachy.patch b/6.0/0001-cachy.patch similarity index 99% rename from 6.0/testing/0001-cachy.patch rename to 6.0/0001-cachy.patch index e26e3cb0..1a8f1378 100644 --- a/6.0/testing/0001-cachy.patch +++ b/6.0/0001-cachy.patch @@ -1,7 +1,7 @@ -From 79b0b16073aa7852df60ff5820f471c0ad0078d7 Mon Sep 17 00:00:00 2001 +From 4ee5774d519ab3d21a214f4aa94e3f2ddc6ceb81 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Tue, 27 Sep 2022 15:12:20 +0200 -Subject: [PATCH 01/13] cachy +Subject: [PATCH 01/16] cachy Signed-off-by: Peter Jung --- diff --git a/6.0/testing/0002-bbr2.patch b/6.0/0002-bbr2.patch similarity index 99% rename from 6.0/testing/0002-bbr2.patch rename to 6.0/0002-bbr2.patch index 7f2c945c..9ca52cf6 100644 --- a/6.0/testing/0002-bbr2.patch +++ b/6.0/0002-bbr2.patch @@ -1,7 +1,7 @@ -From cdfac99ee700654cacf05cbbfb83ca7907b0dcc7 Mon Sep 17 00:00:00 2001 +From 0feaada45827f920b03a53edea1d34597614db84 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 5 Sep 2022 08:34:43 +0200 -Subject: [PATCH 02/13] bbr2 +Subject: [PATCH 02/16] bbr2 Signed-off-by: Peter Jung --- diff --git a/6.0/0002-cachy.patch b/6.0/0002-cachy.patch deleted file mode 100644 index bcf81eab..00000000 --- a/6.0/0002-cachy.patch +++ /dev/null @@ -1,6610 +0,0 @@ -From 4ce1238c4f6da720e90328477839811957f4f32f Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:15:39 +0200 -Subject: [PATCH 02/13] cachy - -Signed-off-by: Peter Jung ---- - .gitignore | 1 + - Documentation/admin-guide/mm/damon/index.rst | 6 +- - Documentation/admin-guide/mm/damon/start.rst | 13 +- - Documentation/admin-guide/mm/damon/usage.rst | 5 + - Documentation/admin-guide/mm/ksm.rst | 36 + - Documentation/admin-guide/pm/amd-pstate.rst | 19 + - Makefile | 2 + - arch/alpha/kernel/syscalls/syscall.tbl | 1 + - arch/arc/configs/axs101_defconfig | 1 + - arch/arc/configs/axs103_defconfig | 1 + - arch/arc/configs/axs103_smp_defconfig | 1 + - arch/arc/configs/haps_hs_defconfig | 1 + - arch/arc/configs/haps_hs_smp_defconfig | 1 + - arch/arc/configs/hsdk_defconfig | 1 + - arch/arc/configs/nsim_700_defconfig | 1 + - arch/arc/configs/nsimosci_defconfig | 1 + - arch/arc/configs/nsimosci_hs_defconfig | 1 + - arch/arc/configs/nsimosci_hs_smp_defconfig | 1 + - arch/arc/configs/tb10x_defconfig | 1 + - arch/arc/configs/vdk_hs38_defconfig | 1 + - arch/arc/configs/vdk_hs38_smp_defconfig | 1 + - arch/arm/tools/syscall.tbl | 1 + - arch/arm64/include/asm/unistd.h | 2 +- - arch/arm64/include/asm/unistd32.h | 2 + - arch/ia64/kernel/syscalls/syscall.tbl | 1 + - arch/m68k/kernel/syscalls/syscall.tbl | 1 + - arch/microblaze/kernel/syscalls/syscall.tbl | 1 + - arch/mips/kernel/syscalls/syscall_n32.tbl | 1 + - arch/mips/kernel/syscalls/syscall_n64.tbl | 1 + - arch/mips/kernel/syscalls/syscall_o32.tbl | 1 + - arch/parisc/kernel/syscalls/syscall.tbl | 1 + - arch/powerpc/kernel/syscalls/syscall.tbl | 1 + - arch/s390/kernel/syscalls/syscall.tbl | 1 + - arch/sh/kernel/syscalls/syscall.tbl | 1 + - arch/sparc/kernel/syscalls/syscall.tbl | 1 + - arch/x86/Kconfig.cpu | 332 +++++- - arch/x86/Makefile | 40 +- - arch/x86/Makefile.postlink | 41 + - arch/x86/boot/compressed/Makefile | 10 +- - arch/x86/entry/syscalls/syscall_32.tbl | 1 + - arch/x86/entry/syscalls/syscall_64.tbl | 1 + - arch/x86/include/asm/msr-index.h | 7 + - arch/x86/include/asm/topology.h | 1 + - arch/x86/include/asm/vdso/processor.h | 2 +- - arch/x86/include/asm/vermagic.h | 66 ++ - arch/x86/kernel/alternative.c | 2 + - arch/x86/kernel/cpu/intel_epb.c | 4 + - arch/x86/kernel/cpu/microcode/core.c | 40 +- - arch/x86/kernel/cpu/microcode/intel.c | 14 +- - arch/x86/kernel/itmt.c | 29 +- - arch/x86/kernel/msr.c | 2 +- - arch/x86/kernel/tsc.c | 3 + - arch/x86/mm/fault.c | 4 +- - arch/xtensa/kernel/syscalls/syscall.tbl | 1 + - block/bfq-cgroup.c | 5 - - block/bfq-iosched.c | 20 +- - block/bfq-iosched.h | 18 +- - block/bfq-wf2q.c | 9 +- - block/blk-core.c | 3 + - block/elevator.c | 7 +- - drivers/acpi/cppc_acpi.c | 131 ++- - drivers/ata/libahci.c | 4 +- - drivers/base/arch_topology.c | 2 +- - drivers/base/firmware_loader/main.c | 2 + - drivers/block/zram/Kconfig | 18 + - drivers/block/zram/zram_drv.c | 39 + - drivers/cpufreq/amd-pstate.c | 1003 ++++++++++++++++- - drivers/cpufreq/cppc_cpufreq.c | 2 +- - drivers/cpufreq/intel_pstate.c | 7 + - drivers/idle/intel_idle.c | 44 +- - drivers/input/serio/i8042.c | 10 +- - drivers/md/dm-crypt.c | 5 + - drivers/net/dummy.c | 2 +- - drivers/nvme/host/core.c | 2 +- - drivers/pci/pci.c | 2 +- - drivers/powercap/intel_rapl_common.c | 2 +- - drivers/soundwire/bus.h | 2 +- - drivers/thermal/intel/intel_powerclamp.c | 10 + - fs/proc/base.c | 15 + - fs/xattr.c | 15 +- - include/acpi/cppc_acpi.h | 17 + - include/linux/ipc_namespace.h | 5 +- - include/linux/jbd2.h | 2 +- - include/linux/ksm.h | 4 + - include/linux/mm_types.h | 5 + - include/linux/page_counter.h | 1 + - include/linux/pagemap.h | 2 +- - include/linux/sched.h | 1 + - include/linux/syscalls.h | 1 + - include/linux/user_namespace.h | 4 + - include/linux/wait.h | 2 + - include/uapi/asm-generic/unistd.h | 5 +- - include/uapi/linux/if_bonding.h | 2 +- - init/Kconfig | 26 + - init/do_mounts.c | 16 +- - kernel/Kconfig.hz | 24 + - kernel/fork.c | 14 + - kernel/locking/rwsem.c | 4 +- - kernel/module/internal.h | 2 + - kernel/module/main.c | 1 + - kernel/module/procfs.c | 13 + - kernel/module/signing.c | 4 + - kernel/sched/core.c | 19 +- - kernel/sched/debug.c | 1 + - kernel/sched/fair.c | 20 +- - kernel/sched/wait.c | 24 + - kernel/sys_ni.c | 1 + - kernel/sysctl.c | 22 + - kernel/user_namespace.c | 7 + - kernel/watchdog.c | 2 +- - lib/raid6/algos.c | 4 +- - lib/string.c | 62 +- - lib/zstd/compress/zstd_compress.c | 2 +- - lib/zstd/compress/zstd_double_fast.c | 61 +- - lib/zstd/compress/zstd_fast.c | 69 +- - lib/zstd/compress/zstd_lazy.c | 223 ++-- - mm/compaction.c | 6 +- - mm/damon/Kconfig | 3 + - mm/damon/core-test.h | 29 +- - mm/damon/core.c | 45 +- - mm/damon/dbgfs.c | 2 +- - mm/damon/lru_sort.c | 41 +- - mm/damon/vaddr.c | 18 +- - mm/ksm.c | 351 +++++- - mm/madvise.c | 115 ++ - mm/memcontrol.c | 2 +- - mm/page-writeback.c | 8 + - mm/page_alloc.c | 5 +- - mm/swap.c | 5 + - mm/vmpressure.c | 4 + - mm/vmscan.c | 4 + - net/ipv4/inet_connection_sock.c | 2 +- - net/ipv4/tcp.c | 4 +- - tools/testing/selftests/damon/Makefile | 1 + - .../debugfs_duplicate_context_creation.sh | 27 + - tools/testing/selftests/vm/.gitignore | 1 + - tools/testing/selftests/vm/Makefile | 1 + - tools/testing/selftests/vm/test-ksm-auto.c | 273 +++++ - 138 files changed, 3281 insertions(+), 429 deletions(-) - create mode 100644 arch/x86/Makefile.postlink - create mode 100644 tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh - create mode 100644 tools/testing/selftests/vm/test-ksm-auto.c - -diff --git a/.gitignore b/.gitignore -index 265959544978..cd4ef88584ea 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -37,6 +37,7 @@ - *.o - *.o.* - *.patch -+*.relocs - *.s - *.so - *.so.dbg -diff --git a/Documentation/admin-guide/mm/damon/index.rst b/Documentation/admin-guide/mm/damon/index.rst -index 05500042f777..33d37bb2fb4e 100644 ---- a/Documentation/admin-guide/mm/damon/index.rst -+++ b/Documentation/admin-guide/mm/damon/index.rst -@@ -1,8 +1,8 @@ - .. SPDX-License-Identifier: GPL-2.0 - --======================== --Monitoring Data Accesses --======================== -+========================== -+DAMON: Data Access MONitor -+========================== - - :doc:`DAMON ` allows light-weight data access monitoring. - Using DAMON, users can analyze the memory access patterns of their systems and -diff --git a/Documentation/admin-guide/mm/damon/start.rst b/Documentation/admin-guide/mm/damon/start.rst -index 4d5ca2c46288..9f88afc734da 100644 ---- a/Documentation/admin-guide/mm/damon/start.rst -+++ b/Documentation/admin-guide/mm/damon/start.rst -@@ -29,16 +29,9 @@ called DAMON Operator (DAMO). It is available at - https://github.com/awslabs/damo. The examples below assume that ``damo`` is on - your ``$PATH``. It's not mandatory, though. - --Because DAMO is using the debugfs interface (refer to :doc:`usage` for the --detail) of DAMON, you should ensure debugfs is mounted. Mount it manually as --below:: -- -- # mount -t debugfs none /sys/kernel/debug/ -- --or append the following line to your ``/etc/fstab`` file so that your system --can automatically mount debugfs upon booting:: -- -- debugfs /sys/kernel/debug debugfs defaults 0 0 -+Because DAMO is using the sysfs interface (refer to :doc:`usage` for the -+detail) of DAMON, you should ensure :doc:`sysfs ` is -+mounted. - - - Recording Data Access Patterns -diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst -index ca91ecc29078..b47b0cbbd491 100644 ---- a/Documentation/admin-guide/mm/damon/usage.rst -+++ b/Documentation/admin-guide/mm/damon/usage.rst -@@ -393,6 +393,11 @@ the files as above. Above is only for an example. - debugfs Interface - ================= - -+.. note:: -+ -+ DAMON debugfs interface will be removed after next LTS kernel is released, so -+ users should move to the :ref:`sysfs interface `. -+ - DAMON exports eight files, ``attrs``, ``target_ids``, ``init_regions``, - ``schemes``, ``monitor_on``, ``kdamond_pid``, ``mk_contexts`` and - ``rm_contexts`` under its debugfs directory, ``/damon/``. -diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst -index b244f0202a03..fb6ba2002a4b 100644 ---- a/Documentation/admin-guide/mm/ksm.rst -+++ b/Documentation/admin-guide/mm/ksm.rst -@@ -184,6 +184,42 @@ The maximum possible ``pages_sharing/pages_shared`` ratio is limited by the - ``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must - be increased accordingly. - -+Monitoring KSM profit -+===================== -+ -+KSM can save memory by merging identical pages, but also can consume -+additional memory, because it needs to generate a number of rmap_items to -+save each scanned page's brief rmap information. Some of these pages may -+be merged, but some may not be abled to be merged after being checked -+several times, which are unprofitable memory consumed. -+ -+1) How to determine whether KSM save memory or consume memory in system-wide -+ range? Here is a simple approximate calculation for reference:: -+ -+ general_profit =~ pages_sharing * sizeof(page) - (all_rmap_items) * -+ sizeof(rmap_item); -+ -+ where all_rmap_items can be easily obtained by summing ``pages_sharing``, -+ ``pages_shared``, ``pages_unshared`` and ``pages_volatile``. -+ -+2) The KSM profit inner a single process can be similarly obtained by the -+ following approximate calculation:: -+ -+ process_profit =~ ksm_merging_pages * sizeof(page) - -+ ksm_rmap_items * sizeof(rmap_item). -+ -+ where ksm_merging_pages is shown under the directory ``/proc//``, -+ and ksm_rmap_items is shown in ``/proc//ksm_stat``. -+ -+From the perspective of application, a high ratio of ``ksm_rmap_items`` to -+``ksm_merging_pages`` means a bad madvise-applied policy, so developers or -+administrators have to rethink how to change madvise policy. Giving an example -+for reference, a page's size is usually 4K, and the rmap_item's size is -+separately 32B on 32-bit CPU architecture and 64B on 64-bit CPU architecture. -+so if the ``ksm_rmap_items/ksm_merging_pages`` ratio exceeds 64 on 64-bit CPU -+or exceeds 128 on 32-bit CPU, then the app's madvise policy should be dropped, -+because the ksm profit is approximately zero or negative. -+ - Monitoring KSM events - ===================== - -diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst -index 83b58eb4ab4d..d0f0e115013b 100644 ---- a/Documentation/admin-guide/pm/amd-pstate.rst -+++ b/Documentation/admin-guide/pm/amd-pstate.rst -@@ -261,6 +261,25 @@ lowest non-linear performance in `AMD CPPC Performance Capability - `_.) - This attribute is read-only. - -+``energy_performance_available_preferences`` -+ -+All the supported EPP preference could be selected, List of the strings that -+can be set to the ``energy_performance_preference`` attribute -+those different profiles represent different energy vs efficiency hints provided -+to low-level firmware -+however, the ``default`` represents the epp value is set by platform firmware -+This attribute is read-only. -+ -+``energy_performance_preference`` -+ -+The current energy performance preference can be read from this attribute. -+and user can change current preference according to energy or performance needs -+Please get all support profiles list from -+``energy_performance_available_preferences`` attribute, all the profiles are -+integer values defined between 0 to 255 when EPP feature is enabled by platform -+firmware, if EPP feature is disabled, driver will ignore the written value -+This attribute is read-write. -+ - Other performance and frequency values can be read back from - ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. - -diff --git a/Makefile b/Makefile -index 647a42a1f800..5c327c29ef12 100644 ---- a/Makefile -+++ b/Makefile -@@ -758,6 +758,8 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) - - ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE - KBUILD_CFLAGS += -O2 -+else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 -+KBUILD_CFLAGS += -O3 - else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE - KBUILD_CFLAGS += -Os - endif -diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl -index 3515bc4f16a4..00ff721da300 100644 ---- a/arch/alpha/kernel/syscalls/syscall.tbl -+++ b/arch/alpha/kernel/syscalls/syscall.tbl -@@ -490,3 +490,4 @@ - 558 common process_mrelease sys_process_mrelease - 559 common futex_waitv sys_futex_waitv - 560 common set_mempolicy_home_node sys_ni_syscall -+561 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig -index e31a8ebc3ecc..0016149f9583 100644 ---- a/arch/arc/configs/axs101_defconfig -+++ b/arch/arc/configs/axs101_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig -index e0e8567f0d75..5b031582a1cf 100644 ---- a/arch/arc/configs/axs103_defconfig -+++ b/arch/arc/configs/axs103_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig -index fcbc952bc75b..d4eec39e0112 100644 ---- a/arch/arc/configs/axs103_smp_defconfig -+++ b/arch/arc/configs/axs103_smp_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig -index d87ad7e88d62..7337cdf4ffdd 100644 ---- a/arch/arc/configs/haps_hs_defconfig -+++ b/arch/arc/configs/haps_hs_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EXPERT=y - CONFIG_PERF_EVENTS=y - # CONFIG_COMPAT_BRK is not set -diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig -index 8d82cdb7f86a..bc927221afc0 100644 ---- a/arch/arc/configs/haps_hs_smp_defconfig -+++ b/arch/arc/configs/haps_hs_smp_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig -index f856b03e0fb5..aa000075a575 100644 ---- a/arch/arc/configs/hsdk_defconfig -+++ b/arch/arc/configs/hsdk_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y - CONFIG_BLK_DEV_RAM=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig -index a1ce12bf5b16..326f6cde7826 100644 ---- a/arch/arc/configs/nsim_700_defconfig -+++ b/arch/arc/configs/nsim_700_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig -index ca10f4a2c823..bf39a0091679 100644 ---- a/arch/arc/configs/nsimosci_defconfig -+++ b/arch/arc/configs/nsimosci_defconfig -@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig -index 31b6ec3683c6..7121bd71c543 100644 ---- a/arch/arc/configs/nsimosci_hs_defconfig -+++ b/arch/arc/configs/nsimosci_hs_defconfig -@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig -index 41a0037f48a5..f9863b294a70 100644 ---- a/arch/arc/configs/nsimosci_hs_smp_defconfig -+++ b/arch/arc/configs/nsimosci_hs_smp_defconfig -@@ -8,6 +8,7 @@ CONFIG_IKCONFIG_PROC=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_PERF_EVENTS=y - # CONFIG_COMPAT_BRK is not set - CONFIG_KPROBES=y -diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig -index d93b65008d4a..a12656ec0072 100644 ---- a/arch/arc/configs/tb10x_defconfig -+++ b/arch/arc/configs/tb10x_defconfig -@@ -14,6 +14,7 @@ CONFIG_INITRAMFS_SOURCE="../tb10x-rootfs.cpio" - CONFIG_INITRAMFS_ROOT_UID=2100 - CONFIG_INITRAMFS_ROOT_GID=501 - # CONFIG_RD_GZIP is not set -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - # CONFIG_AIO is not set - CONFIG_EMBEDDED=y -diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig -index 0c3b21416819..d7c858df520c 100644 ---- a/arch/arc/configs/vdk_hs38_defconfig -+++ b/arch/arc/configs/vdk_hs38_defconfig -@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_IKCONFIG=y - CONFIG_IKCONFIG_PROC=y - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig -index f9ad9d3ee702..015c1d43889e 100644 ---- a/arch/arc/configs/vdk_hs38_smp_defconfig -+++ b/arch/arc/configs/vdk_hs38_smp_defconfig -@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_IKCONFIG=y - CONFIG_IKCONFIG_PROC=y - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl -index ac964612d8b0..90933eabe115 100644 ---- a/arch/arm/tools/syscall.tbl -+++ b/arch/arm/tools/syscall.tbl -@@ -464,3 +464,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h -index 037feba03a51..64a514f90131 100644 ---- a/arch/arm64/include/asm/unistd.h -+++ b/arch/arm64/include/asm/unistd.h -@@ -39,7 +39,7 @@ - #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) - #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) - --#define __NR_compat_syscalls 451 -+#define __NR_compat_syscalls 452 - #endif - - #define __ARCH_WANT_SYS_CLONE -diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h -index 604a2053d006..91f2bb7199af 100644 ---- a/arch/arm64/include/asm/unistd32.h -+++ b/arch/arm64/include/asm/unistd32.h -@@ -907,6 +907,8 @@ __SYSCALL(__NR_process_mrelease, sys_process_mrelease) - __SYSCALL(__NR_futex_waitv, sys_futex_waitv) - #define __NR_set_mempolicy_home_node 450 - __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) -+#define __NR_pmadv_ksm 451 -+__SYSCALL(__NR_pmadv_ksm, sys_pmadv_ksm) - - /* - * Please add new compat syscalls above this comment and update -diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl -index 78b1d03e86e1..79ad5a5682b3 100644 ---- a/arch/ia64/kernel/syscalls/syscall.tbl -+++ b/arch/ia64/kernel/syscalls/syscall.tbl -@@ -371,3 +371,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl -index b1f3940bc298..5ccf925567da 100644 ---- a/arch/m68k/kernel/syscalls/syscall.tbl -+++ b/arch/m68k/kernel/syscalls/syscall.tbl -@@ -450,3 +450,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl -index 820145e47350..6b76208597f3 100644 ---- a/arch/microblaze/kernel/syscalls/syscall.tbl -+++ b/arch/microblaze/kernel/syscalls/syscall.tbl -@@ -456,3 +456,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl -index 253ff994ed2e..e4aeedb17c38 100644 ---- a/arch/mips/kernel/syscalls/syscall_n32.tbl -+++ b/arch/mips/kernel/syscalls/syscall_n32.tbl -@@ -389,3 +389,4 @@ - 448 n32 process_mrelease sys_process_mrelease - 449 n32 futex_waitv sys_futex_waitv - 450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node -+451 n32 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl -index 3f1886ad9d80..fe88db51efa0 100644 ---- a/arch/mips/kernel/syscalls/syscall_n64.tbl -+++ b/arch/mips/kernel/syscalls/syscall_n64.tbl -@@ -365,3 +365,4 @@ - 448 n64 process_mrelease sys_process_mrelease - 449 n64 futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 n64 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl -index 8f243e35a7b2..674cb940bd15 100644 ---- a/arch/mips/kernel/syscalls/syscall_o32.tbl -+++ b/arch/mips/kernel/syscalls/syscall_o32.tbl -@@ -438,3 +438,4 @@ - 448 o32 process_mrelease sys_process_mrelease - 449 o32 futex_waitv sys_futex_waitv - 450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node -+451 o32 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl -index 8a99c998da9b..429b129d5d46 100644 ---- a/arch/parisc/kernel/syscalls/syscall.tbl -+++ b/arch/parisc/kernel/syscalls/syscall.tbl -@@ -448,3 +448,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl -index 2600b4237292..bb2f71a36941 100644 ---- a/arch/powerpc/kernel/syscalls/syscall.tbl -+++ b/arch/powerpc/kernel/syscalls/syscall.tbl -@@ -530,3 +530,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl -index 799147658dee..1cd523748bd2 100644 ---- a/arch/s390/kernel/syscalls/syscall.tbl -+++ b/arch/s390/kernel/syscalls/syscall.tbl -@@ -453,3 +453,4 @@ - 448 common process_mrelease sys_process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm sys_pmadv_ksm -diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl -index 2de85c977f54..cfc75fa43eae 100644 ---- a/arch/sh/kernel/syscalls/syscall.tbl -+++ b/arch/sh/kernel/syscalls/syscall.tbl -@@ -453,3 +453,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl -index 4398cc6fb68d..d2c0a6426f6b 100644 ---- a/arch/sparc/kernel/syscalls/syscall.tbl -+++ b/arch/sparc/kernel/syscalls/syscall.tbl -@@ -496,3 +496,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu -index 542377cd419d..22b919cdb6d1 100644 ---- a/arch/x86/Kconfig.cpu -+++ b/arch/x86/Kconfig.cpu -@@ -157,7 +157,7 @@ config MPENTIUM4 - - - config MK6 -- bool "K6/K6-II/K6-III" -+ bool "AMD K6/K6-II/K6-III" - depends on X86_32 - help - Select this for an AMD K6-family processor. Enables use of -@@ -165,7 +165,7 @@ config MK6 - flags to GCC. - - config MK7 -- bool "Athlon/Duron/K7" -+ bool "AMD Athlon/Duron/K7" - depends on X86_32 - help - Select this for an AMD Athlon K7-family processor. Enables use of -@@ -173,12 +173,98 @@ config MK7 - flags to GCC. - - config MK8 -- bool "Opteron/Athlon64/Hammer/K8" -+ bool "AMD Opteron/Athlon64/Hammer/K8" - help - Select this for an AMD Opteron or Athlon64 Hammer-family processor. - Enables use of some extended instructions, and passes appropriate - optimization flags to GCC. - -+config MK8SSE3 -+ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" -+ help -+ Select this for improved AMD Opteron or Athlon64 Hammer-family processors. -+ Enables use of some extended instructions, and passes appropriate -+ optimization flags to GCC. -+ -+config MK10 -+ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" -+ help -+ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, -+ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. -+ Enables use of some extended instructions, and passes appropriate -+ optimization flags to GCC. -+ -+config MBARCELONA -+ bool "AMD Barcelona" -+ help -+ Select this for AMD Family 10h Barcelona processors. -+ -+ Enables -march=barcelona -+ -+config MBOBCAT -+ bool "AMD Bobcat" -+ help -+ Select this for AMD Family 14h Bobcat processors. -+ -+ Enables -march=btver1 -+ -+config MJAGUAR -+ bool "AMD Jaguar" -+ help -+ Select this for AMD Family 16h Jaguar processors. -+ -+ Enables -march=btver2 -+ -+config MBULLDOZER -+ bool "AMD Bulldozer" -+ help -+ Select this for AMD Family 15h Bulldozer processors. -+ -+ Enables -march=bdver1 -+ -+config MPILEDRIVER -+ bool "AMD Piledriver" -+ help -+ Select this for AMD Family 15h Piledriver processors. -+ -+ Enables -march=bdver2 -+ -+config MSTEAMROLLER -+ bool "AMD Steamroller" -+ help -+ Select this for AMD Family 15h Steamroller processors. -+ -+ Enables -march=bdver3 -+ -+config MEXCAVATOR -+ bool "AMD Excavator" -+ help -+ Select this for AMD Family 15h Excavator processors. -+ -+ Enables -march=bdver4 -+ -+config MZEN -+ bool "AMD Zen" -+ help -+ Select this for AMD Family 17h Zen processors. -+ -+ Enables -march=znver1 -+ -+config MZEN2 -+ bool "AMD Zen 2" -+ help -+ Select this for AMD Family 17h Zen 2 processors. -+ -+ Enables -march=znver2 -+ -+config MZEN3 -+ bool "AMD Zen 3" -+ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ help -+ Select this for AMD Family 19h Zen 3 processors. -+ -+ Enables -march=znver3 -+ - config MCRUSOE - bool "Crusoe" - depends on X86_32 -@@ -270,7 +356,7 @@ config MPSC - in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. - - config MCORE2 -- bool "Core 2/newer Xeon" -+ bool "Intel Core 2" - help - - Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and -@@ -278,6 +364,8 @@ config MCORE2 - family in /proc/cpuinfo. Newer ones have 6 and older ones 15 - (not a typo) - -+ Enables -march=core2 -+ - config MATOM - bool "Intel Atom" - help -@@ -287,6 +375,182 @@ config MATOM - accordingly optimized code. Use a recent GCC with specific Atom - support in order to fully benefit from selecting this option. - -+config MNEHALEM -+ bool "Intel Nehalem" -+ select X86_P6_NOP -+ help -+ -+ Select this for 1st Gen Core processors in the Nehalem family. -+ -+ Enables -march=nehalem -+ -+config MWESTMERE -+ bool "Intel Westmere" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Westmere formerly Nehalem-C family. -+ -+ Enables -march=westmere -+ -+config MSILVERMONT -+ bool "Intel Silvermont" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Silvermont platform. -+ -+ Enables -march=silvermont -+ -+config MGOLDMONT -+ bool "Intel Goldmont" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Goldmont platform including Apollo Lake and Denverton. -+ -+ Enables -march=goldmont -+ -+config MGOLDMONTPLUS -+ bool "Intel Goldmont Plus" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Goldmont Plus platform including Gemini Lake. -+ -+ Enables -march=goldmont-plus -+ -+config MSANDYBRIDGE -+ bool "Intel Sandy Bridge" -+ select X86_P6_NOP -+ help -+ -+ Select this for 2nd Gen Core processors in the Sandy Bridge family. -+ -+ Enables -march=sandybridge -+ -+config MIVYBRIDGE -+ bool "Intel Ivy Bridge" -+ select X86_P6_NOP -+ help -+ -+ Select this for 3rd Gen Core processors in the Ivy Bridge family. -+ -+ Enables -march=ivybridge -+ -+config MHASWELL -+ bool "Intel Haswell" -+ select X86_P6_NOP -+ help -+ -+ Select this for 4th Gen Core processors in the Haswell family. -+ -+ Enables -march=haswell -+ -+config MBROADWELL -+ bool "Intel Broadwell" -+ select X86_P6_NOP -+ help -+ -+ Select this for 5th Gen Core processors in the Broadwell family. -+ -+ Enables -march=broadwell -+ -+config MSKYLAKE -+ bool "Intel Skylake" -+ select X86_P6_NOP -+ help -+ -+ Select this for 6th Gen Core processors in the Skylake family. -+ -+ Enables -march=skylake -+ -+config MSKYLAKEX -+ bool "Intel Skylake X" -+ select X86_P6_NOP -+ help -+ -+ Select this for 6th Gen Core processors in the Skylake X family. -+ -+ Enables -march=skylake-avx512 -+ -+config MCANNONLAKE -+ bool "Intel Cannon Lake" -+ select X86_P6_NOP -+ help -+ -+ Select this for 8th Gen Core processors -+ -+ Enables -march=cannonlake -+ -+config MICELAKE -+ bool "Intel Ice Lake" -+ select X86_P6_NOP -+ help -+ -+ Select this for 10th Gen Core processors in the Ice Lake family. -+ -+ Enables -march=icelake-client -+ -+config MCASCADELAKE -+ bool "Intel Cascade Lake" -+ select X86_P6_NOP -+ help -+ -+ Select this for Xeon processors in the Cascade Lake family. -+ -+ Enables -march=cascadelake -+ -+config MCOOPERLAKE -+ bool "Intel Cooper Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) -+ select X86_P6_NOP -+ help -+ -+ Select this for Xeon processors in the Cooper Lake family. -+ -+ Enables -march=cooperlake -+ -+config MTIGERLAKE -+ bool "Intel Tiger Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) -+ select X86_P6_NOP -+ help -+ -+ Select this for third-generation 10 nm process processors in the Tiger Lake family. -+ -+ Enables -march=tigerlake -+ -+config MSAPPHIRERAPIDS -+ bool "Intel Sapphire Rapids" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help -+ -+ Select this for third-generation 10 nm process processors in the Sapphire Rapids family. -+ -+ Enables -march=sapphirerapids -+ -+config MROCKETLAKE -+ bool "Intel Rocket Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help -+ -+ Select this for eleventh-generation processors in the Rocket Lake family. -+ -+ Enables -march=rocketlake -+ -+config MALDERLAKE -+ bool "Intel Alder Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help -+ -+ Select this for twelfth-generation processors in the Alder Lake family. -+ -+ Enables -march=alderlake -+ - config GENERIC_CPU - bool "Generic-x86-64" - depends on X86_64 -@@ -294,6 +558,50 @@ config GENERIC_CPU - Generic x86-64 CPU. - Run equally well on all x86-64 CPUs. - -+config GENERIC_CPU2 -+ bool "Generic-x86-64-v2" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64 CPU. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v2. -+ -+config GENERIC_CPU3 -+ bool "Generic-x86-64-v3" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64-v3 CPU with v3 instructions. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v3. -+ -+config GENERIC_CPU4 -+ bool "Generic-x86-64-v4" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64 CPU with v4 instructions. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v4. -+ -+config MNATIVE_INTEL -+ bool "Intel-Native optimizations autodetected by the compiler" -+ help -+ -+ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects -+ the optimum settings to use based on your processor. Do NOT use this -+ for AMD CPUs. Intel Only! -+ -+ Enables -march=native -+ -+config MNATIVE_AMD -+ bool "AMD-Native optimizations autodetected by the compiler" -+ help -+ -+ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects -+ the optimum settings to use based on your processor. Do NOT use this -+ for Intel CPUs. AMD Only! -+ -+ Enables -march=native -+ - endchoice - - config X86_GENERIC -@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT - config X86_L1_CACHE_SHIFT - int - default "7" if MPENTIUM4 || MPSC -- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU -+ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 - default "4" if MELAN || M486SX || M486 || MGEODEGX1 - default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX - -@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 - - config X86_INTEL_USERCOPY - def_bool y -- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 -+ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL - - config X86_USE_PPRO_CHECKSUM - def_bool y -- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM -+ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD - - # - # P6_NOPs are a relatively minor optimization that require a family >= -@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM - config X86_P6_NOP - def_bool y - depends on X86_64 -- depends on (MCORE2 || MPENTIUM4 || MPSC) -+ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) - - config X86_TSC - def_bool y -- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 -+ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 - - config X86_CMPXCHG64 - def_bool y -- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 -+ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD - - # this should be set for all -march=.. options where the compiler - # generates cmov. - config X86_CMOV - def_bool y -- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) -+ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) - - config X86_MINIMUM_CPU_FAMILY - int - default "64" if X86_64 -- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) -+ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) - default "5" if X86_32 && X86_CMPXCHG64 - default "4" - -diff --git a/arch/x86/Makefile b/arch/x86/Makefile -index bafbd905e6e7..476ab926c33e 100644 ---- a/arch/x86/Makefile -+++ b/arch/x86/Makefile -@@ -150,8 +150,44 @@ else - # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) - cflags-$(CONFIG_MK8) += -march=k8 - cflags-$(CONFIG_MPSC) += -march=nocona -- cflags-$(CONFIG_MCORE2) += -march=core2 -- cflags-$(CONFIG_MATOM) += -march=atom -+ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3 -+ cflags-$(CONFIG_MK10) += -march=amdfam10 -+ cflags-$(CONFIG_MBARCELONA) += -march=barcelona -+ cflags-$(CONFIG_MBOBCAT) += -march=btver1 -+ cflags-$(CONFIG_MJAGUAR) += -march=btver2 -+ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1 -+ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm -+ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm -+ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm -+ cflags-$(CONFIG_MZEN) += -march=znver1 -+ cflags-$(CONFIG_MZEN2) += -march=znver2 -+ cflags-$(CONFIG_MZEN3) += -march=znver3 -+ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native -+ cflags-$(CONFIG_MNATIVE_AMD) += -march=native -+ cflags-$(CONFIG_MATOM) += -march=bonnell -+ cflags-$(CONFIG_MCORE2) += -march=core2 -+ cflags-$(CONFIG_MNEHALEM) += -march=nehalem -+ cflags-$(CONFIG_MWESTMERE) += -march=westmere -+ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont -+ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont -+ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus -+ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge -+ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge -+ cflags-$(CONFIG_MHASWELL) += -march=haswell -+ cflags-$(CONFIG_MBROADWELL) += -march=broadwell -+ cflags-$(CONFIG_MSKYLAKE) += -march=skylake -+ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512 -+ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake -+ cflags-$(CONFIG_MICELAKE) += -march=icelake-client -+ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake -+ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake -+ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake -+ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids -+ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake -+ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake -+ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2 -+ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3 -+ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4 - cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic - KBUILD_CFLAGS += $(cflags-y) - -diff --git a/arch/x86/Makefile.postlink b/arch/x86/Makefile.postlink -new file mode 100644 -index 000000000000..4650aaf6d8b3 ---- /dev/null -+++ b/arch/x86/Makefile.postlink -@@ -0,0 +1,41 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# =========================================================================== -+# Post-link x86 pass -+# =========================================================================== -+# -+# 1. Separate relocations from vmlinux into vmlinux.relocs. -+# 2. Strip relocations from vmlinux. -+ -+PHONY := __archpost -+__archpost: -+ -+-include include/config/auto.conf -+include scripts/Kbuild.include -+ -+CMD_RELOCS = arch/x86/tools/relocs -+quiet_cmd_relocs = RELOCS $@.relocs -+ cmd_relocs = $(CMD_RELOCS) $@ > $@.relocs;$(CMD_RELOCS) --abs-relocs $@ -+ -+quiet_cmd_strip_relocs = RSTRIP $@ -+ cmd_strip_relocs = objcopy --remove-relocations='*' $@ -+ -+# `@true` prevents complaint when there is nothing to be done -+ -+vmlinux: FORCE -+ @true -+ifeq ($(CONFIG_X86_NEED_RELOCS),y) -+ $(call cmd,relocs) -+ $(call cmd,strip_relocs) -+endif -+ -+%.ko: FORCE -+ @true -+ -+clean: -+ @rm -f vmlinux.relocs -+ -+PHONY += FORCE clean -+ -+FORCE: -+ -+.PHONY: $(PHONY) -diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile -index 35ce1a64068b..eba7709d75ae 100644 ---- a/arch/x86/boot/compressed/Makefile -+++ b/arch/x86/boot/compressed/Makefile -@@ -120,14 +120,12 @@ $(obj)/vmlinux.bin: vmlinux FORCE - - targets += $(patsubst $(obj)/%,%,$(vmlinux-objs-y)) vmlinux.bin.all vmlinux.relocs - --CMD_RELOCS = arch/x86/tools/relocs --quiet_cmd_relocs = RELOCS $@ -- cmd_relocs = $(CMD_RELOCS) $< > $@;$(CMD_RELOCS) --abs-relocs $< --$(obj)/vmlinux.relocs: vmlinux FORCE -- $(call if_changed,relocs) -+# vmlinux.relocs is created by the vmlinux postlink step. -+vmlinux.relocs: vmlinux -+ @true - - vmlinux.bin.all-y := $(obj)/vmlinux.bin --vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs -+vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += vmlinux.relocs - - $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,gzip) -diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl -index 320480a8db4f..331aaf1a782f 100644 ---- a/arch/x86/entry/syscalls/syscall_32.tbl -+++ b/arch/x86/entry/syscalls/syscall_32.tbl -@@ -455,3 +455,4 @@ - 448 i386 process_mrelease sys_process_mrelease - 449 i386 futex_waitv sys_futex_waitv - 450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node -+451 i386 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl -index c84d12608cd2..14902db4c01f 100644 ---- a/arch/x86/entry/syscalls/syscall_64.tbl -+++ b/arch/x86/entry/syscalls/syscall_64.tbl -@@ -372,6 +372,7 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm - - # - # Due to a historical design error, certain syscalls are numbered differently -diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h -index 6674bdb096f3..e278872e9187 100644 ---- a/arch/x86/include/asm/msr-index.h -+++ b/arch/x86/include/asm/msr-index.h -@@ -569,6 +569,7 @@ - #define MSR_AMD_CPPC_CAP2 0xc00102b2 - #define MSR_AMD_CPPC_REQ 0xc00102b3 - #define MSR_AMD_CPPC_STATUS 0xc00102b4 -+#define MSR_AMD_CPPC_HW_CTL 0xc0010015 - - #define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) - #define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) -@@ -579,12 +580,18 @@ - #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) - #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) - #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) -+#define AMD_CPPC_PRECISION_BOOST_BIT 25 -+#define AMD_CPPC_PRECISION_BOOST_ENABLED BIT_ULL(AMD_CPPC_PRECISION_BOOST_BIT) - - /* AMD Performance Counter Global Status and Control MSRs */ - #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 - #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 - #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 - -+#define AMD_CPPC_EPP_PERFORMANCE 0x00 -+#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 -+#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF -+#define AMD_CPPC_EPP_POWERSAVE 0xFF - /* Fam 17h MSRs */ - #define MSR_F17H_IRPERF 0xc00000e9 - -diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h -index 458c891a8273..d86eb1ebf59f 100644 ---- a/arch/x86/include/asm/topology.h -+++ b/arch/x86/include/asm/topology.h -@@ -175,6 +175,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled; - - /* Interface to set priority of a cpu */ - void sched_set_itmt_core_prio(int prio, int core_cpu); -+void sched_set_itmt_power_ratio(int power_ratio, int core_cpu); - - /* Interface to notify scheduler that system supports ITMT */ - int sched_set_itmt_support(void); -diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h -index 57b1a7034c64..e2c45674f989 100644 ---- a/arch/x86/include/asm/vdso/processor.h -+++ b/arch/x86/include/asm/vdso/processor.h -@@ -10,7 +10,7 @@ - /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ - static __always_inline void rep_nop(void) - { -- asm volatile("rep; nop" ::: "memory"); -+ asm volatile("lfence" ::: "memory"); - } - - static __always_inline void cpu_relax(void) -diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h -index 75884d2cdec3..4e6a08d4c7e5 100644 ---- a/arch/x86/include/asm/vermagic.h -+++ b/arch/x86/include/asm/vermagic.h -@@ -17,6 +17,48 @@ - #define MODULE_PROC_FAMILY "586MMX " - #elif defined CONFIG_MCORE2 - #define MODULE_PROC_FAMILY "CORE2 " -+#elif defined CONFIG_MNATIVE_INTEL -+#define MODULE_PROC_FAMILY "NATIVE_INTEL " -+#elif defined CONFIG_MNATIVE_AMD -+#define MODULE_PROC_FAMILY "NATIVE_AMD " -+#elif defined CONFIG_MNEHALEM -+#define MODULE_PROC_FAMILY "NEHALEM " -+#elif defined CONFIG_MWESTMERE -+#define MODULE_PROC_FAMILY "WESTMERE " -+#elif defined CONFIG_MSILVERMONT -+#define MODULE_PROC_FAMILY "SILVERMONT " -+#elif defined CONFIG_MGOLDMONT -+#define MODULE_PROC_FAMILY "GOLDMONT " -+#elif defined CONFIG_MGOLDMONTPLUS -+#define MODULE_PROC_FAMILY "GOLDMONTPLUS " -+#elif defined CONFIG_MSANDYBRIDGE -+#define MODULE_PROC_FAMILY "SANDYBRIDGE " -+#elif defined CONFIG_MIVYBRIDGE -+#define MODULE_PROC_FAMILY "IVYBRIDGE " -+#elif defined CONFIG_MHASWELL -+#define MODULE_PROC_FAMILY "HASWELL " -+#elif defined CONFIG_MBROADWELL -+#define MODULE_PROC_FAMILY "BROADWELL " -+#elif defined CONFIG_MSKYLAKE -+#define MODULE_PROC_FAMILY "SKYLAKE " -+#elif defined CONFIG_MSKYLAKEX -+#define MODULE_PROC_FAMILY "SKYLAKEX " -+#elif defined CONFIG_MCANNONLAKE -+#define MODULE_PROC_FAMILY "CANNONLAKE " -+#elif defined CONFIG_MICELAKE -+#define MODULE_PROC_FAMILY "ICELAKE " -+#elif defined CONFIG_MCASCADELAKE -+#define MODULE_PROC_FAMILY "CASCADELAKE " -+#elif defined CONFIG_MCOOPERLAKE -+#define MODULE_PROC_FAMILY "COOPERLAKE " -+#elif defined CONFIG_MTIGERLAKE -+#define MODULE_PROC_FAMILY "TIGERLAKE " -+#elif defined CONFIG_MSAPPHIRERAPIDS -+#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " -+#elif defined CONFIG_ROCKETLAKE -+#define MODULE_PROC_FAMILY "ROCKETLAKE " -+#elif defined CONFIG_MALDERLAKE -+#define MODULE_PROC_FAMILY "ALDERLAKE " - #elif defined CONFIG_MATOM - #define MODULE_PROC_FAMILY "ATOM " - #elif defined CONFIG_M686 -@@ -35,6 +77,30 @@ - #define MODULE_PROC_FAMILY "K7 " - #elif defined CONFIG_MK8 - #define MODULE_PROC_FAMILY "K8 " -+#elif defined CONFIG_MK8SSE3 -+#define MODULE_PROC_FAMILY "K8SSE3 " -+#elif defined CONFIG_MK10 -+#define MODULE_PROC_FAMILY "K10 " -+#elif defined CONFIG_MBARCELONA -+#define MODULE_PROC_FAMILY "BARCELONA " -+#elif defined CONFIG_MBOBCAT -+#define MODULE_PROC_FAMILY "BOBCAT " -+#elif defined CONFIG_MBULLDOZER -+#define MODULE_PROC_FAMILY "BULLDOZER " -+#elif defined CONFIG_MPILEDRIVER -+#define MODULE_PROC_FAMILY "PILEDRIVER " -+#elif defined CONFIG_MSTEAMROLLER -+#define MODULE_PROC_FAMILY "STEAMROLLER " -+#elif defined CONFIG_MJAGUAR -+#define MODULE_PROC_FAMILY "JAGUAR " -+#elif defined CONFIG_MEXCAVATOR -+#define MODULE_PROC_FAMILY "EXCAVATOR " -+#elif defined CONFIG_MZEN -+#define MODULE_PROC_FAMILY "ZEN " -+#elif defined CONFIG_MZEN2 -+#define MODULE_PROC_FAMILY "ZEN2 " -+#elif defined CONFIG_MZEN3 -+#define MODULE_PROC_FAMILY "ZEN3 " - #elif defined CONFIG_MELAN - #define MODULE_PROC_FAMILY "ELAN " - #elif defined CONFIG_MCRUSOE -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index 62f6b8b7c4a5..f9c9b5850847 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -936,7 +936,9 @@ void __init alternative_instructions(void) - * Then patch alternatives, such that those paravirt calls that are in - * alternatives can be overwritten by their immediate fragments. - */ -+ printk("clr: Applying alternatives\n"); - apply_alternatives(__alt_instructions, __alt_instructions_end); -+ printk("clr: Applying alternatives done\n"); - - apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); - -diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c -index fbaf12e43f41..c8c2d6f1a8ac 100644 ---- a/arch/x86/kernel/cpu/intel_epb.c -+++ b/arch/x86/kernel/cpu/intel_epb.c -@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev, - if (ret < 0) - return ret; - -+ /* update the ITMT scheduler logic to use the power policy data */ -+ /* scale the val up by 2 so the range is 224 - 256 */ -+ sched_set_itmt_power_ratio(256 - val * 2, cpu); -+ - return count; - } - -diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c -index ad57e0e4d674..e2ed33c2bd4d 100644 ---- a/arch/x86/kernel/cpu/microcode/core.c -+++ b/arch/x86/kernel/cpu/microcode/core.c -@@ -44,6 +44,8 @@ - - static struct microcode_ops *microcode_ops; - static bool dis_ucode_ldr = true; -+bool ucode_rollback = false; -+int enable_rollback = 0; - - bool initrd_gone; - -@@ -80,6 +82,26 @@ static u32 final_levels[] = { - 0, /* T-101 terminator */ - }; - -+static int __init ucode_setup(char *str) -+{ -+ if (!str) -+ return -EINVAL; -+ -+ while (*str) { -+ if (!strncmp(str, "rollback", 8)) { -+ enable_rollback = 1; -+ pr_info("Microcode Rollback Enabled\n"); -+ } -+ str += strcspn(str, ","); -+ while (*str == ',') -+ str++; -+ } -+ return 0; -+} -+ -+__setup("ucode=", ucode_setup); -+ -+ - /* - * Check the current patch level on this CPU. - * -@@ -512,6 +534,7 @@ static ssize_t reload_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) - { -+ struct cpuinfo_x86 *c = &boot_cpu_data; - enum ucode_state tmp_ret = UCODE_OK; - int bsp = boot_cpu_data.cpu_index; - unsigned long val; -@@ -521,7 +544,7 @@ static ssize_t reload_store(struct device *dev, - if (ret) - return ret; - -- if (val != 1) -+ if (!val || val > 2) - return size; - - cpus_read_lock(); -@@ -529,6 +552,20 @@ static ssize_t reload_store(struct device *dev, - ret = check_online_cpus(); - if (ret) - goto put; -+ /* -+ * Check if the vendor is Intel to permit reloading -+ * microcode even if the revision is unchanged. -+ * This is typically used during development of microcode -+ * and changing rev is a pain. -+ */ -+ if ((val == 2) && ((c->x86_vendor != X86_VENDOR_INTEL) || -+ !enable_rollback)) -+ return size; -+ else if (val == 2) { -+ mutex_lock(µcode_mutex); -+ ucode_rollback = true; -+ mutex_unlock(µcode_mutex); -+ } - - tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); - if (tmp_ret != UCODE_NEW) -@@ -539,6 +576,7 @@ static ssize_t reload_store(struct device *dev, - mutex_unlock(µcode_mutex); - - put: -+ ucode_rollback = false; - cpus_read_unlock(); - - if (ret == 0) -diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c -index 025c8f0cd948..60473f577a25 100644 ---- a/arch/x86/kernel/cpu/microcode/intel.c -+++ b/arch/x86/kernel/cpu/microcode/intel.c -@@ -44,6 +44,7 @@ static struct microcode_intel *intel_ucode_patch; - - /* last level cache size per core */ - static int llc_size_per_core; -+extern bool ucode_rollback; - - /* - * Returns 1 if update has been found, 0 otherwise. -@@ -80,7 +81,7 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev - { - struct microcode_header_intel *mc_hdr = mc; - -- if (mc_hdr->rev <= new_rev) -+ if (!ucode_rollback && mc_hdr->rev <= new_rev) - return 0; - - return find_matching_signature(mc, csig, cpf); -@@ -120,7 +121,7 @@ static void save_microcode_patch(struct ucode_cpu_info *uci, void *data, unsigne - if (find_matching_signature(data, sig, pf)) { - prev_found = true; - -- if (mc_hdr->rev <= mc_saved_hdr->rev) -+ if (!ucode_rollback && mc_hdr->rev <= mc_saved_hdr->rev) - continue; - - p = memdup_patch(data, size); -@@ -649,7 +650,7 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci) - - phdr = (struct microcode_header_intel *)iter->data; - -- if (phdr->rev <= uci->cpu_sig.rev) -+ if (!ucode_rollback && phdr->rev <= uci->cpu_sig.rev) - continue; - - if (!find_matching_signature(phdr, -@@ -734,10 +735,11 @@ static enum ucode_state apply_microcode_intel(int cpu) - * already. - */ - rev = intel_get_microcode_revision(); -- if (rev >= mc->hdr.rev) { -+ if (!ucode_rollback && rev >= mc->hdr.rev) { - ret = UCODE_OK; - goto out; -- } -+ } else if (ucode_rollback) -+ ret = UCODE_OK; - - /* - * Writeback and invalidate caches before updating microcode to avoid -@@ -756,7 +758,7 @@ static enum ucode_state apply_microcode_intel(int cpu) - return UCODE_ERROR; - } - -- if (bsp && rev != prev_rev) { -+ if (bsp && ((rev != prev_rev) || ucode_rollback)) { - pr_info("updated to revision 0x%x, date = %04x-%02x-%02x\n", - rev, - mc->hdr.date & 0xffff, -diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c -index 9ff480e94511..d4326e050fb7 100644 ---- a/arch/x86/kernel/itmt.c -+++ b/arch/x86/kernel/itmt.c -@@ -25,6 +25,7 @@ - - static DEFINE_MUTEX(itmt_update_mutex); - DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); -+DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio); - - /* Boolean to track if system has ITMT capabilities */ - static bool __read_mostly sched_itmt_capable; -@@ -169,7 +170,12 @@ void sched_clear_itmt_support(void) - - int arch_asym_cpu_priority(int cpu) - { -- return per_cpu(sched_core_priority, cpu); -+ int power_ratio = per_cpu(sched_power_ratio, cpu); -+ -+ /* a power ratio of 0 (uninitialized) is assumed to be maximum */ -+ if (power_ratio == 0) -+ power_ratio = 256 - 2 * 6; -+ return per_cpu(sched_core_priority, cpu) * power_ratio / 256; - } - - /** -@@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) - i++; - } - } -+ -+/** -+ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT -+ * @power_ratio: The power scaling ratio [1..256] for the core -+ * @core_cpu: The cpu number associated with the core -+ * -+ * Set a scaling to the cpu performance based on long term power -+ * settings (like EPB). -+ * -+ * Note this is for the policy not for the actual dynamic frequency; -+ * the frequency will increase itself as workloads run on a core. -+ */ -+ -+void sched_set_itmt_power_ratio(int power_ratio, int core_cpu) -+{ -+ int cpu; -+ -+ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) { -+ per_cpu(sched_power_ratio, cpu) = power_ratio; -+ } -+} -diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c -index ed8ac6bcbafb..d6fc5bdb0246 100644 ---- a/arch/x86/kernel/msr.c -+++ b/arch/x86/kernel/msr.c -@@ -48,7 +48,7 @@ enum allow_write_msrs { - MSR_WRITES_DEFAULT, - }; - --static enum allow_write_msrs allow_writes = MSR_WRITES_DEFAULT; -+static enum allow_write_msrs allow_writes = MSR_WRITES_ON; - - static ssize_t msr_read(struct file *file, char __user *buf, - size_t count, loff_t *ppos) -diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c -index cafacb2e58cc..c2f80184fd33 100644 ---- a/arch/x86/kernel/tsc.c -+++ b/arch/x86/kernel/tsc.c -@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void) - if (!constant_tsc || !mask) - return 0; - -+ if (cpu != 0) -+ return cpu_data(0).loops_per_jiffy; -+ - sibling = cpumask_any_but(mask, cpu); - if (sibling < nr_cpu_ids) - return cpu_data(sibling).loops_per_jiffy; -diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c -index fa71a5d12e87..c4bbd8c66134 100644 ---- a/arch/x86/mm/fault.c -+++ b/arch/x86/mm/fault.c -@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, - if (!printk_ratelimit()) - return; - -- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx", -+ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i", - loglvl, tsk->comm, task_pid_nr(tsk), address, -- (void *)regs->ip, (void *)regs->sp, error_code); -+ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id()); - - print_vma_addr(KERN_CONT " in ", regs->ip); - -diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl -index 52c94ab5c205..1518e261d882 100644 ---- a/arch/xtensa/kernel/syscalls/syscall.tbl -+++ b/arch/xtensa/kernel/syscalls/syscall.tbl -@@ -421,3 +421,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c -index 30b15a9a47c4..144bca006463 100644 ---- a/block/bfq-cgroup.c -+++ b/block/bfq-cgroup.c -@@ -254,17 +254,12 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - - #else /* CONFIG_BFQ_CGROUP_DEBUG */ - --void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -- blk_opf_t opf) { } - void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { } - void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { } - void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - u64 io_start_time_ns, blk_opf_t opf) { } - void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } --void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } --void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } - void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } --void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } - - #endif /* CONFIG_BFQ_CGROUP_DEBUG */ - -diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c -index c740b41fe0a4..5ea6245f0208 100644 ---- a/block/bfq-iosched.c -+++ b/block/bfq-iosched.c -@@ -1925,7 +1925,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, - bfqq->service_from_backlogged = 0; - bfq_clear_bfqq_softrt_update(bfqq); - -- bfq_add_bfqq_busy(bfqd, bfqq); -+ bfq_add_bfqq_busy(bfqq); - - /* - * Expire in-service queue if preemption may be needed for -@@ -2419,7 +2419,7 @@ static void bfq_remove_request(struct request_queue *q, - bfqq->next_rq = NULL; - - if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { -- bfq_del_bfqq_busy(bfqd, bfqq, false); -+ bfq_del_bfqq_busy(bfqq, false); - /* - * bfqq emptied. In normal operation, when - * bfqq is empty, bfqq->entity.service and -@@ -3098,7 +3098,7 @@ void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) - */ - if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && - bfqq != bfqd->in_service_queue) -- bfq_del_bfqq_busy(bfqd, bfqq, false); -+ bfq_del_bfqq_busy(bfqq, false); - - bfq_reassign_last_bfqq(bfqq, NULL); - -@@ -3908,7 +3908,7 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, - */ - bfqq->budget_timeout = jiffies; - -- bfq_del_bfqq_busy(bfqd, bfqq, true); -+ bfq_del_bfqq_busy(bfqq, true); - } else { - bfq_requeue_bfqq(bfqd, bfqq, true); - /* -@@ -5255,9 +5255,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) - struct hlist_node *n; - struct bfq_group *bfqg = bfqq_group(bfqq); - -- if (bfqq->bfqd) -- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", -- bfqq, bfqq->ref); -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); - - bfqq->ref--; - if (bfqq->ref) -@@ -5321,7 +5319,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) - hlist_del_init(&item->woken_list_node); - } - -- if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) -+ if (bfqq->bfqd->last_completed_rq_bfqq == bfqq) - bfqq->bfqd->last_completed_rq_bfqq = NULL; - - kmem_cache_free(bfq_pool, bfqq); -@@ -7463,6 +7461,7 @@ MODULE_ALIAS("bfq-iosched"); - static int __init bfq_init(void) - { - int ret; -+ char msg[60] = "BFQ I/O-scheduler: BFQ-CachyOS v5.19"; - - #ifdef CONFIG_BFQ_GROUP_IOSCHED - ret = blkcg_policy_register(&blkcg_policy_bfq); -@@ -7494,6 +7493,11 @@ static int __init bfq_init(void) - if (ret) - goto slab_kill; - -+#ifdef CONFIG_BFQ_GROUP_IOSCHED -+ strcat(msg, " (with cgroups support)"); -+#endif -+ pr_info("%s", msg); -+ - return 0; - - slab_kill: -diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h -index ad8e513d7e87..64ee618064ba 100644 ---- a/block/bfq-iosched.h -+++ b/block/bfq-iosched.h -@@ -993,20 +993,23 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); - /* ---------------- cgroups-support interface ---------------- */ - - void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq); --void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -- blk_opf_t opf); - void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf); - void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf); - void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - u64 io_start_time_ns, blk_opf_t opf); - void bfqg_stats_update_dequeue(struct bfq_group *bfqg); --void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); --void bfqg_stats_update_idle_time(struct bfq_group *bfqg); - void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg); --void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); - void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct bfq_group *bfqg); - -+#ifdef CONFIG_BFQ_CGROUP_DEBUG -+void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -+ blk_opf_t opf); -+void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); -+void bfqg_stats_update_idle_time(struct bfq_group *bfqg); -+void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); -+#endif -+ - void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg); - void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio); - void bfq_end_wr_async(struct bfq_data *bfqd); -@@ -1077,9 +1080,8 @@ void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); - void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - bool expiration); --void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -- bool expiration); --void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); -+void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration); -+void bfq_add_bfqq_busy(struct bfq_queue *bfqq); - - /* --------------- end of interface of B-WF2Q+ ---------------- */ - -diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c -index 983413cdefad..8fc3da4c23bb 100644 ---- a/block/bfq-wf2q.c -+++ b/block/bfq-wf2q.c -@@ -1651,9 +1651,10 @@ void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - * the service tree. As a special case, it can be invoked during an - * expiration. - */ --void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -- bool expiration) -+void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration) - { -+ struct bfq_data *bfqd = bfqq->bfqd; -+ - bfq_log_bfqq(bfqd, bfqq, "del from busy"); - - bfq_clear_bfqq_busy(bfqq); -@@ -1674,8 +1675,10 @@ void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, - /* - * Called when an inactive queue receives a new request. - */ --void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+void bfq_add_bfqq_busy(struct bfq_queue *bfqq) - { -+ struct bfq_data *bfqd = bfqq->bfqd; -+ - bfq_log_bfqq(bfqd, bfqq, "add to busy"); - - bfq_activate_bfqq(bfqd, bfqq); -diff --git a/block/blk-core.c b/block/blk-core.c -index 651057c4146b..88873f267b2a 100644 ---- a/block/blk-core.c -+++ b/block/blk-core.c -@@ -742,6 +742,9 @@ void submit_bio_noacct(struct bio *bio) - status = BLK_STS_OK; - goto end_io; - } -+ -+ if (bio->bi_opf & REQ_PREFLUSH) -+ current->fsync_count++; - } - - if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) -diff --git a/block/elevator.c b/block/elevator.c -index c319765892bb..b2687032a0e0 100644 ---- a/block/elevator.c -+++ b/block/elevator.c -@@ -640,8 +640,13 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) - - if (q->nr_hw_queues != 1 && - !blk_mq_is_shared_tags(q->tag_set->flags)) -+#if defined(CONFIG_CACHY) && defined(CONFIG_MQ_IOSCHED_KYBER) -+ return elevator_get(q, "kyber", false); -+#elif defined(CONFIG_CACHY) -+ return elevator_get(q, "mq-deadline", false); -+#else - return NULL; -- -+#endif - return elevator_get(q, "mq-deadline", false); - } - -diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c -index 1e15a9f25ae9..4801300d3c77 100644 ---- a/drivers/acpi/cppc_acpi.c -+++ b/drivers/acpi/cppc_acpi.c -@@ -424,6 +424,9 @@ bool acpi_cpc_valid(void) - struct cpc_desc *cpc_ptr; - int cpu; - -+ if (acpi_disabled) -+ return false; -+ - for_each_present_cpu(cpu) { - cpc_ptr = per_cpu(cpc_desc_ptr, cpu); - if (!cpc_ptr) -@@ -1320,6 +1323,132 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) - } - EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs); - -+/** -+ * cppc_get_epp_caps - Get the energy preference register value. -+ * @cpunum: CPU from which to get epp preference level. -+ * @perf_caps: Return address. -+ * -+ * Return: 0 for success, -EIO otherwise. -+ */ -+int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) -+{ -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); -+ struct cpc_register_resource *energy_perf_reg; -+ u64 energy_perf; -+ -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpunum); -+ return -ENODEV; -+ } -+ -+ energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; -+ -+ if (!CPC_SUPPORTED(energy_perf_reg)) -+ pr_warn("energy perf reg update is unsupported!\n"); -+ -+ if (CPC_IN_PCC(energy_perf_reg)) { -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = 0; -+ -+ if (pcc_ss_id < 0) -+ return -EIO; -+ -+ pcc_ss_data = pcc_data[pcc_ss_id]; -+ -+ down_write(&pcc_ss_data->pcc_lock); -+ -+ if (send_pcc_cmd(pcc_ss_id, CMD_READ) >= 0) { -+ cpc_read(cpunum, energy_perf_reg, &energy_perf); -+ perf_caps->energy_perf = energy_perf; -+ } else { -+ ret = -EIO; -+ } -+ -+ up_write(&pcc_ss_data->pcc_lock); -+ -+ return ret; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(cppc_get_epp_caps); -+ -+int cppc_set_auto_epp(int cpu, bool enable) -+{ -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); -+ struct cpc_register_resource *auto_sel_reg; -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = -EINVAL; -+ -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpu); -+ return -EINVAL; -+ } -+ -+ auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; -+ -+ if (CPC_IN_PCC(auto_sel_reg)) { -+ if (pcc_ss_id < 0) -+ return -EIO; -+ -+ ret = cpc_write(cpu, auto_sel_reg, enable); -+ if (ret) -+ return ret; -+ -+ pcc_ss_data = pcc_data[pcc_ss_id]; -+ -+ down_write(&pcc_ss_data->pcc_lock); -+ /* after writing CPC, transfer the ownership of PCC to platform */ -+ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); -+ up_write(&pcc_ss_data->pcc_lock); -+ return ret; -+ } -+ -+ return cpc_write(cpu, auto_sel_reg, enable); -+} -+EXPORT_SYMBOL_GPL(cppc_set_auto_epp); -+ -+/* -+ * Set Energy Performance Preference Register value through -+ * Performance Controls Interface -+ */ -+int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) -+{ -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); -+ struct cpc_register_resource *epp_set_reg; -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = -EINVAL; -+ -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpu); -+ return -EINVAL; -+ } -+ -+ epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; -+ -+ if (CPC_IN_PCC(epp_set_reg)) { -+ if (pcc_ss_id < 0) -+ return -EIO; -+ -+ ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); -+ if (ret) -+ return ret; -+ -+ pcc_ss_data = pcc_data[pcc_ss_id]; -+ -+ down_write(&pcc_ss_data->pcc_lock); -+ /* after writing CPC, transfer the ownership of PCC to platform */ -+ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); -+ up_write(&pcc_ss_data->pcc_lock); -+ } -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(cppc_set_epp_perf); -+ - /** - * cppc_set_enable - Set to enable CPPC on the processor by writing the - * Continuous Performance Control package EnableRegister field. -@@ -1355,7 +1484,7 @@ int cppc_set_enable(int cpu, bool enable) - pcc_ss_data = pcc_data[pcc_ss_id]; - - down_write(&pcc_ss_data->pcc_lock); -- /* after writing CPC, transfer the ownership of PCC to platfrom */ -+ /* after writing CPC, transfer the ownership of PCC to platform */ - ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); - up_write(&pcc_ss_data->pcc_lock); - return ret; -diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c -index cf8c7fd59ada..ad9bb5353dd3 100644 ---- a/drivers/ata/libahci.c -+++ b/drivers/ata/libahci.c -@@ -33,14 +33,14 @@ - #include "libata.h" - - static int ahci_skip_host_reset; --int ahci_ignore_sss; -+int ahci_ignore_sss=1; - EXPORT_SYMBOL_GPL(ahci_ignore_sss); - - module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444); - MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)"); - - module_param_named(ignore_sss, ahci_ignore_sss, int, 0444); --MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)"); -+MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])"); - - static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy, - unsigned hints); -diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c -index 46cbe4471e78..dd90591e51ba 100644 ---- a/drivers/base/arch_topology.c -+++ b/drivers/base/arch_topology.c -@@ -353,7 +353,7 @@ void topology_init_cpu_capacity_cppc(void) - struct cppc_perf_caps perf_caps; - int cpu; - -- if (likely(acpi_disabled || !acpi_cpc_valid())) -+ if (likely(!acpi_cpc_valid())) - return; - - raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), -diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c -index 7c3590fd97c2..bb4880e10581 100644 ---- a/drivers/base/firmware_loader/main.c -+++ b/drivers/base/firmware_loader/main.c -@@ -470,6 +470,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, - static char fw_path_para[256]; - static const char * const fw_path[] = { - fw_path_para, -+ "/etc/firmware/" UTS_RELEASE, -+ "/etc/firmware", - "/lib/firmware/updates/" UTS_RELEASE, - "/lib/firmware/updates", - "/lib/firmware/" UTS_RELEASE, -diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig -index d4100b0c083e..8739ea13161f 100644 ---- a/drivers/block/zram/Kconfig -+++ b/drivers/block/zram/Kconfig -@@ -78,3 +78,21 @@ config ZRAM_MEMORY_TRACKING - /sys/kernel/debug/zram/zramX/block_state. - - See Documentation/admin-guide/blockdev/zram.rst for more information. -+ -+config ZRAM_ENTROPY -+ bool "Use entropy optimization for zram" -+ depends on ZRAM && ZRAM_DEF_COMP_ZSTD -+ help -+ With this feature, entropy will be calculated for each page. -+ Pages above ZRAM_ENTROPY_THRESHOLD entropy will be -+ stored uncompressed. Use this feature if you need a performance -+ boost and a small loss in compression. -+ -+config ZRAM_ENTROPY_THRESHOLD -+ int -+ depends on ZRAM && ZRAM_ENTROPY -+ default 100000 if ZRAM_DEF_COMP_ZSTD -+ help -+ Pages with entropy above ZRAM_ENTROPY_THRESHOLD will be stored -+ uncompressed. The default value was chosen as a result a lot of -+ experiments. You can try set your own value. -diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c -index 226ea76cc819..d02e37e7afd9 100644 ---- a/drivers/block/zram/zram_drv.c -+++ b/drivers/block/zram/zram_drv.c -@@ -1347,6 +1347,35 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, - return ret; - } - -+ -+#ifdef CONFIG_ZRAM_ENTROPY -+static inline u32 ilog2_w(u64 n) -+{ -+ return ilog2(n * n * n * n); -+} -+ -+static inline s32 shannon_entropy(const u8 *src) -+{ -+ s32 entropy_sum = 0; -+ u32 sz_base, i; -+ u16 entropy_count[256] = { 0 }; -+ -+ for (i = 0; i < PAGE_SIZE; ++i) -+ entropy_count[src[i]]++; -+ -+ sz_base = ilog2_w(PAGE_SIZE); -+ for (i = 0; i < ARRAY_SIZE(entropy_count); ++i) { -+ if (entropy_count[i] > 0) { -+ s32 p = entropy_count[i]; -+ -+ entropy_sum += p * (sz_base - ilog2_w((u64)p)); -+ } -+ } -+ -+ return entropy_sum; -+} -+#endif -+ - static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - u32 index, struct bio *bio) - { -@@ -1373,7 +1402,17 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - compress_again: - zstrm = zcomp_stream_get(zram->comp); - src = kmap_atomic(page); -+ -+#ifdef CONFIG_ZRAM_ENTROPY -+ /* Just save this page uncompressible */ -+ if (shannon_entropy((const u8 *)src) > CONFIG_ZRAM_ENTROPY_THRESHOLD) -+ comp_len = PAGE_SIZE; -+ else -+ ret = zcomp_compress(zstrm, src, &comp_len); -+#else - ret = zcomp_compress(zstrm, src, &comp_len); -+#endif -+ - kunmap_atomic(src); - - if (unlikely(ret)) { -diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c -index 9ac75c1cde9c..e8cd2d2b7cf8 100644 ---- a/drivers/cpufreq/amd-pstate.c -+++ b/drivers/cpufreq/amd-pstate.c -@@ -31,23 +31,18 @@ - #include - #include - #include --#include - #include - #include - #include - #include - --#include - #include - - #include --#include --#include --#include - #include "amd-pstate-trace.h" - --#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 --#define AMD_PSTATE_TRANSITION_DELAY 500 -+#define AMD_PSTATE_TRANSITION_LATENCY 20000 -+#define AMD_PSTATE_TRANSITION_DELAY 1000 - - /* - * TODO: We need more time to fine tune processors with shared memory solution -@@ -63,7 +58,13 @@ module_param(shared_mem, bool, 0444); - MODULE_PARM_DESC(shared_mem, - "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); - --static struct cpufreq_driver amd_pstate_driver; -+static bool epp_enabled = true; -+module_param(epp_enabled, bool, 0444); -+MODULE_PARM_DESC(epp_enabled, -+ "load amd_pstate or amd_pstate_epp (true = amd_pstate_epp driver instance (default), false = amd_pstate driver instance)"); -+ -+static struct cpufreq_driver *default_pstate_driver; -+static struct amd_cpudata **all_cpu_data; - - /** - * struct amd_aperf_mperf -@@ -75,6 +76,7 @@ struct amd_aperf_mperf { - u64 aperf; - u64 mperf; - u64 tsc; -+ u64 time; - }; - - /** -@@ -97,6 +99,20 @@ struct amd_aperf_mperf { - * @prev: Last Aperf/Mperf/tsc count value read from register - * @freq: current cpu frequency value - * @boost_supported: check whether the Processor or SBIOS supports boost mode -+ * @precision_boost_off: the core performance boost disabled state -+ * @cppc_hw_conf_cached: the cached hardware configuration register -+ * @epp_powersave: Last saved CPPC energy performance preference -+ * when policy switched to performance -+ * @epp_policy: Last saved policy used to set energy-performance preference -+ * @epp_cached: Cached CPPC energy-performance preference value -+ * @policy: Cpufreq policy value -+ * @sched_flags: Store scheduler flags for possible cross CPU update -+ * @update_util_set: CPUFreq utility callback is set -+ * @last_update: Time stamp of the last performance state update -+ * @cppc_boost_min: Last CPPC boosted min performance state -+ * @cppc_cap1_cached: Cached value of the last CPPC Capabilities MSR -+ * @update_util: Cpufreq utility callback information -+ * @suspended: Whether or not the driver has been suspended. - * - * The amd_cpudata is key private data for each CPU thread in AMD P-State, and - * represents all the attributes and goals that AMD P-State requests at runtime. -@@ -120,10 +136,200 @@ struct amd_cpudata { - struct amd_aperf_mperf cur; - struct amd_aperf_mperf prev; - -- u64 freq; -+ u64 freq; - bool boost_supported; -+ bool precision_boost_off; -+ u64 cppc_hw_conf_cached; -+ -+ /* EPP feature related attributes*/ -+ s16 epp_powersave; -+ s16 epp_policy; -+ s16 epp_cached; -+ u32 policy; -+ u32 sched_flags; -+ bool update_util_set; -+ u64 last_update; -+ u64 last_io_update; -+ u32 cppc_boost_min; -+ u64 cppc_cap1_cached; -+ struct update_util_data update_util; -+ struct amd_aperf_mperf sample; -+ bool suspended; -+}; -+ -+/** -+ * struct amd_pstate_params - global parameters for the performance control -+ * @ cppc_boost_disabled Wheter or not the core performance boost disabled -+ */ -+struct amd_pstate_params { -+ bool cppc_boost_disabled; -+}; -+ -+/* -+ * AMD Energy Preference Performance (EPP) -+ * The EPP is used in the CCLK DPM controller to drive -+ * the frequency that a core is going to operate during -+ * short periods of activity. EPP values will be utilized for -+ * different OS profiles (balanced, performance, power savings) -+ * display strings corresponding to EPP index in the -+ * energy_perf_strings[] -+ * index String -+ *------------------------------------- -+ * 0 default -+ * 1 performance -+ * 2 balance_performance -+ * 3 balance_power -+ * 4 power -+ */ -+ enum energy_perf_value_index { -+ EPP_INDEX_DEFAULT = 0, -+ EPP_INDEX_PERFORMANCE, -+ EPP_INDEX_BALANCE_PERFORMANCE, -+ EPP_INDEX_BALANCE_POWERSAVE, -+ EPP_INDEX_POWERSAVE, -+}; -+ -+static const char * const energy_perf_strings[] = { -+ [EPP_INDEX_DEFAULT] = "default", -+ [EPP_INDEX_PERFORMANCE] = "performance", -+ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", -+ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", -+ [EPP_INDEX_POWERSAVE] = "power", -+ NULL - }; - -+static unsigned int epp_values[] = { -+ [EPP_INDEX_DEFAULT] = 0, -+ [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, -+ [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, -+ [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, -+ [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, -+}; -+ -+static struct amd_pstate_params global; -+ -+static DEFINE_MUTEX(amd_pstate_limits_lock); -+static DEFINE_MUTEX(amd_pstate_driver_lock); -+static DEFINE_SPINLOCK(amd_pstate_cpu_lock); -+ -+static bool cppc_boost __read_mostly; -+struct kobject *amd_pstate_kobj; -+ -+#ifdef CONFIG_ACPI_CPPC_LIB -+static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) -+{ -+ s16 epp; -+ struct cppc_perf_caps perf_caps; -+ int ret; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!cppc_req_cached) { -+ epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, -+ &cppc_req_cached); -+ if (epp) -+ return epp; -+ } -+ epp = (cppc_req_cached >> 24) & 0xFF; -+ } else { -+ ret = cppc_get_epp_caps(cpudata->cpu, &perf_caps); -+ if (ret < 0) { -+ pr_debug("Could not retrieve energy perf value (%d)\n", ret); -+ return -EIO; -+ } -+ epp = (s16) perf_caps.energy_perf; -+ } -+ -+ return epp; -+} -+#endif -+ -+static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata, int *raw_epp) -+{ -+ s16 epp; -+ int index = -EINVAL; -+ -+ *raw_epp = 0; -+ epp = amd_pstate_get_epp(cpudata, 0); -+ if (epp < 0) -+ return epp; -+ -+ switch (epp) { -+ case AMD_CPPC_EPP_PERFORMANCE: -+ index = EPP_INDEX_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_PERFORMANCE: -+ index = EPP_INDEX_BALANCE_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_POWERSAVE: -+ index = EPP_INDEX_BALANCE_POWERSAVE; -+ break; -+ case AMD_CPPC_EPP_POWERSAVE: -+ index = EPP_INDEX_POWERSAVE; -+ break; -+ default: -+ *raw_epp = epp; -+ index = 0; -+ } -+ -+ return index; -+} -+ -+#ifdef CONFIG_ACPI_CPPC_LIB -+static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) -+{ -+ int ret; -+ struct cppc_perf_ctrls perf_ctrls; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ u64 value = READ_ONCE(cpudata->cppc_req_cached); -+ -+ value &= ~GENMASK_ULL(31, 24); -+ value |= (u64)epp << 24; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (!ret) { -+ cpudata->epp_cached = epp; -+ } -+ } else { -+ perf_ctrls.energy_perf = epp; -+ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls); -+ if (ret){ -+ pr_debug("failed to set energy perf value (%d)\n", ret); -+ return ret; -+ } -+ cpudata->epp_cached = epp; -+ } -+ return ret; -+} -+ -+static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, -+ int pref_index, bool use_raw, -+ u32 raw_epp) -+{ -+ int epp = -EINVAL; -+ int ret; -+ -+ if (!pref_index) { -+ pr_debug("EPP pref_index is invaid\n"); -+ return -EINVAL; -+ } -+ -+ if (use_raw) -+ epp = raw_epp; -+ else if (epp == -EINVAL) -+ epp = epp_values[pref_index]; -+ -+ if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ pr_debug("EPP cannot be set under performance policy\n"); -+ return -EBUSY; -+ } -+ -+ ret = amd_pstate_set_epp(cpudata, epp); -+ -+ return ret; -+} -+#endif -+ - static inline int pstate_enable(bool enable) - { - return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); -@@ -131,12 +337,26 @@ static inline int pstate_enable(bool enable) - - static int cppc_enable(bool enable) - { -+ struct cppc_perf_ctrls perf_ctrls; - int cpu, ret = 0; - - for_each_present_cpu(cpu) { - ret = cppc_set_enable(cpu, enable); - if (ret) - return ret; -+ -+ if (epp_enabled) { -+ /* Enable autonomous mode for EPP */ -+ ret = cppc_set_auto_epp(cpu, enable); -+ if (ret) -+ return ret; -+ -+ /* Set zero to desired perf to allow EPP firmware control*/ -+ perf_ctrls.desired_perf = 0; -+ ret = cppc_set_perf(cpu, &perf_ctrls); -+ if (ret) -+ return ret; -+ } - } - - return ret; -@@ -269,6 +489,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, - u64 prev = READ_ONCE(cpudata->cppc_req_cached); - u64 value = prev; - -+ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - value &= ~AMD_CPPC_MIN_PERF(~0L); - value |= AMD_CPPC_MIN_PERF(min_perf); - -@@ -312,7 +533,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, - return -ENODEV; - - cap_perf = READ_ONCE(cpudata->highest_perf); -- min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); -+ min_perf = READ_ONCE(cpudata->lowest_perf); - max_perf = cap_perf; - - freqs.old = policy->cur; -@@ -357,8 +578,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, - if (max_perf < min_perf) - max_perf = min_perf; - -- des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); -- - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); - } - -@@ -438,18 +657,27 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) - { - struct amd_cpudata *cpudata = policy->driver_data; - int ret; -+ u64 value; - - if (!cpudata->boost_supported) { - pr_err("Boost mode is not supported by this processor or SBIOS\n"); - return -EINVAL; - } - -- if (state) -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); -+ if (ret) -+ return ret; -+ -+ if (state) { -+ value |= AMD_CPPC_PRECISION_BOOST_ENABLED; - policy->cpuinfo.max_freq = cpudata->max_freq; -- else -+ } else { -+ value &= ~AMD_CPPC_PRECISION_BOOST_ENABLED; - policy->cpuinfo.max_freq = cpudata->nominal_freq; -- -+ } - policy->max = policy->cpuinfo.max_freq; -+ WRITE_ONCE(cpudata->cppc_hw_conf_cached, value); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, value); - - ret = freq_qos_update_request(&cpudata->req[1], - policy->cpuinfo.max_freq); -@@ -470,7 +698,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) - return; - - cpudata->boost_supported = true; -- amd_pstate_driver.boost_enabled = true; -+ default_pstate_driver->boost_enabled = true; - } - - static int amd_pstate_cpu_init(struct cpufreq_policy *policy) -@@ -478,6 +706,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; - struct device *dev; - struct amd_cpudata *cpudata; -+ u64 value; - - dev = get_cpu_device(policy->cpu); - if (!dev) -@@ -541,7 +770,17 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; - - policy->driver_data = cpudata; -+ if (!shared_mem) { -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); -+ if (ret) -+ return ret; -+ cpudata->precision_boost_off = value & AMD_CPPC_PRECISION_BOOST_ENABLED; - -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ } - amd_pstate_boost_init(cpudata); - - return 0; -@@ -555,9 +794,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - - static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) - { -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - freq_qos_remove_request(&cpudata->req[1]); - freq_qos_remove_request(&cpudata->req[0]); -@@ -599,9 +836,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, - char *buf) - { - int max_freq; -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - max_freq = amd_get_max_freq(cpudata); - if (max_freq < 0) -@@ -614,9 +849,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli - char *buf) - { - int freq; -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - freq = amd_get_lowest_nonlinear_freq(cpudata); - if (freq < 0) -@@ -640,10 +873,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, - return sprintf(&buf[0], "%u\n", perf); - } - -+static ssize_t show_energy_performance_available_preferences( -+ struct cpufreq_policy *policy, char *buf) -+{ -+ int i = 0; -+ int ret = 0; -+ -+ while (energy_perf_strings[i] != NULL) -+ ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); -+ -+ ret += sprintf(&buf[ret], "\n"); -+ -+ return ret; -+} -+ -+static ssize_t store_energy_performance_preference( -+ struct cpufreq_policy *policy, const char *buf, size_t count) -+{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ char str_preference[21]; -+ bool raw = false; -+ ssize_t ret; -+ u32 epp = 0; -+ -+ ret = sscanf(buf, "%20s", str_preference); -+ if (ret != 1) -+ return -EINVAL; -+ -+ ret = match_string(energy_perf_strings, -1, str_preference); -+ if (ret < 0) { -+ ret = kstrtouint(buf, 10, &epp); -+ if (ret) -+ return ret; -+ -+ if ((epp > 255) || (epp < 0)) -+ return -EINVAL; -+ -+ raw = true; -+ } -+ -+ mutex_lock(&amd_pstate_limits_lock); -+ ret = amd_pstate_set_energy_pref_index(cpudata, ret, raw, epp); -+ mutex_unlock(&amd_pstate_limits_lock); -+ -+ return ret ?: count; -+} -+ -+static ssize_t show_energy_performance_preference( -+ struct cpufreq_policy *policy, char *buf) -+{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ int preference, raw_epp; -+ -+ preference = amd_pstate_get_energy_pref_index(cpudata, &raw_epp); -+ if (preference < 0) -+ return preference; -+ -+ if (raw_epp) -+ return sprintf(buf, "%d\n", raw_epp); -+ else -+ return sprintf(buf, "%s\n", energy_perf_strings[preference]); -+} -+ -+static void amd_pstate_update_policies(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) -+ cpufreq_update_policy(cpu); -+} -+ -+static ssize_t show_pstate_dynamic_boost(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", cppc_boost); -+} -+ -+static ssize_t store_pstate_dynamic_boost(struct kobject *a, -+ struct kobj_attribute *b, -+ const char *buf, size_t count) -+{ -+ unsigned int input; -+ int ret; -+ -+ ret = kstrtouint(buf, 10, &input); -+ if (ret) -+ return ret; -+ -+ mutex_lock(&amd_pstate_driver_lock); -+ cppc_boost = !!input; -+ amd_pstate_update_policies(); -+ mutex_unlock(&amd_pstate_driver_lock); -+ -+ return count; -+} -+ - cpufreq_freq_attr_ro(amd_pstate_max_freq); - cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); - - cpufreq_freq_attr_ro(amd_pstate_highest_perf); -+cpufreq_freq_attr_rw(energy_performance_preference); -+cpufreq_freq_attr_ro(energy_performance_available_preferences); -+define_one_global_rw(pstate_dynamic_boost); - - static struct freq_attr *amd_pstate_attr[] = { - &amd_pstate_max_freq, -@@ -652,6 +983,548 @@ static struct freq_attr *amd_pstate_attr[] = { - NULL, - }; - -+static struct freq_attr *amd_pstate_epp_attr[] = { -+ &amd_pstate_max_freq, -+ &amd_pstate_lowest_nonlinear_freq, -+ &amd_pstate_highest_perf, -+ &energy_performance_preference, -+ &energy_performance_available_preferences, -+ NULL, -+}; -+ -+static struct attribute *pstate_global_attributes[] = { -+ &pstate_dynamic_boost.attr, -+ NULL -+}; -+ -+static const struct attribute_group amd_pstate_global_attr_group = { -+ .attrs = pstate_global_attributes, -+}; -+ -+static inline void update_boost_state(void) -+{ -+ u64 misc_en; -+ struct amd_cpudata *cpudata; -+ -+ cpudata = all_cpu_data[0]; -+ rdmsrl(MSR_AMD_CPPC_HW_CTL, misc_en); -+ global.cppc_boost_disabled = misc_en & AMD_CPPC_PRECISION_BOOST_ENABLED; -+} -+ -+static int amd_pstate_init_cpu(unsigned int cpunum) -+{ -+ struct amd_cpudata *cpudata; -+ -+ cpudata = all_cpu_data[cpunum]; -+ if (!cpudata) { -+ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); -+ if (!cpudata) -+ return -ENOMEM; -+ WRITE_ONCE(all_cpu_data[cpunum], cpudata); -+ -+ cpudata->cpu = cpunum; -+ } -+ cpudata->epp_powersave = -EINVAL; -+ cpudata->epp_policy = 0; -+ pr_debug("controlling: cpu %d\n", cpunum); -+ return 0; -+} -+ -+static int __amd_pstate_cpu_init(struct cpufreq_policy *policy) -+{ -+ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; -+ struct amd_cpudata *cpudata; -+ struct device *dev; -+ int rc; -+ u64 value; -+ -+ rc = amd_pstate_init_cpu(policy->cpu); -+ if (rc) -+ return rc; -+ -+ cpudata = all_cpu_data[policy->cpu]; -+ -+ dev = get_cpu_device(policy->cpu); -+ if (!dev) -+ goto free_cpudata1; -+ -+ rc = amd_pstate_init_perf(cpudata); -+ if (rc) -+ goto free_cpudata1; -+ -+ min_freq = amd_get_min_freq(cpudata); -+ max_freq = amd_get_max_freq(cpudata); -+ nominal_freq = amd_get_nominal_freq(cpudata); -+ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); -+ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { -+ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", -+ min_freq, max_freq); -+ ret = -EINVAL; -+ goto free_cpudata1; -+ } -+ -+ policy->min = min_freq; -+ policy->max = max_freq; -+ -+ policy->cpuinfo.min_freq = min_freq; -+ policy->cpuinfo.max_freq = max_freq; -+ /* It will be updated by governor */ -+ policy->cur = policy->cpuinfo.min_freq; -+ -+ /* Initial processor data capability frequencies */ -+ cpudata->max_freq = max_freq; -+ cpudata->min_freq = min_freq; -+ cpudata->nominal_freq = nominal_freq; -+ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; -+ -+ policy->driver_data = cpudata; -+ -+ update_boost_state(); -+ cpudata->epp_cached = amd_pstate_get_epp(cpudata, value); -+ -+ policy->min = policy->cpuinfo.min_freq; -+ policy->max = policy->cpuinfo.max_freq; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) -+ policy->fast_switch_possible = true; -+ -+ if (!shared_mem && boot_cpu_has(X86_FEATURE_CPPC)) { -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_cap1_cached, value); -+ } -+ amd_pstate_boost_init(cpudata); -+ -+ return 0; -+ -+free_cpudata1: -+ kfree(cpudata); -+ return ret; -+} -+ -+static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) -+{ -+ int ret; -+ -+ ret = __amd_pstate_cpu_init(policy); -+ if (ret) -+ return ret; -+ /* -+ * Set the policy to powersave to provide a valid fallback value in case -+ * the default cpufreq governor is neither powersave nor performance. -+ */ -+ policy->policy = CPUFREQ_POLICY_POWERSAVE; -+ -+ return 0; -+} -+ -+static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) -+{ -+ pr_debug("amd-pstate: CPU %d exiting\n", policy->cpu); -+ policy->fast_switch_possible = false; -+ return 0; -+} -+ -+static void amd_pstate_update_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); -+ -+ if (!policy) -+ return; -+ -+ refresh_frequency_limits(policy); -+ cpufreq_cpu_release(policy); -+} -+ -+static void amd_pstate_epp_update_limits(unsigned int cpu) -+{ -+ mutex_lock(&amd_pstate_driver_lock); -+ update_boost_state(); -+ if (global.cppc_boost_disabled) { -+ for_each_possible_cpu(cpu) -+ amd_pstate_update_max_freq(cpu); -+ } else { -+ cpufreq_update_policy(cpu); -+ } -+ mutex_unlock(&amd_pstate_driver_lock); -+} -+ -+static int cppc_boost_hold_time_ns = 3 * NSEC_PER_MSEC; -+ -+static inline void amd_pstate_boost_up(struct amd_cpudata *cpudata) -+{ -+ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); -+ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); -+ u32 max_limit = (hwp_req & 0xff); -+ u32 min_limit = (hwp_req & 0xff00) >> 8; -+ u32 boost_level1; -+ -+ /* If max and min are equal or already at max, nothing to boost */ -+ if (max_limit == min_limit) -+ return; -+ -+ /* Set boost max and min to initial value */ -+ if (!cpudata->cppc_boost_min) -+ cpudata->cppc_boost_min = min_limit; -+ -+ boost_level1 = ((AMD_CPPC_NOMINAL_PERF(hwp_cap) + min_limit) >> 1); -+ -+ if (cpudata->cppc_boost_min < boost_level1) -+ cpudata->cppc_boost_min = boost_level1; -+ else if (cpudata->cppc_boost_min < AMD_CPPC_NOMINAL_PERF(hwp_cap)) -+ cpudata->cppc_boost_min = AMD_CPPC_NOMINAL_PERF(hwp_cap); -+ else if (cpudata->cppc_boost_min == AMD_CPPC_NOMINAL_PERF(hwp_cap)) -+ cpudata->cppc_boost_min = max_limit; -+ else -+ return; -+ -+ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); -+ hwp_req |= AMD_CPPC_MIN_PERF(cpudata->cppc_boost_min); -+ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); -+ cpudata->last_update = cpudata->sample.time; -+} -+ -+static inline void amd_pstate_boost_down(struct amd_cpudata *cpudata) -+{ -+ bool expired; -+ -+ if (cpudata->cppc_boost_min) { -+ expired = time_after64(cpudata->sample.time, cpudata->last_update + -+ cppc_boost_hold_time_ns); -+ -+ if (expired) { -+ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, cpudata->cppc_req_cached); -+ cpudata->cppc_boost_min = 0; -+ } -+ } -+ -+ cpudata->last_update = cpudata->sample.time; -+} -+ -+static inline void amd_pstate_boost_update_util(struct amd_cpudata *cpudata, -+ u64 time) -+{ -+ cpudata->sample.time = time; -+ if (smp_processor_id() != cpudata->cpu) -+ return; -+ -+ if (cpudata->sched_flags & SCHED_CPUFREQ_IOWAIT) { -+ bool do_io = false; -+ -+ cpudata->sched_flags = 0; -+ /* -+ * Set iowait_boost flag and update time. Since IO WAIT flag -+ * is set all the time, we can't just conclude that there is -+ * some IO bound activity is scheduled on this CPU with just -+ * one occurrence. If we receive at least two in two -+ * consecutive ticks, then we treat as boost candidate. -+ * This is leveraged from Intel Pstate driver. -+ */ -+ if (time_before64(time, cpudata->last_io_update + 2 * TICK_NSEC)) -+ do_io = true; -+ -+ cpudata->last_io_update = time; -+ -+ if (do_io) -+ amd_pstate_boost_up(cpudata); -+ -+ } else { -+ amd_pstate_boost_down(cpudata); -+ } -+} -+ -+static inline void amd_pstate_cppc_update_hook(struct update_util_data *data, -+ u64 time, unsigned int flags) -+{ -+ struct amd_cpudata *cpudata = container_of(data, -+ struct amd_cpudata, update_util); -+ -+ cpudata->sched_flags |= flags; -+ -+ if (smp_processor_id() == cpudata->cpu) -+ amd_pstate_boost_update_util(cpudata, time); -+} -+ -+static void amd_pstate_clear_update_util_hook(unsigned int cpu) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu]; -+ -+ if (!cpudata->update_util_set) -+ return; -+ -+ cpufreq_remove_update_util_hook(cpu); -+ cpudata->update_util_set = false; -+ synchronize_rcu(); -+} -+ -+static void amd_pstate_set_update_util_hook(unsigned int cpu_num) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu_num]; -+ -+ if (!cppc_boost) { -+ if (cpudata->update_util_set) -+ amd_pstate_clear_update_util_hook(cpudata->cpu); -+ return; -+ } -+ -+ if (cpudata->update_util_set) -+ return; -+ -+ cpudata->sample.time = 0; -+ cpufreq_add_update_util_hook(cpu_num, &cpudata->update_util, -+ amd_pstate_cppc_update_hook); -+ cpudata->update_util_set = true; -+} -+ -+static void amd_pstate_epp_init(unsigned int cpu) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu]; -+ u32 max_perf, min_perf; -+ u64 value; -+ s16 epp; -+ int ret; -+ -+ max_perf = READ_ONCE(cpudata->highest_perf); -+ min_perf = READ_ONCE(cpudata->lowest_perf); -+ -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) -+ min_perf = max_perf; -+ -+ /* Initial min/max values for CPPC Performance Controls Register */ -+ value &= ~AMD_CPPC_MIN_PERF(~0L); -+ value |= AMD_CPPC_MIN_PERF(min_perf); -+ -+ value &= ~AMD_CPPC_MAX_PERF(~0L); -+ value |= AMD_CPPC_MAX_PERF(max_perf); -+ -+ /* CPPC EPP feature require to set zero to the desire perf bit */ -+ value &= ~AMD_CPPC_DES_PERF(~0L); -+ value |= AMD_CPPC_DES_PERF(0); -+ -+ if (cpudata->epp_policy == cpudata->policy) -+ goto skip_epp; -+ -+ cpudata->epp_policy = cpudata->policy; -+ -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ epp = amd_pstate_get_epp(cpudata, value); -+ cpudata->epp_powersave = epp; -+ if (epp < 0) -+ goto skip_epp; -+ -+ epp = 0; -+ } else { -+ if (cpudata->epp_powersave < 0) -+ goto skip_epp; -+ /* Get BIOS pre-defined epp value */ -+ epp = amd_pstate_get_epp(cpudata, value); -+ if (epp) -+ goto skip_epp; -+ epp = cpudata->epp_powersave; -+ } -+ /* Set initial EPP value */ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ value &= ~GENMASK_ULL(31, 24); -+ value |= (u64)epp << 24; -+ } -+ -+skip_epp: -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (!ret) { -+ cpudata->epp_cached = epp; -+ } -+} -+ -+static void amd_pstate_set_max_limits(struct amd_cpudata *cpudata) -+{ -+ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); -+ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); -+ u32 max_limit = (hwp_cap >> 24) & 0xff; -+ -+ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); -+ hwp_req |= AMD_CPPC_MIN_PERF(max_limit); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); -+} -+ -+static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata; -+ -+ if (!policy->cpuinfo.max_freq) -+ return -ENODEV; -+ -+ pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", -+ policy->cpuinfo.max_freq, policy->max); -+ -+ cpudata = all_cpu_data[policy->cpu]; -+ cpudata->policy = policy->policy; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ mutex_lock(&amd_pstate_limits_lock); -+ -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ amd_pstate_clear_update_util_hook(policy->cpu); -+ amd_pstate_set_max_limits(cpudata); -+ } else { -+ amd_pstate_set_update_util_hook(policy->cpu); -+ } -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) -+ amd_pstate_epp_init(policy->cpu); -+ -+ mutex_unlock(&amd_pstate_limits_lock); -+ } -+ -+ return 0; -+} -+ -+static void amd_pstate_epp_reenable(struct amd_cpudata * cpudata) -+{ -+ struct cppc_perf_ctrls perf_ctrls; -+ u64 value, max_perf; -+ int ret; -+ -+ ret = amd_pstate_enable(true); -+ if (ret) -+ pr_err("failed to enable amd pstate during resume, return %d\n", ret); -+ -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ max_perf = READ_ONCE(cpudata->highest_perf); -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ } else { -+ perf_ctrls.max_perf = max_perf; -+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); -+ cppc_set_perf(cpudata->cpu, &perf_ctrls); -+ } -+} -+ -+static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ -+ pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); -+ -+ if (epp_enabled) { -+ amd_pstate_epp_reenable(cpudata); -+ cpudata->suspended = false; -+ } -+ -+ return 0; -+} -+ -+static void amd_pstate_epp_offline(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ struct cppc_perf_ctrls perf_ctrls; -+ int min_perf; -+ u64 value; -+ -+ min_perf = READ_ONCE(cpudata->lowest_perf); -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ -+ mutex_lock(&amd_pstate_limits_lock); -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; -+ -+ /* Set max perf same as min perf */ -+ value &= ~AMD_CPPC_MAX_PERF(~0L); -+ value |= AMD_CPPC_MAX_PERF(min_perf); -+ value &= ~AMD_CPPC_MIN_PERF(~0L); -+ value |= AMD_CPPC_MIN_PERF(min_perf); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ } else { -+ perf_ctrls.desired_perf = 0; -+ perf_ctrls.max_perf = min_perf; -+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(AMD_CPPC_EPP_POWERSAVE); -+ cppc_set_perf(cpudata->cpu, &perf_ctrls); -+ } -+ mutex_unlock(&amd_pstate_limits_lock); -+} -+ -+static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ -+ pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); -+ -+ if (cpudata->suspended) -+ return 0; -+ -+ if (epp_enabled) -+ amd_pstate_epp_offline(policy); -+ -+ return 0; -+} -+ -+static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) -+{ -+ amd_pstate_clear_update_util_hook(policy->cpu); -+ -+ return amd_pstate_cpu_offline(policy); -+} -+ -+static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ int ret; -+ -+ pr_debug("AMD CPU Core %d suspending\n", cpudata->cpu); -+ -+ cpudata->suspended = true; -+ -+ /* disable CPPC in lowlevel firmware */ -+ ret = amd_pstate_enable(false); -+ if (ret) -+ pr_err("failed to disable amd pstate during suspend, return %d\n", ret); -+ -+ return 0; -+} -+ -+static int amd_pstate_epp_resume(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ -+ pr_debug("AMD CPU Core %d resuming\n", cpudata->cpu); -+ -+ if (cpudata->suspended && epp_enabled) { -+ mutex_lock(&amd_pstate_limits_lock); -+ -+ /* enable amd pstate from suspend state*/ -+ amd_pstate_epp_reenable(cpudata); -+ -+ mutex_unlock(&amd_pstate_limits_lock); -+ } -+ -+ cpudata->suspended = false; -+ -+ return 0; -+} -+ -+static void amd_pstate_verify_cpu_policy(struct amd_cpudata *cpudata, -+ struct cpufreq_policy_data *policy) -+{ -+ update_boost_state(); -+ cpufreq_verify_within_cpu_limits(policy); -+} -+ -+static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) -+{ -+ amd_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); -+ pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min ); -+ return 0; -+} -+ - static struct cpufreq_driver amd_pstate_driver = { - .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, - .verify = amd_pstate_verify, -@@ -662,18 +1535,34 @@ static struct cpufreq_driver amd_pstate_driver = { - .resume = amd_pstate_cpu_resume, - .set_boost = amd_pstate_set_boost, - .name = "amd-pstate", -- .attr = amd_pstate_attr, -+ .attr = amd_pstate_attr, -+}; -+ -+static struct cpufreq_driver amd_pstate_epp_driver = { -+ .flags = CPUFREQ_CONST_LOOPS, -+ .verify = amd_pstate_epp_verify_policy, -+ .setpolicy = amd_pstate_epp_set_policy, -+ .init = amd_pstate_epp_cpu_init, -+ .exit = amd_pstate_epp_cpu_exit, -+ .update_limits = amd_pstate_epp_update_limits, -+ .offline = amd_pstate_epp_cpu_offline, -+ .online = amd_pstate_epp_cpu_online, -+ .suspend = amd_pstate_epp_suspend, -+ .resume = amd_pstate_epp_resume, -+ .name = "amd_pstate_epp", -+ .attr = amd_pstate_epp_attr, - }; - - static int __init amd_pstate_init(void) - { -+ static struct amd_cpudata **cpudata; - int ret; - - if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) - return -ENODEV; - - if (!acpi_cpc_valid()) { -- pr_debug("the _CPC object is not present in SBIOS\n"); -+ pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); - return -ENODEV; - } - -@@ -681,10 +1570,25 @@ static int __init amd_pstate_init(void) - if (cpufreq_get_current_driver()) - return -EEXIST; - -+ cpudata = vzalloc(array_size(sizeof(void *), num_possible_cpus())); -+ if (!cpudata) -+ return -ENOMEM; -+ WRITE_ONCE(all_cpu_data, cpudata); -+ -+ if (epp_enabled) { -+ pr_info("AMD CPPC loading with amd_pstate_epp driver instance.\n"); -+ default_pstate_driver = &amd_pstate_epp_driver; -+ } else { -+ pr_info("AMD CPPC loading with amd_pstate driver instance.\n"); -+ default_pstate_driver = &amd_pstate_driver; -+ } -+ - /* capability check */ - if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!epp_enabled) { -+ default_pstate_driver->adjust_perf = amd_pstate_adjust_perf; -+ } - pr_debug("AMD CPPC MSR based functionality is supported\n"); -- amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; - } else if (shared_mem) { - static_call_update(amd_pstate_enable, cppc_enable); - static_call_update(amd_pstate_init_perf, cppc_init_perf); -@@ -701,19 +1605,56 @@ static int __init amd_pstate_init(void) - return ret; - } - -- ret = cpufreq_register_driver(&amd_pstate_driver); -+ ret = cpufreq_register_driver(default_pstate_driver); - if (ret) -- pr_err("failed to register amd_pstate_driver with return %d\n", -+ pr_err("failed to register amd pstate driver with return %d\n", -+ ret); -+ -+ amd_pstate_kobj = kobject_create_and_add("amd-pstate", &cpu_subsys.dev_root->kobj); -+ if (!amd_pstate_kobj) { -+ pr_err("amd-pstate: Global sysfs registration failed.\n"); -+ } -+ -+ ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); -+ if (ret) { -+ pr_err("amd-pstate: Sysfs attribute export failed with error %d.\n", - ret); -+ } - - return ret; - } - -+static inline void amd_pstate_kobj_cleanup(struct kobject *kobj) -+{ -+ kobject_del(kobj); -+ kobject_put(kobj); -+} -+ - static void __exit amd_pstate_exit(void) - { -- cpufreq_unregister_driver(&amd_pstate_driver); -+ unsigned int cpu; -+ -+ cpufreq_unregister_driver(default_pstate_driver); - - amd_pstate_enable(false); -+ -+ sysfs_remove_group(amd_pstate_kobj, &amd_pstate_global_attr_group); -+ amd_pstate_kobj_cleanup(amd_pstate_kobj); -+ -+ cpus_read_lock(); -+ for_each_online_cpu(cpu) { -+ if (all_cpu_data[cpu]) { -+ if (default_pstate_driver == &amd_pstate_epp_driver) -+ amd_pstate_clear_update_util_hook(cpu); -+ -+ spin_lock(&amd_pstate_cpu_lock); -+ kfree(all_cpu_data[cpu]); -+ WRITE_ONCE(all_cpu_data[cpu], NULL); -+ spin_unlock(&amd_pstate_cpu_lock); -+ } -+ } -+ cpus_read_unlock(); -+ - } - - module_init(amd_pstate_init); -diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c -index 24eaf0ec344d..9adb7612993e 100644 ---- a/drivers/cpufreq/cppc_cpufreq.c -+++ b/drivers/cpufreq/cppc_cpufreq.c -@@ -947,7 +947,7 @@ static int __init cppc_cpufreq_init(void) - { - int ret; - -- if ((acpi_disabled) || !acpi_cpc_valid()) -+ if (!acpi_cpc_valid()) - return -ENODEV; - - cppc_check_hisi_workaround(); -diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c -index 57cdb3679885..50881662b45b 100644 ---- a/drivers/cpufreq/intel_pstate.c -+++ b/drivers/cpufreq/intel_pstate.c -@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu) - * update them at any time after it has been called. - */ - sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); -+ /* -+ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff. -+ * In this case we can't use CPPC.highest_perf to enable ITMT. -+ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide. -+ */ -+ if (cppc_perf.highest_perf == 0xff) -+ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); - - if (max_highest_perf <= min_highest_perf) { - if (cppc_perf.highest_perf > max_highest_perf) -diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c -index 3e101719689a..746138abc3fe 100644 ---- a/drivers/idle/intel_idle.c -+++ b/drivers/idle/intel_idle.c -@@ -578,7 +578,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -586,7 +586,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 33, -- .target_residency = 100, -+ .target_residency = 900, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -594,7 +594,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 133, -- .target_residency = 400, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -602,7 +602,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 166, -- .target_residency = 500, -+ .target_residency = 1500, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -610,7 +610,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 300, -- .target_residency = 900, -+ .target_residency = 2000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -618,7 +618,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 600, -- .target_residency = 1800, -+ .target_residency = 5000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -626,7 +626,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 2600, -- .target_residency = 7700, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -646,7 +646,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -654,7 +654,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 40, -- .target_residency = 100, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -662,7 +662,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 133, -- .target_residency = 400, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -670,7 +670,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 166, -- .target_residency = 500, -+ .target_residency = 2000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -678,7 +678,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 300, -- .target_residency = 900, -+ .target_residency = 4000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -686,7 +686,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 600, -- .target_residency = 1800, -+ .target_residency = 7000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -694,7 +694,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 2600, -- .target_residency = 7700, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -715,7 +715,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -723,7 +723,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 70, -- .target_residency = 100, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -731,7 +731,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 85, -- .target_residency = 200, -+ .target_residency = 600, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -739,7 +739,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x33", - .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 124, -- .target_residency = 800, -+ .target_residency = 3000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -747,7 +747,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 200, -- .target_residency = 800, -+ .target_residency = 3200, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -755,7 +755,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 480, -- .target_residency = 5000, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -763,7 +763,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 890, -- .target_residency = 5000, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -784,7 +784,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 300, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c -index 3fc0a89cc785..a7c103f9dfd3 100644 ---- a/drivers/input/serio/i8042.c -+++ b/drivers/input/serio/i8042.c -@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void) - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - i8042_ctr &= ~I8042_CTR_KBDINT; - i8042_ctr |= I8042_CTR_KBDDIS; -- pr_err("Failed to enable KBD port\n"); -+ pr_info("Failed to enable KBD port\n"); - return -EIO; - } - -@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void) - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - i8042_ctr &= ~I8042_CTR_AUXINT; - i8042_ctr |= I8042_CTR_AUXDIS; -- pr_err("Failed to enable AUX port\n"); -+ pr_info("Failed to enable AUX port\n"); - return -EIO; - } - -@@ -732,7 +732,7 @@ static int i8042_check_mux(void) - i8042_ctr &= ~I8042_CTR_AUXINT; - - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { -- pr_err("Failed to disable AUX port, can't use MUX\n"); -+ pr_info("Failed to disable AUX port, can't use MUX\n"); - return -EIO; - } - -@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void) - do { - - if (i8042_command(¶m, I8042_CMD_CTL_TEST)) { -- pr_err("i8042 controller selftest timeout\n"); -+ pr_info("i8042 controller selftest timeout\n"); - return -ENODEV; - } - -@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void) - pr_info("giving up on controller selftest, continuing anyway...\n"); - return 0; - #else -- pr_err("i8042 controller selftest failed\n"); -+ pr_info("i8042 controller selftest failed\n"); - return -EIO; - #endif - } -diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c -index 159c6806c19b..a201f1c2fce5 100644 ---- a/drivers/md/dm-crypt.c -+++ b/drivers/md/dm-crypt.c -@@ -3137,6 +3137,11 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar - } - } - -+#ifdef CONFIG_CACHY -+ set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); -+ set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); -+#endif -+ - return 0; - } - -diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c -index f82ad7419508..5e8faa70aad6 100644 ---- a/drivers/net/dummy.c -+++ b/drivers/net/dummy.c -@@ -43,7 +43,7 @@ - - #define DRV_NAME "dummy" - --static int numdummies = 1; -+static int numdummies = 0; - - /* fake multicast ability */ - static void set_multicast_list(struct net_device *dev) -diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c -index 66446f1e06cf..c65b03f91ecf 100644 ---- a/drivers/nvme/host/core.c -+++ b/drivers/nvme/host/core.c -@@ -58,7 +58,7 @@ static u8 nvme_max_retries = 5; - module_param_named(max_retries, nvme_max_retries, byte, 0644); - MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); - --static unsigned long default_ps_max_latency_us = 100000; -+static unsigned long default_ps_max_latency_us = 200; - module_param(default_ps_max_latency_us, ulong, 0644); - MODULE_PARM_DESC(default_ps_max_latency_us, - "max power saving latency for new devices; use PM QOS to change per device"); -diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c -index 95bc329e74c0..73114509ddff 100644 ---- a/drivers/pci/pci.c -+++ b/drivers/pci/pci.c -@@ -62,7 +62,7 @@ struct pci_pme_device { - struct pci_dev *dev; - }; - --#define PME_TIMEOUT 1000 /* How long between PME checks */ -+#define PME_TIMEOUT 4000 /* How long between PME checks */ - - static void pci_dev_d3_sleep(struct pci_dev *dev) - { -diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c -index 21d624f9f5fb..aad4e9578d89 100644 ---- a/drivers/powercap/intel_rapl_common.c -+++ b/drivers/powercap/intel_rapl_common.c -@@ -1515,7 +1515,7 @@ static int __init rapl_init(void) - - id = x86_match_cpu(rapl_ids); - if (!id) { -- pr_err("driver does not support CPU family %d model %d\n", -+ pr_info("driver does not support CPU family %d model %d\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); - - return -ENODEV; -diff --git a/drivers/soundwire/bus.h b/drivers/soundwire/bus.h -index 7631ef5e71fb..d3ed828daac0 100644 ---- a/drivers/soundwire/bus.h -+++ b/drivers/soundwire/bus.h -@@ -5,7 +5,7 @@ - #define __SDW_BUS_H - - #define DEFAULT_BANK_SWITCH_TIMEOUT 3000 --#define DEFAULT_PROBE_TIMEOUT 2000 -+#define DEFAULT_PROBE_TIMEOUT 10000 - - u64 sdw_dmi_override_adr(struct sdw_bus *bus, u64 addr); - -diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c -index c841ab37e7c6..fe3d190ab12b 100644 ---- a/drivers/thermal/intel/intel_powerclamp.c -+++ b/drivers/thermal/intel/intel_powerclamp.c -@@ -644,6 +644,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = { - .set_cur_state = powerclamp_set_cur_state, - }; - -+static const struct x86_cpu_id amd_cpu[] = { -+ { X86_VENDOR_AMD }, -+ {}, -+}; -+ - static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = { - X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL), - {} -@@ -653,6 +658,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); - static int __init powerclamp_probe(void) - { - -+ if (x86_match_cpu(amd_cpu)){ -+ pr_info("Intel PowerClamp does not support AMD CPUs\n"); -+ return -ENODEV; -+ } -+ - if (!x86_match_cpu(intel_powerclamp_ids)) { - pr_err("CPU does not support MWAIT\n"); - return -ENODEV; -diff --git a/fs/proc/base.c b/fs/proc/base.c -index 93f7e3d971e4..812a42947afd 100644 ---- a/fs/proc/base.c -+++ b/fs/proc/base.c -@@ -3196,6 +3196,19 @@ static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace * - - return 0; - } -+static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, -+ struct pid *pid, struct task_struct *task) -+{ -+ struct mm_struct *mm; -+ -+ mm = get_task_mm(task); -+ if (mm) { -+ seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); -+ mmput(mm); -+ } -+ -+ return 0; -+} - #endif /* CONFIG_KSM */ - - #ifdef CONFIG_STACKLEAK_METRICS -@@ -3331,6 +3344,7 @@ static const struct pid_entry tgid_base_stuff[] = { - #endif - #ifdef CONFIG_KSM - ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), -+ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), - #endif - }; - -@@ -3668,6 +3682,7 @@ static const struct pid_entry tid_base_stuff[] = { - #endif - #ifdef CONFIG_KSM - ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), -+ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), - #endif - }; - -diff --git a/fs/xattr.c b/fs/xattr.c -index a1f4998bc6be..020e0b5ea5c6 100644 ---- a/fs/xattr.c -+++ b/fs/xattr.c -@@ -122,16 +122,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, - } - - /* -- * In the user.* namespace, only regular files and directories can have -- * extended attributes. For sticky directories, only the owner and -- * privileged users can write attributes. -+ * In the user.* namespace, only regular files, symbolic links, and -+ * directories can have extended attributes. For symbolic links and -+ * sticky directories, only the owner and privileged users can write -+ * attributes. - */ - if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { -- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) -+ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode)) - return (mask & MAY_WRITE) ? -EPERM : -ENODATA; -- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && -- (mask & MAY_WRITE) && -- !inode_owner_or_capable(mnt_userns, inode)) -+ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX)) -+ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE) -+ && !inode_owner_or_capable(mnt_userns, inode)) - return -EPERM; - } - -diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h -index f73d357ecdf5..41d151a6d52e 100644 ---- a/include/acpi/cppc_acpi.h -+++ b/include/acpi/cppc_acpi.h -@@ -108,12 +108,14 @@ struct cppc_perf_caps { - u32 lowest_nonlinear_perf; - u32 lowest_freq; - u32 nominal_freq; -+ u32 energy_perf; - }; - - struct cppc_perf_ctrls { - u32 max_perf; - u32 min_perf; - u32 desired_perf; -+ u32 energy_perf; - }; - - struct cppc_perf_fb_ctrs { -@@ -148,6 +150,9 @@ extern bool cpc_ffh_supported(void); - extern bool cpc_supported_by_cpu(void); - extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); - extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); -+extern int cppc_set_auto_epp(int cpu, bool enable); -+extern int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps); -+extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); - #else /* !CONFIG_ACPI_CPPC_LIB */ - static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) - { -@@ -197,6 +202,18 @@ static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) - { - return -ENOTSUPP; - } -+static inline int cppc_set_auto_epp(int cpu, bool enable) -+{ -+ return -ENOTSUPP; -+} -+static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) -+{ -+ return -ENOTSUPP; -+} -+static inline int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) -+{ -+ return -ENOTSUPP; -+} - #endif /* !CONFIG_ACPI_CPPC_LIB */ - - #endif /* _CPPC_ACPI_H*/ -diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h -index e3e8c8662b49..9209856b6644 100644 ---- a/include/linux/ipc_namespace.h -+++ b/include/linux/ipc_namespace.h -@@ -36,8 +36,6 @@ struct ipc_namespace { - unsigned int msg_ctlmax; - unsigned int msg_ctlmnb; - unsigned int msg_ctlmni; -- atomic_t msg_bytes; -- atomic_t msg_hdrs; - - size_t shm_ctlmax; - size_t shm_ctlall; -@@ -77,6 +75,9 @@ struct ipc_namespace { - struct llist_node mnt_llist; - - struct ns_common ns; -+ int padding[16]; -+ atomic_t msg_bytes; -+ atomic_t msg_hdrs; - } __randomize_layout; - - extern struct ipc_namespace init_ipc_ns; -diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h -index 0b7242370b56..16b8fc483b3d 100644 ---- a/include/linux/jbd2.h -+++ b/include/linux/jbd2.h -@@ -45,7 +45,7 @@ - /* - * The default maximum commit age, in seconds. - */ --#define JBD2_DEFAULT_MAX_COMMIT_AGE 5 -+#define JBD2_DEFAULT_MAX_COMMIT_AGE 30 - - #ifdef CONFIG_JBD2_DEBUG - /* -diff --git a/include/linux/ksm.h b/include/linux/ksm.h -index 0b4f17418f64..e24cf9144b24 100644 ---- a/include/linux/ksm.h -+++ b/include/linux/ksm.h -@@ -19,6 +19,10 @@ struct stable_node; - struct mem_cgroup; - - #ifdef CONFIG_KSM -+int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, -+ unsigned long *vm_flags); -+int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, -+ unsigned long end, unsigned long *vm_flags); - int ksm_madvise(struct vm_area_struct *vma, unsigned long start, - unsigned long end, int advice, unsigned long *vm_flags); - int __ksm_enter(struct mm_struct *mm); -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index cf97f3884fda..8ac6d7e53b17 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -671,6 +671,11 @@ struct mm_struct { - * merging. - */ - unsigned long ksm_merging_pages; -+ /* -+ * Represent how many pages are checked for ksm merging -+ * including merged and not merged. -+ */ -+ unsigned long ksm_rmap_items; - #endif - } __randomize_layout; - -diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h -index 679591301994..685193470d64 100644 ---- a/include/linux/page_counter.h -+++ b/include/linux/page_counter.h -@@ -8,6 +8,7 @@ - - struct page_counter { - atomic_long_t usage; -+ unsigned long padding[7]; - unsigned long min; - unsigned long low; - unsigned long high; -diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index 0178b2040ea3..39f05e00dac4 100644 ---- a/include/linux/pagemap.h -+++ b/include/linux/pagemap.h -@@ -1183,7 +1183,7 @@ struct readahead_control { - ._index = i, \ - } - --#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) -+#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) - - void page_cache_ra_unbounded(struct readahead_control *, - unsigned long nr_to_read, unsigned long lookahead_count); -diff --git a/include/linux/sched.h b/include/linux/sched.h -index e7b2f8a5c711..367b30ed77cb 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -1052,6 +1052,7 @@ struct task_struct { - /* Cached requested key. */ - struct key *cached_requested_key; - #endif -+ int fsync_count; - - /* - * executable name, excluding path. -diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h -index a34b0f9a9972..82afad91de9f 100644 ---- a/include/linux/syscalls.h -+++ b/include/linux/syscalls.h -@@ -917,6 +917,7 @@ asmlinkage long sys_madvise(unsigned long start, size_t len, int behavior); - asmlinkage long sys_process_madvise(int pidfd, const struct iovec __user *vec, - size_t vlen, int behavior, unsigned int flags); - asmlinkage long sys_process_mrelease(int pidfd, unsigned int flags); -+asmlinkage long sys_pmadv_ksm(int pidfd, int behavior, unsigned int flags); - asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size, - unsigned long prot, unsigned long pgoff, - unsigned long flags); -diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h -index 33a4240e6a6f..82213f9c4c17 100644 ---- a/include/linux/user_namespace.h -+++ b/include/linux/user_namespace.h -@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns, - - #ifdef CONFIG_USER_NS - -+extern int unprivileged_userns_clone; -+ - static inline struct user_namespace *get_user_ns(struct user_namespace *ns) - { - if (ns) -@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); - struct ns_common *ns_get_owner(struct ns_common *ns); - #else - -+#define unprivileged_userns_clone 0 -+ - static inline struct user_namespace *get_user_ns(struct user_namespace *ns) - { - return &init_user_ns; -diff --git a/include/linux/wait.h b/include/linux/wait.h -index 58cfbf81447c..d5664b5ae3e1 100644 ---- a/include/linux/wait.h -+++ b/include/linux/wait.h -@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) - - extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); -+extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - -@@ -1164,6 +1165,7 @@ do { \ - */ - void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); -+void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); -diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h -index 45fa180cc56a..40f7e6d04af0 100644 ---- a/include/uapi/asm-generic/unistd.h -+++ b/include/uapi/asm-generic/unistd.h -@@ -886,8 +886,11 @@ __SYSCALL(__NR_futex_waitv, sys_futex_waitv) - #define __NR_set_mempolicy_home_node 450 - __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) - -+#define __NR_pmadv_ksm 451 -+__SYSCALL(__NR_pmadv_ksm, sys_pmadv_ksm) -+ - #undef __NR_syscalls --#define __NR_syscalls 451 -+#define __NR_syscalls 452 - - /* - * 32 bit systems traditionally used different -diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h -index d174914a837d..bf8e2af101a3 100644 ---- a/include/uapi/linux/if_bonding.h -+++ b/include/uapi/linux/if_bonding.h -@@ -82,7 +82,7 @@ - #define BOND_STATE_ACTIVE 0 /* link is active */ - #define BOND_STATE_BACKUP 1 /* link is backup */ - --#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */ -+#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */ - - #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */ - -diff --git a/init/Kconfig b/init/Kconfig -index 532362fcfe31..442a945ca6ae 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -112,6 +112,10 @@ config THREAD_INFO_IN_TASK - - menu "General setup" - -+config CACHY -+ bool "Some kernel tweaks by CachyOS" -+ default y -+ - config BROKEN - bool - -@@ -1241,6 +1245,22 @@ config USER_NS - - If unsure, say N. - -+config USER_NS_UNPRIVILEGED -+ bool "Allow unprivileged users to create namespaces" -+ default y -+ depends on USER_NS -+ help -+ When disabled, unprivileged users will not be able to create -+ new namespaces. Allowing users to create their own namespaces -+ has been part of several recent local privilege escalation -+ exploits, so if you need user namespaces but are -+ paranoid^Wsecurity-conscious you want to disable this. -+ -+ This setting can be overridden at runtime via the -+ kernel.unprivileged_userns_clone sysctl. -+ -+ If unsure, say Y. -+ - config PID_NS - bool "PID Namespaces" - default y -@@ -1407,6 +1427,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE - with the "-O2" compiler flag for best performance and most - helpful compile-time warnings. - -+config CC_OPTIMIZE_FOR_PERFORMANCE_O3 -+ bool "Optimize more for performance (-O3)" -+ help -+ Choosing this option will pass "-O3" to your compiler to optimize -+ the kernel yet more for performance. -+ - config CC_OPTIMIZE_FOR_SIZE - bool "Optimize for size (-Os)" - help -diff --git a/init/do_mounts.c b/init/do_mounts.c -index 7058e14ad5f7..ce4e841ea705 100644 ---- a/init/do_mounts.c -+++ b/init/do_mounts.c -@@ -283,8 +283,18 @@ dev_t name_to_dev_t(const char *name) - if (strcmp(name, "/dev/ram") == 0) - return Root_RAM0; - #ifdef CONFIG_BLOCK -- if (strncmp(name, "PARTUUID=", 9) == 0) -- return devt_from_partuuid(name + 9); -+ if (strncmp(name, "PARTUUID=", 9) == 0) { -+ dev_t res; -+ int needtowait = 40<<1; -+ res = devt_from_partuuid(name + 9); -+ while (!res && needtowait) { -+ /* waiting 0.5 sec */ -+ msleep(500); -+ res = devt_from_partuuid(name + 9); -+ needtowait--; -+ } -+ return res; -+ } - if (strncmp(name, "PARTLABEL=", 10) == 0) - return devt_from_partlabel(name + 10); - if (strncmp(name, "/dev/", 5) == 0) -@@ -612,7 +622,9 @@ void __init prepare_namespace(void) - * For example, it is not atypical to wait 5 seconds here - * for the touchpad of a laptop to initialize. - */ -+ async_synchronize_full(); - wait_for_device_probe(); -+ async_synchronize_full(); - - md_run_setup(); - -diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz -index 38ef6d06888e..0f78364efd4f 100644 ---- a/kernel/Kconfig.hz -+++ b/kernel/Kconfig.hz -@@ -40,6 +40,27 @@ choice - on SMP and NUMA systems and exactly dividing by both PAL and - NTSC frame rates for video and multimedia work. - -+ config HZ_500 -+ bool "500 HZ" -+ help -+ 500 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. -+ -+ config HZ_600 -+ bool "600 HZ" -+ help -+ 600 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. -+ -+ config HZ_750 -+ bool "750 HZ" -+ help -+ 750 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. -+ - config HZ_1000 - bool "1000 HZ" - help -@@ -53,6 +74,9 @@ config HZ - default 100 if HZ_100 - default 250 if HZ_250 - default 300 if HZ_300 -+ default 500 if HZ_500 -+ default 600 if HZ_600 -+ default 750 if HZ_750 - default 1000 if HZ_1000 - - config SCHED_HRTICK -diff --git a/kernel/fork.c b/kernel/fork.c -index 2b6bd511c6ed..704fe6bc9cb4 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -99,6 +99,10 @@ - #include - #include - -+#ifdef CONFIG_USER_NS -+#include -+#endif -+ - #include - #include - #include -@@ -2009,6 +2013,10 @@ static __latent_entropy struct task_struct *copy_process( - if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) - return ERR_PTR(-EINVAL); - -+ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) -+ if (!capable(CAP_SYS_ADMIN)) -+ return ERR_PTR(-EPERM); -+ - /* - * Thread groups must share signals as well, and detached threads - * can only be started up within the thread group. -@@ -3159,6 +3167,12 @@ int ksys_unshare(unsigned long unshare_flags) - if (unshare_flags & CLONE_NEWNS) - unshare_flags |= CLONE_FS; - -+ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { -+ err = -EPERM; -+ if (!capable(CAP_SYS_ADMIN)) -+ goto bad_unshare_out; -+ } -+ - err = check_unshare_flags(unshare_flags); - if (err) - goto bad_unshare_out; -diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c -index 65f0262f635e..8a8f77930818 100644 ---- a/kernel/locking/rwsem.c -+++ b/kernel/locking/rwsem.c -@@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) - struct task_struct *new, *owner; - unsigned long flags, new_flags; - enum owner_state state; -+ int i = 0; - - lockdep_assert_preemption_disabled(); - -@@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) - break; - } - -- cpu_relax(); -+ if (i++ > 1000) -+ cpu_relax(); - } - - return state; -diff --git a/kernel/module/internal.h b/kernel/module/internal.h -index 680d980a4fb2..8a3abfff9fe9 100644 ---- a/kernel/module/internal.h -+++ b/kernel/module/internal.h -@@ -53,6 +53,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; - extern const s32 __start___kcrctab[]; - extern const s32 __start___kcrctab_gpl[]; - -+extern struct boot_params boot_params; -+ - struct load_info { - const char *name; - /* pointer to module in temporary copy, freed at end of load_module() */ -diff --git a/kernel/module/main.c b/kernel/module/main.c -index a4e4d84b6f4e..0e698f5bee86 100644 ---- a/kernel/module/main.c -+++ b/kernel/module/main.c -@@ -53,6 +53,7 @@ - #include - #include - #include -+#include - #include - #include "internal.h" - -diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c -index cf5b9f1e6ec4..80260fa4dac5 100644 ---- a/kernel/module/procfs.c -+++ b/kernel/module/procfs.c -@@ -141,6 +141,19 @@ static const struct proc_ops modules_proc_ops = { - static int __init proc_modules_init(void) - { - proc_create("modules", 0, NULL, &modules_proc_ops); -+ -+#ifdef CONFIG_MODULE_SIG_FORCE -+ switch (boot_params.secure_boot) { -+ case efi_secureboot_mode_unset: -+ case efi_secureboot_mode_unknown: -+ case efi_secureboot_mode_disabled: -+ /* -+ * sig_unenforce is only applied if SecureBoot is not -+ * enabled. -+ */ -+ sig_enforce = !sig_unenforce; -+ } -+#endif - return 0; - } - module_init(proc_modules_init); -diff --git a/kernel/module/signing.c b/kernel/module/signing.c -index a2ff4242e623..876e93758e91 100644 ---- a/kernel/module/signing.c -+++ b/kernel/module/signing.c -@@ -21,6 +21,10 @@ - - static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE); - module_param(sig_enforce, bool_enable_only, 0644); -+/* Allow disabling module signature requirement by adding boot param */ -+static bool sig_unenforce = false; -+module_param(sig_unenforce, bool_enable_only, 0644); -+ - - /* - * Export sig_enforce kernel cmdline parameter to allow other subsystems rely -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index ee28253c9ac0..f2534e712a89 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -138,6 +138,18 @@ __read_mostly int sysctl_resched_latency_warn_ms = 100; - __read_mostly int sysctl_resched_latency_warn_once = 1; - #endif /* CONFIG_SCHED_DEBUG */ - -+/* -+ * Choose the yield level that will perform. -+ * 0: No yield. -+ * 1: Yield only to better priority/deadline tasks. -+ * 2: Re-queue current tasks. (default CFS) -+ */ -+#ifdef CONFIG_CACHY -+__read_mostly int sysctl_sched_yield_type = 1; -+#else -+__read_mostly int sysctl_sched_yield_type = 0; -+#endif -+ - /* - * Number of tasks to iterate in a single balance run. - * Limited because this is done with IRQs disabled. -@@ -8266,10 +8278,15 @@ static void do_sched_yield(void) - struct rq_flags rf; - struct rq *rq; - -+ if (!sysctl_sched_yield_type) -+ return; -+ - rq = this_rq_lock_irq(&rf); - - schedstat_inc(rq->yld_count); -- current->sched_class->yield_task(rq); -+ -+ if (sysctl_sched_yield_type > 1) -+ current->sched_class->yield_task(rq); - - preempt_disable(); - rq_unlock_irq(rq, &rf); -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 667876da8382..13fbcd19b916 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -959,6 +959,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - PN(se.exec_start); - PN(se.vruntime); - PN(se.sum_exec_runtime); -+ P(fsync_count); - - nr_switches = p->nvcsw + p->nivcsw; - -diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 914096c5b1ae..74d773040b54 100644 ---- a/kernel/sched/fair.c -+++ b/kernel/sched/fair.c -@@ -68,9 +68,13 @@ - * - * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_latency = 3000000ULL; -+static unsigned int normalized_sysctl_sched_latency = 3000000ULL; -+#else - unsigned int sysctl_sched_latency = 6000000ULL; - static unsigned int normalized_sysctl_sched_latency = 6000000ULL; -- -+#endif - /* - * The initial- and re-scaling of tunables is configurable - * -@@ -89,8 +93,13 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; - * - * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_min_granularity = 400000ULL; -+static unsigned int normalized_sysctl_sched_min_granularity = 400000ULL; -+#else - unsigned int sysctl_sched_min_granularity = 750000ULL; - static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; -+#endif - - /* - * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. -@@ -120,8 +129,13 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; - * - * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_wakeup_granularity = 500000UL; -+static unsigned int normalized_sysctl_sched_wakeup_granularity = 500000UL; -+#else - unsigned int sysctl_sched_wakeup_granularity = 1000000UL; - static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; -+#endif - - const_debug unsigned int sysctl_sched_migration_cost = 500000UL; - -@@ -174,8 +188,12 @@ int __weak arch_asym_cpu_priority(int cpu) - * - * (default: 5 msec, units: microseconds) - */ -+#ifdef CONFIG_CACHY -+static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; -+#else - static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; - #endif -+#endif - - #ifdef CONFIG_SYSCTL - static struct ctl_table sched_fair_sysctls[] = { -diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c -index 9860bb9a847c..c7f045873a5f 100644 ---- a/kernel/sched/wait.c -+++ b/kernel/sched/wait.c -@@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ - } - EXPORT_SYMBOL_GPL(add_wait_queue_priority); - -+void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) -+{ -+ unsigned long flags; -+ -+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; -+ spin_lock_irqsave(&wq_head->lock, flags); -+ __add_wait_queue(wq_head, wq_entry); -+ spin_unlock_irqrestore(&wq_head->lock, flags); -+} -+EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); -+ - void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) - { - unsigned long flags; -@@ -289,6 +300,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent - } - EXPORT_SYMBOL(prepare_to_wait_exclusive); - -+void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) -+{ -+ unsigned long flags; -+ -+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; -+ spin_lock_irqsave(&wq_head->lock, flags); -+ if (list_empty(&wq_entry->entry)) -+ __add_wait_queue(wq_head, wq_entry); -+ set_current_state(state); -+ spin_unlock_irqrestore(&wq_head->lock, flags); -+} -+EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); -+ - void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) - { - wq_entry->flags = flags; -diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c -index 860b2dcf3ac4..810e1fcaff94 100644 ---- a/kernel/sys_ni.c -+++ b/kernel/sys_ni.c -@@ -292,6 +292,7 @@ COND_SYSCALL(mincore); - COND_SYSCALL(madvise); - COND_SYSCALL(process_madvise); - COND_SYSCALL(process_mrelease); -+COND_SYSCALL(pmadv_ksm); - COND_SYSCALL(remap_file_pages); - COND_SYSCALL(mbind); - COND_SYSCALL(get_mempolicy); -diff --git a/kernel/sysctl.c b/kernel/sysctl.c -index 205d605cacc5..4432b6f15890 100644 ---- a/kernel/sysctl.c -+++ b/kernel/sysctl.c -@@ -89,8 +89,12 @@ - #ifdef CONFIG_PERF_EVENTS - static const int six_hundred_forty_kb = 640 * 1024; - #endif -+#ifdef CONFIG_USER_NS -+#include -+#endif - - -+extern int sysctl_sched_yield_type; - static const int ngroups_max = NGROUPS_MAX; - static const int cap_last_cap = CAP_LAST_CAP; - -@@ -1649,6 +1653,24 @@ static struct ctl_table kern_table[] = { - .mode = 0644, - .proc_handler = proc_dointvec, - }, -+ { -+ .procname = "yield_type", -+ .data = &sysctl_sched_yield_type, -+ .maxlen = sizeof (int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_TWO, -+ }, -+#ifdef CONFIG_USER_NS -+ { -+ .procname = "unprivileged_userns_clone", -+ .data = &unprivileged_userns_clone, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ }, -+#endif - #ifdef CONFIG_PROC_SYSCTL - { - .procname = "tainted", -diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c -index 5481ba44a8d6..423ab2563ad7 100644 ---- a/kernel/user_namespace.c -+++ b/kernel/user_namespace.c -@@ -21,6 +21,13 @@ - #include - #include - -+/* sysctl */ -+#ifdef CONFIG_USER_NS_UNPRIVILEGED -+int unprivileged_userns_clone = 1; -+#else -+int unprivileged_userns_clone; -+#endif -+ - static struct kmem_cache *user_ns_cachep __read_mostly; - static DEFINE_MUTEX(userns_state_mutex); - -diff --git a/kernel/watchdog.c b/kernel/watchdog.c -index 8e61f21e7e33..be1439d38f26 100644 ---- a/kernel/watchdog.c -+++ b/kernel/watchdog.c -@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled; - int __read_mostly watchdog_user_enabled = 1; - int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT; - int __read_mostly soft_watchdog_user_enabled = 1; --int __read_mostly watchdog_thresh = 10; -+int __read_mostly watchdog_thresh = 40; - static int __read_mostly nmi_watchdog_available; - - struct cpumask watchdog_cpumask __read_mostly; -diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c -index 39b74221f4a7..ec3eab8cd6b1 100644 ---- a/lib/raid6/algos.c -+++ b/lib/raid6/algos.c -@@ -128,8 +128,10 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) -- if (!(*algo)->valid || (*algo)->valid()) -+ if (!(*algo)->valid || (*algo)->valid()) { - best = *algo; -+ break; -+ } - - if (best) { - raid6_2data_recov = best->data2; -diff --git a/lib/string.c b/lib/string.c -index 6f334420f687..2e60e9e40535 100644 ---- a/lib/string.c -+++ b/lib/string.c -@@ -866,24 +866,61 @@ char *strnstr(const char *s1, const char *s2, size_t len) - EXPORT_SYMBOL(strnstr); - #endif - -+#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 -+ -+#define MEMCHR_MASK_GEN(mask) (mask *= 0x0101010101010101ULL) -+ -+#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) -+ -+#define MEMCHR_MASK_GEN(mask) \ -+ do { \ -+ mask *= 0x01010101; \ -+ mask |= mask << 32; \ -+ } while (0) -+ -+#else -+ -+#define MEMCHR_MASK_GEN(mask) \ -+ do { \ -+ mask |= mask << 8; \ -+ mask |= mask << 16; \ -+ mask |= mask << 32; \ -+ } while (0) -+ -+#endif -+ - #ifndef __HAVE_ARCH_MEMCHR - /** - * memchr - Find a character in an area of memory. -- * @s: The memory area -+ * @p: The memory area - * @c: The byte to search for -- * @n: The size of the area. -+ * @length: The size of the area. - * - * returns the address of the first occurrence of @c, or %NULL - * if @c is not found - */ --void *memchr(const void *s, int c, size_t n) -+void *memchr(const void *p, int c, unsigned long length) - { -- const unsigned char *p = s; -- while (n-- != 0) { -- if ((unsigned char)c == *p++) { -- return (void *)(p - 1); -+ u64 mask, val; -+ const void *end = p + length; -+ -+ c &= 0xff; -+ if (p <= end - 8) { -+ mask = c; -+ MEMCHR_MASK_GEN(mask); -+ -+ for (; p <= end - 8; p += 8) { -+ val = *(u64 *)p ^ mask; -+ if ((val + 0xfefefefefefefeffu) & -+ (~val & 0x8080808080808080u)) -+ break; - } - } -+ -+ for (; p < end; p++) -+ if (*(unsigned char *)p == c) -+ return (void *)p; -+ - return NULL; - } - EXPORT_SYMBOL(memchr); -@@ -919,16 +956,7 @@ void *memchr_inv(const void *start, int c, size_t bytes) - return check_bytes8(start, value, bytes); - - value64 = value; --#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 -- value64 *= 0x0101010101010101ULL; --#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) -- value64 *= 0x01010101; -- value64 |= value64 << 32; --#else -- value64 |= value64 << 8; -- value64 |= value64 << 16; -- value64 |= value64 << 32; --#endif -+ MEMCHR_MASK_GEN(value64); - - prefix = (unsigned long)start % 8; - if (prefix) { -diff --git a/lib/zstd/compress/zstd_compress.c b/lib/zstd/compress/zstd_compress.c -index a4e916008b3a..73fff4c60149 100644 ---- a/lib/zstd/compress/zstd_compress.c -+++ b/lib/zstd/compress/zstd_compress.c -@@ -4441,7 +4441,7 @@ static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, - size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { - size_t offsetBound; - U32 windowSize = 1 << windowLog; -- /* posInSrc represents the amount of data the the decoder would decode up to this point. -+ /* posInSrc represents the amount of data the decoder would decode up to this point. - * As long as the amount of data decoded is less than or equal to window size, offsets may be - * larger than the total length of output decoded in order to reference the dict, even larger than - * window size. After output surpasses windowSize, we're limited to windowSize offsets again. -diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c -index b0424d23ac57..fb941a5b70f5 100644 ---- a/lib/zstd/compress/zstd_double_fast.c -+++ b/lib/zstd/compress/zstd_double_fast.c -@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(dictMode, mls) \ -+ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \ -+ } -+ -+ZSTD_GEN_FN(noDict, 4) -+ZSTD_GEN_FN(noDict, 5) -+ZSTD_GEN_FN(noDict, 6) -+ZSTD_GEN_FN(noDict, 7) -+ -+ZSTD_GEN_FN(dictMatchState, 4) -+ZSTD_GEN_FN(dictMatchState, 5) -+ZSTD_GEN_FN(dictMatchState, 6) -+ZSTD_GEN_FN(dictMatchState, 7) -+ -+#undef ZSTD_GEN_FN -+ - - size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - - /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ - if (prefixStartIndex == dictStartIndex) -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - } - - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN -+ - size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); - } - } -diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c -index 96b7d48e2868..e0652e31d421 100644 ---- a/lib/zstd/compress/zstd_fast.c -+++ b/lib/zstd/compress/zstd_fast.c -@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN - - size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN -+ - size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( - - /* switch to "regular" variant if extDict is invalidated due to maxDistance */ - if (prefixStartIndex == dictStartIndex) -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); -+ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_extDict_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN - - size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize); - } - } -diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c -index fb54d4e28a2b..1db22db5b1e9 100644 ---- a/lib/zstd/compress/zstd_lazy.c -+++ b/lib/zstd/compress/zstd_lazy.c -@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, - } - - --static size_t --ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); -- } --} -- -- --static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); -- } --} -- -- --static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); -- } --} -- -- -- - /* ********************************* - * Hash Chain - ***********************************/ -@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B - - /* inlining is important to hardwire a hot branch (template emulation) */ - FORCE_INLINE_TEMPLATE --size_t ZSTD_HcFindBestMatch_generic ( -+size_t ZSTD_HcFindBestMatch( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, -@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic ( - return ml; - } - -+typedef size_t (*searchMax_f)( -+ ZSTD_matchState_t* ms, -+ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - --FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); -- } --} -+/* -+ * This struct contains the functions necessary for lazy to search. -+ * Currently, that is only searchMax. However, it is still valuable to have the -+ * VTable because this makes it easier to add more functions to the VTable later. -+ */ -+typedef struct { -+ searchMax_f searchMax; -+} ZSTD_LazyVTable; -+ -+#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \ -+ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, \ -+ const BYTE* ip, const BYTE* const iLimit, \ -+ size_t* offsetPtr) \ -+ { \ -+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ -+ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ -+ } \ -+ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ -+ ZSTD_BtFindBestMatch_##dictMode##_##mls \ -+ }; - -+#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \ -+ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, \ -+ const BYTE* ip, const BYTE* const iLimit, \ -+ size_t* offsetPtr) \ -+ { \ -+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ -+ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ -+ } \ -+ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ -+ ZSTD_HcFindBestMatch_##dictMode##_##mls \ -+ }; - --static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); -+#define ZSTD_FOR_EACH_MLS(X, dictMode) \ -+ X(dictMode, 4) \ -+ X(dictMode, 5) \ -+ X(dictMode, 6) -+ -+#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ -+ X(__VA_ARGS__, noDict) \ -+ X(__VA_ARGS__, extDict) \ -+ X(__VA_ARGS__, dictMatchState) \ -+ X(__VA_ARGS__, dedicatedDictSearch) -+ -+/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ -+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) -+/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ -+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) -+ -+#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ -+ { \ -+ &ZSTD_BtVTable_##dictMode##_4, \ -+ &ZSTD_BtVTable_##dictMode##_5, \ -+ &ZSTD_BtVTable_##dictMode##_6 \ - } --} -- - --static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); -+#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ -+ { \ -+ &ZSTD_HcVTable_##dictMode##_4, \ -+ &ZSTD_HcVTable_##dictMode##_5, \ -+ &ZSTD_HcVTable_##dictMode##_6 \ - } --} - -- --FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); -+#define GEN_ZSTD_VTABLE_ARRAY(X) \ -+ { \ -+ X(noDict), \ -+ X(extDict), \ -+ X(dictMatchState), \ -+ X(dedicatedDictSearch) \ - } --} -- - - /* ******************************* - * Common parser - lazy strategy - *********************************/ - typedef enum { search_hashChain, search_binaryTree } searchMethod_e; - -+static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) -+{ -+ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ -+ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); -+ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); -+ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ -+ -+ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); -+ switch (searchMethod) { -+ case search_hashChain: -+ return hcVTables[dictMode][mls - 4]; -+ case search_binaryTree: -+ return btVTables[dictMode][mls - 4]; -+ default: -+ return NULL; -+ } -+} -+ - FORCE_INLINE_TEMPLATE size_t - ZSTD_compressBlock_lazy_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, -@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic( - const U32 prefixLowestIndex = ms->window.dictLimit; - const BYTE* const prefixLowest = base + prefixLowestIndex; - -- typedef size_t (*searchMax_f)( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -- - /* - * This table is indexed first by the four ZSTD_dictMode_e values, and then - * by the two searchMethod_e values. NULLs are placed for configurations - * that should never occur (extDict modes go to the other implementation - * below and there is no DDSS for binary tree search yet). - */ -- const searchMax_f searchFuncs[4][2] = { -- { -- ZSTD_HcFindBestMatch_selectMLS, -- ZSTD_BtFindBestMatch_selectMLS -- }, -- { -- NULL, -- NULL -- }, -- { -- ZSTD_HcFindBestMatch_dictMatchState_selectMLS, -- ZSTD_BtFindBestMatch_dictMatchState_selectMLS -- }, -- { -- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, -- NULL -- } -- }; -- -- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; -+ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; - U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; - - const int isDMS = dictMode == ZSTD_dictMatchState; -@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( - const BYTE* const dictStart = dictBase + ms->window.lowLimit; - const U32 windowLog = ms->cParams.windowLog; - -- typedef size_t (*searchMax_f)( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; -+ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; - - U32 offset_1 = rep[0], offset_2 = rep[1]; - -diff --git a/mm/compaction.c b/mm/compaction.c -index 640fa76228dd..01a661f5ecac 100644 ---- a/mm/compaction.c -+++ b/mm/compaction.c -@@ -1727,7 +1727,7 @@ typedef enum { - * Allow userspace to control policy on scanning the unevictable LRU for - * compactable pages. - */ --#ifdef CONFIG_PREEMPT_RT -+#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) - int sysctl_compact_unevictable_allowed __read_mostly = 0; - #else - int sysctl_compact_unevictable_allowed __read_mostly = 1; -@@ -2719,7 +2719,11 @@ static void compact_nodes(void) - * aggressively the kernel should compact memory in the - * background. It takes values in the range [0, 100]. - */ -+#ifdef CONFIG_CACHY -+unsigned int __read_mostly sysctl_compaction_proactiveness; -+#else - unsigned int __read_mostly sysctl_compaction_proactiveness = 20; -+#endif - - int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, - void *buffer, size_t *length, loff_t *ppos) -diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig -index 66265e3a9c65..7821fcb3f258 100644 ---- a/mm/damon/Kconfig -+++ b/mm/damon/Kconfig -@@ -68,6 +68,9 @@ config DAMON_DBGFS - - If unsure, say N. - -+ This will be removed after >5.15.y LTS kernel is released, so users -+ should move to the sysfs interface (DAMON_SYSFS). -+ - config DAMON_DBGFS_KUNIT_TEST - bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS - depends on DAMON_DBGFS && KUNIT=y -diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h -index 573669566f84..3db9b7368756 100644 ---- a/mm/damon/core-test.h -+++ b/mm/damon/core-test.h -@@ -126,7 +126,7 @@ static void damon_test_split_at(struct kunit *test) - t = damon_new_target(); - r = damon_new_region(0, 100); - damon_add_region(r, t); -- damon_split_region_at(c, t, r, 25); -+ damon_split_region_at(t, r, 25); - KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); - KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); - -@@ -219,14 +219,14 @@ static void damon_test_split_regions_of(struct kunit *test) - t = damon_new_target(); - r = damon_new_region(0, 22); - damon_add_region(r, t); -- damon_split_regions_of(c, t, 2); -+ damon_split_regions_of(t, 2); - KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u); - damon_free_target(t); - - t = damon_new_target(); - r = damon_new_region(0, 220); - damon_add_region(r, t); -- damon_split_regions_of(c, t, 4); -+ damon_split_regions_of(t, 4); - KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u); - damon_free_target(t); - damon_destroy_ctx(c); -@@ -267,6 +267,28 @@ static void damon_test_ops_registration(struct kunit *test) - KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL); - } - -+static void damon_test_set_regions(struct kunit *test) -+{ -+ struct damon_target *t = damon_new_target(); -+ struct damon_region *r1 = damon_new_region(4, 16); -+ struct damon_region *r2 = damon_new_region(24, 32); -+ struct damon_addr_range range = {.start = 8, .end = 28}; -+ unsigned long expects[] = {8, 16, 16, 24, 24, 28}; -+ int expect_idx = 0; -+ struct damon_region *r; -+ -+ damon_add_region(r1, t); -+ damon_add_region(r2, t); -+ damon_set_regions(t, &range, 1); -+ -+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); -+ damon_for_each_region(r, t) { -+ KUNIT_EXPECT_EQ(test, r->ar.start, expects[expect_idx++]); -+ KUNIT_EXPECT_EQ(test, r->ar.end, expects[expect_idx++]); -+ } -+ damon_destroy_target(t); -+} -+ - static struct kunit_case damon_test_cases[] = { - KUNIT_CASE(damon_test_target), - KUNIT_CASE(damon_test_regions), -@@ -276,6 +298,7 @@ static struct kunit_case damon_test_cases[] = { - KUNIT_CASE(damon_test_merge_regions_of), - KUNIT_CASE(damon_test_split_regions_of), - KUNIT_CASE(damon_test_ops_registration), -+ KUNIT_CASE(damon_test_set_regions), - {}, - }; - -diff --git a/mm/damon/core.c b/mm/damon/core.c -index 7d25dc582fe3..1c7366bf477d 100644 ---- a/mm/damon/core.c -+++ b/mm/damon/core.c -@@ -168,6 +168,27 @@ static bool damon_intersect(struct damon_region *r, - return !(r->ar.end <= re->start || re->end <= r->ar.start); - } - -+/* -+ * Fill holes in regions with new regions. -+ */ -+static void damon_fill_regions_holes(struct damon_region *first, -+ struct damon_region *last, struct damon_target *t) -+{ -+ struct damon_region *r = first; -+ -+ damon_for_each_region_from(r, t) { -+ struct damon_region *next, *newr; -+ -+ if (r == last) -+ break; -+ next = damon_next_region(r); -+ if (r->ar.end != next->ar.start) { -+ newr = damon_new_region(r->ar.end, next->ar.start); -+ damon_insert_region(newr, r, next, t); -+ } -+ } -+} -+ - /* - * damon_set_regions() - Set regions of a target for given address ranges. - * @t: the given target. -@@ -225,6 +246,9 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, - first->ar.start = ALIGN_DOWN(range->start, - DAMON_MIN_REGION); - last->ar.end = ALIGN(range->end, DAMON_MIN_REGION); -+ -+ /* fill possible holes in the range */ -+ damon_fill_regions_holes(first, last, t); - } - } - return 0; -@@ -658,9 +682,8 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) - } - } - --static void damon_split_region_at(struct damon_ctx *ctx, -- struct damon_target *t, struct damon_region *r, -- unsigned long sz_r); -+static void damon_split_region_at(struct damon_target *t, -+ struct damon_region *r, unsigned long sz_r); - - static bool __damos_valid_target(struct damon_region *r, struct damos *s) - { -@@ -726,7 +749,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, - continue; - sz = DAMON_MIN_REGION; - } -- damon_split_region_at(c, t, r, sz); -+ damon_split_region_at(t, r, sz); - r = damon_next_region(r); - sz = r->ar.end - r->ar.start; - } -@@ -745,7 +768,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, - DAMON_MIN_REGION); - if (!sz) - goto update_stat; -- damon_split_region_at(c, t, r, sz); -+ damon_split_region_at(t, r, sz); - } - ktime_get_coarse_ts64(&begin); - sz_applied = c->ops.apply_scheme(c, t, r, s); -@@ -928,9 +951,8 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, - * r the region to be split - * sz_r size of the first sub-region that will be made - */ --static void damon_split_region_at(struct damon_ctx *ctx, -- struct damon_target *t, struct damon_region *r, -- unsigned long sz_r) -+static void damon_split_region_at(struct damon_target *t, -+ struct damon_region *r, unsigned long sz_r) - { - struct damon_region *new; - -@@ -947,8 +969,7 @@ static void damon_split_region_at(struct damon_ctx *ctx, - } - - /* Split every region in the given target into 'nr_subs' regions */ --static void damon_split_regions_of(struct damon_ctx *ctx, -- struct damon_target *t, int nr_subs) -+static void damon_split_regions_of(struct damon_target *t, int nr_subs) - { - struct damon_region *r, *next; - unsigned long sz_region, sz_sub = 0; -@@ -969,7 +990,7 @@ static void damon_split_regions_of(struct damon_ctx *ctx, - if (sz_sub == 0 || sz_sub >= sz_region) - continue; - -- damon_split_region_at(ctx, t, r, sz_sub); -+ damon_split_region_at(t, r, sz_sub); - sz_region = sz_sub; - } - } -@@ -1004,7 +1025,7 @@ static void kdamond_split_regions(struct damon_ctx *ctx) - nr_subregions = 3; - - damon_for_each_target(t, ctx) -- damon_split_regions_of(ctx, t, nr_subregions); -+ damon_split_regions_of(t, nr_subregions); - - last_nr_regions = nr_regions; - } -diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c -index cfdf63132d5a..3b55a1b219b5 100644 ---- a/mm/damon/dbgfs.c -+++ b/mm/damon/dbgfs.c -@@ -1044,7 +1044,7 @@ static int __init __damon_dbgfs_init(void) - fops[i]); - dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]); - -- dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL); -+ dbgfs_dirs = kmalloc(sizeof(dbgfs_root), GFP_KERNEL); - if (!dbgfs_dirs) { - debugfs_remove(dbgfs_root); - return -ENOMEM; -diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c -index 9de6f00a71c5..a3674532fa67 100644 ---- a/mm/damon/lru_sort.c -+++ b/mm/damon/lru_sort.c -@@ -257,18 +257,13 @@ module_param(nr_cold_quota_exceeds, ulong, 0400); - static struct damon_ctx *ctx; - static struct damon_target *target; - --struct damon_lru_sort_ram_walk_arg { -- unsigned long start; -- unsigned long end; --}; -- - static int walk_system_ram(struct resource *res, void *arg) - { -- struct damon_lru_sort_ram_walk_arg *a = arg; -+ struct damon_addr_range *r = arg; - -- if (a->end - a->start < resource_size(res)) { -- a->start = res->start; -- a->end = res->end; -+ if (r->end - r->start < resource_size(res)) { -+ r->start = res->start; -+ r->end = res->end; - } - return 0; - } -@@ -277,16 +272,12 @@ static int walk_system_ram(struct resource *res, void *arg) - * Find biggest 'System RAM' resource and store its start and end address in - * @start and @end, respectively. If no System RAM is found, returns false. - */ --static bool get_monitoring_region(unsigned long *start, unsigned long *end) -+static bool get_monitoring_region(struct damon_addr_range *range) - { -- struct damon_lru_sort_ram_walk_arg arg = {}; -- -- walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); -- if (arg.end <= arg.start) -+ walk_system_ram_res(0, ULONG_MAX, range, walk_system_ram); -+ if (range->end <= range->start) - return false; - -- *start = arg.start; -- *end = arg.end; - return true; - } - -@@ -378,6 +369,16 @@ static int damon_lru_sort_apply_parameters(void) - unsigned int hot_thres, cold_thres; - int err = 0; - -+ if (monitor_region_start > monitor_region_end) -+ return -EINVAL; -+ if (!monitor_region_end) -+ return -EINVAL; -+ -+ addr_range.start = monitor_region_start; -+ addr_range.end = monitor_region_end; -+ if (!get_monitoring_region(&addr_range)) -+ return -EINVAL; -+ - err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, - min_nr_regions, max_nr_regions); - if (err) -@@ -401,14 +402,6 @@ static int damon_lru_sort_apply_parameters(void) - return -ENOMEM; - damon_add_scheme(ctx, scheme); - -- if (monitor_region_start > monitor_region_end) -- return -EINVAL; -- if (!monitor_region_start && !monitor_region_end && -- !get_monitoring_region(&monitor_region_start, -- &monitor_region_end)) -- return -EINVAL; -- addr_range.start = monitor_region_start; -- addr_range.end = monitor_region_end; - return damon_set_regions(target, &addr_range, 1); - } - -diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c -index 3c7b9d6dca95..cc04d467ba23 100644 ---- a/mm/damon/vaddr.c -+++ b/mm/damon/vaddr.c -@@ -302,9 +302,14 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, - pte_t *pte; - spinlock_t *ptl; - -- if (pmd_huge(*pmd)) { -+ if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(walk->mm, pmd); -- if (pmd_huge(*pmd)) { -+ if (!pmd_present(*pmd)) { -+ spin_unlock(ptl); -+ return 0; -+ } -+ -+ if (pmd_trans_huge(*pmd)) { - damon_pmdp_mkold(pmd, walk->mm, addr); - spin_unlock(ptl); - return 0; -@@ -429,9 +434,14 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, - struct damon_young_walk_private *priv = walk->private; - - #ifdef CONFIG_TRANSPARENT_HUGEPAGE -- if (pmd_huge(*pmd)) { -+ if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(walk->mm, pmd); -- if (!pmd_huge(*pmd)) { -+ if (!pmd_present(*pmd)) { -+ spin_unlock(ptl); -+ return 0; -+ } -+ -+ if (!pmd_trans_huge(*pmd)) { - spin_unlock(ptl); - goto regular_page; - } -diff --git a/mm/ksm.c b/mm/ksm.c -index 42ab153335a2..f3e6eaf437ae 100644 ---- a/mm/ksm.c -+++ b/mm/ksm.c -@@ -131,6 +131,10 @@ struct mm_slot { - * @address: the next address inside that to be scanned - * @rmap_list: link to the next rmap to be scanned in the rmap_list - * @seqnr: count of completed full scans (needed when removing unstable node) -+ * @new_ksmpages: count of the new merged KSM pages in the current scanning -+ * of mm_lists (cleared after every turn of ksm_do_scan() ends) -+ * @prev_ksmpages: the record of the new merged KSM pages in the last turn of -+ * scanning by ksm_do_scan(). - * - * There is only the one ksm_scan instance of this cursor structure. - */ -@@ -139,6 +143,8 @@ struct ksm_scan { - unsigned long address; - struct rmap_item **rmap_list; - unsigned long seqnr; -+ unsigned long new_ksmpages; -+ unsigned long prev_ksmpages; - }; - - /** -@@ -277,6 +283,33 @@ static unsigned int zero_checksum __read_mostly; - /* Whether to merge empty (zeroed) pages with actual zero pages */ - static bool ksm_use_zero_pages __read_mostly; - -+/* -+ * Work in auto-mode. -+ * The multiplicative factor of pages_to_scan. -+ * Real pages to scan equals to the product of scanning_factor -+ * and pages_to_scan -+ */ -+#define INIT_SCANNING_FACTOR 1 -+static unsigned int scanning_factor = INIT_SCANNING_FACTOR; -+ -+/* The upper limit of scanning_factor */ -+#define DEFAULT_MAX_SCANNING_FACTOR 16 -+static unsigned int max_scanning_factor = DEFAULT_MAX_SCANNING_FACTOR; -+ -+/* -+ * Work in auto mode. -+ * Value: 0~100. Default 20 means "20%". When free memory is lower -+ * than this total memory * ksm_auto_threshold/100, auto_triggered -+ * will be set true. -+ */ -+unsigned int ksm_auto_threshold = 20; -+ -+/* Work in auto-mode. Whether trigger ksmd to compare and merge pages */ -+static bool auto_triggered; -+ -+/* Count of times that ksmd is triggered due to low free memory */ -+static unsigned long triggered_times; -+ - #ifdef CONFIG_NUMA - /* Zeroed when merging across nodes is not allowed */ - static unsigned int ksm_merge_across_nodes = 1; -@@ -290,6 +323,7 @@ static int ksm_nr_node_ids = 1; - #define KSM_RUN_MERGE 1 - #define KSM_RUN_UNMERGE 2 - #define KSM_RUN_OFFLINE 4 -+#define KSM_RUN_AUTO 8 - static unsigned long ksm_run = KSM_RUN_STOP; - static void wait_while_offlining(void); - -@@ -387,6 +421,7 @@ static inline struct rmap_item *alloc_rmap_item(void) - static inline void free_rmap_item(struct rmap_item *rmap_item) - { - ksm_rmap_items--; -+ rmap_item->mm->ksm_rmap_items--; - rmap_item->mm = NULL; /* debug safety */ - kmem_cache_free(rmap_item_cache, rmap_item); - } -@@ -2020,6 +2055,8 @@ static void stable_tree_append(struct rmap_item *rmap_item, - rmap_item->address |= STABLE_FLAG; - hlist_add_head(&rmap_item->hlist, &stable_node->hlist); - -+ ksm_scan.new_ksmpages++; -+ - if (rmap_item->hlist.next) - ksm_pages_sharing++; - else -@@ -2219,6 +2256,7 @@ static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, - if (rmap_item) { - /* It has already been zeroed */ - rmap_item->mm = mm_slot->mm; -+ rmap_item->mm->ksm_rmap_items++; - rmap_item->address = addr; - rmap_item->rmap_list = *rmap_list; - *rmap_list = rmap_item; -@@ -2399,16 +2437,107 @@ static void ksm_do_scan(unsigned int scan_npages) - rmap_item = scan_get_next_rmap_item(&page); - if (!rmap_item) - return; -- cmp_and_merge_page(page, rmap_item); -+ if (ksm_run & KSM_RUN_AUTO && !auto_triggered) { -+ /* -+ * This should happens only when ksm_run is KSM_RUN_AUTO -+ * and free memory threshold still not reached. -+ * The reason to calculate it's checksum is to reduce the -+ * waiting time the rmap_item is added to unstable tree. -+ */ -+ rmap_item->oldchecksum = calc_checksum(page); -+ } else -+ cmp_and_merge_page(page, rmap_item); -+ - put_page(page); - } - } - -+#define RIGHT_SHIFT_FOUR_BIT 4 -+/* Work in auto mode, should reset auto_triggered ? */ -+static bool should_stop_ksmd_to_merge(void) -+{ -+ unsigned long total_ram_pages, free_pages; -+ unsigned int threshold; -+ -+ total_ram_pages = totalram_pages(); -+ free_pages = global_zone_page_state(NR_FREE_PAGES); -+ threshold = READ_ONCE(ksm_auto_threshold); -+ -+ return free_pages > (total_ram_pages * threshold / 100) + -+ (total_ram_pages >> RIGHT_SHIFT_FOUR_BIT); -+} -+ -+/* Work in auto mode, should ksmd start to merge ? */ -+static bool should_trigger_ksmd_to_merge(void) -+{ -+ unsigned long total_ram_pages, free_pages; -+ unsigned int threshold; -+ -+ total_ram_pages = totalram_pages(); -+ free_pages = global_zone_page_state(NR_FREE_PAGES); -+ threshold = READ_ONCE(ksm_auto_threshold); -+ -+ return free_pages < (total_ram_pages * threshold / 100); -+} -+ -+static inline void trigger_ksmd_to_merge(void) -+{ -+ if (!auto_triggered) { -+ triggered_times++; -+ auto_triggered = true; -+ } -+} -+ -+static inline void stop_ksmd_to_merge(void) -+{ -+ if (auto_triggered) -+ auto_triggered = false; -+} -+ - static int ksmd_should_run(void) - { -- return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list); -+ if (!list_empty(&ksm_mm_head.mm_list)) -+ return ksm_run & KSM_RUN_AUTO || ksm_run & KSM_RUN_MERGE; -+ return 0; - } - -+/* -+ * Work in auto mode, the scan-enhanced algorithm. -+ * current_factor: the current scanning_factor. -+ * return: the scanning_factor caculated by scan-enhanced algorithm. -+ */ -+static unsigned int scan_enhanced_algorithm(unsigned int current_factor) -+{ -+ unsigned int next_factor; -+ unsigned int max, min; -+ -+ /* -+ * The calculation is divied into three cases as follows: -+ * -+ * Case 1: when new_ksmpages > prev_ksmpages * 1/2, get the -+ * next factor by double the current factor. -+ * Case 2: when 0 < new_ksmpages < prev_ksmpages * 1/2, keep -+ * the factor unchanged. -+ * Case 3: when new_ksmpages equals 0, then get the next -+ * factor by halfing the current factor. -+ */ -+ max = READ_ONCE(max_scanning_factor); -+ min = INIT_SCANNING_FACTOR; -+ if (ksm_scan.new_ksmpages * 2 > ksm_scan.prev_ksmpages) { -+ next_factor = current_factor << 1; /* Doubling */ -+ if (next_factor > max) -+ next_factor = max; -+ } else if (ksm_scan.new_ksmpages == 0) { -+ next_factor = current_factor >> 1; /* Halfing */ -+ next_factor = next_factor < min ? min : next_factor; -+ } else -+ next_factor = current_factor; -+ -+ return next_factor; -+} -+ -+#define SLOW_SCAN_PAGES 5 /* Used when ksmd is not triggered to merge*/ -+ - static int ksm_scan_thread(void *nothing) - { - unsigned int sleep_ms; -@@ -2419,17 +2548,41 @@ static int ksm_scan_thread(void *nothing) - while (!kthread_should_stop()) { - mutex_lock(&ksm_thread_mutex); - wait_while_offlining(); -- if (ksmd_should_run()) -- ksm_do_scan(ksm_thread_pages_to_scan); -+ if (ksmd_should_run()) { -+ if (ksm_run & KSM_RUN_AUTO) { -+ if (!auto_triggered) -+ ksm_do_scan(SLOW_SCAN_PAGES); -+ else -+ ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor); -+ -+ scanning_factor = scan_enhanced_algorithm(scanning_factor); -+ /* -+ * Reset ksm_scan.new_ksmpages after -+ * updating scanning_factor by scan_enhanced_algorithm. -+ */ -+ ksm_scan.new_ksmpages = 0; -+ -+ if (should_trigger_ksmd_to_merge()) -+ trigger_ksmd_to_merge(); -+ else if (should_stop_ksmd_to_merge()) -+ stop_ksmd_to_merge(); -+ } else -+ ksm_do_scan(ksm_thread_pages_to_scan); -+ } - mutex_unlock(&ksm_thread_mutex); - - try_to_freeze(); - - if (ksmd_should_run()) { - sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs); -- wait_event_interruptible_timeout(ksm_iter_wait, -- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -- msecs_to_jiffies(sleep_ms)); -+ if (sleep_ms >= 1000) -+ wait_event_interruptible_timeout(ksm_iter_wait, -+ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -+ msecs_to_jiffies(round_jiffies_relative(sleep_ms))); -+ else -+ wait_event_interruptible_timeout(ksm_iter_wait, -+ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -+ msecs_to_jiffies(sleep_ms)); - } else { - wait_event_freezable(ksm_thread_wait, - ksmd_should_run() || kthread_should_stop()); -@@ -2438,54 +2591,78 @@ static int ksm_scan_thread(void *nothing) - return 0; - } - --int ksm_madvise(struct vm_area_struct *vma, unsigned long start, -- unsigned long end, int advice, unsigned long *vm_flags) -+int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, -+ unsigned long *vm_flags) - { -- struct mm_struct *mm = vma->vm_mm; - int err; - -- switch (advice) { -- case MADV_MERGEABLE: -- /* -- * Be somewhat over-protective for now! -- */ -- if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | -- VM_PFNMAP | VM_IO | VM_DONTEXPAND | -- VM_HUGETLB | VM_MIXEDMAP)) -- return 0; /* just ignore the advice */ -+ /* -+ * Be somewhat over-protective for now! -+ */ -+ if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | -+ VM_PFNMAP | VM_IO | VM_DONTEXPAND | -+ VM_HUGETLB | VM_MIXEDMAP)) -+ return 0; /* just ignore the advice */ - -- if (vma_is_dax(vma)) -- return 0; -+ if (vma_is_dax(vma)) -+ return 0; - - #ifdef VM_SAO - if (*vm_flags & VM_SAO) - return 0; - #endif - #ifdef VM_SPARC_ADI -- if (*vm_flags & VM_SPARC_ADI) -- return 0; -+ if (*vm_flags & VM_SPARC_ADI) -+ return 0; - #endif - -- if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { -- err = __ksm_enter(mm); -- if (err) -- return err; -- } -+ if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { -+ err = __ksm_enter(mm); -+ if (err) -+ return err; -+ } - -- *vm_flags |= VM_MERGEABLE; -- break; -+ *vm_flags |= VM_MERGEABLE; - -- case MADV_UNMERGEABLE: -- if (!(*vm_flags & VM_MERGEABLE)) -- return 0; /* just ignore the advice */ -+ return 0; -+} - -- if (vma->anon_vma) { -- err = unmerge_ksm_pages(vma, start, end); -- if (err) -- return err; -- } -+int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, -+ unsigned long end, unsigned long *vm_flags) -+{ -+ int err; -+ -+ if (!(*vm_flags & VM_MERGEABLE)) -+ return 0; /* just ignore the advice */ -+ -+ if (vma->anon_vma) { -+ err = unmerge_ksm_pages(vma, start, end); -+ if (err) -+ return err; -+ } -+ -+ *vm_flags &= ~VM_MERGEABLE; -+ -+ return 0; -+} -+ -+int ksm_madvise(struct vm_area_struct *vma, unsigned long start, -+ unsigned long end, int advice, unsigned long *vm_flags) -+{ -+ struct mm_struct *mm = vma->vm_mm; -+ int err; - -- *vm_flags &= ~VM_MERGEABLE; -+ switch (advice) { -+ case MADV_MERGEABLE: -+ err = ksm_madvise_merge(mm, vma, vm_flags); -+ if (err) -+ return err; -+ break; -+ -+ case MADV_UNMERGEABLE: -+ err = ksm_madvise_unmerge(vma, start, end, vm_flags); -+ if (err) -+ return err; - break; - } - -@@ -2891,6 +3068,34 @@ static ssize_t pages_to_scan_store(struct kobject *kobj, - } - KSM_ATTR(pages_to_scan); - -+static ssize_t max_scanning_factor_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sysfs_emit(buf, "%u\n", max_scanning_factor); -+} -+ -+static ssize_t max_scanning_factor_store(struct kobject *kobj, -+ struct kobj_attribute *attr, -+ const char *buf, size_t count) -+{ -+ unsigned int value, max; -+ int err; -+ -+ err = kstrtouint(buf, 10, &value); -+ if (err) -+ return -EINVAL; -+ -+ max = totalram_pages() / ksm_thread_pages_to_scan; -+ -+ if (value < 1 && value > max) -+ return -EINVAL; -+ -+ max_scanning_factor = value; -+ -+ return count; -+} -+KSM_ATTR(max_scanning_factor); -+ - static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr, - char *buf) - { -@@ -2906,7 +3111,7 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, - err = kstrtouint(buf, 10, &flags); - if (err) - return -EINVAL; -- if (flags > KSM_RUN_UNMERGE) -+ if (flags > KSM_RUN_UNMERGE && flags != KSM_RUN_AUTO) - return -EINVAL; - - /* -@@ -2932,13 +3137,73 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, - } - mutex_unlock(&ksm_thread_mutex); - -- if (flags & KSM_RUN_MERGE) -+ if (flags & KSM_RUN_MERGE || flags & KSM_RUN_AUTO) - wake_up_interruptible(&ksm_thread_wait); - - return count; - } - KSM_ATTR(run); - -+static ssize_t ksmd_status_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ int len = 0; -+ unsigned int mergeable_mms = 0; -+ struct list_head *pos; -+ -+ list_for_each(pos, &ksm_mm_head.mm_list) -+ mergeable_mms++; -+ -+ if (ksm_run & KSM_RUN_AUTO) { -+ len += sysfs_emit_at(buf, len, "mode: auto\n"); -+ len += sysfs_emit_at(buf, len, "auto_triggered: %d\n", -+ auto_triggered); -+ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", -+ mergeable_mms); -+ len += sysfs_emit_at(buf, len, "scanning_factor: %u\n", -+ scanning_factor); -+ len += sysfs_emit_at(buf, len, "triggered_times: %lu\n", -+ triggered_times); -+ } else if (ksm_run & KSM_RUN_MERGE) { -+ len += sysfs_emit_at(buf, len, "mode: on\n"); -+ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", -+ mergeable_mms); -+ } else if (ksm_run & KSM_RUN_UNMERGE) -+ len += sysfs_emit_at(buf, len, "mode: unmerge\n"); -+ else -+ len += sysfs_emit_at(buf, len, "mode: off\n"); -+ -+ -+ return len; -+} -+KSM_ATTR_RO(ksmd_status); -+ -+static ssize_t auto_threshold_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sysfs_emit(buf, "%u\n", ksm_auto_threshold); -+} -+ -+static ssize_t auto_threshold_store(struct kobject *kobj, -+ struct kobj_attribute *attr, -+ const char *buf, size_t count) -+{ -+ unsigned int value; -+ int err; -+ -+ err = kstrtouint(buf, 10, &value); -+ if (err) -+ return -EINVAL; -+ -+ if (value > 100) -+ return -EINVAL; -+ -+ ksm_auto_threshold = value; -+ -+ return count; -+} -+KSM_ATTR(auto_threshold); -+ - #ifdef CONFIG_NUMA - static ssize_t merge_across_nodes_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -@@ -3148,7 +3413,10 @@ KSM_ATTR_RO(full_scans); - static struct attribute *ksm_attrs[] = { - &sleep_millisecs_attr.attr, - &pages_to_scan_attr.attr, -+ &max_scanning_factor_attr.attr, - &run_attr.attr, -+ &ksmd_status_attr.attr, -+ &auto_threshold_attr.attr, - &pages_shared_attr.attr, - &pages_sharing_attr.attr, - &pages_unshared_attr.attr, -@@ -3180,6 +3448,7 @@ static int __init ksm_init(void) - zero_checksum = calc_checksum(ZERO_PAGE(0)); - /* Default to false for backwards compatibility */ - ksm_use_zero_pages = false; -+ auto_triggered = false; - - err = ksm_slab_init(); - if (err) -diff --git a/mm/madvise.c b/mm/madvise.c -index 5f0f0948a50e..82f6ea8c493d 100644 ---- a/mm/madvise.c -+++ b/mm/madvise.c -@@ -1173,6 +1173,10 @@ process_madvise_behavior_valid(int behavior) - case MADV_COLD: - case MADV_PAGEOUT: - case MADV_WILLNEED: -+#ifdef CONFIG_KSM -+ case MADV_MERGEABLE: -+ case MADV_UNMERGEABLE: -+#endif - return true; - default: - return false; -@@ -1493,3 +1497,114 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, - out: - return ret; - } -+ -+SYSCALL_DEFINE3(pmadv_ksm, int, pidfd, int, behaviour, unsigned int, flags) -+{ -+#ifdef CONFIG_KSM -+ ssize_t ret; -+ struct pid *pid; -+ struct task_struct *task; -+ struct mm_struct *mm; -+ unsigned int f_flags; -+ struct vm_area_struct *vma; -+ -+ if (flags != 0) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ switch (behaviour) { -+ case MADV_MERGEABLE: -+ case MADV_UNMERGEABLE: -+ break; -+ default: -+ ret = -EINVAL; -+ goto out; -+ break; -+ } -+ -+ pid = pidfd_get_pid(pidfd, &f_flags); -+ if (IS_ERR(pid)) { -+ ret = PTR_ERR(pid); -+ goto out; -+ } -+ -+ task = get_pid_task(pid, PIDTYPE_PID); -+ if (!task) { -+ ret = -ESRCH; -+ goto put_pid; -+ } -+ -+ /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */ -+ mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); -+ if (IS_ERR_OR_NULL(mm)) { -+ ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH; -+ goto release_task; -+ } -+ -+ /* Require CAP_SYS_NICE for influencing process performance. */ -+ if (!capable(CAP_SYS_NICE)) { -+ ret = -EPERM; -+ goto release_mm; -+ } -+ -+ if (mmap_write_lock_killable(mm)) { -+ ret = -EINTR; -+ goto release_mm; -+ } -+ -+ for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ switch (behaviour) { -+ case MADV_MERGEABLE: -+ ret = ksm_madvise_merge(vma->vm_mm, vma, &vma->vm_flags); -+ break; -+ case MADV_UNMERGEABLE: -+ ret = ksm_madvise_unmerge(vma, vma->vm_start, vma->vm_end, &vma->vm_flags); -+ break; -+ default: -+ /* look, ma, no brain */ -+ break; -+ } -+ if (ret) -+ break; -+ } -+ -+ mmap_write_unlock(mm); -+ -+release_mm: -+ mmput(mm); -+release_task: -+ put_task_struct(task); -+put_pid: -+ put_pid(pid); -+out: -+ return ret; -+#else /* CONFIG_KSM */ -+ return -ENOSYS; -+#endif /* CONFIG_KSM */ -+} -+ -+#ifdef CONFIG_KSM -+static ssize_t ksm_show(struct kobject *kobj, struct kobj_attribute *attr, -+ char *buf) -+{ -+ return sprintf(buf, "%u\n", __NR_pmadv_ksm); -+} -+static struct kobj_attribute pmadv_ksm_attr = __ATTR_RO(ksm); -+ -+static struct attribute *pmadv_sysfs_attrs[] = { -+ &pmadv_ksm_attr.attr, -+ NULL, -+}; -+ -+static const struct attribute_group pmadv_sysfs_attr_group = { -+ .attrs = pmadv_sysfs_attrs, -+ .name = "pmadv", -+}; -+ -+static int __init pmadv_sysfs_init(void) -+{ -+ return sysfs_create_group(kernel_kobj, &pmadv_sysfs_attr_group); -+} -+subsys_initcall(pmadv_sysfs_init); -+#endif /* CONFIG_KSM */ -diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index b69979c9ced5..7eadbafc006b 100644 ---- a/mm/memcontrol.c -+++ b/mm/memcontrol.c -@@ -625,7 +625,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) - cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); - - x = __this_cpu_add_return(stats_updates, abs(val)); -- if (x > MEMCG_CHARGE_BATCH) { -+ if (x > MEMCG_CHARGE_BATCH * 128) { - /* - * If stats_flush_threshold exceeds the threshold - * (>num_online_cpus()), cgroup stats update will be triggered -diff --git a/mm/page-writeback.c b/mm/page-writeback.c -index 032a7bf8d259..cbcecd565c16 100644 ---- a/mm/page-writeback.c -+++ b/mm/page-writeback.c -@@ -70,7 +70,11 @@ static long ratelimit_pages = 32; - /* - * Start background writeback (via writeback threads) at this percentage - */ -+#ifdef CONFIG_CACHY -+static int dirty_background_ratio = 5; -+#else - static int dirty_background_ratio = 10; -+#endif - - /* - * dirty_background_bytes starts at 0 (disabled) so that it is a function of -@@ -98,7 +102,11 @@ static unsigned long vm_dirty_bytes; - /* - * The interval between `kupdate'-style writebacks - */ -+#ifdef CONFIG_CACHY -+unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */ -+#else - unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ -+#endif - - EXPORT_SYMBOL_GPL(dirty_writeback_interval); - -diff --git a/mm/page_alloc.c b/mm/page_alloc.c -index e5486d47406e..cf131d6e08fb 100644 ---- a/mm/page_alloc.c -+++ b/mm/page_alloc.c -@@ -6982,11 +6982,11 @@ static int zone_batchsize(struct zone *zone) - - /* - * The number of pages to batch allocate is either ~0.1% -- * of the zone or 1MB, whichever is smaller. The batch -+ * of the zone or 4MB, whichever is smaller. The batch - * size is striking a balance between allocation latency - * and zone lock contention. - */ -- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); -+ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); - batch /= 4; /* We effectively *= 4 below */ - if (batch < 1) - batch = 1; -@@ -7064,6 +7064,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) - * historical relationship between high and batch. - */ - high = max(high, batch << 2); -+ high = max(high, 1024); - - return high; - #else -diff --git a/mm/swap.c b/mm/swap.c -index 9cee7f6a3809..186b4e5dcecf 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -1070,6 +1070,10 @@ EXPORT_SYMBOL(pagevec_lookup_range_tag); - */ - void __init swap_setup(void) - { -+#ifdef CONFIG_CACHY -+ /* Only swap-in pages requested, avoid readahead */ -+ page_cluster = 0; -+#else - unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); - - /* Use a smaller cluster for small-memory machines */ -@@ -1081,4 +1085,5 @@ void __init swap_setup(void) - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ -+#endif - } -diff --git a/mm/vmpressure.c b/mm/vmpressure.c -index b52644771cc4..11a4b0e3b583 100644 ---- a/mm/vmpressure.c -+++ b/mm/vmpressure.c -@@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; - * essence, they are percents: the higher the value, the more number - * unsuccessful reclaims there were. - */ -+#ifdef CONFIG_CACHY -+static const unsigned int vmpressure_level_med = 65; -+#else - static const unsigned int vmpressure_level_med = 60; -+#endif - static const unsigned int vmpressure_level_critical = 95; - - /* -diff --git a/mm/vmscan.c b/mm/vmscan.c -index b2b1431352dc..0fc65ace3a4e 100644 ---- a/mm/vmscan.c -+++ b/mm/vmscan.c -@@ -178,7 +178,11 @@ struct scan_control { - /* - * From 0 .. 200. Higher means more swappy. - */ -+#ifdef CONFIG_CACHY -+int vm_swappiness = 20; -+#else - int vm_swappiness = 60; -+#endif - - static void set_task_reclaim_state(struct task_struct *task, - struct reclaim_state *rs) -diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c -index eb31c7158b39..8cefa0b62fb9 100644 ---- a/net/ipv4/inet_connection_sock.c -+++ b/net/ipv4/inet_connection_sock.c -@@ -445,7 +445,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) - * having to remove and re-insert us on the wait queue. - */ - for (;;) { -- prepare_to_wait_exclusive(sk_sleep(sk), &wait, -+ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, - TASK_INTERRUPTIBLE); - release_sock(sk); - if (reqsk_queue_empty(&icsk->icsk_accept_queue)) -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index b735776c0225..2fcd440df1f6 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -4790,8 +4790,8 @@ void __init tcp_init(void) - tcp_init_mem(); - /* Set per-socket limits to no more than 1/128 the pressure threshold */ - limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); -- max_wshare = min(4UL*1024*1024, limit); -- max_rshare = min(6UL*1024*1024, limit); -+ max_wshare = min(16UL*1024*1024, limit); -+ max_rshare = min(16UL*1024*1024, limit); - - init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; - init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; -diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile -index 0470c5f3e690..a1fa2eff8192 100644 ---- a/tools/testing/selftests/damon/Makefile -+++ b/tools/testing/selftests/damon/Makefile -@@ -6,6 +6,7 @@ TEST_GEN_FILES += huge_count_read_write - TEST_FILES = _chk_dependency.sh _debugfs_common.sh - TEST_PROGS = debugfs_attrs.sh debugfs_schemes.sh debugfs_target_ids.sh - TEST_PROGS += debugfs_empty_targets.sh debugfs_huge_count_read_write.sh -+TEST_PROGS += debugfs_duplicate_context_creation.sh - TEST_PROGS += sysfs.sh - - include ../lib.mk -diff --git a/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh b/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh -new file mode 100644 -index 000000000000..4a76e37ef16b ---- /dev/null -+++ b/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh -@@ -0,0 +1,27 @@ -+#!/bin/bash -+# SPDX-License-Identifier: GPL-2.0 -+ -+source _debugfs_common.sh -+ -+# Test duplicated context creation -+# ================================ -+ -+if ! echo foo > "$DBGFS/mk_contexts" -+then -+ echo "context creation failed" -+ exit 1 -+fi -+ -+if echo foo > "$DBGFS/mk_contexts" -+then -+ echo "duplicate context creation success" -+ exit 1 -+fi -+ -+if ! echo foo > "$DBGFS/rm_contexts" -+then -+ echo "context deletion failed" -+ exit 1 -+fi -+ -+exit 0 -diff --git a/tools/testing/selftests/vm/.gitignore b/tools/testing/selftests/vm/.gitignore -index 31e5eea2a9b9..1cd8816c055d 100644 ---- a/tools/testing/selftests/vm/.gitignore -+++ b/tools/testing/selftests/vm/.gitignore -@@ -34,3 +34,4 @@ local_config.* - soft-dirty - split_huge_page_test - ksm_tests -+test-ksm-auto -diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile -index d9fa6a9ea584..002d9482c37f 100644 ---- a/tools/testing/selftests/vm/Makefile -+++ b/tools/testing/selftests/vm/Makefile -@@ -54,6 +54,7 @@ TEST_GEN_FILES += userfaultfd - TEST_GEN_PROGS += soft-dirty - TEST_GEN_PROGS += split_huge_page_test - TEST_GEN_FILES += ksm_tests -+TEST_GEN_FILES += test-ksm-auto - - ifeq ($(MACHINE),x86_64) - CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32) -diff --git a/tools/testing/selftests/vm/test-ksm-auto.c b/tools/testing/selftests/vm/test-ksm-auto.c -new file mode 100644 -index 000000000000..0d71593e008e ---- /dev/null -+++ b/tools/testing/selftests/vm/test-ksm-auto.c -@@ -0,0 +1,273 @@ -+// SPDX-License-Identifier: GPL-2.0 -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define KSM_CLEAR_MODE "2\n" -+#define KSM_NORMAL_MODE "1\n" -+#define KSM_AUTO_MODE "8\n" -+ -+#define PAGESIZE (4*1024) -+/* Don't change the value, it will afffect the result */ -+#define TOTAL_MADVISE_SIZE (300*1024*1024) -+ -+char *ksm_run_file = "/sys/kernel/mm/ksm/run"; -+char *ksm_auto_threshold_file = "/sys/kernel/mm/ksm/auto_threshold"; -+char *ksm_pages_volatile_file = "/sys/kernel/mm/ksm/pages_volatile"; -+char *ksm_pages_sharing_file = "/sys/kernel/mm/ksm/pages_sharing"; -+ -+#define SHAPE_FULL 1 -+#define SHAPE_SPARSE 2 -+/* They are related to the shape of memory */ -+int final_pages[3] = {0, 76500, 42}; -+ -+static char *mmap_and_madvise(long long size, int advise) -+{ -+ char *ptr; -+ int err; -+ -+ err = 0; -+ -+ ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); -+ if (!ptr) -+ return NULL; -+ -+ err = madvise(ptr, size, advise); -+ if (err) { -+ perror("Madvise failed\n"); -+ free(ptr); -+ return NULL; -+ } -+ -+ return ptr; -+} -+ -+void make_samepage_ares(char *ptr, int size, int shape_type) -+{ -+ int i, j; -+ char rnd_num; -+ -+ switch (shape_type) { -+ case SHAPE_FULL: -+ for (i = 0; i < (size / PAGESIZE); i++) -+ memset(ptr + (i * PAGESIZE), 0x1, PAGESIZE); -+ break; -+ case SHAPE_SPARSE: -+ /* Make pages different */ -+ j = 0; -+ for (i = 1; i < (size / PAGESIZE); i++) { -+ ptr[i * PAGESIZE + (j%PAGESIZE)] = j%127 + 1; -+ j++; -+ } -+ for (i = 0; i < (size / PAGESIZE); i += 1800) -+ memset(ptr + (i * PAGESIZE), -1, PAGESIZE); -+ } -+ -+ return; -+} -+ -+int read_file(char *file, char *buffer, int buf_len) -+{ -+ FILE *fp; -+ size_t result; -+ long lSize; -+ -+ fp = fopen(file, "r"); -+ if (!fp) -+ return -1; -+ -+ fseek(fp, 0, SEEK_END); -+ lSize = ftell(fp); -+ rewind(fp); -+ -+ memset(buffer, 0, buf_len); -+ result = fread(buffer, 1, buf_len, fp); -+ if (result == 0) -+ return -1; -+ -+ fclose(fp); -+ -+ return 0; -+} -+ -+int write_file(char *file, const char *buffer, int len) -+{ -+ FILE *fp; -+ size_t result; -+ -+ fp = fopen(file, "w+"); -+ if (!fp) -+ return -1; -+ -+ result = fwrite(buffer, len, 1, fp); -+ if (result == 0) -+ return -1; -+ -+ fclose(fp); -+ -+ return 0; -+} -+ -+static inline void get_orig_info(int *run, int *auto_threshold) -+{ -+ char buffer[50]; -+ -+ /* Read the original state of ksm/run */ -+ if (read_file(ksm_run_file, buffer, sizeof(buffer))) { -+ printf("read file %s failed\n", ksm_run_file); -+ exit(1); -+ } -+ *run = atoi(buffer); -+ -+ if (read_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { -+ printf("read file: %s failed\n", ksm_auto_threshold_file); -+ exit(1); -+ } -+ *auto_threshold = atoi(buffer); -+} -+ -+static inline void restore_orig_state(int run, int auto_threshold) -+{ -+ char buffer[50]; -+ -+ /* restore the original state */ -+ memset(buffer, 0, sizeof(buffer)); -+ snprintf(buffer, sizeof(buffer) - 1, "%d\n", run); -+ if (write_file(ksm_run_file, buffer, sizeof(buffer))) { -+ printf("write file %s failed\n", ksm_run_file); -+ exit(1); -+ } -+ -+ memset(buffer, 0, sizeof(buffer)); -+ snprintf(buffer, sizeof(buffer) - 1, "%d\n", auto_threshold); -+ if (write_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { -+ printf("write file %s failed\n", ksm_run_file); -+ exit(1); -+ } -+} -+ -+void set_ksmd_run_mode(char *mode) -+{ -+ if (write_file(ksm_run_file, mode, 2)) { -+ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); -+ exit(1); -+ } -+} -+ -+static inline void wait_ksmpages_converged(int final_pages) -+{ -+ int pages_sharing; -+ char buffer[50]; -+ -+ for (;;) { -+ if (read_file(ksm_pages_sharing_file, buffer, sizeof(buffer))) { -+ printf("read file %s failed\n", ksm_pages_sharing_file); -+ exit(1); -+ } -+ -+ pages_sharing = atoi(buffer); -+ if (pages_sharing >= final_pages) -+ break; -+ } -+} -+ -+void print_shape(int shape_type) -+{ -+ switch (shape_type) { -+ case SHAPE_FULL: -+ printf("Now the shape of memory area is full-samepages:\n"); -+ printf("[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n\n"); -+ break; -+ case SHAPE_SPARSE: -+ printf("Now the shape of memory area is sparse-samepages:\n"); -+ printf("[xx] [xx] [xx] \n\n"); -+ break; -+ } -+} -+ -+void print_ksmd_cpu_comsuption(void) -+{ -+ system("(ps x| grep \"ksmd\" | grep -v grep | awk \'{print $1}\' |" -+ " xargs -i cat /proc/{}/stat) | awk \'{print \"ksm current " -+ "cpu total slice: \" $14+$15+$16+$17}\'"); -+} -+ -+void test_ksmd_performance(char *madvise_area, int shape_type) -+{ -+ struct timeval tv_start, tv_end; -+ -+ make_samepage_ares(madvise_area, TOTAL_MADVISE_SIZE, shape_type); -+ print_shape(shape_type); -+ -+ /********* Start to time ksmd's normal-run mode **********/ -+ printf("Start to test normal-run ksmd...\n"); -+ -+ print_ksmd_cpu_comsuption(); -+ -+ set_ksmd_run_mode(KSM_CLEAR_MODE); -+ set_ksmd_run_mode(KSM_NORMAL_MODE); -+ -+ gettimeofday(&tv_start, NULL); -+ -+ wait_ksmpages_converged(final_pages[shape_type]); -+ -+ gettimeofday(&tv_end, NULL); -+ printf("ksm normal-run's merging time: %lf seconds\n", -+ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - -+ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); -+ -+ /******* Start to time ksmd's auto-run mode **********/ -+ print_ksmd_cpu_comsuption(); -+ -+ printf("Start to test auto-run ksmd...\n"); -+ set_ksmd_run_mode(KSM_CLEAR_MODE); -+ set_ksmd_run_mode(KSM_AUTO_MODE); -+ if (write_file(ksm_auto_threshold_file, "99\n", 2)) -+ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); -+ -+ gettimeofday(&tv_start, NULL); -+ -+ wait_ksmpages_converged(shape_type); -+ -+ gettimeofday(&tv_end, NULL); -+ printf("ksm auto-run's merging time: %lf seconds\n", -+ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - -+ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); -+ -+ print_ksmd_cpu_comsuption(); -+} -+ -+int main(int argc, char **argv) -+{ -+ char *madvise_area; -+ int orig_run, orig_auto_threshold; -+ -+ /* Get the original state of ksm */ -+ get_orig_info(&orig_run, &orig_auto_threshold); -+ printf("Now we mmap 300MB anouymous memory for testing.\n" -+ "There are two type of TEST which have different shape of\n" -+ "samepage areas.\n" -+ "Note: the test requires no other MERGEABLE-madvised vm areas\n" -+ "in system than the areas our testing process allocs.\n"); -+ madvise_area = mmap_and_madvise(TOTAL_MADVISE_SIZE, MADV_MERGEABLE); -+ if (!madvise_area) { -+ printf("madvise failed\n"); -+ exit(1); -+ } -+ -+ printf("\n****************** TEST 1 ******************\n"); -+ test_ksmd_performance(madvise_area, SHAPE_FULL); -+ printf("\n****************** TEST 2 ******************\n"); -+ test_ksmd_performance(madvise_area, SHAPE_SPARSE); -+ -+ /* Restore the original state */ -+ restore_orig_state(orig_run, orig_auto_threshold); -+ -+ return 0; -+} --- -2.37.3 - diff --git a/6.0/0003-futex-winesync.patch b/6.0/0003-futex-winesync.patch index 43bc1afe..55e9c9f9 100644 --- a/6.0/0003-futex-winesync.patch +++ b/6.0/0003-futex-winesync.patch @@ -1,7 +1,7 @@ -From 862003890a3fa4ec3ebea18593a68d14abcb8bb9 Mon Sep 17 00:00:00 2001 +From 3a2a43e0dc41577b2d9262692c628362129d539d Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Sun, 25 Sep 2022 23:49:46 +0200 -Subject: [PATCH 03/13] futex-winesync +Subject: [PATCH 03/16] futex-winesync Signed-off-by: Peter Jung --- @@ -3398,5 +3398,5 @@ index 000000000000..169e922484b0 + +TEST_HARNESS_MAIN -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 diff --git a/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch b/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch index 1eb45788..2151f8e8 100644 --- a/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch +++ b/6.0/0004-Introducing-OpenVPN-Data-Channel-Offload.patch @@ -1,7 +1,7 @@ -From f56b86ff7aa71d9627939c0cbfa928f23042d902 Mon Sep 17 00:00:00 2001 +From 0905ce4d17bc19b8ec54ef87ed8f42e365a2bcc2 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 5 Aug 2022 19:33:47 +0200 -Subject: [PATCH 04/13] Introducing-OpenVPN-Data-Channel-Offload +Subject: [PATCH 04/16] Introducing-OpenVPN-Data-Channel-Offload Signed-off-by: Peter Jung --- @@ -6163,5 +6163,5 @@ index 4828794efcf8..8008c762e6b8 100644 #endif /* _UAPI_LINUX_UDP_H */ -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 diff --git a/6.0/0005-mm-demotion-Memory-tiers-and-demotion.patch b/6.0/0005-mm-demotion-Memory-tiers-and-demotion.patch new file mode 100644 index 00000000..9e2dadb5 --- /dev/null +++ b/6.0/0005-mm-demotion-Memory-tiers-and-demotion.patch @@ -0,0 +1,1882 @@ +From 14903eee0b5577711272732705260cb83e5e0777 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:26:01 +0200 +Subject: [PATCH 05/16] mm/demotion: Memory tiers and demotion + +The current kernel has the basic memory tiering support: Inactive pages on +a higher tier NUMA node can be migrated (demoted) to a lower tier NUMA +node to make room for new allocations on the higher tier NUMA node. +Frequently accessed pages on a lower tier NUMA node can be migrated +(promoted) to a higher tier NUMA node to improve the performance. + +In the current kernel, memory tiers are defined implicitly via a demotion +path relationship between NUMA nodes, which is created during the kernel +initialization and updated when a NUMA node is hot-added or hot-removed. +The current implementation puts all nodes with CPU into the highest tier, +and builds the tier hierarchy tier-by-tier by establishing the per-node +demotion targets based on the distances between nodes. + +This current memory tier kernel implementation needs to be improved for +several important use cases: + +* The current tier initialization code always initializes each + memory-only NUMA node into a lower tier. But a memory-only NUMA node + may have a high performance memory device (e.g. a DRAM-backed + memory-only node on a virtual machine) and that should be put into a + higher tier. + +* The current tier hierarchy always puts CPU nodes into the top tier. + But on a system with HBM (e.g. GPU memory) devices, these memory-only + HBM NUMA nodes should be in the top tier, and DRAM nodes with CPUs are + better to be placed into the next lower tier. + +* Also because the current tier hierarchy always puts CPU nodes into the + top tier, when a CPU is hot-added (or hot-removed) and triggers a memory + node from CPU-less into a CPU node (or vice versa), the memory tier + hierarchy gets changed, even though no memory node is added or removed. + This can make the tier hierarchy unstable and make it difficult to + support tier-based memory accounting. + +* A higher tier node can only be demoted to nodes with shortest distance + on the next lower tier as defined by the demotion path, not any other + node from any lower tier. This strict, demotion order does not work in + all use cases (e.g. some use cases may want to allow cross-socket + demotion to another node in the same demotion tier as a fallback when + the preferred demotion node is out of space), and has resulted in the + feature request for an interface to override the system-wide, per-node + demotion order from the userspace. This demotion order is also + inconsistent with the page allocation fallback order when all the nodes + in a higher tier are out of space: The page allocation can fall back to + any node from any lower tier, whereas the demotion order doesn't allow + that. + +This patch series make the creation of memory tiers explicit under the +control of device driver. + +Memory Tier Initialization +========================== + +Linux kernel presents memory devices as NUMA nodes and each memory device +is of a specific type. The memory type of a device is represented by its +abstract distance. A memory tier corresponds to a range of abstract +distance. This allows for classifying memory devices with a specific +performance range into a memory tier. + +By default, all memory nodes are assigned to the default tier with +abstract distance 512. + +A device driver can move its memory nodes from the default tier. For +example, PMEM can move its memory nodes below the default tier, whereas +GPU can move its memory nodes above the default tier. + +The kernel initialization code makes the decision on which exact tier a +memory node should be assigned to based on the requests from the device +drivers as well as the memory device hardware information provided by the +firmware. + +Hot-adding/removing CPUs doesn't affect memory tier hierarchy. + +This patch (of 10): + +In the current kernel, memory tiers are defined implicitly via a demotion +path relationship between NUMA nodes, which is created during the kernel +initialization and updated when a NUMA node is hot-added or hot-removed. +The current implementation puts all nodes with CPU into the highest tier, +and builds the tier hierarchy by establishing the per-node demotion +targets based on the distances between nodes. + +This current memory tier kernel implementation needs to be improved for +several important use cases, + +The current tier initialization code always initializes each memory-only +NUMA node into a lower tier. But a memory-only NUMA node may have a high +performance memory device (e.g. a DRAM-backed memory-only node on a +virtual machine) that should be put into a higher tier. + +The current tier hierarchy always puts CPU nodes into the top tier. But +on a system with HBM or GPU devices, the memory-only NUMA nodes mapping +these devices should be in the top tier, and DRAM nodes with CPUs are +better to be placed into the next lower tier. + +With current kernel higher tier node can only be demoted to nodes with +shortest distance on the next lower tier as defined by the demotion path, +not any other node from any lower tier. This strict, demotion order does +not work in all use cases (e.g. some use cases may want to allow +cross-socket demotion to another node in the same demotion tier as a +fallback when the preferred demotion node is out of space), This demotion +order is also inconsistent with the page allocation fallback order when +all the nodes in a higher tier are out of space: The page allocation can +fall back to any node from any lower tier, whereas the demotion order +doesn't allow that. + +This patch series address the above by defining memory tiers explicitly. + +Linux kernel presents memory devices as NUMA nodes and each memory device +is of a specific type. The memory type of a device is represented by its +abstract distance. A memory tier corresponds to a range of abstract +distance. This allows for classifying memory devices with a specific +performance range into a memory tier. + +This patch configures the range/chunk size to be 128. The default DRAM +abstract distance is 512. We can have 4 memory tiers below the default +DRAM with abstract distance range 0 - 127, 127 - 255, 256- 383, 384 - 511. +Faster memory devices can be placed in these faster(higher) memory tiers. +Slower memory devices like persistent memory will have abstract distance +higher than the default DRAM level. + +Signed-off-by: Peter Jung +--- + .../ABI/testing/sysfs-kernel-mm-memory-tiers | 25 + + drivers/dax/kmem.c | 42 +- + include/linux/memory-tiers.h | 102 +++ + include/linux/migrate.h | 15 - + include/linux/mmzone.h | 3 + + include/linux/node.h | 5 - + include/linux/nodemask.h | 15 +- + kernel/sched/fair.c | 1 + + mm/Makefile | 1 + + mm/huge_memory.c | 1 + + mm/memory-tiers.c | 732 ++++++++++++++++++ + mm/memory.c | 1 + + mm/migrate.c | 453 +---------- + mm/mprotect.c | 1 + + mm/vmscan.c | 59 +- + mm/vmstat.c | 4 - + 16 files changed, 963 insertions(+), 497 deletions(-) + create mode 100644 Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers + create mode 100644 include/linux/memory-tiers.h + create mode 100644 mm/memory-tiers.c + +diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +new file mode 100644 +index 000000000000..45985e411f13 +--- /dev/null ++++ b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +@@ -0,0 +1,25 @@ ++What: /sys/devices/virtual/memory_tiering/ ++Date: August 2022 ++Contact: Linux memory management mailing list ++Description: A collection of all the memory tiers allocated. ++ ++ Individual memory tier details are contained in subdirectories ++ named by the abstract distance of the memory tier. ++ ++ /sys/devices/virtual/memory_tiering/memory_tierN/ ++ ++ ++What: /sys/devices/virtual/memory_tiering/memory_tierN/ ++ /sys/devices/virtual/memory_tiering/memory_tierN/nodes ++Date: August 2022 ++Contact: Linux memory management mailing list ++Description: Directory with details of a specific memory tier ++ ++ This is the directory containing information about a particular ++ memory tier, memtierN, where N is derived based on abstract distance. ++ ++ A smaller value of N implies a higher (faster) memory tier in the ++ hierarchy. ++ ++ nodes: NUMA nodes that are part of this memory tier. ++ +diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c +index a37622060fff..4852a2dbdb27 100644 +--- a/drivers/dax/kmem.c ++++ b/drivers/dax/kmem.c +@@ -11,9 +11,17 @@ + #include + #include + #include ++#include + #include "dax-private.h" + #include "bus.h" + ++/* ++ * Default abstract distance assigned to the NUMA node onlined ++ * by DAX/kmem if the low level platform driver didn't initialize ++ * one for this NUMA node. ++ */ ++#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) ++ + /* Memory resource name used for add_memory_driver_managed(). */ + static const char *kmem_name; + /* Set if any memory will remain added when the driver will be unloaded. */ +@@ -41,6 +49,7 @@ struct dax_kmem_data { + struct resource *res[]; + }; + ++static struct memory_dev_type *dax_slowmem_type; + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + { + struct device *dev = &dev_dax->dev; +@@ -79,11 +88,13 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + return -EINVAL; + } + ++ init_node_memory_type(numa_node, dax_slowmem_type); ++ ++ rc = -ENOMEM; + data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL); + if (!data) +- return -ENOMEM; ++ goto err_dax_kmem_data; + +- rc = -ENOMEM; + data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); + if (!data->res_name) + goto err_res_name; +@@ -155,6 +166,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + kfree(data->res_name); + err_res_name: + kfree(data); ++err_dax_kmem_data: ++ clear_node_memory_type(numa_node, dax_slowmem_type); + return rc; + } + +@@ -162,6 +175,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + static void dev_dax_kmem_remove(struct dev_dax *dev_dax) + { + int i, success = 0; ++ int node = dev_dax->target_node; + struct device *dev = &dev_dax->dev; + struct dax_kmem_data *data = dev_get_drvdata(dev); + +@@ -198,6 +212,14 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) + kfree(data->res_name); + kfree(data); + dev_set_drvdata(dev, NULL); ++ /* ++ * Clear the memtype association on successful unplug. ++ * If not, we have memory blocks left which can be ++ * offlined/onlined later. We need to keep memory_dev_type ++ * for that. This implies this reference will be around ++ * till next reboot. ++ */ ++ clear_node_memory_type(node, dax_slowmem_type); + } + } + #else +@@ -228,9 +250,22 @@ static int __init dax_kmem_init(void) + if (!kmem_name) + return -ENOMEM; + ++ dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE); ++ if (IS_ERR(dax_slowmem_type)) { ++ rc = PTR_ERR(dax_slowmem_type); ++ goto err_dax_slowmem_type; ++ } ++ + rc = dax_driver_register(&device_dax_kmem_driver); + if (rc) +- kfree_const(kmem_name); ++ goto error_dax_driver; ++ ++ return rc; ++ ++error_dax_driver: ++ destroy_memory_type(dax_slowmem_type); ++err_dax_slowmem_type: ++ kfree_const(kmem_name); + return rc; + } + +@@ -239,6 +274,7 @@ static void __exit dax_kmem_exit(void) + dax_driver_unregister(&device_dax_kmem_driver); + if (!any_hotremove_failed) + kfree_const(kmem_name); ++ destroy_memory_type(dax_slowmem_type); + } + + MODULE_AUTHOR("Intel Corporation"); +diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h +new file mode 100644 +index 000000000000..965009aa01d7 +--- /dev/null ++++ b/include/linux/memory-tiers.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_MEMORY_TIERS_H ++#define _LINUX_MEMORY_TIERS_H ++ ++#include ++#include ++#include ++#include ++/* ++ * Each tier cover a abstrace distance chunk size of 128 ++ */ ++#define MEMTIER_CHUNK_BITS 7 ++#define MEMTIER_CHUNK_SIZE (1 << MEMTIER_CHUNK_BITS) ++/* ++ * Smaller abstract distance values imply faster (higher) memory tiers. Offset ++ * the DRAM adistance so that we can accommodate devices with a slightly lower ++ * adistance value (slightly faster) than default DRAM adistance to be part of ++ * the same memory tier. ++ */ ++#define MEMTIER_ADISTANCE_DRAM ((4 * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1)) ++#define MEMTIER_HOTPLUG_PRIO 100 ++ ++struct memory_tier; ++struct memory_dev_type { ++ /* list of memory types that are part of same tier as this type */ ++ struct list_head tier_sibiling; ++ /* abstract distance for this specific memory type */ ++ int adistance; ++ /* Nodes of same abstract distance */ ++ nodemask_t nodes; ++ struct kref kref; ++}; ++ ++#ifdef CONFIG_NUMA ++extern bool numa_demotion_enabled; ++struct memory_dev_type *alloc_memory_type(int adistance); ++void destroy_memory_type(struct memory_dev_type *memtype); ++void init_node_memory_type(int node, struct memory_dev_type *default_type); ++void clear_node_memory_type(int node, struct memory_dev_type *memtype); ++#ifdef CONFIG_MIGRATION ++int next_demotion_node(int node); ++void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); ++bool node_is_toptier(int node); ++#else ++static inline int next_demotion_node(int node) ++{ ++ return NUMA_NO_NODE; ++} ++ ++static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ *targets = NODE_MASK_NONE; ++} ++ ++static inline bool node_is_toptier(int node) ++{ ++ return true; ++} ++#endif ++ ++#else ++ ++#define numa_demotion_enabled false ++/* ++ * CONFIG_NUMA implementation returns non NULL error. ++ */ ++static inline struct memory_dev_type *alloc_memory_type(int adistance) ++{ ++ return NULL; ++} ++ ++static inline void destroy_memory_type(struct memory_dev_type *memtype) ++{ ++ ++} ++ ++static inline void init_node_memory_type(int node, struct memory_dev_type *default_type) ++{ ++ ++} ++ ++static inline void clear_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ ++} ++ ++static inline int next_demotion_node(int node) ++{ ++ return NUMA_NO_NODE; ++} ++ ++static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ *targets = NODE_MASK_NONE; ++} ++ ++static inline bool node_is_toptier(int node) ++{ ++ return true; ++} ++#endif /* CONFIG_NUMA */ ++#endif /* _LINUX_MEMORY_TIERS_H */ +diff --git a/include/linux/migrate.h b/include/linux/migrate.h +index 22c0a0cf5e0c..704a04f5a074 100644 +--- a/include/linux/migrate.h ++++ b/include/linux/migrate.h +@@ -100,21 +100,6 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping, + + #endif /* CONFIG_MIGRATION */ + +-#if defined(CONFIG_MIGRATION) && defined(CONFIG_NUMA) +-extern void set_migration_target_nodes(void); +-extern void migrate_on_reclaim_init(void); +-extern bool numa_demotion_enabled; +-extern int next_demotion_node(int node); +-#else +-static inline void set_migration_target_nodes(void) {} +-static inline void migrate_on_reclaim_init(void) {} +-static inline int next_demotion_node(int node) +-{ +- return NUMA_NO_NODE; +-} +-#define numa_demotion_enabled false +-#endif +- + #ifdef CONFIG_COMPACTION + bool PageMovable(struct page *page); + void __SetPageMovable(struct page *page, const struct movable_operations *ops); +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index e24b40c52468..7d78133fe8dd 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -1012,6 +1012,9 @@ typedef struct pglist_data { + /* Per-node vmstats */ + struct per_cpu_nodestat __percpu *per_cpu_nodestats; + atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; ++#ifdef CONFIG_NUMA ++ struct memory_tier __rcu *memtier; ++#endif + } pg_data_t; + + #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) +diff --git a/include/linux/node.h b/include/linux/node.h +index 40d641a8bfb0..9ec680dd607f 100644 +--- a/include/linux/node.h ++++ b/include/linux/node.h +@@ -185,9 +185,4 @@ static inline void register_hugetlbfs_with_node(node_registration_func_t reg, + + #define to_node(device) container_of(device, struct node, dev) + +-static inline bool node_is_toptier(int node) +-{ +- return node_state(node, N_CPU); +-} +- + #endif /* _LINUX_NODE_H_ */ +diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h +index 4b71a96190a8..ac5b6a371be5 100644 +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -504,12 +504,21 @@ static inline int num_node_state(enum node_states state) + static inline int node_random(const nodemask_t *maskp) + { + #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) +- int w, bit = NUMA_NO_NODE; ++ int w, bit; + + w = nodes_weight(*maskp); +- if (w) ++ switch (w) { ++ case 0: ++ bit = NUMA_NO_NODE; ++ break; ++ case 1: ++ bit = first_node(*maskp); ++ break; ++ default: + bit = bitmap_ord_to_pos(maskp->bits, +- get_random_int() % w, MAX_NUMNODES); ++ get_random_int() % w, MAX_NUMNODES); ++ break; ++ } + return bit; + #else + return 0; +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index 74d773040b54..1d3e53fed1a4 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -40,6 +40,7 @@ + + #include + #include ++#include + #include + #include + #include +diff --git a/mm/Makefile b/mm/Makefile +index 9a564f836403..488f604e77e0 100644 +--- a/mm/Makefile ++++ b/mm/Makefile +@@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/ + obj-$(CONFIG_FAILSLAB) += failslab.o + obj-$(CONFIG_MEMTEST) += memtest.o + obj-$(CONFIG_MIGRATION) += migrate.o ++obj-$(CONFIG_NUMA) += memory-tiers.o + obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o + obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o + obj-$(CONFIG_PAGE_COUNTER) += page_counter.o +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index e9414ee57c5b..6eb4b1799b79 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include + + #include + #include +diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c +new file mode 100644 +index 000000000000..f116b7b6333e +--- /dev/null ++++ b/mm/memory-tiers.c +@@ -0,0 +1,732 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "internal.h" ++ ++struct memory_tier { ++ /* hierarchy of memory tiers */ ++ struct list_head list; ++ /* list of all memory types part of this tier */ ++ struct list_head memory_types; ++ /* ++ * start value of abstract distance. memory tier maps ++ * an abstract distance range, ++ * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE ++ */ ++ int adistance_start; ++ struct device dev; ++ /* All the nodes that are part of all the lower memory tiers. */ ++ nodemask_t lower_tier_mask; ++}; ++ ++struct demotion_nodes { ++ nodemask_t preferred; ++}; ++ ++struct node_memory_type_map { ++ struct memory_dev_type *memtype; ++ int map_count; ++}; ++ ++static DEFINE_MUTEX(memory_tier_lock); ++static LIST_HEAD(memory_tiers); ++static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; ++static struct memory_dev_type *default_dram_type; ++ ++static struct bus_type memory_tier_subsys = { ++ .name = "memory_tiering", ++ .dev_name = "memory_tier", ++}; ++ ++#ifdef CONFIG_MIGRATION ++static int top_tier_adistance; ++/* ++ * node_demotion[] examples: ++ * ++ * Example 1: ++ * ++ * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes. ++ * ++ * node distances: ++ * node 0 1 2 3 ++ * 0 10 20 30 40 ++ * 1 20 10 40 30 ++ * 2 30 40 10 40 ++ * 3 40 30 40 10 ++ * ++ * memory_tiers0 = 0-1 ++ * memory_tiers1 = 2-3 ++ * ++ * node_demotion[0].preferred = 2 ++ * node_demotion[1].preferred = 3 ++ * node_demotion[2].preferred = ++ * node_demotion[3].preferred = ++ * ++ * Example 2: ++ * ++ * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node. ++ * ++ * node distances: ++ * node 0 1 2 ++ * 0 10 20 30 ++ * 1 20 10 30 ++ * 2 30 30 10 ++ * ++ * memory_tiers0 = 0-2 ++ * ++ * node_demotion[0].preferred = ++ * node_demotion[1].preferred = ++ * node_demotion[2].preferred = ++ * ++ * Example 3: ++ * ++ * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node. ++ * ++ * node distances: ++ * node 0 1 2 ++ * 0 10 20 30 ++ * 1 20 10 40 ++ * 2 30 40 10 ++ * ++ * memory_tiers0 = 1 ++ * memory_tiers1 = 0 ++ * memory_tiers2 = 2 ++ * ++ * node_demotion[0].preferred = 2 ++ * node_demotion[1].preferred = 0 ++ * node_demotion[2].preferred = ++ * ++ */ ++static struct demotion_nodes *node_demotion __read_mostly; ++#endif /* CONFIG_MIGRATION */ ++ ++static inline struct memory_tier *to_memory_tier(struct device *device) ++{ ++ return container_of(device, struct memory_tier, dev); ++} ++ ++static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier) ++{ ++ nodemask_t nodes = NODE_MASK_NONE; ++ struct memory_dev_type *memtype; ++ ++ list_for_each_entry(memtype, &memtier->memory_types, tier_sibiling) ++ nodes_or(nodes, nodes, memtype->nodes); ++ ++ return nodes; ++} ++ ++static void memory_tier_device_release(struct device *dev) ++{ ++ struct memory_tier *tier = to_memory_tier(dev); ++ /* ++ * synchronize_rcu in clear_node_memory_tier makes sure ++ * we don't have rcu access to this memory tier. ++ */ ++ kfree(tier); ++} ++ ++static ssize_t nodes_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ int ret; ++ nodemask_t nmask; ++ ++ mutex_lock(&memory_tier_lock); ++ nmask = get_memtier_nodemask(to_memory_tier(dev)); ++ ret = sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&nmask)); ++ mutex_unlock(&memory_tier_lock); ++ return ret; ++} ++static DEVICE_ATTR_RO(nodes); ++ ++static struct attribute *memtier_dev_attrs[] = { ++ &dev_attr_nodes.attr, ++ NULL ++}; ++ ++static const struct attribute_group memtier_dev_group = { ++ .attrs = memtier_dev_attrs, ++}; ++ ++static const struct attribute_group *memtier_dev_groups[] = { ++ &memtier_dev_group, ++ NULL ++}; ++ ++static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype) ++{ ++ int ret; ++ bool found_slot = false; ++ struct memory_tier *memtier, *new_memtier; ++ int adistance = memtype->adistance; ++ unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE; ++ ++ lockdep_assert_held_once(&memory_tier_lock); ++ ++ adistance = round_down(adistance, memtier_adistance_chunk_size); ++ /* ++ * If the memtype is already part of a memory tier, ++ * just return that. ++ */ ++ if (!list_empty(&memtype->tier_sibiling)) { ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ if (adistance == memtier->adistance_start) ++ return memtier; ++ } ++ WARN_ON(1); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ if (adistance == memtier->adistance_start) { ++ goto link_memtype; ++ } else if (adistance < memtier->adistance_start) { ++ found_slot = true; ++ break; ++ } ++ } ++ ++ new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL); ++ if (!new_memtier) ++ return ERR_PTR(-ENOMEM); ++ ++ new_memtier->adistance_start = adistance; ++ INIT_LIST_HEAD(&new_memtier->list); ++ INIT_LIST_HEAD(&new_memtier->memory_types); ++ if (found_slot) ++ list_add_tail(&new_memtier->list, &memtier->list); ++ else ++ list_add_tail(&new_memtier->list, &memory_tiers); ++ ++ new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS; ++ new_memtier->dev.bus = &memory_tier_subsys; ++ new_memtier->dev.release = memory_tier_device_release; ++ new_memtier->dev.groups = memtier_dev_groups; ++ ++ ret = device_register(&new_memtier->dev); ++ if (ret) { ++ list_del(&memtier->list); ++ put_device(&memtier->dev); ++ return ERR_PTR(ret); ++ } ++ memtier = new_memtier; ++ ++link_memtype: ++ list_add(&memtype->tier_sibiling, &memtier->memory_types); ++ return memtier; ++} ++ ++static struct memory_tier *__node_get_memory_tier(int node) ++{ ++ pg_data_t *pgdat; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return NULL; ++ /* ++ * Since we hold memory_tier_lock, we can avoid ++ * RCU read locks when accessing the details. No ++ * parallel updates are possible here. ++ */ ++ return rcu_dereference_check(pgdat->memtier, ++ lockdep_is_held(&memory_tier_lock)); ++} ++ ++#ifdef CONFIG_MIGRATION ++bool node_is_toptier(int node) ++{ ++ bool toptier; ++ pg_data_t *pgdat; ++ struct memory_tier *memtier; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return false; ++ ++ rcu_read_lock(); ++ memtier = rcu_dereference(pgdat->memtier); ++ if (!memtier) { ++ toptier = true; ++ goto out; ++ } ++ if (memtier->adistance_start <= top_tier_adistance) ++ toptier = true; ++ else ++ toptier = false; ++out: ++ rcu_read_unlock(); ++ return toptier; ++} ++ ++void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ struct memory_tier *memtier; ++ ++ /* ++ * pg_data_t.memtier updates includes a synchronize_rcu() ++ * which ensures that we either find NULL or a valid memtier ++ * in NODE_DATA. protect the access via rcu_read_lock(); ++ */ ++ rcu_read_lock(); ++ memtier = rcu_dereference(pgdat->memtier); ++ if (memtier) ++ *targets = memtier->lower_tier_mask; ++ else ++ *targets = NODE_MASK_NONE; ++ rcu_read_unlock(); ++} ++ ++/** ++ * next_demotion_node() - Get the next node in the demotion path ++ * @node: The starting node to lookup the next node ++ * ++ * Return: node id for next memory node in the demotion path hierarchy ++ * from @node; NUMA_NO_NODE if @node is terminal. This does not keep ++ * @node online or guarantee that it *continues* to be the next demotion ++ * target. ++ */ ++int next_demotion_node(int node) ++{ ++ struct demotion_nodes *nd; ++ int target; ++ ++ if (!node_demotion) ++ return NUMA_NO_NODE; ++ ++ nd = &node_demotion[node]; ++ ++ /* ++ * node_demotion[] is updated without excluding this ++ * function from running. ++ * ++ * Make sure to use RCU over entire code blocks if ++ * node_demotion[] reads need to be consistent. ++ */ ++ rcu_read_lock(); ++ /* ++ * If there are multiple target nodes, just select one ++ * target node randomly. ++ * ++ * In addition, we can also use round-robin to select ++ * target node, but we should introduce another variable ++ * for node_demotion[] to record last selected target node, ++ * that may cause cache ping-pong due to the changing of ++ * last target node. Or introducing per-cpu data to avoid ++ * caching issue, which seems more complicated. So selecting ++ * target node randomly seems better until now. ++ */ ++ target = node_random(&nd->preferred); ++ rcu_read_unlock(); ++ ++ return target; ++} ++ ++static void disable_all_demotion_targets(void) ++{ ++ struct memory_tier *memtier; ++ int node; ++ ++ for_each_node_state(node, N_MEMORY) { ++ node_demotion[node].preferred = NODE_MASK_NONE; ++ /* ++ * We are holding memory_tier_lock, it is safe ++ * to access pgda->memtier. ++ */ ++ memtier = __node_get_memory_tier(node); ++ if (memtier) ++ memtier->lower_tier_mask = NODE_MASK_NONE; ++ } ++ /* ++ * Ensure that the "disable" is visible across the system. ++ * Readers will see either a combination of before+disable ++ * state or disable+after. They will never see before and ++ * after state together. ++ */ ++ synchronize_rcu(); ++} ++ ++/* ++ * Find an automatic demotion target for all memory ++ * nodes. Failing here is OK. It might just indicate ++ * being at the end of a chain. ++ */ ++static void establish_demotion_targets(void) ++{ ++ struct memory_tier *memtier; ++ struct demotion_nodes *nd; ++ int target = NUMA_NO_NODE, node; ++ int distance, best_distance; ++ nodemask_t tier_nodes, lower_tier; ++ ++ lockdep_assert_held_once(&memory_tier_lock); ++ ++ if (!node_demotion || !IS_ENABLED(CONFIG_MIGRATION)) ++ return; ++ ++ disable_all_demotion_targets(); ++ ++ for_each_node_state(node, N_MEMORY) { ++ best_distance = -1; ++ nd = &node_demotion[node]; ++ ++ memtier = __node_get_memory_tier(node); ++ if (!memtier || list_is_last(&memtier->list, &memory_tiers)) ++ continue; ++ /* ++ * Get the lower memtier to find the demotion node list. ++ */ ++ memtier = list_next_entry(memtier, list); ++ tier_nodes = get_memtier_nodemask(memtier); ++ /* ++ * find_next_best_node, use 'used' nodemask as a skip list. ++ * Add all memory nodes except the selected memory tier ++ * nodelist to skip list so that we find the best node from the ++ * memtier nodelist. ++ */ ++ nodes_andnot(tier_nodes, node_states[N_MEMORY], tier_nodes); ++ ++ /* ++ * Find all the nodes in the memory tier node list of same best distance. ++ * add them to the preferred mask. We randomly select between nodes ++ * in the preferred mask when allocating pages during demotion. ++ */ ++ do { ++ target = find_next_best_node(node, &tier_nodes); ++ if (target == NUMA_NO_NODE) ++ break; ++ ++ distance = node_distance(node, target); ++ if (distance == best_distance || best_distance == -1) { ++ best_distance = distance; ++ node_set(target, nd->preferred); ++ } else { ++ break; ++ } ++ } while (1); ++ } ++ /* ++ * Promotion is allowed from a memory tier to higher ++ * memory tier only if the memory tier doesn't include ++ * compute. We want to skip promotion from a memory tier, ++ * if any node that is part of the memory tier have CPUs. ++ * Once we detect such a memory tier, we consider that tier ++ * as top tiper from which promotion is not allowed. ++ */ ++ list_for_each_entry_reverse(memtier, &memory_tiers, list) { ++ tier_nodes = get_memtier_nodemask(memtier); ++ nodes_and(tier_nodes, node_states[N_CPU], tier_nodes); ++ if (!nodes_empty(tier_nodes)) { ++ /* ++ * abstract distance below the max value of this memtier ++ * is considered toptier. ++ */ ++ top_tier_adistance = memtier->adistance_start + ++ MEMTIER_CHUNK_SIZE - 1; ++ break; ++ } ++ } ++ /* ++ * Now build the lower_tier mask for each node collecting node mask from ++ * all memory tier below it. This allows us to fallback demotion page ++ * allocation to a set of nodes that is closer the above selected ++ * perferred node. ++ */ ++ lower_tier = node_states[N_MEMORY]; ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ /* ++ * Keep removing current tier from lower_tier nodes, ++ * This will remove all nodes in current and above ++ * memory tier from the lower_tier mask. ++ */ ++ tier_nodes = get_memtier_nodemask(memtier); ++ nodes_andnot(lower_tier, lower_tier, tier_nodes); ++ memtier->lower_tier_mask = lower_tier; ++ } ++} ++ ++#else ++static inline void disable_all_demotion_targets(void) {} ++static inline void establish_demotion_targets(void) {} ++#endif /* CONFIG_MIGRATION */ ++ ++static inline void __init_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ if (!node_memory_types[node].memtype) ++ node_memory_types[node].memtype = memtype; ++ /* ++ * for each device getting added in the same NUMA node ++ * with this specific memtype, bump the map count. We ++ * Only take memtype device reference once, so that ++ * changing a node memtype can be done by droping the ++ * only reference count taken here. ++ */ ++ ++ if (node_memory_types[node].memtype == memtype) { ++ if (!node_memory_types[node].map_count++) ++ kref_get(&memtype->kref); ++ } ++} ++ ++static struct memory_tier *set_node_memory_tier(int node) ++{ ++ struct memory_tier *memtier; ++ struct memory_dev_type *memtype; ++ pg_data_t *pgdat = NODE_DATA(node); ++ ++ ++ lockdep_assert_held_once(&memory_tier_lock); ++ ++ if (!node_state(node, N_MEMORY)) ++ return ERR_PTR(-EINVAL); ++ ++ __init_node_memory_type(node, default_dram_type); ++ ++ memtype = node_memory_types[node].memtype; ++ node_set(node, memtype->nodes); ++ memtier = find_create_memory_tier(memtype); ++ if (!IS_ERR(memtier)) ++ rcu_assign_pointer(pgdat->memtier, memtier); ++ return memtier; ++} ++ ++static void destroy_memory_tier(struct memory_tier *memtier) ++{ ++ list_del(&memtier->list); ++ device_unregister(&memtier->dev); ++} ++ ++static bool clear_node_memory_tier(int node) ++{ ++ bool cleared = false; ++ pg_data_t *pgdat; ++ struct memory_tier *memtier; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return false; ++ ++ /* ++ * Make sure that anybody looking at NODE_DATA who finds ++ * a valid memtier finds memory_dev_types with nodes still ++ * linked to the memtier. We achieve this by waiting for ++ * rcu read section to finish using synchronize_rcu. ++ * This also enables us to free the destroyed memory tier ++ * with kfree instead of kfree_rcu ++ */ ++ memtier = __node_get_memory_tier(node); ++ if (memtier) { ++ struct memory_dev_type *memtype; ++ ++ rcu_assign_pointer(pgdat->memtier, NULL); ++ synchronize_rcu(); ++ memtype = node_memory_types[node].memtype; ++ node_clear(node, memtype->nodes); ++ if (nodes_empty(memtype->nodes)) { ++ list_del_init(&memtype->tier_sibiling); ++ if (list_empty(&memtier->memory_types)) ++ destroy_memory_tier(memtier); ++ } ++ cleared = true; ++ } ++ return cleared; ++} ++ ++static void release_memtype(struct kref *kref) ++{ ++ struct memory_dev_type *memtype; ++ ++ memtype = container_of(kref, struct memory_dev_type, kref); ++ kfree(memtype); ++} ++ ++struct memory_dev_type *alloc_memory_type(int adistance) ++{ ++ struct memory_dev_type *memtype; ++ ++ memtype = kmalloc(sizeof(*memtype), GFP_KERNEL); ++ if (!memtype) ++ return ERR_PTR(-ENOMEM); ++ ++ memtype->adistance = adistance; ++ INIT_LIST_HEAD(&memtype->tier_sibiling); ++ memtype->nodes = NODE_MASK_NONE; ++ kref_init(&memtype->kref); ++ return memtype; ++} ++EXPORT_SYMBOL_GPL(alloc_memory_type); ++ ++void destroy_memory_type(struct memory_dev_type *memtype) ++{ ++ kref_put(&memtype->kref, release_memtype); ++} ++EXPORT_SYMBOL_GPL(destroy_memory_type); ++ ++void init_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ ++ mutex_lock(&memory_tier_lock); ++ __init_node_memory_type(node, memtype); ++ mutex_unlock(&memory_tier_lock); ++} ++EXPORT_SYMBOL_GPL(init_node_memory_type); ++ ++void clear_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ mutex_lock(&memory_tier_lock); ++ if (node_memory_types[node].memtype == memtype) ++ node_memory_types[node].map_count--; ++ /* ++ * If we umapped all the attached devices to this node, ++ * clear the node memory type. ++ */ ++ if (!node_memory_types[node].map_count) { ++ node_memory_types[node].memtype = NULL; ++ kref_put(&memtype->kref, release_memtype); ++ } ++ mutex_unlock(&memory_tier_lock); ++} ++EXPORT_SYMBOL_GPL(clear_node_memory_type); ++ ++static int __meminit memtier_hotplug_callback(struct notifier_block *self, ++ unsigned long action, void *_arg) ++{ ++ struct memory_tier *memtier; ++ struct memory_notify *arg = _arg; ++ ++ /* ++ * Only update the node migration order when a node is ++ * changing status, like online->offline. ++ */ ++ if (arg->status_change_nid < 0) ++ return notifier_from_errno(0); ++ ++ switch (action) { ++ case MEM_OFFLINE: ++ mutex_lock(&memory_tier_lock); ++ if (clear_node_memory_tier(arg->status_change_nid)) ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ break; ++ case MEM_ONLINE: ++ mutex_lock(&memory_tier_lock); ++ memtier = set_node_memory_tier(arg->status_change_nid); ++ if (!IS_ERR(memtier)) ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ break; ++ } ++ ++ return notifier_from_errno(0); ++} ++ ++static int __init memory_tier_init(void) ++{ ++ int ret, node; ++ struct memory_tier *memtier; ++ ++ ret = subsys_virtual_register(&memory_tier_subsys, NULL); ++ if (ret) ++ panic("%s() failed to register memory tier subsystem\n", __func__); ++ ++#ifdef CONFIG_MIGRATION ++ node_demotion = kcalloc(nr_node_ids, sizeof(struct demotion_nodes), ++ GFP_KERNEL); ++ WARN_ON(!node_demotion); ++#endif ++ mutex_lock(&memory_tier_lock); ++ /* ++ * For now we can have 4 faster memory tiers with smaller adistance ++ * than default DRAM tier. ++ */ ++ default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM); ++ if (!default_dram_type) ++ panic("%s() failed to allocate default DRAM tier\n", __func__); ++ ++ /* ++ * Look at all the existing N_MEMORY nodes and add them to ++ * default memory tier or to a tier if we already have memory ++ * types assigned. ++ */ ++ for_each_node_state(node, N_MEMORY) { ++ memtier = set_node_memory_tier(node); ++ if (IS_ERR(memtier)) ++ /* ++ * Continue with memtiers we are able to setup ++ */ ++ break; ++ } ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ ++ hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRIO); ++ return 0; ++} ++subsys_initcall(memory_tier_init); ++ ++bool numa_demotion_enabled = false; ++ ++#ifdef CONFIG_MIGRATION ++#ifdef CONFIG_SYSFS ++static ssize_t numa_demotion_enabled_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%s\n", ++ numa_demotion_enabled ? "true" : "false"); ++} ++ ++static ssize_t numa_demotion_enabled_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ ssize_t ret; ++ ++ ret = kstrtobool(buf, &numa_demotion_enabled); ++ if (ret) ++ return ret; ++ ++ return count; ++} ++ ++static struct kobj_attribute numa_demotion_enabled_attr = ++ __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, ++ numa_demotion_enabled_store); ++ ++static struct attribute *numa_attrs[] = { ++ &numa_demotion_enabled_attr.attr, ++ NULL, ++}; ++ ++static const struct attribute_group numa_attr_group = { ++ .attrs = numa_attrs, ++}; ++ ++static int __init numa_init_sysfs(void) ++{ ++ int err; ++ struct kobject *numa_kobj; ++ ++ numa_kobj = kobject_create_and_add("numa", mm_kobj); ++ if (!numa_kobj) { ++ pr_err("failed to create numa kobject\n"); ++ return -ENOMEM; ++ } ++ err = sysfs_create_group(numa_kobj, &numa_attr_group); ++ if (err) { ++ pr_err("failed to register numa group\n"); ++ goto delete_obj; ++ } ++ return 0; ++ ++delete_obj: ++ kobject_put(numa_kobj); ++ return err; ++} ++subsys_initcall(numa_init_sysfs); ++#endif /* CONFIG_SYSFS */ ++#endif +diff --git a/mm/memory.c b/mm/memory.c +index 4ba73f5aa8bb..3a3d8721bf4c 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -66,6 +66,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/mm/migrate.c b/mm/migrate.c +index 6a1597c92261..55e7718cfe45 100644 +--- a/mm/migrate.c ++++ b/mm/migrate.c +@@ -50,6 +50,7 @@ + #include + #include + #include ++#include + + #include + +@@ -2170,456 +2171,4 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, + return 0; + } + #endif /* CONFIG_NUMA_BALANCING */ +- +-/* +- * node_demotion[] example: +- * +- * Consider a system with two sockets. Each socket has +- * three classes of memory attached: fast, medium and slow. +- * Each memory class is placed in its own NUMA node. The +- * CPUs are placed in the node with the "fast" memory. The +- * 6 NUMA nodes (0-5) might be split among the sockets like +- * this: +- * +- * Socket A: 0, 1, 2 +- * Socket B: 3, 4, 5 +- * +- * When Node 0 fills up, its memory should be migrated to +- * Node 1. When Node 1 fills up, it should be migrated to +- * Node 2. The migration path start on the nodes with the +- * processors (since allocations default to this node) and +- * fast memory, progress through medium and end with the +- * slow memory: +- * +- * 0 -> 1 -> 2 -> stop +- * 3 -> 4 -> 5 -> stop +- * +- * This is represented in the node_demotion[] like this: +- * +- * { nr=1, nodes[0]=1 }, // Node 0 migrates to 1 +- * { nr=1, nodes[0]=2 }, // Node 1 migrates to 2 +- * { nr=0, nodes[0]=-1 }, // Node 2 does not migrate +- * { nr=1, nodes[0]=4 }, // Node 3 migrates to 4 +- * { nr=1, nodes[0]=5 }, // Node 4 migrates to 5 +- * { nr=0, nodes[0]=-1 }, // Node 5 does not migrate +- * +- * Moreover some systems may have multiple slow memory nodes. +- * Suppose a system has one socket with 3 memory nodes, node 0 +- * is fast memory type, and node 1/2 both are slow memory +- * type, and the distance between fast memory node and slow +- * memory node is same. So the migration path should be: +- * +- * 0 -> 1/2 -> stop +- * +- * This is represented in the node_demotion[] like this: +- * { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2 +- * { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate +- * { nr=0, nodes[0]=-1, }, // Node 2 does not migrate +- */ +- +-/* +- * Writes to this array occur without locking. Cycles are +- * not allowed: Node X demotes to Y which demotes to X... +- * +- * If multiple reads are performed, a single rcu_read_lock() +- * must be held over all reads to ensure that no cycles are +- * observed. +- */ +-#define DEFAULT_DEMOTION_TARGET_NODES 15 +- +-#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES +-#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1) +-#else +-#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES +-#endif +- +-struct demotion_nodes { +- unsigned short nr; +- short nodes[DEMOTION_TARGET_NODES]; +-}; +- +-static struct demotion_nodes *node_demotion __read_mostly; +- +-/** +- * next_demotion_node() - Get the next node in the demotion path +- * @node: The starting node to lookup the next node +- * +- * Return: node id for next memory node in the demotion path hierarchy +- * from @node; NUMA_NO_NODE if @node is terminal. This does not keep +- * @node online or guarantee that it *continues* to be the next demotion +- * target. +- */ +-int next_demotion_node(int node) +-{ +- struct demotion_nodes *nd; +- unsigned short target_nr, index; +- int target; +- +- if (!node_demotion) +- return NUMA_NO_NODE; +- +- nd = &node_demotion[node]; +- +- /* +- * node_demotion[] is updated without excluding this +- * function from running. RCU doesn't provide any +- * compiler barriers, so the READ_ONCE() is required +- * to avoid compiler reordering or read merging. +- * +- * Make sure to use RCU over entire code blocks if +- * node_demotion[] reads need to be consistent. +- */ +- rcu_read_lock(); +- target_nr = READ_ONCE(nd->nr); +- +- switch (target_nr) { +- case 0: +- target = NUMA_NO_NODE; +- goto out; +- case 1: +- index = 0; +- break; +- default: +- /* +- * If there are multiple target nodes, just select one +- * target node randomly. +- * +- * In addition, we can also use round-robin to select +- * target node, but we should introduce another variable +- * for node_demotion[] to record last selected target node, +- * that may cause cache ping-pong due to the changing of +- * last target node. Or introducing per-cpu data to avoid +- * caching issue, which seems more complicated. So selecting +- * target node randomly seems better until now. +- */ +- index = get_random_int() % target_nr; +- break; +- } +- +- target = READ_ONCE(nd->nodes[index]); +- +-out: +- rcu_read_unlock(); +- return target; +-} +- +-/* Disable reclaim-based migration. */ +-static void __disable_all_migrate_targets(void) +-{ +- int node, i; +- +- if (!node_demotion) +- return; +- +- for_each_online_node(node) { +- node_demotion[node].nr = 0; +- for (i = 0; i < DEMOTION_TARGET_NODES; i++) +- node_demotion[node].nodes[i] = NUMA_NO_NODE; +- } +-} +- +-static void disable_all_migrate_targets(void) +-{ +- __disable_all_migrate_targets(); +- +- /* +- * Ensure that the "disable" is visible across the system. +- * Readers will see either a combination of before+disable +- * state or disable+after. They will never see before and +- * after state together. +- * +- * The before+after state together might have cycles and +- * could cause readers to do things like loop until this +- * function finishes. This ensures they can only see a +- * single "bad" read and would, for instance, only loop +- * once. +- */ +- synchronize_rcu(); +-} +- +-/* +- * Find an automatic demotion target for 'node'. +- * Failing here is OK. It might just indicate +- * being at the end of a chain. +- */ +-static int establish_migrate_target(int node, nodemask_t *used, +- int best_distance) +-{ +- int migration_target, index, val; +- struct demotion_nodes *nd; +- +- if (!node_demotion) +- return NUMA_NO_NODE; +- +- nd = &node_demotion[node]; +- +- migration_target = find_next_best_node(node, used); +- if (migration_target == NUMA_NO_NODE) +- return NUMA_NO_NODE; +- +- /* +- * If the node has been set a migration target node before, +- * which means it's the best distance between them. Still +- * check if this node can be demoted to other target nodes +- * if they have a same best distance. +- */ +- if (best_distance != -1) { +- val = node_distance(node, migration_target); +- if (val > best_distance) +- goto out_clear; +- } +- +- index = nd->nr; +- if (WARN_ONCE(index >= DEMOTION_TARGET_NODES, +- "Exceeds maximum demotion target nodes\n")) +- goto out_clear; +- +- nd->nodes[index] = migration_target; +- nd->nr++; +- +- return migration_target; +-out_clear: +- node_clear(migration_target, *used); +- return NUMA_NO_NODE; +-} +- +-/* +- * When memory fills up on a node, memory contents can be +- * automatically migrated to another node instead of +- * discarded at reclaim. +- * +- * Establish a "migration path" which will start at nodes +- * with CPUs and will follow the priorities used to build the +- * page allocator zonelists. +- * +- * The difference here is that cycles must be avoided. If +- * node0 migrates to node1, then neither node1, nor anything +- * node1 migrates to can migrate to node0. Also one node can +- * be migrated to multiple nodes if the target nodes all have +- * a same best-distance against the source node. +- * +- * This function can run simultaneously with readers of +- * node_demotion[]. However, it can not run simultaneously +- * with itself. Exclusion is provided by memory hotplug events +- * being single-threaded. +- */ +-static void __set_migration_target_nodes(void) +-{ +- nodemask_t next_pass; +- nodemask_t this_pass; +- nodemask_t used_targets = NODE_MASK_NONE; +- int node, best_distance; +- +- /* +- * Avoid any oddities like cycles that could occur +- * from changes in the topology. This will leave +- * a momentary gap when migration is disabled. +- */ +- disable_all_migrate_targets(); +- +- /* +- * Allocations go close to CPUs, first. Assume that +- * the migration path starts at the nodes with CPUs. +- */ +- next_pass = node_states[N_CPU]; +-again: +- this_pass = next_pass; +- next_pass = NODE_MASK_NONE; +- /* +- * To avoid cycles in the migration "graph", ensure +- * that migration sources are not future targets by +- * setting them in 'used_targets'. Do this only +- * once per pass so that multiple source nodes can +- * share a target node. +- * +- * 'used_targets' will become unavailable in future +- * passes. This limits some opportunities for +- * multiple source nodes to share a destination. +- */ +- nodes_or(used_targets, used_targets, this_pass); +- +- for_each_node_mask(node, this_pass) { +- best_distance = -1; +- +- /* +- * Try to set up the migration path for the node, and the target +- * migration nodes can be multiple, so doing a loop to find all +- * the target nodes if they all have a best node distance. +- */ +- do { +- int target_node = +- establish_migrate_target(node, &used_targets, +- best_distance); +- +- if (target_node == NUMA_NO_NODE) +- break; +- +- if (best_distance == -1) +- best_distance = node_distance(node, target_node); +- +- /* +- * Visit targets from this pass in the next pass. +- * Eventually, every node will have been part of +- * a pass, and will become set in 'used_targets'. +- */ +- node_set(target_node, next_pass); +- } while (1); +- } +- /* +- * 'next_pass' contains nodes which became migration +- * targets in this pass. Make additional passes until +- * no more migrations targets are available. +- */ +- if (!nodes_empty(next_pass)) +- goto again; +-} +- +-/* +- * For callers that do not hold get_online_mems() already. +- */ +-void set_migration_target_nodes(void) +-{ +- get_online_mems(); +- __set_migration_target_nodes(); +- put_online_mems(); +-} +- +-/* +- * This leaves migrate-on-reclaim transiently disabled between +- * the MEM_GOING_OFFLINE and MEM_OFFLINE events. This runs +- * whether reclaim-based migration is enabled or not, which +- * ensures that the user can turn reclaim-based migration at +- * any time without needing to recalculate migration targets. +- * +- * These callbacks already hold get_online_mems(). That is why +- * __set_migration_target_nodes() can be used as opposed to +- * set_migration_target_nodes(). +- */ +-#ifdef CONFIG_MEMORY_HOTPLUG +-static int __meminit migrate_on_reclaim_callback(struct notifier_block *self, +- unsigned long action, void *_arg) +-{ +- struct memory_notify *arg = _arg; +- +- /* +- * Only update the node migration order when a node is +- * changing status, like online->offline. This avoids +- * the overhead of synchronize_rcu() in most cases. +- */ +- if (arg->status_change_nid < 0) +- return notifier_from_errno(0); +- +- switch (action) { +- case MEM_GOING_OFFLINE: +- /* +- * Make sure there are not transient states where +- * an offline node is a migration target. This +- * will leave migration disabled until the offline +- * completes and the MEM_OFFLINE case below runs. +- */ +- disable_all_migrate_targets(); +- break; +- case MEM_OFFLINE: +- case MEM_ONLINE: +- /* +- * Recalculate the target nodes once the node +- * reaches its final state (online or offline). +- */ +- __set_migration_target_nodes(); +- break; +- case MEM_CANCEL_OFFLINE: +- /* +- * MEM_GOING_OFFLINE disabled all the migration +- * targets. Reenable them. +- */ +- __set_migration_target_nodes(); +- break; +- case MEM_GOING_ONLINE: +- case MEM_CANCEL_ONLINE: +- break; +- } +- +- return notifier_from_errno(0); +-} +-#endif +- +-void __init migrate_on_reclaim_init(void) +-{ +- node_demotion = kcalloc(nr_node_ids, +- sizeof(struct demotion_nodes), +- GFP_KERNEL); +- WARN_ON(!node_demotion); +-#ifdef CONFIG_MEMORY_HOTPLUG +- hotplug_memory_notifier(migrate_on_reclaim_callback, 100); +-#endif +- /* +- * At this point, all numa nodes with memory/CPus have their state +- * properly set, so we can build the demotion order now. +- * Let us hold the cpu_hotplug lock just, as we could possibily have +- * CPU hotplug events during boot. +- */ +- cpus_read_lock(); +- set_migration_target_nodes(); +- cpus_read_unlock(); +-} +- +-bool numa_demotion_enabled = false; +- +-#ifdef CONFIG_SYSFS +-static ssize_t numa_demotion_enabled_show(struct kobject *kobj, +- struct kobj_attribute *attr, char *buf) +-{ +- return sysfs_emit(buf, "%s\n", +- numa_demotion_enabled ? "true" : "false"); +-} +- +-static ssize_t numa_demotion_enabled_store(struct kobject *kobj, +- struct kobj_attribute *attr, +- const char *buf, size_t count) +-{ +- ssize_t ret; +- +- ret = kstrtobool(buf, &numa_demotion_enabled); +- if (ret) +- return ret; +- +- return count; +-} +- +-static struct kobj_attribute numa_demotion_enabled_attr = +- __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, +- numa_demotion_enabled_store); +- +-static struct attribute *numa_attrs[] = { +- &numa_demotion_enabled_attr.attr, +- NULL, +-}; +- +-static const struct attribute_group numa_attr_group = { +- .attrs = numa_attrs, +-}; +- +-static int __init numa_init_sysfs(void) +-{ +- int err; +- struct kobject *numa_kobj; +- +- numa_kobj = kobject_create_and_add("numa", mm_kobj); +- if (!numa_kobj) { +- pr_err("failed to create numa kobject\n"); +- return -ENOMEM; +- } +- err = sysfs_create_group(numa_kobj, &numa_attr_group); +- if (err) { +- pr_err("failed to register numa group\n"); +- goto delete_obj; +- } +- return 0; +- +-delete_obj: +- kobject_put(numa_kobj); +- return err; +-} +-subsys_initcall(numa_init_sysfs); +-#endif /* CONFIG_SYSFS */ + #endif /* CONFIG_NUMA */ +diff --git a/mm/mprotect.c b/mm/mprotect.c +index bc6bddd156ca..eb8982bde7ee 100644 +--- a/mm/mprotect.c ++++ b/mm/mprotect.c +@@ -31,6 +31,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/mm/vmscan.c b/mm/vmscan.c +index 0fc65ace3a4e..e673be68cea3 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -43,6 +43,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -1524,21 +1525,34 @@ static void folio_check_dirty_writeback(struct folio *folio, + mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); + } + +-static struct page *alloc_demote_page(struct page *page, unsigned long node) ++static struct page *alloc_demote_page(struct page *page, unsigned long private) + { +- struct migration_target_control mtc = { +- /* +- * Allocate from 'node', or fail quickly and quietly. +- * When this happens, 'page' will likely just be discarded +- * instead of migrated. +- */ +- .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | +- __GFP_THISNODE | __GFP_NOWARN | +- __GFP_NOMEMALLOC | GFP_NOWAIT, +- .nid = node +- }; ++ struct page *target_page; ++ nodemask_t *allowed_mask; ++ struct migration_target_control *mtc; ++ ++ mtc = (struct migration_target_control *)private; ++ ++ allowed_mask = mtc->nmask; ++ /* ++ * make sure we allocate from the target node first also trying to ++ * demote or reclaim pages from the target node via kswapd if we are ++ * low on free memory on target node. If we don't do this and if ++ * we have free memory on the slower(lower) memtier, we would start ++ * allocating pages from slower(lower) memory tiers without even forcing ++ * a demotion of cold pages from the target memtier. This can result ++ * in the kernel placing hot pages in slower(lower) memory tiers. ++ */ ++ mtc->nmask = NULL; ++ mtc->gfp_mask |= __GFP_THISNODE; ++ target_page = alloc_migration_target(page, (unsigned long)mtc); ++ if (target_page) ++ return target_page; + +- return alloc_migration_target(page, (unsigned long)&mtc); ++ mtc->gfp_mask &= ~__GFP_THISNODE; ++ mtc->nmask = allowed_mask; ++ ++ return alloc_migration_target(page, (unsigned long)mtc); + } + + /* +@@ -1551,6 +1565,19 @@ static unsigned int demote_page_list(struct list_head *demote_pages, + { + int target_nid = next_demotion_node(pgdat->node_id); + unsigned int nr_succeeded; ++ nodemask_t allowed_mask; ++ ++ struct migration_target_control mtc = { ++ /* ++ * Allocate from 'node', or fail quickly and quietly. ++ * When this happens, 'page' will likely just be discarded ++ * instead of migrated. ++ */ ++ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | ++ __GFP_NOMEMALLOC | GFP_NOWAIT, ++ .nid = target_nid, ++ .nmask = &allowed_mask ++ }; + + if (list_empty(demote_pages)) + return 0; +@@ -1558,10 +1585,12 @@ static unsigned int demote_page_list(struct list_head *demote_pages, + if (target_nid == NUMA_NO_NODE) + return 0; + ++ node_get_allowed_targets(pgdat, &allowed_mask); ++ + /* Demotion ignores all cpuset and mempolicy settings */ + migrate_pages(demote_pages, alloc_demote_page, NULL, +- target_nid, MIGRATE_ASYNC, MR_DEMOTION, +- &nr_succeeded); ++ (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, ++ &nr_succeeded); + + if (current_is_kswapd()) + __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded); +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 90af9a8572f5..0b8098e8296b 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -28,7 +28,6 @@ + #include + #include + #include +-#include + + #include "internal.h" + +@@ -2067,7 +2066,6 @@ static int vmstat_cpu_online(unsigned int cpu) + + if (!node_state(cpu_to_node(cpu), N_CPU)) { + node_set_state(cpu_to_node(cpu), N_CPU); +- set_migration_target_nodes(); + } + + return 0; +@@ -2092,7 +2090,6 @@ static int vmstat_cpu_dead(unsigned int cpu) + return 0; + + node_clear_state(node, N_CPU); +- set_migration_target_nodes(); + + return 0; + } +@@ -2125,7 +2122,6 @@ void __init init_mm_internals(void) + + start_shepherd_timer(); + #endif +- migrate_on_reclaim_init(); + #ifdef CONFIG_PROC_FS + proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); + proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); +-- +2.38.0.rc1.8.g2a7d63a245 + diff --git a/6.0/0006-mm-cleanup.patch b/6.0/0006-mm-cleanup.patch deleted file mode 100644 index dcf34d62..00000000 --- a/6.0/0006-mm-cleanup.patch +++ /dev/null @@ -1,322 +0,0 @@ -From 440d9b1f159e8842799696a54fd5ddb51d545bc9 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:18:41 +0200 -Subject: [PATCH 06/13] mm-cleanup - -Signed-off-by: Peter Jung ---- - include/linux/gfp.h | 23 ------------- - include/linux/memory_hotplug.h | 8 ----- - include/linux/mm.h | 2 +- - include/linux/mmzone.h | 12 ------- - mm/internal.h | 3 -- - mm/page_alloc.c | 63 ++++++++++++++++------------------ - 6 files changed, 31 insertions(+), 80 deletions(-) - -diff --git a/include/linux/gfp.h b/include/linux/gfp.h -index f314be58fa77..5c7d752b18d9 100644 ---- a/include/linux/gfp.h -+++ b/include/linux/gfp.h -@@ -33,29 +33,6 @@ static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) - return !!(gfp_flags & __GFP_DIRECT_RECLAIM); - } - --/** -- * gfpflags_normal_context - is gfp_flags a normal sleepable context? -- * @gfp_flags: gfp_flags to test -- * -- * Test whether @gfp_flags indicates that the allocation is from the -- * %current context and allowed to sleep. -- * -- * An allocation being allowed to block doesn't mean it owns the %current -- * context. When direct reclaim path tries to allocate memory, the -- * allocation context is nested inside whatever %current was doing at the -- * time of the original allocation. The nested allocation may be allowed -- * to block but modifying anything %current owns can corrupt the outer -- * context's expectations. -- * -- * %true result from this function indicates that the allocation context -- * can sleep and use anything that's associated with %current. -- */ --static inline bool gfpflags_normal_context(const gfp_t gfp_flags) --{ -- return (gfp_flags & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC)) == -- __GFP_DIRECT_RECLAIM; --} -- - #ifdef CONFIG_HIGHMEM - #define OPT_ZONE_HIGHMEM ZONE_HIGHMEM - #else -diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h -index e0b2209ab71c..7ac0e5c78648 100644 ---- a/include/linux/memory_hotplug.h -+++ b/include/linux/memory_hotplug.h -@@ -44,11 +44,6 @@ extern void arch_refresh_nodedata(int nid, pg_data_t *pgdat); - ({ \ - memblock_alloc(sizeof(*pgdat), SMP_CACHE_BYTES); \ - }) --/* -- * This definition is just for error path in node hotadd. -- * For node hotremove, we have to replace this. -- */ --#define generic_free_nodedata(pgdat) kfree(pgdat) - - extern pg_data_t *node_data[]; - static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) -@@ -64,9 +59,6 @@ static inline pg_data_t *generic_alloc_nodedata(int nid) - BUG(); - return NULL; - } --static inline void generic_free_nodedata(pg_data_t *pgdat) --{ --} - static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) - { - } -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 88976a521ef5..6c61497f0167 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -3016,7 +3016,7 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, - unsigned long address, unsigned long size, - pte_fn_t fn, void *data); - --extern void init_mem_debugging_and_hardening(void); -+extern void __init init_mem_debugging_and_hardening(void); - #ifdef CONFIG_PAGE_POISONING - extern void __kernel_poison_pages(struct page *page, int numpages); - extern void __kernel_unpoison_pages(struct page *page, int numpages); -diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index 1543001feba9..21d39f152d0c 100644 ---- a/include/linux/mmzone.h -+++ b/include/linux/mmzone.h -@@ -577,13 +577,6 @@ enum zone_watermarks { - #define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1)) - #define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP) - --/* -- * Shift to encode migratetype and order in the same integer, with order -- * in the least significant bits. -- */ --#define NR_PCP_ORDER_WIDTH 8 --#define NR_PCP_ORDER_MASK ((1<_watermark[WMARK_MIN] + z->watermark_boost) - #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost) - #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost) -@@ -1241,11 +1234,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) - return pgdat->node_start_pfn + pgdat->node_spanned_pages; - } - --static inline bool pgdat_is_empty(pg_data_t *pgdat) --{ -- return !pgdat->node_start_pfn && !pgdat->node_spanned_pages; --} -- - #include - - void build_all_zonelists(pg_data_t *pgdat); -diff --git a/mm/internal.h b/mm/internal.h -index a1fddea6b34f..dd8600d71afe 100644 ---- a/mm/internal.h -+++ b/mm/internal.h -@@ -366,7 +366,6 @@ extern int user_min_free_kbytes; - extern void free_unref_page(struct page *page, unsigned int order); - extern void free_unref_page_list(struct list_head *list); - --extern void zone_pcp_update(struct zone *zone, int cpu_online); - extern void zone_pcp_reset(struct zone *zone); - extern void zone_pcp_disable(struct zone *zone); - extern void zone_pcp_enable(struct zone *zone); -@@ -861,8 +860,6 @@ int migrate_device_coherent_page(struct page *page); - */ - struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags); - --DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); -- - extern bool mirrored_kernelcore; - - static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) -diff --git a/mm/page_alloc.c b/mm/page_alloc.c -index cf131d6e08fb..292ed1bb6a5a 100644 ---- a/mm/page_alloc.c -+++ b/mm/page_alloc.c -@@ -870,7 +870,8 @@ static inline bool set_page_guard(struct zone *zone, struct page *page, - INIT_LIST_HEAD(&page->buddy_list); - set_page_private(page, order); - /* Guard pages are not available for any usage */ -- __mod_zone_freepage_state(zone, -(1 << order), migratetype); -+ if (!is_migrate_isolate(migratetype)) -+ __mod_zone_freepage_state(zone, -(1 << order), migratetype); - - return true; - } -@@ -900,7 +901,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, - * order of appearance. So we need to first gather the full picture of what was - * enabled, and then make decisions. - */ --void init_mem_debugging_and_hardening(void) -+void __init init_mem_debugging_and_hardening(void) - { - bool page_poisoning_requested = false; - -@@ -1105,7 +1106,7 @@ static inline void __free_one_page(struct page *page, - int migratetype, fpi_t fpi_flags) - { - struct capture_control *capc = task_capc(zone); -- unsigned long buddy_pfn; -+ unsigned long buddy_pfn = 0; - unsigned long combined_pfn; - struct page *buddy; - bool to_tail; -@@ -1575,7 +1576,6 @@ static void free_pcppages_bulk(struct zone *zone, int count, - - order = pindex_to_order(pindex); - nr_pages = 1 << order; -- BUILD_BUG_ON(MAX_ORDER >= (1< PAGE_ALLOC_COSTLY_ORDER, gfp_mask); -+ WARN_ON_ONCE_GFP(costly_order, gfp_mask); - - /* - * Help non-failing allocations by giving them access to memory -@@ -6507,7 +6502,7 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta - #define BOOT_PAGESET_BATCH 1 - static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); - static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); --DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); -+static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); - - static void __build_all_zonelists(void *data) - { -@@ -6810,7 +6805,7 @@ void __ref memmap_init_zone_device(struct zone *zone, - unsigned long start = jiffies; - int nid = pgdat->node_id; - -- if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) -+ if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) - return; - - /* -@@ -6986,7 +6981,7 @@ static int zone_batchsize(struct zone *zone) - * size is striking a balance between allocation latency - * and zone lock contention. - */ -- batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); -+ batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); - batch /= 4; /* We effectively *= 4 below */ - if (batch < 1) - batch = 1; -@@ -7171,6 +7166,17 @@ void __meminit setup_zone_pageset(struct zone *zone) - zone_set_pageset_high_and_batch(zone, 0); - } - -+/* -+ * The zone indicated has a new number of managed_pages; batch sizes and percpu -+ * page high values need to be recalculated. -+ */ -+static void zone_pcp_update(struct zone *zone, int cpu_online) -+{ -+ mutex_lock(&pcp_batch_high_lock); -+ zone_set_pageset_high_and_batch(zone, cpu_online); -+ mutex_unlock(&pcp_batch_high_lock); -+} -+ - /* - * Allocate per cpu pagesets and initialize them. - * Before this call only boot pagesets were available. -@@ -8461,8 +8467,8 @@ void __init mem_init_print_info(void) - #endif - ")\n", - K(nr_free_pages()), K(physpages), -- codesize >> 10, datasize >> 10, rosize >> 10, -- (init_data_size + init_code_size) >> 10, bss_size >> 10, -+ codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, -+ (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, - K(physpages - totalram_pages() - totalcma_pages), - K(totalcma_pages) - #ifdef CONFIG_HIGHMEM -@@ -8987,8 +8993,8 @@ void *__init alloc_large_system_hash(const char *tablename, - numentries -= arch_reserved_kernel_pages(); - - /* It isn't necessary when PAGE_SIZE >= 1MB */ -- if (PAGE_SHIFT < 20) -- numentries = round_up(numentries, (1<<20)/PAGE_SIZE); -+ if (PAGE_SIZE < SZ_1M) -+ numentries = round_up(numentries, SZ_1M / PAGE_SIZE); - - #if __BITS_PER_LONG > 32 - if (!high_limit) { -@@ -9412,17 +9418,6 @@ void free_contig_range(unsigned long pfn, unsigned long nr_pages) - } - EXPORT_SYMBOL(free_contig_range); - --/* -- * The zone indicated has a new number of managed_pages; batch sizes and percpu -- * page high values need to be recalculated. -- */ --void zone_pcp_update(struct zone *zone, int cpu_online) --{ -- mutex_lock(&pcp_batch_high_lock); -- zone_set_pageset_high_and_batch(zone, cpu_online); -- mutex_unlock(&pcp_batch_high_lock); --} -- - /* - * Effectively disable pcplists for the zone by setting the high limit to 0 - * and draining all cpus. A concurrent page freeing on another CPU that's about -@@ -9455,9 +9450,11 @@ void zone_pcp_reset(struct zone *zone) - drain_zonestat(zone, pzstats); - } - free_percpu(zone->per_cpu_pageset); -- free_percpu(zone->per_cpu_zonestats); - zone->per_cpu_pageset = &boot_pageset; -- zone->per_cpu_zonestats = &boot_zonestats; -+ if (zone->per_cpu_zonestats != &boot_zonestats) { -+ free_percpu(zone->per_cpu_zonestats); -+ zone->per_cpu_zonestats = &boot_zonestats; -+ } - } - } - --- -2.37.3 - diff --git a/6.0/0006-mm-khugepaged-add-struct-collapse_control.patch b/6.0/0006-mm-khugepaged-add-struct-collapse_control.patch new file mode 100644 index 00000000..616c261c --- /dev/null +++ b/6.0/0006-mm-khugepaged-add-struct-collapse_control.patch @@ -0,0 +1,2743 @@ +From 30817d963bfdddf095e330e41317c9efceec642a Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:26:29 +0200 +Subject: [PATCH 06/16] mm/khugepaged: add struct collapse_control + +Signed-off-by: Peter Jung +--- + arch/alpha/include/uapi/asm/mman.h | 2 + + arch/mips/include/uapi/asm/mman.h | 2 + + arch/parisc/include/uapi/asm/mman.h | 2 + + arch/xtensa/include/uapi/asm/mman.h | 2 + + fs/proc/task_mmu.c | 2 +- + include/linux/huge_mm.h | 23 +- + include/trace/events/huge_memory.h | 1 + + include/uapi/asm-generic/mman-common.h | 2 + + mm/huge_memory.c | 32 +- + mm/internal.h | 2 +- + mm/khugepaged.c | 763 +++++++++++-------- + mm/ksm.c | 10 + + mm/madvise.c | 9 +- + mm/memory.c | 4 +- + mm/rmap.c | 15 +- + tools/include/uapi/asm-generic/mman-common.h | 2 + + tools/testing/selftests/vm/khugepaged.c | 563 ++++++++------ + 17 files changed, 825 insertions(+), 611 deletions(-) + +diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h +index 4aa996423b0d..763929e814e9 100644 +--- a/arch/alpha/include/uapi/asm/mman.h ++++ b/arch/alpha/include/uapi/asm/mman.h +@@ -76,6 +76,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h +index 1be428663c10..c6e1fc77c996 100644 +--- a/arch/mips/include/uapi/asm/mman.h ++++ b/arch/mips/include/uapi/asm/mman.h +@@ -103,6 +103,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h +index a7ea3204a5fa..22133a6a506e 100644 +--- a/arch/parisc/include/uapi/asm/mman.h ++++ b/arch/parisc/include/uapi/asm/mman.h +@@ -70,6 +70,8 @@ + #define MADV_WIPEONFORK 71 /* Zero memory on fork, child only */ + #define MADV_KEEPONFORK 72 /* Undo MADV_WIPEONFORK */ + ++#define MADV_COLLAPSE 73 /* Synchronous hugepage collapse */ ++ + #define MADV_HWPOISON 100 /* poison a page for testing */ + #define MADV_SOFT_OFFLINE 101 /* soft offline page for testing */ + +diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h +index 7966a58af472..1ff0c858544f 100644 +--- a/arch/xtensa/include/uapi/asm/mman.h ++++ b/arch/xtensa/include/uapi/asm/mman.h +@@ -111,6 +111,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c +index 4e0023643f8b..482f91577f8c 100644 +--- a/fs/proc/task_mmu.c ++++ b/fs/proc/task_mmu.c +@@ -864,7 +864,7 @@ static int show_smap(struct seq_file *m, void *v) + __show_smap(m, &mss, false); + + seq_printf(m, "THPeligible: %d\n", +- hugepage_vma_check(vma, vma->vm_flags, true, false)); ++ hugepage_vma_check(vma, vma->vm_flags, true, false, true)); + + if (arch_pkeys_enabled()) + seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); +diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h +index 768e5261fdae..38265f9f782e 100644 +--- a/include/linux/huge_mm.h ++++ b/include/linux/huge_mm.h +@@ -168,9 +168,8 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) + !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); + } + +-bool hugepage_vma_check(struct vm_area_struct *vma, +- unsigned long vm_flags, +- bool smaps, bool in_pf); ++bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, ++ bool smaps, bool in_pf, bool enforce_sysfs); + + #define transparent_hugepage_use_zero_page() \ + (transparent_hugepage_flags & \ +@@ -219,6 +218,9 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, + + int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, + int advice); ++int madvise_collapse(struct vm_area_struct *vma, ++ struct vm_area_struct **prev, ++ unsigned long start, unsigned long end); + void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, + unsigned long end, long adjust_next); + spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma); +@@ -321,8 +323,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, + } + + static inline bool hugepage_vma_check(struct vm_area_struct *vma, +- unsigned long vm_flags, +- bool smaps, bool in_pf) ++ unsigned long vm_flags, bool smaps, ++ bool in_pf, bool enforce_sysfs) + { + return false; + } +@@ -362,9 +364,16 @@ static inline void split_huge_pmd_address(struct vm_area_struct *vma, + static inline int hugepage_madvise(struct vm_area_struct *vma, + unsigned long *vm_flags, int advice) + { +- BUG(); +- return 0; ++ return -EINVAL; + } ++ ++static inline int madvise_collapse(struct vm_area_struct *vma, ++ struct vm_area_struct **prev, ++ unsigned long start, unsigned long end) ++{ ++ return -EINVAL; ++} ++ + static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, + unsigned long start, + unsigned long end, +diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h +index d651f3437367..55392bf30a03 100644 +--- a/include/trace/events/huge_memory.h ++++ b/include/trace/events/huge_memory.h +@@ -11,6 +11,7 @@ + EM( SCAN_FAIL, "failed") \ + EM( SCAN_SUCCEED, "succeeded") \ + EM( SCAN_PMD_NULL, "pmd_null") \ ++ EM( SCAN_PMD_MAPPED, "page_pmd_mapped") \ + EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \ + EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ + EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ +diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h +index 6c1aa92a92e4..6ce1f1ceb432 100644 +--- a/include/uapi/asm-generic/mman-common.h ++++ b/include/uapi/asm-generic/mman-common.h +@@ -77,6 +77,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 6eb4b1799b79..42cdc3338adc 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -71,9 +71,8 @@ static atomic_t huge_zero_refcount; + struct page *huge_zero_page __read_mostly; + unsigned long huge_zero_pfn __read_mostly = ~0UL; + +-bool hugepage_vma_check(struct vm_area_struct *vma, +- unsigned long vm_flags, +- bool smaps, bool in_pf) ++bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, ++ bool smaps, bool in_pf, bool enforce_sysfs) + { + if (!vma->vm_mm) /* vdso */ + return false; +@@ -122,11 +121,10 @@ bool hugepage_vma_check(struct vm_area_struct *vma, + if (!in_pf && shmem_file(vma->vm_file)) + return shmem_huge_enabled(vma); + +- if (!hugepage_flags_enabled()) +- return false; +- +- /* THP settings require madvise. */ +- if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always()) ++ /* Enforce sysfs THP requirements as necessary */ ++ if (enforce_sysfs && ++ (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) && ++ !hugepage_flags_always()))) + return false; + + /* Only regular file is valid */ +@@ -2289,25 +2287,11 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, + void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, + bool freeze, struct folio *folio) + { +- pgd_t *pgd; +- p4d_t *p4d; +- pud_t *pud; +- pmd_t *pmd; +- +- pgd = pgd_offset(vma->vm_mm, address); +- if (!pgd_present(*pgd)) +- return; ++ pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); + +- p4d = p4d_offset(pgd, address); +- if (!p4d_present(*p4d)) ++ if (!pmd) + return; + +- pud = pud_offset(p4d, address); +- if (!pud_present(*pud)) +- return; +- +- pmd = pmd_offset(pud, address); +- + __split_huge_pmd(vma, pmd, address, freeze, folio); + } + +diff --git a/mm/internal.h b/mm/internal.h +index 785409805ed7..55ce10e4d0c0 100644 +--- a/mm/internal.h ++++ b/mm/internal.h +@@ -187,7 +187,7 @@ extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason + /* + * in mm/rmap.c: + */ +-extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); ++pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); + + /* + * in mm/page_alloc.c +diff --git a/mm/khugepaged.c b/mm/khugepaged.c +index 01f71786d530..5f7c60b8b269 100644 +--- a/mm/khugepaged.c ++++ b/mm/khugepaged.c +@@ -28,6 +28,7 @@ enum scan_result { + SCAN_FAIL, + SCAN_SUCCEED, + SCAN_PMD_NULL, ++ SCAN_PMD_MAPPED, + SCAN_EXCEED_NONE_PTE, + SCAN_EXCEED_SWAP_PTE, + SCAN_EXCEED_SHARED_PTE, +@@ -73,6 +74,8 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); + * default collapse hugepages if there is at least one pte mapped like + * it would have happened if the vma was large enough during page + * fault. ++ * ++ * Note that these are only respected if collapse was initiated by khugepaged. + */ + static unsigned int khugepaged_max_ptes_none __read_mostly; + static unsigned int khugepaged_max_ptes_swap __read_mostly; +@@ -85,6 +88,16 @@ static struct kmem_cache *mm_slot_cache __read_mostly; + + #define MAX_PTE_MAPPED_THP 8 + ++struct collapse_control { ++ bool is_khugepaged; ++ ++ /* Num pages scanned per node */ ++ u32 node_load[MAX_NUMNODES]; ++ ++ /* Last target selected in hpage_collapse_find_target_node() */ ++ int last_target_node; ++}; ++ + /** + * struct mm_slot - hash lookup from mm to mm_slot + * @hash: hash collision list +@@ -425,7 +438,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm, + hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); + } + +-static inline int khugepaged_test_exit(struct mm_struct *mm) ++static inline int hpage_collapse_test_exit(struct mm_struct *mm) + { + return atomic_read(&mm->mm_users) == 0; + } +@@ -440,7 +453,7 @@ void __khugepaged_enter(struct mm_struct *mm) + return; + + /* __khugepaged_exit() must not run from under us */ +- VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); ++ VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); + if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { + free_mm_slot(mm_slot); + return; +@@ -466,7 +479,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, + { + if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && + hugepage_flags_enabled()) { +- if (hugepage_vma_check(vma, vm_flags, false, false)) ++ if (hugepage_vma_check(vma, vm_flags, false, false, true)) + __khugepaged_enter(vma->vm_mm); + } + } +@@ -492,11 +505,10 @@ void __khugepaged_exit(struct mm_struct *mm) + } else if (mm_slot) { + /* + * This is required to serialize against +- * khugepaged_test_exit() (which is guaranteed to run +- * under mmap sem read mode). Stop here (after we +- * return all pagetables will be destroyed) until +- * khugepaged has finished working on the pagetables +- * under the mmap_lock. ++ * hpage_collapse_test_exit() (which is guaranteed to run ++ * under mmap sem read mode). Stop here (after we return all ++ * pagetables will be destroyed) until khugepaged has finished ++ * working on the pagetables under the mmap_lock. + */ + mmap_write_lock(mm); + mmap_write_unlock(mm); +@@ -546,11 +558,12 @@ static bool is_refcount_suitable(struct page *page) + static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + unsigned long address, + pte_t *pte, ++ struct collapse_control *cc, + struct list_head *compound_pagelist) + { + struct page *page = NULL; + pte_t *_pte; +- int none_or_zero = 0, shared = 0, result = 0, referenced = 0; ++ int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; + bool writable = false; + + for (_pte = pte; _pte < pte + HPAGE_PMD_NR; +@@ -558,8 +571,10 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + pte_t pteval = *_pte; + if (pte_none(pteval) || (pte_present(pteval) && + is_zero_pfn(pte_pfn(pteval)))) { ++ ++none_or_zero; + if (!userfaultfd_armed(vma) && +- ++none_or_zero <= khugepaged_max_ptes_none) { ++ (!cc->is_khugepaged || ++ none_or_zero <= khugepaged_max_ptes_none)) { + continue; + } else { + result = SCAN_EXCEED_NONE_PTE; +@@ -579,11 +594,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + + VM_BUG_ON_PAGE(!PageAnon(page), page); + +- if (page_mapcount(page) > 1 && +- ++shared > khugepaged_max_ptes_shared) { +- result = SCAN_EXCEED_SHARED_PTE; +- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); +- goto out; ++ if (page_mapcount(page) > 1) { ++ ++shared; ++ if (cc->is_khugepaged && ++ shared > khugepaged_max_ptes_shared) { ++ result = SCAN_EXCEED_SHARED_PTE; ++ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); ++ goto out; ++ } + } + + if (PageCompound(page)) { +@@ -646,10 +664,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + if (PageCompound(page)) + list_add_tail(&page->lru, compound_pagelist); + next: +- /* There should be enough young pte to collapse the page */ +- if (pte_young(pteval) || +- page_is_young(page) || PageReferenced(page) || +- mmu_notifier_test_young(vma->vm_mm, address)) ++ /* ++ * If collapse was initiated by khugepaged, check that there is ++ * enough young pte to justify collapsing the page ++ */ ++ if (cc->is_khugepaged && ++ (pte_young(pteval) || page_is_young(page) || ++ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, ++ address))) + referenced++; + + if (pte_write(pteval)) +@@ -658,19 +680,19 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + + if (unlikely(!writable)) { + result = SCAN_PAGE_RO; +- } else if (unlikely(!referenced)) { ++ } else if (unlikely(cc->is_khugepaged && !referenced)) { + result = SCAN_LACK_REFERENCED_PAGE; + } else { + result = SCAN_SUCCEED; + trace_mm_collapse_huge_page_isolate(page, none_or_zero, + referenced, writable, result); +- return 1; ++ return result; + } + out: + release_pte_pages(pte, _pte, compound_pagelist); + trace_mm_collapse_huge_page_isolate(page, none_or_zero, + referenced, writable, result); +- return 0; ++ return result; + } + + static void __collapse_huge_page_copy(pte_t *pte, struct page *page, +@@ -735,9 +757,12 @@ static void khugepaged_alloc_sleep(void) + remove_wait_queue(&khugepaged_wait, &wait); + } + +-static int khugepaged_node_load[MAX_NUMNODES]; ++struct collapse_control khugepaged_collapse_control = { ++ .is_khugepaged = true, ++ .last_target_node = NUMA_NO_NODE, ++}; + +-static bool khugepaged_scan_abort(int nid) ++static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) + { + int i; + +@@ -749,11 +774,11 @@ static bool khugepaged_scan_abort(int nid) + return false; + + /* If there is a count for this node already, it must be acceptable */ +- if (khugepaged_node_load[nid]) ++ if (cc->node_load[nid]) + return false; + + for (i = 0; i < MAX_NUMNODES; i++) { +- if (!khugepaged_node_load[i]) ++ if (!cc->node_load[i]) + continue; + if (node_distance(nid, i) > node_reclaim_distance) + return true; +@@ -772,146 +797,62 @@ static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) + } + + #ifdef CONFIG_NUMA +-static int khugepaged_find_target_node(void) ++static int hpage_collapse_find_target_node(struct collapse_control *cc) + { +- static int last_khugepaged_target_node = NUMA_NO_NODE; + int nid, target_node = 0, max_value = 0; + + /* find first node with max normal pages hit */ + for (nid = 0; nid < MAX_NUMNODES; nid++) +- if (khugepaged_node_load[nid] > max_value) { +- max_value = khugepaged_node_load[nid]; ++ if (cc->node_load[nid] > max_value) { ++ max_value = cc->node_load[nid]; + target_node = nid; + } + + /* do some balance if several nodes have the same hit record */ +- if (target_node <= last_khugepaged_target_node) +- for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; +- nid++) +- if (max_value == khugepaged_node_load[nid]) { ++ if (target_node <= cc->last_target_node) ++ for (nid = cc->last_target_node + 1; nid < MAX_NUMNODES; ++ nid++) ++ if (max_value == cc->node_load[nid]) { + target_node = nid; + break; + } + +- last_khugepaged_target_node = target_node; ++ cc->last_target_node = target_node; + return target_node; + } +- +-static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) ++#else ++static int hpage_collapse_find_target_node(struct collapse_control *cc) + { +- if (IS_ERR(*hpage)) { +- if (!*wait) +- return false; +- +- *wait = false; +- *hpage = NULL; +- khugepaged_alloc_sleep(); +- } else if (*hpage) { +- put_page(*hpage); +- *hpage = NULL; +- } +- +- return true; ++ return 0; + } ++#endif + +-static struct page * +-khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) ++static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node) + { +- VM_BUG_ON_PAGE(*hpage, *hpage); +- + *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); + if (unlikely(!*hpage)) { + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); +- *hpage = ERR_PTR(-ENOMEM); +- return NULL; ++ return false; + } + + prep_transhuge_page(*hpage); + count_vm_event(THP_COLLAPSE_ALLOC); +- return *hpage; +-} +-#else +-static int khugepaged_find_target_node(void) +-{ +- return 0; +-} +- +-static inline struct page *alloc_khugepaged_hugepage(void) +-{ +- struct page *page; +- +- page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), +- HPAGE_PMD_ORDER); +- if (page) +- prep_transhuge_page(page); +- return page; +-} +- +-static struct page *khugepaged_alloc_hugepage(bool *wait) +-{ +- struct page *hpage; +- +- do { +- hpage = alloc_khugepaged_hugepage(); +- if (!hpage) { +- count_vm_event(THP_COLLAPSE_ALLOC_FAILED); +- if (!*wait) +- return NULL; +- +- *wait = false; +- khugepaged_alloc_sleep(); +- } else +- count_vm_event(THP_COLLAPSE_ALLOC); +- } while (unlikely(!hpage) && likely(hugepage_flags_enabled())); +- +- return hpage; +-} +- +-static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) +-{ +- /* +- * If the hpage allocated earlier was briefly exposed in page cache +- * before collapse_file() failed, it is possible that racing lookups +- * have not yet completed, and would then be unpleasantly surprised by +- * finding the hpage reused for the same mapping at a different offset. +- * Just release the previous allocation if there is any danger of that. +- */ +- if (*hpage && page_count(*hpage) > 1) { +- put_page(*hpage); +- *hpage = NULL; +- } +- +- if (!*hpage) +- *hpage = khugepaged_alloc_hugepage(wait); +- +- if (unlikely(!*hpage)) +- return false; +- + return true; + } + +-static struct page * +-khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) +-{ +- VM_BUG_ON(!*hpage); +- +- return *hpage; +-} +-#endif +- + /* + * If mmap_lock temporarily dropped, revalidate vma + * before taking mmap_lock. +- * Return 0 if succeeds, otherwise return none-zero +- * value (scan code). ++ * Returns enum scan_result value. + */ + + static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, +- struct vm_area_struct **vmap) ++ struct vm_area_struct **vmap, ++ struct collapse_control *cc) + { + struct vm_area_struct *vma; + +- if (unlikely(khugepaged_test_exit(mm))) ++ if (unlikely(hpage_collapse_test_exit(mm))) + return SCAN_ANY_PROCESS; + + *vmap = vma = find_vma(mm, address); +@@ -920,7 +861,8 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, + + if (!transhuge_vma_suitable(vma, address)) + return SCAN_ADDRESS_RANGE; +- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, ++ cc->is_khugepaged)) + return SCAN_VMA_CHECK; + /* + * Anon VMA expected, the address may be unmapped then +@@ -931,21 +873,60 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, + */ + if (!vma->anon_vma || !vma_is_anonymous(vma)) + return SCAN_VMA_CHECK; +- return 0; ++ return SCAN_SUCCEED; ++} ++ ++static int find_pmd_or_thp_or_none(struct mm_struct *mm, ++ unsigned long address, ++ pmd_t **pmd) ++{ ++ pmd_t pmde; ++ ++ *pmd = mm_find_pmd(mm, address); ++ if (!*pmd) ++ return SCAN_PMD_NULL; ++ ++ pmde = pmd_read_atomic(*pmd); ++ ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ ++ barrier(); ++#endif ++ if (!pmd_present(pmde)) ++ return SCAN_PMD_NULL; ++ if (pmd_trans_huge(pmde)) ++ return SCAN_PMD_MAPPED; ++ if (pmd_bad(pmde)) ++ return SCAN_PMD_NULL; ++ return SCAN_SUCCEED; ++} ++ ++static int check_pmd_still_valid(struct mm_struct *mm, ++ unsigned long address, ++ pmd_t *pmd) ++{ ++ pmd_t *new_pmd; ++ int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); ++ ++ if (result != SCAN_SUCCEED) ++ return result; ++ if (new_pmd != pmd) ++ return SCAN_FAIL; ++ return SCAN_SUCCEED; + } + + /* + * Bring missing pages in from swap, to complete THP collapse. +- * Only done if khugepaged_scan_pmd believes it is worthwhile. ++ * Only done if hpage_collapse_scan_pmd believes it is worthwhile. + * + * Called and returns without pte mapped or spinlocks held. + * Note that if false is returned, mmap_lock will be released. + */ + +-static bool __collapse_huge_page_swapin(struct mm_struct *mm, +- struct vm_area_struct *vma, +- unsigned long haddr, pmd_t *pmd, +- int referenced) ++static int __collapse_huge_page_swapin(struct mm_struct *mm, ++ struct vm_area_struct *vma, ++ unsigned long haddr, pmd_t *pmd, ++ int referenced) + { + int swapped_in = 0; + vm_fault_t ret = 0; +@@ -976,12 +957,13 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, + */ + if (ret & VM_FAULT_RETRY) { + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); +- return false; ++ /* Likely, but not guaranteed, that page lock failed */ ++ return SCAN_PAGE_LOCK; + } + if (ret & VM_FAULT_ERROR) { + mmap_read_unlock(mm); + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); +- return false; ++ return SCAN_FAIL; + } + swapped_in++; + } +@@ -991,30 +973,41 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, + lru_add_drain(); + + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); +- return true; ++ return SCAN_SUCCEED; + } + +-static void collapse_huge_page(struct mm_struct *mm, +- unsigned long address, +- struct page **hpage, +- int node, int referenced, int unmapped) ++static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, ++ struct collapse_control *cc) ++{ ++ /* Only allocate from the target node */ ++ gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : ++ GFP_TRANSHUGE) | __GFP_THISNODE; ++ int node = hpage_collapse_find_target_node(cc); ++ ++ if (!hpage_collapse_alloc_page(hpage, gfp, node)) ++ return SCAN_ALLOC_HUGE_PAGE_FAIL; ++ if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) ++ return SCAN_CGROUP_CHARGE_FAIL; ++ count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); ++ return SCAN_SUCCEED; ++} ++ ++static int collapse_huge_page(struct mm_struct *mm, unsigned long address, ++ int referenced, int unmapped, ++ struct collapse_control *cc) + { + LIST_HEAD(compound_pagelist); + pmd_t *pmd, _pmd; + pte_t *pte; + pgtable_t pgtable; +- struct page *new_page; ++ struct page *hpage; + spinlock_t *pmd_ptl, *pte_ptl; +- int isolated = 0, result = 0; ++ int result = SCAN_FAIL; + struct vm_area_struct *vma; + struct mmu_notifier_range range; +- gfp_t gfp; + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + +- /* Only allocate from the target node */ +- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; +- + /* + * Before allocating the hugepage, release the mmap_lock read lock. + * The allocation can take potentially a long time if it involves +@@ -1022,40 +1015,34 @@ static void collapse_huge_page(struct mm_struct *mm, + * that. We will recheck the vma after taking it again in write mode. + */ + mmap_read_unlock(mm); +- new_page = khugepaged_alloc_page(hpage, gfp, node); +- if (!new_page) { +- result = SCAN_ALLOC_HUGE_PAGE_FAIL; +- goto out_nolock; +- } + +- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { +- result = SCAN_CGROUP_CHARGE_FAIL; ++ result = alloc_charge_hpage(&hpage, mm, cc); ++ if (result != SCAN_SUCCEED) + goto out_nolock; +- } +- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); + + mmap_read_lock(mm); +- result = hugepage_vma_revalidate(mm, address, &vma); +- if (result) { ++ result = hugepage_vma_revalidate(mm, address, &vma, cc); ++ if (result != SCAN_SUCCEED) { + mmap_read_unlock(mm); + goto out_nolock; + } + +- pmd = mm_find_pmd(mm, address); +- if (!pmd) { +- result = SCAN_PMD_NULL; ++ result = find_pmd_or_thp_or_none(mm, address, &pmd); ++ if (result != SCAN_SUCCEED) { + mmap_read_unlock(mm); + goto out_nolock; + } + +- /* +- * __collapse_huge_page_swapin will return with mmap_lock released +- * when it fails. So we jump out_nolock directly in that case. +- * Continuing to collapse causes inconsistency. +- */ +- if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, +- pmd, referenced)) { +- goto out_nolock; ++ if (unmapped) { ++ /* ++ * __collapse_huge_page_swapin will return with mmap_lock ++ * released when it fails. So we jump out_nolock directly in ++ * that case. Continuing to collapse causes inconsistency. ++ */ ++ result = __collapse_huge_page_swapin(mm, vma, address, pmd, ++ referenced); ++ if (result != SCAN_SUCCEED) ++ goto out_nolock; + } + + mmap_read_unlock(mm); +@@ -1065,11 +1052,12 @@ static void collapse_huge_page(struct mm_struct *mm, + * handled by the anon_vma lock + PG_lock. + */ + mmap_write_lock(mm); +- result = hugepage_vma_revalidate(mm, address, &vma); +- if (result) ++ result = hugepage_vma_revalidate(mm, address, &vma, cc); ++ if (result != SCAN_SUCCEED) + goto out_up_write; + /* check if the pmd is still valid */ +- if (mm_find_pmd(mm, address) != pmd) ++ result = check_pmd_still_valid(mm, address, pmd); ++ if (result != SCAN_SUCCEED) + goto out_up_write; + + anon_vma_lock_write(vma->anon_vma); +@@ -1093,11 +1081,11 @@ static void collapse_huge_page(struct mm_struct *mm, + mmu_notifier_invalidate_range_end(&range); + + spin_lock(pte_ptl); +- isolated = __collapse_huge_page_isolate(vma, address, pte, +- &compound_pagelist); ++ result = __collapse_huge_page_isolate(vma, address, pte, cc, ++ &compound_pagelist); + spin_unlock(pte_ptl); + +- if (unlikely(!isolated)) { ++ if (unlikely(result != SCAN_SUCCEED)) { + pte_unmap(pte); + spin_lock(pmd_ptl); + BUG_ON(!pmd_none(*pmd)); +@@ -1109,7 +1097,6 @@ static void collapse_huge_page(struct mm_struct *mm, + pmd_populate(mm, pmd, pmd_pgtable(_pmd)); + spin_unlock(pmd_ptl); + anon_vma_unlock_write(vma->anon_vma); +- result = SCAN_FAIL; + goto out_up_write; + } + +@@ -1119,8 +1106,8 @@ static void collapse_huge_page(struct mm_struct *mm, + */ + anon_vma_unlock_write(vma->anon_vma); + +- __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, +- &compound_pagelist); ++ __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, ++ &compound_pagelist); + pte_unmap(pte); + /* + * spin_lock() below is not the equivalent of smp_wmb(), but +@@ -1128,42 +1115,43 @@ static void collapse_huge_page(struct mm_struct *mm, + * avoid the copy_huge_page writes to become visible after + * the set_pmd_at() write. + */ +- __SetPageUptodate(new_page); ++ __SetPageUptodate(hpage); + pgtable = pmd_pgtable(_pmd); + +- _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); ++ _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); + _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); + + spin_lock(pmd_ptl); + BUG_ON(!pmd_none(*pmd)); +- page_add_new_anon_rmap(new_page, vma, address); +- lru_cache_add_inactive_or_unevictable(new_page, vma); ++ page_add_new_anon_rmap(hpage, vma, address); ++ lru_cache_add_inactive_or_unevictable(hpage, vma); + pgtable_trans_huge_deposit(mm, pmd, pgtable); + set_pmd_at(mm, address, pmd, _pmd); + update_mmu_cache_pmd(vma, address, pmd); + spin_unlock(pmd_ptl); + +- *hpage = NULL; ++ hpage = NULL; + +- khugepaged_pages_collapsed++; + result = SCAN_SUCCEED; + out_up_write: + mmap_write_unlock(mm); + out_nolock: +- if (!IS_ERR_OR_NULL(*hpage)) +- mem_cgroup_uncharge(page_folio(*hpage)); +- trace_mm_collapse_huge_page(mm, isolated, result); +- return; ++ if (hpage) { ++ mem_cgroup_uncharge(page_folio(hpage)); ++ put_page(hpage); ++ } ++ trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); ++ return result; + } + +-static int khugepaged_scan_pmd(struct mm_struct *mm, +- struct vm_area_struct *vma, +- unsigned long address, +- struct page **hpage) ++static int hpage_collapse_scan_pmd(struct mm_struct *mm, ++ struct vm_area_struct *vma, ++ unsigned long address, bool *mmap_locked, ++ struct collapse_control *cc) + { + pmd_t *pmd; + pte_t *pte, *_pte; +- int ret = 0, result = 0, referenced = 0; ++ int result = SCAN_FAIL, referenced = 0; + int none_or_zero = 0, shared = 0; + struct page *page = NULL; + unsigned long _address; +@@ -1173,19 +1161,19 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + +- pmd = mm_find_pmd(mm, address); +- if (!pmd) { +- result = SCAN_PMD_NULL; ++ result = find_pmd_or_thp_or_none(mm, address, &pmd); ++ if (result != SCAN_SUCCEED) + goto out; +- } + +- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); ++ memset(cc->node_load, 0, sizeof(cc->node_load)); + pte = pte_offset_map_lock(mm, pmd, address, &ptl); + for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; + _pte++, _address += PAGE_SIZE) { + pte_t pteval = *_pte; + if (is_swap_pte(pteval)) { +- if (++unmapped <= khugepaged_max_ptes_swap) { ++ ++unmapped; ++ if (!cc->is_khugepaged || ++ unmapped <= khugepaged_max_ptes_swap) { + /* + * Always be strict with uffd-wp + * enabled swap entries. Please see +@@ -1203,8 +1191,10 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + } + } + if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { ++ ++none_or_zero; + if (!userfaultfd_armed(vma) && +- ++none_or_zero <= khugepaged_max_ptes_none) { ++ (!cc->is_khugepaged || ++ none_or_zero <= khugepaged_max_ptes_none)) { + continue; + } else { + result = SCAN_EXCEED_NONE_PTE; +@@ -1234,27 +1224,30 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + goto out_unmap; + } + +- if (page_mapcount(page) > 1 && +- ++shared > khugepaged_max_ptes_shared) { +- result = SCAN_EXCEED_SHARED_PTE; +- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); +- goto out_unmap; ++ if (page_mapcount(page) > 1) { ++ ++shared; ++ if (cc->is_khugepaged && ++ shared > khugepaged_max_ptes_shared) { ++ result = SCAN_EXCEED_SHARED_PTE; ++ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); ++ goto out_unmap; ++ } + } + + page = compound_head(page); + + /* + * Record which node the original page is from and save this +- * information to khugepaged_node_load[]. ++ * information to cc->node_load[]. + * Khugepaged will allocate hugepage from the node has the max + * hit record. + */ + node = page_to_nid(page); +- if (khugepaged_scan_abort(node)) { ++ if (hpage_collapse_scan_abort(node, cc)) { + result = SCAN_SCAN_ABORT; + goto out_unmap; + } +- khugepaged_node_load[node]++; ++ cc->node_load[node]++; + if (!PageLRU(page)) { + result = SCAN_PAGE_LRU; + goto out_unmap; +@@ -1289,31 +1282,38 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + result = SCAN_PAGE_COUNT; + goto out_unmap; + } +- if (pte_young(pteval) || +- page_is_young(page) || PageReferenced(page) || +- mmu_notifier_test_young(vma->vm_mm, address)) ++ ++ /* ++ * If collapse was initiated by khugepaged, check that there is ++ * enough young pte to justify collapsing the page ++ */ ++ if (cc->is_khugepaged && ++ (pte_young(pteval) || page_is_young(page) || ++ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, ++ address))) + referenced++; + } + if (!writable) { + result = SCAN_PAGE_RO; +- } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { ++ } else if (cc->is_khugepaged && ++ (!referenced || ++ (unmapped && referenced < HPAGE_PMD_NR / 2))) { + result = SCAN_LACK_REFERENCED_PAGE; + } else { + result = SCAN_SUCCEED; +- ret = 1; + } + out_unmap: + pte_unmap_unlock(pte, ptl); +- if (ret) { +- node = khugepaged_find_target_node(); ++ if (result == SCAN_SUCCEED) { ++ result = collapse_huge_page(mm, address, referenced, ++ unmapped, cc); + /* collapse_huge_page will return with the mmap_lock released */ +- collapse_huge_page(mm, address, hpage, node, +- referenced, unmapped); ++ *mmap_locked = false; + } + out: + trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, + none_or_zero, result, unmapped); +- return ret; ++ return result; + } + + static void collect_mm_slot(struct mm_slot *mm_slot) +@@ -1322,7 +1322,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) + + lockdep_assert_held(&khugepaged_mm_lock); + +- if (khugepaged_test_exit(mm)) { ++ if (hpage_collapse_test_exit(mm)) { + /* free mm_slot */ + hash_del(&mm_slot->hash); + list_del(&mm_slot->mm_node); +@@ -1400,12 +1400,13 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) + return; + + /* +- * This vm_flags may not have VM_HUGEPAGE if the page was not +- * collapsed by this mm. But we can still collapse if the page is +- * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() +- * will not fail the vma for missing VM_HUGEPAGE ++ * If we are here, we've succeeded in replacing all the native pages ++ * in the page cache with a single hugepage. If a mm were to fault-in ++ * this memory (mapped by a suitably aligned VMA), we'd get the hugepage ++ * and map it by a PMD, regardless of sysfs THP settings. As such, let's ++ * analogously elide sysfs THP settings here. + */ +- if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false)) ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) + return; + + /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ +@@ -1420,8 +1421,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) + if (!PageHead(hpage)) + goto drop_hpage; + +- pmd = mm_find_pmd(mm, haddr); +- if (!pmd) ++ if (find_pmd_or_thp_or_none(mm, haddr, &pmd) != SCAN_SUCCEED) + goto drop_hpage; + + start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); +@@ -1495,7 +1495,7 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) + if (!mmap_write_trylock(mm)) + return; + +- if (unlikely(khugepaged_test_exit(mm))) ++ if (unlikely(hpage_collapse_test_exit(mm))) + goto out; + + for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) +@@ -1539,8 +1539,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + if (vma->vm_end < addr + HPAGE_PMD_SIZE) + continue; + mm = vma->vm_mm; +- pmd = mm_find_pmd(mm, addr); +- if (!pmd) ++ if (find_pmd_or_thp_or_none(mm, addr, &pmd) != SCAN_SUCCEED) + continue; + /* + * We need exclusive mmap_lock to retract page table. +@@ -1558,7 +1557,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + * it'll always mapped in small page size for uffd-wp + * registered ranges. + */ +- if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) ++ if (!hpage_collapse_test_exit(mm) && ++ !userfaultfd_wp(vma)) + collapse_and_free_pmd(mm, vma, addr, pmd); + mmap_write_unlock(mm); + } else { +@@ -1575,8 +1575,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + * @mm: process address space where collapse happens + * @file: file that collapse on + * @start: collapse start address +- * @hpage: new allocated huge page for collapse +- * @node: appointed node the new huge page allocate from ++ * @cc: collapse context and scratchpad + * + * Basic scheme is simple, details are more complex: + * - allocate and lock a new huge page; +@@ -1593,13 +1592,11 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + * + restore gaps in the page cache; + * + unlock and free huge page; + */ +-static void collapse_file(struct mm_struct *mm, +- struct file *file, pgoff_t start, +- struct page **hpage, int node) ++static int collapse_file(struct mm_struct *mm, struct file *file, ++ pgoff_t start, struct collapse_control *cc) + { + struct address_space *mapping = file->f_mapping; +- gfp_t gfp; +- struct page *new_page; ++ struct page *hpage; + pgoff_t index, end = start + HPAGE_PMD_NR; + LIST_HEAD(pagelist); + XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); +@@ -1610,20 +1607,9 @@ static void collapse_file(struct mm_struct *mm, + VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); + VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + +- /* Only allocate from the target node */ +- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; +- +- new_page = khugepaged_alloc_page(hpage, gfp, node); +- if (!new_page) { +- result = SCAN_ALLOC_HUGE_PAGE_FAIL; +- goto out; +- } +- +- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { +- result = SCAN_CGROUP_CHARGE_FAIL; ++ result = alloc_charge_hpage(&hpage, mm, cc); ++ if (result != SCAN_SUCCEED) + goto out; +- } +- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); + + /* + * Ensure we have slots for all the pages in the range. This is +@@ -1641,14 +1627,14 @@ static void collapse_file(struct mm_struct *mm, + } + } while (1); + +- __SetPageLocked(new_page); ++ __SetPageLocked(hpage); + if (is_shmem) +- __SetPageSwapBacked(new_page); +- new_page->index = start; +- new_page->mapping = mapping; ++ __SetPageSwapBacked(hpage); ++ hpage->index = start; ++ hpage->mapping = mapping; + + /* +- * At this point the new_page is locked and not up-to-date. ++ * At this point the hpage is locked and not up-to-date. + * It's safe to insert it into the page cache, because nobody would + * be able to map it or use it in another way until we unlock it. + */ +@@ -1676,7 +1662,7 @@ static void collapse_file(struct mm_struct *mm, + result = SCAN_FAIL; + goto xa_locked; + } +- xas_store(&xas, new_page); ++ xas_store(&xas, hpage); + nr_none++; + continue; + } +@@ -1818,19 +1804,19 @@ static void collapse_file(struct mm_struct *mm, + list_add_tail(&page->lru, &pagelist); + + /* Finally, replace with the new page. */ +- xas_store(&xas, new_page); ++ xas_store(&xas, hpage); + continue; + out_unlock: + unlock_page(page); + put_page(page); + goto xa_unlocked; + } +- nr = thp_nr_pages(new_page); ++ nr = thp_nr_pages(hpage); + + if (is_shmem) +- __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); ++ __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); + else { +- __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); ++ __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); + filemap_nr_thps_inc(mapping); + /* + * Paired with smp_mb() in do_dentry_open() to ensure +@@ -1841,21 +1827,21 @@ static void collapse_file(struct mm_struct *mm, + smp_mb(); + if (inode_is_open_for_write(mapping->host)) { + result = SCAN_FAIL; +- __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); ++ __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); + filemap_nr_thps_dec(mapping); + goto xa_locked; + } + } + + if (nr_none) { +- __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); ++ __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); + /* nr_none is always 0 for non-shmem. */ +- __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); ++ __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); + } + + /* Join all the small entries into a single multi-index entry */ + xas_set_order(&xas, start, HPAGE_PMD_ORDER); +- xas_store(&xas, new_page); ++ xas_store(&xas, hpage); + xa_locked: + xas_unlock_irq(&xas); + xa_unlocked: +@@ -1877,11 +1863,11 @@ static void collapse_file(struct mm_struct *mm, + index = start; + list_for_each_entry_safe(page, tmp, &pagelist, lru) { + while (index < page->index) { +- clear_highpage(new_page + (index % HPAGE_PMD_NR)); ++ clear_highpage(hpage + (index % HPAGE_PMD_NR)); + index++; + } +- copy_highpage(new_page + (page->index % HPAGE_PMD_NR), +- page); ++ copy_highpage(hpage + (page->index % HPAGE_PMD_NR), ++ page); + list_del(&page->lru); + page->mapping = NULL; + page_ref_unfreeze(page, 1); +@@ -1892,23 +1878,22 @@ static void collapse_file(struct mm_struct *mm, + index++; + } + while (index < end) { +- clear_highpage(new_page + (index % HPAGE_PMD_NR)); ++ clear_highpage(hpage + (index % HPAGE_PMD_NR)); + index++; + } + +- SetPageUptodate(new_page); +- page_ref_add(new_page, HPAGE_PMD_NR - 1); ++ SetPageUptodate(hpage); ++ page_ref_add(hpage, HPAGE_PMD_NR - 1); + if (is_shmem) +- set_page_dirty(new_page); +- lru_cache_add(new_page); ++ set_page_dirty(hpage); ++ lru_cache_add(hpage); + + /* + * Remove pte page tables, so we can re-fault the page as huge. + */ + retract_page_tables(mapping, start); +- *hpage = NULL; +- +- khugepaged_pages_collapsed++; ++ unlock_page(hpage); ++ hpage = NULL; + } else { + struct page *page; + +@@ -1947,19 +1932,23 @@ static void collapse_file(struct mm_struct *mm, + VM_BUG_ON(nr_none); + xas_unlock_irq(&xas); + +- new_page->mapping = NULL; ++ hpage->mapping = NULL; + } + +- unlock_page(new_page); ++ if (hpage) ++ unlock_page(hpage); + out: + VM_BUG_ON(!list_empty(&pagelist)); +- if (!IS_ERR_OR_NULL(*hpage)) +- mem_cgroup_uncharge(page_folio(*hpage)); ++ if (hpage) { ++ mem_cgroup_uncharge(page_folio(hpage)); ++ put_page(hpage); ++ } + /* TODO: tracepoints */ ++ return result; + } + +-static void khugepaged_scan_file(struct mm_struct *mm, +- struct file *file, pgoff_t start, struct page **hpage) ++static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, ++ pgoff_t start, struct collapse_control *cc) + { + struct page *page = NULL; + struct address_space *mapping = file->f_mapping; +@@ -1970,14 +1959,16 @@ static void khugepaged_scan_file(struct mm_struct *mm, + + present = 0; + swap = 0; +- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); ++ memset(cc->node_load, 0, sizeof(cc->node_load)); + rcu_read_lock(); + xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { + if (xas_retry(&xas, page)) + continue; + + if (xa_is_value(page)) { +- if (++swap > khugepaged_max_ptes_swap) { ++ ++swap; ++ if (cc->is_khugepaged && ++ swap > khugepaged_max_ptes_swap) { + result = SCAN_EXCEED_SWAP_PTE; + count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); + break; +@@ -1995,11 +1986,11 @@ static void khugepaged_scan_file(struct mm_struct *mm, + } + + node = page_to_nid(page); +- if (khugepaged_scan_abort(node)) { ++ if (hpage_collapse_scan_abort(node, cc)) { + result = SCAN_SCAN_ABORT; + break; + } +- khugepaged_node_load[node]++; ++ cc->node_load[node]++; + + if (!PageLRU(page)) { + result = SCAN_PAGE_LRU; +@@ -2028,20 +2019,21 @@ static void khugepaged_scan_file(struct mm_struct *mm, + rcu_read_unlock(); + + if (result == SCAN_SUCCEED) { +- if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { ++ if (cc->is_khugepaged && ++ present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { + result = SCAN_EXCEED_NONE_PTE; + count_vm_event(THP_SCAN_EXCEED_NONE_PTE); + } else { +- node = khugepaged_find_target_node(); +- collapse_file(mm, file, start, hpage, node); ++ result = collapse_file(mm, file, start, cc); + } + } + + /* TODO: tracepoints */ ++ return result; + } + #else +-static void khugepaged_scan_file(struct mm_struct *mm, +- struct file *file, pgoff_t start, struct page **hpage) ++static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, ++ pgoff_t start, struct collapse_control *cc) + { + BUILD_BUG(); + } +@@ -2051,8 +2043,8 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) + } + #endif + +-static unsigned int khugepaged_scan_mm_slot(unsigned int pages, +- struct page **hpage) ++static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, ++ struct collapse_control *cc) + __releases(&khugepaged_mm_lock) + __acquires(&khugepaged_mm_lock) + { +@@ -2063,6 +2055,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + + VM_BUG_ON(!pages); + lockdep_assert_held(&khugepaged_mm_lock); ++ *result = SCAN_FAIL; + + if (khugepaged_scan.mm_slot) + mm_slot = khugepaged_scan.mm_slot; +@@ -2083,7 +2076,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + vma = NULL; + if (unlikely(!mmap_read_trylock(mm))) + goto breakouterloop_mmap_lock; +- if (likely(!khugepaged_test_exit(mm))) ++ if (likely(!hpage_collapse_test_exit(mm))) + vma = find_vma(mm, khugepaged_scan.address); + + progress++; +@@ -2091,11 +2084,11 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + unsigned long hstart, hend; + + cond_resched(); +- if (unlikely(khugepaged_test_exit(mm))) { ++ if (unlikely(hpage_collapse_test_exit(mm))) { + progress++; + break; + } +- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) { ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { + skip: + progress++; + continue; +@@ -2109,9 +2102,10 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); + + while (khugepaged_scan.address < hend) { +- int ret; ++ bool mmap_locked = true; ++ + cond_resched(); +- if (unlikely(khugepaged_test_exit(mm))) ++ if (unlikely(hpage_collapse_test_exit(mm))) + goto breakouterloop; + + VM_BUG_ON(khugepaged_scan.address < hstart || +@@ -2123,19 +2117,29 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + khugepaged_scan.address); + + mmap_read_unlock(mm); +- ret = 1; +- khugepaged_scan_file(mm, file, pgoff, hpage); ++ *result = khugepaged_scan_file(mm, file, pgoff, ++ cc); ++ mmap_locked = false; + fput(file); + } else { +- ret = khugepaged_scan_pmd(mm, vma, +- khugepaged_scan.address, +- hpage); ++ *result = hpage_collapse_scan_pmd(mm, vma, ++ khugepaged_scan.address, ++ &mmap_locked, ++ cc); + } ++ if (*result == SCAN_SUCCEED) ++ ++khugepaged_pages_collapsed; + /* move to next address */ + khugepaged_scan.address += HPAGE_PMD_SIZE; + progress += HPAGE_PMD_NR; +- if (ret) +- /* we released mmap_lock so break loop */ ++ if (!mmap_locked) ++ /* ++ * We released mmap_lock so break loop. Note ++ * that we drop mmap_lock before all hugepage ++ * allocations, so if allocation fails, we are ++ * guaranteed to break here and report the ++ * correct result back to caller. ++ */ + goto breakouterloop_mmap_lock; + if (progress >= pages) + goto breakouterloop; +@@ -2151,7 +2155,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + * Release the current mm_slot if this mm is about to die, or + * if we scanned all vmas of this mm. + */ +- if (khugepaged_test_exit(mm) || !vma) { ++ if (hpage_collapse_test_exit(mm) || !vma) { + /* + * Make sure that if mm_users is reaching zero while + * khugepaged runs here, khugepaged_exit will find +@@ -2185,19 +2189,16 @@ static int khugepaged_wait_event(void) + kthread_should_stop(); + } + +-static void khugepaged_do_scan(void) ++static void khugepaged_do_scan(struct collapse_control *cc) + { +- struct page *hpage = NULL; + unsigned int progress = 0, pass_through_head = 0; + unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); + bool wait = true; ++ int result = SCAN_SUCCEED; + + lru_add_drain_all(); + +- while (progress < pages) { +- if (!khugepaged_prealloc_page(&hpage, &wait)) +- break; +- ++ while (true) { + cond_resched(); + + if (unlikely(kthread_should_stop() || try_to_freeze())) +@@ -2209,14 +2210,25 @@ static void khugepaged_do_scan(void) + if (khugepaged_has_work() && + pass_through_head < 2) + progress += khugepaged_scan_mm_slot(pages - progress, +- &hpage); ++ &result, cc); + else + progress = pages; + spin_unlock(&khugepaged_mm_lock); +- } + +- if (!IS_ERR_OR_NULL(hpage)) +- put_page(hpage); ++ if (progress >= pages) ++ break; ++ ++ if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { ++ /* ++ * If fail to allocate the first time, try to sleep for ++ * a while. When hit again, cancel the scan. ++ */ ++ if (!wait) ++ break; ++ wait = false; ++ khugepaged_alloc_sleep(); ++ } ++ } + } + + static bool khugepaged_should_wakeup(void) +@@ -2253,7 +2265,7 @@ static int khugepaged(void *none) + set_user_nice(current, MAX_NICE); + + while (!kthread_should_stop()) { +- khugepaged_do_scan(); ++ khugepaged_do_scan(&khugepaged_collapse_control); + khugepaged_wait_work(); + } + +@@ -2352,3 +2364,120 @@ void khugepaged_min_free_kbytes_update(void) + set_recommended_min_free_kbytes(); + mutex_unlock(&khugepaged_mutex); + } ++ ++static int madvise_collapse_errno(enum scan_result r) ++{ ++ /* ++ * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide ++ * actionable feedback to caller, so they may take an appropriate ++ * fallback measure depending on the nature of the failure. ++ */ ++ switch (r) { ++ case SCAN_ALLOC_HUGE_PAGE_FAIL: ++ return -ENOMEM; ++ case SCAN_CGROUP_CHARGE_FAIL: ++ return -EBUSY; ++ /* Resource temporary unavailable - trying again might succeed */ ++ case SCAN_PAGE_LOCK: ++ case SCAN_PAGE_LRU: ++ return -EAGAIN; ++ /* ++ * Other: Trying again likely not to succeed / error intrinsic to ++ * specified memory range. khugepaged likely won't be able to collapse ++ * either. ++ */ ++ default: ++ return -EINVAL; ++ } ++} ++ ++int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, ++ unsigned long start, unsigned long end) ++{ ++ struct collapse_control *cc; ++ struct mm_struct *mm = vma->vm_mm; ++ unsigned long hstart, hend, addr; ++ int thps = 0, last_fail = SCAN_FAIL; ++ bool mmap_locked = true; ++ ++ BUG_ON(vma->vm_start > start); ++ BUG_ON(vma->vm_end < end); ++ ++ *prev = vma; ++ ++ /* TODO: Support file/shmem */ ++ if (!vma->anon_vma || !vma_is_anonymous(vma)) ++ return -EINVAL; ++ ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) ++ return -EINVAL; ++ ++ cc = kmalloc(sizeof(*cc), GFP_KERNEL); ++ if (!cc) ++ return -ENOMEM; ++ cc->is_khugepaged = false; ++ cc->last_target_node = NUMA_NO_NODE; ++ ++ mmgrab(mm); ++ lru_add_drain_all(); ++ ++ hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; ++ hend = end & HPAGE_PMD_MASK; ++ ++ for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { ++ int result = SCAN_FAIL; ++ ++ if (!mmap_locked) { ++ cond_resched(); ++ mmap_read_lock(mm); ++ mmap_locked = true; ++ result = hugepage_vma_revalidate(mm, addr, &vma, cc); ++ if (result != SCAN_SUCCEED) { ++ last_fail = result; ++ goto out_nolock; ++ } ++ } ++ mmap_assert_locked(mm); ++ memset(cc->node_load, 0, sizeof(cc->node_load)); ++ result = hpage_collapse_scan_pmd(mm, vma, addr, &mmap_locked, ++ cc); ++ if (!mmap_locked) ++ *prev = NULL; /* Tell caller we dropped mmap_lock */ ++ ++ switch (result) { ++ case SCAN_SUCCEED: ++ case SCAN_PMD_MAPPED: ++ ++thps; ++ break; ++ /* Whitelisted set of results where continuing OK */ ++ case SCAN_PMD_NULL: ++ case SCAN_PTE_NON_PRESENT: ++ case SCAN_PTE_UFFD_WP: ++ case SCAN_PAGE_RO: ++ case SCAN_LACK_REFERENCED_PAGE: ++ case SCAN_PAGE_NULL: ++ case SCAN_PAGE_COUNT: ++ case SCAN_PAGE_LOCK: ++ case SCAN_PAGE_COMPOUND: ++ case SCAN_PAGE_LRU: ++ last_fail = result; ++ break; ++ default: ++ last_fail = result; ++ /* Other error, exit */ ++ goto out_maybelock; ++ } ++ } ++ ++out_maybelock: ++ /* Caller expects us to hold mmap_lock on return */ ++ if (!mmap_locked) ++ mmap_read_lock(mm); ++out_nolock: ++ mmap_assert_locked(mm); ++ mmdrop(mm); ++ kfree(cc); ++ ++ return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 ++ : madvise_collapse_errno(last_fail); ++} +diff --git a/mm/ksm.c b/mm/ksm.c +index 4e64adb9adee..dde27d7e92d4 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -1134,6 +1134,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, + { + struct mm_struct *mm = vma->vm_mm; + pmd_t *pmd; ++ pmd_t pmde; + pte_t *ptep; + pte_t newpte; + spinlock_t *ptl; +@@ -1148,6 +1149,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, + pmd = mm_find_pmd(mm, addr); + if (!pmd) + goto out; ++ /* ++ * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() ++ * without holding anon_vma lock for write. So when looking for a ++ * genuine pmde (in which to find pte), test present and !THP together. ++ */ ++ pmde = *pmd; ++ barrier(); ++ if (!pmd_present(pmde) || pmd_trans_huge(pmde)) ++ goto out; + + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, + addr + PAGE_SIZE); +diff --git a/mm/madvise.c b/mm/madvise.c +index 5f0f0948a50e..af97100a0727 100644 +--- a/mm/madvise.c ++++ b/mm/madvise.c +@@ -59,6 +59,7 @@ static int madvise_need_mmap_write(int behavior) + case MADV_FREE: + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: ++ case MADV_COLLAPSE: + return 0; + default: + /* be safe, default to 1. list exceptions explicitly */ +@@ -1057,6 +1058,8 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, + if (error) + goto out; + break; ++ case MADV_COLLAPSE: ++ return madvise_collapse(vma, prev, start, end); + } + + anon_name = anon_vma_name(vma); +@@ -1150,6 +1153,7 @@ madvise_behavior_valid(int behavior) + #ifdef CONFIG_TRANSPARENT_HUGEPAGE + case MADV_HUGEPAGE: + case MADV_NOHUGEPAGE: ++ case MADV_COLLAPSE: + #endif + case MADV_DONTDUMP: + case MADV_DODUMP: +@@ -1166,13 +1170,13 @@ madvise_behavior_valid(int behavior) + } + } + +-static bool +-process_madvise_behavior_valid(int behavior) ++static bool process_madvise_behavior_valid(int behavior) + { + switch (behavior) { + case MADV_COLD: + case MADV_PAGEOUT: + case MADV_WILLNEED: ++ case MADV_COLLAPSE: + return true; + default: + return false; +@@ -1339,6 +1343,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, + * MADV_NOHUGEPAGE - mark the given range as not worth being backed by + * transparent huge pages so the existing pages will not be + * coalesced into THP and new pages will not be allocated as THP. ++ * MADV_COLLAPSE - synchronously coalesce pages into new THP. + * MADV_DONTDUMP - the application wants to prevent pages in the given range + * from being included in its core dump. + * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. +diff --git a/mm/memory.c b/mm/memory.c +index 3a3d8721bf4c..e58d5d522467 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -4986,7 +4986,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, + return VM_FAULT_OOM; + retry_pud: + if (pud_none(*vmf.pud) && +- hugepage_vma_check(vma, vm_flags, false, true)) { ++ hugepage_vma_check(vma, vm_flags, false, true, true)) { + ret = create_huge_pud(&vmf); + if (!(ret & VM_FAULT_FALLBACK)) + return ret; +@@ -5020,7 +5020,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, + goto retry_pud; + + if (pmd_none(*vmf.pmd) && +- hugepage_vma_check(vma, vm_flags, false, true)) { ++ hugepage_vma_check(vma, vm_flags, false, true, true)) { + ret = create_huge_pmd(&vmf); + if (!(ret & VM_FAULT_FALLBACK)) + return ret; +diff --git a/mm/rmap.c b/mm/rmap.c +index 93d5a6f793d2..9af08343ce55 100644 +--- a/mm/rmap.c ++++ b/mm/rmap.c +@@ -770,13 +770,17 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) + return vma_address(page, vma); + } + ++/* ++ * Returns the actual pmd_t* where we expect 'address' to be mapped from, or ++ * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* ++ * represents. ++ */ + pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) + { + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; +- pmd_t pmde; + + pgd = pgd_offset(mm, address); + if (!pgd_present(*pgd)) +@@ -791,15 +795,6 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) + goto out; + + pmd = pmd_offset(pud, address); +- /* +- * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() +- * without holding anon_vma lock for write. So when looking for a +- * genuine pmde (in which to find pte), test present and !THP together. +- */ +- pmde = *pmd; +- barrier(); +- if (!pmd_present(pmde) || pmd_trans_huge(pmde)) +- pmd = NULL; + out: + return pmd; + } +diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h +index 6c1aa92a92e4..6ce1f1ceb432 100644 +--- a/tools/include/uapi/asm-generic/mman-common.h ++++ b/tools/include/uapi/asm-generic/mman-common.h +@@ -77,6 +77,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/tools/testing/selftests/vm/khugepaged.c b/tools/testing/selftests/vm/khugepaged.c +index 155120b67a16..b77b1e28cdb3 100644 +--- a/tools/testing/selftests/vm/khugepaged.c ++++ b/tools/testing/selftests/vm/khugepaged.c +@@ -14,6 +14,9 @@ + #ifndef MADV_PAGEOUT + #define MADV_PAGEOUT 21 + #endif ++#ifndef MADV_COLLAPSE ++#define MADV_COLLAPSE 25 ++#endif + + #define BASE_ADDR ((void *)(1UL << 30)) + static unsigned long hpage_pmd_size; +@@ -23,6 +26,11 @@ static int hpage_pmd_nr; + #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" + #define PID_SMAPS "/proc/self/smaps" + ++struct collapse_context { ++ void (*collapse)(const char *msg, char *p, int nr_hpages, bool expect); ++ bool enforce_pte_scan_limits; ++}; ++ + enum thp_enabled { + THP_ALWAYS, + THP_MADVISE, +@@ -90,18 +98,6 @@ struct settings { + struct khugepaged_settings khugepaged; + }; + +-static struct settings default_settings = { +- .thp_enabled = THP_MADVISE, +- .thp_defrag = THP_DEFRAG_ALWAYS, +- .shmem_enabled = SHMEM_NEVER, +- .use_zero_page = 0, +- .khugepaged = { +- .defrag = 1, +- .alloc_sleep_millisecs = 10, +- .scan_sleep_millisecs = 10, +- }, +-}; +- + static struct settings saved_settings; + static bool skip_settings_restore; + +@@ -279,6 +275,39 @@ static void write_settings(struct settings *settings) + write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); + } + ++#define MAX_SETTINGS_DEPTH 4 ++static struct settings settings_stack[MAX_SETTINGS_DEPTH]; ++static int settings_index; ++ ++static struct settings *current_settings(void) ++{ ++ if (!settings_index) { ++ printf("Fail: No settings set"); ++ exit(EXIT_FAILURE); ++ } ++ return settings_stack + settings_index - 1; ++} ++ ++static void push_settings(struct settings *settings) ++{ ++ if (settings_index >= MAX_SETTINGS_DEPTH) { ++ printf("Fail: Settings stack exceeded"); ++ exit(EXIT_FAILURE); ++ } ++ settings_stack[settings_index++] = *settings; ++ write_settings(current_settings()); ++} ++ ++static void pop_settings(void) ++{ ++ if (settings_index <= 0) { ++ printf("Fail: Settings stack empty"); ++ exit(EXIT_FAILURE); ++ } ++ --settings_index; ++ write_settings(current_settings()); ++} ++ + static void restore_settings(int sig) + { + if (skip_settings_restore) +@@ -322,14 +351,6 @@ static void save_settings(void) + signal(SIGQUIT, restore_settings); + } + +-static void adjust_settings(void) +-{ +- +- printf("Adjust settings..."); +- write_settings(&default_settings); +- success("OK"); +-} +- + #define MAX_LINE_LENGTH 500 + + static bool check_for_pattern(FILE *fp, char *pattern, char *buf) +@@ -341,7 +362,7 @@ static bool check_for_pattern(FILE *fp, char *pattern, char *buf) + return false; + } + +-static bool check_huge(void *addr) ++static bool check_huge(void *addr, int nr_hpages) + { + bool thp = false; + int ret; +@@ -366,7 +387,7 @@ static bool check_huge(void *addr) + goto err_out; + + ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB", +- hpage_pmd_size >> 10); ++ nr_hpages * (hpage_pmd_size >> 10)); + if (ret >= MAX_LINE_LENGTH) { + printf("%s: Pattern is too long\n", __func__); + exit(EXIT_FAILURE); +@@ -434,12 +455,12 @@ static bool check_swap(void *addr, unsigned long size) + return swap; + } + +-static void *alloc_mapping(void) ++static void *alloc_mapping(int nr) + { + void *p; + +- p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, +- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); ++ p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, ++ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (p != BASE_ADDR) { + printf("Failed to allocate VMA at %p\n", BASE_ADDR); + exit(EXIT_FAILURE); +@@ -456,6 +477,25 @@ static void fill_memory(int *p, unsigned long start, unsigned long end) + p[i * page_size / sizeof(*p)] = i + 0xdead0000; + } + ++/* ++ * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with ++ * validate_memory()'able contents. ++ */ ++static void *alloc_hpage(void) ++{ ++ void *p; ++ ++ p = alloc_mapping(1); ++ printf("Allocate huge page..."); ++ madvise(p, hpage_pmd_size, MADV_HUGEPAGE); ++ fill_memory(p, 0, hpage_pmd_size); ++ if (check_huge(p, 1)) ++ success("OK"); ++ else ++ fail("Fail"); ++ return p; ++} ++ + static void validate_memory(int *p, unsigned long start, unsigned long end) + { + int i; +@@ -469,26 +509,59 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) + } + } + ++static void madvise_collapse(const char *msg, char *p, int nr_hpages, ++ bool expect) ++{ ++ int ret; ++ struct settings settings = *current_settings(); ++ ++ printf("%s...", msg); ++ /* Sanity check */ ++ if (!check_huge(p, 0)) { ++ printf("Unexpected huge page\n"); ++ exit(EXIT_FAILURE); ++ } ++ ++ /* ++ * Prevent khugepaged interference and tests that MADV_COLLAPSE ++ * ignores /sys/kernel/mm/transparent_hugepage/enabled ++ */ ++ settings.thp_enabled = THP_NEVER; ++ push_settings(&settings); ++ ++ /* Clear VM_NOHUGEPAGE */ ++ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); ++ ret = madvise(p, nr_hpages * hpage_pmd_size, MADV_COLLAPSE); ++ if (((bool)ret) == expect) ++ fail("Fail: Bad return value"); ++ else if (check_huge(p, nr_hpages) != expect) ++ fail("Fail: check_huge()"); ++ else ++ success("OK"); ++ ++ pop_settings(); ++} ++ + #define TICK 500000 +-static bool wait_for_scan(const char *msg, char *p) ++static bool wait_for_scan(const char *msg, char *p, int nr_hpages) + { + int full_scans; + int timeout = 6; /* 3 seconds */ + + /* Sanity check */ +- if (check_huge(p)) { ++ if (!check_huge(p, 0)) { + printf("Unexpected huge page\n"); + exit(EXIT_FAILURE); + } + +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); ++ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); + + /* Wait until the second full_scan completed */ + full_scans = read_num("khugepaged/full_scans") + 2; + + printf("%s...", msg); + while (timeout--) { +- if (check_huge(p)) ++ if (check_huge(p, nr_hpages)) + break; + if (read_num("khugepaged/full_scans") >= full_scans) + break; +@@ -496,121 +569,121 @@ static bool wait_for_scan(const char *msg, char *p) + usleep(TICK); + } + +- madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); ++ madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); + + return timeout == -1; + } + ++static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, ++ bool expect) ++{ ++ if (wait_for_scan(msg, p, nr_hpages)) { ++ if (expect) ++ fail("Timeout"); ++ else ++ success("OK"); ++ return; ++ } else if (check_huge(p, nr_hpages) == expect) { ++ success("OK"); ++ } else { ++ fail("Fail"); ++ } ++} ++ + static void alloc_at_fault(void) + { +- struct settings settings = default_settings; ++ struct settings settings = *current_settings(); + char *p; + + settings.thp_enabled = THP_ALWAYS; +- write_settings(&settings); ++ push_settings(&settings); + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + *p = 1; + printf("Allocate huge page on fault..."); +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); + +- write_settings(&default_settings); ++ pop_settings(); + + madvise(p, page_size, MADV_DONTNEED); + printf("Split huge PMD on MADV_DONTNEED..."); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + munmap(p, hpage_pmd_size); + } + +-static void collapse_full(void) ++static void collapse_full(struct collapse_context *c) + { + void *p; +- +- p = alloc_mapping(); +- fill_memory(p, 0, hpage_pmd_size); +- if (wait_for_scan("Collapse fully populated PTE table", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- validate_memory(p, 0, hpage_pmd_size); +- munmap(p, hpage_pmd_size); ++ int nr_hpages = 4; ++ unsigned long size = nr_hpages * hpage_pmd_size; ++ ++ p = alloc_mapping(nr_hpages); ++ fill_memory(p, 0, size); ++ c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, ++ true); ++ validate_memory(p, 0, size); ++ munmap(p, size); + } + +-static void collapse_empty(void) ++static void collapse_empty(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); +- if (wait_for_scan("Do not collapse empty PTE table", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- fail("Fail"); +- else +- success("OK"); ++ p = alloc_mapping(1); ++ c->collapse("Do not collapse empty PTE table", p, 1, false); + munmap(p, hpage_pmd_size); + } + +-static void collapse_single_pte_entry(void) ++static void collapse_single_pte_entry(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + fill_memory(p, 0, page_size); +- if (wait_for_scan("Collapse PTE table with single PTE entry present", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table with single PTE entry present", p, ++ 1, true); + validate_memory(p, 0, page_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_max_ptes_none(void) ++static void collapse_max_ptes_none(struct collapse_context *c) + { + int max_ptes_none = hpage_pmd_nr / 2; +- struct settings settings = default_settings; ++ struct settings settings = *current_settings(); + void *p; + + settings.khugepaged.max_ptes_none = max_ptes_none; +- write_settings(&settings); ++ push_settings(&settings); + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + + fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); +- if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- fail("Fail"); +- else +- success("OK"); ++ c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, ++ !c->enforce_pte_scan_limits); + validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); + +- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); +- if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); ++ if (c->enforce_pte_scan_limits) { ++ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); ++ c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ++ true); ++ validate_memory(p, 0, ++ (hpage_pmd_nr - max_ptes_none) * page_size); ++ } + + munmap(p, hpage_pmd_size); +- write_settings(&default_settings); ++ pop_settings(); + } + +-static void collapse_swapin_single_pte(void) ++static void collapse_swapin_single_pte(struct collapse_context *c) + { + void *p; +- p = alloc_mapping(); ++ p = alloc_mapping(1); + fill_memory(p, 0, hpage_pmd_size); + + printf("Swapout one page..."); +@@ -625,23 +698,18 @@ static void collapse_swapin_single_pte(void) + goto out; + } + +- if (wait_for_scan("Collapse with swapping in single PTE entry", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse with swapping in single PTE entry", p, 1, true); + validate_memory(p, 0, hpage_pmd_size); + out: + munmap(p, hpage_pmd_size); + } + +-static void collapse_max_ptes_swap(void) ++static void collapse_max_ptes_swap(struct collapse_context *c) + { + int max_ptes_swap = read_num("khugepaged/max_ptes_swap"); + void *p; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + + fill_memory(p, 0, hpage_pmd_size); + printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); +@@ -656,115 +724,83 @@ static void collapse_max_ptes_swap(void) + goto out; + } + +- if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- fail("Fail"); +- else +- success("OK"); ++ c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ++ !c->enforce_pte_scan_limits); + validate_memory(p, 0, hpage_pmd_size); + +- fill_memory(p, 0, hpage_pmd_size); +- printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); +- if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { +- perror("madvise(MADV_PAGEOUT)"); +- exit(EXIT_FAILURE); +- } +- if (check_swap(p, max_ptes_swap * page_size)) { +- success("OK"); +- } else { +- fail("Fail"); +- goto out; +- } ++ if (c->enforce_pte_scan_limits) { ++ fill_memory(p, 0, hpage_pmd_size); ++ printf("Swapout %d of %d pages...", max_ptes_swap, ++ hpage_pmd_nr); ++ if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { ++ perror("madvise(MADV_PAGEOUT)"); ++ exit(EXIT_FAILURE); ++ } ++ if (check_swap(p, max_ptes_swap * page_size)) { ++ success("OK"); ++ } else { ++ fail("Fail"); ++ goto out; ++ } + +- if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- validate_memory(p, 0, hpage_pmd_size); ++ c->collapse("Collapse with max_ptes_swap pages swapped out", p, ++ 1, true); ++ validate_memory(p, 0, hpage_pmd_size); ++ } + out: + munmap(p, hpage_pmd_size); + } + +-static void collapse_single_pte_entry_compound(void) ++static void collapse_single_pte_entry_compound(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ p = alloc_hpage(); + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); +- + printf("Split huge page leaving single PTE mapping compound page..."); + madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table with single PTE mapping compound page", ++ p, 1, true); + validate_memory(p, 0, page_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_full_of_compound(void) ++static void collapse_full_of_compound(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- ++ p = alloc_hpage(); + printf("Split huge page leaving single PTE page table full of compound pages..."); + madvise(p, page_size, MADV_NOHUGEPAGE); + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Collapse PTE table full of compound pages", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table full of compound pages", p, 1, true); + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_compound_extreme(void) ++static void collapse_compound_extreme(struct collapse_context *c) + { + void *p; + int i; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + for (i = 0; i < hpage_pmd_nr; i++) { + printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", + i + 1, hpage_pmd_nr); + + madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); + fill_memory(BASE_ADDR, 0, hpage_pmd_size); +- if (!check_huge(BASE_ADDR)) { ++ if (!check_huge(BASE_ADDR, 1)) { + printf("Failed to allocate huge page\n"); + exit(EXIT_FAILURE); + } +@@ -793,32 +829,28 @@ static void collapse_compound_extreme(void) + + munmap(BASE_ADDR, hpage_pmd_size); + fill_memory(p, 0, hpage_pmd_size); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Collapse PTE table full of different compound pages", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table full of different compound pages", p, 1, ++ true); + + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_fork(void) ++static void collapse_fork(struct collapse_context *c) + { + int wstatus; + void *p; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + + printf("Allocate small page..."); + fill_memory(p, 0, page_size); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); +@@ -829,19 +861,14 @@ static void collapse_fork(void) + skip_settings_restore = true; + exit_status = 0; + +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + + fill_memory(p, page_size, 2 * page_size); +- +- if (wait_for_scan("Collapse PTE table with single page shared with parent process", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table with single page shared with parent process", ++ p, 1, true); + + validate_memory(p, 0, page_size); + munmap(p, hpage_pmd_size); +@@ -852,7 +879,7 @@ static void collapse_fork(void) + exit_status += WEXITSTATUS(wstatus); + + printf("Check if parent still has small page..."); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); +@@ -860,28 +887,19 @@ static void collapse_fork(void) + munmap(p, hpage_pmd_size); + } + +-static void collapse_fork_compound(void) ++static void collapse_fork_compound(struct collapse_context *c) + { + int wstatus; + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- ++ p = alloc_hpage(); + printf("Share huge page over fork()..."); + if (!fork()) { + /* Do not touch settings on child exit */ + skip_settings_restore = true; + exit_status = 0; + +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -889,21 +907,17 @@ static void collapse_fork_compound(void) + printf("Split huge page PMD in child process..."); + madvise(p, page_size, MADV_NOHUGEPAGE); + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + fill_memory(p, 0, page_size); + + write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); +- if (wait_for_scan("Collapse PTE table full of compound pages in child", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table full of compound pages in child", ++ p, 1, true); + write_num("khugepaged/max_ptes_shared", +- default_settings.khugepaged.max_ptes_shared); ++ current_settings()->khugepaged.max_ptes_shared); + + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); +@@ -914,7 +928,7 @@ static void collapse_fork_compound(void) + exit_status += WEXITSTATUS(wstatus); + + printf("Check if parent still has huge page..."); +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -922,29 +936,20 @@ static void collapse_fork_compound(void) + munmap(p, hpage_pmd_size); + } + +-static void collapse_max_ptes_shared() ++static void collapse_max_ptes_shared(struct collapse_context *c) + { + int max_ptes_shared = read_num("khugepaged/max_ptes_shared"); + int wstatus; + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- ++ p = alloc_hpage(); + printf("Share huge page over fork()..."); + if (!fork()) { + /* Do not touch settings on child exit */ + skip_settings_restore = true; + exit_status = 0; + +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -952,33 +957,27 @@ static void collapse_max_ptes_shared() + printf("Trigger CoW on page %d of %d...", + hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); + fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p)) +- fail("Timeout"); +- else if (!check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- +- printf("Trigger CoW on page %d of %d...", +- hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); +- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); +- if (!check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- +- +- if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Maybe collapse with max_ptes_shared exceeded", p, ++ 1, !c->enforce_pte_scan_limits); ++ ++ if (c->enforce_pte_scan_limits) { ++ printf("Trigger CoW on page %d of %d...", ++ hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); ++ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * ++ page_size); ++ if (check_huge(p, 0)) ++ success("OK"); ++ else ++ fail("Fail"); ++ ++ c->collapse("Collapse with max_ptes_shared PTEs shared", ++ p, 1, true); ++ } + + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); +@@ -989,7 +988,7 @@ static void collapse_max_ptes_shared() + exit_status += WEXITSTATUS(wstatus); + + printf("Check if parent still has huge page..."); +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -997,8 +996,52 @@ static void collapse_max_ptes_shared() + munmap(p, hpage_pmd_size); + } + +-int main(void) ++static void madvise_collapse_existing_thps(void) + { ++ void *p; ++ int err; ++ ++ p = alloc_mapping(1); ++ fill_memory(p, 0, hpage_pmd_size); ++ ++ printf("Collapse fully populated PTE table..."); ++ /* ++ * Note that we don't set MADV_HUGEPAGE here, which ++ * also tests that VM_HUGEPAGE isn't required for ++ * MADV_COLLAPSE in "madvise" mode. ++ */ ++ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); ++ if (err == 0 && check_huge(p, 1)) { ++ success("OK"); ++ printf("Re-collapse PMD-mapped hugepage"); ++ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); ++ if (err == 0 && check_huge(p, 1)) ++ success("OK"); ++ else ++ fail("Fail"); ++ } else { ++ fail("Fail"); ++ } ++ validate_memory(p, 0, hpage_pmd_size); ++ munmap(p, hpage_pmd_size); ++} ++ ++int main(int argc, const char **argv) ++{ ++ struct collapse_context c; ++ struct settings default_settings = { ++ .thp_enabled = THP_MADVISE, ++ .thp_defrag = THP_DEFRAG_ALWAYS, ++ .shmem_enabled = SHMEM_NEVER, ++ .use_zero_page = 0, ++ .khugepaged = { ++ .defrag = 1, ++ .alloc_sleep_millisecs = 10, ++ .scan_sleep_millisecs = 10, ++ }, ++ }; ++ const char *tests = argc == 1 ? "all" : argv[1]; ++ + setbuf(stdout, NULL); + + page_size = getpagesize(); +@@ -1011,21 +1054,47 @@ int main(void) + default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8; + + save_settings(); +- adjust_settings(); ++ push_settings(&default_settings); + + alloc_at_fault(); +- collapse_full(); +- collapse_empty(); +- collapse_single_pte_entry(); +- collapse_max_ptes_none(); +- collapse_swapin_single_pte(); +- collapse_max_ptes_swap(); +- collapse_single_pte_entry_compound(); +- collapse_full_of_compound(); +- collapse_compound_extreme(); +- collapse_fork(); +- collapse_fork_compound(); +- collapse_max_ptes_shared(); ++ ++ if (!strcmp(tests, "khugepaged") || !strcmp(tests, "all")) { ++ printf("\n*** Testing context: khugepaged ***\n"); ++ c.collapse = &khugepaged_collapse; ++ c.enforce_pte_scan_limits = true; ++ ++ collapse_full(&c); ++ collapse_empty(&c); ++ collapse_single_pte_entry(&c); ++ collapse_max_ptes_none(&c); ++ collapse_swapin_single_pte(&c); ++ collapse_max_ptes_swap(&c); ++ collapse_single_pte_entry_compound(&c); ++ collapse_full_of_compound(&c); ++ collapse_compound_extreme(&c); ++ collapse_fork(&c); ++ collapse_fork_compound(&c); ++ collapse_max_ptes_shared(&c); ++ } ++ if (!strcmp(tests, "madvise") || !strcmp(tests, "all")) { ++ printf("\n*** Testing context: madvise ***\n"); ++ c.collapse = &madvise_collapse; ++ c.enforce_pte_scan_limits = false; ++ ++ collapse_full(&c); ++ collapse_empty(&c); ++ collapse_single_pte_entry(&c); ++ collapse_max_ptes_none(&c); ++ collapse_swapin_single_pte(&c); ++ collapse_max_ptes_swap(&c); ++ collapse_single_pte_entry_compound(&c); ++ collapse_full_of_compound(&c); ++ collapse_compound_extreme(&c); ++ collapse_fork(&c); ++ collapse_fork_compound(&c); ++ collapse_max_ptes_shared(&c); ++ madvise_collapse_existing_thps(); ++ } + + restore_settings(0); + } +-- +2.38.0.rc1.8.g2a7d63a245 + diff --git a/6.0/0005-lru.patch b/6.0/0007-mm-multi-gen-LRU.patch similarity index 99% rename from 6.0/0005-lru.patch rename to 6.0/0007-mm-multi-gen-LRU.patch index acde1681..e5ef646e 100644 --- a/6.0/0005-lru.patch +++ b/6.0/0007-mm-multi-gen-LRU.patch @@ -1,7 +1,7 @@ -From 92568143bea4e1fe29fa24b6c871ce02fb2b00d2 Mon Sep 17 00:00:00 2001 +From 3430d4868012555c67c2ec34b073b0e4ecda986d Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Mon, 26 Sep 2022 00:17:26 +0200 -Subject: [PATCH 05/13] lru +Date: Wed, 28 Sep 2022 00:26:48 +0200 +Subject: [PATCH 07/16] mm: multi-gen LRU Signed-off-by: Peter Jung --- @@ -960,12 +960,12 @@ index 7b25b53c474a..4949eda9a9a2 100644 list_del(&folio->lru); update_lru_size(lruvec, lru, folio_zonenum(folio), diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 8ac6d7e53b17..15afe5e0c4c2 100644 +index cf97f3884fda..e1797813cc2c 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h -@@ -677,6 +677,22 @@ struct mm_struct { +@@ -672,6 +672,22 @@ struct mm_struct { */ - unsigned long ksm_rmap_items; + unsigned long ksm_merging_pages; #endif +#ifdef CONFIG_LRU_GEN + struct { @@ -986,7 +986,7 @@ index 8ac6d7e53b17..15afe5e0c4c2 100644 } __randomize_layout; /* -@@ -703,6 +719,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) +@@ -698,6 +714,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) return (struct cpumask *)&mm->cpu_bitmap; } @@ -1054,7 +1054,7 @@ index 8ac6d7e53b17..15afe5e0c4c2 100644 extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index e24b40c52468..1543001feba9 100644 +index 7d78133fe8dd..e5c240eed6af 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -306,6 +306,8 @@ static inline bool is_active_lru(enum lru_list lru) @@ -1309,7 +1309,7 @@ index e24b40c52468..1543001feba9 100644 /* Per-node vmstats */ diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h -index 4b71a96190a8..3a0eec9f2faa 100644 +index ac5b6a371be5..e66742db741c 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) @@ -1583,10 +1583,10 @@ index 0331f1461f81..96cd3ae25c6f 100644 endmenu diff --git a/mm/huge_memory.c b/mm/huge_memory.c -index e9414ee57c5b..1eee94e39d0c 100644 +index 42cdc3338adc..786497dd5f26 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c -@@ -2438,7 +2438,8 @@ static void __split_huge_page_tail(struct page *head, int tail, +@@ -2423,7 +2423,8 @@ static void __split_huge_page_tail(struct page *head, int tail, #ifdef CONFIG_64BIT (1L << PG_arch_2) | #endif @@ -1597,7 +1597,7 @@ index e9414ee57c5b..1eee94e39d0c 100644 /* ->mapping in first tail page is compound_mapcount */ VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, diff --git a/mm/internal.h b/mm/internal.h -index 785409805ed7..a1fddea6b34f 100644 +index 55ce10e4d0c0..cf134d58fd6d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -83,6 +83,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); @@ -1676,10 +1676,10 @@ index 7eadbafc006b..48588e89867a 100644 .post_attach = mem_cgroup_move_task, .dfl_cftypes = memory_files, diff --git a/mm/memory.c b/mm/memory.c -index 4ba73f5aa8bb..47daee6b421f 100644 +index e58d5d522467..bc4dc2e45dcc 100644 --- a/mm/memory.c +++ b/mm/memory.c -@@ -125,18 +125,6 @@ int randomize_va_space __read_mostly = +@@ -126,18 +126,6 @@ int randomize_va_space __read_mostly = 2; #endif @@ -1698,7 +1698,7 @@ index 4ba73f5aa8bb..47daee6b421f 100644 #ifndef arch_wants_old_prefaulted_pte static inline bool arch_wants_old_prefaulted_pte(void) { -@@ -2870,7 +2858,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, +@@ -2871,7 +2859,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, * On architectures with software "accessed" bits, we would * take a double page fault, so mark it accessed here. */ @@ -1707,7 +1707,7 @@ index 4ba73f5aa8bb..47daee6b421f 100644 pte_t entry; vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); -@@ -5114,6 +5102,27 @@ static inline void mm_account_fault(struct pt_regs *regs, +@@ -5115,6 +5103,27 @@ static inline void mm_account_fault(struct pt_regs *regs, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } @@ -1735,7 +1735,7 @@ index 4ba73f5aa8bb..47daee6b421f 100644 /* * By the time we get here, we already hold the mm semaphore * -@@ -5145,11 +5154,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, +@@ -5146,11 +5155,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, if (flags & FAULT_FLAG_USER) mem_cgroup_enter_user_fault(); @@ -1788,10 +1788,10 @@ index 0ae7571e35ab..68e1511be12d 100644 #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) diff --git a/mm/rmap.c b/mm/rmap.c -index 93d5a6f793d2..9e0ce48bca08 100644 +index 9af08343ce55..3080edeaa3ad 100644 --- a/mm/rmap.c +++ b/mm/rmap.c -@@ -833,6 +833,12 @@ static bool folio_referenced_one(struct folio *folio, +@@ -828,6 +828,12 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte) { @@ -1912,10 +1912,10 @@ index 186b4e5dcecf..027943b341a0 100644 folio_get(folio); diff --git a/mm/vmscan.c b/mm/vmscan.c -index 0fc65ace3a4e..0ae7458b1e55 100644 +index e673be68cea3..feb8416d8edd 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c -@@ -49,6 +49,10 @@ +@@ -50,6 +50,10 @@ #include #include #include @@ -1926,7 +1926,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 #include #include -@@ -129,6 +133,12 @@ struct scan_control { +@@ -130,6 +134,12 @@ struct scan_control { /* Always discard instead of demoting to lower tier memory */ unsigned int no_demotion:1; @@ -1939,7 +1939,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 /* Allocation order */ s8 order; -@@ -1338,9 +1348,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, +@@ -1339,9 +1349,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, if (folio_test_swapcache(folio)) { swp_entry_t swap = folio_swap_entry(folio); @@ -1952,7 +1952,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 __delete_from_swap_cache(folio, swap, shadow); xa_unlock_irq(&mapping->i_pages); put_swap_page(&folio->page, swap); -@@ -1637,6 +1649,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, +@@ -1666,6 +1678,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, if (!sc->may_unmap && folio_mapped(folio)) goto keep_locked; @@ -1964,7 +1964,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 /* * The number of dirty pages determines if a node is marked * reclaim_congested. kswapd will stall and start writing -@@ -2732,6 +2749,112 @@ enum scan_balance { +@@ -2761,6 +2778,112 @@ enum scan_balance { SCAN_FILE, }; @@ -2077,7 +2077,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 /* * Determine how aggressively the anon and file LRU lists should be * scanned. -@@ -2951,159 +3074,2912 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, +@@ -2980,159 +3103,2912 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, return can_demote(pgdat->node_id, sc); } @@ -5118,7 +5118,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 * where always a non-zero amount of pages were scanned. */ if (!nr_reclaimed) -@@ -3201,109 +6077,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) +@@ -3230,109 +6106,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) unsigned long nr_reclaimed, nr_scanned; struct lruvec *target_lruvec; bool reclaimable = false; @@ -5229,7 +5229,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 shrink_node_memcgs(pgdat, sc); -@@ -3561,11 +6344,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) +@@ -3590,11 +6373,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) struct lruvec *target_lruvec; unsigned long refaults; @@ -5246,7 +5246,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 } /* -@@ -3927,12 +6713,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, +@@ -3956,12 +6742,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, } #endif @@ -5265,7 +5265,7 @@ index 0fc65ace3a4e..0ae7458b1e55 100644 if (!can_age_anon_pages(pgdat, sc)) return; -@@ -4252,12 +7042,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) +@@ -4281,12 +7071,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) sc.may_swap = !nr_boost_reclaim; /* @@ -5433,5 +5433,5 @@ index a5e84862fc86..ae7e984b23c6 100644 rcu_read_lock(); /* -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 diff --git a/6.0/0007-rtw88.patch b/6.0/0007-rtw88.patch deleted file mode 100644 index 0c3bd7a7..00000000 --- a/6.0/0007-rtw88.patch +++ /dev/null @@ -1,2488 +0,0 @@ -From b0023af1c32f89b1b3b4c888d850eb2f77cffd99 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Fri, 19 Aug 2022 17:06:47 +0200 -Subject: [PATCH 07/13] rtw88 - -Signed-off-by: Peter Jung ---- - drivers/net/wireless/realtek/rtw88/Kconfig | 47 + - drivers/net/wireless/realtek/rtw88/Makefile | 14 + - drivers/net/wireless/realtek/rtw88/coex.c | 3 +- - drivers/net/wireless/realtek/rtw88/debug.c | 15 + - drivers/net/wireless/realtek/rtw88/fw.c | 13 +- - drivers/net/wireless/realtek/rtw88/hci.h | 9 +- - drivers/net/wireless/realtek/rtw88/mac.c | 3 + - drivers/net/wireless/realtek/rtw88/mac80211.c | 2 +- - drivers/net/wireless/realtek/rtw88/main.c | 9 +- - drivers/net/wireless/realtek/rtw88/main.h | 11 +- - drivers/net/wireless/realtek/rtw88/phy.c | 6 +- - drivers/net/wireless/realtek/rtw88/ps.c | 2 +- - drivers/net/wireless/realtek/rtw88/reg.h | 1 + - drivers/net/wireless/realtek/rtw88/rtw8723d.c | 19 + - drivers/net/wireless/realtek/rtw88/rtw8723d.h | 1 + - .../net/wireless/realtek/rtw88/rtw8723du.c | 40 + - .../net/wireless/realtek/rtw88/rtw8723du.h | 13 + - drivers/net/wireless/realtek/rtw88/rtw8821c.c | 23 + - drivers/net/wireless/realtek/rtw88/rtw8821c.h | 21 + - .../net/wireless/realtek/rtw88/rtw8821cu.c | 69 ++ - .../net/wireless/realtek/rtw88/rtw8821cu.h | 15 + - drivers/net/wireless/realtek/rtw88/rtw8822b.c | 19 + - .../net/wireless/realtek/rtw88/rtw8822bu.c | 62 + - .../net/wireless/realtek/rtw88/rtw8822bu.h | 15 + - drivers/net/wireless/realtek/rtw88/rtw8822c.c | 24 + - .../net/wireless/realtek/rtw88/rtw8822cu.c | 40 + - .../net/wireless/realtek/rtw88/rtw8822cu.h | 15 + - drivers/net/wireless/realtek/rtw88/tx.h | 31 + - drivers/net/wireless/realtek/rtw88/usb.c | 1051 +++++++++++++++++ - drivers/net/wireless/realtek/rtw88/usb.h | 109 ++ - drivers/net/wireless/realtek/rtw88/util.c | 92 ++ - drivers/net/wireless/realtek/rtw88/util.h | 12 +- - 32 files changed, 1770 insertions(+), 36 deletions(-) - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8723du.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8723du.h - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8821cu.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8821cu.h - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822bu.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822bu.h - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822cu.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822cu.h - create mode 100644 drivers/net/wireless/realtek/rtw88/usb.c - create mode 100644 drivers/net/wireless/realtek/rtw88/usb.h - -diff --git a/drivers/net/wireless/realtek/rtw88/Kconfig b/drivers/net/wireless/realtek/rtw88/Kconfig -index e3d7cb6c1290..651ab56d9c6b 100644 ---- a/drivers/net/wireless/realtek/rtw88/Kconfig -+++ b/drivers/net/wireless/realtek/rtw88/Kconfig -@@ -16,6 +16,9 @@ config RTW88_CORE - config RTW88_PCI - tristate - -+config RTW88_USB -+ tristate -+ - config RTW88_8822B - tristate - -@@ -39,6 +42,17 @@ config RTW88_8822BE - - 802.11ac PCIe wireless network adapter - -+config RTW88_8822BU -+ tristate "Realtek 8822BU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8822B -+ help -+ Select this option will enable support for 8822BU chipset -+ -+ 802.11ac USB wireless network adapter -+ - config RTW88_8822CE - tristate "Realtek 8822CE PCI wireless network adapter" - depends on PCI -@@ -50,6 +64,17 @@ config RTW88_8822CE - - 802.11ac PCIe wireless network adapter - -+config RTW88_8822CU -+ tristate "Realtek 8822CU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8822C -+ help -+ Select this option will enable support for 8822CU chipset -+ -+ 802.11ac USB wireless network adapter -+ - config RTW88_8723DE - tristate "Realtek 8723DE PCI wireless network adapter" - depends on PCI -@@ -61,6 +86,17 @@ config RTW88_8723DE - - 802.11n PCIe wireless network adapter - -+config RTW88_8723DU -+ tristate "Realtek 8723DU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8723D -+ help -+ Select this option will enable support for 8723DU chipset -+ -+ 802.11n USB wireless network adapter -+ - config RTW88_8821CE - tristate "Realtek 8821CE PCI wireless network adapter" - depends on PCI -@@ -72,6 +108,17 @@ config RTW88_8821CE - - 802.11ac PCIe wireless network adapter - -+config RTW88_8821CU -+ tristate "Realtek 8821CU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8821C -+ help -+ Select this option will enable support for 8821CU chipset -+ -+ 802.11ac USB wireless network adapter -+ - config RTW88_DEBUG - bool "Realtek rtw88 debug support" - depends on RTW88_CORE -diff --git a/drivers/net/wireless/realtek/rtw88/Makefile b/drivers/net/wireless/realtek/rtw88/Makefile -index 834c66ec0af9..e0950dbc2565 100644 ---- a/drivers/net/wireless/realtek/rtw88/Makefile -+++ b/drivers/net/wireless/realtek/rtw88/Makefile -@@ -26,23 +26,37 @@ rtw88_8822b-objs := rtw8822b.o rtw8822b_table.o - obj-$(CONFIG_RTW88_8822BE) += rtw88_8822be.o - rtw88_8822be-objs := rtw8822be.o - -+obj-$(CONFIG_RTW88_8822BU) += rtw88_8822bu.o -+rtw88_8822bu-objs := rtw8822bu.o -+ - obj-$(CONFIG_RTW88_8822C) += rtw88_8822c.o - rtw88_8822c-objs := rtw8822c.o rtw8822c_table.o - - obj-$(CONFIG_RTW88_8822CE) += rtw88_8822ce.o - rtw88_8822ce-objs := rtw8822ce.o - -+obj-$(CONFIG_RTW88_8822CU) += rtw88_8822cu.o -+rtw88_8822cu-objs := rtw8822cu.o -+ - obj-$(CONFIG_RTW88_8723D) += rtw88_8723d.o - rtw88_8723d-objs := rtw8723d.o rtw8723d_table.o - - obj-$(CONFIG_RTW88_8723DE) += rtw88_8723de.o - rtw88_8723de-objs := rtw8723de.o - -+obj-$(CONFIG_RTW88_8723DU) += rtw88_8723du.o -+rtw88_8723du-objs := rtw8723du.o -+ - obj-$(CONFIG_RTW88_8821C) += rtw88_8821c.o - rtw88_8821c-objs := rtw8821c.o rtw8821c_table.o - - obj-$(CONFIG_RTW88_8821CE) += rtw88_8821ce.o - rtw88_8821ce-objs := rtw8821ce.o - -+obj-$(CONFIG_RTW88_8821CU) += rtw88_8821cu.o -+rtw88_8821cu-objs := rtw8821cu.o -+ - obj-$(CONFIG_RTW88_PCI) += rtw88_pci.o -+obj-$(CONFIG_RTW88_USB) += rtw88_usb.o - rtw88_pci-objs := pci.o -+rtw88_usb-objs := usb.o -diff --git a/drivers/net/wireless/realtek/rtw88/coex.c b/drivers/net/wireless/realtek/rtw88/coex.c -index cac053f485c3..b156f8c48ffb 100644 ---- a/drivers/net/wireless/realtek/rtw88/coex.c -+++ b/drivers/net/wireless/realtek/rtw88/coex.c -@@ -633,7 +633,7 @@ static struct sk_buff *rtw_coex_info_request(struct rtw_dev *rtwdev, - struct rtw_coex *coex = &rtwdev->coex; - struct sk_buff *skb_resp = NULL; - -- mutex_lock(&coex->mutex); -+ lockdep_assert_held(&rtwdev->mutex); - - rtw_fw_query_bt_mp_info(rtwdev, req); - -@@ -650,7 +650,6 @@ static struct sk_buff *rtw_coex_info_request(struct rtw_dev *rtwdev, - } - - out: -- mutex_unlock(&coex->mutex); - return skb_resp; - } - -diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c -index 7cde6bcf253b..28db952be4ff 100644 ---- a/drivers/net/wireless/realtek/rtw88/debug.c -+++ b/drivers/net/wireless/realtek/rtw88/debug.c -@@ -144,7 +144,9 @@ static int rtw_debugfs_get_rf_read(struct seq_file *m, void *v) - addr = debugfs_priv->rf_addr; - mask = debugfs_priv->rf_mask; - -+ mutex_lock(&rtwdev->mutex); - val = rtw_read_rf(rtwdev, path, addr, mask); -+ mutex_unlock(&rtwdev->mutex); - - seq_printf(m, "rf_read path:%d addr:0x%08x mask:0x%08x val=0x%08x\n", - path, addr, mask, val); -@@ -390,7 +392,9 @@ static ssize_t rtw_debugfs_set_h2c(struct file *filp, - return -EINVAL; - } - -+ mutex_lock(&rtwdev->mutex); - rtw_fw_h2c_cmd_dbg(rtwdev, param); -+ mutex_unlock(&rtwdev->mutex); - - return count; - } -@@ -414,7 +418,9 @@ static ssize_t rtw_debugfs_set_rf_write(struct file *filp, - return count; - } - -+ mutex_lock(&rtwdev->mutex); - rtw_write_rf(rtwdev, path, addr, mask, val); -+ mutex_unlock(&rtwdev->mutex); - rtw_dbg(rtwdev, RTW_DBG_DEBUGFS, - "write_rf path:%d addr:0x%08x mask:0x%08x, val:0x%08x\n", - path, addr, mask, val); -@@ -519,6 +525,8 @@ static int rtw_debug_get_rf_dump(struct seq_file *m, void *v) - u32 addr, offset, data; - u8 path; - -+ mutex_lock(&rtwdev->mutex); -+ - for (path = 0; path < rtwdev->hal.rf_path_num; path++) { - seq_printf(m, "RF path:%d\n", path); - for (addr = 0; addr < 0x100; addr += 4) { -@@ -533,6 +541,8 @@ static int rtw_debug_get_rf_dump(struct seq_file *m, void *v) - seq_puts(m, "\n"); - } - -+ mutex_unlock(&rtwdev->mutex); -+ - return 0; - } - -@@ -828,7 +838,9 @@ static int rtw_debugfs_get_coex_info(struct seq_file *m, void *v) - struct rtw_debugfs_priv *debugfs_priv = m->private; - struct rtw_dev *rtwdev = debugfs_priv->rtwdev; - -+ mutex_lock(&rtwdev->mutex); - rtw_coex_display_coex_info(rtwdev, m); -+ mutex_unlock(&rtwdev->mutex); - - return 0; - } -@@ -1023,6 +1035,8 @@ static void dump_gapk_status(struct rtw_dev *rtwdev, struct seq_file *m) - dm_info->dm_flags & BIT(RTW_DM_CAP_TXGAPK) ? '-' : '+', - rtw_dm_cap_strs[RTW_DM_CAP_TXGAPK]); - -+ mutex_lock(&rtwdev->mutex); -+ - for (path = 0; path < rtwdev->hal.rf_path_num; path++) { - val = rtw_read_rf(rtwdev, path, RF_GAINTX, RFREG_MASK); - seq_printf(m, "path %d:\n0x%x = 0x%x\n", path, RF_GAINTX, val); -@@ -1032,6 +1046,7 @@ static void dump_gapk_status(struct rtw_dev *rtwdev, struct seq_file *m) - txgapk->rf3f_fs[path][i], i); - seq_puts(m, "\n"); - } -+ mutex_unlock(&rtwdev->mutex); - } - - static int rtw_debugfs_get_dm_cap(struct seq_file *m, void *v) -diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c -index 4fdab0329695..5e912421408e 100644 ---- a/drivers/net/wireless/realtek/rtw88/fw.c -+++ b/drivers/net/wireless/realtek/rtw88/fw.c -@@ -320,7 +320,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - h2c[3], h2c[2], h2c[1], h2c[0], - h2c[7], h2c[6], h2c[5], h2c[4]); - -- spin_lock(&rtwdev->h2c.lock); -+ lockdep_assert_held(&rtwdev->mutex); - - box = rtwdev->h2c.last_box_num; - switch (box) { -@@ -342,7 +342,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - break; - default: - WARN(1, "invalid h2c mail box number\n"); -- goto out; -+ return; - } - - ret = read_poll_timeout_atomic(rtw_read8, box_state, -@@ -351,7 +351,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - - if (ret) { - rtw_err(rtwdev, "failed to send h2c command\n"); -- goto out; -+ return; - } - - for (idx = 0; idx < 4; idx++) -@@ -361,9 +361,6 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - - if (++rtwdev->h2c.last_box_num >= 4) - rtwdev->h2c.last_box_num = 0; -- --out: -- spin_unlock(&rtwdev->h2c.lock); - } - - void rtw_fw_h2c_cmd_dbg(struct rtw_dev *rtwdev, u8 *h2c) -@@ -375,15 +372,13 @@ static void rtw_fw_send_h2c_packet(struct rtw_dev *rtwdev, u8 *h2c_pkt) - { - int ret; - -- spin_lock(&rtwdev->h2c.lock); -+ lockdep_assert_held(&rtwdev->mutex); - - FW_OFFLOAD_H2C_SET_SEQ_NUM(h2c_pkt, rtwdev->h2c.seq); - ret = rtw_hci_write_data_h2c(rtwdev, h2c_pkt, H2C_PKT_SIZE); - if (ret) - rtw_err(rtwdev, "failed to send h2c packet\n"); - rtwdev->h2c.seq++; -- -- spin_unlock(&rtwdev->h2c.lock); - } - - void -diff --git a/drivers/net/wireless/realtek/rtw88/hci.h b/drivers/net/wireless/realtek/rtw88/hci.h -index 4c6fc6fb3f83..830d7532f2a3 100644 ---- a/drivers/net/wireless/realtek/rtw88/hci.h -+++ b/drivers/net/wireless/realtek/rtw88/hci.h -@@ -166,12 +166,11 @@ static inline u32 - rtw_read_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, - u32 addr, u32 mask) - { -- unsigned long flags; - u32 val; - -- spin_lock_irqsave(&rtwdev->rf_lock, flags); -+ lockdep_assert_held(&rtwdev->mutex); -+ - val = rtwdev->chip->ops->read_rf(rtwdev, rf_path, addr, mask); -- spin_unlock_irqrestore(&rtwdev->rf_lock, flags); - - return val; - } -@@ -180,11 +179,9 @@ static inline void - rtw_write_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, - u32 addr, u32 mask, u32 data) - { -- unsigned long flags; -+ lockdep_assert_held(&rtwdev->mutex); - -- spin_lock_irqsave(&rtwdev->rf_lock, flags); - rtwdev->chip->ops->write_rf(rtwdev, rf_path, addr, mask, data); -- spin_unlock_irqrestore(&rtwdev->rf_lock, flags); - } - - static inline u32 -diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c -index caf2603da2d6..c3c24850a427 100644 ---- a/drivers/net/wireless/realtek/rtw88/mac.c -+++ b/drivers/net/wireless/realtek/rtw88/mac.c -@@ -1032,6 +1032,9 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev) - if (rtw_chip_wcpu_11ac(rtwdev)) - rtw_write32(rtwdev, REG_H2CQ_CSR, BIT_H2CQ_FULL); - -+ if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB) -+ rtw_write8_set(rtwdev, REG_TXDMA_PQ_MAP, BIT_RXDMA_ARBBW_EN); -+ - return 0; - } - -diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c -index c7b98a0599d5..bc30400e67a1 100644 ---- a/drivers/net/wireless/realtek/rtw88/mac80211.c -+++ b/drivers/net/wireless/realtek/rtw88/mac80211.c -@@ -483,8 +483,8 @@ static int rtw_ops_sta_remove(struct ieee80211_hw *hw, - { - struct rtw_dev *rtwdev = hw->priv; - -- rtw_fw_beacon_filter_config(rtwdev, false, vif); - mutex_lock(&rtwdev->mutex); -+ rtw_fw_beacon_filter_config(rtwdev, false, vif); - rtw_sta_remove(rtwdev, sta, true); - mutex_unlock(&rtwdev->mutex); - -diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c -index 76dc9da88f6c..0edf22d9c1ef 100644 ---- a/drivers/net/wireless/realtek/rtw88/main.c -+++ b/drivers/net/wireless/realtek/rtw88/main.c -@@ -1733,6 +1733,10 @@ static int rtw_chip_parameter_setup(struct rtw_dev *rtwdev) - rtwdev->hci.rpwm_addr = 0x03d9; - rtwdev->hci.cpwm_addr = 0x03da; - break; -+ case RTW_HCI_TYPE_USB: -+ rtwdev->hci.rpwm_addr = 0xfe58; -+ rtwdev->hci.cpwm_addr = 0xfe57; -+ break; - default: - rtw_err(rtwdev, "unsupported hci type\n"); - return -EINVAL; -@@ -2017,13 +2021,10 @@ int rtw_core_init(struct rtw_dev *rtwdev) - skb_queue_head_init(&rtwdev->coex.queue); - skb_queue_head_init(&rtwdev->tx_report.queue); - -- spin_lock_init(&rtwdev->rf_lock); -- spin_lock_init(&rtwdev->h2c.lock); - spin_lock_init(&rtwdev->txq_lock); - spin_lock_init(&rtwdev->tx_report.q_lock); - - mutex_init(&rtwdev->mutex); -- mutex_init(&rtwdev->coex.mutex); - mutex_init(&rtwdev->hal.tx_power_mutex); - - init_waitqueue_head(&rtwdev->coex.wait); -@@ -2091,7 +2092,6 @@ void rtw_core_deinit(struct rtw_dev *rtwdev) - } - - mutex_destroy(&rtwdev->mutex); -- mutex_destroy(&rtwdev->coex.mutex); - mutex_destroy(&rtwdev->hal.tx_power_mutex); - } - EXPORT_SYMBOL(rtw_core_deinit); -@@ -2132,6 +2132,7 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) - hw->wiphy->available_antennas_rx = hal->antenna_rx; - - hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS | -+ WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL | - WIPHY_FLAG_TDLS_EXTERNAL_SETUP; - - hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR; -diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h -index 7db627fc26be..663b8dc453da 100644 ---- a/drivers/net/wireless/realtek/rtw88/main.h -+++ b/drivers/net/wireless/realtek/rtw88/main.h -@@ -875,6 +875,10 @@ struct rtw_chip_ops { - bool is_tx2_path); - void (*config_txrx_mode)(struct rtw_dev *rtwdev, u8 tx_path, - u8 rx_path, bool is_tx2_path); -+ /* for USB/SDIO only */ -+ void (*fill_txdesc_checksum)(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc); - - /* for coex */ - void (*coex_set_init)(struct rtw_dev *rtwdev); -@@ -1504,8 +1508,6 @@ struct rtw_coex_stat { - }; - - struct rtw_coex { -- /* protects coex info request section */ -- struct mutex mutex; - struct sk_buff_head queue; - wait_queue_head_t wait; - -@@ -1994,9 +1996,6 @@ struct rtw_dev { - /* ensures exclusive access from mac80211 callbacks */ - struct mutex mutex; - -- /* read/write rf register */ -- spinlock_t rf_lock; -- - /* watch dog every 2 sec */ - struct delayed_work watch_dog_work; - u32 watch_dog_cnt; -@@ -2022,8 +2021,6 @@ struct rtw_dev { - struct { - /* incicate the mail box to use with fw */ - u8 last_box_num; -- /* protect to send h2c to fw */ -- spinlock_t lock; - u32 seq; - } h2c; - -diff --git a/drivers/net/wireless/realtek/rtw88/phy.c b/drivers/net/wireless/realtek/rtw88/phy.c -index 8982e0c98dac..f6de6568f962 100644 ---- a/drivers/net/wireless/realtek/rtw88/phy.c -+++ b/drivers/net/wireless/realtek/rtw88/phy.c -@@ -300,7 +300,7 @@ static void rtw_phy_stat_rssi(struct rtw_dev *rtwdev) - - data.rtwdev = rtwdev; - data.min_rssi = U8_MAX; -- rtw_iterate_stas_atomic(rtwdev, rtw_phy_stat_rssi_iter, &data); -+ rtw_iterate_stas(rtwdev, rtw_phy_stat_rssi_iter, &data); - - dm_info->pre_min_rssi = dm_info->min_rssi; - dm_info->min_rssi = data.min_rssi; -@@ -544,7 +544,7 @@ static void rtw_phy_ra_info_update(struct rtw_dev *rtwdev) - if (rtwdev->watch_dog_cnt & 0x3) - return; - -- rtw_iterate_stas_atomic(rtwdev, rtw_phy_ra_info_update_iter, rtwdev); -+ rtw_iterate_stas(rtwdev, rtw_phy_ra_info_update_iter, rtwdev); - } - - static u32 rtw_phy_get_rrsr_mask(struct rtw_dev *rtwdev, u8 rate_idx) -@@ -597,7 +597,7 @@ static void rtw_phy_rrsr_update(struct rtw_dev *rtwdev) - struct rtw_dm_info *dm_info = &rtwdev->dm_info; - - dm_info->rrsr_mask_min = RRSR_RATE_ORDER_MAX; -- rtw_iterate_stas_atomic(rtwdev, rtw_phy_rrsr_mask_min_iter, rtwdev); -+ rtw_iterate_stas(rtwdev, rtw_phy_rrsr_mask_min_iter, rtwdev); - rtw_write32(rtwdev, REG_RRSR, dm_info->rrsr_val_init & dm_info->rrsr_mask_min); - } - -diff --git a/drivers/net/wireless/realtek/rtw88/ps.c b/drivers/net/wireless/realtek/rtw88/ps.c -index bfa64c038f5f..a7213ff2c224 100644 ---- a/drivers/net/wireless/realtek/rtw88/ps.c -+++ b/drivers/net/wireless/realtek/rtw88/ps.c -@@ -58,7 +58,7 @@ int rtw_leave_ips(struct rtw_dev *rtwdev) - return ret; - } - -- rtw_iterate_vifs_atomic(rtwdev, rtw_restore_port_cfg_iter, rtwdev); -+ rtw_iterate_vifs(rtwdev, rtw_restore_port_cfg_iter, rtwdev); - - rtw_coex_ips_notify(rtwdev, COEX_IPS_LEAVE); - -diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h -index 03bd8dc53f72..8852b24d6c2a 100644 ---- a/drivers/net/wireless/realtek/rtw88/reg.h -+++ b/drivers/net/wireless/realtek/rtw88/reg.h -@@ -184,6 +184,7 @@ - #define BIT_TXDMA_VIQ_MAP(x) \ - (((x) & BIT_MASK_TXDMA_VIQ_MAP) << BIT_SHIFT_TXDMA_VIQ_MAP) - #define REG_TXDMA_PQ_MAP 0x010C -+#define BIT_RXDMA_ARBBW_EN BIT(0) - #define BIT_SHIFT_TXDMA_BEQ_MAP 8 - #define BIT_MASK_TXDMA_BEQ_MAP 0x3 - #define BIT_TXDMA_BEQ_MAP(x) \ -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.c b/drivers/net/wireless/realtek/rtw88/rtw8723d.c -index 993bd6b1d723..d99c10f34f92 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8723d.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.c -@@ -210,6 +210,12 @@ static void rtw8723de_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8723du_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8723d_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - static int rtw8723d_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - { - struct rtw_efuse *efuse = &rtwdev->efuse; -@@ -239,6 +245,9 @@ static int rtw8723d_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8723de_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8723du_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -1945,6 +1954,15 @@ static void rtw8723d_pwr_track(struct rtw_dev *rtwdev) - dm_info->pwr_trk_triggered = false; - } - -+static void rtw8723d_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */ -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static struct rtw_chip_ops rtw8723d_ops = { - .phy_set_param = rtw8723d_phy_set_param, - .read_efuse = rtw8723d_read_efuse, -@@ -1965,6 +1983,7 @@ static struct rtw_chip_ops rtw8723d_ops = { - .config_bfee = NULL, - .set_gid_table = NULL, - .cfg_csi_rate = NULL, -+ .fill_txdesc_checksum = rtw8723d_fill_txdesc_checksum, - - .coex_set_init = rtw8723d_coex_cfg_init, - .coex_set_ant_switch = NULL, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.h b/drivers/net/wireless/realtek/rtw88/rtw8723d.h -index 4641f6e047b4..7d9fc3dde97d 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8723d.h -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.h -@@ -70,6 +70,7 @@ struct rtw8723d_efuse { - u8 country_code[2]; - u8 res[3]; - struct rtw8723de_efuse e; -+ struct rtw8723de_efuse u; - }; - - extern const struct rtw_chip_info rtw8723d_hw_spec; -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723du.c b/drivers/net/wireless/realtek/rtw88/rtw8723du.c -new file mode 100644 -index 000000000000..910f64c16813 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723du.c -@@ -0,0 +1,40 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8723du.h" -+#include "usb.h" -+ -+static const struct usb_device_id rtw_8723du_id_table[] = { -+ /* -+ * ULLI : -+ * ID found in rtw8822bu sources -+ */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xD723, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8723d_hw_spec) }, /* 8723DU 1*1 */ -+ { }, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8723du_id_table); -+ -+static int rtw8723du_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8723du_driver = { -+ .name = "rtw_8723du", -+ .id_table = rtw_8723du_id_table, -+ .probe = rtw8723du_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8723du_driver); -+ -+MODULE_AUTHOR("Hans Ulli Kroll "); -+MODULE_DESCRIPTION("Realtek 802.11n wireless 8723du driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723du.h b/drivers/net/wireless/realtek/rtw88/rtw8723du.h -new file mode 100644 -index 000000000000..2e069f65c055 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723du.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8723DU_H_ -+#define __RTW_8723DU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+ -+extern struct rtw_chip_info rtw8723d_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.c b/drivers/net/wireless/realtek/rtw88/rtw8821c.c -index 025262a8970e..5b4ea9990b78 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8821c.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.c -@@ -26,6 +26,12 @@ static void rtw8821ce_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8821cu_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8821c_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - enum rtw8821ce_rf_set { - SWITCH_TO_BTG, - SWITCH_TO_WLG, -@@ -68,6 +74,9 @@ static int rtw8821c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8821ce_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8821cu_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -1148,6 +1157,18 @@ static void rtw8821c_phy_cck_pd_set(struct rtw_dev *rtwdev, u8 new_lvl) - dm_info->cck_pd_default + new_lvl * 2); - } - -+static void rtw8821c_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ size_t words; -+ -+ words = (pkt_info->pkt_offset * 8 + chip->tx_pkt_desc_sz) / 2; -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8821c[] = { - {0x0086, - RTW_PWR_CUT_ALL_MSK, -@@ -1521,6 +1542,7 @@ static const struct rtw_rfe_def rtw8821c_rfe_defs[] = { - [2] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2), - [4] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2), - [6] = RTW_DEF_RFE(8821c, 0, 0), -+ [34] = RTW_DEF_RFE(8821c, 0, 0), - }; - - static struct rtw_hw_reg rtw8821c_dig[] = { -@@ -1595,6 +1617,7 @@ static struct rtw_chip_ops rtw8821c_ops = { - .config_bfee = rtw8821c_bf_config_bfee, - .set_gid_table = rtw_bf_set_gid_table, - .cfg_csi_rate = rtw_bf_cfg_csi_rate, -+ .fill_txdesc_checksum = rtw8821c_fill_txdesc_checksum, - - .coex_set_init = rtw8821c_coex_cfg_init, - .coex_set_ant_switch = rtw8821c_coex_cfg_ant_switch, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.h b/drivers/net/wireless/realtek/rtw88/rtw8821c.h -index 2698801fc35d..71ce01623d49 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8821c.h -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.h -@@ -9,6 +9,26 @@ - - #define RCR_VHT_ACK BIT(26) - -+struct rtw8821cu_efuse { -+ u8 res4[4]; /* 0xd0 */ -+ u8 usb_optional_function; -+ u8 res5[0x1e]; -+ u8 res6[2]; -+ u8 serial[0x0b]; /* 0xf5 */ -+ u8 vid; /* 0x100 */ -+ u8 res7; -+ u8 pid; -+ u8 res8[4]; -+ u8 mac_addr[ETH_ALEN]; /* 0x107 */ -+ u8 res9[2]; -+ u8 vendor_name[0x07]; -+ u8 res10[2]; -+ u8 device_name[0x14]; -+ u8 res11[0xcf]; -+ u8 package_type; /* 0x1fb */ -+ u8 res12[0x4]; -+}; -+ - struct rtw8821ce_efuse { - u8 mac_addr[ETH_ALEN]; /* 0xd0 */ - u8 vender_id[2]; -@@ -73,6 +93,7 @@ struct rtw8821c_efuse { - u8 res[3]; - union { - struct rtw8821ce_efuse e; -+ struct rtw8821cu_efuse u; - }; - }; - -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c -new file mode 100644 -index 000000000000..e6710c5ebdfc ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c -@@ -0,0 +1,69 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8821cu.h" -+#include "usb.h" -+ -+static const struct usb_device_id rtw_8821cu_id_table[] = { -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xb82b, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xb820, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC821, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC820, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC82A, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC82B, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC811, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8811CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0x8811, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8811CU */ -+ /*=== Customer ID ===*/ -+ { USB_DEVICE(0x0bda, 0x2006), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Totolink */ -+ { USB_DEVICE(0x0bda, 0xc811), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Simplecom NW602 */ -+ {}, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8821cu_id_table); -+ -+static int rtw_8821cu_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8821cu_driver = { -+ .name = "rtw_8821cu", -+ .id_table = rtw_8821cu_id_table, -+ .probe = rtw_8821cu_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8821cu_driver); -+ -+MODULE_AUTHOR("Hans Ulli Kroll "); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8821cu driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.h b/drivers/net/wireless/realtek/rtw88/rtw8821cu.h -new file mode 100644 -index 000000000000..bddbd96aa45f ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8821CU_H_ -+#define __RTW_8821CU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+#define RTW_USB_PRODUCT_ID_REALTEK_8811C 0xC811 -+#define RTW_USB_PRODUCT_ID_REALTEK_8821C 0xC81C -+ -+extern struct rtw_chip_info rtw8821c_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c -index 321848870561..fd1e9a47ccef 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c -@@ -26,6 +26,12 @@ static void rtw8822be_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8822bu_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8822b_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - { - struct rtw_efuse *efuse = &rtwdev->efuse; -@@ -56,6 +62,9 @@ static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8822be_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8822bu_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -1588,6 +1597,15 @@ static void rtw8822b_adaptivity(struct rtw_dev *rtwdev) - rtw_phy_set_edcca_th(rtwdev, l2h, h2l); - } - -+static void rtw8822b_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */ -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8822b[] = { - {0x0086, - RTW_PWR_CUT_ALL_MSK, -@@ -2163,6 +2181,7 @@ static struct rtw_chip_ops rtw8822b_ops = { - .cfg_csi_rate = rtw_bf_cfg_csi_rate, - .adaptivity_init = rtw8822b_adaptivity_init, - .adaptivity = rtw8822b_adaptivity, -+ .fill_txdesc_checksum = rtw8822b_fill_txdesc_checksum, - - .coex_set_init = rtw8822b_coex_cfg_init, - .coex_set_ant_switch = rtw8822b_coex_cfg_ant_switch, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bu.c b/drivers/net/wireless/realtek/rtw88/rtw8822bu.c -new file mode 100644 -index 000000000000..5becebdc3247 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822bu.c -@@ -0,0 +1,62 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8822bu.h" -+#include "usb.h" -+ -+#define RTW_USB_VENDER_ID_EDIMAX 0x7392 -+ -+static const struct usb_device_id rtw_8822bu_id_table[] = { -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8812B, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8822B, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDER_ID_EDIMAX, -+ 0xB822, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDER_ID_EDIMAX, -+ 0xC822, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x0b05, 0x184c), /* ASUS AC53 Nano */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x0b05, 0x1841), /* ASUS AC55 B1 */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x2001, 0x331c), /* D-Link DWA-182 rev D1 */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x13b1, 0x0043), /* Linksys WUSB6400M */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x2357, 0x012D), /* TP-Link AC1300 T3U */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x2357, 0x0138), /* TP-Link AC1300 T3U */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ {}, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8822bu_id_table); -+ -+static int rtw8822bu_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8822bu_driver = { -+ .name = "rtw_8822bu", -+ .id_table = rtw_8822bu_id_table, -+ .probe = rtw8822bu_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8822bu_driver); -+ -+MODULE_AUTHOR("Realtek Corporation"); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822bu driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bu.h b/drivers/net/wireless/realtek/rtw88/rtw8822bu.h -new file mode 100644 -index 000000000000..20f01ecd7441 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822bu.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8822BU_H_ -+#define __RTW_8822BU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+#define RTW_USB_PRODUCT_ID_REALTEK_8812B 0xB812 -+#define RTW_USB_PRODUCT_ID_REALTEK_8822B 0xB82C -+ -+extern struct rtw_chip_info rtw8822b_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c -index 09f9e4adcf34..c0283c8278da 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c -@@ -29,6 +29,12 @@ static void rtw8822ce_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8822cu_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8822c_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - { - struct rtw_efuse *efuse = &rtwdev->efuse; -@@ -58,6 +64,9 @@ static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8822ce_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8822cu_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -4557,6 +4566,18 @@ static void rtw8822c_adaptivity(struct rtw_dev *rtwdev) - rtw_phy_set_edcca_th(rtwdev, l2h, h2l); - } - -+static void rtw8822c_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ size_t words; -+ -+ words = (pkt_info->pkt_offset * 8 + chip->tx_pkt_desc_sz) / 2; -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8822c[] = { - {0x0086, - RTW_PWR_CUT_ALL_MSK, -@@ -4895,6 +4916,8 @@ static const struct rtw_rfe_def rtw8822c_rfe_defs[] = { - [0] = RTW_DEF_RFE(8822c, 0, 0), - [1] = RTW_DEF_RFE(8822c, 0, 0), - [2] = RTW_DEF_RFE(8822c, 0, 0), -+ [3] = RTW_DEF_RFE(8822c, 0, 0), -+ [4] = RTW_DEF_RFE(8822c, 0, 0), - [5] = RTW_DEF_RFE(8822c, 0, 5), - [6] = RTW_DEF_RFE(8822c, 0, 0), - }; -@@ -4978,6 +5001,7 @@ static struct rtw_chip_ops rtw8822c_ops = { - .cfo_track = rtw8822c_cfo_track, - .config_tx_path = rtw8822c_config_tx_path, - .config_txrx_mode = rtw8822c_config_trx_mode, -+ .fill_txdesc_checksum = rtw8822c_fill_txdesc_checksum, - - .coex_set_init = rtw8822c_coex_cfg_init, - .coex_set_ant_switch = NULL, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.c b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c -new file mode 100644 -index 000000000000..36dc734f76eb ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c -@@ -0,0 +1,40 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8822cu.h" -+#include "usb.h" -+ -+static const struct usb_device_id rtw_8822cu_id_table[] = { -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8822C, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8812C, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, -+ {}, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8822cu_id_table); -+ -+static int rtw8822bu_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8822cu_driver = { -+ .name = "rtw_8822cu", -+ .id_table = rtw_8822cu_id_table, -+ .probe = rtw8822bu_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8822cu_driver); -+ -+MODULE_AUTHOR("Realtek Corporation"); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822cu driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.h b/drivers/net/wireless/realtek/rtw88/rtw8822cu.h -new file mode 100644 -index 000000000000..16afe22a8216 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8822CU_H_ -+#define __RTW_8822CU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+#define RTW_USB_PRODUCT_ID_REALTEK_8812C 0xC812 -+#define RTW_USB_PRODUCT_ID_REALTEK_8822C 0xC82C -+ -+extern struct rtw_chip_info rtw8822c_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/tx.h b/drivers/net/wireless/realtek/rtw88/tx.h -index 8419603adce4..d283bb05d1dd 100644 ---- a/drivers/net/wireless/realtek/rtw88/tx.h -+++ b/drivers/net/wireless/realtek/rtw88/tx.h -@@ -71,6 +71,14 @@ - le32p_replace_bits((__le32 *)(txdesc) + 0x03, value, BIT(15)) - #define SET_TX_DESC_BT_NULL(txdesc, value) \ - le32p_replace_bits((__le32 *)(txdesc) + 0x02, value, BIT(23)) -+#define SET_TX_DESC_TXDESC_CHECKSUM(txdesc, value) \ -+ le32p_replace_bits((__le32 *)(txdesc) + 0x07, value, GENMASK(15, 0)) -+#define SET_TX_DESC_DMA_TXAGG_NUM(txdesc, value) \ -+ le32p_replace_bits((__le32 *)(txdesc) + 0x07, value, GENMASK(31, 24)) -+#define GET_TX_DESC_PKT_OFFSET(txdesc) \ -+ le32_get_bits(*((__le32 *)(txdesc) + 0x01), GENMASK(28, 24)) -+#define GET_TX_DESC_QSEL(txdesc) \ -+ le32_get_bits(*((__le32 *)(txdesc) + 0x01), GENMASK(12, 8)) - - enum rtw_tx_desc_queue_select { - TX_DESC_QSEL_TID0 = 0, -@@ -123,4 +131,27 @@ rtw_tx_write_data_h2c_get(struct rtw_dev *rtwdev, - struct rtw_tx_pkt_info *pkt_info, - u8 *buf, u32 size); - -+static inline -+void fill_txdesc_checksum_common(u8 *txdesc, size_t words) -+{ -+ __le16 chksum = 0; -+ __le16 *data = (__le16 *)(txdesc); -+ -+ SET_TX_DESC_TXDESC_CHECKSUM(txdesc, 0x0000); -+ -+ while (words--) -+ chksum ^= *data++; -+ -+ SET_TX_DESC_TXDESC_CHECKSUM(txdesc, __le16_to_cpu(chksum)); -+} -+ -+static inline void rtw_tx_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ -+ chip->ops->fill_txdesc_checksum(rtwdev, pkt_info, txdesc); -+} -+ - #endif -diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c -new file mode 100644 -index 000000000000..7641ea6f6ad1 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/usb.c -@@ -0,0 +1,1051 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include -+#include "main.h" -+#include "debug.h" -+#include "reg.h" -+#include "tx.h" -+#include "rx.h" -+#include "fw.h" -+#include "ps.h" -+#include "usb.h" -+ -+#define RTW_USB_MAX_RXQ_LEN 128 -+ -+struct rtw_usb_txcb { -+ struct rtw_dev *rtwdev; -+ struct sk_buff_head tx_ack_queue; -+}; -+ -+static void rtw_usb_fill_tx_checksum(struct rtw_usb *rtwusb, -+ struct sk_buff *skb, int agg_num) -+{ -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct rtw_tx_pkt_info pkt_info; -+ -+ SET_TX_DESC_DMA_TXAGG_NUM(skb->data, agg_num); -+ pkt_info.pkt_offset = GET_TX_DESC_PKT_OFFSET(skb->data); -+ rtw_tx_fill_txdesc_checksum(rtwdev, &pkt_info, skb->data); -+} -+ -+static void usbctrl_async_callback(struct urb *urb) -+{ -+ /* free dr */ -+ kfree(urb->setup_packet); -+ /* free databuf */ -+ kfree(urb->transfer_buffer); -+} -+ -+static int usbctrl_vendorreq_async_write(struct usb_device *udev, u8 request, -+ u16 value, u16 index, void *pdata, -+ u16 len) -+{ -+ int rc; -+ unsigned int pipe; -+ u8 reqtype; -+ struct usb_ctrlrequest *dr; -+ struct urb *urb; -+ const u16 databuf_maxlen = RTW_USB_VENQT_MAX_BUF_SIZE; -+ u8 *databuf; -+ -+ if (WARN_ON_ONCE(len > databuf_maxlen)) -+ len = databuf_maxlen; -+ -+ pipe = usb_sndctrlpipe(udev, 0); /* write_out */ -+ reqtype = RTW_USB_CMD_WRITE; -+ -+ dr = kzalloc(sizeof(*dr), GFP_ATOMIC); -+ if (!dr) -+ return -ENOMEM; -+ -+ databuf = kmemdup(pdata, len, GFP_ATOMIC); -+ if (!databuf) { -+ kfree(dr); -+ return -ENOMEM; -+ } -+ -+ urb = usb_alloc_urb(0, GFP_ATOMIC); -+ if (!urb) { -+ kfree(databuf); -+ kfree(dr); -+ return -ENOMEM; -+ } -+ -+ dr->bRequestType = reqtype; -+ dr->bRequest = request; -+ dr->wValue = cpu_to_le16(value); -+ dr->wIndex = cpu_to_le16(index); -+ dr->wLength = cpu_to_le16(len); -+ -+ usb_fill_control_urb(urb, udev, pipe, -+ (unsigned char *)dr, databuf, len, -+ usbctrl_async_callback, NULL); -+ rc = usb_submit_urb(urb, GFP_ATOMIC); -+ if (rc < 0) { -+ kfree(databuf); -+ kfree(dr); -+ } -+ -+ usb_free_urb(urb); -+ -+ return rc; -+} -+ -+static u32 rtw_usb_read_sync(struct rtw_dev *rtwdev, u32 addr, u16 len) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *udev = rtwusb->udev; -+ __le32 *data; -+ unsigned long flags; -+ int ret; -+ static int count; -+ -+ spin_lock_irqsave(&rtwusb->usb_lock, flags); -+ -+ if (++rtwusb->usb_data_index >= RTW_USB_MAX_RX_COUNT) -+ rtwusb->usb_data_index = 0; -+ data = &rtwusb->usb_data[rtwusb->usb_data_index]; -+ -+ spin_unlock_irqrestore(&rtwusb->usb_lock, flags); -+ -+ ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), -+ RTW_USB_CMD_REQ, RTW_USB_CMD_READ, addr, -+ RTW_USB_VENQT_CMD_IDX, data, len, 1000); -+ if (ret < 0 && ret != -ENODEV && count++ < 4) -+ rtw_err(rtwdev, "reg 0x%x, usbctrl_vendorreq failed with %d\n", -+ addr, ret); -+ -+ return le32_to_cpu(*data); -+} -+ -+static u8 rtw_usb_read8_sync(struct rtw_dev *rtwdev, u32 addr) -+{ -+ return (u8)rtw_usb_read_sync(rtwdev, addr, 1); -+} -+ -+static u16 rtw_usb_read16_sync(struct rtw_dev *rtwdev, u32 addr) -+{ -+ return (u16)rtw_usb_read_sync(rtwdev, addr, 2); -+} -+ -+static u32 rtw_usb_read32_sync(struct rtw_dev *rtwdev, u32 addr) -+{ -+ return (u32)rtw_usb_read_sync(rtwdev, addr, 4); -+} -+ -+static void rtw_usb_write_async(struct rtw_dev *rtwdev, u32 addr, u32 val, -+ u16 len) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *udev = rtwusb->udev; -+ u8 request; -+ u16 wvalue; -+ u16 index; -+ __le32 data; -+ -+ request = RTW_USB_CMD_REQ; -+ index = RTW_USB_VENQT_CMD_IDX; /* n/a */ -+ wvalue = (u16)(addr & 0x0000ffff); -+ data = cpu_to_le32(val); -+ usbctrl_vendorreq_async_write(udev, request, wvalue, index, &data, len); -+} -+ -+static void rtw_usb_write8_async(struct rtw_dev *rtwdev, u32 addr, u8 val) -+{ -+ rtw_usb_write_async(rtwdev, addr, val, 1); -+} -+ -+static void rtw_usb_write16_async(struct rtw_dev *rtwdev, u32 addr, u16 val) -+{ -+ rtw_usb_write_async(rtwdev, addr, val, 2); -+} -+ -+static void rtw_usb_write32_async(struct rtw_dev *rtwdev, u32 addr, u32 val) -+{ -+ rtw_usb_write_async(rtwdev, addr, val, 4); -+} -+ -+static int rtw_usb_parse(struct rtw_dev *rtwdev, -+ struct usb_interface *interface) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_host_interface *host_interface = &interface->altsetting[0]; -+ struct usb_interface_descriptor *interface_desc = &host_interface->desc; -+ struct usb_endpoint_descriptor *endpoint; -+ struct usb_device *usbd = interface_to_usbdev(interface); -+ int num_out_pipes = 0; -+ int i; -+ u8 num; -+ -+ for (i = 0; i < interface_desc->bNumEndpoints; i++) { -+ endpoint = &host_interface->endpoint[i].desc; -+ num = usb_endpoint_num(endpoint); -+ -+ if (usb_endpoint_dir_in(endpoint) && -+ usb_endpoint_xfer_bulk(endpoint)) { -+ if (rtwusb->pipe_in) { -+ rtw_err(rtwdev, "IN pipes overflow\n"); -+ return -EINVAL; -+ } -+ -+ rtwusb->pipe_in = num; -+ } -+ -+ if (usb_endpoint_dir_in(endpoint) && -+ usb_endpoint_xfer_int(endpoint)) { -+ if (rtwusb->pipe_interrupt) { -+ rtw_err(rtwdev, "INT pipes overflow\n"); -+ return -EINVAL; -+ } -+ -+ rtwusb->pipe_interrupt = num; -+ } -+ -+ if (usb_endpoint_dir_out(endpoint) && -+ usb_endpoint_xfer_bulk(endpoint)) { -+ if (num_out_pipes >= ARRAY_SIZE(rtwusb->out_ep)) { -+ rtw_err(rtwdev, "OUT pipes overflow\n"); -+ return -EINVAL; -+ } -+ -+ rtwusb->out_ep[num_out_pipes++] = num; -+ } -+ } -+ -+ switch (usbd->speed) { -+ case USB_SPEED_LOW: -+ case USB_SPEED_FULL: -+ rtwusb->bulkout_size = RTW_USB_FULL_SPEED_BULK_SIZE; -+ break; -+ case USB_SPEED_HIGH: -+ rtwusb->bulkout_size = RTW_USB_HIGH_SPEED_BULK_SIZE; -+ break; -+ case USB_SPEED_SUPER: -+ rtwusb->bulkout_size = RTW_USB_SUPER_SPEED_BULK_SIZE; -+ break; -+ default: -+ rtw_err(rtwdev, "failed to detect usb speed\n"); -+ return -EINVAL; -+ } -+ -+ rtwdev->hci.bulkout_num = num_out_pipes; -+ -+ switch (num_out_pipes) { -+ case 4: -+ case 3: -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID4] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID5] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID6] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID7] = 1; -+ break; -+ case 2: -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 1; -+ break; -+ case 1: -+ break; -+ default: -+ rtw_err(rtwdev, "failed to get out_pipes(%d)\n", num_out_pipes); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+static void rtw_usb_txcb_enqueue(struct rtw_usb_txcb *txcb, struct sk_buff *skb) -+{ -+ skb_queue_tail(&txcb->tx_ack_queue, skb); -+} -+ -+static void rtw_usb_tx_agg_skb(struct rtw_usb *rtwusb, struct sk_buff_head *list, -+ struct sk_buff *skb_head, struct sk_buff *skb, -+ struct rtw_usb_txcb *txcb) -+{ -+ struct sk_buff *skb_iter; -+ unsigned long flags; -+ u8 *data_ptr; -+ int agg_num = 0, len, max_len; -+ -+ data_ptr = skb_head->data; -+ skb_iter = skb; -+ -+ while (skb_iter) { -+ memcpy(data_ptr, skb_iter->data, skb_iter->len); -+ len = ALIGN(skb_iter->len, 8); -+ skb_put(skb_head, len); -+ data_ptr += len; -+ agg_num++; -+ -+ rtw_usb_txcb_enqueue(txcb, skb_iter); -+ -+ spin_lock_irqsave(&list->lock, flags); -+ -+ skb_iter = skb_peek(list); -+ max_len = RTW_USB_MAX_XMITBUF_SZ - skb_head->len; -+ -+ if (skb_iter && skb_iter->len < max_len) -+ __skb_unlink(skb_iter, list); -+ else -+ skb_iter = NULL; -+ spin_unlock_irqrestore(&list->lock, flags); -+ } -+ -+ if (agg_num > 1) -+ rtw_usb_fill_tx_checksum(rtwusb, skb_head, agg_num); -+} -+ -+static void rtw_usb_indicate_tx_status(struct rtw_dev *rtwdev, -+ struct sk_buff *skb) -+{ -+ struct ieee80211_hw *hw = rtwdev->hw; -+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); -+ struct rtw_usb_tx_data *tx_data = rtw_usb_get_tx_data(skb); -+ -+ /* enqueue to wait for tx report */ -+ if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { -+ rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); -+ return; -+ } -+ -+ /* always ACK for others, then they won't be marked as drop */ -+ ieee80211_tx_info_clear_status(info); -+ if (info->flags & IEEE80211_TX_CTL_NO_ACK) -+ info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; -+ else -+ info->flags |= IEEE80211_TX_STAT_ACK; -+ -+ ieee80211_tx_status_irqsafe(hw, skb); -+} -+ -+static void rtw_usb_write_port_tx_complete(struct urb *urb) -+{ -+ struct rtw_usb_txcb *txcb = urb->context; -+ struct rtw_dev *rtwdev = txcb->rtwdev; -+ -+ while (true) { -+ struct sk_buff *skb = skb_dequeue(&txcb->tx_ack_queue); -+ if (!skb) -+ break; -+ -+ if (GET_TX_DESC_QSEL(skb->data) <= TX_DESC_QSEL_TID7) -+ rtw_usb_indicate_tx_status(rtwdev, skb); -+ else -+ dev_kfree_skb_any(skb); -+ } -+ -+ kfree(txcb); -+} -+ -+static int rtw_usb_write_port(struct rtw_dev *rtwdev, u8 qsel, struct sk_buff *skb, -+ usb_complete_t cb, void *context) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *usbd = rtwusb->udev; -+ struct urb *urb; -+ unsigned int pipe; -+ int ret; -+ int ep = rtwusb->qsel_to_ep[qsel]; -+ -+ pipe = usb_sndbulkpipe(usbd, rtwusb->out_ep[ep]); -+ urb = usb_alloc_urb(0, GFP_ATOMIC); -+ if (!urb) -+ return -ENOMEM; -+ -+ usb_fill_bulk_urb(urb, usbd, pipe, skb->data, skb->len, cb, context); -+ ret = usb_submit_urb(urb, GFP_ATOMIC); -+ -+ usb_free_urb(urb); -+ -+ return ret; -+} -+ -+static struct sk_buff *rtw_usb_tx_agg_check(struct rtw_usb *rtwusb, -+ struct sk_buff *skb, -+ int index, -+ struct rtw_usb_txcb *txcb) -+{ -+ struct sk_buff_head *list; -+ struct sk_buff *skb_head; -+ -+ list = &rtwusb->tx_queue[index]; -+ if (skb_queue_empty(list)) -+ return NULL; -+ -+ skb_head = dev_alloc_skb(RTW_USB_MAX_XMITBUF_SZ); -+ if (!skb_head) -+ return NULL; -+ -+ rtw_usb_tx_agg_skb(rtwusb, list, skb_head, skb, txcb); -+ -+ return skb_head; -+} -+ -+static void rtw_usb_tx_agg(struct rtw_usb *rtwusb, struct sk_buff *skb, int index) -+{ -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct sk_buff *skb_head; -+ struct rtw_usb_txcb *txcb; -+ u8 qsel; -+ -+ txcb = kmalloc(sizeof(*txcb), GFP_ATOMIC); -+ if (!txcb) -+ return; -+ -+ txcb->rtwdev = rtwdev; -+ skb_queue_head_init(&txcb->tx_ack_queue); -+ -+ skb_head = rtw_usb_tx_agg_check(rtwusb, skb, index, txcb); -+ if (!skb_head) { -+ skb_head = skb; -+ rtw_usb_txcb_enqueue(txcb, skb); -+ } -+ -+ qsel = GET_TX_DESC_QSEL(skb->data); -+ -+ rtw_usb_write_port(rtwdev, qsel, skb_head, -+ rtw_usb_write_port_tx_complete, txcb); -+ -+ if (skb_head != skb) -+ dev_kfree_skb(skb_head); -+} -+ -+static void rtw_usb_tx_handler(struct work_struct *work) -+{ -+ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, tx_work); -+ struct sk_buff *skb; -+ int index, limit; -+ -+ for (index = ARRAY_SIZE(rtwusb->tx_queue) - 1; index >= 0; index--) { -+ for (limit = 0; limit < 200; limit++) { -+ skb = skb_dequeue(&rtwusb->tx_queue[index]); -+ if (skb) -+ rtw_usb_tx_agg(rtwusb, skb, index); -+ else -+ break; -+ } -+ } -+} -+ -+static void rtw_usb_tx_queue_purge(struct rtw_usb *rtwusb) -+{ -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) -+ skb_queue_purge(&rtwusb->tx_queue[i]); -+} -+ -+static void rtw_usb_write_port_complete(struct urb *urb) -+{ -+ struct sk_buff *skb = urb->context; -+ -+ dev_kfree_skb_any(skb); -+} -+ -+static int rtw_usb_write_data(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *buf) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct sk_buff *skb; -+ unsigned int desclen, headsize, size; -+ u8 qsel; -+ int ret = 0; -+ -+ size = pkt_info->tx_pkt_size; -+ qsel = pkt_info->qsel; -+ desclen = chip->tx_pkt_desc_sz; -+ headsize = pkt_info->offset ? pkt_info->offset : desclen; -+ -+ skb = dev_alloc_skb(headsize + size); -+ if (unlikely(!skb)) -+ return -ENOMEM; -+ -+ skb_reserve(skb, headsize); -+ skb_put_data(skb, buf, size); -+ skb_push(skb, headsize); -+ memset(skb->data, 0, headsize); -+ rtw_tx_fill_tx_desc(pkt_info, skb); -+ rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, skb->data); -+ -+ ret = rtw_usb_write_port(rtwdev, qsel, skb, -+ rtw_usb_write_port_complete, skb); -+ if (unlikely(ret)) -+ rtw_err(rtwdev, "failed to do USB write, ret=%d\n", ret); -+ -+ return ret; -+} -+ -+static int rtw_usb_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, -+ u32 size) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct rtw_usb *rtwusb; -+ struct rtw_tx_pkt_info pkt_info = {0}; -+ u32 len, desclen; -+ -+ if (unlikely(!rtwdev)) -+ return -EINVAL; -+ -+ rtwusb = rtw_get_usb_priv(rtwdev); -+ if (unlikely(!rtwusb)) -+ return -EINVAL; -+ -+ pkt_info.tx_pkt_size = size; -+ pkt_info.qsel = TX_DESC_QSEL_BEACON; -+ -+ desclen = chip->tx_pkt_desc_sz; -+ len = desclen + size; -+ if (len % rtwusb->bulkout_size == 0) { -+ len += RTW_USB_PACKET_OFFSET_SZ; -+ pkt_info.offset = desclen + RTW_USB_PACKET_OFFSET_SZ; -+ pkt_info.pkt_offset = 1; -+ } else { -+ pkt_info.offset = desclen; -+ } -+ -+ return rtw_usb_write_data(rtwdev, &pkt_info, buf); -+} -+ -+static int rtw_usb_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) -+{ -+ struct rtw_tx_pkt_info pkt_info = {0}; -+ -+ pkt_info.tx_pkt_size = size; -+ pkt_info.qsel = TX_DESC_QSEL_H2C; -+ -+ return rtw_usb_write_data(rtwdev, &pkt_info, buf); -+} -+ -+static u8 rtw_usb_tx_queue_mapping_to_qsel(struct sk_buff *skb) -+{ -+ struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; -+ __le16 fc = hdr->frame_control; -+ u8 qsel; -+ -+ if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc))) -+ qsel = TX_DESC_QSEL_MGMT; -+ else if (skb_get_queue_mapping(skb) <= IEEE80211_AC_BK) -+ qsel = skb->priority; -+ else -+ qsel = TX_DESC_QSEL_BEACON; -+ -+ return qsel; -+} -+ -+static int rtw_usb_tx_write(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ struct sk_buff *skb) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct rtw_usb_tx_data *tx_data; -+ u8 *pkt_desc; -+ int ep; -+ -+ pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); -+ memset(pkt_desc, 0, chip->tx_pkt_desc_sz); -+ pkt_info->qsel = rtw_usb_tx_queue_mapping_to_qsel(skb); -+ ep = rtwusb->qsel_to_ep[pkt_info->qsel]; -+ rtw_tx_fill_tx_desc(pkt_info, skb); -+ rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, skb->data); -+ tx_data = rtw_usb_get_tx_data(skb); -+ tx_data->sn = pkt_info->sn; -+ -+ skb_queue_tail(&rtwusb->tx_queue[ep], skb); -+ -+ return 0; -+} -+ -+static void rtw_usb_tx_kick_off(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ queue_work(rtwusb->txwq, &rtwusb->tx_work); -+} -+ -+static void rtw_usb_rx_handler(struct work_struct *work) -+{ -+ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, rx_work); -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct rtw_rx_pkt_stat pkt_stat; -+ struct ieee80211_rx_status rx_status; -+ struct sk_buff *skb; -+ u32 pkt_desc_sz = chip->rx_pkt_desc_sz; -+ u32 pkt_offset; -+ u8 *rx_desc; -+ int limit; -+ -+ for (limit = 0; limit < 200; limit++) { -+ skb = skb_dequeue(&rtwusb->rx_queue); -+ if (!skb) -+ break; -+ -+ rx_desc = skb->data; -+ chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, -+ &rx_status); -+ pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + -+ pkt_stat.shift; -+ -+ if (pkt_stat.is_c2h) { -+ skb_put(skb, pkt_stat.pkt_len + pkt_offset); -+ rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, -+ skb); -+ continue; -+ } -+ -+ if (skb_queue_len(&rtwusb->rx_queue) >= RTW_USB_MAX_RXQ_LEN) { -+ rtw_err(rtwdev, "failed to get rx_queue, overflow\n"); -+ dev_kfree_skb_any(skb); -+ continue; -+ } -+ -+ skb_put(skb, pkt_stat.pkt_len); -+ skb_reserve(skb, pkt_offset); -+ memcpy(skb->cb, &rx_status, sizeof(rx_status)); -+ ieee80211_rx_irqsafe(rtwdev->hw, skb); -+ } -+} -+ -+static void rtw_usb_rx_data_put(struct rtw_usb *rtwusb, -+ struct rx_usb_ctrl_block *rxcb) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_move(&rxcb->list, &rtwusb->rx_data_free); -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+} -+ -+static void rtw_usb_read_port_complete(struct urb *urb) -+{ -+ struct rx_usb_ctrl_block *rxcb = urb->context; -+ struct rtw_dev *rtwdev = rxcb->rtwdev; -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct sk_buff *skb = rxcb->rx_skb; -+ -+ if (urb->status == 0) { -+ if (urb->actual_length >= RTW_USB_MAX_RECVBUF_SZ || -+ urb->actual_length < 24) { -+ rtw_err(rtwdev, "failed to get urb length:%d\n", -+ urb->actual_length); -+ if (skb) -+ dev_kfree_skb_any(skb); -+ } else { -+ skb_queue_tail(&rtwusb->rx_queue, skb); -+ queue_work(rtwusb->rxwq, &rtwusb->rx_work); -+ } -+ -+ rtw_usb_rx_data_put(rtwusb, rxcb); -+ queue_work(rtwusb->rxwq, &rtwusb->rx_refill_work); -+ } else { -+ switch (urb->status) { -+ case -EINVAL: -+ case -EPIPE: -+ case -ENODEV: -+ case -ESHUTDOWN: -+ case -ENOENT: -+ case -EPROTO: -+ case -EILSEQ: -+ case -ETIME: -+ case -ECOMM: -+ case -EOVERFLOW: -+ case -EINPROGRESS: -+ break; -+ default: -+ rtw_err(rtwdev, "status unknown=%d\n", urb->status); -+ break; -+ } -+ if (skb) -+ dev_kfree_skb_any(skb); -+ } -+} -+ -+static void rtw_usb_rx_refill_work(struct work_struct *work) -+{ -+ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, rx_refill_work); -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct rx_usb_ctrl_block *rxcb; -+ unsigned long flags; -+ int error; -+ -+ do { -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ -+ rxcb = list_first_entry_or_null(&rtwusb->rx_data_free, -+ struct rx_usb_ctrl_block, list); -+ -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ if (!rxcb) -+ return; -+ -+ rxcb->rx_skb = alloc_skb(RTW_USB_MAX_RECVBUF_SZ, GFP_KERNEL); -+ if (!rxcb->rx_skb) { -+ rtw_err(rtwdev, "could not allocate rx skbuff\n"); -+ return; -+ } -+ -+ usb_fill_bulk_urb(rxcb->rx_urb, rtwusb->udev, -+ usb_rcvbulkpipe(rtwusb->udev, rtwusb->pipe_in), -+ rxcb->rx_skb->data, RTW_USB_MAX_RECVBUF_SZ, -+ rtw_usb_read_port_complete, rxcb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_move(&rxcb->list, &rtwusb->rx_data_used); -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ -+ error = usb_submit_urb(rxcb->rx_urb, GFP_KERNEL); -+ if (error) { -+ kfree_skb(rxcb->rx_skb); -+ if (error != -ENODEV) -+ rtw_err(rtwdev, "Err sending rx data urb %d\n", -+ error); -+ rtw_usb_rx_data_put(rtwusb, rxcb); -+ -+ return; -+ } -+ } while (true); -+} -+ -+static void rtw_usb_cancel_rx_bufs(struct rtw_usb *rtwusb) -+{ -+ struct rx_usb_ctrl_block *rxcb; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ -+ while (true) { -+ rxcb = list_first_entry_or_null(&rtwusb->rx_data_used, -+ struct rx_usb_ctrl_block, list); -+ -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ -+ if (!rxcb) -+ break; -+ -+ usb_kill_urb(rxcb->rx_urb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_move(&rxcb->list, &rtwusb->rx_data_free); -+ } -+} -+ -+static void rtw_usb_free_rx_bufs(struct rtw_usb *rtwusb) -+{ -+ struct rx_usb_ctrl_block *rxcb; -+ unsigned long flags; -+ -+ rtw_usb_cancel_rx_bufs(rtwusb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ -+ while (true) { -+ rxcb = list_first_entry_or_null(&rtwusb->rx_data_free, struct rx_usb_ctrl_block, list); -+ -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ -+ if (!rxcb) -+ break; -+ -+ usb_free_urb(rxcb->rx_urb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_del(&rxcb->list); -+ } -+} -+ -+static int rtw_usb_alloc_rx_bufs(struct rtw_usb *rtwusb) -+{ -+ int i; -+ -+ for (i = 0; i < RTW_USB_RXCB_NUM; i++) { -+ struct rx_usb_ctrl_block *rxcb = &rtwusb->rx_cb[i]; -+ -+ rxcb->rtwdev = rtwusb->rtwdev; -+ rxcb->rx_urb = usb_alloc_urb(0, GFP_KERNEL); -+ if (!rxcb->rx_urb) -+ goto err; -+ list_add_tail(&rxcb->list, &rtwusb->rx_data_free); -+ } -+ -+ return 0; -+err: -+ rtw_usb_free_rx_bufs(rtwusb); -+ return -ENOMEM; -+} -+ -+static int rtw_usb_setup(struct rtw_dev *rtwdev) -+{ -+ /* empty function for rtw_hci_ops */ -+ return 0; -+} -+ -+static int rtw_usb_start(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ queue_work(rtwusb->rxwq, &rtwusb->rx_refill_work); -+ -+ return 0; -+} -+ -+static void rtw_usb_stop(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ cancel_work_sync(&rtwusb->rx_refill_work); -+ rtw_usb_cancel_rx_bufs(rtwusb); -+} -+ -+static void rtw_usb_deep_ps(struct rtw_dev *rtwdev, bool enter) -+{ -+ /* empty function for rtw_hci_ops */ -+} -+ -+static void rtw_usb_link_ps(struct rtw_dev *rtwdev, bool enter) -+{ -+ /* empty function for rtw_hci_ops */ -+} -+ -+static void rtw_usb_interface_cfg(struct rtw_dev *rtwdev) -+{ -+ /* empty function for rtw_hci_ops */ -+} -+ -+static struct rtw_hci_ops rtw_usb_ops = { -+ .tx_write = rtw_usb_tx_write, -+ .tx_kick_off = rtw_usb_tx_kick_off, -+ .setup = rtw_usb_setup, -+ .start = rtw_usb_start, -+ .stop = rtw_usb_stop, -+ .deep_ps = rtw_usb_deep_ps, -+ .link_ps = rtw_usb_link_ps, -+ .interface_cfg = rtw_usb_interface_cfg, -+ -+ .write8 = rtw_usb_write8_async, -+ .write16 = rtw_usb_write16_async, -+ .write32 = rtw_usb_write32_async, -+ .read8 = rtw_usb_read8_sync, -+ .read16 = rtw_usb_read16_sync, -+ .read32 = rtw_usb_read32_sync, -+ -+ .write_data_rsvd_page = rtw_usb_write_data_rsvd_page, -+ .write_data_h2c = rtw_usb_write_data_h2c, -+}; -+ -+static int rtw_usb_init_rx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ rtwusb->rxwq = create_singlethread_workqueue("rtw88_usb: rx wq"); -+ if (!rtwusb->rxwq) { -+ rtw_err(rtwdev, "failed to create RX work queue\n"); -+ return -ENOMEM; -+ } -+ -+ skb_queue_head_init(&rtwusb->rx_queue); -+ -+ INIT_WORK(&rtwusb->rx_work, rtw_usb_rx_handler); -+ -+ return 0; -+} -+ -+static void rtw_usb_deinit_rx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ skb_queue_purge(&rtwusb->rx_queue); -+ -+ flush_workqueue(rtwusb->rxwq); -+ destroy_workqueue(rtwusb->rxwq); -+} -+ -+static int rtw_usb_init_tx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ int i; -+ -+ rtwusb->txwq = create_singlethread_workqueue("rtw88_usb: tx wq"); -+ if (!rtwusb->txwq) { -+ rtw_err(rtwdev, "failed to create TX work queue\n"); -+ return -ENOMEM; -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) -+ skb_queue_head_init(&rtwusb->tx_queue[i]); -+ -+ INIT_WORK(&rtwusb->tx_work, rtw_usb_tx_handler); -+ -+ return 0; -+} -+ -+static void rtw_usb_deinit_tx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ rtw_usb_tx_queue_purge(rtwusb); -+ flush_workqueue(rtwusb->txwq); -+ destroy_workqueue(rtwusb->txwq); -+} -+ -+static int rtw_usb_intf_init(struct rtw_dev *rtwdev, -+ struct usb_interface *intf) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *udev = usb_get_dev(interface_to_usbdev(intf)); -+ int ret; -+ -+ rtwusb->udev = udev; -+ ret = rtw_usb_parse(rtwdev, intf); -+ if (ret) -+ return ret; -+ -+ rtwusb->usb_data = kcalloc(RTW_USB_MAX_RX_COUNT, sizeof(u32), -+ GFP_KERNEL); -+ if (!rtwusb->usb_data) -+ return -ENOMEM; -+ -+ usb_set_intfdata(intf, rtwdev->hw); -+ -+ SET_IEEE80211_DEV(rtwdev->hw, &intf->dev); -+ spin_lock_init(&rtwusb->usb_lock); -+ -+ return 0; -+} -+ -+static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev, -+ struct usb_interface *intf) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ usb_put_dev(rtwusb->udev); -+ usb_set_intfdata(intf, NULL); -+} -+ -+int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) -+{ -+ struct rtw_dev *rtwdev; -+ struct ieee80211_hw *hw; -+ struct rtw_usb *rtwusb; -+ int drv_data_size; -+ int ret; -+ -+ drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_usb); -+ hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); -+ if (!hw) -+ return -ENOMEM; -+ -+ rtwdev = hw->priv; -+ rtwdev->hw = hw; -+ rtwdev->dev = &intf->dev; -+ rtwdev->chip = (struct rtw_chip_info *)id->driver_info; -+ rtwdev->hci.ops = &rtw_usb_ops; -+ rtwdev->hci.type = RTW_HCI_TYPE_USB; -+ -+ rtwusb = rtw_get_usb_priv(rtwdev); -+ rtwusb->rtwdev = rtwdev; -+ -+ INIT_WORK(&rtwusb->rx_refill_work, rtw_usb_rx_refill_work); -+ INIT_LIST_HEAD(&rtwusb->rx_data_free); -+ INIT_LIST_HEAD(&rtwusb->rx_data_used); -+ spin_lock_init(&rtwusb->rx_data_list_lock); -+ -+ ret = rtw_usb_alloc_rx_bufs(rtwusb); -+ if (ret) -+ return ret; -+ -+ ret = rtw_core_init(rtwdev); -+ if (ret) -+ goto err_release_hw; -+ -+ ret = rtw_usb_intf_init(rtwdev, intf); -+ if (ret) { -+ rtw_err(rtwdev, "failed to init USB interface\n"); -+ goto err_deinit_core; -+ } -+ -+ ret = rtw_usb_init_tx(rtwdev); -+ if (ret) { -+ rtw_err(rtwdev, "failed to init USB TX\n"); -+ goto err_destroy_usb; -+ } -+ -+ ret = rtw_usb_init_rx(rtwdev); -+ if (ret) { -+ rtw_err(rtwdev, "failed to init USB RX\n"); -+ goto err_destroy_txwq; -+ } -+ -+ ret = rtw_chip_info_setup(rtwdev); -+ if (ret) { -+ rtw_err(rtwdev, "failed to setup chip information\n"); -+ goto err_destroy_rxwq; -+ } -+ -+ ret = rtw_register_hw(rtwdev, rtwdev->hw); -+ if (ret) { -+ rtw_err(rtwdev, "failed to register hw\n"); -+ goto err_destroy_rxwq; -+ } -+ -+ return 0; -+ -+err_destroy_rxwq: -+ rtw_usb_deinit_rx(rtwdev); -+ -+err_destroy_txwq: -+ rtw_usb_deinit_tx(rtwdev); -+ -+err_destroy_usb: -+ rtw_usb_intf_deinit(rtwdev, intf); -+ -+err_deinit_core: -+ rtw_core_deinit(rtwdev); -+ -+err_release_hw: -+ ieee80211_free_hw(hw); -+ -+ return ret; -+} -+EXPORT_SYMBOL(rtw_usb_probe); -+ -+void rtw_usb_disconnect(struct usb_interface *intf) -+{ -+ struct ieee80211_hw *hw = usb_get_intfdata(intf); -+ struct rtw_dev *rtwdev; -+ struct rtw_usb *rtwusb; -+ -+ if (!hw) -+ return; -+ -+ rtwdev = hw->priv; -+ rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ rtw_unregister_hw(rtwdev, hw); -+ rtw_usb_deinit_tx(rtwdev); -+ rtw_usb_deinit_rx(rtwdev); -+ -+ if (rtwusb->udev->state != USB_STATE_NOTATTACHED) -+ usb_reset_device(rtwusb->udev); -+ -+ rtw_usb_free_rx_bufs(rtwusb); -+ -+ rtw_usb_intf_deinit(rtwdev, intf); -+ rtw_core_deinit(rtwdev); -+ ieee80211_free_hw(hw); -+} -+EXPORT_SYMBOL(rtw_usb_disconnect); -+ -+MODULE_AUTHOR("Realtek Corporation"); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless USB driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/usb.h b/drivers/net/wireless/realtek/rtw88/usb.h -new file mode 100644 -index 000000000000..4d714372f265 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/usb.h -@@ -0,0 +1,109 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_USB_H_ -+#define __RTW_USB_H_ -+ -+#define FW_8192C_START_ADDRESS 0x1000 -+#define FW_8192C_END_ADDRESS 0x5FFF -+ -+#define RTW_USB_MAX_RX_COUNT 100 -+#define RTW_USB_VENQT_MAX_BUF_SIZE 254 -+#define MAX_USBCTRL_VENDORREQ_TIMES 10 -+ -+#define RTW_USB_CMD_READ 0xc0 -+#define RTW_USB_CMD_WRITE 0x40 -+#define RTW_USB_CMD_REQ 0x05 -+ -+#define RTW_USB_VENQT_CMD_IDX 0x00 -+ -+#define RTW_USB_SUPER_SPEED_BULK_SIZE 1024 -+#define RTW_USB_HIGH_SPEED_BULK_SIZE 512 -+#define RTW_USB_FULL_SPEED_BULK_SIZE 64 -+ -+#define RTW_USB_TX_SEL_HQ BIT(0) -+#define RTW_USB_TX_SEL_LQ BIT(1) -+#define RTW_USB_TX_SEL_NQ BIT(2) -+#define RTW_USB_TX_SEL_EQ BIT(3) -+ -+#define RTW_USB_BULK_IN_ADDR 0x80 -+#define RTW_USB_INT_IN_ADDR 0x81 -+ -+#define RTW_USB_HW_QUEUE_ENTRY 8 -+ -+#define RTW_USB_PACKET_OFFSET_SZ 8 -+#define RTW_USB_MAX_XMITBUF_SZ (1592 * 3) -+#define RTW_USB_MAX_RECVBUF_SZ 32768 -+ -+#define RTW_USB_RECVBUFF_ALIGN_SZ 8 -+ -+#define RTW_USB_RXAGG_SIZE 6 -+#define RTW_USB_RXAGG_TIMEOUT 10 -+ -+#define RTW_USB_RXCB_NUM 4 -+ -+#define RTW_USB_EP_MAX 4 -+ -+#define TX_DESC_QSEL_MAX 20 -+ -+static inline struct rtw_usb *rtw_get_usb_priv(struct rtw_dev *rtwdev) -+{ -+ return (struct rtw_usb *)rtwdev->priv; -+} -+ -+struct rx_usb_ctrl_block { -+ struct rtw_dev *rtwdev; -+ struct urb *rx_urb; -+ struct sk_buff *rx_skb; -+ struct list_head list; -+}; -+ -+struct rtw_usb_tx_data { -+ u8 sn; -+}; -+ -+struct rtw_usb { -+ struct rtw_dev *rtwdev; -+ struct usb_device *udev; -+ -+ spinlock_t rx_data_list_lock; -+ struct work_struct rx_refill_work; -+ struct list_head rx_data_free; -+ struct list_head rx_data_used; -+ -+ spinlock_t usb_lock; -+ __le32 *usb_data; -+ int usb_data_index; -+ -+ u32 bulkout_size; -+ u8 pipe_interrupt; -+ u8 pipe_in; -+ u8 out_ep[RTW_USB_EP_MAX]; -+ u8 qsel_to_ep[TX_DESC_QSEL_MAX]; -+ u8 usb_txagg_num; -+ -+ struct workqueue_struct *txwq, *rxwq; -+ -+ struct sk_buff_head tx_queue[RTW_USB_EP_MAX]; -+ struct work_struct tx_work; -+ -+ struct rx_usb_ctrl_block rx_cb[RTW_USB_RXCB_NUM]; -+ struct sk_buff_head rx_queue; -+ struct work_struct rx_work; -+}; -+ -+static inline struct rtw_usb_tx_data *rtw_usb_get_tx_data(struct sk_buff *skb) -+{ -+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); -+ -+ BUILD_BUG_ON(sizeof(struct rtw_usb_tx_data) > -+ sizeof(info->status.status_driver_data)); -+ -+ return (struct rtw_usb_tx_data *)info->status.status_driver_data; -+} -+ -+int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id); -+void rtw_usb_disconnect(struct usb_interface *intf); -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/util.c b/drivers/net/wireless/realtek/rtw88/util.c -index 2c515af214e7..db55dbd5c533 100644 ---- a/drivers/net/wireless/realtek/rtw88/util.c -+++ b/drivers/net/wireless/realtek/rtw88/util.c -@@ -105,3 +105,95 @@ void rtw_desc_to_mcsrate(u16 rate, u8 *mcs, u8 *nss) - *mcs = rate - DESC_RATEMCS0; - } - } -+ -+struct rtw_stas_entry { -+ struct list_head list; -+ struct ieee80211_sta *sta; -+}; -+ -+struct rtw_iter_stas_data { -+ struct rtw_dev *rtwdev; -+ struct list_head list; -+}; -+ -+void rtw_collect_sta_iter(void *data, struct ieee80211_sta *sta) -+{ -+ struct rtw_iter_stas_data *iter_stas = data; -+ struct rtw_stas_entry *stas_entry; -+ -+ stas_entry = kmalloc(sizeof(*stas_entry), GFP_ATOMIC); -+ if (!stas_entry) -+ return; -+ -+ stas_entry->sta = sta; -+ list_add_tail(&stas_entry->list, &iter_stas->list); -+} -+ -+void rtw_iterate_stas(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, -+ struct ieee80211_sta *sta), -+ void *data) -+{ -+ struct rtw_iter_stas_data iter_data; -+ struct rtw_stas_entry *sta_entry, *tmp; -+ -+ iter_data.rtwdev = rtwdev; -+ INIT_LIST_HEAD(&iter_data.list); -+ -+ ieee80211_iterate_stations_atomic(rtwdev->hw, rtw_collect_sta_iter, -+ &iter_data); -+ -+ list_for_each_entry_safe(sta_entry, tmp, &iter_data.list, -+ list) { -+ list_del_init(&sta_entry->list); -+ iterator(data, sta_entry->sta); -+ kfree(sta_entry); -+ } -+} -+ -+struct rtw_vifs_entry { -+ struct list_head list; -+ struct ieee80211_vif *vif; -+ u8 mac[ETH_ALEN]; -+}; -+ -+struct rtw_iter_vifs_data { -+ struct rtw_dev *rtwdev; -+ struct list_head list; -+}; -+ -+void rtw_collect_vif_iter(void *data, u8 *mac, struct ieee80211_vif *vif) -+{ -+ struct rtw_iter_vifs_data *iter_stas = data; -+ struct rtw_vifs_entry *vifs_entry; -+ -+ vifs_entry = kmalloc(sizeof(*vifs_entry), GFP_ATOMIC); -+ if (!vifs_entry) -+ return; -+ -+ vifs_entry->vif = vif; -+ ether_addr_copy(vifs_entry->mac, mac); -+ list_add_tail(&vifs_entry->list, &iter_stas->list); -+} -+ -+void rtw_iterate_vifs(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, u8 *mac, -+ struct ieee80211_vif *vif), -+ void *data) -+{ -+ struct rtw_iter_vifs_data iter_data; -+ struct rtw_vifs_entry *vif_entry, *tmp; -+ -+ iter_data.rtwdev = rtwdev; -+ INIT_LIST_HEAD(&iter_data.list); -+ -+ ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, -+ IEEE80211_IFACE_ITER_NORMAL, rtw_collect_vif_iter, &iter_data); -+ -+ list_for_each_entry_safe(vif_entry, tmp, &iter_data.list, -+ list) { -+ list_del_init(&vif_entry->list); -+ iterator(data, vif_entry->mac, vif_entry->vif); -+ kfree(vif_entry); -+ } -+} -diff --git a/drivers/net/wireless/realtek/rtw88/util.h b/drivers/net/wireless/realtek/rtw88/util.h -index 0c23b5069be0..dc8965525400 100644 ---- a/drivers/net/wireless/realtek/rtw88/util.h -+++ b/drivers/net/wireless/realtek/rtw88/util.h -@@ -7,9 +7,6 @@ - - struct rtw_dev; - --#define rtw_iterate_vifs(rtwdev, iterator, data) \ -- ieee80211_iterate_active_interfaces(rtwdev->hw, \ -- IEEE80211_IFACE_ITER_NORMAL, iterator, data) - #define rtw_iterate_vifs_atomic(rtwdev, iterator, data) \ - ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, \ - IEEE80211_IFACE_ITER_NORMAL, iterator, data) -@@ -20,6 +17,15 @@ struct rtw_dev; - #define rtw_iterate_keys_rcu(rtwdev, vif, iterator, data) \ - ieee80211_iter_keys_rcu((rtwdev)->hw, vif, iterator, data) - -+void rtw_iterate_vifs(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, u8 *mac, -+ struct ieee80211_vif *vif), -+ void *data); -+void rtw_iterate_stas(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, -+ struct ieee80211_sta *sta), -+ void *data); -+ - static inline u8 *get_hdr_bssid(struct ieee80211_hdr *hdr) - { - __le16 fc = hdr->frame_control; --- -2.37.3 - diff --git a/6.0/testing/0005-maple-lru-khugepage.patch b/6.0/0008-Introducing-the-Maple-Tree.patch similarity index 88% rename from 6.0/testing/0005-maple-lru-khugepage.patch rename to 6.0/0008-Introducing-the-Maple-Tree.patch index fd3995c3..ce059afb 100644 --- a/6.0/testing/0005-maple-lru-khugepage.patch +++ b/6.0/0008-Introducing-the-Maple-Tree.patch @@ -1,24 +1,96 @@ -From c8708680c88426cc0d0f2769502a803364b1e120 Mon Sep 17 00:00:00 2001 +From f7046da0d2b40d6725122f9d3ed897a12a8fda63 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Tue, 27 Sep 2022 18:01:21 +0200 -Subject: [PATCH 05/13] maple-lru-khugepage +Date: Wed, 28 Sep 2022 00:27:32 +0200 +Subject: [PATCH 08/16] Introducing the Maple Tree + +The maple tree is an RCU-safe range based B-tree designed to use modern +processor cache efficiently. There are a number of places in the kernel +that a non-overlapping range-based tree would be beneficial, especially +one with a simple interface. If you use an rbtree with other data +structures to improve performance or an interval tree to track +non-overlapping ranges, then this is for you. + +The tree has a branching factor of 10 for non-leaf nodes and 16 for leaf +nodes. With the increased branching factor, it is significantly shorter +than the rbtree so it has fewer cache misses. The removal of the linked +list between subsequent entries also reduces the cache misses and the need +to pull in the previous and next VMA during many tree alterations. + +The first user that is covered in this patch set is the vm_area_struct, +where three data structures are replaced by the maple tree: the augmented +rbtree, the vma cache, and the linked list of VMAs in the mm_struct. The +long term goal is to reduce or remove the mmap_lock contention. + +The plan is to get to the point where we use the maple tree in RCU mode. +Readers will not block for writers. A single write operation will be +allowed at a time. A reader re-walks if stale data is encountered. VMAs +would be RCU enabled and this mode would be entered once multiple tasks +are using the mm_struct. + +Davidlor said + +: Yes I like the maple tree, and at this stage I don't think we can ask for +: more from this series wrt the MM - albeit there seems to still be some +: folks reporting breakage. Fundamentally I see Liam's work to (re)move +: complexity out of the MM (not to say that the actual maple tree is not +: complex) by consolidating the three complimentary data structures very +: much worth it considering performance does not take a hit. This was very +: much a turn off with the range locking approach, which worst case scenario +: incurred in prohibitive overhead. Also as Liam and Matthew have +: mentioned, RCU opens up a lot of nice performance opportunities, and in +: addition academia[1] has shown outstanding scalability of address spaces +: with the foundation of replacing the locked rbtree with RCU aware trees. + +A similar work has been discovered in the academic press + + https://pdos.csail.mit.edu/papers/rcuvm:asplos12.pdf + +Sheer coincidence. We designed our tree with the intention of solving the +hardest problem first. Upon settling on a b-tree variant and a rough +outline, we researched ranged based b-trees and RCU b-trees and did find +that article. So it was nice to find reassurances that we were on the +right path, but our design choice of using ranges made that paper unusable +for us. + +This patch (of 70): + +The maple tree is an RCU-safe range based B-tree designed to use modern +processor cache efficiently. There are a number of places in the kernel +that a non-overlapping range-based tree would be beneficial, especially +one with a simple interface. If you use an rbtree with other data +structures to improve performance or an interval tree to track +non-overlapping ranges, then this is for you. + +The tree has a branching factor of 10 for non-leaf nodes and 16 for leaf +nodes. With the increased branching factor, it is significantly shorter +than the rbtree so it has fewer cache misses. The removal of the linked +list between subsequent entries also reduces the cache misses and the need +to pull in the previous and next VMA during many tree alterations. + +The first user that is covered in this patch set is the vm_area_struct, +where three data structures are replaced by the maple tree: the augmented +rbtree, the vma cache, and the linked list of VMAs in the mm_struct. The +long term goal is to reduce or remove the mmap_lock contention. + +The plan is to get to the point where we use the maple tree in RCU mode. +Readers will not block for writers. A single write operation will be +allowed at a time. A reader re-walks if stale data is encountered. VMAs +would be RCU enabled and this mode would be entered once multiple tasks +are using the mm_struct. + +There is additional BUG_ON() calls added within the tree, most of which +are in debug code. These will be replaced with a WARN_ON() call in the +future. There is also additional BUG_ON() calls within the code which +will also be reduced in number at a later date. These exist to catch +things such as out-of-range accesses which would crash anyways. Signed-off-by: Peter Jung --- - Documentation/admin-guide/mm/index.rst | 1 + - Documentation/admin-guide/mm/multigen_lru.rst | 162 + Documentation/core-api/index.rst | 1 + Documentation/core-api/maple_tree.rst | 217 + - Documentation/mm/index.rst | 1 + - Documentation/mm/multigen_lru.rst | 159 + MAINTAINERS | 12 + - arch/Kconfig | 8 + - arch/alpha/include/uapi/asm/mman.h | 2 + - arch/arm64/include/asm/pgtable.h | 15 +- arch/arm64/kernel/elfcore.c | 16 +- arch/arm64/kernel/vdso.c | 3 +- - arch/mips/include/uapi/asm/mman.h | 2 + - arch/parisc/include/uapi/asm/mman.h | 2 + arch/parisc/kernel/cache.c | 9 +- arch/powerpc/kernel/vdso.c | 6 +- arch/powerpc/mm/book3s32/tlb.c | 11 +- @@ -27,12 +99,8 @@ Signed-off-by: Peter Jung arch/s390/kernel/vdso.c | 3 +- arch/s390/mm/gmap.c | 6 +- arch/um/kernel/tlb.c | 14 +- - arch/x86/Kconfig | 1 + arch/x86/entry/vdso/vma.c | 9 +- - arch/x86/include/asm/pgtable.h | 9 +- arch/x86/kernel/tboot.c | 2 +- - arch/x86/mm/pgtable.c | 5 +- - arch/xtensa/include/uapi/asm/mman.h | 2 + arch/xtensa/kernel/syscall.c | 18 +- drivers/firmware/efi/efi.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 14 +- @@ -40,88 +108,64 @@ Signed-off-by: Peter Jung drivers/tee/optee/call.c | 18 +- drivers/xen/privcmd.c | 2 +- fs/coredump.c | 34 +- - fs/exec.c | 14 +- - fs/fuse/dev.c | 3 +- + fs/exec.c | 12 +- fs/proc/base.c | 5 +- fs/proc/internal.h | 2 +- - fs/proc/task_mmu.c | 76 +- + fs/proc/task_mmu.c | 74 +- fs/proc/task_nommu.c | 45 +- fs/userfaultfd.c | 62 +- - include/linux/cgroup.h | 15 +- - include/linux/huge_mm.h | 23 +- include/linux/maple_tree.h | 685 + - include/linux/memcontrol.h | 36 + - include/linux/mm.h | 83 +- - include/linux/mm_inline.h | 231 +- - include/linux/mm_types.h | 119 +- + include/linux/mm.h | 78 +- + include/linux/mm_types.h | 43 +- include/linux/mm_types_task.h | 12 - - include/linux/mmzone.h | 216 + - include/linux/nodemask.h | 1 + - include/linux/page-flags-layout.h | 16 +- - include/linux/page-flags.h | 4 +- - include/linux/pgtable.h | 17 +- - include/linux/sched.h | 5 +- - include/linux/swap.h | 4 + + include/linux/sched.h | 1 - include/linux/userfaultfd_k.h | 7 +- include/linux/vm_event_item.h | 4 - include/linux/vmacache.h | 28 - include/linux/vmstat.h | 6 - - include/trace/events/huge_memory.h | 1 + include/trace/events/maple_tree.h | 123 + include/trace/events/mmap.h | 73 + - include/uapi/asm-generic/mman-common.h | 2 + init/main.c | 2 + ipc/shm.c | 21 +- kernel/acct.c | 11 +- - kernel/bounds.c | 7 + kernel/bpf/task_iter.c | 10 +- - kernel/cgroup/cgroup-internal.h | 1 - kernel/debug/debug_core.c | 12 - kernel/events/core.c | 3 +- kernel/events/uprobes.c | 9 +- - kernel/exit.c | 1 + - kernel/fork.c | 71 +- - kernel/sched/core.c | 1 + + kernel/fork.c | 62 +- kernel/sched/fair.c | 10 +- lib/Kconfig.debug | 17 +- lib/Makefile | 2 +- lib/maple_tree.c | 7130 +++ lib/test_maple_tree.c | 38307 ++++++++++++++++ - mm/Kconfig | 26 + mm/Makefile | 2 +- mm/damon/vaddr-test.h | 36 +- mm/damon/vaddr.c | 53 +- mm/debug.c | 14 +- mm/gup.c | 7 +- - mm/huge_memory.c | 39 +- + mm/huge_memory.c | 4 +- mm/init-mm.c | 4 +- - mm/internal.h | 11 +- - mm/khugepaged.c | 772 +- - mm/ksm.c | 28 +- - mm/madvise.c | 11 +- - mm/memcontrol.c | 34 +- - mm/memory.c | 77 +- + mm/internal.h | 8 +- + mm/khugepaged.c | 11 +- + mm/ksm.c | 18 +- + mm/madvise.c | 2 +- + mm/memcontrol.c | 6 +- + mm/memory.c | 34 +- mm/mempolicy.c | 56 +- mm/mlock.c | 37 +- - mm/mm_init.c | 6 +- mm/mmap.c | 2154 +- - mm/mmzone.c | 2 + mm/mprotect.c | 8 +- mm/mremap.c | 22 +- mm/msync.c | 2 +- mm/nommu.c | 260 +- mm/oom_kill.c | 3 +- mm/pagewalk.c | 2 +- - mm/rmap.c | 21 +- - mm/swap.c | 54 +- mm/swapfile.c | 4 +- mm/util.c | 32 - mm/vmacache.c | 117 - - mm/vmscan.c | 3247 +- + mm/vmscan.c | 12 +- mm/vmstat.c | 4 - - mm/workingset.c | 110 +- tools/include/linux/slab.h | 4 + - tools/include/uapi/asm-generic/mman-common.h | 2 + tools/testing/radix-tree/.gitignore | 2 + tools/testing/radix-tree/Makefile | 9 +- tools/testing/radix-tree/generated/autoconf.h | 1 + @@ -131,11 +175,8 @@ Signed-off-by: Peter Jung tools/testing/radix-tree/linux/maple_tree.h | 7 + tools/testing/radix-tree/maple.c | 59 + .../radix-tree/trace/events/maple_tree.h | 5 + - tools/testing/selftests/vm/khugepaged.c | 563 +- - 127 files changed, 53649 insertions(+), 2781 deletions(-) - create mode 100644 Documentation/admin-guide/mm/multigen_lru.rst + 89 files changed, 48582 insertions(+), 1894 deletions(-) create mode 100644 Documentation/core-api/maple_tree.rst - create mode 100644 Documentation/mm/multigen_lru.rst create mode 100644 include/linux/maple_tree.h delete mode 100644 include/linux/vmacache.h create mode 100644 include/trace/events/maple_tree.h @@ -146,186 +187,6 @@ Signed-off-by: Peter Jung create mode 100644 tools/testing/radix-tree/maple.c create mode 100644 tools/testing/radix-tree/trace/events/maple_tree.h -diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst -index 1bd11118dfb1..d1064e0ba34a 100644 ---- a/Documentation/admin-guide/mm/index.rst -+++ b/Documentation/admin-guide/mm/index.rst -@@ -32,6 +32,7 @@ the Linux memory management. - idle_page_tracking - ksm - memory-hotplug -+ multigen_lru - nommu-mmap - numa_memory_policy - numaperf -diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst -new file mode 100644 -index 000000000000..33e068830497 ---- /dev/null -+++ b/Documentation/admin-guide/mm/multigen_lru.rst -@@ -0,0 +1,162 @@ -+.. SPDX-License-Identifier: GPL-2.0 -+ -+============= -+Multi-Gen LRU -+============= -+The multi-gen LRU is an alternative LRU implementation that optimizes -+page reclaim and improves performance under memory pressure. Page -+reclaim decides the kernel's caching policy and ability to overcommit -+memory. It directly impacts the kswapd CPU usage and RAM efficiency. -+ -+Quick start -+=========== -+Build the kernel with the following configurations. -+ -+* ``CONFIG_LRU_GEN=y`` -+* ``CONFIG_LRU_GEN_ENABLED=y`` -+ -+All set! -+ -+Runtime options -+=============== -+``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the -+following subsections. -+ -+Kill switch -+----------- -+``enabled`` accepts different values to enable or disable the -+following components. Its default value depends on -+``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled -+unless some of them have unforeseen side effects. Writing to -+``enabled`` has no effect when a component is not supported by the -+hardware, and valid values will be accepted even when the main switch -+is off. -+ -+====== =============================================================== -+Values Components -+====== =============================================================== -+0x0001 The main switch for the multi-gen LRU. -+0x0002 Clearing the accessed bit in leaf page table entries in large -+ batches, when MMU sets it (e.g., on x86). This behavior can -+ theoretically worsen lock contention (mmap_lock). If it is -+ disabled, the multi-gen LRU will suffer a minor performance -+ degradation for workloads that contiguously map hot pages, -+ whose accessed bits can be otherwise cleared by fewer larger -+ batches. -+0x0004 Clearing the accessed bit in non-leaf page table entries as -+ well, when MMU sets it (e.g., on x86). This behavior was not -+ verified on x86 varieties other than Intel and AMD. If it is -+ disabled, the multi-gen LRU will suffer a negligible -+ performance degradation. -+[yYnN] Apply to all the components above. -+====== =============================================================== -+ -+E.g., -+:: -+ -+ echo y >/sys/kernel/mm/lru_gen/enabled -+ cat /sys/kernel/mm/lru_gen/enabled -+ 0x0007 -+ echo 5 >/sys/kernel/mm/lru_gen/enabled -+ cat /sys/kernel/mm/lru_gen/enabled -+ 0x0005 -+ -+Thrashing prevention -+-------------------- -+Personal computers are more sensitive to thrashing because it can -+cause janks (lags when rendering UI) and negatively impact user -+experience. The multi-gen LRU offers thrashing prevention to the -+majority of laptop and desktop users who do not have ``oomd``. -+ -+Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of -+``N`` milliseconds from getting evicted. The OOM killer is triggered -+if this working set cannot be kept in memory. In other words, this -+option works as an adjustable pressure relief valve, and when open, it -+terminates applications that are hopefully not being used. -+ -+Based on the average human detectable lag (~100ms), ``N=1000`` usually -+eliminates intolerable janks due to thrashing. Larger values like -+``N=3000`` make janks less noticeable at the risk of premature OOM -+kills. -+ -+The default value ``0`` means disabled. -+ -+Experimental features -+===================== -+``/sys/kernel/debug/lru_gen`` accepts commands described in the -+following subsections. Multiple command lines are supported, so does -+concatenation with delimiters ``,`` and ``;``. -+ -+``/sys/kernel/debug/lru_gen_full`` provides additional stats for -+debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from -+evicted generations in this file. -+ -+Working set estimation -+---------------------- -+Working set estimation measures how much memory an application needs -+in a given time interval, and it is usually done with little impact on -+the performance of the application. E.g., data centers want to -+optimize job scheduling (bin packing) to improve memory utilizations. -+When a new job comes in, the job scheduler needs to find out whether -+each server it manages can allocate a certain amount of memory for -+this new job before it can pick a candidate. To do so, the job -+scheduler needs to estimate the working sets of the existing jobs. -+ -+When it is read, ``lru_gen`` returns a histogram of numbers of pages -+accessed over different time intervals for each memcg and node. -+``MAX_NR_GENS`` decides the number of bins for each histogram. The -+histograms are noncumulative. -+:: -+ -+ memcg memcg_id memcg_path -+ node node_id -+ min_gen_nr age_in_ms nr_anon_pages nr_file_pages -+ ... -+ max_gen_nr age_in_ms nr_anon_pages nr_file_pages -+ -+Each bin contains an estimated number of pages that have been accessed -+within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages -+and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of -+the former is the largest and that of the latter is the smallest. -+ -+Users can write the following command to ``lru_gen`` to create a new -+generation ``max_gen_nr+1``: -+ -+ ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` -+ -+``can_swap`` defaults to the swap setting and, if it is set to ``1``, -+it forces the scan of anon pages when swap is off, and vice versa. -+``force_scan`` defaults to ``1`` and, if it is set to ``0``, it -+employs heuristics to reduce the overhead, which is likely to reduce -+the coverage as well. -+ -+A typical use case is that a job scheduler runs this command at a -+certain time interval to create new generations, and it ranks the -+servers it manages based on the sizes of their cold pages defined by -+this time interval. -+ -+Proactive reclaim -+----------------- -+Proactive reclaim induces page reclaim when there is no memory -+pressure. It usually targets cold pages only. E.g., when a new job -+comes in, the job scheduler wants to proactively reclaim cold pages on -+the server it selected, to improve the chance of successfully landing -+this new job. -+ -+Users can write the following command to ``lru_gen`` to evict -+generations less than or equal to ``min_gen_nr``. -+ -+ ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` -+ -+``min_gen_nr`` should be less than ``max_gen_nr-1``, since -+``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to -+the active list) and therefore cannot be evicted. ``swappiness`` -+overrides the default value in ``/proc/sys/vm/swappiness``. -+``nr_to_reclaim`` limits the number of pages to evict. -+ -+A typical use case is that a job scheduler runs this command before it -+tries to land a new job on a server. If it fails to materialize enough -+cold pages because of the overestimation, it retries on the next -+server according to the ranking result obtained from the working set -+estimation step. This less forceful approach limits the impacts on the -+existing jobs. diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index dc95df462eea..1da6a4fac664 100644 --- a/Documentation/core-api/index.rst @@ -561,183 +422,6 @@ index 000000000000..45defcf15da7 + +.. kernel-doc:: include/linux/maple_tree.h +.. kernel-doc:: lib/maple_tree.c -diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst -index 575ccd40e30c..4aa12b8be278 100644 ---- a/Documentation/mm/index.rst -+++ b/Documentation/mm/index.rst -@@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. - ksm - memory-model - mmu_notifier -+ multigen_lru - numa - overcommit-accounting - page_migration -diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst -new file mode 100644 -index 000000000000..d7062c6a8946 ---- /dev/null -+++ b/Documentation/mm/multigen_lru.rst -@@ -0,0 +1,159 @@ -+.. SPDX-License-Identifier: GPL-2.0 -+ -+============= -+Multi-Gen LRU -+============= -+The multi-gen LRU is an alternative LRU implementation that optimizes -+page reclaim and improves performance under memory pressure. Page -+reclaim decides the kernel's caching policy and ability to overcommit -+memory. It directly impacts the kswapd CPU usage and RAM efficiency. -+ -+Design overview -+=============== -+Objectives -+---------- -+The design objectives are: -+ -+* Good representation of access recency -+* Try to profit from spatial locality -+* Fast paths to make obvious choices -+* Simple self-correcting heuristics -+ -+The representation of access recency is at the core of all LRU -+implementations. In the multi-gen LRU, each generation represents a -+group of pages with similar access recency. Generations establish a -+(time-based) common frame of reference and therefore help make better -+choices, e.g., between different memcgs on a computer or different -+computers in a data center (for job scheduling). -+ -+Exploiting spatial locality improves efficiency when gathering the -+accessed bit. A rmap walk targets a single page and does not try to -+profit from discovering a young PTE. A page table walk can sweep all -+the young PTEs in an address space, but the address space can be too -+sparse to make a profit. The key is to optimize both methods and use -+them in combination. -+ -+Fast paths reduce code complexity and runtime overhead. Unmapped pages -+do not require TLB flushes; clean pages do not require writeback. -+These facts are only helpful when other conditions, e.g., access -+recency, are similar. With generations as a common frame of reference, -+additional factors stand out. But obvious choices might not be good -+choices; thus self-correction is necessary. -+ -+The benefits of simple self-correcting heuristics are self-evident. -+Again, with generations as a common frame of reference, this becomes -+attainable. Specifically, pages in the same generation can be -+categorized based on additional factors, and a feedback loop can -+statistically compare the refault percentages across those categories -+and infer which of them are better choices. -+ -+Assumptions -+----------- -+The protection of hot pages and the selection of cold pages are based -+on page access channels and patterns. There are two access channels: -+ -+* Accesses through page tables -+* Accesses through file descriptors -+ -+The protection of the former channel is by design stronger because: -+ -+1. The uncertainty in determining the access patterns of the former -+ channel is higher due to the approximation of the accessed bit. -+2. The cost of evicting the former channel is higher due to the TLB -+ flushes required and the likelihood of encountering the dirty bit. -+3. The penalty of underprotecting the former channel is higher because -+ applications usually do not prepare themselves for major page -+ faults like they do for blocked I/O. E.g., GUI applications -+ commonly use dedicated I/O threads to avoid blocking rendering -+ threads. -+ -+There are also two access patterns: -+ -+* Accesses exhibiting temporal locality -+* Accesses not exhibiting temporal locality -+ -+For the reasons listed above, the former channel is assumed to follow -+the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is -+present, and the latter channel is assumed to follow the latter -+pattern unless outlying refaults have been observed. -+ -+Workflow overview -+================= -+Evictable pages are divided into multiple generations for each -+``lruvec``. The youngest generation number is stored in -+``lrugen->max_seq`` for both anon and file types as they are aged on -+an equal footing. The oldest generation numbers are stored in -+``lrugen->min_seq[]`` separately for anon and file types as clean file -+pages can be evicted regardless of swap constraints. These three -+variables are monotonically increasing. -+ -+Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` -+bits in order to fit into the gen counter in ``folio->flags``. Each -+truncated generation number is an index to ``lrugen->lists[]``. The -+sliding window technique is used to track at least ``MIN_NR_GENS`` and -+at most ``MAX_NR_GENS`` generations. The gen counter stores a value -+within ``[1, MAX_NR_GENS]`` while a page is on one of -+``lrugen->lists[]``; otherwise it stores zero. -+ -+Each generation is divided into multiple tiers. A page accessed ``N`` -+times through file descriptors is in tier ``order_base_2(N)``. Unlike -+generations, tiers do not have dedicated ``lrugen->lists[]``. In -+contrast to moving across generations, which requires the LRU lock, -+moving across tiers only involves atomic operations on -+``folio->flags`` and therefore has a negligible cost. A feedback loop -+modeled after the PID controller monitors refaults over all the tiers -+from anon and file types and decides which tiers from which types to -+evict or protect. -+ -+There are two conceptually independent procedures: the aging and the -+eviction. They form a closed-loop system, i.e., the page reclaim. -+ -+Aging -+----- -+The aging produces young generations. Given an ``lruvec``, it -+increments ``max_seq`` when ``max_seq-min_seq+1`` approaches -+``MIN_NR_GENS``. The aging promotes hot pages to the youngest -+generation when it finds them accessed through page tables; the -+demotion of cold pages happens consequently when it increments -+``max_seq``. The aging uses page table walks and rmap walks to find -+young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` -+and calls ``walk_page_range()`` with each ``mm_struct`` on this list -+to scan PTEs, and after each iteration, it increments ``max_seq``. For -+the latter, when the eviction walks the rmap and finds a young PTE, -+the aging scans the adjacent PTEs. For both, on finding a young PTE, -+the aging clears the accessed bit and updates the gen counter of the -+page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. -+ -+Eviction -+-------- -+The eviction consumes old generations. Given an ``lruvec``, it -+increments ``min_seq`` when ``lrugen->lists[]`` indexed by -+``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to -+evict from, it first compares ``min_seq[]`` to select the older type. -+If both types are equally old, it selects the one whose first tier has -+a lower refault percentage. The first tier contains single-use -+unmapped clean pages, which are the best bet. The eviction sorts a -+page according to its gen counter if the aging has found this page -+accessed through page tables and updated its gen counter. It also -+moves a page to the next generation, i.e., ``min_seq+1``, if this page -+was accessed multiple times through file descriptors and the feedback -+loop has detected outlying refaults from the tier this page is in. To -+this end, the feedback loop uses the first tier as the baseline, for -+the reason stated earlier. -+ -+Summary -+------- -+The multi-gen LRU can be disassembled into the following parts: -+ -+* Generations -+* Rmap walks -+* Page table walks -+* Bloom filters -+* PID controller -+ -+The aging and the eviction form a producer-consumer model; -+specifically, the latter drives the former by the sliding window over -+generations. Within the aging, rmap walks drive page table walks by -+inserting hot densely populated page tables to the Bloom filters. -+Within the eviction, the PID controller uses refaults as the feedback -+to select types to evict and tiers to protect. diff --git a/MAINTAINERS b/MAINTAINERS index a29c9731350c..96a09757feb3 100644 --- a/MAINTAINERS @@ -761,69 +445,6 @@ index a29c9731350c..96a09757feb3 100644 MARDUK (CREATOR CI40) DEVICE TREE SUPPORT M: Rahul Bedarkar L: linux-mips@vger.kernel.org -diff --git a/arch/Kconfig b/arch/Kconfig -index 8b311e400ec1..bf19a84fffa2 100644 ---- a/arch/Kconfig -+++ b/arch/Kconfig -@@ -1418,6 +1418,14 @@ config DYNAMIC_SIGFRAME - config HAVE_ARCH_NODE_DEV_GROUP - bool - -+config ARCH_HAS_NONLEAF_PMD_YOUNG -+ bool -+ help -+ Architectures that select this option are capable of setting the -+ accessed bit in non-leaf PMD entries when using them as part of linear -+ address translations. Page table walkers that clear the accessed bit -+ may use this capability to reduce their search space. -+ - source "kernel/gcov/Kconfig" - - source "scripts/gcc-plugins/Kconfig" -diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h -index 4aa996423b0d..763929e814e9 100644 ---- a/arch/alpha/include/uapi/asm/mman.h -+++ b/arch/alpha/include/uapi/asm/mman.h -@@ -76,6 +76,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h -index b5df82aa99e6..71a1af42f0e8 100644 ---- a/arch/arm64/include/asm/pgtable.h -+++ b/arch/arm64/include/asm/pgtable.h -@@ -1082,24 +1082,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, - * page after fork() + CoW for pfn mappings. We don't always have a - * hardware-managed access flag on arm64. - */ --static inline bool arch_faults_on_old_pte(void) --{ -- /* The register read below requires a stable CPU to make any sense */ -- cant_migrate(); -- -- return !cpu_has_hw_af(); --} --#define arch_faults_on_old_pte arch_faults_on_old_pte -+#define arch_has_hw_pte_young cpu_has_hw_af - - /* - * Experimentally, it's cheap to set the access flag in hardware and we - * benefit from prefaulting mappings as 'old' to start with. - */ --static inline bool arch_wants_old_prefaulted_pte(void) --{ -- return !arch_faults_on_old_pte(); --} --#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte -+#define arch_wants_old_prefaulted_pte cpu_has_hw_af - - static inline bool pud_sect_supported(void) - { diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c index 98d67444a5b6..27ef7ad3ffd2 100644 --- a/arch/arm64/kernel/elfcore.c @@ -901,32 +522,6 @@ index a61fc4f989b3..a8388af62b99 100644 unsigned long size = vma->vm_end - vma->vm_start; if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) -diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h -index 1be428663c10..c6e1fc77c996 100644 ---- a/arch/mips/include/uapi/asm/mman.h -+++ b/arch/mips/include/uapi/asm/mman.h -@@ -103,6 +103,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h -index a7ea3204a5fa..22133a6a506e 100644 ---- a/arch/parisc/include/uapi/asm/mman.h -+++ b/arch/parisc/include/uapi/asm/mman.h -@@ -70,6 +70,8 @@ - #define MADV_WIPEONFORK 71 /* Zero memory on fork, child only */ - #define MADV_KEEPONFORK 72 /* Undo MADV_WIPEONFORK */ - -+#define MADV_COLLAPSE 73 /* Synchronous hugepage collapse */ -+ - #define MADV_HWPOISON 100 /* poison a page for testing */ - #define MADV_SOFT_OFFLINE 101 /* soft offline page for testing */ - diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 3feb7694e0ca..1d3b8bc8a623 100644 --- a/arch/parisc/kernel/cache.c @@ -1137,18 +732,6 @@ index bc38f79ca3a3..ad449173a1a1 100644 - vma = vma->vm_next; - } } -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index f9920f1341c8..674d694a665e 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -85,6 +85,7 @@ config X86 - select ARCH_HAS_PMEM_API if X86_64 - select ARCH_HAS_PTE_DEVMAP if X86_64 - select ARCH_HAS_PTE_SPECIAL -+ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 - select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 - select ARCH_HAS_COPY_MC if X86_64 - select ARCH_HAS_SET_MEMORY diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 1000d457c332..6292b960037b 100644 --- a/arch/x86/entry/vdso/vma.c @@ -1191,34 +774,6 @@ index 1000d457c332..6292b960037b 100644 if (vma_is_special_mapping(vma, &vdso_mapping) || vma_is_special_mapping(vma, &vvar_mapping)) { mmap_write_unlock(mm); -diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h -index 44e2d6f1dbaa..5059799bebe3 100644 ---- a/arch/x86/include/asm/pgtable.h -+++ b/arch/x86/include/asm/pgtable.h -@@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) - - static inline int pmd_bad(pmd_t pmd) - { -- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; -+ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != -+ (_KERNPG_TABLE & ~_PAGE_ACCESSED); - } - - static inline unsigned long pages_to_mb(unsigned long npg) -@@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) - return boot_cpu_has_bug(X86_BUG_L1TF); - } - --#define arch_faults_on_old_pte arch_faults_on_old_pte --static inline bool arch_faults_on_old_pte(void) -+#define arch_has_hw_pte_young arch_has_hw_pte_young -+static inline bool arch_has_hw_pte_young(void) - { -- return false; -+ return true; - } - - #ifdef CONFIG_PAGE_TABLE_CHECK diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index 3bacd935f840..4c1bcb6053fc 100644 --- a/arch/x86/kernel/tboot.c @@ -1232,42 +787,6 @@ index 3bacd935f840..4c1bcb6053fc 100644 .pgd = swapper_pg_dir, .mm_users = ATOMIC_INIT(2), .mm_count = ATOMIC_INIT(1), -diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c -index a932d7712d85..8525f2876fb4 100644 ---- a/arch/x86/mm/pgtable.c -+++ b/arch/x86/mm/pgtable.c -@@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, - return ret; - } - --#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) - int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmdp) - { -@@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, - - return ret; - } -+#endif -+ -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE - int pudp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pud_t *pudp) - { -diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h -index 7966a58af472..1ff0c858544f 100644 ---- a/arch/xtensa/include/uapi/asm/mman.h -+++ b/arch/xtensa/include/uapi/asm/mman.h -@@ -111,6 +111,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - diff --git a/arch/xtensa/kernel/syscall.c b/arch/xtensa/kernel/syscall.c index 201356faa7e6..b3c2450d6f23 100644 --- a/arch/xtensa/kernel/syscall.c @@ -1576,7 +1095,7 @@ index 9f4aae202109..35f2af85b9bc 100644 mmap_write_unlock(mm); diff --git a/fs/exec.c b/fs/exec.c -index d046dbb9cbd0..3a7bae4e16e5 100644 +index c67b12f0f577..3a7bae4e16e5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,7 +28,6 @@ @@ -1629,39 +1148,15 @@ index d046dbb9cbd0..3a7bae4e16e5 100644 } tlb_finish_mmu(&tlb); -@@ -1011,6 +1013,7 @@ static int exec_mmap(struct mm_struct *mm) - active_mm = tsk->active_mm; - tsk->active_mm = mm; - tsk->mm = mm; -+ lru_gen_add_mm(mm); - /* - * This prevents preemption while active_mm is being loaded and - * it and mm are being updated, which could cause problems for -@@ -1023,9 +1026,8 @@ static int exec_mmap(struct mm_struct *mm) +@@ -1024,8 +1026,6 @@ static int exec_mmap(struct mm_struct *mm) activate_mm(active_mm, mm); if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) local_irq_enable(); - tsk->mm->vmacache_seqnum = 0; - vmacache_flush(tsk); task_unlock(tsk); -+ lru_gen_use_mm(mm); + lru_gen_use_mm(mm); if (old_mm) { - mmap_read_unlock(old_mm); - BUG_ON(active_mm != old_mm); -diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c -index 51897427a534..b4a6e0a1b945 100644 ---- a/fs/fuse/dev.c -+++ b/fs/fuse/dev.c -@@ -776,7 +776,8 @@ static int fuse_check_page(struct page *page) - 1 << PG_active | - 1 << PG_workingset | - 1 << PG_reclaim | -- 1 << PG_waiters))) { -+ 1 << PG_waiters | -+ LRU_GEN_MASK | LRU_REFS_MASK))) { - dump_page(page, "fuse: trying to steal weird page"); - return 1; - } diff --git a/fs/proc/base.c b/fs/proc/base.c index 93f7e3d971e4..12885a75913f 100644 --- a/fs/proc/base.c @@ -1699,7 +1194,7 @@ index 06a80f78433d..f03000764ce5 100644 #ifdef CONFIG_NUMA struct mempolicy *task_mempolicy; diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c -index 4e0023643f8b..7a10fe08beb3 100644 +index 482f91577f8c..7a10fe08beb3 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,5 @@ @@ -1778,15 +1273,6 @@ index 4e0023643f8b..7a10fe08beb3 100644 } static void m_stop(struct seq_file *m, void *v) -@@ -864,7 +867,7 @@ static int show_smap(struct seq_file *m, void *v) - __show_smap(m, &mss, false); - - seq_printf(m, "THPeligible: %d\n", -- hugepage_vma_check(vma, vma->vm_flags, true, false)); -+ hugepage_vma_check(vma, vma->vm_flags, true, false, true)); - - if (arch_pkeys_enabled()) - seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); @@ -877,16 +880,16 @@ static int show_smaps_rollup(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; @@ -2193,102 +1679,6 @@ index 175de70e3adf..f662cede4488 100644 out_unlock: mmap_write_unlock(mm); mmput(mm); -diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h -index ac5d0515680e..9179463c3c9f 100644 ---- a/include/linux/cgroup.h -+++ b/include/linux/cgroup.h -@@ -432,6 +432,18 @@ static inline void cgroup_put(struct cgroup *cgrp) - css_put(&cgrp->self); - } - -+extern struct mutex cgroup_mutex; -+ -+static inline void cgroup_lock(void) -+{ -+ mutex_lock(&cgroup_mutex); -+} -+ -+static inline void cgroup_unlock(void) -+{ -+ mutex_unlock(&cgroup_mutex); -+} -+ - /** - * task_css_set_check - obtain a task's css_set with extra access conditions - * @task: the task to obtain css_set for -@@ -446,7 +458,6 @@ static inline void cgroup_put(struct cgroup *cgrp) - * as locks used during the cgroup_subsys::attach() methods. - */ - #ifdef CONFIG_PROVE_RCU --extern struct mutex cgroup_mutex; - extern spinlock_t css_set_lock; - #define task_css_set_check(task, __c) \ - rcu_dereference_check((task)->cgroups, \ -@@ -708,6 +719,8 @@ struct cgroup; - static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } - static inline void css_get(struct cgroup_subsys_state *css) {} - static inline void css_put(struct cgroup_subsys_state *css) {} -+static inline void cgroup_lock(void) {} -+static inline void cgroup_unlock(void) {} - static inline int cgroup_attach_task_all(struct task_struct *from, - struct task_struct *t) { return 0; } - static inline int cgroupstats_build(struct cgroupstats *stats, -diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h -index 768e5261fdae..38265f9f782e 100644 ---- a/include/linux/huge_mm.h -+++ b/include/linux/huge_mm.h -@@ -168,9 +168,8 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) - !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); - } - --bool hugepage_vma_check(struct vm_area_struct *vma, -- unsigned long vm_flags, -- bool smaps, bool in_pf); -+bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, -+ bool smaps, bool in_pf, bool enforce_sysfs); - - #define transparent_hugepage_use_zero_page() \ - (transparent_hugepage_flags & \ -@@ -219,6 +218,9 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, - - int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, - int advice); -+int madvise_collapse(struct vm_area_struct *vma, -+ struct vm_area_struct **prev, -+ unsigned long start, unsigned long end); - void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, - unsigned long end, long adjust_next); - spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma); -@@ -321,8 +323,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, - } - - static inline bool hugepage_vma_check(struct vm_area_struct *vma, -- unsigned long vm_flags, -- bool smaps, bool in_pf) -+ unsigned long vm_flags, bool smaps, -+ bool in_pf, bool enforce_sysfs) - { - return false; - } -@@ -362,9 +364,16 @@ static inline void split_huge_pmd_address(struct vm_area_struct *vma, - static inline int hugepage_madvise(struct vm_area_struct *vma, - unsigned long *vm_flags, int advice) - { -- BUG(); -- return 0; -+ return -EINVAL; - } -+ -+static inline int madvise_collapse(struct vm_area_struct *vma, -+ struct vm_area_struct **prev, -+ unsigned long start, unsigned long end) -+{ -+ return -EINVAL; -+} -+ - static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, - unsigned long start, - unsigned long end, diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h new file mode 100644 index 000000000000..2effab72add1 @@ -2980,83 +2370,8 @@ index 000000000000..2effab72add1 +#endif /* CONFIG_DEBUG_MAPLE_TREE */ + +#endif /*_LINUX_MAPLE_TREE_H */ -diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h -index 6257867fbf95..207cfd3b42e5 100644 ---- a/include/linux/memcontrol.h -+++ b/include/linux/memcontrol.h -@@ -350,6 +350,11 @@ struct mem_cgroup { - struct deferred_split deferred_split_queue; - #endif - -+#ifdef CONFIG_LRU_GEN -+ /* per-memcg mm_struct list */ -+ struct lru_gen_mm_list mm_list; -+#endif -+ - struct mem_cgroup_per_node *nodeinfo[]; - }; - -@@ -444,6 +449,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - * - * For a kmem folio a caller should hold an rcu read lock to protect memcg - * associated with a kmem folio from being released. -@@ -505,6 +511,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - * - * For a kmem page a caller should hold an rcu read lock to protect memcg - * associated with a kmem page from being released. -@@ -959,6 +966,23 @@ void unlock_page_memcg(struct page *page); - - void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); - -+/* try to stablize folio_memcg() for all the pages in a memcg */ -+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) -+{ -+ rcu_read_lock(); -+ -+ if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) -+ return true; -+ -+ rcu_read_unlock(); -+ return false; -+} -+ -+static inline void mem_cgroup_unlock_pages(void) -+{ -+ rcu_read_unlock(); -+} -+ - /* idx can be of type enum memcg_stat_item or node_stat_item */ - static inline void mod_memcg_state(struct mem_cgroup *memcg, - int idx, int val) -@@ -1433,6 +1457,18 @@ static inline void folio_memcg_unlock(struct folio *folio) - { - } - -+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) -+{ -+ /* to match folio_memcg_rcu() */ -+ rcu_read_lock(); -+ return true; -+} -+ -+static inline void mem_cgroup_unlock_pages(void) -+{ -+ rcu_read_unlock(); -+} -+ - static inline void mem_cgroup_handle_over_high(void) - { - } diff --git a/include/linux/mm.h b/include/linux/mm.h -index 21f8b27bd9fd..9ac0e02e2238 100644 +index 88976a521ef5..9ac0e02e2238 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -661,6 +661,38 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) @@ -3098,19 +2413,7 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 #ifdef CONFIG_SHMEM /* * The vma_is_shmem is not inline because it is used only by slow -@@ -1465,6 +1497,11 @@ static inline unsigned long folio_pfn(struct folio *folio) - return page_to_pfn(&folio->page); - } - -+static inline struct folio *pfn_folio(unsigned long pfn) -+{ -+ return page_folio(pfn_to_page(pfn)); -+} -+ - static inline atomic_t *folio_pincount_ptr(struct folio *folio) - { - return &folio_page(folio, 1)->compound_pincount; -@@ -1795,8 +1832,9 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, +@@ -1800,8 +1832,9 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, unsigned long size); void zap_page_range(struct vm_area_struct *vma, unsigned long address, unsigned long size); @@ -3122,7 +2425,7 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 struct mmu_notifier_range; -@@ -2593,14 +2631,15 @@ extern int __split_vma(struct mm_struct *, struct vm_area_struct *, +@@ -2598,14 +2631,15 @@ extern int __split_vma(struct mm_struct *, struct vm_area_struct *, extern int split_vma(struct mm_struct *, struct vm_area_struct *, unsigned long addr, int new_below); extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); @@ -3140,7 +2443,7 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 static inline int check_data_rlimit(unsigned long rlim, unsigned long new, unsigned long start, -@@ -2648,8 +2687,9 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, +@@ -2653,8 +2687,9 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); @@ -3152,7 +2455,7 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 extern int do_munmap(struct mm_struct *, unsigned long, size_t, struct list_head *uf); extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); -@@ -2716,26 +2756,12 @@ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long add +@@ -2721,26 +2756,12 @@ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long add extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, struct vm_area_struct **pprev); @@ -3183,7 +2486,7 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 /** * vma_lookup() - Find a VMA at a specific address -@@ -2747,12 +2773,7 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, +@@ -2752,12 +2773,7 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) { @@ -3197,7 +2500,7 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 } static inline unsigned long vm_start_gap(struct vm_area_struct *vma) -@@ -2788,7 +2809,7 @@ static inline unsigned long vma_pages(struct vm_area_struct *vma) +@@ -2793,7 +2809,7 @@ static inline unsigned long vma_pages(struct vm_area_struct *vma) static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, unsigned long vm_start, unsigned long vm_end) { @@ -3206,284 +2509,8 @@ index 21f8b27bd9fd..9ac0e02e2238 100644 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) vma = NULL; -diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h -index 7b25b53c474a..4949eda9a9a2 100644 ---- a/include/linux/mm_inline.h -+++ b/include/linux/mm_inline.h -@@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) - return folio_is_file_lru(page_folio(page)); - } - --static __always_inline void update_lru_size(struct lruvec *lruvec, -+static __always_inline void __update_lru_size(struct lruvec *lruvec, - enum lru_list lru, enum zone_type zid, - long nr_pages) - { - struct pglist_data *pgdat = lruvec_pgdat(lruvec); - -+ lockdep_assert_held(&lruvec->lru_lock); -+ WARN_ON_ONCE(nr_pages != (int)nr_pages); -+ - __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); - __mod_zone_page_state(&pgdat->node_zones[zid], - NR_ZONE_LRU_BASE + lru, nr_pages); -+} -+ -+static __always_inline void update_lru_size(struct lruvec *lruvec, -+ enum lru_list lru, enum zone_type zid, -+ long nr_pages) -+{ -+ __update_lru_size(lruvec, lru, zid, nr_pages); - #ifdef CONFIG_MEMCG - mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); - #endif -@@ -94,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) - return lru; - } - -+#ifdef CONFIG_LRU_GEN -+ -+#ifdef CONFIG_LRU_GEN_ENABLED -+static inline bool lru_gen_enabled(void) -+{ -+ DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); -+ -+ return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); -+} -+#else -+static inline bool lru_gen_enabled(void) -+{ -+ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); -+ -+ return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); -+} -+#endif -+ -+static inline bool lru_gen_in_fault(void) -+{ -+ return current->in_lru_fault; -+} -+ -+static inline int lru_gen_from_seq(unsigned long seq) -+{ -+ return seq % MAX_NR_GENS; -+} -+ -+static inline int lru_hist_from_seq(unsigned long seq) -+{ -+ return seq % NR_HIST_GENS; -+} -+ -+static inline int lru_tier_from_refs(int refs) -+{ -+ VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); -+ -+ /* see the comment in folio_lru_refs() */ -+ return order_base_2(refs + 1); -+} -+ -+static inline int folio_lru_refs(struct folio *folio) -+{ -+ unsigned long flags = READ_ONCE(folio->flags); -+ bool workingset = flags & BIT(PG_workingset); -+ -+ /* -+ * Return the number of accesses beyond PG_referenced, i.e., N-1 if the -+ * total number of accesses is N>1, since N=0,1 both map to the first -+ * tier. lru_tier_from_refs() will account for this off-by-one. Also see -+ * the comment on MAX_NR_TIERS. -+ */ -+ return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; -+} -+ -+static inline int folio_lru_gen(struct folio *folio) -+{ -+ unsigned long flags = READ_ONCE(folio->flags); -+ -+ return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+} -+ -+static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) -+{ -+ unsigned long max_seq = lruvec->lrugen.max_seq; -+ -+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); -+ -+ /* see the comment on MIN_NR_GENS */ -+ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); -+} -+ -+static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, -+ int old_gen, int new_gen) -+{ -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); -+ -+ if (old_gen >= 0) -+ WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], -+ lrugen->nr_pages[old_gen][type][zone] - delta); -+ if (new_gen >= 0) -+ WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], -+ lrugen->nr_pages[new_gen][type][zone] + delta); -+ -+ /* addition */ -+ if (old_gen < 0) { -+ if (lru_gen_is_active(lruvec, new_gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, delta); -+ return; -+ } -+ -+ /* deletion */ -+ if (new_gen < 0) { -+ if (lru_gen_is_active(lruvec, old_gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, -delta); -+ return; -+ } -+ -+ /* promotion */ -+ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { -+ __update_lru_size(lruvec, lru, zone, -delta); -+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); -+ } -+ -+ /* demotion requires isolation, e.g., lru_deactivate_fn() */ -+ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); -+} -+ -+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ unsigned long seq; -+ unsigned long flags; -+ int gen = folio_lru_gen(folio); -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); -+ -+ if (folio_test_unevictable(folio) || !lrugen->enabled) -+ return false; -+ /* -+ * There are three common cases for this page: -+ * 1. If it's hot, e.g., freshly faulted in or previously hot and -+ * migrated, add it to the youngest generation. -+ * 2. If it's cold but can't be evicted immediately, i.e., an anon page -+ * not in swapcache or a dirty page pending writeback, add it to the -+ * second oldest generation. -+ * 3. Everything else (clean, cold) is added to the oldest generation. -+ */ -+ if (folio_test_active(folio)) -+ seq = lrugen->max_seq; -+ else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || -+ (folio_test_reclaim(folio) && -+ (folio_test_dirty(folio) || folio_test_writeback(folio)))) -+ seq = lrugen->min_seq[type] + 1; -+ else -+ seq = lrugen->min_seq[type]; -+ -+ gen = lru_gen_from_seq(seq); -+ flags = (gen + 1UL) << LRU_GEN_PGOFF; -+ /* see the comment on MIN_NR_GENS about PG_active */ -+ set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); -+ -+ lru_gen_update_size(lruvec, folio, -1, gen); -+ /* for folio_rotate_reclaimable() */ -+ if (reclaiming) -+ list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); -+ else -+ list_add(&folio->lru, &lrugen->lists[gen][type][zone]); -+ -+ return true; -+} -+ -+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ unsigned long flags; -+ int gen = folio_lru_gen(folio); -+ -+ if (gen < 0) -+ return false; -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ -+ /* for folio_migrate_flags() */ -+ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; -+ flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); -+ gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+ -+ lru_gen_update_size(lruvec, folio, gen, -1); -+ list_del(&folio->lru); -+ -+ return true; -+} -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline bool lru_gen_enabled(void) -+{ -+ return false; -+} -+ -+static inline bool lru_gen_in_fault(void) -+{ -+ return false; -+} -+ -+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ return false; -+} -+ -+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ return false; -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ - static __always_inline - void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_add_folio(lruvec, folio, false)) -+ return; -+ - update_lru_size(lruvec, lru, folio_zonenum(folio), - folio_nr_pages(folio)); - if (lru != LRU_UNEVICTABLE) -@@ -116,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_add_folio(lruvec, folio, true)) -+ return; -+ - update_lru_size(lruvec, lru, folio_zonenum(folio), - folio_nr_pages(folio)); - /* This is not expected to be used on LRU_UNEVICTABLE */ -@@ -133,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_del_folio(lruvec, folio, false)) -+ return; -+ - if (lru != LRU_UNEVICTABLE) - list_del(&folio->lru); - update_lru_size(lruvec, lru, folio_zonenum(folio), diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index cf97f3884fda..5e32211cb5a9 100644 +index e1797813cc2c..5e32211cb5a9 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -9,6 +9,7 @@ @@ -3535,30 +2562,7 @@ index cf97f3884fda..5e32211cb5a9 100644 pgd_t * pgd; #ifdef CONFIG_MEMBARRIER -@@ -672,6 +655,22 @@ struct mm_struct { - */ - unsigned long ksm_merging_pages; - #endif -+#ifdef CONFIG_LRU_GEN -+ struct { -+ /* this mm_struct is on lru_gen_mm_list */ -+ struct list_head list; -+ /* -+ * Set when switching to this mm_struct, as a hint of -+ * whether it has been used since the last time per-node -+ * page table walkers cleared the corresponding bits. -+ */ -+ unsigned long bitmap; -+#ifdef CONFIG_MEMCG -+ /* points to the memcg of "owner" above */ -+ struct mem_cgroup *memcg; -+#endif -+ } lru_gen; -+#endif /* CONFIG_LRU_GEN */ - } __randomize_layout; - - /* -@@ -681,6 +680,7 @@ struct mm_struct { +@@ -697,6 +680,7 @@ struct mm_struct { unsigned long cpu_bitmap[]; }; @@ -3566,70 +2570,10 @@ index cf97f3884fda..5e32211cb5a9 100644 extern struct mm_struct init_mm; /* Pointer magic because the dynamic array size confuses some compilers. */ -@@ -698,6 +698,87 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) - return (struct cpumask *)&mm->cpu_bitmap; - } +@@ -774,6 +758,27 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) + + #endif /* CONFIG_LRU_GEN */ -+#ifdef CONFIG_LRU_GEN -+ -+struct lru_gen_mm_list { -+ /* mm_struct list for page table walkers */ -+ struct list_head fifo; -+ /* protects the list above */ -+ spinlock_t lock; -+}; -+ -+void lru_gen_add_mm(struct mm_struct *mm); -+void lru_gen_del_mm(struct mm_struct *mm); -+#ifdef CONFIG_MEMCG -+void lru_gen_migrate_mm(struct mm_struct *mm); -+#endif -+ -+static inline void lru_gen_init_mm(struct mm_struct *mm) -+{ -+ INIT_LIST_HEAD(&mm->lru_gen.list); -+ mm->lru_gen.bitmap = 0; -+#ifdef CONFIG_MEMCG -+ mm->lru_gen.memcg = NULL; -+#endif -+} -+ -+static inline void lru_gen_use_mm(struct mm_struct *mm) -+{ -+ /* -+ * When the bitmap is set, page reclaim knows this mm_struct has been -+ * used since the last time it cleared the bitmap. So it might be worth -+ * walking the page tables of this mm_struct to clear the accessed bit. -+ */ -+ WRITE_ONCE(mm->lru_gen.bitmap, -1); -+} -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline void lru_gen_add_mm(struct mm_struct *mm) -+{ -+} -+ -+static inline void lru_gen_del_mm(struct mm_struct *mm) -+{ -+} -+ -+#ifdef CONFIG_MEMCG -+static inline void lru_gen_migrate_mm(struct mm_struct *mm) -+{ -+} -+#endif -+ -+static inline void lru_gen_init_mm(struct mm_struct *mm) -+{ -+} -+ -+static inline void lru_gen_use_mm(struct mm_struct *mm) -+{ -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ +struct vma_iterator { + struct ma_state mas; +}; @@ -3677,382 +2621,8 @@ index c1bc6731125c..0bb4b6da9993 100644 /* * When updating this, please also update struct resident_page_types[] in * kernel/fork.c -diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index e24b40c52468..1543001feba9 100644 ---- a/include/linux/mmzone.h -+++ b/include/linux/mmzone.h -@@ -306,6 +306,8 @@ static inline bool is_active_lru(enum lru_list lru) - return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); - } - -+#define WORKINGSET_ANON 0 -+#define WORKINGSET_FILE 1 - #define ANON_AND_FILE 2 - - enum lruvec_flags { -@@ -314,6 +316,207 @@ enum lruvec_flags { - */ - }; - -+#endif /* !__GENERATING_BOUNDS_H */ -+ -+/* -+ * Evictable pages are divided into multiple generations. The youngest and the -+ * oldest generation numbers, max_seq and min_seq, are monotonically increasing. -+ * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An -+ * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the -+ * corresponding generation. The gen counter in folio->flags stores gen+1 while -+ * a page is on one of lrugen->lists[]. Otherwise it stores 0. -+ * -+ * A page is added to the youngest generation on faulting. The aging needs to -+ * check the accessed bit at least twice before handing this page over to the -+ * eviction. The first check takes care of the accessed bit set on the initial -+ * fault; the second check makes sure this page hasn't been used since then. -+ * This process, AKA second chance, requires a minimum of two generations, -+ * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive -+ * LRU, e.g., /proc/vmstat, these two generations are considered active; the -+ * rest of generations, if they exist, are considered inactive. See -+ * lru_gen_is_active(). -+ * -+ * PG_active is always cleared while a page is on one of lrugen->lists[] so that -+ * the aging needs not to worry about it. And it's set again when a page -+ * considered active is isolated for non-reclaiming purposes, e.g., migration. -+ * See lru_gen_add_folio() and lru_gen_del_folio(). -+ * -+ * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the -+ * number of categories of the active/inactive LRU when keeping track of -+ * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits -+ * in folio->flags. -+ */ -+#define MIN_NR_GENS 2U -+#define MAX_NR_GENS 4U -+ -+/* -+ * Each generation is divided into multiple tiers. A page accessed N times -+ * through file descriptors is in tier order_base_2(N). A page in the first tier -+ * (N=0,1) is marked by PG_referenced unless it was faulted in through page -+ * tables or read ahead. A page in any other tier (N>1) is marked by -+ * PG_referenced and PG_workingset. This implies a minimum of two tiers is -+ * supported without using additional bits in folio->flags. -+ * -+ * In contrast to moving across generations which requires the LRU lock, moving -+ * across tiers only involves atomic operations on folio->flags and therefore -+ * has a negligible cost in the buffered access path. In the eviction path, -+ * comparisons of refaulted/(evicted+protected) from the first tier and the -+ * rest infer whether pages accessed multiple times through file descriptors -+ * are statistically hot and thus worth protecting. -+ * -+ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the -+ * number of categories of the active/inactive LRU when keeping track of -+ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in -+ * folio->flags. -+ */ -+#define MAX_NR_TIERS 4U -+ -+#ifndef __GENERATING_BOUNDS_H -+ -+struct lruvec; -+struct page_vma_mapped_walk; -+ -+#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) -+#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) -+ -+#ifdef CONFIG_LRU_GEN -+ -+enum { -+ LRU_GEN_ANON, -+ LRU_GEN_FILE, -+}; -+ -+enum { -+ LRU_GEN_CORE, -+ LRU_GEN_MM_WALK, -+ LRU_GEN_NONLEAF_YOUNG, -+ NR_LRU_GEN_CAPS -+}; -+ -+#define MIN_LRU_BATCH BITS_PER_LONG -+#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) -+ -+/* whether to keep historical stats from evicted generations */ -+#ifdef CONFIG_LRU_GEN_STATS -+#define NR_HIST_GENS MAX_NR_GENS -+#else -+#define NR_HIST_GENS 1U -+#endif -+ -+/* -+ * The youngest generation number is stored in max_seq for both anon and file -+ * types as they are aged on an equal footing. The oldest generation numbers are -+ * stored in min_seq[] separately for anon and file types as clean file pages -+ * can be evicted regardless of swap constraints. -+ * -+ * Normally anon and file min_seq are in sync. But if swapping is constrained, -+ * e.g., out of swap space, file min_seq is allowed to advance and leave anon -+ * min_seq behind. -+ * -+ * The number of pages in each generation is eventually consistent and therefore -+ * can be transiently negative when reset_batch_size() is pending. -+ */ -+struct lru_gen_struct { -+ /* the aging increments the youngest generation number */ -+ unsigned long max_seq; -+ /* the eviction increments the oldest generation numbers */ -+ unsigned long min_seq[ANON_AND_FILE]; -+ /* the birth time of each generation in jiffies */ -+ unsigned long timestamps[MAX_NR_GENS]; -+ /* the multi-gen LRU lists, lazily sorted on eviction */ -+ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* the multi-gen LRU sizes, eventually consistent */ -+ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* the exponential moving average of refaulted */ -+ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; -+ /* the exponential moving average of evicted+protected */ -+ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; -+ /* the first tier doesn't need protection, hence the minus one */ -+ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; -+ /* can be modified without holding the LRU lock */ -+ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; -+ atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; -+ /* whether the multi-gen LRU is enabled */ -+ bool enabled; -+}; -+ -+enum { -+ MM_LEAF_TOTAL, /* total leaf entries */ -+ MM_LEAF_OLD, /* old leaf entries */ -+ MM_LEAF_YOUNG, /* young leaf entries */ -+ MM_NONLEAF_TOTAL, /* total non-leaf entries */ -+ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ -+ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ -+ NR_MM_STATS -+}; -+ -+/* double-buffering Bloom filters */ -+#define NR_BLOOM_FILTERS 2 -+ -+struct lru_gen_mm_state { -+ /* set to max_seq after each iteration */ -+ unsigned long seq; -+ /* where the current iteration continues (inclusive) */ -+ struct list_head *head; -+ /* where the last iteration ended (exclusive) */ -+ struct list_head *tail; -+ /* to wait for the last page table walker to finish */ -+ struct wait_queue_head wait; -+ /* Bloom filters flip after each iteration */ -+ unsigned long *filters[NR_BLOOM_FILTERS]; -+ /* the mm stats for debugging */ -+ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; -+ /* the number of concurrent page table walkers */ -+ int nr_walkers; -+}; -+ -+struct lru_gen_mm_walk { -+ /* the lruvec under reclaim */ -+ struct lruvec *lruvec; -+ /* unstable max_seq from lru_gen_struct */ -+ unsigned long max_seq; -+ /* the next address within an mm to scan */ -+ unsigned long next_addr; -+ /* to batch promoted pages */ -+ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* to batch the mm stats */ -+ int mm_stats[NR_MM_STATS]; -+ /* total batched items */ -+ int batched; -+ bool can_swap; -+ bool force_scan; -+}; -+ -+void lru_gen_init_lruvec(struct lruvec *lruvec); -+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); -+ -+#ifdef CONFIG_MEMCG -+void lru_gen_init_memcg(struct mem_cgroup *memcg); -+void lru_gen_exit_memcg(struct mem_cgroup *memcg); -+#endif -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline void lru_gen_init_lruvec(struct lruvec *lruvec) -+{ -+} -+ -+static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) -+{ -+} -+ -+#ifdef CONFIG_MEMCG -+static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) -+{ -+} -+ -+static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) -+{ -+} -+#endif -+ -+#endif /* CONFIG_LRU_GEN */ -+ - struct lruvec { - struct list_head lists[NR_LRU_LISTS]; - /* per lruvec lru_lock for memcg */ -@@ -331,6 +534,12 @@ struct lruvec { - unsigned long refaults[ANON_AND_FILE]; - /* Various lruvec state flags (enum lruvec_flags) */ - unsigned long flags; -+#ifdef CONFIG_LRU_GEN -+ /* evictable pages divided into generations */ -+ struct lru_gen_struct lrugen; -+ /* to concurrently iterate lru_gen_mm_list */ -+ struct lru_gen_mm_state mm_state; -+#endif - #ifdef CONFIG_MEMCG - struct pglist_data *pgdat; - #endif -@@ -746,6 +955,8 @@ static inline bool zone_is_empty(struct zone *zone) - #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) - #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) - #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) -+#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) -+#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) - - /* - * Define the bit shifts to access each section. For non-existent -@@ -1007,6 +1218,11 @@ typedef struct pglist_data { - - unsigned long flags; - -+#ifdef CONFIG_LRU_GEN -+ /* kswap mm walk data */ -+ struct lru_gen_mm_walk mm_walk; -+#endif -+ - ZONE_PADDING(_pad2_) - - /* Per-node vmstats */ -diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h -index 4b71a96190a8..3a0eec9f2faa 100644 ---- a/include/linux/nodemask.h -+++ b/include/linux/nodemask.h -@@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) - #define first_online_node 0 - #define first_memory_node 0 - #define next_online_node(nid) (MAX_NUMNODES) -+#define next_memory_node(nid) (MAX_NUMNODES) - #define nr_node_ids 1U - #define nr_online_nodes 1U - -diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h -index ef1e3e736e14..7d79818dc065 100644 ---- a/include/linux/page-flags-layout.h -+++ b/include/linux/page-flags-layout.h -@@ -55,7 +55,8 @@ - #define SECTIONS_WIDTH 0 - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ -+ <= BITS_PER_LONG - NR_PAGEFLAGS - #define NODES_WIDTH NODES_SHIFT - #elif defined(CONFIG_SPARSEMEM_VMEMMAP) - #error "Vmemmap: No space for nodes field in page flags" -@@ -89,8 +90,8 @@ - #define LAST_CPUPID_SHIFT 0 - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ -- <= BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ -+ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS - #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT - #else - #define LAST_CPUPID_WIDTH 0 -@@ -100,10 +101,15 @@ - #define LAST_CPUPID_NOT_IN_PAGE_FLAGS - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ -- > BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ -+ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS - #error "Not enough bits in page flags" - #endif - -+/* see the comment on MAX_NR_TIERS */ -+#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ -+ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ -+ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) -+ - #endif - #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ -diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h -index 465ff35a8c00..0b0ae5084e60 100644 ---- a/include/linux/page-flags.h -+++ b/include/linux/page-flags.h -@@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) - 1UL << PG_private | 1UL << PG_private_2 | \ - 1UL << PG_writeback | 1UL << PG_reserved | \ - 1UL << PG_slab | 1UL << PG_active | \ -- 1UL << PG_unevictable | __PG_MLOCKED) -+ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) - - /* - * Flags checked when a page is prepped for return by the page allocator. -@@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) - * alloc-free cycle to prevent from reusing the page. - */ - #define PAGE_FLAGS_CHECK_AT_PREP \ -- (PAGEFLAGS_MASK & ~__PG_HWPOISON) -+ ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) - - #define PAGE_FLAGS_PRIVATE \ - (1UL << PG_private | 1UL << PG_private_2) -diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h -index 014ee8f0fbaa..d9095251bffd 100644 ---- a/include/linux/pgtable.h -+++ b/include/linux/pgtable.h -@@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - #endif - - #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG --#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) - static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) -@@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - BUILD_BUG(); - return 0; - } --#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -+#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ - #endif - - #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -@@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, - #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - #endif - -+#ifndef arch_has_hw_pte_young -+/* -+ * Return whether the accessed bit is supported on the local CPU. -+ * -+ * This stub assumes accessing through an old PTE triggers a page fault. -+ * Architectures that automatically set the access bit should overwrite it. -+ */ -+static inline bool arch_has_hw_pte_young(void) -+{ -+ return false; -+} -+#endif -+ - #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR - static inline pte_t ptep_get_and_clear(struct mm_struct *mm, - unsigned long address, diff --git a/include/linux/sched.h b/include/linux/sched.h -index 367b30ed77cb..2cc9429d3585 100644 +index 070b15ba2e1e..2cc9429d3585 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -861,7 +861,6 @@ struct task_struct { @@ -4063,32 +2633,6 @@ index 367b30ed77cb..2cc9429d3585 100644 #ifdef SPLIT_RSS_COUNTING struct task_rss_stat rss_stat; -@@ -914,6 +913,10 @@ struct task_struct { - #ifdef CONFIG_MEMCG - unsigned in_user_fault:1; - #endif -+#ifdef CONFIG_LRU_GEN -+ /* whether the LRU algorithm may apply to this access */ -+ unsigned in_lru_fault:1; -+#endif - #ifdef CONFIG_COMPAT_BRK - unsigned brk_randomized:1; - #endif -diff --git a/include/linux/swap.h b/include/linux/swap.h -index 43150b9bbc5c..6308150b234a 100644 ---- a/include/linux/swap.h -+++ b/include/linux/swap.h -@@ -162,6 +162,10 @@ union swap_header { - */ - struct reclaim_state { - unsigned long reclaimed_slab; -+#ifdef CONFIG_LRU_GEN -+ /* per-thread mm walk data */ -+ struct lru_gen_mm_walk *mm_walk; -+#endif - }; - - #ifdef __KERNEL__ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index e1b8a915e9e9..f07e6998bb68 100644 --- a/include/linux/userfaultfd_k.h @@ -4180,18 +2724,6 @@ index bfe38869498d..19cf5b6892ce 100644 #define __count_zid_vm_events(item, zid, delta) \ __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) -diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h -index d651f3437367..55392bf30a03 100644 ---- a/include/trace/events/huge_memory.h -+++ b/include/trace/events/huge_memory.h -@@ -11,6 +11,7 @@ - EM( SCAN_FAIL, "failed") \ - EM( SCAN_SUCCEED, "succeeded") \ - EM( SCAN_PMD_NULL, "pmd_null") \ -+ EM( SCAN_PMD_MAPPED, "page_pmd_mapped") \ - EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \ - EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ - EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ diff --git a/include/trace/events/maple_tree.h b/include/trace/events/maple_tree.h new file mode 100644 index 000000000000..2be403bdc2bd @@ -4405,19 +2937,6 @@ index 4661f7ba07c0..216de5f03621 100644 #endif /* This part must be outside protection */ -diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h -index 6c1aa92a92e4..6ce1f1ceb432 100644 ---- a/include/uapi/asm-generic/mman-common.h -+++ b/include/uapi/asm-generic/mman-common.h -@@ -77,6 +77,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - diff --git a/init/main.c b/init/main.c index 1fe7942f5d4a..df800fc61b2a 100644 --- a/init/main.c @@ -4534,24 +3053,6 @@ index 13706356ec54..62200d799b9b 100644 } spin_lock_irq(¤t->sighand->siglock); -diff --git a/kernel/bounds.c b/kernel/bounds.c -index 9795d75b09b2..b529182e8b04 100644 ---- a/kernel/bounds.c -+++ b/kernel/bounds.c -@@ -22,6 +22,13 @@ int main(void) - DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); - #endif - DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); -+#ifdef CONFIG_LRU_GEN -+ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); -+ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); -+#else -+ DEFINE(LRU_GEN_WIDTH, 0); -+ DEFINE(__LRU_REFS_WIDTH, 0); -+#endif - /* End of constants */ - - return 0; diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 8c921799def4..1c8debd42dc9 100644 --- a/kernel/bpf/task_iter.c @@ -4589,18 +3090,6 @@ index 8c921799def4..1c8debd42dc9 100644 break; } if (!curr_vma) { -diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h -index 36b740cb3d59..63dc3e82be4f 100644 ---- a/kernel/cgroup/cgroup-internal.h -+++ b/kernel/cgroup/cgroup-internal.h -@@ -164,7 +164,6 @@ struct cgroup_mgctx { - #define DEFINE_CGROUP_MGCTX(name) \ - struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) - --extern struct mutex cgroup_mutex; - extern spinlock_t css_set_lock; - extern struct cgroup_subsys *cgroup_subsys[]; - extern struct list_head cgroup_roots; diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 7beceb447211..d5e9ccde3ab8 100644 --- a/kernel/debug/debug_core.c @@ -4688,20 +3177,8 @@ index 2eaa327f8158..401bc2d24ce0 100644 if (!valid_vma(vma, false)) continue; /* -diff --git a/kernel/exit.c b/kernel/exit.c -index 84021b24f79e..98a33bd7c25c 100644 ---- a/kernel/exit.c -+++ b/kernel/exit.c -@@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm) - goto retry; - } - WRITE_ONCE(mm->owner, c); -+ lru_gen_migrate_mm(mm); - task_unlock(c); - put_task_struct(c); - } diff --git a/kernel/fork.c b/kernel/fork.c -index 704fe6bc9cb4..1add76edca04 100644 +index b15aaae04403..1add76edca04 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -43,7 +43,6 @@ @@ -4836,23 +3313,7 @@ index 704fe6bc9cb4..1add76edca04 100644 atomic_set(&mm->mm_users, 1); atomic_set(&mm->mm_count, 1); seqcount_init(&mm->write_protect_seq); -@@ -1156,6 +1154,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - goto fail_nocontext; - - mm->user_ns = get_user_ns(user_ns); -+ lru_gen_init_mm(mm); - return mm; - - fail_nocontext: -@@ -1198,6 +1197,7 @@ static inline void __mmput(struct mm_struct *mm) - } - if (mm->binfmt) - module_put(mm->binfmt->module); -+ lru_gen_del_mm(mm); - mmdrop(mm); - } - -@@ -1289,13 +1289,16 @@ int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) +@@ -1291,13 +1289,16 @@ int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) /* Forbid mm->exe_file change if old file still mapped. */ old_exe_file = get_mm_exe_file(mm); if (old_exe_file) { @@ -4871,7 +3332,7 @@ index 704fe6bc9cb4..1add76edca04 100644 } mmap_read_unlock(mm); fput(old_exe_file); -@@ -1571,9 +1574,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) +@@ -1573,9 +1574,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) if (!oldmm) return 0; @@ -4881,37 +3342,11 @@ index 704fe6bc9cb4..1add76edca04 100644 if (clone_flags & CLONE_VM) { mmget(oldmm); mm = oldmm; -@@ -2700,6 +2700,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) - get_task_struct(p); - } - -+ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { -+ /* lock the task to synchronize with memcg migration */ -+ task_lock(p); -+ lru_gen_add_mm(p->mm); -+ task_unlock(p); -+ } -+ - wake_up_new_task(p); - - /* forking complete and child started to run, tell ptracer */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index f2534e712a89..265944f14948 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -5178,6 +5178,7 @@ context_switch(struct rq *rq, struct task_struct *prev, - * finish_task_switch()'s mmdrop(). - */ - switch_mm_irqs_off(prev->active_mm, next->mm, next); -+ lru_gen_use_mm(next->mm); - - if (!prev->mm) { // from kernel - /* will mmdrop() in finish_task_switch(). */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 74d773040b54..b544cd26794a 100644 +index 1d3e53fed1a4..4a6e22f96ede 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c -@@ -2783,6 +2783,7 @@ static void task_numa_work(struct callback_head *work) +@@ -2784,6 +2784,7 @@ static void task_numa_work(struct callback_head *work) struct task_struct *p = current; struct mm_struct *mm = p->mm; u64 runtime = p->se.sum_exec_runtime; @@ -4919,7 +3354,7 @@ index 74d773040b54..b544cd26794a 100644 struct vm_area_struct *vma; unsigned long start, end; unsigned long nr_pte_updates = 0; -@@ -2839,13 +2840,16 @@ static void task_numa_work(struct callback_head *work) +@@ -2840,13 +2841,16 @@ static void task_numa_work(struct callback_head *work) if (!mmap_read_trylock(mm)) return; @@ -50438,45 +48873,8 @@ index 000000000000..4f69e009a015 +module_exit(maple_tree_harvest); +MODULE_AUTHOR("Liam R. Howlett "); +MODULE_LICENSE("GPL"); -diff --git a/mm/Kconfig b/mm/Kconfig -index 0331f1461f81..96cd3ae25c6f 100644 ---- a/mm/Kconfig -+++ b/mm/Kconfig -@@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP - purposes. It is required to enable userfaultfd write protection on - file-backed memory types like shmem and hugetlbfs. - -+# multi-gen LRU { -+config LRU_GEN -+ bool "Multi-Gen LRU" -+ depends on MMU -+ # make sure folio->flags has enough spare bits -+ depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP -+ help -+ A high performance LRU implementation to overcommit memory. See -+ Documentation/admin-guide/mm/multigen_lru.rst for details. -+ -+config LRU_GEN_ENABLED -+ bool "Enable by default" -+ depends on LRU_GEN -+ help -+ This option enables the multi-gen LRU by default. -+ -+config LRU_GEN_STATS -+ bool "Full stats for debugging" -+ depends on LRU_GEN -+ help -+ Do not enable this option unless you plan to look at historical stats -+ from evicted generations for debugging purpose. -+ -+ This option has a per-memcg and per-node memory overhead. -+# } -+ - source "mm/damon/Kconfig" - - endmenu diff --git a/mm/Makefile b/mm/Makefile -index 9a564f836403..8083fa85a348 100644 +index 488f604e77e0..a731d1decbb1 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -52,7 +52,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ @@ -50708,66 +49106,10 @@ index 5abdaf487460..5f3c464dbce1 100644 /* * Set [nstart; nend) to intersection of desired address diff --git a/mm/huge_memory.c b/mm/huge_memory.c -index e9414ee57c5b..cd0eed90a865 100644 +index 786497dd5f26..cca500fcfb64 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c -@@ -70,9 +70,8 @@ static atomic_t huge_zero_refcount; - struct page *huge_zero_page __read_mostly; - unsigned long huge_zero_pfn __read_mostly = ~0UL; - --bool hugepage_vma_check(struct vm_area_struct *vma, -- unsigned long vm_flags, -- bool smaps, bool in_pf) -+bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, -+ bool smaps, bool in_pf, bool enforce_sysfs) - { - if (!vma->vm_mm) /* vdso */ - return false; -@@ -121,11 +120,10 @@ bool hugepage_vma_check(struct vm_area_struct *vma, - if (!in_pf && shmem_file(vma->vm_file)) - return shmem_huge_enabled(vma); - -- if (!hugepage_flags_enabled()) -- return false; -- -- /* THP settings require madvise. */ -- if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always()) -+ /* Enforce sysfs THP requirements as necessary */ -+ if (enforce_sysfs && -+ (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) && -+ !hugepage_flags_always()))) - return false; - - /* Only regular file is valid */ -@@ -2288,25 +2286,11 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, - void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, - bool freeze, struct folio *folio) - { -- pgd_t *pgd; -- p4d_t *p4d; -- pud_t *pud; -- pmd_t *pmd; -- -- pgd = pgd_offset(vma->vm_mm, address); -- if (!pgd_present(*pgd)) -- return; -+ pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); - -- p4d = p4d_offset(pgd, address); -- if (!p4d_present(*p4d)) -+ if (!pmd) - return; - -- pud = pud_offset(p4d, address); -- if (!pud_present(*pud)) -- return; -- -- pmd = pmd_offset(pud, address); -- - __split_huge_pmd(vma, pmd, address, freeze, folio); - } - -@@ -2334,11 +2318,11 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, +@@ -2319,11 +2319,11 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, split_huge_pmd_if_needed(vma, end); /* @@ -50781,16 +49123,6 @@ index e9414ee57c5b..cd0eed90a865 100644 unsigned long nstart = next->vm_start; nstart += adjust_next; split_huge_pmd_if_needed(next, nstart); -@@ -2438,7 +2422,8 @@ static void __split_huge_page_tail(struct page *head, int tail, - #ifdef CONFIG_64BIT - (1L << PG_arch_2) | - #endif -- (1L << PG_dirty))); -+ (1L << PG_dirty) | -+ LRU_GEN_MASK | LRU_REFS_MASK)); - - /* ->mapping in first tail page is compound_mapcount */ - VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, diff --git a/mm/init-mm.c b/mm/init-mm.c index fbe7844d0912..c9327abb771c 100644 --- a/mm/init-mm.c @@ -50813,14 +49145,12 @@ index fbe7844d0912..c9327abb771c 100644 .mm_users = ATOMIC_INIT(2), .mm_count = ATOMIC_INIT(1), diff --git a/mm/internal.h b/mm/internal.h -index 785409805ed7..0f106a3982e7 100644 +index cf134d58fd6d..0f106a3982e7 100644 --- a/mm/internal.h +++ b/mm/internal.h -@@ -83,9 +83,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); - void folio_rotate_reclaimable(struct folio *folio); - bool __folio_end_writeback(struct folio *folio); +@@ -85,8 +85,9 @@ bool __folio_end_writeback(struct folio *folio); void deactivate_file_folio(struct folio *folio); -+void folio_activate(struct folio *folio); + void folio_activate(struct folio *folio); -void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, - unsigned long floor, unsigned long ceiling); @@ -50830,16 +49160,7 @@ index 785409805ed7..0f106a3982e7 100644 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); struct zap_details; -@@ -187,7 +189,7 @@ extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason - /* - * in mm/rmap.c: - */ --extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); -+pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); - - /* - * in mm/page_alloc.c -@@ -479,9 +481,6 @@ static inline bool is_data_mapping(vm_flags_t flags) +@@ -480,9 +481,6 @@ static inline bool is_data_mapping(vm_flags_t flags) } /* mm/util.c */ @@ -50850,815 +49171,9 @@ index 785409805ed7..0f106a3982e7 100644 #ifdef CONFIG_MMU diff --git a/mm/khugepaged.c b/mm/khugepaged.c -index 01f71786d530..df890338daed 100644 +index 5f7c60b8b269..df890338daed 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c -@@ -28,6 +28,7 @@ enum scan_result { - SCAN_FAIL, - SCAN_SUCCEED, - SCAN_PMD_NULL, -+ SCAN_PMD_MAPPED, - SCAN_EXCEED_NONE_PTE, - SCAN_EXCEED_SWAP_PTE, - SCAN_EXCEED_SHARED_PTE, -@@ -73,6 +74,8 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); - * default collapse hugepages if there is at least one pte mapped like - * it would have happened if the vma was large enough during page - * fault. -+ * -+ * Note that these are only respected if collapse was initiated by khugepaged. - */ - static unsigned int khugepaged_max_ptes_none __read_mostly; - static unsigned int khugepaged_max_ptes_swap __read_mostly; -@@ -85,6 +88,16 @@ static struct kmem_cache *mm_slot_cache __read_mostly; - - #define MAX_PTE_MAPPED_THP 8 - -+struct collapse_control { -+ bool is_khugepaged; -+ -+ /* Num pages scanned per node */ -+ u32 node_load[MAX_NUMNODES]; -+ -+ /* Last target selected in hpage_collapse_find_target_node() */ -+ int last_target_node; -+}; -+ - /** - * struct mm_slot - hash lookup from mm to mm_slot - * @hash: hash collision list -@@ -425,7 +438,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm, - hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); - } - --static inline int khugepaged_test_exit(struct mm_struct *mm) -+static inline int hpage_collapse_test_exit(struct mm_struct *mm) - { - return atomic_read(&mm->mm_users) == 0; - } -@@ -440,7 +453,7 @@ void __khugepaged_enter(struct mm_struct *mm) - return; - - /* __khugepaged_exit() must not run from under us */ -- VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); -+ VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); - if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { - free_mm_slot(mm_slot); - return; -@@ -466,7 +479,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, - { - if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - hugepage_flags_enabled()) { -- if (hugepage_vma_check(vma, vm_flags, false, false)) -+ if (hugepage_vma_check(vma, vm_flags, false, false, true)) - __khugepaged_enter(vma->vm_mm); - } - } -@@ -492,11 +505,10 @@ void __khugepaged_exit(struct mm_struct *mm) - } else if (mm_slot) { - /* - * This is required to serialize against -- * khugepaged_test_exit() (which is guaranteed to run -- * under mmap sem read mode). Stop here (after we -- * return all pagetables will be destroyed) until -- * khugepaged has finished working on the pagetables -- * under the mmap_lock. -+ * hpage_collapse_test_exit() (which is guaranteed to run -+ * under mmap sem read mode). Stop here (after we return all -+ * pagetables will be destroyed) until khugepaged has finished -+ * working on the pagetables under the mmap_lock. - */ - mmap_write_lock(mm); - mmap_write_unlock(mm); -@@ -546,11 +558,12 @@ static bool is_refcount_suitable(struct page *page) - static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - unsigned long address, - pte_t *pte, -+ struct collapse_control *cc, - struct list_head *compound_pagelist) - { - struct page *page = NULL; - pte_t *_pte; -- int none_or_zero = 0, shared = 0, result = 0, referenced = 0; -+ int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; - bool writable = false; - - for (_pte = pte; _pte < pte + HPAGE_PMD_NR; -@@ -558,8 +571,10 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - pte_t pteval = *_pte; - if (pte_none(pteval) || (pte_present(pteval) && - is_zero_pfn(pte_pfn(pteval)))) { -+ ++none_or_zero; - if (!userfaultfd_armed(vma) && -- ++none_or_zero <= khugepaged_max_ptes_none) { -+ (!cc->is_khugepaged || -+ none_or_zero <= khugepaged_max_ptes_none)) { - continue; - } else { - result = SCAN_EXCEED_NONE_PTE; -@@ -579,11 +594,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - - VM_BUG_ON_PAGE(!PageAnon(page), page); - -- if (page_mapcount(page) > 1 && -- ++shared > khugepaged_max_ptes_shared) { -- result = SCAN_EXCEED_SHARED_PTE; -- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -- goto out; -+ if (page_mapcount(page) > 1) { -+ ++shared; -+ if (cc->is_khugepaged && -+ shared > khugepaged_max_ptes_shared) { -+ result = SCAN_EXCEED_SHARED_PTE; -+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -+ goto out; -+ } - } - - if (PageCompound(page)) { -@@ -646,10 +664,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - if (PageCompound(page)) - list_add_tail(&page->lru, compound_pagelist); - next: -- /* There should be enough young pte to collapse the page */ -- if (pte_young(pteval) || -- page_is_young(page) || PageReferenced(page) || -- mmu_notifier_test_young(vma->vm_mm, address)) -+ /* -+ * If collapse was initiated by khugepaged, check that there is -+ * enough young pte to justify collapsing the page -+ */ -+ if (cc->is_khugepaged && -+ (pte_young(pteval) || page_is_young(page) || -+ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, -+ address))) - referenced++; - - if (pte_write(pteval)) -@@ -658,19 +680,19 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - - if (unlikely(!writable)) { - result = SCAN_PAGE_RO; -- } else if (unlikely(!referenced)) { -+ } else if (unlikely(cc->is_khugepaged && !referenced)) { - result = SCAN_LACK_REFERENCED_PAGE; - } else { - result = SCAN_SUCCEED; - trace_mm_collapse_huge_page_isolate(page, none_or_zero, - referenced, writable, result); -- return 1; -+ return result; - } - out: - release_pte_pages(pte, _pte, compound_pagelist); - trace_mm_collapse_huge_page_isolate(page, none_or_zero, - referenced, writable, result); -- return 0; -+ return result; - } - - static void __collapse_huge_page_copy(pte_t *pte, struct page *page, -@@ -735,9 +757,12 @@ static void khugepaged_alloc_sleep(void) - remove_wait_queue(&khugepaged_wait, &wait); - } - --static int khugepaged_node_load[MAX_NUMNODES]; -+struct collapse_control khugepaged_collapse_control = { -+ .is_khugepaged = true, -+ .last_target_node = NUMA_NO_NODE, -+}; - --static bool khugepaged_scan_abort(int nid) -+static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) - { - int i; - -@@ -749,11 +774,11 @@ static bool khugepaged_scan_abort(int nid) - return false; - - /* If there is a count for this node already, it must be acceptable */ -- if (khugepaged_node_load[nid]) -+ if (cc->node_load[nid]) - return false; - - for (i = 0; i < MAX_NUMNODES; i++) { -- if (!khugepaged_node_load[i]) -+ if (!cc->node_load[i]) - continue; - if (node_distance(nid, i) > node_reclaim_distance) - return true; -@@ -772,146 +797,62 @@ static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) - } - - #ifdef CONFIG_NUMA --static int khugepaged_find_target_node(void) -+static int hpage_collapse_find_target_node(struct collapse_control *cc) - { -- static int last_khugepaged_target_node = NUMA_NO_NODE; - int nid, target_node = 0, max_value = 0; - - /* find first node with max normal pages hit */ - for (nid = 0; nid < MAX_NUMNODES; nid++) -- if (khugepaged_node_load[nid] > max_value) { -- max_value = khugepaged_node_load[nid]; -+ if (cc->node_load[nid] > max_value) { -+ max_value = cc->node_load[nid]; - target_node = nid; - } - - /* do some balance if several nodes have the same hit record */ -- if (target_node <= last_khugepaged_target_node) -- for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; -- nid++) -- if (max_value == khugepaged_node_load[nid]) { -+ if (target_node <= cc->last_target_node) -+ for (nid = cc->last_target_node + 1; nid < MAX_NUMNODES; -+ nid++) -+ if (max_value == cc->node_load[nid]) { - target_node = nid; - break; - } - -- last_khugepaged_target_node = target_node; -+ cc->last_target_node = target_node; - return target_node; - } -- --static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) -+#else -+static int hpage_collapse_find_target_node(struct collapse_control *cc) - { -- if (IS_ERR(*hpage)) { -- if (!*wait) -- return false; -- -- *wait = false; -- *hpage = NULL; -- khugepaged_alloc_sleep(); -- } else if (*hpage) { -- put_page(*hpage); -- *hpage = NULL; -- } -- -- return true; -+ return 0; - } -+#endif - --static struct page * --khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) -+static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node) - { -- VM_BUG_ON_PAGE(*hpage, *hpage); -- - *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); - if (unlikely(!*hpage)) { - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); -- *hpage = ERR_PTR(-ENOMEM); -- return NULL; -+ return false; - } - - prep_transhuge_page(*hpage); - count_vm_event(THP_COLLAPSE_ALLOC); -- return *hpage; --} --#else --static int khugepaged_find_target_node(void) --{ -- return 0; --} -- --static inline struct page *alloc_khugepaged_hugepage(void) --{ -- struct page *page; -- -- page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), -- HPAGE_PMD_ORDER); -- if (page) -- prep_transhuge_page(page); -- return page; --} -- --static struct page *khugepaged_alloc_hugepage(bool *wait) --{ -- struct page *hpage; -- -- do { -- hpage = alloc_khugepaged_hugepage(); -- if (!hpage) { -- count_vm_event(THP_COLLAPSE_ALLOC_FAILED); -- if (!*wait) -- return NULL; -- -- *wait = false; -- khugepaged_alloc_sleep(); -- } else -- count_vm_event(THP_COLLAPSE_ALLOC); -- } while (unlikely(!hpage) && likely(hugepage_flags_enabled())); -- -- return hpage; --} -- --static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) --{ -- /* -- * If the hpage allocated earlier was briefly exposed in page cache -- * before collapse_file() failed, it is possible that racing lookups -- * have not yet completed, and would then be unpleasantly surprised by -- * finding the hpage reused for the same mapping at a different offset. -- * Just release the previous allocation if there is any danger of that. -- */ -- if (*hpage && page_count(*hpage) > 1) { -- put_page(*hpage); -- *hpage = NULL; -- } -- -- if (!*hpage) -- *hpage = khugepaged_alloc_hugepage(wait); -- -- if (unlikely(!*hpage)) -- return false; -- - return true; - } - --static struct page * --khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) --{ -- VM_BUG_ON(!*hpage); -- -- return *hpage; --} --#endif -- - /* - * If mmap_lock temporarily dropped, revalidate vma - * before taking mmap_lock. -- * Return 0 if succeeds, otherwise return none-zero -- * value (scan code). -+ * Returns enum scan_result value. - */ - - static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, -- struct vm_area_struct **vmap) -+ struct vm_area_struct **vmap, -+ struct collapse_control *cc) - { - struct vm_area_struct *vma; - -- if (unlikely(khugepaged_test_exit(mm))) -+ if (unlikely(hpage_collapse_test_exit(mm))) - return SCAN_ANY_PROCESS; - - *vmap = vma = find_vma(mm, address); -@@ -920,7 +861,8 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, - - if (!transhuge_vma_suitable(vma, address)) - return SCAN_ADDRESS_RANGE; -- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, -+ cc->is_khugepaged)) - return SCAN_VMA_CHECK; - /* - * Anon VMA expected, the address may be unmapped then -@@ -931,21 +873,60 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, - */ - if (!vma->anon_vma || !vma_is_anonymous(vma)) - return SCAN_VMA_CHECK; -- return 0; -+ return SCAN_SUCCEED; -+} -+ -+static int find_pmd_or_thp_or_none(struct mm_struct *mm, -+ unsigned long address, -+ pmd_t **pmd) -+{ -+ pmd_t pmde; -+ -+ *pmd = mm_find_pmd(mm, address); -+ if (!*pmd) -+ return SCAN_PMD_NULL; -+ -+ pmde = pmd_read_atomic(*pmd); -+ -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ -+ barrier(); -+#endif -+ if (!pmd_present(pmde)) -+ return SCAN_PMD_NULL; -+ if (pmd_trans_huge(pmde)) -+ return SCAN_PMD_MAPPED; -+ if (pmd_bad(pmde)) -+ return SCAN_PMD_NULL; -+ return SCAN_SUCCEED; -+} -+ -+static int check_pmd_still_valid(struct mm_struct *mm, -+ unsigned long address, -+ pmd_t *pmd) -+{ -+ pmd_t *new_pmd; -+ int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); -+ -+ if (result != SCAN_SUCCEED) -+ return result; -+ if (new_pmd != pmd) -+ return SCAN_FAIL; -+ return SCAN_SUCCEED; - } - - /* - * Bring missing pages in from swap, to complete THP collapse. -- * Only done if khugepaged_scan_pmd believes it is worthwhile. -+ * Only done if hpage_collapse_scan_pmd believes it is worthwhile. - * - * Called and returns without pte mapped or spinlocks held. - * Note that if false is returned, mmap_lock will be released. - */ - --static bool __collapse_huge_page_swapin(struct mm_struct *mm, -- struct vm_area_struct *vma, -- unsigned long haddr, pmd_t *pmd, -- int referenced) -+static int __collapse_huge_page_swapin(struct mm_struct *mm, -+ struct vm_area_struct *vma, -+ unsigned long haddr, pmd_t *pmd, -+ int referenced) - { - int swapped_in = 0; - vm_fault_t ret = 0; -@@ -976,12 +957,13 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, - */ - if (ret & VM_FAULT_RETRY) { - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); -- return false; -+ /* Likely, but not guaranteed, that page lock failed */ -+ return SCAN_PAGE_LOCK; - } - if (ret & VM_FAULT_ERROR) { - mmap_read_unlock(mm); - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); -- return false; -+ return SCAN_FAIL; - } - swapped_in++; - } -@@ -991,30 +973,41 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, - lru_add_drain(); - - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); -- return true; -+ return SCAN_SUCCEED; - } - --static void collapse_huge_page(struct mm_struct *mm, -- unsigned long address, -- struct page **hpage, -- int node, int referenced, int unmapped) -+static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, -+ struct collapse_control *cc) -+{ -+ /* Only allocate from the target node */ -+ gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : -+ GFP_TRANSHUGE) | __GFP_THISNODE; -+ int node = hpage_collapse_find_target_node(cc); -+ -+ if (!hpage_collapse_alloc_page(hpage, gfp, node)) -+ return SCAN_ALLOC_HUGE_PAGE_FAIL; -+ if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) -+ return SCAN_CGROUP_CHARGE_FAIL; -+ count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); -+ return SCAN_SUCCEED; -+} -+ -+static int collapse_huge_page(struct mm_struct *mm, unsigned long address, -+ int referenced, int unmapped, -+ struct collapse_control *cc) - { - LIST_HEAD(compound_pagelist); - pmd_t *pmd, _pmd; - pte_t *pte; - pgtable_t pgtable; -- struct page *new_page; -+ struct page *hpage; - spinlock_t *pmd_ptl, *pte_ptl; -- int isolated = 0, result = 0; -+ int result = SCAN_FAIL; - struct vm_area_struct *vma; - struct mmu_notifier_range range; -- gfp_t gfp; - - VM_BUG_ON(address & ~HPAGE_PMD_MASK); - -- /* Only allocate from the target node */ -- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; -- - /* - * Before allocating the hugepage, release the mmap_lock read lock. - * The allocation can take potentially a long time if it involves -@@ -1022,40 +1015,34 @@ static void collapse_huge_page(struct mm_struct *mm, - * that. We will recheck the vma after taking it again in write mode. - */ - mmap_read_unlock(mm); -- new_page = khugepaged_alloc_page(hpage, gfp, node); -- if (!new_page) { -- result = SCAN_ALLOC_HUGE_PAGE_FAIL; -- goto out_nolock; -- } - -- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { -- result = SCAN_CGROUP_CHARGE_FAIL; -+ result = alloc_charge_hpage(&hpage, mm, cc); -+ if (result != SCAN_SUCCEED) - goto out_nolock; -- } -- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); - - mmap_read_lock(mm); -- result = hugepage_vma_revalidate(mm, address, &vma); -- if (result) { -+ result = hugepage_vma_revalidate(mm, address, &vma, cc); -+ if (result != SCAN_SUCCEED) { - mmap_read_unlock(mm); - goto out_nolock; - } - -- pmd = mm_find_pmd(mm, address); -- if (!pmd) { -- result = SCAN_PMD_NULL; -+ result = find_pmd_or_thp_or_none(mm, address, &pmd); -+ if (result != SCAN_SUCCEED) { - mmap_read_unlock(mm); - goto out_nolock; - } - -- /* -- * __collapse_huge_page_swapin will return with mmap_lock released -- * when it fails. So we jump out_nolock directly in that case. -- * Continuing to collapse causes inconsistency. -- */ -- if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, -- pmd, referenced)) { -- goto out_nolock; -+ if (unmapped) { -+ /* -+ * __collapse_huge_page_swapin will return with mmap_lock -+ * released when it fails. So we jump out_nolock directly in -+ * that case. Continuing to collapse causes inconsistency. -+ */ -+ result = __collapse_huge_page_swapin(mm, vma, address, pmd, -+ referenced); -+ if (result != SCAN_SUCCEED) -+ goto out_nolock; - } - - mmap_read_unlock(mm); -@@ -1065,11 +1052,12 @@ static void collapse_huge_page(struct mm_struct *mm, - * handled by the anon_vma lock + PG_lock. - */ - mmap_write_lock(mm); -- result = hugepage_vma_revalidate(mm, address, &vma); -- if (result) -+ result = hugepage_vma_revalidate(mm, address, &vma, cc); -+ if (result != SCAN_SUCCEED) - goto out_up_write; - /* check if the pmd is still valid */ -- if (mm_find_pmd(mm, address) != pmd) -+ result = check_pmd_still_valid(mm, address, pmd); -+ if (result != SCAN_SUCCEED) - goto out_up_write; - - anon_vma_lock_write(vma->anon_vma); -@@ -1093,11 +1081,11 @@ static void collapse_huge_page(struct mm_struct *mm, - mmu_notifier_invalidate_range_end(&range); - - spin_lock(pte_ptl); -- isolated = __collapse_huge_page_isolate(vma, address, pte, -- &compound_pagelist); -+ result = __collapse_huge_page_isolate(vma, address, pte, cc, -+ &compound_pagelist); - spin_unlock(pte_ptl); - -- if (unlikely(!isolated)) { -+ if (unlikely(result != SCAN_SUCCEED)) { - pte_unmap(pte); - spin_lock(pmd_ptl); - BUG_ON(!pmd_none(*pmd)); -@@ -1109,7 +1097,6 @@ static void collapse_huge_page(struct mm_struct *mm, - pmd_populate(mm, pmd, pmd_pgtable(_pmd)); - spin_unlock(pmd_ptl); - anon_vma_unlock_write(vma->anon_vma); -- result = SCAN_FAIL; - goto out_up_write; - } - -@@ -1119,8 +1106,8 @@ static void collapse_huge_page(struct mm_struct *mm, - */ - anon_vma_unlock_write(vma->anon_vma); - -- __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, -- &compound_pagelist); -+ __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, -+ &compound_pagelist); - pte_unmap(pte); - /* - * spin_lock() below is not the equivalent of smp_wmb(), but -@@ -1128,42 +1115,43 @@ static void collapse_huge_page(struct mm_struct *mm, - * avoid the copy_huge_page writes to become visible after - * the set_pmd_at() write. - */ -- __SetPageUptodate(new_page); -+ __SetPageUptodate(hpage); - pgtable = pmd_pgtable(_pmd); - -- _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); -+ _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); - _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); - - spin_lock(pmd_ptl); - BUG_ON(!pmd_none(*pmd)); -- page_add_new_anon_rmap(new_page, vma, address); -- lru_cache_add_inactive_or_unevictable(new_page, vma); -+ page_add_new_anon_rmap(hpage, vma, address); -+ lru_cache_add_inactive_or_unevictable(hpage, vma); - pgtable_trans_huge_deposit(mm, pmd, pgtable); - set_pmd_at(mm, address, pmd, _pmd); - update_mmu_cache_pmd(vma, address, pmd); - spin_unlock(pmd_ptl); - -- *hpage = NULL; -+ hpage = NULL; - -- khugepaged_pages_collapsed++; - result = SCAN_SUCCEED; - out_up_write: - mmap_write_unlock(mm); - out_nolock: -- if (!IS_ERR_OR_NULL(*hpage)) -- mem_cgroup_uncharge(page_folio(*hpage)); -- trace_mm_collapse_huge_page(mm, isolated, result); -- return; -+ if (hpage) { -+ mem_cgroup_uncharge(page_folio(hpage)); -+ put_page(hpage); -+ } -+ trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); -+ return result; - } - --static int khugepaged_scan_pmd(struct mm_struct *mm, -- struct vm_area_struct *vma, -- unsigned long address, -- struct page **hpage) -+static int hpage_collapse_scan_pmd(struct mm_struct *mm, -+ struct vm_area_struct *vma, -+ unsigned long address, bool *mmap_locked, -+ struct collapse_control *cc) - { - pmd_t *pmd; - pte_t *pte, *_pte; -- int ret = 0, result = 0, referenced = 0; -+ int result = SCAN_FAIL, referenced = 0; - int none_or_zero = 0, shared = 0; - struct page *page = NULL; - unsigned long _address; -@@ -1173,19 +1161,19 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - - VM_BUG_ON(address & ~HPAGE_PMD_MASK); - -- pmd = mm_find_pmd(mm, address); -- if (!pmd) { -- result = SCAN_PMD_NULL; -+ result = find_pmd_or_thp_or_none(mm, address, &pmd); -+ if (result != SCAN_SUCCEED) - goto out; -- } - -- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); -+ memset(cc->node_load, 0, sizeof(cc->node_load)); - pte = pte_offset_map_lock(mm, pmd, address, &ptl); - for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; - _pte++, _address += PAGE_SIZE) { - pte_t pteval = *_pte; - if (is_swap_pte(pteval)) { -- if (++unmapped <= khugepaged_max_ptes_swap) { -+ ++unmapped; -+ if (!cc->is_khugepaged || -+ unmapped <= khugepaged_max_ptes_swap) { - /* - * Always be strict with uffd-wp - * enabled swap entries. Please see -@@ -1203,8 +1191,10 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - } - } - if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { -+ ++none_or_zero; - if (!userfaultfd_armed(vma) && -- ++none_or_zero <= khugepaged_max_ptes_none) { -+ (!cc->is_khugepaged || -+ none_or_zero <= khugepaged_max_ptes_none)) { - continue; - } else { - result = SCAN_EXCEED_NONE_PTE; -@@ -1234,27 +1224,30 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - goto out_unmap; - } - -- if (page_mapcount(page) > 1 && -- ++shared > khugepaged_max_ptes_shared) { -- result = SCAN_EXCEED_SHARED_PTE; -- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -- goto out_unmap; -+ if (page_mapcount(page) > 1) { -+ ++shared; -+ if (cc->is_khugepaged && -+ shared > khugepaged_max_ptes_shared) { -+ result = SCAN_EXCEED_SHARED_PTE; -+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -+ goto out_unmap; -+ } - } - - page = compound_head(page); - - /* - * Record which node the original page is from and save this -- * information to khugepaged_node_load[]. -+ * information to cc->node_load[]. - * Khugepaged will allocate hugepage from the node has the max - * hit record. - */ - node = page_to_nid(page); -- if (khugepaged_scan_abort(node)) { -+ if (hpage_collapse_scan_abort(node, cc)) { - result = SCAN_SCAN_ABORT; - goto out_unmap; - } -- khugepaged_node_load[node]++; -+ cc->node_load[node]++; - if (!PageLRU(page)) { - result = SCAN_PAGE_LRU; - goto out_unmap; -@@ -1289,31 +1282,38 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - result = SCAN_PAGE_COUNT; - goto out_unmap; - } -- if (pte_young(pteval) || -- page_is_young(page) || PageReferenced(page) || -- mmu_notifier_test_young(vma->vm_mm, address)) -+ -+ /* -+ * If collapse was initiated by khugepaged, check that there is -+ * enough young pte to justify collapsing the page -+ */ -+ if (cc->is_khugepaged && -+ (pte_young(pteval) || page_is_young(page) || -+ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, -+ address))) - referenced++; - } - if (!writable) { - result = SCAN_PAGE_RO; -- } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { -+ } else if (cc->is_khugepaged && -+ (!referenced || -+ (unmapped && referenced < HPAGE_PMD_NR / 2))) { - result = SCAN_LACK_REFERENCED_PAGE; - } else { - result = SCAN_SUCCEED; -- ret = 1; - } - out_unmap: - pte_unmap_unlock(pte, ptl); -- if (ret) { -- node = khugepaged_find_target_node(); -+ if (result == SCAN_SUCCEED) { -+ result = collapse_huge_page(mm, address, referenced, -+ unmapped, cc); - /* collapse_huge_page will return with the mmap_lock released */ -- collapse_huge_page(mm, address, hpage, node, -- referenced, unmapped); -+ *mmap_locked = false; - } - out: - trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, - none_or_zero, result, unmapped); -- return ret; -+ return result; - } - - static void collect_mm_slot(struct mm_slot *mm_slot) -@@ -1322,7 +1322,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) - - lockdep_assert_held(&khugepaged_mm_lock); - -- if (khugepaged_test_exit(mm)) { -+ if (hpage_collapse_test_exit(mm)) { - /* free mm_slot */ - hash_del(&mm_slot->hash); - list_del(&mm_slot->mm_node); @@ -1387,7 +1387,7 @@ static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *v void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) { @@ -51668,337 +49183,7 @@ index 01f71786d530..df890338daed 100644 struct page *hpage; pte_t *start_pte, *pte; pmd_t *pmd; -@@ -1400,12 +1400,13 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) - return; - - /* -- * This vm_flags may not have VM_HUGEPAGE if the page was not -- * collapsed by this mm. But we can still collapse if the page is -- * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() -- * will not fail the vma for missing VM_HUGEPAGE -+ * If we are here, we've succeeded in replacing all the native pages -+ * in the page cache with a single hugepage. If a mm were to fault-in -+ * this memory (mapped by a suitably aligned VMA), we'd get the hugepage -+ * and map it by a PMD, regardless of sysfs THP settings. As such, let's -+ * analogously elide sysfs THP settings here. - */ -- if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false)) -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) - return; - - /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ -@@ -1420,8 +1421,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) - if (!PageHead(hpage)) - goto drop_hpage; - -- pmd = mm_find_pmd(mm, haddr); -- if (!pmd) -+ if (find_pmd_or_thp_or_none(mm, haddr, &pmd) != SCAN_SUCCEED) - goto drop_hpage; - - start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); -@@ -1495,7 +1495,7 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) - if (!mmap_write_trylock(mm)) - return; - -- if (unlikely(khugepaged_test_exit(mm))) -+ if (unlikely(hpage_collapse_test_exit(mm))) - goto out; - - for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) -@@ -1539,8 +1539,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - if (vma->vm_end < addr + HPAGE_PMD_SIZE) - continue; - mm = vma->vm_mm; -- pmd = mm_find_pmd(mm, addr); -- if (!pmd) -+ if (find_pmd_or_thp_or_none(mm, addr, &pmd) != SCAN_SUCCEED) - continue; - /* - * We need exclusive mmap_lock to retract page table. -@@ -1558,7 +1557,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - * it'll always mapped in small page size for uffd-wp - * registered ranges. - */ -- if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) -+ if (!hpage_collapse_test_exit(mm) && -+ !userfaultfd_wp(vma)) - collapse_and_free_pmd(mm, vma, addr, pmd); - mmap_write_unlock(mm); - } else { -@@ -1575,8 +1575,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - * @mm: process address space where collapse happens - * @file: file that collapse on - * @start: collapse start address -- * @hpage: new allocated huge page for collapse -- * @node: appointed node the new huge page allocate from -+ * @cc: collapse context and scratchpad - * - * Basic scheme is simple, details are more complex: - * - allocate and lock a new huge page; -@@ -1593,13 +1592,11 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - * + restore gaps in the page cache; - * + unlock and free huge page; - */ --static void collapse_file(struct mm_struct *mm, -- struct file *file, pgoff_t start, -- struct page **hpage, int node) -+static int collapse_file(struct mm_struct *mm, struct file *file, -+ pgoff_t start, struct collapse_control *cc) - { - struct address_space *mapping = file->f_mapping; -- gfp_t gfp; -- struct page *new_page; -+ struct page *hpage; - pgoff_t index, end = start + HPAGE_PMD_NR; - LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); -@@ -1610,20 +1607,9 @@ static void collapse_file(struct mm_struct *mm, - VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); - -- /* Only allocate from the target node */ -- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; -- -- new_page = khugepaged_alloc_page(hpage, gfp, node); -- if (!new_page) { -- result = SCAN_ALLOC_HUGE_PAGE_FAIL; -+ result = alloc_charge_hpage(&hpage, mm, cc); -+ if (result != SCAN_SUCCEED) - goto out; -- } -- -- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { -- result = SCAN_CGROUP_CHARGE_FAIL; -- goto out; -- } -- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); - - /* - * Ensure we have slots for all the pages in the range. This is -@@ -1641,14 +1627,14 @@ static void collapse_file(struct mm_struct *mm, - } - } while (1); - -- __SetPageLocked(new_page); -+ __SetPageLocked(hpage); - if (is_shmem) -- __SetPageSwapBacked(new_page); -- new_page->index = start; -- new_page->mapping = mapping; -+ __SetPageSwapBacked(hpage); -+ hpage->index = start; -+ hpage->mapping = mapping; - - /* -- * At this point the new_page is locked and not up-to-date. -+ * At this point the hpage is locked and not up-to-date. - * It's safe to insert it into the page cache, because nobody would - * be able to map it or use it in another way until we unlock it. - */ -@@ -1676,7 +1662,7 @@ static void collapse_file(struct mm_struct *mm, - result = SCAN_FAIL; - goto xa_locked; - } -- xas_store(&xas, new_page); -+ xas_store(&xas, hpage); - nr_none++; - continue; - } -@@ -1818,19 +1804,19 @@ static void collapse_file(struct mm_struct *mm, - list_add_tail(&page->lru, &pagelist); - - /* Finally, replace with the new page. */ -- xas_store(&xas, new_page); -+ xas_store(&xas, hpage); - continue; - out_unlock: - unlock_page(page); - put_page(page); - goto xa_unlocked; - } -- nr = thp_nr_pages(new_page); -+ nr = thp_nr_pages(hpage); - - if (is_shmem) -- __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); -+ __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); - else { -- __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); -+ __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); - filemap_nr_thps_inc(mapping); - /* - * Paired with smp_mb() in do_dentry_open() to ensure -@@ -1841,21 +1827,21 @@ static void collapse_file(struct mm_struct *mm, - smp_mb(); - if (inode_is_open_for_write(mapping->host)) { - result = SCAN_FAIL; -- __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); -+ __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); - filemap_nr_thps_dec(mapping); - goto xa_locked; - } - } - - if (nr_none) { -- __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); -+ __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); - /* nr_none is always 0 for non-shmem. */ -- __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); -+ __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); - } - - /* Join all the small entries into a single multi-index entry */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); -- xas_store(&xas, new_page); -+ xas_store(&xas, hpage); - xa_locked: - xas_unlock_irq(&xas); - xa_unlocked: -@@ -1877,11 +1863,11 @@ static void collapse_file(struct mm_struct *mm, - index = start; - list_for_each_entry_safe(page, tmp, &pagelist, lru) { - while (index < page->index) { -- clear_highpage(new_page + (index % HPAGE_PMD_NR)); -+ clear_highpage(hpage + (index % HPAGE_PMD_NR)); - index++; - } -- copy_highpage(new_page + (page->index % HPAGE_PMD_NR), -- page); -+ copy_highpage(hpage + (page->index % HPAGE_PMD_NR), -+ page); - list_del(&page->lru); - page->mapping = NULL; - page_ref_unfreeze(page, 1); -@@ -1892,23 +1878,22 @@ static void collapse_file(struct mm_struct *mm, - index++; - } - while (index < end) { -- clear_highpage(new_page + (index % HPAGE_PMD_NR)); -+ clear_highpage(hpage + (index % HPAGE_PMD_NR)); - index++; - } - -- SetPageUptodate(new_page); -- page_ref_add(new_page, HPAGE_PMD_NR - 1); -+ SetPageUptodate(hpage); -+ page_ref_add(hpage, HPAGE_PMD_NR - 1); - if (is_shmem) -- set_page_dirty(new_page); -- lru_cache_add(new_page); -+ set_page_dirty(hpage); -+ lru_cache_add(hpage); - - /* - * Remove pte page tables, so we can re-fault the page as huge. - */ - retract_page_tables(mapping, start); -- *hpage = NULL; -- -- khugepaged_pages_collapsed++; -+ unlock_page(hpage); -+ hpage = NULL; - } else { - struct page *page; - -@@ -1947,19 +1932,23 @@ static void collapse_file(struct mm_struct *mm, - VM_BUG_ON(nr_none); - xas_unlock_irq(&xas); - -- new_page->mapping = NULL; -+ hpage->mapping = NULL; - } - -- unlock_page(new_page); -+ if (hpage) -+ unlock_page(hpage); - out: - VM_BUG_ON(!list_empty(&pagelist)); -- if (!IS_ERR_OR_NULL(*hpage)) -- mem_cgroup_uncharge(page_folio(*hpage)); -+ if (hpage) { -+ mem_cgroup_uncharge(page_folio(hpage)); -+ put_page(hpage); -+ } - /* TODO: tracepoints */ -+ return result; - } - --static void khugepaged_scan_file(struct mm_struct *mm, -- struct file *file, pgoff_t start, struct page **hpage) -+static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, -+ pgoff_t start, struct collapse_control *cc) - { - struct page *page = NULL; - struct address_space *mapping = file->f_mapping; -@@ -1970,14 +1959,16 @@ static void khugepaged_scan_file(struct mm_struct *mm, - - present = 0; - swap = 0; -- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); -+ memset(cc->node_load, 0, sizeof(cc->node_load)); - rcu_read_lock(); - xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { - if (xas_retry(&xas, page)) - continue; - - if (xa_is_value(page)) { -- if (++swap > khugepaged_max_ptes_swap) { -+ ++swap; -+ if (cc->is_khugepaged && -+ swap > khugepaged_max_ptes_swap) { - result = SCAN_EXCEED_SWAP_PTE; - count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); - break; -@@ -1995,11 +1986,11 @@ static void khugepaged_scan_file(struct mm_struct *mm, - } - - node = page_to_nid(page); -- if (khugepaged_scan_abort(node)) { -+ if (hpage_collapse_scan_abort(node, cc)) { - result = SCAN_SCAN_ABORT; - break; - } -- khugepaged_node_load[node]++; -+ cc->node_load[node]++; - - if (!PageLRU(page)) { - result = SCAN_PAGE_LRU; -@@ -2028,20 +2019,21 @@ static void khugepaged_scan_file(struct mm_struct *mm, - rcu_read_unlock(); - - if (result == SCAN_SUCCEED) { -- if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { -+ if (cc->is_khugepaged && -+ present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { - result = SCAN_EXCEED_NONE_PTE; - count_vm_event(THP_SCAN_EXCEED_NONE_PTE); - } else { -- node = khugepaged_find_target_node(); -- collapse_file(mm, file, start, hpage, node); -+ result = collapse_file(mm, file, start, cc); - } - } - - /* TODO: tracepoints */ -+ return result; - } - #else --static void khugepaged_scan_file(struct mm_struct *mm, -- struct file *file, pgoff_t start, struct page **hpage) -+static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, -+ pgoff_t start, struct collapse_control *cc) - { - BUILD_BUG(); - } -@@ -2051,11 +2043,12 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) - } - #endif - --static unsigned int khugepaged_scan_mm_slot(unsigned int pages, -- struct page **hpage) -+static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, -+ struct collapse_control *cc) +@@ -2048,6 +2048,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, __releases(&khugepaged_mm_lock) __acquires(&khugepaged_mm_lock) { @@ -52006,19 +49191,11 @@ index 01f71786d530..df890338daed 100644 struct mm_slot *mm_slot; struct mm_struct *mm; struct vm_area_struct *vma; -@@ -2063,6 +2056,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - - VM_BUG_ON(!pages); - lockdep_assert_held(&khugepaged_mm_lock); -+ *result = SCAN_FAIL; - - if (khugepaged_scan.mm_slot) - mm_slot = khugepaged_scan.mm_slot; -@@ -2083,19 +2077,21 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, +@@ -2076,11 +2077,13 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, vma = NULL; if (unlikely(!mmap_read_trylock(mm))) goto breakouterloop_mmap_lock; -- if (likely(!khugepaged_test_exit(mm))) +- if (likely(!hpage_collapse_test_exit(mm))) - vma = find_vma(mm, khugepaged_scan.address); progress++; @@ -52031,312 +49208,36 @@ index 01f71786d530..df890338daed 100644 unsigned long hstart, hend; cond_resched(); -- if (unlikely(khugepaged_test_exit(mm))) { -+ if (unlikely(hpage_collapse_test_exit(mm))) { - progress++; - break; - } -- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) { -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { - skip: - progress++; - continue; -@@ -2109,9 +2105,10 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); - - while (khugepaged_scan.address < hend) { -- int ret; -+ bool mmap_locked = true; -+ - cond_resched(); -- if (unlikely(khugepaged_test_exit(mm))) -+ if (unlikely(hpage_collapse_test_exit(mm))) - goto breakouterloop; - - VM_BUG_ON(khugepaged_scan.address < hstart || -@@ -2123,19 +2120,29 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - khugepaged_scan.address); - - mmap_read_unlock(mm); -- ret = 1; -- khugepaged_scan_file(mm, file, pgoff, hpage); -+ *result = khugepaged_scan_file(mm, file, pgoff, -+ cc); -+ mmap_locked = false; - fput(file); - } else { -- ret = khugepaged_scan_pmd(mm, vma, -- khugepaged_scan.address, -- hpage); -+ *result = hpage_collapse_scan_pmd(mm, vma, -+ khugepaged_scan.address, -+ &mmap_locked, -+ cc); - } -+ if (*result == SCAN_SUCCEED) -+ ++khugepaged_pages_collapsed; - /* move to next address */ - khugepaged_scan.address += HPAGE_PMD_SIZE; - progress += HPAGE_PMD_NR; -- if (ret) -- /* we released mmap_lock so break loop */ -+ if (!mmap_locked) -+ /* -+ * We released mmap_lock so break loop. Note -+ * that we drop mmap_lock before all hugepage -+ * allocations, so if allocation fails, we are -+ * guaranteed to break here and report the -+ * correct result back to caller. -+ */ - goto breakouterloop_mmap_lock; - if (progress >= pages) - goto breakouterloop; -@@ -2151,7 +2158,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - * Release the current mm_slot if this mm is about to die, or - * if we scanned all vmas of this mm. - */ -- if (khugepaged_test_exit(mm) || !vma) { -+ if (hpage_collapse_test_exit(mm) || !vma) { - /* - * Make sure that if mm_users is reaching zero while - * khugepaged runs here, khugepaged_exit will find -@@ -2185,19 +2192,16 @@ static int khugepaged_wait_event(void) - kthread_should_stop(); - } - --static void khugepaged_do_scan(void) -+static void khugepaged_do_scan(struct collapse_control *cc) - { -- struct page *hpage = NULL; - unsigned int progress = 0, pass_through_head = 0; - unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); - bool wait = true; -+ int result = SCAN_SUCCEED; - - lru_add_drain_all(); - -- while (progress < pages) { -- if (!khugepaged_prealloc_page(&hpage, &wait)) -- break; -- -+ while (true) { - cond_resched(); - - if (unlikely(kthread_should_stop() || try_to_freeze())) -@@ -2209,14 +2213,25 @@ static void khugepaged_do_scan(void) - if (khugepaged_has_work() && - pass_through_head < 2) - progress += khugepaged_scan_mm_slot(pages - progress, -- &hpage); -+ &result, cc); - else - progress = pages; - spin_unlock(&khugepaged_mm_lock); -- } +diff --git a/mm/ksm.c b/mm/ksm.c +index dde27d7e92d4..b7ee3c87a059 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -981,11 +981,13 @@ static int unmerge_and_remove_all_rmap_items(void) + struct mm_slot, mm_list); + spin_unlock(&ksm_mmlist_lock); -- if (!IS_ERR_OR_NULL(hpage)) -- put_page(hpage); -+ if (progress >= pages) -+ break; +- for (mm_slot = ksm_scan.mm_slot; +- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { ++ for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; ++ mm_slot = ksm_scan.mm_slot) { ++ VMA_ITERATOR(vmi, mm_slot->mm, 0); + -+ if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { -+ /* -+ * If fail to allocate the first time, try to sleep for -+ * a while. When hit again, cancel the scan. -+ */ -+ if (!wait) -+ break; -+ wait = false; -+ khugepaged_alloc_sleep(); -+ } -+ } - } - - static bool khugepaged_should_wakeup(void) -@@ -2253,7 +2268,7 @@ static int khugepaged(void *none) - set_user_nice(current, MAX_NICE); + mm = mm_slot->mm; + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (ksm_test_exit(mm)) + break; + if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) +@@ -2242,6 +2244,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + struct mm_slot *slot; + struct vm_area_struct *vma; + struct rmap_item *rmap_item; ++ struct vma_iterator vmi; + int nid; - while (!kthread_should_stop()) { -- khugepaged_do_scan(); -+ khugepaged_do_scan(&khugepaged_collapse_control); - khugepaged_wait_work(); - } - -@@ -2352,3 +2367,120 @@ void khugepaged_min_free_kbytes_update(void) - set_recommended_min_free_kbytes(); - mutex_unlock(&khugepaged_mutex); - } -+ -+static int madvise_collapse_errno(enum scan_result r) -+{ -+ /* -+ * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide -+ * actionable feedback to caller, so they may take an appropriate -+ * fallback measure depending on the nature of the failure. -+ */ -+ switch (r) { -+ case SCAN_ALLOC_HUGE_PAGE_FAIL: -+ return -ENOMEM; -+ case SCAN_CGROUP_CHARGE_FAIL: -+ return -EBUSY; -+ /* Resource temporary unavailable - trying again might succeed */ -+ case SCAN_PAGE_LOCK: -+ case SCAN_PAGE_LRU: -+ return -EAGAIN; -+ /* -+ * Other: Trying again likely not to succeed / error intrinsic to -+ * specified memory range. khugepaged likely won't be able to collapse -+ * either. -+ */ -+ default: -+ return -EINVAL; -+ } -+} -+ -+int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, -+ unsigned long start, unsigned long end) -+{ -+ struct collapse_control *cc; -+ struct mm_struct *mm = vma->vm_mm; -+ unsigned long hstart, hend, addr; -+ int thps = 0, last_fail = SCAN_FAIL; -+ bool mmap_locked = true; -+ -+ BUG_ON(vma->vm_start > start); -+ BUG_ON(vma->vm_end < end); -+ -+ *prev = vma; -+ -+ /* TODO: Support file/shmem */ -+ if (!vma->anon_vma || !vma_is_anonymous(vma)) -+ return -EINVAL; -+ -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) -+ return -EINVAL; -+ -+ cc = kmalloc(sizeof(*cc), GFP_KERNEL); -+ if (!cc) -+ return -ENOMEM; -+ cc->is_khugepaged = false; -+ cc->last_target_node = NUMA_NO_NODE; -+ -+ mmgrab(mm); -+ lru_add_drain_all(); -+ -+ hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; -+ hend = end & HPAGE_PMD_MASK; -+ -+ for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { -+ int result = SCAN_FAIL; -+ -+ if (!mmap_locked) { -+ cond_resched(); -+ mmap_read_lock(mm); -+ mmap_locked = true; -+ result = hugepage_vma_revalidate(mm, addr, &vma, cc); -+ if (result != SCAN_SUCCEED) { -+ last_fail = result; -+ goto out_nolock; -+ } -+ } -+ mmap_assert_locked(mm); -+ memset(cc->node_load, 0, sizeof(cc->node_load)); -+ result = hpage_collapse_scan_pmd(mm, vma, addr, &mmap_locked, -+ cc); -+ if (!mmap_locked) -+ *prev = NULL; /* Tell caller we dropped mmap_lock */ -+ -+ switch (result) { -+ case SCAN_SUCCEED: -+ case SCAN_PMD_MAPPED: -+ ++thps; -+ break; -+ /* Whitelisted set of results where continuing OK */ -+ case SCAN_PMD_NULL: -+ case SCAN_PTE_NON_PRESENT: -+ case SCAN_PTE_UFFD_WP: -+ case SCAN_PAGE_RO: -+ case SCAN_LACK_REFERENCED_PAGE: -+ case SCAN_PAGE_NULL: -+ case SCAN_PAGE_COUNT: -+ case SCAN_PAGE_LOCK: -+ case SCAN_PAGE_COMPOUND: -+ case SCAN_PAGE_LRU: -+ last_fail = result; -+ break; -+ default: -+ last_fail = result; -+ /* Other error, exit */ -+ goto out_maybelock; -+ } -+ } -+ -+out_maybelock: -+ /* Caller expects us to hold mmap_lock on return */ -+ if (!mmap_locked) -+ mmap_read_lock(mm); -+out_nolock: -+ mmap_assert_locked(mm); -+ mmdrop(mm); -+ kfree(cc); -+ -+ return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 -+ : madvise_collapse_errno(last_fail); -+} -diff --git a/mm/ksm.c b/mm/ksm.c -index 4e64adb9adee..b7ee3c87a059 100644 ---- a/mm/ksm.c -+++ b/mm/ksm.c -@@ -981,11 +981,13 @@ static int unmerge_and_remove_all_rmap_items(void) - struct mm_slot, mm_list); - spin_unlock(&ksm_mmlist_lock); - -- for (mm_slot = ksm_scan.mm_slot; -- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { -+ for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; -+ mm_slot = ksm_scan.mm_slot) { -+ VMA_ITERATOR(vmi, mm_slot->mm, 0); -+ - mm = mm_slot->mm; - mmap_read_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (ksm_test_exit(mm)) - break; - if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) -@@ -1134,6 +1136,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, - { - struct mm_struct *mm = vma->vm_mm; - pmd_t *pmd; -+ pmd_t pmde; - pte_t *ptep; - pte_t newpte; - spinlock_t *ptl; -@@ -1148,6 +1151,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, - pmd = mm_find_pmd(mm, addr); - if (!pmd) - goto out; -+ /* -+ * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() -+ * without holding anon_vma lock for write. So when looking for a -+ * genuine pmde (in which to find pte), test present and !THP together. -+ */ -+ pmde = *pmd; -+ barrier(); -+ if (!pmd_present(pmde) || pmd_trans_huge(pmde)) -+ goto out; - - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, - addr + PAGE_SIZE); -@@ -2232,6 +2244,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) - struct mm_slot *slot; - struct vm_area_struct *vma; - struct rmap_item *rmap_item; -+ struct vma_iterator vmi; - int nid; - - if (list_empty(&ksm_mm_head.mm_list)) -@@ -2290,13 +2303,13 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + if (list_empty(&ksm_mm_head.mm_list)) +@@ -2300,13 +2303,13 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) } mm = slot->mm; @@ -52354,7 +49255,7 @@ index 4e64adb9adee..b7ee3c87a059 100644 if (!(vma->vm_flags & VM_MERGEABLE)) continue; if (ksm_scan.address < vma->vm_start) -@@ -2334,6 +2347,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) +@@ -2344,6 +2347,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) } if (ksm_test_exit(mm)) { @@ -52363,51 +49264,10 @@ index 4e64adb9adee..b7ee3c87a059 100644 ksm_scan.rmap_list = &slot->rmap_list; } diff --git a/mm/madvise.c b/mm/madvise.c -index 5f0f0948a50e..682e1d161aef 100644 +index af97100a0727..682e1d161aef 100644 --- a/mm/madvise.c +++ b/mm/madvise.c -@@ -59,6 +59,7 @@ static int madvise_need_mmap_write(int behavior) - case MADV_FREE: - case MADV_POPULATE_READ: - case MADV_POPULATE_WRITE: -+ case MADV_COLLAPSE: - return 0; - default: - /* be safe, default to 1. list exceptions explicitly */ -@@ -1057,6 +1058,8 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, - if (error) - goto out; - break; -+ case MADV_COLLAPSE: -+ return madvise_collapse(vma, prev, start, end); - } - - anon_name = anon_vma_name(vma); -@@ -1150,6 +1153,7 @@ madvise_behavior_valid(int behavior) - #ifdef CONFIG_TRANSPARENT_HUGEPAGE - case MADV_HUGEPAGE: - case MADV_NOHUGEPAGE: -+ case MADV_COLLAPSE: - #endif - case MADV_DONTDUMP: - case MADV_DODUMP: -@@ -1166,13 +1170,13 @@ madvise_behavior_valid(int behavior) - } - } - --static bool --process_madvise_behavior_valid(int behavior) -+static bool process_madvise_behavior_valid(int behavior) - { - switch (behavior) { - case MADV_COLD: - case MADV_PAGEOUT: - case MADV_WILLNEED: -+ case MADV_COLLAPSE: - return true; - default: - return false; -@@ -1238,7 +1242,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, +@@ -1242,7 +1242,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, if (start >= end) break; if (prev) @@ -52416,43 +49276,11 @@ index 5f0f0948a50e..682e1d161aef 100644 else /* madvise_remove dropped mmap_lock */ vma = find_vma(mm, start); } -@@ -1339,6 +1343,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, - * MADV_NOHUGEPAGE - mark the given range as not worth being backed by - * transparent huge pages so the existing pages will not be - * coalesced into THP and new pages will not be allocated as THP. -+ * MADV_COLLAPSE - synchronously coalesce pages into new THP. - * MADV_DONTDUMP - the application wants to prevent pages in the given range - * from being included in its core dump. - * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index 7eadbafc006b..454a283b9e85 100644 +index 48588e89867a..454a283b9e85 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c -@@ -2789,6 +2789,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - */ - folio->memcg_data = (unsigned long)memcg; - } -@@ -5170,6 +5171,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) - - static void mem_cgroup_free(struct mem_cgroup *memcg) - { -+ lru_gen_exit_memcg(memcg); - memcg_wb_domain_exit(memcg); - __mem_cgroup_free(memcg); - } -@@ -5228,6 +5230,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) - memcg->deferred_split_queue.split_queue_len = 0; - #endif - idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); -+ lru_gen_init_memcg(memcg); - return memcg; - fail: - mem_cgroup_id_remove(memcg); -@@ -5871,7 +5874,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) +@@ -5874,7 +5874,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) unsigned long precharge; mmap_read_lock(mm); @@ -52461,7 +49289,7 @@ index 7eadbafc006b..454a283b9e85 100644 mmap_read_unlock(mm); precharge = mc.precharge; -@@ -6169,9 +6172,7 @@ static void mem_cgroup_move_charge(void) +@@ -6172,9 +6172,7 @@ static void mem_cgroup_move_charge(void) * When we have consumed all precharges and failed in doing * additional charge, the page walk just aborts. */ @@ -52472,69 +49300,11 @@ index 7eadbafc006b..454a283b9e85 100644 mmap_read_unlock(mc.mm); atomic_dec(&mc.from->moving_account); } -@@ -6196,6 +6197,30 @@ static void mem_cgroup_move_task(void) - } - #endif - -+#ifdef CONFIG_LRU_GEN -+static void mem_cgroup_attach(struct cgroup_taskset *tset) -+{ -+ struct task_struct *task; -+ struct cgroup_subsys_state *css; -+ -+ /* find the first leader if there is any */ -+ cgroup_taskset_for_each_leader(task, css, tset) -+ break; -+ -+ if (!task) -+ return; -+ -+ task_lock(task); -+ if (task->mm && READ_ONCE(task->mm->owner) == task) -+ lru_gen_migrate_mm(task->mm); -+ task_unlock(task); -+} -+#else -+static void mem_cgroup_attach(struct cgroup_taskset *tset) -+{ -+} -+#endif /* CONFIG_LRU_GEN */ -+ - static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) - { - if (value == PAGE_COUNTER_MAX) -@@ -6601,6 +6626,7 @@ struct cgroup_subsys memory_cgrp_subsys = { - .css_reset = mem_cgroup_css_reset, - .css_rstat_flush = mem_cgroup_css_rstat_flush, - .can_attach = mem_cgroup_can_attach, -+ .attach = mem_cgroup_attach, - .cancel_attach = mem_cgroup_cancel_attach, - .post_attach = mem_cgroup_move_task, - .dfl_cftypes = memory_files, diff --git a/mm/memory.c b/mm/memory.c -index 4ba73f5aa8bb..4fd25d3aba64 100644 +index bc4dc2e45dcc..acc2e88f4984 100644 --- a/mm/memory.c +++ b/mm/memory.c -@@ -125,18 +125,6 @@ int randomize_va_space __read_mostly = - 2; - #endif - --#ifndef arch_faults_on_old_pte --static inline bool arch_faults_on_old_pte(void) --{ -- /* -- * Those arches which don't have hw access flag feature need to -- * implement their own helper. By default, "true" means pagefault -- * will be hit on old pte. -- */ -- return true; --} --#endif -- - #ifndef arch_wants_old_prefaulted_pte - static inline bool arch_wants_old_prefaulted_pte(void) - { -@@ -402,12 +390,21 @@ void free_pgd_range(struct mmu_gather *tlb, +@@ -391,12 +391,21 @@ void free_pgd_range(struct mmu_gather *tlb, } while (pgd++, addr = next, addr != end); } @@ -52560,7 +49330,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 /* * Hide vma from rmap and truncate_pagecache before freeing -@@ -426,7 +423,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, +@@ -415,7 +424,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, while (next && next->vm_start <= vma->vm_end + PMD_SIZE && !is_vm_hugetlb_page(next)) { vma = next; @@ -52569,7 +49339,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 unlink_anon_vmas(vma); unlink_file_vma(vma); } -@@ -434,7 +431,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, +@@ -423,7 +432,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, floor, next ? next->vm_start : ceiling); } vma = next; @@ -52578,7 +49348,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 } void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) -@@ -1698,6 +1695,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, +@@ -1687,6 +1696,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, /** * unmap_vmas - unmap a range of memory covered by a list of vma's * @tlb: address of the caller's struct mmu_gather @@ -52586,7 +49356,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 * @vma: the starting vma * @start_addr: virtual address at which to start unmapping * @end_addr: virtual address at which to end unmapping -@@ -1713,7 +1711,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, +@@ -1702,7 +1712,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, * ensure that any thus-far unmapped pages are flushed before unmap_vmas() * drops the lock and schedules. */ @@ -52595,7 +49365,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 struct vm_area_struct *vma, unsigned long start_addr, unsigned long end_addr) { -@@ -1723,12 +1721,14 @@ void unmap_vmas(struct mmu_gather *tlb, +@@ -1712,12 +1722,14 @@ void unmap_vmas(struct mmu_gather *tlb, /* Careful - we need to zap private pages too! */ .even_cows = true, }; @@ -52611,7 +49381,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 mmu_notifier_invalidate_range_end(&range); } -@@ -1743,8 +1743,11 @@ void unmap_vmas(struct mmu_gather *tlb, +@@ -1732,8 +1744,11 @@ void unmap_vmas(struct mmu_gather *tlb, void zap_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long size) { @@ -52623,7 +49393,7 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, -@@ -1752,8 +1755,9 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, +@@ -1741,8 +1756,9 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); @@ -52634,77 +49404,6 @@ index 4ba73f5aa8bb..4fd25d3aba64 100644 mmu_notifier_invalidate_range_end(&range); tlb_finish_mmu(&tlb); } -@@ -2870,7 +2874,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, - * On architectures with software "accessed" bits, we would - * take a double page fault, so mark it accessed here. - */ -- if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { -+ if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { - pte_t entry; - - vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); -@@ -4985,7 +4989,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, - return VM_FAULT_OOM; - retry_pud: - if (pud_none(*vmf.pud) && -- hugepage_vma_check(vma, vm_flags, false, true)) { -+ hugepage_vma_check(vma, vm_flags, false, true, true)) { - ret = create_huge_pud(&vmf); - if (!(ret & VM_FAULT_FALLBACK)) - return ret; -@@ -5019,7 +5023,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, - goto retry_pud; - - if (pmd_none(*vmf.pmd) && -- hugepage_vma_check(vma, vm_flags, false, true)) { -+ hugepage_vma_check(vma, vm_flags, false, true, true)) { - ret = create_huge_pmd(&vmf); - if (!(ret & VM_FAULT_FALLBACK)) - return ret; -@@ -5114,6 +5118,27 @@ static inline void mm_account_fault(struct pt_regs *regs, - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); - } - -+#ifdef CONFIG_LRU_GEN -+static void lru_gen_enter_fault(struct vm_area_struct *vma) -+{ -+ /* the LRU algorithm doesn't apply to sequential or random reads */ -+ current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); -+} -+ -+static void lru_gen_exit_fault(void) -+{ -+ current->in_lru_fault = false; -+} -+#else -+static void lru_gen_enter_fault(struct vm_area_struct *vma) -+{ -+} -+ -+static void lru_gen_exit_fault(void) -+{ -+} -+#endif /* CONFIG_LRU_GEN */ -+ - /* - * By the time we get here, we already hold the mm semaphore - * -@@ -5145,11 +5170,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, - if (flags & FAULT_FLAG_USER) - mem_cgroup_enter_user_fault(); - -+ lru_gen_enter_fault(vma); -+ - if (unlikely(is_vm_hugetlb_page(vma))) - ret = hugetlb_fault(vma->vm_mm, vma, address, flags); - else - ret = __handle_mm_fault(vma, address, flags); - -+ lru_gen_exit_fault(); -+ - if (flags & FAULT_FLAG_USER) { - mem_cgroup_exit_user_fault(); - /* diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b73d3248d976..6c27acb6cd63 100644 --- a/mm/mempolicy.c @@ -52970,29 +49669,6 @@ index b14e929084cc..7032f6dd0ce1 100644 cond_resched(); } out: -diff --git a/mm/mm_init.c b/mm/mm_init.c -index 9ddaf0e1b0ab..0d7b2bd2454a 100644 ---- a/mm/mm_init.c -+++ b/mm/mm_init.c -@@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) - - shift = 8 * sizeof(unsigned long); - width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH -- - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; -+ - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; - mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", -- "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", -+ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", - SECTIONS_WIDTH, - NODES_WIDTH, - ZONES_WIDTH, - LAST_CPUPID_WIDTH, - KASAN_TAG_WIDTH, -+ LRU_GEN_WIDTH, -+ LRU_REFS_WIDTH, - NR_PAGEFLAGS); - mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", - "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", diff --git a/mm/mmap.c b/mm/mmap.c index 9d780f415be3..6fc054c55cd8 100644 --- a/mm/mmap.c @@ -55911,24 +52587,11 @@ index 9d780f415be3..6fc054c55cd8 100644 if (vma->anon_vma) list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) vm_unlock_anon_vma(avc->anon_vma); -diff --git a/mm/mmzone.c b/mm/mmzone.c -index 0ae7571e35ab..68e1511be12d 100644 ---- a/mm/mmzone.c -+++ b/mm/mmzone.c -@@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) - * Poison its list head, so that any operations on it would crash. - */ - list_del(&lruvec->lists[LRU_UNEVICTABLE]); -+ -+ lru_gen_init_lruvec(lruvec); - } - - #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) diff --git a/mm/mprotect.c b/mm/mprotect.c -index bc6bddd156ca..bf030bbe7e97 100644 +index eb8982bde7ee..2bb8613cd295 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c -@@ -669,6 +669,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, +@@ -670,6 +670,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, const bool rier = (current->personality & READ_IMPLIES_EXEC) && (prot & PROT_READ); struct mmu_gather tlb; @@ -55936,7 +52599,7 @@ index bc6bddd156ca..bf030bbe7e97 100644 start = untagged_addr(start); -@@ -700,7 +701,8 @@ static int do_mprotect_pkey(unsigned long start, size_t len, +@@ -701,7 +702,8 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) goto out; @@ -55946,7 +52609,7 @@ index bc6bddd156ca..bf030bbe7e97 100644 error = -ENOMEM; if (!vma) goto out; -@@ -726,7 +728,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, +@@ -727,7 +729,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (start > vma->vm_start) prev = vma; else @@ -55955,7 +52618,7 @@ index bc6bddd156ca..bf030bbe7e97 100644 tlb_gather_mmu(&tlb, current->mm); for (nstart = start ; ; ) { -@@ -789,7 +791,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, +@@ -790,7 +792,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (nstart >= end) break; @@ -56533,165 +53196,6 @@ index fa7a3d21a751..c09bb9c01f7e 100644 err = walk_page_test(start, next, &walk); if (err > 0) { -diff --git a/mm/rmap.c b/mm/rmap.c -index 93d5a6f793d2..3080edeaa3ad 100644 ---- a/mm/rmap.c -+++ b/mm/rmap.c -@@ -770,13 +770,17 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) - return vma_address(page, vma); - } - -+/* -+ * Returns the actual pmd_t* where we expect 'address' to be mapped from, or -+ * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* -+ * represents. -+ */ - pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) - { - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd = NULL; -- pmd_t pmde; - - pgd = pgd_offset(mm, address); - if (!pgd_present(*pgd)) -@@ -791,15 +795,6 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) - goto out; - - pmd = pmd_offset(pud, address); -- /* -- * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() -- * without holding anon_vma lock for write. So when looking for a -- * genuine pmde (in which to find pte), test present and !THP together. -- */ -- pmde = *pmd; -- barrier(); -- if (!pmd_present(pmde) || pmd_trans_huge(pmde)) -- pmd = NULL; - out: - return pmd; - } -@@ -833,6 +828,12 @@ static bool folio_referenced_one(struct folio *folio, - } - - if (pvmw.pte) { -+ if (lru_gen_enabled() && pte_young(*pvmw.pte) && -+ !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { -+ lru_gen_look_around(&pvmw); -+ referenced++; -+ } -+ - if (ptep_clear_flush_young_notify(vma, address, - pvmw.pte)) { - /* -diff --git a/mm/swap.c b/mm/swap.c -index 186b4e5dcecf..027943b341a0 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu) - folio_batch_move_lru(fbatch, folio_activate_fn); - } - --static void folio_activate(struct folio *folio) -+void folio_activate(struct folio *folio) - { - if (folio_test_lru(folio) && !folio_test_active(folio) && - !folio_test_unevictable(folio)) { -@@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu) - { - } - --static void folio_activate(struct folio *folio) -+void folio_activate(struct folio *folio) - { - struct lruvec *lruvec; - -@@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio) - local_unlock(&cpu_fbatches.lock); - } - -+#ifdef CONFIG_LRU_GEN -+static void folio_inc_refs(struct folio *folio) -+{ -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); -+ -+ if (folio_test_unevictable(folio)) -+ return; -+ -+ if (!folio_test_referenced(folio)) { -+ folio_set_referenced(folio); -+ return; -+ } -+ -+ if (!folio_test_workingset(folio)) { -+ folio_set_workingset(folio); -+ return; -+ } -+ -+ /* see the comment on MAX_NR_TIERS */ -+ do { -+ new_flags = old_flags & LRU_REFS_MASK; -+ if (new_flags == LRU_REFS_MASK) -+ break; -+ -+ new_flags += BIT(LRU_REFS_PGOFF); -+ new_flags |= old_flags & ~LRU_REFS_MASK; -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); -+} -+#else -+static void folio_inc_refs(struct folio *folio) -+{ -+} -+#endif /* CONFIG_LRU_GEN */ -+ - /* - * Mark a page as having seen activity. - * -@@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio) - */ - void folio_mark_accessed(struct folio *folio) - { -+ if (lru_gen_enabled()) { -+ folio_inc_refs(folio); -+ return; -+ } -+ - if (!folio_test_referenced(folio)) { - folio_set_referenced(folio); - } else if (folio_test_unevictable(folio)) { -@@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio) - folio_test_unevictable(folio), folio); - VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); - -+ /* see the comment in lru_gen_add_folio() */ -+ if (lru_gen_enabled() && !folio_test_unevictable(folio) && -+ lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) -+ folio_set_active(folio); -+ - folio_get(folio); - local_lock(&cpu_fbatches.lock); - fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); -@@ -575,7 +619,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) - - static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) - { -- if (folio_test_active(folio) && !folio_test_unevictable(folio)) { -+ if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { - long nr_pages = folio_nr_pages(folio); - - lruvec_del_folio(lruvec, folio); -@@ -688,8 +732,8 @@ void deactivate_page(struct page *page) - { - struct folio *folio = page_folio(page); - -- if (folio_test_lru(folio) && folio_test_active(folio) && -- !folio_test_unevictable(folio)) { -+ if (folio_test_lru(folio) && !folio_test_unevictable(folio) && -+ (folio_test_active(folio) || lru_gen_enabled())) { - struct folio_batch *fbatch; - - folio_get(folio); diff --git a/mm/swapfile.c b/mm/swapfile.c index 1fdccd2f1422..5c8681a3f1d9 100644 --- a/mm/swapfile.c @@ -56881,3726 +53385,229 @@ index 01a6e6688ec1..000000000000 -} -#endif diff --git a/mm/vmscan.c b/mm/vmscan.c -index 0fc65ace3a4e..f38fd15935df 100644 +index feb8416d8edd..f85a9c915d75 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c -@@ -49,6 +49,10 @@ - #include - #include - #include -+#include -+#include -+#include -+#include - - #include - #include -@@ -129,6 +133,12 @@ struct scan_control { - /* Always discard instead of demoting to lower tier memory */ - unsigned int no_demotion:1; - -+#ifdef CONFIG_LRU_GEN -+ /* help kswapd make better choices among multiple memcgs */ -+ unsigned int memcgs_need_aging:1; -+ unsigned long last_reclaimed; -+#endif -+ - /* Allocation order */ - s8 order; - -@@ -1338,9 +1348,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, - - if (folio_test_swapcache(folio)) { - swp_entry_t swap = folio_swap_entry(folio); -- mem_cgroup_swapout(folio, swap); -+ -+ /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ - if (reclaimed && !mapping_exiting(mapping)) - shadow = workingset_eviction(folio, target_memcg); -+ mem_cgroup_swapout(folio, swap); - __delete_from_swap_cache(folio, swap, shadow); - xa_unlock_irq(&mapping->i_pages); - put_swap_page(&folio->page, swap); -@@ -1637,6 +1649,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, - if (!sc->may_unmap && folio_mapped(folio)) - goto keep_locked; - -+ /* folio_update_gen() tried to promote this page? */ -+ if (lru_gen_enabled() && !ignore_references && -+ folio_mapped(folio) && folio_test_referenced(folio)) -+ goto keep_locked; -+ - /* - * The number of dirty pages determines if a node is marked - * reclaim_congested. kswapd will stall and start writing -@@ -2732,6 +2749,112 @@ enum scan_balance { - SCAN_FILE, - }; - -+static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) -+{ -+ unsigned long file; -+ struct lruvec *target_lruvec; -+ -+ if (lru_gen_enabled()) -+ return; -+ -+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); -+ -+ /* -+ * Flush the memory cgroup stats, so that we read accurate per-memcg -+ * lruvec stats for heuristics. -+ */ -+ mem_cgroup_flush_stats(); -+ -+ /* -+ * Determine the scan balance between anon and file LRUs. -+ */ -+ spin_lock_irq(&target_lruvec->lru_lock); -+ sc->anon_cost = target_lruvec->anon_cost; -+ sc->file_cost = target_lruvec->file_cost; -+ spin_unlock_irq(&target_lruvec->lru_lock); -+ -+ /* -+ * Target desirable inactive:active list ratios for the anon -+ * and file LRU lists. -+ */ -+ if (!sc->force_deactivate) { -+ unsigned long refaults; -+ -+ /* -+ * When refaults are being observed, it means a new -+ * workingset is being established. Deactivate to get -+ * rid of any stale active pages quickly. -+ */ -+ refaults = lruvec_page_state(target_lruvec, -+ WORKINGSET_ACTIVATE_ANON); -+ if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || -+ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) -+ sc->may_deactivate |= DEACTIVATE_ANON; -+ else -+ sc->may_deactivate &= ~DEACTIVATE_ANON; -+ -+ refaults = lruvec_page_state(target_lruvec, -+ WORKINGSET_ACTIVATE_FILE); -+ if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || -+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) -+ sc->may_deactivate |= DEACTIVATE_FILE; -+ else -+ sc->may_deactivate &= ~DEACTIVATE_FILE; -+ } else -+ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; -+ -+ /* -+ * If we have plenty of inactive file pages that aren't -+ * thrashing, try to reclaim those first before touching -+ * anonymous pages. -+ */ -+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); -+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) -+ sc->cache_trim_mode = 1; -+ else -+ sc->cache_trim_mode = 0; -+ -+ /* -+ * Prevent the reclaimer from falling into the cache trap: as -+ * cache pages start out inactive, every cache fault will tip -+ * the scan balance towards the file LRU. And as the file LRU -+ * shrinks, so does the window for rotation from references. -+ * This means we have a runaway feedback loop where a tiny -+ * thrashing file LRU becomes infinitely more attractive than -+ * anon pages. Try to detect this based on file LRU size. -+ */ -+ if (!cgroup_reclaim(sc)) { -+ unsigned long total_high_wmark = 0; -+ unsigned long free, anon; -+ int z; -+ -+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); -+ file = node_page_state(pgdat, NR_ACTIVE_FILE) + -+ node_page_state(pgdat, NR_INACTIVE_FILE); -+ -+ for (z = 0; z < MAX_NR_ZONES; z++) { -+ struct zone *zone = &pgdat->node_zones[z]; -+ -+ if (!managed_zone(zone)) -+ continue; -+ -+ total_high_wmark += high_wmark_pages(zone); -+ } -+ -+ /* -+ * Consider anon: if that's low too, this isn't a -+ * runaway file reclaim problem, but rather just -+ * extreme pressure. Reclaim as per usual then. -+ */ -+ anon = node_page_state(pgdat, NR_INACTIVE_ANON); -+ -+ sc->file_is_tiny = -+ file + free <= total_high_wmark && -+ !(sc->may_deactivate & DEACTIVATE_ANON) && -+ anon >> sc->priority; -+ } -+} -+ - /* - * Determine how aggressively the anon and file LRU lists should be - * scanned. -@@ -2951,159 +3074,2906 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, - return can_demote(pgdat->node_id, sc); - } - --static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) --{ -- unsigned long nr[NR_LRU_LISTS]; -- unsigned long targets[NR_LRU_LISTS]; -- unsigned long nr_to_scan; -- enum lru_list lru; -- unsigned long nr_reclaimed = 0; -- unsigned long nr_to_reclaim = sc->nr_to_reclaim; -- struct blk_plug plug; -- bool scan_adjusted; -+#ifdef CONFIG_LRU_GEN - -- get_scan_count(lruvec, sc, nr); -+#ifdef CONFIG_LRU_GEN_ENABLED -+DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); -+#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) -+#else -+DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); -+#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) -+#endif - -- /* Record the original scan target for proportional adjustments later */ -- memcpy(targets, nr, sizeof(nr)); -+/****************************************************************************** -+ * shorthand helpers -+ ******************************************************************************/ - -- /* -- * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal -- * event that can occur when there is little memory pressure e.g. -- * multiple streaming readers/writers. Hence, we do not abort scanning -- * when the requested number of pages are reclaimed when scanning at -- * DEF_PRIORITY on the assumption that the fact we are direct -- * reclaiming implies that kswapd is not keeping up and it is best to -- * do a batch of work at once. For memcg reclaim one check is made to -- * abort proportional reclaim if either the file or anon lru has already -- * dropped to zero at the first pass. -- */ -- scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && -- sc->priority == DEF_PRIORITY); -+#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) - -- blk_start_plug(&plug); -- while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || -- nr[LRU_INACTIVE_FILE]) { -- unsigned long nr_anon, nr_file, percentage; -- unsigned long nr_scanned; -+#define DEFINE_MAX_SEQ(lruvec) \ -+ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) - -- for_each_evictable_lru(lru) { -- if (nr[lru]) { -- nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); -- nr[lru] -= nr_to_scan; -+#define DEFINE_MIN_SEQ(lruvec) \ -+ unsigned long min_seq[ANON_AND_FILE] = { \ -+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ -+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ -+ } - -- nr_reclaimed += shrink_list(lru, nr_to_scan, -- lruvec, sc); -- } -- } -+#define for_each_gen_type_zone(gen, type, zone) \ -+ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ -+ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ -+ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) +@@ -3778,23 +3778,17 @@ static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk + { + unsigned long start = round_up(*vm_end, size); + unsigned long end = (start | ~mask) + 1; ++ VMA_ITERATOR(vmi, args->mm, start); -- cond_resched(); -+static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) -+{ -+ struct pglist_data *pgdat = NODE_DATA(nid); + VM_WARN_ON_ONCE(mask & size); + VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); -- if (nr_reclaimed < nr_to_reclaim || scan_adjusted) +- while (args->vma) { +- if (start >= args->vma->vm_end) { +- args->vma = args->vma->vm_next; - continue; -+#ifdef CONFIG_MEMCG -+ if (memcg) { -+ struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; - -- /* -- * For kswapd and memcg, reclaim at least the number of pages -- * requested. Ensure that the anon and file LRUs are scanned -- * proportionally what was requested by get_scan_count(). We -- * stop reclaiming one LRU and reduce the amount scanning -- * proportional to the original scan target. -- */ -- nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; -- nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; -+ /* for hotadd_new_pgdat() */ -+ if (!lruvec->pgdat) -+ lruvec->pgdat = pgdat; - -- /* -- * It's just vindictive to attack the larger once the smaller -- * has gone to zero. And given the way we stop scanning the -- * smaller below, this makes sure that we only make one nudge -- * towards proportionality once we've got nr_to_reclaim. -- */ -- if (!nr_file || !nr_anon) -- break; -+ return lruvec; -+ } -+#endif -+ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); - -- if (nr_file > nr_anon) { -- unsigned long scan_target = targets[LRU_INACTIVE_ANON] + -- targets[LRU_ACTIVE_ANON] + 1; -- lru = LRU_BASE; -- percentage = nr_anon * 100 / scan_target; -- } else { -- unsigned long scan_target = targets[LRU_INACTIVE_FILE] + -- targets[LRU_ACTIVE_FILE] + 1; -- lru = LRU_FILE; -- percentage = nr_file * 100 / scan_target; - } -+ return pgdat ? &pgdat->__lruvec : NULL; -+} +- ++ for_each_vma(vmi, args->vma) { + if (end && end <= args->vma->vm_start) + return false; -- /* Stop scanning the smaller of the LRU */ -- nr[lru] = 0; -- nr[lru + LRU_ACTIVE] = 0; -+static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(lruvec); +- if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { +- args->vma = args->vma->vm_next; ++ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) + continue; +- } -- /* -- * Recalculate the other LRU scan count based on its original -- * scan target and the percentage scanning already complete -- */ -- lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; -- nr_scanned = targets[lru] - nr[lru]; -- nr[lru] = targets[lru] * (100 - percentage) / 100; -- nr[lru] -= min(nr[lru], nr_scanned); -+ if (!can_demote(pgdat->node_id, sc) && -+ mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) -+ return 0; + *vm_start = max(start, args->vma->vm_start); + *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 0b8098e8296b..33091a67627e 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -1388,10 +1388,6 @@ const char * const vmstat_text[] = { + "nr_tlb_local_flush_one", + #endif /* CONFIG_DEBUG_TLBFLUSH */ -- lru += LRU_ACTIVE; -- nr_scanned = targets[lru] - nr[lru]; -- nr[lru] = targets[lru] * (100 - percentage) / 100; -- nr[lru] -= min(nr[lru], nr_scanned); -+ return mem_cgroup_swappiness(memcg); -+} +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- "vmacache_find_calls", +- "vmacache_find_hits", +-#endif + #ifdef CONFIG_SWAP + "swap_ra", + "swap_ra_hit", +diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h +index 0616409513eb..311759ea25e9 100644 +--- a/tools/include/linux/slab.h ++++ b/tools/include/linux/slab.h +@@ -41,4 +41,8 @@ struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, + unsigned int align, unsigned int flags, + void (*ctor)(void *)); -- scan_adjusted = true; -- } -- blk_finish_plug(&plug); -- sc->nr_reclaimed += nr_reclaimed; -+static int get_nr_gens(struct lruvec *lruvec, int type) -+{ -+ return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; -+} ++void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); ++int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, ++ void **list); ++ + #endif /* _TOOLS_SLAB_H */ +diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore +index d971516401e6..c901d96dd013 100644 +--- a/tools/testing/radix-tree/.gitignore ++++ b/tools/testing/radix-tree/.gitignore +@@ -6,3 +6,5 @@ main + multiorder + radix-tree.c + xarray ++maple ++ma_xa_benchmark +diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile +index c4ea4fbb0bfc..89d613e0505b 100644 +--- a/tools/testing/radix-tree/Makefile ++++ b/tools/testing/radix-tree/Makefile +@@ -4,9 +4,9 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ + -fsanitize=undefined + LDFLAGS += -fsanitize=address -fsanitize=undefined + LDLIBS+= -lpthread -lurcu +-TARGETS = main idr-test multiorder xarray ++TARGETS = main idr-test multiorder xarray maple + CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \ +- slab.o ++ slab.o maple.o + OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ + regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ + iteration_check_2.o benchmark.o +@@ -29,6 +29,8 @@ idr-test: idr-test.o $(CORE_OFILES) -- /* -- * Even if we did not try to evict anon pages at all, we want to -- * rebalance the anon lru active/inactive ratio. -- */ -- if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && -- inactive_is_low(lruvec, LRU_INACTIVE_ANON)) -- shrink_active_list(SWAP_CLUSTER_MAX, lruvec, -- sc, LRU_ACTIVE_ANON); -+static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) -+{ -+ /* see the comment on lru_gen_struct */ -+ return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && -+ get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && -+ get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; - } + xarray: $(CORE_OFILES) --/* Use reclaim/compaction for costly allocs or under memory pressure */ --static bool in_reclaim_compaction(struct scan_control *sc) -+/****************************************************************************** -+ * mm_struct list -+ ******************************************************************************/ ++maple: $(CORE_OFILES) + -+static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) - { -- if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && -- (sc->order > PAGE_ALLOC_COSTLY_ORDER || -- sc->priority < DEF_PRIORITY - 2)) -- return true; -+ static struct lru_gen_mm_list mm_list = { -+ .fifo = LIST_HEAD_INIT(mm_list.fifo), -+ .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), -+ }; + multiorder: multiorder.o $(CORE_OFILES) -- return false; -+#ifdef CONFIG_MEMCG -+ if (memcg) -+ return &memcg->mm_list; -+#endif -+ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); -+ -+ return &mm_list; - } + clean: +@@ -40,6 +42,7 @@ $(OFILES): Makefile *.h */*.h generated/map-shift.h \ + ../../include/linux/*.h \ + ../../include/asm/*.h \ + ../../../include/linux/xarray.h \ ++ ../../../include/linux/maple_tree.h \ + ../../../include/linux/radix-tree.h \ + ../../../include/linux/idr.h --/* -- * Reclaim/compaction is used for high-order allocation requests. It reclaims -- * order-0 pages before compacting the zone. should_continue_reclaim() returns -- * true if more pages should be reclaimed such that when the page allocator -- * calls try_to_compact_pages() that it will have enough free pages to succeed. -- * It will give up earlier than that if there is difficulty reclaiming pages. -- */ --static inline bool should_continue_reclaim(struct pglist_data *pgdat, -- unsigned long nr_reclaimed, -- struct scan_control *sc) -+void lru_gen_add_mm(struct mm_struct *mm) - { -- unsigned long pages_for_compaction; -- unsigned long inactive_lru_pages; -- int z; -+ int nid; -+ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); - -- /* If not in reclaim/compaction mode, stop */ -- if (!in_reclaim_compaction(sc)) -- return false; -+ VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); -+#ifdef CONFIG_MEMCG -+ VM_WARN_ON_ONCE(mm->lru_gen.memcg); -+ mm->lru_gen.memcg = memcg; -+#endif -+ spin_lock(&mm_list->lock); +@@ -51,6 +54,8 @@ idr.c: ../../../lib/idr.c -- /* -- * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX -- * number of pages that were scanned. This will return to the caller -- * with the risk reclaim/compaction and the resulting allocation attempt -- * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL -- * allocations through requiring that the full LRU list has been scanned -- * first, by assuming that zero delta of sc->nr_scanned means full LRU -- * scan, but that approximation was wrong, and there were corner cases -+ for_each_node_state(nid, N_MEMORY) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ if (!lruvec) -+ continue; -+ -+ /* the first addition since the last iteration */ -+ if (lruvec->mm_state.tail == &mm_list->fifo) -+ lruvec->mm_state.tail = &mm->lru_gen.list; -+ } -+ -+ list_add_tail(&mm->lru_gen.list, &mm_list->fifo); + xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c + ++maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c + -+ spin_unlock(&mm_list->lock); + generated/map-shift.h: + @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ + echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ +diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h +index 2218b3cc184e..e7da80350236 100644 +--- a/tools/testing/radix-tree/generated/autoconf.h ++++ b/tools/testing/radix-tree/generated/autoconf.h +@@ -1 +1,2 @@ + #define CONFIG_XARRAY_MULTI 1 ++#define CONFIG_64BIT 1 +diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c +index d5c1bcba86fe..2048d12c31df 100644 +--- a/tools/testing/radix-tree/linux.c ++++ b/tools/testing/radix-tree/linux.c +@@ -23,15 +23,47 @@ struct kmem_cache { + int nr_objs; + void *objs; + void (*ctor)(void *); ++ unsigned int non_kernel; ++ unsigned long nr_allocated; ++ unsigned long nr_tallocated; + }; + ++void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) ++{ ++ cachep->non_kernel = val; +} + -+void lru_gen_del_mm(struct mm_struct *mm) ++unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) +{ -+ int nid; -+ struct lru_gen_mm_list *mm_list; -+ struct mem_cgroup *memcg = NULL; -+ -+ if (list_empty(&mm->lru_gen.list)) -+ return; -+ -+#ifdef CONFIG_MEMCG -+ memcg = mm->lru_gen.memcg; -+#endif -+ mm_list = get_mm_list(memcg); -+ -+ spin_lock(&mm_list->lock); -+ -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ if (!lruvec) -+ continue; -+ -+ /* where the last iteration ended (exclusive) */ -+ if (lruvec->mm_state.tail == &mm->lru_gen.list) -+ lruvec->mm_state.tail = lruvec->mm_state.tail->next; -+ -+ /* where the current iteration continues (inclusive) */ -+ if (lruvec->mm_state.head != &mm->lru_gen.list) -+ continue; -+ -+ lruvec->mm_state.head = lruvec->mm_state.head->next; -+ /* the deletion ends the current iteration */ -+ if (lruvec->mm_state.head == &mm_list->fifo) -+ WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); -+ } -+ -+ list_del_init(&mm->lru_gen.list); -+ -+ spin_unlock(&mm_list->lock); -+ -+#ifdef CONFIG_MEMCG -+ mem_cgroup_put(mm->lru_gen.memcg); -+ mm->lru_gen.memcg = NULL; -+#endif ++ return cachep->size * cachep->nr_allocated; +} + -+#ifdef CONFIG_MEMCG -+void lru_gen_migrate_mm(struct mm_struct *mm) ++unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) +{ -+ struct mem_cgroup *memcg; -+ struct task_struct *task = rcu_dereference_protected(mm->owner, true); -+ -+ VM_WARN_ON_ONCE(task->mm != mm); -+ lockdep_assert_held(&task->alloc_lock); -+ -+ /* for mm_update_next_owner() */ -+ if (mem_cgroup_disabled()) -+ return; -+ -+ rcu_read_lock(); -+ memcg = mem_cgroup_from_task(task); -+ rcu_read_unlock(); -+ if (memcg == mm->lru_gen.memcg) -+ return; -+ -+ VM_WARN_ON_ONCE(!mm->lru_gen.memcg); -+ VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); -+ -+ lru_gen_del_mm(mm); -+ lru_gen_add_mm(mm); ++ return cachep->nr_allocated; +} -+#endif -+ -+/* -+ * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when -+ * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of -+ * bits in a bitmap, k is the number of hash functions and n is the number of -+ * inserted items. -+ * -+ * Page table walkers use one of the two filters to reduce their search space. -+ * To get rid of non-leaf entries that no longer have enough leaf entries, the -+ * aging uses the double-buffering technique to flip to the other filter each -+ * time it produces a new generation. For non-leaf entries that have enough -+ * leaf entries, the aging carries them over to the next generation in -+ * walk_pmd_range(); the eviction also report them when walking the rmap -+ * in lru_gen_look_around(). -+ * -+ * For future optimizations: -+ * 1. It's not necessary to keep both filters all the time. The spare one can be -+ * freed after the RCU grace period and reallocated if needed again. -+ * 2. And when reallocating, it's worth scaling its size according to the number -+ * of inserted entries in the other filter, to reduce the memory overhead on -+ * small systems and false positives on large systems. -+ * 3. Jenkins' hash function is an alternative to Knuth's. -+ */ -+#define BLOOM_FILTER_SHIFT 15 + -+static inline int filter_gen_from_seq(unsigned long seq) ++unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) +{ -+ return seq % NR_BLOOM_FILTERS; ++ return cachep->nr_tallocated; +} + -+static void get_item_key(void *item, int *key) ++void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) +{ -+ u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); -+ -+ BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); -+ -+ key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); -+ key[1] = hash >> BLOOM_FILTER_SHIFT; ++ cachep->nr_tallocated = 0; +} + -+static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) -+{ -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); + void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, + int gfp) + { + void *p; + +- if (!(gfp & __GFP_DIRECT_RECLAIM)) +- return NULL; ++ if (!(gfp & __GFP_DIRECT_RECLAIM)) { ++ if (!cachep->non_kernel) ++ return NULL; + -+ filter = lruvec->mm_state.filters[gen]; -+ if (filter) { -+ bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); -+ return; ++ cachep->non_kernel--; + } -+ -+ filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), -+ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); -+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter); + + pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs) { +@@ -53,19 +85,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, + memset(p, 0, cachep->size); + } + ++ uatomic_inc(&cachep->nr_allocated); + uatomic_inc(&nr_allocated); ++ uatomic_inc(&cachep->nr_tallocated); + if (kmalloc_verbose) + printf("Allocating %p from slab\n", p); + return p; + } + +-void kmem_cache_free(struct kmem_cache *cachep, void *objp) ++void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) + { + assert(objp); + uatomic_dec(&nr_allocated); ++ uatomic_dec(&cachep->nr_allocated); + if (kmalloc_verbose) + printf("Freeing %p to slab\n", objp); +- pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs > 10 || cachep->align) { + memset(objp, POISON_FREE, cachep->size); + free(objp); +@@ -75,9 +109,80 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) + node->parent = cachep->objs; + cachep->objs = node; + } +} + -+static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) ++void kmem_cache_free(struct kmem_cache *cachep, void *objp) +{ -+ int key[2]; -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); -+ -+ filter = READ_ONCE(lruvec->mm_state.filters[gen]); -+ if (!filter) -+ return; -+ -+ get_item_key(item, key); ++ pthread_mutex_lock(&cachep->lock); ++ kmem_cache_free_locked(cachep, objp); + pthread_mutex_unlock(&cachep->lock); + } + ++void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) ++{ ++ if (kmalloc_verbose) ++ pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); + -+ if (!test_bit(key[0], filter)) -+ set_bit(key[0], filter); -+ if (!test_bit(key[1], filter)) -+ set_bit(key[1], filter); ++ pthread_mutex_lock(&cachep->lock); ++ for (int i = 0; i < size; i++) ++ kmem_cache_free_locked(cachep, list[i]); ++ pthread_mutex_unlock(&cachep->lock); +} + -+static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) ++int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, ++ void **p) +{ -+ int key[2]; -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); -+ -+ filter = READ_ONCE(lruvec->mm_state.filters[gen]); -+ if (!filter) -+ return true; -+ -+ get_item_key(item, key); -+ -+ return test_bit(key[0], filter) && test_bit(key[1], filter); -+} -+ -+static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) -+{ -+ int i; -+ int hist; -+ -+ lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); -+ -+ if (walk) { -+ hist = lru_hist_from_seq(walk->max_seq); -+ -+ for (i = 0; i < NR_MM_STATS; i++) { -+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], -+ lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); -+ walk->mm_stats[i] = 0; -+ } -+ } -+ -+ if (NR_HIST_GENS > 1 && last) { -+ hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); -+ -+ for (i = 0; i < NR_MM_STATS; i++) -+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); -+ } -+} -+ -+static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) -+{ -+ int type; -+ unsigned long size = 0; -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); -+ -+ if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) -+ return true; -+ -+ clear_bit(key, &mm->lru_gen.bitmap); -+ -+ for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { -+ size += type ? get_mm_counter(mm, MM_FILEPAGES) : -+ get_mm_counter(mm, MM_ANONPAGES) + -+ get_mm_counter(mm, MM_SHMEMPAGES); -+ } -+ -+ if (size < MIN_LRU_BATCH) -+ return true; -+ -+ return !mmget_not_zero(mm); -+} -+ -+static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, -+ struct mm_struct **iter) -+{ -+ bool first = false; -+ bool last = true; -+ struct mm_struct *mm = NULL; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); -+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; -+ -+ /* -+ * There are four interesting cases for this page table walker: -+ * 1. It tries to start a new iteration of mm_list with a stale max_seq; -+ * there is nothing left to do. -+ * 2. It's the first of the current generation, and it needs to reset -+ * the Bloom filter for the next generation. -+ * 3. It reaches the end of mm_list, and it needs to increment -+ * mm_state->seq; the iteration is done. -+ * 4. It's the last of the current generation, and it needs to reset the -+ * mm stats counters for the next generation. -+ */ -+ spin_lock(&mm_list->lock); -+ -+ VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); -+ VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); -+ VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); -+ -+ if (walk->max_seq <= mm_state->seq) { -+ if (!*iter) -+ last = false; -+ goto done; -+ } -+ -+ if (!mm_state->nr_walkers) { -+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); -+ -+ mm_state->head = mm_list->fifo.next; -+ first = true; -+ } -+ -+ while (!mm && mm_state->head != &mm_list->fifo) { -+ mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); -+ -+ mm_state->head = mm_state->head->next; -+ -+ /* force scan for those added after the last iteration */ -+ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { -+ mm_state->tail = mm_state->head; -+ walk->force_scan = true; -+ } -+ -+ if (should_skip_mm(mm, walk)) -+ mm = NULL; -+ } -+ -+ if (mm_state->head == &mm_list->fifo) -+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); -+done: -+ if (*iter && !mm) -+ mm_state->nr_walkers--; -+ if (!*iter && mm) -+ mm_state->nr_walkers++; -+ -+ if (mm_state->nr_walkers) -+ last = false; -+ -+ if (*iter || last) -+ reset_mm_stats(lruvec, walk, last); -+ -+ spin_unlock(&mm_list->lock); -+ -+ if (mm && first) -+ reset_bloom_filter(lruvec, walk->max_seq + 1); -+ -+ if (*iter) -+ mmput_async(*iter); -+ -+ *iter = mm; -+ -+ return last; -+} -+ -+static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) -+{ -+ bool success = false; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); -+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; -+ -+ spin_lock(&mm_list->lock); -+ -+ VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); -+ -+ if (max_seq > mm_state->seq && !mm_state->nr_walkers) { -+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); -+ -+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); -+ reset_mm_stats(lruvec, NULL, true); -+ success = true; -+ } -+ -+ spin_unlock(&mm_list->lock); -+ -+ return success; -+} -+ -+/****************************************************************************** -+ * refault feedback loop -+ ******************************************************************************/ -+ -+/* -+ * A feedback loop based on Proportional-Integral-Derivative (PID) controller. -+ * -+ * The P term is refaulted/(evicted+protected) from a tier in the generation -+ * currently being evicted; the I term is the exponential moving average of the -+ * P term over the generations previously evicted, using the smoothing factor -+ * 1/2; the D term isn't supported. -+ * -+ * The setpoint (SP) is always the first tier of one type; the process variable -+ * (PV) is either any tier of the other type or any other tier of the same -+ * type. -+ * -+ * The error is the difference between the SP and the PV; the correction is to -+ * turn off protection when SP>PV or turn on protection when SPlrugen; -+ int hist = lru_hist_from_seq(lrugen->min_seq[type]); -+ -+ pos->refaulted = lrugen->avg_refaulted[type][tier] + -+ atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ pos->total = lrugen->avg_total[type][tier] + -+ atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ pos->total += lrugen->protected[hist][type][tier - 1]; -+ pos->gain = gain; -+} -+ -+static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) -+{ -+ int hist, tier; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; -+ unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; -+ -+ lockdep_assert_held(&lruvec->lru_lock); -+ -+ if (!carryover && !clear) -+ return; -+ -+ hist = lru_hist_from_seq(seq); -+ -+ for (tier = 0; tier < MAX_NR_TIERS; tier++) { -+ if (carryover) { -+ unsigned long sum; -+ -+ sum = lrugen->avg_refaulted[type][tier] + -+ atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); -+ -+ sum = lrugen->avg_total[type][tier] + -+ atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ sum += lrugen->protected[hist][type][tier - 1]; -+ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); -+ } -+ -+ if (clear) { -+ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); -+ atomic_long_set(&lrugen->evicted[hist][type][tier], 0); -+ if (tier) -+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); -+ } -+ } -+} -+ -+static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) -+{ -+ /* -+ * Return true if the PV has a limited number of refaults or a lower -+ * refaulted/total than the SP. -+ */ -+ return pv->refaulted < MIN_LRU_BATCH || -+ pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= -+ (sp->refaulted + 1) * pv->total * pv->gain; -+} -+ -+/****************************************************************************** -+ * the aging -+ ******************************************************************************/ -+ -+/* promote pages accessed through page tables */ -+static int folio_update_gen(struct folio *folio, int gen) -+{ -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); -+ -+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(!rcu_read_lock_held()); -+ -+ do { -+ /* lru_gen_del_folio() has isolated this page? */ -+ if (!(old_flags & LRU_GEN_MASK)) { -+ /* for shrink_page_list() */ -+ new_flags = old_flags | BIT(PG_referenced); -+ continue; -+ } -+ -+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); -+ new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); -+ -+ return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+} -+ -+/* protect pages accessed multiple times through file descriptors */ -+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ int type = folio_is_file_lru(folio); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); -+ -+ VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); -+ -+ do { -+ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+ /* folio_update_gen() has promoted this page? */ -+ if (new_gen >= 0 && new_gen != old_gen) -+ return new_gen; -+ -+ new_gen = (old_gen + 1) % MAX_NR_GENS; -+ -+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); -+ new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; -+ /* for folio_end_writeback() */ -+ if (reclaiming) -+ new_flags |= BIT(PG_reclaim); -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); -+ -+ lru_gen_update_size(lruvec, folio, old_gen, new_gen); -+ -+ return new_gen; -+} -+ -+static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, -+ int old_gen, int new_gen) -+{ -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ -+ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); -+ -+ walk->batched++; -+ -+ walk->nr_pages[old_gen][type][zone] -= delta; -+ walk->nr_pages[new_gen][type][zone] += delta; -+} -+ -+static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) -+{ -+ int gen, type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ walk->batched = 0; -+ -+ for_each_gen_type_zone(gen, type, zone) { -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ int delta = walk->nr_pages[gen][type][zone]; -+ -+ if (!delta) -+ continue; -+ -+ walk->nr_pages[gen][type][zone] = 0; -+ WRITE_ONCE(lrugen->nr_pages[gen][type][zone], -+ lrugen->nr_pages[gen][type][zone] + delta); -+ -+ if (lru_gen_is_active(lruvec, gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, delta); -+ } -+} -+ -+static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) -+{ -+ struct address_space *mapping; -+ struct vm_area_struct *vma = args->vma; -+ struct lru_gen_mm_walk *walk = args->private; -+ -+ if (!vma_is_accessible(vma)) -+ return true; -+ -+ if (is_vm_hugetlb_page(vma)) -+ return true; -+ -+ if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) -+ return true; -+ -+ if (vma == get_gate_vma(vma->vm_mm)) -+ return true; -+ -+ if (vma_is_anonymous(vma)) -+ return !walk->can_swap; -+ -+ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) -+ return true; -+ -+ mapping = vma->vm_file->f_mapping; -+ if (mapping_unevictable(mapping)) -+ return true; -+ -+ if (shmem_mapping(mapping)) -+ return !walk->can_swap; -+ -+ /* to exclude special mappings like dax, etc. */ -+ return !mapping->a_ops->read_folio; -+} -+ -+/* -+ * Some userspace memory allocators map many single-page VMAs. Instead of -+ * returning back to the PGD table for each of such VMAs, finish an entire PMD -+ * table to reduce zigzags and improve cache performance. -+ */ -+static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, -+ unsigned long *vm_start, unsigned long *vm_end) -+{ -+ unsigned long start = round_up(*vm_end, size); -+ unsigned long end = (start | ~mask) + 1; -+ VMA_ITERATOR(vmi, args->mm, start); -+ -+ VM_WARN_ON_ONCE(mask & size); -+ VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); -+ -+ for_each_vma(vmi, args->vma) { -+ if (end && end <= args->vma->vm_start) -+ return false; -+ -+ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) -+ continue; -+ -+ *vm_start = max(start, args->vma->vm_start); -+ *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; -+ -+ return true; -+ } -+ -+ return false; -+} -+ -+static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) -+{ -+ unsigned long pfn = pte_pfn(pte); -+ -+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); -+ -+ if (!pte_present(pte) || is_zero_pfn(pfn)) -+ return -1; -+ -+ if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) -+ return -1; -+ -+ if (WARN_ON_ONCE(!pfn_valid(pfn))) -+ return -1; -+ -+ return pfn; -+} -+ -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -+static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) -+{ -+ unsigned long pfn = pmd_pfn(pmd); -+ -+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); -+ -+ if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) -+ return -1; -+ -+ if (WARN_ON_ONCE(pmd_devmap(pmd))) -+ return -1; -+ -+ if (WARN_ON_ONCE(!pfn_valid(pfn))) -+ return -1; -+ -+ return pfn; -+} -+#endif -+ -+static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, -+ struct pglist_data *pgdat, bool can_swap) -+{ -+ struct folio *folio; -+ -+ /* try to avoid unnecessary memory loads */ -+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) -+ return NULL; -+ -+ folio = pfn_folio(pfn); -+ if (folio_nid(folio) != pgdat->node_id) -+ return NULL; -+ -+ if (folio_memcg_rcu(folio) != memcg) -+ return NULL; -+ -+ /* file VMAs can contain anon pages from COW */ -+ if (!folio_is_file_lru(folio) && !can_swap) -+ return NULL; -+ -+ return folio; -+} -+ -+static bool suitable_to_scan(int total, int young) -+{ -+ int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); -+ -+ /* suitable if the average number of young PTEs per cacheline is >=1 */ -+ return young * n >= total; -+} -+ -+static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, -+ struct mm_walk *args) -+{ -+ int i; -+ pte_t *pte; -+ spinlock_t *ptl; -+ unsigned long addr; -+ int total = 0; -+ int young = 0; -+ struct lru_gen_mm_walk *walk = args->private; -+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); -+ -+ VM_WARN_ON_ONCE(pmd_leaf(*pmd)); -+ -+ ptl = pte_lockptr(args->mm, pmd); -+ if (!spin_trylock(ptl)) -+ return false; -+ -+ arch_enter_lazy_mmu_mode(); -+ -+ pte = pte_offset_map(pmd, start & PMD_MASK); -+restart: -+ for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { -+ unsigned long pfn; -+ struct folio *folio; -+ -+ total++; -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ -+ pfn = get_pte_pfn(pte[i], args->vma, addr); -+ if (pfn == -1) -+ continue; -+ -+ if (!pte_young(pte[i])) { -+ walk->mm_stats[MM_LEAF_OLD]++; -+ continue; -+ } -+ -+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); -+ if (!folio) -+ continue; -+ -+ if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) -+ VM_WARN_ON_ONCE(true); -+ -+ young++; -+ walk->mm_stats[MM_LEAF_YOUNG]++; -+ -+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); -+ -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen >= 0 && old_gen != new_gen) -+ update_batch_size(walk, folio, old_gen, new_gen); -+ } -+ -+ if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) -+ goto restart; -+ -+ pte_unmap(pte); -+ -+ arch_leave_lazy_mmu_mode(); -+ spin_unlock(ptl); -+ -+ return suitable_to_scan(total, young); -+} -+ -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, -+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) -+{ -+ int i; -+ pmd_t *pmd; -+ spinlock_t *ptl; -+ struct lru_gen_mm_walk *walk = args->private; -+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); -+ -+ VM_WARN_ON_ONCE(pud_leaf(*pud)); -+ -+ /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ -+ if (*start == -1) { -+ *start = next; -+ return; -+ } -+ -+ i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); -+ if (i && i <= MIN_LRU_BATCH) { -+ __set_bit(i - 1, bitmap); -+ return; -+ } -+ -+ pmd = pmd_offset(pud, *start); -+ -+ ptl = pmd_lockptr(args->mm, pmd); -+ if (!spin_trylock(ptl)) -+ goto done; -+ -+ arch_enter_lazy_mmu_mode(); -+ -+ do { -+ unsigned long pfn; -+ struct folio *folio; -+ unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; -+ -+ pfn = get_pmd_pfn(pmd[i], vma, addr); -+ if (pfn == -1) -+ goto next; -+ -+ if (!pmd_trans_huge(pmd[i])) { -+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && -+ get_cap(LRU_GEN_NONLEAF_YOUNG)) -+ pmdp_test_and_clear_young(vma, addr, pmd + i); -+ goto next; -+ } -+ -+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); -+ if (!folio) -+ goto next; -+ -+ if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) -+ goto next; -+ -+ walk->mm_stats[MM_LEAF_YOUNG]++; -+ -+ if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); -+ -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen >= 0 && old_gen != new_gen) -+ update_batch_size(walk, folio, old_gen, new_gen); -+next: -+ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; -+ } while (i <= MIN_LRU_BATCH); -+ -+ arch_leave_lazy_mmu_mode(); -+ spin_unlock(ptl); -+done: -+ *start = -1; -+ bitmap_zero(bitmap, MIN_LRU_BATCH); -+} -+#else -+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, -+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) -+{ -+} -+#endif -+ -+static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, -+ struct mm_walk *args) -+{ -+ int i; -+ pmd_t *pmd; -+ unsigned long next; -+ unsigned long addr; -+ struct vm_area_struct *vma; -+ unsigned long pos = -1; -+ struct lru_gen_mm_walk *walk = args->private; -+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; -+ -+ VM_WARN_ON_ONCE(pud_leaf(*pud)); -+ -+ /* -+ * Finish an entire PMD in two passes: the first only reaches to PTE -+ * tables to avoid taking the PMD lock; the second, if necessary, takes -+ * the PMD lock to clear the accessed bit in PMD entries. -+ */ -+ pmd = pmd_offset(pud, start & PUD_MASK); -+restart: -+ /* walk_pte_range() may call get_next_vma() */ -+ vma = args->vma; -+ for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { -+ pmd_t val = pmd_read_atomic(pmd + i); -+ -+ /* for pmd_read_atomic() */ -+ barrier(); -+ -+ next = pmd_addr_end(addr, end); -+ -+ if (!pmd_present(val) || is_huge_zero_pmd(val)) { -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ continue; -+ } -+ -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ if (pmd_trans_huge(val)) { -+ unsigned long pfn = pmd_pfn(val); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ -+ if (!pmd_young(val)) { -+ walk->mm_stats[MM_LEAF_OLD]++; -+ continue; -+ } -+ -+ /* try to avoid unnecessary memory loads */ -+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) -+ continue; -+ -+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); -+ continue; -+ } -+#endif -+ walk->mm_stats[MM_NONLEAF_TOTAL]++; -+ -+#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG -+ if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { -+ if (!pmd_young(val)) -+ continue; -+ -+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); -+ } -+#endif -+ if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) -+ continue; -+ -+ walk->mm_stats[MM_NONLEAF_FOUND]++; -+ -+ if (!walk_pte_range(&val, addr, next, args)) -+ continue; -+ -+ walk->mm_stats[MM_NONLEAF_ADDED]++; -+ -+ /* carry over to the next generation */ -+ update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); -+ } -+ -+ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); -+ -+ if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) -+ goto restart; -+} -+ -+static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, -+ struct mm_walk *args) -+{ -+ int i; -+ pud_t *pud; -+ unsigned long addr; -+ unsigned long next; -+ struct lru_gen_mm_walk *walk = args->private; -+ -+ VM_WARN_ON_ONCE(p4d_leaf(*p4d)); -+ -+ pud = pud_offset(p4d, start & P4D_MASK); -+restart: -+ for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { -+ pud_t val = READ_ONCE(pud[i]); -+ -+ next = pud_addr_end(addr, end); -+ -+ if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) -+ continue; -+ -+ walk_pmd_range(&val, addr, next, args); -+ -+ /* a racy check to curtail the waiting time */ -+ if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) -+ return 1; -+ -+ if (need_resched() || walk->batched >= MAX_LRU_BATCH) { -+ end = (addr | ~PUD_MASK) + 1; -+ goto done; -+ } -+ } -+ -+ if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) -+ goto restart; -+ -+ end = round_up(end, P4D_SIZE); -+done: -+ if (!end || !args->vma) -+ return 1; -+ -+ walk->next_addr = max(end, args->vma->vm_start); -+ -+ return -EAGAIN; -+} -+ -+static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) -+{ -+ static const struct mm_walk_ops mm_walk_ops = { -+ .test_walk = should_skip_vma, -+ .p4d_entry = walk_pud_range, -+ }; -+ -+ int err; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ -+ walk->next_addr = FIRST_USER_ADDRESS; -+ -+ do { -+ err = -EBUSY; -+ -+ /* folio_update_gen() requires stable folio_memcg() */ -+ if (!mem_cgroup_trylock_pages(memcg)) -+ break; -+ -+ /* the caller might be holding the lock for write */ -+ if (mmap_read_trylock(mm)) { -+ err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); -+ -+ mmap_read_unlock(mm); -+ } -+ -+ mem_cgroup_unlock_pages(); -+ -+ if (walk->batched) { -+ spin_lock_irq(&lruvec->lru_lock); -+ reset_batch_size(lruvec, walk); -+ spin_unlock_irq(&lruvec->lru_lock); -+ } -+ -+ cond_resched(); -+ } while (err == -EAGAIN); -+} -+ -+static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) -+{ -+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; -+ -+ if (pgdat && current_is_kswapd()) { -+ VM_WARN_ON_ONCE(walk); -+ -+ walk = &pgdat->mm_walk; -+ } else if (!pgdat && !walk) { -+ VM_WARN_ON_ONCE(current_is_kswapd()); -+ -+ walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); -+ } -+ -+ current->reclaim_state->mm_walk = walk; -+ -+ return walk; -+} -+ -+static void clear_mm_walk(void) -+{ -+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; -+ -+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); -+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); -+ -+ current->reclaim_state->mm_walk = NULL; -+ -+ if (!current_is_kswapd()) -+ kfree(walk); -+} -+ -+static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) -+{ -+ int zone; -+ int remaining = MAX_LRU_BATCH; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ -+ if (type == LRU_GEN_ANON && !can_swap) -+ goto done; -+ -+ /* prevent cold/hot inversion if force_scan is true */ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ struct list_head *head = &lrugen->lists[old_gen][type][zone]; -+ -+ while (!list_empty(head)) { -+ struct folio *folio = lru_to_folio(head); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); -+ -+ new_gen = folio_inc_gen(lruvec, folio, false); -+ list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); -+ -+ if (!--remaining) -+ return false; -+ } -+ } -+done: -+ reset_ctrl_pos(lruvec, type, true); -+ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); -+ -+ return true; -+} -+ -+static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) -+{ -+ int gen, type, zone; -+ bool success = false; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ DEFINE_MIN_SEQ(lruvec); -+ -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ -+ /* find the oldest populated generation */ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { -+ gen = lru_gen_from_seq(min_seq[type]); -+ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ if (!list_empty(&lrugen->lists[gen][type][zone])) -+ goto next; -+ } -+ -+ min_seq[type]++; -+ } -+next: -+ ; -+ } -+ -+ /* see the comment on lru_gen_struct */ -+ if (can_swap) { -+ min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); -+ min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); -+ } -+ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ if (min_seq[type] == lrugen->min_seq[type]) -+ continue; -+ -+ reset_ctrl_pos(lruvec, type, true); -+ WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); -+ success = true; -+ } -+ -+ return success; -+} -+ -+static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) -+{ -+ int prev, next; -+ int type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ -+ for (type = ANON_AND_FILE - 1; type >= 0; type--) { -+ if (get_nr_gens(lruvec, type) != MAX_NR_GENS) -+ continue; -+ -+ VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); -+ -+ while (!inc_min_seq(lruvec, type, can_swap)) { -+ spin_unlock_irq(&lruvec->lru_lock); -+ cond_resched(); -+ spin_lock_irq(&lruvec->lru_lock); -+ } -+ } -+ -+ /* -+ * Update the active/inactive LRU sizes for compatibility. Both sides of -+ * the current max_seq need to be covered, since max_seq+1 can overlap -+ * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do -+ * overlap, cold/hot inversion happens. -+ */ -+ prev = lru_gen_from_seq(lrugen->max_seq - 1); -+ next = lru_gen_from_seq(lrugen->max_seq + 1); -+ -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ long delta = lrugen->nr_pages[prev][type][zone] - -+ lrugen->nr_pages[next][type][zone]; -+ -+ if (!delta) -+ continue; -+ -+ __update_lru_size(lruvec, lru, zone, delta); -+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); -+ } -+ } -+ -+ for (type = 0; type < ANON_AND_FILE; type++) -+ reset_ctrl_pos(lruvec, type, false); -+ -+ WRITE_ONCE(lrugen->timestamps[next], jiffies); -+ /* make sure preceding modifications appear */ -+ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+} -+ -+static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, -+ struct scan_control *sc, bool can_swap, bool force_scan) -+{ -+ bool success; -+ struct lru_gen_mm_walk *walk; -+ struct mm_struct *mm = NULL; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); -+ -+ /* see the comment in iterate_mm_list() */ -+ if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { -+ success = false; -+ goto done; -+ } -+ -+ /* -+ * If the hardware doesn't automatically set the accessed bit, fallback -+ * to lru_gen_look_around(), which only clears the accessed bit in a -+ * handful of PTEs. Spreading the work out over a period of time usually -+ * is less efficient, but it avoids bursty page faults. -+ */ -+ if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { -+ success = iterate_mm_list_nowalk(lruvec, max_seq); -+ goto done; -+ } -+ -+ walk = set_mm_walk(NULL); -+ if (!walk) { -+ success = iterate_mm_list_nowalk(lruvec, max_seq); -+ goto done; -+ } -+ -+ walk->lruvec = lruvec; -+ walk->max_seq = max_seq; -+ walk->can_swap = can_swap; -+ walk->force_scan = force_scan; -+ -+ do { -+ success = iterate_mm_list(lruvec, walk, &mm); -+ if (mm) -+ walk_mm(lruvec, mm, walk); -+ -+ cond_resched(); -+ } while (mm); -+done: -+ if (!success) { -+ if (sc->priority <= DEF_PRIORITY - 2) -+ wait_event_killable(lruvec->mm_state.wait, -+ max_seq < READ_ONCE(lrugen->max_seq)); -+ -+ return max_seq < READ_ONCE(lrugen->max_seq); -+ } -+ -+ VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); -+ -+ inc_max_seq(lruvec, can_swap, force_scan); -+ /* either this sees any waiters or they will see updated max_seq */ -+ if (wq_has_sleeper(&lruvec->mm_state.wait)) -+ wake_up_all(&lruvec->mm_state.wait); -+ -+ wakeup_flusher_threads(WB_REASON_VMSCAN); -+ -+ return true; -+} -+ -+static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, -+ struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) -+{ -+ int gen, type, zone; -+ unsigned long old = 0; -+ unsigned long young = 0; -+ unsigned long total = 0; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ unsigned long seq; -+ -+ for (seq = min_seq[type]; seq <= max_seq; seq++) { -+ unsigned long size = 0; -+ -+ gen = lru_gen_from_seq(seq); -+ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) -+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); -+ -+ total += size; -+ if (seq == max_seq) -+ young += size; -+ else if (seq + MIN_NR_GENS == max_seq) -+ old += size; -+ } -+ } -+ -+ /* try to scrape all its memory if this memcg was deleted */ -+ *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; -+ -+ /* -+ * The aging tries to be lazy to reduce the overhead, while the eviction -+ * stalls when the number of generations reaches MIN_NR_GENS. Hence, the -+ * ideal number of generations is MIN_NR_GENS+1. -+ */ -+ if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) -+ return true; -+ if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) -+ return false; -+ -+ /* -+ * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) -+ * of the total number of pages for each generation. A reasonable range -+ * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The -+ * aging cares about the upper bound of hot pages, while the eviction -+ * cares about the lower bound of cold pages. -+ */ -+ if (young * MIN_NR_GENS > total) -+ return true; -+ if (old * (MIN_NR_GENS + 2) < total) -+ return true; -+ -+ return false; -+} -+ -+static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) -+{ -+ bool need_aging; -+ unsigned long nr_to_scan; -+ int swappiness = get_swappiness(lruvec, sc); -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ VM_WARN_ON_ONCE(sc->memcg_low_reclaim); -+ -+ mem_cgroup_calculate_protection(NULL, memcg); -+ -+ if (mem_cgroup_below_min(memcg)) -+ return false; -+ -+ need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); -+ -+ if (min_ttl) { -+ int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); -+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); -+ -+ if (time_is_after_jiffies(birth + min_ttl)) -+ return false; -+ -+ /* the size is likely too small to be helpful */ -+ if (!nr_to_scan && sc->priority != DEF_PRIORITY) -+ return false; -+ } -+ -+ if (need_aging) -+ try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); -+ -+ return true; -+} -+ -+/* to protect the working set of the last N jiffies */ -+static unsigned long lru_gen_min_ttl __read_mostly; -+ -+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -+{ -+ struct mem_cgroup *memcg; -+ bool success = false; -+ unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); -+ -+ VM_WARN_ON_ONCE(!current_is_kswapd()); -+ -+ sc->last_reclaimed = sc->nr_reclaimed; -+ -+ /* -+ * To reduce the chance of going into the aging path, which can be -+ * costly, optimistically skip it if the flag below was cleared in the -+ * eviction path. This improves the overall performance when multiple -+ * memcgs are available. -+ */ -+ if (!sc->memcgs_need_aging) { -+ sc->memcgs_need_aging = true; -+ return; -+ } -+ -+ set_mm_walk(pgdat); -+ -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ -+ if (age_lruvec(lruvec, sc, min_ttl)) -+ success = true; -+ -+ cond_resched(); -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+ -+ clear_mm_walk(); -+ -+ /* check the order to exclude compaction-induced reclaim */ -+ if (success || !min_ttl || sc->order) -+ return; -+ -+ /* -+ * The main goal is to OOM kill if every generation from all memcgs is -+ * younger than min_ttl. However, another possibility is all memcgs are -+ * either below min or empty. -+ */ -+ if (mutex_trylock(&oom_lock)) { -+ struct oom_control oc = { -+ .gfp_mask = sc->gfp_mask, -+ }; -+ -+ out_of_memory(&oc); -+ -+ mutex_unlock(&oom_lock); -+ } -+} -+ -+/* -+ * This function exploits spatial locality when shrink_page_list() walks the -+ * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If -+ * the scan was done cacheline efficiently, it adds the PMD entry pointing to -+ * the PTE table to the Bloom filter. This forms a feedback loop between the -+ * eviction and the aging. -+ */ -+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) -+{ -+ int i; -+ pte_t *pte; -+ unsigned long start; -+ unsigned long end; -+ unsigned long addr; -+ struct lru_gen_mm_walk *walk; -+ int young = 0; -+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; -+ struct folio *folio = pfn_folio(pvmw->pfn); -+ struct mem_cgroup *memcg = folio_memcg(folio); -+ struct pglist_data *pgdat = folio_pgdat(folio); -+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ DEFINE_MAX_SEQ(lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(max_seq); -+ -+ lockdep_assert_held(pvmw->ptl); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); -+ -+ if (spin_is_contended(pvmw->ptl)) -+ return; -+ -+ /* avoid taking the LRU lock under the PTL when possible */ -+ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; -+ -+ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); -+ end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; -+ -+ if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { -+ if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) -+ end = start + MIN_LRU_BATCH * PAGE_SIZE; -+ else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) -+ start = end - MIN_LRU_BATCH * PAGE_SIZE; -+ else { -+ start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; -+ end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; -+ } -+ } -+ -+ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; -+ -+ rcu_read_lock(); -+ arch_enter_lazy_mmu_mode(); -+ -+ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { -+ unsigned long pfn; -+ -+ pfn = get_pte_pfn(pte[i], pvmw->vma, addr); -+ if (pfn == -1) -+ continue; -+ -+ if (!pte_young(pte[i])) -+ continue; -+ -+ folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); -+ if (!folio) -+ continue; -+ -+ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) -+ VM_WARN_ON_ONCE(true); -+ -+ young++; -+ -+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); -+ -+ old_gen = folio_lru_gen(folio); -+ if (old_gen < 0) -+ folio_set_referenced(folio); -+ else if (old_gen != new_gen) -+ __set_bit(i, bitmap); -+ } -+ -+ arch_leave_lazy_mmu_mode(); -+ rcu_read_unlock(); -+ -+ /* feedback from rmap walkers to page table walkers */ -+ if (suitable_to_scan(i, young)) -+ update_bloom_filter(lruvec, max_seq, pvmw->pmd); -+ -+ if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { -+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { -+ folio = pfn_folio(pte_pfn(pte[i])); -+ folio_activate(folio); -+ } -+ return; -+ } -+ -+ /* folio_update_gen() requires stable folio_memcg() */ -+ if (!mem_cgroup_trylock_pages(memcg)) -+ return; -+ -+ if (!walk) { -+ spin_lock_irq(&lruvec->lru_lock); -+ new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); -+ } -+ -+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { -+ folio = pfn_folio(pte_pfn(pte[i])); -+ if (folio_memcg_rcu(folio) != memcg) -+ continue; -+ -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen < 0 || old_gen == new_gen) -+ continue; -+ -+ if (walk) -+ update_batch_size(walk, folio, old_gen, new_gen); -+ else -+ lru_gen_update_size(lruvec, folio, old_gen, new_gen); -+ } -+ -+ if (!walk) -+ spin_unlock_irq(&lruvec->lru_lock); -+ -+ mem_cgroup_unlock_pages(); -+} -+ -+/****************************************************************************** -+ * the eviction -+ ******************************************************************************/ -+ -+static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) -+{ -+ bool success; -+ int gen = folio_lru_gen(folio); -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ int refs = folio_lru_refs(folio); -+ int tier = lru_tier_from_refs(refs); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); -+ -+ /* unevictable */ -+ if (!folio_evictable(folio)) { -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ folio_set_unevictable(folio); -+ lruvec_add_folio(lruvec, folio); -+ __count_vm_events(UNEVICTABLE_PGCULLED, delta); -+ return true; -+ } -+ -+ /* dirty lazyfree */ -+ if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ folio_set_swapbacked(folio); -+ lruvec_add_folio_tail(lruvec, folio); -+ return true; -+ } -+ -+ /* promoted */ -+ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { -+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); -+ return true; -+ } -+ -+ /* protected */ -+ if (tier > tier_idx) { -+ int hist = lru_hist_from_seq(lrugen->min_seq[type]); -+ -+ gen = folio_inc_gen(lruvec, folio, false); -+ list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); -+ -+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], -+ lrugen->protected[hist][type][tier - 1] + delta); -+ __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); -+ return true; -+ } -+ -+ /* waiting for writeback */ -+ if (folio_test_locked(folio) || folio_test_writeback(folio) || -+ (type == LRU_GEN_FILE && folio_test_dirty(folio))) { -+ gen = folio_inc_gen(lruvec, folio, true); -+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); -+ return true; -+ } -+ -+ return false; -+} -+ -+static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) -+{ -+ bool success; -+ -+ /* unmapping inhibited */ -+ if (!sc->may_unmap && folio_mapped(folio)) -+ return false; -+ -+ /* swapping inhibited */ -+ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && -+ (folio_test_dirty(folio) || -+ (folio_test_anon(folio) && !folio_test_swapcache(folio)))) -+ return false; -+ -+ /* raced with release_pages() */ -+ if (!folio_try_get(folio)) -+ return false; -+ -+ /* raced with another isolation */ -+ if (!folio_test_clear_lru(folio)) { -+ folio_put(folio); -+ return false; -+ } -+ -+ /* see the comment on MAX_NR_TIERS */ -+ if (!folio_test_referenced(folio)) -+ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); -+ -+ /* for shrink_page_list() */ -+ folio_clear_reclaim(folio); -+ folio_clear_referenced(folio); -+ -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ -+ return true; -+} -+ -+static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, -+ int type, int tier, struct list_head *list) -+{ -+ int gen, zone; -+ enum vm_event_item item; -+ int sorted = 0; -+ int scanned = 0; -+ int isolated = 0; -+ int remaining = MAX_LRU_BATCH; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ -+ VM_WARN_ON_ONCE(!list_empty(list)); -+ -+ if (get_nr_gens(lruvec, type) == MIN_NR_GENS) -+ return 0; -+ -+ gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ -+ for (zone = sc->reclaim_idx; zone >= 0; zone--) { -+ LIST_HEAD(moved); -+ int skipped = 0; -+ struct list_head *head = &lrugen->lists[gen][type][zone]; -+ -+ while (!list_empty(head)) { -+ struct folio *folio = lru_to_folio(head); -+ int delta = folio_nr_pages(folio); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); -+ -+ scanned += delta; -+ -+ if (sort_folio(lruvec, folio, tier)) -+ sorted += delta; -+ else if (isolate_folio(lruvec, folio, sc)) { -+ list_add(&folio->lru, list); -+ isolated += delta; -+ } else { -+ list_move(&folio->lru, &moved); -+ skipped += delta; -+ } -+ -+ if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) -+ break; -+ } -+ -+ if (skipped) { -+ list_splice(&moved, head); -+ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); -+ } -+ -+ if (!remaining || isolated >= MIN_LRU_BATCH) -+ break; -+ } -+ -+ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; -+ if (!cgroup_reclaim(sc)) { -+ __count_vm_events(item, isolated); -+ __count_vm_events(PGREFILL, sorted); -+ } -+ __count_memcg_events(memcg, item, isolated); -+ __count_memcg_events(memcg, PGREFILL, sorted); -+ __count_vm_events(PGSCAN_ANON + type, isolated); -+ -+ /* -+ * There might not be eligible pages due to reclaim_idx, may_unmap and -+ * may_writepage. Check the remaining to prevent livelock if it's not -+ * making progress. -+ */ -+ return isolated || !remaining ? scanned : 0; -+} -+ -+static int get_tier_idx(struct lruvec *lruvec, int type) -+{ -+ int tier; -+ struct ctrl_pos sp, pv; -+ -+ /* -+ * To leave a margin for fluctuations, use a larger gain factor (1:2). -+ * This value is chosen because any other tier would have at least twice -+ * as many refaults as the first tier. -+ */ -+ read_ctrl_pos(lruvec, type, 0, 1, &sp); -+ for (tier = 1; tier < MAX_NR_TIERS; tier++) { -+ read_ctrl_pos(lruvec, type, tier, 2, &pv); -+ if (!positive_ctrl_err(&sp, &pv)) -+ break; -+ } -+ -+ return tier - 1; -+} -+ -+static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) -+{ -+ int type, tier; -+ struct ctrl_pos sp, pv; -+ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; -+ -+ /* -+ * Compare the first tier of anon with that of file to determine which -+ * type to scan. Also need to compare other tiers of the selected type -+ * with the first tier of the other type to determine the last tier (of -+ * the selected type) to evict. -+ */ -+ read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); -+ read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); -+ type = positive_ctrl_err(&sp, &pv); -+ -+ read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); -+ for (tier = 1; tier < MAX_NR_TIERS; tier++) { -+ read_ctrl_pos(lruvec, type, tier, gain[type], &pv); -+ if (!positive_ctrl_err(&sp, &pv)) -+ break; -+ } -+ -+ *tier_idx = tier - 1; -+ -+ return type; -+} -+ -+static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, -+ int *type_scanned, struct list_head *list) -+{ -+ int i; -+ int type; -+ int scanned; -+ int tier = -1; -+ DEFINE_MIN_SEQ(lruvec); -+ -+ /* -+ * Try to make the obvious choice first. When anon and file are both -+ * available from the same generation, interpret swappiness 1 as file -+ * first and 200 as anon first. -+ */ -+ if (!swappiness) -+ type = LRU_GEN_FILE; -+ else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) -+ type = LRU_GEN_ANON; -+ else if (swappiness == 1) -+ type = LRU_GEN_FILE; -+ else if (swappiness == 200) -+ type = LRU_GEN_ANON; -+ else -+ type = get_type_to_scan(lruvec, swappiness, &tier); -+ -+ for (i = !swappiness; i < ANON_AND_FILE; i++) { -+ if (tier < 0) -+ tier = get_tier_idx(lruvec, type); -+ -+ scanned = scan_folios(lruvec, sc, type, tier, list); -+ if (scanned) -+ break; -+ -+ type = !type; -+ tier = -1; -+ } -+ -+ *type_scanned = type; -+ -+ return scanned; -+} -+ -+static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, -+ bool *need_swapping) -+{ -+ int type; -+ int scanned; -+ int reclaimed; -+ LIST_HEAD(list); -+ struct folio *folio; -+ enum vm_event_item item; -+ struct reclaim_stat stat; -+ struct lru_gen_mm_walk *walk; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(lruvec); -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); -+ -+ scanned += try_to_inc_min_seq(lruvec, swappiness); -+ -+ if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) -+ scanned = 0; -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+ -+ if (list_empty(&list)) -+ return scanned; -+ -+ reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); -+ -+ list_for_each_entry(folio, &list, lru) { -+ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ -+ if (folio_test_workingset(folio)) -+ folio_set_referenced(folio); -+ -+ /* don't add rejected pages to the oldest generation */ -+ if (folio_test_reclaim(folio) && -+ (folio_test_dirty(folio) || folio_test_writeback(folio))) -+ folio_clear_active(folio); -+ else -+ folio_set_active(folio); -+ } -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ move_pages_to_lru(lruvec, &list); -+ -+ walk = current->reclaim_state->mm_walk; -+ if (walk && walk->batched) -+ reset_batch_size(lruvec, walk); -+ -+ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; -+ if (!cgroup_reclaim(sc)) -+ __count_vm_events(item, reclaimed); -+ __count_memcg_events(memcg, item, reclaimed); -+ __count_vm_events(PGSTEAL_ANON + type, reclaimed); -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+ -+ mem_cgroup_uncharge_list(&list); -+ free_unref_page_list(&list); -+ -+ sc->nr_reclaimed += reclaimed; -+ -+ if (need_swapping && type == LRU_GEN_ANON) -+ *need_swapping = true; -+ -+ return scanned; -+} -+ -+/* -+ * For future optimizations: -+ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg -+ * reclaim. -+ */ -+static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, -+ bool can_swap, bool *need_aging) -+{ -+ unsigned long nr_to_scan; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (mem_cgroup_below_min(memcg) || -+ (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) -+ return 0; -+ -+ *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); -+ if (!*need_aging) -+ return nr_to_scan; -+ -+ /* skip the aging path at the default priority */ -+ if (sc->priority == DEF_PRIORITY) -+ goto done; -+ -+ /* leave the work to lru_gen_age_node() */ -+ if (current_is_kswapd()) -+ return 0; -+ -+ if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) -+ return nr_to_scan; -+done: -+ return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; -+} -+ -+static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, -+ struct scan_control *sc, bool need_swapping) -+{ -+ int i; -+ DEFINE_MAX_SEQ(lruvec); -+ -+ if (!current_is_kswapd()) { -+ /* age each memcg once to ensure fairness */ -+ if (max_seq - seq > 1) -+ return true; -+ -+ /* over-swapping can increase allocation latency */ -+ if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) -+ return true; -+ -+ /* give this thread a chance to exit and free its memory */ -+ if (fatal_signal_pending(current)) { -+ sc->nr_reclaimed += MIN_LRU_BATCH; -+ return true; -+ } -+ -+ if (cgroup_reclaim(sc)) -+ return false; -+ } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) -+ return false; -+ -+ /* keep scanning at low priorities to ensure fairness */ -+ if (sc->priority > DEF_PRIORITY - 2) -+ return false; -+ -+ /* -+ * A minimum amount of work was done under global memory pressure. For -+ * kswapd, it may be overshooting. For direct reclaim, the target isn't -+ * met, and yet the allocation may still succeed, since kswapd may have -+ * caught up. In either case, it's better to stop now, and restart if -+ * necessary. -+ */ -+ for (i = 0; i <= sc->reclaim_idx; i++) { -+ unsigned long wmark; -+ struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; -+ -+ if (!managed_zone(zone)) -+ continue; -+ -+ wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); -+ if (wmark > zone_page_state(zone, NR_FREE_PAGES)) -+ return false; -+ } -+ -+ sc->nr_reclaimed += MIN_LRU_BATCH; -+ -+ return true; -+} -+ -+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ struct blk_plug plug; -+ bool need_aging = false; -+ bool need_swapping = false; -+ unsigned long scanned = 0; -+ unsigned long reclaimed = sc->nr_reclaimed; -+ DEFINE_MAX_SEQ(lruvec); -+ -+ lru_add_drain(); -+ -+ blk_start_plug(&plug); -+ -+ set_mm_walk(lruvec_pgdat(lruvec)); -+ -+ while (true) { -+ int delta; -+ int swappiness; -+ unsigned long nr_to_scan; -+ -+ if (sc->may_swap) -+ swappiness = get_swappiness(lruvec, sc); -+ else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) -+ swappiness = 1; -+ else -+ swappiness = 0; -+ -+ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); -+ if (!nr_to_scan) -+ goto done; -+ -+ delta = evict_folios(lruvec, sc, swappiness, &need_swapping); -+ if (!delta) -+ goto done; -+ -+ scanned += delta; -+ if (scanned >= nr_to_scan) -+ break; -+ -+ if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) -+ break; -+ -+ cond_resched(); -+ } -+ -+ /* see the comment in lru_gen_age_node() */ -+ if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) -+ sc->memcgs_need_aging = false; -+done: -+ clear_mm_walk(); -+ -+ blk_finish_plug(&plug); -+} -+ -+/****************************************************************************** -+ * state change -+ ******************************************************************************/ -+ -+static bool __maybe_unused state_is_valid(struct lruvec *lruvec) -+{ -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ if (lrugen->enabled) { -+ enum lru_list lru; -+ -+ for_each_evictable_lru(lru) { -+ if (!list_empty(&lruvec->lists[lru])) -+ return false; -+ } -+ } else { -+ int gen, type, zone; -+ -+ for_each_gen_type_zone(gen, type, zone) { -+ if (!list_empty(&lrugen->lists[gen][type][zone])) -+ return false; -+ } -+ } -+ -+ return true; -+} -+ -+static bool fill_evictable(struct lruvec *lruvec) -+{ -+ enum lru_list lru; -+ int remaining = MAX_LRU_BATCH; -+ -+ for_each_evictable_lru(lru) { -+ int type = is_file_lru(lru); -+ bool active = is_active_lru(lru); -+ struct list_head *head = &lruvec->lists[lru]; -+ -+ while (!list_empty(head)) { -+ bool success; -+ struct folio *folio = lru_to_folio(head); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); -+ -+ lruvec_del_folio(lruvec, folio); -+ success = lru_gen_add_folio(lruvec, folio, false); -+ VM_WARN_ON_ONCE(!success); -+ -+ if (!--remaining) -+ return false; -+ } -+ } -+ -+ return true; -+} -+ -+static bool drain_evictable(struct lruvec *lruvec) -+{ -+ int gen, type, zone; -+ int remaining = MAX_LRU_BATCH; -+ -+ for_each_gen_type_zone(gen, type, zone) { -+ struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; -+ -+ while (!list_empty(head)) { -+ bool success; -+ struct folio *folio = lru_to_folio(head); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); -+ -+ success = lru_gen_del_folio(lruvec, folio, false); -+ VM_WARN_ON_ONCE(!success); -+ lruvec_add_folio(lruvec, folio); -+ -+ if (!--remaining) -+ return false; -+ } -+ } -+ -+ return true; -+} -+ -+static void lru_gen_change_state(bool enabled) -+{ -+ static DEFINE_MUTEX(state_mutex); -+ -+ struct mem_cgroup *memcg; -+ -+ cgroup_lock(); -+ cpus_read_lock(); -+ get_online_mems(); -+ mutex_lock(&state_mutex); -+ -+ if (enabled == lru_gen_enabled()) -+ goto unlock; -+ -+ if (enabled) -+ static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); -+ else -+ static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); -+ -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ int nid; -+ -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ if (!lruvec) -+ continue; -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ VM_WARN_ON_ONCE(!state_is_valid(lruvec)); -+ -+ lruvec->lrugen.enabled = enabled; -+ -+ while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { -+ spin_unlock_irq(&lruvec->lru_lock); -+ cond_resched(); -+ spin_lock_irq(&lruvec->lru_lock); -+ } -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+ } -+ -+ cond_resched(); -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+unlock: -+ mutex_unlock(&state_mutex); -+ put_online_mems(); -+ cpus_read_unlock(); -+ cgroup_unlock(); -+} -+ -+/****************************************************************************** -+ * sysfs interface -+ ******************************************************************************/ -+ -+static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, -+ const char *buf, size_t len) -+{ -+ unsigned int msecs; -+ -+ if (kstrtouint(buf, 0, &msecs)) -+ return -EINVAL; -+ -+ WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); -+ -+ return len; -+} -+ -+static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( -+ min_ttl_ms, 0644, show_min_ttl, store_min_ttl -+); -+ -+static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -+{ -+ unsigned int caps = 0; -+ -+ if (get_cap(LRU_GEN_CORE)) -+ caps |= BIT(LRU_GEN_CORE); -+ -+ if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) -+ caps |= BIT(LRU_GEN_MM_WALK); -+ -+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) -+ caps |= BIT(LRU_GEN_NONLEAF_YOUNG); -+ -+ return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, -+ const char *buf, size_t len) -+{ -+ int i; -+ unsigned int caps; -+ -+ if (tolower(*buf) == 'n') -+ caps = 0; -+ else if (tolower(*buf) == 'y') -+ caps = -1; -+ else if (kstrtouint(buf, 0, &caps)) -+ return -EINVAL; -+ -+ for (i = 0; i < NR_LRU_GEN_CAPS; i++) { -+ bool enabled = caps & BIT(i); -+ -+ if (i == LRU_GEN_CORE) -+ lru_gen_change_state(enabled); -+ else if (enabled) -+ static_branch_enable(&lru_gen_caps[i]); -+ else -+ static_branch_disable(&lru_gen_caps[i]); -+ } -+ -+ return len; -+} -+ -+static struct kobj_attribute lru_gen_enabled_attr = __ATTR( -+ enabled, 0644, show_enabled, store_enabled -+); -+ -+static struct attribute *lru_gen_attrs[] = { -+ &lru_gen_min_ttl_attr.attr, -+ &lru_gen_enabled_attr.attr, -+ NULL -+}; -+ -+static struct attribute_group lru_gen_attr_group = { -+ .name = "lru_gen", -+ .attrs = lru_gen_attrs, -+}; -+ -+/****************************************************************************** -+ * debugfs interface -+ ******************************************************************************/ -+ -+static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) -+{ -+ struct mem_cgroup *memcg; -+ loff_t nr_to_skip = *pos; -+ -+ m->private = kvmalloc(PATH_MAX, GFP_KERNEL); -+ if (!m->private) -+ return ERR_PTR(-ENOMEM); -+ -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ int nid; -+ -+ for_each_node_state(nid, N_MEMORY) { -+ if (!nr_to_skip--) -+ return get_lruvec(memcg, nid); -+ } -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+ -+ return NULL; -+} -+ -+static void lru_gen_seq_stop(struct seq_file *m, void *v) -+{ -+ if (!IS_ERR_OR_NULL(v)) -+ mem_cgroup_iter_break(NULL, lruvec_memcg(v)); -+ -+ kvfree(m->private); -+ m->private = NULL; -+} -+ -+static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) -+{ -+ int nid = lruvec_pgdat(v)->node_id; -+ struct mem_cgroup *memcg = lruvec_memcg(v); -+ -+ ++*pos; -+ -+ nid = next_memory_node(nid); -+ if (nid == MAX_NUMNODES) { -+ memcg = mem_cgroup_iter(NULL, memcg, NULL); -+ if (!memcg) -+ return NULL; -+ -+ nid = first_memory_node; -+ } -+ -+ return get_lruvec(memcg, nid); -+} -+ -+static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, -+ unsigned long max_seq, unsigned long *min_seq, -+ unsigned long seq) -+{ -+ int i; -+ int type, tier; -+ int hist = lru_hist_from_seq(seq); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ for (tier = 0; tier < MAX_NR_TIERS; tier++) { -+ seq_printf(m, " %10d", tier); -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ const char *s = " "; -+ unsigned long n[3] = {}; -+ -+ if (seq == max_seq) { -+ s = "RT "; -+ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); -+ n[1] = READ_ONCE(lrugen->avg_total[type][tier]); -+ } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { -+ s = "rep"; -+ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); -+ } -+ -+ for (i = 0; i < 3; i++) -+ seq_printf(m, " %10lu%c", n[i], s[i]); -+ } -+ seq_putc(m, '\n'); -+ } -+ -+ seq_puts(m, " "); -+ for (i = 0; i < NR_MM_STATS; i++) { -+ const char *s = " "; -+ unsigned long n = 0; -+ -+ if (seq == max_seq && NR_HIST_GENS == 1) { -+ s = "LOYNFA"; -+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); -+ } else if (seq != max_seq && NR_HIST_GENS > 1) { -+ s = "loynfa"; -+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); -+ } -+ -+ seq_printf(m, " %10lu%c", n, s[i]); -+ } -+ seq_putc(m, '\n'); -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static int lru_gen_seq_show(struct seq_file *m, void *v) -+{ -+ unsigned long seq; -+ bool full = !debugfs_real_fops(m->file)->write; -+ struct lruvec *lruvec = v; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int nid = lruvec_pgdat(lruvec)->node_id; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (nid == first_memory_node) { -+ const char *path = memcg ? m->private : ""; -+ -+#ifdef CONFIG_MEMCG -+ if (memcg) -+ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); -+#endif -+ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); -+ } -+ -+ seq_printf(m, " node %5d\n", nid); -+ -+ if (!full) -+ seq = min_seq[LRU_GEN_ANON]; -+ else if (max_seq >= MAX_NR_GENS) -+ seq = max_seq - MAX_NR_GENS + 1; -+ else -+ seq = 0; -+ -+ for (; seq <= max_seq; seq++) { -+ int type, zone; -+ int gen = lru_gen_from_seq(seq); -+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); -+ -+ seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); -+ -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ unsigned long size = 0; -+ char mark = full && seq < min_seq[type] ? 'x' : ' '; -+ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) -+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); -+ -+ seq_printf(m, " %10lu%c", size, mark); -+ } -+ -+ seq_putc(m, '\n'); -+ -+ if (full) -+ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); -+ } -+ -+ return 0; -+} -+ -+static const struct seq_operations lru_gen_seq_ops = { -+ .start = lru_gen_seq_start, -+ .stop = lru_gen_seq_stop, -+ .next = lru_gen_seq_next, -+ .show = lru_gen_seq_show, -+}; -+ -+static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, -+ bool can_swap, bool force_scan) -+{ -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (seq < max_seq) -+ return 0; -+ -+ if (seq > max_seq) -+ return -EINVAL; -+ -+ if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) -+ return -ERANGE; -+ -+ try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); -+ -+ return 0; -+} -+ -+static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, -+ int swappiness, unsigned long nr_to_reclaim) -+{ -+ DEFINE_MAX_SEQ(lruvec); -+ -+ if (seq + MIN_NR_GENS > max_seq) -+ return -EINVAL; -+ -+ sc->nr_reclaimed = 0; -+ -+ while (!signal_pending(current)) { -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (seq < min_seq[!swappiness]) -+ return 0; -+ -+ if (sc->nr_reclaimed >= nr_to_reclaim) -+ return 0; -+ -+ if (!evict_folios(lruvec, sc, swappiness, NULL)) -+ return 0; -+ -+ cond_resched(); -+ } -+ -+ return -EINTR; -+} -+ -+static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, -+ struct scan_control *sc, int swappiness, unsigned long opt) -+{ -+ struct lruvec *lruvec; -+ int err = -EINVAL; -+ struct mem_cgroup *memcg = NULL; -+ -+ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) -+ return -EINVAL; -+ -+ if (!mem_cgroup_disabled()) { -+ rcu_read_lock(); -+ memcg = mem_cgroup_from_id(memcg_id); -+#ifdef CONFIG_MEMCG -+ if (memcg && !css_tryget(&memcg->css)) -+ memcg = NULL; -+#endif -+ rcu_read_unlock(); -+ -+ if (!memcg) -+ return -EINVAL; -+ } -+ -+ if (memcg_id != mem_cgroup_id(memcg)) -+ goto done; -+ -+ lruvec = get_lruvec(memcg, nid); -+ -+ if (swappiness < 0) -+ swappiness = get_swappiness(lruvec, sc); -+ else if (swappiness > 200) -+ goto done; -+ -+ switch (cmd) { -+ case '+': -+ err = run_aging(lruvec, seq, sc, swappiness, opt); -+ break; -+ case '-': -+ err = run_eviction(lruvec, seq, sc, swappiness, opt); -+ break; -+ } -+done: -+ mem_cgroup_put(memcg); -+ -+ return err; -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, -+ size_t len, loff_t *pos) -+{ -+ void *buf; -+ char *cur, *next; -+ unsigned int flags; -+ struct blk_plug plug; -+ int err = -EINVAL; -+ struct scan_control sc = { -+ .may_writepage = true, -+ .may_unmap = true, -+ .may_swap = true, -+ .reclaim_idx = MAX_NR_ZONES - 1, -+ .gfp_mask = GFP_KERNEL, -+ }; -+ -+ buf = kvmalloc(len + 1, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, src, len)) { -+ kvfree(buf); -+ return -EFAULT; -+ } -+ -+ set_task_reclaim_state(current, &sc.reclaim_state); -+ flags = memalloc_noreclaim_save(); -+ blk_start_plug(&plug); -+ if (!set_mm_walk(NULL)) { -+ err = -ENOMEM; -+ goto done; -+ } -+ -+ next = buf; -+ next[len] = '\0'; -+ -+ while ((cur = strsep(&next, ",;\n"))) { -+ int n; -+ int end; -+ char cmd; -+ unsigned int memcg_id; -+ unsigned int nid; -+ unsigned long seq; -+ unsigned int swappiness = -1; -+ unsigned long opt = -1; -+ -+ cur = skip_spaces(cur); -+ if (!*cur) -+ continue; -+ -+ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, -+ &seq, &end, &swappiness, &end, &opt, &end); -+ if (n < 4 || cur[end]) { -+ err = -EINVAL; -+ break; -+ } -+ -+ err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); -+ if (err) -+ break; -+ } -+done: -+ clear_mm_walk(); -+ blk_finish_plug(&plug); -+ memalloc_noreclaim_restore(flags); -+ set_task_reclaim_state(current, NULL); -+ -+ kvfree(buf); -+ -+ return err ? : len; -+} -+ -+static int lru_gen_seq_open(struct inode *inode, struct file *file) -+{ -+ return seq_open(file, &lru_gen_seq_ops); -+} -+ -+static const struct file_operations lru_gen_rw_fops = { -+ .open = lru_gen_seq_open, -+ .read = seq_read, -+ .write = lru_gen_seq_write, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+static const struct file_operations lru_gen_ro_fops = { -+ .open = lru_gen_seq_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+/****************************************************************************** -+ * initialization -+ ******************************************************************************/ -+ -+void lru_gen_init_lruvec(struct lruvec *lruvec) -+{ -+ int i; -+ int gen, type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ lrugen->max_seq = MIN_NR_GENS + 1; -+ lrugen->enabled = lru_gen_enabled(); -+ -+ for (i = 0; i <= MIN_NR_GENS + 1; i++) -+ lrugen->timestamps[i] = jiffies; -+ -+ for_each_gen_type_zone(gen, type, zone) -+ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); -+ -+ lruvec->mm_state.seq = MIN_NR_GENS; -+ init_waitqueue_head(&lruvec->mm_state.wait); -+} -+ -+#ifdef CONFIG_MEMCG -+void lru_gen_init_memcg(struct mem_cgroup *memcg) -+{ -+ INIT_LIST_HEAD(&memcg->mm_list.fifo); -+ spin_lock_init(&memcg->mm_list.lock); -+} -+ -+void lru_gen_exit_memcg(struct mem_cgroup *memcg) -+{ -+ int i; -+ int nid; -+ -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, -+ sizeof(lruvec->lrugen.nr_pages))); -+ -+ for (i = 0; i < NR_BLOOM_FILTERS; i++) { -+ bitmap_free(lruvec->mm_state.filters[i]); -+ lruvec->mm_state.filters[i] = NULL; -+ } -+ } -+} -+#endif -+ -+static int __init init_lru_gen(void) -+{ -+ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); -+ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); -+ -+ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) -+ pr_err("lru_gen: failed to create sysfs group\n"); -+ -+ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); -+ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); -+ -+ return 0; -+}; -+late_initcall(init_lru_gen); -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -+{ -+} -+ -+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ -+static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ unsigned long nr[NR_LRU_LISTS]; -+ unsigned long targets[NR_LRU_LISTS]; -+ unsigned long nr_to_scan; -+ enum lru_list lru; -+ unsigned long nr_reclaimed = 0; -+ unsigned long nr_to_reclaim = sc->nr_to_reclaim; -+ bool proportional_reclaim; -+ struct blk_plug plug; -+ -+ if (lru_gen_enabled()) { -+ lru_gen_shrink_lruvec(lruvec, sc); -+ return; -+ } -+ -+ get_scan_count(lruvec, sc, nr); -+ -+ /* Record the original scan target for proportional adjustments later */ -+ memcpy(targets, nr, sizeof(nr)); -+ -+ /* -+ * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal -+ * event that can occur when there is little memory pressure e.g. -+ * multiple streaming readers/writers. Hence, we do not abort scanning -+ * when the requested number of pages are reclaimed when scanning at -+ * DEF_PRIORITY on the assumption that the fact we are direct -+ * reclaiming implies that kswapd is not keeping up and it is best to -+ * do a batch of work at once. For memcg reclaim one check is made to -+ * abort proportional reclaim if either the file or anon lru has already -+ * dropped to zero at the first pass. -+ */ -+ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && -+ sc->priority == DEF_PRIORITY); -+ -+ blk_start_plug(&plug); -+ while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || -+ nr[LRU_INACTIVE_FILE]) { -+ unsigned long nr_anon, nr_file, percentage; -+ unsigned long nr_scanned; -+ -+ for_each_evictable_lru(lru) { -+ if (nr[lru]) { -+ nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); -+ nr[lru] -= nr_to_scan; -+ -+ nr_reclaimed += shrink_list(lru, nr_to_scan, -+ lruvec, sc); -+ } -+ } -+ -+ cond_resched(); -+ -+ if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) -+ continue; -+ -+ /* -+ * For kswapd and memcg, reclaim at least the number of pages -+ * requested. Ensure that the anon and file LRUs are scanned -+ * proportionally what was requested by get_scan_count(). We -+ * stop reclaiming one LRU and reduce the amount scanning -+ * proportional to the original scan target. -+ */ -+ nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; -+ nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; -+ -+ /* -+ * It's just vindictive to attack the larger once the smaller -+ * has gone to zero. And given the way we stop scanning the -+ * smaller below, this makes sure that we only make one nudge -+ * towards proportionality once we've got nr_to_reclaim. -+ */ -+ if (!nr_file || !nr_anon) -+ break; -+ -+ if (nr_file > nr_anon) { -+ unsigned long scan_target = targets[LRU_INACTIVE_ANON] + -+ targets[LRU_ACTIVE_ANON] + 1; -+ lru = LRU_BASE; -+ percentage = nr_anon * 100 / scan_target; -+ } else { -+ unsigned long scan_target = targets[LRU_INACTIVE_FILE] + -+ targets[LRU_ACTIVE_FILE] + 1; -+ lru = LRU_FILE; -+ percentage = nr_file * 100 / scan_target; -+ } -+ -+ /* Stop scanning the smaller of the LRU */ -+ nr[lru] = 0; -+ nr[lru + LRU_ACTIVE] = 0; -+ -+ /* -+ * Recalculate the other LRU scan count based on its original -+ * scan target and the percentage scanning already complete -+ */ -+ lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; -+ nr_scanned = targets[lru] - nr[lru]; -+ nr[lru] = targets[lru] * (100 - percentage) / 100; -+ nr[lru] -= min(nr[lru], nr_scanned); -+ -+ lru += LRU_ACTIVE; -+ nr_scanned = targets[lru] - nr[lru]; -+ nr[lru] = targets[lru] * (100 - percentage) / 100; -+ nr[lru] -= min(nr[lru], nr_scanned); -+ } -+ blk_finish_plug(&plug); -+ sc->nr_reclaimed += nr_reclaimed; -+ -+ /* -+ * Even if we did not try to evict anon pages at all, we want to -+ * rebalance the anon lru active/inactive ratio. -+ */ -+ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && -+ inactive_is_low(lruvec, LRU_INACTIVE_ANON)) -+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec, -+ sc, LRU_ACTIVE_ANON); -+} -+ -+/* Use reclaim/compaction for costly allocs or under memory pressure */ -+static bool in_reclaim_compaction(struct scan_control *sc) -+{ -+ if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && -+ (sc->order > PAGE_ALLOC_COSTLY_ORDER || -+ sc->priority < DEF_PRIORITY - 2)) -+ return true; -+ -+ return false; -+} -+ -+/* -+ * Reclaim/compaction is used for high-order allocation requests. It reclaims -+ * order-0 pages before compacting the zone. should_continue_reclaim() returns -+ * true if more pages should be reclaimed such that when the page allocator -+ * calls try_to_compact_pages() that it will have enough free pages to succeed. -+ * It will give up earlier than that if there is difficulty reclaiming pages. -+ */ -+static inline bool should_continue_reclaim(struct pglist_data *pgdat, -+ unsigned long nr_reclaimed, -+ struct scan_control *sc) -+{ -+ unsigned long pages_for_compaction; -+ unsigned long inactive_lru_pages; -+ int z; -+ -+ /* If not in reclaim/compaction mode, stop */ -+ if (!in_reclaim_compaction(sc)) -+ return false; -+ -+ /* -+ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX -+ * number of pages that were scanned. This will return to the caller -+ * with the risk reclaim/compaction and the resulting allocation attempt -+ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL -+ * allocations through requiring that the full LRU list has been scanned -+ * first, by assuming that zero delta of sc->nr_scanned means full LRU -+ * scan, but that approximation was wrong, and there were corner cases - * where always a non-zero amount of pages were scanned. - */ - if (!nr_reclaimed) -@@ -3201,109 +6071,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) - unsigned long nr_reclaimed, nr_scanned; - struct lruvec *target_lruvec; - bool reclaimable = false; -- unsigned long file; - - target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); - - again: -- /* -- * Flush the memory cgroup stats, so that we read accurate per-memcg -- * lruvec stats for heuristics. -- */ -- mem_cgroup_flush_stats(); -- - memset(&sc->nr, 0, sizeof(sc->nr)); - - nr_reclaimed = sc->nr_reclaimed; - nr_scanned = sc->nr_scanned; - -- /* -- * Determine the scan balance between anon and file LRUs. -- */ -- spin_lock_irq(&target_lruvec->lru_lock); -- sc->anon_cost = target_lruvec->anon_cost; -- sc->file_cost = target_lruvec->file_cost; -- spin_unlock_irq(&target_lruvec->lru_lock); -- -- /* -- * Target desirable inactive:active list ratios for the anon -- * and file LRU lists. -- */ -- if (!sc->force_deactivate) { -- unsigned long refaults; -- -- refaults = lruvec_page_state(target_lruvec, -- WORKINGSET_ACTIVATE_ANON); -- if (refaults != target_lruvec->refaults[0] || -- inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) -- sc->may_deactivate |= DEACTIVATE_ANON; -- else -- sc->may_deactivate &= ~DEACTIVATE_ANON; -- -- /* -- * When refaults are being observed, it means a new -- * workingset is being established. Deactivate to get -- * rid of any stale active pages quickly. -- */ -- refaults = lruvec_page_state(target_lruvec, -- WORKINGSET_ACTIVATE_FILE); -- if (refaults != target_lruvec->refaults[1] || -- inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) -- sc->may_deactivate |= DEACTIVATE_FILE; -- else -- sc->may_deactivate &= ~DEACTIVATE_FILE; -- } else -- sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; -- -- /* -- * If we have plenty of inactive file pages that aren't -- * thrashing, try to reclaim those first before touching -- * anonymous pages. -- */ -- file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); -- if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) -- sc->cache_trim_mode = 1; -- else -- sc->cache_trim_mode = 0; -- -- /* -- * Prevent the reclaimer from falling into the cache trap: as -- * cache pages start out inactive, every cache fault will tip -- * the scan balance towards the file LRU. And as the file LRU -- * shrinks, so does the window for rotation from references. -- * This means we have a runaway feedback loop where a tiny -- * thrashing file LRU becomes infinitely more attractive than -- * anon pages. Try to detect this based on file LRU size. -- */ -- if (!cgroup_reclaim(sc)) { -- unsigned long total_high_wmark = 0; -- unsigned long free, anon; -- int z; -- -- free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); -- file = node_page_state(pgdat, NR_ACTIVE_FILE) + -- node_page_state(pgdat, NR_INACTIVE_FILE); -- -- for (z = 0; z < MAX_NR_ZONES; z++) { -- struct zone *zone = &pgdat->node_zones[z]; -- if (!managed_zone(zone)) -- continue; -- -- total_high_wmark += high_wmark_pages(zone); -- } -- -- /* -- * Consider anon: if that's low too, this isn't a -- * runaway file reclaim problem, but rather just -- * extreme pressure. Reclaim as per usual then. -- */ -- anon = node_page_state(pgdat, NR_INACTIVE_ANON); -- -- sc->file_is_tiny = -- file + free <= total_high_wmark && -- !(sc->may_deactivate & DEACTIVATE_ANON) && -- anon >> sc->priority; -- } -+ prepare_scan_count(pgdat, sc); - - shrink_node_memcgs(pgdat, sc); - -@@ -3561,11 +6338,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) - struct lruvec *target_lruvec; - unsigned long refaults; - -+ if (lru_gen_enabled()) -+ return; -+ - target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); -- target_lruvec->refaults[0] = refaults; -+ target_lruvec->refaults[WORKINGSET_ANON] = refaults; - refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); -- target_lruvec->refaults[1] = refaults; -+ target_lruvec->refaults[WORKINGSET_FILE] = refaults; - } - - /* -@@ -3927,12 +6707,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - } - #endif - --static void age_active_anon(struct pglist_data *pgdat, -- struct scan_control *sc) -+static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) - { - struct mem_cgroup *memcg; - struct lruvec *lruvec; - -+ if (lru_gen_enabled()) { -+ lru_gen_age_node(pgdat, sc); -+ return; -+ } -+ - if (!can_age_anon_pages(pgdat, sc)) - return; - -@@ -4252,12 +7036,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) - sc.may_swap = !nr_boost_reclaim; - - /* -- * Do some background aging of the anon list, to give -- * pages a chance to be referenced before reclaiming. All -- * pages are rotated regardless of classzone as this is -- * about consistent aging. -+ * Do some background aging, to give pages a chance to be -+ * referenced before reclaiming. All pages are rotated -+ * regardless of classzone as this is about consistent aging. - */ -- age_active_anon(pgdat, &sc); -+ kswapd_age_node(pgdat, &sc); - - /* - * If we're getting trouble reclaiming, start doing writepage -diff --git a/mm/vmstat.c b/mm/vmstat.c -index 90af9a8572f5..4b85d3d5f7d7 100644 ---- a/mm/vmstat.c -+++ b/mm/vmstat.c -@@ -1389,10 +1389,6 @@ const char * const vmstat_text[] = { - "nr_tlb_local_flush_one", - #endif /* CONFIG_DEBUG_TLBFLUSH */ - --#ifdef CONFIG_DEBUG_VM_VMACACHE -- "vmacache_find_calls", -- "vmacache_find_hits", --#endif - #ifdef CONFIG_SWAP - "swap_ra", - "swap_ra_hit", -diff --git a/mm/workingset.c b/mm/workingset.c -index a5e84862fc86..ae7e984b23c6 100644 ---- a/mm/workingset.c -+++ b/mm/workingset.c -@@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; - static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, - bool workingset) - { -- eviction >>= bucket_order; - eviction &= EVICTION_MASK; - eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; - eviction = (eviction << NODES_SHIFT) | pgdat->node_id; -@@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, - - *memcgidp = memcgid; - *pgdat = NODE_DATA(nid); -- *evictionp = entry << bucket_order; -+ *evictionp = entry; - *workingsetp = workingset; - } - -+#ifdef CONFIG_LRU_GEN -+ -+static void *lru_gen_eviction(struct folio *folio) -+{ -+ int hist; -+ unsigned long token; -+ unsigned long min_seq; -+ struct lruvec *lruvec; -+ struct lru_gen_struct *lrugen; -+ int type = folio_is_file_lru(folio); -+ int delta = folio_nr_pages(folio); -+ int refs = folio_lru_refs(folio); -+ int tier = lru_tier_from_refs(refs); -+ struct mem_cgroup *memcg = folio_memcg(folio); -+ struct pglist_data *pgdat = folio_pgdat(folio); -+ -+ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); -+ -+ lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ lrugen = &lruvec->lrugen; -+ min_seq = READ_ONCE(lrugen->min_seq[type]); -+ token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); -+ -+ hist = lru_hist_from_seq(min_seq); -+ atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); -+ -+ return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); -+} -+ -+static void lru_gen_refault(struct folio *folio, void *shadow) -+{ -+ int hist, tier, refs; -+ int memcg_id; -+ bool workingset; -+ unsigned long token; -+ unsigned long min_seq; -+ struct lruvec *lruvec; -+ struct lru_gen_struct *lrugen; -+ struct mem_cgroup *memcg; -+ struct pglist_data *pgdat; -+ int type = folio_is_file_lru(folio); -+ int delta = folio_nr_pages(folio); -+ -+ unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); -+ -+ if (pgdat != folio_pgdat(folio)) -+ return; -+ -+ rcu_read_lock(); -+ -+ memcg = folio_memcg_rcu(folio); -+ if (memcg_id != mem_cgroup_id(memcg)) -+ goto unlock; -+ -+ lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ lrugen = &lruvec->lrugen; -+ -+ min_seq = READ_ONCE(lrugen->min_seq[type]); -+ if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) -+ goto unlock; -+ -+ hist = lru_hist_from_seq(min_seq); -+ /* see the comment in folio_lru_refs() */ -+ refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; -+ tier = lru_tier_from_refs(refs); -+ -+ atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); -+ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); -+ -+ /* -+ * Count the following two cases as stalls: -+ * 1. For pages accessed through page tables, hotter pages pushed out -+ * hot pages which refaulted immediately. -+ * 2. For pages accessed multiple times through file descriptors, -+ * numbers of accesses might have been out of the range. -+ */ -+ if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { -+ folio_set_workingset(folio); -+ mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); -+ } -+unlock: -+ rcu_read_unlock(); -+} -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static void *lru_gen_eviction(struct folio *folio) -+{ -+ return NULL; -+} -+ -+static void lru_gen_refault(struct folio *folio, void *shadow) -+{ -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ - /** - * workingset_age_nonresident - age non-resident entries as LRU ages - * @lruvec: the lruvec that was aged -@@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) - VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); - VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); - -+ if (lru_gen_enabled()) -+ return lru_gen_eviction(folio); -+ - lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - /* XXX: target_memcg can be NULL, go through lruvec */ - memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); - eviction = atomic_long_read(&lruvec->nonresident_age); -+ eviction >>= bucket_order; - workingset_age_nonresident(lruvec, folio_nr_pages(folio)); - return pack_shadow(memcgid, pgdat, eviction, - folio_test_workingset(folio)); -@@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) - int memcgid; - long nr; - -+ if (lru_gen_enabled()) { -+ lru_gen_refault(folio, shadow); -+ return; -+ } -+ - unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); -+ eviction <<= bucket_order; - - rcu_read_lock(); - /* -diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h -index 0616409513eb..311759ea25e9 100644 ---- a/tools/include/linux/slab.h -+++ b/tools/include/linux/slab.h -@@ -41,4 +41,8 @@ struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, - unsigned int align, unsigned int flags, - void (*ctor)(void *)); - -+void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); -+int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, -+ void **list); -+ - #endif /* _TOOLS_SLAB_H */ -diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h -index 6c1aa92a92e4..6ce1f1ceb432 100644 ---- a/tools/include/uapi/asm-generic/mman-common.h -+++ b/tools/include/uapi/asm-generic/mman-common.h -@@ -77,6 +77,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore -index d971516401e6..c901d96dd013 100644 ---- a/tools/testing/radix-tree/.gitignore -+++ b/tools/testing/radix-tree/.gitignore -@@ -6,3 +6,5 @@ main - multiorder - radix-tree.c - xarray -+maple -+ma_xa_benchmark -diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile -index c4ea4fbb0bfc..89d613e0505b 100644 ---- a/tools/testing/radix-tree/Makefile -+++ b/tools/testing/radix-tree/Makefile -@@ -4,9 +4,9 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ - -fsanitize=undefined - LDFLAGS += -fsanitize=address -fsanitize=undefined - LDLIBS+= -lpthread -lurcu --TARGETS = main idr-test multiorder xarray -+TARGETS = main idr-test multiorder xarray maple - CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \ -- slab.o -+ slab.o maple.o - OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ - regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ - iteration_check_2.o benchmark.o -@@ -29,6 +29,8 @@ idr-test: idr-test.o $(CORE_OFILES) - - xarray: $(CORE_OFILES) - -+maple: $(CORE_OFILES) -+ - multiorder: multiorder.o $(CORE_OFILES) - - clean: -@@ -40,6 +42,7 @@ $(OFILES): Makefile *.h */*.h generated/map-shift.h \ - ../../include/linux/*.h \ - ../../include/asm/*.h \ - ../../../include/linux/xarray.h \ -+ ../../../include/linux/maple_tree.h \ - ../../../include/linux/radix-tree.h \ - ../../../include/linux/idr.h - -@@ -51,6 +54,8 @@ idr.c: ../../../lib/idr.c - - xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c - -+maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c -+ - generated/map-shift.h: - @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ - echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ -diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h -index 2218b3cc184e..e7da80350236 100644 ---- a/tools/testing/radix-tree/generated/autoconf.h -+++ b/tools/testing/radix-tree/generated/autoconf.h -@@ -1 +1,2 @@ - #define CONFIG_XARRAY_MULTI 1 -+#define CONFIG_64BIT 1 -diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c -index d5c1bcba86fe..2048d12c31df 100644 ---- a/tools/testing/radix-tree/linux.c -+++ b/tools/testing/radix-tree/linux.c -@@ -23,15 +23,47 @@ struct kmem_cache { - int nr_objs; - void *objs; - void (*ctor)(void *); -+ unsigned int non_kernel; -+ unsigned long nr_allocated; -+ unsigned long nr_tallocated; - }; - -+void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) -+{ -+ cachep->non_kernel = val; -+} -+ -+unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) -+{ -+ return cachep->size * cachep->nr_allocated; -+} -+ -+unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) -+{ -+ return cachep->nr_allocated; -+} -+ -+unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) -+{ -+ return cachep->nr_tallocated; -+} -+ -+void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) -+{ -+ cachep->nr_tallocated = 0; -+} -+ - void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, - int gfp) - { - void *p; - -- if (!(gfp & __GFP_DIRECT_RECLAIM)) -- return NULL; -+ if (!(gfp & __GFP_DIRECT_RECLAIM)) { -+ if (!cachep->non_kernel) -+ return NULL; -+ -+ cachep->non_kernel--; -+ } - - pthread_mutex_lock(&cachep->lock); - if (cachep->nr_objs) { -@@ -53,19 +85,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, - memset(p, 0, cachep->size); - } - -+ uatomic_inc(&cachep->nr_allocated); - uatomic_inc(&nr_allocated); -+ uatomic_inc(&cachep->nr_tallocated); - if (kmalloc_verbose) - printf("Allocating %p from slab\n", p); - return p; - } - --void kmem_cache_free(struct kmem_cache *cachep, void *objp) -+void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) - { - assert(objp); - uatomic_dec(&nr_allocated); -+ uatomic_dec(&cachep->nr_allocated); - if (kmalloc_verbose) - printf("Freeing %p to slab\n", objp); -- pthread_mutex_lock(&cachep->lock); - if (cachep->nr_objs > 10 || cachep->align) { - memset(objp, POISON_FREE, cachep->size); - free(objp); -@@ -75,9 +109,80 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) - node->parent = cachep->objs; - cachep->objs = node; - } -+} -+ -+void kmem_cache_free(struct kmem_cache *cachep, void *objp) -+{ -+ pthread_mutex_lock(&cachep->lock); -+ kmem_cache_free_locked(cachep, objp); - pthread_mutex_unlock(&cachep->lock); - } - -+void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) -+{ -+ if (kmalloc_verbose) -+ pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); -+ -+ pthread_mutex_lock(&cachep->lock); -+ for (int i = 0; i < size; i++) -+ kmem_cache_free_locked(cachep, list[i]); -+ pthread_mutex_unlock(&cachep->lock); -+} -+ -+int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, -+ void **p) -+{ -+ size_t i; ++ size_t i; + + if (kmalloc_verbose) + pr_debug("Bulk alloc %lu\n", size); @@ -60821,929 +53828,6 @@ index 000000000000..97d0e1ddcf08 +#define trace_ma_op(a, b) do {} while (0) +#define trace_ma_read(a, b) do {} while (0) +#define trace_ma_write(a, b, c, d) do {} while (0) -diff --git a/tools/testing/selftests/vm/khugepaged.c b/tools/testing/selftests/vm/khugepaged.c -index 155120b67a16..b77b1e28cdb3 100644 ---- a/tools/testing/selftests/vm/khugepaged.c -+++ b/tools/testing/selftests/vm/khugepaged.c -@@ -14,6 +14,9 @@ - #ifndef MADV_PAGEOUT - #define MADV_PAGEOUT 21 - #endif -+#ifndef MADV_COLLAPSE -+#define MADV_COLLAPSE 25 -+#endif - - #define BASE_ADDR ((void *)(1UL << 30)) - static unsigned long hpage_pmd_size; -@@ -23,6 +26,11 @@ static int hpage_pmd_nr; - #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" - #define PID_SMAPS "/proc/self/smaps" - -+struct collapse_context { -+ void (*collapse)(const char *msg, char *p, int nr_hpages, bool expect); -+ bool enforce_pte_scan_limits; -+}; -+ - enum thp_enabled { - THP_ALWAYS, - THP_MADVISE, -@@ -90,18 +98,6 @@ struct settings { - struct khugepaged_settings khugepaged; - }; - --static struct settings default_settings = { -- .thp_enabled = THP_MADVISE, -- .thp_defrag = THP_DEFRAG_ALWAYS, -- .shmem_enabled = SHMEM_NEVER, -- .use_zero_page = 0, -- .khugepaged = { -- .defrag = 1, -- .alloc_sleep_millisecs = 10, -- .scan_sleep_millisecs = 10, -- }, --}; -- - static struct settings saved_settings; - static bool skip_settings_restore; - -@@ -279,6 +275,39 @@ static void write_settings(struct settings *settings) - write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); - } - -+#define MAX_SETTINGS_DEPTH 4 -+static struct settings settings_stack[MAX_SETTINGS_DEPTH]; -+static int settings_index; -+ -+static struct settings *current_settings(void) -+{ -+ if (!settings_index) { -+ printf("Fail: No settings set"); -+ exit(EXIT_FAILURE); -+ } -+ return settings_stack + settings_index - 1; -+} -+ -+static void push_settings(struct settings *settings) -+{ -+ if (settings_index >= MAX_SETTINGS_DEPTH) { -+ printf("Fail: Settings stack exceeded"); -+ exit(EXIT_FAILURE); -+ } -+ settings_stack[settings_index++] = *settings; -+ write_settings(current_settings()); -+} -+ -+static void pop_settings(void) -+{ -+ if (settings_index <= 0) { -+ printf("Fail: Settings stack empty"); -+ exit(EXIT_FAILURE); -+ } -+ --settings_index; -+ write_settings(current_settings()); -+} -+ - static void restore_settings(int sig) - { - if (skip_settings_restore) -@@ -322,14 +351,6 @@ static void save_settings(void) - signal(SIGQUIT, restore_settings); - } - --static void adjust_settings(void) --{ -- -- printf("Adjust settings..."); -- write_settings(&default_settings); -- success("OK"); --} -- - #define MAX_LINE_LENGTH 500 - - static bool check_for_pattern(FILE *fp, char *pattern, char *buf) -@@ -341,7 +362,7 @@ static bool check_for_pattern(FILE *fp, char *pattern, char *buf) - return false; - } - --static bool check_huge(void *addr) -+static bool check_huge(void *addr, int nr_hpages) - { - bool thp = false; - int ret; -@@ -366,7 +387,7 @@ static bool check_huge(void *addr) - goto err_out; - - ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB", -- hpage_pmd_size >> 10); -+ nr_hpages * (hpage_pmd_size >> 10)); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); - exit(EXIT_FAILURE); -@@ -434,12 +455,12 @@ static bool check_swap(void *addr, unsigned long size) - return swap; - } - --static void *alloc_mapping(void) -+static void *alloc_mapping(int nr) - { - void *p; - -- p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, -- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); -+ p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, -+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (p != BASE_ADDR) { - printf("Failed to allocate VMA at %p\n", BASE_ADDR); - exit(EXIT_FAILURE); -@@ -456,6 +477,25 @@ static void fill_memory(int *p, unsigned long start, unsigned long end) - p[i * page_size / sizeof(*p)] = i + 0xdead0000; - } - -+/* -+ * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with -+ * validate_memory()'able contents. -+ */ -+static void *alloc_hpage(void) -+{ -+ void *p; -+ -+ p = alloc_mapping(1); -+ printf("Allocate huge page..."); -+ madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -+ fill_memory(p, 0, hpage_pmd_size); -+ if (check_huge(p, 1)) -+ success("OK"); -+ else -+ fail("Fail"); -+ return p; -+} -+ - static void validate_memory(int *p, unsigned long start, unsigned long end) - { - int i; -@@ -469,26 +509,59 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) - } - } - -+static void madvise_collapse(const char *msg, char *p, int nr_hpages, -+ bool expect) -+{ -+ int ret; -+ struct settings settings = *current_settings(); -+ -+ printf("%s...", msg); -+ /* Sanity check */ -+ if (!check_huge(p, 0)) { -+ printf("Unexpected huge page\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ /* -+ * Prevent khugepaged interference and tests that MADV_COLLAPSE -+ * ignores /sys/kernel/mm/transparent_hugepage/enabled -+ */ -+ settings.thp_enabled = THP_NEVER; -+ push_settings(&settings); -+ -+ /* Clear VM_NOHUGEPAGE */ -+ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); -+ ret = madvise(p, nr_hpages * hpage_pmd_size, MADV_COLLAPSE); -+ if (((bool)ret) == expect) -+ fail("Fail: Bad return value"); -+ else if (check_huge(p, nr_hpages) != expect) -+ fail("Fail: check_huge()"); -+ else -+ success("OK"); -+ -+ pop_settings(); -+} -+ - #define TICK 500000 --static bool wait_for_scan(const char *msg, char *p) -+static bool wait_for_scan(const char *msg, char *p, int nr_hpages) - { - int full_scans; - int timeout = 6; /* 3 seconds */ - - /* Sanity check */ -- if (check_huge(p)) { -+ if (!check_huge(p, 0)) { - printf("Unexpected huge page\n"); - exit(EXIT_FAILURE); - } - -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -+ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); - - /* Wait until the second full_scan completed */ - full_scans = read_num("khugepaged/full_scans") + 2; - - printf("%s...", msg); - while (timeout--) { -- if (check_huge(p)) -+ if (check_huge(p, nr_hpages)) - break; - if (read_num("khugepaged/full_scans") >= full_scans) - break; -@@ -496,121 +569,121 @@ static bool wait_for_scan(const char *msg, char *p) - usleep(TICK); - } - -- madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -+ madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); - - return timeout == -1; - } - -+static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, -+ bool expect) -+{ -+ if (wait_for_scan(msg, p, nr_hpages)) { -+ if (expect) -+ fail("Timeout"); -+ else -+ success("OK"); -+ return; -+ } else if (check_huge(p, nr_hpages) == expect) { -+ success("OK"); -+ } else { -+ fail("Fail"); -+ } -+} -+ - static void alloc_at_fault(void) - { -- struct settings settings = default_settings; -+ struct settings settings = *current_settings(); - char *p; - - settings.thp_enabled = THP_ALWAYS; -- write_settings(&settings); -+ push_settings(&settings); - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - *p = 1; - printf("Allocate huge page on fault..."); -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); - -- write_settings(&default_settings); -+ pop_settings(); - - madvise(p, page_size, MADV_DONTNEED); - printf("Split huge PMD on MADV_DONTNEED..."); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - munmap(p, hpage_pmd_size); - } - --static void collapse_full(void) -+static void collapse_full(struct collapse_context *c) - { - void *p; -- -- p = alloc_mapping(); -- fill_memory(p, 0, hpage_pmd_size); -- if (wait_for_scan("Collapse fully populated PTE table", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- validate_memory(p, 0, hpage_pmd_size); -- munmap(p, hpage_pmd_size); -+ int nr_hpages = 4; -+ unsigned long size = nr_hpages * hpage_pmd_size; -+ -+ p = alloc_mapping(nr_hpages); -+ fill_memory(p, 0, size); -+ c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, -+ true); -+ validate_memory(p, 0, size); -+ munmap(p, size); - } - --static void collapse_empty(void) -+static void collapse_empty(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -- if (wait_for_scan("Do not collapse empty PTE table", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- fail("Fail"); -- else -- success("OK"); -+ p = alloc_mapping(1); -+ c->collapse("Do not collapse empty PTE table", p, 1, false); - munmap(p, hpage_pmd_size); - } - --static void collapse_single_pte_entry(void) -+static void collapse_single_pte_entry(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - fill_memory(p, 0, page_size); -- if (wait_for_scan("Collapse PTE table with single PTE entry present", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table with single PTE entry present", p, -+ 1, true); - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_max_ptes_none(void) -+static void collapse_max_ptes_none(struct collapse_context *c) - { - int max_ptes_none = hpage_pmd_nr / 2; -- struct settings settings = default_settings; -+ struct settings settings = *current_settings(); - void *p; - - settings.khugepaged.max_ptes_none = max_ptes_none; -- write_settings(&settings); -+ push_settings(&settings); - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); -- if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- fail("Fail"); -- else -- success("OK"); -+ c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, -+ !c->enforce_pte_scan_limits); - validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); - -- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); -- if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); -+ if (c->enforce_pte_scan_limits) { -+ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); -+ c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, -+ true); -+ validate_memory(p, 0, -+ (hpage_pmd_nr - max_ptes_none) * page_size); -+ } - - munmap(p, hpage_pmd_size); -- write_settings(&default_settings); -+ pop_settings(); - } - --static void collapse_swapin_single_pte(void) -+static void collapse_swapin_single_pte(struct collapse_context *c) - { - void *p; -- p = alloc_mapping(); -+ p = alloc_mapping(1); - fill_memory(p, 0, hpage_pmd_size); - - printf("Swapout one page..."); -@@ -625,23 +698,18 @@ static void collapse_swapin_single_pte(void) - goto out; - } - -- if (wait_for_scan("Collapse with swapping in single PTE entry", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse with swapping in single PTE entry", p, 1, true); - validate_memory(p, 0, hpage_pmd_size); - out: - munmap(p, hpage_pmd_size); - } - --static void collapse_max_ptes_swap(void) -+static void collapse_max_ptes_swap(struct collapse_context *c) - { - int max_ptes_swap = read_num("khugepaged/max_ptes_swap"); - void *p; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - - fill_memory(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); -@@ -656,115 +724,83 @@ static void collapse_max_ptes_swap(void) - goto out; - } - -- if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- fail("Fail"); -- else -- success("OK"); -+ c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, -+ !c->enforce_pte_scan_limits); - validate_memory(p, 0, hpage_pmd_size); - -- fill_memory(p, 0, hpage_pmd_size); -- printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); -- if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { -- perror("madvise(MADV_PAGEOUT)"); -- exit(EXIT_FAILURE); -- } -- if (check_swap(p, max_ptes_swap * page_size)) { -- success("OK"); -- } else { -- fail("Fail"); -- goto out; -- } -+ if (c->enforce_pte_scan_limits) { -+ fill_memory(p, 0, hpage_pmd_size); -+ printf("Swapout %d of %d pages...", max_ptes_swap, -+ hpage_pmd_nr); -+ if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { -+ perror("madvise(MADV_PAGEOUT)"); -+ exit(EXIT_FAILURE); -+ } -+ if (check_swap(p, max_ptes_swap * page_size)) { -+ success("OK"); -+ } else { -+ fail("Fail"); -+ goto out; -+ } - -- if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- validate_memory(p, 0, hpage_pmd_size); -+ c->collapse("Collapse with max_ptes_swap pages swapped out", p, -+ 1, true); -+ validate_memory(p, 0, hpage_pmd_size); -+ } - out: - munmap(p, hpage_pmd_size); - } - --static void collapse_single_pte_entry_compound(void) -+static void collapse_single_pte_entry_compound(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ p = alloc_hpage(); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -- - printf("Split huge page leaving single PTE mapping compound page..."); - madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table with single PTE mapping compound page", -+ p, 1, true); - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_full_of_compound(void) -+static void collapse_full_of_compound(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -+ p = alloc_hpage(); - printf("Split huge page leaving single PTE page table full of compound pages..."); - madvise(p, page_size, MADV_NOHUGEPAGE); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Collapse PTE table full of compound pages", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table full of compound pages", p, 1, true); - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_compound_extreme(void) -+static void collapse_compound_extreme(struct collapse_context *c) - { - void *p; - int i; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - for (i = 0; i < hpage_pmd_nr; i++) { - printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", - i + 1, hpage_pmd_nr); - - madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(BASE_ADDR, 0, hpage_pmd_size); -- if (!check_huge(BASE_ADDR)) { -+ if (!check_huge(BASE_ADDR, 1)) { - printf("Failed to allocate huge page\n"); - exit(EXIT_FAILURE); - } -@@ -793,32 +829,28 @@ static void collapse_compound_extreme(void) - - munmap(BASE_ADDR, hpage_pmd_size); - fill_memory(p, 0, hpage_pmd_size); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Collapse PTE table full of different compound pages", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table full of different compound pages", p, 1, -+ true); - - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_fork(void) -+static void collapse_fork(struct collapse_context *c) - { - int wstatus; - void *p; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - - printf("Allocate small page..."); - fill_memory(p, 0, page_size); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); -@@ -829,19 +861,14 @@ static void collapse_fork(void) - skip_settings_restore = true; - exit_status = 0; - -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - - fill_memory(p, page_size, 2 * page_size); -- -- if (wait_for_scan("Collapse PTE table with single page shared with parent process", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table with single page shared with parent process", -+ p, 1, true); - - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); -@@ -852,7 +879,7 @@ static void collapse_fork(void) - exit_status += WEXITSTATUS(wstatus); - - printf("Check if parent still has small page..."); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); -@@ -860,28 +887,19 @@ static void collapse_fork(void) - munmap(p, hpage_pmd_size); - } - --static void collapse_fork_compound(void) -+static void collapse_fork_compound(struct collapse_context *c) - { - int wstatus; - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -+ p = alloc_hpage(); - printf("Share huge page over fork()..."); - if (!fork()) { - /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -889,21 +907,17 @@ static void collapse_fork_compound(void) - printf("Split huge page PMD in child process..."); - madvise(p, page_size, MADV_NOHUGEPAGE); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - fill_memory(p, 0, page_size); - - write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); -- if (wait_for_scan("Collapse PTE table full of compound pages in child", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table full of compound pages in child", -+ p, 1, true); - write_num("khugepaged/max_ptes_shared", -- default_settings.khugepaged.max_ptes_shared); -+ current_settings()->khugepaged.max_ptes_shared); - - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); -@@ -914,7 +928,7 @@ static void collapse_fork_compound(void) - exit_status += WEXITSTATUS(wstatus); - - printf("Check if parent still has huge page..."); -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -922,29 +936,20 @@ static void collapse_fork_compound(void) - munmap(p, hpage_pmd_size); - } - --static void collapse_max_ptes_shared() -+static void collapse_max_ptes_shared(struct collapse_context *c) - { - int max_ptes_shared = read_num("khugepaged/max_ptes_shared"); - int wstatus; - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -+ p = alloc_hpage(); - printf("Share huge page over fork()..."); - if (!fork()) { - /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -952,33 +957,27 @@ static void collapse_max_ptes_shared() - printf("Trigger CoW on page %d of %d...", - hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p)) -- fail("Timeout"); -- else if (!check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -- printf("Trigger CoW on page %d of %d...", -- hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); -- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); -- if (!check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -- -- if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Maybe collapse with max_ptes_shared exceeded", p, -+ 1, !c->enforce_pte_scan_limits); -+ -+ if (c->enforce_pte_scan_limits) { -+ printf("Trigger CoW on page %d of %d...", -+ hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); -+ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * -+ page_size); -+ if (check_huge(p, 0)) -+ success("OK"); -+ else -+ fail("Fail"); -+ -+ c->collapse("Collapse with max_ptes_shared PTEs shared", -+ p, 1, true); -+ } - - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); -@@ -989,7 +988,7 @@ static void collapse_max_ptes_shared() - exit_status += WEXITSTATUS(wstatus); - - printf("Check if parent still has huge page..."); -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -997,8 +996,52 @@ static void collapse_max_ptes_shared() - munmap(p, hpage_pmd_size); - } - --int main(void) -+static void madvise_collapse_existing_thps(void) - { -+ void *p; -+ int err; -+ -+ p = alloc_mapping(1); -+ fill_memory(p, 0, hpage_pmd_size); -+ -+ printf("Collapse fully populated PTE table..."); -+ /* -+ * Note that we don't set MADV_HUGEPAGE here, which -+ * also tests that VM_HUGEPAGE isn't required for -+ * MADV_COLLAPSE in "madvise" mode. -+ */ -+ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); -+ if (err == 0 && check_huge(p, 1)) { -+ success("OK"); -+ printf("Re-collapse PMD-mapped hugepage"); -+ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); -+ if (err == 0 && check_huge(p, 1)) -+ success("OK"); -+ else -+ fail("Fail"); -+ } else { -+ fail("Fail"); -+ } -+ validate_memory(p, 0, hpage_pmd_size); -+ munmap(p, hpage_pmd_size); -+} -+ -+int main(int argc, const char **argv) -+{ -+ struct collapse_context c; -+ struct settings default_settings = { -+ .thp_enabled = THP_MADVISE, -+ .thp_defrag = THP_DEFRAG_ALWAYS, -+ .shmem_enabled = SHMEM_NEVER, -+ .use_zero_page = 0, -+ .khugepaged = { -+ .defrag = 1, -+ .alloc_sleep_millisecs = 10, -+ .scan_sleep_millisecs = 10, -+ }, -+ }; -+ const char *tests = argc == 1 ? "all" : argv[1]; -+ - setbuf(stdout, NULL); - - page_size = getpagesize(); -@@ -1011,21 +1054,47 @@ int main(void) - default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8; - - save_settings(); -- adjust_settings(); -+ push_settings(&default_settings); - - alloc_at_fault(); -- collapse_full(); -- collapse_empty(); -- collapse_single_pte_entry(); -- collapse_max_ptes_none(); -- collapse_swapin_single_pte(); -- collapse_max_ptes_swap(); -- collapse_single_pte_entry_compound(); -- collapse_full_of_compound(); -- collapse_compound_extreme(); -- collapse_fork(); -- collapse_fork_compound(); -- collapse_max_ptes_shared(); -+ -+ if (!strcmp(tests, "khugepaged") || !strcmp(tests, "all")) { -+ printf("\n*** Testing context: khugepaged ***\n"); -+ c.collapse = &khugepaged_collapse; -+ c.enforce_pte_scan_limits = true; -+ -+ collapse_full(&c); -+ collapse_empty(&c); -+ collapse_single_pte_entry(&c); -+ collapse_max_ptes_none(&c); -+ collapse_swapin_single_pte(&c); -+ collapse_max_ptes_swap(&c); -+ collapse_single_pte_entry_compound(&c); -+ collapse_full_of_compound(&c); -+ collapse_compound_extreme(&c); -+ collapse_fork(&c); -+ collapse_fork_compound(&c); -+ collapse_max_ptes_shared(&c); -+ } -+ if (!strcmp(tests, "madvise") || !strcmp(tests, "all")) { -+ printf("\n*** Testing context: madvise ***\n"); -+ c.collapse = &madvise_collapse; -+ c.enforce_pte_scan_limits = false; -+ -+ collapse_full(&c); -+ collapse_empty(&c); -+ collapse_single_pte_entry(&c); -+ collapse_max_ptes_none(&c); -+ collapse_swapin_single_pte(&c); -+ collapse_max_ptes_swap(&c); -+ collapse_single_pte_entry_compound(&c); -+ collapse_full_of_compound(&c); -+ collapse_compound_extreme(&c); -+ collapse_fork(&c); -+ collapse_fork_compound(&c); -+ collapse_max_ptes_shared(&c); -+ madvise_collapse_existing_thps(); -+ } - - restore_settings(0); - } -- 2.38.0.rc1.8.g2a7d63a245 diff --git a/6.0/0008-rcu.patch b/6.0/0008-rcu.patch deleted file mode 100644 index f4427f3e..00000000 --- a/6.0/0008-rcu.patch +++ /dev/null @@ -1,74 +0,0 @@ -From 13477a1f4474e7a9cf46017e0f8081dd095433bf Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Wed, 14 Sep 2022 14:40:34 +0200 -Subject: [PATCH 08/13] rcu - -Signed-off-by: Peter Jung ---- - kernel/rcu/tree_nocb.h | 34 +++++++++++----------------------- - 1 file changed, 11 insertions(+), 23 deletions(-) - -diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h -index a8f574d8850d..4017ebecec91 100644 ---- a/kernel/rcu/tree_nocb.h -+++ b/kernel/rcu/tree_nocb.h -@@ -1210,45 +1210,33 @@ EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload); - void __init rcu_init_nohz(void) - { - int cpu; -- bool need_rcu_nocb_mask = false; -- bool offload_all = false; - struct rcu_data *rdp; -- --#if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) -- if (!rcu_state.nocb_is_setup) { -- need_rcu_nocb_mask = true; -- offload_all = true; -- } --#endif /* #if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) */ -+ const struct cpumask *cpumask = NULL; - - #if defined(CONFIG_NO_HZ_FULL) -- if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) { -- need_rcu_nocb_mask = true; -- offload_all = false; /* NO_HZ_FULL has its own mask. */ -- } --#endif /* #if defined(CONFIG_NO_HZ_FULL) */ -+ if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) -+ cpumask = tick_nohz_full_mask; -+#endif -+ -+ if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) && -+ !rcu_state.nocb_is_setup && !cpumask) -+ cpumask = cpu_possible_mask; - -- if (need_rcu_nocb_mask) { -+ if (cpumask) { - if (!cpumask_available(rcu_nocb_mask)) { - if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) { - pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n"); - return; - } - } -+ -+ cpumask_or(rcu_nocb_mask, rcu_nocb_mask, cpumask); - rcu_state.nocb_is_setup = true; - } - - if (!rcu_state.nocb_is_setup) - return; - --#if defined(CONFIG_NO_HZ_FULL) -- if (tick_nohz_full_running) -- cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask); --#endif /* #if defined(CONFIG_NO_HZ_FULL) */ -- -- if (offload_all) -- cpumask_setall(rcu_nocb_mask); -- - if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) { - pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n"); - cpumask_and(rcu_nocb_mask, cpu_possible_mask, --- -2.37.3 - diff --git a/6.0/0009-lrng.patch b/6.0/0009-lrng.patch deleted file mode 100644 index 67f9e1d8..00000000 --- a/6.0/0009-lrng.patch +++ /dev/null @@ -1,10599 +0,0 @@ -From 0499cabe43b65283662b91133be9c798aa6ebaff Mon Sep 17 00:00:00 2001 -From: Piotr Gorski -Date: Tue, 6 Sep 2022 20:04:11 +0200 -Subject: [PATCH 09/13] lrng - -Signed-off-by: Piotr Gorski ---- - MAINTAINERS | 7 + - crypto/drbg.c | 16 +- - crypto/jitterentropy-kcapi.c | 3 +- - crypto/jitterentropy.c | 2 +- - drivers/char/Kconfig | 2 + - drivers/char/Makefile | 5 +- - drivers/char/lrng/Kconfig | 907 ++++++++++++++++++ - drivers/char/lrng/Makefile | 39 + - drivers/char/lrng/lrng_definitions.h | 151 +++ - drivers/char/lrng/lrng_drng_atomic.c | 171 ++++ - drivers/char/lrng/lrng_drng_atomic.h | 23 + - drivers/char/lrng/lrng_drng_chacha20.c | 195 ++++ - drivers/char/lrng/lrng_drng_chacha20.h | 42 + - drivers/char/lrng/lrng_drng_drbg.c | 179 ++++ - drivers/char/lrng/lrng_drng_drbg.h | 13 + - drivers/char/lrng/lrng_drng_kcapi.c | 208 ++++ - drivers/char/lrng/lrng_drng_kcapi.h | 13 + - drivers/char/lrng/lrng_drng_mgr.c | 657 +++++++++++++ - drivers/char/lrng/lrng_drng_mgr.h | 86 ++ - drivers/char/lrng/lrng_es_aux.c | 335 +++++++ - drivers/char/lrng/lrng_es_aux.h | 44 + - drivers/char/lrng/lrng_es_cpu.c | 280 ++++++ - drivers/char/lrng/lrng_es_cpu.h | 17 + - drivers/char/lrng/lrng_es_irq.c | 727 ++++++++++++++ - drivers/char/lrng/lrng_es_irq.h | 24 + - drivers/char/lrng/lrng_es_jent.c | 136 +++ - drivers/char/lrng/lrng_es_jent.h | 17 + - drivers/char/lrng/lrng_es_krng.c | 100 ++ - drivers/char/lrng/lrng_es_krng.h | 17 + - drivers/char/lrng/lrng_es_mgr.c | 460 +++++++++ - drivers/char/lrng/lrng_es_mgr.h | 51 + - drivers/char/lrng/lrng_es_mgr_cb.h | 87 ++ - drivers/char/lrng/lrng_es_sched.c | 562 +++++++++++ - drivers/char/lrng/lrng_es_sched.h | 20 + - drivers/char/lrng/lrng_es_timer_common.c | 144 +++ - drivers/char/lrng/lrng_es_timer_common.h | 83 ++ - drivers/char/lrng/lrng_hash_kcapi.c | 140 +++ - drivers/char/lrng/lrng_health.c | 420 ++++++++ - drivers/char/lrng/lrng_health.h | 42 + - drivers/char/lrng/lrng_interface_aux.c | 126 +++ - drivers/char/lrng/lrng_interface_dev.c | 35 + - drivers/char/lrng/lrng_interface_dev_common.c | 314 ++++++ - drivers/char/lrng/lrng_interface_dev_common.h | 51 + - drivers/char/lrng/lrng_interface_hwrand.c | 68 ++ - drivers/char/lrng/lrng_interface_kcapi.c | 129 +++ - .../char/lrng/lrng_interface_random_kernel.c | 214 +++++ - .../char/lrng/lrng_interface_random_kernel.h | 15 + - .../char/lrng/lrng_interface_random_user.c | 104 ++ - drivers/char/lrng/lrng_numa.c | 124 +++ - drivers/char/lrng/lrng_numa.h | 15 + - drivers/char/lrng/lrng_proc.c | 74 ++ - drivers/char/lrng/lrng_proc.h | 15 + - drivers/char/lrng/lrng_selftest.c | 397 ++++++++ - drivers/char/lrng/lrng_sha.h | 14 + - drivers/char/lrng/lrng_sha1.c | 88 ++ - drivers/char/lrng/lrng_sha256.c | 72 ++ - drivers/char/lrng/lrng_switch.c | 286 ++++++ - drivers/char/lrng/lrng_sysctl.c | 140 +++ - drivers/char/lrng/lrng_sysctl.h | 15 + - drivers/char/lrng/lrng_testing.c | 901 +++++++++++++++++ - drivers/char/lrng/lrng_testing.h | 85 ++ - include/crypto/drbg.h | 7 + - .../crypto/internal}/jitterentropy.h | 0 - include/linux/lrng.h | 251 +++++ - kernel/sched/core.c | 3 + - 65 files changed, 9958 insertions(+), 10 deletions(-) - create mode 100644 drivers/char/lrng/Kconfig - create mode 100644 drivers/char/lrng/Makefile - create mode 100644 drivers/char/lrng/lrng_definitions.h - create mode 100644 drivers/char/lrng/lrng_drng_atomic.c - create mode 100644 drivers/char/lrng/lrng_drng_atomic.h - create mode 100644 drivers/char/lrng/lrng_drng_chacha20.c - create mode 100644 drivers/char/lrng/lrng_drng_chacha20.h - create mode 100644 drivers/char/lrng/lrng_drng_drbg.c - create mode 100644 drivers/char/lrng/lrng_drng_drbg.h - create mode 100644 drivers/char/lrng/lrng_drng_kcapi.c - create mode 100644 drivers/char/lrng/lrng_drng_kcapi.h - create mode 100644 drivers/char/lrng/lrng_drng_mgr.c - create mode 100644 drivers/char/lrng/lrng_drng_mgr.h - create mode 100644 drivers/char/lrng/lrng_es_aux.c - create mode 100644 drivers/char/lrng/lrng_es_aux.h - create mode 100644 drivers/char/lrng/lrng_es_cpu.c - create mode 100644 drivers/char/lrng/lrng_es_cpu.h - create mode 100644 drivers/char/lrng/lrng_es_irq.c - create mode 100644 drivers/char/lrng/lrng_es_irq.h - create mode 100644 drivers/char/lrng/lrng_es_jent.c - create mode 100644 drivers/char/lrng/lrng_es_jent.h - create mode 100644 drivers/char/lrng/lrng_es_krng.c - create mode 100644 drivers/char/lrng/lrng_es_krng.h - create mode 100644 drivers/char/lrng/lrng_es_mgr.c - create mode 100644 drivers/char/lrng/lrng_es_mgr.h - create mode 100644 drivers/char/lrng/lrng_es_mgr_cb.h - create mode 100644 drivers/char/lrng/lrng_es_sched.c - create mode 100644 drivers/char/lrng/lrng_es_sched.h - create mode 100644 drivers/char/lrng/lrng_es_timer_common.c - create mode 100644 drivers/char/lrng/lrng_es_timer_common.h - create mode 100644 drivers/char/lrng/lrng_hash_kcapi.c - create mode 100644 drivers/char/lrng/lrng_health.c - create mode 100644 drivers/char/lrng/lrng_health.h - create mode 100644 drivers/char/lrng/lrng_interface_aux.c - create mode 100644 drivers/char/lrng/lrng_interface_dev.c - create mode 100644 drivers/char/lrng/lrng_interface_dev_common.c - create mode 100644 drivers/char/lrng/lrng_interface_dev_common.h - create mode 100644 drivers/char/lrng/lrng_interface_hwrand.c - create mode 100644 drivers/char/lrng/lrng_interface_kcapi.c - create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.c - create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.h - create mode 100644 drivers/char/lrng/lrng_interface_random_user.c - create mode 100644 drivers/char/lrng/lrng_numa.c - create mode 100644 drivers/char/lrng/lrng_numa.h - create mode 100644 drivers/char/lrng/lrng_proc.c - create mode 100644 drivers/char/lrng/lrng_proc.h - create mode 100644 drivers/char/lrng/lrng_selftest.c - create mode 100644 drivers/char/lrng/lrng_sha.h - create mode 100644 drivers/char/lrng/lrng_sha1.c - create mode 100644 drivers/char/lrng/lrng_sha256.c - create mode 100644 drivers/char/lrng/lrng_switch.c - create mode 100644 drivers/char/lrng/lrng_sysctl.c - create mode 100644 drivers/char/lrng/lrng_sysctl.h - create mode 100644 drivers/char/lrng/lrng_testing.c - create mode 100644 drivers/char/lrng/lrng_testing.h - rename {crypto => include/crypto/internal}/jitterentropy.h (100%) - create mode 100644 include/linux/lrng.h - -diff --git a/MAINTAINERS b/MAINTAINERS -index a29c9731350c..d7de955b50bb 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -11741,6 +11741,13 @@ F: Documentation/litmus-tests/ - F: Documentation/memory-barriers.txt - F: tools/memory-model/ - -+LINUX RANDOM NUMBER GENERATOR (LRNG) DRIVER -+M: Stephan Mueller -+S: Maintained -+W: https://www.chronox.de/lrng.html -+F: drivers/char/lrng/ -+F: include/linux/lrng.h -+ - LIS3LV02D ACCELEROMETER DRIVER - M: Eric Piel - S: Maintained -diff --git a/crypto/drbg.c b/crypto/drbg.c -index 177983b6ae38..ea79a31014a4 100644 ---- a/crypto/drbg.c -+++ b/crypto/drbg.c -@@ -115,7 +115,7 @@ - * the SHA256 / AES 256 over other ciphers. Thus, the favored - * DRBGs are the latest entries in this array. - */ --static const struct drbg_core drbg_cores[] = { -+const struct drbg_core drbg_cores[] = { - #ifdef CONFIG_CRYPTO_DRBG_CTR - { - .flags = DRBG_CTR | DRBG_STRENGTH128, -@@ -192,6 +192,7 @@ static const struct drbg_core drbg_cores[] = { - }, - #endif /* CONFIG_CRYPTO_DRBG_HMAC */ - }; -+EXPORT_SYMBOL(drbg_cores); - - static int drbg_uninstantiate(struct drbg_state *drbg); - -@@ -207,7 +208,7 @@ static int drbg_uninstantiate(struct drbg_state *drbg); - * Return: normalized strength in *bytes* value or 32 as default - * to counter programming errors - */ --static inline unsigned short drbg_sec_strength(drbg_flag_t flags) -+unsigned short drbg_sec_strength(drbg_flag_t flags) - { - switch (flags & DRBG_STRENGTH_MASK) { - case DRBG_STRENGTH128: -@@ -220,6 +221,7 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) - return 32; - } - } -+EXPORT_SYMBOL(drbg_sec_strength); - - /* - * FIPS 140-2 continuous self test for the noise source -@@ -1252,7 +1254,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, - } - - /* Free all substructures in a DRBG state without the DRBG state structure */ --static inline void drbg_dealloc_state(struct drbg_state *drbg) -+void drbg_dealloc_state(struct drbg_state *drbg) - { - if (!drbg) - return; -@@ -1273,12 +1275,13 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) - drbg->fips_primed = false; - } - } -+EXPORT_SYMBOL(drbg_dealloc_state); - - /* - * Allocate all sub-structures for a DRBG state. - * The DRBG state structure must already be allocated. - */ --static inline int drbg_alloc_state(struct drbg_state *drbg) -+int drbg_alloc_state(struct drbg_state *drbg) - { - int ret = -ENOMEM; - unsigned int sb_size = 0; -@@ -1359,6 +1362,7 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) - drbg_dealloc_state(drbg); - return ret; - } -+EXPORT_SYMBOL(drbg_alloc_state); - - /************************************************************************* - * DRBG interface functions -@@ -1895,8 +1899,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, - * - * return: flags - */ --static inline void drbg_convert_tfm_core(const char *cra_driver_name, -- int *coreref, bool *pr) -+void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, bool *pr) - { - int i = 0; - size_t start = 0; -@@ -1923,6 +1926,7 @@ static inline void drbg_convert_tfm_core(const char *cra_driver_name, - } - } - } -+EXPORT_SYMBOL(drbg_convert_tfm_core); - - static int drbg_kcapi_init(struct crypto_tfm *tfm) - { -diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c -index 2d115bec15ae..e7dac734d237 100644 ---- a/crypto/jitterentropy-kcapi.c -+++ b/crypto/jitterentropy-kcapi.c -@@ -42,8 +42,7 @@ - #include - #include - #include -- --#include "jitterentropy.h" -+#include - - /*************************************************************************** - * Helper function -diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c -index 93bff3213823..81b80a4d3d3a 100644 ---- a/crypto/jitterentropy.c -+++ b/crypto/jitterentropy.c -@@ -133,7 +133,7 @@ struct rand_data { - #define JENT_ENTROPY_SAFETY_FACTOR 64 - - #include --#include "jitterentropy.h" -+#include - - /*************************************************************************** - * Adaptive Proportion Test -diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig -index 0f378d29dab0..39c91f09cbe0 100644 ---- a/drivers/char/Kconfig -+++ b/drivers/char/Kconfig -@@ -459,4 +459,6 @@ config RANDOM_TRUST_BOOTLOADER - believe its RNG facilities may be faulty. This may also be configured - at boot time with "random.trust_bootloader=on/off". - -+source "drivers/char/lrng/Kconfig" -+ - endmenu -diff --git a/drivers/char/Makefile b/drivers/char/Makefile -index 1b35d1724565..0d56f0a24fcd 100644 ---- a/drivers/char/Makefile -+++ b/drivers/char/Makefile -@@ -3,7 +3,8 @@ - # Makefile for the kernel character device drivers. - # - --obj-y += mem.o random.o -+obj-y += mem.o -+obj-$(CONFIG_RANDOM_DEFAULT_IMPL) += random.o - obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o - obj-y += misc.o - obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o -@@ -45,3 +46,5 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o - obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/ - obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o - obj-$(CONFIG_ADI) += adi.o -+ -+obj-$(CONFIG_LRNG) += lrng/ -diff --git a/drivers/char/lrng/Kconfig b/drivers/char/lrng/Kconfig -new file mode 100644 -index 000000000000..bfd6b3ea4f1b ---- /dev/null -+++ b/drivers/char/lrng/Kconfig -@@ -0,0 +1,907 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# Linux Random Number Generator configuration -+# -+ -+config RANDOM_DEFAULT_IMPL -+ bool "Kernel RNG Default Implementation" -+ default y -+ help -+ The default random number generator as provided with -+ drivers/char/random.c is selected with this option. -+ -+config LRNG -+ bool "Linux Random Number Generator" -+ default n -+ select CRYPTO_LIB_SHA256 if CRYPTO -+ help -+ The Linux Random Number Generator (LRNG) generates entropy -+ from different entropy sources. Each entropy source can -+ be enabled and configured independently. The interrupt -+ entropy source can be configured to be SP800-90B compliant. -+ The entire LRNG can be configured to be SP800-90C compliant. -+ Runtime-switchable cryptographic support is available. -+ The LRNG delivers significant entropy during boot. -+ -+ The LRNG also provides compliance to SP800-90A/B/C. -+ -+menu "Linux Random Number Generator Configuration" -+ depends on LRNG -+ -+if LRNG -+ -+config LRNG_SHA256 -+ bool -+ default y if CRYPTO_LIB_SHA256 -+ -+config LRNG_SHA1 -+ bool -+ default y if !CRYPTO_LIB_SHA256 -+ -+config LRNG_COMMON_DEV_IF -+ bool -+ -+config LRNG_DRNG_ATOMIC -+ bool -+ select LRNG_DRNG_CHACHA20 -+ -+config LRNG_SYSCTL -+ bool -+ depends on SYSCTL -+ -+config LRNG_RANDOM_IF -+ bool -+ default n if RANDOM_DEFAULT_IMPL -+ default y if !RANDOM_DEFAULT_IMPL -+ select LRNG_COMMON_DEV_IF -+ select LRNG_DRNG_ATOMIC -+ select LRNG_SYSCTL -+ -+menu "LRNG Interfaces" -+ -+config LRNG_KCAPI_IF -+ tristate "Interface with Kernel Crypto API" -+ depends on CRYPTO_RNG -+ help -+ The LRNG can be registered with the kernel crypto API's -+ random number generator framework. This offers a random -+ number generator with the name "lrng" and a priority that -+ is intended to be higher than the existing RNG -+ implementations. -+ -+config LRNG_HWRAND_IF -+ tristate "Interface with Hardware Random Number Generator Framework" -+ depends on HW_RANDOM -+ select LRNG_DRNG_ATOMIC -+ help -+ The LRNG can be registered with the hardware random number -+ generator framework. This offers a random number generator -+ with the name "lrng" that is accessible via the framework. -+ For example it allows pulling data from the LRNG via the -+ /dev/hwrng file. -+ -+config LRNG_DEV_IF -+ bool "Character device file interface" -+ select LRNG_COMMON_DEV_IF -+ help -+ The LRNG can create a character device file that operates -+ identically to /dev/random including IOCTL, read and write -+ operations. -+ -+endmenu # "LRNG Interfaces" -+ -+menu "Entropy Source Configuration" -+ -+config LRNG_RUNTIME_ES_CONFIG -+ bool "Enable runtime configuration of entropy sources" -+ help -+ When enabling this option, the LRNG provides the mechanism -+ allowing to alter the entropy rate of each entropy source -+ during boot time and runtime. -+ -+ Each entropy source allows its entropy rate changed with -+ a kernel command line option. When not providing any -+ option, the default specified during kernel compilation -+ is applied. -+ -+comment "Common Timer-based Entropy Source Configuration" -+ -+config LRNG_IRQ_DFLT_TIMER_ES -+ bool -+ -+config LRNG_SCHED_DFLT_TIMER_ES -+ bool -+ -+config LRNG_TIMER_COMMON -+ bool -+ -+choice -+ prompt "Default Timer-based Entropy Source" -+ default LRNG_IRQ_DFLT_TIMER_ES -+ depends on LRNG_TIMER_COMMON -+ help -+ Select the timer-based entropy source that is credited -+ with entropy. The other timer-based entropy sources may -+ be operational and provide data, but are credited with no -+ entropy. -+ -+ config LRNG_IRQ_DFLT_TIMER_ES -+ bool "Interrupt Entropy Source" -+ depends on LRNG_IRQ -+ help -+ The interrupt entropy source is selected as a timer-based -+ entropy source to provide entropy. -+ -+ config LRNG_SCHED_DFLT_TIMER_ES -+ bool "Scheduler Entropy Source" -+ depends on LRNG_SCHED -+ help -+ The scheduler entropy source is selected as timer-based -+ entropy source to provide entropy. -+endchoice -+ -+choice -+ prompt "LRNG Entropy Collection Pool Size" -+ default LRNG_COLLECTION_SIZE_1024 -+ depends on LRNG_TIMER_COMMON -+ help -+ Select the size of the LRNG entropy collection pool -+ storing data for the interrupt as well as the scheduler -+ entropy sources without performing a compression -+ operation. The larger the collection size is, the faster -+ the average interrupt handling will be. The collection -+ size represents the number of bytes of the per-CPU memory -+ used to batch up entropy event data. -+ -+ The default value is good for regular operations. Choose -+ larger sizes for servers that have no memory limitations. -+ If runtime memory is precious, choose a smaller size. -+ -+ The collection size is unrelated to the entropy rate -+ or the amount of entropy the LRNG can process. -+ -+ config LRNG_COLLECTION_SIZE_32 -+ depends on LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ depends on !LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION -+ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES -+ bool "32 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_256 -+ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES -+ bool "256 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_512 -+ bool "512 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_1024 -+ bool "1024 interrupt events (default)" -+ -+ config LRNG_COLLECTION_SIZE_2048 -+ bool "2048 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_4096 -+ bool "4096 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_8192 -+ bool "8192 interrupt events" -+ -+endchoice -+ -+config LRNG_COLLECTION_SIZE -+ int -+ default 32 if LRNG_COLLECTION_SIZE_32 -+ default 256 if LRNG_COLLECTION_SIZE_256 -+ default 512 if LRNG_COLLECTION_SIZE_512 -+ default 1024 if LRNG_COLLECTION_SIZE_1024 -+ default 2048 if LRNG_COLLECTION_SIZE_2048 -+ default 4096 if LRNG_COLLECTION_SIZE_4096 -+ default 8192 if LRNG_COLLECTION_SIZE_8192 -+ -+config LRNG_HEALTH_TESTS -+ bool "Enable internal entropy source online health tests" -+ depends on LRNG_TIMER_COMMON -+ help -+ The online health tests applied to the interrupt entropy -+ source and to the scheduler entropy source to validate -+ the noise source at runtime for fatal errors. These tests -+ include SP800-90B compliant tests which are invoked if -+ the system is booted with fips=1. In case of fatal errors -+ during active SP800-90B tests, the issue is logged and -+ the noise data is discarded. These tests are required for -+ full compliance of the interrupt entropy source with -+ SP800-90B. -+ -+ If both, the scheduler and the interrupt entropy sources, -+ are enabled, the health tests for both are applied -+ independent of each other. -+ -+ If unsure, say Y. -+ -+config LRNG_RCT_BROKEN -+ bool "SP800-90B RCT with dangerous low cutoff value" -+ depends on LRNG_HEALTH_TESTS -+ depends on BROKEN -+ default n -+ help -+ This option enables a dangerously low SP800-90B repetitive -+ count test (RCT) cutoff value which makes it very likely -+ that the RCT is triggered to raise a self test failure. -+ -+ This option is ONLY intended for developers wanting to -+ test the effectiveness of the SP800-90B RCT health test. -+ -+ If unsure, say N. -+ -+config LRNG_APT_BROKEN -+ bool "SP800-90B APT with dangerous low cutoff value" -+ depends on LRNG_HEALTH_TESTS -+ depends on BROKEN -+ default n -+ help -+ This option enables a dangerously low SP800-90B adaptive -+ proportion test (APT) cutoff value which makes it very -+ likely that the APT is triggered to raise a self test -+ failure. -+ -+ This option is ONLY intended for developers wanting to -+ test the effectiveness of the SP800-90B APT health test. -+ -+ If unsure, say N. -+ -+# Default taken from SP800-90B sec 4.4.1 - significance level 2^-30 -+config LRNG_RCT_CUTOFF -+ int -+ default 31 if !LRNG_RCT_BROKEN -+ default 1 if LRNG_RCT_BROKEN -+ -+# Default taken from SP800-90B sec 4.4.2 - significance level 2^-30 -+config LRNG_APT_CUTOFF -+ int -+ default 325 if !LRNG_APT_BROKEN -+ default 32 if LRNG_APT_BROKEN -+ -+comment "Interrupt Entropy Source" -+ -+config LRNG_IRQ -+ bool "Enable Interrupt Entropy Source as LRNG Seed Source" -+ default y -+ depends on !RANDOM_DEFAULT_IMPL -+ select LRNG_TIMER_COMMON -+ help -+ The LRNG models an entropy source based on the timing of the -+ occurrence of interrupts. Enable this option to enable this -+ IRQ entropy source. -+ -+ The IRQ entropy source is triggered every time an interrupt -+ arrives and thus causes the interrupt handler to execute -+ slightly longer. Disabling the IRQ entropy source implies -+ that the performance penalty on the interrupt handler added -+ by the LRNG is eliminated. Yet, this entropy source is -+ considered to be an internal entropy source of the LRNG. -+ Thus, only disable it if you ensured that other entropy -+ sources are available that supply the LRNG with entropy. -+ -+ If you disable the IRQ entropy source, you MUST ensure -+ one or more entropy sources collectively have the -+ capability to deliver sufficient entropy with one invocation -+ at a rate compliant to the security strength of the DRNG -+ (usually 256 bits of entropy). In addition, if those -+ entropy sources do not deliver sufficient entropy during -+ first request, the reseed must be triggered from user -+ space or kernel space when sufficient entropy is considered -+ to be present. -+ -+ If unsure, say Y. -+ -+choice -+ prompt "Continuous entropy compression boot time setting" -+ default LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ depends on LRNG_IRQ -+ help -+ Select the default behavior of the interrupt entropy source -+ continuous compression operation. -+ -+ The LRNG IRQ ES collects entropy data during each interrupt. -+ For performance reasons, a amount of entropy data defined by -+ the LRNG entropy collection pool size is concatenated into -+ an array. When that array is filled up, a hash is calculated -+ to compress the entropy. That hash is calculated in -+ interrupt context. -+ -+ In case such hash calculation in interrupt context is deemed -+ too time-consuming, the continuous compression operation -+ can be disabled. If disabled, the collection of entropy will -+ not trigger a hash compression operation in interrupt context. -+ The compression happens only when the DRNG is reseeded which is -+ in process context. This implies that old entropy data -+ collected after the last DRNG-reseed is overwritten with newer -+ entropy data once the collection pool is full instead of -+ retaining its entropy with the compression operation. -+ -+ config LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ bool "Enable continuous compression (default)" -+ -+ config LRNG_CONTINUOUS_COMPRESSION_DISABLED -+ bool "Disable continuous compression" -+ -+endchoice -+ -+config LRNG_ENABLE_CONTINUOUS_COMPRESSION -+ bool -+ default y if LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ default n if LRNG_CONTINUOUS_COMPRESSION_DISABLED -+ -+config LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION -+ bool "Runtime-switchable continuous entropy compression" -+ depends on LRNG_IRQ -+ help -+ Per default, the interrupt entropy source continuous -+ compression operation behavior is hard-wired into the kernel. -+ Enable this option to allow it to be configurable at boot time. -+ -+ To modify the default behavior of the continuous -+ compression operation, use the kernel command line option -+ of lrng_sw_noise.lrng_pcpu_continuous_compression. -+ -+ If unsure, say N. -+ -+config LRNG_IRQ_ENTROPY_RATE -+ int "Interrupt Entropy Source Entropy Rate" -+ depends on LRNG_IRQ -+ range 256 4294967295 if LRNG_IRQ_DFLT_TIMER_ES -+ range 4294967295 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES -+ default 256 if LRNG_IRQ_DFLT_TIMER_ES -+ default 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES -+ help -+ The LRNG will collect the configured number of interrupts to -+ obtain 256 bits of entropy. This value can be set to any between -+ 256 and 4294967295. The LRNG guarantees that this value is not -+ lower than 256. This lower limit implies that one interrupt event -+ is credited with one bit of entropy. This value is subject to the -+ increase by the oversampling factor, if no high-resolution timer -+ is found. -+ -+ In order to effectively disable the interrupt entropy source, -+ the option has to be set to 4294967295. In this case, the -+ interrupt entropy source will still deliver data but without -+ being credited with entropy. -+ -+comment "Jitter RNG Entropy Source" -+ -+config LRNG_JENT -+ bool "Enable Jitter RNG as LRNG Seed Source" -+ depends on CRYPTO -+ select CRYPTO_JITTERENTROPY -+ help -+ The LRNG may use the Jitter RNG as entropy source. Enabling -+ this option enables the use of the Jitter RNG. Its default -+ entropy level is 16 bits of entropy per 256 data bits delivered -+ by the Jitter RNG. This entropy level can be changed at boot -+ time or at runtime with the lrng_base.jitterrng configuration -+ variable. -+ -+config LRNG_JENT_ENTROPY_RATE -+ int "Jitter RNG Entropy Source Entropy Rate" -+ depends on LRNG_JENT -+ range 0 256 -+ default 16 -+ help -+ The option defines the amount of entropy the LRNG applies to 256 -+ bits of data obtained from the Jitter RNG entropy source. The -+ LRNG enforces the limit that this value must be in the range -+ between 0 and 256. -+ -+ When configuring this value to 0, the Jitter RNG entropy source -+ will provide 256 bits of data without being credited to contain -+ entropy. -+ -+comment "CPU Entropy Source" -+ -+config LRNG_CPU -+ bool "Enable CPU Entropy Source as LRNG Seed Source" -+ default y -+ help -+ Current CPUs commonly contain entropy sources which can be -+ used to seed the LRNG. For example, the Intel RDSEED -+ instruction, or the POWER DARN instruction will be sourced -+ to seed the LRNG if this option is enabled. -+ -+ Note, if this option is enabled and the underlying CPU -+ does not offer such entropy source, the LRNG will automatically -+ detect this and ignore the hardware. -+ -+config LRNG_CPU_FULL_ENT_MULTIPLIER -+ int -+ default 1 if !LRNG_TEST_CPU_ES_COMPRESSION -+ default 123 if LRNG_TEST_CPU_ES_COMPRESSION -+ -+config LRNG_CPU_ENTROPY_RATE -+ int "CPU Entropy Source Entropy Rate" -+ depends on LRNG_CPU -+ range 0 256 -+ default 8 -+ help -+ The option defines the amount of entropy the LRNG applies to 256 -+ bits of data obtained from the CPU entropy source. The LRNG -+ enforces the limit that this value must be in the range between -+ 0 and 256. -+ -+ When configuring this value to 0, the CPU entropy source will -+ provide 256 bits of data without being credited to contain -+ entropy. -+ -+ Note, this option is overwritten when the option -+ CONFIG_RANDOM_TRUST_CPU is set. -+ -+comment "Scheduler Entropy Source" -+ -+config LRNG_SCHED -+ bool "Enable Scheduer Entropy Source as LRNG Seed Source" -+ select LRNG_TIMER_COMMON -+ help -+ The LRNG models an entropy source based on the timing of the -+ occurrence of scheduler-triggered context switches. Enable -+ this option to enable this scheduler entropy source. -+ -+ The scheduler entropy source is triggered every time a -+ context switch is triggered thus causes the scheduler to -+ execute slightly longer. Disabling the scheduler entropy -+ source implies that the performance penalty on the scheduler -+ added by the LRNG is eliminated. Yet, this entropy source is -+ considered to be an internal entropy source of the LRNG. -+ Thus, only disable it if you ensured that other entropy -+ sources are available that supply the LRNG with entropy. -+ -+ If you disable the scheduler entropy source, you MUST -+ ensure one or more entropy sources collectively have the -+ capability to deliver sufficient entropy with one invocation -+ at a rate compliant to the security strength of the DRNG -+ (usually 256 bits of entropy). In addition, if those -+ entropy sources do not deliver sufficient entropy during -+ first request, the reseed must be triggered from user -+ space or kernel space when sufficient entropy is considered -+ to be present. -+ -+ If unsure, say Y. -+ -+config LRNG_SCHED_ENTROPY_RATE -+ int "Scheduler Entropy Source Entropy Rate" -+ depends on LRNG_SCHED -+ range 256 4294967295 if LRNG_SCHED_DFLT_TIMER_ES -+ range 4294967295 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES -+ default 256 if LRNG_SCHED_DFLT_TIMER_ES -+ default 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES -+ help -+ The LRNG will collect the configured number of context switches -+ triggered by the scheduler to obtain 256 bits of entropy. This -+ value can be set to any between 256 and 4294967295. The LRNG -+ guarantees that this value is not lower than 256. This lower -+ limit implies that one interrupt event is credited with one bit -+ of entropy. This value is subject to the increase by the -+ oversampling factor, if no high-resolution timer is found. -+ -+ In order to effectively disable the scheduler entropy source, -+ the option has to be set to 4294967295. In this case, the -+ scheduler entropy source will still deliver data but without -+ being credited with entropy. -+ -+comment "Kernel RNG Entropy Source" -+ -+config LRNG_KERNEL_RNG -+ bool "Enable Kernel RNG as LRNG Seed Source" -+ depends on RANDOM_DEFAULT_IMPL -+ help -+ The LRNG may use the kernel RNG (random.c) as entropy -+ source. -+ -+config LRNG_KERNEL_RNG_ENTROPY_RATE -+ int "Kernel RNG Entropy Source Entropy Rate" -+ depends on LRNG_KERNEL_RNG -+ range 0 256 -+ default 256 -+ help -+ The option defines the amount of entropy the LRNG applies to 256 -+ bits of data obtained from the kernel RNG entropy source. The -+ LRNG enforces the limit that this value must be in the range -+ between 0 and 256. -+ -+ When configuring this value to 0, the kernel RNG entropy source -+ will provide 256 bits of data without being credited to contain -+ entropy. -+ -+ Note: This value is set to 0 automatically when booting the -+ kernel in FIPS mode (with fips=1 kernel command line option). -+ This is due to the fact that random.c is not SP800-90B -+ compliant. -+ -+endmenu # "Entropy Source Configuration" -+ -+config LRNG_DRNG_CHACHA20 -+ tristate -+ -+config LRNG_DRBG -+ tristate -+ depends on CRYPTO -+ select CRYPTO_DRBG_MENU -+ -+config LRNG_DRNG_KCAPI -+ tristate -+ depends on CRYPTO -+ select CRYPTO_RNG -+ -+config LRNG_SWITCH -+ bool -+ -+menuconfig LRNG_SWITCH_HASH -+ bool "Support conditioning hash runtime switching" -+ select LRNG_SWITCH -+ help -+ The LRNG uses a default message digest. With this -+ configuration option other message digests can be selected -+ and loaded at runtime. -+ -+if LRNG_SWITCH_HASH -+ -+config LRNG_HASH_KCAPI -+ tristate "Kernel crypto API hashing support for LRNG" -+ select CRYPTO_HASH -+ select CRYPTO_SHA512 -+ help -+ Enable the kernel crypto API support for entropy compression -+ and conditioning functions. -+ -+endif # LRNG_SWITCH_HASH -+ -+menuconfig LRNG_SWITCH_DRNG -+ bool "Support DRNG runtime switching" -+ select LRNG_SWITCH -+ help -+ The LRNG uses a default DRNG With this configuration -+ option other DRNGs or message digests can be selected and -+ loaded at runtime. -+ -+if LRNG_SWITCH_DRNG -+ -+config LRNG_SWITCH_DRNG_CHACHA20 -+ tristate "ChaCha20-based DRNG support for LRNG" -+ depends on !LRNG_DFLT_DRNG_CHACHA20 -+ select LRNG_DRNG_CHACHA20 -+ help -+ Enable the ChaCha20-based DRNG. This DRNG implementation -+ does not depend on the kernel crypto API presence. -+ -+config LRNG_SWITCH_DRBG -+ tristate "SP800-90A support for the LRNG" -+ depends on !LRNG_DFLT_DRNG_DRBG -+ select LRNG_DRBG -+ help -+ Enable the SP800-90A DRBG support for the LRNG. Once the -+ module is loaded, output from /dev/random, /dev/urandom, -+ getrandom(2), or get_random_bytes_full is provided by a DRBG. -+ -+config LRNG_SWITCH_DRNG_KCAPI -+ tristate "Kernel Crypto API support for the LRNG" -+ depends on !LRNG_DFLT_DRNG_KCAPI -+ depends on !LRNG_SWITCH_DRBG -+ select LRNG_DRNG_KCAPI -+ help -+ Enable the support for generic pseudo-random number -+ generators offered by the kernel crypto API with the -+ LRNG. Once the module is loaded, output from /dev/random, -+ /dev/urandom, getrandom(2), or get_random_bytes is -+ provided by the selected kernel crypto API RNG. -+ -+endif # LRNG_SWITCH_DRNG -+ -+choice -+ prompt "LRNG Default DRNG" -+ default LRNG_DFLT_DRNG_CHACHA20 -+ help -+ Select the default deterministic random number generator -+ that is used by the LRNG. When enabling the switchable -+ cryptographic mechanism support, this DRNG can be -+ replaced at runtime. -+ -+ config LRNG_DFLT_DRNG_CHACHA20 -+ bool "ChaCha20-based DRNG" -+ select LRNG_DRNG_CHACHA20 -+ -+ config LRNG_DFLT_DRNG_DRBG -+ bool "SP800-90A DRBG" -+ select LRNG_DRBG -+ -+ config LRNG_DFLT_DRNG_KCAPI -+ bool "Kernel Crypto API DRNG" -+ select LRNG_DRNG_KCAPI -+endchoice -+ -+menuconfig LRNG_TESTING_MENU -+ bool "LRNG testing interfaces" -+ depends on DEBUG_FS -+ help -+ Enable one or more of the following test interfaces. -+ -+ If unsure, say N. -+ -+if LRNG_TESTING_MENU -+ -+config LRNG_TESTING -+ bool -+ -+config LRNG_TESTING_RECORDING -+ bool -+ -+comment "Interrupt Entropy Source Test Interfaces" -+ -+config LRNG_RAW_HIRES_ENTROPY -+ bool "Interface to obtain raw unprocessed IRQ noise source data" -+ default y -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned high resolution time stamp noise that -+ is collected by the LRNG for statistical analysis. Extracted -+ noise data is not used to seed the LRNG. -+ -+ The raw noise data can be obtained using the lrng_raw_hires -+ debugfs file. Using the option lrng_testing.boot_raw_hires_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_JIFFIES_ENTROPY -+ bool "Entropy test interface to Jiffies of IRQ noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned Jiffies that is collected by -+ the LRNG for statistical analysis. This data is used for -+ seeding the LRNG if a high-resolution time stamp is not -+ available. If a high-resolution time stamp is detected, -+ the Jiffies value is not collected by the LRNG and no -+ data is provided via the test interface. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the lrng_raw_jiffies -+ debugfs file. Using the option lrng_testing.boot_raw_jiffies_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_IRQ_ENTROPY -+ bool "Entropy test interface to IRQ number noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned interrupt number that is collected by -+ the LRNG for statistical analysis. Extracted noise data is -+ not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the lrng_raw_irq -+ debugfs file. Using the option lrng_testing.boot_raw_irq_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_RETIP_ENTROPY -+ bool "Entropy test interface to RETIP value of IRQ noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned return instruction pointer value -+ that is collected by the LRNG for statistical analysis. -+ Extracted noise data is not used to seed the random number -+ generator. -+ -+ The raw noise data can be obtained using the lrng_raw_retip -+ debugfs file. Using the option lrng_testing.boot_raw_retip_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_REGS_ENTROPY -+ bool "Entropy test interface to IRQ register value noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned interrupt register value that is -+ collected by the LRNG for statistical analysis. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the lrng_raw_regs -+ debugfs file. Using the option lrng_testing.boot_raw_regs_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_ARRAY -+ bool "Test interface to LRNG raw entropy IRQ storage array" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw noise data that is collected by the LRNG -+ in the per-CPU array for statistical analysis. The purpose -+ of this interface is to verify that the array handling code -+ truly only concatenates data and provides the same entropy -+ rate as the raw unconditioned noise source when assessing -+ the collected data byte-wise. -+ -+ The data can be obtained using the lrng_raw_array debugfs -+ file. Using the option lrng_testing.boot_raw_array=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_IRQ_PERF -+ bool "LRNG interrupt entropy source performance monitor" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ With this option, the performance monitor of the LRNG -+ interrupt handling code is enabled. The file provides -+ the execution time of the interrupt handler in -+ cycles. -+ -+ The interrupt performance data can be obtained using -+ the lrng_irq_perf debugfs file. Using the option -+ lrng_testing.boot_irq_perf=1 the performance data of -+ the first 1000 entropy events since boot can be sampled. -+ -+comment "Scheduler Entropy Source Test Interfaces" -+ -+config LRNG_RAW_SCHED_HIRES_ENTROPY -+ bool "Interface to obtain raw unprocessed scheduler noise source data" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned high resolution time stamp noise that -+ is collected by the LRNG for the Scheduler-based noise source -+ for statistical analysis. Extracted noise data is not used to -+ seed the LRNG. -+ -+ The raw noise data can be obtained using the lrng_raw_sched_hires -+ debugfs file. Using the option -+ lrng_testing.boot_raw_sched_hires_test=1 the raw noise of the -+ first 1000 entropy events since boot can be sampled. -+ -+config LRNG_RAW_SCHED_PID_ENTROPY -+ bool "Entropy test interface to PID value" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned PID value that is collected by the -+ LRNG for statistical analysis. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the -+ lrng_raw_sched_pid debugfs file. Using the option -+ lrng_testing.boot_raw_sched_pid_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_SCHED_START_TIME_ENTROPY -+ bool "Entropy test interface to task start time value" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned task start time value that is collected -+ by the LRNG for statistical analysis. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the -+ lrng_raw_sched_starttime debugfs file. Using the option -+ lrng_testing.boot_raw_sched_starttime_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+ -+config LRNG_RAW_SCHED_NVCSW_ENTROPY -+ bool "Entropy test interface to task context switch numbers" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned task numbers of context switches that -+ are collected by the LRNG for statistical analysis. Extracted -+ noise data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the -+ lrng_raw_sched_nvcsw debugfs file. Using the option -+ lrng_testing.boot_raw_sched_nvcsw_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_SCHED_PERF -+ bool "LRNG scheduler entropy source performance monitor" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ With this option, the performance monitor of the LRNG -+ scheduler event handling code is enabled. The file provides -+ the execution time of the interrupt handler in cycles. -+ -+ The scheduler performance data can be obtained using -+ the lrng_sched_perf debugfs file. Using the option -+ lrng_testing.boot_sched_perf=1 the performance data of -+ the first 1000 entropy events since boot can be sampled. -+ -+comment "Auxiliary Test Interfaces" -+ -+config LRNG_ACVT_HASH -+ bool "Enable LRNG ACVT Hash interface" -+ select LRNG_TESTING -+ help -+ With this option, the LRNG built-in hash function used for -+ auxiliary pool management and prior to switching the -+ cryptographic backends is made available for ACVT. The -+ interface allows writing of the data to be hashed -+ into the interface. The read operation triggers the hash -+ operation to generate message digest. -+ -+ The ACVT interface is available with the lrng_acvt_hash -+ debugfs file. -+ -+config LRNG_RUNTIME_MAX_WO_RESEED_CONFIG -+ bool "Enable runtime configuration of max reseed threshold" -+ help -+ When enabling this option, the LRNG provides an interface -+ allowing the setting of the maximum number of DRNG generate -+ operations without a reseed that has full entropy. The -+ interface is lrng_drng.max_wo_reseed. -+ -+config LRNG_TEST_CPU_ES_COMPRESSION -+ bool "Force CPU ES compression operation" -+ help -+ When enabling this option, the CPU ES compression operation -+ is forced by setting an arbitrary value > 1 for the data -+ multiplier even when the CPU ES would deliver full entropy. -+ This allows testing of the compression operation. It -+ therefore forces to pull more data from the CPU ES -+ than what may be required. -+ -+endif #LRNG_TESTING_MENU -+ -+config LRNG_SELFTEST -+ bool "Enable power-on and on-demand self-tests" -+ help -+ The power-on self-tests are executed during boot time -+ covering the ChaCha20 DRNG, the hash operation used for -+ processing the entropy pools and the auxiliary pool, and -+ the time stamp management of the LRNG. -+ -+ The on-demand self-tests are triggered by writing any -+ value into the SysFS file selftest_status. At the same -+ time, when reading this file, the test status is -+ returned. A zero indicates that all tests were executed -+ successfully. -+ -+ If unsure, say Y. -+ -+if LRNG_SELFTEST -+ -+config LRNG_SELFTEST_PANIC -+ bool "Panic the kernel upon self-test failure" -+ help -+ If the option is enabled, the kernel is terminated if an -+ LRNG power-on self-test failure is detected. -+ -+endif # LRNG_SELFTEST -+ -+endif # LRNG -+ -+endmenu # LRNG -diff --git a/drivers/char/lrng/Makefile b/drivers/char/lrng/Makefile -new file mode 100644 -index 000000000000..f61fc40f4620 ---- /dev/null -+++ b/drivers/char/lrng/Makefile -@@ -0,0 +1,39 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# Makefile for the Entropy Source and DRNG Manager. -+# -+ -+obj-y += lrng_es_mgr.o lrng_drng_mgr.o \ -+ lrng_es_aux.o -+obj-$(CONFIG_LRNG_SHA256) += lrng_sha256.o -+obj-$(CONFIG_LRNG_SHA1) += lrng_sha1.o -+ -+obj-$(CONFIG_SYSCTL) += lrng_proc.o -+obj-$(CONFIG_LRNG_SYSCTL) += lrng_sysctl.o -+obj-$(CONFIG_NUMA) += lrng_numa.o -+ -+obj-$(CONFIG_LRNG_SWITCH) += lrng_switch.o -+obj-$(CONFIG_LRNG_HASH_KCAPI) += lrng_hash_kcapi.o -+obj-$(CONFIG_LRNG_DRNG_CHACHA20) += lrng_drng_chacha20.o -+obj-$(CONFIG_LRNG_DRBG) += lrng_drng_drbg.o -+obj-$(CONFIG_LRNG_DRNG_KCAPI) += lrng_drng_kcapi.o -+obj-$(CONFIG_LRNG_DRNG_ATOMIC) += lrng_drng_atomic.o -+ -+obj-$(CONFIG_LRNG_TIMER_COMMON) += lrng_es_timer_common.o -+obj-$(CONFIG_LRNG_IRQ) += lrng_es_irq.o -+obj-$(CONFIG_LRNG_KERNEL_RNG) += lrng_es_krng.o -+obj-$(CONFIG_LRNG_SCHED) += lrng_es_sched.o -+obj-$(CONFIG_LRNG_CPU) += lrng_es_cpu.o -+obj-$(CONFIG_LRNG_JENT) += lrng_es_jent.o -+ -+obj-$(CONFIG_LRNG_HEALTH_TESTS) += lrng_health.o -+obj-$(CONFIG_LRNG_TESTING) += lrng_testing.o -+obj-$(CONFIG_LRNG_SELFTEST) += lrng_selftest.o -+ -+obj-$(CONFIG_LRNG_COMMON_DEV_IF) += lrng_interface_dev_common.o -+obj-$(CONFIG_LRNG_RANDOM_IF) += lrng_interface_random_user.o \ -+ lrng_interface_random_kernel.o \ -+ lrng_interface_aux.o -+obj-$(CONFIG_LRNG_KCAPI_IF) += lrng_interface_kcapi.o -+obj-$(CONFIG_LRNG_DEV_IF) += lrng_interface_dev.o -+obj-$(CONFIG_LRNG_HWRAND_IF) += lrng_interface_hwrand.o -diff --git a/drivers/char/lrng/lrng_definitions.h b/drivers/char/lrng/lrng_definitions.h -new file mode 100644 -index 000000000000..50f87c40f701 ---- /dev/null -+++ b/drivers/char/lrng/lrng_definitions.h -@@ -0,0 +1,151 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DEFINITIONS_H -+#define _LRNG_DEFINITIONS_H -+ -+#include -+#include -+#include -+ -+/*************************** General LRNG parameter ***************************/ -+ -+/* -+ * Specific settings for different use cases -+ */ -+#ifdef CONFIG_CRYPTO_FIPS -+# define LRNG_OVERSAMPLE_ES_BITS 64 -+# define LRNG_SEED_BUFFER_INIT_ADD_BITS 128 -+#else /* CONFIG_CRYPTO_FIPS */ -+# define LRNG_OVERSAMPLE_ES_BITS 0 -+# define LRNG_SEED_BUFFER_INIT_ADD_BITS 0 -+#endif /* CONFIG_CRYPTO_FIPS */ -+ -+/* Security strength of LRNG -- this must match DRNG security strength */ -+#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32 -+#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8) -+#define LRNG_DRNG_INIT_SEED_SIZE_BITS \ -+ (LRNG_DRNG_SECURITY_STRENGTH_BITS + LRNG_SEED_BUFFER_INIT_ADD_BITS) -+#define LRNG_DRNG_INIT_SEED_SIZE_BYTES (LRNG_DRNG_INIT_SEED_SIZE_BITS >> 3) -+ -+/* -+ * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is -+ * considered a safer margin. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_DRNG_MAX_REQSIZE (1<<12) -+ -+/* -+ * SP800-90A defines a maximum number of requests between reseeds of 2^48. -+ * The given value is considered a much safer margin, balancing requests for -+ * frequent reseeds with the need to conserve entropy. This value MUST NOT be -+ * larger than INT_MAX because it is used in an atomic_t. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_DRNG_RESEED_THRESH (1<<20) -+ -+/* -+ * Maximum DRNG generation operations without reseed having full entropy -+ * This value defines the absolute maximum value of DRNG generation operations -+ * without a reseed holding full entropy. LRNG_DRNG_RESEED_THRESH is the -+ * threshold when a new reseed is attempted. But it is possible that this fails -+ * to deliver full entropy. In this case the DRNG will continue to provide data -+ * even though it was not reseeded with full entropy. To avoid in the extreme -+ * case that no reseed is performed for too long, this threshold is enforced. -+ * If that absolute low value is reached, the LRNG is marked as not operational. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_DRNG_MAX_WITHOUT_RESEED (1<<30) -+ -+/* -+ * Min required seed entropy is 128 bits covering the minimum entropy -+ * requirement of SP800-131A and the German BSI's TR02102. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS -+#define LRNG_MIN_SEED_ENTROPY_BITS 128 -+#define LRNG_INIT_ENTROPY_BITS 32 -+ -+/* -+ * Wakeup value -+ * -+ * This value is allowed to be changed but must not be larger than the -+ * digest size of the hash operation used update the aux_pool. -+ */ -+#ifdef CONFIG_LRNG_SHA256 -+# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE -+#else -+# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE -+#endif -+#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE -+ -+/* -+ * If the switching support is configured, we must provide support up to -+ * the largest digest size. Without switching support, we know it is only -+ * the built-in digest size. -+ */ -+#ifdef CONFIG_LRNG_SWITCH -+# define LRNG_MAX_DIGESTSIZE 64 -+#else -+# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE -+#endif -+ -+/* -+ * Oversampling factor of timer-based events to obtain -+ * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a -+ * high-resolution time stamp is not available. In this case, jiffies and -+ * register contents are used to fill the entropy pool. These noise sources -+ * are much less entropic than the high-resolution timer. The entropy content -+ * is the entropy content assumed with LRNG_[IRQ|SCHED]_ENTROPY_BITS divided by -+ * LRNG_ES_OVERSAMPLING_FACTOR. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_ES_OVERSAMPLING_FACTOR 10 -+ -+/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */ -+#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN -+ -+/* -+ * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK -+ * as it will be casted into a struct shash_desc. -+ */ -+#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE) -+ -+/****************************** Helper code ***********************************/ -+ -+static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits) -+{ -+ /* Obtain entropy statement */ -+ ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS; -+ /* Cap entropy to buffer size in bits */ -+ ent_bits = min_t(u32, ent_bits, requested_bits); -+ return ent_bits; -+} -+ -+/* Convert entropy in bits into nr. of events with the same entropy content. */ -+static inline u32 lrng_entropy_to_data(u32 entropy_bits, u32 entropy_rate) -+{ -+ return ((entropy_bits * entropy_rate) / -+ LRNG_DRNG_SECURITY_STRENGTH_BITS); -+} -+ -+/* Convert number of events into entropy value. */ -+static inline u32 lrng_data_to_entropy(u32 num, u32 entropy_rate) -+{ -+ return ((num * LRNG_DRNG_SECURITY_STRENGTH_BITS) / -+ entropy_rate); -+} -+ -+static inline u32 atomic_read_u32(atomic_t *v) -+{ -+ return (u32)atomic_read(v); -+} -+ -+#endif /* _LRNG_DEFINITIONS_H */ -diff --git a/drivers/char/lrng/lrng_drng_atomic.c b/drivers/char/lrng/lrng_drng_atomic.c -new file mode 100644 -index 000000000000..c112366677b6 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_atomic.c -@@ -0,0 +1,171 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG DRNG for atomic contexts -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include "lrng_drng_atomic.h" -+#include "lrng_drng_chacha20.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_sha.h" -+ -+static struct chacha20_state chacha20_atomic = { -+ LRNG_CC20_INIT_RFC7539(.block) -+}; -+ -+/* -+ * DRNG usable in atomic context. This DRNG will always use the ChaCha20 -+ * DRNG. It will never benefit from a DRNG switch like the "regular" DRNG. If -+ * there was no DRNG switch, the atomic DRNG is identical to the "regular" DRNG. -+ * -+ * The reason for having this is due to the fact that DRNGs other than -+ * the ChaCha20 DRNG may sleep. -+ */ -+static struct lrng_drng lrng_drng_atomic = { -+ LRNG_DRNG_STATE_INIT(lrng_drng_atomic, -+ &chacha20_atomic, NULL, -+ &lrng_cc20_drng_cb, &lrng_sha_hash_cb), -+ .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_atomic.spin_lock) -+}; -+ -+void lrng_drng_atomic_reset(void) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); -+ lrng_drng_reset(&lrng_drng_atomic); -+ spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); -+} -+ -+void lrng_drng_atomic_force_reseed(void) -+{ -+ lrng_drng_atomic.force_reseed = lrng_drng_atomic.fully_seeded; -+} -+ -+static bool lrng_drng_atomic_must_reseed(struct lrng_drng *drng) -+{ -+ return (!drng->fully_seeded || -+ atomic_read(&lrng_drng_atomic.requests) == 0 || -+ drng->force_reseed || -+ time_after(jiffies, -+ drng->last_seeded + lrng_drng_reseed_max_time * HZ)); -+} -+ -+void lrng_drng_atomic_seed_drng(struct lrng_drng *regular_drng) -+{ -+ u8 seedbuf[LRNG_DRNG_SECURITY_STRENGTH_BYTES] -+ __aligned(LRNG_KCAPI_ALIGN); -+ int ret; -+ -+ if (!lrng_drng_atomic_must_reseed(&lrng_drng_atomic)) -+ return; -+ -+ /* -+ * Reseed atomic DRNG another DRNG "regular" while this regular DRNG -+ * is reseeded. Therefore, this code operates in non-atomic context and -+ * thus can use the lrng_drng_get function to get random numbers from -+ * the just freshly seeded DRNG. -+ */ -+ ret = lrng_drng_get(regular_drng, seedbuf, sizeof(seedbuf)); -+ -+ if (ret < 0) { -+ pr_warn("Error generating random numbers for atomic DRNG: %d\n", -+ ret); -+ } else { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); -+ lrng_drng_inject(&lrng_drng_atomic, seedbuf, ret, -+ regular_drng->fully_seeded, "atomic"); -+ spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); -+ } -+ memzero_explicit(&seedbuf, sizeof(seedbuf)); -+} -+ -+void lrng_drng_atomic_seed_es(void) -+{ -+ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); -+ struct lrng_drng *drng = &lrng_drng_atomic; -+ unsigned long flags; -+ u32 requested_bits = LRNG_MIN_SEED_ENTROPY_BITS; -+ -+ if (lrng_drng_atomic.fully_seeded) -+ return; -+ -+ /* -+ * When the LRNG reached the minimally seeded level, leave 256 bits of -+ * entropy for the regular DRNG. In addition ensure that additional -+ * 256 bits are available for the atomic DRNG to get to the fully -+ * seeded stage of the LRNG. -+ */ -+ if (lrng_state_min_seeded()) { -+ u32 avail_bits = lrng_avail_entropy(); -+ -+ requested_bits = -+ (avail_bits >= 2 * LRNG_FULL_SEED_ENTROPY_BITS) ? -+ LRNG_FULL_SEED_ENTROPY_BITS : 0; -+ -+ if (!requested_bits) -+ return; -+ } -+ -+ pr_debug("atomic DRNG seeding attempt to pull %u bits of entropy directly from entropy sources\n", -+ requested_bits); -+ -+ lrng_fill_seed_buffer(&seedbuf, requested_bits); -+ spin_lock_irqsave(&drng->spin_lock, flags); -+ lrng_drng_inject(&lrng_drng_atomic, (u8 *)&seedbuf, sizeof(seedbuf), -+ lrng_fully_seeded(drng->fully_seeded, -+ lrng_entropy_rate_eb(&seedbuf)), -+ "atomic"); -+ spin_unlock_irqrestore(&drng->spin_lock, flags); -+ lrng_init_ops(&seedbuf); -+ -+ /* -+ * Do not call lrng_init_ops(seedbuf) here as the atomic DRNG does not -+ * serve common users. -+ */ -+ memzero_explicit(&seedbuf, sizeof(seedbuf)); -+} -+ -+static void lrng_drng_atomic_get(u8 *outbuf, u32 outbuflen) -+{ -+ struct lrng_drng *drng = &lrng_drng_atomic; -+ unsigned long flags; -+ -+ if (!outbuf || !outbuflen) -+ return; -+ -+ outbuflen = min_t(size_t, outbuflen, INT_MAX); -+ -+ while (outbuflen) { -+ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); -+ int ret; -+ -+ atomic_dec(&drng->requests); -+ -+ spin_lock_irqsave(&drng->spin_lock, flags); -+ ret = drng->drng_cb->drng_generate(drng->drng, outbuf, todo); -+ spin_unlock_irqrestore(&drng->spin_lock, flags); -+ if (ret <= 0) { -+ pr_warn("getting random data from DRNG failed (%d)\n", -+ ret); -+ return; -+ } -+ outbuflen -= ret; -+ outbuf += ret; -+ } -+} -+ -+void lrng_get_random_bytes(void *buf, int nbytes) -+{ -+ lrng_drng_atomic_get((u8 *)buf, (u32)nbytes); -+ lrng_debug_report_seedlevel("lrng_get_random_bytes"); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes); -diff --git a/drivers/char/lrng/lrng_drng_atomic.h b/drivers/char/lrng/lrng_drng_atomic.h -new file mode 100644 -index 000000000000..d5d24fc94f2e ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_atomic.h -@@ -0,0 +1,23 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DRNG_ATOMIC_H -+#define _LRNG_DRNG_ATOMIC_H -+ -+#include "lrng_drng_mgr.h" -+ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+void lrng_drng_atomic_reset(void); -+void lrng_drng_atomic_seed_drng(struct lrng_drng *drng); -+void lrng_drng_atomic_seed_es(void); -+void lrng_drng_atomic_force_reseed(void); -+#else /* CONFIG_LRNG_DRNG_ATOMIC */ -+static inline void lrng_drng_atomic_reset(void) { } -+static inline void lrng_drng_atomic_seed_drng(struct lrng_drng *drng) { } -+static inline void lrng_drng_atomic_seed_es(void) { } -+static inline void lrng_drng_atomic_force_reseed(void) { } -+#endif /* CONFIG_LRNG_DRNG_ATOMIC */ -+ -+#endif /* _LRNG_DRNG_ATOMIC_H */ -diff --git a/drivers/char/lrng/lrng_drng_chacha20.c b/drivers/char/lrng/lrng_drng_chacha20.c -new file mode 100644 -index 000000000000..31be102e3007 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_chacha20.c -@@ -0,0 +1,195 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the cryptographic primitives using -+ * ChaCha20 cipher implementations. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_chacha20.h" -+ -+/******************************* ChaCha20 DRNG *******************************/ -+ -+#define CHACHA_BLOCK_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) -+ -+/* -+ * Update of the ChaCha20 state by either using an unused buffer part or by -+ * generating one ChaCha20 block which is half of the state of the ChaCha20. -+ * The block is XORed into the key part of the state. This shall ensure -+ * backtracking resistance as well as a proper mix of the ChaCha20 state once -+ * the key is injected. -+ */ -+static void lrng_chacha20_update(struct chacha20_state *chacha20_state, -+ __le32 *buf, u32 used_words) -+{ -+ struct chacha20_block *chacha20 = &chacha20_state->block; -+ u32 i; -+ __le32 tmp[CHACHA_BLOCK_WORDS]; -+ -+ BUILD_BUG_ON(sizeof(struct chacha20_block) != CHACHA_BLOCK_SIZE); -+ BUILD_BUG_ON(CHACHA_BLOCK_SIZE != 2 * CHACHA_KEY_SIZE); -+ -+ if (used_words > CHACHA_KEY_SIZE_WORDS) { -+ chacha20_block(&chacha20->constants[0], (u8 *)tmp); -+ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) -+ chacha20->key.u[i] ^= le32_to_cpu(tmp[i]); -+ memzero_explicit(tmp, sizeof(tmp)); -+ } else { -+ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) -+ chacha20->key.u[i] ^= le32_to_cpu(buf[i + used_words]); -+ } -+ -+ /* Deterministic increment of nonce as required in RFC 7539 chapter 4 */ -+ chacha20->nonce[0]++; -+ if (chacha20->nonce[0] == 0) { -+ chacha20->nonce[1]++; -+ if (chacha20->nonce[1] == 0) -+ chacha20->nonce[2]++; -+ } -+ -+ /* Leave counter untouched as it is start value is undefined in RFC */ -+} -+ -+/* -+ * Seed the ChaCha20 DRNG by injecting the input data into the key part of -+ * the ChaCha20 state. If the input data is longer than the ChaCha20 key size, -+ * perform a ChaCha20 operation after processing of key size input data. -+ * This operation shall spread out the entropy into the ChaCha20 state before -+ * new entropy is injected into the key part. -+ */ -+static int lrng_cc20_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) -+{ -+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; -+ struct chacha20_block *chacha20 = &chacha20_state->block; -+ -+ while (inbuflen) { -+ u32 i, todo = min_t(u32, inbuflen, CHACHA_KEY_SIZE); -+ -+ for (i = 0; i < todo; i++) -+ chacha20->key.b[i] ^= inbuf[i]; -+ -+ /* Break potential dependencies between the inbuf key blocks */ -+ lrng_chacha20_update(chacha20_state, NULL, -+ CHACHA_BLOCK_WORDS); -+ inbuf += todo; -+ inbuflen -= todo; -+ } -+ -+ return 0; -+} -+ -+/* -+ * Chacha20 DRNG generation of random numbers: the stream output of ChaCha20 -+ * is the random number. After the completion of the generation of the -+ * stream, the entire ChaCha20 state is updated. -+ * -+ * Note, as the ChaCha20 implements a 32 bit counter, we must ensure -+ * that this function is only invoked for at most 2^32 - 1 ChaCha20 blocks -+ * before a reseed or an update happens. This is ensured by the variable -+ * outbuflen which is a 32 bit integer defining the number of bytes to be -+ * generated by the ChaCha20 DRNG. At the end of this function, an update -+ * operation is invoked which implies that the 32 bit counter will never be -+ * overflown in this implementation. -+ */ -+static int lrng_cc20_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) -+{ -+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; -+ struct chacha20_block *chacha20 = &chacha20_state->block; -+ __le32 aligned_buf[CHACHA_BLOCK_WORDS]; -+ u32 ret = outbuflen, used = CHACHA_BLOCK_WORDS; -+ int zeroize_buf = 0; -+ -+ while (outbuflen >= CHACHA_BLOCK_SIZE) { -+ chacha20_block(&chacha20->constants[0], outbuf); -+ outbuf += CHACHA_BLOCK_SIZE; -+ outbuflen -= CHACHA_BLOCK_SIZE; -+ } -+ -+ if (outbuflen) { -+ chacha20_block(&chacha20->constants[0], (u8 *)aligned_buf); -+ memcpy(outbuf, aligned_buf, outbuflen); -+ used = ((outbuflen + sizeof(aligned_buf[0]) - 1) / -+ sizeof(aligned_buf[0])); -+ zeroize_buf = 1; -+ } -+ -+ lrng_chacha20_update(chacha20_state, aligned_buf, used); -+ -+ if (zeroize_buf) -+ memzero_explicit(aligned_buf, sizeof(aligned_buf)); -+ -+ return ret; -+} -+ -+/* -+ * Allocation of the DRNG state -+ */ -+static void *lrng_cc20_drng_alloc(u32 sec_strength) -+{ -+ struct chacha20_state *state = NULL; -+ -+ if (sec_strength > CHACHA_KEY_SIZE) { -+ pr_err("Security strength of ChaCha20 DRNG (%u bits) lower than requested by LRNG (%u bits)\n", -+ CHACHA_KEY_SIZE * 8, sec_strength * 8); -+ return ERR_PTR(-EINVAL); -+ } -+ if (sec_strength < CHACHA_KEY_SIZE) -+ pr_warn("Security strength of ChaCha20 DRNG (%u bits) higher than requested by LRNG (%u bits)\n", -+ CHACHA_KEY_SIZE * 8, sec_strength * 8); -+ -+ state = kmalloc(sizeof(struct chacha20_state), GFP_KERNEL); -+ if (!state) -+ return ERR_PTR(-ENOMEM); -+ pr_debug("memory for ChaCha20 core allocated\n"); -+ -+ lrng_cc20_init_rfc7539(&state->block); -+ -+ return state; -+} -+ -+static void lrng_cc20_drng_dealloc(void *drng) -+{ -+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; -+ -+ pr_debug("ChaCha20 core zeroized and freed\n"); -+ kfree_sensitive(chacha20_state); -+} -+ -+static const char *lrng_cc20_drng_name(void) -+{ -+ return "ChaCha20 DRNG"; -+} -+ -+const struct lrng_drng_cb lrng_cc20_drng_cb = { -+ .drng_name = lrng_cc20_drng_name, -+ .drng_alloc = lrng_cc20_drng_alloc, -+ .drng_dealloc = lrng_cc20_drng_dealloc, -+ .drng_seed = lrng_cc20_drng_seed_helper, -+ .drng_generate = lrng_cc20_drng_generate_helper, -+}; -+ -+#if !defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) && \ -+ !defined(CONFIG_LRNG_DRNG_ATOMIC) -+static int __init lrng_cc20_drng_init(void) -+{ -+ return lrng_set_drng_cb(&lrng_cc20_drng_cb); -+} -+ -+static void __exit lrng_cc20_drng_exit(void) -+{ -+ lrng_set_drng_cb(NULL); -+} -+ -+late_initcall(lrng_cc20_drng_init); -+module_exit(lrng_cc20_drng_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - ChaCha20-based DRNG backend"); -+#endif /* CONFIG_LRNG_DFLT_DRNG_CHACHA20 */ -diff --git a/drivers/char/lrng/lrng_drng_chacha20.h b/drivers/char/lrng/lrng_drng_chacha20.h -new file mode 100644 -index 000000000000..fee6571281b6 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_chacha20.h -@@ -0,0 +1,42 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG ChaCha20 definitions -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_CHACHA20_H -+#define _LRNG_CHACHA20_H -+ -+#include -+ -+/* State according to RFC 7539 section 2.3 */ -+struct chacha20_block { -+ u32 constants[4]; -+ union { -+#define CHACHA_KEY_SIZE_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) -+ u32 u[CHACHA_KEY_SIZE_WORDS]; -+ u8 b[CHACHA_KEY_SIZE]; -+ } key; -+ u32 counter; -+ u32 nonce[3]; -+}; -+ -+struct chacha20_state { -+ struct chacha20_block block; -+}; -+ -+static inline void lrng_cc20_init_rfc7539(struct chacha20_block *chacha20) -+{ -+ chacha_init_consts(chacha20->constants); -+} -+ -+#define LRNG_CC20_INIT_RFC7539(x) \ -+ x.constants[0] = 0x61707865, \ -+ x.constants[1] = 0x3320646e, \ -+ x.constants[2] = 0x79622d32, \ -+ x.constants[3] = 0x6b206574, -+ -+extern const struct lrng_drng_cb lrng_cc20_drng_cb; -+ -+#endif /* _LRNG_CHACHA20_H */ -diff --git a/drivers/char/lrng/lrng_drng_drbg.c b/drivers/char/lrng/lrng_drng_drbg.c -new file mode 100644 -index 000000000000..1feec2c095a5 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_drbg.c -@@ -0,0 +1,179 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the cryptographic primitives using the -+ * kernel crypto API and its DRBG. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_drbg.h" -+ -+/* -+ * Define a DRBG plus a hash / MAC used to extract data from the entropy pool. -+ * For LRNG_HASH_NAME you can use a hash or a MAC (HMAC or CMAC) of your choice -+ * (Note, you should use the suggested selections below -- using SHA-1 or MD5 -+ * is not wise). The idea is that the used cipher primitive can be selected to -+ * be the same as used for the DRBG. I.e. the LRNG only uses one cipher -+ * primitive using the same cipher implementation with the options offered in -+ * the following. This means, if the CTR DRBG is selected and AES-NI is present, -+ * both the CTR DRBG and the selected cmac(aes) use AES-NI. -+ * -+ * The security strengths of the DRBGs are all 256 bits according to -+ * SP800-57 section 5.6.1. -+ * -+ * This definition is allowed to be changed. -+ */ -+#ifdef CONFIG_CRYPTO_DRBG_CTR -+static unsigned int lrng_drbg_type = 0; -+#elif defined CONFIG_CRYPTO_DRBG_HMAC -+static unsigned int lrng_drbg_type = 1; -+#elif defined CONFIG_CRYPTO_DRBG_HASH -+static unsigned int lrng_drbg_type = 2; -+#else -+#error "Unknown DRBG in use" -+#endif -+ -+/* The parameter must be r/o in sysfs as otherwise races appear. */ -+module_param(lrng_drbg_type, uint, 0444); -+MODULE_PARM_DESC(lrng_drbg_type, "DRBG type used for LRNG (0->CTR_DRBG, 1->HMAC_DRBG, 2->Hash_DRBG)"); -+ -+struct lrng_drbg { -+ const char *hash_name; -+ const char *drbg_core; -+}; -+ -+static const struct lrng_drbg lrng_drbg_types[] = { -+ { /* CTR_DRBG with AES-256 using derivation function */ -+ .drbg_core = "drbg_nopr_ctr_aes256", -+ }, { /* HMAC_DRBG with SHA-512 */ -+ .drbg_core = "drbg_nopr_hmac_sha512", -+ }, { /* Hash_DRBG with SHA-512 using derivation function */ -+ .drbg_core = "drbg_nopr_sha512" -+ } -+}; -+ -+static int lrng_drbg_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) -+{ -+ struct drbg_state *drbg = (struct drbg_state *)drng; -+ LIST_HEAD(seedlist); -+ struct drbg_string data; -+ int ret; -+ -+ drbg_string_fill(&data, inbuf, inbuflen); -+ list_add_tail(&data.list, &seedlist); -+ ret = drbg->d_ops->update(drbg, &seedlist, drbg->seeded); -+ -+ if (ret >= 0) -+ drbg->seeded = true; -+ -+ return ret; -+} -+ -+static int lrng_drbg_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) -+{ -+ struct drbg_state *drbg = (struct drbg_state *)drng; -+ -+ return drbg->d_ops->generate(drbg, outbuf, outbuflen, NULL); -+} -+ -+static void *lrng_drbg_drng_alloc(u32 sec_strength) -+{ -+ struct drbg_state *drbg; -+ int coreref = -1; -+ bool pr = false; -+ int ret; -+ -+ drbg_convert_tfm_core(lrng_drbg_types[lrng_drbg_type].drbg_core, -+ &coreref, &pr); -+ if (coreref < 0) -+ return ERR_PTR(-EFAULT); -+ -+ drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); -+ if (!drbg) -+ return ERR_PTR(-ENOMEM); -+ -+ drbg->core = &drbg_cores[coreref]; -+ drbg->seeded = false; -+ ret = drbg_alloc_state(drbg); -+ if (ret) -+ goto err; -+ -+ if (sec_strength > drbg_sec_strength(drbg->core->flags)) { -+ pr_err("Security strength of DRBG (%u bits) lower than requested by LRNG (%u bits)\n", -+ drbg_sec_strength(drbg->core->flags) * 8, -+ sec_strength * 8); -+ goto dealloc; -+ } -+ -+ if (sec_strength < drbg_sec_strength(drbg->core->flags)) -+ pr_warn("Security strength of DRBG (%u bits) higher than requested by LRNG (%u bits)\n", -+ drbg_sec_strength(drbg->core->flags) * 8, -+ sec_strength * 8); -+ -+ pr_info("DRBG with %s core allocated\n", drbg->core->backend_cra_name); -+ -+ return drbg; -+ -+dealloc: -+ if (drbg->d_ops) -+ drbg->d_ops->crypto_fini(drbg); -+ drbg_dealloc_state(drbg); -+err: -+ kfree(drbg); -+ return ERR_PTR(-EINVAL); -+} -+ -+static void lrng_drbg_drng_dealloc(void *drng) -+{ -+ struct drbg_state *drbg = (struct drbg_state *)drng; -+ -+ if (drbg && drbg->d_ops) -+ drbg->d_ops->crypto_fini(drbg); -+ drbg_dealloc_state(drbg); -+ kfree_sensitive(drbg); -+ pr_info("DRBG deallocated\n"); -+} -+ -+static const char *lrng_drbg_name(void) -+{ -+ return lrng_drbg_types[lrng_drbg_type].drbg_core; -+} -+ -+const struct lrng_drng_cb lrng_drbg_cb = { -+ .drng_name = lrng_drbg_name, -+ .drng_alloc = lrng_drbg_drng_alloc, -+ .drng_dealloc = lrng_drbg_drng_dealloc, -+ .drng_seed = lrng_drbg_drng_seed_helper, -+ .drng_generate = lrng_drbg_drng_generate_helper, -+}; -+ -+#ifndef CONFIG_LRNG_DFLT_DRNG_DRBG -+static int __init lrng_drbg_init(void) -+{ -+ if (lrng_drbg_type >= ARRAY_SIZE(lrng_drbg_types)) { -+ pr_err("lrng_drbg_type parameter too large (given %u - max: %lu)", -+ lrng_drbg_type, -+ (unsigned long)ARRAY_SIZE(lrng_drbg_types) - 1); -+ return -EAGAIN; -+ } -+ return lrng_set_drng_cb(&lrng_drbg_cb); -+} -+ -+static void __exit lrng_drbg_exit(void) -+{ -+ lrng_set_drng_cb(NULL); -+} -+ -+late_initcall(lrng_drbg_init); -+module_exit(lrng_drbg_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - SP800-90A DRBG backend"); -+#endif /* CONFIG_LRNG_DFLT_DRNG_DRBG */ -diff --git a/drivers/char/lrng/lrng_drng_drbg.h b/drivers/char/lrng/lrng_drng_drbg.h -new file mode 100644 -index 000000000000..b3d556caf294 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_drbg.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG SP800-90A definitions -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DRBG_H -+#define _LRNG_DRBG_H -+ -+extern const struct lrng_drng_cb lrng_drbg_cb; -+ -+#endif /* _LRNG_DRBG_H */ -diff --git a/drivers/char/lrng/lrng_drng_kcapi.c b/drivers/char/lrng/lrng_drng_kcapi.c -new file mode 100644 -index 000000000000..a204bcf52a9a ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_kcapi.c -@@ -0,0 +1,208 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the cryptographic primitives using the -+ * kernel crypto API. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_kcapi.h" -+ -+static char *drng_name = NULL; -+module_param(drng_name, charp, 0444); -+MODULE_PARM_DESC(drng_name, "Kernel crypto API name of DRNG"); -+ -+static char *seed_hash = NULL; -+module_param(seed_hash, charp, 0444); -+MODULE_PARM_DESC(seed_hash, -+ "Kernel crypto API name of hash with output size equal to seedsize of DRNG to bring seed string to the size required by the DRNG"); -+ -+struct lrng_drng_info { -+ struct crypto_rng *kcapi_rng; -+ struct crypto_shash *hash_tfm; -+}; -+ -+static int lrng_kcapi_drng_seed_helper(void *drng, const u8 *inbuf, -+ u32 inbuflen) -+{ -+ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; -+ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; -+ struct crypto_shash *hash_tfm = lrng_drng_info->hash_tfm; -+ SHASH_DESC_ON_STACK(shash, hash_tfm); -+ u32 digestsize; -+ u8 digest[HASH_MAX_DIGESTSIZE] __aligned(8); -+ int ret; -+ -+ if (!hash_tfm) -+ return crypto_rng_reset(kcapi_rng, inbuf, inbuflen); -+ -+ shash->tfm = hash_tfm; -+ digestsize = crypto_shash_digestsize(hash_tfm); -+ -+ ret = crypto_shash_digest(shash, inbuf, inbuflen, digest); -+ shash_desc_zero(shash); -+ if (ret) -+ return ret; -+ -+ ret = crypto_rng_reset(kcapi_rng, digest, digestsize); -+ if (ret) -+ return ret; -+ -+ memzero_explicit(digest, digestsize); -+ return 0; -+} -+ -+static int lrng_kcapi_drng_generate_helper(void *drng, u8 *outbuf, -+ u32 outbuflen) -+{ -+ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; -+ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; -+ int ret = crypto_rng_get_bytes(kcapi_rng, outbuf, outbuflen); -+ -+ if (ret < 0) -+ return ret; -+ -+ return outbuflen; -+} -+ -+static void *lrng_kcapi_drng_alloc(u32 sec_strength) -+{ -+ struct lrng_drng_info *lrng_drng_info; -+ struct crypto_rng *kcapi_rng; -+ u32 time = random_get_entropy(); -+ int seedsize, rv; -+ void *ret = ERR_PTR(-ENOMEM); -+ -+ if (!drng_name) { -+ pr_err("DRNG name missing\n"); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ if (!memcmp(drng_name, "stdrng", 6) || -+ !memcmp(drng_name, "lrng", 4) || -+ !memcmp(drng_name, "drbg", 4) || -+ !memcmp(drng_name, "jitterentropy_rng", 17)) { -+ pr_err("Refusing to load the requested random number generator\n"); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ lrng_drng_info = kzalloc(sizeof(*lrng_drng_info), GFP_KERNEL); -+ if (!lrng_drng_info) -+ return ERR_PTR(-ENOMEM); -+ -+ kcapi_rng = crypto_alloc_rng(drng_name, 0, 0); -+ if (IS_ERR(kcapi_rng)) { -+ pr_err("DRNG %s cannot be allocated\n", drng_name); -+ ret = ERR_CAST(kcapi_rng); -+ goto free; -+ } -+ -+ lrng_drng_info->kcapi_rng = kcapi_rng; -+ -+ seedsize = crypto_rng_seedsize(kcapi_rng); -+ if (seedsize) { -+ struct crypto_shash *hash_tfm; -+ -+ if (!seed_hash) { -+ switch (seedsize) { -+ case 32: -+ seed_hash = "sha256"; -+ break; -+ case 48: -+ seed_hash = "sha384"; -+ break; -+ case 64: -+ seed_hash = "sha512"; -+ break; -+ default: -+ pr_err("Seed size %d cannot be processed\n", -+ seedsize); -+ goto dealloc; -+ } -+ } -+ -+ hash_tfm = crypto_alloc_shash(seed_hash, 0, 0); -+ if (IS_ERR(hash_tfm)) { -+ ret = ERR_CAST(hash_tfm); -+ goto dealloc; -+ } -+ -+ if (seedsize != crypto_shash_digestsize(hash_tfm)) { -+ pr_err("Seed hash output size not equal to DRNG seed size\n"); -+ crypto_free_shash(hash_tfm); -+ ret = ERR_PTR(-EINVAL); -+ goto dealloc; -+ } -+ -+ lrng_drng_info->hash_tfm = hash_tfm; -+ -+ pr_info("Seed hash %s allocated\n", seed_hash); -+ } -+ -+ rv = lrng_kcapi_drng_seed_helper(lrng_drng_info, (u8 *)(&time), -+ sizeof(time)); -+ if (rv) { -+ ret = ERR_PTR(rv); -+ goto dealloc; -+ } -+ -+ pr_info("Kernel crypto API DRNG %s allocated\n", drng_name); -+ -+ return lrng_drng_info; -+ -+dealloc: -+ crypto_free_rng(kcapi_rng); -+free: -+ kfree(lrng_drng_info); -+ return ret; -+} -+ -+static void lrng_kcapi_drng_dealloc(void *drng) -+{ -+ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; -+ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; -+ -+ crypto_free_rng(kcapi_rng); -+ if (lrng_drng_info->hash_tfm) -+ crypto_free_shash(lrng_drng_info->hash_tfm); -+ kfree(lrng_drng_info); -+ pr_info("DRNG %s deallocated\n", drng_name); -+} -+ -+static const char *lrng_kcapi_drng_name(void) -+{ -+ return drng_name; -+} -+ -+const struct lrng_drng_cb lrng_kcapi_drng_cb = { -+ .drng_name = lrng_kcapi_drng_name, -+ .drng_alloc = lrng_kcapi_drng_alloc, -+ .drng_dealloc = lrng_kcapi_drng_dealloc, -+ .drng_seed = lrng_kcapi_drng_seed_helper, -+ .drng_generate = lrng_kcapi_drng_generate_helper, -+}; -+ -+#ifndef CONFIG_LRNG_DFLT_DRNG_KCAPI -+static int __init lrng_kcapi_init(void) -+{ -+ return lrng_set_drng_cb(&lrng_kcapi_drng_cb); -+} -+static void __exit lrng_kcapi_exit(void) -+{ -+ lrng_set_drng_cb(NULL); -+} -+ -+late_initcall(lrng_kcapi_init); -+module_exit(lrng_kcapi_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - kernel crypto API DRNG backend"); -+#endif /* CONFIG_LRNG_DFLT_DRNG_KCAPI */ -diff --git a/drivers/char/lrng/lrng_drng_kcapi.h b/drivers/char/lrng/lrng_drng_kcapi.h -new file mode 100644 -index 000000000000..5db25aaf830c ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_kcapi.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG kernel crypto API DRNG definition -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_KCAPI_DRNG_H -+#define _LRNG_KCAPI_DRNG_H -+ -+extern const struct lrng_drng_cb lrng_kcapi_drng_cb; -+ -+#endif /* _LRNG_KCAPI_DRNG_H */ -diff --git a/drivers/char/lrng/lrng_drng_mgr.c b/drivers/char/lrng/lrng_drng_mgr.c -new file mode 100644 -index 000000000000..e9e08c186530 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_mgr.c -@@ -0,0 +1,657 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG DRNG management -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_atomic.h" -+#include "lrng_drng_chacha20.h" -+#include "lrng_drng_drbg.h" -+#include "lrng_drng_kcapi.h" -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_random_kernel.h" -+#include "lrng_numa.h" -+#include "lrng_sha.h" -+ -+/* -+ * Maximum number of seconds between DRNG reseed intervals of the DRNG. Note, -+ * this is enforced with the next request of random numbers from the -+ * DRNG. Setting this value to zero implies a reseeding attempt before every -+ * generated random number. -+ */ -+int lrng_drng_reseed_max_time = 600; -+ -+/* -+ * Is LRNG for general-purpose use (i.e. is at least the lrng_drng_init -+ * fully allocated)? -+ */ -+static atomic_t lrng_avail = ATOMIC_INIT(0); -+ -+/* Guard protecting all crypto callback update operation of all DRNGs. */ -+DEFINE_MUTEX(lrng_crypto_cb_update); -+ -+/* -+ * Default hash callback that provides the crypto primitive right from the -+ * kernel start. It must not perform any memory allocation operation, but -+ * simply perform the hash calculation. -+ */ -+const struct lrng_hash_cb *lrng_default_hash_cb = &lrng_sha_hash_cb; -+ -+/* -+ * Default DRNG callback that provides the crypto primitive which is -+ * allocated either during late kernel boot stage. So, it is permissible for -+ * the callback to perform memory allocation operations. -+ */ -+const struct lrng_drng_cb *lrng_default_drng_cb = -+#if defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) -+ &lrng_cc20_drng_cb; -+#elif defined(CONFIG_LRNG_DFLT_DRNG_DRBG) -+ &lrng_drbg_cb; -+#elif defined(CONFIG_LRNG_DFLT_DRNG_KCAPI) -+ &lrng_kcapi_drng_cb; -+#else -+#error "Unknown default DRNG selected" -+#endif -+ -+/* DRNG for non-atomic use cases */ -+static struct lrng_drng lrng_drng_init = { -+ LRNG_DRNG_STATE_INIT(lrng_drng_init, NULL, NULL, NULL, -+ &lrng_sha_hash_cb), -+ .lock = __MUTEX_INITIALIZER(lrng_drng_init.lock), -+}; -+ -+/* Prediction-resistance DRNG: only deliver as much data as received entropy */ -+static struct lrng_drng lrng_drng_pr = { -+ LRNG_DRNG_STATE_INIT(lrng_drng_pr, NULL, NULL, NULL, -+ &lrng_sha_hash_cb), -+ .lock = __MUTEX_INITIALIZER(lrng_drng_pr.lock), -+}; -+ -+static u32 max_wo_reseed = LRNG_DRNG_MAX_WITHOUT_RESEED; -+#ifdef CONFIG_LRNG_RUNTIME_MAX_WO_RESEED_CONFIG -+module_param(max_wo_reseed, uint, 0444); -+MODULE_PARM_DESC(max_wo_reseed, -+ "Maximum number of DRNG generate operation without full reseed\n"); -+#endif -+ -+/* Wait queue to wait until the LRNG is initialized - can freely be used */ -+DECLARE_WAIT_QUEUE_HEAD(lrng_init_wait); -+ -+/********************************** Helper ************************************/ -+ -+bool lrng_get_available(void) -+{ -+ return likely(atomic_read(&lrng_avail)); -+} -+ -+struct lrng_drng *lrng_drng_init_instance(void) -+{ -+ return &lrng_drng_init; -+} -+ -+struct lrng_drng *lrng_drng_pr_instance(void) -+{ -+ return &lrng_drng_pr; -+} -+ -+struct lrng_drng *lrng_drng_node_instance(void) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ int node = numa_node_id(); -+ -+ if (lrng_drng && lrng_drng[node]) -+ return lrng_drng[node]; -+ -+ return lrng_drng_init_instance(); -+} -+ -+void lrng_drng_reset(struct lrng_drng *drng) -+{ -+ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); -+ atomic_set(&drng->requests_since_fully_seeded, 0); -+ drng->last_seeded = jiffies; -+ drng->fully_seeded = false; -+ drng->force_reseed = true; -+ pr_debug("reset DRNG\n"); -+} -+ -+/* Initialize the DRNG, except the mutex lock */ -+int lrng_drng_alloc_common(struct lrng_drng *drng, -+ const struct lrng_drng_cb *drng_cb) -+{ -+ if (!drng || !drng_cb) -+ return -EINVAL; -+ if (!IS_ERR_OR_NULL(drng->drng)) -+ return 0; -+ -+ drng->drng_cb = drng_cb; -+ drng->drng = drng_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); -+ if (IS_ERR(drng->drng)) -+ return PTR_ERR(drng->drng); -+ -+ lrng_drng_reset(drng); -+ return 0; -+} -+ -+/* Initialize the default DRNG during boot and perform its seeding */ -+int lrng_drng_initalize(void) -+{ -+ int ret; -+ -+ if (lrng_get_available()) -+ return 0; -+ -+ /* Catch programming error */ -+ WARN_ON(lrng_drng_init.hash_cb != lrng_default_hash_cb); -+ -+ mutex_lock(&lrng_drng_init.lock); -+ if (lrng_get_available()) { -+ mutex_unlock(&lrng_drng_init.lock); -+ return 0; -+ } -+ -+ /* Initialize the PR DRNG inside init lock as it guards lrng_avail. */ -+ mutex_lock(&lrng_drng_pr.lock); -+ ret = lrng_drng_alloc_common(&lrng_drng_pr, lrng_default_drng_cb); -+ mutex_unlock(&lrng_drng_pr.lock); -+ -+ if (!ret) { -+ ret = lrng_drng_alloc_common(&lrng_drng_init, -+ lrng_default_drng_cb); -+ if (!ret) -+ atomic_set(&lrng_avail, 1); -+ } -+ mutex_unlock(&lrng_drng_init.lock); -+ if (ret) -+ return ret; -+ -+ pr_debug("LRNG for general use is available\n"); -+ -+ /* Seed the DRNG with any entropy available */ -+ if (lrng_pool_trylock()) { -+ pr_info("Initial DRNG initialized triggering first seeding\n"); -+ lrng_drng_seed_work(NULL); -+ } else { -+ pr_info("Initial DRNG initialized without seeding\n"); -+ } -+ -+ return 0; -+} -+ -+static int __init lrng_drng_make_available(void) -+{ -+ return lrng_drng_initalize(); -+} -+late_initcall(lrng_drng_make_available); -+ -+bool lrng_sp80090c_compliant(void) -+{ -+ /* SP800-90C compliant oversampling is only requested in FIPS mode */ -+ return fips_enabled; -+} -+ -+bool lrng_ntg1_compliant(void) -+{ -+ /* Implies using of /dev/random with O_SYNC */ -+ return true; -+} -+ -+/************************* Random Number Generation ***************************/ -+ -+/* Inject a data buffer into the DRNG - caller must hold its lock */ -+void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, -+ bool fully_seeded, const char *drng_type) -+{ -+ BUILD_BUG_ON(LRNG_DRNG_RESEED_THRESH > INT_MAX); -+ pr_debug("seeding %s DRNG with %u bytes\n", drng_type, inbuflen); -+ if (drng->drng_cb->drng_seed(drng->drng, inbuf, inbuflen) < 0) { -+ pr_warn("seeding of %s DRNG failed\n", drng_type); -+ drng->force_reseed = true; -+ } else { -+ int gc = LRNG_DRNG_RESEED_THRESH - atomic_read(&drng->requests); -+ -+ pr_debug("%s DRNG stats since last seeding: %lu secs; generate calls: %d\n", -+ drng_type, -+ (time_after(jiffies, drng->last_seeded) ? -+ (jiffies - drng->last_seeded) : 0) / HZ, gc); -+ -+ /* Count the numbers of generate ops since last fully seeded */ -+ if (fully_seeded) -+ atomic_set(&drng->requests_since_fully_seeded, 0); -+ else -+ atomic_add(gc, &drng->requests_since_fully_seeded); -+ -+ drng->last_seeded = jiffies; -+ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); -+ drng->force_reseed = false; -+ -+ if (!drng->fully_seeded) { -+ drng->fully_seeded = fully_seeded; -+ if (drng->fully_seeded) -+ pr_debug("%s DRNG fully seeded\n", drng_type); -+ } -+ } -+} -+ -+/* -+ * Perform the seeding of the DRNG with data from entropy source. -+ * The function returns the entropy injected into the DRNG in bits. -+ */ -+static u32 lrng_drng_seed_es_nolock(struct lrng_drng *drng) -+{ -+ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); -+ u32 collected_entropy; -+ -+ lrng_fill_seed_buffer(&seedbuf, -+ lrng_get_seed_entropy_osr(drng->fully_seeded)); -+ -+ collected_entropy = lrng_entropy_rate_eb(&seedbuf); -+ lrng_drng_inject(drng, (u8 *)&seedbuf, sizeof(seedbuf), -+ lrng_fully_seeded(drng->fully_seeded, -+ collected_entropy), -+ "regular"); -+ -+ /* Set the seeding state of the LRNG */ -+ lrng_init_ops(&seedbuf); -+ -+ memzero_explicit(&seedbuf, sizeof(seedbuf)); -+ -+ return collected_entropy; -+} -+ -+static void lrng_drng_seed_es(struct lrng_drng *drng) -+{ -+ mutex_lock(&drng->lock); -+ lrng_drng_seed_es_nolock(drng); -+ mutex_unlock(&drng->lock); -+} -+ -+static void lrng_drng_seed(struct lrng_drng *drng) -+{ -+ BUILD_BUG_ON(LRNG_MIN_SEED_ENTROPY_BITS > -+ LRNG_DRNG_SECURITY_STRENGTH_BITS); -+ -+ if (lrng_get_available()) { -+ /* (Re-)Seed DRNG */ -+ lrng_drng_seed_es(drng); -+ /* (Re-)Seed atomic DRNG from regular DRNG */ -+ lrng_drng_atomic_seed_drng(drng); -+ } else { -+ /* -+ * If no-one is waiting for the DRNG, seed the atomic DRNG -+ * directly from the entropy sources. -+ */ -+ if (!wq_has_sleeper(&lrng_init_wait)) -+ lrng_drng_atomic_seed_es(); -+ else -+ lrng_init_ops(NULL); -+ } -+} -+ -+static void _lrng_drng_seed_work(struct lrng_drng *drng, u32 node) -+{ -+ pr_debug("reseed triggered by system events for DRNG on NUMA node %d\n", -+ node); -+ lrng_drng_seed(drng); -+ if (drng->fully_seeded) { -+ /* Prevent reseed storm */ -+ drng->last_seeded += node * 100 * HZ; -+ } -+} -+ -+/* -+ * DRNG reseed trigger: Kernel thread handler triggered by the schedule_work() -+ */ -+void lrng_drng_seed_work(struct work_struct *dummy) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ u32 node; -+ -+ if (lrng_drng) { -+ for_each_online_node(node) { -+ struct lrng_drng *drng = lrng_drng[node]; -+ -+ if (drng && !drng->fully_seeded) { -+ _lrng_drng_seed_work(drng, node); -+ goto out; -+ } -+ } -+ } else { -+ if (!lrng_drng_init.fully_seeded) { -+ _lrng_drng_seed_work(&lrng_drng_init, 0); -+ goto out; -+ } -+ } -+ -+ if (!lrng_drng_pr.fully_seeded) { -+ _lrng_drng_seed_work(&lrng_drng_pr, 0); -+ goto out; -+ } -+ -+ lrng_pool_all_numa_nodes_seeded(true); -+ -+out: -+ /* Allow the seeding operation to be called again */ -+ lrng_pool_unlock(); -+} -+ -+/* Force all DRNGs to reseed before next generation */ -+void lrng_drng_force_reseed(void) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ u32 node; -+ -+ /* -+ * If the initial DRNG is over the reseed threshold, allow a forced -+ * reseed only for the initial DRNG as this is the fallback for all. It -+ * must be kept seeded before all others to keep the LRNG operational. -+ */ -+ if (!lrng_drng || -+ (atomic_read_u32(&lrng_drng_init.requests_since_fully_seeded) > -+ LRNG_DRNG_RESEED_THRESH)) { -+ lrng_drng_init.force_reseed = lrng_drng_init.fully_seeded; -+ pr_debug("force reseed of initial DRNG\n"); -+ return; -+ } -+ for_each_online_node(node) { -+ struct lrng_drng *drng = lrng_drng[node]; -+ -+ if (!drng) -+ continue; -+ -+ drng->force_reseed = drng->fully_seeded; -+ pr_debug("force reseed of DRNG on node %u\n", node); -+ } -+ lrng_drng_atomic_force_reseed(); -+} -+EXPORT_SYMBOL(lrng_drng_force_reseed); -+ -+static bool lrng_drng_must_reseed(struct lrng_drng *drng) -+{ -+ return (atomic_dec_and_test(&drng->requests) || -+ drng->force_reseed || -+ time_after(jiffies, -+ drng->last_seeded + lrng_drng_reseed_max_time * HZ)); -+} -+ -+/* -+ * lrng_drng_get() - Get random data out of the DRNG which is reseeded -+ * frequently. -+ * -+ * @drng: DRNG instance -+ * @outbuf: buffer for storing random data -+ * @outbuflen: length of outbuf -+ * -+ * Return: -+ * * < 0 in error case (DRNG generation or update failed) -+ * * >=0 returning the returned number of bytes -+ */ -+int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen) -+{ -+ u32 processed = 0; -+ bool pr = (drng == &lrng_drng_pr) ? true : false; -+ -+ if (!outbuf || !outbuflen) -+ return 0; -+ -+ if (!lrng_get_available()) -+ return -EOPNOTSUPP; -+ -+ outbuflen = min_t(size_t, outbuflen, INT_MAX); -+ -+ /* If DRNG operated without proper reseed for too long, block LRNG */ -+ BUILD_BUG_ON(LRNG_DRNG_MAX_WITHOUT_RESEED < LRNG_DRNG_RESEED_THRESH); -+ if (atomic_read_u32(&drng->requests_since_fully_seeded) > max_wo_reseed) -+ lrng_unset_fully_seeded(drng); -+ -+ while (outbuflen) { -+ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); -+ int ret; -+ -+ /* In normal operation, check whether to reseed */ -+ if (!pr && lrng_drng_must_reseed(drng)) { -+ if (!lrng_pool_trylock()) { -+ drng->force_reseed = true; -+ } else { -+ lrng_drng_seed(drng); -+ lrng_pool_unlock(); -+ } -+ } -+ -+ mutex_lock(&drng->lock); -+ -+ if (pr) { -+ /* If async reseed did not deliver entropy, try now */ -+ if (!drng->fully_seeded) { -+ u32 coll_ent_bits; -+ -+ /* If we cannot get the pool lock, try again. */ -+ if (!lrng_pool_trylock()) { -+ mutex_unlock(&drng->lock); -+ continue; -+ } -+ -+ coll_ent_bits = lrng_drng_seed_es_nolock(drng); -+ -+ lrng_pool_unlock(); -+ -+ /* If no new entropy was received, stop now. */ -+ if (!coll_ent_bits) { -+ mutex_unlock(&drng->lock); -+ goto out; -+ } -+ -+ /* Produce no more data than received entropy */ -+ todo = min_t(u32, todo, coll_ent_bits >> 3); -+ } -+ -+ /* Do not produce more than DRNG security strength */ -+ todo = min_t(u32, todo, lrng_security_strength() >> 3); -+ } -+ ret = drng->drng_cb->drng_generate(drng->drng, -+ outbuf + processed, todo); -+ -+ mutex_unlock(&drng->lock); -+ if (ret <= 0) { -+ pr_warn("getting random data from DRNG failed (%d)\n", -+ ret); -+ return -EFAULT; -+ } -+ processed += ret; -+ outbuflen -= ret; -+ -+ if (pr) { -+ /* Force the async reseed for PR DRNG */ -+ lrng_unset_fully_seeded(drng); -+ if (outbuflen) -+ cond_resched(); -+ } -+ } -+ -+out: -+ return processed; -+} -+ -+int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *drng = &lrng_drng_init; -+ int ret, node = numa_node_id(); -+ -+ might_sleep(); -+ -+ if (pr) -+ drng = &lrng_drng_pr; -+ else if (lrng_drng && lrng_drng[node] && lrng_drng[node]->fully_seeded) -+ drng = lrng_drng[node]; -+ -+ ret = lrng_drng_initalize(); -+ if (ret) -+ return ret; -+ -+ return lrng_drng_get(drng, outbuf, outbuflen); -+} -+ -+/* Reset LRNG such that all existing entropy is gone */ -+static void _lrng_reset(struct work_struct *work) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ -+ if (!lrng_drng) { -+ mutex_lock(&lrng_drng_init.lock); -+ lrng_drng_reset(&lrng_drng_init); -+ mutex_unlock(&lrng_drng_init.lock); -+ } else { -+ u32 node; -+ -+ for_each_online_node(node) { -+ struct lrng_drng *drng = lrng_drng[node]; -+ -+ if (!drng) -+ continue; -+ mutex_lock(&drng->lock); -+ lrng_drng_reset(drng); -+ mutex_unlock(&drng->lock); -+ } -+ } -+ -+ mutex_lock(&lrng_drng_pr.lock); -+ lrng_drng_reset(&lrng_drng_pr); -+ mutex_unlock(&lrng_drng_pr.lock); -+ -+ lrng_drng_atomic_reset(); -+ lrng_set_entropy_thresh(LRNG_INIT_ENTROPY_BITS); -+ -+ lrng_reset_state(); -+} -+ -+static DECLARE_WORK(lrng_reset_work, _lrng_reset); -+ -+void lrng_reset(void) -+{ -+ schedule_work(&lrng_reset_work); -+} -+ -+/******************* Generic LRNG kernel output interfaces ********************/ -+ -+static int lrng_drng_sleep_while_not_all_nodes_seeded(unsigned int nonblock) -+{ -+ if (lrng_pool_all_numa_nodes_seeded_get()) -+ return 0; -+ if (nonblock) -+ return -EAGAIN; -+ wait_event_interruptible(lrng_init_wait, -+ lrng_pool_all_numa_nodes_seeded_get()); -+ return 0; -+} -+ -+int lrng_drng_sleep_while_nonoperational(int nonblock) -+{ -+ if (likely(lrng_state_operational())) -+ return 0; -+ if (nonblock) -+ return -EAGAIN; -+ return wait_event_interruptible(lrng_init_wait, -+ lrng_state_operational()); -+} -+ -+int lrng_drng_sleep_while_non_min_seeded(void) -+{ -+ if (likely(lrng_state_min_seeded())) -+ return 0; -+ return wait_event_interruptible(lrng_init_wait, -+ lrng_state_min_seeded()); -+} -+ -+ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags) -+{ -+ struct entropy_buf *eb = (struct entropy_buf *)(buf + 2); -+ u64 buflen = sizeof(struct entropy_buf) + 2 * sizeof(u64); -+ u64 collected_bits = 0; -+ int ret; -+ -+ /* Ensure buffer is aligned as required */ -+ BUILD_BUG_ON(sizeof(buflen) < LRNG_KCAPI_ALIGN); -+ if (nbytes < sizeof(buflen)) -+ return -EINVAL; -+ -+ /* Write buffer size into first word */ -+ buf[0] = buflen; -+ if (nbytes < buflen) -+ return -EMSGSIZE; -+ -+ ret = lrng_drng_sleep_while_not_all_nodes_seeded( -+ flags & LRNG_GET_SEED_NONBLOCK); -+ if (ret) -+ return ret; -+ -+ /* Try to get the pool lock and sleep on it to get it. */ -+ lrng_pool_lock(); -+ -+ /* If an LRNG DRNG becomes unseeded, give this DRNG precedence. */ -+ if (!lrng_pool_all_numa_nodes_seeded_get()) { -+ lrng_pool_unlock(); -+ return 0; -+ } -+ -+ /* -+ * Try to get seed data - a rarely used busyloop is cheaper than a wait -+ * queue that is constantly woken up by the hot code path of -+ * lrng_init_ops. -+ */ -+ for (;;) { -+ lrng_fill_seed_buffer(eb, -+ lrng_get_seed_entropy_osr(flags & -+ LRNG_GET_SEED_FULLY_SEEDED)); -+ collected_bits = lrng_entropy_rate_eb(eb); -+ -+ /* Break the collection loop if we got entropy, ... */ -+ if (collected_bits || -+ /* ... a DRNG becomes unseeded, give DRNG precedence, ... */ -+ !lrng_pool_all_numa_nodes_seeded_get() || -+ /* ... if the caller does not want a blocking behavior. */ -+ (flags & LRNG_GET_SEED_NONBLOCK)) -+ break; -+ -+ schedule(); -+ } -+ -+ lrng_pool_unlock(); -+ -+ /* Write collected entropy size into second word */ -+ buf[1] = collected_bits; -+ -+ return (ssize_t)buflen; -+} -+ -+void lrng_get_random_bytes_full(void *buf, int nbytes) -+{ -+ lrng_drng_sleep_while_nonoperational(0); -+ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes_full); -+ -+void lrng_get_random_bytes_min(void *buf, int nbytes) -+{ -+ lrng_drng_sleep_while_non_min_seeded(); -+ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes_min); -+ -+int lrng_get_random_bytes_pr(void *buf, int nbytes) -+{ -+ lrng_drng_sleep_while_nonoperational(0); -+ return lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, true); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes_pr); -diff --git a/drivers/char/lrng/lrng_drng_mgr.h b/drivers/char/lrng/lrng_drng_mgr.h -new file mode 100644 -index 000000000000..ec120e4980b5 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_mgr.h -@@ -0,0 +1,86 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DRNG_H -+#define _LRNG_DRNG_H -+ -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+ -+extern struct wait_queue_head lrng_init_wait; -+extern int lrng_drng_reseed_max_time; -+extern struct mutex lrng_crypto_cb_update; -+extern const struct lrng_drng_cb *lrng_default_drng_cb; -+extern const struct lrng_hash_cb *lrng_default_hash_cb; -+ -+/* DRNG state handle */ -+struct lrng_drng { -+ void *drng; /* DRNG handle */ -+ void *hash; /* Hash handle */ -+ const struct lrng_drng_cb *drng_cb; /* DRNG callbacks */ -+ const struct lrng_hash_cb *hash_cb; /* Hash callbacks */ -+ atomic_t requests; /* Number of DRNG requests */ -+ atomic_t requests_since_fully_seeded; /* Number DRNG requests since -+ * last fully seeded -+ */ -+ unsigned long last_seeded; /* Last time it was seeded */ -+ bool fully_seeded; /* Is DRNG fully seeded? */ -+ bool force_reseed; /* Force a reseed */ -+ -+ rwlock_t hash_lock; /* Lock hash_cb replacement */ -+ /* Lock write operations on DRNG state, DRNG replacement of drng_cb */ -+ struct mutex lock; /* Non-atomic DRNG operation */ -+ spinlock_t spin_lock; /* Atomic DRNG operation */ -+}; -+ -+#define LRNG_DRNG_STATE_INIT(x, d, h, d_cb, h_cb) \ -+ .drng = d, \ -+ .hash = h, \ -+ .drng_cb = d_cb, \ -+ .hash_cb = h_cb, \ -+ .requests = ATOMIC_INIT(LRNG_DRNG_RESEED_THRESH),\ -+ .requests_since_fully_seeded = ATOMIC_INIT(0), \ -+ .last_seeded = 0, \ -+ .fully_seeded = false, \ -+ .force_reseed = true, \ -+ .hash_lock = __RW_LOCK_UNLOCKED(x.hash_lock) -+ -+struct lrng_drng *lrng_drng_init_instance(void); -+struct lrng_drng *lrng_drng_pr_instance(void); -+struct lrng_drng *lrng_drng_node_instance(void); -+ -+void lrng_reset(void); -+int lrng_drng_alloc_common(struct lrng_drng *drng, -+ const struct lrng_drng_cb *crypto_cb); -+int lrng_drng_initalize(void); -+bool lrng_sp80090c_compliant(void); -+bool lrng_ntg1_compliant(void); -+bool lrng_get_available(void); -+void lrng_drng_reset(struct lrng_drng *drng); -+void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, -+ bool fully_seeded, const char *drng_type); -+int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen); -+int lrng_drng_sleep_while_nonoperational(int nonblock); -+int lrng_drng_sleep_while_non_min_seeded(void); -+int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr); -+void lrng_drng_seed_work(struct work_struct *dummy); -+void lrng_drng_force_reseed(void); -+ -+static inline u32 lrng_compress_osr(void) -+{ -+ return lrng_sp80090c_compliant() ? LRNG_OVERSAMPLE_ES_BITS : 0; -+} -+ -+static inline u32 lrng_reduce_by_osr(u32 entropy_bits) -+{ -+ u32 osr_bits = lrng_compress_osr(); -+ -+ return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0; -+} -+ -+#endif /* _LRNG_DRNG_H */ -diff --git a/drivers/char/lrng/lrng_es_aux.c b/drivers/char/lrng/lrng_es_aux.c -new file mode 100644 -index 000000000000..245bc829998b ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_aux.c -@@ -0,0 +1,335 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Auxiliary entropy pool -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_sysctl.h" -+ -+/* -+ * This is the auxiliary pool -+ * -+ * The aux pool array is aligned to 8 bytes to comfort the kernel crypto API -+ * cipher implementations of the hash functions used to read the pool: for some -+ * accelerated implementations, we need an alignment to avoid a realignment -+ * which involves memcpy(). The alignment to 8 bytes should satisfy all crypto -+ * implementations. -+ */ -+struct lrng_pool { -+ u8 aux_pool[LRNG_POOL_SIZE]; /* Aux pool: digest state */ -+ atomic_t aux_entropy_bits; -+ atomic_t digestsize; /* Digest size of used hash */ -+ bool initialized; /* Aux pool initialized? */ -+ -+ /* Serialize read of entropy pool and update of aux pool */ -+ spinlock_t lock; -+}; -+ -+static struct lrng_pool lrng_pool __aligned(LRNG_KCAPI_ALIGN) = { -+ .aux_entropy_bits = ATOMIC_INIT(0), -+ .digestsize = ATOMIC_INIT(LRNG_ATOMIC_DIGEST_SIZE), -+ .initialized = false, -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_pool.lock) -+}; -+ -+/********************************** Helper ***********************************/ -+ -+/* Entropy in bits present in aux pool */ -+static u32 lrng_aux_avail_entropy(u32 __unused) -+{ -+ /* Cap available entropy with max entropy */ -+ u32 avail_bits = min_t(u32, lrng_get_digestsize(), -+ atomic_read_u32(&lrng_pool.aux_entropy_bits)); -+ -+ /* Consider oversampling rate due to aux pool conditioning */ -+ return lrng_reduce_by_osr(avail_bits); -+} -+ -+/* Set the digest size of the used hash in bytes */ -+static void lrng_set_digestsize(u32 digestsize) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ u32 ent_bits = atomic_xchg_relaxed(&pool->aux_entropy_bits, 0), -+ old_digestsize = lrng_get_digestsize(); -+ -+ atomic_set(&lrng_pool.digestsize, digestsize); -+ -+ /* -+ * Update the write wakeup threshold which must not be larger -+ * than the digest size of the current conditioning hash. -+ */ -+ digestsize = lrng_reduce_by_osr(digestsize << 3); -+ lrng_sysctl_update_max_write_thresh(digestsize); -+ lrng_write_wakeup_bits = digestsize; -+ -+ /* -+ * In case the new digest is larger than the old one, cap the available -+ * entropy to the old message digest used to process the existing data. -+ */ -+ ent_bits = min_t(u32, ent_bits, old_digestsize); -+ atomic_add(ent_bits, &pool->aux_entropy_bits); -+} -+ -+static int __init lrng_init_wakeup_bits(void) -+{ -+ u32 digestsize = lrng_reduce_by_osr(lrng_get_digestsize()); -+ -+ lrng_sysctl_update_max_write_thresh(digestsize); -+ lrng_write_wakeup_bits = digestsize; -+ return 0; -+} -+core_initcall(lrng_init_wakeup_bits); -+ -+/* Obtain the digest size provided by the used hash in bits */ -+u32 lrng_get_digestsize(void) -+{ -+ return atomic_read_u32(&lrng_pool.digestsize) << 3; -+} -+ -+/* Set entropy content in user-space controllable aux pool */ -+void lrng_pool_set_entropy(u32 entropy_bits) -+{ -+ atomic_set(&lrng_pool.aux_entropy_bits, entropy_bits); -+} -+ -+static void lrng_aux_reset(void) -+{ -+ lrng_pool_set_entropy(0); -+} -+ -+/* -+ * Replace old with new hash for auxiliary pool handling -+ * -+ * Assumption: the caller must guarantee that the new_cb is available during the -+ * entire operation (e.g. it must hold the write lock against pointer updating). -+ */ -+static int -+lrng_aux_switch_hash(struct lrng_drng *drng, int __unused, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ struct lrng_drng *init_drng = lrng_drng_init_instance(); -+ struct lrng_pool *pool = &lrng_pool; -+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ int ret; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ if (unlikely(!pool->initialized)) -+ return 0; -+ -+ /* We only switch if the processed DRNG is the initial DRNG. */ -+ if (init_drng != drng) -+ return 0; -+ -+ /* Get the aux pool hash with old digest ... */ -+ ret = old_cb->hash_final(shash, digest) ?: -+ /* ... re-initialize the hash with the new digest ... */ -+ new_cb->hash_init(shash, new_hash) ?: -+ /* -+ * ... feed the old hash into the new state. We may feed -+ * uninitialized memory into the new state, but this is -+ * considered no issue and even good as we have some more -+ * uncertainty here. -+ */ -+ new_cb->hash_update(shash, digest, sizeof(digest)); -+ if (!ret) { -+ lrng_set_digestsize(new_cb->hash_digestsize(new_hash)); -+ pr_debug("Re-initialize aux entropy pool with hash %s\n", -+ new_cb->hash_name()); -+ } -+ -+ memzero_explicit(digest, sizeof(digest)); -+ return ret; -+} -+ -+/* Insert data into auxiliary pool by using the hash update function. */ -+static int -+lrng_aux_pool_insert_locked(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ const struct lrng_hash_cb *hash_cb; -+ unsigned long flags; -+ void *hash; -+ int ret; -+ -+ entropy_bits = min_t(u32, entropy_bits, inbuflen << 3); -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ if (unlikely(!pool->initialized)) { -+ ret = hash_cb->hash_init(shash, hash); -+ if (ret) -+ goto out; -+ pool->initialized = true; -+ } -+ -+ ret = hash_cb->hash_update(shash, inbuf, inbuflen); -+ if (ret) -+ goto out; -+ -+ /* -+ * Cap the available entropy to the hash output size compliant to -+ * SP800-90B section 3.1.5.1 table 1. -+ */ -+ entropy_bits += atomic_read_u32(&pool->aux_entropy_bits); -+ atomic_set(&pool->aux_entropy_bits, -+ min_t(u32, entropy_bits, -+ hash_cb->hash_digestsize(hash) << 3)); -+ -+out: -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ return ret; -+} -+ -+int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ unsigned long flags; -+ int ret; -+ -+ spin_lock_irqsave(&pool->lock, flags); -+ ret = lrng_aux_pool_insert_locked(inbuf, inbuflen, entropy_bits); -+ spin_unlock_irqrestore(&pool->lock, flags); -+ -+ lrng_es_add_entropy(); -+ -+ return ret; -+} -+EXPORT_SYMBOL(lrng_pool_insert_aux); -+ -+/************************* Get data from entropy pool *************************/ -+ -+/* -+ * Get auxiliary entropy pool and its entropy content for seed buffer. -+ * Caller must hold lrng_pool.pool->lock. -+ * @outbuf: buffer to store data in with size requested_bits -+ * @requested_bits: Requested amount of entropy -+ * @return: amount of entropy in outbuf in bits. -+ */ -+static u32 lrng_aux_get_pool(u8 *outbuf, u32 requested_bits) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ const struct lrng_hash_cb *hash_cb; -+ unsigned long flags; -+ void *hash; -+ u32 collected_ent_bits, returned_ent_bits, unused_bits = 0, -+ digestsize, digestsize_bits, requested_bits_osr; -+ u8 aux_output[LRNG_MAX_DIGESTSIZE]; -+ -+ if (unlikely(!pool->initialized)) -+ return 0; -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ digestsize = hash_cb->hash_digestsize(hash); -+ digestsize_bits = digestsize << 3; -+ -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(digestsize_bits, requested_bits); -+ -+ /* Ensure that no more than the size of aux_pool can be requested */ -+ requested_bits = min_t(u32, requested_bits, (LRNG_MAX_DIGESTSIZE << 3)); -+ requested_bits_osr = requested_bits + lrng_compress_osr(); -+ -+ /* Cap entropy with entropy counter from aux pool and the used digest */ -+ collected_ent_bits = min_t(u32, digestsize_bits, -+ atomic_xchg_relaxed(&pool->aux_entropy_bits, 0)); -+ -+ /* We collected too much entropy and put the overflow back */ -+ if (collected_ent_bits > requested_bits_osr) { -+ /* Amount of bits we collected too much */ -+ unused_bits = collected_ent_bits - requested_bits_osr; -+ /* Put entropy back */ -+ atomic_add(unused_bits, &pool->aux_entropy_bits); -+ /* Fix collected entropy */ -+ collected_ent_bits = requested_bits_osr; -+ } -+ -+ /* Apply oversampling: discount requested oversampling rate */ -+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); -+ -+ pr_debug("obtained %u bits by collecting %u bits of entropy from aux pool, %u bits of entropy remaining\n", -+ returned_ent_bits, collected_ent_bits, unused_bits); -+ -+ /* Get the digest for the aux pool to be returned to the caller ... */ -+ if (hash_cb->hash_final(shash, aux_output) || -+ /* -+ * ... and re-initialize the aux state. Do not add the aux pool -+ * digest for backward secrecy as it will be added with the -+ * insertion of the complete seed buffer after it has been filled. -+ */ -+ hash_cb->hash_init(shash, hash)) { -+ returned_ent_bits = 0; -+ } else { -+ /* -+ * Do not truncate the output size exactly to collected_ent_bits -+ * as the aux pool may contain data that is not credited with -+ * entropy, but we want to use them to stir the DRNG state. -+ */ -+ memcpy(outbuf, aux_output, requested_bits >> 3); -+ } -+ -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ memzero_explicit(aux_output, digestsize); -+ return returned_ent_bits; -+} -+ -+static void lrng_aux_get_backtrack(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ unsigned long flags; -+ -+ /* Ensure aux pool extraction and backtracking op are atomic */ -+ spin_lock_irqsave(&pool->lock, flags); -+ -+ eb->e_bits[lrng_ext_es_aux] = lrng_aux_get_pool(eb->e[lrng_ext_es_aux], -+ requested_bits); -+ -+ /* Mix the extracted data back into pool for backtracking resistance */ -+ if (lrng_aux_pool_insert_locked((u8 *)eb, -+ sizeof(struct entropy_buf), 0)) -+ pr_warn("Backtracking resistance operation failed\n"); -+ -+ spin_unlock_irqrestore(&pool->lock, flags); -+} -+ -+static void lrng_aux_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for operating entropy pool: %s\n" -+ " Available entropy: %u\n", -+ lrng_drng_init->hash_cb->hash_name(), -+ lrng_aux_avail_entropy(0)); -+} -+ -+struct lrng_es_cb lrng_es_aux = { -+ .name = "Auxiliary", -+ .get_ent = lrng_aux_get_backtrack, -+ .curr_entropy = lrng_aux_avail_entropy, -+ .max_entropy = lrng_get_digestsize, -+ .state = lrng_aux_es_state, -+ .reset = lrng_aux_reset, -+ .switch_hash = lrng_aux_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_aux.h b/drivers/char/lrng/lrng_es_aux.h -new file mode 100644 -index 000000000000..bc41e6474aad ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_aux.h -@@ -0,0 +1,44 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_AUX_H -+#define _LRNG_ES_AUX_H -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_mgr_cb.h" -+ -+u32 lrng_get_digestsize(void); -+void lrng_pool_set_entropy(u32 entropy_bits); -+int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits); -+ -+extern struct lrng_es_cb lrng_es_aux; -+ -+/****************************** Helper code ***********************************/ -+ -+/* Obtain the security strength of the LRNG in bits */ -+static inline u32 lrng_security_strength(void) -+{ -+ /* -+ * We use a hash to read the entropy in the entropy pool. According to -+ * SP800-90B table 1, the entropy can be at most the digest size. -+ * Considering this together with the last sentence in section 3.1.5.1.2 -+ * the security strength of a (approved) hash is equal to its output -+ * size. On the other hand the entropy cannot be larger than the -+ * security strength of the used DRBG. -+ */ -+ return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize()); -+} -+ -+static inline u32 lrng_get_seed_entropy_osr(bool fully_seeded) -+{ -+ u32 requested_bits = lrng_security_strength(); -+ -+ /* Apply oversampling during initialization according to SP800-90C */ -+ if (lrng_sp80090c_compliant() && !fully_seeded) -+ requested_bits += LRNG_SEED_BUFFER_INIT_ADD_BITS; -+ return requested_bits; -+} -+ -+#endif /* _LRNG_ES_AUX_H */ -diff --git a/drivers/char/lrng/lrng_es_cpu.c b/drivers/char/lrng/lrng_es_cpu.c -new file mode 100644 -index 000000000000..f982cc31df4e ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_cpu.c -@@ -0,0 +1,280 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Fast Entropy Source: CPU-based entropy source -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_cpu.h" -+ -+/* -+ * Estimated entropy of data is a 32th of LRNG_DRNG_SECURITY_STRENGTH_BITS. -+ * As we have no ability to review the implementation of those noise sources, -+ * it is prudent to have a conservative estimate here. -+ */ -+#define LRNG_ARCHRANDOM_DEFAULT_STRENGTH CONFIG_LRNG_CPU_ENTROPY_RATE -+#define LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH LRNG_DRNG_SECURITY_STRENGTH_BITS -+#ifdef CONFIG_RANDOM_TRUST_CPU -+static u32 cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; -+#else -+static u32 cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; -+#endif -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(cpu_entropy, uint, 0644); -+MODULE_PARM_DESC(cpu_entropy, "Entropy in bits of 256 data bits from CPU noise source (e.g. RDSEED)"); -+#endif -+ -+static int __init lrng_parse_trust_cpu(char *arg) -+{ -+ int ret; -+ bool trust_cpu = false; -+ -+ ret = kstrtobool(arg, &trust_cpu); -+ if (ret) -+ return ret; -+ -+ if (trust_cpu) { -+ cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; -+ lrng_es_add_entropy(); -+ } else { -+ cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; -+ } -+ -+ return 0; -+} -+early_param("random.trust_cpu", lrng_parse_trust_cpu); -+ -+static u32 lrng_cpu_entropylevel(u32 requested_bits) -+{ -+ return lrng_fast_noise_entropylevel(cpu_entropy, requested_bits); -+} -+ -+static u32 lrng_cpu_poolsize(void) -+{ -+ return lrng_cpu_entropylevel(lrng_security_strength()); -+} -+ -+static u32 lrng_get_cpu_data(u8 *outbuf, u32 requested_bits) -+{ -+ size_t longs = 0; -+ u32 i, req = requested_bits >> 3; -+ -+ /* operate on full blocks */ -+ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BYTES % sizeof(unsigned long)); -+ BUILD_BUG_ON(LRNG_SEED_BUFFER_INIT_ADD_BITS % sizeof(unsigned long)); -+ /* ensure we have aligned buffers */ -+ BUILD_BUG_ON(LRNG_KCAPI_ALIGN % sizeof(unsigned long)); -+ -+ for (i = 0; i < req; i += longs) { -+ longs = arch_get_random_seed_longs( -+ (unsigned long *)(outbuf + i), req - i); -+ if (longs) -+ continue; -+ longs = arch_get_random_longs((unsigned long *)(outbuf + i), -+ req - i); -+ if (!longs) { -+ cpu_entropy = 0; -+ return 0; -+ } -+ } -+ -+ return requested_bits; -+} -+ -+static u32 lrng_get_cpu_data_compress(u8 *outbuf, u32 requested_bits, -+ u32 data_multiplier) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb; -+ struct lrng_drng *drng = lrng_drng_node_instance(); -+ unsigned long flags; -+ u32 ent_bits = 0, i, partial_bits = 0, digestsize, digestsize_bits, -+ full_bits; -+ void *hash; -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ digestsize = hash_cb->hash_digestsize(hash); -+ digestsize_bits = digestsize << 3; -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(digestsize_bits, requested_bits); -+ full_bits = requested_bits * data_multiplier; -+ -+ /* Calculate oversampling for SP800-90C */ -+ if (lrng_sp80090c_compliant()) { -+ /* Complete amount of bits to be pulled */ -+ full_bits += LRNG_OVERSAMPLE_ES_BITS * data_multiplier; -+ /* Full blocks that will be pulled */ -+ data_multiplier = full_bits / requested_bits; -+ /* Partial block in bits to be pulled */ -+ partial_bits = full_bits - (data_multiplier * requested_bits); -+ } -+ -+ if (hash_cb->hash_init(shash, hash)) -+ goto out; -+ -+ /* Hash all data from the CPU entropy source */ -+ for (i = 0; i < data_multiplier; i++) { -+ ent_bits = lrng_get_cpu_data(outbuf, requested_bits); -+ if (!ent_bits) -+ goto out; -+ -+ if (hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) -+ goto err; -+ } -+ -+ /* Hash partial block, if applicable */ -+ ent_bits = lrng_get_cpu_data(outbuf, partial_bits); -+ if (ent_bits && -+ hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) -+ goto err; -+ -+ pr_debug("pulled %u bits from CPU RNG entropy source\n", full_bits); -+ ent_bits = requested_bits; -+ -+ /* Generate the compressed data to be returned to the caller */ -+ if (requested_bits < digestsize_bits) { -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ -+ if (hash_cb->hash_final(shash, digest)) -+ goto err; -+ -+ /* Truncate output data to requested size */ -+ memcpy(outbuf, digest, requested_bits >> 3); -+ memzero_explicit(digest, digestsize); -+ } else { -+ if (hash_cb->hash_final(shash, outbuf)) -+ goto err; -+ } -+ -+out: -+ hash_cb->hash_desc_zero(shash); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ return ent_bits; -+ -+err: -+ ent_bits = 0; -+ goto out; -+} -+ -+/* -+ * If CPU entropy source requires does not return full entropy, return the -+ * multiplier of how much data shall be sampled from it. -+ */ -+static u32 lrng_cpu_multiplier(void) -+{ -+ static u32 data_multiplier = 0; -+ unsigned long v; -+ -+ if (data_multiplier > 0) -+ return data_multiplier; -+ -+ if (IS_ENABLED(CONFIG_X86) && !arch_get_random_seed_longs(&v, 1)) { -+ /* -+ * Intel SPEC: pulling 512 blocks from RDRAND ensures -+ * one reseed making it logically equivalent to RDSEED. -+ */ -+ data_multiplier = 512; -+ } else if (IS_ENABLED(CONFIG_PPC)) { -+ /* -+ * PowerISA defines DARN to deliver at least 0.5 bits of -+ * entropy per data bit. -+ */ -+ data_multiplier = 2; -+ } else if (IS_ENABLED(CONFIG_RISCV)) { -+ /* -+ * riscv-crypto-spec-scalar-1.0.0-rc6.pdf section 4.2 defines -+ * this requirement. -+ */ -+ data_multiplier = 2; -+ } else { -+ /* CPU provides full entropy */ -+ data_multiplier = CONFIG_LRNG_CPU_FULL_ENT_MULTIPLIER; -+ } -+ return data_multiplier; -+} -+ -+static int -+lrng_cpu_switch_hash(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ u32 digestsize, multiplier; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ digestsize = lrng_get_digestsize(); -+ multiplier = lrng_cpu_multiplier(); -+ -+ /* -+ * It would be security violation if the new digestsize is smaller than -+ * the set CPU entropy rate. -+ */ -+ WARN_ON(multiplier > 1 && digestsize < cpu_entropy); -+ cpu_entropy = min_t(u32, digestsize, cpu_entropy); -+ return 0; -+} -+ -+/* -+ * lrng_get_arch() - Get CPU entropy source entropy -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: requested entropy in bits -+ */ -+static void lrng_cpu_get(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ u32 ent_bits, data_multiplier = lrng_cpu_multiplier(); -+ -+ if (data_multiplier <= 1) { -+ ent_bits = lrng_get_cpu_data(eb->e[lrng_ext_es_cpu], -+ requested_bits); -+ } else { -+ ent_bits = lrng_get_cpu_data_compress(eb->e[lrng_ext_es_cpu], -+ requested_bits, -+ data_multiplier); -+ } -+ -+ ent_bits = lrng_cpu_entropylevel(ent_bits); -+ pr_debug("obtained %u bits of entropy from CPU RNG entropy source\n", -+ ent_bits); -+ eb->e_bits[lrng_ext_es_cpu] = ent_bits; -+} -+ -+static void lrng_cpu_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ u32 data_multiplier = lrng_cpu_multiplier(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for compressing data: %s\n" -+ " Available entropy: %u\n" -+ " Data multiplier: %u\n", -+ (data_multiplier <= 1) ? -+ "N/A" : lrng_drng_init->hash_cb->hash_name(), -+ lrng_cpu_poolsize(), -+ data_multiplier); -+} -+ -+struct lrng_es_cb lrng_es_cpu = { -+ .name = "CPU", -+ .get_ent = lrng_cpu_get, -+ .curr_entropy = lrng_cpu_entropylevel, -+ .max_entropy = lrng_cpu_poolsize, -+ .state = lrng_cpu_es_state, -+ .reset = NULL, -+ .switch_hash = lrng_cpu_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_cpu.h b/drivers/char/lrng/lrng_es_cpu.h -new file mode 100644 -index 000000000000..8dbb4d9a2926 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_cpu.h -@@ -0,0 +1,17 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_CPU_H -+#define _LRNG_ES_CPU_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_CPU -+ -+extern struct lrng_es_cb lrng_es_cpu; -+ -+#endif /* CONFIG_LRNG_CPU */ -+ -+#endif /* _LRNG_ES_CPU_H */ -diff --git a/drivers/char/lrng/lrng_es_irq.c b/drivers/char/lrng/lrng_es_irq.c -new file mode 100644 -index 000000000000..97c5c1d26447 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_irq.c -@@ -0,0 +1,727 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Interrupt data collection -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_timer_common.h" -+#include "lrng_health.h" -+#include "lrng_numa.h" -+#include "lrng_testing.h" -+ -+/* -+ * Number of interrupts to be recorded to assume that DRNG security strength -+ * bits of entropy are received. -+ * Note: a value below the DRNG security strength should not be defined as this -+ * may imply the DRNG can never be fully seeded in case other noise -+ * sources are unavailable. -+ */ -+#define LRNG_IRQ_ENTROPY_BITS LRNG_UINT32_C(CONFIG_LRNG_IRQ_ENTROPY_RATE) -+ -+ -+/* Number of interrupts required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ -+static u32 lrng_irq_entropy_bits = LRNG_IRQ_ENTROPY_BITS; -+ -+static u32 irq_entropy __read_mostly = LRNG_IRQ_ENTROPY_BITS; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(irq_entropy, uint, 0444); -+MODULE_PARM_DESC(irq_entropy, -+ "How many interrupts must be collected for obtaining 256 bits of entropy\n"); -+#endif -+ -+/* Per-CPU array holding concatenated IRQ entropy events */ -+static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_irq_array) -+ __aligned(LRNG_KCAPI_ALIGN); -+static DEFINE_PER_CPU(u32, lrng_irq_array_ptr) = 0; -+static DEFINE_PER_CPU(atomic_t, lrng_irq_array_irqs) = ATOMIC_INIT(0); -+ -+/* -+ * The entropy collection is performed by executing the following steps: -+ * 1. fill up the per-CPU array holding the time stamps -+ * 2. once the per-CPU array is full, a compression of the data into -+ * the entropy pool is performed - this happens in interrupt context -+ * -+ * If step 2 is not desired in interrupt context, the following boolean -+ * needs to be set to false. This implies that old entropy data in the -+ * per-CPU array collected since the last DRNG reseed is overwritten with -+ * new entropy data instead of retaining the entropy with the compression -+ * operation. -+ * -+ * Impact on entropy: -+ * -+ * If continuous compression is enabled, the maximum entropy that is collected -+ * per CPU between DRNG reseeds is equal to the digest size of the used hash. -+ * -+ * If continuous compression is disabled, the maximum number of entropy events -+ * that can be collected per CPU is equal to LRNG_DATA_ARRAY_SIZE. This amount -+ * of events is converted into an entropy statement which then represents the -+ * maximum amount of entropy collectible per CPU between DRNG reseeds. -+ */ -+static bool lrng_irq_continuous_compression __read_mostly = -+ IS_ENABLED(CONFIG_LRNG_ENABLE_CONTINUOUS_COMPRESSION); -+ -+#ifdef CONFIG_LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION -+module_param(lrng_irq_continuous_compression, bool, 0444); -+MODULE_PARM_DESC(lrng_irq_continuous_compression, -+ "Perform entropy compression if per-CPU entropy data array is full\n"); -+#endif -+ -+/* -+ * Per-CPU entropy pool with compressed entropy event -+ * -+ * The per-CPU entropy pool is defined as the hash state. New data is simply -+ * inserted into the entropy pool by performing a hash update operation. -+ * To read the entropy pool, a hash final must be invoked. However, before -+ * the entropy pool is released again after a hash final, the hash init must -+ * be performed. -+ */ -+static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_irq_pool) -+ __aligned(LRNG_KCAPI_ALIGN); -+/* -+ * Lock to allow other CPUs to read the pool - as this is only done during -+ * reseed which is infrequent, this lock is hardly contended. -+ */ -+static DEFINE_PER_CPU(spinlock_t, lrng_irq_lock); -+static DEFINE_PER_CPU(bool, lrng_irq_lock_init) = false; -+ -+static bool lrng_irq_pool_online(int cpu) -+{ -+ return per_cpu(lrng_irq_lock_init, cpu); -+} -+ -+static void __init lrng_irq_check_compression_state(void) -+{ -+ /* One pool must hold sufficient entropy for disabled compression */ -+ if (!lrng_irq_continuous_compression) { -+ u32 max_ent = min_t(u32, lrng_get_digestsize(), -+ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, -+ lrng_irq_entropy_bits)); -+ if (max_ent < lrng_security_strength()) { -+ pr_warn("Force continuous compression operation to ensure LRNG can hold enough entropy\n"); -+ lrng_irq_continuous_compression = true; -+ } -+ } -+} -+ -+void __init lrng_irq_es_init(bool highres_timer) -+{ -+ /* Set a minimum number of interrupts that must be collected */ -+ irq_entropy = max_t(u32, LRNG_IRQ_ENTROPY_BITS, irq_entropy); -+ -+ if (highres_timer) { -+ lrng_irq_entropy_bits = irq_entropy; -+ } else { -+ u32 new_entropy = irq_entropy * LRNG_ES_OVERSAMPLING_FACTOR; -+ -+ lrng_irq_entropy_bits = (irq_entropy < new_entropy) ? -+ new_entropy : irq_entropy; -+ pr_warn("operating without high-resolution timer and applying IRQ oversampling factor %u\n", -+ LRNG_ES_OVERSAMPLING_FACTOR); -+ } -+ -+ lrng_irq_check_compression_state(); -+} -+ -+/* -+ * Reset all per-CPU pools - reset entropy estimator but leave the pool data -+ * that may or may not have entropy unchanged. -+ */ -+static void lrng_irq_reset(void) -+{ -+ int cpu; -+ -+ /* Trigger GCD calculation anew. */ -+ lrng_gcd_set(0); -+ -+ for_each_online_cpu(cpu) -+ atomic_set(per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); -+} -+ -+static u32 lrng_irq_avail_pool_size(void) -+{ -+ u32 max_size = 0, max_pool = lrng_get_digestsize(); -+ int cpu; -+ -+ if (!lrng_irq_continuous_compression) -+ max_pool = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); -+ -+ for_each_online_cpu(cpu) { -+ if (lrng_irq_pool_online(cpu)) -+ max_size += max_pool; -+ } -+ -+ return max_size; -+} -+ -+/* Return entropy of unused IRQs present in all per-CPU pools. */ -+static u32 lrng_irq_avail_entropy(u32 __unused) -+{ -+ u32 digestsize_irqs, irq = 0; -+ int cpu; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) -+ return 0; -+ -+ /* Obtain the cap of maximum numbers of IRQs we count */ -+ digestsize_irqs = lrng_entropy_to_data(lrng_get_digestsize(), -+ lrng_irq_entropy_bits); -+ if (!lrng_irq_continuous_compression) { -+ /* Cap to max. number of IRQs the array can hold */ -+ digestsize_irqs = min_t(u32, digestsize_irqs, -+ LRNG_DATA_NUM_VALUES); -+ } -+ -+ for_each_online_cpu(cpu) { -+ if (!lrng_irq_pool_online(cpu)) -+ continue; -+ irq += min_t(u32, digestsize_irqs, -+ atomic_read_u32(per_cpu_ptr(&lrng_irq_array_irqs, -+ cpu))); -+ } -+ -+ /* Consider oversampling rate */ -+ return lrng_reduce_by_osr(lrng_data_to_entropy(irq, -+ lrng_irq_entropy_bits)); -+} -+ -+/* -+ * Trigger a switch of the hash implementation for the per-CPU pool. -+ * -+ * For each per-CPU pool, obtain the message digest with the old hash -+ * implementation, initialize the per-CPU pool again with the new hash -+ * implementation and inject the message digest into the new state. -+ * -+ * Assumption: the caller must guarantee that the new_cb is available during the -+ * entire operation (e.g. it must hold the lock against pointer updating). -+ */ -+static int -+lrng_irq_switch_hash(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ u32 digestsize_irqs, found_irqs; -+ int ret = 0, cpu; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ for_each_online_cpu(cpu) { -+ struct shash_desc *pcpu_shash; -+ -+ /* -+ * Only switch the per-CPU pools for the current node because -+ * the hash_cb only applies NUMA-node-wide. -+ */ -+ if (cpu_to_node(cpu) != node || !lrng_irq_pool_online(cpu)) -+ continue; -+ -+ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, -+ cpu); -+ -+ digestsize_irqs = old_cb->hash_digestsize(pcpu_shash); -+ digestsize_irqs = lrng_entropy_to_data(digestsize_irqs << 3, -+ lrng_irq_entropy_bits); -+ -+ if (pcpu_shash->tfm == new_hash) -+ continue; -+ -+ /* Get the per-CPU pool hash with old digest ... */ -+ ret = old_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash with the new digest ... */ -+ new_cb->hash_init(pcpu_shash, new_hash) ?: -+ /* -+ * ... feed the old hash into the new state. We may feed -+ * uninitialized memory into the new state, but this is -+ * considered no issue and even good as we have some more -+ * uncertainty here. -+ */ -+ new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); -+ if (ret) -+ goto out; -+ -+ /* -+ * In case the new digest is larger than the old one, cap -+ * the available entropy to the old message digest used to -+ * process the existing data. -+ */ -+ found_irqs = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); -+ found_irqs = min_t(u32, found_irqs, digestsize_irqs); -+ atomic_add_return_relaxed(found_irqs, -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu)); -+ -+ pr_debug("Re-initialize per-CPU interrupt entropy pool for CPU %d on NUMA node %d with hash %s\n", -+ cpu, node, new_cb->hash_name()); -+ } -+ -+out: -+ memzero_explicit(digest, sizeof(digest)); -+ return ret; -+} -+ -+/* -+ * When reading the per-CPU message digest, make sure we use the crypto -+ * callbacks defined for the NUMA node the per-CPU pool is defined for because -+ * the LRNG crypto switch support is only atomic per NUMA node. -+ */ -+static u32 -+lrng_irq_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, -+ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) -+{ -+ struct shash_desc *pcpu_shash = -+ (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, cpu); -+ spinlock_t *lock = per_cpu_ptr(&lrng_irq_lock, cpu); -+ unsigned long flags; -+ u32 digestsize_irqs, found_irqs; -+ -+ /* Lock guarding against reading / writing to per-CPU pool */ -+ spin_lock_irqsave(lock, flags); -+ -+ *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); -+ digestsize_irqs = lrng_entropy_to_data(*digestsize << 3, -+ lrng_irq_entropy_bits); -+ -+ /* Obtain entropy statement like for the entropy pool */ -+ found_irqs = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); -+ /* Cap to maximum amount of data we can hold in hash */ -+ found_irqs = min_t(u32, found_irqs, digestsize_irqs); -+ -+ /* Cap to maximum amount of data we can hold in array */ -+ if (!lrng_irq_continuous_compression) -+ found_irqs = min_t(u32, found_irqs, LRNG_DATA_NUM_VALUES); -+ -+ /* Store all not-yet compressed data in data array into hash, ... */ -+ if (pcpu_hash_cb->hash_update(pcpu_shash, -+ (u8 *)per_cpu_ptr(lrng_irq_array, cpu), -+ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: -+ /* ... get the per-CPU pool digest, ... */ -+ pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash, ... */ -+ pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: -+ /* ... feed the old hash into the new state. */ -+ pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) -+ found_irqs = 0; -+ -+ spin_unlock_irqrestore(lock, flags); -+ return found_irqs; -+} -+ -+/* -+ * Hash all per-CPU pools and return the digest to be used as seed data for -+ * seeding a DRNG. The caller must guarantee backtracking resistance. -+ * The function will only copy as much data as entropy is available into the -+ * caller-provided output buffer. -+ * -+ * This function handles the translation from the number of received interrupts -+ * into an entropy statement. The conversion depends on LRNG_IRQ_ENTROPY_BITS -+ * which defines how many interrupts must be received to obtain 256 bits of -+ * entropy. With this value, the function lrng_data_to_entropy converts a given -+ * data size (received interrupts, requested amount of data, etc.) into an -+ * entropy statement. lrng_entropy_to_data does the reverse. -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: Requested amount of entropy -+ * @fully_seeded: indicator whether LRNG is fully seeded -+ */ -+static void lrng_irq_pool_hash(struct entropy_buf *eb, u32 requested_bits, -+ bool fully_seeded) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb; -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ unsigned long flags, flags2; -+ u32 found_irqs, collected_irqs = 0, collected_ent_bits, requested_irqs, -+ returned_ent_bits; -+ int ret, cpu; -+ void *hash; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) { -+ eb->e_bits[lrng_int_es_irq] = 0; -+ return; -+ } -+ -+ /* Lock guarding replacement of per-NUMA hash */ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ /* The hash state of filled with all per-CPU pool hashes. */ -+ ret = hash_cb->hash_init(shash, hash); -+ if (ret) -+ goto err; -+ -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); -+ requested_irqs = lrng_entropy_to_data(requested_bits + -+ lrng_compress_osr(), -+ lrng_irq_entropy_bits); -+ -+ /* -+ * Harvest entropy from each per-CPU hash state - even though we may -+ * have collected sufficient entropy, we will hash all per-CPU pools. -+ */ -+ for_each_online_cpu(cpu) { -+ struct lrng_drng *pcpu_drng = drng; -+ u32 digestsize, pcpu_unused_irqs = 0; -+ int node = cpu_to_node(cpu); -+ -+ /* If pool is not online, then no entropy is present. */ -+ if (!lrng_irq_pool_online(cpu)) -+ continue; -+ -+ if (lrng_drng && lrng_drng[node]) -+ pcpu_drng = lrng_drng[node]; -+ -+ if (pcpu_drng == drng) { -+ found_irqs = lrng_irq_pool_hash_one(hash_cb, hash, -+ cpu, digest, -+ &digestsize); -+ } else { -+ read_lock_irqsave(&pcpu_drng->hash_lock, flags2); -+ found_irqs = -+ lrng_irq_pool_hash_one(pcpu_drng->hash_cb, -+ pcpu_drng->hash, cpu, -+ digest, &digestsize); -+ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); -+ } -+ -+ /* Inject the digest into the state of all per-CPU pools */ -+ ret = hash_cb->hash_update(shash, digest, digestsize); -+ if (ret) -+ goto err; -+ -+ collected_irqs += found_irqs; -+ if (collected_irqs > requested_irqs) { -+ pcpu_unused_irqs = collected_irqs - requested_irqs; -+ atomic_add_return_relaxed(pcpu_unused_irqs, -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu)); -+ collected_irqs = requested_irqs; -+ } -+ pr_debug("%u interrupts used from entropy pool of CPU %d, %u interrupts remain unused\n", -+ found_irqs - pcpu_unused_irqs, cpu, pcpu_unused_irqs); -+ } -+ -+ ret = hash_cb->hash_final(shash, digest); -+ if (ret) -+ goto err; -+ -+ collected_ent_bits = lrng_data_to_entropy(collected_irqs, -+ lrng_irq_entropy_bits); -+ /* Apply oversampling: discount requested oversampling rate */ -+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); -+ -+ pr_debug("obtained %u bits by collecting %u bits of entropy from entropy pool noise source\n", -+ returned_ent_bits, collected_ent_bits); -+ -+ /* -+ * Truncate to available entropy as implicitly allowed by SP800-90B -+ * section 3.1.5.1.1 table 1 which awards truncated hashes full -+ * entropy. -+ * -+ * During boot time, we read requested_bits data with -+ * returned_ent_bits entropy. In case our conservative entropy -+ * estimate underestimates the available entropy we can transport as -+ * much available entropy as possible. -+ */ -+ memcpy(eb->e[lrng_int_es_irq], digest, -+ fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); -+ eb->e_bits[lrng_int_es_irq] = returned_ent_bits; -+ -+out: -+ hash_cb->hash_desc_zero(shash); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ memzero_explicit(digest, sizeof(digest)); -+ return; -+ -+err: -+ eb->e_bits[lrng_int_es_irq] = 0; -+ goto out; -+} -+ -+/* Compress the lrng_irq_array array into lrng_irq_pool */ -+static void lrng_irq_array_compress(void) -+{ -+ struct shash_desc *shash = -+ (struct shash_desc *)this_cpu_ptr(lrng_irq_pool); -+ struct lrng_drng *drng = lrng_drng_node_instance(); -+ const struct lrng_hash_cb *hash_cb; -+ spinlock_t *lock = this_cpu_ptr(&lrng_irq_lock); -+ unsigned long flags, flags2; -+ void *hash; -+ bool init = false; -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ if (unlikely(!this_cpu_read(lrng_irq_lock_init))) { -+ init = true; -+ spin_lock_init(lock); -+ this_cpu_write(lrng_irq_lock_init, true); -+ pr_debug("Initializing per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n", -+ raw_smp_processor_id(), numa_node_id(), -+ hash_cb->hash_name()); -+ } -+ -+ spin_lock_irqsave(lock, flags2); -+ -+ if (unlikely(init) && hash_cb->hash_init(shash, hash)) { -+ this_cpu_write(lrng_irq_lock_init, false); -+ pr_warn("Initialization of hash failed\n"); -+ } else if (lrng_irq_continuous_compression) { -+ /* Add entire per-CPU data array content into entropy pool. */ -+ if (hash_cb->hash_update(shash, -+ (u8 *)this_cpu_ptr(lrng_irq_array), -+ LRNG_DATA_ARRAY_SIZE * sizeof(u32))) -+ pr_warn_ratelimited("Hashing of entropy data failed\n"); -+ } -+ -+ spin_unlock_irqrestore(lock, flags2); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+} -+ -+/* Compress data array into hash */ -+static void lrng_irq_array_to_hash(u32 ptr) -+{ -+ u32 *array = this_cpu_ptr(lrng_irq_array); -+ -+ /* -+ * During boot time the hash operation is triggered more often than -+ * during regular operation. -+ */ -+ if (unlikely(!lrng_state_fully_seeded())) { -+ if ((ptr & 31) && (ptr < LRNG_DATA_WORD_MASK)) -+ return; -+ } else if (ptr < LRNG_DATA_WORD_MASK) { -+ return; -+ } -+ -+ if (lrng_raw_array_entropy_store(*array)) { -+ u32 i; -+ -+ /* -+ * If we fed even a part of the array to external analysis, we -+ * mark that the entire array and the per-CPU pool to have no -+ * entropy. This is due to the non-IID property of the data as -+ * we do not fully know whether the existing dependencies -+ * diminish the entropy beyond to what we expect it has. -+ */ -+ atomic_set(this_cpu_ptr(&lrng_irq_array_irqs), 0); -+ -+ for (i = 1; i < LRNG_DATA_ARRAY_SIZE; i++) -+ lrng_raw_array_entropy_store(*(array + i)); -+ } else { -+ lrng_irq_array_compress(); -+ /* Ping pool handler about received entropy */ -+ if (lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) -+ lrng_es_add_entropy(); -+ } -+} -+ -+/* -+ * Concatenate full 32 bit word at the end of time array even when current -+ * ptr is not aligned to sizeof(data). -+ */ -+static void _lrng_irq_array_add_u32(u32 data) -+{ -+ /* Increment pointer by number of slots taken for input value */ -+ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_irq_array_ptr, -+ LRNG_DATA_SLOTS_PER_UINT); -+ unsigned int pre_array; -+ -+ /* -+ * This function injects a unit into the array - guarantee that -+ * array unit size is equal to data type of input data. -+ */ -+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != (sizeof(data) << 3)); -+ -+ /* -+ * The following logic requires at least two units holding -+ * the data as otherwise the pointer would immediately wrap when -+ * injection an u32 word. -+ */ -+ BUILD_BUG_ON(LRNG_DATA_NUM_VALUES <= LRNG_DATA_SLOTS_PER_UINT); -+ -+ lrng_data_split_u32(&ptr, &pre_ptr, &mask); -+ -+ /* MSB of data go into previous unit */ -+ pre_array = lrng_data_idx2array(pre_ptr); -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_irq_array[pre_array], ~(0xffffffff & ~mask)); -+ this_cpu_or(lrng_irq_array[pre_array], data & ~mask); -+ -+ /* Invoke compression as we just filled data array completely */ -+ if (unlikely(pre_ptr > ptr)) -+ lrng_irq_array_to_hash(LRNG_DATA_WORD_MASK); -+ -+ /* LSB of data go into current unit */ -+ this_cpu_write(lrng_irq_array[lrng_data_idx2array(ptr)], -+ data & mask); -+ -+ if (likely(pre_ptr <= ptr)) -+ lrng_irq_array_to_hash(ptr); -+} -+ -+/* Concatenate a 32-bit word at the end of the per-CPU array */ -+void lrng_irq_array_add_u32(u32 data) -+{ -+ /* -+ * Disregard entropy-less data without continuous compression to -+ * avoid it overwriting data with entropy when array ptr wraps. -+ */ -+ if (lrng_irq_continuous_compression) -+ _lrng_irq_array_add_u32(data); -+} -+ -+/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ -+static void lrng_irq_array_add_slot(u32 data) -+{ -+ /* Get slot */ -+ u32 ptr = this_cpu_inc_return(lrng_irq_array_ptr) & -+ LRNG_DATA_WORD_MASK; -+ unsigned int array = lrng_data_idx2array(ptr); -+ unsigned int slot = lrng_data_idx2slot(ptr); -+ -+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS % LRNG_DATA_SLOTSIZE_BITS); -+ /* Ensure consistency of values */ -+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != -+ sizeof(lrng_irq_array[0]) << 3); -+ -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_irq_array[array], -+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, -+ slot))); -+ /* Store data into slot */ -+ this_cpu_or(lrng_irq_array[array], lrng_data_slot_val(data, slot)); -+ -+ lrng_irq_array_to_hash(ptr); -+} -+ -+static void -+lrng_time_process_common(u32 time, void(*add_time)(u32 data)) -+{ -+ enum lrng_health_res health_test; -+ -+ if (lrng_raw_hires_entropy_store(time)) -+ return; -+ -+ health_test = lrng_health_test(time, lrng_int_es_irq); -+ if (health_test > lrng_health_fail_use) -+ return; -+ -+ if (health_test == lrng_health_pass) -+ atomic_inc_return(this_cpu_ptr(&lrng_irq_array_irqs)); -+ -+ add_time(time); -+} -+ -+/* -+ * Batching up of entropy in per-CPU array before injecting into entropy pool. -+ */ -+static void lrng_time_process(void) -+{ -+ u32 now_time = random_get_entropy(); -+ -+ if (unlikely(!lrng_gcd_tested())) { -+ /* When GCD is unknown, we process the full time stamp */ -+ lrng_time_process_common(now_time, _lrng_irq_array_add_u32); -+ lrng_gcd_add_value(now_time); -+ } else { -+ /* GCD is known and applied */ -+ lrng_time_process_common((now_time / lrng_gcd_get()) & -+ LRNG_DATA_SLOTSIZE_MASK, -+ lrng_irq_array_add_slot); -+ } -+ -+ lrng_perf_time(now_time); -+} -+ -+/* Hot code path - Callback for interrupt handler */ -+void add_interrupt_randomness(int irq) -+{ -+ if (lrng_highres_timer()) { -+ lrng_time_process(); -+ } else { -+ struct pt_regs *regs = get_irq_regs(); -+ static atomic_t reg_idx = ATOMIC_INIT(0); -+ u64 ip; -+ u32 tmp; -+ -+ if (regs) { -+ u32 *ptr = (u32 *)regs; -+ int reg_ptr = atomic_add_return_relaxed(1, ®_idx); -+ size_t n = (sizeof(struct pt_regs) / sizeof(u32)); -+ -+ ip = instruction_pointer(regs); -+ tmp = *(ptr + (reg_ptr % n)); -+ tmp = lrng_raw_regs_entropy_store(tmp) ? 0 : tmp; -+ _lrng_irq_array_add_u32(tmp); -+ } else { -+ ip = _RET_IP_; -+ } -+ -+ lrng_time_process(); -+ -+ /* -+ * The XOR operation combining the different values is not -+ * considered to destroy entropy since the entirety of all -+ * processed values delivers the entropy (and not each -+ * value separately of the other values). -+ */ -+ tmp = lrng_raw_jiffies_entropy_store(jiffies) ? 0 : jiffies; -+ tmp ^= lrng_raw_irq_entropy_store(irq) ? 0 : irq; -+ tmp ^= lrng_raw_retip_entropy_store(ip) ? 0 : ip; -+ tmp ^= ip >> 32; -+ _lrng_irq_array_add_u32(tmp); -+ } -+} -+EXPORT_SYMBOL(add_interrupt_randomness); -+ -+static void lrng_irq_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for operating entropy pool: %s\n" -+ " Available entropy: %u\n" -+ " per-CPU interrupt collection size: %u\n" -+ " Standards compliance: %s\n" -+ " High-resolution timer: %s\n" -+ " Continuous compression: %s\n", -+ lrng_drng_init->hash_cb->hash_name(), -+ lrng_irq_avail_entropy(0), -+ LRNG_DATA_NUM_VALUES, -+ lrng_sp80090b_compliant(lrng_int_es_irq) ? "SP800-90B " : "", -+ lrng_highres_timer() ? "true" : "false", -+ lrng_irq_continuous_compression ? "true" : "false"); -+} -+ -+struct lrng_es_cb lrng_es_irq = { -+ .name = "IRQ", -+ .get_ent = lrng_irq_pool_hash, -+ .curr_entropy = lrng_irq_avail_entropy, -+ .max_entropy = lrng_irq_avail_pool_size, -+ .state = lrng_irq_es_state, -+ .reset = lrng_irq_reset, -+ .switch_hash = lrng_irq_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_irq.h b/drivers/char/lrng/lrng_es_irq.h -new file mode 100644 -index 000000000000..2cd746611cf0 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_irq.h -@@ -0,0 +1,24 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_IRQ_H -+#define _LRNG_ES_IRQ_H -+ -+#include -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_IRQ -+void lrng_irq_es_init(bool highres_timer); -+void lrng_irq_array_add_u32(u32 data); -+ -+extern struct lrng_es_cb lrng_es_irq; -+ -+#else /* CONFIG_LRNG_IRQ */ -+static inline void lrng_irq_es_init(bool highres_timer) { } -+static inline void lrng_irq_array_add_u32(u32 data) { } -+#endif /* CONFIG_LRNG_IRQ */ -+ -+#endif /* _LRNG_ES_IRQ_H */ -diff --git a/drivers/char/lrng/lrng_es_jent.c b/drivers/char/lrng/lrng_es_jent.c -new file mode 100644 -index 000000000000..dbb2fce591f0 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_jent.c -@@ -0,0 +1,136 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Fast Entropy Source: Jitter RNG -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_jent.h" -+ -+/* -+ * Estimated entropy of data is a 16th of LRNG_DRNG_SECURITY_STRENGTH_BITS. -+ * Albeit a full entropy assessment is provided for the noise source indicating -+ * that it provides high entropy rates and considering that it deactivates -+ * when it detects insufficient hardware, the chosen under estimation of -+ * entropy is considered to be acceptable to all reviewers. -+ */ -+static u32 jent_entropy = CONFIG_LRNG_JENT_ENTROPY_RATE; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(jent_entropy, uint, 0644); -+MODULE_PARM_DESC(jent_entropy, "Entropy in bits of 256 data bits from Jitter RNG noise source"); -+#endif -+ -+static bool lrng_jent_initialized = false; -+static struct rand_data *lrng_jent_state; -+ -+static int __init lrng_jent_initialize(void) -+{ -+ /* Initialize the Jitter RNG after the clocksources are initialized. */ -+ if (jent_entropy_init() || -+ (lrng_jent_state = jent_entropy_collector_alloc(1, 0)) == NULL) { -+ jent_entropy = 0; -+ pr_info("Jitter RNG unusable on current system\n"); -+ return 0; -+ } -+ lrng_jent_initialized = true; -+ pr_debug("Jitter RNG working on current system\n"); -+ -+ /* -+ * In FIPS mode, the Jitter RNG is defined to have full of entropy -+ * unless a different value has been specified at the command line -+ * (i.e. the user overrides the default), and the default value is -+ * larger than zero (if it is zero, it is assumed that an RBG2(P) or -+ * RBG2(NP) construction is attempted that intends to exclude the -+ * Jitter RNG). -+ */ -+ if (fips_enabled && -+ CONFIG_LRNG_JENT_ENTROPY_RATE > 0 && -+ jent_entropy == CONFIG_LRNG_JENT_ENTROPY_RATE) -+ jent_entropy = LRNG_DRNG_SECURITY_STRENGTH_BITS; -+ -+ lrng_drng_force_reseed(); -+ if (jent_entropy) -+ lrng_es_add_entropy(); -+ -+ return 0; -+} -+device_initcall(lrng_jent_initialize); -+ -+static u32 lrng_jent_entropylevel(u32 requested_bits) -+{ -+ return lrng_fast_noise_entropylevel(lrng_jent_initialized ? -+ jent_entropy : 0, requested_bits); -+} -+ -+static u32 lrng_jent_poolsize(void) -+{ -+ return lrng_jent_entropylevel(lrng_security_strength()); -+} -+ -+/* -+ * lrng_get_jent() - Get Jitter RNG entropy -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: requested entropy in bits -+ */ -+static void lrng_jent_get(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ int ret; -+ u32 ent_bits = lrng_jent_entropylevel(requested_bits); -+ unsigned long flags; -+ static DEFINE_SPINLOCK(lrng_jent_lock); -+ -+ spin_lock_irqsave(&lrng_jent_lock, flags); -+ -+ if (!lrng_jent_initialized) { -+ spin_unlock_irqrestore(&lrng_jent_lock, flags); -+ goto err; -+ } -+ -+ ret = jent_read_entropy(lrng_jent_state, eb->e[lrng_ext_es_jitter], -+ requested_bits >> 3); -+ spin_unlock_irqrestore(&lrng_jent_lock, flags); -+ -+ if (ret) { -+ pr_debug("Jitter RNG failed with %d\n", ret); -+ goto err; -+ } -+ -+ pr_debug("obtained %u bits of entropy from Jitter RNG noise source\n", -+ ent_bits); -+ -+ eb->e_bits[lrng_ext_es_jitter] = ent_bits; -+ return; -+ -+err: -+ eb->e_bits[lrng_ext_es_jitter] = 0; -+} -+ -+static void lrng_jent_es_state(unsigned char *buf, size_t buflen) -+{ -+ snprintf(buf, buflen, -+ " Available entropy: %u\n" -+ " Enabled: %s\n", -+ lrng_jent_poolsize(), -+ lrng_jent_initialized ? "true" : "false"); -+} -+ -+struct lrng_es_cb lrng_es_jent = { -+ .name = "JitterRNG", -+ .get_ent = lrng_jent_get, -+ .curr_entropy = lrng_jent_entropylevel, -+ .max_entropy = lrng_jent_poolsize, -+ .state = lrng_jent_es_state, -+ .reset = NULL, -+ .switch_hash = NULL, -+}; -diff --git a/drivers/char/lrng/lrng_es_jent.h b/drivers/char/lrng/lrng_es_jent.h -new file mode 100644 -index 000000000000..32882d4bdf99 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_jent.h -@@ -0,0 +1,17 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_JENT_H -+#define _LRNG_ES_JENT_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_JENT -+ -+extern struct lrng_es_cb lrng_es_jent; -+ -+#endif /* CONFIG_LRNG_JENT */ -+ -+#endif /* _LRNG_ES_JENT_H */ -diff --git a/drivers/char/lrng/lrng_es_krng.c b/drivers/char/lrng/lrng_es_krng.c -new file mode 100644 -index 000000000000..519ba640cc75 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_krng.c -@@ -0,0 +1,100 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Fast Entropy Source: Linux kernel RNG (random.c) -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_krng.h" -+ -+static u32 krng_entropy = CONFIG_LRNG_KERNEL_RNG_ENTROPY_RATE; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(krng_entropy, uint, 0644); -+MODULE_PARM_DESC(krng_entropy, "Entropy in bits of 256 data bits from the kernel RNG noise source"); -+#endif -+ -+static atomic_t lrng_krng_initial_rate = ATOMIC_INIT(0); -+ -+static u32 lrng_krng_fips_entropylevel(u32 entropylevel) -+{ -+ return fips_enabled ? 0 : entropylevel; -+} -+ -+static int lrng_krng_adjust_entropy(void) -+{ -+ u32 entropylevel; -+ -+ krng_entropy = atomic_read_u32(&lrng_krng_initial_rate); -+ -+ entropylevel = lrng_krng_fips_entropylevel(krng_entropy); -+ pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", -+ entropylevel); -+ lrng_drng_force_reseed(); -+ if (entropylevel) -+ lrng_es_add_entropy(); -+ return 0; -+} -+ -+static u32 lrng_krng_entropylevel(u32 requested_bits) -+{ -+ static bool init = false; -+ -+ if (unlikely(!init) && rng_is_initialized()) { -+ init = true; -+ lrng_krng_adjust_entropy(); -+ } -+ -+ return lrng_fast_noise_entropylevel( -+ lrng_krng_fips_entropylevel(krng_entropy), requested_bits); -+} -+ -+static u32 lrng_krng_poolsize(void) -+{ -+ return lrng_krng_entropylevel(lrng_security_strength()); -+} -+ -+/* -+ * lrng_krng_get() - Get kernel RNG entropy -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: requested entropy in bits -+ */ -+static void lrng_krng_get(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ u32 ent_bits = lrng_krng_entropylevel(requested_bits); -+ -+ get_random_bytes(eb->e[lrng_ext_es_krng], requested_bits >> 3); -+ -+ pr_debug("obtained %u bits of entropy from kernel RNG noise source\n", -+ ent_bits); -+ -+ eb->e_bits[lrng_ext_es_krng] = ent_bits; -+} -+ -+static void lrng_krng_es_state(unsigned char *buf, size_t buflen) -+{ -+ snprintf(buf, buflen, -+ " Available entropy: %u\n" -+ " Entropy Rate per 256 data bits: %u\n", -+ lrng_krng_poolsize(), -+ lrng_krng_entropylevel(256)); -+} -+ -+struct lrng_es_cb lrng_es_krng = { -+ .name = "KernelRNG", -+ .get_ent = lrng_krng_get, -+ .curr_entropy = lrng_krng_entropylevel, -+ .max_entropy = lrng_krng_poolsize, -+ .state = lrng_krng_es_state, -+ .reset = NULL, -+ .switch_hash = NULL, -+}; -diff --git a/drivers/char/lrng/lrng_es_krng.h b/drivers/char/lrng/lrng_es_krng.h -new file mode 100644 -index 000000000000..cf982b9eea05 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_krng.h -@@ -0,0 +1,17 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_RANDOM_H -+#define _LRNG_ES_RANDOM_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_KERNEL_RNG -+ -+extern struct lrng_es_cb lrng_es_krng; -+ -+#endif /* CONFIG_LRNG_KERNEL_RNG */ -+ -+#endif /* _LRNG_ES_RANDOM_H */ -diff --git a/drivers/char/lrng/lrng_es_mgr.c b/drivers/char/lrng/lrng_es_mgr.c -new file mode 100644 -index 000000000000..23eba26b9f45 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_mgr.c -@@ -0,0 +1,460 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Entropy sources management -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_cpu.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_jent.h" -+#include "lrng_es_krng.h" -+#include "lrng_es_mgr.h" -+#include "lrng_es_sched.h" -+#include "lrng_interface_dev_common.h" -+#include "lrng_interface_random_kernel.h" -+ -+struct lrng_state { -+ bool can_invalidate; /* Can invalidate batched entropy? */ -+ bool perform_seedwork; /* Can seed work be performed? */ -+ bool lrng_operational; /* Is DRNG operational? */ -+ bool lrng_fully_seeded; /* Is DRNG fully seeded? */ -+ bool lrng_min_seeded; /* Is DRNG minimally seeded? */ -+ bool all_online_numa_node_seeded;/* All NUMA DRNGs seeded? */ -+ -+ /* -+ * To ensure that external entropy providers cannot dominate the -+ * internal noise sources but yet cannot be dominated by internal -+ * noise sources, the following booleans are intended to allow -+ * external to provide seed once when a DRNG reseed occurs. This -+ * triggering of external noise source is performed even when the -+ * entropy pool has sufficient entropy. -+ */ -+ -+ atomic_t boot_entropy_thresh; /* Reseed threshold */ -+ struct mutex reseed_in_progress; /* Flag for on executing reseed */ -+ struct work_struct lrng_seed_work; /* (re)seed work queue */ -+}; -+ -+static struct lrng_state lrng_state = { -+ false, false, false, false, false, false, -+ .boot_entropy_thresh = ATOMIC_INIT(LRNG_INIT_ENTROPY_BITS), -+ .reseed_in_progress = -+ __MUTEX_INITIALIZER(lrng_state.reseed_in_progress), -+}; -+ -+/* -+ * If the entropy count falls under this number of bits, then we -+ * should wake up processes which are selecting or polling on write -+ * access to /dev/random. -+ */ -+u32 lrng_write_wakeup_bits = (LRNG_WRITE_WAKEUP_ENTROPY << 3); -+ -+/* -+ * The entries must be in the same order as defined by enum lrng_internal_es and -+ * enum lrng_external_es -+ */ -+struct lrng_es_cb *lrng_es[] = { -+#ifdef CONFIG_LRNG_IRQ -+ &lrng_es_irq, -+#endif -+#ifdef CONFIG_LRNG_SCHED -+ &lrng_es_sched, -+#endif -+#ifdef CONFIG_LRNG_JENT -+ &lrng_es_jent, -+#endif -+#ifdef CONFIG_LRNG_CPU -+ &lrng_es_cpu, -+#endif -+#ifdef CONFIG_LRNG_KERNEL_RNG -+ &lrng_es_krng, -+#endif -+ &lrng_es_aux -+}; -+ -+/********************************** Helper ***********************************/ -+ -+void lrng_debug_report_seedlevel(const char *name) -+{ -+#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM -+ static void *previous = NULL; -+ void *caller = (void *) _RET_IP_; -+ -+ if (READ_ONCE(previous) == caller) -+ return; -+ -+ if (!lrng_state_min_seeded()) -+ pr_notice("%pS %s called without reaching minimally seeded level (available entropy %u)\n", -+ caller, name, lrng_avail_entropy()); -+ -+ WRITE_ONCE(previous, caller); -+#endif -+} -+ -+/* -+ * Reading of the LRNG pool is only allowed by one caller. The reading is -+ * only performed to (re)seed DRNGs. Thus, if this "lock" is already taken, -+ * the reseeding operation is in progress. The caller is not intended to wait -+ * but continue with its other operation. -+ */ -+int lrng_pool_trylock(void) -+{ -+ return mutex_trylock(&lrng_state.reseed_in_progress); -+} -+ -+void lrng_pool_lock(void) -+{ -+ mutex_lock(&lrng_state.reseed_in_progress); -+} -+ -+void lrng_pool_unlock(void) -+{ -+ mutex_unlock(&lrng_state.reseed_in_progress); -+} -+ -+/* Set new entropy threshold for reseeding during boot */ -+void lrng_set_entropy_thresh(u32 new_entropy_bits) -+{ -+ atomic_set(&lrng_state.boot_entropy_thresh, new_entropy_bits); -+} -+ -+/* -+ * Reset LRNG state - the entropy counters are reset, but the data that may -+ * or may not have entropy remains in the pools as this data will not hurt. -+ */ -+void lrng_reset_state(void) -+{ -+ u32 i; -+ -+ for_each_lrng_es(i) { -+ if (lrng_es[i]->reset) -+ lrng_es[i]->reset(); -+ } -+ lrng_state.lrng_operational = false; -+ lrng_state.lrng_fully_seeded = false; -+ lrng_state.lrng_min_seeded = false; -+ lrng_state.all_online_numa_node_seeded = false; -+ pr_debug("reset LRNG\n"); -+} -+ -+/* Set flag that all DRNGs are fully seeded */ -+void lrng_pool_all_numa_nodes_seeded(bool set) -+{ -+ lrng_state.all_online_numa_node_seeded = set; -+ if (set) -+ wake_up_all(&lrng_init_wait); -+} -+ -+bool lrng_pool_all_numa_nodes_seeded_get(void) -+{ -+ return lrng_state.all_online_numa_node_seeded; -+} -+ -+/* Return boolean whether LRNG reached minimally seed level */ -+bool lrng_state_min_seeded(void) -+{ -+ return lrng_state.lrng_min_seeded; -+} -+ -+/* Return boolean whether LRNG reached fully seed level */ -+bool lrng_state_fully_seeded(void) -+{ -+ return lrng_state.lrng_fully_seeded; -+} -+ -+/* Return boolean whether LRNG is considered fully operational */ -+bool lrng_state_operational(void) -+{ -+ return lrng_state.lrng_operational; -+} -+ -+static void lrng_init_wakeup(void) -+{ -+ wake_up_all(&lrng_init_wait); -+ lrng_init_wakeup_dev(); -+} -+ -+bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy) -+{ -+ return (collected_entropy >= lrng_get_seed_entropy_osr(fully_seeded)); -+} -+ -+u32 lrng_entropy_rate_eb(struct entropy_buf *eb) -+{ -+ u32 i, collected_entropy = 0; -+ -+ for_each_lrng_es(i) -+ collected_entropy += eb->e_bits[i]; -+ -+ return collected_entropy; -+} -+ -+/* Mark one DRNG as not fully seeded */ -+void lrng_unset_fully_seeded(struct lrng_drng *drng) -+{ -+ drng->fully_seeded = false; -+ lrng_pool_all_numa_nodes_seeded(false); -+ -+ /* -+ * The init DRNG instance must always be fully seeded as this instance -+ * is the fall-back if any of the per-NUMA node DRNG instances is -+ * insufficiently seeded. Thus, we mark the entire LRNG as -+ * non-operational if the initial DRNG becomes not fully seeded. -+ */ -+ if (drng == lrng_drng_init_instance() && lrng_state_operational()) { -+ pr_debug("LRNG set to non-operational\n"); -+ lrng_state.lrng_operational = false; -+ lrng_state.lrng_fully_seeded = false; -+ -+ /* If sufficient entropy is available, reseed now. */ -+ lrng_es_add_entropy(); -+ } -+} -+ -+/* Policy to enable LRNG operational mode */ -+static void lrng_set_operational(void) -+{ -+ /* -+ * LRNG is operational if the initial DRNG is fully seeded. This state -+ * can only occur if either the external entropy sources provided -+ * sufficient entropy, or the SP800-90B startup test completed for -+ * the internal ES to supply also entropy data. -+ */ -+ if (lrng_state.lrng_fully_seeded) { -+ lrng_state.lrng_operational = true; -+ lrng_init_wakeup(); -+ pr_info("LRNG fully operational\n"); -+ } -+} -+ -+static u32 lrng_avail_entropy_thresh(void) -+{ -+ u32 ent_thresh = lrng_security_strength(); -+ -+ /* -+ * Apply oversampling during initialization according to SP800-90C as -+ * we request a larger buffer from the ES. -+ */ -+ if (lrng_sp80090c_compliant() && -+ !lrng_state.all_online_numa_node_seeded) -+ ent_thresh += LRNG_SEED_BUFFER_INIT_ADD_BITS; -+ -+ return ent_thresh; -+} -+ -+/* Available entropy in the entire LRNG considering all entropy sources */ -+u32 lrng_avail_entropy(void) -+{ -+ u32 i, ent = 0, ent_thresh = lrng_avail_entropy_thresh(); -+ -+ BUILD_BUG_ON(ARRAY_SIZE(lrng_es) != lrng_ext_es_last); -+ for_each_lrng_es(i) -+ ent += lrng_es[i]->curr_entropy(ent_thresh); -+ return ent; -+} -+ -+u32 lrng_avail_entropy_aux(void) -+{ -+ u32 ent_thresh = lrng_avail_entropy_thresh(); -+ -+ return lrng_es[lrng_ext_es_aux]->curr_entropy(ent_thresh); -+} -+ -+/* -+ * lrng_init_ops() - Set seed stages of LRNG -+ * -+ * Set the slow noise source reseed trigger threshold. The initial threshold -+ * is set to the minimum data size that can be read from the pool: a word. Upon -+ * reaching this value, the next seed threshold of 128 bits is set followed -+ * by 256 bits. -+ * -+ * @eb: buffer containing the size of entropy currently injected into DRNG - if -+ * NULL, the function obtains the available entropy from the ES. -+ */ -+void lrng_init_ops(struct entropy_buf *eb) -+{ -+ struct lrng_state *state = &lrng_state; -+ u32 i, requested_bits, seed_bits = 0; -+ -+ if (state->lrng_operational) -+ return; -+ -+ requested_bits = lrng_get_seed_entropy_osr( -+ state->all_online_numa_node_seeded); -+ -+ if (eb) { -+ for_each_lrng_es(i) -+ seed_bits += eb->e_bits[i]; -+ } else { -+ u32 ent_thresh = lrng_avail_entropy_thresh(); -+ -+ for_each_lrng_es(i) -+ seed_bits += lrng_es[i]->curr_entropy(ent_thresh); -+ } -+ -+ /* DRNG is seeded with full security strength */ -+ if (state->lrng_fully_seeded) { -+ lrng_set_operational(); -+ lrng_set_entropy_thresh(requested_bits); -+ } else if (lrng_fully_seeded(state->all_online_numa_node_seeded, -+ seed_bits)) { -+ if (state->can_invalidate) -+ invalidate_batched_entropy(); -+ -+ state->lrng_fully_seeded = true; -+ lrng_set_operational(); -+ state->lrng_min_seeded = true; -+ pr_info("LRNG fully seeded with %u bits of entropy\n", -+ seed_bits); -+ lrng_set_entropy_thresh(requested_bits); -+ } else if (!state->lrng_min_seeded) { -+ -+ /* DRNG is seeded with at least 128 bits of entropy */ -+ if (seed_bits >= LRNG_MIN_SEED_ENTROPY_BITS) { -+ if (state->can_invalidate) -+ invalidate_batched_entropy(); -+ -+ state->lrng_min_seeded = true; -+ pr_info("LRNG minimally seeded with %u bits of entropy\n", -+ seed_bits); -+ lrng_set_entropy_thresh(requested_bits); -+ lrng_init_wakeup(); -+ -+ /* DRNG is seeded with at least LRNG_INIT_ENTROPY_BITS bits */ -+ } else if (seed_bits >= LRNG_INIT_ENTROPY_BITS) { -+ pr_info("LRNG initial entropy level %u bits of entropy\n", -+ seed_bits); -+ lrng_set_entropy_thresh(LRNG_MIN_SEED_ENTROPY_BITS); -+ } -+ } -+} -+ -+int __init lrng_rand_initialize(void) -+{ -+ struct seed { -+ ktime_t time; -+ unsigned long data[((LRNG_MAX_DIGESTSIZE + -+ sizeof(unsigned long) - 1) / -+ sizeof(unsigned long))]; -+ struct new_utsname utsname; -+ } seed __aligned(LRNG_KCAPI_ALIGN); -+ size_t longs = 0; -+ unsigned int i; -+ -+ seed.time = ktime_get_real(); -+ -+ for (i = 0; i < ARRAY_SIZE(seed.data); i += longs) { -+#ifdef CONFIG_LRNG_RANDOM_IF -+ longs = arch_get_random_seed_longs_early( -+ seed.data + i, ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+ longs = arch_get_random_longs_early(seed.data + i, -+ ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+#else -+ longs = arch_get_random_seed_longs(seed.data + i, -+ ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+ longs = arch_get_random_longs(seed.data + i, -+ ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+#endif -+ seed.data[i] = random_get_entropy(); -+ longs = 1; -+ } -+ memcpy(&seed.utsname, utsname(), sizeof(*(utsname()))); -+ -+ lrng_pool_insert_aux((u8 *)&seed, sizeof(seed), 0); -+ memzero_explicit(&seed, sizeof(seed)); -+ -+ /* Initialize the seed work queue */ -+ INIT_WORK(&lrng_state.lrng_seed_work, lrng_drng_seed_work); -+ lrng_state.perform_seedwork = true; -+ -+ invalidate_batched_entropy(); -+ -+ lrng_state.can_invalidate = true; -+ -+ return 0; -+} -+ -+#ifndef CONFIG_LRNG_RANDOM_IF -+early_initcall(lrng_rand_initialize); -+#endif -+ -+/* Interface requesting a reseed of the DRNG */ -+void lrng_es_add_entropy(void) -+{ -+ /* -+ * Once all DRNGs are fully seeded, the system-triggered arrival of -+ * entropy will not cause any reseeding any more. -+ */ -+ if (likely(lrng_state.all_online_numa_node_seeded)) -+ return; -+ -+ /* Only trigger the DRNG reseed if we have collected entropy. */ -+ if (lrng_avail_entropy() < -+ atomic_read_u32(&lrng_state.boot_entropy_thresh)) -+ return; -+ -+ /* Ensure that the seeding only occurs once at any given time. */ -+ if (!lrng_pool_trylock()) -+ return; -+ -+ /* Seed the DRNG with any available noise. */ -+ if (lrng_state.perform_seedwork) -+ schedule_work(&lrng_state.lrng_seed_work); -+ else -+ lrng_drng_seed_work(NULL); -+} -+ -+/* Fill the seed buffer with data from the noise sources */ -+void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits) -+{ -+ struct lrng_state *state = &lrng_state; -+ u32 i, req_ent = lrng_sp80090c_compliant() ? -+ lrng_security_strength() : LRNG_MIN_SEED_ENTROPY_BITS; -+ -+ /* Guarantee that requested bits is a multiple of bytes */ -+ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BITS % 8); -+ -+ /* always reseed the DRNG with the current time stamp */ -+ eb->now = random_get_entropy(); -+ -+ /* -+ * Require at least 128 bits of entropy for any reseed. If the LRNG is -+ * operated SP800-90C compliant we want to comply with SP800-90A section -+ * 9.2 mandating that DRNG is reseeded with the security strength. -+ */ -+ if (state->lrng_fully_seeded && (lrng_avail_entropy() < req_ent)) { -+ for_each_lrng_es(i) -+ eb->e_bits[i] = 0; -+ -+ goto wakeup; -+ } -+ -+ /* Concatenate the output of the entropy sources. */ -+ for_each_lrng_es(i) { -+ lrng_es[i]->get_ent(eb, requested_bits, -+ state->lrng_fully_seeded); -+ } -+ -+ /* allow external entropy provider to provide seed */ -+ lrng_state_exseed_allow_all(); -+ -+wakeup: -+ lrng_writer_wakeup(); -+} -diff --git a/drivers/char/lrng/lrng_es_mgr.h b/drivers/char/lrng/lrng_es_mgr.h -new file mode 100644 -index 000000000000..f9bbd4863120 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_mgr.h -@@ -0,0 +1,51 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_MGR_H -+#define _LRNG_ES_MGR_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+/*************************** General LRNG parameter ***************************/ -+ -+#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */ -+ -+/* Helper to concatenate a macro with an integer type */ -+#define LRNG_PASTER(x, y) x ## y -+#define LRNG_UINT32_C(x) LRNG_PASTER(x, U) -+ -+/************************* Entropy sources management *************************/ -+ -+extern struct lrng_es_cb *lrng_es[]; -+ -+#define for_each_lrng_es(ctr) \ -+ for ((ctr) = 0; (ctr) < lrng_ext_es_last; (ctr)++) -+ -+bool lrng_pool_all_numa_nodes_seeded_get(void); -+bool lrng_state_min_seeded(void); -+void lrng_debug_report_seedlevel(const char *name); -+int lrng_rand_initialize(void); -+bool lrng_state_operational(void); -+ -+extern u32 lrng_write_wakeup_bits; -+void lrng_set_entropy_thresh(u32 new); -+u32 lrng_avail_entropy(void); -+u32 lrng_avail_entropy_aux(void); -+void lrng_reset_state(void); -+ -+bool lrng_state_fully_seeded(void); -+ -+int lrng_pool_trylock(void); -+void lrng_pool_lock(void); -+void lrng_pool_unlock(void); -+void lrng_pool_all_numa_nodes_seeded(bool set); -+ -+bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy); -+u32 lrng_entropy_rate_eb(struct entropy_buf *eb); -+void lrng_unset_fully_seeded(struct lrng_drng *drng); -+void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits); -+void lrng_init_ops(struct entropy_buf *eb); -+ -+#endif /* _LRNG_ES_MGR_H */ -diff --git a/drivers/char/lrng/lrng_es_mgr_cb.h b/drivers/char/lrng/lrng_es_mgr_cb.h -new file mode 100644 -index 000000000000..08b24e1b7766 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_mgr_cb.h -@@ -0,0 +1,87 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ * -+ * Definition of an entropy source. -+ */ -+ -+#ifndef _LRNG_ES_MGR_CB_H -+#define _LRNG_ES_MGR_CB_H -+ -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_drng_mgr.h" -+ -+enum lrng_internal_es { -+#ifdef CONFIG_LRNG_IRQ -+ lrng_int_es_irq, /* IRQ-based entropy source */ -+#endif -+#ifdef CONFIG_LRNG_SCHED -+ lrng_int_es_sched, /* Scheduler entropy source */ -+#endif -+ lrng_int_es_last, /* MUST be the last entry */ -+}; -+ -+enum lrng_external_es { -+ lrng_ext_link = lrng_int_es_last - 1, /* Link entry */ -+#ifdef CONFIG_LRNG_JENT -+ lrng_ext_es_jitter, /* Jitter RNG */ -+#endif -+#ifdef CONFIG_LRNG_CPU -+ lrng_ext_es_cpu, /* CPU-based, e.g. RDSEED */ -+#endif -+#ifdef CONFIG_LRNG_KERNEL_RNG -+ lrng_ext_es_krng, /* random.c */ -+#endif -+ lrng_ext_es_aux, /* MUST BE LAST ES! */ -+ lrng_ext_es_last /* MUST be the last entry */ -+}; -+ -+struct entropy_buf { -+ u8 e[lrng_ext_es_last][LRNG_DRNG_INIT_SEED_SIZE_BYTES]; -+ u32 now, e_bits[lrng_ext_es_last]; -+}; -+ -+/* -+ * struct lrng_es_cb - callback defining an entropy source -+ * @name: Name of the entropy source. -+ * @get_ent: Fetch entropy into the entropy_buf. The ES shall only deliver -+ * data if its internal initialization is complete, including any -+ * SP800-90B startup testing or similar. -+ * @curr_entropy: Return amount of currently available entropy. -+ * @max_entropy: Maximum amount of entropy the entropy source is able to -+ * maintain. -+ * @state: Buffer with human-readable ES state. -+ * @reset: Reset entropy source (drop all entropy and reinitialize). -+ * This callback may be NULL. -+ * @switch_hash: callback to switch from an old hash callback definition to -+ * a new one. This callback may be NULL. -+ */ -+struct lrng_es_cb { -+ const char *name; -+ void (*get_ent)(struct entropy_buf *eb, u32 requested_bits, -+ bool fully_seeded); -+ u32 (*curr_entropy)(u32 requested_bits); -+ u32 (*max_entropy)(void); -+ void (*state)(unsigned char *buf, size_t buflen); -+ void (*reset)(void); -+ int (*switch_hash)(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb); -+}; -+ -+/* Allow entropy sources to tell the ES manager that new entropy is there */ -+void lrng_es_add_entropy(void); -+ -+/* Cap to maximum entropy that can ever be generated with given hash */ -+#define lrng_cap_requested(__digestsize_bits, __requested_bits) \ -+ do { \ -+ if (__digestsize_bits < __requested_bits) { \ -+ pr_debug("Cannot satisfy requested entropy %u due to insufficient hash size %u\n",\ -+ __requested_bits, __digestsize_bits); \ -+ __requested_bits = __digestsize_bits; \ -+ } \ -+ } while (0) -+ -+#endif /* _LRNG_ES_MGR_CB_H */ -diff --git a/drivers/char/lrng/lrng_es_sched.c b/drivers/char/lrng/lrng_es_sched.c -new file mode 100644 -index 000000000000..c3abaa7ff310 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_sched.c -@@ -0,0 +1,562 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Scheduler-based data collection -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_sched.h" -+#include "lrng_es_timer_common.h" -+#include "lrng_health.h" -+#include "lrng_numa.h" -+#include "lrng_testing.h" -+ -+/* -+ * Number of scheduler-based context switches to be recorded to assume that -+ * DRNG security strength bits of entropy are received. -+ * Note: a value below the DRNG security strength should not be defined as this -+ * may imply the DRNG can never be fully seeded in case other noise -+ * sources are unavailable. -+ */ -+#define LRNG_SCHED_ENTROPY_BITS \ -+ LRNG_UINT32_C(CONFIG_LRNG_SCHED_ENTROPY_RATE) -+ -+/* Number of events required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ -+static u32 lrng_sched_entropy_bits = LRNG_SCHED_ENTROPY_BITS; -+ -+static u32 sched_entropy __read_mostly = LRNG_SCHED_ENTROPY_BITS; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(sched_entropy, uint, 0444); -+MODULE_PARM_DESC(sched_entropy, -+ "How many scheduler-based context switches must be collected for obtaining 256 bits of entropy\n"); -+#endif -+ -+/* Per-CPU array holding concatenated entropy events */ -+static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_sched_array) -+ __aligned(LRNG_KCAPI_ALIGN); -+static DEFINE_PER_CPU(u32, lrng_sched_array_ptr) = 0; -+static DEFINE_PER_CPU(atomic_t, lrng_sched_array_events) = ATOMIC_INIT(0); -+ -+/* -+ * Per-CPU entropy pool with compressed entropy event -+ * -+ * The per-CPU entropy pool is defined as the hash state. New data is simply -+ * inserted into the entropy pool by performing a hash update operation. -+ * To read the entropy pool, a hash final must be invoked. However, before -+ * the entropy pool is released again after a hash final, the hash init must -+ * be performed. -+ */ -+static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_sched_pool) -+ __aligned(LRNG_KCAPI_ALIGN); -+/* -+ * Lock to allow other CPUs to read the pool - as this is only done during -+ * reseed which is infrequent, this lock is hardly contended. -+ */ -+static DEFINE_PER_CPU(spinlock_t, lrng_sched_lock); -+static DEFINE_PER_CPU(bool, lrng_sched_lock_init) = false; -+ -+static bool lrng_sched_pool_online(int cpu) -+{ -+ return per_cpu(lrng_sched_lock_init, cpu); -+} -+ -+static void __init lrng_sched_check_compression_state(void) -+{ -+ /* One pool should hold sufficient entropy for disabled compression */ -+ u32 max_ent = min_t(u32, lrng_get_digestsize(), -+ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, -+ lrng_sched_entropy_bits)); -+ if (max_ent < lrng_security_strength()) { -+ pr_devel("Scheduler entropy source will never provide %u bits of entropy required for fully seeding the DRNG all by itself\n", -+ lrng_security_strength()); -+ } -+} -+ -+void __init lrng_sched_es_init(bool highres_timer) -+{ -+ /* Set a minimum number of scheduler events that must be collected */ -+ sched_entropy = max_t(u32, LRNG_SCHED_ENTROPY_BITS, sched_entropy); -+ -+ if (highres_timer) { -+ lrng_sched_entropy_bits = sched_entropy; -+ } else { -+ u32 new_entropy = sched_entropy * LRNG_ES_OVERSAMPLING_FACTOR; -+ -+ lrng_sched_entropy_bits = (sched_entropy < new_entropy) ? -+ new_entropy : sched_entropy; -+ pr_warn("operating without high-resolution timer and applying oversampling factor %u\n", -+ LRNG_ES_OVERSAMPLING_FACTOR); -+ } -+ -+ lrng_sched_check_compression_state(); -+} -+ -+static u32 lrng_sched_avail_pool_size(void) -+{ -+ u32 max_pool = lrng_get_digestsize(), -+ max_size = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); -+ int cpu; -+ -+ for_each_online_cpu(cpu) -+ max_size += max_pool; -+ -+ return max_size; -+} -+ -+/* Return entropy of unused scheduler events present in all per-CPU pools. */ -+static u32 lrng_sched_avail_entropy(u32 __unused) -+{ -+ u32 digestsize_events, events = 0; -+ int cpu; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) -+ return 0; -+ -+ /* Obtain the cap of maximum numbers of scheduler events we count */ -+ digestsize_events = lrng_entropy_to_data(lrng_get_digestsize(), -+ lrng_sched_entropy_bits); -+ /* Cap to max. number of scheduler events the array can hold */ -+ digestsize_events = min_t(u32, digestsize_events, LRNG_DATA_NUM_VALUES); -+ -+ for_each_online_cpu(cpu) { -+ events += min_t(u32, digestsize_events, -+ atomic_read_u32(per_cpu_ptr(&lrng_sched_array_events, -+ cpu))); -+ } -+ -+ /* Consider oversampling rate */ -+ return lrng_reduce_by_osr( -+ lrng_data_to_entropy(events, lrng_sched_entropy_bits)); -+} -+ -+/* -+ * Reset all per-CPU pools - reset entropy estimator but leave the pool data -+ * that may or may not have entropy unchanged. -+ */ -+static void lrng_sched_reset(void) -+{ -+ int cpu; -+ -+ /* Trigger GCD calculation anew. */ -+ lrng_gcd_set(0); -+ -+ for_each_online_cpu(cpu) -+ atomic_set(per_cpu_ptr(&lrng_sched_array_events, cpu), 0); -+} -+ -+/* -+ * Trigger a switch of the hash implementation for the per-CPU pool. -+ * -+ * For each per-CPU pool, obtain the message digest with the old hash -+ * implementation, initialize the per-CPU pool again with the new hash -+ * implementation and inject the message digest into the new state. -+ * -+ * Assumption: the caller must guarantee that the new_cb is available during the -+ * entire operation (e.g. it must hold the lock against pointer updating). -+ */ -+static int -+lrng_sched_switch_hash(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ u32 digestsize_events, found_events; -+ int ret = 0, cpu; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ for_each_online_cpu(cpu) { -+ struct shash_desc *pcpu_shash; -+ -+ /* -+ * Only switch the per-CPU pools for the current node because -+ * the hash_cb only applies NUMA-node-wide. -+ */ -+ if (cpu_to_node(cpu) != node || !lrng_sched_pool_online(cpu)) -+ continue; -+ -+ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, -+ cpu); -+ -+ digestsize_events = old_cb->hash_digestsize(pcpu_shash); -+ digestsize_events = lrng_entropy_to_data(digestsize_events << 3, -+ lrng_sched_entropy_bits); -+ -+ if (pcpu_shash->tfm == new_hash) -+ continue; -+ -+ /* Get the per-CPU pool hash with old digest ... */ -+ ret = old_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash with the new digest ... */ -+ new_cb->hash_init(pcpu_shash, new_hash) ?: -+ /* -+ * ... feed the old hash into the new state. We may feed -+ * uninitialized memory into the new state, but this is -+ * considered no issue and even good as we have some more -+ * uncertainty here. -+ */ -+ new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); -+ if (ret) -+ goto out; -+ -+ /* -+ * In case the new digest is larger than the old one, cap -+ * the available entropy to the old message digest used to -+ * process the existing data. -+ */ -+ found_events = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_sched_array_events, cpu), 0); -+ found_events = min_t(u32, found_events, digestsize_events); -+ atomic_add_return_relaxed(found_events, -+ per_cpu_ptr(&lrng_sched_array_events, cpu)); -+ -+ pr_debug("Re-initialize per-CPU scheduler entropy pool for CPU %d on NUMA node %d with hash %s\n", -+ cpu, node, new_cb->hash_name()); -+ } -+ -+out: -+ memzero_explicit(digest, sizeof(digest)); -+ return ret; -+} -+ -+static u32 -+lrng_sched_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, -+ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) -+{ -+ struct shash_desc *pcpu_shash = -+ (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, cpu); -+ spinlock_t *lock = per_cpu_ptr(&lrng_sched_lock, cpu); -+ unsigned long flags; -+ u32 digestsize_events, found_events; -+ -+ if (unlikely(!per_cpu(lrng_sched_lock_init, cpu))) { -+ if (pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash)) { -+ pr_warn("Initialization of hash failed\n"); -+ return 0; -+ } -+ spin_lock_init(lock); -+ per_cpu(lrng_sched_lock_init, cpu) = true; -+ pr_debug("Initializing per-CPU scheduler entropy pool for CPU %d with hash %s\n", -+ raw_smp_processor_id(), pcpu_hash_cb->hash_name()); -+ } -+ -+ /* Lock guarding against reading / writing to per-CPU pool */ -+ spin_lock_irqsave(lock, flags); -+ -+ *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); -+ digestsize_events = lrng_entropy_to_data(*digestsize << 3, -+ lrng_sched_entropy_bits); -+ -+ /* Obtain entropy statement like for the entropy pool */ -+ found_events = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_sched_array_events, cpu), 0); -+ /* Cap to maximum amount of data we can hold in hash */ -+ found_events = min_t(u32, found_events, digestsize_events); -+ -+ /* Cap to maximum amount of data we can hold in array */ -+ found_events = min_t(u32, found_events, LRNG_DATA_NUM_VALUES); -+ -+ /* Store all not-yet compressed data in data array into hash, ... */ -+ if (pcpu_hash_cb->hash_update(pcpu_shash, -+ (u8 *)per_cpu_ptr(lrng_sched_array, cpu), -+ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: -+ /* ... get the per-CPU pool digest, ... */ -+ pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash, ... */ -+ pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: -+ /* ... feed the old hash into the new state. */ -+ pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) -+ found_events = 0; -+ -+ spin_unlock_irqrestore(lock, flags); -+ return found_events; -+} -+ -+/* -+ * Hash all per-CPU arrays and return the digest to be used as seed data for -+ * seeding a DRNG. The caller must guarantee backtracking resistance. -+ * The function will only copy as much data as entropy is available into the -+ * caller-provided output buffer. -+ * -+ * This function handles the translation from the number of received scheduler -+ * events into an entropy statement. The conversion depends on -+ * LRNG_SCHED_ENTROPY_BITS which defines how many scheduler events must be -+ * received to obtain 256 bits of entropy. With this value, the function -+ * lrng_data_to_entropy converts a given data size (received scheduler events, -+ * requested amount of data, etc.) into an entropy statement. -+ * lrng_entropy_to_data does the reverse. -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: Requested amount of entropy -+ * @fully_seeded: indicator whether LRNG is fully seeded -+ */ -+static void lrng_sched_pool_hash(struct entropy_buf *eb, u32 requested_bits, -+ bool fully_seeded) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb; -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ unsigned long flags, flags2; -+ u32 found_events, collected_events = 0, collected_ent_bits, -+ requested_events, returned_ent_bits; -+ int ret, cpu; -+ void *hash; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) { -+ eb->e_bits[lrng_int_es_sched] = 0; -+ return; -+ } -+ -+ /* Lock guarding replacement of per-NUMA hash */ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ /* The hash state of filled with all per-CPU pool hashes. */ -+ ret = hash_cb->hash_init(shash, hash); -+ if (ret) -+ goto err; -+ -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); -+ requested_events = lrng_entropy_to_data(requested_bits + -+ lrng_compress_osr(), -+ lrng_sched_entropy_bits); -+ -+ /* -+ * Harvest entropy from each per-CPU hash state - even though we may -+ * have collected sufficient entropy, we will hash all per-CPU pools. -+ */ -+ for_each_online_cpu(cpu) { -+ struct lrng_drng *pcpu_drng = drng; -+ u32 digestsize, unused_events = 0; -+ int node = cpu_to_node(cpu); -+ -+ if (lrng_drng && lrng_drng[node]) -+ pcpu_drng = lrng_drng[node]; -+ -+ if (pcpu_drng == drng) { -+ found_events = lrng_sched_pool_hash_one(hash_cb, hash, -+ cpu, digest, -+ &digestsize); -+ } else { -+ read_lock_irqsave(&pcpu_drng->hash_lock, flags2); -+ found_events = -+ lrng_sched_pool_hash_one(pcpu_drng->hash_cb, -+ pcpu_drng->hash, cpu, -+ digest, &digestsize); -+ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); -+ } -+ -+ /* Store all not-yet compressed data in data array into hash */ -+ ret = hash_cb->hash_update(shash, digest, digestsize); -+ if (ret) -+ goto err; -+ -+ collected_events += found_events; -+ if (collected_events > requested_events) { -+ unused_events = collected_events - requested_events; -+ atomic_add_return_relaxed(unused_events, -+ per_cpu_ptr(&lrng_sched_array_events, cpu)); -+ collected_events = requested_events; -+ } -+ pr_debug("%u scheduler-based events used from entropy array of CPU %d, %u scheduler-based events remain unused\n", -+ found_events - unused_events, cpu, unused_events); -+ } -+ -+ ret = hash_cb->hash_final(shash, digest); -+ if (ret) -+ goto err; -+ -+ collected_ent_bits = lrng_data_to_entropy(collected_events, -+ lrng_sched_entropy_bits); -+ /* Apply oversampling: discount requested oversampling rate */ -+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); -+ -+ pr_debug("obtained %u bits by collecting %u bits of entropy from scheduler-based noise source\n", -+ returned_ent_bits, collected_ent_bits); -+ -+ /* -+ * Truncate to available entropy as implicitly allowed by SP800-90B -+ * section 3.1.5.1.1 table 1 which awards truncated hashes full -+ * entropy. -+ * -+ * During boot time, we read requested_bits data with -+ * returned_ent_bits entropy. In case our conservative entropy -+ * estimate underestimates the available entropy we can transport as -+ * much available entropy as possible. -+ */ -+ memcpy(eb->e[lrng_int_es_sched], digest, -+ fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); -+ eb->e_bits[lrng_int_es_sched] = returned_ent_bits; -+ -+out: -+ hash_cb->hash_desc_zero(shash); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ memzero_explicit(digest, sizeof(digest)); -+ return; -+ -+err: -+ eb->e_bits[lrng_int_es_sched] = 0; -+ goto out; -+} -+ -+/* -+ * Concatenate full 32 bit word at the end of time array even when current -+ * ptr is not aligned to sizeof(data). -+ */ -+static void lrng_sched_array_add_u32(u32 data) -+{ -+ /* Increment pointer by number of slots taken for input value */ -+ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_sched_array_ptr, -+ LRNG_DATA_SLOTS_PER_UINT); -+ unsigned int pre_array; -+ -+ lrng_data_split_u32(&ptr, &pre_ptr, &mask); -+ -+ /* MSB of data go into previous unit */ -+ pre_array = lrng_data_idx2array(pre_ptr); -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_sched_array[pre_array], ~(0xffffffff & ~mask)); -+ this_cpu_or(lrng_sched_array[pre_array], data & ~mask); -+ -+ /* -+ * Continuous compression is not allowed for scheduler noise source, -+ * so do not call lrng_sched_array_to_hash here. -+ */ -+ -+ /* LSB of data go into current unit */ -+ this_cpu_write(lrng_sched_array[lrng_data_idx2array(ptr)], -+ data & mask); -+} -+ -+/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ -+static void lrng_sched_array_add_slot(u32 data) -+{ -+ /* Get slot */ -+ u32 ptr = this_cpu_inc_return(lrng_sched_array_ptr) & -+ LRNG_DATA_WORD_MASK; -+ unsigned int array = lrng_data_idx2array(ptr); -+ unsigned int slot = lrng_data_idx2slot(ptr); -+ -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_sched_array[array], -+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, -+ slot))); -+ /* Store data into slot */ -+ this_cpu_or(lrng_sched_array[array], lrng_data_slot_val(data, slot)); -+ -+ /* -+ * Continuous compression is not allowed for scheduler noise source, -+ * so do not call lrng_sched_array_to_hash here. -+ */ -+} -+ -+static void -+lrng_time_process_common(u32 time, void(*add_time)(u32 data)) -+{ -+ enum lrng_health_res health_test; -+ -+ if (lrng_raw_sched_hires_entropy_store(time)) -+ return; -+ -+ health_test = lrng_health_test(time, lrng_int_es_sched); -+ if (health_test > lrng_health_fail_use) -+ return; -+ -+ if (health_test == lrng_health_pass) -+ atomic_inc_return(this_cpu_ptr(&lrng_sched_array_events)); -+ -+ add_time(time); -+ -+ /* -+ * We cannot call lrng_es_add_entropy() as this would call a schedule -+ * operation that is not permissible in scheduler context. -+ * As the scheduler ES provides a high bandwidth of entropy, we assume -+ * that other reseed triggers happen to pick up the scheduler ES -+ * entropy in due time. -+ */ -+} -+ -+/* Batching up of entropy in per-CPU array */ -+static void lrng_sched_time_process(void) -+{ -+ u32 now_time = random_get_entropy(); -+ -+ if (unlikely(!lrng_gcd_tested())) { -+ /* When GCD is unknown, we process the full time stamp */ -+ lrng_time_process_common(now_time, lrng_sched_array_add_u32); -+ lrng_gcd_add_value(now_time); -+ } else { -+ /* GCD is known and applied */ -+ lrng_time_process_common((now_time / lrng_gcd_get()) & -+ LRNG_DATA_SLOTSIZE_MASK, -+ lrng_sched_array_add_slot); -+ } -+ -+ lrng_sched_perf_time(now_time); -+} -+ -+void add_sched_randomness(const struct task_struct *p, int cpu) -+{ -+ if (lrng_highres_timer()) { -+ lrng_sched_time_process(); -+ } else { -+ u32 tmp = cpu; -+ -+ tmp ^= lrng_raw_sched_pid_entropy_store(p->pid) ? -+ 0 : (u32)p->pid; -+ tmp ^= lrng_raw_sched_starttime_entropy_store(p->start_time) ? -+ 0 : (u32)p->start_time; -+ tmp ^= lrng_raw_sched_nvcsw_entropy_store(p->nvcsw) ? -+ 0 : (u32)p->nvcsw; -+ -+ lrng_sched_time_process(); -+ lrng_sched_array_add_u32(tmp); -+ } -+} -+EXPORT_SYMBOL(add_sched_randomness); -+ -+static void lrng_sched_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for operating entropy pool: %s\n" -+ " Available entropy: %u\n" -+ " per-CPU scheduler event collection size: %u\n" -+ " Standards compliance: %s\n" -+ " High-resolution timer: %s\n", -+ lrng_drng_init->hash_cb->hash_name(), -+ lrng_sched_avail_entropy(0), -+ LRNG_DATA_NUM_VALUES, -+ lrng_sp80090b_compliant(lrng_int_es_sched) ? "SP800-90B " : "", -+ lrng_highres_timer() ? "true" : "false"); -+} -+ -+struct lrng_es_cb lrng_es_sched = { -+ .name = "Scheduler", -+ .get_ent = lrng_sched_pool_hash, -+ .curr_entropy = lrng_sched_avail_entropy, -+ .max_entropy = lrng_sched_avail_pool_size, -+ .state = lrng_sched_es_state, -+ .reset = lrng_sched_reset, -+ .switch_hash = lrng_sched_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_sched.h b/drivers/char/lrng/lrng_es_sched.h -new file mode 100644 -index 000000000000..f1e596dd89d9 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_sched.h -@@ -0,0 +1,20 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_SCHED_H -+#define _LRNG_ES_SCHED_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_SCHED -+void lrng_sched_es_init(bool highres_timer); -+ -+extern struct lrng_es_cb lrng_es_sched; -+ -+#else /* CONFIG_LRNG_SCHED */ -+static inline void lrng_sched_es_init(bool highres_timer) { } -+#endif /* CONFIG_LRNG_SCHED */ -+ -+#endif /* _LRNG_ES_SCHED_H */ -diff --git a/drivers/char/lrng/lrng_es_timer_common.c b/drivers/char/lrng/lrng_es_timer_common.c -new file mode 100644 -index 000000000000..70f3ff074fe6 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_timer_common.c -@@ -0,0 +1,144 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Interrupt data collection -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_es_irq.h" -+#include "lrng_es_sched.h" -+#include "lrng_es_timer_common.h" -+#include "lrng_health.h" -+ -+/* Is high-resolution timer present? */ -+static bool lrng_highres_timer_val = false; -+ -+/* Number of time stamps analyzed to calculate a GCD */ -+#define LRNG_GCD_WINDOW_SIZE 100 -+static u32 lrng_gcd_history[LRNG_GCD_WINDOW_SIZE]; -+static atomic_t lrng_gcd_history_ptr = ATOMIC_INIT(-1); -+ -+/* The common divisor for all timestamps */ -+static u32 lrng_gcd_timer = 0; -+ -+bool lrng_gcd_tested(void) -+{ -+ return (lrng_gcd_timer != 0); -+} -+ -+u32 lrng_gcd_get(void) -+{ -+ return lrng_gcd_timer; -+} -+ -+/* Set the GCD for use in IRQ ES - if 0, the GCD calculation is restarted. */ -+void lrng_gcd_set(u32 running_gcd) -+{ -+ lrng_gcd_timer = running_gcd; -+ /* Ensure that update to global variable lrng_gcd_timer is visible */ -+ mb(); -+} -+ -+static void lrng_gcd_set_check(u32 running_gcd) -+{ -+ if (!lrng_gcd_tested()) { -+ lrng_gcd_set(running_gcd); -+ pr_debug("Setting GCD to %u\n", running_gcd); -+ } -+} -+ -+u32 lrng_gcd_analyze(u32 *history, size_t nelem) -+{ -+ u32 running_gcd = 0; -+ size_t i; -+ -+ /* Now perform the analysis on the accumulated time data. */ -+ for (i = 0; i < nelem; i++) { -+ /* -+ * NOTE: this would be the place to add more analysis on the -+ * appropriateness of the timer like checking the presence -+ * of sufficient variations in the timer. -+ */ -+ -+ /* -+ * This calculates the gcd of all the time values. that is -+ * gcd(time_1, time_2, ..., time_nelem) -+ * -+ * Some timers increment by a fixed (non-1) amount each step. -+ * This code checks for such increments, and allows the library -+ * to output the number of such changes have occurred. -+ */ -+ running_gcd = (u32)gcd(history[i], running_gcd); -+ -+ /* Zeroize data */ -+ history[i] = 0; -+ } -+ -+ return running_gcd; -+} -+ -+void lrng_gcd_add_value(u32 time) -+{ -+ u32 ptr = (u32)atomic_inc_return_relaxed(&lrng_gcd_history_ptr); -+ -+ if (ptr < LRNG_GCD_WINDOW_SIZE) { -+ lrng_gcd_history[ptr] = time; -+ } else if (ptr == LRNG_GCD_WINDOW_SIZE) { -+ u32 gcd = lrng_gcd_analyze(lrng_gcd_history, -+ LRNG_GCD_WINDOW_SIZE); -+ -+ if (!gcd) -+ gcd = 1; -+ -+ /* -+ * Ensure that we have variations in the time stamp below the -+ * given value. This is just a safety measure to prevent the GCD -+ * becoming too large. -+ */ -+ if (gcd >= 1000) { -+ pr_warn("calculated GCD is larger than expected: %u\n", -+ gcd); -+ gcd = 1000; -+ } -+ -+ /* Adjust all deltas by the observed (small) common factor. */ -+ lrng_gcd_set_check(gcd); -+ atomic_set(&lrng_gcd_history_ptr, 0); -+ } -+} -+ -+/* Return boolean whether LRNG identified presence of high-resolution timer */ -+bool lrng_highres_timer(void) -+{ -+ return lrng_highres_timer_val; -+} -+ -+static int __init lrng_init_time_source(void) -+{ -+ if ((random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK) || -+ (random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK)) { -+ /* -+ * As the highres timer is identified here, previous interrupts -+ * obtained during boot time are treated like a lowres-timer -+ * would have been present. -+ */ -+ lrng_highres_timer_val = true; -+ } else { -+ lrng_health_disable(); -+ lrng_highres_timer_val = false; -+ } -+ -+ lrng_irq_es_init(lrng_highres_timer_val); -+ lrng_sched_es_init(lrng_highres_timer_val); -+ -+ /* Ensure that changes to global variables are visible */ -+ mb(); -+ -+ return 0; -+} -+core_initcall(lrng_init_time_source); -diff --git a/drivers/char/lrng/lrng_es_timer_common.h b/drivers/char/lrng/lrng_es_timer_common.h -new file mode 100644 -index 000000000000..b45b9f683dea ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_timer_common.h -@@ -0,0 +1,83 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG Slow Noise Source: Time stamp array handling -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_TIMER_COMMON_H -+#define _LRNG_ES_TIMER_COMMON_H -+ -+bool lrng_gcd_tested(void); -+void lrng_gcd_set(u32 running_gcd); -+u32 lrng_gcd_get(void); -+u32 lrng_gcd_analyze(u32 *history, size_t nelem); -+void lrng_gcd_add_value(u32 time); -+bool lrng_highres_timer(void); -+ -+/* -+ * To limit the impact on the interrupt handling, the LRNG concatenates -+ * entropic LSB parts of the time stamps in a per-CPU array and only -+ * injects them into the entropy pool when the array is full. -+ */ -+ -+/* Store multiple integers in one u32 */ -+#define LRNG_DATA_SLOTSIZE_BITS (8) -+#define LRNG_DATA_SLOTSIZE_MASK ((1 << LRNG_DATA_SLOTSIZE_BITS) - 1) -+#define LRNG_DATA_ARRAY_MEMBER_BITS (4 << 3) /* ((sizeof(u32)) << 3) */ -+#define LRNG_DATA_SLOTS_PER_UINT (LRNG_DATA_ARRAY_MEMBER_BITS / \ -+ LRNG_DATA_SLOTSIZE_BITS) -+ -+/* -+ * Number of time values to store in the array - in small environments -+ * only one atomic_t variable per CPU is used. -+ */ -+#define LRNG_DATA_NUM_VALUES (CONFIG_LRNG_COLLECTION_SIZE) -+/* Mask of LSB of time stamp to store */ -+#define LRNG_DATA_WORD_MASK (LRNG_DATA_NUM_VALUES - 1) -+ -+#define LRNG_DATA_SLOTS_MASK (LRNG_DATA_SLOTS_PER_UINT - 1) -+#define LRNG_DATA_ARRAY_SIZE (LRNG_DATA_NUM_VALUES / \ -+ LRNG_DATA_SLOTS_PER_UINT) -+ -+/* Starting bit index of slot */ -+static inline unsigned int lrng_data_slot2bitindex(unsigned int slot) -+{ -+ return (LRNG_DATA_SLOTSIZE_BITS * slot); -+} -+ -+/* Convert index into the array index */ -+static inline unsigned int lrng_data_idx2array(unsigned int idx) -+{ -+ return idx / LRNG_DATA_SLOTS_PER_UINT; -+} -+ -+/* Convert index into the slot of a given array index */ -+static inline unsigned int lrng_data_idx2slot(unsigned int idx) -+{ -+ return idx & LRNG_DATA_SLOTS_MASK; -+} -+ -+/* Convert value into slot value */ -+static inline unsigned int lrng_data_slot_val(unsigned int val, -+ unsigned int slot) -+{ -+ return val << lrng_data_slot2bitindex(slot); -+} -+ -+/* -+ * Return the pointers for the previous and current units to inject a u32 into. -+ * Also return the mask which the u32 word is to be processed. -+ */ -+static inline void lrng_data_split_u32(u32 *ptr, u32 *pre_ptr, u32 *mask) -+{ -+ /* ptr to previous unit */ -+ *pre_ptr = (*ptr - LRNG_DATA_SLOTS_PER_UINT) & LRNG_DATA_WORD_MASK; -+ *ptr &= LRNG_DATA_WORD_MASK; -+ -+ /* mask to split data into the two parts for the two units */ -+ *mask = ((1 << (*pre_ptr & (LRNG_DATA_SLOTS_PER_UINT - 1)) * -+ LRNG_DATA_SLOTSIZE_BITS)) - 1; -+} -+ -+#endif /* _LRNG_ES_TIMER_COMMON_H */ -\ No newline at end of file -diff --git a/drivers/char/lrng/lrng_hash_kcapi.c b/drivers/char/lrng/lrng_hash_kcapi.c -new file mode 100644 -index 000000000000..13e62db9b6c8 ---- /dev/null -+++ b/drivers/char/lrng/lrng_hash_kcapi.c -@@ -0,0 +1,140 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for providing the hash primitive using the kernel crypto API. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+ -+static char *lrng_hash_name = "sha512"; -+ -+/* The parameter must be r/o in sysfs as otherwise races appear. */ -+module_param(lrng_hash_name, charp, 0444); -+MODULE_PARM_DESC(lrng_hash_name, "Kernel crypto API hash name"); -+ -+struct lrng_hash_info { -+ struct crypto_shash *tfm; -+}; -+ -+static const char *lrng_kcapi_hash_name(void) -+{ -+ return lrng_hash_name; -+} -+ -+static void _lrng_kcapi_hash_free(struct lrng_hash_info *lrng_hash) -+{ -+ struct crypto_shash *tfm = lrng_hash->tfm; -+ -+ crypto_free_shash(tfm); -+ kfree(lrng_hash); -+} -+ -+static void *lrng_kcapi_hash_alloc(const char *name) -+{ -+ struct lrng_hash_info *lrng_hash; -+ struct crypto_shash *tfm; -+ int ret; -+ -+ if (!name) { -+ pr_err("Hash name missing\n"); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ tfm = crypto_alloc_shash(name, 0, 0); -+ if (IS_ERR(tfm)) { -+ pr_err("could not allocate hash %s\n", name); -+ return ERR_CAST(tfm); -+ } -+ -+ ret = sizeof(struct lrng_hash_info); -+ lrng_hash = kmalloc(ret, GFP_KERNEL); -+ if (!lrng_hash) { -+ crypto_free_shash(tfm); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ lrng_hash->tfm = tfm; -+ -+ pr_info("Hash %s allocated\n", name); -+ -+ return lrng_hash; -+} -+ -+static void *lrng_kcapi_hash_name_alloc(void) -+{ -+ return lrng_kcapi_hash_alloc(lrng_kcapi_hash_name()); -+} -+ -+static u32 lrng_kcapi_hash_digestsize(void *hash) -+{ -+ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; -+ struct crypto_shash *tfm = lrng_hash->tfm; -+ -+ return crypto_shash_digestsize(tfm); -+} -+ -+static void lrng_kcapi_hash_dealloc(void *hash) -+{ -+ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; -+ -+ _lrng_kcapi_hash_free(lrng_hash); -+ pr_info("Hash deallocated\n"); -+} -+ -+static int lrng_kcapi_hash_init(struct shash_desc *shash, void *hash) -+{ -+ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; -+ struct crypto_shash *tfm = lrng_hash->tfm; -+ -+ shash->tfm = tfm; -+ return crypto_shash_init(shash); -+} -+ -+static int lrng_kcapi_hash_update(struct shash_desc *shash, const u8 *inbuf, -+ u32 inbuflen) -+{ -+ return crypto_shash_update(shash, inbuf, inbuflen); -+} -+ -+static int lrng_kcapi_hash_final(struct shash_desc *shash, u8 *digest) -+{ -+ return crypto_shash_final(shash, digest); -+} -+ -+static void lrng_kcapi_hash_zero(struct shash_desc *shash) -+{ -+ shash_desc_zero(shash); -+} -+ -+static const struct lrng_hash_cb lrng_kcapi_hash_cb = { -+ .hash_name = lrng_kcapi_hash_name, -+ .hash_alloc = lrng_kcapi_hash_name_alloc, -+ .hash_dealloc = lrng_kcapi_hash_dealloc, -+ .hash_digestsize = lrng_kcapi_hash_digestsize, -+ .hash_init = lrng_kcapi_hash_init, -+ .hash_update = lrng_kcapi_hash_update, -+ .hash_final = lrng_kcapi_hash_final, -+ .hash_desc_zero = lrng_kcapi_hash_zero, -+}; -+ -+static int __init lrng_kcapi_init(void) -+{ -+ return lrng_set_hash_cb(&lrng_kcapi_hash_cb); -+} -+ -+static void __exit lrng_kcapi_exit(void) -+{ -+ lrng_set_hash_cb(NULL); -+} -+ -+late_initcall(lrng_kcapi_init); -+module_exit(lrng_kcapi_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - Kernel crypto API hash backend"); -diff --git a/drivers/char/lrng/lrng_health.c b/drivers/char/lrng/lrng_health.c -new file mode 100644 -index 000000000000..a60c8378eea5 ---- /dev/null -+++ b/drivers/char/lrng/lrng_health.c -@@ -0,0 +1,420 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Entropy Source and DRNG Manager (LRNG) Health Testing -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_es_mgr.h" -+#include "lrng_health.h" -+ -+/* Stuck Test */ -+struct lrng_stuck_test { -+ u32 last_time; /* Stuck test: time of previous IRQ */ -+ u32 last_delta; /* Stuck test: delta of previous IRQ */ -+ u32 last_delta2; /* Stuck test: 2. time derivation of prev IRQ */ -+}; -+ -+/* Repetition Count Test */ -+struct lrng_rct { -+ atomic_t rct_count; /* Number of stuck values */ -+}; -+ -+/* Adaptive Proportion Test */ -+struct lrng_apt { -+ /* Data window size */ -+#define LRNG_APT_WINDOW_SIZE 512 -+ /* LSB of time stamp to process */ -+#define LRNG_APT_LSB 16 -+#define LRNG_APT_WORD_MASK (LRNG_APT_LSB - 1) -+ atomic_t apt_count; /* APT counter */ -+ atomic_t apt_base; /* APT base reference */ -+ -+ atomic_t apt_trigger; -+ bool apt_base_set; /* Is APT base set? */ -+}; -+ -+/* Health data collected for one entropy source */ -+struct lrng_health_es_state { -+ struct lrng_rct rct; -+ struct lrng_apt apt; -+ -+ /* SP800-90B startup health tests */ -+#define LRNG_SP80090B_STARTUP_SAMPLES 1024 -+#define LRNG_SP80090B_STARTUP_BLOCKS ((LRNG_SP80090B_STARTUP_SAMPLES + \ -+ LRNG_APT_WINDOW_SIZE - 1) / \ -+ LRNG_APT_WINDOW_SIZE) -+ bool sp80090b_startup_done; -+ atomic_t sp80090b_startup_blocks; -+}; -+ -+#define LRNG_HEALTH_ES_INIT(x) \ -+ x.rct.rct_count = ATOMIC_INIT(0), \ -+ x.apt.apt_count = ATOMIC_INIT(0), \ -+ x.apt.apt_base = ATOMIC_INIT(-1), \ -+ x.apt.apt_trigger = ATOMIC_INIT(LRNG_APT_WINDOW_SIZE), \ -+ x.apt.apt_base_set = false, \ -+ x.sp80090b_startup_blocks = ATOMIC_INIT(LRNG_SP80090B_STARTUP_BLOCKS), \ -+ x.sp80090b_startup_done = false, -+ -+/* The health test code must operate lock-less */ -+struct lrng_health { -+ bool health_test_enabled; -+ struct lrng_health_es_state es_state[lrng_int_es_last]; -+}; -+ -+static struct lrng_health lrng_health = { -+ .health_test_enabled = true, -+ -+#ifdef CONFIG_LRNG_IRQ -+ LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_irq]) -+#endif -+#ifdef CONFIG_LRNG_SCHED -+ LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_sched]) -+#endif -+}; -+ -+static DEFINE_PER_CPU(struct lrng_stuck_test[lrng_int_es_last], -+ lrng_stuck_test_array); -+ -+static bool lrng_sp80090b_health_requested(void) -+{ -+ /* Health tests are only requested in FIPS mode */ -+ return fips_enabled; -+} -+ -+static bool lrng_sp80090b_health_enabled(void) -+{ -+ struct lrng_health *health = &lrng_health; -+ -+ return lrng_sp80090b_health_requested() && health->health_test_enabled; -+} -+ -+/*************************************************************************** -+ * SP800-90B Compliance -+ * -+ * If the Linux-RNG is booted into FIPS mode, the following interfaces -+ * provide an SP800-90B compliant noise source: -+ * -+ * * /dev/random -+ * * getrandom(2) -+ * * get_random_bytes_full -+ * -+ * All other interfaces, including /dev/urandom or get_random_bytes without -+ * the add_random_ready_callback cannot claim to use an SP800-90B compliant -+ * noise source. -+ ***************************************************************************/ -+ -+/* -+ * Perform SP800-90B startup testing -+ */ -+static void lrng_sp80090b_startup(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ if (!es_state->sp80090b_startup_done && -+ atomic_dec_and_test(&es_state->sp80090b_startup_blocks)) { -+ es_state->sp80090b_startup_done = true; -+ pr_info("SP800-90B startup health tests for internal entropy source %u completed\n", -+ es); -+ lrng_drng_force_reseed(); -+ -+ /* -+ * We cannot call lrng_es_add_entropy() as this may cause a -+ * schedule operation while in scheduler context for the -+ * scheduler ES. -+ */ -+ } -+} -+ -+/* -+ * Handle failure of SP800-90B startup testing -+ */ -+static void lrng_sp80090b_startup_failure(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ /* Reset of LRNG and its entropy - NOTE: we are in atomic context */ -+ lrng_reset(); -+ -+ /* -+ * Reset the SP800-90B startup test. -+ * -+ * NOTE SP800-90B section 4.3 bullet 4 does not specify what -+ * exactly is to be done in case of failure! Thus, we do what -+ * makes sense, i.e. restarting the health test and thus gating -+ * the output function of /dev/random and getrandom(2). -+ */ -+ atomic_set(&es_state->sp80090b_startup_blocks, -+ LRNG_SP80090B_STARTUP_BLOCKS); -+} -+ -+/* -+ * Handle failure of SP800-90B runtime testing -+ */ -+static void lrng_sp80090b_runtime_failure(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ lrng_sp80090b_startup_failure(health, es); -+ es_state->sp80090b_startup_done = false; -+} -+ -+static void lrng_sp80090b_failure(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ if (es_state->sp80090b_startup_done) { -+ pr_err("SP800-90B runtime health test failure for internal entropy source %u - invalidating all existing entropy and initiate SP800-90B startup\n", es); -+ lrng_sp80090b_runtime_failure(health, es); -+ } else { -+ pr_err("SP800-90B startup test failure for internal entropy source %u - resetting\n", es); -+ lrng_sp80090b_startup_failure(health, es); -+ } -+} -+ -+bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) -+{ -+ struct lrng_health *health = &lrng_health; -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ if (!lrng_sp80090b_health_enabled()) -+ return true; -+ -+ return es_state->sp80090b_startup_done; -+} -+ -+bool lrng_sp80090b_compliant(enum lrng_internal_es es) -+{ -+ struct lrng_health *health = &lrng_health; -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ return lrng_sp80090b_health_enabled() && -+ es_state->sp80090b_startup_done; -+} -+ -+/*************************************************************************** -+ * Adaptive Proportion Test -+ * -+ * This test complies with SP800-90B section 4.4.2. -+ ***************************************************************************/ -+ -+/* -+ * Reset the APT counter -+ * -+ * @health [in] Reference to health state -+ */ -+static void lrng_apt_reset(struct lrng_apt *apt, unsigned int time_masked) -+{ -+ /* Reset APT */ -+ atomic_set(&apt->apt_count, 0); -+ atomic_set(&apt->apt_base, time_masked); -+} -+ -+static void lrng_apt_restart(struct lrng_apt *apt) -+{ -+ atomic_set(&apt->apt_trigger, LRNG_APT_WINDOW_SIZE); -+} -+ -+/* -+ * Insert a new entropy event into APT -+ * -+ * This function does is void as it does not decide about the fate of a time -+ * stamp. An APT failure can only happen at the same time of a stuck test -+ * failure. Thus, the stuck failure will already decide how the time stamp -+ * is handled. -+ * -+ * @health [in] Reference to health state -+ * @now_time [in] Time stamp to process -+ */ -+static void lrng_apt_insert(struct lrng_health *health, -+ unsigned int now_time, enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ struct lrng_apt *apt = &es_state->apt; -+ -+ if (!lrng_sp80090b_health_requested()) -+ return; -+ -+ now_time &= LRNG_APT_WORD_MASK; -+ -+ /* Initialization of APT */ -+ if (!apt->apt_base_set) { -+ atomic_set(&apt->apt_base, now_time); -+ apt->apt_base_set = true; -+ return; -+ } -+ -+ if (now_time == (unsigned int)atomic_read(&apt->apt_base)) { -+ u32 apt_val = (u32)atomic_inc_return_relaxed(&apt->apt_count); -+ -+ if (apt_val >= CONFIG_LRNG_APT_CUTOFF) -+ lrng_sp80090b_failure(health, es); -+ } -+ -+ if (atomic_dec_and_test(&apt->apt_trigger)) { -+ lrng_apt_restart(apt); -+ lrng_apt_reset(apt, now_time); -+ lrng_sp80090b_startup(health, es); -+ } -+} -+ -+/*************************************************************************** -+ * Repetition Count Test -+ * -+ * The LRNG uses an enhanced version of the Repetition Count Test -+ * (RCT) specified in SP800-90B section 4.4.1. Instead of counting identical -+ * back-to-back values, the input to the RCT is the counting of the stuck -+ * values while filling the entropy pool. -+ * -+ * The RCT is applied with an alpha of 2^-30 compliant to FIPS 140-2 IG 9.8. -+ * -+ * During the counting operation, the LRNG always calculates the RCT -+ * cut-off value of C. If that value exceeds the allowed cut-off value, -+ * the LRNG will invalidate all entropy for the entropy pool which implies -+ * that no data can be extracted from the entropy pool unless new entropy -+ * is received. -+ ***************************************************************************/ -+ -+/* -+ * Hot code path - Insert data for Repetition Count Test -+ * -+ * @health: Reference to health information -+ * @stuck: Decision of stuck test -+ */ -+static void lrng_rct(struct lrng_health *health, enum lrng_internal_es es, -+ int stuck) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ struct lrng_rct *rct = &es_state->rct; -+ -+ if (!lrng_sp80090b_health_requested()) -+ return; -+ -+ if (stuck) { -+ u32 rct_count = atomic_add_return_relaxed(1, &rct->rct_count); -+ -+ /* -+ * The cutoff value is based on the following consideration: -+ * alpha = 2^-30 as recommended in FIPS 140-2 IG 9.8. -+ * In addition, we imply an entropy value H of 1 bit as this -+ * is the minimum entropy required to provide full entropy. -+ * -+ * Note, rct_count (which equals to value B in the -+ * pseudo code of SP800-90B section 4.4.1) starts with zero. -+ * Hence we need to subtract one from the cutoff value as -+ * calculated following SP800-90B. -+ */ -+ if (rct_count >= CONFIG_LRNG_RCT_CUTOFF) { -+ atomic_set(&rct->rct_count, 0); -+ -+ /* -+ * APT must start anew as we consider all previously -+ * recorded data to contain no entropy. -+ */ -+ lrng_apt_restart(&es_state->apt); -+ -+ lrng_sp80090b_failure(health, es); -+ } -+ } else { -+ atomic_set(&rct->rct_count, 0); -+ } -+} -+ -+/*************************************************************************** -+ * Stuck Test -+ * -+ * Checking the: -+ * 1st derivative of the event occurrence (time delta) -+ * 2nd derivative of the event occurrence (delta of time deltas) -+ * 3rd derivative of the event occurrence (delta of delta of time deltas) -+ * -+ * All values must always be non-zero. The stuck test is only valid disabled if -+ * high-resolution time stamps are identified after initialization. -+ ***************************************************************************/ -+ -+static u32 lrng_delta(u32 prev, u32 next) -+{ -+ /* -+ * Note that this (unsigned) subtraction does yield the correct value -+ * in the wraparound-case, i.e. when next < prev. -+ */ -+ return (next - prev); -+} -+ -+/* -+ * Hot code path -+ * -+ * @health: Reference to health information -+ * @now: Event time -+ * @return: 0 event occurrence not stuck (good time stamp) -+ * != 0 event occurrence stuck (reject time stamp) -+ */ -+static int lrng_irq_stuck(enum lrng_internal_es es, u32 now_time) -+{ -+ struct lrng_stuck_test *stuck = this_cpu_ptr(lrng_stuck_test_array); -+ u32 delta = lrng_delta(stuck[es].last_time, now_time); -+ u32 delta2 = lrng_delta(stuck[es].last_delta, delta); -+ u32 delta3 = lrng_delta(stuck[es].last_delta2, delta2); -+ -+ stuck[es].last_time = now_time; -+ stuck[es].last_delta = delta; -+ stuck[es].last_delta2 = delta2; -+ -+ if (!delta || !delta2 || !delta3) -+ return 1; -+ -+ return 0; -+} -+ -+/*************************************************************************** -+ * Health test interfaces -+ ***************************************************************************/ -+ -+/* -+ * Disable all health tests -+ */ -+void lrng_health_disable(void) -+{ -+ struct lrng_health *health = &lrng_health; -+ -+ health->health_test_enabled = false; -+ -+ if (lrng_sp80090b_health_requested()) -+ pr_warn("SP800-90B compliance requested but the Linux RNG is NOT SP800-90B compliant\n"); -+} -+ -+/* -+ * Hot code path - Perform health test on time stamp received from an event -+ * -+ * @now_time Time stamp -+ */ -+enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es) -+{ -+ struct lrng_health *health = &lrng_health; -+ int stuck; -+ -+ if (!health->health_test_enabled) -+ return lrng_health_pass; -+ -+ lrng_apt_insert(health, now_time, es); -+ -+ stuck = lrng_irq_stuck(es, now_time); -+ lrng_rct(health, es, stuck); -+ if (stuck) { -+ /* SP800-90B disallows using a failing health test time stamp */ -+ return lrng_sp80090b_health_requested() ? -+ lrng_health_fail_drop : lrng_health_fail_use; -+ } -+ -+ return lrng_health_pass; -+} -diff --git a/drivers/char/lrng/lrng_health.h b/drivers/char/lrng/lrng_health.h -new file mode 100644 -index 000000000000..4f9f5033fc30 ---- /dev/null -+++ b/drivers/char/lrng/lrng_health.h -@@ -0,0 +1,42 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_HEALTH_H -+#define _LRNG_HEALTH_H -+ -+#include "lrng_es_mgr.h" -+ -+enum lrng_health_res { -+ lrng_health_pass, /* Health test passes on time stamp */ -+ lrng_health_fail_use, /* Time stamp unhealthy, but mix in */ -+ lrng_health_fail_drop /* Time stamp unhealthy, drop it */ -+}; -+ -+#ifdef CONFIG_LRNG_HEALTH_TESTS -+bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es); -+bool lrng_sp80090b_compliant(enum lrng_internal_es es); -+ -+enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es); -+void lrng_health_disable(void); -+#else /* CONFIG_LRNG_HEALTH_TESTS */ -+static inline bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) -+{ -+ return true; -+} -+ -+static inline bool lrng_sp80090b_compliant(enum lrng_internal_es es) -+{ -+ return false; -+} -+ -+static inline enum lrng_health_res -+lrng_health_test(u32 now_time, enum lrng_internal_es es) -+{ -+ return lrng_health_pass; -+} -+static inline void lrng_health_disable(void) { } -+#endif /* CONFIG_LRNG_HEALTH_TESTS */ -+ -+#endif /* _LRNG_HEALTH_H */ -diff --git a/drivers/char/lrng/lrng_interface_aux.c b/drivers/char/lrng/lrng_interface_aux.c -new file mode 100644 -index 000000000000..0eb49bd6b48f ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_aux.c -@@ -0,0 +1,126 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG auxiliary interfaces -+ * -+ * Copyright (C) 2022 Stephan Mueller -+ * Copyright (C) 2017 Jason A. Donenfeld . All -+ * Rights Reserved. -+ * Copyright (C) 2016 Jason Cooper -+ */ -+ -+#include -+#include -+#include -+ -+#include "lrng_es_mgr.h" -+#include "lrng_interface_random_kernel.h" -+ -+struct batched_entropy { -+ union { -+ u64 entropy_u64[LRNG_DRNG_BLOCKSIZE / sizeof(u64)]; -+ u32 entropy_u32[LRNG_DRNG_BLOCKSIZE / sizeof(u32)]; -+ }; -+ unsigned int position; -+ spinlock_t batch_lock; -+}; -+ -+/* -+ * Get a random word for internal kernel use only. The quality of the random -+ * number is as good as /dev/urandom, but there is no backtrack protection, -+ * with the goal of being quite fast and not depleting entropy. -+ */ -+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { -+ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), -+}; -+ -+u64 get_random_u64(void) -+{ -+ u64 ret; -+ unsigned long flags; -+ struct batched_entropy *batch; -+ -+ lrng_debug_report_seedlevel("get_random_u64"); -+ -+ batch = raw_cpu_ptr(&batched_entropy_u64); -+ spin_lock_irqsave(&batch->batch_lock, flags); -+ if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { -+ lrng_get_random_bytes(batch->entropy_u64, LRNG_DRNG_BLOCKSIZE); -+ batch->position = 0; -+ } -+ ret = batch->entropy_u64[batch->position++]; -+ spin_unlock_irqrestore(&batch->batch_lock, flags); -+ return ret; -+} -+EXPORT_SYMBOL(get_random_u64); -+ -+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { -+ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), -+}; -+ -+u32 get_random_u32(void) -+{ -+ u32 ret; -+ unsigned long flags; -+ struct batched_entropy *batch; -+ -+ lrng_debug_report_seedlevel("get_random_u32"); -+ -+ batch = raw_cpu_ptr(&batched_entropy_u32); -+ spin_lock_irqsave(&batch->batch_lock, flags); -+ if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { -+ lrng_get_random_bytes(batch->entropy_u32, LRNG_DRNG_BLOCKSIZE); -+ batch->position = 0; -+ } -+ ret = batch->entropy_u32[batch->position++]; -+ spin_unlock_irqrestore(&batch->batch_lock, flags); -+ return ret; -+} -+EXPORT_SYMBOL(get_random_u32); -+ -+#ifdef CONFIG_SMP -+/* -+ * This function is called when the CPU is coming up, with entry -+ * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP. -+ */ -+int random_prepare_cpu(unsigned int cpu) -+{ -+ /* -+ * When the cpu comes back online, immediately invalidate all batches, -+ * so that we serve fresh randomness. -+ */ -+ per_cpu_ptr(&batched_entropy_u32, cpu)->position = 0; -+ per_cpu_ptr(&batched_entropy_u64, cpu)->position = 0; -+ return 0; -+} -+ -+int random_online_cpu(unsigned int cpu) -+{ -+ return 0; -+} -+#endif -+ -+/* -+ * It's important to invalidate all potential batched entropy that might -+ * be stored before the crng is initialized, which we can do lazily by -+ * simply resetting the counter to zero so that it's re-extracted on the -+ * next usage. -+ */ -+void invalidate_batched_entropy(void) -+{ -+ int cpu; -+ unsigned long flags; -+ -+ for_each_possible_cpu(cpu) { -+ struct batched_entropy *batched_entropy; -+ -+ batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); -+ spin_lock_irqsave(&batched_entropy->batch_lock, flags); -+ batched_entropy->position = 0; -+ spin_unlock(&batched_entropy->batch_lock); -+ -+ batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); -+ spin_lock(&batched_entropy->batch_lock); -+ batched_entropy->position = 0; -+ spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); -+ } -+} -diff --git a/drivers/char/lrng/lrng_interface_dev.c b/drivers/char/lrng/lrng_interface_dev.c -new file mode 100644 -index 000000000000..e60060d402b3 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_dev.c -@@ -0,0 +1,35 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG user space device file interface -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+ -+#include "lrng_interface_dev_common.h" -+ -+static const struct file_operations lrng_fops = { -+ .read = lrng_drng_read_block, -+ .write = lrng_drng_write, -+ .poll = lrng_random_poll, -+ .unlocked_ioctl = lrng_ioctl, -+ .compat_ioctl = compat_ptr_ioctl, -+ .fasync = lrng_fasync, -+ .llseek = noop_llseek, -+}; -+ -+static struct miscdevice lrng_miscdev = { -+ .minor = MISC_DYNAMIC_MINOR, -+ .name = "lrng", -+ .nodename = "lrng", -+ .fops = &lrng_fops, -+ .mode = 0666 -+}; -+ -+static int __init lrng_dev_if_mod_init(void) -+{ -+ return misc_register(&lrng_miscdev); -+} -+device_initcall(lrng_dev_if_mod_init); -diff --git a/drivers/char/lrng/lrng_interface_dev_common.c b/drivers/char/lrng/lrng_interface_dev_common.c -new file mode 100644 -index 000000000000..09d0563074a4 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_dev_common.c -@@ -0,0 +1,314 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG User and kernel space interfaces -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+ -+DECLARE_WAIT_QUEUE_HEAD(lrng_write_wait); -+static struct fasync_struct *fasync; -+ -+static bool lrng_seed_hw = true; /* Allow HW to provide seed */ -+static bool lrng_seed_user = true; /* Allow user space to provide seed */ -+ -+/********************************** Helper ***********************************/ -+ -+static u32 lrng_get_aux_ent(void) -+{ -+ return lrng_es[lrng_ext_es_aux]->curr_entropy(0); -+} -+ -+/* Is the DRNG seed level too low? */ -+bool lrng_need_entropy(void) -+{ -+ return (lrng_get_aux_ent() < lrng_write_wakeup_bits); -+} -+ -+void lrng_writer_wakeup(void) -+{ -+ if (lrng_need_entropy() && wq_has_sleeper(&lrng_write_wait)) { -+ wake_up_interruptible(&lrng_write_wait); -+ kill_fasync(&fasync, SIGIO, POLL_OUT); -+ } -+} -+ -+void lrng_init_wakeup_dev(void) -+{ -+ kill_fasync(&fasync, SIGIO, POLL_IN); -+} -+ -+/* External entropy provider is allowed to provide seed data */ -+bool lrng_state_exseed_allow(enum lrng_external_noise_source source) -+{ -+ if (source == lrng_noise_source_hw) -+ return lrng_seed_hw; -+ return lrng_seed_user; -+} -+ -+/* Enable / disable external entropy provider to furnish seed */ -+void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) -+{ -+ /* -+ * If the LRNG is not yet operational, allow all entropy sources -+ * to deliver data unconditionally to get fully seeded asap. -+ */ -+ if (!lrng_state_operational()) -+ return; -+ -+ if (source == lrng_noise_source_hw) -+ lrng_seed_hw = type; -+ else -+ lrng_seed_user = type; -+} -+ -+void lrng_state_exseed_allow_all(void) -+{ -+ lrng_state_exseed_set(lrng_noise_source_hw, true); -+ lrng_state_exseed_set(lrng_noise_source_user, true); -+} -+ -+/************************ LRNG user output interfaces *************************/ -+ -+ssize_t lrng_read_seed(char __user *buf, size_t nbytes, unsigned int flags) -+{ -+ ssize_t ret = 0; -+ u64 t[(sizeof(struct entropy_buf) + 3 * sizeof(u64) - 1) / sizeof(u64)]; -+ -+ memset(t, 0, sizeof(t)); -+ ret = lrng_get_seed(t, min_t(size_t, nbytes, sizeof(t)), flags); -+ if (ret == -EMSGSIZE && copy_to_user(buf, t, sizeof(u64))) { -+ ret = -EFAULT; -+ } else if (ret > 0 && copy_to_user(buf, t, ret)) { -+ ret = -EFAULT; -+ } -+ memzero_explicit(t, sizeof(t)); -+ -+ return ret; -+} -+ -+ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr) -+{ -+ ssize_t ret = 0; -+ u8 tmpbuf[LRNG_DRNG_BLOCKSIZE] __aligned(LRNG_KCAPI_ALIGN); -+ u8 *tmp_large = NULL, *tmp = tmpbuf; -+ u32 tmplen = sizeof(tmpbuf); -+ -+ if (nbytes == 0) -+ return 0; -+ -+ /* -+ * Satisfy large read requests -- as the common case are smaller -+ * request sizes, such as 16 or 32 bytes, avoid a kmalloc overhead for -+ * those by using the stack variable of tmpbuf. -+ */ -+ if (!CONFIG_BASE_SMALL && (nbytes > sizeof(tmpbuf))) { -+ tmplen = min_t(u32, nbytes, LRNG_DRNG_MAX_REQSIZE); -+ tmp_large = kmalloc(tmplen + LRNG_KCAPI_ALIGN, GFP_KERNEL); -+ if (!tmp_large) -+ tmplen = sizeof(tmpbuf); -+ else -+ tmp = PTR_ALIGN(tmp_large, LRNG_KCAPI_ALIGN); -+ } -+ -+ while (nbytes) { -+ u32 todo = min_t(u32, nbytes, tmplen); -+ int rc = 0; -+ -+ /* Reschedule if we received a large request. */ -+ if ((tmp_large) && need_resched()) { -+ if (signal_pending(current)) { -+ if (ret == 0) -+ ret = -ERESTARTSYS; -+ break; -+ } -+ schedule(); -+ } -+ -+ rc = lrng_drng_get_sleep(tmp, todo, pr); -+ if (rc <= 0) { -+ if (rc < 0) -+ ret = rc; -+ break; -+ } -+ if (copy_to_user(buf, tmp, rc)) { -+ ret = -EFAULT; -+ break; -+ } -+ -+ nbytes -= rc; -+ buf += rc; -+ ret += rc; -+ } -+ -+ /* Wipe data just returned from memory */ -+ if (tmp_large) -+ kfree_sensitive(tmp_large); -+ else -+ memzero_explicit(tmpbuf, sizeof(tmpbuf)); -+ -+ return ret; -+} -+ -+ssize_t lrng_read_common_block(int nonblock, int pr, -+ char __user *buf, size_t nbytes) -+{ -+ int ret; -+ -+ if (nbytes == 0) -+ return 0; -+ -+ ret = lrng_drng_sleep_while_nonoperational(nonblock); -+ if (ret) -+ return ret; -+ -+ return lrng_read_common(buf, nbytes, !!pr); -+} -+ -+ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, -+ loff_t *ppos) -+{ -+ return lrng_read_common_block(file->f_flags & O_NONBLOCK, -+ file->f_flags & O_SYNC, buf, nbytes); -+} -+ -+__poll_t lrng_random_poll(struct file *file, poll_table *wait) -+{ -+ __poll_t mask; -+ -+ poll_wait(file, &lrng_init_wait, wait); -+ poll_wait(file, &lrng_write_wait, wait); -+ mask = 0; -+ if (lrng_state_operational()) -+ mask |= EPOLLIN | EPOLLRDNORM; -+ if (lrng_need_entropy() || -+ lrng_state_exseed_allow(lrng_noise_source_user)) { -+ lrng_state_exseed_set(lrng_noise_source_user, false); -+ mask |= EPOLLOUT | EPOLLWRNORM; -+ } -+ return mask; -+} -+ -+ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, -+ u32 entropy_bits) -+{ -+ ssize_t ret = 0; -+ u8 buf[64] __aligned(LRNG_KCAPI_ALIGN); -+ const char __user *p = buffer; -+ u32 orig_entropy_bits = entropy_bits; -+ -+ if (!lrng_get_available()) { -+ ret = lrng_drng_initalize(); -+ if (!ret) -+ return ret; -+ } -+ -+ count = min_t(size_t, count, INT_MAX); -+ while (count > 0) { -+ size_t bytes = min_t(size_t, count, sizeof(buf)); -+ u32 ent = min_t(u32, bytes<<3, entropy_bits); -+ -+ if (copy_from_user(&buf, p, bytes)) -+ return -EFAULT; -+ /* Inject data into entropy pool */ -+ lrng_pool_insert_aux(buf, bytes, ent); -+ -+ count -= bytes; -+ p += bytes; -+ ret += bytes; -+ entropy_bits -= ent; -+ -+ cond_resched(); -+ } -+ -+ /* Force reseed of DRNG during next data request. */ -+ if (!orig_entropy_bits) -+ lrng_drng_force_reseed(); -+ -+ return ret; -+} -+ -+ssize_t lrng_drng_write(struct file *file, const char __user *buffer, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_drng_write_common(buffer, count, 0); -+} -+ -+long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg) -+{ -+ u32 digestsize_bits; -+ int size, ent_count_bits; -+ int __user *p = (int __user *)arg; -+ -+ switch (cmd) { -+ case RNDGETENTCNT: -+ ent_count_bits = lrng_avail_entropy_aux(); -+ if (put_user(ent_count_bits, p)) -+ return -EFAULT; -+ return 0; -+ case RNDADDTOENTCNT: -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ if (get_user(ent_count_bits, p)) -+ return -EFAULT; -+ ent_count_bits = (int)lrng_get_aux_ent() + ent_count_bits; -+ if (ent_count_bits < 0) -+ ent_count_bits = 0; -+ digestsize_bits = lrng_get_digestsize(); -+ if (ent_count_bits > digestsize_bits) -+ ent_count_bits = digestsize_bits; -+ lrng_pool_set_entropy(ent_count_bits); -+ return 0; -+ case RNDADDENTROPY: -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ if (get_user(ent_count_bits, p++)) -+ return -EFAULT; -+ if (ent_count_bits < 0) -+ return -EINVAL; -+ if (get_user(size, p++)) -+ return -EFAULT; -+ if (size < 0) -+ return -EINVAL; -+ /* there cannot be more entropy than data */ -+ ent_count_bits = min(ent_count_bits, size<<3); -+ return lrng_drng_write_common((const char __user *)p, size, -+ ent_count_bits); -+ case RNDZAPENTCNT: -+ case RNDCLEARPOOL: -+ /* Clear the entropy pool counter. */ -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ lrng_pool_set_entropy(0); -+ return 0; -+ case RNDRESEEDCRNG: -+ /* -+ * We leave the capability check here since it is present -+ * in the upstream's RNG implementation. Yet, user space -+ * can trigger a reseed as easy as writing into /dev/random -+ * or /dev/urandom where no privilege is needed. -+ */ -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ /* Force a reseed of all DRNGs */ -+ lrng_drng_force_reseed(); -+ return 0; -+ default: -+ return -EINVAL; -+ } -+} -+EXPORT_SYMBOL(lrng_ioctl); -+ -+int lrng_fasync(int fd, struct file *filp, int on) -+{ -+ return fasync_helper(fd, filp, on, &fasync); -+} -diff --git a/drivers/char/lrng/lrng_interface_dev_common.h b/drivers/char/lrng/lrng_interface_dev_common.h -new file mode 100644 -index 000000000000..9e6603ad8af4 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_dev_common.h -@@ -0,0 +1,51 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_INTERFACE_DEV_COMMON_H -+#define _LRNG_INTERFACE_DEV_COMMON_H -+ -+#include -+#include -+ -+/******************* Upstream functions hooked into the LRNG ******************/ -+enum lrng_external_noise_source { -+ lrng_noise_source_hw, -+ lrng_noise_source_user -+}; -+ -+#ifdef CONFIG_LRNG_COMMON_DEV_IF -+void lrng_writer_wakeup(void); -+void lrng_init_wakeup_dev(void); -+void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type); -+void lrng_state_exseed_allow_all(void); -+#else /* CONFIG_LRNG_COMMON_DEV_IF */ -+static inline void lrng_writer_wakeup(void) { } -+static inline void lrng_init_wakeup_dev(void) { } -+static inline void -+lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) { } -+static inline void lrng_state_exseed_allow_all(void) { } -+#endif /* CONFIG_LRNG_COMMON_DEV_IF */ -+ -+/****** Downstream service functions to actual interface implementations ******/ -+ -+bool lrng_state_exseed_allow(enum lrng_external_noise_source source); -+int lrng_fasync(int fd, struct file *filp, int on); -+long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg); -+ssize_t lrng_drng_write(struct file *file, const char __user *buffer, -+ size_t count, loff_t *ppos); -+ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, -+ u32 entropy_bits); -+__poll_t lrng_random_poll(struct file *file, poll_table *wait); -+ssize_t lrng_read_common_block(int nonblock, int pr, -+ char __user *buf, size_t nbytes); -+ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, -+ loff_t *ppos); -+ssize_t lrng_read_seed(char __user *buf, size_t nbytes, unsigned int flags); -+ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr); -+bool lrng_need_entropy(void); -+ -+extern struct wait_queue_head lrng_write_wait; -+ -+#endif /* _LRNG_INTERFACE_DEV_COMMON_H */ -diff --git a/drivers/char/lrng/lrng_interface_hwrand.c b/drivers/char/lrng/lrng_interface_hwrand.c -new file mode 100644 -index 000000000000..e841eea13348 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_hwrand.c -@@ -0,0 +1,68 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG interface with the HW-Random framework -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+ -+static int lrng_hwrand_if_random(struct hwrng *rng, void *buf, size_t max, -+ bool wait) -+{ -+ /* -+ * lrng_get_random_bytes_full not called as we cannot block. -+ * -+ * Note: We should either adjust .quality below depending on -+ * rng_is_initialized() or block here, but neither is not supported by -+ * the hw_rand framework. -+ */ -+ lrng_get_random_bytes(buf, max); -+ return (int)max; -+} -+ -+static struct hwrng lrng_hwrand = { -+ .name = "lrng", -+ .init = NULL, -+ .cleanup = NULL, -+ .read = lrng_hwrand_if_random, -+ -+ /* -+ * We set .quality only in case the LRNG does not provide the common -+ * interfaces or does not use the legacy RNG as entropy source. This -+ * shall avoid that the LRNG automatically spawns the hw_rand -+ * framework's hwrng kernel thread to feed data into -+ * add_hwgenerator_randomness. When the LRNG implements the common -+ * interfaces, this function feeds the data directly into the LRNG. -+ * If the LRNG uses the legacy RNG as entropy source, -+ * add_hwgenerator_randomness is implemented by the legacy RNG, but -+ * still eventually feeds the data into the LRNG. We should avoid such -+ * circular loops. -+ * -+ * We can specify full entropy here, because the LRNG is designed -+ * to provide full entropy. -+ */ -+#if !defined(CONFIG_LRNG_RANDOM_IF) && \ -+ !defined(CONFIG_LRNG_KERNEL_RNG) -+ .quality = 1024, -+#endif -+}; -+ -+static int __init lrng_hwrand_if_mod_init(void) -+{ -+ return hwrng_register(&lrng_hwrand); -+} -+ -+static void __exit lrng_hwrand_if_mod_exit(void) -+{ -+ hwrng_unregister(&lrng_hwrand); -+} -+ -+module_init(lrng_hwrand_if_mod_init); -+module_exit(lrng_hwrand_if_mod_exit); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager HW-Random Interface"); -diff --git a/drivers/char/lrng/lrng_interface_kcapi.c b/drivers/char/lrng/lrng_interface_kcapi.c -new file mode 100644 -index 000000000000..4cb511f8088e ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_kcapi.c -@@ -0,0 +1,129 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG interface with the RNG framework of the kernel crypto API -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+ -+static int lrng_kcapi_if_init(struct crypto_tfm *tfm) -+{ -+ return 0; -+} -+ -+static void lrng_kcapi_if_cleanup(struct crypto_tfm *tfm) { } -+ -+static int lrng_kcapi_if_reseed(const u8 *src, unsigned int slen) -+{ -+ int ret; -+ -+ if (!slen) -+ return 0; -+ -+ /* Insert caller-provided data without crediting entropy */ -+ ret = lrng_pool_insert_aux((u8 *)src, slen, 0); -+ if (ret) -+ return ret; -+ -+ /* Make sure the new data is immediately available to DRNG */ -+ lrng_drng_force_reseed(); -+ -+ return 0; -+} -+ -+static int lrng_kcapi_if_random(struct crypto_rng *tfm, -+ const u8 *src, unsigned int slen, -+ u8 *rdata, unsigned int dlen) -+{ -+ int ret = lrng_kcapi_if_reseed(src, slen); -+ -+ if (!ret) -+ lrng_get_random_bytes_full(rdata, dlen); -+ -+ return ret; -+} -+ -+static int lrng_kcapi_if_reset(struct crypto_rng *tfm, -+ const u8 *seed, unsigned int slen) -+{ -+ return lrng_kcapi_if_reseed(seed, slen); -+} -+ -+static struct rng_alg lrng_alg = { -+ .generate = lrng_kcapi_if_random, -+ .seed = lrng_kcapi_if_reset, -+ .seedsize = 0, -+ .base = { -+ .cra_name = "stdrng", -+ .cra_driver_name = "lrng", -+ .cra_priority = 500, -+ .cra_ctxsize = 0, -+ .cra_module = THIS_MODULE, -+ .cra_init = lrng_kcapi_if_init, -+ .cra_exit = lrng_kcapi_if_cleanup, -+ -+ } -+}; -+ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+static int lrng_kcapi_if_random_atomic(struct crypto_rng *tfm, -+ const u8 *src, unsigned int slen, -+ u8 *rdata, unsigned int dlen) -+{ -+ int ret = lrng_kcapi_if_reseed(src, slen); -+ -+ if (!ret) -+ lrng_get_random_bytes(rdata, dlen); -+ -+ return ret; -+} -+ -+static struct rng_alg lrng_alg_atomic = { -+ .generate = lrng_kcapi_if_random_atomic, -+ .seed = lrng_kcapi_if_reset, -+ .seedsize = 0, -+ .base = { -+ .cra_name = "lrng_atomic", -+ .cra_driver_name = "lrng_atomic", -+ .cra_priority = 100, -+ .cra_ctxsize = 0, -+ .cra_module = THIS_MODULE, -+ .cra_init = lrng_kcapi_if_init, -+ .cra_exit = lrng_kcapi_if_cleanup, -+ -+ } -+}; -+#endif /* CONFIG_LRNG_DRNG_ATOMIC */ -+ -+static int __init lrng_kcapi_if_mod_init(void) -+{ -+ return -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+ crypto_register_rng(&lrng_alg_atomic) ?: -+#endif -+ crypto_register_rng(&lrng_alg); -+} -+ -+static void __exit lrng_kcapi_if_mod_exit(void) -+{ -+ crypto_unregister_rng(&lrng_alg); -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+ crypto_unregister_rng(&lrng_alg_atomic); -+#endif -+} -+ -+module_init(lrng_kcapi_if_mod_init); -+module_exit(lrng_kcapi_if_mod_exit); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager kernel crypto API RNG framework interface"); -+MODULE_ALIAS_CRYPTO("lrng"); -+MODULE_ALIAS_CRYPTO("lrng_atomic"); -+MODULE_ALIAS_CRYPTO("stdrng"); -diff --git a/drivers/char/lrng/lrng_interface_random_kernel.c b/drivers/char/lrng/lrng_interface_random_kernel.c -new file mode 100644 -index 000000000000..471390bfd70c ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_random_kernel.c -@@ -0,0 +1,214 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Kernel space interfaces API/ABI compliant to linux/random.h -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+#include "lrng_interface_random_kernel.h" -+ -+/********************************** Helper ***********************************/ -+ -+static bool lrng_trust_bootloader __initdata = -+ IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER); -+ -+static int __init lrng_parse_trust_bootloader(char *arg) -+{ -+ return kstrtobool(arg, &lrng_trust_bootloader); -+} -+early_param("random.trust_bootloader", lrng_parse_trust_bootloader); -+ -+int __init random_init(const char *command_line) -+{ -+ int ret = lrng_rand_initialize(); -+ -+ lrng_pool_insert_aux(command_line, strlen(command_line), 0); -+ return ret; -+} -+ -+/************************ LRNG kernel input interfaces ************************/ -+ -+/* -+ * add_hwgenerator_randomness() - Interface for in-kernel drivers of true -+ * hardware RNGs. -+ * -+ * Those devices may produce endless random bits and will be throttled -+ * when our pool is full. -+ * -+ * @buffer: buffer holding the entropic data from HW noise sources to be used to -+ * insert into entropy pool. -+ * @count: length of buffer -+ * @entropy_bits: amount of entropy in buffer (value is in bits) -+ */ -+void add_hwgenerator_randomness(const void *buffer, size_t count, -+ size_t entropy_bits) -+{ -+ /* -+ * Suspend writing if we are fully loaded with entropy. -+ * We'll be woken up again once below lrng_write_wakeup_thresh, -+ * or when the calling thread is about to terminate. -+ */ -+ wait_event_interruptible(lrng_write_wait, -+ lrng_need_entropy() || -+ lrng_state_exseed_allow(lrng_noise_source_hw) || -+ kthread_should_stop()); -+ lrng_state_exseed_set(lrng_noise_source_hw, false); -+ lrng_pool_insert_aux(buffer, count, entropy_bits); -+} -+EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); -+ -+/* -+ * add_bootloader_randomness() - Handle random seed passed by bootloader. -+ * -+ * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise -+ * it would be regarded as device data. -+ * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. -+ * -+ * @buf: buffer holding the entropic data from HW noise sources to be used to -+ * insert into entropy pool. -+ * @size: length of buffer -+ */ -+void __init add_bootloader_randomness(const void *buf, size_t size) -+{ -+ lrng_pool_insert_aux(buf, size, lrng_trust_bootloader ? size * 8 : 0); -+} -+ -+/* -+ * Callback for HID layer -- use the HID event values to stir the entropy pool -+ */ -+void add_input_randomness(unsigned int type, unsigned int code, -+ unsigned int value) -+{ -+ static unsigned char last_value; -+ -+ /* ignore autorepeat and the like */ -+ if (value == last_value) -+ return; -+ -+ last_value = value; -+ -+ lrng_irq_array_add_u32((type << 4) ^ code ^ (code >> 4) ^ value); -+} -+EXPORT_SYMBOL_GPL(add_input_randomness); -+ -+/* -+ * add_device_randomness() - Add device- or boot-specific data to the entropy -+ * pool to help initialize it. -+ * -+ * None of this adds any entropy; it is meant to avoid the problem of -+ * the entropy pool having similar initial state across largely -+ * identical devices. -+ * -+ * @buf: buffer holding the entropic data from HW noise sources to be used to -+ * insert into entropy pool. -+ * @size: length of buffer -+ */ -+void add_device_randomness(const void *buf, size_t size) -+{ -+ lrng_pool_insert_aux((u8 *)buf, size, 0); -+} -+EXPORT_SYMBOL(add_device_randomness); -+ -+#ifdef CONFIG_BLOCK -+void rand_initialize_disk(struct gendisk *disk) { } -+void add_disk_randomness(struct gendisk *disk) { } -+EXPORT_SYMBOL(add_disk_randomness); -+#endif -+ -+#ifndef CONFIG_LRNG_IRQ -+void add_interrupt_randomness(int irq) { } -+EXPORT_SYMBOL(add_interrupt_randomness); -+#endif -+ -+#if IS_ENABLED(CONFIG_VMGENID) -+static BLOCKING_NOTIFIER_HEAD(lrng_vmfork_chain); -+ -+/* -+ * Handle a new unique VM ID, which is unique, not secret, so we -+ * don't credit it, but we do immediately force a reseed after so -+ * that it's used by the crng posthaste. -+ */ -+void add_vmfork_randomness(const void *unique_vm_id, size_t size) -+{ -+ add_device_randomness(unique_vm_id, size); -+ if (lrng_state_operational()) -+ lrng_drng_force_reseed(); -+ blocking_notifier_call_chain(&lrng_vmfork_chain, 0, NULL); -+} -+#if IS_MODULE(CONFIG_VMGENID) -+EXPORT_SYMBOL_GPL(add_vmfork_randomness); -+#endif -+ -+int register_random_vmfork_notifier(struct notifier_block *nb) -+{ -+ return blocking_notifier_chain_register(&lrng_vmfork_chain, nb); -+} -+EXPORT_SYMBOL_GPL(register_random_vmfork_notifier); -+ -+int unregister_random_vmfork_notifier(struct notifier_block *nb) -+{ -+ return blocking_notifier_chain_unregister(&lrng_vmfork_chain, nb); -+} -+EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier); -+#endif -+ -+/*********************** LRNG kernel output interfaces ************************/ -+ -+/* -+ * get_random_bytes() - Provider of cryptographic strong random numbers for -+ * kernel-internal usage. -+ * -+ * This function is appropriate for all in-kernel use cases. However, -+ * it will always use the ChaCha20 DRNG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+void get_random_bytes(void *buf, size_t nbytes) -+{ -+ lrng_get_random_bytes(buf, nbytes); -+} -+EXPORT_SYMBOL(get_random_bytes); -+ -+/* -+ * wait_for_random_bytes() - Wait for the LRNG to be seeded and thus -+ * guaranteed to supply cryptographically secure random numbers. -+ * -+ * This applies to: the /dev/urandom device, the get_random_bytes function, -+ * and the get_random_{u32,u64,int,long} family of functions. Using any of -+ * these functions without first calling this function forfeits the guarantee -+ * of security. -+ * -+ * Return: -+ * * 0 if the LRNG has been seeded. -+ * * -ERESTARTSYS if the function was interrupted by a signal. -+ */ -+int wait_for_random_bytes(void) -+{ -+ return lrng_drng_sleep_while_non_min_seeded(); -+} -+EXPORT_SYMBOL(wait_for_random_bytes); -+ -+/* -+ * Returns whether or not the LRNG has been seeded. -+ * -+ * Returns: true if the urandom pool has been seeded. -+ * false if the urandom pool has not been seeded. -+ */ -+bool rng_is_initialized(void) -+{ -+ return lrng_state_operational(); -+} -+EXPORT_SYMBOL(rng_is_initialized); -diff --git a/drivers/char/lrng/lrng_interface_random_kernel.h b/drivers/char/lrng/lrng_interface_random_kernel.h -new file mode 100644 -index 000000000000..f4d09d1fbb14 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_random_kernel.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_INTERFACE_RANDOM_H -+#define _LRNG_INTERFACE_RANDOM_H -+ -+#ifdef CONFIG_LRNG_RANDOM_IF -+void invalidate_batched_entropy(void); -+#else /* CONFIG_LRNG_RANDOM_IF */ -+static inline void invalidate_batched_entropy(void) { } -+#endif /* CONFIG_LRNG_RANDOM_IF */ -+ -+#endif /* _LRNG_INTERFACE_RANDOM_H */ -diff --git a/drivers/char/lrng/lrng_interface_random_user.c b/drivers/char/lrng/lrng_interface_random_user.c -new file mode 100644 -index 000000000000..d12e883804d9 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_random_user.c -@@ -0,0 +1,104 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Common user space interfaces compliant to random(4), random(7) and -+ * getrandom(2) man pages. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+ -+static ssize_t lrng_drng_read(struct file *file, char __user *buf, -+ size_t nbytes, loff_t *ppos) -+{ -+ if (!lrng_state_min_seeded()) -+ pr_notice_ratelimited("%s - use of insufficiently seeded DRNG (%zu bytes read)\n", -+ current->comm, nbytes); -+ else if (!lrng_state_operational()) -+ pr_debug_ratelimited("%s - use of not fully seeded DRNG (%zu bytes read)\n", -+ current->comm, nbytes); -+ -+ return lrng_read_common(buf, nbytes, false); -+} -+ -+const struct file_operations random_fops = { -+ .read = lrng_drng_read_block, -+ .write = lrng_drng_write, -+ .poll = lrng_random_poll, -+ .unlocked_ioctl = lrng_ioctl, -+ .compat_ioctl = compat_ptr_ioctl, -+ .fasync = lrng_fasync, -+ .llseek = noop_llseek, -+}; -+ -+const struct file_operations urandom_fops = { -+ .read = lrng_drng_read, -+ .write = lrng_drng_write, -+ .unlocked_ioctl = lrng_ioctl, -+ .compat_ioctl = compat_ptr_ioctl, -+ .fasync = lrng_fasync, -+ .llseek = noop_llseek, -+}; -+ -+/* -+ * GRND_SEED -+ * -+ * This flag requests to provide the data directly from the entropy sources. -+ * -+ * The behavior of the call is exactly as outlined for the function -+ * lrng_get_seed in lrng.h. -+ */ -+#define GRND_SEED 0x0010 -+ -+/* -+ * GRND_FULLY_SEEDED -+ * -+ * This flag indicates whether the caller wants to reseed a DRNG that is already -+ * fully seeded. See esdm_get_seed in lrng.h for details. -+ */ -+#define GRND_FULLY_SEEDED 0x0020 -+ -+SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, -+ unsigned int, flags) -+{ -+ if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE| -+ GRND_SEED|GRND_FULLY_SEEDED)) -+ return -EINVAL; -+ -+ /* -+ * Requesting insecure and blocking randomness at the same time makes -+ * no sense. -+ */ -+ if ((flags & -+ (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) -+ return -EINVAL; -+ if ((flags & -+ (GRND_INSECURE|GRND_SEED)) == (GRND_INSECURE|GRND_SEED)) -+ return -EINVAL; -+ if ((flags & -+ (GRND_RANDOM|GRND_SEED)) == (GRND_RANDOM|GRND_SEED)) -+ return -EINVAL; -+ -+ if (count > INT_MAX) -+ count = INT_MAX; -+ -+ if (flags & GRND_INSECURE) { -+ return lrng_drng_read(NULL, buf, count, NULL); -+ } else if (flags & GRND_SEED) { -+ unsigned int seed_flags = (flags & GRND_NONBLOCK) ? -+ LRNG_GET_SEED_NONBLOCK : 0; -+ -+ seed_flags |= (flags & GRND_FULLY_SEEDED) ? -+ LRNG_GET_SEED_FULLY_SEEDED : 0; -+ return lrng_read_seed(buf, count, seed_flags); -+ } -+ -+ return lrng_read_common_block(flags & GRND_NONBLOCK, -+ flags & GRND_RANDOM, buf, count); -+} -diff --git a/drivers/char/lrng/lrng_numa.c b/drivers/char/lrng/lrng_numa.c -new file mode 100644 -index 000000000000..d74dd8df2843 ---- /dev/null -+++ b/drivers/char/lrng/lrng_numa.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG NUMA support -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_mgr.h" -+#include "lrng_numa.h" -+#include "lrng_proc.h" -+ -+static struct lrng_drng **lrng_drng __read_mostly = NULL; -+ -+struct lrng_drng **lrng_drng_instances(void) -+{ -+ /* counterpart to cmpxchg_release in _lrng_drngs_numa_alloc */ -+ return READ_ONCE(lrng_drng); -+} -+ -+/* Allocate the data structures for the per-NUMA node DRNGs */ -+static void _lrng_drngs_numa_alloc(struct work_struct *work) -+{ -+ struct lrng_drng **drngs; -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ u32 node; -+ bool init_drng_used = false; -+ -+ mutex_lock(&lrng_crypto_cb_update); -+ -+ /* per-NUMA-node DRNGs are already present */ -+ if (lrng_drng) -+ goto unlock; -+ -+ /* Make sure the initial DRNG is initialized and its drng_cb is set */ -+ if (lrng_drng_initalize()) -+ goto err; -+ -+ drngs = kcalloc(nr_node_ids, sizeof(void *), GFP_KERNEL|__GFP_NOFAIL); -+ for_each_online_node(node) { -+ struct lrng_drng *drng; -+ -+ if (!init_drng_used) { -+ drngs[node] = lrng_drng_init; -+ init_drng_used = true; -+ continue; -+ } -+ -+ drng = kmalloc_node(sizeof(struct lrng_drng), -+ GFP_KERNEL|__GFP_NOFAIL, node); -+ memset(drng, 0, sizeof(lrng_drng)); -+ -+ if (lrng_drng_alloc_common(drng, lrng_drng_init->drng_cb)) { -+ kfree(drng); -+ goto err; -+ } -+ -+ drng->hash_cb = lrng_drng_init->hash_cb; -+ drng->hash = lrng_drng_init->hash_cb->hash_alloc(); -+ if (IS_ERR(drng->hash)) { -+ lrng_drng_init->drng_cb->drng_dealloc(drng->drng); -+ kfree(drng); -+ goto err; -+ } -+ -+ mutex_init(&drng->lock); -+ rwlock_init(&drng->hash_lock); -+ -+ /* -+ * No reseeding of NUMA DRNGs from previous DRNGs as this -+ * would complicate the code. Let it simply reseed. -+ */ -+ drngs[node] = drng; -+ -+ lrng_pool_inc_numa_node(); -+ pr_info("DRNG and entropy pool read hash for NUMA node %d allocated\n", -+ node); -+ } -+ -+ /* counterpart to READ_ONCE in lrng_drng_instances */ -+ if (!cmpxchg_release(&lrng_drng, NULL, drngs)) { -+ lrng_pool_all_numa_nodes_seeded(false); -+ goto unlock; -+ } -+ -+err: -+ for_each_online_node(node) { -+ struct lrng_drng *drng = drngs[node]; -+ -+ if (drng == lrng_drng_init) -+ continue; -+ -+ if (drng) { -+ drng->hash_cb->hash_dealloc(drng->hash); -+ drng->drng_cb->drng_dealloc(drng->drng); -+ kfree(drng); -+ } -+ } -+ kfree(drngs); -+ -+unlock: -+ mutex_unlock(&lrng_crypto_cb_update); -+} -+ -+static DECLARE_WORK(lrng_drngs_numa_alloc_work, _lrng_drngs_numa_alloc); -+ -+static void lrng_drngs_numa_alloc(void) -+{ -+ schedule_work(&lrng_drngs_numa_alloc_work); -+} -+ -+static int __init lrng_numa_init(void) -+{ -+ lrng_drngs_numa_alloc(); -+ return 0; -+} -+ -+late_initcall(lrng_numa_init); -diff --git a/drivers/char/lrng/lrng_numa.h b/drivers/char/lrng/lrng_numa.h -new file mode 100644 -index 000000000000..dc8dff9816ee ---- /dev/null -+++ b/drivers/char/lrng/lrng_numa.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_NUMA_H -+#define _LRNG_NUMA_H -+ -+#ifdef CONFIG_NUMA -+struct lrng_drng **lrng_drng_instances(void); -+#else /* CONFIG_NUMA */ -+static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; } -+#endif /* CONFIG_NUMA */ -+ -+#endif /* _LRNG_NUMA_H */ -diff --git a/drivers/char/lrng/lrng_proc.c b/drivers/char/lrng/lrng_proc.c -new file mode 100644 -index 000000000000..92329cc6ef28 ---- /dev/null -+++ b/drivers/char/lrng/lrng_proc.c -@@ -0,0 +1,74 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG proc interfaces -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_proc.h" -+ -+/* Number of online DRNGs */ -+static u32 numa_drngs = 1; -+ -+void lrng_pool_inc_numa_node(void) -+{ -+ numa_drngs++; -+} -+ -+static int lrng_proc_type_show(struct seq_file *m, void *v) -+{ -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ unsigned char buf[270]; -+ u32 i; -+ -+ mutex_lock(&lrng_drng_init->lock); -+ snprintf(buf, sizeof(buf), -+ "DRNG name: %s\n" -+ "LRNG security strength in bits: %d\n" -+ "Number of DRNG instances: %u\n" -+ "Standards compliance: %s%s\n" -+ "LRNG minimally seeded: %s\n" -+ "LRNG fully seeded: %s\n" -+ "LRNG entropy level: %u\n", -+ lrng_drng_init->drng_cb->drng_name(), -+ lrng_security_strength(), -+ numa_drngs, -+ lrng_sp80090c_compliant() ? "SP800-90C " : "", -+ lrng_ntg1_compliant() ? "NTG.1 " : "", -+ lrng_state_min_seeded() ? "true" : "false", -+ lrng_state_fully_seeded() ? "true" : "false", -+ lrng_avail_entropy()); -+ seq_write(m, buf, strlen(buf)); -+ -+ for_each_lrng_es(i) { -+ snprintf(buf, sizeof(buf), -+ "Entropy Source %u properties:\n" -+ " Name: %s\n", -+ i, lrng_es[i]->name); -+ seq_write(m, buf, strlen(buf)); -+ -+ buf[0] = '\0'; -+ lrng_es[i]->state(buf, sizeof(buf)); -+ seq_write(m, buf, strlen(buf)); -+ } -+ -+ mutex_unlock(&lrng_drng_init->lock); -+ -+ return 0; -+} -+ -+static int __init lrng_proc_type_init(void) -+{ -+ proc_create_single("lrng_type", 0444, NULL, &lrng_proc_type_show); -+ return 0; -+} -+ -+module_init(lrng_proc_type_init); -diff --git a/drivers/char/lrng/lrng_proc.h b/drivers/char/lrng/lrng_proc.h -new file mode 100644 -index 000000000000..c653274f1954 ---- /dev/null -+++ b/drivers/char/lrng/lrng_proc.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_PROC_H -+#define _LRNG_PROC_H -+ -+#ifdef CONFIG_SYSCTL -+void lrng_pool_inc_numa_node(void); -+#else -+static inline void lrng_pool_inc_numa_node(void) { } -+#endif -+ -+#endif /* _LRNG_PROC_H */ -diff --git a/drivers/char/lrng/lrng_selftest.c b/drivers/char/lrng/lrng_selftest.c -new file mode 100644 -index 000000000000..15f1e4a2a719 ---- /dev/null -+++ b/drivers/char/lrng/lrng_selftest.c -@@ -0,0 +1,397 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG power-on and on-demand self-test -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+/* -+ * In addition to the self-tests below, the following LRNG components -+ * are covered with self-tests during regular operation: -+ * -+ * * power-on self-test: SP800-90A DRBG provided by the Linux kernel crypto API -+ * * power-on self-test: PRNG provided by the Linux kernel crypto API -+ * * runtime test: Raw noise source data testing including SP800-90B compliant -+ * tests when enabling CONFIG_LRNG_HEALTH_TESTS -+ * -+ * Additional developer tests present with LRNG code: -+ * * SP800-90B APT and RCT test enforcement validation when enabling -+ * CONFIG_LRNG_APT_BROKEN or CONFIG_LRNG_RCT_BROKEN. -+ * * Collection of raw entropy from the interrupt noise source when enabling -+ * CONFIG_LRNG_TESTING and pulling the data from the kernel with the provided -+ * interface. -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "lrng_drng_chacha20.h" -+#include "lrng_sha.h" -+ -+#define LRNG_SELFTEST_PASSED 0 -+#define LRNG_SEFLTEST_ERROR_TIME (1 << 0) -+#define LRNG_SEFLTEST_ERROR_CHACHA20 (1 << 1) -+#define LRNG_SEFLTEST_ERROR_HASH (1 << 2) -+#define LRNG_SEFLTEST_ERROR_GCD (1 << 3) -+#define LRNG_SELFTEST_NOT_EXECUTED 0xffffffff -+ -+#ifdef CONFIG_LRNG_TIMER_COMMON -+ -+#include "lrng_es_timer_common.h" -+ -+static u32 lrng_data_selftest_ptr = 0; -+static u32 lrng_data_selftest[LRNG_DATA_ARRAY_SIZE]; -+ -+static void lrng_data_process_selftest_insert(u32 time) -+{ -+ u32 ptr = lrng_data_selftest_ptr++ & LRNG_DATA_WORD_MASK; -+ unsigned int array = lrng_data_idx2array(ptr); -+ unsigned int slot = lrng_data_idx2slot(ptr); -+ -+ /* zeroization of slot to ensure the following OR adds the data */ -+ lrng_data_selftest[array] &= -+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, -+ slot)); -+ lrng_data_selftest[array] |= -+ lrng_data_slot_val(time & LRNG_DATA_SLOTSIZE_MASK, slot); -+} -+ -+static void lrng_data_process_selftest_u32(u32 data) -+{ -+ u32 pre_ptr, ptr, mask; -+ unsigned int pre_array; -+ -+ /* Increment pointer by number of slots taken for input value */ -+ lrng_data_selftest_ptr += LRNG_DATA_SLOTS_PER_UINT; -+ -+ /* ptr to current unit */ -+ ptr = lrng_data_selftest_ptr; -+ -+ lrng_data_split_u32(&ptr, &pre_ptr, &mask); -+ -+ /* MSB of data go into previous unit */ -+ pre_array = lrng_data_idx2array(pre_ptr); -+ /* zeroization of slot to ensure the following OR adds the data */ -+ lrng_data_selftest[pre_array] &= ~(0xffffffff & ~mask); -+ lrng_data_selftest[pre_array] |= data & ~mask; -+ -+ /* LSB of data go into current unit */ -+ lrng_data_selftest[lrng_data_idx2array(ptr)] = data & mask; -+} -+ -+static unsigned int lrng_data_process_selftest(void) -+{ -+ u32 time; -+ u32 idx_zero_compare = (0 << 0) | (1 << 8) | (2 << 16) | (3 << 24); -+ u32 idx_one_compare = (4 << 0) | (5 << 8) | (6 << 16) | (7 << 24); -+ u32 idx_last_compare = -+ (((LRNG_DATA_NUM_VALUES - 4) & LRNG_DATA_SLOTSIZE_MASK) << 0) | -+ (((LRNG_DATA_NUM_VALUES - 3) & LRNG_DATA_SLOTSIZE_MASK) << 8) | -+ (((LRNG_DATA_NUM_VALUES - 2) & LRNG_DATA_SLOTSIZE_MASK) << 16) | -+ (((LRNG_DATA_NUM_VALUES - 1) & LRNG_DATA_SLOTSIZE_MASK) << 24); -+ -+ (void)idx_one_compare; -+ -+ /* "poison" the array to verify the operation of the zeroization */ -+ lrng_data_selftest[0] = 0xffffffff; -+ lrng_data_selftest[1] = 0xffffffff; -+ -+ lrng_data_process_selftest_insert(0); -+ /* -+ * Note, when using lrng_data_process_u32() on unaligned ptr, -+ * the first slots will go into next word, and the last slots go -+ * into the previous word. -+ */ -+ lrng_data_process_selftest_u32((4 << 0) | (1 << 8) | (2 << 16) | -+ (3 << 24)); -+ lrng_data_process_selftest_insert(5); -+ lrng_data_process_selftest_insert(6); -+ lrng_data_process_selftest_insert(7); -+ -+ if ((lrng_data_selftest[0] != idx_zero_compare) || -+ (lrng_data_selftest[1] != idx_one_compare)) -+ goto err; -+ -+ /* Reset for next test */ -+ lrng_data_selftest[0] = 0; -+ lrng_data_selftest[1] = 0; -+ lrng_data_selftest_ptr = 0; -+ -+ for (time = 0; time < LRNG_DATA_NUM_VALUES; time++) -+ lrng_data_process_selftest_insert(time); -+ -+ if ((lrng_data_selftest[0] != idx_zero_compare) || -+ (lrng_data_selftest[1] != idx_one_compare) || -+ (lrng_data_selftest[LRNG_DATA_ARRAY_SIZE - 1] != idx_last_compare)) -+ goto err; -+ -+ return LRNG_SELFTEST_PASSED; -+ -+err: -+ pr_err("LRNG data array self-test FAILED\n"); -+ return LRNG_SEFLTEST_ERROR_TIME; -+} -+ -+static unsigned int lrng_gcd_selftest(void) -+{ -+ u32 history[10]; -+ unsigned int i; -+ -+#define LRNG_GCD_SELFTEST 3 -+ for (i = 0; i < ARRAY_SIZE(history); i++) -+ history[i] = i * LRNG_GCD_SELFTEST; -+ -+ if (lrng_gcd_analyze(history, ARRAY_SIZE(history)) == LRNG_GCD_SELFTEST) -+ return LRNG_SELFTEST_PASSED; -+ -+ pr_err("LRNG GCD self-test FAILED\n"); -+ return LRNG_SEFLTEST_ERROR_GCD; -+} -+ -+#else /* CONFIG_LRNG_TIMER_COMMON */ -+ -+static unsigned int lrng_data_process_selftest(void) -+{ -+ return LRNG_SELFTEST_PASSED; -+} -+ -+static unsigned int lrng_gcd_selftest(void) -+{ -+ return LRNG_SELFTEST_PASSED; -+} -+ -+#endif /* CONFIG_LRNG_TIMER_COMMON */ -+ -+/* The test vectors are taken from crypto/testmgr.h */ -+static unsigned int lrng_hash_selftest(void) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; -+ static const u8 lrng_hash_selftest_result[] = -+#ifdef CONFIG_CRYPTO_LIB_SHA256 -+ { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, -+ 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, -+ 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, -+ 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad }; -+#else /* CONFIG_CRYPTO_LIB_SHA256 */ -+ { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e, -+ 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d }; -+#endif /* CONFIG_CRYPTO_LIB_SHA256 */ -+ static const u8 hash_input[] = { 0x61, 0x62, 0x63 }; /* "abc" */ -+ u8 digest[sizeof(lrng_hash_selftest_result)] __aligned(sizeof(u32)); -+ -+ if (sizeof(digest) != hash_cb->hash_digestsize(NULL)) -+ return LRNG_SEFLTEST_ERROR_HASH; -+ -+ if (!hash_cb->hash_init(shash, NULL) && -+ !hash_cb->hash_update(shash, hash_input, -+ sizeof(hash_input)) && -+ !hash_cb->hash_final(shash, digest) && -+ !memcmp(digest, lrng_hash_selftest_result, sizeof(digest))) -+ return 0; -+ -+ pr_err("LRNG %s Hash self-test FAILED\n", hash_cb->hash_name()); -+ return LRNG_SEFLTEST_ERROR_HASH; -+} -+ -+#ifdef CONFIG_LRNG_DRNG_CHACHA20 -+ -+static void lrng_selftest_bswap32(u32 *ptr, u32 words) -+{ -+ u32 i; -+ -+ /* Byte-swap data which is an LE representation */ -+ for (i = 0; i < words; i++) { -+ __le32 *p = (__le32 *)ptr; -+ -+ *p = cpu_to_le32(*ptr); -+ ptr++; -+ } -+} -+ -+/* -+ * The test vectors were generated using the ChaCha20 DRNG from -+ * https://www.chronox.de/chacha20.html -+ */ -+static unsigned int lrng_chacha20_drng_selftest(void) -+{ -+ const struct lrng_drng_cb *drng_cb = &lrng_cc20_drng_cb; -+ u8 seed[CHACHA_KEY_SIZE * 2] = { -+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, -+ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, -+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, -+ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, -+ }; -+ struct chacha20_block chacha20; -+ int ret; -+ u8 outbuf[CHACHA_KEY_SIZE * 2] __aligned(sizeof(u32)); -+ -+ /* -+ * Expected result when ChaCha20 DRNG state is zero: -+ * * constants are set to "expand 32-byte k" -+ * * remaining state is 0 -+ * and pulling one half ChaCha20 DRNG block. -+ */ -+ static const u8 expected_halfblock[CHACHA_KEY_SIZE] = { -+ 0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, -+ 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86, 0xbd, 0x28, -+ 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, -+ 0xa8, 0x36, 0xef, 0xcc, 0x8b, 0x77, 0x0d, 0xc7 }; -+ -+ /* -+ * Expected result when ChaCha20 DRNG state is zero: -+ * * constants are set to "expand 32-byte k" -+ * * remaining state is 0 -+ * followed by a reseed with two keyblocks -+ * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ * 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, -+ * 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, -+ * 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, -+ * 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f -+ * and pulling one ChaCha20 DRNG block. -+ */ -+ static const u8 expected_oneblock[CHACHA_KEY_SIZE * 2] = { -+ 0xe3, 0xb0, 0x8a, 0xcc, 0x34, 0xc3, 0x17, 0x0e, -+ 0xc3, 0xd8, 0xc3, 0x40, 0xe7, 0x73, 0xe9, 0x0d, -+ 0xd1, 0x62, 0xa3, 0x5d, 0x7d, 0xf2, 0xf1, 0x4a, -+ 0x24, 0x42, 0xb7, 0x1e, 0xb0, 0x05, 0x17, 0x07, -+ 0xb9, 0x35, 0x10, 0x69, 0x8b, 0x46, 0xfb, 0x51, -+ 0xe9, 0x91, 0x3f, 0x46, 0xf2, 0x4d, 0xea, 0xd0, -+ 0x81, 0xc1, 0x1b, 0xa9, 0x5d, 0x52, 0x91, 0x5f, -+ 0xcd, 0xdc, 0xc6, 0xd6, 0xc3, 0x7c, 0x50, 0x23 }; -+ -+ /* -+ * Expected result when ChaCha20 DRNG state is zero: -+ * * constants are set to "expand 32-byte k" -+ * * remaining state is 0 -+ * followed by a reseed with one key block plus one byte -+ * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ * 0x20 -+ * and pulling less than one ChaCha20 DRNG block. -+ */ -+ static const u8 expected_block_nonalinged[CHACHA_KEY_SIZE + 4] = { -+ 0x9c, 0xfc, 0x5e, 0x31, 0x21, 0x62, 0x11, 0x85, -+ 0xd3, 0x77, 0xd3, 0x69, 0x0f, 0xa8, 0x16, 0x55, -+ 0xb4, 0x4c, 0xf6, 0x52, 0xf3, 0xa8, 0x37, 0x99, -+ 0x38, 0x76, 0xa0, 0x66, 0xec, 0xbb, 0xce, 0xa9, -+ 0x9c, 0x95, 0xa1, 0xfd }; -+ -+ BUILD_BUG_ON(sizeof(seed) % sizeof(u32)); -+ -+ memset(&chacha20, 0, sizeof(chacha20)); -+ lrng_cc20_init_rfc7539(&chacha20); -+ lrng_selftest_bswap32((u32 *)seed, sizeof(seed) / sizeof(u32)); -+ -+ /* Generate with zero state */ -+ ret = drng_cb->drng_generate(&chacha20, outbuf, -+ sizeof(expected_halfblock)); -+ if (ret != sizeof(expected_halfblock)) -+ goto err; -+ if (memcmp(outbuf, expected_halfblock, sizeof(expected_halfblock))) -+ goto err; -+ -+ /* Clear state of DRNG */ -+ memset(&chacha20.key.u[0], 0, 48); -+ -+ /* Reseed with 2 key blocks */ -+ ret = drng_cb->drng_seed(&chacha20, seed, sizeof(expected_oneblock)); -+ if (ret < 0) -+ goto err; -+ ret = drng_cb->drng_generate(&chacha20, outbuf, -+ sizeof(expected_oneblock)); -+ if (ret != sizeof(expected_oneblock)) -+ goto err; -+ if (memcmp(outbuf, expected_oneblock, sizeof(expected_oneblock))) -+ goto err; -+ -+ /* Clear state of DRNG */ -+ memset(&chacha20.key.u[0], 0, 48); -+ -+ /* Reseed with 1 key block and one byte */ -+ ret = drng_cb->drng_seed(&chacha20, seed, -+ sizeof(expected_block_nonalinged)); -+ if (ret < 0) -+ goto err; -+ ret = drng_cb->drng_generate(&chacha20, outbuf, -+ sizeof(expected_block_nonalinged)); -+ if (ret != sizeof(expected_block_nonalinged)) -+ goto err; -+ if (memcmp(outbuf, expected_block_nonalinged, -+ sizeof(expected_block_nonalinged))) -+ goto err; -+ -+ return LRNG_SELFTEST_PASSED; -+ -+err: -+ pr_err("LRNG ChaCha20 DRNG self-test FAILED\n"); -+ return LRNG_SEFLTEST_ERROR_CHACHA20; -+} -+ -+#else /* CONFIG_LRNG_DRNG_CHACHA20 */ -+ -+static unsigned int lrng_chacha20_drng_selftest(void) -+{ -+ return LRNG_SELFTEST_PASSED; -+} -+ -+#endif /* CONFIG_LRNG_DRNG_CHACHA20 */ -+ -+static unsigned int lrng_selftest_status = LRNG_SELFTEST_NOT_EXECUTED; -+ -+static int lrng_selftest(void) -+{ -+ unsigned int ret = lrng_data_process_selftest(); -+ -+ ret |= lrng_chacha20_drng_selftest(); -+ ret |= lrng_hash_selftest(); -+ ret |= lrng_gcd_selftest(); -+ -+ if (ret) { -+ if (IS_ENABLED(CONFIG_LRNG_SELFTEST_PANIC)) -+ panic("LRNG self-tests failed: %u\n", ret); -+ } else { -+ pr_info("LRNG self-tests passed\n"); -+ } -+ -+ lrng_selftest_status = ret; -+ -+ if (lrng_selftest_status) -+ return -EFAULT; -+ return 0; -+} -+ -+#ifdef CONFIG_SYSFS -+/* Re-perform self-test when any value is written to the sysfs file. */ -+static int lrng_selftest_sysfs_set(const char *val, -+ const struct kernel_param *kp) -+{ -+ return lrng_selftest(); -+} -+ -+static const struct kernel_param_ops lrng_selftest_sysfs = { -+ .set = lrng_selftest_sysfs_set, -+ .get = param_get_uint, -+}; -+module_param_cb(selftest_status, &lrng_selftest_sysfs, &lrng_selftest_status, -+ 0644); -+#endif /* CONFIG_SYSFS */ -+ -+static int __init lrng_selftest_init(void) -+{ -+ return lrng_selftest(); -+} -+ -+module_init(lrng_selftest_init); -diff --git a/drivers/char/lrng/lrng_sha.h b/drivers/char/lrng/lrng_sha.h -new file mode 100644 -index 000000000000..d2f134f54773 ---- /dev/null -+++ b/drivers/char/lrng/lrng_sha.h -@@ -0,0 +1,14 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG SHA definition usable in atomic contexts right from the start of the -+ * kernel. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_SHA_H -+#define _LRNG_SHA_H -+ -+extern const struct lrng_hash_cb lrng_sha_hash_cb; -+ -+#endif /* _LRNG_SHA_H */ -diff --git a/drivers/char/lrng/lrng_sha1.c b/drivers/char/lrng/lrng_sha1.c -new file mode 100644 -index 000000000000..9cbc7a6fee49 ---- /dev/null -+++ b/drivers/char/lrng/lrng_sha1.c -@@ -0,0 +1,88 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the SHA-1 implementation that can be used -+ * without the kernel crypto API available including during early boot and in -+ * atomic contexts. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "lrng_sha.h" -+ -+/* -+ * If the SHA-256 support is not compiled, we fall back to SHA-1 that is always -+ * compiled and present in the kernel. -+ */ -+static u32 lrng_sha1_hash_digestsize(void *hash) -+{ -+ return SHA1_DIGEST_SIZE; -+} -+ -+static void lrng_sha1_block_fn(struct sha1_state *sctx, const u8 *src, -+ int blocks) -+{ -+ u32 temp[SHA1_WORKSPACE_WORDS]; -+ -+ while (blocks--) { -+ sha1_transform(sctx->state, src, temp); -+ src += SHA1_BLOCK_SIZE; -+ } -+ memzero_explicit(temp, sizeof(temp)); -+} -+ -+static int lrng_sha1_hash_init(struct shash_desc *shash, void *hash) -+{ -+ /* -+ * We do not need a TFM - we only need sufficient space for -+ * struct sha1_state on the stack. -+ */ -+ sha1_base_init(shash); -+ return 0; -+} -+ -+static int lrng_sha1_hash_update(struct shash_desc *shash, -+ const u8 *inbuf, u32 inbuflen) -+{ -+ return sha1_base_do_update(shash, inbuf, inbuflen, lrng_sha1_block_fn); -+} -+ -+static int lrng_sha1_hash_final(struct shash_desc *shash, u8 *digest) -+{ -+ return sha1_base_do_finalize(shash, lrng_sha1_block_fn) ?: -+ sha1_base_finish(shash, digest); -+} -+ -+static const char *lrng_sha1_hash_name(void) -+{ -+ return "SHA-1"; -+} -+ -+static void lrng_sha1_hash_desc_zero(struct shash_desc *shash) -+{ -+ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha1_state)); -+} -+ -+static void *lrng_sha1_hash_alloc(void) -+{ -+ pr_info("Hash %s allocated\n", lrng_sha1_hash_name()); -+ return NULL; -+} -+ -+static void lrng_sha1_hash_dealloc(void *hash) { } -+ -+const struct lrng_hash_cb lrng_sha_hash_cb = { -+ .hash_name = lrng_sha1_hash_name, -+ .hash_alloc = lrng_sha1_hash_alloc, -+ .hash_dealloc = lrng_sha1_hash_dealloc, -+ .hash_digestsize = lrng_sha1_hash_digestsize, -+ .hash_init = lrng_sha1_hash_init, -+ .hash_update = lrng_sha1_hash_update, -+ .hash_final = lrng_sha1_hash_final, -+ .hash_desc_zero = lrng_sha1_hash_desc_zero, -+}; -diff --git a/drivers/char/lrng/lrng_sha256.c b/drivers/char/lrng/lrng_sha256.c -new file mode 100644 -index 000000000000..50705351a71c ---- /dev/null -+++ b/drivers/char/lrng/lrng_sha256.c -@@ -0,0 +1,72 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the SHA-256 implementation that can be used -+ * without the kernel crypto API available including during early boot and in -+ * atomic contexts. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_sha.h" -+ -+static u32 lrng_sha256_hash_digestsize(void *hash) -+{ -+ return SHA256_DIGEST_SIZE; -+} -+ -+static int lrng_sha256_hash_init(struct shash_desc *shash, void *hash) -+{ -+ /* -+ * We do not need a TFM - we only need sufficient space for -+ * struct sha256_state on the stack. -+ */ -+ sha256_init(shash_desc_ctx(shash)); -+ return 0; -+} -+ -+static int lrng_sha256_hash_update(struct shash_desc *shash, -+ const u8 *inbuf, u32 inbuflen) -+{ -+ sha256_update(shash_desc_ctx(shash), inbuf, inbuflen); -+ return 0; -+} -+ -+static int lrng_sha256_hash_final(struct shash_desc *shash, u8 *digest) -+{ -+ sha256_final(shash_desc_ctx(shash), digest); -+ return 0; -+} -+ -+static const char *lrng_sha256_hash_name(void) -+{ -+ return "SHA-256"; -+} -+ -+static void lrng_sha256_hash_desc_zero(struct shash_desc *shash) -+{ -+ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha256_state)); -+} -+ -+static void *lrng_sha256_hash_alloc(void) -+{ -+ pr_info("Hash %s allocated\n", lrng_sha256_hash_name()); -+ return NULL; -+} -+ -+static void lrng_sha256_hash_dealloc(void *hash) { } -+ -+const struct lrng_hash_cb lrng_sha_hash_cb = { -+ .hash_name = lrng_sha256_hash_name, -+ .hash_alloc = lrng_sha256_hash_alloc, -+ .hash_dealloc = lrng_sha256_hash_dealloc, -+ .hash_digestsize = lrng_sha256_hash_digestsize, -+ .hash_init = lrng_sha256_hash_init, -+ .hash_update = lrng_sha256_hash_update, -+ .hash_final = lrng_sha256_hash_final, -+ .hash_desc_zero = lrng_sha256_hash_desc_zero, -+}; -diff --git a/drivers/char/lrng/lrng_switch.c b/drivers/char/lrng/lrng_switch.c -new file mode 100644 -index 000000000000..aae75594926f ---- /dev/null -+++ b/drivers/char/lrng/lrng_switch.c -@@ -0,0 +1,286 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG DRNG switching support -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+#include "lrng_numa.h" -+ -+static int __maybe_unused -+lrng_hash_switch(struct lrng_drng *drng_store, const void *cb, int node) -+{ -+ const struct lrng_hash_cb *new_cb = (const struct lrng_hash_cb *)cb; -+ const struct lrng_hash_cb *old_cb = drng_store->hash_cb; -+ unsigned long flags; -+ u32 i; -+ void *new_hash, *old_hash; -+ int ret; -+ -+ if (node == -1) -+ return 0; -+ -+ new_hash = new_cb->hash_alloc(); -+ old_hash = drng_store->hash; -+ -+ if (IS_ERR(new_hash)) { -+ pr_warn("could not allocate new LRNG pool hash (%ld)\n", -+ PTR_ERR(new_hash)); -+ return PTR_ERR(new_hash); -+ } -+ -+ if (new_cb->hash_digestsize(new_hash) > LRNG_MAX_DIGESTSIZE) { -+ pr_warn("digest size of newly requested hash too large\n"); -+ new_cb->hash_dealloc(new_hash); -+ return -EINVAL; -+ } -+ -+ write_lock_irqsave(&drng_store->hash_lock, flags); -+ -+ /* Trigger the switch for each entropy source */ -+ for_each_lrng_es(i) { -+ if (!lrng_es[i]->switch_hash) -+ continue; -+ ret = lrng_es[i]->switch_hash(drng_store, node, new_cb, -+ new_hash, old_cb); -+ if (ret) { -+ u32 j; -+ -+ /* Revert all already executed operations */ -+ for (j = 0; j < i; j++) { -+ if (!lrng_es[j]->switch_hash) -+ continue; -+ WARN_ON(lrng_es[j]->switch_hash(drng_store, -+ node, old_cb, -+ old_hash, -+ new_cb)); -+ } -+ goto err; -+ } -+ } -+ -+ drng_store->hash = new_hash; -+ drng_store->hash_cb = new_cb; -+ old_cb->hash_dealloc(old_hash); -+ pr_info("Conditioning function allocated for DRNG for NUMA node %d\n", -+ node); -+ -+err: -+ write_unlock_irqrestore(&drng_store->hash_lock, flags); -+ return ret; -+} -+ -+static int __maybe_unused -+lrng_drng_switch(struct lrng_drng *drng_store, const void *cb, int node) -+{ -+ const struct lrng_drng_cb *new_cb = (const struct lrng_drng_cb *)cb; -+ const struct lrng_drng_cb *old_cb = drng_store->drng_cb; -+ int ret; -+ u8 seed[LRNG_DRNG_SECURITY_STRENGTH_BYTES]; -+ void *new_drng = new_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); -+ void *old_drng = drng_store->drng; -+ u32 current_security_strength; -+ bool reset_drng = !lrng_get_available(); -+ -+ if (IS_ERR(new_drng)) { -+ pr_warn("could not allocate new DRNG for NUMA node %d (%ld)\n", -+ node, PTR_ERR(new_drng)); -+ return PTR_ERR(new_drng); -+ } -+ -+ current_security_strength = lrng_security_strength(); -+ mutex_lock(&drng_store->lock); -+ -+ /* -+ * Pull from existing DRNG to seed new DRNG regardless of seed status -+ * of old DRNG -- the entropy state for the DRNG is left unchanged which -+ * implies that als the new DRNG is reseeded when deemed necessary. This -+ * seeding of the new DRNG shall only ensure that the new DRNG has the -+ * same entropy as the old DRNG. -+ */ -+ ret = old_cb->drng_generate(old_drng, seed, sizeof(seed)); -+ mutex_unlock(&drng_store->lock); -+ -+ if (ret < 0) { -+ reset_drng = true; -+ pr_warn("getting random data from DRNG failed for NUMA node %d (%d)\n", -+ node, ret); -+ } else { -+ /* seed new DRNG with data */ -+ ret = new_cb->drng_seed(new_drng, seed, ret); -+ memzero_explicit(seed, sizeof(seed)); -+ if (ret < 0) { -+ reset_drng = true; -+ pr_warn("seeding of new DRNG failed for NUMA node %d (%d)\n", -+ node, ret); -+ } else { -+ pr_debug("seeded new DRNG of NUMA node %d instance from old DRNG instance\n", -+ node); -+ } -+ } -+ -+ mutex_lock(&drng_store->lock); -+ -+ if (reset_drng) -+ lrng_drng_reset(drng_store); -+ -+ drng_store->drng = new_drng; -+ drng_store->drng_cb = new_cb; -+ -+ /* Reseed if previous LRNG security strength was insufficient */ -+ if (current_security_strength < lrng_security_strength()) -+ drng_store->force_reseed = true; -+ -+ /* Force oversampling seeding as we initialize DRNG */ -+ if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) -+ lrng_unset_fully_seeded(drng_store); -+ -+ if (lrng_state_min_seeded()) -+ lrng_set_entropy_thresh(lrng_get_seed_entropy_osr( -+ drng_store->fully_seeded)); -+ -+ old_cb->drng_dealloc(old_drng); -+ -+ pr_info("DRNG of NUMA node %d switched\n", node); -+ -+ mutex_unlock(&drng_store->lock); -+ return ret; -+} -+ -+/* -+ * Switch the existing DRNG and hash instances with new using the new crypto -+ * callbacks. The caller must hold the lrng_crypto_cb_update lock. -+ */ -+static int lrng_switch(const void *cb, -+ int(*switcher)(struct lrng_drng *drng_store, -+ const void *cb, int node)) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ struct lrng_drng *lrng_drng_pr = lrng_drng_pr_instance(); -+ int ret = 0; -+ -+ if (lrng_drng) { -+ u32 node; -+ -+ for_each_online_node(node) { -+ if (lrng_drng[node]) -+ ret |= switcher(lrng_drng[node], cb, node); -+ } -+ } else { -+ ret |= switcher(lrng_drng_init, cb, 0); -+ } -+ -+ ret |= switcher(lrng_drng_pr, cb, -1); -+ -+ return ret; -+} -+ -+/* -+ * lrng_set_drng_cb - Register new cryptographic callback functions for DRNG -+ * The registering implies that all old DRNG states are replaced with new -+ * DRNG states. -+ * -+ * drng_cb: Callback functions to be registered -- if NULL, use the default -+ * callbacks defined at compile time. -+ * -+ * Return: -+ * * 0 on success -+ * * < 0 on error -+ */ -+int lrng_set_drng_cb(const struct lrng_drng_cb *drng_cb) -+{ -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ int ret; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH_DRNG)) -+ return -EOPNOTSUPP; -+ -+ if (!drng_cb) -+ drng_cb = lrng_default_drng_cb; -+ -+ mutex_lock(&lrng_crypto_cb_update); -+ -+ /* -+ * If a callback other than the default is set, allow it only to be -+ * set back to the default callback. This ensures that multiple -+ * different callbacks can be registered at the same time. If a -+ * callback different from the current callback and the default -+ * callback shall be set, the current callback must be deregistered -+ * (e.g. the kernel module providing it must be unloaded) and the new -+ * implementation can be registered. -+ */ -+ if ((drng_cb != lrng_default_drng_cb) && -+ (lrng_drng_init->drng_cb != lrng_default_drng_cb)) { -+ pr_warn("disallow setting new DRNG callbacks, unload the old callbacks first!\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ ret = lrng_switch(drng_cb, lrng_drng_switch); -+ /* The switch may imply new entropy due to larger DRNG sec strength. */ -+ if (!ret) -+ lrng_es_add_entropy(); -+ -+out: -+ mutex_unlock(&lrng_crypto_cb_update); -+ return ret; -+} -+EXPORT_SYMBOL(lrng_set_drng_cb); -+ -+/* -+ * lrng_set_hash_cb - Register new cryptographic callback functions for hash -+ * The registering implies that all old hash states are replaced with new -+ * hash states. -+ * -+ * @hash_cb: Callback functions to be registered -- if NULL, use the default -+ * callbacks defined at compile time. -+ * -+ * Return: -+ * * 0 on success -+ * * < 0 on error -+ */ -+int lrng_set_hash_cb(const struct lrng_hash_cb *hash_cb) -+{ -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ int ret; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH_HASH)) -+ return -EOPNOTSUPP; -+ -+ if (!hash_cb) -+ hash_cb = lrng_default_hash_cb; -+ -+ mutex_lock(&lrng_crypto_cb_update); -+ -+ /* Comment from lrng_set_drng_cb applies. */ -+ if ((hash_cb != lrng_default_hash_cb) && -+ (lrng_drng_init->hash_cb != lrng_default_hash_cb)) { -+ pr_warn("disallow setting new hash callbacks, unload the old callbacks first!\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ ret = lrng_switch(hash_cb, lrng_hash_switch); -+ /* -+ * The switch may imply new entropy due to larger digest size. But -+ * it may also offer more room in the aux pool which means we ping -+ * any waiting entropy providers. -+ */ -+ if (!ret) { -+ lrng_es_add_entropy(); -+ lrng_writer_wakeup(); -+ } -+ -+out: -+ mutex_unlock(&lrng_crypto_cb_update); -+ return ret; -+} -+EXPORT_SYMBOL(lrng_set_hash_cb); -diff --git a/drivers/char/lrng/lrng_sysctl.c b/drivers/char/lrng/lrng_sysctl.c -new file mode 100644 -index 000000000000..ecdd96a842b4 ---- /dev/null -+++ b/drivers/char/lrng/lrng_sysctl.c -@@ -0,0 +1,140 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG sysctl interfaces -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_mgr.h" -+#include "lrng_sysctl.h" -+ -+/* -+ * This function is used to return both the bootid UUID, and random -+ * UUID. The difference is in whether table->data is NULL; if it is, -+ * then a new UUID is generated and returned to the user. -+ * -+ * If the user accesses this via the proc interface, the UUID will be -+ * returned as an ASCII string in the standard UUID format; if via the -+ * sysctl system call, as 16 bytes of binary data. -+ */ -+static int lrng_sysctl_do_uuid(struct ctl_table *table, int write, -+ void *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table fake_table; -+ unsigned char buf[64], tmp_uuid[16], *uuid; -+ -+ uuid = table->data; -+ if (!uuid) { -+ uuid = tmp_uuid; -+ generate_random_uuid(uuid); -+ } else { -+ static DEFINE_SPINLOCK(bootid_spinlock); -+ -+ spin_lock(&bootid_spinlock); -+ if (!uuid[8]) -+ generate_random_uuid(uuid); -+ spin_unlock(&bootid_spinlock); -+ } -+ -+ sprintf(buf, "%pU", uuid); -+ -+ fake_table.data = buf; -+ fake_table.maxlen = sizeof(buf); -+ -+ return proc_dostring(&fake_table, write, buffer, lenp, ppos); -+} -+ -+static int lrng_sysctl_do_entropy(struct ctl_table *table, int write, -+ void *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table fake_table; -+ int entropy_count = lrng_avail_entropy_aux(); -+ -+ fake_table.data = &entropy_count; -+ fake_table.maxlen = sizeof(entropy_count); -+ -+ return proc_dointvec(&fake_table, write, buffer, lenp, ppos); -+} -+ -+static int lrng_sysctl_do_poolsize(struct ctl_table *table, int write, -+ void *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table fake_table; -+ u32 entropy_count = lrng_es[lrng_ext_es_aux]->max_entropy(); -+ -+ fake_table.data = &entropy_count; -+ fake_table.maxlen = sizeof(entropy_count); -+ -+ return proc_dointvec(&fake_table, write, buffer, lenp, ppos); -+} -+ -+static int lrng_min_write_thresh; -+static int lrng_max_write_thresh = (LRNG_WRITE_WAKEUP_ENTROPY << 3); -+static char lrng_sysctl_bootid[16]; -+static int lrng_drng_reseed_max_min; -+ -+void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) -+{ -+ lrng_max_write_thresh = (int)new_digestsize; -+ /* Ensure that changes to the global variable are visible */ -+ mb(); -+} -+ -+static struct ctl_table random_table[] = { -+ { -+ .procname = "poolsize", -+ .maxlen = sizeof(int), -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_poolsize, -+ }, -+ { -+ .procname = "entropy_avail", -+ .maxlen = sizeof(int), -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_entropy, -+ }, -+ { -+ .procname = "write_wakeup_threshold", -+ .data = &lrng_write_wakeup_bits, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec_minmax, -+ .extra1 = &lrng_min_write_thresh, -+ .extra2 = &lrng_max_write_thresh, -+ }, -+ { -+ .procname = "boot_id", -+ .data = &lrng_sysctl_bootid, -+ .maxlen = 16, -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_uuid, -+ }, -+ { -+ .procname = "uuid", -+ .maxlen = 16, -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_uuid, -+ }, -+ { -+ .procname = "urandom_min_reseed_secs", -+ .data = &lrng_drng_reseed_max_time, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ .extra1 = &lrng_drng_reseed_max_min, -+ }, -+ { } -+}; -+ -+static int __init random_sysctls_init(void) -+{ -+ register_sysctl_init("kernel/random", random_table); -+ return 0; -+} -+device_initcall(random_sysctls_init); -diff --git a/drivers/char/lrng/lrng_sysctl.h b/drivers/char/lrng/lrng_sysctl.h -new file mode 100644 -index 000000000000..4b487e5077ed ---- /dev/null -+++ b/drivers/char/lrng/lrng_sysctl.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_SYSCTL_H -+#define _LRNG_SYSCTL_H -+ -+#ifdef CONFIG_LRNG_SYSCTL -+void lrng_sysctl_update_max_write_thresh(u32 new_digestsize); -+#else -+static inline void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) { } -+#endif -+ -+#endif /* _LRNG_SYSCTL_H */ -diff --git a/drivers/char/lrng/lrng_testing.c b/drivers/char/lrng/lrng_testing.c -new file mode 100644 -index 000000000000..101140085d81 ---- /dev/null -+++ b/drivers/char/lrng/lrng_testing.c -@@ -0,0 +1,901 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG testing interfaces to obtain raw entropy -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_drng_chacha20.h" -+#include "lrng_sha.h" -+#include "lrng_testing.h" -+ -+#if defined(CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY) || \ -+ defined(CONFIG_LRNG_RAW_SCHED_PID_ENTROPY) || \ -+ defined(CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY) || \ -+ defined(CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY) || \ -+ defined(CONFIG_LRNG_SCHED_PERF) -+#define LRNG_TESTING_USE_BUSYLOOP -+#endif -+ -+#ifdef CONFIG_LRNG_TESTING_RECORDING -+ -+#define LRNG_TESTING_RINGBUFFER_SIZE 1024 -+#define LRNG_TESTING_RINGBUFFER_MASK (LRNG_TESTING_RINGBUFFER_SIZE - 1) -+ -+struct lrng_testing { -+ u32 lrng_testing_rb[LRNG_TESTING_RINGBUFFER_SIZE]; -+ u32 rb_reader; -+ atomic_t rb_writer; -+ atomic_t lrng_testing_enabled; -+ spinlock_t lock; -+ wait_queue_head_t read_wait; -+}; -+ -+/*************************** Generic Data Handling ****************************/ -+ -+/* -+ * boot variable: -+ * 0 ==> No boot test, gathering of runtime data allowed -+ * 1 ==> Boot test enabled and ready for collecting data, gathering runtime -+ * data is disabled -+ * 2 ==> Boot test completed and disabled, gathering of runtime data is -+ * disabled -+ */ -+ -+static void lrng_testing_reset(struct lrng_testing *data) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&data->lock, flags); -+ data->rb_reader = 0; -+ atomic_set(&data->rb_writer, 0); -+ spin_unlock_irqrestore(&data->lock, flags); -+} -+ -+static void lrng_testing_init(struct lrng_testing *data, u32 boot) -+{ -+ /* -+ * The boot time testing implies we have a running test. If the -+ * caller wants to clear it, he has to unset the boot_test flag -+ * at runtime via sysfs to enable regular runtime testing -+ */ -+ if (boot) -+ return; -+ -+ lrng_testing_reset(data); -+ atomic_set(&data->lrng_testing_enabled, 1); -+ pr_warn("Enabling data collection\n"); -+} -+ -+static void lrng_testing_fini(struct lrng_testing *data, u32 boot) -+{ -+ /* If we have boot data, we do not reset yet to allow data to be read */ -+ if (boot) -+ return; -+ -+ atomic_set(&data->lrng_testing_enabled, 0); -+ lrng_testing_reset(data); -+ pr_warn("Disabling data collection\n"); -+} -+ -+static bool lrng_testing_store(struct lrng_testing *data, u32 value, -+ u32 *boot) -+{ -+ unsigned long flags; -+ -+ if (!atomic_read(&data->lrng_testing_enabled) && (*boot != 1)) -+ return false; -+ -+ spin_lock_irqsave(&data->lock, flags); -+ -+ /* -+ * Disable entropy testing for boot time testing after ring buffer -+ * is filled. -+ */ -+ if (*boot) { -+ if (((u32)atomic_read(&data->rb_writer)) > -+ LRNG_TESTING_RINGBUFFER_SIZE) { -+ *boot = 2; -+ pr_warn_once("One time data collection test disabled\n"); -+ spin_unlock_irqrestore(&data->lock, flags); -+ return false; -+ } -+ -+ if (atomic_read(&data->rb_writer) == 1) -+ pr_warn("One time data collection test enabled\n"); -+ } -+ -+ data->lrng_testing_rb[((u32)atomic_read(&data->rb_writer)) & -+ LRNG_TESTING_RINGBUFFER_MASK] = value; -+ atomic_inc(&data->rb_writer); -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+#ifndef LRNG_TESTING_USE_BUSYLOOP -+ if (wq_has_sleeper(&data->read_wait)) -+ wake_up_interruptible(&data->read_wait); -+#endif -+ -+ return true; -+} -+ -+static bool lrng_testing_have_data(struct lrng_testing *data) -+{ -+ return ((((u32)atomic_read(&data->rb_writer)) & -+ LRNG_TESTING_RINGBUFFER_MASK) != -+ (data->rb_reader & LRNG_TESTING_RINGBUFFER_MASK)); -+} -+ -+static int lrng_testing_reader(struct lrng_testing *data, u32 *boot, -+ u8 *outbuf, u32 outbuflen) -+{ -+ unsigned long flags; -+ int collected_data = 0; -+ -+ lrng_testing_init(data, *boot); -+ -+ while (outbuflen) { -+ u32 writer = (u32)atomic_read(&data->rb_writer); -+ -+ spin_lock_irqsave(&data->lock, flags); -+ -+ /* We have no data or reached the writer. */ -+ if (!writer || (writer == data->rb_reader)) { -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+ /* -+ * Now we gathered all boot data, enable regular data -+ * collection. -+ */ -+ if (*boot) { -+ *boot = 0; -+ goto out; -+ } -+ -+#ifdef LRNG_TESTING_USE_BUSYLOOP -+ while (!lrng_testing_have_data(data)) -+ ; -+#else -+ wait_event_interruptible(data->read_wait, -+ lrng_testing_have_data(data)); -+#endif -+ if (signal_pending(current)) { -+ collected_data = -ERESTARTSYS; -+ goto out; -+ } -+ -+ continue; -+ } -+ -+ /* We copy out word-wise */ -+ if (outbuflen < sizeof(u32)) { -+ spin_unlock_irqrestore(&data->lock, flags); -+ goto out; -+ } -+ -+ memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], -+ sizeof(u32)); -+ data->rb_reader++; -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+ outbuf += sizeof(u32); -+ outbuflen -= sizeof(u32); -+ collected_data += sizeof(u32); -+ } -+ -+out: -+ lrng_testing_fini(data, *boot); -+ return collected_data; -+} -+ -+static int lrng_testing_extract_user(struct file *file, char __user *buf, -+ size_t nbytes, loff_t *ppos, -+ int (*reader)(u8 *outbuf, u32 outbuflen)) -+{ -+ u8 *tmp, *tmp_aligned; -+ int ret = 0, large_request = (nbytes > 256); -+ -+ if (!nbytes) -+ return 0; -+ -+ /* -+ * The intention of this interface is for collecting at least -+ * 1000 samples due to the SP800-90B requirements. So, we make no -+ * effort in avoiding allocating more memory that actually needed -+ * by the user. Hence, we allocate sufficient memory to always hold -+ * that amount of data. -+ */ -+ tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); -+ if (!tmp) -+ return -ENOMEM; -+ -+ tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); -+ -+ while (nbytes) { -+ int i; -+ -+ if (large_request && need_resched()) { -+ if (signal_pending(current)) { -+ if (ret == 0) -+ ret = -ERESTARTSYS; -+ break; -+ } -+ schedule(); -+ } -+ -+ i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); -+ i = reader(tmp_aligned, i); -+ if (i <= 0) { -+ if (i < 0) -+ ret = i; -+ break; -+ } -+ if (copy_to_user(buf, tmp_aligned, i)) { -+ ret = -EFAULT; -+ break; -+ } -+ -+ nbytes -= i; -+ buf += i; -+ ret += i; -+ } -+ -+ kfree_sensitive(tmp); -+ -+ if (ret > 0) -+ *ppos += ret; -+ -+ return ret; -+} -+ -+#endif /* CONFIG_LRNG_TESTING_RECORDING */ -+ -+/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+ -+static u32 boot_raw_hires_test = 0; -+module_param(boot_raw_hires_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); -+ -+static struct lrng_testing lrng_raw_hires = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) -+}; -+ -+bool lrng_raw_hires_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); -+} -+ -+static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_hires_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_hires_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_hires_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+ -+/********************* Raw Jiffies Entropy Data Handling **********************/ -+ -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+ -+static u32 boot_raw_jiffies_test = 0; -+module_param(boot_raw_jiffies_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); -+ -+static struct lrng_testing lrng_raw_jiffies = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) -+}; -+ -+bool lrng_raw_jiffies_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_jiffies, value, -+ &boot_raw_jiffies_test); -+} -+ -+static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_jiffies_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_jiffies_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_jiffies_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+ -+/************************** Raw IRQ Data Handling ****************************/ -+ -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+ -+static u32 boot_raw_irq_test = 0; -+module_param(boot_raw_irq_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); -+ -+static struct lrng_testing lrng_raw_irq = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) -+}; -+ -+bool lrng_raw_irq_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); -+} -+ -+static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_irq_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_irq_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_irq_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+ -+/************************ Raw _RET_IP_ Data Handling **************************/ -+ -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+ -+static u32 boot_raw_retip_test = 0; -+module_param(boot_raw_retip_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); -+ -+static struct lrng_testing lrng_raw_retip = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) -+}; -+ -+bool lrng_raw_retip_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); -+} -+ -+static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_retip_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_retip_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_retip_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+ -+/********************** Raw IRQ register Data Handling ************************/ -+ -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+ -+static u32 boot_raw_regs_test = 0; -+module_param(boot_raw_regs_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); -+ -+static struct lrng_testing lrng_raw_regs = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) -+}; -+ -+bool lrng_raw_regs_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); -+} -+ -+static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_regs_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_regs_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_regs_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+ -+/********************** Raw Entropy Array Data Handling ***********************/ -+ -+#ifdef CONFIG_LRNG_RAW_ARRAY -+ -+static u32 boot_raw_array = 0; -+module_param(boot_raw_array, uint, 0644); -+MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); -+ -+static struct lrng_testing lrng_raw_array = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) -+}; -+ -+bool lrng_raw_array_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); -+} -+ -+static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_raw_array_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_array_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_array_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_array_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_ARRAY */ -+ -+/******************** Interrupt Performance Data Handling *********************/ -+ -+#ifdef CONFIG_LRNG_IRQ_PERF -+ -+static u32 boot_irq_perf = 0; -+module_param(boot_irq_perf, uint, 0644); -+MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); -+ -+static struct lrng_testing lrng_irq_perf = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) -+}; -+ -+bool lrng_perf_time(u32 start) -+{ -+ return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, -+ &boot_irq_perf); -+} -+ -+static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_irq_perf_reader); -+} -+ -+static const struct file_operations lrng_irq_perf_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_irq_perf_read, -+}; -+ -+#endif /* CONFIG_LRNG_IRQ_PERF */ -+ -+/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+ -+static u32 boot_raw_sched_hires_test = 0; -+module_param(boot_raw_sched_hires_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); -+ -+static struct lrng_testing lrng_raw_sched_hires = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), -+ .read_wait = -+ __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) -+}; -+ -+bool lrng_raw_sched_hires_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_hires, value, -+ &boot_raw_sched_hires_test); -+} -+ -+static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_hires, -+ &boot_raw_sched_hires_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_hires_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_hires_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_hires_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+ -+/******************** Interrupt Performance Data Handling *********************/ -+ -+#ifdef CONFIG_LRNG_SCHED_PERF -+ -+static u32 boot_sched_perf = 0; -+module_param(boot_sched_perf, uint, 0644); -+MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); -+ -+static struct lrng_testing lrng_sched_perf = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) -+}; -+ -+bool lrng_sched_perf_time(u32 start) -+{ -+ return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, -+ &boot_sched_perf); -+} -+ -+static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_sched_perf_reader); -+} -+ -+static const struct file_operations lrng_sched_perf_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_sched_perf_read, -+}; -+ -+#endif /* CONFIG_LRNG_SCHED_PERF */ -+ -+/*************** Raw Scheduler task_struct->pid Data Handling *****************/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+ -+static u32 boot_raw_sched_pid_test = 0; -+module_param(boot_raw_sched_pid_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); -+ -+static struct lrng_testing lrng_raw_sched_pid = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) -+}; -+ -+bool lrng_raw_sched_pid_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_pid, value, -+ &boot_raw_sched_pid_test); -+} -+ -+static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_pid, -+ &boot_raw_sched_pid_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_pid_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_pid_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_pid_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+ -+ -+/*********** Raw Scheduler task_struct->start_time Data Handling **************/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+ -+static u32 boot_raw_sched_starttime_test = 0; -+module_param(boot_raw_sched_starttime_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); -+ -+static struct lrng_testing lrng_raw_sched_starttime = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) -+}; -+ -+bool lrng_raw_sched_starttime_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_starttime, value, -+ &boot_raw_sched_starttime_test); -+} -+ -+static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_starttime, -+ &boot_raw_sched_starttime_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_starttime_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_starttime_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_starttime_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+ -+/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+ -+static u32 boot_raw_sched_nvcsw_test = 0; -+module_param(boot_raw_sched_nvcsw_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); -+ -+static struct lrng_testing lrng_raw_sched_nvcsw = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) -+}; -+ -+bool lrng_raw_sched_nvcsw_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_nvcsw, value, -+ &boot_raw_sched_nvcsw_test); -+} -+ -+static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_nvcsw, -+ &boot_raw_sched_nvcsw_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_nvcsw_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_nvcsw_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_nvcsw_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+ -+/*********************************** ACVT ************************************/ -+ -+#ifdef CONFIG_LRNG_ACVT_HASH -+ -+/* maximum amount of data to be hashed as defined by ACVP */ -+#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) -+ -+/* -+ * As we use static variables to store the data, it is clear that the -+ * test interface is only able to handle single threaded testing. This is -+ * considered to be sufficient for testing. If multi-threaded use of the -+ * ACVT test interface would be performed, the caller would get garbage -+ * but the kernel operation is unaffected by this. -+ */ -+static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] -+ __aligned(LRNG_KCAPI_ALIGN); -+static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); -+static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; -+ -+static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, -+ size_t nbytes, loff_t *ppos) -+{ -+ if (nbytes > LRNG_ACVT_MAX_SHA_MSG) -+ return -EINVAL; -+ -+ atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); -+ -+ return simple_write_to_buffer(lrng_acvt_hash_data, -+ LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); -+} -+ -+static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; -+ ssize_t ret; -+ -+ if (count > LRNG_ATOMIC_DIGEST_SIZE) -+ return -EINVAL; -+ -+ ret = hash_cb->hash_init(shash, NULL) ?: -+ hash_cb->hash_update(shash, lrng_acvt_hash_data, -+ atomic_read_u32(&lrng_acvt_hash_data_size)) ?: -+ hash_cb->hash_final(shash, lrng_acvt_hash_digest); -+ if (ret) -+ return ret; -+ -+ return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, -+ sizeof(lrng_acvt_hash_digest)); -+} -+ -+static const struct file_operations lrng_acvt_hash_fops = { -+ .owner = THIS_MODULE, -+ .open = simple_open, -+ .llseek = default_llseek, -+ .read = lrng_acvt_hash_read, -+ .write = lrng_acvt_hash_write, -+}; -+ -+#endif /* CONFIG_LRNG_ACVT_DRNG */ -+ -+/************************************************************************** -+ * Debugfs interface -+ **************************************************************************/ -+ -+static int __init lrng_raw_init(void) -+{ -+ struct dentry *lrng_raw_debugfs_root; -+ -+ lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_hires", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_hires_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_jiffies_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, -+ NULL, &lrng_raw_irq_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_retip", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_retip_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_regs", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_regs_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_ARRAY -+ debugfs_create_file_unsafe("lrng_raw_array", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_array_fops); -+#endif -+#ifdef CONFIG_LRNG_IRQ_PERF -+ debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, -+ NULL, &lrng_irq_perf_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, -+ lrng_raw_debugfs_root, -+ NULL, &lrng_raw_sched_hires_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_pid_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_starttime_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_nvcsw_fops); -+#endif -+#ifdef CONFIG_LRNG_SCHED_PERF -+ debugfs_create_file_unsafe("lrng_sched_perf", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_sched_perf_fops); -+#endif -+#ifdef CONFIG_LRNG_ACVT_HASH -+ debugfs_create_file_unsafe("lrng_acvt_hash", 0600, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_acvt_hash_fops); -+#endif -+ -+ return 0; -+} -+ -+module_init(lrng_raw_init); -diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h -new file mode 100644 -index 000000000000..672a7fca4595 ---- /dev/null -+++ b/drivers/char/lrng/lrng_testing.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_TESTING_H -+#define _LRNG_TESTING_H -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+bool lrng_raw_hires_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+bool lrng_raw_jiffies_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+bool lrng_raw_irq_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+bool lrng_raw_retip_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+bool lrng_raw_regs_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_ARRAY -+bool lrng_raw_array_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_ARRAY */ -+static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_ARRAY */ -+ -+#ifdef CONFIG_LRNG_IRQ_PERF -+bool lrng_perf_time(u32 start); -+#else /* CONFIG_LRNG_IRQ_PERF */ -+static inline bool lrng_perf_time(u32 start) { return false; } -+#endif /*CONFIG_LRNG_IRQ_PERF */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+bool lrng_raw_sched_hires_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+static inline bool -+lrng_raw_sched_hires_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+bool lrng_raw_sched_pid_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+static inline bool -+lrng_raw_sched_pid_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+bool lrng_raw_sched_starttime_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+static inline bool -+lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+bool lrng_raw_sched_nvcsw_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+static inline bool -+lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_SCHED_PERF -+bool lrng_sched_perf_time(u32 start); -+#else /* CONFIG_LRNG_SCHED_PERF */ -+static inline bool lrng_sched_perf_time(u32 start) { return false; } -+#endif /*CONFIG_LRNG_SCHED_PERF */ -+ -+#endif /* _LRNG_TESTING_H */ -diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h -index af5ad51d3eef..b12ae9bdebf4 100644 ---- a/include/crypto/drbg.h -+++ b/include/crypto/drbg.h -@@ -283,4 +283,11 @@ enum drbg_prefixes { - DRBG_PREFIX3 - }; - -+extern int drbg_alloc_state(struct drbg_state *drbg); -+extern void drbg_dealloc_state(struct drbg_state *drbg); -+extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, -+ bool *pr); -+extern const struct drbg_core drbg_cores[]; -+extern unsigned short drbg_sec_strength(drbg_flag_t flags); -+ - #endif /* _DRBG_H */ -diff --git a/crypto/jitterentropy.h b/include/crypto/internal/jitterentropy.h -similarity index 100% -rename from crypto/jitterentropy.h -rename to include/crypto/internal/jitterentropy.h -diff --git a/include/linux/lrng.h b/include/linux/lrng.h -new file mode 100644 -index 000000000000..c0d31a03d51f ---- /dev/null -+++ b/include/linux/lrng.h -@@ -0,0 +1,251 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_H -+#define _LRNG_H -+ -+#include -+#include -+#include -+#include -+ -+/* -+ * struct lrng_drng_cb - cryptographic callback functions defining a DRNG -+ * @drng_name Name of DRNG -+ * @drng_alloc: Allocate DRNG -- the provided integer should be used for -+ * sanity checks. -+ * return: allocated data structure or PTR_ERR on error -+ * @drng_dealloc: Deallocate DRNG -+ * @drng_seed: Seed the DRNG with data of arbitrary length drng: is -+ * pointer to data structure allocated with drng_alloc -+ * return: >= 0 on success, < 0 on error -+ * @drng_generate: Generate random numbers from the DRNG with arbitrary -+ * length -+ */ -+struct lrng_drng_cb { -+ const char *(*drng_name)(void); -+ void *(*drng_alloc)(u32 sec_strength); -+ void (*drng_dealloc)(void *drng); -+ int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); -+ int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); -+}; -+ -+/* -+ * struct lrng_hash_cb - cryptographic callback functions defining a hash -+ * @hash_name Name of Hash used for reading entropy pool arbitrary -+ * length -+ * @hash_alloc: Allocate the hash for reading the entropy pool -+ * return: allocated data structure (NULL is success too) -+ * or ERR_PTR on error -+ * @hash_dealloc: Deallocate Hash -+ * @hash_digestsize: Return the digestsize for the used hash to read out -+ * entropy pool -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: size of digest of hash in bytes -+ * @hash_init: Initialize hash -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_update: Update hash operation -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_final Final hash operation -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_desc_zero Zeroization of hash state buffer -+ * -+ * Assumptions: -+ * -+ * 1. Hash operation will not sleep -+ * 2. The hash' volatile state information is provided with *shash by caller. -+ */ -+struct lrng_hash_cb { -+ const char *(*hash_name)(void); -+ void *(*hash_alloc)(void); -+ void (*hash_dealloc)(void *hash); -+ u32 (*hash_digestsize)(void *hash); -+ int (*hash_init)(struct shash_desc *shash, void *hash); -+ int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, -+ u32 inbuflen); -+ int (*hash_final)(struct shash_desc *shash, u8 *digest); -+ void (*hash_desc_zero)(struct shash_desc *shash); -+}; -+ -+/* Register cryptographic backend */ -+#ifdef CONFIG_LRNG_SWITCH -+int lrng_set_drng_cb(const struct lrng_drng_cb *cb); -+int lrng_set_hash_cb(const struct lrng_hash_cb *cb); -+#else /* CONFIG_LRNG_SWITCH */ -+static inline int -+lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } -+static inline int -+lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } -+#endif /* CONFIG_LRNG_SWITCH */ -+ -+/* Callback to feed events to the scheduler entropy source */ -+#ifdef CONFIG_LRNG_SCHED -+extern void add_sched_randomness(const struct task_struct *p, int cpu); -+#else -+static inline void -+add_sched_randomness(const struct task_struct *p, int cpu) { } -+#endif -+ -+/* -+ * lrng_get_random_bytes() - Provider of cryptographic strong random numbers -+ * for kernel-internal usage. -+ * -+ * This function is appropriate for in-kernel use cases operating in atomic -+ * contexts. It will always use the ChaCha20 DRNG and it may be the case that -+ * it is not fully seeded when being used. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+void lrng_get_random_bytes(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_random_bytes_full() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from a fully initialized LRNG. -+ * -+ * This function will always return random numbers from a fully seeded and -+ * fully initialized LRNG. -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG -+void lrng_get_random_bytes_full(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_random_bytes_min() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from at least a minimally seeded -+ * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C -+ * oversampling applied in FIPS mode is not applied yet). -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG -+void lrng_get_random_bytes_min(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_random_bytes_pr() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from a fully initialized LRNG and -+ * requiring a reseed from the entropy sources before. -+ * -+ * This function will always return random numbers from a fully seeded and -+ * fully initialized LRNG. -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * This call only returns no more data than entropy was pulled from the -+ * entropy sources. Thus, it is likely that this call returns less data -+ * than requested by the caller. Also, the caller MUST be prepared that this -+ * call returns 0 bytes, i.e. it did not generate data. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ * -+ * @return: positive number indicates amount of generated bytes, < 0 on error -+ */ -+#ifdef CONFIG_LRNG -+int lrng_get_random_bytes_pr(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_seed() - Fill buffer with data from entropy sources -+ * -+ * This call allows accessing the entropy sources directly and fill the buffer -+ * with data from all available entropy sources. This filled buffer is -+ * identical to the temporary seed buffer used by the LRNG to seed its DRNGs. -+ * -+ * The call is to allows users to seed their DRNG directly from the entropy -+ * sources in case the caller does not want to use the LRNG's DRNGs. This -+ * buffer can be directly used to seed the caller's DRNG from. -+ * -+ * The call blocks as long as one LRNG DRNG is not yet fully seeded. If -+ * LRNG_GET_SEED_NONBLOCK is specified, it does not block in this case, but -+ * returns with an error. -+ * -+ * Considering SP800-90C, there is a differentiation between the seeding -+ * requirements during instantiating a DRNG and at runtime of the DRNG. When -+ * specifying LRNG_GET_SEED_FULLY_SEEDED the caller indicates the DRNG was -+ * already fully seeded and the regular amount of entropy is requested. -+ * Otherwise, the LRNG will obtain the entropy rate required for initial -+ * seeding. The following minimum entropy rates will be obtained: -+ * -+ * * FIPS mode: -+ * * Initial seeding: 384 bits of entropy -+ * * Runtime seeding: 256 bits of entropy -+ * * Non-FIPS mode: -+ * * 128 bits of entropy in any case -+ * -+ * Albeit these are minimum entropy rates, the LRNG tries to request the -+ * given amount of entropy from each entropy source individually. If the -+ * minimum amount of entropy cannot be obtained collectively by all entropy -+ * sources, the LRNG will not fill the buffer. -+ * -+ * The return data in buf is structurally equivalent to the following -+ * definition: -+ * -+ * struct { -+ * u64 seedlen; -+ * u64 entropy_rate; -+ * struct entropy_buf seed; -+ * } __attribute((__packed__)); -+ * -+ * As struct entropy_buf is not known outsize of the LRNG, the LRNG fills -+ * seedlen first with the size of struct entropy_buf. If the caller-provided -+ * buffer buf is smaller than u64, then -EINVAL is returned -+ * and buf is not touched. If it is u64 or larger but smaller -+ * than the size of the structure above, -EMSGSIZE is returned and seedlen -+ * is filled with the size of the buffer. Finally, if buf is large -+ * enough to hold all data, it is filled with the seed data and the seedlen -+ * is set to sizeof(struct entropy_buf). The entropy rate is returned with -+ * the variable entropy_rate and provides the value in bits. -+ * -+ * The seed buffer is the data that should be handed to the caller's DRNG as -+ * seed data. -+ * -+ * @buf [out] Buffer to be filled with data from the entropy sources - note, the -+ * buffer is marked as u64 to ensure it is aligned to 64 bits. -+ * @nbytes [in] Size of the buffer allocated by the caller - this value -+ * provides size of @param buf in bytes. -+ * @flags [in] Flags field to adjust the behavior -+ * -+ * @return -EINVAL or -EMSGSIZE indicating the buffer is too small, -EAGAIN when -+ * the call would block, but NONBLOCK is specified, > 0 the size of -+ * the filled buffer. -+ */ -+#ifdef CONFIG_LRNG -+enum lrng_get_seed_flags { -+ LRNG_GET_SEED_NONBLOCK = 0x0001, /**< Do not block the call */ -+ LRNG_GET_SEED_FULLY_SEEDED = 0x0002, /**< DRNG is fully seeded */ -+}; -+ -+ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags); -+#endif -+ -+#endif /* _LRNG_H */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 265944f14948..e9f0ec6c9aeb 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -6,6 +6,7 @@ - * - * Copyright (C) 1991-2002 Linus Torvalds - */ -+#include - #include - #include - #include -@@ -3586,6 +3587,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) - { - struct rq *rq; - -+ add_sched_randomness(p, cpu); -+ - if (!schedstat_enabled()) - return; - --- -2.37.3 - diff --git a/6.0/testing/0006-mm-cleanup.patch b/6.0/0009-mm-cleanup.patch similarity index 98% rename from 6.0/testing/0006-mm-cleanup.patch rename to 6.0/0009-mm-cleanup.patch index 81738cc8..abd6edb2 100644 --- a/6.0/testing/0006-mm-cleanup.patch +++ b/6.0/0009-mm-cleanup.patch @@ -1,7 +1,7 @@ -From ee83b3c6bc5cedb0d989a4f52368fd31c0736f28 Mon Sep 17 00:00:00 2001 +From 79eeeac092d265211e4f6ce60f69ad549d8a201c Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 Sep 2022 00:18:41 +0200 -Subject: [PATCH 06/13] mm-cleanup +Subject: [PATCH 09/16] mm-cleanup Signed-off-by: Peter Jung --- @@ -87,7 +87,7 @@ index 9ac0e02e2238..9c3d3474033f 100644 extern void __kernel_poison_pages(struct page *page, int numpages); extern void __kernel_unpoison_pages(struct page *page, int numpages); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index 1543001feba9..21d39f152d0c 100644 +index e5c240eed6af..b4c82726816b 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -577,13 +577,6 @@ enum zone_watermarks { @@ -104,7 +104,7 @@ index 1543001feba9..21d39f152d0c 100644 #define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost) #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost) #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost) -@@ -1241,11 +1234,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) +@@ -1244,11 +1237,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) return pgdat->node_start_pfn + pgdat->node_spanned_pages; } diff --git a/6.0/0010-per-VMA-locks-proposal.patch b/6.0/0010-per-VMA-locks-proposal.patch deleted file mode 100644 index 7d615c03..00000000 --- a/6.0/0010-per-VMA-locks-proposal.patch +++ /dev/null @@ -1,1442 +0,0 @@ -From 87a1aea3e04d22f00c7eca68492ba64e6cad4efd Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 12 Sep 2022 11:33:01 +0200 -Subject: [PATCH 10/13] per-VMA locks proposal - -Signed-off-by: Peter Jung ---- - arch/arm64/Kconfig | 1 + - arch/arm64/mm/fault.c | 36 +++++++++ - arch/powerpc/mm/fault.c | 41 ++++++++++ - arch/powerpc/platforms/powernv/Kconfig | 1 + - arch/powerpc/platforms/pseries/Kconfig | 1 + - arch/x86/Kconfig | 1 + - arch/x86/mm/fault.c | 36 +++++++++ - drivers/gpu/drm/i915/i915_gpu_error.c | 4 +- - fs/proc/task_mmu.c | 1 + - fs/userfaultfd.c | 6 ++ - include/linux/mm.h | 104 ++++++++++++++++++++++++- - include/linux/mm_types.h | 33 ++++++-- - include/linux/mmap_lock.h | 37 ++++++--- - include/linux/vm_event_item.h | 6 ++ - include/linux/vmstat.h | 6 ++ - kernel/fork.c | 75 +++++++++++++++++- - mm/Kconfig | 13 ++++ - mm/Kconfig.debug | 8 ++ - mm/init-mm.c | 6 ++ - mm/internal.h | 4 +- - mm/khugepaged.c | 1 + - mm/madvise.c | 1 + - mm/memory.c | 82 ++++++++++++++++--- - mm/mempolicy.c | 6 +- - mm/mlock.c | 2 + - mm/mmap.c | 60 ++++++++++---- - mm/mprotect.c | 1 + - mm/mremap.c | 1 + - mm/nommu.c | 2 + - mm/oom_kill.c | 3 +- - mm/vmstat.c | 6 ++ - 31 files changed, 531 insertions(+), 54 deletions(-) - -diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig -index 1ce7685ad5de..fb11380b49dd 100644 ---- a/arch/arm64/Kconfig -+++ b/arch/arm64/Kconfig -@@ -93,6 +93,7 @@ config ARM64 - select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 - select ARCH_SUPPORTS_NUMA_BALANCING - select ARCH_SUPPORTS_PAGE_TABLE_CHECK -+ select ARCH_SUPPORTS_PER_VMA_LOCK - select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT - select ARCH_WANT_DEFAULT_BPF_JIT - select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT -diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c -index c33f1fad2745..f05ce40ff32b 100644 ---- a/arch/arm64/mm/fault.c -+++ b/arch/arm64/mm/fault.c -@@ -525,6 +525,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, - unsigned long vm_flags; - unsigned int mm_flags = FAULT_FLAG_DEFAULT; - unsigned long addr = untagged_addr(far); -+#ifdef CONFIG_PER_VMA_LOCK -+ struct vm_area_struct *vma; -+#endif - - if (kprobe_page_fault(regs, esr)) - return 0; -@@ -575,6 +578,36 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, - - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); - -+#ifdef CONFIG_PER_VMA_LOCK -+ if (!(mm_flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) -+ goto lock_mmap; -+ -+ vma = find_and_lock_anon_vma(mm, addr); -+ if (!vma) -+ goto lock_mmap; -+ -+ if (!(vma->vm_flags & vm_flags)) { -+ vma_read_unlock(vma); -+ goto lock_mmap; -+ } -+ fault = handle_mm_fault(vma, addr & PAGE_MASK, -+ mm_flags | FAULT_FLAG_VMA_LOCK, regs); -+ vma_read_unlock(vma); -+ -+ if (!(fault & VM_FAULT_RETRY)) { -+ count_vm_vma_lock_event(VMA_LOCK_SUCCESS); -+ goto done; -+ } -+ count_vm_vma_lock_event(VMA_LOCK_RETRY); -+ -+ /* Quick path to respond to signals */ -+ if (fault_signal_pending(fault, regs)) { -+ if (!user_mode(regs)) -+ goto no_context; -+ return 0; -+ } -+lock_mmap: -+#endif /* CONFIG_PER_VMA_LOCK */ - /* - * As per x86, we may deadlock here. However, since the kernel only - * validly references user space from well defined areas of the code, -@@ -618,6 +651,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, - } - mmap_read_unlock(mm); - -+#ifdef CONFIG_PER_VMA_LOCK -+done: -+#endif - /* - * Handle the "normal" (no error) case first. - */ -diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c -index 014005428687..c92bdfcd1796 100644 ---- a/arch/powerpc/mm/fault.c -+++ b/arch/powerpc/mm/fault.c -@@ -450,6 +450,44 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, - if (is_exec) - flags |= FAULT_FLAG_INSTRUCTION; - -+#ifdef CONFIG_PER_VMA_LOCK -+ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) -+ goto lock_mmap; -+ -+ vma = find_and_lock_anon_vma(mm, address); -+ if (!vma) -+ goto lock_mmap; -+ -+ if (unlikely(access_pkey_error(is_write, is_exec, -+ (error_code & DSISR_KEYFAULT), vma))) { -+ int rc = bad_access_pkey(regs, address, vma); -+ -+ vma_read_unlock(vma); -+ return rc; -+ } -+ -+ if (unlikely(access_error(is_write, is_exec, vma))) { -+ int rc = bad_access(regs, address); -+ -+ vma_read_unlock(vma); -+ return rc; -+ } -+ -+ fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); -+ vma_read_unlock(vma); -+ -+ if (!(fault & VM_FAULT_RETRY)) { -+ count_vm_vma_lock_event(VMA_LOCK_SUCCESS); -+ goto done; -+ } -+ count_vm_vma_lock_event(VMA_LOCK_RETRY); -+ -+ if (fault_signal_pending(fault, regs)) -+ return user_mode(regs) ? 0 : SIGBUS; -+ -+lock_mmap: -+#endif /* CONFIG_PER_VMA_LOCK */ -+ - /* When running in the kernel we expect faults to occur only to - * addresses in user space. All other faults represent errors in the - * kernel and should generate an OOPS. Unfortunately, in the case of an -@@ -526,6 +564,9 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, - - mmap_read_unlock(current->mm); - -+#ifdef CONFIG_PER_VMA_LOCK -+done: -+#endif - if (unlikely(fault & VM_FAULT_ERROR)) - return mm_fault_error(regs, address, fault); - -diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig -index ae248a161b43..70a46acc70d6 100644 ---- a/arch/powerpc/platforms/powernv/Kconfig -+++ b/arch/powerpc/platforms/powernv/Kconfig -@@ -16,6 +16,7 @@ config PPC_POWERNV - select PPC_DOORBELL - select MMU_NOTIFIER - select FORCE_SMP -+ select ARCH_SUPPORTS_PER_VMA_LOCK - default y - - config OPAL_PRD -diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig -index fb6499977f99..7d13a2de3475 100644 ---- a/arch/powerpc/platforms/pseries/Kconfig -+++ b/arch/powerpc/platforms/pseries/Kconfig -@@ -21,6 +21,7 @@ config PPC_PSERIES - select HOTPLUG_CPU - select FORCE_SMP - select SWIOTLB -+ select ARCH_SUPPORTS_PER_VMA_LOCK - default y - - config PARAVIRT_SPINLOCKS -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index 674d694a665e..fa41ba3e880f 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -27,6 +27,7 @@ config X86_64 - # Options that are inherently 64-bit kernel only: - select ARCH_HAS_GIGANTIC_PAGE - select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 -+ select ARCH_SUPPORTS_PER_VMA_LOCK - select ARCH_USE_CMPXCHG_LOCKREF - select HAVE_ARCH_SOFT_DIRTY - select MODULES_USE_ELF_RELA -diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c -index c4bbd8c66134..81458c2cf4d9 100644 ---- a/arch/x86/mm/fault.c -+++ b/arch/x86/mm/fault.c -@@ -19,6 +19,7 @@ - #include /* faulthandler_disabled() */ - #include /* efi_crash_gracefully_on_page_fault()*/ - #include -+#include /* find_and_lock_vma() */ - - #include /* boot_cpu_has, ... */ - #include /* dotraplinkage, ... */ -@@ -1323,6 +1324,38 @@ void do_user_addr_fault(struct pt_regs *regs, - } - #endif - -+#ifdef CONFIG_PER_VMA_LOCK -+ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) -+ goto lock_mmap; -+ -+ vma = find_and_lock_anon_vma(mm, address); -+ if (!vma) -+ goto lock_mmap; -+ -+ if (unlikely(access_error(error_code, vma))) { -+ vma_read_unlock(vma); -+ goto lock_mmap; -+ } -+ fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); -+ vma_read_unlock(vma); -+ -+ if (!(fault & VM_FAULT_RETRY)) { -+ count_vm_vma_lock_event(VMA_LOCK_SUCCESS); -+ goto done; -+ } -+ count_vm_vma_lock_event(VMA_LOCK_RETRY); -+ -+ /* Quick path to respond to signals */ -+ if (fault_signal_pending(fault, regs)) { -+ if (!user_mode(regs)) -+ kernelmode_fixup_or_oops(regs, error_code, address, -+ SIGBUS, BUS_ADRERR, -+ ARCH_DEFAULT_PKEY); -+ return; -+ } -+lock_mmap: -+#endif /* CONFIG_PER_VMA_LOCK */ -+ - /* - * Kernel-mode access to the user address space should only occur - * on well-defined single instructions listed in the exception -@@ -1423,6 +1456,9 @@ void do_user_addr_fault(struct pt_regs *regs, - } - - mmap_read_unlock(mm); -+#ifdef CONFIG_PER_VMA_LOCK -+done: -+#endif - if (likely(!(fault & VM_FAULT_ERROR))) - return; - -diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c -index 32e92651ef7c..fc94985c95c8 100644 ---- a/drivers/gpu/drm/i915/i915_gpu_error.c -+++ b/drivers/gpu/drm/i915/i915_gpu_error.c -@@ -507,7 +507,7 @@ static void error_print_context(struct drm_i915_error_state_buf *m, - } - - static struct i915_vma_coredump * --__find_vma(struct i915_vma_coredump *vma, const char *name) -+__i915_find_vma(struct i915_vma_coredump *vma, const char *name) - { - while (vma) { - if (strcmp(vma->name, name) == 0) -@@ -521,7 +521,7 @@ __find_vma(struct i915_vma_coredump *vma, const char *name) - struct i915_vma_coredump * - intel_gpu_error_find_batch(const struct intel_engine_coredump *ee) - { -- return __find_vma(ee->vma, "batch"); -+ return __i915_find_vma(ee->vma, "batch"); - } - - static void error_print_engine(struct drm_i915_error_state_buf *m, -diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c -index 4e0023643f8b..ceffa5c2c650 100644 ---- a/fs/proc/task_mmu.c -+++ b/fs/proc/task_mmu.c -@@ -1285,6 +1285,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (!(vma->vm_flags & VM_SOFTDIRTY)) - continue; -+ vma_mark_locked(vma); - vma->vm_flags &= ~VM_SOFTDIRTY; - vma_set_page_prot(vma); - } -diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c -index 175de70e3adf..fe557b3d1c07 100644 ---- a/fs/userfaultfd.c -+++ b/fs/userfaultfd.c -@@ -620,6 +620,7 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, - mmap_write_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) - if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { -+ vma_mark_locked(vma); - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - } -@@ -653,6 +654,7 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) - - octx = vma->vm_userfaultfd_ctx.ctx; - if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) { -+ vma_mark_locked(vma); - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - return 0; -@@ -734,6 +736,7 @@ void mremap_userfaultfd_prep(struct vm_area_struct *vma, - atomic_inc(&ctx->mmap_changing); - } else { - /* Drop uffd context if remap feature not enabled */ -+ vma_mark_locked(vma); - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - } -@@ -891,6 +894,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) - vma = prev; - else - prev = vma; -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - } -@@ -1449,6 +1453,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - * the next vma was merged into the current one and - * the current one has not been updated yet. - */ -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx.ctx = ctx; - -@@ -1630,6 +1635,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - * the next vma was merged into the current one and - * the current one has not been updated yet. - */ -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 6c61497f0167..f15e26670abf 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -249,6 +249,7 @@ void setup_initial_init_mm(void *start_code, void *end_code, - struct vm_area_struct *vm_area_alloc(struct mm_struct *); - struct vm_area_struct *vm_area_dup(struct vm_area_struct *); - void vm_area_free(struct vm_area_struct *); -+void drain_free_vmas(struct mm_struct *mm); - - #ifndef CONFIG_MMU - extern struct rb_root nommu_region_tree; -@@ -466,7 +467,8 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags) - { FAULT_FLAG_USER, "USER" }, \ - { FAULT_FLAG_REMOTE, "REMOTE" }, \ - { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ -- { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" } -+ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ -+ { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } - - /* - * vm_fault is filled by the pagefault handler and passed to the vma's -@@ -611,6 +613,94 @@ struct vm_operations_struct { - unsigned long addr); - }; - -+#ifdef CONFIG_PER_VMA_LOCK -+static inline void vma_init_lock(struct vm_area_struct *vma) -+{ -+ init_rwsem(&vma->lock); -+ vma->vm_lock_seq = -1; -+} -+ -+static inline void vma_mark_locked(struct vm_area_struct *vma) -+{ -+ int mm_lock_seq; -+ -+ mmap_assert_write_locked(vma->vm_mm); -+ -+ /* -+ * current task is holding mmap_write_lock, both vma->vm_lock_seq and -+ * mm->mm_lock_seq can't be concurrently modified. -+ */ -+ mm_lock_seq = READ_ONCE(vma->vm_mm->mm_lock_seq); -+ if (vma->vm_lock_seq == mm_lock_seq) -+ return; -+ -+ down_write(&vma->lock); -+ vma->vm_lock_seq = mm_lock_seq; -+ up_write(&vma->lock); -+} -+ -+static inline bool vma_read_trylock(struct vm_area_struct *vma) -+{ -+ if (unlikely(down_read_trylock(&vma->lock) == 0)) -+ return false; -+ -+ /* -+ * Overflow might produce false locked result but it's not critical. -+ * False unlocked result is critical but is impossible because we -+ * modify and check vma->vm_lock_seq under vma->lock protection and -+ * mm->mm_lock_seq modification invalidates all existing locks. -+ */ -+ if (vma->vm_lock_seq == READ_ONCE(vma->vm_mm->mm_lock_seq)) { -+ up_read(&vma->lock); -+ return false; -+ } -+ return true; -+} -+ -+static inline void vma_read_unlock(struct vm_area_struct *vma) -+{ -+ up_read(&vma->lock); -+} -+ -+static inline void vma_assert_locked(struct vm_area_struct *vma) -+{ -+ lockdep_assert_held(&vma->lock); -+ VM_BUG_ON_VMA(!rwsem_is_locked(&vma->lock), vma); -+} -+ -+static inline void vma_assert_write_locked(struct vm_area_struct *vma, int pos) -+{ -+ mmap_assert_write_locked(vma->vm_mm); -+ /* -+ * current task is holding mmap_write_lock, both vma->vm_lock_seq and -+ * mm->mm_lock_seq can't be concurrently modified. -+ */ -+ VM_BUG_ON_VMA(vma->vm_lock_seq != READ_ONCE(vma->vm_mm->mm_lock_seq), vma); -+} -+ -+static inline void vma_assert_no_reader(struct vm_area_struct *vma) -+{ -+ VM_BUG_ON_VMA(rwsem_is_locked(&vma->lock) && -+ vma->vm_lock_seq != READ_ONCE(vma->vm_mm->mm_lock_seq), -+ vma); -+} -+ -+struct vm_area_struct *find_and_lock_anon_vma(struct mm_struct *mm, -+ unsigned long address); -+ -+#else /* CONFIG_PER_VMA_LOCK */ -+ -+static inline void vma_init_lock(struct vm_area_struct *vma) {} -+static inline void vma_mark_locked(struct vm_area_struct *vma) {} -+static inline bool vma_read_trylock(struct vm_area_struct *vma) -+ { return false; } -+static inline void vma_read_unlock(struct vm_area_struct *vma) {} -+static inline void vma_assert_locked(struct vm_area_struct *vma) {} -+static inline void vma_assert_write_locked(struct vm_area_struct *vma, int pos) {} -+static inline void vma_assert_no_reader(struct vm_area_struct *vma) {} -+ -+#endif /* CONFIG_PER_VMA_LOCK */ -+ - static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) - { - static const struct vm_operations_struct dummy_vm_ops = {}; -@@ -619,6 +709,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) - vma->vm_mm = mm; - vma->vm_ops = &dummy_vm_ops; - INIT_LIST_HEAD(&vma->anon_vma_chain); -+ vma_init_lock(vma); - } - - static inline void vma_set_anonymous(struct vm_area_struct *vma) -@@ -1801,7 +1892,7 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, - void zap_page_range(struct vm_area_struct *vma, unsigned long address, - unsigned long size); - void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, -- unsigned long start, unsigned long end); -+ unsigned long start, unsigned long end, bool lock_vma); - - struct mmu_notifier_range; - -@@ -2717,7 +2808,14 @@ extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); - #endif - - /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ --extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); -+extern struct vm_area_struct *__find_vma(struct mm_struct *mm, unsigned long addr); -+static inline -+struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) -+{ -+ mmap_assert_locked(mm); -+ return __find_vma(mm, addr); -+} -+ - extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, - struct vm_area_struct **pprev); - -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 15afe5e0c4c2..3a4b9b5e534d 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -403,12 +403,22 @@ struct anon_vma_name { - struct vm_area_struct { - /* The first cache line has the info for VMA tree walking. */ - -- unsigned long vm_start; /* Our start address within vm_mm. */ -- unsigned long vm_end; /* The first byte after our end address -- within vm_mm. */ -+ union { -+ struct { -+ /* VMA covers [vm_start; vm_end) addresses within mm */ -+ unsigned long vm_start, vm_end; - -- /* linked list of VM areas per task, sorted by address */ -- struct vm_area_struct *vm_next, *vm_prev; -+ /* linked list of VMAs per task, sorted by address */ -+ struct vm_area_struct *vm_next, *vm_prev; -+ }; -+#ifdef CONFIG_PER_VMA_LOCK -+ struct { -+ struct list_head vm_free_list; -+ /* Used for deferred freeing. */ -+ struct rcu_head vm_rcu; -+ }; -+#endif -+ }; - - struct rb_node vm_rb; - -@@ -480,6 +490,10 @@ struct vm_area_struct { - struct mempolicy *vm_policy; /* NUMA policy for the VMA */ - #endif - struct vm_userfaultfd_ctx vm_userfaultfd_ctx; -+#ifdef CONFIG_PER_VMA_LOCK -+ struct rw_semaphore lock; -+ int vm_lock_seq; -+#endif - } __randomize_layout; - - struct kioctx_table; -@@ -561,6 +575,14 @@ struct mm_struct { - * init_mm.mmlist, and are protected - * by mmlist_lock - */ -+#ifdef CONFIG_PER_VMA_LOCK -+ int mm_lock_seq; -+ struct { -+ struct list_head head; -+ spinlock_t lock; -+ int size; -+ } vma_free_list; -+#endif - - - unsigned long hiwater_rss; /* High-watermark of RSS usage */ -@@ -954,6 +976,7 @@ enum fault_flag { - FAULT_FLAG_INTERRUPTIBLE = 1 << 9, - FAULT_FLAG_UNSHARE = 1 << 10, - FAULT_FLAG_ORIG_PTE_VALID = 1 << 11, -+ FAULT_FLAG_VMA_LOCK = 1 << 12, - }; - - typedef unsigned int __bitwise zap_flags_t; -diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h -index 96e113e23d04..a391ae226564 100644 ---- a/include/linux/mmap_lock.h -+++ b/include/linux/mmap_lock.h -@@ -60,6 +60,29 @@ static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) - - #endif /* CONFIG_TRACING */ - -+static inline void mmap_assert_locked(struct mm_struct *mm) -+{ -+ lockdep_assert_held(&mm->mmap_lock); -+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); -+} -+ -+static inline void mmap_assert_write_locked(struct mm_struct *mm) -+{ -+ lockdep_assert_held_write(&mm->mmap_lock); -+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); -+} -+ -+#ifdef CONFIG_PER_VMA_LOCK -+static inline void vma_mark_unlocked_all(struct mm_struct *mm) -+{ -+ mmap_assert_write_locked(mm); -+ /* No races during update due to exclusive mmap_lock being held */ -+ WRITE_ONCE(mm->mm_lock_seq, mm->mm_lock_seq + 1); -+} -+#else -+static inline void vma_mark_unlocked_all(struct mm_struct *mm) {} -+#endif -+ - static inline void mmap_init_lock(struct mm_struct *mm) - { - init_rwsem(&mm->mmap_lock); -@@ -102,12 +125,14 @@ static inline bool mmap_write_trylock(struct mm_struct *mm) - static inline void mmap_write_unlock(struct mm_struct *mm) - { - __mmap_lock_trace_released(mm, true); -+ vma_mark_unlocked_all(mm); - up_write(&mm->mmap_lock); - } - - static inline void mmap_write_downgrade(struct mm_struct *mm) - { - __mmap_lock_trace_acquire_returned(mm, false, true); -+ vma_mark_unlocked_all(mm); - downgrade_write(&mm->mmap_lock); - } - -@@ -150,18 +175,6 @@ static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) - up_read_non_owner(&mm->mmap_lock); - } - --static inline void mmap_assert_locked(struct mm_struct *mm) --{ -- lockdep_assert_held(&mm->mmap_lock); -- VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); --} -- --static inline void mmap_assert_write_locked(struct mm_struct *mm) --{ -- lockdep_assert_held_write(&mm->mmap_lock); -- VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); --} -- - static inline int mmap_lock_is_contended(struct mm_struct *mm) - { - return rwsem_is_contended(&mm->mmap_lock); -diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h -index f3fc36cd2276..a325783ed05d 100644 ---- a/include/linux/vm_event_item.h -+++ b/include/linux/vm_event_item.h -@@ -150,6 +150,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, - #ifdef CONFIG_X86 - DIRECT_MAP_LEVEL2_SPLIT, - DIRECT_MAP_LEVEL3_SPLIT, -+#endif -+#ifdef CONFIG_PER_VMA_LOCK_STATS -+ VMA_LOCK_SUCCESS, -+ VMA_LOCK_ABORT, -+ VMA_LOCK_RETRY, -+ VMA_LOCK_MISS, - #endif - NR_VM_EVENT_ITEMS - }; -diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h -index bfe38869498d..0c2611899cfc 100644 ---- a/include/linux/vmstat.h -+++ b/include/linux/vmstat.h -@@ -131,6 +131,12 @@ static inline void vm_events_fold_cpu(int cpu) - #define count_vm_vmacache_event(x) do {} while (0) - #endif - -+#ifdef CONFIG_PER_VMA_LOCK_STATS -+#define count_vm_vma_lock_event(x) count_vm_event(x) -+#else -+#define count_vm_vma_lock_event(x) do {} while (0) -+#endif -+ - #define __count_zid_vm_events(item, zid, delta) \ - __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) - -diff --git a/kernel/fork.c b/kernel/fork.c -index b15aaae04403..ac29435e00ad 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -479,18 +479,83 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) - */ - *new = data_race(*orig); - INIT_LIST_HEAD(&new->anon_vma_chain); -+ vma_init_lock(new); - new->vm_next = new->vm_prev = NULL; - dup_anon_vma_name(orig, new); - } - return new; - } - -+#ifdef CONFIG_PER_VMA_LOCK -+static inline void __vm_area_free(struct vm_area_struct *vma) -+{ -+ /* The vma should either have no lock holders or be write-locked. */ -+ vma_assert_no_reader(vma); -+ kmem_cache_free(vm_area_cachep, vma); -+} -+ -+static void vma_free_rcu_callback(struct rcu_head *head) -+{ -+ struct vm_area_struct *first_vma; -+ struct vm_area_struct *vma, *vma2; -+ -+ first_vma = container_of(head, struct vm_area_struct, vm_rcu); -+ list_for_each_entry_safe(vma, vma2, &first_vma->vm_free_list, vm_free_list) -+ __vm_area_free(vma); -+ __vm_area_free(first_vma); -+} -+ -+void drain_free_vmas(struct mm_struct *mm) -+{ -+ struct vm_area_struct *first_vma; -+ LIST_HEAD(to_destroy); -+ -+ spin_lock(&mm->vma_free_list.lock); -+ list_splice_init(&mm->vma_free_list.head, &to_destroy); -+ mm->vma_free_list.size = 0; -+ spin_unlock(&mm->vma_free_list.lock); -+ -+ if (list_empty(&to_destroy)) -+ return; -+ -+ first_vma = list_first_entry(&to_destroy, struct vm_area_struct, vm_free_list); -+ /* Remove the head which is allocated on the stack */ -+ list_del(&to_destroy); -+ -+ call_rcu(&first_vma->vm_rcu, vma_free_rcu_callback); -+} -+ -+#define VM_AREA_FREE_LIST_MAX 32 -+ -+void vm_area_free(struct vm_area_struct *vma) -+{ -+ struct mm_struct *mm = vma->vm_mm; -+ bool drain; -+ -+ free_anon_vma_name(vma); -+ -+ spin_lock(&mm->vma_free_list.lock); -+ list_add(&vma->vm_free_list, &mm->vma_free_list.head); -+ mm->vma_free_list.size++; -+ drain = mm->vma_free_list.size > VM_AREA_FREE_LIST_MAX; -+ spin_unlock(&mm->vma_free_list.lock); -+ -+ if (drain) -+ drain_free_vmas(mm); -+} -+ -+#else /* CONFIG_PER_VMA_LOCK */ -+ -+void drain_free_vmas(struct mm_struct *mm) {} -+ - void vm_area_free(struct vm_area_struct *vma) - { - free_anon_vma_name(vma); - kmem_cache_free(vm_area_cachep, vma); - } - -+#endif /* CONFIG_PER_VMA_LOCK */ -+ - static void account_kernel_stack(struct task_struct *tsk, int account) - { - if (IS_ENABLED(CONFIG_VMAP_STACK)) { -@@ -699,8 +764,10 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - rb_parent = &tmp->vm_rb; - - mm->map_count++; -- if (!(tmp->vm_flags & VM_WIPEONFORK)) -+ if (!(tmp->vm_flags & VM_WIPEONFORK)) { -+ vma_mark_locked(mpnt); - retval = copy_page_range(tmp, mpnt); -+ } - - if (tmp->vm_ops && tmp->vm_ops->open) - tmp->vm_ops->open(tmp); -@@ -1121,6 +1188,12 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - seqcount_init(&mm->write_protect_seq); - mmap_init_lock(mm); - INIT_LIST_HEAD(&mm->mmlist); -+#ifdef CONFIG_PER_VMA_LOCK -+ WRITE_ONCE(mm->mm_lock_seq, 0); -+ INIT_LIST_HEAD(&mm->vma_free_list.head); -+ spin_lock_init(&mm->vma_free_list.lock); -+ mm->vma_free_list.size = 0; -+#endif - mm_pgtables_bytes_init(mm); - mm->map_count = 0; - mm->locked_vm = 0; -diff --git a/mm/Kconfig b/mm/Kconfig -index 96cd3ae25c6f..6c8f47269dbd 100644 ---- a/mm/Kconfig -+++ b/mm/Kconfig -@@ -1150,6 +1150,19 @@ config LRU_GEN_STATS - This option has a per-memcg and per-node memory overhead. - # } - -+config ARCH_SUPPORTS_PER_VMA_LOCK -+ def_bool n -+ -+config PER_VMA_LOCK -+ bool "Per-vma locking support" -+ default y -+ depends on ARCH_SUPPORTS_PER_VMA_LOCK && MMU && SMP -+ help -+ Allow per-vma locking during page fault handling. -+ -+ This feature allows locking each virtual memory area separately when -+ handling page faults instead of taking mmap_lock. -+ - source "mm/damon/Kconfig" - - endmenu -diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug -index ce8dded36de9..075642763a03 100644 ---- a/mm/Kconfig.debug -+++ b/mm/Kconfig.debug -@@ -207,3 +207,11 @@ config PTDUMP_DEBUGFS - kernel. - - If in doubt, say N. -+ -+ -+config PER_VMA_LOCK_STATS -+ bool "Statistics for per-vma locks" -+ depends on PER_VMA_LOCK -+ help -+ Statistics for per-vma locks. -+ If in doubt, say N. -diff --git a/mm/init-mm.c b/mm/init-mm.c -index fbe7844d0912..7b6d2460545f 100644 ---- a/mm/init-mm.c -+++ b/mm/init-mm.c -@@ -37,6 +37,12 @@ struct mm_struct init_mm = { - .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), - .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), - .mmlist = LIST_HEAD_INIT(init_mm.mmlist), -+#ifdef CONFIG_PER_VMA_LOCK -+ .mm_lock_seq = 0, -+ .vma_free_list.head = LIST_HEAD_INIT(init_mm.vma_free_list.head), -+ .vma_free_list.lock = __SPIN_LOCK_UNLOCKED(init_mm.vma_free_list.lock), -+ .vma_free_list.size = 0, -+#endif - .user_ns = &init_user_ns, - .cpu_bitmap = CPU_BITS_NONE, - #ifdef CONFIG_IOMMU_SVA -diff --git a/mm/internal.h b/mm/internal.h -index dd8600d71afe..dfb0b35eff76 100644 ---- a/mm/internal.h -+++ b/mm/internal.h -@@ -86,14 +86,14 @@ void deactivate_file_folio(struct folio *folio); - void folio_activate(struct folio *folio); - - void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, -- unsigned long floor, unsigned long ceiling); -+ unsigned long floor, unsigned long ceiling, bool lock_vma); - void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); - - struct zap_details; - void unmap_page_range(struct mmu_gather *tlb, - struct vm_area_struct *vma, - unsigned long addr, unsigned long end, -- struct zap_details *details); -+ struct zap_details *details, bool lock_vma); - - void page_cache_ra_order(struct readahead_control *, struct file_ra_state *, - unsigned int order); -diff --git a/mm/khugepaged.c b/mm/khugepaged.c -index 01f71786d530..030680633989 100644 ---- a/mm/khugepaged.c -+++ b/mm/khugepaged.c -@@ -1072,6 +1072,7 @@ static void collapse_huge_page(struct mm_struct *mm, - if (mm_find_pmd(mm, address) != pmd) - goto out_up_write; - -+ vma_mark_locked(vma); - anon_vma_lock_write(vma->anon_vma); - - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, -diff --git a/mm/madvise.c b/mm/madvise.c -index 82f6ea8c493d..3efc8a07ebb0 100644 ---- a/mm/madvise.c -+++ b/mm/madvise.c -@@ -181,6 +181,7 @@ static int madvise_update_vma(struct vm_area_struct *vma, - /* - * vm_flags is protected by the mmap_lock held in write mode. - */ -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - if (!vma->vm_file) { - error = replace_anon_vma_name(vma, anon_name); -diff --git a/mm/memory.c b/mm/memory.c -index 47daee6b421f..a9640d62b7a9 100644 ---- a/mm/memory.c -+++ b/mm/memory.c -@@ -391,7 +391,7 @@ void free_pgd_range(struct mmu_gather *tlb, - } - - void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, -- unsigned long floor, unsigned long ceiling) -+ unsigned long floor, unsigned long ceiling, bool lock_vma) - { - while (vma) { - struct vm_area_struct *next = vma->vm_next; -@@ -401,6 +401,8 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - * Hide vma from rmap and truncate_pagecache before freeing - * pgtables - */ -+ if (lock_vma) -+ vma_mark_locked(vma); - unlink_anon_vmas(vma); - unlink_file_vma(vma); - -@@ -415,6 +417,8 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - && !is_vm_hugetlb_page(next)) { - vma = next; - next = vma->vm_next; -+ if (lock_vma) -+ vma_mark_locked(vma); - unlink_anon_vmas(vma); - unlink_file_vma(vma); - } -@@ -1619,12 +1623,16 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, - void unmap_page_range(struct mmu_gather *tlb, - struct vm_area_struct *vma, - unsigned long addr, unsigned long end, -- struct zap_details *details) -+ struct zap_details *details, -+ bool lock_vma) - { - pgd_t *pgd; - unsigned long next; - - BUG_ON(addr >= end); -+ if (lock_vma) -+ vma_mark_locked(vma); -+ - tlb_start_vma(tlb, vma); - pgd = pgd_offset(vma->vm_mm, addr); - do { -@@ -1640,7 +1648,7 @@ void unmap_page_range(struct mmu_gather *tlb, - static void unmap_single_vma(struct mmu_gather *tlb, - struct vm_area_struct *vma, unsigned long start_addr, - unsigned long end_addr, -- struct zap_details *details) -+ struct zap_details *details, bool lock_vma) - { - unsigned long start = max(vma->vm_start, start_addr); - unsigned long end; -@@ -1679,7 +1687,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, - i_mmap_unlock_write(vma->vm_file->f_mapping); - } - } else -- unmap_page_range(tlb, vma, start, end, details); -+ unmap_page_range(tlb, vma, start, end, details, lock_vma); - } - } - -@@ -1703,7 +1711,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, - */ - void unmap_vmas(struct mmu_gather *tlb, - struct vm_area_struct *vma, unsigned long start_addr, -- unsigned long end_addr) -+ unsigned long end_addr, bool lock_vma) - { - struct mmu_notifier_range range; - struct zap_details details = { -@@ -1716,7 +1724,8 @@ void unmap_vmas(struct mmu_gather *tlb, - start_addr, end_addr); - mmu_notifier_invalidate_range_start(&range); - for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) -- unmap_single_vma(tlb, vma, start_addr, end_addr, &details); -+ unmap_single_vma(tlb, vma, start_addr, end_addr, &details, -+ lock_vma); - mmu_notifier_invalidate_range_end(&range); - } - -@@ -1741,7 +1750,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, - update_hiwater_rss(vma->vm_mm); - mmu_notifier_invalidate_range_start(&range); - for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) -- unmap_single_vma(&tlb, vma, start, range.end, NULL); -+ unmap_single_vma(&tlb, vma, start, range.end, NULL, false); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -@@ -1756,7 +1765,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, - * The range must fit into one VMA. - */ - static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, -- unsigned long size, struct zap_details *details) -+ unsigned long size, struct zap_details *details, bool lock_vma) - { - struct mmu_notifier_range range; - struct mmu_gather tlb; -@@ -1767,7 +1776,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr - tlb_gather_mmu(&tlb, vma->vm_mm); - update_hiwater_rss(vma->vm_mm); - mmu_notifier_invalidate_range_start(&range); -- unmap_single_vma(&tlb, vma, address, range.end, details); -+ unmap_single_vma(&tlb, vma, address, range.end, details, lock_vma); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -@@ -1790,7 +1799,7 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, - !(vma->vm_flags & VM_PFNMAP)) - return; - -- zap_page_range_single(vma, address, size, NULL); -+ zap_page_range_single(vma, address, size, NULL, true); - } - EXPORT_SYMBOL_GPL(zap_vma_ptes); - -@@ -3471,7 +3480,8 @@ static void unmap_mapping_range_vma(struct vm_area_struct *vma, - unsigned long start_addr, unsigned long end_addr, - struct zap_details *details) - { -- zap_page_range_single(vma, start_addr, end_addr - start_addr, details); -+ zap_page_range_single(vma, start_addr, end_addr - start_addr, details, -+ false); - } - - static inline void unmap_mapping_range_tree(struct rb_root_cached *root, -@@ -3716,6 +3726,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) - vm_fault_t ret = 0; - void *shadow = NULL; - -+ if (vmf->flags & FAULT_FLAG_VMA_LOCK) { -+ ret = VM_FAULT_RETRY; -+ goto out; -+ } -+ - if (!pte_unmap_same(vmf)) - goto out; - -@@ -5181,6 +5196,51 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, - } - EXPORT_SYMBOL_GPL(handle_mm_fault); - -+#ifdef CONFIG_PER_VMA_LOCK -+static inline struct vm_area_struct *find_vma_under_rcu(struct mm_struct *mm, -+ unsigned long address) -+{ -+ struct vm_area_struct *vma = __find_vma(mm, address); -+ -+ if (!vma || vma->vm_start > address) -+ return NULL; -+ -+ if (!vma_is_anonymous(vma)) -+ return NULL; -+ -+ if (!vma_read_trylock(vma)) { -+ count_vm_vma_lock_event(VMA_LOCK_ABORT); -+ return NULL; -+ } -+ -+ /* Check if the VMA got isolated after we found it */ -+ if (RB_EMPTY_NODE(&vma->vm_rb)) { -+ vma_read_unlock(vma); -+ count_vm_vma_lock_event(VMA_LOCK_MISS); -+ return NULL; -+ } -+ -+ return vma; -+} -+ -+/* -+ * Lookup and lock and anonymous VMA. Returned VMA is guaranteed to be stable -+ * and not isolated. If the VMA is not found of is being modified the function -+ * returns NULL. -+ */ -+struct vm_area_struct *find_and_lock_anon_vma(struct mm_struct *mm, -+ unsigned long address) -+{ -+ struct vm_area_struct *vma; -+ -+ rcu_read_lock(); -+ vma = find_vma_under_rcu(mm, address); -+ rcu_read_unlock(); -+ -+ return vma; -+} -+#endif /* CONFIG_PER_VMA_LOCK */ -+ - #ifndef __PAGETABLE_P4D_FOLDED - /* - * Allocate p4d page table. -diff --git a/mm/mempolicy.c b/mm/mempolicy.c -index b73d3248d976..6be1e5c75556 100644 ---- a/mm/mempolicy.c -+++ b/mm/mempolicy.c -@@ -383,8 +383,10 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) - struct vm_area_struct *vma; - - mmap_write_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) -+ for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ vma_mark_locked(vma); - mpol_rebind_policy(vma->vm_policy, new); -+ } - mmap_write_unlock(mm); - } - -@@ -632,6 +634,7 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, - struct mmu_gather tlb; - int nr_updated; - -+ vma_mark_locked(vma); - tlb_gather_mmu(&tlb, vma->vm_mm); - - nr_updated = change_protection(&tlb, vma, addr, end, PAGE_NONE, -@@ -765,6 +768,7 @@ static int vma_replace_policy(struct vm_area_struct *vma, - if (IS_ERR(new)) - return PTR_ERR(new); - -+ vma_mark_locked(vma); - if (vma->vm_ops && vma->vm_ops->set_policy) { - err = vma->vm_ops->set_policy(vma, new); - if (err) -diff --git a/mm/mlock.c b/mm/mlock.c -index b14e929084cc..f62e1a4d05f2 100644 ---- a/mm/mlock.c -+++ b/mm/mlock.c -@@ -380,6 +380,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, - */ - if (newflags & VM_LOCKED) - newflags |= VM_IO; -+ vma_mark_locked(vma); - WRITE_ONCE(vma->vm_flags, newflags); - - lru_add_drain(); -@@ -456,6 +457,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, - - if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) { - /* No work to do, and mlocking twice would be wrong */ -+ vma_mark_locked(vma); - vma->vm_flags = newflags; - } else { - mlock_vma_pages_range(vma, start, end, newflags); -diff --git a/mm/mmap.c b/mm/mmap.c -index 9d780f415be3..d61b7ef84ba6 100644 ---- a/mm/mmap.c -+++ b/mm/mmap.c -@@ -79,7 +79,7 @@ core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); - - static void unmap_region(struct mm_struct *mm, - struct vm_area_struct *vma, struct vm_area_struct *prev, -- unsigned long start, unsigned long end); -+ unsigned long start, unsigned long end, bool lock_vma); - - static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) - { -@@ -421,12 +421,14 @@ static inline void vma_rb_insert(struct vm_area_struct *vma, - - static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) - { -+ vma_mark_locked(vma); - /* - * Note rb_erase_augmented is a fairly large inline function, - * so make sure we instantiate it only once with our desired - * augmented rbtree callbacks. - */ - rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); -+ RB_CLEAR_NODE(&vma->vm_rb); - } - - static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, -@@ -710,6 +712,10 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - long adjust_next = 0; - int remove_next = 0; - -+ vma_mark_locked(vma); -+ if (next) -+ vma_mark_locked(next); -+ - if (next && !insert) { - struct vm_area_struct *exporter = NULL, *importer = NULL; - -@@ -754,8 +760,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - * If next doesn't have anon_vma, import from vma after - * next, if the vma overlaps with it. - */ -- if (remove_next == 2 && !next->anon_vma) -+ if (remove_next == 2 && !next->anon_vma) { - exporter = next->vm_next; -+ if (exporter) -+ vma_mark_locked(exporter); -+ } - - } else if (end > next->vm_start) { - /* -@@ -931,6 +940,8 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - * "vma->vm_next" gap must be updated. - */ - next = vma->vm_next; -+ if (next) -+ vma_mark_locked(next); - } else { - /* - * For the scope of the comment "next" and -@@ -1138,10 +1149,17 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, - if (vm_flags & VM_SPECIAL) - return NULL; - -+ if (prev) -+ vma_mark_locked(prev); - next = vma_next(mm, prev); - area = next; -- if (area && area->vm_end == end) /* cases 6, 7, 8 */ -+ if (area) -+ vma_mark_locked(area); -+ if (area && area->vm_end == end) { /* cases 6, 7, 8 */ - next = next->vm_next; -+ if (next) -+ vma_mark_locked(next); -+ } - - /* verify some invariant that must be enforced by the caller */ - VM_WARN_ON(prev && addr <= prev->vm_start); -@@ -1818,6 +1836,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, - out: - perf_event_mmap(vma); - -+ vma_mark_locked(vma); - vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); - if (vm_flags & VM_LOCKED) { - if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || -@@ -1849,7 +1868,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, - vma->vm_file = NULL; - - /* Undo any partial mapping done by a device driver. */ -- unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); -+ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end, true); - if (vm_flags & VM_SHARED) - mapping_unmap_writable(file->f_mapping); - free_vma: -@@ -2250,12 +2269,11 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - EXPORT_SYMBOL(get_unmapped_area); - - /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ --struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) -+struct vm_area_struct *__find_vma(struct mm_struct *mm, unsigned long addr) - { - struct rb_node *rb_node; - struct vm_area_struct *vma; - -- mmap_assert_locked(mm); - /* Check the cache first. */ - vma = vmacache_find(mm, addr); - if (likely(vma)) -@@ -2281,8 +2299,7 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) - vmacache_update(addr, vma); - return vma; - } -- --EXPORT_SYMBOL(find_vma); -+EXPORT_SYMBOL(__find_vma); - - /* - * Same as find_vma, but also return a pointer to the previous VMA in *pprev. -@@ -2611,7 +2628,7 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) - */ - static void unmap_region(struct mm_struct *mm, - struct vm_area_struct *vma, struct vm_area_struct *prev, -- unsigned long start, unsigned long end) -+ unsigned long start, unsigned long end, bool lock_vma) - { - struct vm_area_struct *next = vma_next(mm, prev); - struct mmu_gather tlb; -@@ -2619,9 +2636,10 @@ static void unmap_region(struct mm_struct *mm, - lru_add_drain(); - tlb_gather_mmu(&tlb, mm); - update_hiwater_rss(mm); -- unmap_vmas(&tlb, vma, start, end); -+ unmap_vmas(&tlb, vma, start, end, lock_vma); - free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, -- next ? next->vm_start : USER_PGTABLES_CEILING); -+ next ? next->vm_start : USER_PGTABLES_CEILING, -+ lock_vma); - tlb_finish_mmu(&tlb); - } - -@@ -2662,10 +2680,14 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, - * VM_GROWSUP VMA. Such VMAs can change their size under - * down_read(mmap_lock) and collide with the VMA we are about to unmap. - */ -- if (vma && (vma->vm_flags & VM_GROWSDOWN)) -+ if (vma && (vma->vm_flags & VM_GROWSDOWN)) { -+ vma_mark_locked(vma); - return false; -- if (prev && (prev->vm_flags & VM_GROWSUP)) -+ } -+ if (prev && (prev->vm_flags & VM_GROWSUP)) { -+ vma_mark_locked(prev); - return false; -+ } - return true; - } - -@@ -2679,6 +2701,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *new; - int err; - -+ vma_mark_locked(vma); - if (vma->vm_ops && vma->vm_ops->may_split) { - err = vma->vm_ops->may_split(vma, addr); - if (err) -@@ -2833,7 +2856,7 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - if (downgrade) - mmap_write_downgrade(mm); - -- unmap_region(mm, vma, prev, start, end); -+ unmap_region(mm, vma, prev, start, end, !downgrade); - - /* Fix up all other VM information */ - remove_vma_list(mm, vma); -@@ -3113,8 +3136,8 @@ void exit_mmap(struct mm_struct *mm) - tlb_gather_mmu_fullmm(&tlb, mm); - /* update_hiwater_rss(mm) here? but nobody should be looking */ - /* Use -1 here to ensure all VMAs in the mm are unmapped */ -- unmap_vmas(&tlb, vma, 0, -1); -- free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); -+ unmap_vmas(&tlb, vma, 0, -1, true); -+ free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING, true); - tlb_finish_mmu(&tlb); - - /* Walk the list again, actually closing and freeing it. */ -@@ -3126,6 +3149,7 @@ void exit_mmap(struct mm_struct *mm) - } - mm->mmap = NULL; - mmap_write_unlock(mm); -+ drain_free_vmas(mm); - vm_unacct_memory(nr_accounted); - } - -@@ -3232,6 +3256,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - get_file(new_vma->vm_file); - if (new_vma->vm_ops && new_vma->vm_ops->open) - new_vma->vm_ops->open(new_vma); -+ vma_mark_locked(new_vma); - vma_link(mm, new_vma, prev, rb_link, rb_parent); - *need_rmap_locks = false; - } -@@ -3514,6 +3539,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) - * hugetlb mapping); - * - all i_mmap_rwsem locks; - * - all anon_vma->rwseml -+ * - all vmas marked locked - * - * We can take all locks within these types randomly because the VM code - * doesn't nest them and we protected from parallel mm_take_all_locks() by -@@ -3555,6 +3581,7 @@ int mm_take_all_locks(struct mm_struct *mm) - if (vma->anon_vma) - list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - vm_lock_anon_vma(mm, avc->anon_vma); -+ vma_mark_locked(vma); - } - - return 0; -@@ -3612,6 +3639,7 @@ void mm_drop_all_locks(struct mm_struct *mm) - mmap_assert_write_locked(mm); - BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); - -+ vma_mark_unlocked_all(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (vma->anon_vma) - list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) -diff --git a/mm/mprotect.c b/mm/mprotect.c -index bc6bddd156ca..df47fc21b0e4 100644 ---- a/mm/mprotect.c -+++ b/mm/mprotect.c -@@ -621,6 +621,7 @@ mprotect_fixup(struct mmu_gather *tlb, struct vm_area_struct *vma, - * vm_flags and vm_page_prot are protected by the mmap_lock - * held in write mode. - */ -+ vma_mark_locked(vma); - vma->vm_flags = newflags; - /* - * We want to check manually if we can change individual PTEs writable -diff --git a/mm/mremap.c b/mm/mremap.c -index b522cd0259a0..bdbf96254e43 100644 ---- a/mm/mremap.c -+++ b/mm/mremap.c -@@ -620,6 +620,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, - return -ENOMEM; - } - -+ vma_mark_locked(vma); - new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); - new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, - &need_rmap_locks); -diff --git a/mm/nommu.c b/mm/nommu.c -index e819cbc21b39..ff9933e57501 100644 ---- a/mm/nommu.c -+++ b/mm/nommu.c -@@ -622,6 +622,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) - struct mm_struct *mm = vma->vm_mm; - struct task_struct *curr = current; - -+ vma_mark_locked(vma); - mm->map_count--; - for (i = 0; i < VMACACHE_SIZE; i++) { - /* if the vma is cached, invalidate the entire cache */ -@@ -644,6 +645,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) - - /* remove from the MM's tree and list */ - rb_erase(&vma->vm_rb, &mm->mm_rb); -+ RB_CLEAR_NODE(&vma->vm_rb); - - __vma_unlink_list(mm, vma); - } -diff --git a/mm/oom_kill.c b/mm/oom_kill.c -index 3c6cf9e3cd66..6ffa7c511aa3 100644 ---- a/mm/oom_kill.c -+++ b/mm/oom_kill.c -@@ -549,7 +549,8 @@ bool __oom_reap_task_mm(struct mm_struct *mm) - ret = false; - continue; - } -- unmap_page_range(&tlb, vma, range.start, range.end, NULL); -+ unmap_page_range(&tlb, vma, range.start, range.end, -+ NULL, false); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -diff --git a/mm/vmstat.c b/mm/vmstat.c -index 90af9a8572f5..3f3804c846a6 100644 ---- a/mm/vmstat.c -+++ b/mm/vmstat.c -@@ -1411,6 +1411,12 @@ const char * const vmstat_text[] = { - "direct_map_level2_splits", - "direct_map_level3_splits", - #endif -+#ifdef CONFIG_PER_VMA_LOCK_STATS -+ "vma_lock_success", -+ "vma_lock_abort", -+ "vma_lock_retry", -+ "vma_lock_miss", -+#endif - #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ - }; - #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ --- -2.37.3 - diff --git a/6.0/testing/0007-rtw88.patch b/6.0/0010-rtw88.patch similarity index 99% rename from 6.0/testing/0007-rtw88.patch rename to 6.0/0010-rtw88.patch index 2a51a3da..849bfd6a 100644 --- a/6.0/testing/0007-rtw88.patch +++ b/6.0/0010-rtw88.patch @@ -1,7 +1,7 @@ -From 4400a1f1fd24bc600b4557fda2baee2850e5e693 Mon Sep 17 00:00:00 2001 +From 6257c94a850dc4b3faa5a55be5831de4f8777cac Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 19 Aug 2022 17:06:47 +0200 -Subject: [PATCH 07/13] rtw88 +Subject: [PATCH 10/16] rtw88 Signed-off-by: Peter Jung --- diff --git a/6.0/0011-folios.patch b/6.0/0011-folios.patch deleted file mode 100644 index a87565c8..00000000 --- a/6.0/0011-folios.patch +++ /dev/null @@ -1,2014 +0,0 @@ -From 4ba2e168370145273187f242e0a85ce3842e25fa Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 19 Sep 2022 14:40:14 +0200 -Subject: [PATCH 11/13] folios - -Signed-off-by: Peter Jung ---- - fs/afs/write.c | 114 +++++++++++++++++---------------- - fs/btrfs/extent_io.c | 57 +++++++++-------- - fs/ceph/addr.c | 138 ++++++++++++++++++++-------------------- - fs/cifs/file.c | 33 +++++++++- - fs/ext4/inode.c | 55 ++++++++-------- - fs/f2fs/checkpoint.c | 49 +++++++------- - fs/f2fs/compress.c | 13 ++-- - fs/f2fs/data.c | 69 ++++++++++---------- - fs/f2fs/f2fs.h | 5 +- - fs/f2fs/node.c | 72 +++++++++++---------- - fs/gfs2/aops.c | 64 ++++++++++--------- - fs/nilfs2/btree.c | 14 ++-- - fs/nilfs2/page.c | 59 ++++++++--------- - fs/nilfs2/segment.c | 44 +++++++------ - include/linux/pagemap.h | 32 +++++++--- - include/linux/pagevec.h | 8 --- - mm/filemap.c | 87 ++++++++++++------------- - mm/page-writeback.c | 44 +++++++------ - mm/swap.c | 10 --- - 19 files changed, 507 insertions(+), 460 deletions(-) - -diff --git a/fs/afs/write.c b/fs/afs/write.c -index 9ebdd36eaf2f..c17dbd82a38c 100644 ---- a/fs/afs/write.c -+++ b/fs/afs/write.c -@@ -699,82 +699,86 @@ static int afs_writepages_region(struct address_space *mapping, - loff_t start, loff_t end, loff_t *_next) - { - struct folio *folio; -- struct page *head_page; -+ struct folio_batch fbatch; - ssize_t ret; -+ unsigned int i; - int n, skips = 0; - - _enter("%llx,%llx,", start, end); -+ folio_batch_init(&fbatch); - - do { - pgoff_t index = start / PAGE_SIZE; - -- n = find_get_pages_range_tag(mapping, &index, end / PAGE_SIZE, -- PAGECACHE_TAG_DIRTY, 1, &head_page); -+ n = filemap_get_folios_tag(mapping, &index, end / PAGE_SIZE, -+ PAGECACHE_TAG_DIRTY, &fbatch); -+ - if (!n) - break; -+ for (i = 0; i < n; i++) { -+ folio = fbatch.folios[i]; -+ start = folio_pos(folio); /* May regress with THPs */ - -- folio = page_folio(head_page); -- start = folio_pos(folio); /* May regress with THPs */ -- -- _debug("wback %lx", folio_index(folio)); -+ _debug("wback %lx", folio_index(folio)); - -- /* At this point we hold neither the i_pages lock nor the -- * page lock: the page may be truncated or invalidated -- * (changing page->mapping to NULL), or even swizzled -- * back from swapper_space to tmpfs file mapping -- */ -- if (wbc->sync_mode != WB_SYNC_NONE) { -- ret = folio_lock_killable(folio); -- if (ret < 0) { -- folio_put(folio); -- return ret; -- } -- } else { -- if (!folio_trylock(folio)) { -- folio_put(folio); -- return 0; -+ /* At this point we hold neither the i_pages lock nor the -+ * page lock: the page may be truncated or invalidated -+ * (changing page->mapping to NULL), or even swizzled -+ * back from swapper_space to tmpfs file mapping -+ */ -+ if (wbc->sync_mode != WB_SYNC_NONE) { -+ ret = folio_lock_killable(folio); -+ if (ret < 0) { -+ folio_batch_release(&fbatch); -+ return ret; -+ } -+ } else { -+ if (!folio_trylock(folio)) -+ continue; - } -- } - -- if (folio_mapping(folio) != mapping || -- !folio_test_dirty(folio)) { -- start += folio_size(folio); -- folio_unlock(folio); -- folio_put(folio); -- continue; -- } -+ if (folio->mapping != mapping || -+ !folio_test_dirty(folio)) { -+ start += folio_size(folio); -+ folio_unlock(folio); -+ continue; -+ } - -- if (folio_test_writeback(folio) || -- folio_test_fscache(folio)) { -- folio_unlock(folio); -- if (wbc->sync_mode != WB_SYNC_NONE) { -- folio_wait_writeback(folio); -+ if (folio_test_writeback(folio) || -+ folio_test_fscache(folio)) { -+ folio_unlock(folio); -+ if (wbc->sync_mode != WB_SYNC_NONE) { -+ folio_wait_writeback(folio); - #ifdef CONFIG_AFS_FSCACHE -- folio_wait_fscache(folio); -+ folio_wait_fscache(folio); - #endif -- } else { -- start += folio_size(folio); -+ } else { -+ start += folio_size(folio); -+ } -+ if (wbc->sync_mode == WB_SYNC_NONE) { -+ if (skips >= 5 || need_resched()) { -+ *_next = start; -+ _leave(" = 0 [%llx]", *_next); -+ return 0; -+ } -+ skips++; -+ } -+ continue; - } -- folio_put(folio); -- if (wbc->sync_mode == WB_SYNC_NONE) { -- if (skips >= 5 || need_resched()) -- break; -- skips++; -+ -+ if (!folio_clear_dirty_for_io(folio)) -+ BUG(); -+ ret = afs_write_back_from_locked_folio(mapping, wbc, -+ folio, start, end); -+ if (ret < 0) { -+ _leave(" = %zd", ret); -+ folio_batch_release(&fbatch); -+ return ret; - } -- continue; -- } - -- if (!folio_clear_dirty_for_io(folio)) -- BUG(); -- ret = afs_write_back_from_locked_folio(mapping, wbc, folio, start, end); -- folio_put(folio); -- if (ret < 0) { -- _leave(" = %zd", ret); -- return ret; -+ start += ret; - } -- -- start += ret; -- -+ folio_batch_release(&fbatch); - cond_resched(); - } while (wbc->nr_to_write > 0); - -diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c -index cf4f19e80e2f..80fe313f8461 100644 ---- a/fs/btrfs/extent_io.c -+++ b/fs/btrfs/extent_io.c -@@ -4844,14 +4844,14 @@ int btree_write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int nr_to_write_done = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - int scanned = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* Start from prev offset */ - end = -1; -@@ -4874,14 +4874,15 @@ int btree_write_cache_pages(struct address_space *mapping, - if (wbc->sync_mode == WB_SYNC_ALL) - tag_pages_for_writeback(mapping, index, end); - while (!done && !nr_to_write_done && (index <= end) && -- (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag))) { -+ (nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch))) { - unsigned i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- ret = submit_eb_page(page, wbc, &epd, &eb_context); -+ ret = submit_eb_page(&folio->page, wbc, &epd, -+ &eb_context); - if (ret == 0) - continue; - if (ret < 0) { -@@ -4896,7 +4897,7 @@ int btree_write_cache_pages(struct address_space *mapping, - */ - nr_to_write_done = wbc->nr_to_write <= 0; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - if (!scanned && !done) { -@@ -4971,8 +4972,8 @@ static int extent_write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int nr_to_write_done = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; -@@ -4992,7 +4993,7 @@ static int extent_write_cache_pages(struct address_space *mapping, - if (!igrab(inode)) - return 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* Start from prev offset */ - end = -1; -@@ -5030,14 +5031,14 @@ static int extent_write_cache_pages(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; - while (!done && !nr_to_write_done && (index <= end) && -- (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, -- &index, end, tag))) { -+ (nr_folios = filemap_get_folios_tag(mapping, &index, -+ end, tag, &fbatch))) { - unsigned i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- done_index = page->index + 1; -+ done_index = folio->index + folio_nr_pages(folio); - /* - * At this point we hold neither the i_pages lock nor - * the page lock: the page may be truncated or -@@ -5045,29 +5046,29 @@ static int extent_write_cache_pages(struct address_space *mapping, - * or even swizzled back from swapper_space to - * tmpfs file mapping - */ -- if (!trylock_page(page)) { -+ if (!folio_trylock(folio)) { - submit_write_bio(epd, 0); -- lock_page(page); -+ folio_lock(folio); - } - -- if (unlikely(page->mapping != mapping)) { -- unlock_page(page); -+ if (unlikely(folio->mapping != mapping)) { -+ folio_unlock(folio); - continue; - } - - if (wbc->sync_mode != WB_SYNC_NONE) { -- if (PageWriteback(page)) -+ if (folio_test_writeback(folio)) - submit_write_bio(epd, 0); -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - } - -- if (PageWriteback(page) || -- !clear_page_dirty_for_io(page)) { -- unlock_page(page); -+ if (folio_test_writeback(folio) || -+ !folio_clear_dirty_for_io(folio)) { -+ folio_unlock(folio); - continue; - } - -- ret = __extent_writepage(page, wbc, epd); -+ ret = __extent_writepage(&folio->page, wbc, epd); - if (ret < 0) { - done = 1; - break; -@@ -5080,7 +5081,7 @@ static int extent_write_cache_pages(struct address_space *mapping, - */ - nr_to_write_done = wbc->nr_to_write <= 0; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - if (!scanned && !done) { -diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c -index dcf701b05cc1..33dbe55b08be 100644 ---- a/fs/ceph/addr.c -+++ b/fs/ceph/addr.c -@@ -792,7 +792,7 @@ static int ceph_writepages_start(struct address_space *mapping, - struct ceph_vino vino = ceph_vino(inode); - pgoff_t index, start_index, end = -1; - struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; -- struct pagevec pvec; -+ struct folio_batch fbatch; - int rc = 0; - unsigned int wsize = i_blocksize(inode); - struct ceph_osd_request *req = NULL; -@@ -821,7 +821,7 @@ static int ceph_writepages_start(struct address_space *mapping, - if (fsc->mount_options->wsize < wsize) - wsize = fsc->mount_options->wsize; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - - start_index = wbc->range_cyclic ? mapping->writeback_index : 0; - index = start_index; -@@ -869,9 +869,9 @@ static int ceph_writepages_start(struct address_space *mapping, - - while (!done && index <= end) { - int num_ops = 0, op_idx; -- unsigned i, pvec_pages, max_pages, locked_pages = 0; -+ unsigned i, nr_folios, max_pages, locked_pages = 0; - struct page **pages = NULL, **data_pages; -- struct page *page; -+ struct folio *folio; - pgoff_t strip_unit_end = 0; - u64 offset = 0, len = 0; - bool from_pool = false; -@@ -879,28 +879,28 @@ static int ceph_writepages_start(struct address_space *mapping, - max_pages = wsize >> PAGE_SHIFT; - - get_more_pages: -- pvec_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, -- end, PAGECACHE_TAG_DIRTY); -- dout("pagevec_lookup_range_tag got %d\n", pvec_pages); -- if (!pvec_pages && !locked_pages) -+ nr_folios = filemap_get_folios_tag(mapping, &index, -+ end, PAGECACHE_TAG_DIRTY, &fbatch); -+ dout("filemap_get_folios_tag got %d\n", nr_folios); -+ if (!nr_folios && !locked_pages) - break; -- for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { -- page = pvec.pages[i]; -- dout("? %p idx %lu\n", page, page->index); -+ for (i = 0; i < nr_folios && locked_pages < max_pages; i++) { -+ folio = fbatch.folios[i]; -+ dout("? %p idx %lu\n", folio, folio->index); - if (locked_pages == 0) -- lock_page(page); /* first page */ -- else if (!trylock_page(page)) -+ folio_lock(folio); /* first folio */ -+ else if (!folio_trylock(folio)) - break; - - /* only dirty pages, or our accounting breaks */ -- if (unlikely(!PageDirty(page)) || -- unlikely(page->mapping != mapping)) { -- dout("!dirty or !mapping %p\n", page); -- unlock_page(page); -+ if (unlikely(!folio_test_dirty(folio)) || -+ unlikely(folio->mapping != mapping)) { -+ dout("!dirty or !mapping %p\n", folio); -+ folio_unlock(folio); - continue; - } - /* only if matching snap context */ -- pgsnapc = page_snap_context(page); -+ pgsnapc = page_snap_context(&folio->page); - if (pgsnapc != snapc) { - dout("page snapc %p %lld != oldest %p %lld\n", - pgsnapc, pgsnapc->seq, snapc, snapc->seq); -@@ -908,11 +908,10 @@ static int ceph_writepages_start(struct address_space *mapping, - !ceph_wbc.head_snapc && - wbc->sync_mode != WB_SYNC_NONE) - should_loop = true; -- unlock_page(page); -+ folio_unlock(folio); - continue; - } -- if (page_offset(page) >= ceph_wbc.i_size) { -- struct folio *folio = page_folio(page); -+ if (folio_pos(folio) >= ceph_wbc.i_size) { - - dout("folio at %lu beyond eof %llu\n", - folio->index, ceph_wbc.i_size); -@@ -924,25 +923,26 @@ static int ceph_writepages_start(struct address_space *mapping, - folio_unlock(folio); - continue; - } -- if (strip_unit_end && (page->index > strip_unit_end)) { -- dout("end of strip unit %p\n", page); -- unlock_page(page); -+ if (strip_unit_end && (folio->index > strip_unit_end)) { -+ dout("end of strip unit %p\n", folio); -+ folio_unlock(folio); - break; - } -- if (PageWriteback(page) || PageFsCache(page)) { -+ if (folio_test_writeback(folio) || -+ folio_test_fscache(folio)) { - if (wbc->sync_mode == WB_SYNC_NONE) { -- dout("%p under writeback\n", page); -- unlock_page(page); -+ dout("%p under writeback\n", folio); -+ folio_unlock(folio); - continue; - } -- dout("waiting on writeback %p\n", page); -- wait_on_page_writeback(page); -- wait_on_page_fscache(page); -+ dout("waiting on writeback %p\n", folio); -+ folio_wait_writeback(folio); -+ folio_wait_fscache(folio); - } - -- if (!clear_page_dirty_for_io(page)) { -- dout("%p !clear_page_dirty_for_io\n", page); -- unlock_page(page); -+ if (!folio_clear_dirty_for_io(folio)) { -+ dout("%p !clear_page_dirty_for_io\n", folio); -+ folio_unlock(folio); - continue; - } - -@@ -958,7 +958,7 @@ static int ceph_writepages_start(struct address_space *mapping, - u32 xlen; - - /* prepare async write request */ -- offset = (u64)page_offset(page); -+ offset = (u64)folio_pos(folio); - ceph_calc_file_object_mapping(&ci->i_layout, - offset, wsize, - &objnum, &objoff, -@@ -966,7 +966,7 @@ static int ceph_writepages_start(struct address_space *mapping, - len = xlen; - - num_ops = 1; -- strip_unit_end = page->index + -+ strip_unit_end = folio->index + - ((len - 1) >> PAGE_SHIFT); - - BUG_ON(pages); -@@ -981,54 +981,53 @@ static int ceph_writepages_start(struct address_space *mapping, - } - - len = 0; -- } else if (page->index != -+ } else if (folio->index != - (offset + len) >> PAGE_SHIFT) { - if (num_ops >= (from_pool ? CEPH_OSD_SLAB_OPS : - CEPH_OSD_MAX_OPS)) { -- redirty_page_for_writepage(wbc, page); -- unlock_page(page); -+ folio_redirty_for_writepage(wbc, folio); -+ folio_unlock(folio); - break; - } - - num_ops++; -- offset = (u64)page_offset(page); -+ offset = (u64)folio_pos(folio); - len = 0; - } - -- /* note position of first page in pvec */ -+ /* note position of first page in fbatch */ - dout("%p will write page %p idx %lu\n", -- inode, page, page->index); -+ inode, folio, folio->index); - - if (atomic_long_inc_return(&fsc->writeback_count) > - CONGESTION_ON_THRESH( - fsc->mount_options->congestion_kb)) - fsc->write_congested = true; - -- pages[locked_pages++] = page; -- pvec.pages[i] = NULL; -+ pages[locked_pages++] = &folio->page; -+ fbatch.folios[i] = NULL; - -- len += thp_size(page); -+ len += folio_size(folio); - } - - /* did we get anything? */ - if (!locked_pages) -- goto release_pvec_pages; -+ goto release_folio_batches; - if (i) { - unsigned j, n = 0; -- /* shift unused page to beginning of pvec */ -- for (j = 0; j < pvec_pages; j++) { -- if (!pvec.pages[j]) -+ /* shift unused folio to the beginning of fbatch */ -+ for (j = 0; j < nr_folios; j++) { -+ if (!fbatch.folios[j]) - continue; - if (n < j) -- pvec.pages[n] = pvec.pages[j]; -+ fbatch.folios[n] = fbatch.folios[j]; - n++; - } -- pvec.nr = n; -- -- if (pvec_pages && i == pvec_pages && -+ fbatch.nr = n; -+ if (nr_folios && i == nr_folios && - locked_pages < max_pages) { -- dout("reached end pvec, trying for more\n"); -- pagevec_release(&pvec); -+ dout("reached end of fbatch, trying for more\n"); -+ folio_batch_release(&fbatch); - goto get_more_pages; - } - } -@@ -1056,7 +1055,7 @@ static int ceph_writepages_start(struct address_space *mapping, - BUG_ON(IS_ERR(req)); - } - BUG_ON(len < page_offset(pages[locked_pages - 1]) + -- thp_size(page) - offset); -+ folio_size(folio) - offset); - - req->r_callback = writepages_finish; - req->r_inode = inode; -@@ -1098,7 +1097,7 @@ static int ceph_writepages_start(struct address_space *mapping, - set_page_writeback(pages[i]); - if (caching) - ceph_set_page_fscache(pages[i]); -- len += thp_size(page); -+ len += folio_size(folio); - } - ceph_fscache_write_to_cache(inode, offset, len, caching); - -@@ -1108,7 +1107,7 @@ static int ceph_writepages_start(struct address_space *mapping, - /* writepages_finish() clears writeback pages - * according to the data length, so make sure - * data length covers all locked pages */ -- u64 min_len = len + 1 - thp_size(page); -+ u64 min_len = len + 1 - folio_size(folio); - len = get_writepages_data_length(inode, pages[i - 1], - offset); - len = max(len, min_len); -@@ -1164,10 +1163,10 @@ static int ceph_writepages_start(struct address_space *mapping, - if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) - done = true; - --release_pvec_pages: -- dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, -- pvec.nr ? pvec.pages[0] : NULL); -- pagevec_release(&pvec); -+release_folio_batches: -+ dout("folio_batch_release on %d batches (%p)", (int) fbatch.nr, -+ fbatch.nr ? fbatch.folios[0] : NULL); -+ folio_batch_release(&fbatch); - } - - if (should_loop && !done) { -@@ -1180,19 +1179,22 @@ static int ceph_writepages_start(struct address_space *mapping, - if (wbc->sync_mode != WB_SYNC_NONE && - start_index == 0 && /* all dirty pages were checked */ - !ceph_wbc.head_snapc) { -- struct page *page; -+ struct folio *folio; - unsigned i, nr; - index = 0; - while ((index <= end) && -- (nr = pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_WRITEBACK))) { -+ (nr = filemap_get_folios_tag(mapping, &index, -+ (pgoff_t)-1, -+ PAGECACHE_TAG_WRITEBACK, -+ &fbatch))) { - for (i = 0; i < nr; i++) { -- page = pvec.pages[i]; -- if (page_snap_context(page) != snapc) -+ folio = fbatch.folios[i]; -+ if (page_snap_context(&folio->page) != -+ snapc) - continue; -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/fs/cifs/file.c b/fs/cifs/file.c -index 6f38b134a346..249a94896f78 100644 ---- a/fs/cifs/file.c -+++ b/fs/cifs/file.c -@@ -2517,14 +2517,41 @@ wdata_alloc_and_fillpages(pgoff_t tofind, struct address_space *mapping, - unsigned int *found_pages) - { - struct cifs_writedata *wdata; -- -+ struct folio_batch fbatch; -+ unsigned int i, idx, p, nr; - wdata = cifs_writedata_alloc((unsigned int)tofind, - cifs_writev_complete); - if (!wdata) - return NULL; - -- *found_pages = find_get_pages_range_tag(mapping, index, end, -- PAGECACHE_TAG_DIRTY, tofind, wdata->pages); -+ folio_batch_init(&fbatch); -+ *found_pages = 0; -+ -+again: -+ nr = filemap_get_folios_tag(mapping, index, end, -+ PAGECACHE_TAG_DIRTY, &fbatch); -+ if (!nr) -+ goto out; /* No dirty pages left in the range */ -+ -+ for (i = 0; i < nr; i++) { -+ struct folio *folio = fbatch.folios[i]; -+ -+ idx = 0; -+ p = folio_nr_pages(folio); -+add_more: -+ wdata->pages[*found_pages] = folio_page(folio, idx); -+ if (++*found_pages == tofind) { -+ folio_batch_release(&fbatch); -+ goto out; -+ } -+ if (++idx < p) { -+ folio_ref_inc(folio); -+ goto add_more; -+ } -+ } -+ folio_batch_release(&fbatch); -+ goto again; -+out: - return wdata; - } - -diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c -index 601214453c3a..fbd876e10a85 100644 ---- a/fs/ext4/inode.c -+++ b/fs/ext4/inode.c -@@ -2565,8 +2565,8 @@ static int ext4_da_writepages_trans_blocks(struct inode *inode) - static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - { - struct address_space *mapping = mpd->inode->i_mapping; -- struct pagevec pvec; -- unsigned int nr_pages; -+ struct folio_batch fbatch; -+ unsigned int nr_folios; - long left = mpd->wbc->nr_to_write; - pgoff_t index = mpd->first_page; - pgoff_t end = mpd->last_page; -@@ -2580,18 +2580,17 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - tag = PAGECACHE_TAG_TOWRITE; - else - tag = PAGECACHE_TAG_DIRTY; -- -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - mpd->map.m_len = 0; - mpd->next_page = index; - while (index <= end) { -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - - /* - * Accumulated enough dirty pages? This doesn't apply -@@ -2605,10 +2604,10 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - goto out; - - /* If we can't merge this page, we are done. */ -- if (mpd->map.m_len > 0 && mpd->next_page != page->index) -+ if (mpd->map.m_len > 0 && mpd->next_page != folio->index) - goto out; - -- lock_page(page); -+ folio_lock(folio); - /* - * If the page is no longer dirty, or its mapping no - * longer corresponds to inode we are writing (which -@@ -2616,16 +2615,16 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - * page is already under writeback and we are not doing - * a data integrity writeback, skip the page - */ -- if (!PageDirty(page) || -- (PageWriteback(page) && -+ if (!folio_test_dirty(folio) || -+ (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || -- unlikely(page->mapping != mapping)) { -- unlock_page(page); -+ unlikely(folio->mapping != mapping)) { -+ folio_unlock(folio); - continue; - } - -- wait_on_page_writeback(page); -- BUG_ON(PageWriteback(page)); -+ folio_wait_writeback(folio); -+ BUG_ON(folio_test_writeback(folio)); - - /* - * Should never happen but for buggy code in -@@ -2636,33 +2635,33 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - * - * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz - */ -- if (!page_has_buffers(page)) { -- ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index); -- ClearPageDirty(page); -- unlock_page(page); -+ if (!folio_buffers(folio)) { -+ ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index); -+ folio_clear_dirty(folio); -+ folio_unlock(folio); - continue; - } - - if (mpd->map.m_len == 0) -- mpd->first_page = page->index; -- mpd->next_page = page->index + 1; -+ mpd->first_page = folio->index; -+ mpd->next_page = folio->index + folio_nr_pages(folio); - /* Add all dirty buffers to mpd */ -- lblk = ((ext4_lblk_t)page->index) << -+ lblk = ((ext4_lblk_t)folio->index) << - (PAGE_SHIFT - blkbits); -- head = page_buffers(page); -+ head = folio_buffers(folio); - err = mpage_process_page_bufs(mpd, head, head, lblk); - if (err <= 0) - goto out; - err = 0; -- left--; -+ left -= folio_nr_pages(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - mpd->scanned_until_end = 1; - return 0; - out: -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - return err; - } - -diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c -index 8259e0fa97e1..9f6694f7d723 100644 ---- a/fs/f2fs/checkpoint.c -+++ b/fs/f2fs/checkpoint.c -@@ -377,59 +377,62 @@ long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type, - { - struct address_space *mapping = META_MAPPING(sbi); - pgoff_t index = 0, prev = ULONG_MAX; -- struct pagevec pvec; -+ struct folio_batch fbatch; - long nwritten = 0; -- int nr_pages; -+ int nr_folios; - struct writeback_control wbc = { - .for_reclaim = 0, - }; - struct blk_plug plug; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - - blk_start_plug(&plug); - -- while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(mapping, &index, -+ (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- if (prev == ULONG_MAX) -- prev = page->index - 1; -- if (nr_to_write != LONG_MAX && page->index != prev + 1) { -- pagevec_release(&pvec); -+ if (nr_to_write != LONG_MAX && i != 0 && -+ folio->index != prev + -+ folio_nr_pages(fbatch.folios[i-1])) { -+ folio_batch_release(&fbatch); - goto stop; - } - -- lock_page(page); -+ folio_lock(folio); - -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- f2fs_wait_on_page_writeback(page, META, true, true); -+ f2fs_wait_on_page_writeback(&folio->page, META, -+ true, true); - -- if (!clear_page_dirty_for_io(page)) -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - -- if (__f2fs_write_meta_page(page, &wbc, io_type)) { -- unlock_page(page); -+ if (__f2fs_write_meta_page(&folio->page, &wbc, -+ io_type)) { -+ folio_unlock(folio); - break; - } -- nwritten++; -- prev = page->index; -+ nwritten += folio_nr_pages(folio); -+ prev = folio->index; - if (unlikely(nwritten >= nr_to_write)) - break; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - stop: -@@ -1381,7 +1384,7 @@ static void commit_checkpoint(struct f2fs_sb_info *sbi, - }; - - /* -- * pagevec_lookup_tag and lock_page again will take -+ * filemap_get_folios_tag and lock_page again will take - * some extra time. Therefore, f2fs_update_meta_pages and - * f2fs_sync_meta_pages are combined in this function. - */ -diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c -index 70e97075e535..e1bd2e859f64 100644 ---- a/fs/f2fs/compress.c -+++ b/fs/f2fs/compress.c -@@ -841,10 +841,11 @@ bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) - return is_page_in_cluster(cc, index); - } - --bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, -- int index, int nr_pages, bool uptodate) -+bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, -+ struct folio_batch *fbatch, -+ int index, int nr_folios, bool uptodate) - { -- unsigned long pgidx = pages[index]->index; -+ unsigned long pgidx = fbatch->folios[index]->index; - int i = uptodate ? 0 : 1; - - /* -@@ -854,13 +855,13 @@ bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, - if (uptodate && (pgidx % cc->cluster_size)) - return false; - -- if (nr_pages - index < cc->cluster_size) -+ if (nr_folios - index < cc->cluster_size) - return false; - - for (; i < cc->cluster_size; i++) { -- if (pages[index + i]->index != pgidx + i) -+ if (fbatch->folios[index + i]->index != pgidx + i) - return false; -- if (uptodate && !PageUptodate(pages[index + i])) -+ if (uptodate && !folio_test_uptodate(fbatch->folios[index + i])) - return false; - } - -diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c -index aa3ccddfa037..b14747598b39 100644 ---- a/fs/f2fs/data.c -+++ b/fs/f2fs/data.c -@@ -2917,7 +2917,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - { - int ret = 0; - int done = 0, retry = 0; -- struct page *pages[F2FS_ONSTACK_PAGES]; -+ struct folio_batch fbatch; - struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); - struct bio *bio = NULL; - sector_t last_block; -@@ -2938,7 +2938,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - .private = NULL, - }; - #endif -- int nr_pages; -+ int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; -@@ -2948,6 +2948,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - int submitted = 0; - int i; - -+ folio_batch_init(&fbatch); -+ - if (get_dirty_pages(mapping->host) <= - SM_I(F2FS_M_SB(mapping))->min_hot_blocks) - set_inode_flag(mapping->host, FI_HOT_DATA); -@@ -2973,13 +2975,13 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; - while (!done && !retry && (index <= end)) { -- nr_pages = find_get_pages_range_tag(mapping, &index, end, -- tag, F2FS_ONSTACK_PAGES, pages); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - bool need_readd; - readd: - need_readd = false; -@@ -2996,7 +2998,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - } - - if (!f2fs_cluster_can_merge_page(&cc, -- page->index)) { -+ folio->index)) { - ret = f2fs_write_multi_pages(&cc, - &submitted, wbc, io_type); - if (!ret) -@@ -3005,27 +3007,28 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - } - - if (unlikely(f2fs_cp_error(sbi))) -- goto lock_page; -+ goto lock_folio; - - if (!f2fs_cluster_is_empty(&cc)) -- goto lock_page; -+ goto lock_folio; - - if (f2fs_all_cluster_page_ready(&cc, -- pages, i, nr_pages, true)) -- goto lock_page; -+ &fbatch, i, nr_folios, true)) -+ goto lock_folio; - - ret2 = f2fs_prepare_compress_overwrite( - inode, &pagep, -- page->index, &fsdata); -+ folio->index, &fsdata); - if (ret2 < 0) { - ret = ret2; - done = 1; - break; - } else if (ret2 && - (!f2fs_compress_write_end(inode, -- fsdata, page->index, 1) || -+ fsdata, folio->index, 1) || - !f2fs_all_cluster_page_ready(&cc, -- pages, i, nr_pages, false))) { -+ &fbatch, i, nr_folios, -+ false))) { - retry = 1; - break; - } -@@ -3038,46 +3041,47 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - break; - } - #ifdef CONFIG_F2FS_FS_COMPRESSION --lock_page: -+lock_folio: - #endif -- done_index = page->index; -+ done_index = folio->index; - retry_write: -- lock_page(page); -+ folio_lock(folio); - -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- f2fs_wait_on_page_writeback(page, -+ f2fs_wait_on_page_writeback( -+ &folio->page, - DATA, true, true); - else - goto continue_unlock; - } - -- if (!clear_page_dirty_for_io(page)) -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - #ifdef CONFIG_F2FS_FS_COMPRESSION - if (f2fs_compressed_file(inode)) { -- get_page(page); -- f2fs_compress_ctx_add_page(&cc, page); -+ folio_get(folio); -+ f2fs_compress_ctx_add_page(&cc, &folio->page); - continue; - } - #endif -- ret = f2fs_write_single_data_page(page, &submitted, -- &bio, &last_block, wbc, io_type, -- 0, true); -+ ret = f2fs_write_single_data_page(&folio->page, -+ &submitted, &bio, &last_block, -+ wbc, io_type, 0, true); - if (ret == AOP_WRITEPAGE_ACTIVATE) -- unlock_page(page); -+ folio_unlock(folio); - #ifdef CONFIG_F2FS_FS_COMPRESSION - result: - #endif -@@ -3101,7 +3105,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - } - goto next; - } -- done_index = page->index + 1; -+ done_index = folio->index + -+ folio_nr_pages(folio); - done = 1; - break; - } -@@ -3115,7 +3120,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - if (need_readd) - goto readd; - } -- release_pages(pages, nr_pages); -+ folio_batch_release(&fbatch); - cond_resched(); - } - #ifdef CONFIG_F2FS_FS_COMPRESSION -diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h -index 3c7cdb70fe2e..dcb28240f724 100644 ---- a/fs/f2fs/f2fs.h -+++ b/fs/f2fs/f2fs.h -@@ -4196,8 +4196,9 @@ void f2fs_end_read_compressed_page(struct page *page, bool failed, - block_t blkaddr, bool in_task); - bool f2fs_cluster_is_empty(struct compress_ctx *cc); - bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index); --bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, -- int index, int nr_pages, bool uptodate); -+bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, -+ struct folio_batch *fbatch, int index, int nr_folios, -+ bool uptodate); - bool f2fs_sanity_check_cluster(struct dnode_of_data *dn); - void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page); - int f2fs_write_multi_pages(struct compress_ctx *cc, -diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c -index e06a0c478b39..b993be76013e 100644 ---- a/fs/f2fs/node.c -+++ b/fs/f2fs/node.c -@@ -1513,23 +1513,24 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) - static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) - { - pgoff_t index; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct page *last_page = NULL; -- int nr_pages; -+ int nr_folios; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - index = 0; - -- while ((nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, -- PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - - if (unlikely(f2fs_cp_error(sbi))) { - f2fs_put_page(last_page, 0); -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - return ERR_PTR(-EIO); - } - -@@ -1560,7 +1561,7 @@ static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) - last_page = page; - unlock_page(page); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - return last_page; -@@ -1726,12 +1727,12 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - unsigned int *seq_id) - { - pgoff_t index; -- struct pagevec pvec; -+ struct folio_batch fbatch; - int ret = 0; - struct page *last_page = NULL; - bool marked = false; - nid_t ino = inode->i_ino; -- int nr_pages; -+ int nr_folios; - int nwritten = 0; - - if (atomic) { -@@ -1740,20 +1741,21 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - return PTR_ERR_OR_ZERO(last_page); - } - retry: -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - index = 0; - -- while ((nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, -- PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - bool submitted = false; - - if (unlikely(f2fs_cp_error(sbi))) { - f2fs_put_page(last_page, 0); -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - ret = -EIO; - goto out; - } -@@ -1819,7 +1821,7 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - break; - } - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (ret || marked) -@@ -1884,17 +1886,18 @@ static bool flush_dirty_inode(struct page *page) - void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) - { - pgoff_t index = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while ((nr_pages = pagevec_lookup_tag(&pvec, -- NODE_MAPPING(sbi), &index, PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - - if (!IS_DNODE(page)) - continue; -@@ -1921,7 +1924,7 @@ void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) - } - unlock_page(page); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -@@ -1931,23 +1934,24 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, - bool do_balance, enum iostat_type io_type) - { - pgoff_t index; -- struct pagevec pvec; -+ struct folio_batch fbatch; - int step = 0; - int nwritten = 0; - int ret = 0; -- int nr_pages, done = 0; -+ int nr_folios, done = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - - next_step: - index = 0; - -- while (!done && (nr_pages = pagevec_lookup_tag(&pvec, -- NODE_MAPPING(sbi), &index, PAGECACHE_TAG_DIRTY))) { -+ while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), -+ &index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - bool submitted = false; - - /* give a priority to WB_SYNC threads */ -@@ -2022,7 +2026,7 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, - if (--wbc->nr_to_write == 0) - break; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (wbc->nr_to_write == 0) { -diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c -index 05bee80ac7de..8f87c2551a3d 100644 ---- a/fs/gfs2/aops.c -+++ b/fs/gfs2/aops.c -@@ -195,67 +195,71 @@ static int gfs2_writepages(struct address_space *mapping, - } - - /** -- * gfs2_write_jdata_pagevec - Write back a pagevec's worth of pages -+ * gfs2_write_jdata_batch - Write back a folio batch's worth of folios - * @mapping: The mapping - * @wbc: The writeback control -- * @pvec: The vector of pages -- * @nr_pages: The number of pages to write -+ * @fbatch: The batch of folios - * @done_index: Page index - * - * Returns: non-zero if loop should terminate, zero otherwise - */ - --static int gfs2_write_jdata_pagevec(struct address_space *mapping, -+static int gfs2_write_jdata_batch(struct address_space *mapping, - struct writeback_control *wbc, -- struct pagevec *pvec, -- int nr_pages, -+ struct folio_batch *fbatch, - pgoff_t *done_index) - { - struct inode *inode = mapping->host; - struct gfs2_sbd *sdp = GFS2_SB(inode); -- unsigned nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); -+ unsigned nrblocks; - int i; - int ret; -+ int nr_pages = 0; -+ int nr_folios = folio_batch_count(fbatch); -+ -+ for (i = 0; i < nr_folios; i++) -+ nr_pages += folio_nr_pages(fbatch->folios[i]); -+ nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); - - ret = gfs2_trans_begin(sdp, nrblocks, nrblocks); - if (ret < 0) - return ret; - -- for(i = 0; i < nr_pages; i++) { -- struct page *page = pvec->pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch->folios[i]; - -- *done_index = page->index; -+ *done_index = folio->index; - -- lock_page(page); -+ folio_lock(folio); - -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - else - goto continue_unlock; - } - -- BUG_ON(PageWriteback(page)); -- if (!clear_page_dirty_for_io(page)) -+ BUG_ON(folio_test_writeback(folio)); -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - trace_wbc_writepage(wbc, inode_to_bdi(inode)); - -- ret = __gfs2_jdata_writepage(page, wbc); -+ ret = __gfs2_jdata_writepage(&folio->page, wbc); - if (unlikely(ret)) { - if (ret == AOP_WRITEPAGE_ACTIVATE) { -- unlock_page(page); -+ folio_unlock(folio); - ret = 0; - } else { - -@@ -268,7 +272,8 @@ static int gfs2_write_jdata_pagevec(struct address_space *mapping, - * not be suitable for data integrity - * writeout). - */ -- *done_index = page->index + 1; -+ *done_index = folio->index + -+ folio_nr_pages(folio); - ret = 1; - break; - } -@@ -305,8 +310,8 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - { - int ret = 0; - int done = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - pgoff_t writeback_index; - pgoff_t index; - pgoff_t end; -@@ -315,7 +320,7 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - int range_whole = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - writeback_index = mapping->writeback_index; /* prev offset */ - index = writeback_index; -@@ -341,17 +346,18 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; - while (!done && (index <= end)) { -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ if (nr_folios == 0) - break; - -- ret = gfs2_write_jdata_pagevec(mapping, wbc, &pvec, nr_pages, &done_index); -+ ret = gfs2_write_jdata_batch(mapping, wbc, &fbatch, -+ &done_index); - if (ret) - done = 1; - if (ret > 0) - ret = 0; -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c -index 9f4d9432d38a..1e26f32a4e36 100644 ---- a/fs/nilfs2/btree.c -+++ b/fs/nilfs2/btree.c -@@ -2143,7 +2143,7 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, - struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; - struct address_space *btcache = btnc_inode->i_mapping; - struct list_head lists[NILFS_BTREE_LEVEL_MAX]; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct buffer_head *bh, *head; - pgoff_t index = 0; - int level, i; -@@ -2153,19 +2153,19 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, - level++) - INIT_LIST_HEAD(&lists[level]); - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while (pagevec_lookup_tag(&pvec, btcache, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- bh = head = page_buffers(pvec.pages[i]); -+ while (filemap_get_folios_tag(btcache, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ bh = head = folio_buffers(fbatch.folios[i]); - do { - if (buffer_dirty(bh)) - nilfs_btree_add_dirty_buffer(btree, - lists, bh); - } while ((bh = bh->b_this_page) != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c -index 3267e96c256c..b66f4e988016 100644 ---- a/fs/nilfs2/page.c -+++ b/fs/nilfs2/page.c -@@ -240,42 +240,43 @@ static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty) - int nilfs_copy_dirty_pages(struct address_space *dmap, - struct address_space *smap) - { -- struct pagevec pvec; -+ struct folio_batch fbatch; - unsigned int i; - pgoff_t index = 0; - int err = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - repeat: -- if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY)) -+ if (!filemap_get_folios_tag(smap, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) - return 0; - -- for (i = 0; i < pagevec_count(&pvec); i++) { -- struct page *page = pvec.pages[i], *dpage; -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ struct folio *folio = fbatch.folios[i], *dfolio; - -- lock_page(page); -- if (unlikely(!PageDirty(page))) -- NILFS_PAGE_BUG(page, "inconsistent dirty state"); -+ folio_lock(folio); -+ if (unlikely(!folio_test_dirty(folio))) -+ NILFS_PAGE_BUG(&folio->page, "inconsistent dirty state"); - -- dpage = grab_cache_page(dmap, page->index); -- if (unlikely(!dpage)) { -+ dfolio = filemap_grab_folio(dmap, folio->index); -+ if (unlikely(!dfolio)) { - /* No empty page is added to the page cache */ - err = -ENOMEM; -- unlock_page(page); -+ folio_unlock(folio); - break; - } -- if (unlikely(!page_has_buffers(page))) -- NILFS_PAGE_BUG(page, -+ if (unlikely(!folio_buffers(folio))) -+ NILFS_PAGE_BUG(&folio->page, - "found empty page in dat page cache"); - -- nilfs_copy_page(dpage, page, 1); -- __set_page_dirty_nobuffers(dpage); -+ nilfs_copy_page(&dfolio->page, &folio->page, 1); -+ filemap_dirty_folio(folio_mapping(dfolio), dfolio); - -- unlock_page(dpage); -- put_page(dpage); -- unlock_page(page); -+ folio_unlock(dfolio); -+ folio_put(dfolio); -+ folio_unlock(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (likely(!err)) -@@ -357,22 +358,22 @@ void nilfs_copy_back_pages(struct address_space *dmap, - */ - void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent) - { -- struct pagevec pvec; -+ struct folio_batch fbatch; - unsigned int i; - pgoff_t index = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while (pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- struct page *page = pvec.pages[i]; -+ while (filemap_get_folios_tag(mapping, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ struct folio *folio = fbatch.folios[i]; - -- lock_page(page); -- nilfs_clear_dirty_page(page, silent); -- unlock_page(page); -+ folio_lock(folio); -+ nilfs_clear_dirty_page(&folio->page, silent); -+ folio_unlock(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c -index 0afe0832c754..6f2ca279d230 100644 ---- a/fs/nilfs2/segment.c -+++ b/fs/nilfs2/segment.c -@@ -680,7 +680,7 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - loff_t start, loff_t end) - { - struct address_space *mapping = inode->i_mapping; -- struct pagevec pvec; -+ struct folio_batch fbatch; - pgoff_t index = 0, last = ULONG_MAX; - size_t ndirties = 0; - int i; -@@ -694,23 +694,26 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - index = start >> PAGE_SHIFT; - last = end >> PAGE_SHIFT; - } -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - repeat: - if (unlikely(index > last) || -- !pagevec_lookup_range_tag(&pvec, mapping, &index, last, -- PAGECACHE_TAG_DIRTY)) -+ !filemap_get_folios_tag(mapping, &index, last, -+ PAGECACHE_TAG_DIRTY, &fbatch)) - return ndirties; - -- for (i = 0; i < pagevec_count(&pvec); i++) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { - struct buffer_head *bh, *head; -- struct page *page = pvec.pages[i]; -+ struct folio *folio = fbatch.folios[i]; - -- lock_page(page); -- if (!page_has_buffers(page)) -- create_empty_buffers(page, i_blocksize(inode), 0); -- unlock_page(page); -+ folio_lock(folio); -+ head = folio_buffers(folio); -+ if (!head) { -+ create_empty_buffers(&folio->page, i_blocksize(inode), 0); -+ head = folio_buffers(folio); -+ } -+ folio_unlock(folio); - -- bh = head = page_buffers(page); -+ bh = head; - do { - if (!buffer_dirty(bh) || buffer_async_write(bh)) - continue; -@@ -718,13 +721,13 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - list_add_tail(&bh->b_assoc_buffers, listp); - ndirties++; - if (unlikely(ndirties >= nlimit)) { -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - return ndirties; - } - } while (bh = bh->b_this_page, bh != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - goto repeat; - } -@@ -734,20 +737,19 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, - { - struct nilfs_inode_info *ii = NILFS_I(inode); - struct inode *btnc_inode = ii->i_assoc_inode; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct buffer_head *bh, *head; - unsigned int i; - pgoff_t index = 0; - - if (!btnc_inode) - return; -+ folio_batch_init(&fbatch); - -- pagevec_init(&pvec); -- -- while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- bh = head = page_buffers(pvec.pages[i]); -+ while (filemap_get_folios_tag(btnc_inode->i_mapping, &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ bh = head = folio_buffers(fbatch.folios[i]); - do { - if (buffer_dirty(bh) && - !buffer_async_write(bh)) { -@@ -758,7 +760,7 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, - bh = bh->b_this_page; - } while (bh != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index 39f05e00dac4..9fa4c4e0809b 100644 ---- a/include/linux/pagemap.h -+++ b/include/linux/pagemap.h -@@ -547,6 +547,26 @@ static inline struct folio *filemap_lock_folio(struct address_space *mapping, - return __filemap_get_folio(mapping, index, FGP_LOCK, 0); - } - -+/** -+ * filemap_grab_folio - grab a folio from the page cache -+ * @mapping: The address space to search -+ * @index: The page index -+ * -+ * Looks up the page cache entry at @mapping & @index. If no folio is found, -+ * a new folio is created. The folio is locked, marked as accessed, and -+ * returned. -+ * -+ * Return: A found or created folio. NULL if no folio is found and failed to -+ * create a folio. -+ */ -+static inline struct folio *filemap_grab_folio(struct address_space *mapping, -+ pgoff_t index) -+{ -+ return __filemap_get_folio(mapping, index, -+ FGP_LOCK | FGP_ACCESSED | FGP_CREAT, -+ mapping_gfp_mask(mapping)); -+} -+ - /** - * find_get_page - find and get a page reference - * @mapping: the address_space to search -@@ -720,16 +740,8 @@ unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, - pgoff_t end, struct folio_batch *fbatch); - unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, - unsigned int nr_pages, struct page **pages); --unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, -- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages); --static inline unsigned find_get_pages_tag(struct address_space *mapping, -- pgoff_t *index, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages) --{ -- return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag, -- nr_pages, pages); --} -+unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, -+ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch); - - struct page *grab_cache_page_write_begin(struct address_space *mapping, - pgoff_t index); -diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h -index 215eb6c3bdc9..a520632297ac 100644 ---- a/include/linux/pagevec.h -+++ b/include/linux/pagevec.h -@@ -26,14 +26,6 @@ struct pagevec { - }; - - void __pagevec_release(struct pagevec *pvec); --unsigned pagevec_lookup_range_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, pgoff_t end, -- xa_mark_t tag); --static inline unsigned pagevec_lookup_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, xa_mark_t tag) --{ -- return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag); --} - - static inline void pagevec_init(struct pagevec *pvec) - { -diff --git a/mm/filemap.c b/mm/filemap.c -index 15800334147b..b986f246a6ae 100644 ---- a/mm/filemap.c -+++ b/mm/filemap.c -@@ -503,28 +503,30 @@ static void __filemap_fdatawait_range(struct address_space *mapping, - { - pgoff_t index = start_byte >> PAGE_SHIFT; - pgoff_t end = end_byte >> PAGE_SHIFT; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned nr_folios; - - if (end_byte < start_byte) - return; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); -+ - while (index <= end) { - unsigned i; - -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, -- end, PAGECACHE_TAG_WRITEBACK); -- if (!nr_pages) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ PAGECACHE_TAG_WRITEBACK, &fbatch); -+ -+ if (!nr_folios) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- wait_on_page_writeback(page); -- ClearPageError(page); -+ folio_wait_writeback(folio); -+ folio_clear_error(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -@@ -2255,64 +2257,57 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, - EXPORT_SYMBOL(find_get_pages_contig); - - /** -- * find_get_pages_range_tag - Find and return head pages matching @tag. -- * @mapping: the address_space to search -- * @index: the starting page index -- * @end: The final page index (inclusive) -- * @tag: the tag index -- * @nr_pages: the maximum number of pages -- * @pages: where the resulting pages are placed -+ * filemap_get_folios_tag - Get a batch of folios matching @tag. -+ * @mapping: The address_space to search -+ * @start: The starting page index -+ * @end: The final page index (inclusive) -+ * @tag: The tag index -+ * @fbatch: The batch to fill - * -- * Like find_get_pages_range(), except we only return head pages which are -- * tagged with @tag. @index is updated to the index immediately after the -- * last page we return, ready for the next iteration. -+ * Same as filemap_get_folios, but only returning folios tagged with @tag - * -- * Return: the number of pages which were found. -+ * Return: The number of folios found -+ * Also update @start to index the next folio for traversal - */ --unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, -- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages) -+unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, -+ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) - { -- XA_STATE(xas, &mapping->i_pages, *index); -+ XA_STATE(xas, &mapping->i_pages, *start); - struct folio *folio; -- unsigned ret = 0; -- -- if (unlikely(!nr_pages)) -- return 0; - - rcu_read_lock(); -- while ((folio = find_get_entry(&xas, end, tag))) { -- /* -- * Shadow entries should never be tagged, but this iteration -+ while ((folio = find_get_entry(&xas, end, tag)) != NULL) { -+ /* Shadow entries should never be tagged, but this iteration - * is lockless so there is a window for page reclaim to evict -- * a page we saw tagged. Skip over it. -+ * a page we saw tagged. Skip over it. - */ - if (xa_is_value(folio)) - continue; -+ if (!folio_batch_add(fbatch, folio)) { -+ unsigned long nr = folio_nr_pages(folio); - -- pages[ret] = &folio->page; -- if (++ret == nr_pages) { -- *index = folio->index + folio_nr_pages(folio); -+ if (folio_test_hugetlb(folio)) -+ nr = 1; -+ *start = folio->index + nr; - goto out; - } - } -- - /* -- * We come here when we got to @end. We take care to not overflow the -- * index @index as it confuses some of the callers. This breaks the -- * iteration when there is a page at index -1 but that is already -- * broken anyway. -+ * We come here when there is no page beyond @end. We take care to not -+ * overflow the index @start as it confuses some of the callers. This -+ * breaks the iteration when there is a page at index -1 but that is -+ * already broke anyway. - */ - if (end == (pgoff_t)-1) -- *index = (pgoff_t)-1; -+ *start = (pgoff_t)-1; - else -- *index = end + 1; -+ *start = end + 1; - out: - rcu_read_unlock(); - -- return ret; -+ return folio_batch_count(fbatch); - } --EXPORT_SYMBOL(find_get_pages_range_tag); -+EXPORT_SYMBOL(filemap_get_folios_tag); - - /* - * CD/DVDs are error prone. When a medium error occurs, the driver may fail -diff --git a/mm/page-writeback.c b/mm/page-writeback.c -index cbcecd565c16..8f20c037a87a 100644 ---- a/mm/page-writeback.c -+++ b/mm/page-writeback.c -@@ -2293,15 +2293,15 @@ int write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int error; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; - int range_whole = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* prev offset */ - end = -1; -@@ -2321,17 +2321,18 @@ int write_cache_pages(struct address_space *mapping, - while (!done && (index <= end)) { - int i; - -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- done_index = page->index; -+ done_index = folio->index; - -- lock_page(page); -+ folio_lock(folio); - - /* - * Page truncated or invalidated. We can freely skip it -@@ -2341,30 +2342,30 @@ int write_cache_pages(struct address_space *mapping, - * even if there is now a new, dirty page at the same - * pagecache address. - */ -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - else - goto continue_unlock; - } - -- BUG_ON(PageWriteback(page)); -- if (!clear_page_dirty_for_io(page)) -+ BUG_ON(folio_test_writeback(folio)); -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); -- error = (*writepage)(page, wbc, data); -+ error = writepage(&folio->page, wbc, data); - if (unlikely(error)) { - /* - * Handle errors according to the type of -@@ -2379,11 +2380,12 @@ int write_cache_pages(struct address_space *mapping, - * the first error. - */ - if (error == AOP_WRITEPAGE_ACTIVATE) { -- unlock_page(page); -+ folio_unlock(folio); - error = 0; - } else if (wbc->sync_mode != WB_SYNC_ALL) { - ret = error; -- done_index = page->index + 1; -+ done_index = folio->index + -+ folio_nr_pages(folio); - done = 1; - break; - } -@@ -2403,7 +2405,7 @@ int write_cache_pages(struct address_space *mapping, - break; - } - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/mm/swap.c b/mm/swap.c -index 027943b341a0..29bf97746da0 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -1099,16 +1099,6 @@ void folio_batch_remove_exceptionals(struct folio_batch *fbatch) - fbatch->nr = j; - } - --unsigned pagevec_lookup_range_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, pgoff_t end, -- xa_mark_t tag) --{ -- pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, -- PAGEVEC_SIZE, pvec->pages); -- return pagevec_count(pvec); --} --EXPORT_SYMBOL(pagevec_lookup_range_tag); -- - /* - * Perform any setup for the swap system - */ --- -2.37.3 - diff --git a/6.0/testing/0008-rcu.patch b/6.0/0011-rcu.patch similarity index 96% rename from 6.0/testing/0008-rcu.patch rename to 6.0/0011-rcu.patch index 2b6391cc..14d5e359 100644 --- a/6.0/testing/0008-rcu.patch +++ b/6.0/0011-rcu.patch @@ -1,7 +1,7 @@ -From bc469031493ed241d08816cacca6881482c8a46e Mon Sep 17 00:00:00 2001 +From 953761366f999b9035f8fff70c214426ad9f027b Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Wed, 14 Sep 2022 14:40:34 +0200 -Subject: [PATCH 08/13] rcu +Subject: [PATCH 11/16] rcu Signed-off-by: Peter Jung --- diff --git a/6.0/0012-fixes.patch b/6.0/0012-fixes.patch deleted file mode 100644 index c26c6a42..00000000 --- a/6.0/0012-fixes.patch +++ /dev/null @@ -1,1692 +0,0 @@ -From b639b03341d179c471fc8a536c7840d5d20e1016 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 19 Sep 2022 14:42:00 +0200 -Subject: [PATCH 12/13] fixes - -Signed-off-by: Peter Jung ---- - Documentation/ABI/stable/sysfs-block | 10 + - .../testing/sysfs-class-led-trigger-blkdev | 68 + - Documentation/leds/index.rst | 1 + - Documentation/leds/ledtrig-blkdev.rst | 155 +++ - arch/x86/events/amd/uncore.c | 1 + - arch/x86/kernel/alternative.c | 9 + - arch/x86/net/bpf_jit_comp.c | 4 +- - drivers/leds/trigger/Kconfig | 9 + - drivers/leds/trigger/Makefile | 1 + - drivers/leds/trigger/ledtrig-blkdev.c | 1177 +++++++++++++++++ - include/linux/pageblock-flags.h | 2 +- - include/net/bluetooth/rfcomm.h | 3 + - net/bluetooth/rfcomm/core.c | 2 + - net/bluetooth/rfcomm/sock.c | 34 +- - tools/objtool/check.c | 3 - - 15 files changed, 1462 insertions(+), 17 deletions(-) - create mode 100644 Documentation/ABI/testing/sysfs-class-led-trigger-blkdev - create mode 100644 Documentation/leds/ledtrig-blkdev.rst - create mode 100644 drivers/leds/trigger/ledtrig-blkdev.c - -diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block -index cd14ecb3c9a5..d11d04d804fe 100644 ---- a/Documentation/ABI/stable/sysfs-block -+++ b/Documentation/ABI/stable/sysfs-block -@@ -101,6 +101,16 @@ Description: - devices that support receiving integrity metadata. - - -+What: /sys/block//linked_leds -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Directory that contains symbolic links to all LEDs that -+ are associated with (linked to) this block device by the -+ blkdev LED trigger. Only present when at least one LED -+ is linked. (See Documentation/leds/ledtrig-blkdev.rst.) -+ -+ - What: /sys/block///alignment_offset - Date: April 2009 - Contact: Martin K. Petersen -diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev -new file mode 100644 -index 000000000000..d8d403569b21 ---- /dev/null -+++ b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev -@@ -0,0 +1,68 @@ -+What: /sys/class/leds//blink_time -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Time (in milliseconds) that the LED will be on during a single -+ "blink". -+ -+What: /sys/class/leds//check_interval -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Frequency (in milliseconds) with which block devices linked to -+ this LED will be checked for activity and the LED will -+ (potentially) be blinked. -+ -+What: /sys/class/leds//blink_on_read -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to read activity on any of its linked block devices. -+ -+What: /sys/class/leds//blink_on_write -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to write activity on any of its linked block devices. -+ -+What: /sys/class/leds//blink_on_discard -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to discard activity on any of its linked block devices. -+ -+What: /sys/class/leds//blink_on_flush -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to cache flush activity on any of its linked block devices. -+ -+What: /sys/class/leds//link_dev_by_path -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Associate a block device with this LED by writing the path to -+ the device special file (e.g. /dev/sda) to this attribute. -+ Symbolic links are followed. -+ -+What: /sys/class/leds//unlink_dev_by_path -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Remove the association between this LED and a block device by -+ writing the path to the device special file (e.g. /dev/sda) to -+ this attribute. Symbolic links are followed. -+ -+What: /sys/class/leds//linked_devices -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Directory containing links to all block devices that are -+ associated with this LED. (Note that the names of the -+ symbolic links in this directory are *kernel* names, which -+ may not match the device special file paths written to -+ link_device and unlink_device.) -diff --git a/Documentation/leds/index.rst b/Documentation/leds/index.rst -index e5d63b940045..e3c24e468cbc 100644 ---- a/Documentation/leds/index.rst -+++ b/Documentation/leds/index.rst -@@ -10,6 +10,7 @@ LEDs - leds-class - leds-class-flash - leds-class-multicolor -+ ledtrig-blkdev - ledtrig-oneshot - ledtrig-transient - ledtrig-usbport -diff --git a/Documentation/leds/ledtrig-blkdev.rst b/Documentation/leds/ledtrig-blkdev.rst -new file mode 100644 -index 000000000000..ae92aa559257 ---- /dev/null -+++ b/Documentation/leds/ledtrig-blkdev.rst -@@ -0,0 +1,155 @@ -+.. SPDX-License-Identifier: GPL-2.0 -+ -+================================= -+Block Device (blkdev) LED Trigger -+================================= -+ -+Available when ``CONFIG_LEDS_TRIGGER_BLKDEV=y`` or -+``CONFIG_LEDS_TRIGGER_BLKDEV=m``. -+ -+See also: -+ -+* ``Documentation/ABI/testing/sysfs-class-led-trigger-blkdev`` -+* ``Documentation/ABI/stable/sysfs-block`` (``/sys/block//linked_leds``) -+ -+Overview -+======== -+ -+.. note:: -+ The examples below use ```` to refer to the name of a -+ system-specific LED. If no suitable LED is available on a test -+ system (in a virtual machine, for example), it is possible to -+ use a userspace LED. (See ``Documentation/leds/uleds.rst``.) -+ -+Verify that the ``blkdev`` LED trigger is available:: -+ -+ # grep blkdev /sys/class/leds//trigger -+ ... rfkill-none blkdev -+ -+(If the previous command produces no output, you may need to load the trigger -+module - ``modprobe ledtrig_blkdev``. If the module is not available, check -+the value of ``CONFIG_LEDS_TRIGGER_BLKDEV`` in your kernel configuration.) -+ -+Associate the LED with the ``blkdev`` LED trigger:: -+ -+ # echo blkdev > /sys/class/leds//trigger -+ -+ # cat /sys/class/leds//trigger -+ ... rfkill-none [blkdev] -+ -+Note that several new device attributes are available in the -+``/sys/class/leds/`` directory. -+ -+* ``link_dev_by_path`` and ``unlink_dev_by_path`` are used to manage the set of -+ block devices associated with this LED. The LED will blink in response to -+ read or write activity on its linked devices. -+ -+* ``blink_on_read``, ``blink_on_write``, ``blink_on_discard``, and -+ ``blink_on_flush`` are boolean values that determine whether the LED will -+ blink when a particular type of activity is detected on one of its linked -+ block devices. -+ -+* ``blink_time`` is the duration (in milliseconds) of each blink of this LED. -+ (The minimum value is 10 milliseconds.) -+ -+* ``check_interval`` is the frequency (in milliseconds) with which block devices -+ linked to this LED will be checked for activity and the LED blinked (if the -+ correct type of activity has occurred). -+ -+* The ``linked_devices`` directory will contain a symbolic link to every device -+ that is associated with this LED. -+ -+Link a block device to the LED:: -+ -+ # echo /dev/sda > /sys/class/leds//link_dev_by_path -+ -+ # ls /sys/class/leds//linked_devices -+ sda -+ -+(The value written to ``link_dev_by_path`` must be the path of the device -+special file, such as ``/dev/sda``, that represents the block device - or the -+path of a symbolic link to such a device special file.) -+ -+Activity on the device will now cause the LED to blink. The duration of each -+blink (in milliseconds) can be adjusted by setting -+``/sys/class/leds//blink_time``. (But see **check_interval and -+blink_time** below.) -+ -+Associate a second device with the LED:: -+ -+ # echo /dev/sdb > /sys/class/leds//link_dev_by_path -+ -+ # ls /sys/class/leds//linked_devices -+ sda sdb -+ -+When a block device is linked to one or more LEDs, the LEDs are linked from -+the device's ``linked_leds`` directory:: -+ -+ # ls /sys/class/block/sd{a,b}/linked_leds -+ /sys/class/block/sda/linked_leds: -+ -+ -+ /sys/class/block/sdb/linked_leds: -+ -+ -+(The ``linked_leds`` directory only exists when the block device is linked to -+at least one LED.) -+ -+``check_interval`` and ``blink_time`` -+===================================== -+ -+* By default, linked block devices are checked for activity every 100 -+ milliseconds. This frequency can be changed for an LED via the -+ ``/sys/class/leds//check_interval`` attribute. (The minimum value is 25 -+ milliseconds.) -+ -+* All block devices associated with an LED are checked for activity every -+ ``check_interval`` milliseconds, and a blink is triggered if the correct type -+ of activity (as determined by the LED's ``blink_on_*`` attributes) is -+ detected. The duration of an LED's blink is determined by its ``blink_time`` -+ attribute. Thus (when the correct type of activity is detected), the LED will -+ be on for ``blink_time`` milliseconds and off for -+ ``check_interval - blink_time`` milliseconds. -+ -+* The LED subsystem ignores new blink requests for an LED that is already in -+ in the process of blinking, so setting a ``blink_time`` greater than or equal -+ to ``check_interval`` will cause some blinks to be missed. -+ -+* Because of processing times, scheduling latencies, etc., avoiding missed -+ blinks actually requires a difference of at least a few milliseconds between -+ the ``blink_time`` and ``check_interval``. The required difference is likely -+ to vary from system to system. As a reference, a Thecus N5550 NAS requires a -+ difference of 7 milliseconds (e.g. ``check_interval == 100``, -+ ``blink_time == 93``). -+ -+* The default values (``check_interval == 100``, ``blink_time == 75``) cause the -+ LED associated with a continuously active device to blink rapidly. For a more -+ "always on" effect, increase the ``blink_time`` (but not too much; see the -+ previous bullet). -+ -+Other Notes -+=========== -+ -+* Many (possibly all) types of block devices work with this trigger, including: -+ -+ * SCSI (including SATA and USB) hard disk drives and SSDs -+ * SCSI (including SATA and USB) optical drives -+ * NVMe SSDs -+ * SD cards -+ * loopback block devices (``/dev/loop*``) -+ * device mapper devices, such as LVM logical volumes -+ * MD RAID devices -+ * zRAM compressed RAM-disks -+ * partitions on block devics that support them -+ -+* The names of the symbolic links in ``/sys/class/leds//linked_devices`` -+ are **kernel** names, which may not match the paths used for -+ ``link_dev_by_path`` and ``unlink_dev_by_path``. This is most likely when a -+ symbolic link is used to refer to the device (as is common with logical -+ volumes), but it can be true for any device, because nothing prevents the -+ creation of device special files with arbitrary names (e.g. -+ ``sudo mknod /foo b 8 0``). -+ -+* The ``blkdev`` LED trigger supports many-to-many device/LED associations. -+ A device can be associated with multiple LEDs, and an LED can be associated -+ with multiple devices. -diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c -index d568afc705d2..83f15fe411b3 100644 ---- a/arch/x86/events/amd/uncore.c -+++ b/arch/x86/events/amd/uncore.c -@@ -553,6 +553,7 @@ static void uncore_clean_online(void) - - hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { - hlist_del(&uncore->node); -+ kfree(uncore->events); - kfree(uncore); - } - } -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index f9c9b5850847..3582e61ddce6 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) - return ret; - i += ret; - -+ /* -+ * The compiler is supposed to EMIT an INT3 after every unconditional -+ * JMP instruction due to AMD BTC. However, if the compiler is too old -+ * or SLS isn't enabled, we still need an INT3 after indirect JMPs -+ * even on Intel. -+ */ -+ if (op == JMP32_INSN_OPCODE && i < insn->length) -+ bytes[i++] = INT3_INSN_OPCODE; -+ - for (; i < insn->length;) - bytes[i++] = BYTES_NOP1; - -diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c -index c1f6c1c51d99..4922517ddb0d 100644 ---- a/arch/x86/net/bpf_jit_comp.c -+++ b/arch/x86/net/bpf_jit_comp.c -@@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) - OPTIMIZER_HIDE_VAR(reg); - emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); - } else { -- EMIT2(0xFF, 0xE0 + reg); -+ EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ -+ if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS)) -+ EMIT1(0xCC); /* int3 */ - } - - *pprog = prog; -diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig -index dc6816d36d06..bda249068182 100644 ---- a/drivers/leds/trigger/Kconfig -+++ b/drivers/leds/trigger/Kconfig -@@ -154,4 +154,13 @@ config LEDS_TRIGGER_TTY - - When build as a module this driver will be called ledtrig-tty. - -+config LEDS_TRIGGER_BLKDEV -+ tristate "LED Trigger for block devices" -+ depends on BLOCK -+ help -+ The blkdev LED trigger allows LEDs to be controlled by block device -+ activity (reads and writes). -+ -+ See Documentation/leds/ledtrig-blkdev.rst. -+ - endif # LEDS_TRIGGERS -diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile -index 25c4db97cdd4..d53bab5d93f1 100644 ---- a/drivers/leds/trigger/Makefile -+++ b/drivers/leds/trigger/Makefile -@@ -16,3 +16,4 @@ obj-$(CONFIG_LEDS_TRIGGER_NETDEV) += ledtrig-netdev.o - obj-$(CONFIG_LEDS_TRIGGER_PATTERN) += ledtrig-pattern.o - obj-$(CONFIG_LEDS_TRIGGER_AUDIO) += ledtrig-audio.o - obj-$(CONFIG_LEDS_TRIGGER_TTY) += ledtrig-tty.o -+obj-$(CONFIG_LEDS_TRIGGER_BLKDEV) += ledtrig-blkdev.o -diff --git a/drivers/leds/trigger/ledtrig-blkdev.c b/drivers/leds/trigger/ledtrig-blkdev.c -new file mode 100644 -index 000000000000..0d2eb36fbe38 ---- /dev/null -+++ b/drivers/leds/trigger/ledtrig-blkdev.c -@@ -0,0 +1,1177 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * Block device LED trigger -+ * -+ * Copyright 2021-2022 Ian Pilcher -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+/** -+ * DOC: Overview -+ * -+ * The ``blkdev`` LED trigger works by periodically checking the activity -+ * counters of block devices that have been linked to one or more LEDs and -+ * blinking those LED(s) if the correct type of activity has occurred. The -+ * periodic check is scheduled with the Linux kernel's deferred work facility. -+ * -+ * Trigger-specific data about block devices and LEDs is stored in two data -+ * structures --- &struct blkdev_trig_bdev (a "BTB") and &struct blkdev_trig_led -+ * (a "BTL"). Each structure contains a &struct xarray that holds links to any -+ * linked devices of the other type. I.e. &blkdev_trig_bdev.linked_btls -+ * contains links to all BTLs whose LEDs have been linked to the BTB's block -+ * device, and &blkdev_trig_led.linked_btbs contains links to all BTBs whose -+ * block devices have been linked to the BTL's LED. Thus, a block device can -+ * be linked to more than one LED, and an LED can be linked to more than one -+ * block device. -+ */ -+ -+/* Default, minimum & maximum blink duration (milliseconds) */ -+#define BLKDEV_TRIG_BLINK_DEF 75 -+#define BLKDEV_TRIG_BLINK_MIN 10 -+#define BLKDEV_TRIG_BLINK_MAX 86400000 /* 24 hours */ -+ -+/* Default, minimum & maximum activity check interval (milliseconds) */ -+#define BLKDEV_TRIG_CHECK_DEF 100 -+#define BLKDEV_TRIG_CHECK_MIN 25 -+#define BLKDEV_TRIG_CHECK_MAX 86400000 /* 24 hours */ -+ -+/* -+ * If blkdev_trig_check() can't lock the mutex, how long to wait before trying -+ * again (milliseconds) -+ */ -+#define BLKDEV_TRIG_CHECK_RETRY 5 -+ -+/* Mode argument for calls to blkdev_get_by_path() and blkdev_put() */ -+#define BLKDEV_TRIG_FMODE 0 -+ -+/** -+ * struct blkdev_trig_bdev - Trigger-specific data about a block device. -+ * @last_checked: Time (in jiffies) at which the trigger last checked this -+ * block device for activity. -+ * @last_activity: Time (in jiffies) at which the trigger last detected -+ * activity of each type. -+ * @ios: Activity counter values for each type, corresponding to -+ * the timestamps in &last_activity. -+ * @index: &xarray index, so the BTB can be included in one or more -+ * &blkdev_trig_led.linked_btbs. -+ * @bdev: The block device. -+ * @linked_btls: The BTLs that represent the LEDs linked to the BTB's -+ * block device. -+ * -+ * Every block device linked to at least one LED gets a "BTB." A BTB is created -+ * when a block device that is not currently linked to any LEDs is linked to an -+ * LED. -+ * -+ * A BTB is freed when one of the following occurs: -+ * -+ * * The number of LEDs linked to the block device becomes zero, because it has -+ * been unlinked from its last LED using the trigger's &sysfs interface. -+ * -+ * * The number of LEDs linked to the block device becomes zero, because the -+ * last LED to which it was linked has been disassociated from the trigger -+ * (which happens automatically if the LED device is removed from the system). -+ * -+ * * The BTB's block device is removed from the system. To accomodate this -+ * scenario, BTB's are created as device resources, so that the release -+ * function will be called by the driver core when the device is removed. -+ */ -+struct blkdev_trig_bdev { -+ unsigned long last_checked; -+ unsigned long last_activity[NR_STAT_GROUPS]; -+ unsigned long ios[NR_STAT_GROUPS]; -+ unsigned long index; -+ struct block_device *bdev; -+ struct xarray linked_btls; -+}; -+ -+/** -+ * struct blkdev_trig_led - Trigger-specific data about an LED. -+ * @last_checked: Time (in jiffies) at which the trigger last checked the -+ * the block devices linked to this LED for activity. -+ * @index: &xarray index, so the BTL can be included in one or more -+ * &blkdev_trig_bdev.linked_btls. -+ * @mode: Bitmask for types of block device activity that will -+ * cause this LED to blink --- reads, writes, discards, -+ * etc. -+ * @led: The LED device. -+ * @blink_msec: Duration of a blink (milliseconds). -+ * @check_jiffies: Frequency with which block devices linked to this LED -+ * should be checked for activity (jiffies). -+ * @linked_btbs: The BTBs that represent the block devices linked to the -+ * BTL's LED. -+ * @all_btls_node: The BTL's node in the module's list of all BTLs. -+ * -+ * Every LED associated with the block device trigger gets a "BTL." A BTL is -+ * created when the trigger is "activated" on an LED (usually by writing -+ * ``blkdev`` to the LED's &sysfs &trigger attribute). A BTL is freed wnen its -+ * LED is disassociated from the trigger, either through the trigger's &sysfs -+ * interface or because the LED device is removed from the system. -+ */ -+struct blkdev_trig_led { -+ unsigned long last_checked; -+ unsigned long index; -+ unsigned long mode; /* must be ulong for atomic bit ops */ -+ struct led_classdev *led; -+ unsigned int blink_msec; -+ unsigned int check_jiffies; -+ struct xarray linked_btbs; -+ struct hlist_node all_btls_node; -+}; -+ -+/* Protects everything except atomic LED attributes */ -+static DEFINE_MUTEX(blkdev_trig_mutex); -+ -+/* BTB device resource release function */ -+static void blkdev_trig_btb_release(struct device *dev, void *res); -+ -+/* Index for next BTB or BTL */ -+static unsigned long blkdev_trig_next_index; -+ -+/* All LEDs associated with the trigger */ -+static HLIST_HEAD(blkdev_trig_all_btls); -+ -+/* Delayed work to periodically check for activity & blink LEDs */ -+static void blkdev_trig_check(struct work_struct *work); -+static DECLARE_DELAYED_WORK(blkdev_trig_work, blkdev_trig_check); -+ -+/* When is the delayed work scheduled to run next (jiffies) */ -+static unsigned long blkdev_trig_next_check; -+ -+/* Total number of BTB-to-BTL links */ -+static unsigned int blkdev_trig_link_count; -+ -+/* Empty sysfs attribute list for next 2 declarations */ -+static struct attribute *blkdev_trig_attrs_empty[] = { NULL }; -+ -+/* linked_leds sysfs directory for block devs linked to 1 or more LEDs */ -+static const struct attribute_group blkdev_trig_linked_leds = { -+ .name = "linked_leds", -+ .attrs = blkdev_trig_attrs_empty, -+}; -+ -+/* linked_devices sysfs directory for each LED associated with the trigger */ -+static const struct attribute_group blkdev_trig_linked_devs = { -+ .name = "linked_devices", -+ .attrs = blkdev_trig_attrs_empty, -+}; -+ -+ -+/* -+ * -+ * Delayed work to check for activity & blink LEDs -+ * -+ */ -+ -+/** -+ * blkdev_trig_blink() - Blink an LED, if the correct type of activity has -+ * occurred on the block device. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: &true if the LED is blinked, &false if not. -+ */ -+static bool blkdev_trig_blink(const struct blkdev_trig_led *btl, -+ const struct blkdev_trig_bdev *btb) -+{ -+ unsigned long mode, mask, delay_on, delay_off; -+ enum stat_group i; -+ -+ mode = READ_ONCE(btl->mode); -+ -+ for (i = STAT_READ, mask = 1; i <= STAT_FLUSH; ++i, mask <<= 1) { -+ -+ if (!(mode & mask)) -+ continue; -+ -+ if (time_before_eq(btb->last_activity[i], btl->last_checked)) -+ continue; -+ -+ delay_on = READ_ONCE(btl->blink_msec); -+ delay_off = 1; /* 0 leaves LED turned on */ -+ -+ led_blink_set_oneshot(btl->led, &delay_on, &delay_off, 0); -+ return true; -+ } -+ -+ return false; -+} -+ -+/** -+ * blkdev_trig_update_btb() - Update a BTB's activity counters and timestamps. -+ * @btb: The BTB -+ * @now: Timestamp (in jiffies) -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_update_btb(struct blkdev_trig_bdev *btb, -+ unsigned long now) -+{ -+ unsigned long new_ios; -+ enum stat_group i; -+ -+ for (i = STAT_READ; i <= STAT_FLUSH; ++i) { -+ -+ new_ios = part_stat_read(btb->bdev, ios[i]); -+ -+ if (new_ios != btb->ios[i]) { -+ btb->ios[i] = new_ios; -+ btb->last_activity[i] = now; -+ } -+ } -+ -+ btb->last_checked = now; -+} -+ -+/** -+ * blkdev_trig_check() - Check linked devices for activity and blink LEDs. -+ * @work: Delayed work (&blkdev_trig_work) -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_check(struct work_struct *work) -+{ -+ struct blkdev_trig_led *btl; -+ struct blkdev_trig_bdev *btb; -+ unsigned long index, delay, now, led_check, led_delay; -+ bool blinked; -+ -+ if (!mutex_trylock(&blkdev_trig_mutex)) { -+ delay = msecs_to_jiffies(BLKDEV_TRIG_CHECK_RETRY); -+ goto exit_reschedule; -+ } -+ -+ now = jiffies; -+ delay = ULONG_MAX; -+ -+ hlist_for_each_entry (btl, &blkdev_trig_all_btls, all_btls_node) { -+ -+ led_check = btl->last_checked + btl->check_jiffies; -+ -+ if (time_before_eq(led_check, now)) { -+ -+ blinked = false; -+ -+ xa_for_each (&btl->linked_btbs, index, btb) { -+ -+ if (btb->last_checked != now) -+ blkdev_trig_update_btb(btb, now); -+ if (!blinked) -+ blinked = blkdev_trig_blink(btl, btb); -+ } -+ -+ btl->last_checked = now; -+ led_delay = btl->check_jiffies; -+ -+ } else { -+ led_delay = led_check - now; -+ } -+ -+ if (led_delay < delay) -+ delay = led_delay; -+ } -+ -+ mutex_unlock(&blkdev_trig_mutex); -+ -+exit_reschedule: -+ WARN_ON_ONCE(delay == ULONG_MAX); -+ WARN_ON_ONCE(!schedule_delayed_work(&blkdev_trig_work, delay)); -+} -+ -+/** -+ * blkdev_trig_sched_led() - Set the schedule of the delayed work when a new -+ * LED is added to the schedule. -+ * @btl: The BTL that represents the LED -+ * -+ * Called when the number of block devices to which an LED is linked becomes -+ * non-zero. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_sched_led(const struct blkdev_trig_led *btl) -+{ -+ unsigned long delay = READ_ONCE(btl->check_jiffies); -+ unsigned long check_by = jiffies + delay; -+ -+ /* -+ * If no other LED-to-block device links exist, simply schedule the -+ * delayed work according to this LED's check_interval attribute -+ * (check_jiffies). -+ */ -+ if (blkdev_trig_link_count == 0) { -+ WARN_ON(!schedule_delayed_work(&blkdev_trig_work, delay)); -+ blkdev_trig_next_check = check_by; -+ return; -+ } -+ -+ /* -+ * If the next check is already scheduled to occur soon enough to -+ * accomodate this LED's check_interval, the schedule doesn't need -+ * to be changed. -+ */ -+ if (time_after_eq(check_by, blkdev_trig_next_check)) -+ return; -+ -+ /* -+ * Modify the schedule, so that the delayed work runs soon enough for -+ * this LED. -+ */ -+ WARN_ON(!mod_delayed_work(system_wq, &blkdev_trig_work, delay)); -+ blkdev_trig_next_check = check_by; -+} -+ -+ -+/* -+ * -+ * Linking and unlinking LEDs and block devices -+ * -+ */ -+ -+/** -+ * blkdev_trig_link() - Link a block device to an LED. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: &0 on success, negative &errno on error. -+ */ -+static int blkdev_trig_link(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ bool led_first_link; -+ int err; -+ -+ led_first_link = xa_empty(&btl->linked_btbs); -+ -+ err = xa_insert(&btb->linked_btls, btl->index, btl, GFP_KERNEL); -+ if (err) -+ return err; -+ -+ err = xa_insert(&btl->linked_btbs, btb->index, btb, GFP_KERNEL); -+ if (err) -+ goto error_erase_btl; -+ -+ /* Create /sys/class/block//linked_leds/ symlink */ -+ err = sysfs_add_link_to_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ &btl->led->dev->kobj, btl->led->name); -+ if (err) -+ goto error_erase_btb; -+ -+ /* Create /sys/class/leds//linked_devices/ symlink */ -+ err = sysfs_add_link_to_group(&btl->led->dev->kobj, -+ blkdev_trig_linked_devs.name, -+ bdev_kobj(btb->bdev), -+ dev_name(&btb->bdev->bd_device)); -+ if (err) -+ goto error_remove_symlink; -+ -+ /* -+ * If this is the first block device linked to this LED, the delayed -+ * work schedule may need to be changed. -+ */ -+ if (led_first_link) -+ blkdev_trig_sched_led(btl); -+ -+ ++blkdev_trig_link_count; -+ -+ return 0; -+ -+error_remove_symlink: -+ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ btl->led->name); -+error_erase_btb: -+ xa_erase(&btl->linked_btbs, btb->index); -+error_erase_btl: -+ xa_erase(&btb->linked_btls, btl->index); -+ return err; -+} -+ -+/** -+ * blkdev_trig_put_btb() - Remove and free a BTB, if it is no longer needed. -+ * @btb: The BTB -+ * -+ * Does nothing if the BTB (block device) is still linked to at least one LED. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_put_btb(struct blkdev_trig_bdev *btb) -+{ -+ struct block_device *bdev = btb->bdev; -+ int err; -+ -+ if (xa_empty(&btb->linked_btls)) { -+ -+ sysfs_remove_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); -+ err = devres_destroy(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ WARN_ON(err); -+ } -+} -+ -+/** -+ * _blkdev_trig_unlink_always() - Perform the unconditionally required steps of -+ * unlinking a block device from an LED. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * When a block device is unlinked from an LED, certain steps must be performed -+ * only if the block device is **not** being released. This function performs -+ * those steps that are **always** required, whether or not the block device is -+ * being released. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void _blkdev_trig_unlink_always(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ --blkdev_trig_link_count; -+ -+ if (blkdev_trig_link_count == 0) -+ WARN_ON(!cancel_delayed_work_sync(&blkdev_trig_work)); -+ -+ xa_erase(&btb->linked_btls, btl->index); -+ xa_erase(&btl->linked_btbs, btb->index); -+ -+ /* Remove /sys/class/leds//linked_devices/ symlink */ -+ sysfs_remove_link_from_group(&btl->led->dev->kobj, -+ blkdev_trig_linked_devs.name, -+ dev_name(&btb->bdev->bd_device)); -+} -+ -+/** -+ * blkdev_trig_unlink_norelease() - Unlink an LED from a block device that is -+ * **not** being released. -+ * @btl: The BTL that represents the LED. -+ * @btb: The BTB that represents the block device. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_unlink_norelease(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ _blkdev_trig_unlink_always(btl, btb); -+ -+ /* Remove /sys/class/block//linked_leds/ symlink */ -+ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ btl->led->name); -+ -+ blkdev_trig_put_btb(btb); -+} -+ -+/** -+ * blkdev_trig_unlink_release() - Unlink an LED from a block device that is -+ * being released. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_unlink_release(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ _blkdev_trig_unlink_always(btl, btb); -+ -+ /* -+ * If the BTB is being released, the driver core has already removed the -+ * device's attribute groups, and the BTB will be freed automatically, -+ * so there's nothing else to do. -+ */ -+} -+ -+ -+/* -+ * -+ * BTB creation -+ * -+ */ -+ -+/** -+ * blkdev_trig_btb_release() - BTB device resource release function. -+ * @dev: The block device -+ * @res: The BTB -+ * -+ * Called by the driver core when a block device with a BTB is removed. -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_btb_release(struct device *dev, void *res) -+{ -+ struct blkdev_trig_bdev *btb = res; -+ struct blkdev_trig_led *btl; -+ unsigned long index; -+ -+ mutex_lock(&blkdev_trig_mutex); -+ -+ xa_for_each (&btb->linked_btls, index, btl) -+ blkdev_trig_unlink_release(btl, btb); -+ -+ mutex_unlock(&blkdev_trig_mutex); -+} -+ -+/** -+ * blkdev_trig_get_bdev() - Get a block device by path. -+ * @path: The value written to an LED's &link_dev_by_path or -+ * &unlink_dev_by_path attribute, which should be the path to a -+ * special file that represents a block device -+ * @len: The number of characters in &path (not including its -+ * terminating null) -+ * -+ * The caller must call blkdev_put() when finished with the device. -+ * -+ * Context: Process context. -+ * Return: The block device, or an error pointer. -+ */ -+static struct block_device *blkdev_trig_get_bdev(const char *path, size_t len) -+{ -+ struct block_device *bdev; -+ char *buf; -+ -+ buf = kmemdup(path, len + 1, GFP_KERNEL); /* +1 to include null */ -+ if (buf == NULL) -+ return ERR_PTR(-ENOMEM); -+ -+ bdev = blkdev_get_by_path(strim(buf), BLKDEV_TRIG_FMODE, THIS_MODULE); -+ kfree(buf); -+ return bdev; -+} -+ -+/** -+ * blkdev_trig_get_btb() - Find or create the BTB for a block device. -+ * @path: The value written to an LED's &link_dev_by_path attribute, -+ * which should be the path to a special file that represents a -+ * block device -+ * @len: The number of characters in &path -+ * -+ * If a new BTB is created, because the block device was not previously linked -+ * to any LEDs, the block device's &linked_leds &sysfs directory is created. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: Pointer to the BTB, error pointer on error. -+ */ -+static struct blkdev_trig_bdev *blkdev_trig_get_btb(const char *path, -+ size_t len) -+{ -+ struct block_device *bdev; -+ struct blkdev_trig_bdev *btb; -+ int err; -+ -+ bdev = blkdev_trig_get_bdev(path, len); -+ if (IS_ERR(bdev)) -+ return ERR_CAST(bdev); -+ -+ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ if (btb != NULL) { -+ err = 0; -+ goto exit_put_bdev; -+ } -+ -+ if (blkdev_trig_next_index == ULONG_MAX) { -+ err = -EOVERFLOW; -+ goto exit_put_bdev; -+ } -+ -+ btb = devres_alloc(blkdev_trig_btb_release, sizeof(*btb), GFP_KERNEL); -+ if (btb == NULL) { -+ err = -ENOMEM; -+ goto exit_put_bdev; -+ } -+ -+ err = sysfs_create_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); -+ if (err) -+ goto exit_free_btb; -+ -+ btb->index = blkdev_trig_next_index++; -+ btb->bdev = bdev; -+ xa_init(&btb->linked_btls); -+ -+ /* Populate BTB activity counters */ -+ blkdev_trig_update_btb(btb, jiffies); -+ -+ devres_add(&bdev->bd_device, btb); -+ -+exit_free_btb: -+ if (err) -+ devres_free(btb); -+exit_put_bdev: -+ blkdev_put(bdev, BLKDEV_TRIG_FMODE); -+ return err ? ERR_PTR(err) : btb; -+} -+ -+ -+/* -+ * -+ * Activating and deactivating the trigger on an LED -+ * -+ */ -+ -+/** -+ * blkdev_trig_activate() - Called by the LEDs subsystem when an LED is -+ * associated with the trigger. -+ * @led: The LED -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &0 on success, negative &errno on error. -+ */ -+static int blkdev_trig_activate(struct led_classdev *led) -+{ -+ struct blkdev_trig_led *btl; -+ int err; -+ -+ btl = kzalloc(sizeof(*btl), GFP_KERNEL); -+ if (btl == NULL) -+ return -ENOMEM; -+ -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ goto exit_free; -+ -+ if (blkdev_trig_next_index == ULONG_MAX) { -+ err = -EOVERFLOW; -+ goto exit_unlock; -+ } -+ -+ btl->index = blkdev_trig_next_index++; -+ btl->last_checked = jiffies; -+ btl->mode = -1; /* set all bits */ -+ btl->led = led; -+ btl->blink_msec = BLKDEV_TRIG_BLINK_DEF; -+ btl->check_jiffies = msecs_to_jiffies(BLKDEV_TRIG_CHECK_DEF); -+ xa_init(&btl->linked_btbs); -+ -+ hlist_add_head(&btl->all_btls_node, &blkdev_trig_all_btls); -+ led_set_trigger_data(led, btl); -+ -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+exit_free: -+ if (err) -+ kfree(btl); -+ return err; -+} -+ -+/** -+ * blkdev_trig_deactivate() - Called by the the LEDs subsystem when an LED is -+ * disassociated from the trigger. -+ * @led: The LED -+ * -+ * The LEDs subsystem also calls this function when an LED associated with the -+ * trigger is removed or when the trigger is unregistered (if the module is -+ * unloaded). -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_deactivate(struct led_classdev *led) -+{ -+ struct blkdev_trig_led *btl = led_get_trigger_data(led); -+ struct blkdev_trig_bdev *btb; -+ unsigned long index; -+ -+ mutex_lock(&blkdev_trig_mutex); -+ -+ xa_for_each (&btl->linked_btbs, index, btb) -+ blkdev_trig_unlink_norelease(btl, btb); -+ -+ hlist_del(&btl->all_btls_node); -+ kfree(btl); -+ -+ mutex_unlock(&blkdev_trig_mutex); -+} -+ -+ -+/* -+ * -+ * Link-related attribute store functions -+ * -+ */ -+ -+/** -+ * link_dev_by_path_store() - &link_dev_by_path device attribute store function. -+ * @dev: The LED device -+ * @attr: The &link_dev_by_path attribute (&dev_attr_link_dev_by_path) -+ * @buf: The value written to the attribute, which should be the path to -+ * a special file that represents a block device to be linked to -+ * the LED (e.g. ``/dev/sda``) -+ * @count: The number of characters in &buf -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t link_dev_by_path_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ struct blkdev_trig_bdev *btb; -+ int err; -+ -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ return err; -+ -+ btb = blkdev_trig_get_btb(buf, count); -+ if (IS_ERR(btb)) { -+ err = PTR_ERR(btb); -+ goto exit_unlock; -+ } -+ -+ if (xa_load(&btb->linked_btls, btl->index) != NULL) { -+ err = -EEXIST; -+ goto exit_put_btb; -+ } -+ -+ err = blkdev_trig_link(btl, btb); -+ -+exit_put_btb: -+ if (err) -+ blkdev_trig_put_btb(btb); -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+ return err ? : count; -+} -+ -+/** -+ * unlink_dev_by_path_store() - &unlink_dev_by_path device attribute store -+ * function. -+ * @dev: The LED device -+ * @attr: The &unlink_dev_by_path attribute (&dev_attr_unlink_dev_by_path) -+ * @buf: The value written to the attribute, which should be the path to -+ * a special file that represents a block device to be unlinked -+ * from the LED (e.g. ``/dev/sda``) -+ * @count: The number of characters in &buf -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t unlink_dev_by_path_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ struct block_device *bdev; -+ struct blkdev_trig_bdev *btb; -+ int err; -+ -+ bdev = blkdev_trig_get_bdev(buf, count); -+ if (IS_ERR(bdev)) -+ return PTR_ERR(bdev); -+ -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ goto exit_put_bdev; -+ -+ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ if (btb == NULL) { -+ err = -EUNATCH; /* bdev isn't linked to any LED */ -+ goto exit_unlock; -+ } -+ -+ if (xa_load(&btb->linked_btls, btl->index) == NULL) { -+ err = -EUNATCH; /* bdev isn't linked to this LED */ -+ goto exit_unlock; -+ } -+ -+ blkdev_trig_unlink_norelease(btl, btb); -+ -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+exit_put_bdev: -+ blkdev_put(bdev, BLKDEV_TRIG_FMODE); -+ return err ? : count; -+} -+ -+ -+/* -+ * -+ * Atomic attribute show & store functions -+ * -+ */ -+ -+/** -+ * blink_time_show() - &blink_time device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_time attribute (&dev_attr_blink_time) -+ * @buf: Output buffer -+ * -+ * Writes the value of &blkdev_trig_led.blink_msec to &buf. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_time_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ const struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ -+ return sprintf(buf, "%u\n", READ_ONCE(btl->blink_msec)); -+} -+ -+/** -+ * blink_time_store() - &blink_time device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_time attribute (&dev_attr_blink_time) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets &blkdev_trig_led.blink_msec to the value in &buf. -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_time_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ unsigned int value; -+ int err; -+ -+ err = kstrtouint(buf, 0, &value); -+ if (err) -+ return err; -+ -+ if (value < BLKDEV_TRIG_BLINK_MIN || value > BLKDEV_TRIG_BLINK_MAX) -+ return -ERANGE; -+ -+ WRITE_ONCE(btl->blink_msec, value); -+ return count; -+} -+ -+/** -+ * check_interval_show() - &check_interval device attribute show function. -+ * @dev: The LED device -+ * @attr: The &check_interval attribute (&dev_attr_check_interval) -+ * @buf: Output buffer -+ * -+ * Writes the value of &blkdev_trig_led.check_jiffies (converted to -+ * milliseconds) to &buf. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t check_interval_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ -+ return sprintf(buf, "%u\n", -+ jiffies_to_msecs(READ_ONCE(btl->check_jiffies))); -+} -+ -+/** -+ * check_interval_store() - &check_interval device attribute store function -+ * @dev: The LED device -+ * @attr: The &check_interval attribute (&dev_attr_check_interval) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets &blkdev_trig_led.check_jiffies to the value in &buf (after converting -+ * from milliseconds). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t check_interval_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *led = led_trigger_get_drvdata(dev); -+ unsigned int value; -+ int err; -+ -+ err = kstrtouint(buf, 0, &value); -+ if (err) -+ return err; -+ -+ if (value < BLKDEV_TRIG_CHECK_MIN || value > BLKDEV_TRIG_CHECK_MAX) -+ return -ERANGE; -+ -+ WRITE_ONCE(led->check_jiffies, msecs_to_jiffies(value)); -+ -+ return count; -+} -+ -+/** -+ * blkdev_trig_mode_show() - Helper for boolean attribute show functions. -+ * @led: The LED -+ * @buf: Output buffer -+ * @bit: Which bit to show -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static int blkdev_trig_mode_show(const struct blkdev_trig_led *led, char *buf, -+ enum stat_group bit) -+{ -+ return sprintf(buf, READ_ONCE(led->mode) & (1 << bit) ? "Y\n" : "N\n"); -+} -+ -+/** -+ * blkdev_trig_mode_store() - Helper for boolean attribute store functions. -+ * @led: The LED -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * @bit: Which bit to set -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static int blkdev_trig_mode_store(struct blkdev_trig_led *led, -+ const char *buf, size_t count, -+ enum stat_group bit) -+{ -+ bool set; -+ int err; -+ -+ err = kstrtobool(buf, &set); -+ if (err) -+ return err; -+ -+ if (set) -+ set_bit(bit, &led->mode); -+ else -+ clear_bit(bit, &led->mode); -+ -+ return count; -+} -+ -+/** -+ * blink_on_read_show() - &blink_on_read device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_READ bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_read_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_READ); -+} -+ -+/** -+ * blink_on_read_store() - &blink_on_read device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_READ bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_read_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_READ); -+} -+ -+/** -+ * blink_on_write_show() - &blink_on_write device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_WRITE bit in -+ * in &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_write_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_WRITE); -+} -+ -+/** -+ * blink_on_write_store() - &blink_on_write device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_WRITE bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_write_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_WRITE); -+} -+ -+/** -+ * blink_on_flush_show() - &blink_on_flush device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending whether the &STAT_FLUSH bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_flush_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_FLUSH); -+} -+ -+/** -+ * blink_on_flush_store() - &blink_on_flush device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_FLUSH bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_flush_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_FLUSH); -+} -+ -+/** -+ * blink_on_discard_show() - &blink_on_discard device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_DISCARD bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_discard_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_DISCARD); -+} -+ -+/** -+ * blink_on_discard_store() - &blink_on_discard device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_DISCARD bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_discard_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_DISCARD); -+} -+ -+/* Device attributes */ -+static DEVICE_ATTR_WO(link_dev_by_path); -+static DEVICE_ATTR_WO(unlink_dev_by_path); -+static DEVICE_ATTR_RW(blink_time); -+static DEVICE_ATTR_RW(check_interval); -+static DEVICE_ATTR_RW(blink_on_read); -+static DEVICE_ATTR_RW(blink_on_write); -+static DEVICE_ATTR_RW(blink_on_flush); -+static DEVICE_ATTR_RW(blink_on_discard); -+ -+/* Device attributes in LED directory (/sys/class/leds//...) */ -+static struct attribute *blkdev_trig_attrs[] = { -+ &dev_attr_link_dev_by_path.attr, -+ &dev_attr_unlink_dev_by_path.attr, -+ &dev_attr_blink_time.attr, -+ &dev_attr_check_interval.attr, -+ &dev_attr_blink_on_read.attr, -+ &dev_attr_blink_on_write.attr, -+ &dev_attr_blink_on_flush.attr, -+ &dev_attr_blink_on_discard.attr, -+ NULL -+}; -+ -+/* Unnamed attribute group == no subdirectory */ -+static const struct attribute_group blkdev_trig_attr_group = { -+ .attrs = blkdev_trig_attrs, -+}; -+ -+/* Attribute groups for the trigger */ -+static const struct attribute_group *blkdev_trig_attr_groups[] = { -+ &blkdev_trig_attr_group, /* /sys/class/leds//... */ -+ &blkdev_trig_linked_devs, /* /sys/class/leds//linked_devices/ */ -+ NULL -+}; -+ -+/* Trigger registration data */ -+static struct led_trigger blkdev_trig_trigger = { -+ .name = "blkdev", -+ .activate = blkdev_trig_activate, -+ .deactivate = blkdev_trig_deactivate, -+ .groups = blkdev_trig_attr_groups, -+}; -+ -+/** -+ * blkdev_trig_init() - Block device LED trigger initialization. -+ * -+ * Registers the ``blkdev`` LED trigger. -+ * -+ * Return: &0 on success, negative &errno on failure. -+ */ -+static int __init blkdev_trig_init(void) -+{ -+ return led_trigger_register(&blkdev_trig_trigger); -+} -+module_init(blkdev_trig_init); -+ -+/** -+ * blkdev_trig_exit() - Block device LED trigger module exit. -+ * -+ * Unregisters the ``blkdev`` LED trigger. -+ */ -+static void __exit blkdev_trig_exit(void) -+{ -+ led_trigger_unregister(&blkdev_trig_trigger); -+} -+module_exit(blkdev_trig_exit); -+ -+MODULE_DESCRIPTION("Block device LED trigger"); -+MODULE_AUTHOR("Ian Pilcher "); -+MODULE_LICENSE("GPL v2"); -diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h -index 83c7248053a1..d2b8741eabf7 100644 ---- a/include/linux/pageblock-flags.h -+++ b/include/linux/pageblock-flags.h -@@ -48,7 +48,7 @@ extern unsigned int pageblock_order; - #else /* CONFIG_HUGETLB_PAGE */ - - /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ --#define pageblock_order (MAX_ORDER-1) -+#define pageblock_order PAGE_ALLOC_COSTLY_ORDER - - #endif /* CONFIG_HUGETLB_PAGE */ - -diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h -index 99d26879b02a..a92799fc5e74 100644 ---- a/include/net/bluetooth/rfcomm.h -+++ b/include/net/bluetooth/rfcomm.h -@@ -171,6 +171,7 @@ struct rfcomm_dlc { - struct rfcomm_session *session; - struct sk_buff_head tx_queue; - struct timer_list timer; -+ struct work_struct state_change_work; - - struct mutex lock; - unsigned long state; -@@ -186,6 +187,7 @@ struct rfcomm_dlc { - u8 sec_level; - u8 role_switch; - u32 defer_setup; -+ int err; - - uint mtu; - uint cfc; -@@ -310,6 +312,7 @@ struct rfcomm_pinfo { - u8 role_switch; - }; - -+void __rfcomm_sk_state_change(struct work_struct *work); - int rfcomm_init_sockets(void); - void rfcomm_cleanup_sockets(void); - -diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c -index 7324764384b6..c6494e85cd68 100644 ---- a/net/bluetooth/rfcomm/core.c -+++ b/net/bluetooth/rfcomm/core.c -@@ -289,6 +289,7 @@ static void rfcomm_dlc_clear_state(struct rfcomm_dlc *d) - d->flags = 0; - d->mscex = 0; - d->sec_level = BT_SECURITY_LOW; -+ d->err = 0; - d->mtu = RFCOMM_DEFAULT_MTU; - d->v24_sig = RFCOMM_V24_RTC | RFCOMM_V24_RTR | RFCOMM_V24_DV; - -@@ -306,6 +307,7 @@ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio) - timer_setup(&d->timer, rfcomm_dlc_timeout, 0); - - skb_queue_head_init(&d->tx_queue); -+ INIT_WORK(&d->state_change_work, __rfcomm_sk_state_change); - mutex_init(&d->lock); - refcount_set(&d->refcnt, 1); - -diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c -index 4bf4ea6cbb5e..4850dafbaa05 100644 ---- a/net/bluetooth/rfcomm/sock.c -+++ b/net/bluetooth/rfcomm/sock.c -@@ -61,19 +61,22 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) - rfcomm_dlc_throttle(d); - } - --static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) -+void __rfcomm_sk_state_change(struct work_struct *work) - { -+ struct rfcomm_dlc *d = container_of(work, struct rfcomm_dlc, -+ state_change_work); - struct sock *sk = d->owner, *parent; - - if (!sk) - return; - -- BT_DBG("dlc %p state %ld err %d", d, d->state, err); -- - lock_sock(sk); -+ rfcomm_dlc_lock(d); - -- if (err) -- sk->sk_err = err; -+ BT_DBG("dlc %p state %ld err %d", d, d->state, d->err); -+ -+ if (d->err) -+ sk->sk_err = d->err; - - sk->sk_state = d->state; - -@@ -91,15 +94,22 @@ static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) - sk->sk_state_change(sk); - } - -+ rfcomm_dlc_unlock(d); - release_sock(sk); -+ sock_put(sk); -+} - -- if (parent && sock_flag(sk, SOCK_ZAPPED)) { -- /* We have to drop DLC lock here, otherwise -- * rfcomm_sock_destruct() will dead lock. */ -- rfcomm_dlc_unlock(d); -- rfcomm_sock_kill(sk); -- rfcomm_dlc_lock(d); -- } -+static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) -+{ -+ struct sock *sk = d->owner; -+ -+ if (!sk) -+ return; -+ -+ d->err = err; -+ sock_hold(sk); -+ if (!schedule_work(&d->state_change_work)) -+ sock_put(sk); - } - - /* ---- Socket functions ---- */ -diff --git a/tools/objtool/check.c b/tools/objtool/check.c -index e55fdf952a3a..445741f36d6d 100644 ---- a/tools/objtool/check.c -+++ b/tools/objtool/check.c -@@ -2104,9 +2104,6 @@ static int read_noendbr_hints(struct objtool_file *file) - return -1; - } - -- if (insn->type == INSN_ENDBR) -- WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); -- - insn->noendbr = 1; - } - --- -2.37.3 - diff --git a/6.0/testing/0009-lrng.patch b/6.0/0012-lrng.patch similarity index 99% rename from 6.0/testing/0009-lrng.patch rename to 6.0/0012-lrng.patch index 41df366a..f38bea07 100644 --- a/6.0/testing/0009-lrng.patch +++ b/6.0/0012-lrng.patch @@ -1,7 +1,7 @@ -From 3c963a73af0422079aabc28646162c1ae73cfad2 Mon Sep 17 00:00:00 2001 +From e2af20ddb7f4e410c25c3deb9dd579d56e340a0b Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Tue, 6 Sep 2022 20:04:11 +0200 -Subject: [PATCH 09/13] lrng +Subject: [PATCH 12/16] lrng Signed-off-by: Piotr Gorski --- diff --git a/6.0/testing/0010-folios.patch b/6.0/0013-folios.patch similarity index 99% rename from 6.0/testing/0010-folios.patch rename to 6.0/0013-folios.patch index 20d07e31..8c63af3e 100644 --- a/6.0/testing/0010-folios.patch +++ b/6.0/0013-folios.patch @@ -1,7 +1,7 @@ -From 0d38511c9f1327acc3ab92d5ff32d615ef59fb70 Mon Sep 17 00:00:00 2001 +From e1f1e6838dfabd0b23fc9a7ee4dc0d0a91d27680 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Sep 2022 14:40:14 +0200 -Subject: [PATCH 10/13] folios +Subject: [PATCH 13/16] folios Signed-off-by: Peter Jung --- diff --git a/6.0/0013-kallsyms.patch b/6.0/0013-kallsyms.patch deleted file mode 100644 index f132a89f..00000000 --- a/6.0/0013-kallsyms.patch +++ /dev/null @@ -1,1141 +0,0 @@ -From 6e51044a3fe99b75493fedb88cd391598f2c1d95 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:19:51 +0200 -Subject: [PATCH 13/13] kallsyms - -Signed-off-by: Peter Jung ---- - include/linux/kallsyms.h | 12 +- - include/linux/module.h | 4 +- - init/Kconfig | 13 ++ - kernel/Makefile | 1 + - kernel/kallsyms.c | 195 +++++++++++++++-- - kernel/kallsyms_internal.h | 1 + - kernel/kallsyms_selftest.c | 421 +++++++++++++++++++++++++++++++++++++ - kernel/livepatch/core.c | 31 ++- - kernel/module/kallsyms.c | 15 +- - kernel/trace/ftrace.c | 3 +- - scripts/kallsyms.c | 88 +++++--- - 11 files changed, 717 insertions(+), 67 deletions(-) - create mode 100644 kernel/kallsyms_selftest.c - -diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h -index ad39636e0c3f..c7219d74e290 100644 ---- a/include/linux/kallsyms.h -+++ b/include/linux/kallsyms.h -@@ -66,9 +66,11 @@ static inline void *dereference_symbol_descriptor(void *ptr) - } - - #ifdef CONFIG_KALLSYMS --int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, -- unsigned long), -+unsigned long kallsyms_sym_address(int idx); -+int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), - void *data); -+int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data); - - /* Lookup the address for a symbol. Returns 0 if not found. */ - unsigned long kallsyms_lookup_name(const char *name); -@@ -168,6 +170,12 @@ static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct - { - return -EOPNOTSUPP; - } -+ -+static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data) -+{ -+ return -EOPNOTSUPP; -+} - #endif /*CONFIG_KALLSYMS*/ - - static inline void print_ip_sym(const char *loglvl, unsigned long ip) -diff --git a/include/linux/module.h b/include/linux/module.h -index 518296ea7f73..6e1a531d78e7 100644 ---- a/include/linux/module.h -+++ b/include/linux/module.h -@@ -879,8 +879,8 @@ static inline bool module_sig_ok(struct module *module) - } - #endif /* CONFIG_MODULE_SIG */ - --int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, -- struct module *, unsigned long), -+int module_kallsyms_on_each_symbol(const char *modname, -+ int (*fn)(void *, const char *, unsigned long), - void *data); - - #endif /* _LINUX_MODULE_H */ -diff --git a/init/Kconfig b/init/Kconfig -index 442a945ca6ae..b3a9ec8aa753 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1742,6 +1742,19 @@ config KALLSYMS - symbolic stack backtraces. This increases the size of the kernel - somewhat, as all symbols have to be loaded into the kernel image. - -+config KALLSYMS_SELFTEST -+ bool "Test the basic functions and performance of kallsyms" -+ depends on KALLSYMS -+ default n -+ help -+ Test the basic functions and performance of some interfaces, such as -+ kallsyms_lookup_name. It also calculates the compression rate of the -+ kallsyms compression algorithm for the current symbol set. -+ -+ Start self-test automatically after system startup. Suggest executing -+ "dmesg | grep kallsyms_selftest" to collect test results. "finish" is -+ displayed in the last line, indicating that the test is complete. -+ - config KALLSYMS_ALL - bool "Include all symbols in kallsyms" - depends on DEBUG_KERNEL && KALLSYMS -diff --git a/kernel/Makefile b/kernel/Makefile -index 318789c728d3..122a5fed457b 100644 ---- a/kernel/Makefile -+++ b/kernel/Makefile -@@ -68,6 +68,7 @@ endif - obj-$(CONFIG_UID16) += uid16.o - obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o - obj-$(CONFIG_KALLSYMS) += kallsyms.o -+obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o - obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o - obj-$(CONFIG_CRASH_CORE) += crash_core.o - obj-$(CONFIG_KEXEC_CORE) += kexec_core.o -diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c -index 3e7e2c2ad2f7..1512db69aa0a 100644 ---- a/kernel/kallsyms.c -+++ b/kernel/kallsyms.c -@@ -87,6 +87,86 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, - return off; - } - -+static unsigned char *find_token(unsigned char *str, int len, -+ const unsigned char *token) -+{ -+ int i; -+ -+ for (i = 0; i < len - 1; i++) { -+ if (str[i] == token[0] && str[i+1] == token[1]) -+ return &str[i]; -+ } -+ return NULL; -+} -+ -+static int kallsyms_compress_symbol_name(const char *name, char *buf, size_t size) -+{ -+ int i, j, n, len; -+ unsigned char *p1, *p2; -+ const unsigned char *token; -+ -+ len = strscpy(buf, name, size); -+ if (WARN_ON_ONCE(len <= 0)) -+ return 0; -+ -+ /* -+ * For each entry in kallsyms_best_token_table[], the storage -+ * format is: -+ * 1. For tokens that cannot be used to compress characters, the value -+ * at [j] is 0, and the value at [j+1] is the number of consecutive -+ * tokens with this feature. -+ * 2. For each token: the larger the token value, the higher the -+ * frequency, and the lower the index. -+ * -+ * ------------------------------- -+ * | j | [j] [j+1] | token | -+ * -----|---------------|---------| -+ * | 0 | ?? ?? | 255 | -+ * | 2 | ?? ?? | 254 | -+ * | ... | ?? ?? | ... | -+ * | n-2 | ?? ?? | x | -+ * | n | 00 len | x-1 | -+ * | n+2 | ?? ?? | x-1-len | -+ * above '??' is non-zero -+ */ -+ for (i = 255, j = 0; i >= 0; i--, j += 2) { -+ if (!kallsyms_best_token_table[j]) { -+ i -= kallsyms_best_token_table[j + 1]; -+ if (i < 0) -+ break; -+ j += 2; -+ } -+ token = &kallsyms_best_token_table[j]; -+ -+ p1 = buf; -+ -+ /* find the token on the symbol */ -+ p2 = find_token(p1, len, token); -+ if (!p2) -+ continue; -+ -+ n = len; -+ -+ do { -+ *p2 = i; -+ p2++; -+ n -= (p2 - p1); -+ memmove(p2, p2 + 1, n); -+ p1 = p2; -+ len--; -+ -+ if (n < 2) -+ break; -+ -+ /* find the token on the symbol */ -+ p2 = find_token(p1, n, token); -+ -+ } while (p2); -+ } -+ -+ return len; -+} -+ - /* - * Get symbol type information. This is encoded as a single char at the - * beginning of the symbol name. -@@ -128,7 +208,7 @@ static unsigned int get_symbol_offset(unsigned long pos) - return name - kallsyms_names; - } - --static unsigned long kallsyms_sym_address(int idx) -+unsigned long kallsyms_sym_address(int idx) - { - if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE)) - return kallsyms_addresses[idx]; -@@ -186,26 +266,95 @@ static bool cleanup_symbol_name(char *s) - return false; - } - -+static int kallsyms_lookup_clang_name(unsigned char *namebuf, int len, -+ const char *name, -+ unsigned long *addr) -+{ -+ unsigned long i; -+ unsigned int off; -+ -+ if (!IS_ENABLED(CONFIG_LTO_CLANG)) -+ return -ENOENT; -+ -+ for (i = 0, off = 0; i < kallsyms_num_syms; i++) { -+ off = kallsyms_expand_symbol(off, namebuf, len); -+ -+ if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) { -+ *addr = kallsyms_sym_address(i); -+ return 0; -+ } -+ } -+ -+ return -ENOENT; -+} -+ -+static int __kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, -+ unsigned int *index, -+ unsigned int *offset, -+ unsigned long *addr) -+{ -+ unsigned int i = *index; -+ unsigned int off = *offset; -+ unsigned int name_len; -+ const unsigned char *name; -+ -+ for (; len && i < kallsyms_num_syms; i++) { -+ /* -+ * For each entry in kallsyms_names[], the storage format is: -+ * ---------------------------- -+ * | len(1) | type(1) | name(x) | -+ * ---------------------------- -+ * -+ * Number of bytes in parentheses, and: len = 1 + x -+ */ -+ name_len = kallsyms_names[off] - 1; -+ name = &kallsyms_names[off + 2]; -+ off += name_len + 2; -+ -+ if (name_len != len) -+ continue; -+ -+ if (!memcmp(name, namebuf, len)) { -+ /* Prepare for the next iteration */ -+ *index = i + 1; -+ *offset = off; -+ -+ *addr = kallsyms_sym_address(i); -+ return 0; -+ } -+ } -+ -+ return -ENOENT; -+} -+ -+static int kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, -+ unsigned long *addr) -+{ -+ unsigned int i = 0, off = 0; -+ -+ return __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, addr); -+} -+ - /* Lookup the address for this symbol. Returns 0 if not found. */ - unsigned long kallsyms_lookup_name(const char *name) - { - char namebuf[KSYM_NAME_LEN]; -- unsigned long i; -- unsigned int off; -+ unsigned long addr; -+ int ret, len; - - /* Skip the search for empty string. */ - if (!*name) - return 0; - -- for (i = 0, off = 0; i < kallsyms_num_syms; i++) { -- off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); -+ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); -+ ret = kallsyms_lookup_compressed_name(namebuf, len, &addr); -+ if (!ret) -+ return addr; - -- if (strcmp(namebuf, name) == 0) -- return kallsyms_sym_address(i); -+ ret = kallsyms_lookup_clang_name(namebuf, len, name, &addr); -+ if (!ret) -+ return addr; - -- if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) -- return kallsyms_sym_address(i); -- } - return module_kallsyms_lookup_name(name); - } - -@@ -213,8 +362,7 @@ unsigned long kallsyms_lookup_name(const char *name) - * Iterate over all symbols in vmlinux. For symbols from modules use - * module_kallsyms_on_each_symbol instead. - */ --int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, -- unsigned long), -+int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), - void *data) - { - char namebuf[KSYM_NAME_LEN]; -@@ -224,7 +372,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - - for (i = 0, off = 0; i < kallsyms_num_syms; i++) { - off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); -- ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); -+ ret = fn(data, namebuf, kallsyms_sym_address(i)); - if (ret != 0) - return ret; - cond_resched(); -@@ -232,6 +380,27 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - return 0; - } - -+int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data) -+{ -+ int ret, len; -+ unsigned long addr; -+ unsigned int i = 0, off = 0; -+ char namebuf[KSYM_NAME_LEN]; -+ -+ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); -+ do { -+ ret = __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, &addr); -+ if (ret) -+ return 0; /* end of lookup */ -+ -+ ret = fn(data, addr); -+ cond_resched(); -+ } while (!ret); -+ -+ return ret; -+} -+ - static unsigned long get_symbol_pos(unsigned long addr, - unsigned long *symbolsize, - unsigned long *offset) -diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h -index 2d0c6f2f0243..d9672ede8cfc 100644 ---- a/kernel/kallsyms_internal.h -+++ b/kernel/kallsyms_internal.h -@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; - extern const u16 kallsyms_token_index[] __weak; - - extern const unsigned int kallsyms_markers[] __weak; -+extern const unsigned char kallsyms_best_token_table[] __weak; - - #endif // LINUX_KALLSYMS_INTERNAL_H_ -diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c -new file mode 100644 -index 000000000000..f7538a70d36c ---- /dev/null -+++ b/kernel/kallsyms_selftest.c -@@ -0,0 +1,421 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Test the function and performance of kallsyms -+ * -+ * Copyright (C) Huawei Technologies Co., Ltd., 2022 -+ * -+ * Authors: Zhen Lei Huawei -+ */ -+ -+#define pr_fmt(fmt) "kallsyms_selftest: " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "kallsyms_internal.h" -+ -+ -+#define MAX_NUM_OF_RECORDS 64 -+ -+struct test_stat { -+ int min; -+ int max; -+ int save_cnt; -+ int real_cnt; -+ u64 sum; -+ char *name; -+ unsigned long addr; -+ unsigned long addrs[MAX_NUM_OF_RECORDS]; -+}; -+ -+struct test_item { -+ char *name; -+ unsigned long addr; -+}; -+ -+#define ITEM_FUNC(s) \ -+ { \ -+ .name = #s, \ -+ .addr = (unsigned long)s, \ -+ } -+ -+#define ITEM_DATA(s) \ -+ { \ -+ .name = #s, \ -+ .addr = (unsigned long)&s, \ -+ } -+ -+static int test_var_bss_static; -+static int test_var_data_static = 1; -+int test_var_bss; -+int test_var_data = 1; -+ -+static int test_func_static(void) -+{ -+ test_var_bss_static++; -+ test_var_data_static++; -+ -+ return 0; -+} -+ -+int test_func(void) -+{ -+ return test_func_static(); -+} -+ -+__weak int test_func_weak(void) -+{ -+ test_var_bss++; -+ test_var_data++; -+ return 0; -+} -+ -+static struct test_item test_items[] = { -+ ITEM_FUNC(test_func_static), -+ ITEM_FUNC(test_func), -+ ITEM_FUNC(test_func_weak), -+ ITEM_FUNC(vmalloc), -+ ITEM_FUNC(vfree), -+#ifdef CONFIG_KALLSYMS_ALL -+ ITEM_DATA(test_var_bss_static), -+ ITEM_DATA(test_var_data_static), -+ ITEM_DATA(test_var_bss), -+ ITEM_DATA(test_var_data), -+ ITEM_DATA(vmap_area_list), -+#endif -+}; -+ -+static char stub_name[KSYM_NAME_LEN]; -+ -+static int stat_symbol_len(void *data, const char *name, unsigned long addr) -+{ -+ *(u32 *)data += strlen(name); -+ -+ return 0; -+} -+ -+static void test_kallsyms_compression_ratio(void) -+{ -+ int i; -+ const u8 *name; -+ u32 pos; -+ u32 ratio, total_size, total_len = 0; -+ -+ kallsyms_on_each_symbol(stat_symbol_len, &total_len); -+ -+ /* -+ * A symbol name cannot start with a number. This stub name helps us -+ * traverse the entire symbol table without finding a match. It's used -+ * for subsequent performance tests, and its length is the average -+ * length of all symbol names. -+ */ -+ memset(stub_name, '4', sizeof(stub_name)); -+ pos = total_len / kallsyms_num_syms; -+ stub_name[pos] = 0; -+ -+ pos = kallsyms_num_syms - 1; -+ name = &kallsyms_names[kallsyms_markers[pos >> 8]]; -+ for (i = 0; i <= (pos & 0xff); i++) -+ name = name + (*name) + 1; -+ -+ /* -+ * 1. The length fields is not counted -+ * 2. The memory occupied by array kallsyms_token_table[] and -+ * kallsyms_token_index[] needs to be counted. -+ */ -+ total_size = (name - kallsyms_names) - kallsyms_num_syms; -+ pos = kallsyms_token_index[0xff]; -+ total_size += pos + strlen(&kallsyms_token_table[pos]) + 1; -+ total_size += 0x100 * sizeof(u16); -+ -+ pr_info(" ---------------------------------------------------------\n"); -+ pr_info("| nr_symbols | compressed size | original size | ratio(%%) |\n"); -+ pr_info("|---------------------------------------------------------|\n"); -+ ratio = 10000ULL * total_size / total_len; -+ pr_info("| %10d | %10d | %10d | %2d.%-2d |\n", -+ kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100); -+ pr_info(" ---------------------------------------------------------\n"); -+} -+ -+static int lookup_name(void *data, const char *name, unsigned long addr) -+{ -+ u64 t0, t1, t; -+ unsigned long flags; -+ struct test_stat *stat = (struct test_stat *)data; -+ -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ (void)kallsyms_lookup_name(name); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ -+ t = t1 - t0; -+ if (t < stat->min) -+ stat->min = t; -+ -+ if (t > stat->max) -+ stat->max = t; -+ -+ stat->real_cnt++; -+ stat->sum += t; -+ -+ return 0; -+} -+ -+static void test_perf_kallsyms_lookup_name(void) -+{ -+ struct test_stat stat; -+ -+ memset(&stat, 0, sizeof(stat)); -+ stat.min = INT_MAX; -+ kallsyms_on_each_symbol(lookup_name, &stat); -+ pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt); -+ pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n", -+ stat.min, stat.max, stat.sum / stat.real_cnt); -+} -+ -+static int find_symbol(void *data, const char *name, unsigned long addr) -+{ -+ struct test_stat *stat = (struct test_stat *)data; -+ -+ if (strcmp(name, stat->name) == 0) { -+ stat->real_cnt++; -+ stat->addr = addr; -+ -+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { -+ stat->addrs[stat->save_cnt] = addr; -+ stat->save_cnt++; -+ } -+ -+ if (stat->real_cnt == stat->max) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void test_perf_kallsyms_on_each_symbol(void) -+{ -+ u64 t0, t1; -+ unsigned long flags; -+ struct test_stat stat; -+ -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = stub_name; -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ kallsyms_on_each_symbol(find_symbol, &stat); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0); -+} -+ -+static int match_symbol(void *data, unsigned long addr) -+{ -+ struct test_stat *stat = (struct test_stat *)data; -+ -+ stat->real_cnt++; -+ stat->addr = addr; -+ -+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { -+ stat->addrs[stat->save_cnt] = addr; -+ stat->save_cnt++; -+ } -+ -+ if (stat->real_cnt == stat->max) -+ return 1; -+ -+ return 0; -+} -+ -+static void test_perf_kallsyms_on_each_match_symbol(void) -+{ -+ u64 t0, t1; -+ unsigned long flags; -+ struct test_stat stat; -+ -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = stub_name; -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0); -+} -+ -+static int test_kallsyms_basic_function(void) -+{ -+ int i, j, ret; -+ int next = 0, nr_failed = 0; -+ char *prefix; -+ unsigned short rand; -+ unsigned long addr; -+ char namebuf[KSYM_NAME_LEN]; -+ struct test_stat stat, stat1, stat2; -+ -+ prefix = "kallsyms_lookup_name() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ addr = kallsyms_lookup_name(test_items[i].name); -+ if (addr != test_items[i].addr) { -+ nr_failed++; -+ pr_info("%s %s failed: addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, addr, test_items[i].addr); -+ } -+ } -+ -+ prefix = "kallsyms_on_each_symbol() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = test_items[i].name; -+ kallsyms_on_each_symbol(find_symbol, &stat); -+ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { -+ nr_failed++; -+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, -+ stat.real_cnt, stat.addr, test_items[i].addr); -+ } -+ } -+ -+ prefix = "kallsyms_on_each_match_symbol() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = test_items[i].name; -+ kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, &stat); -+ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { -+ nr_failed++; -+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, -+ stat.real_cnt, stat.addr, test_items[i].addr); -+ } -+ } -+ -+ if (nr_failed) -+ return -ESRCH; -+ -+ for (i = 0; i < kallsyms_num_syms; i++) { -+ addr = kallsyms_sym_address(i); -+ if (!is_ksym_addr(addr)) -+ continue; -+ -+ ret = lookup_symbol_name(addr, namebuf); -+ if (unlikely(ret)) { -+ namebuf[0] = 0; -+ goto failed; -+ } -+ -+ stat.addr = kallsyms_lookup_name(namebuf); -+ -+ memset(&stat1, 0, sizeof(stat1)); -+ stat1.max = INT_MAX; -+ kallsyms_on_each_match_symbol(match_symbol, namebuf, &stat1); -+ -+ /* -+ * kallsyms_on_each_symbol() is too slow, randomly select some -+ * symbols for test. -+ */ -+ if (i >= next) { -+ memset(&stat2, 0, sizeof(stat2)); -+ stat2.max = INT_MAX; -+ stat2.name = namebuf; -+ kallsyms_on_each_symbol(find_symbol, &stat2); -+ -+ /* -+ * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol() -+ * need to get the same traversal result. -+ */ -+ if (stat1.addr != stat2.addr || -+ stat1.real_cnt != stat2.real_cnt || -+ memcmp(stat1.addrs, stat2.addrs, -+ stat1.save_cnt * sizeof(stat1.addrs[0]))) -+ goto failed; -+ -+ /* -+ * The average of random increments is 128, that is, one of -+ * them is tested every 128 symbols. -+ */ -+ get_random_bytes(&rand, sizeof(rand)); -+ next = i + (rand & 0xff) + 1; -+ } -+ -+ /* Need to be found at least once */ -+ if (!stat1.real_cnt) -+ goto failed; -+ -+ /* -+ * kallsyms_lookup_name() returns the address of the first -+ * symbol found and cannot be NULL. -+ */ -+ if (!stat.addr || stat.addr != stat1.addrs[0]) -+ goto failed; -+ -+ /* -+ * If the addresses of all matching symbols are recorded, the -+ * target address needs to be exist. -+ */ -+ if (stat1.real_cnt <= MAX_NUM_OF_RECORDS) { -+ for (j = 0; j < stat1.save_cnt; j++) { -+ if (stat1.addrs[j] == addr) -+ break; -+ } -+ -+ if (j == stat1.save_cnt) -+ goto failed; -+ } -+ } -+ -+ return 0; -+ -+failed: -+ pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr); -+ return -ESRCH; -+} -+ -+static int test_entry(void *p) -+{ -+ int ret; -+ -+ do { -+ schedule_timeout(5 * HZ); -+ } while (system_state != SYSTEM_RUNNING); -+ -+ pr_info("start\n"); -+ ret = test_kallsyms_basic_function(); -+ if (ret) { -+ pr_info("abort\n"); -+ return 0; -+ } -+ -+ test_kallsyms_compression_ratio(); -+ test_perf_kallsyms_lookup_name(); -+ test_perf_kallsyms_on_each_symbol(); -+ test_perf_kallsyms_on_each_match_symbol(); -+ pr_info("finish\n"); -+ -+ return 0; -+} -+ -+static int __init kallsyms_test_init(void) -+{ -+ struct task_struct *t; -+ -+ t = kthread_create(test_entry, NULL, "kallsyms_test"); -+ if (IS_ERR(t)) { -+ pr_info("Create kallsyms selftest task failed\n"); -+ return PTR_ERR(t); -+ } -+ kthread_bind(t, 0); -+ wake_up_process(t); -+ -+ return 0; -+} -+late_initcall(kallsyms_test_init); -diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c -index bc475e62279d..074a91e2cfd5 100644 ---- a/kernel/livepatch/core.c -+++ b/kernel/livepatch/core.c -@@ -118,26 +118,36 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, - } - - struct klp_find_arg { -- const char *objname; - const char *name; - unsigned long addr; - unsigned long count; - unsigned long pos; - }; - --static int klp_find_callback(void *data, const char *name, -- struct module *mod, unsigned long addr) -+static int klp_find_callback(void *data, const char *name, unsigned long addr) - { - struct klp_find_arg *args = data; - -- if ((mod && !args->objname) || (!mod && args->objname)) -- return 0; -- - if (strcmp(args->name, name)) - return 0; - -- if (args->objname && strcmp(args->objname, mod->name)) -- return 0; -+ args->addr = addr; -+ args->count++; -+ -+ /* -+ * Finish the search when the symbol is found for the desired position -+ * or the position is not defined for a non-unique symbol. -+ */ -+ if ((args->pos && (args->count == args->pos)) || -+ (!args->pos && (args->count > 1))) -+ return 1; -+ -+ return 0; -+} -+ -+static int klp_match_callback(void *data, unsigned long addr) -+{ -+ struct klp_find_arg *args = data; - - args->addr = addr; - args->count++; -@@ -157,7 +167,6 @@ static int klp_find_object_symbol(const char *objname, const char *name, - unsigned long sympos, unsigned long *addr) - { - struct klp_find_arg args = { -- .objname = objname, - .name = name, - .addr = 0, - .count = 0, -@@ -165,9 +174,9 @@ static int klp_find_object_symbol(const char *objname, const char *name, - }; - - if (objname) -- module_kallsyms_on_each_symbol(klp_find_callback, &args); -+ module_kallsyms_on_each_symbol(objname, klp_find_callback, &args); - else -- kallsyms_on_each_symbol(klp_find_callback, &args); -+ kallsyms_on_each_match_symbol(klp_match_callback, name, &args); - - /* - * Ensure an address was found. If sympos is 0, ensure symbol is unique; -diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c -index f5c5c9175333..329cef573675 100644 ---- a/kernel/module/kallsyms.c -+++ b/kernel/module/kallsyms.c -@@ -495,8 +495,8 @@ unsigned long module_kallsyms_lookup_name(const char *name) - } - - #ifdef CONFIG_LIVEPATCH --int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, -- struct module *, unsigned long), -+int module_kallsyms_on_each_symbol(const char *modname, -+ int (*fn)(void *, const char *, unsigned long), - void *data) - { - struct module *mod; -@@ -510,6 +510,9 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, - if (mod->state == MODULE_STATE_UNFORMED) - continue; - -+ if (strcmp(modname, mod->name)) -+ continue; -+ - /* Use rcu_dereference_sched() to remain compliant with the sparse tool */ - preempt_disable(); - kallsyms = rcu_dereference_sched(mod->kallsyms); -@@ -522,10 +525,16 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, - continue; - - ret = fn(data, kallsyms_symbol_name(kallsyms, i), -- mod, kallsyms_symbol_value(sym)); -+ kallsyms_symbol_value(sym)); - if (ret != 0) - goto out; - } -+ -+ /* -+ * The given module is found, the subsequent modules do not -+ * need to be compared. -+ */ -+ break; - } - out: - mutex_unlock(&module_mutex); -diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c -index 439e2ab6905e..f135a0a334a3 100644 ---- a/kernel/trace/ftrace.c -+++ b/kernel/trace/ftrace.c -@@ -8250,8 +8250,7 @@ struct kallsyms_data { - size_t found; - }; - --static int kallsyms_callback(void *data, const char *name, -- struct module *mod, unsigned long addr) -+static int kallsyms_callback(void *data, const char *name, unsigned long addr) - { - struct kallsyms_data *args = data; - const char **sym; -diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c -index f18e6dfc68c5..40a6fe6d14ef 100644 ---- a/scripts/kallsyms.c -+++ b/scripts/kallsyms.c -@@ -34,7 +34,8 @@ struct sym_entry { - unsigned int len; - unsigned int start_pos; - unsigned int percpu_absolute; -- unsigned char sym[]; -+ unsigned char type; -+ unsigned char name[]; - }; - - struct addr_range { -@@ -75,11 +76,6 @@ static void usage(void) - exit(1); - } - --static char *sym_name(const struct sym_entry *s) --{ -- return (char *)s->sym + 1; --} -- - static bool is_ignored_symbol(const char *name, char type) - { - /* Symbol names that exactly match to the following are ignored.*/ -@@ -227,11 +223,7 @@ static struct sym_entry *read_symbol(FILE *in) - check_symbol_range(name, addr, text_ranges, ARRAY_SIZE(text_ranges)); - check_symbol_range(name, addr, &percpu_range, 1); - -- /* include the type field in the symbol name, so that it gets -- * compressed together */ -- -- len = strlen(name) + 1; -- -+ len = strlen(name); - sym = malloc(sizeof(*sym) + len + 1); - if (!sym) { - fprintf(stderr, "kallsyms failure: " -@@ -240,8 +232,8 @@ static struct sym_entry *read_symbol(FILE *in) - } - sym->addr = addr; - sym->len = len; -- sym->sym[0] = type; -- strcpy(sym_name(sym), name); -+ sym->type = type; -+ strcpy((char *)sym->name, name); - sym->percpu_absolute = 0; - - return sym; -@@ -265,7 +257,7 @@ static int symbol_in_range(const struct sym_entry *s, - - static int symbol_valid(const struct sym_entry *s) - { -- const char *name = sym_name(s); -+ const char *name = (char *)s->name; - - /* if --all-symbols is not specified, then symbols outside the text - * and inittext sections are discarded */ -@@ -471,12 +463,18 @@ static void write_src(void) - if ((i & 0xFF) == 0) - markers[i >> 8] = off; - -- printf("\t.byte 0x%02x", table[i]->len); -+ /* -+ * Store the symbol type togerher with symbol name. -+ * It helps to reduce the size. -+ */ -+ printf("\t.byte 0x%02x", table[i]->len + 1); -+ printf(", 0x%02x", table[i]->type); - for (k = 0; k < table[i]->len; k++) -- printf(", 0x%02x", table[i]->sym[k]); -+ printf(", 0x%02x", table[i]->name[k]); - printf("\n"); - -- off += table[i]->len + 1; -+ /* fields 'len' and 'type' occupy one byte each */ -+ off += table[i]->len + 1 + 1; - } - printf("\n"); - -@@ -501,6 +499,24 @@ static void write_src(void) - for (i = 0; i < 256; i++) - printf("\t.short\t%d\n", best_idx[i]); - printf("\n"); -+ -+ output_label("kallsyms_best_token_table"); -+ for (i = 255, k = 0; (int)i >= 0; i--) { -+ if (best_table_len[i] <= 1) { -+ k++; -+ continue; -+ } -+ -+ if (k) { -+ printf("\t.byte 0x00, 0x%02x\n", k); -+ k = 0; -+ } -+ -+ printf("\t.byte 0x%02x, 0x%02x\n", best_table[i][0], best_table[i][1]); -+ } -+ if (k) -+ printf("\t.byte 0x00, 0x%02x\n", k); -+ printf("\n"); - } - - -@@ -525,12 +541,12 @@ static void forget_symbol(const unsigned char *symbol, int len) - } - - /* do the initial token count */ --static void build_initial_tok_table(void) -+static void build_initial_token_table(void) - { - unsigned int i; - - for (i = 0; i < table_cnt; i++) -- learn_symbol(table[i]->sym, table[i]->len); -+ learn_symbol(table[i]->name, table[i]->len); - } - - static unsigned char *find_token(unsigned char *str, int len, -@@ -555,14 +571,14 @@ static void compress_symbols(const unsigned char *str, int idx) - for (i = 0; i < table_cnt; i++) { - - len = table[i]->len; -- p1 = table[i]->sym; -+ p1 = table[i]->name; - - /* find the token on the symbol */ - p2 = find_token(p1, len, str); - if (!p2) continue; - - /* decrease the counts for this symbol's tokens */ -- forget_symbol(table[i]->sym, len); -+ forget_symbol(table[i]->name, len); - - size = len; - -@@ -584,7 +600,7 @@ static void compress_symbols(const unsigned char *str, int idx) - table[i]->len = len; - - /* increase the counts for this symbol's new tokens */ -- learn_symbol(table[i]->sym, len); -+ learn_symbol(table[i]->name, len); - } - } - -@@ -637,20 +653,24 @@ static void optimize_result(void) - /* start by placing the symbols that are actually used on the table */ - static void insert_real_symbols_in_table(void) - { -- unsigned int i, j, c; -+ unsigned int i, j; -+ unsigned char c; - - for (i = 0; i < table_cnt; i++) { - for (j = 0; j < table[i]->len; j++) { -- c = table[i]->sym[j]; -- best_table[c][0]=c; -- best_table_len[c]=1; -+ c = table[i]->name[j]; -+ best_table[c][0] = c; -+ best_table_len[c] = 1; - } -+ c = table[i]->type; -+ best_table[c][0] = c; -+ best_table_len[c] = 1; - } - } - - static void optimize_token_table(void) - { -- build_initial_tok_table(); -+ build_initial_token_table(); - - insert_real_symbols_in_table(); - -@@ -660,8 +680,8 @@ static void optimize_token_table(void) - /* guess for "linker script provide" symbol */ - static int may_be_linker_script_provide_symbol(const struct sym_entry *se) - { -- const char *symbol = sym_name(se); -- int len = se->len - 1; -+ const char *symbol = (char *)se->name; -+ int len = se->len; - - if (len < 8) - return 0; -@@ -705,8 +725,8 @@ static int compare_symbols(const void *a, const void *b) - return -1; - - /* sort by "weakness" type */ -- wa = (sa->sym[0] == 'w') || (sa->sym[0] == 'W'); -- wb = (sb->sym[0] == 'w') || (sb->sym[0] == 'W'); -+ wa = (sa->type == 'w') || (sa->type == 'W'); -+ wb = (sb->type == 'w') || (sb->type == 'W'); - if (wa != wb) - return wa - wb; - -@@ -717,8 +737,8 @@ static int compare_symbols(const void *a, const void *b) - return wa - wb; - - /* sort by the number of prefix underscores */ -- wa = strspn(sym_name(sa), "_"); -- wb = strspn(sym_name(sb), "_"); -+ wa = strspn((char *)sa->name, "_"); -+ wb = strspn((char *)sb->name, "_"); - if (wa != wb) - return wa - wb; - -@@ -742,7 +762,7 @@ static void make_percpus_absolute(void) - * ensure consistent behavior compared to older - * versions of this tool. - */ -- table[i]->sym[0] = 'A'; -+ table[i]->type = 'A'; - table[i]->percpu_absolute = 1; - } - } --- -2.37.3 - diff --git a/6.0/testing/0011-fixes.patch b/6.0/0014-fixes.patch similarity index 99% rename from 6.0/testing/0011-fixes.patch rename to 6.0/0014-fixes.patch index 05681187..3eae2eb5 100644 --- a/6.0/testing/0011-fixes.patch +++ b/6.0/0014-fixes.patch @@ -1,7 +1,7 @@ -From d30c8771521031923117e12623f501a6cf17ec66 Mon Sep 17 00:00:00 2001 +From da70f4396195cb2e56bcfe68c95ea4e31c933e6b Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Sep 2022 14:42:00 +0200 -Subject: [PATCH 11/13] fixes +Subject: [PATCH 14/16] fixes Signed-off-by: Peter Jung --- diff --git a/6.0/testing/0012-kallsyms.patch b/6.0/0015-kallsyms.patch similarity index 99% rename from 6.0/testing/0012-kallsyms.patch rename to 6.0/0015-kallsyms.patch index 90196ecf..abc26630 100644 --- a/6.0/testing/0012-kallsyms.patch +++ b/6.0/0015-kallsyms.patch @@ -1,7 +1,7 @@ -From 0d94dd6f072f2cf4c73fba0862b5c877828269d6 Mon Sep 17 00:00:00 2001 +From 1c95ad8820155c71485f71b29697ed823bcce3b2 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 Sep 2022 00:19:51 +0200 -Subject: [PATCH 12/13] kallsyms +Subject: [PATCH 15/16] kallsyms Signed-off-by: Peter Jung --- diff --git a/6.0/testing/0013-bitmap.patch b/6.0/0016-bitmap.patch similarity index 98% rename from 6.0/testing/0013-bitmap.patch rename to 6.0/0016-bitmap.patch index f510d2a6..cb445319 100644 --- a/6.0/testing/0013-bitmap.patch +++ b/6.0/0016-bitmap.patch @@ -1,7 +1,7 @@ -From 1c53b3346447afadb2f1fbc5c94c11ef14500a15 Mon Sep 17 00:00:00 2001 +From 2fc2cb736eb578dcdd96ebc321ef6fe31971e7a3 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Tue, 27 Sep 2022 18:01:48 +0200 -Subject: [PATCH 13/13] bitmap +Date: Wed, 28 Sep 2022 00:34:04 +0200 +Subject: [PATCH 16/16] bitmap Signed-off-by: Peter Jung --- @@ -16,7 +16,7 @@ Signed-off-by: Peter Jung include/linux/cpumask.h | 114 +++++++++---- include/linux/find.h | 272 ++++++++++++++++++++++++++----- include/linux/netdevice.h | 10 +- - include/linux/nodemask.h | 3 +- + include/linux/nodemask.h | 27 ++-- kernel/smp.c | 6 +- lib/Kconfig | 9 ++ lib/bitmap.c | 68 +++----- @@ -26,7 +26,7 @@ Signed-off-by: Peter Jung lib/test_bitmap.c | 291 +++++++++++++++++++++++++++++++++- tools/include/linux/find.h | 61 ++----- tools/lib/find_bit.c | 149 ++++++++--------- - 21 files changed, 950 insertions(+), 365 deletions(-) + 21 files changed, 962 insertions(+), 377 deletions(-) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 8f5c2f9a1a83..18a81edd3ac5 100644 @@ -825,19 +825,43 @@ index 05d6f3facd5a..4d6d5a2dd82e 100644 if (src1p && src2p) return find_next_and_bit(src1p, src2p, nr_bits, n + 1); diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h -index 3a0eec9f2faa..ef7c6fc71e10 100644 +index e66742db741c..0294f2de2fb4 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h -@@ -509,8 +509,7 @@ static inline int node_random(const nodemask_t *maskp) - - w = nodes_weight(*maskp); - if (w) +@@ -507,20 +507,19 @@ static inline int node_random(const nodemask_t *maskp) + #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) + int w, bit; + +- w = nodes_weight(*maskp); +- switch (w) { +- case 0: +- bit = NUMA_NO_NODE; +- break; +- case 1: +- bit = first_node(*maskp); +- break; +- default: - bit = bitmap_ord_to_pos(maskp->bits, -- get_random_int() % w, MAX_NUMNODES); +- get_random_int() % w, MAX_NUMNODES); +- break; +- } +- return bit; ++ w = nodes_weight(*maskp); ++ switch (w) { ++ case 0: ++ bit = NUMA_NO_NODE; ++ break; ++ case 1: ++ bit = first_node(*maskp); ++ break; ++ default: + bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_int() % w); - return bit; ++ break; ++ } ++ return bit; #else return 0; + #endif diff --git a/kernel/smp.c b/kernel/smp.c index 650810a6f29b..661d09ae5d6a 100644 --- a/kernel/smp.c diff --git a/6.0/all/0001-cachyos-base-all-exp.patch b/6.0/all/0001-cachyos-base-all-exp.patch deleted file mode 100644 index 4dd4a440..00000000 --- a/6.0/all/0001-cachyos-base-all-exp.patch +++ /dev/null @@ -1,100390 +0,0 @@ -From 79b0b16073aa7852df60ff5820f471c0ad0078d7 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Tue, 27 Sep 2022 15:12:20 +0200 -Subject: [PATCH 01/13] cachy - -Signed-off-by: Peter Jung ---- - .gitignore | 1 + - Documentation/admin-guide/pm/amd-pstate.rst | 19 + - Makefile | 2 + - arch/arc/configs/axs101_defconfig | 1 + - arch/arc/configs/axs103_defconfig | 1 + - arch/arc/configs/axs103_smp_defconfig | 1 + - arch/arc/configs/haps_hs_defconfig | 1 + - arch/arc/configs/haps_hs_smp_defconfig | 1 + - arch/arc/configs/hsdk_defconfig | 1 + - arch/arc/configs/nsim_700_defconfig | 1 + - arch/arc/configs/nsimosci_defconfig | 1 + - arch/arc/configs/nsimosci_hs_defconfig | 1 + - arch/arc/configs/nsimosci_hs_smp_defconfig | 1 + - arch/arc/configs/tb10x_defconfig | 1 + - arch/arc/configs/vdk_hs38_defconfig | 1 + - arch/arc/configs/vdk_hs38_smp_defconfig | 1 + - arch/x86/Kconfig.cpu | 332 +++++- - arch/x86/Makefile | 40 +- - arch/x86/Makefile.postlink | 41 + - arch/x86/boot/compressed/Makefile | 10 +- - arch/x86/include/asm/msr-index.h | 7 + - arch/x86/include/asm/topology.h | 1 + - arch/x86/include/asm/vdso/processor.h | 2 +- - arch/x86/include/asm/vermagic.h | 66 ++ - arch/x86/kernel/alternative.c | 2 + - arch/x86/kernel/cpu/intel_epb.c | 4 + - arch/x86/kernel/cpu/microcode/core.c | 40 +- - arch/x86/kernel/cpu/microcode/intel.c | 14 +- - arch/x86/kernel/itmt.c | 29 +- - arch/x86/kernel/msr.c | 2 +- - arch/x86/kernel/tsc.c | 3 + - arch/x86/mm/fault.c | 4 +- - block/bfq-cgroup.c | 5 - - block/bfq-iosched.c | 20 +- - block/bfq-iosched.h | 18 +- - block/bfq-wf2q.c | 9 +- - block/blk-core.c | 3 + - block/elevator.c | 7 +- - drivers/Makefile | 15 +- - drivers/acpi/cppc_acpi.c | 131 ++- - drivers/ata/libahci.c | 4 +- - drivers/base/arch_topology.c | 2 +- - drivers/base/firmware_loader/main.c | 2 + - drivers/block/zram/Kconfig | 18 + - drivers/block/zram/zram_drv.c | 39 + - drivers/cpufreq/amd-pstate.c | 1003 ++++++++++++++++++- - drivers/cpufreq/cppc_cpufreq.c | 2 +- - drivers/cpufreq/intel_pstate.c | 7 + - drivers/idle/intel_idle.c | 44 +- - drivers/input/serio/i8042.c | 10 +- - drivers/md/dm-crypt.c | 5 + - drivers/net/dummy.c | 2 +- - drivers/nvme/host/core.c | 2 +- - drivers/pci/pci.c | 2 +- - drivers/powercap/intel_rapl_common.c | 2 +- - drivers/thermal/intel/intel_powerclamp.c | 10 + - fs/xattr.c | 15 +- - include/acpi/cppc_acpi.h | 17 + - include/linux/ipc_namespace.h | 5 +- - include/linux/jbd2.h | 2 +- - include/linux/page_counter.h | 1 + - include/linux/pagemap.h | 2 +- - include/linux/sched.h | 1 + - include/linux/user_namespace.h | 4 + - include/linux/wait.h | 2 + - include/uapi/linux/if_bonding.h | 2 +- - init/Kconfig | 26 + - init/do_mounts.c | 16 +- - kernel/Kconfig.hz | 24 + - kernel/fork.c | 14 + - kernel/locking/rwsem.c | 4 +- - kernel/module/internal.h | 2 + - kernel/module/main.c | 1 + - kernel/module/procfs.c | 13 + - kernel/module/signing.c | 4 + - kernel/sched/core.c | 19 +- - kernel/sched/debug.c | 1 + - kernel/sched/fair.c | 20 +- - kernel/sched/wait.c | 24 + - kernel/sysctl.c | 22 + - kernel/user_namespace.c | 7 + - kernel/watchdog.c | 2 +- - lib/Kconfig | 8 +- - lib/raid6/algos.c | 4 +- - lib/string.c | 62 +- - lib/zstd/Makefile | 16 +- - lib/zstd/common/entropy_common.c | 4 +- - lib/zstd/common/zstd_common.c | 7 + - lib/zstd/compress/zstd_double_fast.c | 61 +- - lib/zstd/compress/zstd_fast.c | 69 +- - lib/zstd/compress/zstd_lazy.c | 223 ++--- - mm/compaction.c | 6 +- - mm/ksm.c | 11 +- - mm/memcontrol.c | 2 +- - mm/page-writeback.c | 8 + - mm/page_alloc.c | 5 +- - mm/swap.c | 5 + - mm/vmpressure.c | 4 + - mm/vmscan.c | 4 + - net/ipv4/inet_connection_sock.c | 2 +- - net/ipv4/tcp.c | 4 +- - 101 files changed, 2400 insertions(+), 349 deletions(-) - create mode 100644 arch/x86/Makefile.postlink - -diff --git a/.gitignore b/.gitignore -index 265959544978..cd4ef88584ea 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -37,6 +37,7 @@ - *.o - *.o.* - *.patch -+*.relocs - *.s - *.so - *.so.dbg -diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst -index 83b58eb4ab4d..d0f0e115013b 100644 ---- a/Documentation/admin-guide/pm/amd-pstate.rst -+++ b/Documentation/admin-guide/pm/amd-pstate.rst -@@ -261,6 +261,25 @@ lowest non-linear performance in `AMD CPPC Performance Capability - `_.) - This attribute is read-only. - -+``energy_performance_available_preferences`` -+ -+All the supported EPP preference could be selected, List of the strings that -+can be set to the ``energy_performance_preference`` attribute -+those different profiles represent different energy vs efficiency hints provided -+to low-level firmware -+however, the ``default`` represents the epp value is set by platform firmware -+This attribute is read-only. -+ -+``energy_performance_preference`` -+ -+The current energy performance preference can be read from this attribute. -+and user can change current preference according to energy or performance needs -+Please get all support profiles list from -+``energy_performance_available_preferences`` attribute, all the profiles are -+integer values defined between 0 to 255 when EPP feature is enabled by platform -+firmware, if EPP feature is disabled, driver will ignore the written value -+This attribute is read-write. -+ - Other performance and frequency values can be read back from - ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. - -diff --git a/Makefile b/Makefile -index 647a42a1f800..5c327c29ef12 100644 ---- a/Makefile -+++ b/Makefile -@@ -758,6 +758,8 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) - - ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE - KBUILD_CFLAGS += -O2 -+else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 -+KBUILD_CFLAGS += -O3 - else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE - KBUILD_CFLAGS += -Os - endif -diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig -index e31a8ebc3ecc..0016149f9583 100644 ---- a/arch/arc/configs/axs101_defconfig -+++ b/arch/arc/configs/axs101_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig -index e0e8567f0d75..5b031582a1cf 100644 ---- a/arch/arc/configs/axs103_defconfig -+++ b/arch/arc/configs/axs103_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig -index fcbc952bc75b..d4eec39e0112 100644 ---- a/arch/arc/configs/axs103_smp_defconfig -+++ b/arch/arc/configs/axs103_smp_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig -index d87ad7e88d62..7337cdf4ffdd 100644 ---- a/arch/arc/configs/haps_hs_defconfig -+++ b/arch/arc/configs/haps_hs_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EXPERT=y - CONFIG_PERF_EVENTS=y - # CONFIG_COMPAT_BRK is not set -diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig -index 8d82cdb7f86a..bc927221afc0 100644 ---- a/arch/arc/configs/haps_hs_smp_defconfig -+++ b/arch/arc/configs/haps_hs_smp_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig -index f856b03e0fb5..aa000075a575 100644 ---- a/arch/arc/configs/hsdk_defconfig -+++ b/arch/arc/configs/hsdk_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y - CONFIG_BLK_DEV_RAM=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig -index a1ce12bf5b16..326f6cde7826 100644 ---- a/arch/arc/configs/nsim_700_defconfig -+++ b/arch/arc/configs/nsim_700_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig -index ca10f4a2c823..bf39a0091679 100644 ---- a/arch/arc/configs/nsimosci_defconfig -+++ b/arch/arc/configs/nsimosci_defconfig -@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig -index 31b6ec3683c6..7121bd71c543 100644 ---- a/arch/arc/configs/nsimosci_hs_defconfig -+++ b/arch/arc/configs/nsimosci_hs_defconfig -@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig -index 41a0037f48a5..f9863b294a70 100644 ---- a/arch/arc/configs/nsimosci_hs_smp_defconfig -+++ b/arch/arc/configs/nsimosci_hs_smp_defconfig -@@ -8,6 +8,7 @@ CONFIG_IKCONFIG_PROC=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_PERF_EVENTS=y - # CONFIG_COMPAT_BRK is not set - CONFIG_KPROBES=y -diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig -index d93b65008d4a..a12656ec0072 100644 ---- a/arch/arc/configs/tb10x_defconfig -+++ b/arch/arc/configs/tb10x_defconfig -@@ -14,6 +14,7 @@ CONFIG_INITRAMFS_SOURCE="../tb10x-rootfs.cpio" - CONFIG_INITRAMFS_ROOT_UID=2100 - CONFIG_INITRAMFS_ROOT_GID=501 - # CONFIG_RD_GZIP is not set -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - # CONFIG_AIO is not set - CONFIG_EMBEDDED=y -diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig -index 0c3b21416819..d7c858df520c 100644 ---- a/arch/arc/configs/vdk_hs38_defconfig -+++ b/arch/arc/configs/vdk_hs38_defconfig -@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_IKCONFIG=y - CONFIG_IKCONFIG_PROC=y - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig -index f9ad9d3ee702..015c1d43889e 100644 ---- a/arch/arc/configs/vdk_hs38_smp_defconfig -+++ b/arch/arc/configs/vdk_hs38_smp_defconfig -@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_IKCONFIG=y - CONFIG_IKCONFIG_PROC=y - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu -index 542377cd419d..22b919cdb6d1 100644 ---- a/arch/x86/Kconfig.cpu -+++ b/arch/x86/Kconfig.cpu -@@ -157,7 +157,7 @@ config MPENTIUM4 - - - config MK6 -- bool "K6/K6-II/K6-III" -+ bool "AMD K6/K6-II/K6-III" - depends on X86_32 - help - Select this for an AMD K6-family processor. Enables use of -@@ -165,7 +165,7 @@ config MK6 - flags to GCC. - - config MK7 -- bool "Athlon/Duron/K7" -+ bool "AMD Athlon/Duron/K7" - depends on X86_32 - help - Select this for an AMD Athlon K7-family processor. Enables use of -@@ -173,12 +173,98 @@ config MK7 - flags to GCC. - - config MK8 -- bool "Opteron/Athlon64/Hammer/K8" -+ bool "AMD Opteron/Athlon64/Hammer/K8" - help - Select this for an AMD Opteron or Athlon64 Hammer-family processor. - Enables use of some extended instructions, and passes appropriate - optimization flags to GCC. - -+config MK8SSE3 -+ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" -+ help -+ Select this for improved AMD Opteron or Athlon64 Hammer-family processors. -+ Enables use of some extended instructions, and passes appropriate -+ optimization flags to GCC. -+ -+config MK10 -+ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" -+ help -+ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, -+ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. -+ Enables use of some extended instructions, and passes appropriate -+ optimization flags to GCC. -+ -+config MBARCELONA -+ bool "AMD Barcelona" -+ help -+ Select this for AMD Family 10h Barcelona processors. -+ -+ Enables -march=barcelona -+ -+config MBOBCAT -+ bool "AMD Bobcat" -+ help -+ Select this for AMD Family 14h Bobcat processors. -+ -+ Enables -march=btver1 -+ -+config MJAGUAR -+ bool "AMD Jaguar" -+ help -+ Select this for AMD Family 16h Jaguar processors. -+ -+ Enables -march=btver2 -+ -+config MBULLDOZER -+ bool "AMD Bulldozer" -+ help -+ Select this for AMD Family 15h Bulldozer processors. -+ -+ Enables -march=bdver1 -+ -+config MPILEDRIVER -+ bool "AMD Piledriver" -+ help -+ Select this for AMD Family 15h Piledriver processors. -+ -+ Enables -march=bdver2 -+ -+config MSTEAMROLLER -+ bool "AMD Steamroller" -+ help -+ Select this for AMD Family 15h Steamroller processors. -+ -+ Enables -march=bdver3 -+ -+config MEXCAVATOR -+ bool "AMD Excavator" -+ help -+ Select this for AMD Family 15h Excavator processors. -+ -+ Enables -march=bdver4 -+ -+config MZEN -+ bool "AMD Zen" -+ help -+ Select this for AMD Family 17h Zen processors. -+ -+ Enables -march=znver1 -+ -+config MZEN2 -+ bool "AMD Zen 2" -+ help -+ Select this for AMD Family 17h Zen 2 processors. -+ -+ Enables -march=znver2 -+ -+config MZEN3 -+ bool "AMD Zen 3" -+ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ help -+ Select this for AMD Family 19h Zen 3 processors. -+ -+ Enables -march=znver3 -+ - config MCRUSOE - bool "Crusoe" - depends on X86_32 -@@ -270,7 +356,7 @@ config MPSC - in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. - - config MCORE2 -- bool "Core 2/newer Xeon" -+ bool "Intel Core 2" - help - - Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and -@@ -278,6 +364,8 @@ config MCORE2 - family in /proc/cpuinfo. Newer ones have 6 and older ones 15 - (not a typo) - -+ Enables -march=core2 -+ - config MATOM - bool "Intel Atom" - help -@@ -287,6 +375,182 @@ config MATOM - accordingly optimized code. Use a recent GCC with specific Atom - support in order to fully benefit from selecting this option. - -+config MNEHALEM -+ bool "Intel Nehalem" -+ select X86_P6_NOP -+ help -+ -+ Select this for 1st Gen Core processors in the Nehalem family. -+ -+ Enables -march=nehalem -+ -+config MWESTMERE -+ bool "Intel Westmere" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Westmere formerly Nehalem-C family. -+ -+ Enables -march=westmere -+ -+config MSILVERMONT -+ bool "Intel Silvermont" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Silvermont platform. -+ -+ Enables -march=silvermont -+ -+config MGOLDMONT -+ bool "Intel Goldmont" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Goldmont platform including Apollo Lake and Denverton. -+ -+ Enables -march=goldmont -+ -+config MGOLDMONTPLUS -+ bool "Intel Goldmont Plus" -+ select X86_P6_NOP -+ help -+ -+ Select this for the Intel Goldmont Plus platform including Gemini Lake. -+ -+ Enables -march=goldmont-plus -+ -+config MSANDYBRIDGE -+ bool "Intel Sandy Bridge" -+ select X86_P6_NOP -+ help -+ -+ Select this for 2nd Gen Core processors in the Sandy Bridge family. -+ -+ Enables -march=sandybridge -+ -+config MIVYBRIDGE -+ bool "Intel Ivy Bridge" -+ select X86_P6_NOP -+ help -+ -+ Select this for 3rd Gen Core processors in the Ivy Bridge family. -+ -+ Enables -march=ivybridge -+ -+config MHASWELL -+ bool "Intel Haswell" -+ select X86_P6_NOP -+ help -+ -+ Select this for 4th Gen Core processors in the Haswell family. -+ -+ Enables -march=haswell -+ -+config MBROADWELL -+ bool "Intel Broadwell" -+ select X86_P6_NOP -+ help -+ -+ Select this for 5th Gen Core processors in the Broadwell family. -+ -+ Enables -march=broadwell -+ -+config MSKYLAKE -+ bool "Intel Skylake" -+ select X86_P6_NOP -+ help -+ -+ Select this for 6th Gen Core processors in the Skylake family. -+ -+ Enables -march=skylake -+ -+config MSKYLAKEX -+ bool "Intel Skylake X" -+ select X86_P6_NOP -+ help -+ -+ Select this for 6th Gen Core processors in the Skylake X family. -+ -+ Enables -march=skylake-avx512 -+ -+config MCANNONLAKE -+ bool "Intel Cannon Lake" -+ select X86_P6_NOP -+ help -+ -+ Select this for 8th Gen Core processors -+ -+ Enables -march=cannonlake -+ -+config MICELAKE -+ bool "Intel Ice Lake" -+ select X86_P6_NOP -+ help -+ -+ Select this for 10th Gen Core processors in the Ice Lake family. -+ -+ Enables -march=icelake-client -+ -+config MCASCADELAKE -+ bool "Intel Cascade Lake" -+ select X86_P6_NOP -+ help -+ -+ Select this for Xeon processors in the Cascade Lake family. -+ -+ Enables -march=cascadelake -+ -+config MCOOPERLAKE -+ bool "Intel Cooper Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) -+ select X86_P6_NOP -+ help -+ -+ Select this for Xeon processors in the Cooper Lake family. -+ -+ Enables -march=cooperlake -+ -+config MTIGERLAKE -+ bool "Intel Tiger Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) -+ select X86_P6_NOP -+ help -+ -+ Select this for third-generation 10 nm process processors in the Tiger Lake family. -+ -+ Enables -march=tigerlake -+ -+config MSAPPHIRERAPIDS -+ bool "Intel Sapphire Rapids" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help -+ -+ Select this for third-generation 10 nm process processors in the Sapphire Rapids family. -+ -+ Enables -march=sapphirerapids -+ -+config MROCKETLAKE -+ bool "Intel Rocket Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help -+ -+ Select this for eleventh-generation processors in the Rocket Lake family. -+ -+ Enables -march=rocketlake -+ -+config MALDERLAKE -+ bool "Intel Alder Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help -+ -+ Select this for twelfth-generation processors in the Alder Lake family. -+ -+ Enables -march=alderlake -+ - config GENERIC_CPU - bool "Generic-x86-64" - depends on X86_64 -@@ -294,6 +558,50 @@ config GENERIC_CPU - Generic x86-64 CPU. - Run equally well on all x86-64 CPUs. - -+config GENERIC_CPU2 -+ bool "Generic-x86-64-v2" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64 CPU. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v2. -+ -+config GENERIC_CPU3 -+ bool "Generic-x86-64-v3" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64-v3 CPU with v3 instructions. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v3. -+ -+config GENERIC_CPU4 -+ bool "Generic-x86-64-v4" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64 CPU with v4 instructions. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v4. -+ -+config MNATIVE_INTEL -+ bool "Intel-Native optimizations autodetected by the compiler" -+ help -+ -+ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects -+ the optimum settings to use based on your processor. Do NOT use this -+ for AMD CPUs. Intel Only! -+ -+ Enables -march=native -+ -+config MNATIVE_AMD -+ bool "AMD-Native optimizations autodetected by the compiler" -+ help -+ -+ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects -+ the optimum settings to use based on your processor. Do NOT use this -+ for Intel CPUs. AMD Only! -+ -+ Enables -march=native -+ - endchoice - - config X86_GENERIC -@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT - config X86_L1_CACHE_SHIFT - int - default "7" if MPENTIUM4 || MPSC -- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU -+ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 - default "4" if MELAN || M486SX || M486 || MGEODEGX1 - default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX - -@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 - - config X86_INTEL_USERCOPY - def_bool y -- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 -+ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL - - config X86_USE_PPRO_CHECKSUM - def_bool y -- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM -+ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD - - # - # P6_NOPs are a relatively minor optimization that require a family >= -@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM - config X86_P6_NOP - def_bool y - depends on X86_64 -- depends on (MCORE2 || MPENTIUM4 || MPSC) -+ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) - - config X86_TSC - def_bool y -- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 -+ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 - - config X86_CMPXCHG64 - def_bool y -- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 -+ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD - - # this should be set for all -march=.. options where the compiler - # generates cmov. - config X86_CMOV - def_bool y -- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) -+ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) - - config X86_MINIMUM_CPU_FAMILY - int - default "64" if X86_64 -- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) -+ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) - default "5" if X86_32 && X86_CMPXCHG64 - default "4" - -diff --git a/arch/x86/Makefile b/arch/x86/Makefile -index bafbd905e6e7..476ab926c33e 100644 ---- a/arch/x86/Makefile -+++ b/arch/x86/Makefile -@@ -150,8 +150,44 @@ else - # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) - cflags-$(CONFIG_MK8) += -march=k8 - cflags-$(CONFIG_MPSC) += -march=nocona -- cflags-$(CONFIG_MCORE2) += -march=core2 -- cflags-$(CONFIG_MATOM) += -march=atom -+ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3 -+ cflags-$(CONFIG_MK10) += -march=amdfam10 -+ cflags-$(CONFIG_MBARCELONA) += -march=barcelona -+ cflags-$(CONFIG_MBOBCAT) += -march=btver1 -+ cflags-$(CONFIG_MJAGUAR) += -march=btver2 -+ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1 -+ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm -+ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm -+ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm -+ cflags-$(CONFIG_MZEN) += -march=znver1 -+ cflags-$(CONFIG_MZEN2) += -march=znver2 -+ cflags-$(CONFIG_MZEN3) += -march=znver3 -+ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native -+ cflags-$(CONFIG_MNATIVE_AMD) += -march=native -+ cflags-$(CONFIG_MATOM) += -march=bonnell -+ cflags-$(CONFIG_MCORE2) += -march=core2 -+ cflags-$(CONFIG_MNEHALEM) += -march=nehalem -+ cflags-$(CONFIG_MWESTMERE) += -march=westmere -+ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont -+ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont -+ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus -+ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge -+ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge -+ cflags-$(CONFIG_MHASWELL) += -march=haswell -+ cflags-$(CONFIG_MBROADWELL) += -march=broadwell -+ cflags-$(CONFIG_MSKYLAKE) += -march=skylake -+ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512 -+ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake -+ cflags-$(CONFIG_MICELAKE) += -march=icelake-client -+ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake -+ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake -+ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake -+ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids -+ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake -+ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake -+ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2 -+ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3 -+ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4 - cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic - KBUILD_CFLAGS += $(cflags-y) - -diff --git a/arch/x86/Makefile.postlink b/arch/x86/Makefile.postlink -new file mode 100644 -index 000000000000..b38ffa4defb3 ---- /dev/null -+++ b/arch/x86/Makefile.postlink -@@ -0,0 +1,41 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# =========================================================================== -+# Post-link x86 pass -+# =========================================================================== -+# -+# 1. Separate relocations from vmlinux into vmlinux.relocs. -+# 2. Strip relocations from vmlinux. -+ -+PHONY := __archpost -+__archpost: -+ -+-include include/config/auto.conf -+include scripts/Kbuild.include -+ -+CMD_RELOCS = arch/x86/tools/relocs -+quiet_cmd_relocs = RELOCS $@.relocs -+ cmd_relocs = $(CMD_RELOCS) $@ > $@.relocs;$(CMD_RELOCS) --abs-relocs $@ -+ -+quiet_cmd_strip_relocs = RSTRIP $@ -+ cmd_strip_relocs = $(OBJCOPY) --remove-section='.rel.*' --remove-section='.rel__*' --remove-section='.rela.*' --remove-section='.rela__*' $@ -+ -+# `@true` prevents complaint when there is nothing to be done -+ -+vmlinux: FORCE -+ @true -+ifeq ($(CONFIG_X86_NEED_RELOCS),y) -+ $(call cmd,relocs) -+ $(call cmd,strip_relocs) -+endif -+ -+%.ko: FORCE -+ @true -+ -+clean: -+ @rm -f vmlinux.relocs -+ -+PHONY += FORCE clean -+ -+FORCE: -+ -+.PHONY: $(PHONY) -diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile -index 35ce1a64068b..eba7709d75ae 100644 ---- a/arch/x86/boot/compressed/Makefile -+++ b/arch/x86/boot/compressed/Makefile -@@ -120,14 +120,12 @@ $(obj)/vmlinux.bin: vmlinux FORCE - - targets += $(patsubst $(obj)/%,%,$(vmlinux-objs-y)) vmlinux.bin.all vmlinux.relocs - --CMD_RELOCS = arch/x86/tools/relocs --quiet_cmd_relocs = RELOCS $@ -- cmd_relocs = $(CMD_RELOCS) $< > $@;$(CMD_RELOCS) --abs-relocs $< --$(obj)/vmlinux.relocs: vmlinux FORCE -- $(call if_changed,relocs) -+# vmlinux.relocs is created by the vmlinux postlink step. -+vmlinux.relocs: vmlinux -+ @true - - vmlinux.bin.all-y := $(obj)/vmlinux.bin --vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs -+vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += vmlinux.relocs - - $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,gzip) -diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h -index 6674bdb096f3..e278872e9187 100644 ---- a/arch/x86/include/asm/msr-index.h -+++ b/arch/x86/include/asm/msr-index.h -@@ -569,6 +569,7 @@ - #define MSR_AMD_CPPC_CAP2 0xc00102b2 - #define MSR_AMD_CPPC_REQ 0xc00102b3 - #define MSR_AMD_CPPC_STATUS 0xc00102b4 -+#define MSR_AMD_CPPC_HW_CTL 0xc0010015 - - #define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) - #define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) -@@ -579,12 +580,18 @@ - #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) - #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) - #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) -+#define AMD_CPPC_PRECISION_BOOST_BIT 25 -+#define AMD_CPPC_PRECISION_BOOST_ENABLED BIT_ULL(AMD_CPPC_PRECISION_BOOST_BIT) - - /* AMD Performance Counter Global Status and Control MSRs */ - #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 - #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 - #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 - -+#define AMD_CPPC_EPP_PERFORMANCE 0x00 -+#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 -+#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF -+#define AMD_CPPC_EPP_POWERSAVE 0xFF - /* Fam 17h MSRs */ - #define MSR_F17H_IRPERF 0xc00000e9 - -diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h -index 458c891a8273..d86eb1ebf59f 100644 ---- a/arch/x86/include/asm/topology.h -+++ b/arch/x86/include/asm/topology.h -@@ -175,6 +175,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled; - - /* Interface to set priority of a cpu */ - void sched_set_itmt_core_prio(int prio, int core_cpu); -+void sched_set_itmt_power_ratio(int power_ratio, int core_cpu); - - /* Interface to notify scheduler that system supports ITMT */ - int sched_set_itmt_support(void); -diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h -index 57b1a7034c64..e2c45674f989 100644 ---- a/arch/x86/include/asm/vdso/processor.h -+++ b/arch/x86/include/asm/vdso/processor.h -@@ -10,7 +10,7 @@ - /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ - static __always_inline void rep_nop(void) - { -- asm volatile("rep; nop" ::: "memory"); -+ asm volatile("lfence" ::: "memory"); - } - - static __always_inline void cpu_relax(void) -diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h -index 75884d2cdec3..4e6a08d4c7e5 100644 ---- a/arch/x86/include/asm/vermagic.h -+++ b/arch/x86/include/asm/vermagic.h -@@ -17,6 +17,48 @@ - #define MODULE_PROC_FAMILY "586MMX " - #elif defined CONFIG_MCORE2 - #define MODULE_PROC_FAMILY "CORE2 " -+#elif defined CONFIG_MNATIVE_INTEL -+#define MODULE_PROC_FAMILY "NATIVE_INTEL " -+#elif defined CONFIG_MNATIVE_AMD -+#define MODULE_PROC_FAMILY "NATIVE_AMD " -+#elif defined CONFIG_MNEHALEM -+#define MODULE_PROC_FAMILY "NEHALEM " -+#elif defined CONFIG_MWESTMERE -+#define MODULE_PROC_FAMILY "WESTMERE " -+#elif defined CONFIG_MSILVERMONT -+#define MODULE_PROC_FAMILY "SILVERMONT " -+#elif defined CONFIG_MGOLDMONT -+#define MODULE_PROC_FAMILY "GOLDMONT " -+#elif defined CONFIG_MGOLDMONTPLUS -+#define MODULE_PROC_FAMILY "GOLDMONTPLUS " -+#elif defined CONFIG_MSANDYBRIDGE -+#define MODULE_PROC_FAMILY "SANDYBRIDGE " -+#elif defined CONFIG_MIVYBRIDGE -+#define MODULE_PROC_FAMILY "IVYBRIDGE " -+#elif defined CONFIG_MHASWELL -+#define MODULE_PROC_FAMILY "HASWELL " -+#elif defined CONFIG_MBROADWELL -+#define MODULE_PROC_FAMILY "BROADWELL " -+#elif defined CONFIG_MSKYLAKE -+#define MODULE_PROC_FAMILY "SKYLAKE " -+#elif defined CONFIG_MSKYLAKEX -+#define MODULE_PROC_FAMILY "SKYLAKEX " -+#elif defined CONFIG_MCANNONLAKE -+#define MODULE_PROC_FAMILY "CANNONLAKE " -+#elif defined CONFIG_MICELAKE -+#define MODULE_PROC_FAMILY "ICELAKE " -+#elif defined CONFIG_MCASCADELAKE -+#define MODULE_PROC_FAMILY "CASCADELAKE " -+#elif defined CONFIG_MCOOPERLAKE -+#define MODULE_PROC_FAMILY "COOPERLAKE " -+#elif defined CONFIG_MTIGERLAKE -+#define MODULE_PROC_FAMILY "TIGERLAKE " -+#elif defined CONFIG_MSAPPHIRERAPIDS -+#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " -+#elif defined CONFIG_ROCKETLAKE -+#define MODULE_PROC_FAMILY "ROCKETLAKE " -+#elif defined CONFIG_MALDERLAKE -+#define MODULE_PROC_FAMILY "ALDERLAKE " - #elif defined CONFIG_MATOM - #define MODULE_PROC_FAMILY "ATOM " - #elif defined CONFIG_M686 -@@ -35,6 +77,30 @@ - #define MODULE_PROC_FAMILY "K7 " - #elif defined CONFIG_MK8 - #define MODULE_PROC_FAMILY "K8 " -+#elif defined CONFIG_MK8SSE3 -+#define MODULE_PROC_FAMILY "K8SSE3 " -+#elif defined CONFIG_MK10 -+#define MODULE_PROC_FAMILY "K10 " -+#elif defined CONFIG_MBARCELONA -+#define MODULE_PROC_FAMILY "BARCELONA " -+#elif defined CONFIG_MBOBCAT -+#define MODULE_PROC_FAMILY "BOBCAT " -+#elif defined CONFIG_MBULLDOZER -+#define MODULE_PROC_FAMILY "BULLDOZER " -+#elif defined CONFIG_MPILEDRIVER -+#define MODULE_PROC_FAMILY "PILEDRIVER " -+#elif defined CONFIG_MSTEAMROLLER -+#define MODULE_PROC_FAMILY "STEAMROLLER " -+#elif defined CONFIG_MJAGUAR -+#define MODULE_PROC_FAMILY "JAGUAR " -+#elif defined CONFIG_MEXCAVATOR -+#define MODULE_PROC_FAMILY "EXCAVATOR " -+#elif defined CONFIG_MZEN -+#define MODULE_PROC_FAMILY "ZEN " -+#elif defined CONFIG_MZEN2 -+#define MODULE_PROC_FAMILY "ZEN2 " -+#elif defined CONFIG_MZEN3 -+#define MODULE_PROC_FAMILY "ZEN3 " - #elif defined CONFIG_MELAN - #define MODULE_PROC_FAMILY "ELAN " - #elif defined CONFIG_MCRUSOE -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index 62f6b8b7c4a5..f9c9b5850847 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -936,7 +936,9 @@ void __init alternative_instructions(void) - * Then patch alternatives, such that those paravirt calls that are in - * alternatives can be overwritten by their immediate fragments. - */ -+ printk("clr: Applying alternatives\n"); - apply_alternatives(__alt_instructions, __alt_instructions_end); -+ printk("clr: Applying alternatives done\n"); - - apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); - -diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c -index fbaf12e43f41..c8c2d6f1a8ac 100644 ---- a/arch/x86/kernel/cpu/intel_epb.c -+++ b/arch/x86/kernel/cpu/intel_epb.c -@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev, - if (ret < 0) - return ret; - -+ /* update the ITMT scheduler logic to use the power policy data */ -+ /* scale the val up by 2 so the range is 224 - 256 */ -+ sched_set_itmt_power_ratio(256 - val * 2, cpu); -+ - return count; - } - -diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c -index ad57e0e4d674..e2ed33c2bd4d 100644 ---- a/arch/x86/kernel/cpu/microcode/core.c -+++ b/arch/x86/kernel/cpu/microcode/core.c -@@ -44,6 +44,8 @@ - - static struct microcode_ops *microcode_ops; - static bool dis_ucode_ldr = true; -+bool ucode_rollback = false; -+int enable_rollback = 0; - - bool initrd_gone; - -@@ -80,6 +82,26 @@ static u32 final_levels[] = { - 0, /* T-101 terminator */ - }; - -+static int __init ucode_setup(char *str) -+{ -+ if (!str) -+ return -EINVAL; -+ -+ while (*str) { -+ if (!strncmp(str, "rollback", 8)) { -+ enable_rollback = 1; -+ pr_info("Microcode Rollback Enabled\n"); -+ } -+ str += strcspn(str, ","); -+ while (*str == ',') -+ str++; -+ } -+ return 0; -+} -+ -+__setup("ucode=", ucode_setup); -+ -+ - /* - * Check the current patch level on this CPU. - * -@@ -512,6 +534,7 @@ static ssize_t reload_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) - { -+ struct cpuinfo_x86 *c = &boot_cpu_data; - enum ucode_state tmp_ret = UCODE_OK; - int bsp = boot_cpu_data.cpu_index; - unsigned long val; -@@ -521,7 +544,7 @@ static ssize_t reload_store(struct device *dev, - if (ret) - return ret; - -- if (val != 1) -+ if (!val || val > 2) - return size; - - cpus_read_lock(); -@@ -529,6 +552,20 @@ static ssize_t reload_store(struct device *dev, - ret = check_online_cpus(); - if (ret) - goto put; -+ /* -+ * Check if the vendor is Intel to permit reloading -+ * microcode even if the revision is unchanged. -+ * This is typically used during development of microcode -+ * and changing rev is a pain. -+ */ -+ if ((val == 2) && ((c->x86_vendor != X86_VENDOR_INTEL) || -+ !enable_rollback)) -+ return size; -+ else if (val == 2) { -+ mutex_lock(µcode_mutex); -+ ucode_rollback = true; -+ mutex_unlock(µcode_mutex); -+ } - - tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); - if (tmp_ret != UCODE_NEW) -@@ -539,6 +576,7 @@ static ssize_t reload_store(struct device *dev, - mutex_unlock(µcode_mutex); - - put: -+ ucode_rollback = false; - cpus_read_unlock(); - - if (ret == 0) -diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c -index 025c8f0cd948..60473f577a25 100644 ---- a/arch/x86/kernel/cpu/microcode/intel.c -+++ b/arch/x86/kernel/cpu/microcode/intel.c -@@ -44,6 +44,7 @@ static struct microcode_intel *intel_ucode_patch; - - /* last level cache size per core */ - static int llc_size_per_core; -+extern bool ucode_rollback; - - /* - * Returns 1 if update has been found, 0 otherwise. -@@ -80,7 +81,7 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev - { - struct microcode_header_intel *mc_hdr = mc; - -- if (mc_hdr->rev <= new_rev) -+ if (!ucode_rollback && mc_hdr->rev <= new_rev) - return 0; - - return find_matching_signature(mc, csig, cpf); -@@ -120,7 +121,7 @@ static void save_microcode_patch(struct ucode_cpu_info *uci, void *data, unsigne - if (find_matching_signature(data, sig, pf)) { - prev_found = true; - -- if (mc_hdr->rev <= mc_saved_hdr->rev) -+ if (!ucode_rollback && mc_hdr->rev <= mc_saved_hdr->rev) - continue; - - p = memdup_patch(data, size); -@@ -649,7 +650,7 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci) - - phdr = (struct microcode_header_intel *)iter->data; - -- if (phdr->rev <= uci->cpu_sig.rev) -+ if (!ucode_rollback && phdr->rev <= uci->cpu_sig.rev) - continue; - - if (!find_matching_signature(phdr, -@@ -734,10 +735,11 @@ static enum ucode_state apply_microcode_intel(int cpu) - * already. - */ - rev = intel_get_microcode_revision(); -- if (rev >= mc->hdr.rev) { -+ if (!ucode_rollback && rev >= mc->hdr.rev) { - ret = UCODE_OK; - goto out; -- } -+ } else if (ucode_rollback) -+ ret = UCODE_OK; - - /* - * Writeback and invalidate caches before updating microcode to avoid -@@ -756,7 +758,7 @@ static enum ucode_state apply_microcode_intel(int cpu) - return UCODE_ERROR; - } - -- if (bsp && rev != prev_rev) { -+ if (bsp && ((rev != prev_rev) || ucode_rollback)) { - pr_info("updated to revision 0x%x, date = %04x-%02x-%02x\n", - rev, - mc->hdr.date & 0xffff, -diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c -index 9ff480e94511..d4326e050fb7 100644 ---- a/arch/x86/kernel/itmt.c -+++ b/arch/x86/kernel/itmt.c -@@ -25,6 +25,7 @@ - - static DEFINE_MUTEX(itmt_update_mutex); - DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); -+DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio); - - /* Boolean to track if system has ITMT capabilities */ - static bool __read_mostly sched_itmt_capable; -@@ -169,7 +170,12 @@ void sched_clear_itmt_support(void) - - int arch_asym_cpu_priority(int cpu) - { -- return per_cpu(sched_core_priority, cpu); -+ int power_ratio = per_cpu(sched_power_ratio, cpu); -+ -+ /* a power ratio of 0 (uninitialized) is assumed to be maximum */ -+ if (power_ratio == 0) -+ power_ratio = 256 - 2 * 6; -+ return per_cpu(sched_core_priority, cpu) * power_ratio / 256; - } - - /** -@@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) - i++; - } - } -+ -+/** -+ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT -+ * @power_ratio: The power scaling ratio [1..256] for the core -+ * @core_cpu: The cpu number associated with the core -+ * -+ * Set a scaling to the cpu performance based on long term power -+ * settings (like EPB). -+ * -+ * Note this is for the policy not for the actual dynamic frequency; -+ * the frequency will increase itself as workloads run on a core. -+ */ -+ -+void sched_set_itmt_power_ratio(int power_ratio, int core_cpu) -+{ -+ int cpu; -+ -+ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) { -+ per_cpu(sched_power_ratio, cpu) = power_ratio; -+ } -+} -diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c -index ed8ac6bcbafb..d6fc5bdb0246 100644 ---- a/arch/x86/kernel/msr.c -+++ b/arch/x86/kernel/msr.c -@@ -48,7 +48,7 @@ enum allow_write_msrs { - MSR_WRITES_DEFAULT, - }; - --static enum allow_write_msrs allow_writes = MSR_WRITES_DEFAULT; -+static enum allow_write_msrs allow_writes = MSR_WRITES_ON; - - static ssize_t msr_read(struct file *file, char __user *buf, - size_t count, loff_t *ppos) -diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c -index cafacb2e58cc..c2f80184fd33 100644 ---- a/arch/x86/kernel/tsc.c -+++ b/arch/x86/kernel/tsc.c -@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void) - if (!constant_tsc || !mask) - return 0; - -+ if (cpu != 0) -+ return cpu_data(0).loops_per_jiffy; -+ - sibling = cpumask_any_but(mask, cpu); - if (sibling < nr_cpu_ids) - return cpu_data(sibling).loops_per_jiffy; -diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c -index fa71a5d12e87..c4bbd8c66134 100644 ---- a/arch/x86/mm/fault.c -+++ b/arch/x86/mm/fault.c -@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, - if (!printk_ratelimit()) - return; - -- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx", -+ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i", - loglvl, tsk->comm, task_pid_nr(tsk), address, -- (void *)regs->ip, (void *)regs->sp, error_code); -+ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id()); - - print_vma_addr(KERN_CONT " in ", regs->ip); - -diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c -index 30b15a9a47c4..144bca006463 100644 ---- a/block/bfq-cgroup.c -+++ b/block/bfq-cgroup.c -@@ -254,17 +254,12 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - - #else /* CONFIG_BFQ_CGROUP_DEBUG */ - --void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -- blk_opf_t opf) { } - void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { } - void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { } - void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - u64 io_start_time_ns, blk_opf_t opf) { } - void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } --void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } --void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } - void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } --void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } - - #endif /* CONFIG_BFQ_CGROUP_DEBUG */ - -diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c -index c740b41fe0a4..5ea6245f0208 100644 ---- a/block/bfq-iosched.c -+++ b/block/bfq-iosched.c -@@ -1925,7 +1925,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, - bfqq->service_from_backlogged = 0; - bfq_clear_bfqq_softrt_update(bfqq); - -- bfq_add_bfqq_busy(bfqd, bfqq); -+ bfq_add_bfqq_busy(bfqq); - - /* - * Expire in-service queue if preemption may be needed for -@@ -2419,7 +2419,7 @@ static void bfq_remove_request(struct request_queue *q, - bfqq->next_rq = NULL; - - if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { -- bfq_del_bfqq_busy(bfqd, bfqq, false); -+ bfq_del_bfqq_busy(bfqq, false); - /* - * bfqq emptied. In normal operation, when - * bfqq is empty, bfqq->entity.service and -@@ -3098,7 +3098,7 @@ void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) - */ - if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && - bfqq != bfqd->in_service_queue) -- bfq_del_bfqq_busy(bfqd, bfqq, false); -+ bfq_del_bfqq_busy(bfqq, false); - - bfq_reassign_last_bfqq(bfqq, NULL); - -@@ -3908,7 +3908,7 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, - */ - bfqq->budget_timeout = jiffies; - -- bfq_del_bfqq_busy(bfqd, bfqq, true); -+ bfq_del_bfqq_busy(bfqq, true); - } else { - bfq_requeue_bfqq(bfqd, bfqq, true); - /* -@@ -5255,9 +5255,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) - struct hlist_node *n; - struct bfq_group *bfqg = bfqq_group(bfqq); - -- if (bfqq->bfqd) -- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", -- bfqq, bfqq->ref); -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); - - bfqq->ref--; - if (bfqq->ref) -@@ -5321,7 +5319,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) - hlist_del_init(&item->woken_list_node); - } - -- if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) -+ if (bfqq->bfqd->last_completed_rq_bfqq == bfqq) - bfqq->bfqd->last_completed_rq_bfqq = NULL; - - kmem_cache_free(bfq_pool, bfqq); -@@ -7463,6 +7461,7 @@ MODULE_ALIAS("bfq-iosched"); - static int __init bfq_init(void) - { - int ret; -+ char msg[60] = "BFQ I/O-scheduler: BFQ-CachyOS v5.19"; - - #ifdef CONFIG_BFQ_GROUP_IOSCHED - ret = blkcg_policy_register(&blkcg_policy_bfq); -@@ -7494,6 +7493,11 @@ static int __init bfq_init(void) - if (ret) - goto slab_kill; - -+#ifdef CONFIG_BFQ_GROUP_IOSCHED -+ strcat(msg, " (with cgroups support)"); -+#endif -+ pr_info("%s", msg); -+ - return 0; - - slab_kill: -diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h -index ad8e513d7e87..64ee618064ba 100644 ---- a/block/bfq-iosched.h -+++ b/block/bfq-iosched.h -@@ -993,20 +993,23 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); - /* ---------------- cgroups-support interface ---------------- */ - - void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq); --void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -- blk_opf_t opf); - void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf); - void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf); - void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - u64 io_start_time_ns, blk_opf_t opf); - void bfqg_stats_update_dequeue(struct bfq_group *bfqg); --void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); --void bfqg_stats_update_idle_time(struct bfq_group *bfqg); - void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg); --void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); - void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct bfq_group *bfqg); - -+#ifdef CONFIG_BFQ_CGROUP_DEBUG -+void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -+ blk_opf_t opf); -+void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); -+void bfqg_stats_update_idle_time(struct bfq_group *bfqg); -+void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); -+#endif -+ - void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg); - void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio); - void bfq_end_wr_async(struct bfq_data *bfqd); -@@ -1077,9 +1080,8 @@ void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); - void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - bool expiration); --void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -- bool expiration); --void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); -+void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration); -+void bfq_add_bfqq_busy(struct bfq_queue *bfqq); - - /* --------------- end of interface of B-WF2Q+ ---------------- */ - -diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c -index 983413cdefad..8fc3da4c23bb 100644 ---- a/block/bfq-wf2q.c -+++ b/block/bfq-wf2q.c -@@ -1651,9 +1651,10 @@ void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - * the service tree. As a special case, it can be invoked during an - * expiration. - */ --void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -- bool expiration) -+void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration) - { -+ struct bfq_data *bfqd = bfqq->bfqd; -+ - bfq_log_bfqq(bfqd, bfqq, "del from busy"); - - bfq_clear_bfqq_busy(bfqq); -@@ -1674,8 +1675,10 @@ void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, - /* - * Called when an inactive queue receives a new request. - */ --void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+void bfq_add_bfqq_busy(struct bfq_queue *bfqq) - { -+ struct bfq_data *bfqd = bfqq->bfqd; -+ - bfq_log_bfqq(bfqd, bfqq, "add to busy"); - - bfq_activate_bfqq(bfqd, bfqq); -diff --git a/block/blk-core.c b/block/blk-core.c -index 651057c4146b..88873f267b2a 100644 ---- a/block/blk-core.c -+++ b/block/blk-core.c -@@ -742,6 +742,9 @@ void submit_bio_noacct(struct bio *bio) - status = BLK_STS_OK; - goto end_io; - } -+ -+ if (bio->bi_opf & REQ_PREFLUSH) -+ current->fsync_count++; - } - - if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) -diff --git a/block/elevator.c b/block/elevator.c -index c319765892bb..b2687032a0e0 100644 ---- a/block/elevator.c -+++ b/block/elevator.c -@@ -640,8 +640,13 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) - - if (q->nr_hw_queues != 1 && - !blk_mq_is_shared_tags(q->tag_set->flags)) -+#if defined(CONFIG_CACHY) && defined(CONFIG_MQ_IOSCHED_KYBER) -+ return elevator_get(q, "kyber", false); -+#elif defined(CONFIG_CACHY) -+ return elevator_get(q, "mq-deadline", false); -+#else - return NULL; -- -+#endif - return elevator_get(q, "mq-deadline", false); - } - -diff --git a/drivers/Makefile b/drivers/Makefile -index 057857258bfd..643e2d3dee81 100644 ---- a/drivers/Makefile -+++ b/drivers/Makefile -@@ -59,15 +59,8 @@ obj-y += char/ - # iommu/ comes before gpu as gpu are using iommu controllers - obj-y += iommu/ - --# gpu/ comes after char for AGP vs DRM startup and after iommu --obj-y += gpu/ -- - obj-$(CONFIG_CONNECTOR) += connector/ - --# i810fb and intelfb depend on char/agp/ --obj-$(CONFIG_FB_I810) += video/fbdev/i810/ --obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ -- - obj-$(CONFIG_PARPORT) += parport/ - obj-y += base/ block/ misc/ mfd/ nfc/ - obj-$(CONFIG_LIBNVDIMM) += nvdimm/ -@@ -79,6 +72,14 @@ obj-y += macintosh/ - obj-y += scsi/ - obj-y += nvme/ - obj-$(CONFIG_ATA) += ata/ -+ -+# gpu/ comes after char for AGP vs DRM startup and after iommu -+obj-y += gpu/ -+ -+# i810fb and intelfb depend on char/agp/ -+obj-$(CONFIG_FB_I810) += video/fbdev/i810/ -+obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ -+ - obj-$(CONFIG_TARGET_CORE) += target/ - obj-$(CONFIG_MTD) += mtd/ - obj-$(CONFIG_SPI) += spi/ -diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c -index 1e15a9f25ae9..4801300d3c77 100644 ---- a/drivers/acpi/cppc_acpi.c -+++ b/drivers/acpi/cppc_acpi.c -@@ -424,6 +424,9 @@ bool acpi_cpc_valid(void) - struct cpc_desc *cpc_ptr; - int cpu; - -+ if (acpi_disabled) -+ return false; -+ - for_each_present_cpu(cpu) { - cpc_ptr = per_cpu(cpc_desc_ptr, cpu); - if (!cpc_ptr) -@@ -1320,6 +1323,132 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) - } - EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs); - -+/** -+ * cppc_get_epp_caps - Get the energy preference register value. -+ * @cpunum: CPU from which to get epp preference level. -+ * @perf_caps: Return address. -+ * -+ * Return: 0 for success, -EIO otherwise. -+ */ -+int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) -+{ -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); -+ struct cpc_register_resource *energy_perf_reg; -+ u64 energy_perf; -+ -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpunum); -+ return -ENODEV; -+ } -+ -+ energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; -+ -+ if (!CPC_SUPPORTED(energy_perf_reg)) -+ pr_warn("energy perf reg update is unsupported!\n"); -+ -+ if (CPC_IN_PCC(energy_perf_reg)) { -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = 0; -+ -+ if (pcc_ss_id < 0) -+ return -EIO; -+ -+ pcc_ss_data = pcc_data[pcc_ss_id]; -+ -+ down_write(&pcc_ss_data->pcc_lock); -+ -+ if (send_pcc_cmd(pcc_ss_id, CMD_READ) >= 0) { -+ cpc_read(cpunum, energy_perf_reg, &energy_perf); -+ perf_caps->energy_perf = energy_perf; -+ } else { -+ ret = -EIO; -+ } -+ -+ up_write(&pcc_ss_data->pcc_lock); -+ -+ return ret; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(cppc_get_epp_caps); -+ -+int cppc_set_auto_epp(int cpu, bool enable) -+{ -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); -+ struct cpc_register_resource *auto_sel_reg; -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = -EINVAL; -+ -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpu); -+ return -EINVAL; -+ } -+ -+ auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; -+ -+ if (CPC_IN_PCC(auto_sel_reg)) { -+ if (pcc_ss_id < 0) -+ return -EIO; -+ -+ ret = cpc_write(cpu, auto_sel_reg, enable); -+ if (ret) -+ return ret; -+ -+ pcc_ss_data = pcc_data[pcc_ss_id]; -+ -+ down_write(&pcc_ss_data->pcc_lock); -+ /* after writing CPC, transfer the ownership of PCC to platform */ -+ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); -+ up_write(&pcc_ss_data->pcc_lock); -+ return ret; -+ } -+ -+ return cpc_write(cpu, auto_sel_reg, enable); -+} -+EXPORT_SYMBOL_GPL(cppc_set_auto_epp); -+ -+/* -+ * Set Energy Performance Preference Register value through -+ * Performance Controls Interface -+ */ -+int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) -+{ -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); -+ struct cpc_register_resource *epp_set_reg; -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = -EINVAL; -+ -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpu); -+ return -EINVAL; -+ } -+ -+ epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; -+ -+ if (CPC_IN_PCC(epp_set_reg)) { -+ if (pcc_ss_id < 0) -+ return -EIO; -+ -+ ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); -+ if (ret) -+ return ret; -+ -+ pcc_ss_data = pcc_data[pcc_ss_id]; -+ -+ down_write(&pcc_ss_data->pcc_lock); -+ /* after writing CPC, transfer the ownership of PCC to platform */ -+ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); -+ up_write(&pcc_ss_data->pcc_lock); -+ } -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(cppc_set_epp_perf); -+ - /** - * cppc_set_enable - Set to enable CPPC on the processor by writing the - * Continuous Performance Control package EnableRegister field. -@@ -1355,7 +1484,7 @@ int cppc_set_enable(int cpu, bool enable) - pcc_ss_data = pcc_data[pcc_ss_id]; - - down_write(&pcc_ss_data->pcc_lock); -- /* after writing CPC, transfer the ownership of PCC to platfrom */ -+ /* after writing CPC, transfer the ownership of PCC to platform */ - ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); - up_write(&pcc_ss_data->pcc_lock); - return ret; -diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c -index cf8c7fd59ada..ad9bb5353dd3 100644 ---- a/drivers/ata/libahci.c -+++ b/drivers/ata/libahci.c -@@ -33,14 +33,14 @@ - #include "libata.h" - - static int ahci_skip_host_reset; --int ahci_ignore_sss; -+int ahci_ignore_sss=1; - EXPORT_SYMBOL_GPL(ahci_ignore_sss); - - module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444); - MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)"); - - module_param_named(ignore_sss, ahci_ignore_sss, int, 0444); --MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)"); -+MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])"); - - static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy, - unsigned hints); -diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c -index 46cbe4471e78..dd90591e51ba 100644 ---- a/drivers/base/arch_topology.c -+++ b/drivers/base/arch_topology.c -@@ -353,7 +353,7 @@ void topology_init_cpu_capacity_cppc(void) - struct cppc_perf_caps perf_caps; - int cpu; - -- if (likely(acpi_disabled || !acpi_cpc_valid())) -+ if (likely(!acpi_cpc_valid())) - return; - - raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), -diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c -index 7c3590fd97c2..bb4880e10581 100644 ---- a/drivers/base/firmware_loader/main.c -+++ b/drivers/base/firmware_loader/main.c -@@ -470,6 +470,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, - static char fw_path_para[256]; - static const char * const fw_path[] = { - fw_path_para, -+ "/etc/firmware/" UTS_RELEASE, -+ "/etc/firmware", - "/lib/firmware/updates/" UTS_RELEASE, - "/lib/firmware/updates", - "/lib/firmware/" UTS_RELEASE, -diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig -index d4100b0c083e..8739ea13161f 100644 ---- a/drivers/block/zram/Kconfig -+++ b/drivers/block/zram/Kconfig -@@ -78,3 +78,21 @@ config ZRAM_MEMORY_TRACKING - /sys/kernel/debug/zram/zramX/block_state. - - See Documentation/admin-guide/blockdev/zram.rst for more information. -+ -+config ZRAM_ENTROPY -+ bool "Use entropy optimization for zram" -+ depends on ZRAM && ZRAM_DEF_COMP_ZSTD -+ help -+ With this feature, entropy will be calculated for each page. -+ Pages above ZRAM_ENTROPY_THRESHOLD entropy will be -+ stored uncompressed. Use this feature if you need a performance -+ boost and a small loss in compression. -+ -+config ZRAM_ENTROPY_THRESHOLD -+ int -+ depends on ZRAM && ZRAM_ENTROPY -+ default 100000 if ZRAM_DEF_COMP_ZSTD -+ help -+ Pages with entropy above ZRAM_ENTROPY_THRESHOLD will be stored -+ uncompressed. The default value was chosen as a result a lot of -+ experiments. You can try set your own value. -diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c -index 226ea76cc819..d02e37e7afd9 100644 ---- a/drivers/block/zram/zram_drv.c -+++ b/drivers/block/zram/zram_drv.c -@@ -1347,6 +1347,35 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, - return ret; - } - -+ -+#ifdef CONFIG_ZRAM_ENTROPY -+static inline u32 ilog2_w(u64 n) -+{ -+ return ilog2(n * n * n * n); -+} -+ -+static inline s32 shannon_entropy(const u8 *src) -+{ -+ s32 entropy_sum = 0; -+ u32 sz_base, i; -+ u16 entropy_count[256] = { 0 }; -+ -+ for (i = 0; i < PAGE_SIZE; ++i) -+ entropy_count[src[i]]++; -+ -+ sz_base = ilog2_w(PAGE_SIZE); -+ for (i = 0; i < ARRAY_SIZE(entropy_count); ++i) { -+ if (entropy_count[i] > 0) { -+ s32 p = entropy_count[i]; -+ -+ entropy_sum += p * (sz_base - ilog2_w((u64)p)); -+ } -+ } -+ -+ return entropy_sum; -+} -+#endif -+ - static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - u32 index, struct bio *bio) - { -@@ -1373,7 +1402,17 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - compress_again: - zstrm = zcomp_stream_get(zram->comp); - src = kmap_atomic(page); -+ -+#ifdef CONFIG_ZRAM_ENTROPY -+ /* Just save this page uncompressible */ -+ if (shannon_entropy((const u8 *)src) > CONFIG_ZRAM_ENTROPY_THRESHOLD) -+ comp_len = PAGE_SIZE; -+ else -+ ret = zcomp_compress(zstrm, src, &comp_len); -+#else - ret = zcomp_compress(zstrm, src, &comp_len); -+#endif -+ - kunmap_atomic(src); - - if (unlikely(ret)) { -diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c -index 9ac75c1cde9c..e8cd2d2b7cf8 100644 ---- a/drivers/cpufreq/amd-pstate.c -+++ b/drivers/cpufreq/amd-pstate.c -@@ -31,23 +31,18 @@ - #include - #include - #include --#include - #include - #include - #include - #include - --#include - #include - - #include --#include --#include --#include - #include "amd-pstate-trace.h" - --#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 --#define AMD_PSTATE_TRANSITION_DELAY 500 -+#define AMD_PSTATE_TRANSITION_LATENCY 20000 -+#define AMD_PSTATE_TRANSITION_DELAY 1000 - - /* - * TODO: We need more time to fine tune processors with shared memory solution -@@ -63,7 +58,13 @@ module_param(shared_mem, bool, 0444); - MODULE_PARM_DESC(shared_mem, - "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); - --static struct cpufreq_driver amd_pstate_driver; -+static bool epp_enabled = true; -+module_param(epp_enabled, bool, 0444); -+MODULE_PARM_DESC(epp_enabled, -+ "load amd_pstate or amd_pstate_epp (true = amd_pstate_epp driver instance (default), false = amd_pstate driver instance)"); -+ -+static struct cpufreq_driver *default_pstate_driver; -+static struct amd_cpudata **all_cpu_data; - - /** - * struct amd_aperf_mperf -@@ -75,6 +76,7 @@ struct amd_aperf_mperf { - u64 aperf; - u64 mperf; - u64 tsc; -+ u64 time; - }; - - /** -@@ -97,6 +99,20 @@ struct amd_aperf_mperf { - * @prev: Last Aperf/Mperf/tsc count value read from register - * @freq: current cpu frequency value - * @boost_supported: check whether the Processor or SBIOS supports boost mode -+ * @precision_boost_off: the core performance boost disabled state -+ * @cppc_hw_conf_cached: the cached hardware configuration register -+ * @epp_powersave: Last saved CPPC energy performance preference -+ * when policy switched to performance -+ * @epp_policy: Last saved policy used to set energy-performance preference -+ * @epp_cached: Cached CPPC energy-performance preference value -+ * @policy: Cpufreq policy value -+ * @sched_flags: Store scheduler flags for possible cross CPU update -+ * @update_util_set: CPUFreq utility callback is set -+ * @last_update: Time stamp of the last performance state update -+ * @cppc_boost_min: Last CPPC boosted min performance state -+ * @cppc_cap1_cached: Cached value of the last CPPC Capabilities MSR -+ * @update_util: Cpufreq utility callback information -+ * @suspended: Whether or not the driver has been suspended. - * - * The amd_cpudata is key private data for each CPU thread in AMD P-State, and - * represents all the attributes and goals that AMD P-State requests at runtime. -@@ -120,10 +136,200 @@ struct amd_cpudata { - struct amd_aperf_mperf cur; - struct amd_aperf_mperf prev; - -- u64 freq; -+ u64 freq; - bool boost_supported; -+ bool precision_boost_off; -+ u64 cppc_hw_conf_cached; -+ -+ /* EPP feature related attributes*/ -+ s16 epp_powersave; -+ s16 epp_policy; -+ s16 epp_cached; -+ u32 policy; -+ u32 sched_flags; -+ bool update_util_set; -+ u64 last_update; -+ u64 last_io_update; -+ u32 cppc_boost_min; -+ u64 cppc_cap1_cached; -+ struct update_util_data update_util; -+ struct amd_aperf_mperf sample; -+ bool suspended; -+}; -+ -+/** -+ * struct amd_pstate_params - global parameters for the performance control -+ * @ cppc_boost_disabled Wheter or not the core performance boost disabled -+ */ -+struct amd_pstate_params { -+ bool cppc_boost_disabled; -+}; -+ -+/* -+ * AMD Energy Preference Performance (EPP) -+ * The EPP is used in the CCLK DPM controller to drive -+ * the frequency that a core is going to operate during -+ * short periods of activity. EPP values will be utilized for -+ * different OS profiles (balanced, performance, power savings) -+ * display strings corresponding to EPP index in the -+ * energy_perf_strings[] -+ * index String -+ *------------------------------------- -+ * 0 default -+ * 1 performance -+ * 2 balance_performance -+ * 3 balance_power -+ * 4 power -+ */ -+ enum energy_perf_value_index { -+ EPP_INDEX_DEFAULT = 0, -+ EPP_INDEX_PERFORMANCE, -+ EPP_INDEX_BALANCE_PERFORMANCE, -+ EPP_INDEX_BALANCE_POWERSAVE, -+ EPP_INDEX_POWERSAVE, -+}; -+ -+static const char * const energy_perf_strings[] = { -+ [EPP_INDEX_DEFAULT] = "default", -+ [EPP_INDEX_PERFORMANCE] = "performance", -+ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", -+ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", -+ [EPP_INDEX_POWERSAVE] = "power", -+ NULL - }; - -+static unsigned int epp_values[] = { -+ [EPP_INDEX_DEFAULT] = 0, -+ [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, -+ [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, -+ [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, -+ [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, -+}; -+ -+static struct amd_pstate_params global; -+ -+static DEFINE_MUTEX(amd_pstate_limits_lock); -+static DEFINE_MUTEX(amd_pstate_driver_lock); -+static DEFINE_SPINLOCK(amd_pstate_cpu_lock); -+ -+static bool cppc_boost __read_mostly; -+struct kobject *amd_pstate_kobj; -+ -+#ifdef CONFIG_ACPI_CPPC_LIB -+static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) -+{ -+ s16 epp; -+ struct cppc_perf_caps perf_caps; -+ int ret; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!cppc_req_cached) { -+ epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, -+ &cppc_req_cached); -+ if (epp) -+ return epp; -+ } -+ epp = (cppc_req_cached >> 24) & 0xFF; -+ } else { -+ ret = cppc_get_epp_caps(cpudata->cpu, &perf_caps); -+ if (ret < 0) { -+ pr_debug("Could not retrieve energy perf value (%d)\n", ret); -+ return -EIO; -+ } -+ epp = (s16) perf_caps.energy_perf; -+ } -+ -+ return epp; -+} -+#endif -+ -+static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata, int *raw_epp) -+{ -+ s16 epp; -+ int index = -EINVAL; -+ -+ *raw_epp = 0; -+ epp = amd_pstate_get_epp(cpudata, 0); -+ if (epp < 0) -+ return epp; -+ -+ switch (epp) { -+ case AMD_CPPC_EPP_PERFORMANCE: -+ index = EPP_INDEX_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_PERFORMANCE: -+ index = EPP_INDEX_BALANCE_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_POWERSAVE: -+ index = EPP_INDEX_BALANCE_POWERSAVE; -+ break; -+ case AMD_CPPC_EPP_POWERSAVE: -+ index = EPP_INDEX_POWERSAVE; -+ break; -+ default: -+ *raw_epp = epp; -+ index = 0; -+ } -+ -+ return index; -+} -+ -+#ifdef CONFIG_ACPI_CPPC_LIB -+static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) -+{ -+ int ret; -+ struct cppc_perf_ctrls perf_ctrls; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ u64 value = READ_ONCE(cpudata->cppc_req_cached); -+ -+ value &= ~GENMASK_ULL(31, 24); -+ value |= (u64)epp << 24; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (!ret) { -+ cpudata->epp_cached = epp; -+ } -+ } else { -+ perf_ctrls.energy_perf = epp; -+ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls); -+ if (ret){ -+ pr_debug("failed to set energy perf value (%d)\n", ret); -+ return ret; -+ } -+ cpudata->epp_cached = epp; -+ } -+ return ret; -+} -+ -+static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, -+ int pref_index, bool use_raw, -+ u32 raw_epp) -+{ -+ int epp = -EINVAL; -+ int ret; -+ -+ if (!pref_index) { -+ pr_debug("EPP pref_index is invaid\n"); -+ return -EINVAL; -+ } -+ -+ if (use_raw) -+ epp = raw_epp; -+ else if (epp == -EINVAL) -+ epp = epp_values[pref_index]; -+ -+ if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ pr_debug("EPP cannot be set under performance policy\n"); -+ return -EBUSY; -+ } -+ -+ ret = amd_pstate_set_epp(cpudata, epp); -+ -+ return ret; -+} -+#endif -+ - static inline int pstate_enable(bool enable) - { - return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); -@@ -131,12 +337,26 @@ static inline int pstate_enable(bool enable) - - static int cppc_enable(bool enable) - { -+ struct cppc_perf_ctrls perf_ctrls; - int cpu, ret = 0; - - for_each_present_cpu(cpu) { - ret = cppc_set_enable(cpu, enable); - if (ret) - return ret; -+ -+ if (epp_enabled) { -+ /* Enable autonomous mode for EPP */ -+ ret = cppc_set_auto_epp(cpu, enable); -+ if (ret) -+ return ret; -+ -+ /* Set zero to desired perf to allow EPP firmware control*/ -+ perf_ctrls.desired_perf = 0; -+ ret = cppc_set_perf(cpu, &perf_ctrls); -+ if (ret) -+ return ret; -+ } - } - - return ret; -@@ -269,6 +489,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, - u64 prev = READ_ONCE(cpudata->cppc_req_cached); - u64 value = prev; - -+ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - value &= ~AMD_CPPC_MIN_PERF(~0L); - value |= AMD_CPPC_MIN_PERF(min_perf); - -@@ -312,7 +533,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, - return -ENODEV; - - cap_perf = READ_ONCE(cpudata->highest_perf); -- min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); -+ min_perf = READ_ONCE(cpudata->lowest_perf); - max_perf = cap_perf; - - freqs.old = policy->cur; -@@ -357,8 +578,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, - if (max_perf < min_perf) - max_perf = min_perf; - -- des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); -- - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); - } - -@@ -438,18 +657,27 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) - { - struct amd_cpudata *cpudata = policy->driver_data; - int ret; -+ u64 value; - - if (!cpudata->boost_supported) { - pr_err("Boost mode is not supported by this processor or SBIOS\n"); - return -EINVAL; - } - -- if (state) -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); -+ if (ret) -+ return ret; -+ -+ if (state) { -+ value |= AMD_CPPC_PRECISION_BOOST_ENABLED; - policy->cpuinfo.max_freq = cpudata->max_freq; -- else -+ } else { -+ value &= ~AMD_CPPC_PRECISION_BOOST_ENABLED; - policy->cpuinfo.max_freq = cpudata->nominal_freq; -- -+ } - policy->max = policy->cpuinfo.max_freq; -+ WRITE_ONCE(cpudata->cppc_hw_conf_cached, value); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, value); - - ret = freq_qos_update_request(&cpudata->req[1], - policy->cpuinfo.max_freq); -@@ -470,7 +698,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) - return; - - cpudata->boost_supported = true; -- amd_pstate_driver.boost_enabled = true; -+ default_pstate_driver->boost_enabled = true; - } - - static int amd_pstate_cpu_init(struct cpufreq_policy *policy) -@@ -478,6 +706,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; - struct device *dev; - struct amd_cpudata *cpudata; -+ u64 value; - - dev = get_cpu_device(policy->cpu); - if (!dev) -@@ -541,7 +770,17 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; - - policy->driver_data = cpudata; -+ if (!shared_mem) { -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); -+ if (ret) -+ return ret; -+ cpudata->precision_boost_off = value & AMD_CPPC_PRECISION_BOOST_ENABLED; - -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ } - amd_pstate_boost_init(cpudata); - - return 0; -@@ -555,9 +794,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - - static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) - { -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - freq_qos_remove_request(&cpudata->req[1]); - freq_qos_remove_request(&cpudata->req[0]); -@@ -599,9 +836,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, - char *buf) - { - int max_freq; -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - max_freq = amd_get_max_freq(cpudata); - if (max_freq < 0) -@@ -614,9 +849,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli - char *buf) - { - int freq; -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - freq = amd_get_lowest_nonlinear_freq(cpudata); - if (freq < 0) -@@ -640,10 +873,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, - return sprintf(&buf[0], "%u\n", perf); - } - -+static ssize_t show_energy_performance_available_preferences( -+ struct cpufreq_policy *policy, char *buf) -+{ -+ int i = 0; -+ int ret = 0; -+ -+ while (energy_perf_strings[i] != NULL) -+ ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); -+ -+ ret += sprintf(&buf[ret], "\n"); -+ -+ return ret; -+} -+ -+static ssize_t store_energy_performance_preference( -+ struct cpufreq_policy *policy, const char *buf, size_t count) -+{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ char str_preference[21]; -+ bool raw = false; -+ ssize_t ret; -+ u32 epp = 0; -+ -+ ret = sscanf(buf, "%20s", str_preference); -+ if (ret != 1) -+ return -EINVAL; -+ -+ ret = match_string(energy_perf_strings, -1, str_preference); -+ if (ret < 0) { -+ ret = kstrtouint(buf, 10, &epp); -+ if (ret) -+ return ret; -+ -+ if ((epp > 255) || (epp < 0)) -+ return -EINVAL; -+ -+ raw = true; -+ } -+ -+ mutex_lock(&amd_pstate_limits_lock); -+ ret = amd_pstate_set_energy_pref_index(cpudata, ret, raw, epp); -+ mutex_unlock(&amd_pstate_limits_lock); -+ -+ return ret ?: count; -+} -+ -+static ssize_t show_energy_performance_preference( -+ struct cpufreq_policy *policy, char *buf) -+{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ int preference, raw_epp; -+ -+ preference = amd_pstate_get_energy_pref_index(cpudata, &raw_epp); -+ if (preference < 0) -+ return preference; -+ -+ if (raw_epp) -+ return sprintf(buf, "%d\n", raw_epp); -+ else -+ return sprintf(buf, "%s\n", energy_perf_strings[preference]); -+} -+ -+static void amd_pstate_update_policies(void) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) -+ cpufreq_update_policy(cpu); -+} -+ -+static ssize_t show_pstate_dynamic_boost(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", cppc_boost); -+} -+ -+static ssize_t store_pstate_dynamic_boost(struct kobject *a, -+ struct kobj_attribute *b, -+ const char *buf, size_t count) -+{ -+ unsigned int input; -+ int ret; -+ -+ ret = kstrtouint(buf, 10, &input); -+ if (ret) -+ return ret; -+ -+ mutex_lock(&amd_pstate_driver_lock); -+ cppc_boost = !!input; -+ amd_pstate_update_policies(); -+ mutex_unlock(&amd_pstate_driver_lock); -+ -+ return count; -+} -+ - cpufreq_freq_attr_ro(amd_pstate_max_freq); - cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); - - cpufreq_freq_attr_ro(amd_pstate_highest_perf); -+cpufreq_freq_attr_rw(energy_performance_preference); -+cpufreq_freq_attr_ro(energy_performance_available_preferences); -+define_one_global_rw(pstate_dynamic_boost); - - static struct freq_attr *amd_pstate_attr[] = { - &amd_pstate_max_freq, -@@ -652,6 +983,548 @@ static struct freq_attr *amd_pstate_attr[] = { - NULL, - }; - -+static struct freq_attr *amd_pstate_epp_attr[] = { -+ &amd_pstate_max_freq, -+ &amd_pstate_lowest_nonlinear_freq, -+ &amd_pstate_highest_perf, -+ &energy_performance_preference, -+ &energy_performance_available_preferences, -+ NULL, -+}; -+ -+static struct attribute *pstate_global_attributes[] = { -+ &pstate_dynamic_boost.attr, -+ NULL -+}; -+ -+static const struct attribute_group amd_pstate_global_attr_group = { -+ .attrs = pstate_global_attributes, -+}; -+ -+static inline void update_boost_state(void) -+{ -+ u64 misc_en; -+ struct amd_cpudata *cpudata; -+ -+ cpudata = all_cpu_data[0]; -+ rdmsrl(MSR_AMD_CPPC_HW_CTL, misc_en); -+ global.cppc_boost_disabled = misc_en & AMD_CPPC_PRECISION_BOOST_ENABLED; -+} -+ -+static int amd_pstate_init_cpu(unsigned int cpunum) -+{ -+ struct amd_cpudata *cpudata; -+ -+ cpudata = all_cpu_data[cpunum]; -+ if (!cpudata) { -+ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); -+ if (!cpudata) -+ return -ENOMEM; -+ WRITE_ONCE(all_cpu_data[cpunum], cpudata); -+ -+ cpudata->cpu = cpunum; -+ } -+ cpudata->epp_powersave = -EINVAL; -+ cpudata->epp_policy = 0; -+ pr_debug("controlling: cpu %d\n", cpunum); -+ return 0; -+} -+ -+static int __amd_pstate_cpu_init(struct cpufreq_policy *policy) -+{ -+ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; -+ struct amd_cpudata *cpudata; -+ struct device *dev; -+ int rc; -+ u64 value; -+ -+ rc = amd_pstate_init_cpu(policy->cpu); -+ if (rc) -+ return rc; -+ -+ cpudata = all_cpu_data[policy->cpu]; -+ -+ dev = get_cpu_device(policy->cpu); -+ if (!dev) -+ goto free_cpudata1; -+ -+ rc = amd_pstate_init_perf(cpudata); -+ if (rc) -+ goto free_cpudata1; -+ -+ min_freq = amd_get_min_freq(cpudata); -+ max_freq = amd_get_max_freq(cpudata); -+ nominal_freq = amd_get_nominal_freq(cpudata); -+ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); -+ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { -+ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", -+ min_freq, max_freq); -+ ret = -EINVAL; -+ goto free_cpudata1; -+ } -+ -+ policy->min = min_freq; -+ policy->max = max_freq; -+ -+ policy->cpuinfo.min_freq = min_freq; -+ policy->cpuinfo.max_freq = max_freq; -+ /* It will be updated by governor */ -+ policy->cur = policy->cpuinfo.min_freq; -+ -+ /* Initial processor data capability frequencies */ -+ cpudata->max_freq = max_freq; -+ cpudata->min_freq = min_freq; -+ cpudata->nominal_freq = nominal_freq; -+ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; -+ -+ policy->driver_data = cpudata; -+ -+ update_boost_state(); -+ cpudata->epp_cached = amd_pstate_get_epp(cpudata, value); -+ -+ policy->min = policy->cpuinfo.min_freq; -+ policy->max = policy->cpuinfo.max_freq; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) -+ policy->fast_switch_possible = true; -+ -+ if (!shared_mem && boot_cpu_has(X86_FEATURE_CPPC)) { -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_cap1_cached, value); -+ } -+ amd_pstate_boost_init(cpudata); -+ -+ return 0; -+ -+free_cpudata1: -+ kfree(cpudata); -+ return ret; -+} -+ -+static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) -+{ -+ int ret; -+ -+ ret = __amd_pstate_cpu_init(policy); -+ if (ret) -+ return ret; -+ /* -+ * Set the policy to powersave to provide a valid fallback value in case -+ * the default cpufreq governor is neither powersave nor performance. -+ */ -+ policy->policy = CPUFREQ_POLICY_POWERSAVE; -+ -+ return 0; -+} -+ -+static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) -+{ -+ pr_debug("amd-pstate: CPU %d exiting\n", policy->cpu); -+ policy->fast_switch_possible = false; -+ return 0; -+} -+ -+static void amd_pstate_update_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); -+ -+ if (!policy) -+ return; -+ -+ refresh_frequency_limits(policy); -+ cpufreq_cpu_release(policy); -+} -+ -+static void amd_pstate_epp_update_limits(unsigned int cpu) -+{ -+ mutex_lock(&amd_pstate_driver_lock); -+ update_boost_state(); -+ if (global.cppc_boost_disabled) { -+ for_each_possible_cpu(cpu) -+ amd_pstate_update_max_freq(cpu); -+ } else { -+ cpufreq_update_policy(cpu); -+ } -+ mutex_unlock(&amd_pstate_driver_lock); -+} -+ -+static int cppc_boost_hold_time_ns = 3 * NSEC_PER_MSEC; -+ -+static inline void amd_pstate_boost_up(struct amd_cpudata *cpudata) -+{ -+ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); -+ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); -+ u32 max_limit = (hwp_req & 0xff); -+ u32 min_limit = (hwp_req & 0xff00) >> 8; -+ u32 boost_level1; -+ -+ /* If max and min are equal or already at max, nothing to boost */ -+ if (max_limit == min_limit) -+ return; -+ -+ /* Set boost max and min to initial value */ -+ if (!cpudata->cppc_boost_min) -+ cpudata->cppc_boost_min = min_limit; -+ -+ boost_level1 = ((AMD_CPPC_NOMINAL_PERF(hwp_cap) + min_limit) >> 1); -+ -+ if (cpudata->cppc_boost_min < boost_level1) -+ cpudata->cppc_boost_min = boost_level1; -+ else if (cpudata->cppc_boost_min < AMD_CPPC_NOMINAL_PERF(hwp_cap)) -+ cpudata->cppc_boost_min = AMD_CPPC_NOMINAL_PERF(hwp_cap); -+ else if (cpudata->cppc_boost_min == AMD_CPPC_NOMINAL_PERF(hwp_cap)) -+ cpudata->cppc_boost_min = max_limit; -+ else -+ return; -+ -+ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); -+ hwp_req |= AMD_CPPC_MIN_PERF(cpudata->cppc_boost_min); -+ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); -+ cpudata->last_update = cpudata->sample.time; -+} -+ -+static inline void amd_pstate_boost_down(struct amd_cpudata *cpudata) -+{ -+ bool expired; -+ -+ if (cpudata->cppc_boost_min) { -+ expired = time_after64(cpudata->sample.time, cpudata->last_update + -+ cppc_boost_hold_time_ns); -+ -+ if (expired) { -+ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, cpudata->cppc_req_cached); -+ cpudata->cppc_boost_min = 0; -+ } -+ } -+ -+ cpudata->last_update = cpudata->sample.time; -+} -+ -+static inline void amd_pstate_boost_update_util(struct amd_cpudata *cpudata, -+ u64 time) -+{ -+ cpudata->sample.time = time; -+ if (smp_processor_id() != cpudata->cpu) -+ return; -+ -+ if (cpudata->sched_flags & SCHED_CPUFREQ_IOWAIT) { -+ bool do_io = false; -+ -+ cpudata->sched_flags = 0; -+ /* -+ * Set iowait_boost flag and update time. Since IO WAIT flag -+ * is set all the time, we can't just conclude that there is -+ * some IO bound activity is scheduled on this CPU with just -+ * one occurrence. If we receive at least two in two -+ * consecutive ticks, then we treat as boost candidate. -+ * This is leveraged from Intel Pstate driver. -+ */ -+ if (time_before64(time, cpudata->last_io_update + 2 * TICK_NSEC)) -+ do_io = true; -+ -+ cpudata->last_io_update = time; -+ -+ if (do_io) -+ amd_pstate_boost_up(cpudata); -+ -+ } else { -+ amd_pstate_boost_down(cpudata); -+ } -+} -+ -+static inline void amd_pstate_cppc_update_hook(struct update_util_data *data, -+ u64 time, unsigned int flags) -+{ -+ struct amd_cpudata *cpudata = container_of(data, -+ struct amd_cpudata, update_util); -+ -+ cpudata->sched_flags |= flags; -+ -+ if (smp_processor_id() == cpudata->cpu) -+ amd_pstate_boost_update_util(cpudata, time); -+} -+ -+static void amd_pstate_clear_update_util_hook(unsigned int cpu) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu]; -+ -+ if (!cpudata->update_util_set) -+ return; -+ -+ cpufreq_remove_update_util_hook(cpu); -+ cpudata->update_util_set = false; -+ synchronize_rcu(); -+} -+ -+static void amd_pstate_set_update_util_hook(unsigned int cpu_num) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu_num]; -+ -+ if (!cppc_boost) { -+ if (cpudata->update_util_set) -+ amd_pstate_clear_update_util_hook(cpudata->cpu); -+ return; -+ } -+ -+ if (cpudata->update_util_set) -+ return; -+ -+ cpudata->sample.time = 0; -+ cpufreq_add_update_util_hook(cpu_num, &cpudata->update_util, -+ amd_pstate_cppc_update_hook); -+ cpudata->update_util_set = true; -+} -+ -+static void amd_pstate_epp_init(unsigned int cpu) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu]; -+ u32 max_perf, min_perf; -+ u64 value; -+ s16 epp; -+ int ret; -+ -+ max_perf = READ_ONCE(cpudata->highest_perf); -+ min_perf = READ_ONCE(cpudata->lowest_perf); -+ -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) -+ min_perf = max_perf; -+ -+ /* Initial min/max values for CPPC Performance Controls Register */ -+ value &= ~AMD_CPPC_MIN_PERF(~0L); -+ value |= AMD_CPPC_MIN_PERF(min_perf); -+ -+ value &= ~AMD_CPPC_MAX_PERF(~0L); -+ value |= AMD_CPPC_MAX_PERF(max_perf); -+ -+ /* CPPC EPP feature require to set zero to the desire perf bit */ -+ value &= ~AMD_CPPC_DES_PERF(~0L); -+ value |= AMD_CPPC_DES_PERF(0); -+ -+ if (cpudata->epp_policy == cpudata->policy) -+ goto skip_epp; -+ -+ cpudata->epp_policy = cpudata->policy; -+ -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ epp = amd_pstate_get_epp(cpudata, value); -+ cpudata->epp_powersave = epp; -+ if (epp < 0) -+ goto skip_epp; -+ -+ epp = 0; -+ } else { -+ if (cpudata->epp_powersave < 0) -+ goto skip_epp; -+ /* Get BIOS pre-defined epp value */ -+ epp = amd_pstate_get_epp(cpudata, value); -+ if (epp) -+ goto skip_epp; -+ epp = cpudata->epp_powersave; -+ } -+ /* Set initial EPP value */ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ value &= ~GENMASK_ULL(31, 24); -+ value |= (u64)epp << 24; -+ } -+ -+skip_epp: -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (!ret) { -+ cpudata->epp_cached = epp; -+ } -+} -+ -+static void amd_pstate_set_max_limits(struct amd_cpudata *cpudata) -+{ -+ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); -+ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); -+ u32 max_limit = (hwp_cap >> 24) & 0xff; -+ -+ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); -+ hwp_req |= AMD_CPPC_MIN_PERF(max_limit); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); -+} -+ -+static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata; -+ -+ if (!policy->cpuinfo.max_freq) -+ return -ENODEV; -+ -+ pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", -+ policy->cpuinfo.max_freq, policy->max); -+ -+ cpudata = all_cpu_data[policy->cpu]; -+ cpudata->policy = policy->policy; -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ mutex_lock(&amd_pstate_limits_lock); -+ -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ amd_pstate_clear_update_util_hook(policy->cpu); -+ amd_pstate_set_max_limits(cpudata); -+ } else { -+ amd_pstate_set_update_util_hook(policy->cpu); -+ } -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) -+ amd_pstate_epp_init(policy->cpu); -+ -+ mutex_unlock(&amd_pstate_limits_lock); -+ } -+ -+ return 0; -+} -+ -+static void amd_pstate_epp_reenable(struct amd_cpudata * cpudata) -+{ -+ struct cppc_perf_ctrls perf_ctrls; -+ u64 value, max_perf; -+ int ret; -+ -+ ret = amd_pstate_enable(true); -+ if (ret) -+ pr_err("failed to enable amd pstate during resume, return %d\n", ret); -+ -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ max_perf = READ_ONCE(cpudata->highest_perf); -+ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ } else { -+ perf_ctrls.max_perf = max_perf; -+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); -+ cppc_set_perf(cpudata->cpu, &perf_ctrls); -+ } -+} -+ -+static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ -+ pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); -+ -+ if (epp_enabled) { -+ amd_pstate_epp_reenable(cpudata); -+ cpudata->suspended = false; -+ } -+ -+ return 0; -+} -+ -+static void amd_pstate_epp_offline(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ struct cppc_perf_ctrls perf_ctrls; -+ int min_perf; -+ u64 value; -+ -+ min_perf = READ_ONCE(cpudata->lowest_perf); -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ -+ mutex_lock(&amd_pstate_limits_lock); -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; -+ -+ /* Set max perf same as min perf */ -+ value &= ~AMD_CPPC_MAX_PERF(~0L); -+ value |= AMD_CPPC_MAX_PERF(min_perf); -+ value &= ~AMD_CPPC_MIN_PERF(~0L); -+ value |= AMD_CPPC_MIN_PERF(min_perf); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ } else { -+ perf_ctrls.desired_perf = 0; -+ perf_ctrls.max_perf = min_perf; -+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(AMD_CPPC_EPP_POWERSAVE); -+ cppc_set_perf(cpudata->cpu, &perf_ctrls); -+ } -+ mutex_unlock(&amd_pstate_limits_lock); -+} -+ -+static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ -+ pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); -+ -+ if (cpudata->suspended) -+ return 0; -+ -+ if (epp_enabled) -+ amd_pstate_epp_offline(policy); -+ -+ return 0; -+} -+ -+static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) -+{ -+ amd_pstate_clear_update_util_hook(policy->cpu); -+ -+ return amd_pstate_cpu_offline(policy); -+} -+ -+static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ int ret; -+ -+ pr_debug("AMD CPU Core %d suspending\n", cpudata->cpu); -+ -+ cpudata->suspended = true; -+ -+ /* disable CPPC in lowlevel firmware */ -+ ret = amd_pstate_enable(false); -+ if (ret) -+ pr_err("failed to disable amd pstate during suspend, return %d\n", ret); -+ -+ return 0; -+} -+ -+static int amd_pstate_epp_resume(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ -+ pr_debug("AMD CPU Core %d resuming\n", cpudata->cpu); -+ -+ if (cpudata->suspended && epp_enabled) { -+ mutex_lock(&amd_pstate_limits_lock); -+ -+ /* enable amd pstate from suspend state*/ -+ amd_pstate_epp_reenable(cpudata); -+ -+ mutex_unlock(&amd_pstate_limits_lock); -+ } -+ -+ cpudata->suspended = false; -+ -+ return 0; -+} -+ -+static void amd_pstate_verify_cpu_policy(struct amd_cpudata *cpudata, -+ struct cpufreq_policy_data *policy) -+{ -+ update_boost_state(); -+ cpufreq_verify_within_cpu_limits(policy); -+} -+ -+static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) -+{ -+ amd_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); -+ pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min ); -+ return 0; -+} -+ - static struct cpufreq_driver amd_pstate_driver = { - .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, - .verify = amd_pstate_verify, -@@ -662,18 +1535,34 @@ static struct cpufreq_driver amd_pstate_driver = { - .resume = amd_pstate_cpu_resume, - .set_boost = amd_pstate_set_boost, - .name = "amd-pstate", -- .attr = amd_pstate_attr, -+ .attr = amd_pstate_attr, -+}; -+ -+static struct cpufreq_driver amd_pstate_epp_driver = { -+ .flags = CPUFREQ_CONST_LOOPS, -+ .verify = amd_pstate_epp_verify_policy, -+ .setpolicy = amd_pstate_epp_set_policy, -+ .init = amd_pstate_epp_cpu_init, -+ .exit = amd_pstate_epp_cpu_exit, -+ .update_limits = amd_pstate_epp_update_limits, -+ .offline = amd_pstate_epp_cpu_offline, -+ .online = amd_pstate_epp_cpu_online, -+ .suspend = amd_pstate_epp_suspend, -+ .resume = amd_pstate_epp_resume, -+ .name = "amd_pstate_epp", -+ .attr = amd_pstate_epp_attr, - }; - - static int __init amd_pstate_init(void) - { -+ static struct amd_cpudata **cpudata; - int ret; - - if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) - return -ENODEV; - - if (!acpi_cpc_valid()) { -- pr_debug("the _CPC object is not present in SBIOS\n"); -+ pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); - return -ENODEV; - } - -@@ -681,10 +1570,25 @@ static int __init amd_pstate_init(void) - if (cpufreq_get_current_driver()) - return -EEXIST; - -+ cpudata = vzalloc(array_size(sizeof(void *), num_possible_cpus())); -+ if (!cpudata) -+ return -ENOMEM; -+ WRITE_ONCE(all_cpu_data, cpudata); -+ -+ if (epp_enabled) { -+ pr_info("AMD CPPC loading with amd_pstate_epp driver instance.\n"); -+ default_pstate_driver = &amd_pstate_epp_driver; -+ } else { -+ pr_info("AMD CPPC loading with amd_pstate driver instance.\n"); -+ default_pstate_driver = &amd_pstate_driver; -+ } -+ - /* capability check */ - if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!epp_enabled) { -+ default_pstate_driver->adjust_perf = amd_pstate_adjust_perf; -+ } - pr_debug("AMD CPPC MSR based functionality is supported\n"); -- amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; - } else if (shared_mem) { - static_call_update(amd_pstate_enable, cppc_enable); - static_call_update(amd_pstate_init_perf, cppc_init_perf); -@@ -701,19 +1605,56 @@ static int __init amd_pstate_init(void) - return ret; - } - -- ret = cpufreq_register_driver(&amd_pstate_driver); -+ ret = cpufreq_register_driver(default_pstate_driver); - if (ret) -- pr_err("failed to register amd_pstate_driver with return %d\n", -+ pr_err("failed to register amd pstate driver with return %d\n", -+ ret); -+ -+ amd_pstate_kobj = kobject_create_and_add("amd-pstate", &cpu_subsys.dev_root->kobj); -+ if (!amd_pstate_kobj) { -+ pr_err("amd-pstate: Global sysfs registration failed.\n"); -+ } -+ -+ ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); -+ if (ret) { -+ pr_err("amd-pstate: Sysfs attribute export failed with error %d.\n", - ret); -+ } - - return ret; - } - -+static inline void amd_pstate_kobj_cleanup(struct kobject *kobj) -+{ -+ kobject_del(kobj); -+ kobject_put(kobj); -+} -+ - static void __exit amd_pstate_exit(void) - { -- cpufreq_unregister_driver(&amd_pstate_driver); -+ unsigned int cpu; -+ -+ cpufreq_unregister_driver(default_pstate_driver); - - amd_pstate_enable(false); -+ -+ sysfs_remove_group(amd_pstate_kobj, &amd_pstate_global_attr_group); -+ amd_pstate_kobj_cleanup(amd_pstate_kobj); -+ -+ cpus_read_lock(); -+ for_each_online_cpu(cpu) { -+ if (all_cpu_data[cpu]) { -+ if (default_pstate_driver == &amd_pstate_epp_driver) -+ amd_pstate_clear_update_util_hook(cpu); -+ -+ spin_lock(&amd_pstate_cpu_lock); -+ kfree(all_cpu_data[cpu]); -+ WRITE_ONCE(all_cpu_data[cpu], NULL); -+ spin_unlock(&amd_pstate_cpu_lock); -+ } -+ } -+ cpus_read_unlock(); -+ - } - - module_init(amd_pstate_init); -diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c -index 24eaf0ec344d..9adb7612993e 100644 ---- a/drivers/cpufreq/cppc_cpufreq.c -+++ b/drivers/cpufreq/cppc_cpufreq.c -@@ -947,7 +947,7 @@ static int __init cppc_cpufreq_init(void) - { - int ret; - -- if ((acpi_disabled) || !acpi_cpc_valid()) -+ if (!acpi_cpc_valid()) - return -ENODEV; - - cppc_check_hisi_workaround(); -diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c -index 57cdb3679885..50881662b45b 100644 ---- a/drivers/cpufreq/intel_pstate.c -+++ b/drivers/cpufreq/intel_pstate.c -@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu) - * update them at any time after it has been called. - */ - sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); -+ /* -+ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff. -+ * In this case we can't use CPPC.highest_perf to enable ITMT. -+ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide. -+ */ -+ if (cppc_perf.highest_perf == 0xff) -+ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); - - if (max_highest_perf <= min_highest_perf) { - if (cppc_perf.highest_perf > max_highest_perf) -diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c -index 3e101719689a..746138abc3fe 100644 ---- a/drivers/idle/intel_idle.c -+++ b/drivers/idle/intel_idle.c -@@ -578,7 +578,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -586,7 +586,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 33, -- .target_residency = 100, -+ .target_residency = 900, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -594,7 +594,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 133, -- .target_residency = 400, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -602,7 +602,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 166, -- .target_residency = 500, -+ .target_residency = 1500, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -610,7 +610,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 300, -- .target_residency = 900, -+ .target_residency = 2000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -618,7 +618,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 600, -- .target_residency = 1800, -+ .target_residency = 5000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -626,7 +626,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 2600, -- .target_residency = 7700, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -646,7 +646,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -654,7 +654,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 40, -- .target_residency = 100, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -662,7 +662,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 133, -- .target_residency = 400, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -670,7 +670,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 166, -- .target_residency = 500, -+ .target_residency = 2000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -678,7 +678,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 300, -- .target_residency = 900, -+ .target_residency = 4000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -686,7 +686,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 600, -- .target_residency = 1800, -+ .target_residency = 7000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -694,7 +694,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 2600, -- .target_residency = 7700, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -715,7 +715,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -723,7 +723,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 70, -- .target_residency = 100, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -731,7 +731,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 85, -- .target_residency = 200, -+ .target_residency = 600, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -739,7 +739,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x33", - .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 124, -- .target_residency = 800, -+ .target_residency = 3000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -747,7 +747,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 200, -- .target_residency = 800, -+ .target_residency = 3200, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -755,7 +755,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 480, -- .target_residency = 5000, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -763,7 +763,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 890, -- .target_residency = 5000, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -784,7 +784,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 300, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c -index 3fc0a89cc785..a7c103f9dfd3 100644 ---- a/drivers/input/serio/i8042.c -+++ b/drivers/input/serio/i8042.c -@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void) - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - i8042_ctr &= ~I8042_CTR_KBDINT; - i8042_ctr |= I8042_CTR_KBDDIS; -- pr_err("Failed to enable KBD port\n"); -+ pr_info("Failed to enable KBD port\n"); - return -EIO; - } - -@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void) - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - i8042_ctr &= ~I8042_CTR_AUXINT; - i8042_ctr |= I8042_CTR_AUXDIS; -- pr_err("Failed to enable AUX port\n"); -+ pr_info("Failed to enable AUX port\n"); - return -EIO; - } - -@@ -732,7 +732,7 @@ static int i8042_check_mux(void) - i8042_ctr &= ~I8042_CTR_AUXINT; - - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { -- pr_err("Failed to disable AUX port, can't use MUX\n"); -+ pr_info("Failed to disable AUX port, can't use MUX\n"); - return -EIO; - } - -@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void) - do { - - if (i8042_command(¶m, I8042_CMD_CTL_TEST)) { -- pr_err("i8042 controller selftest timeout\n"); -+ pr_info("i8042 controller selftest timeout\n"); - return -ENODEV; - } - -@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void) - pr_info("giving up on controller selftest, continuing anyway...\n"); - return 0; - #else -- pr_err("i8042 controller selftest failed\n"); -+ pr_info("i8042 controller selftest failed\n"); - return -EIO; - #endif - } -diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c -index 159c6806c19b..a201f1c2fce5 100644 ---- a/drivers/md/dm-crypt.c -+++ b/drivers/md/dm-crypt.c -@@ -3137,6 +3137,11 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar - } - } - -+#ifdef CONFIG_CACHY -+ set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); -+ set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); -+#endif -+ - return 0; - } - -diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c -index f82ad7419508..5e8faa70aad6 100644 ---- a/drivers/net/dummy.c -+++ b/drivers/net/dummy.c -@@ -43,7 +43,7 @@ - - #define DRV_NAME "dummy" - --static int numdummies = 1; -+static int numdummies = 0; - - /* fake multicast ability */ - static void set_multicast_list(struct net_device *dev) -diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c -index 66446f1e06cf..c65b03f91ecf 100644 ---- a/drivers/nvme/host/core.c -+++ b/drivers/nvme/host/core.c -@@ -58,7 +58,7 @@ static u8 nvme_max_retries = 5; - module_param_named(max_retries, nvme_max_retries, byte, 0644); - MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); - --static unsigned long default_ps_max_latency_us = 100000; -+static unsigned long default_ps_max_latency_us = 200; - module_param(default_ps_max_latency_us, ulong, 0644); - MODULE_PARM_DESC(default_ps_max_latency_us, - "max power saving latency for new devices; use PM QOS to change per device"); -diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c -index 95bc329e74c0..73114509ddff 100644 ---- a/drivers/pci/pci.c -+++ b/drivers/pci/pci.c -@@ -62,7 +62,7 @@ struct pci_pme_device { - struct pci_dev *dev; - }; - --#define PME_TIMEOUT 1000 /* How long between PME checks */ -+#define PME_TIMEOUT 4000 /* How long between PME checks */ - - static void pci_dev_d3_sleep(struct pci_dev *dev) - { -diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c -index 21d624f9f5fb..aad4e9578d89 100644 ---- a/drivers/powercap/intel_rapl_common.c -+++ b/drivers/powercap/intel_rapl_common.c -@@ -1515,7 +1515,7 @@ static int __init rapl_init(void) - - id = x86_match_cpu(rapl_ids); - if (!id) { -- pr_err("driver does not support CPU family %d model %d\n", -+ pr_info("driver does not support CPU family %d model %d\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); - - return -ENODEV; -diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c -index c841ab37e7c6..fe3d190ab12b 100644 ---- a/drivers/thermal/intel/intel_powerclamp.c -+++ b/drivers/thermal/intel/intel_powerclamp.c -@@ -644,6 +644,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = { - .set_cur_state = powerclamp_set_cur_state, - }; - -+static const struct x86_cpu_id amd_cpu[] = { -+ { X86_VENDOR_AMD }, -+ {}, -+}; -+ - static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = { - X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL), - {} -@@ -653,6 +658,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); - static int __init powerclamp_probe(void) - { - -+ if (x86_match_cpu(amd_cpu)){ -+ pr_info("Intel PowerClamp does not support AMD CPUs\n"); -+ return -ENODEV; -+ } -+ - if (!x86_match_cpu(intel_powerclamp_ids)) { - pr_err("CPU does not support MWAIT\n"); - return -ENODEV; -diff --git a/fs/xattr.c b/fs/xattr.c -index a1f4998bc6be..020e0b5ea5c6 100644 ---- a/fs/xattr.c -+++ b/fs/xattr.c -@@ -122,16 +122,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, - } - - /* -- * In the user.* namespace, only regular files and directories can have -- * extended attributes. For sticky directories, only the owner and -- * privileged users can write attributes. -+ * In the user.* namespace, only regular files, symbolic links, and -+ * directories can have extended attributes. For symbolic links and -+ * sticky directories, only the owner and privileged users can write -+ * attributes. - */ - if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { -- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) -+ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode)) - return (mask & MAY_WRITE) ? -EPERM : -ENODATA; -- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && -- (mask & MAY_WRITE) && -- !inode_owner_or_capable(mnt_userns, inode)) -+ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX)) -+ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE) -+ && !inode_owner_or_capable(mnt_userns, inode)) - return -EPERM; - } - -diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h -index f73d357ecdf5..41d151a6d52e 100644 ---- a/include/acpi/cppc_acpi.h -+++ b/include/acpi/cppc_acpi.h -@@ -108,12 +108,14 @@ struct cppc_perf_caps { - u32 lowest_nonlinear_perf; - u32 lowest_freq; - u32 nominal_freq; -+ u32 energy_perf; - }; - - struct cppc_perf_ctrls { - u32 max_perf; - u32 min_perf; - u32 desired_perf; -+ u32 energy_perf; - }; - - struct cppc_perf_fb_ctrs { -@@ -148,6 +150,9 @@ extern bool cpc_ffh_supported(void); - extern bool cpc_supported_by_cpu(void); - extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); - extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); -+extern int cppc_set_auto_epp(int cpu, bool enable); -+extern int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps); -+extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); - #else /* !CONFIG_ACPI_CPPC_LIB */ - static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) - { -@@ -197,6 +202,18 @@ static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) - { - return -ENOTSUPP; - } -+static inline int cppc_set_auto_epp(int cpu, bool enable) -+{ -+ return -ENOTSUPP; -+} -+static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) -+{ -+ return -ENOTSUPP; -+} -+static inline int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) -+{ -+ return -ENOTSUPP; -+} - #endif /* !CONFIG_ACPI_CPPC_LIB */ - - #endif /* _CPPC_ACPI_H*/ -diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h -index e3e8c8662b49..9209856b6644 100644 ---- a/include/linux/ipc_namespace.h -+++ b/include/linux/ipc_namespace.h -@@ -36,8 +36,6 @@ struct ipc_namespace { - unsigned int msg_ctlmax; - unsigned int msg_ctlmnb; - unsigned int msg_ctlmni; -- atomic_t msg_bytes; -- atomic_t msg_hdrs; - - size_t shm_ctlmax; - size_t shm_ctlall; -@@ -77,6 +75,9 @@ struct ipc_namespace { - struct llist_node mnt_llist; - - struct ns_common ns; -+ int padding[16]; -+ atomic_t msg_bytes; -+ atomic_t msg_hdrs; - } __randomize_layout; - - extern struct ipc_namespace init_ipc_ns; -diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h -index 0b7242370b56..16b8fc483b3d 100644 ---- a/include/linux/jbd2.h -+++ b/include/linux/jbd2.h -@@ -45,7 +45,7 @@ - /* - * The default maximum commit age, in seconds. - */ --#define JBD2_DEFAULT_MAX_COMMIT_AGE 5 -+#define JBD2_DEFAULT_MAX_COMMIT_AGE 30 - - #ifdef CONFIG_JBD2_DEBUG - /* -diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h -index 679591301994..685193470d64 100644 ---- a/include/linux/page_counter.h -+++ b/include/linux/page_counter.h -@@ -8,6 +8,7 @@ - - struct page_counter { - atomic_long_t usage; -+ unsigned long padding[7]; - unsigned long min; - unsigned long low; - unsigned long high; -diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index 0178b2040ea3..39f05e00dac4 100644 ---- a/include/linux/pagemap.h -+++ b/include/linux/pagemap.h -@@ -1183,7 +1183,7 @@ struct readahead_control { - ._index = i, \ - } - --#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) -+#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) - - void page_cache_ra_unbounded(struct readahead_control *, - unsigned long nr_to_read, unsigned long lookahead_count); -diff --git a/include/linux/sched.h b/include/linux/sched.h -index e7b2f8a5c711..367b30ed77cb 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -1052,6 +1052,7 @@ struct task_struct { - /* Cached requested key. */ - struct key *cached_requested_key; - #endif -+ int fsync_count; - - /* - * executable name, excluding path. -diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h -index 33a4240e6a6f..82213f9c4c17 100644 ---- a/include/linux/user_namespace.h -+++ b/include/linux/user_namespace.h -@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns, - - #ifdef CONFIG_USER_NS - -+extern int unprivileged_userns_clone; -+ - static inline struct user_namespace *get_user_ns(struct user_namespace *ns) - { - if (ns) -@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); - struct ns_common *ns_get_owner(struct ns_common *ns); - #else - -+#define unprivileged_userns_clone 0 -+ - static inline struct user_namespace *get_user_ns(struct user_namespace *ns) - { - return &init_user_ns; -diff --git a/include/linux/wait.h b/include/linux/wait.h -index 58cfbf81447c..d5664b5ae3e1 100644 ---- a/include/linux/wait.h -+++ b/include/linux/wait.h -@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) - - extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); -+extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - -@@ -1164,6 +1165,7 @@ do { \ - */ - void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); -+void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); -diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h -index d174914a837d..bf8e2af101a3 100644 ---- a/include/uapi/linux/if_bonding.h -+++ b/include/uapi/linux/if_bonding.h -@@ -82,7 +82,7 @@ - #define BOND_STATE_ACTIVE 0 /* link is active */ - #define BOND_STATE_BACKUP 1 /* link is backup */ - --#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */ -+#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */ - - #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */ - -diff --git a/init/Kconfig b/init/Kconfig -index 532362fcfe31..442a945ca6ae 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -112,6 +112,10 @@ config THREAD_INFO_IN_TASK - - menu "General setup" - -+config CACHY -+ bool "Some kernel tweaks by CachyOS" -+ default y -+ - config BROKEN - bool - -@@ -1241,6 +1245,22 @@ config USER_NS - - If unsure, say N. - -+config USER_NS_UNPRIVILEGED -+ bool "Allow unprivileged users to create namespaces" -+ default y -+ depends on USER_NS -+ help -+ When disabled, unprivileged users will not be able to create -+ new namespaces. Allowing users to create their own namespaces -+ has been part of several recent local privilege escalation -+ exploits, so if you need user namespaces but are -+ paranoid^Wsecurity-conscious you want to disable this. -+ -+ This setting can be overridden at runtime via the -+ kernel.unprivileged_userns_clone sysctl. -+ -+ If unsure, say Y. -+ - config PID_NS - bool "PID Namespaces" - default y -@@ -1407,6 +1427,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE - with the "-O2" compiler flag for best performance and most - helpful compile-time warnings. - -+config CC_OPTIMIZE_FOR_PERFORMANCE_O3 -+ bool "Optimize more for performance (-O3)" -+ help -+ Choosing this option will pass "-O3" to your compiler to optimize -+ the kernel yet more for performance. -+ - config CC_OPTIMIZE_FOR_SIZE - bool "Optimize for size (-Os)" - help -diff --git a/init/do_mounts.c b/init/do_mounts.c -index 7058e14ad5f7..ce4e841ea705 100644 ---- a/init/do_mounts.c -+++ b/init/do_mounts.c -@@ -283,8 +283,18 @@ dev_t name_to_dev_t(const char *name) - if (strcmp(name, "/dev/ram") == 0) - return Root_RAM0; - #ifdef CONFIG_BLOCK -- if (strncmp(name, "PARTUUID=", 9) == 0) -- return devt_from_partuuid(name + 9); -+ if (strncmp(name, "PARTUUID=", 9) == 0) { -+ dev_t res; -+ int needtowait = 40<<1; -+ res = devt_from_partuuid(name + 9); -+ while (!res && needtowait) { -+ /* waiting 0.5 sec */ -+ msleep(500); -+ res = devt_from_partuuid(name + 9); -+ needtowait--; -+ } -+ return res; -+ } - if (strncmp(name, "PARTLABEL=", 10) == 0) - return devt_from_partlabel(name + 10); - if (strncmp(name, "/dev/", 5) == 0) -@@ -612,7 +622,9 @@ void __init prepare_namespace(void) - * For example, it is not atypical to wait 5 seconds here - * for the touchpad of a laptop to initialize. - */ -+ async_synchronize_full(); - wait_for_device_probe(); -+ async_synchronize_full(); - - md_run_setup(); - -diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz -index 38ef6d06888e..0f78364efd4f 100644 ---- a/kernel/Kconfig.hz -+++ b/kernel/Kconfig.hz -@@ -40,6 +40,27 @@ choice - on SMP and NUMA systems and exactly dividing by both PAL and - NTSC frame rates for video and multimedia work. - -+ config HZ_500 -+ bool "500 HZ" -+ help -+ 500 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. -+ -+ config HZ_600 -+ bool "600 HZ" -+ help -+ 600 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. -+ -+ config HZ_750 -+ bool "750 HZ" -+ help -+ 750 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. -+ - config HZ_1000 - bool "1000 HZ" - help -@@ -53,6 +74,9 @@ config HZ - default 100 if HZ_100 - default 250 if HZ_250 - default 300 if HZ_300 -+ default 500 if HZ_500 -+ default 600 if HZ_600 -+ default 750 if HZ_750 - default 1000 if HZ_1000 - - config SCHED_HRTICK -diff --git a/kernel/fork.c b/kernel/fork.c -index 2b6bd511c6ed..704fe6bc9cb4 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -99,6 +99,10 @@ - #include - #include - -+#ifdef CONFIG_USER_NS -+#include -+#endif -+ - #include - #include - #include -@@ -2009,6 +2013,10 @@ static __latent_entropy struct task_struct *copy_process( - if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) - return ERR_PTR(-EINVAL); - -+ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) -+ if (!capable(CAP_SYS_ADMIN)) -+ return ERR_PTR(-EPERM); -+ - /* - * Thread groups must share signals as well, and detached threads - * can only be started up within the thread group. -@@ -3159,6 +3167,12 @@ int ksys_unshare(unsigned long unshare_flags) - if (unshare_flags & CLONE_NEWNS) - unshare_flags |= CLONE_FS; - -+ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { -+ err = -EPERM; -+ if (!capable(CAP_SYS_ADMIN)) -+ goto bad_unshare_out; -+ } -+ - err = check_unshare_flags(unshare_flags); - if (err) - goto bad_unshare_out; -diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c -index 65f0262f635e..8a8f77930818 100644 ---- a/kernel/locking/rwsem.c -+++ b/kernel/locking/rwsem.c -@@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) - struct task_struct *new, *owner; - unsigned long flags, new_flags; - enum owner_state state; -+ int i = 0; - - lockdep_assert_preemption_disabled(); - -@@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) - break; - } - -- cpu_relax(); -+ if (i++ > 1000) -+ cpu_relax(); - } - - return state; -diff --git a/kernel/module/internal.h b/kernel/module/internal.h -index 680d980a4fb2..8a3abfff9fe9 100644 ---- a/kernel/module/internal.h -+++ b/kernel/module/internal.h -@@ -53,6 +53,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; - extern const s32 __start___kcrctab[]; - extern const s32 __start___kcrctab_gpl[]; - -+extern struct boot_params boot_params; -+ - struct load_info { - const char *name; - /* pointer to module in temporary copy, freed at end of load_module() */ -diff --git a/kernel/module/main.c b/kernel/module/main.c -index a4e4d84b6f4e..0e698f5bee86 100644 ---- a/kernel/module/main.c -+++ b/kernel/module/main.c -@@ -53,6 +53,7 @@ - #include - #include - #include -+#include - #include - #include "internal.h" - -diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c -index cf5b9f1e6ec4..80260fa4dac5 100644 ---- a/kernel/module/procfs.c -+++ b/kernel/module/procfs.c -@@ -141,6 +141,19 @@ static const struct proc_ops modules_proc_ops = { - static int __init proc_modules_init(void) - { - proc_create("modules", 0, NULL, &modules_proc_ops); -+ -+#ifdef CONFIG_MODULE_SIG_FORCE -+ switch (boot_params.secure_boot) { -+ case efi_secureboot_mode_unset: -+ case efi_secureboot_mode_unknown: -+ case efi_secureboot_mode_disabled: -+ /* -+ * sig_unenforce is only applied if SecureBoot is not -+ * enabled. -+ */ -+ sig_enforce = !sig_unenforce; -+ } -+#endif - return 0; - } - module_init(proc_modules_init); -diff --git a/kernel/module/signing.c b/kernel/module/signing.c -index a2ff4242e623..876e93758e91 100644 ---- a/kernel/module/signing.c -+++ b/kernel/module/signing.c -@@ -21,6 +21,10 @@ - - static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE); - module_param(sig_enforce, bool_enable_only, 0644); -+/* Allow disabling module signature requirement by adding boot param */ -+static bool sig_unenforce = false; -+module_param(sig_unenforce, bool_enable_only, 0644); -+ - - /* - * Export sig_enforce kernel cmdline parameter to allow other subsystems rely -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index ee28253c9ac0..f2534e712a89 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -138,6 +138,18 @@ __read_mostly int sysctl_resched_latency_warn_ms = 100; - __read_mostly int sysctl_resched_latency_warn_once = 1; - #endif /* CONFIG_SCHED_DEBUG */ - -+/* -+ * Choose the yield level that will perform. -+ * 0: No yield. -+ * 1: Yield only to better priority/deadline tasks. -+ * 2: Re-queue current tasks. (default CFS) -+ */ -+#ifdef CONFIG_CACHY -+__read_mostly int sysctl_sched_yield_type = 1; -+#else -+__read_mostly int sysctl_sched_yield_type = 0; -+#endif -+ - /* - * Number of tasks to iterate in a single balance run. - * Limited because this is done with IRQs disabled. -@@ -8266,10 +8278,15 @@ static void do_sched_yield(void) - struct rq_flags rf; - struct rq *rq; - -+ if (!sysctl_sched_yield_type) -+ return; -+ - rq = this_rq_lock_irq(&rf); - - schedstat_inc(rq->yld_count); -- current->sched_class->yield_task(rq); -+ -+ if (sysctl_sched_yield_type > 1) -+ current->sched_class->yield_task(rq); - - preempt_disable(); - rq_unlock_irq(rq, &rf); -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 667876da8382..13fbcd19b916 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -959,6 +959,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - PN(se.exec_start); - PN(se.vruntime); - PN(se.sum_exec_runtime); -+ P(fsync_count); - - nr_switches = p->nvcsw + p->nivcsw; - -diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 914096c5b1ae..74d773040b54 100644 ---- a/kernel/sched/fair.c -+++ b/kernel/sched/fair.c -@@ -68,9 +68,13 @@ - * - * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_latency = 3000000ULL; -+static unsigned int normalized_sysctl_sched_latency = 3000000ULL; -+#else - unsigned int sysctl_sched_latency = 6000000ULL; - static unsigned int normalized_sysctl_sched_latency = 6000000ULL; -- -+#endif - /* - * The initial- and re-scaling of tunables is configurable - * -@@ -89,8 +93,13 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; - * - * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_min_granularity = 400000ULL; -+static unsigned int normalized_sysctl_sched_min_granularity = 400000ULL; -+#else - unsigned int sysctl_sched_min_granularity = 750000ULL; - static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; -+#endif - - /* - * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. -@@ -120,8 +129,13 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; - * - * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_wakeup_granularity = 500000UL; -+static unsigned int normalized_sysctl_sched_wakeup_granularity = 500000UL; -+#else - unsigned int sysctl_sched_wakeup_granularity = 1000000UL; - static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; -+#endif - - const_debug unsigned int sysctl_sched_migration_cost = 500000UL; - -@@ -174,8 +188,12 @@ int __weak arch_asym_cpu_priority(int cpu) - * - * (default: 5 msec, units: microseconds) - */ -+#ifdef CONFIG_CACHY -+static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; -+#else - static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; - #endif -+#endif - - #ifdef CONFIG_SYSCTL - static struct ctl_table sched_fair_sysctls[] = { -diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c -index 9860bb9a847c..c7f045873a5f 100644 ---- a/kernel/sched/wait.c -+++ b/kernel/sched/wait.c -@@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ - } - EXPORT_SYMBOL_GPL(add_wait_queue_priority); - -+void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) -+{ -+ unsigned long flags; -+ -+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; -+ spin_lock_irqsave(&wq_head->lock, flags); -+ __add_wait_queue(wq_head, wq_entry); -+ spin_unlock_irqrestore(&wq_head->lock, flags); -+} -+EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); -+ - void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) - { - unsigned long flags; -@@ -289,6 +300,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent - } - EXPORT_SYMBOL(prepare_to_wait_exclusive); - -+void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) -+{ -+ unsigned long flags; -+ -+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; -+ spin_lock_irqsave(&wq_head->lock, flags); -+ if (list_empty(&wq_entry->entry)) -+ __add_wait_queue(wq_head, wq_entry); -+ set_current_state(state); -+ spin_unlock_irqrestore(&wq_head->lock, flags); -+} -+EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); -+ - void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) - { - wq_entry->flags = flags; -diff --git a/kernel/sysctl.c b/kernel/sysctl.c -index 205d605cacc5..4432b6f15890 100644 ---- a/kernel/sysctl.c -+++ b/kernel/sysctl.c -@@ -89,8 +89,12 @@ - #ifdef CONFIG_PERF_EVENTS - static const int six_hundred_forty_kb = 640 * 1024; - #endif -+#ifdef CONFIG_USER_NS -+#include -+#endif - - -+extern int sysctl_sched_yield_type; - static const int ngroups_max = NGROUPS_MAX; - static const int cap_last_cap = CAP_LAST_CAP; - -@@ -1649,6 +1653,24 @@ static struct ctl_table kern_table[] = { - .mode = 0644, - .proc_handler = proc_dointvec, - }, -+ { -+ .procname = "yield_type", -+ .data = &sysctl_sched_yield_type, -+ .maxlen = sizeof (int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_TWO, -+ }, -+#ifdef CONFIG_USER_NS -+ { -+ .procname = "unprivileged_userns_clone", -+ .data = &unprivileged_userns_clone, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ }, -+#endif - #ifdef CONFIG_PROC_SYSCTL - { - .procname = "tainted", -diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c -index 5481ba44a8d6..423ab2563ad7 100644 ---- a/kernel/user_namespace.c -+++ b/kernel/user_namespace.c -@@ -21,6 +21,13 @@ - #include - #include - -+/* sysctl */ -+#ifdef CONFIG_USER_NS_UNPRIVILEGED -+int unprivileged_userns_clone = 1; -+#else -+int unprivileged_userns_clone; -+#endif -+ - static struct kmem_cache *user_ns_cachep __read_mostly; - static DEFINE_MUTEX(userns_state_mutex); - -diff --git a/kernel/watchdog.c b/kernel/watchdog.c -index 8e61f21e7e33..be1439d38f26 100644 ---- a/kernel/watchdog.c -+++ b/kernel/watchdog.c -@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled; - int __read_mostly watchdog_user_enabled = 1; - int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT; - int __read_mostly soft_watchdog_user_enabled = 1; --int __read_mostly watchdog_thresh = 10; -+int __read_mostly watchdog_thresh = 40; - static int __read_mostly nmi_watchdog_available; - - struct cpumask watchdog_cpumask __read_mostly; -diff --git a/lib/Kconfig b/lib/Kconfig -index dc1ab2ed1dc6..3ea8941ab18d 100644 ---- a/lib/Kconfig -+++ b/lib/Kconfig -@@ -343,12 +343,16 @@ config LZ4HC_COMPRESS - config LZ4_DECOMPRESS - tristate - --config ZSTD_COMPRESS -+config ZSTD_COMMON - select XXHASH - tristate - -+config ZSTD_COMPRESS -+ select ZSTD_COMMON -+ tristate -+ - config ZSTD_DECOMPRESS -- select XXHASH -+ select ZSTD_COMMON - tristate - - source "lib/xz/Kconfig" -diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c -index 39b74221f4a7..ec3eab8cd6b1 100644 ---- a/lib/raid6/algos.c -+++ b/lib/raid6/algos.c -@@ -128,8 +128,10 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) -- if (!(*algo)->valid || (*algo)->valid()) -+ if (!(*algo)->valid || (*algo)->valid()) { - best = *algo; -+ break; -+ } - - if (best) { - raid6_2data_recov = best->data2; -diff --git a/lib/string.c b/lib/string.c -index 6f334420f687..2e60e9e40535 100644 ---- a/lib/string.c -+++ b/lib/string.c -@@ -866,24 +866,61 @@ char *strnstr(const char *s1, const char *s2, size_t len) - EXPORT_SYMBOL(strnstr); - #endif - -+#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 -+ -+#define MEMCHR_MASK_GEN(mask) (mask *= 0x0101010101010101ULL) -+ -+#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) -+ -+#define MEMCHR_MASK_GEN(mask) \ -+ do { \ -+ mask *= 0x01010101; \ -+ mask |= mask << 32; \ -+ } while (0) -+ -+#else -+ -+#define MEMCHR_MASK_GEN(mask) \ -+ do { \ -+ mask |= mask << 8; \ -+ mask |= mask << 16; \ -+ mask |= mask << 32; \ -+ } while (0) -+ -+#endif -+ - #ifndef __HAVE_ARCH_MEMCHR - /** - * memchr - Find a character in an area of memory. -- * @s: The memory area -+ * @p: The memory area - * @c: The byte to search for -- * @n: The size of the area. -+ * @length: The size of the area. - * - * returns the address of the first occurrence of @c, or %NULL - * if @c is not found - */ --void *memchr(const void *s, int c, size_t n) -+void *memchr(const void *p, int c, unsigned long length) - { -- const unsigned char *p = s; -- while (n-- != 0) { -- if ((unsigned char)c == *p++) { -- return (void *)(p - 1); -+ u64 mask, val; -+ const void *end = p + length; -+ -+ c &= 0xff; -+ if (p <= end - 8) { -+ mask = c; -+ MEMCHR_MASK_GEN(mask); -+ -+ for (; p <= end - 8; p += 8) { -+ val = *(u64 *)p ^ mask; -+ if ((val + 0xfefefefefefefeffu) & -+ (~val & 0x8080808080808080u)) -+ break; - } - } -+ -+ for (; p < end; p++) -+ if (*(unsigned char *)p == c) -+ return (void *)p; -+ - return NULL; - } - EXPORT_SYMBOL(memchr); -@@ -919,16 +956,7 @@ void *memchr_inv(const void *start, int c, size_t bytes) - return check_bytes8(start, value, bytes); - - value64 = value; --#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 -- value64 *= 0x0101010101010101ULL; --#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) -- value64 *= 0x01010101; -- value64 |= value64 << 32; --#else -- value64 |= value64 << 8; -- value64 |= value64 << 16; -- value64 |= value64 << 32; --#endif -+ MEMCHR_MASK_GEN(value64); - - prefix = (unsigned long)start % 8; - if (prefix) { -diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile -index fc45339fc3a3..440bd0007ae2 100644 ---- a/lib/zstd/Makefile -+++ b/lib/zstd/Makefile -@@ -10,14 +10,10 @@ - # ################################################################ - obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o - obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o -+obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o - - zstd_compress-y := \ - zstd_compress_module.o \ -- common/debug.o \ -- common/entropy_common.o \ -- common/error_private.o \ -- common/fse_decompress.o \ -- common/zstd_common.o \ - compress/fse_compress.o \ - compress/hist.o \ - compress/huf_compress.o \ -@@ -33,12 +29,14 @@ zstd_compress-y := \ - - zstd_decompress-y := \ - zstd_decompress_module.o \ -+ decompress/huf_decompress.o \ -+ decompress/zstd_ddict.o \ -+ decompress/zstd_decompress.o \ -+ decompress/zstd_decompress_block.o \ -+ -+zstd_common-y := \ - common/debug.o \ - common/entropy_common.o \ - common/error_private.o \ - common/fse_decompress.o \ - common/zstd_common.o \ -- decompress/huf_decompress.o \ -- decompress/zstd_ddict.o \ -- decompress/zstd_decompress.o \ -- decompress/zstd_decompress_block.o \ -diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c -index 53b47a2b52ff..f84612627471 100644 ---- a/lib/zstd/common/entropy_common.c -+++ b/lib/zstd/common/entropy_common.c -@@ -15,6 +15,7 @@ - /* ************************************* - * Dependencies - ***************************************/ -+#include - #include "mem.h" - #include "error_private.h" /* ERR_*, ERROR */ - #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ -@@ -239,7 +240,7 @@ size_t FSE_readNCount( - { - return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); - } -- -+EXPORT_SYMBOL_GPL(FSE_readNCount); - - /*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). -@@ -255,6 +256,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); - } -+EXPORT_SYMBOL_GPL(HUF_readStats); - - FORCE_INLINE_TEMPLATE size_t - HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, -diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c -index 3d7e35b309b5..06f62b2026d5 100644 ---- a/lib/zstd/common/zstd_common.c -+++ b/lib/zstd/common/zstd_common.c -@@ -13,6 +13,7 @@ - /*-************************************* - * Dependencies - ***************************************/ -+#include - #define ZSTD_DEPS_NEED_MALLOC - #include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ - #include "error_private.h" -@@ -59,6 +60,7 @@ void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); - } -+EXPORT_SYMBOL_GPL(ZSTD_customMalloc); - - void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) - { -@@ -71,6 +73,7 @@ void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) - } - return ZSTD_calloc(1, size); - } -+EXPORT_SYMBOL_GPL(ZSTD_customCalloc); - - void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) - { -@@ -81,3 +84,7 @@ void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) - ZSTD_free(ptr); - } - } -+EXPORT_SYMBOL_GPL(ZSTD_customFree); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("Zstd Common"); -diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c -index b0424d23ac57..fb941a5b70f5 100644 ---- a/lib/zstd/compress/zstd_double_fast.c -+++ b/lib/zstd/compress/zstd_double_fast.c -@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(dictMode, mls) \ -+ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \ -+ } -+ -+ZSTD_GEN_FN(noDict, 4) -+ZSTD_GEN_FN(noDict, 5) -+ZSTD_GEN_FN(noDict, 6) -+ZSTD_GEN_FN(noDict, 7) -+ -+ZSTD_GEN_FN(dictMatchState, 4) -+ZSTD_GEN_FN(dictMatchState, 5) -+ZSTD_GEN_FN(dictMatchState, 6) -+ZSTD_GEN_FN(dictMatchState, 7) -+ -+#undef ZSTD_GEN_FN -+ - - size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - - /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ - if (prefixStartIndex == dictStartIndex) -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - } - - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN -+ - size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); - } - } -diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c -index 96b7d48e2868..e0652e31d421 100644 ---- a/lib/zstd/compress/zstd_fast.c -+++ b/lib/zstd/compress/zstd_fast.c -@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN - - size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN -+ - size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( - - /* switch to "regular" variant if extDict is invalidated due to maxDistance */ - if (prefixStartIndex == dictStartIndex) -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); -+ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_extDict_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } -+ -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) -+ -+#undef ZSTD_GEN_FN - - size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize); - } - } -diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c -index fb54d4e28a2b..1db22db5b1e9 100644 ---- a/lib/zstd/compress/zstd_lazy.c -+++ b/lib/zstd/compress/zstd_lazy.c -@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, - } - - --static size_t --ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); -- } --} -- -- --static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); -- } --} -- -- --static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); -- } --} -- -- -- - /* ********************************* - * Hash Chain - ***********************************/ -@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B - - /* inlining is important to hardwire a hot branch (template emulation) */ - FORCE_INLINE_TEMPLATE --size_t ZSTD_HcFindBestMatch_generic ( -+size_t ZSTD_HcFindBestMatch( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, -@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic ( - return ml; - } - -+typedef size_t (*searchMax_f)( -+ ZSTD_matchState_t* ms, -+ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - --FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); -- } --} -+/* -+ * This struct contains the functions necessary for lazy to search. -+ * Currently, that is only searchMax. However, it is still valuable to have the -+ * VTable because this makes it easier to add more functions to the VTable later. -+ */ -+typedef struct { -+ searchMax_f searchMax; -+} ZSTD_LazyVTable; -+ -+#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \ -+ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, \ -+ const BYTE* ip, const BYTE* const iLimit, \ -+ size_t* offsetPtr) \ -+ { \ -+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ -+ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ -+ } \ -+ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ -+ ZSTD_BtFindBestMatch_##dictMode##_##mls \ -+ }; - -+#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \ -+ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, \ -+ const BYTE* ip, const BYTE* const iLimit, \ -+ size_t* offsetPtr) \ -+ { \ -+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ -+ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ -+ } \ -+ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ -+ ZSTD_HcFindBestMatch_##dictMode##_##mls \ -+ }; - --static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); -+#define ZSTD_FOR_EACH_MLS(X, dictMode) \ -+ X(dictMode, 4) \ -+ X(dictMode, 5) \ -+ X(dictMode, 6) -+ -+#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ -+ X(__VA_ARGS__, noDict) \ -+ X(__VA_ARGS__, extDict) \ -+ X(__VA_ARGS__, dictMatchState) \ -+ X(__VA_ARGS__, dedicatedDictSearch) -+ -+/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ -+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) -+/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ -+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) -+ -+#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ -+ { \ -+ &ZSTD_BtVTable_##dictMode##_4, \ -+ &ZSTD_BtVTable_##dictMode##_5, \ -+ &ZSTD_BtVTable_##dictMode##_6 \ - } --} -- - --static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); -+#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ -+ { \ -+ &ZSTD_HcVTable_##dictMode##_4, \ -+ &ZSTD_HcVTable_##dictMode##_5, \ -+ &ZSTD_HcVTable_##dictMode##_6 \ - } --} - -- --FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); -+#define GEN_ZSTD_VTABLE_ARRAY(X) \ -+ { \ -+ X(noDict), \ -+ X(extDict), \ -+ X(dictMatchState), \ -+ X(dedicatedDictSearch) \ - } --} -- - - /* ******************************* - * Common parser - lazy strategy - *********************************/ - typedef enum { search_hashChain, search_binaryTree } searchMethod_e; - -+static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) -+{ -+ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ -+ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); -+ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); -+ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ -+ -+ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); -+ switch (searchMethod) { -+ case search_hashChain: -+ return hcVTables[dictMode][mls - 4]; -+ case search_binaryTree: -+ return btVTables[dictMode][mls - 4]; -+ default: -+ return NULL; -+ } -+} -+ - FORCE_INLINE_TEMPLATE size_t - ZSTD_compressBlock_lazy_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, -@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic( - const U32 prefixLowestIndex = ms->window.dictLimit; - const BYTE* const prefixLowest = base + prefixLowestIndex; - -- typedef size_t (*searchMax_f)( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -- - /* - * This table is indexed first by the four ZSTD_dictMode_e values, and then - * by the two searchMethod_e values. NULLs are placed for configurations - * that should never occur (extDict modes go to the other implementation - * below and there is no DDSS for binary tree search yet). - */ -- const searchMax_f searchFuncs[4][2] = { -- { -- ZSTD_HcFindBestMatch_selectMLS, -- ZSTD_BtFindBestMatch_selectMLS -- }, -- { -- NULL, -- NULL -- }, -- { -- ZSTD_HcFindBestMatch_dictMatchState_selectMLS, -- ZSTD_BtFindBestMatch_dictMatchState_selectMLS -- }, -- { -- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, -- NULL -- } -- }; -- -- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; -+ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; - U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; - - const int isDMS = dictMode == ZSTD_dictMatchState; -@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( - const BYTE* const dictStart = dictBase + ms->window.lowLimit; - const U32 windowLog = ms->cParams.windowLog; - -- typedef size_t (*searchMax_f)( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; -+ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; - - U32 offset_1 = rep[0], offset_2 = rep[1]; - -diff --git a/mm/compaction.c b/mm/compaction.c -index 640fa76228dd..01a661f5ecac 100644 ---- a/mm/compaction.c -+++ b/mm/compaction.c -@@ -1727,7 +1727,7 @@ typedef enum { - * Allow userspace to control policy on scanning the unevictable LRU for - * compactable pages. - */ --#ifdef CONFIG_PREEMPT_RT -+#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) - int sysctl_compact_unevictable_allowed __read_mostly = 0; - #else - int sysctl_compact_unevictable_allowed __read_mostly = 1; -@@ -2719,7 +2719,11 @@ static void compact_nodes(void) - * aggressively the kernel should compact memory in the - * background. It takes values in the range [0, 100]. - */ -+#ifdef CONFIG_CACHY -+unsigned int __read_mostly sysctl_compaction_proactiveness; -+#else - unsigned int __read_mostly sysctl_compaction_proactiveness = 20; -+#endif - - int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, - void *buffer, size_t *length, loff_t *ppos) -diff --git a/mm/ksm.c b/mm/ksm.c -index 42ab153335a2..4e64adb9adee 100644 ---- a/mm/ksm.c -+++ b/mm/ksm.c -@@ -2427,9 +2427,14 @@ static int ksm_scan_thread(void *nothing) - - if (ksmd_should_run()) { - sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs); -- wait_event_interruptible_timeout(ksm_iter_wait, -- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -- msecs_to_jiffies(sleep_ms)); -+ if (sleep_ms >= 1000) -+ wait_event_interruptible_timeout(ksm_iter_wait, -+ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -+ msecs_to_jiffies(round_jiffies_relative(sleep_ms))); -+ else -+ wait_event_interruptible_timeout(ksm_iter_wait, -+ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -+ msecs_to_jiffies(sleep_ms)); - } else { - wait_event_freezable(ksm_thread_wait, - ksmd_should_run() || kthread_should_stop()); -diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index b69979c9ced5..7eadbafc006b 100644 ---- a/mm/memcontrol.c -+++ b/mm/memcontrol.c -@@ -625,7 +625,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) - cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); - - x = __this_cpu_add_return(stats_updates, abs(val)); -- if (x > MEMCG_CHARGE_BATCH) { -+ if (x > MEMCG_CHARGE_BATCH * 128) { - /* - * If stats_flush_threshold exceeds the threshold - * (>num_online_cpus()), cgroup stats update will be triggered -diff --git a/mm/page-writeback.c b/mm/page-writeback.c -index 032a7bf8d259..cbcecd565c16 100644 ---- a/mm/page-writeback.c -+++ b/mm/page-writeback.c -@@ -70,7 +70,11 @@ static long ratelimit_pages = 32; - /* - * Start background writeback (via writeback threads) at this percentage - */ -+#ifdef CONFIG_CACHY -+static int dirty_background_ratio = 5; -+#else - static int dirty_background_ratio = 10; -+#endif - - /* - * dirty_background_bytes starts at 0 (disabled) so that it is a function of -@@ -98,7 +102,11 @@ static unsigned long vm_dirty_bytes; - /* - * The interval between `kupdate'-style writebacks - */ -+#ifdef CONFIG_CACHY -+unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */ -+#else - unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ -+#endif - - EXPORT_SYMBOL_GPL(dirty_writeback_interval); - -diff --git a/mm/page_alloc.c b/mm/page_alloc.c -index e5486d47406e..cf131d6e08fb 100644 ---- a/mm/page_alloc.c -+++ b/mm/page_alloc.c -@@ -6982,11 +6982,11 @@ static int zone_batchsize(struct zone *zone) - - /* - * The number of pages to batch allocate is either ~0.1% -- * of the zone or 1MB, whichever is smaller. The batch -+ * of the zone or 4MB, whichever is smaller. The batch - * size is striking a balance between allocation latency - * and zone lock contention. - */ -- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); -+ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); - batch /= 4; /* We effectively *= 4 below */ - if (batch < 1) - batch = 1; -@@ -7064,6 +7064,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) - * historical relationship between high and batch. - */ - high = max(high, batch << 2); -+ high = max(high, 1024); - - return high; - #else -diff --git a/mm/swap.c b/mm/swap.c -index 9cee7f6a3809..186b4e5dcecf 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -1070,6 +1070,10 @@ EXPORT_SYMBOL(pagevec_lookup_range_tag); - */ - void __init swap_setup(void) - { -+#ifdef CONFIG_CACHY -+ /* Only swap-in pages requested, avoid readahead */ -+ page_cluster = 0; -+#else - unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); - - /* Use a smaller cluster for small-memory machines */ -@@ -1081,4 +1085,5 @@ void __init swap_setup(void) - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ -+#endif - } -diff --git a/mm/vmpressure.c b/mm/vmpressure.c -index b52644771cc4..11a4b0e3b583 100644 ---- a/mm/vmpressure.c -+++ b/mm/vmpressure.c -@@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; - * essence, they are percents: the higher the value, the more number - * unsuccessful reclaims there were. - */ -+#ifdef CONFIG_CACHY -+static const unsigned int vmpressure_level_med = 65; -+#else - static const unsigned int vmpressure_level_med = 60; -+#endif - static const unsigned int vmpressure_level_critical = 95; - - /* -diff --git a/mm/vmscan.c b/mm/vmscan.c -index b2b1431352dc..0fc65ace3a4e 100644 ---- a/mm/vmscan.c -+++ b/mm/vmscan.c -@@ -178,7 +178,11 @@ struct scan_control { - /* - * From 0 .. 200. Higher means more swappy. - */ -+#ifdef CONFIG_CACHY -+int vm_swappiness = 20; -+#else - int vm_swappiness = 60; -+#endif - - static void set_task_reclaim_state(struct task_struct *task, - struct reclaim_state *rs) -diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c -index eb31c7158b39..8cefa0b62fb9 100644 ---- a/net/ipv4/inet_connection_sock.c -+++ b/net/ipv4/inet_connection_sock.c -@@ -445,7 +445,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) - * having to remove and re-insert us on the wait queue. - */ - for (;;) { -- prepare_to_wait_exclusive(sk_sleep(sk), &wait, -+ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, - TASK_INTERRUPTIBLE); - release_sock(sk); - if (reqsk_queue_empty(&icsk->icsk_accept_queue)) -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index e373dde1f46f..3fc8ca6a264e 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -4789,8 +4789,8 @@ void __init tcp_init(void) - tcp_init_mem(); - /* Set per-socket limits to no more than 1/128 the pressure threshold */ - limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); -- max_wshare = min(4UL*1024*1024, limit); -- max_rshare = min(6UL*1024*1024, limit); -+ max_wshare = min(16UL*1024*1024, limit); -+ max_rshare = min(16UL*1024*1024, limit); - - init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; - init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; --- -2.38.0.rc1.8.g2a7d63a245 - -From cdfac99ee700654cacf05cbbfb83ca7907b0dcc7 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 5 Sep 2022 08:34:43 +0200 -Subject: [PATCH 02/13] bbr2 - -Signed-off-by: Peter Jung ---- - Documentation/networking/ip-sysctl.rst | 58 + - include/linux/tcp.h | 6 +- - include/net/inet_connection_sock.h | 3 +- - include/net/netns/ipv4.h | 5 + - include/net/tcp.h | 58 +- - include/uapi/linux/inet_diag.h | 33 + - include/uapi/linux/snmp.h | 1 + - net/ipv4/Kconfig | 22 + - net/ipv4/Makefile | 3 +- - net/ipv4/bpf_tcp_ca.c | 12 + - net/ipv4/proc.c | 1 + - net/ipv4/sysctl_net_ipv4.c | 43 + - net/ipv4/tcp.c | 1 + - net/ipv4/tcp_bbr.c | 38 +- - net/ipv4/tcp_bbr2.c | 2692 ++++++++++++++++++++++++ - net/ipv4/tcp_cong.c | 1 + - net/ipv4/tcp_input.c | 27 +- - net/ipv4/tcp_ipv4.c | 7 + - net/ipv4/tcp_output.c | 26 +- - net/ipv4/tcp_plb.c | 100 + - net/ipv4/tcp_rate.c | 30 +- - net/ipv4/tcp_timer.c | 1 + - 22 files changed, 3133 insertions(+), 35 deletions(-) - create mode 100644 net/ipv4/tcp_bbr2.c - create mode 100644 net/ipv4/tcp_plb.c - -diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst -index a759872a2883..f2372f8f860b 100644 ---- a/Documentation/networking/ip-sysctl.rst -+++ b/Documentation/networking/ip-sysctl.rst -@@ -1040,6 +1040,64 @@ tcp_challenge_ack_limit - INTEGER - TCP stack implements per TCP socket limits anyway. - Default: INT_MAX (unlimited) - -+tcp_plb_enabled - BOOLEAN -+ If set, TCP PLB (Protective Load Balancing) is enabled. PLB is -+ described in the following paper: -+ https://doi.org/10.1145/3544216.3544226. Based on PLB parameters, -+ upon sensing sustained congestion, TCP triggers a change in -+ flow label field for outgoing IPv6 packets. A change in flow label -+ field potentially changes the path of outgoing packets for switches -+ that use ECMP/WCMP for routing. -+ -+ Default: 0 -+ -+tcp_plb_cong_thresh - INTEGER -+ Fraction of packets marked with congestion over a round (RTT) to -+ tag that round as congested. This is referred to as K in the PLB paper: -+ https://doi.org/10.1145/3544216.3544226. -+ -+ The 0-1 fraction range is mapped to 0-256 range to avoid floating -+ point operations. For example, 128 means that if at least 50% of -+ the packets in a round were marked as congested then the round -+ will be tagged as congested. -+ -+ Possible Values: 0 - 256 -+ -+ Default: 128 -+ -+tcp_plb_idle_rehash_rounds - INTEGER -+ Number of consecutive congested rounds (RTT) seen after which -+ a rehash can be performed, given there are no packets in flight. -+ This is referred to as M in PLB paper: -+ https://doi.org/10.1145/3544216.3544226. -+ -+ Possible Values: 0 - 31 -+ -+ Default: 3 -+ -+tcp_plb_rehash_rounds - INTEGER -+ Number of consecutive congested rounds (RTT) seen after which -+ a forced rehash can be performed. Be careful when setting this -+ parameter, as a small value increases the risk of retransmissions. -+ This is referred to as N in PLB paper: -+ https://doi.org/10.1145/3544216.3544226. -+ -+ Possible Values: 0 - 31 -+ -+ Default: 12 -+ -+tcp_plb_suspend_rto_sec - INTEGER -+ Time, in seconds, to suspend PLB in event of an RTO. In order to avoid -+ having PLB repath onto a connectivity "black hole", after an RTO a TCP -+ connection suspends PLB repathing for a random duration between 1x and -+ 2x of this parameter. Randomness is added to avoid concurrent rehashing -+ of multiple TCP connections. This should be set corresponding to the -+ amount of time it takes to repair a failed link. -+ -+ Possible Values: 0 - 255 -+ -+ Default: 60 -+ - UDP variables - ============= - -diff --git a/include/linux/tcp.h b/include/linux/tcp.h -index a9fbe22732c3..4034e46093b4 100644 ---- a/include/linux/tcp.h -+++ b/include/linux/tcp.h -@@ -255,7 +255,8 @@ struct tcp_sock { - u8 compressed_ack; - u8 dup_ack_counter:2, - tlp_retrans:1, /* TLP is a retransmission */ -- unused:5; -+ fast_ack_mode:2, /* which fast ack mode ? */ -+ unused:3; - u32 chrono_start; /* Start time in jiffies of a TCP chrono */ - u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ - u8 chrono_type:2, /* current chronograph type */ -@@ -443,6 +444,9 @@ struct tcp_sock { - */ - struct request_sock __rcu *fastopen_rsk; - struct saved_syn *saved_syn; -+ -+/* Rerouting information */ -+ u16 ecn_rehash; /* PLB triggered rehash attempts */ - }; - - enum tsq_enum { -diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h -index ee88f0f1350f..e3075b3f1ece 100644 ---- a/include/net/inet_connection_sock.h -+++ b/include/net/inet_connection_sock.h -@@ -132,7 +132,8 @@ struct inet_connection_sock { - u32 icsk_probes_tstamp; - u32 icsk_user_timeout; - -- u64 icsk_ca_priv[104 / sizeof(u64)]; -+/* XXX inflated by temporary internal debugging info */ -+ u64 icsk_ca_priv[224 / sizeof(u64)]; - #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) - }; - -diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h -index 6320a76cefdc..2e39e07ed41f 100644 ---- a/include/net/netns/ipv4.h -+++ b/include/net/netns/ipv4.h -@@ -181,6 +181,11 @@ struct netns_ipv4 { - unsigned long tfo_active_disable_stamp; - u32 tcp_challenge_timestamp; - u32 tcp_challenge_count; -+ u8 sysctl_tcp_plb_enabled; -+ int sysctl_tcp_plb_cong_thresh; -+ u8 sysctl_tcp_plb_idle_rehash_rounds; -+ u8 sysctl_tcp_plb_rehash_rounds; -+ u8 sysctl_tcp_plb_suspend_rto_sec; - - int sysctl_udp_wmem_min; - int sysctl_udp_rmem_min; -diff --git a/include/net/tcp.h b/include/net/tcp.h -index d10962b9f0d0..d3947345feaa 100644 ---- a/include/net/tcp.h -+++ b/include/net/tcp.h -@@ -372,6 +372,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk, - #define TCP_ECN_QUEUE_CWR 2 - #define TCP_ECN_DEMAND_CWR 4 - #define TCP_ECN_SEEN 8 -+#define TCP_ECN_ECT_PERMANENT 16 - - enum tcp_tw_status { - TCP_TW_SUCCESS = 0, -@@ -816,6 +817,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) - return max_t(s64, t1 - t0, 0); - } - -+static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) -+{ -+ return max_t(s32, t1 - t0, 0); -+} -+ - static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) - { - return tcp_ns_to_ts(skb->skb_mstamp_ns); -@@ -891,9 +897,14 @@ struct tcp_skb_cb { - /* pkts S/ACKed so far upon tx of skb, incl retrans: */ - __u32 delivered; - /* start of send pipeline phase */ -- u64 first_tx_mstamp; -+ u32 first_tx_mstamp; - /* when we reached the "delivered" count */ -- u64 delivered_mstamp; -+ u32 delivered_mstamp; -+#define TCPCB_IN_FLIGHT_BITS 20 -+#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) -+ u32 in_flight:20, /* packets in flight at transmit */ -+ unused2:12; -+ u32 lost; /* packets lost so far upon tx of skb */ - } tx; /* only used for outgoing skbs */ - union { - struct inet_skb_parm h4; -@@ -1019,7 +1030,11 @@ enum tcp_ca_ack_event_flags { - #define TCP_CONG_NON_RESTRICTED 0x1 - /* Requires ECN/ECT set on all packets */ - #define TCP_CONG_NEEDS_ECN 0x2 --#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) -+/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ -+#define TCP_CONG_WANTS_CE_EVENTS 0x4 -+#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ -+ TCP_CONG_NEEDS_ECN | \ -+ TCP_CONG_WANTS_CE_EVENTS) - - union tcp_cc_info; - -@@ -1039,8 +1054,11 @@ struct ack_sample { - */ - struct rate_sample { - u64 prior_mstamp; /* starting timestamp for interval */ -+ u32 prior_lost; /* tp->lost at "prior_mstamp" */ - u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ - u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ -+ u32 tx_in_flight; /* packets in flight at starting timestamp */ -+ s32 lost; /* number of packets lost over interval */ - s32 delivered; /* number of packets delivered over interval */ - s32 delivered_ce; /* number of packets delivered w/ CE marks*/ - long interval_us; /* time for tp->delivered to incr "delivered" */ -@@ -1054,6 +1072,7 @@ struct rate_sample { - bool is_app_limited; /* is sample from packet with bubble in pipe? */ - bool is_retrans; /* is sample from retransmission? */ - bool is_ack_delayed; /* is this (likely) a delayed ACK? */ -+ bool is_ece; /* did this ACK have ECN marked? */ - }; - - struct tcp_congestion_ops { -@@ -1077,8 +1096,11 @@ struct tcp_congestion_ops { - /* hook for packet ack accounting (optional) */ - void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - -- /* override sysctl_tcp_min_tso_segs */ -- u32 (*min_tso_segs)(struct sock *sk); -+ /* pick target number of segments per TSO/GSO skb (optional): */ -+ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); -+ -+ /* react to a specific lost skb (optional) */ -+ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); - - /* call when packets are delivered to update cwnd and pacing rate, - * after all the ca_state processing. (optional) -@@ -1141,6 +1163,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) - } - #endif - -+static inline bool tcp_ca_wants_ce_events(const struct sock *sk) -+{ -+ const struct inet_connection_sock *icsk = inet_csk(sk); -+ -+ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | -+ TCP_CONG_WANTS_CE_EVENTS); -+} -+ - static inline bool tcp_ca_needs_ecn(const struct sock *sk) - { - const struct inet_connection_sock *icsk = inet_csk(sk); -@@ -1160,6 +1190,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) - void tcp_set_ca_state(struct sock *sk, const u8 ca_state); - - /* From tcp_rate.c */ -+void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); - void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); - void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - struct rate_sample *rs); -@@ -2130,6 +2161,23 @@ extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, - extern void tcp_rack_reo_timeout(struct sock *sk); - extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); - -+/* tcp_plb.c */ -+ -+#define TCP_PLB_SCALE 8 /* scaling factor for fractions in PLB (e.g. ce_ratio) */ -+ -+/* State for PLB (Protective Load Balancing) for a single TCP connection. */ -+struct tcp_plb_state { -+ u8 consec_cong_rounds:5, /* consecutive congested rounds */ -+ enabled:1, /* Check if PLB is enabled */ -+ unused:2; -+ u32 pause_until; /* jiffies32 when PLB can resume repathing */ -+}; -+ -+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, -+ const int cong_ratio); -+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); -+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); -+ - /* At how many usecs into the future should the RTO fire? */ - static inline s64 tcp_rto_delta_us(const struct sock *sk) - { -diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h -index 50655de04c9b..0e24f11627d5 100644 ---- a/include/uapi/linux/inet_diag.h -+++ b/include/uapi/linux/inet_diag.h -@@ -231,9 +231,42 @@ struct tcp_bbr_info { - __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ - }; - -+/* Phase as reported in netlink/ss stats. */ -+enum tcp_bbr2_phase { -+ BBR2_PHASE_INVALID = 0, -+ BBR2_PHASE_STARTUP = 1, -+ BBR2_PHASE_DRAIN = 2, -+ BBR2_PHASE_PROBE_RTT = 3, -+ BBR2_PHASE_PROBE_BW_UP = 4, -+ BBR2_PHASE_PROBE_BW_DOWN = 5, -+ BBR2_PHASE_PROBE_BW_CRUISE = 6, -+ BBR2_PHASE_PROBE_BW_REFILL = 7 -+}; -+ -+struct tcp_bbr2_info { -+ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ -+ __u32 bbr_bw_lsb; /* lower 32 bits of bw */ -+ __u32 bbr_bw_msb; /* upper 32 bits of bw */ -+ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ -+ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ -+ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ -+ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ -+ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ -+ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ -+ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ -+ __u8 bbr_mode; /* current bbr_mode in state machine */ -+ __u8 bbr_phase; /* current state machine phase */ -+ __u8 unused1; /* alignment padding; not used yet */ -+ __u8 bbr_version; /* MUST be at this offset in struct */ -+ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ -+ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ -+ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ -+}; -+ - union tcp_cc_info { - struct tcpvegas_info vegas; - struct tcp_dctcp_info dctcp; - struct tcp_bbr_info bbr; -+ struct tcp_bbr2_info bbr2; - }; - #endif /* _UAPI_INET_DIAG_H_ */ -diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h -index 4d7470036a8b..8ce035f1c874 100644 ---- a/include/uapi/linux/snmp.h -+++ b/include/uapi/linux/snmp.h -@@ -292,6 +292,7 @@ enum - LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */ - LINUX_MIB_TCPMIGRATEREQSUCCESS, /* TCPMigrateReqSuccess */ - LINUX_MIB_TCPMIGRATEREQFAILURE, /* TCPMigrateReqFailure */ -+ LINUX_MIB_TCPECNREHASH, /* TCPECNRehash */ - __LINUX_MIB_MAX - }; - -diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig -index e983bb0c5012..d172ffbdf27f 100644 ---- a/net/ipv4/Kconfig -+++ b/net/ipv4/Kconfig -@@ -668,6 +668,24 @@ config TCP_CONG_BBR - AQM schemes that do not provide a delay signal. It requires the fq - ("Fair Queue") pacing packet scheduler. - -+config TCP_CONG_BBR2 -+ tristate "BBR2 TCP" -+ default n -+ help -+ -+ BBR2 TCP congestion control is a model-based congestion control -+ algorithm that aims to maximize network utilization, keep queues and -+ retransmit rates low, and to be able to coexist with Reno/CUBIC in -+ common scenarios. It builds an explicit model of the network path. It -+ tolerates a targeted degree of random packet loss and delay that are -+ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, -+ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can -+ coexist with flows that use loss-based congestion control, and can -+ operate with shallow buffers, deep buffers, bufferbloat, policers, or -+ AQM schemes that do not provide a delay signal. It requires pacing, -+ using either TCP internal pacing or the fq ("Fair Queue") pacing packet -+ scheduler. -+ - choice - prompt "Default TCP congestion control" - default DEFAULT_CUBIC -@@ -705,6 +723,9 @@ choice - config DEFAULT_BBR - bool "BBR" if TCP_CONG_BBR=y - -+ config DEFAULT_BBR2 -+ bool "BBR2" if TCP_CONG_BBR2=y -+ - config DEFAULT_RENO - bool "Reno" - endchoice -@@ -729,6 +750,7 @@ config DEFAULT_TCP_CONG - default "dctcp" if DEFAULT_DCTCP - default "cdg" if DEFAULT_CDG - default "bbr" if DEFAULT_BBR -+ default "bbr2" if DEFAULT_BBR2 - default "cubic" - - config TCP_MD5SIG -diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile -index bbdd9c44f14e..e7a86a50838a 100644 ---- a/net/ipv4/Makefile -+++ b/net/ipv4/Makefile -@@ -10,7 +10,7 @@ obj-y := route.o inetpeer.o protocol.o \ - tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \ - tcp_minisocks.o tcp_cong.o tcp_metrics.o tcp_fastopen.o \ - tcp_rate.o tcp_recovery.o tcp_ulp.o \ -- tcp_offload.o datagram.o raw.o udp.o udplite.o \ -+ tcp_offload.o tcp_plb.o datagram.o raw.o udp.o udplite.o \ - udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \ - fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \ - inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \ -@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o - obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o - obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o - obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o -+obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o - obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o - obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o - obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o -diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c -index 85a9e500c42d..24fcbac984fe 100644 ---- a/net/ipv4/bpf_tcp_ca.c -+++ b/net/ipv4/bpf_tcp_ca.c -@@ -14,6 +14,18 @@ - /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ - extern struct bpf_struct_ops bpf_tcp_congestion_ops; - -+static u32 optional_ops[] = { -+ offsetof(struct tcp_congestion_ops, init), -+ offsetof(struct tcp_congestion_ops, release), -+ offsetof(struct tcp_congestion_ops, set_state), -+ offsetof(struct tcp_congestion_ops, cwnd_event), -+ offsetof(struct tcp_congestion_ops, in_ack_event), -+ offsetof(struct tcp_congestion_ops, pkts_acked), -+ offsetof(struct tcp_congestion_ops, tso_segs), -+ offsetof(struct tcp_congestion_ops, sndbuf_expand), -+ offsetof(struct tcp_congestion_ops, cong_control), -+}; -+ - static u32 unsupported_ops[] = { - offsetof(struct tcp_congestion_ops, get_info), - }; -diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c -index 0088a4c64d77..e0a664b467e0 100644 ---- a/net/ipv4/proc.c -+++ b/net/ipv4/proc.c -@@ -297,6 +297,7 @@ static const struct snmp_mib snmp4_net_list[] = { - SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS), - SNMP_MIB_ITEM("TCPMigrateReqSuccess", LINUX_MIB_TCPMIGRATEREQSUCCESS), - SNMP_MIB_ITEM("TCPMigrateReqFailure", LINUX_MIB_TCPMIGRATEREQFAILURE), -+ SNMP_MIB_ITEM("TCPECNRehash", LINUX_MIB_TCPECNREHASH), - SNMP_MIB_SENTINEL - }; - -diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c -index 5490c285668b..ed35a9485e71 100644 ---- a/net/ipv4/sysctl_net_ipv4.c -+++ b/net/ipv4/sysctl_net_ipv4.c -@@ -39,6 +39,8 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ; - static int one_day_secs = 24 * 3600; - static u32 fib_multipath_hash_fields_all_mask __maybe_unused = - FIB_MULTIPATH_HASH_FIELD_ALL_MASK; -+static int tcp_plb_max_rounds = 31; -+static int tcp_plb_max_cong_thresh = 256; - - /* obsolete */ - static int sysctl_tcp_low_latency __read_mostly; -@@ -1346,6 +1348,47 @@ static struct ctl_table ipv4_net_table[] = { - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_TWO, - }, -+ { -+ .procname = "tcp_plb_enabled", -+ .data = &init_net.ipv4.sysctl_tcp_plb_enabled, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "tcp_plb_cong_thresh", -+ .data = &init_net.ipv4.sysctl_tcp_plb_cong_thresh, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &tcp_plb_max_cong_thresh, -+ }, -+ { -+ .procname = "tcp_plb_idle_rehash_rounds", -+ .data = &init_net.ipv4.sysctl_tcp_plb_idle_rehash_rounds, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra2 = &tcp_plb_max_rounds, -+ }, -+ { -+ .procname = "tcp_plb_rehash_rounds", -+ .data = &init_net.ipv4.sysctl_tcp_plb_rehash_rounds, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra2 = &tcp_plb_max_rounds, -+ }, -+ { -+ .procname = "tcp_plb_suspend_rto_sec", -+ .data = &init_net.ipv4.sysctl_tcp_plb_suspend_rto_sec, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ }, - { } - }; - -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index 3fc8ca6a264e..2fcd440df1f6 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -3187,6 +3187,7 @@ int tcp_disconnect(struct sock *sk, int flags) - tp->rx_opt.dsack = 0; - tp->rx_opt.num_sacks = 0; - tp->rcv_ooopack = 0; -+ tp->fast_ack_mode = 0; - - - /* Clean up fastopen related fields */ -diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c -index 54eec33c6e1c..bfbf158c71f4 100644 ---- a/net/ipv4/tcp_bbr.c -+++ b/net/ipv4/tcp_bbr.c -@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) - sk->sk_pacing_rate = rate; - } - --/* override sysctl_tcp_min_tso_segs */ - static u32 bbr_min_tso_segs(struct sock *sk) - { - return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; - } - -+/* Return the number of segments BBR would like in a TSO/GSO skb, given -+ * a particular max gso size as a constraint. -+ */ -+static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, -+ u32 gso_max_size) -+{ -+ u32 segs; -+ u64 bytes; -+ -+ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ -+ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; -+ -+ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); -+ return segs; -+} -+ -+/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ -+static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) -+{ -+ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); -+} -+ -+/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ - static u32 bbr_tso_segs_goal(struct sock *sk) - { - struct tcp_sock *tp = tcp_sk(sk); -- u32 segs, bytes; -- -- /* Sort of tcp_tso_autosize() but ignoring -- * driver provided sk_gso_max_size. -- */ -- bytes = min_t(unsigned long, -- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), -- GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); -- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - -- return min(segs, 0x7FU); -+ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); - } - - /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { - .undo_cwnd = bbr_undo_cwnd, - .cwnd_event = bbr_cwnd_event, - .ssthresh = bbr_ssthresh, -- .min_tso_segs = bbr_min_tso_segs, -+ .tso_segs = bbr_tso_segs, - .get_info = bbr_get_info, - .set_state = bbr_set_state, - }; -diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c -new file mode 100644 -index 000000000000..2e39f7a353be ---- /dev/null -+++ b/net/ipv4/tcp_bbr2.c -@@ -0,0 +1,2692 @@ -+/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 -+ * -+ * BBRv2 is a model-based congestion control algorithm that aims for low -+ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model -+ * of the network path, it uses measurements of bandwidth and RTT, as well as -+ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that -+ * although it can use ECN or loss signals explicitly, it does not require -+ * either; it can bound its in-flight data based on its estimate of the BDP. -+ * -+ * The model has both higher and lower bounds for the operating range: -+ * lo: bw_lo, inflight_lo: conservative short-term lower bound -+ * hi: bw_hi, inflight_hi: robust long-term upper bound -+ * The bandwidth-probing time scale is (a) extended dynamically based on -+ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by -+ * an interactive wall-clock time-scale to be more scalable and responsive -+ * than Reno and CUBIC. -+ * -+ * Here is a state transition diagram for BBR: -+ * -+ * | -+ * V -+ * +---> STARTUP ----+ -+ * | | | -+ * | V | -+ * | DRAIN ----+ -+ * | | | -+ * | V | -+ * +---> PROBE_BW ----+ -+ * | ^ | | -+ * | | | | -+ * | +----+ | -+ * | | -+ * +---- PROBE_RTT <--+ -+ * -+ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. -+ * When it estimates the pipe is full, it enters DRAIN to drain the queue. -+ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. -+ * A long-lived BBR flow spends the vast majority of its time remaining -+ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth -+ * in a fair manner, with a small, bounded queue. *If* a flow has been -+ * continuously sending for the entire min_rtt window, and hasn't seen an RTT -+ * sample that matches or decreases its min_rtt estimate for 10 seconds, then -+ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe -+ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if -+ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; -+ * otherwise we enter STARTUP to try to fill the pipe. -+ * -+ * BBR is described in detail in: -+ * "BBR: Congestion-Based Congestion Control", -+ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, -+ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. -+ * -+ * There is a public e-mail list for discussing BBR development and testing: -+ * https://groups.google.com/forum/#!forum/bbr-dev -+ * -+ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, -+ * otherwise TCP stack falls back to an internal pacing using one high -+ * resolution timer per TCP socket and may use more resources. -+ */ -+#include -+#include -+#include -+#include -+#include -+ -+#include "tcp_dctcp.h" -+ -+/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth -+ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. -+ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. -+ * Since the minimum window is >=4 packets, the lower bound isn't -+ * an issue. The upper bound isn't an issue with existing technologies. -+ */ -+#define BW_SCALE 24 -+#define BW_UNIT (1 << BW_SCALE) -+ -+#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ -+#define BBR_UNIT (1 << BBR_SCALE) -+ -+#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ -+#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ -+ -+#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ -+ -+/* BBR has the following modes for deciding how fast to send: */ -+enum bbr_mode { -+ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ -+ BBR_DRAIN, /* drain any queue created during startup */ -+ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ -+ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ -+}; -+ -+/* How does the incoming ACK stream relate to our bandwidth probing? */ -+enum bbr_ack_phase { -+ BBR_ACKS_INIT, /* not probing; not getting probe feedback */ -+ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ -+ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ -+ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ -+ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ -+}; -+ -+/* BBR congestion control block */ -+struct bbr { -+ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ -+ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ -+ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ -+ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ -+ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ -+ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ -+ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ -+ u64 cycle_mstamp; /* time of this cycle phase start */ -+ u32 mode:3, /* current bbr_mode in state machine */ -+ prev_ca_state:3, /* CA state on previous ACK */ -+ packet_conservation:1, /* use packet conservation? */ -+ round_start:1, /* start of packet-timed tx->ack round? */ -+ ce_state:1, /* If most recent data has CE bit set */ -+ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ -+ try_fast_path:1, /* can we take fast path? */ -+ unused2:11, -+ idle_restart:1, /* restarting after idle? */ -+ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ -+ cycle_idx:3, /* current index in pacing_gain cycle array */ -+ has_seen_rtt:1; /* have we seen an RTT sample yet? */ -+ u32 pacing_gain:11, /* current gain for setting pacing rate */ -+ cwnd_gain:11, /* current gain for setting cwnd */ -+ full_bw_reached:1, /* reached full bw in Startup? */ -+ full_bw_cnt:2, /* number of rounds without large bw gains */ -+ init_cwnd:7; /* initial cwnd */ -+ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ -+ u32 full_bw; /* recent bw, to estimate if pipe is full */ -+ -+ /* For tracking ACK aggregation: */ -+ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ -+ u16 extra_acked[2]; /* max excess data ACKed in epoch */ -+ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ -+ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ -+ extra_acked_win_idx:1, /* current index in extra_acked array */ -+ /* BBR v2 state: */ -+ unused1:2, -+ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ -+ loss_in_cycle:1, /* packet loss in this cycle? */ -+ ecn_in_cycle:1; /* ECN in this cycle? */ -+ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ -+ u32 undo_bw_lo; /* bw_lo before latest losses */ -+ u32 undo_inflight_lo; /* inflight_lo before latest losses */ -+ u32 undo_inflight_hi; /* inflight_hi before latest losses */ -+ u32 bw_latest; /* max delivered bw in last round trip */ -+ u32 bw_lo; /* lower bound on sending bandwidth */ -+ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ -+ u32 inflight_latest; /* max delivered data in last round trip */ -+ u32 inflight_lo; /* lower bound of inflight data range */ -+ u32 inflight_hi; /* upper bound of inflight data range */ -+ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ -+ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ -+ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ -+ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ -+ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ -+ bw_probe_samples:1, /* rate samples reflect bw probing? */ -+ prev_probe_too_high:1, /* did last PROBE_UP go too high? */ -+ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ -+ rounds_since_probe:8, /* packet-timed rounds since probed bw */ -+ loss_round_start:1, /* loss_round_delivered round trip? */ -+ loss_in_round:1, /* loss marked in this round trip? */ -+ ecn_in_round:1, /* ECN marked in this round trip? */ -+ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ -+ loss_events_in_round:4,/* losses in STARTUP round */ -+ initialized:1; /* has bbr_init() been called? */ -+ u32 alpha_last_delivered; /* tp->delivered at alpha update */ -+ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ -+ struct tcp_plb_state plb; -+ -+ /* Params configurable using setsockopt. Refer to correspoding -+ * module param for detailed description of params. -+ */ -+ struct bbr_params { -+ u32 high_gain:11, /* max allowed value: 2047 */ -+ drain_gain:10, /* max allowed value: 1023 */ -+ cwnd_gain:11; /* max allowed value: 2047 */ -+ u32 cwnd_min_target:4, /* max allowed value: 15 */ -+ min_rtt_win_sec:5, /* max allowed value: 31 */ -+ probe_rtt_mode_ms:9, /* max allowed value: 511 */ -+ full_bw_cnt:3, /* max allowed value: 7 */ -+ cwnd_tso_budget:1, /* allowed values: {0, 1} */ -+ unused3:6, -+ drain_to_target:1, /* boolean */ -+ precise_ece_ack:1, /* boolean */ -+ extra_acked_in_startup:1, /* allowed values: {0, 1} */ -+ fast_path:1; /* boolean */ -+ u32 full_bw_thresh:10, /* max allowed value: 1023 */ -+ startup_cwnd_gain:11, /* max allowed value: 2047 */ -+ bw_probe_pif_gain:9, /* max allowed value: 511 */ -+ usage_based_cwnd:1, /* boolean */ -+ unused2:1; -+ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ -+ refill_add_inc:2; /* max allowed value: 3 */ -+ u16 extra_acked_gain:11, /* max allowed value: 2047 */ -+ extra_acked_win_rtts:5; /* max allowed value: 31*/ -+ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ -+ /* Mostly BBR v2 parameters below here: */ -+ u32 ecn_alpha_gain:8, /* max allowed value: 255 */ -+ ecn_factor:8, /* max allowed value: 255 */ -+ ecn_thresh:8, /* max allowed value: 255 */ -+ beta:8; /* max allowed value: 255 */ -+ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ -+ bw_probe_reno_gain:9, /* max allowed value: 511 */ -+ full_loss_cnt:4; /* max allowed value: 15 */ -+ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ -+ inflight_headroom:8, /* max allowed value: 255 */ -+ loss_thresh:8, /* max allowed value: 255 */ -+ bw_probe_max_rounds:8; /* max allowed value: 255 */ -+ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ -+ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ -+ full_ecn_cnt:2; /* max allowed value: 3 */ -+ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ -+ undo:1, /* boolean */ -+ tso_rtt_shift:4, /* max allowed value: 15 */ -+ unused5:1; -+ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ -+ unused1:14, -+ ecn_alpha_init:9; /* max allowed value: 256 */ -+ } params; -+ -+ struct { -+ u32 snd_isn; /* Initial sequence number */ -+ u32 rs_bw; /* last valid rate sample bw */ -+ u32 target_cwnd; /* target cwnd, based on BDP */ -+ u8 undo:1, /* Undo even happened but not yet logged */ -+ unused:7; -+ char event; /* single-letter event debug codes */ -+ u16 unused2; -+ } debug; -+}; -+ -+struct bbr_context { -+ u32 sample_bw; -+ u32 target_cwnd; -+ u32 log:1; -+}; -+ -+/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ -+static u32 bbr_min_rtt_win_sec = 10; -+/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. -+ * Max allowed value is 511 (0x1FF). -+ */ -+static u32 bbr_probe_rtt_mode_ms = 200; -+/* Window length of probe_rtt_min_us filter (in ms), and consequently the -+ * typical interval between PROBE_RTT mode entries. -+ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC -+ */ -+static u32 bbr_probe_rtt_win_ms = 5000; -+/* Skip TSO below the following bandwidth (bits/sec): */ -+static int bbr_min_tso_rate = 1200000; -+ -+/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting -+ * in bigger TSO bursts. By default we cut the RTT-based allowance in half -+ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance -+ * is below 1500 bytes after 6 * ~500 usec = 3ms. -+ */ -+static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ -+ -+/* Select cwnd TSO budget approach: -+ * 0: padding -+ * 1: flooring -+ */ -+static uint bbr_cwnd_tso_budget = 1; -+ -+/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. -+ * In order to help drive the network toward lower queues and low latency while -+ * maintaining high utilization, the average pacing rate aims to be slightly -+ * lower than the estimated bandwidth. This is an important aspect of the -+ * design. -+ */ -+static const int bbr_pacing_margin_percent = 1; -+ -+/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain -+ * that will allow a smoothly increasing pacing rate that will double each RTT -+ * and send the same number of packets per RTT that an un-paced, slow-starting -+ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). -+ */ -+static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; -+/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ -+static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; -+/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain -+ * the queue created in BBR_STARTUP in a single round. Max allowed value -+ * is 1023 (0x3FF). -+ */ -+static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; -+/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. -+ * Max allowed value is 2047 (0x7FF). -+ */ -+static int bbr_cwnd_gain = BBR_UNIT * 2; -+/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. -+ * Max allowed value for each element is 1023 (0x3FF). -+ */ -+enum bbr_pacing_gain_phase { -+ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ -+ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ -+ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ -+ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ -+}; -+static int bbr_pacing_gain[] = { -+ BBR_UNIT * 5 / 4, /* probe for more available bw */ -+ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ -+ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ -+ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ -+}; -+ -+/* Try to keep at least this many packets in flight, if things go smoothly. For -+ * smooth functioning, a sliding window protocol ACKing every other packet -+ * needs at least 4 packets in flight. Max allowed value is 15 (0xF). -+ */ -+static u32 bbr_cwnd_min_target = 4; -+ -+/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. -+ * Use 0 to disable. Max allowed value is 255. -+ */ -+static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; -+ -+/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ -+/* If bw has increased significantly (1.25x), there may be more bw available. -+ * Max allowed value is 1023 (0x3FF). -+ */ -+static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; -+/* But after 3 rounds w/o significant bw growth, estimate pipe is full. -+ * Max allowed value is 7 (0x7). -+ */ -+static u32 bbr_full_bw_cnt = 3; -+ -+static u32 bbr_flags; /* Debugging related stuff */ -+ -+/* Whether to debug using printk. -+ */ -+static bool bbr_debug_with_printk; -+ -+/* Whether to debug using ftrace event tcp:tcp_bbr_event. -+ * Ignored when bbr_debug_with_printk is set. -+ */ -+static bool bbr_debug_ftrace; -+ -+/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ -+static bool bbr_drain_to_target = true; /* default: enabled */ -+ -+/* Experiment: Flags to control BBR with ECN behavior. -+ */ -+static bool bbr_precise_ece_ack = true; /* default: enabled */ -+ -+/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is -+ * (2^(16+14) B)/(1024 B/packet) = 1M packets. -+ */ -+static u32 bbr_cwnd_warn_val = 1U << 20; -+ -+static u16 bbr_debug_port_mask; -+ -+/* BBR module parameters. These are module parameters only in Google prod. -+ * Upstream these are intentionally not module parameters. -+ */ -+static int bbr_pacing_gain_size = CYCLE_LEN; -+ -+/* Gain factor for adding extra_acked to target cwnd: */ -+static int bbr_extra_acked_gain = 256; -+ -+/* Window length of extra_acked window. Max allowed val is 31. */ -+static u32 bbr_extra_acked_win_rtts = 5; -+ -+/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ -+static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; -+ -+/* Time period for clamping cwnd increment due to ack aggregation */ -+static u32 bbr_extra_acked_max_us = 100 * 1000; -+ -+/* Use extra acked in startup ? -+ * 0: disabled -+ * 1: use latest extra_acked value from 1-2 rtt in startup -+ */ -+static int bbr_extra_acked_in_startup = 1; /* default: enabled */ -+ -+/* Experiment: don't grow cwnd beyond twice of what we just probed. */ -+static bool bbr_usage_based_cwnd; /* default: disabled */ -+ -+/* For lab testing, researchers can enable BBRv2 ECN support with this flag, -+ * when they know that any ECN marks that the connections experience will be -+ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. -+ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on -+ * negotiation or configuration that is outside the scope of the BBRv2 -+ * alpha release. -+ */ -+static bool bbr_ecn_enable = false; -+ -+module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); -+module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); -+module_param_named(high_gain, bbr_high_gain, int, 0644); -+module_param_named(drain_gain, bbr_drain_gain, int, 0644); -+module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); -+module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); -+module_param_array_named(pacing_gain, bbr_pacing_gain, int, -+ &bbr_pacing_gain_size, 0644); -+module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); -+module_param_named(probe_rtt_cwnd_gain, -+ bbr_probe_rtt_cwnd_gain, uint, 0664); -+module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); -+module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); -+module_param_named(flags, bbr_flags, uint, 0644); -+module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); -+module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); -+module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); -+module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); -+module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); -+module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); -+module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); -+module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); -+module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); -+module_param_named(extra_acked_win_rtts, -+ bbr_extra_acked_win_rtts, uint, 0664); -+module_param_named(extra_acked_max_us, -+ bbr_extra_acked_max_us, uint, 0664); -+module_param_named(ack_epoch_acked_reset_thresh, -+ bbr_ack_epoch_acked_reset_thresh, uint, 0664); -+module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); -+module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); -+module_param_named(extra_acked_in_startup, -+ bbr_extra_acked_in_startup, int, 0664); -+module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); -+module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); -+ -+static void bbr2_exit_probe_rtt(struct sock *sk); -+static void bbr2_reset_congestion_signals(struct sock *sk); -+ -+static void bbr_check_probe_rtt_done(struct sock *sk); -+ -+/* Do we estimate that STARTUP filled the pipe? */ -+static bool bbr_full_bw_reached(const struct sock *sk) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ -+ return bbr->full_bw_reached; -+} -+ -+/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ -+static u32 bbr_max_bw(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return max(bbr->bw_hi[0], bbr->bw_hi[1]); -+} -+ -+/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ -+static u32 bbr_bw(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return min(bbr_max_bw(sk), bbr->bw_lo); -+} -+ -+/* Return maximum extra acked in past k-2k round trips, -+ * where k = bbr_extra_acked_win_rtts. -+ */ -+static u16 bbr_extra_acked(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return max(bbr->extra_acked[0], bbr->extra_acked[1]); -+} -+ -+/* Return rate in bytes per second, optionally with a gain. -+ * The order here is chosen carefully to avoid overflow of u64. This should -+ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. -+ */ -+static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, -+ int margin) -+{ -+ unsigned int mss = tcp_sk(sk)->mss_cache; -+ -+ rate *= mss; -+ rate *= gain; -+ rate >>= BBR_SCALE; -+ rate *= USEC_PER_SEC / 100 * (100 - margin); -+ rate >>= BW_SCALE; -+ rate = max(rate, 1ULL); -+ return rate; -+} -+ -+static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) -+{ -+ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); -+} -+ -+static u64 bbr_rate_kbps(struct sock *sk, u64 rate) -+{ -+ rate = bbr_bw_bytes_per_sec(sk, rate); -+ rate *= 8; -+ do_div(rate, 1000); -+ return rate; -+} -+ -+static u32 bbr_tso_segs_goal(struct sock *sk); -+static void bbr_debug(struct sock *sk, u32 acked, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ static const char ca_states[] = { -+ [TCP_CA_Open] = 'O', -+ [TCP_CA_Disorder] = 'D', -+ [TCP_CA_CWR] = 'C', -+ [TCP_CA_Recovery] = 'R', -+ [TCP_CA_Loss] = 'L', -+ }; -+ static const char mode[] = { -+ 'G', /* Growing - BBR_STARTUP */ -+ 'D', /* Drain - BBR_DRAIN */ -+ 'W', /* Window - BBR_PROBE_BW */ -+ 'M', /* Min RTT - BBR_PROBE_RTT */ -+ }; -+ static const char ack_phase[] = { /* bbr_ack_phase strings */ -+ 'I', /* BBR_ACKS_INIT - 'Init' */ -+ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ -+ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ -+ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ -+ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ -+ }; -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ const u32 una = tp->snd_una - bbr->debug.snd_isn; -+ const u32 fack = tcp_highest_sack_seq(tp); -+ const u16 dport = ntohs(inet_sk(sk)->inet_dport); -+ bool is_port_match = (bbr_debug_port_mask && -+ ((dport & bbr_debug_port_mask) == 0)); -+ char debugmsg[320]; -+ -+ if (sk->sk_state == TCP_SYN_SENT) -+ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ -+ -+ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { -+ char addr[INET6_ADDRSTRLEN + 10] = { 0 }; -+ -+ if (sk->sk_family == AF_INET) -+ snprintf(addr, sizeof(addr), "%pI4:%u", -+ &inet_sk(sk)->inet_daddr, dport); -+ else if (sk->sk_family == AF_INET6) -+ snprintf(addr, sizeof(addr), "%pI6:%u", -+ &sk->sk_v6_daddr, dport); -+ -+ WARN_ONCE(1, -+ "BBR %s cwnd alert: %u " -+ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " -+ "bw: %u rtt: %u min_rtt: %u " -+ "acked: %u tso_segs: %u " -+ "bw: %d %ld %d pif: %u\n", -+ addr, tp->snd_cwnd, -+ una, inet_csk(sk)->icsk_ca_state, -+ bbr->pacing_gain, bbr->cwnd_gain, -+ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, -+ acked, bbr_tso_segs_goal(sk), -+ rs->delivered, rs->interval_us, rs->is_retrans, -+ tcp_packets_in_flight(tp)); -+ } -+ -+ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) -+ return; -+ -+ if (!sock_flag(sk, SOCK_DBG) && !is_port_match) -+ return; -+ -+ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) -+ return; -+ -+ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && -+ !(bbr_flags & FLAG_DEBUG_LOOPBACK)) -+ return; -+ -+ snprintf(debugmsg, sizeof(debugmsg) - 1, -+ "BBR %pI4:%-5u %5u,%03u:%-7u %c " -+ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " -+ "bw %llu lb %llu ib %llu qb %llu " -+ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " -+ "lr %d er %d ea %d bwl %lld il %d ih %d c %d " -+ "v %d %c %u %c %s\n", -+ &inet_sk(sk)->inet_daddr, dport, -+ una / 1000, una % 1000, fack - tp->snd_una, -+ ca_states[inet_csk(sk)->icsk_ca_state], -+ bbr->debug.undo ? '@' : mode[bbr->mode], -+ tp->snd_cwnd, -+ bbr_extra_acked(sk), /* br (legacy): extra_acked */ -+ rs->tx_in_flight, /* cr (legacy): tx_inflight */ -+ rs->rtt_us, -+ rs->delivered, -+ rs->interval_us, -+ bbr->min_rtt_us, -+ rs->is_app_limited ? '_' : 'l', -+ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ -+ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ -+ 0ULL, /* lb: [obsolete] */ -+ 0ULL, /* ib: [obsolete] */ -+ div_u64((u64)sk->sk_pacing_rate * 8, 1000), -+ acked, -+ tcp_packets_in_flight(tp), -+ rs->is_ack_delayed ? 'd' : '.', -+ bbr->round_start ? '*' : '.', -+ tp->delivered, tp->lost, -+ tp->app_limited, -+ 0, /* #: [obsolete] */ -+ ctx->target_cwnd, -+ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ -+ ca_states[bbr->prev_ca_state], -+ (rs->lost + rs->delivered) > 0 ? -+ (1000 * rs->lost / -+ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ -+ (rs->delivered) > 0 ? -+ (1000 * rs->delivered_ce / -+ (rs->delivered)) : 0, /* er: ECN rate x1000 */ -+ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ -+ bbr->bw_lo == ~0U ? -+ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ -+ bbr->inflight_lo, /* il */ -+ bbr->inflight_hi, /* ih */ -+ bbr->bw_probe_up_cnt, /* c */ -+ 2, /* v: version */ -+ bbr->debug.event, -+ bbr->cycle_idx, -+ ack_phase[bbr->ack_phase], -+ bbr->bw_probe_samples ? "Y" : "N"); -+ debugmsg[sizeof(debugmsg) - 1] = 0; -+ -+ /* printk takes a higher precedence. */ -+ if (bbr_debug_with_printk) -+ printk(KERN_DEBUG "%s", debugmsg); -+ -+ if (unlikely(bbr->debug.undo)) -+ bbr->debug.undo = 0; -+} -+ -+/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ -+static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) -+{ -+ u64 rate = bw; -+ -+ rate = bbr_rate_bytes_per_sec(sk, rate, gain, -+ bbr_pacing_margin_percent); -+ rate = min_t(u64, rate, sk->sk_max_pacing_rate); -+ return rate; -+} -+ -+/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ -+static void bbr_init_pacing_rate_from_rtt(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw; -+ u32 rtt_us; -+ -+ if (tp->srtt_us) { /* any RTT sample yet? */ -+ rtt_us = max(tp->srtt_us >> 3, 1U); -+ bbr->has_seen_rtt = 1; -+ } else { /* no RTT sample yet */ -+ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ -+ } -+ bw = (u64)tp->snd_cwnd * BW_UNIT; -+ do_div(bw, rtt_us); -+ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); -+} -+ -+/* Pace using current bw estimate and a gain factor. */ -+static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); -+ -+ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) -+ bbr_init_pacing_rate_from_rtt(sk); -+ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) -+ sk->sk_pacing_rate = rate; -+} -+ -+static u32 bbr_min_tso_segs(struct sock *sk) -+{ -+ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; -+} -+ -+/* Return the number of segments BBR would like in a TSO/GSO skb, given -+ * a particular max gso size as a constraint. -+ */ -+static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, -+ u32 gso_max_size) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 segs, r; -+ u64 bytes; -+ -+ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ -+ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; -+ -+ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every -+ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. -+ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) -+ */ -+ if (bbr->params.tso_rtt_shift) { -+ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; -+ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ -+ bytes += GSO_MAX_SIZE >> r; -+ } -+ -+ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); -+ return segs; -+} -+ -+/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ -+static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) -+{ -+ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); -+} -+ -+/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ -+static u32 bbr_tso_segs_goal(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ -+ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); -+} -+ -+/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -+static void bbr_save_cwnd(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) -+ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ -+ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ -+ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); -+} -+ -+static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (event == CA_EVENT_TX_START) { -+ tcp_plb_check_rehash(sk, &bbr->plb); -+ -+ if (!tp->app_limited) -+ return; -+ bbr->idle_restart = 1; -+ bbr->ack_epoch_mstamp = tp->tcp_mstamp; -+ bbr->ack_epoch_acked = 0; -+ /* Avoid pointless buffer overflows: pace at est. bw if we don't -+ * need more speed (we're restarting from idle and app-limited). -+ */ -+ if (bbr->mode == BBR_PROBE_BW) -+ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); -+ else if (bbr->mode == BBR_PROBE_RTT) -+ bbr_check_probe_rtt_done(sk); -+ } else if ((event == CA_EVENT_ECN_IS_CE || -+ event == CA_EVENT_ECN_NO_CE) && -+ bbr_ecn_enable && -+ bbr->params.precise_ece_ack) { -+ u32 state = bbr->ce_state; -+ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); -+ bbr->ce_state = state; -+ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) -+ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); -+ } -+} -+ -+/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: -+ * -+ * bdp = ceil(bw * min_rtt * gain) -+ * -+ * The key factor, gain, controls the amount of queue. While a small gain -+ * builds a smaller queue, it becomes more vulnerable to noise in RTT -+ * measurements (e.g., delayed ACKs or other ACK compression effects). This -+ * noise may cause BBR to under-estimate the rate. -+ */ -+static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 bdp; -+ u64 w; -+ -+ /* If we've never had a valid RTT sample, cap cwnd at the initial -+ * default. This should only happen when the connection is not using TCP -+ * timestamps and has retransmitted all of the SYN/SYNACK/data packets -+ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which -+ * case we need to slow-start up toward something safe: initial cwnd. -+ */ -+ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ -+ return bbr->init_cwnd; /* be safe: cap at initial cwnd */ -+ -+ w = (u64)bw * bbr->min_rtt_us; -+ -+ /* Apply a gain to the given value, remove the BW_SCALE shift, and -+ * round the value up to avoid a negative feedback loop. -+ */ -+ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; -+ -+ return bdp; -+} -+ -+/* To achieve full performance in high-speed paths, we budget enough cwnd to -+ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: -+ * - one skb in sending host Qdisc, -+ * - one skb in sending host TSO/GSO engine -+ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine -+ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because -+ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, -+ * which allows 2 outstanding 2-packet sequences, to try to keep pipe -+ * full even with ACK-every-other-packet delayed ACKs. -+ */ -+static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 tso_segs_goal; -+ -+ tso_segs_goal = 3 * bbr_tso_segs_goal(sk); -+ -+ /* Allow enough full-sized skbs in flight to utilize end systems. */ -+ if (bbr->params.cwnd_tso_budget == 1) { -+ cwnd = max_t(u32, cwnd, tso_segs_goal); -+ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); -+ } else { -+ cwnd += tso_segs_goal; -+ cwnd = (cwnd + 1) & ~1U; -+ } -+ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) -+ cwnd += 2; -+ -+ return cwnd; -+} -+ -+/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ -+static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) -+{ -+ u32 inflight; -+ -+ inflight = bbr_bdp(sk, bw, gain); -+ inflight = bbr_quantization_budget(sk, inflight); -+ -+ return inflight; -+} -+ -+/* With pacing at lower layers, there's often less data "in the network" than -+ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), -+ * we often have several skbs queued in the pacing layer with a pre-scheduled -+ * earliest departure time (EDT). BBR adapts its pacing rate based on the -+ * inflight level that it estimates has already been "baked in" by previous -+ * departure time decisions. We calculate a rough estimate of the number of our -+ * packets that might be in the network at the earliest departure time for the -+ * next skb scheduled: -+ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw -+ * If we're increasing inflight, then we want to know if the transmit of the -+ * EDT skb will push inflight above the target, so inflight_at_edt includes -+ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, -+ * then estimate if inflight will sink too low just before the EDT transmit. -+ */ -+static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 now_ns, edt_ns, interval_us; -+ u32 interval_delivered, inflight_at_edt; -+ -+ now_ns = tp->tcp_clock_cache; -+ edt_ns = max(tp->tcp_wstamp_ns, now_ns); -+ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); -+ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; -+ inflight_at_edt = inflight_now; -+ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ -+ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ -+ if (interval_delivered >= inflight_at_edt) -+ return 0; -+ return inflight_at_edt - interval_delivered; -+} -+ -+/* Find the cwnd increment based on estimate of ack aggregation */ -+static u32 bbr_ack_aggregation_cwnd(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 max_aggr_cwnd, aggr_cwnd = 0; -+ -+ if (bbr->params.extra_acked_gain && -+ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { -+ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) -+ / BW_UNIT; -+ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) -+ >> BBR_SCALE; -+ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); -+ } -+ -+ return aggr_cwnd; -+} -+ -+/* Returns the cwnd for PROBE_RTT mode. */ -+static u32 bbr_probe_rtt_cwnd(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->params.probe_rtt_cwnd_gain == 0) -+ return bbr->params.cwnd_min_target; -+ return max_t(u32, bbr->params.cwnd_min_target, -+ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); -+} -+ -+/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss -+ * has drawn us down below target), or snap down to target if we're above it. -+ */ -+static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, -+ u32 acked, u32 bw, int gain, u32 cwnd, -+ struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; -+ -+ if (!acked) -+ goto done; /* no packet fully ACKed; just apply caps */ -+ -+ target_cwnd = bbr_bdp(sk, bw, gain); -+ -+ /* Increment the cwnd to account for excess ACKed data that seems -+ * due to aggregation (of data and/or ACKs) visible in the ACK stream. -+ */ -+ target_cwnd += bbr_ack_aggregation_cwnd(sk); -+ target_cwnd = bbr_quantization_budget(sk, target_cwnd); -+ -+ /* If we're below target cwnd, slow start cwnd toward target cwnd. */ -+ bbr->debug.target_cwnd = target_cwnd; -+ -+ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ -+ bbr->try_fast_path = 0; -+ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ -+ cwnd += acked; -+ if (cwnd >= target_cwnd) { -+ cwnd = target_cwnd; -+ bbr->try_fast_path = 1; -+ } -+ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { -+ cwnd += acked; -+ } else { -+ bbr->try_fast_path = 1; -+ } -+ -+ /* When growing cwnd, don't grow beyond twice what we just probed. */ -+ if (bbr->params.usage_based_cwnd) { -+ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); -+ cwnd = min(cwnd, max_probe); -+ } -+ -+ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); -+done: -+ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ -+ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ -+ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); -+ -+ ctx->target_cwnd = target_cwnd; -+ ctx->log = (tp->snd_cwnd != prev_cwnd); -+} -+ -+/* See if we have reached next round trip */ -+static void bbr_update_round_start(struct sock *sk, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->round_start = 0; -+ -+ /* See if we've reached the next RTT */ -+ if (rs->interval_us > 0 && -+ !before(rs->prior_delivered, bbr->next_rtt_delivered)) { -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr->round_start = 1; -+ } -+} -+ -+/* Calculate the bandwidth based on how fast packets are delivered */ -+static void bbr_calculate_bw_sample(struct sock *sk, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw = 0; -+ -+ /* Divide delivered by the interval to find a (lower bound) bottleneck -+ * bandwidth sample. Delivered is in packets and interval_us in uS and -+ * ratio will be <<1 for most connections. So delivered is first scaled. -+ * Round up to allow growth at low rates, even with integer division. -+ */ -+ if (rs->interval_us > 0) { -+ if (WARN_ONCE(rs->delivered < 0, -+ "negative delivered: %d interval_us: %ld\n", -+ rs->delivered, rs->interval_us)) -+ return; -+ -+ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); -+ } -+ -+ ctx->sample_bw = bw; -+ bbr->debug.rs_bw = bw; -+} -+ -+/* Estimates the windowed max degree of ack aggregation. -+ * This is used to provision extra in-flight data to keep sending during -+ * inter-ACK silences. -+ * -+ * Degree of ack aggregation is estimated as extra data acked beyond expected. -+ * -+ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" -+ * cwnd += max_extra_acked -+ * -+ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). -+ * Max filter is an approximate sliding window of 5-10 (packet timed) round -+ * trips for non-startup phase, and 1-2 round trips for startup. -+ */ -+static void bbr_update_ack_aggregation(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ u32 epoch_us, expected_acked, extra_acked; -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct tcp_sock *tp = tcp_sk(sk); -+ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; -+ -+ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || -+ rs->delivered < 0 || rs->interval_us <= 0) -+ return; -+ -+ if (bbr->round_start) { -+ bbr->extra_acked_win_rtts = min(0x1F, -+ bbr->extra_acked_win_rtts + 1); -+ if (bbr->params.extra_acked_in_startup && -+ !bbr_full_bw_reached(sk)) -+ extra_acked_win_rtts_thresh = 1; -+ if (bbr->extra_acked_win_rtts >= -+ extra_acked_win_rtts_thresh) { -+ bbr->extra_acked_win_rtts = 0; -+ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? -+ 0 : 1; -+ bbr->extra_acked[bbr->extra_acked_win_idx] = 0; -+ } -+ } -+ -+ /* Compute how many packets we expected to be delivered over epoch. */ -+ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, -+ bbr->ack_epoch_mstamp); -+ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; -+ -+ /* Reset the aggregation epoch if ACK rate is below expected rate or -+ * significantly large no. of ack received since epoch (potentially -+ * quite old epoch). -+ */ -+ if (bbr->ack_epoch_acked <= expected_acked || -+ (bbr->ack_epoch_acked + rs->acked_sacked >= -+ bbr_ack_epoch_acked_reset_thresh)) { -+ bbr->ack_epoch_acked = 0; -+ bbr->ack_epoch_mstamp = tp->delivered_mstamp; -+ expected_acked = 0; -+ } -+ -+ /* Compute excess data delivered, beyond what was expected. */ -+ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, -+ bbr->ack_epoch_acked + rs->acked_sacked); -+ extra_acked = bbr->ack_epoch_acked - expected_acked; -+ extra_acked = min(extra_acked, tp->snd_cwnd); -+ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) -+ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; -+} -+ -+/* Estimate when the pipe is full, using the change in delivery rate: BBR -+ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by -+ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited -+ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the -+ * higher rwin, 3: we get higher delivery rate samples. Or transient -+ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar -+ * design goal, but uses delay and inter-ACK spacing instead of bandwidth. -+ */ -+static void bbr_check_full_bw_reached(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 bw_thresh; -+ -+ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) -+ return; -+ -+ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; -+ if (bbr_max_bw(sk) >= bw_thresh) { -+ bbr->full_bw = bbr_max_bw(sk); -+ bbr->full_bw_cnt = 0; -+ return; -+ } -+ ++bbr->full_bw_cnt; -+ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; -+} -+ -+/* If pipe is probably full, drain the queue and then enter steady-state. */ -+static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { -+ bbr->mode = BBR_DRAIN; /* drain queue we created */ -+ tcp_sk(sk)->snd_ssthresh = -+ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); -+ bbr2_reset_congestion_signals(sk); -+ } /* fall through to check if in-flight is already small: */ -+ if (bbr->mode == BBR_DRAIN && -+ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= -+ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) -+ return true; /* exiting DRAIN now */ -+ return false; -+} -+ -+static void bbr_check_probe_rtt_done(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (!(bbr->probe_rtt_done_stamp && -+ after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) -+ return; -+ -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr2_exit_probe_rtt(sk); -+} -+ -+/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and -+ * periodically drain the bottleneck queue, to converge to measure the true -+ * min_rtt (unloaded propagation delay). This allows the flows to keep queues -+ * small (reducing queuing delay and packet loss) and achieve fairness among -+ * BBR flows. -+ * -+ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, -+ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. -+ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed -+ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and -+ * re-enter the previous mode. BBR uses 200ms to approximately bound the -+ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). -+ * -+ * Note that flows need only pay 2% if they are busy sending over the last 10 -+ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have -+ * natural silences or low-rate periods within 10 seconds where the rate is low -+ * enough for long enough to drain its queue in the bottleneck. We pick up -+ * these min RTT measurements opportunistically with our min_rtt filter. :-) -+ */ -+static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool probe_rtt_expired, min_rtt_expired; -+ u32 expire; -+ -+ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ -+ expire = bbr->probe_rtt_min_stamp + -+ msecs_to_jiffies(bbr->params.probe_rtt_win_ms); -+ probe_rtt_expired = after(tcp_jiffies32, expire); -+ if (rs->rtt_us >= 0 && -+ (rs->rtt_us <= bbr->probe_rtt_min_us || -+ (probe_rtt_expired && !rs->is_ack_delayed))) { -+ bbr->probe_rtt_min_us = rs->rtt_us; -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; -+ } -+ /* Track min RTT seen in the min_rtt_win_sec filter window: */ -+ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; -+ min_rtt_expired = after(tcp_jiffies32, expire); -+ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || -+ min_rtt_expired) { -+ bbr->min_rtt_us = bbr->probe_rtt_min_us; -+ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; -+ } -+ -+ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && -+ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { -+ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ -+ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ -+ bbr->probe_rtt_done_stamp = 0; -+ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; -+ bbr->next_rtt_delivered = tp->delivered; -+ } -+ -+ if (bbr->mode == BBR_PROBE_RTT) { -+ /* Ignore low rate samples during this mode. */ -+ tp->app_limited = -+ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; -+ /* Maintain min packets in flight for max(200 ms, 1 round). */ -+ if (!bbr->probe_rtt_done_stamp && -+ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { -+ bbr->probe_rtt_done_stamp = tcp_jiffies32 + -+ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); -+ bbr->probe_rtt_round_done = 0; -+ bbr->next_rtt_delivered = tp->delivered; -+ } else if (bbr->probe_rtt_done_stamp) { -+ if (bbr->round_start) -+ bbr->probe_rtt_round_done = 1; -+ if (bbr->probe_rtt_round_done) -+ bbr_check_probe_rtt_done(sk); -+ } -+ } -+ /* Restart after idle ends only once we process a new S/ACK for data */ -+ if (rs->delivered > 0) -+ bbr->idle_restart = 0; -+} -+ -+static void bbr_update_gains(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ bbr->pacing_gain = bbr->params.high_gain; -+ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; -+ break; -+ case BBR_DRAIN: -+ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ -+ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ -+ break; -+ case BBR_PROBE_BW: -+ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; -+ bbr->cwnd_gain = bbr->params.cwnd_gain; -+ break; -+ case BBR_PROBE_RTT: -+ bbr->pacing_gain = BBR_UNIT; -+ bbr->cwnd_gain = BBR_UNIT; -+ break; -+ default: -+ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); -+ break; -+ } -+} -+ -+static void bbr_init(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ int i; -+ -+ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); -+ -+ bbr->initialized = 1; -+ bbr->params.high_gain = min(0x7FF, bbr_high_gain); -+ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); -+ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); -+ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); -+ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); -+ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); -+ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); -+ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); -+ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); -+ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); -+ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); -+ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); -+ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; -+ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; -+ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; -+ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); -+ bbr->params.probe_rtt_win_ms = -+ min(0x3FFFU, -+ min_t(u32, bbr_probe_rtt_win_ms, -+ bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); -+ for (i = 0; i < CYCLE_LEN; i++) -+ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); -+ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; -+ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); -+ -+ bbr->debug.snd_isn = tp->snd_una; -+ bbr->debug.target_cwnd = 0; -+ bbr->debug.undo = 0; -+ -+ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); -+ bbr->prior_cwnd = tp->prior_cwnd; -+ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; -+ bbr->next_rtt_delivered = 0; -+ bbr->prev_ca_state = TCP_CA_Open; -+ bbr->packet_conservation = 0; -+ -+ bbr->probe_rtt_done_stamp = 0; -+ bbr->probe_rtt_round_done = 0; -+ bbr->probe_rtt_min_us = tcp_min_rtt(tp); -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; -+ bbr->min_rtt_us = tcp_min_rtt(tp); -+ bbr->min_rtt_stamp = tcp_jiffies32; -+ -+ bbr->has_seen_rtt = 0; -+ bbr_init_pacing_rate_from_rtt(sk); -+ -+ bbr->round_start = 0; -+ bbr->idle_restart = 0; -+ bbr->full_bw_reached = 0; -+ bbr->full_bw = 0; -+ bbr->full_bw_cnt = 0; -+ bbr->cycle_mstamp = 0; -+ bbr->cycle_idx = 0; -+ bbr->mode = BBR_STARTUP; -+ bbr->debug.rs_bw = 0; -+ -+ bbr->ack_epoch_mstamp = tp->tcp_mstamp; -+ bbr->ack_epoch_acked = 0; -+ bbr->extra_acked_win_rtts = 0; -+ bbr->extra_acked_win_idx = 0; -+ bbr->extra_acked[0] = 0; -+ bbr->extra_acked[1] = 0; -+ -+ bbr->ce_state = 0; -+ bbr->prior_rcv_nxt = tp->rcv_nxt; -+ bbr->try_fast_path = 0; -+ -+ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); -+} -+ -+static u32 bbr_sndbuf_expand(struct sock *sk) -+{ -+ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ -+ return 3; -+} -+ -+/* __________________________________________________________________________ -+ * -+ * Functions new to BBR v2 ("bbr") congestion control are below here. -+ * __________________________________________________________________________ -+ */ -+ -+/* Incorporate a new bw sample into the current window of our max filter. */ -+static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); -+} -+ -+/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ -+static void bbr2_advance_bw_hi_filter(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (!bbr->bw_hi[1]) -+ return; /* no samples in this window; remember old window */ -+ bbr->bw_hi[0] = bbr->bw_hi[1]; -+ bbr->bw_hi[1] = 0; -+} -+ -+/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ -+static u32 bbr2_target_inflight(struct sock *sk) -+{ -+ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); -+ -+ return min(bdp, tcp_sk(sk)->snd_cwnd); -+} -+ -+static bool bbr2_is_probing_bandwidth(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ return (bbr->mode == BBR_STARTUP) || -+ (bbr->mode == BBR_PROBE_BW && -+ (bbr->cycle_idx == BBR_BW_PROBE_REFILL || -+ bbr->cycle_idx == BBR_BW_PROBE_UP)); -+} -+ -+/* Has the given amount of time elapsed since we marked the phase start? */ -+static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) -+{ -+ const struct tcp_sock *tp = tcp_sk(sk); -+ const struct bbr *bbr = inet_csk_ca(sk); -+ -+ return tcp_stamp_us_delta(tp->tcp_mstamp, -+ bbr->cycle_mstamp + interval_us) > 0; -+} -+ -+static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->full_bw_reached = 1; -+ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); -+} -+ -+/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ -+static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || -+ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) -+ return; -+ -+ if (ce_ratio >= bbr->params.ecn_thresh) -+ bbr->startup_ecn_rounds++; -+ else -+ bbr->startup_ecn_rounds = 0; -+ -+ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { -+ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ -+ bbr2_handle_queue_too_high_in_startup(sk); -+ return; -+ } -+} -+ -+static int bbr2_update_ecn_alpha(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ s32 delivered, delivered_ce; -+ u64 alpha, ce_ratio; -+ u32 gain; -+ -+ if (bbr->params.ecn_factor == 0) -+ return -1; -+ -+ delivered = tp->delivered - bbr->alpha_last_delivered; -+ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; -+ -+ if (delivered == 0 || /* avoid divide by zero */ -+ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ -+ return -1; -+ -+ /* See if we should use ECN sender logic for this connection. */ -+ if (!bbr->ecn_eligible && bbr_ecn_enable && -+ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || -+ !bbr->params.ecn_max_rtt_us)) -+ bbr->ecn_eligible = 1; -+ -+ ce_ratio = (u64)delivered_ce << BBR_SCALE; -+ do_div(ce_ratio, delivered); -+ gain = bbr->params.ecn_alpha_gain; -+ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; -+ alpha += (gain * ce_ratio) >> BBR_SCALE; -+ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); -+ -+ bbr->alpha_last_delivered = tp->delivered; -+ bbr->alpha_last_delivered_ce = tp->delivered_ce; -+ -+ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); -+ return (int)ce_ratio; -+} -+ -+/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ -+static void bbr2_raise_inflight_hi_slope(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 growth_this_round, cnt; -+ -+ /* Calculate "slope": packets S/Acked per inflight_hi increment. */ -+ growth_this_round = 1 << bbr->bw_probe_up_rounds; -+ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); -+ cnt = tp->snd_cwnd / growth_this_round; -+ cnt = max(cnt, 1U); -+ bbr->bw_probe_up_cnt = cnt; -+ bbr->debug.event = 'G'; /* Grow inflight_hi slope */ -+} -+ -+/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ -+static void bbr2_probe_inflight_hi_upward(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 delta; -+ -+ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { -+ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ -+ return; /* not fully using inflight_hi, so don't grow it */ -+ } -+ -+ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ -+ bbr->bw_probe_up_acks += rs->acked_sacked; -+ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { -+ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; -+ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; -+ bbr->inflight_hi += delta; -+ bbr->debug.event = 'I'; /* Increment inflight_hi */ -+ } -+ -+ if (bbr->round_start) -+ bbr2_raise_inflight_hi_slope(sk); -+} -+ -+/* Does loss/ECN rate for this sample say inflight is "too high"? -+ * This is used by both the bbr_check_loss_too_high_in_startup() function, -+ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which -+ * uses it to notice when loss/ECN rates suggest inflight is too high. -+ */ -+static bool bbr2_is_inflight_too_high(const struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ u32 loss_thresh, ecn_thresh; -+ -+ if (rs->lost > 0 && rs->tx_in_flight) { -+ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> -+ BBR_SCALE; -+ if (rs->lost > loss_thresh) -+ return true; -+ } -+ -+ if (rs->delivered_ce > 0 && rs->delivered > 0 && -+ bbr->ecn_eligible && bbr->params.ecn_thresh) { -+ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> -+ BBR_SCALE; -+ if (rs->delivered_ce >= ecn_thresh) -+ return true; -+ } -+ -+ return false; -+} -+ -+/* Calculate the tx_in_flight level that corresponded to excessive loss. -+ * We find "lost_prefix" segs of the skb where loss rate went too high, -+ * by solving for "lost_prefix" in the following equation: -+ * lost / inflight >= loss_thresh -+ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh -+ * Then we take that equation, convert it to fixed point, and -+ * round up to the nearest packet. -+ */ -+static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, -+ const struct rate_sample *rs, -+ const struct sk_buff *skb) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ u32 loss_thresh = bbr->params.loss_thresh; -+ u32 pcount, divisor, inflight_hi; -+ s32 inflight_prev, lost_prev; -+ u64 loss_budget, lost_prefix; -+ -+ pcount = tcp_skb_pcount(skb); -+ -+ /* How much data was in flight before this skb? */ -+ inflight_prev = rs->tx_in_flight - pcount; -+ if (WARN_ONCE(inflight_prev < 0, -+ "tx_in_flight: %u pcount: %u reneg: %u", -+ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) -+ return ~0U; -+ -+ /* How much inflight data was marked lost before this skb? */ -+ lost_prev = rs->lost - pcount; -+ if (WARN_ON_ONCE(lost_prev < 0)) -+ return ~0U; -+ -+ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ -+ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; -+ loss_budget >>= BBR_SCALE; -+ if (lost_prev >= loss_budget) { -+ lost_prefix = 0; /* previous losses crossed loss_thresh */ -+ } else { -+ lost_prefix = loss_budget - lost_prev; -+ lost_prefix <<= BBR_SCALE; -+ divisor = BBR_UNIT - loss_thresh; -+ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ -+ return ~0U; -+ do_div(lost_prefix, divisor); -+ } -+ -+ inflight_hi = inflight_prev + lost_prefix; -+ return inflight_hi; -+} -+ -+/* If loss/ECN rates during probing indicated we may have overfilled a -+ * buffer, return an operating point that tries to leave unutilized headroom in -+ * the path for other flows, for fairness convergence and lower RTTs and loss. -+ */ -+static u32 bbr2_inflight_with_headroom(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 headroom, headroom_fraction; -+ -+ if (bbr->inflight_hi == ~0U) -+ return ~0U; -+ -+ headroom_fraction = bbr->params.inflight_headroom; -+ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; -+ headroom = max(headroom, 1U); -+ return max_t(s32, bbr->inflight_hi - headroom, -+ bbr->params.cwnd_min_target); -+} -+ -+/* Bound cwnd to a sensible level, based on our current probing state -+ * machine phase and model of a good inflight level (inflight_lo, inflight_hi). -+ */ -+static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 cap; -+ -+ /* tcp_rcv_synsent_state_process() currently calls tcp_ack() -+ * and thus cong_control() without first initializing us(!). -+ */ -+ if (!bbr->initialized) -+ return; -+ -+ cap = ~0U; -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { -+ /* Probe to see if more packets fit in the path. */ -+ cap = bbr->inflight_hi; -+ } else { -+ if (bbr->mode == BBR_PROBE_RTT || -+ (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) -+ cap = bbr2_inflight_with_headroom(sk); -+ } -+ /* Adapt to any loss/ECN since our last bw probe. */ -+ cap = min(cap, bbr->inflight_lo); -+ -+ cap = max_t(u32, cap, bbr->params.cwnd_min_target); -+ tp->snd_cwnd = min(cap, tp->snd_cwnd); -+} -+ -+/* Estimate a short-term lower bound on the capacity available now, based -+ * on measurements of the current delivery process and recent history. When we -+ * are seeing loss/ECN at times when we are not probing bw, then conservatively -+ * move toward flow balance by multiplicatively cutting our short-term -+ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a -+ * multiplicative decrease in order to converge to a lower capacity in time -+ * logarithmic in the magnitude of the decrease. -+ * -+ * However, we do not cut our short-term estimates lower than the current rate -+ * and volume of delivered data from this round trip, since from the current -+ * delivery process we can estimate the measured capacity available now. -+ * -+ * Anything faster than that approach would knowingly risk high loss, which can -+ * cause low bw for Reno/CUBIC and high loss recovery latency for -+ * request/response flows using any congestion control. -+ */ -+static void bbr2_adapt_lower_bounds(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 ecn_cut, ecn_inflight_lo, beta; -+ -+ /* We only use lower-bound estimates when not probing bw. -+ * When probing we need to push inflight higher to probe bw. -+ */ -+ if (bbr2_is_probing_bandwidth(sk)) -+ return; -+ -+ /* ECN response. */ -+ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { -+ /* Reduce inflight to (1 - alpha*ecn_factor). */ -+ ecn_cut = (BBR_UNIT - -+ ((bbr->ecn_alpha * bbr->params.ecn_factor) >> -+ BBR_SCALE)); -+ if (bbr->inflight_lo == ~0U) -+ bbr->inflight_lo = tp->snd_cwnd; -+ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; -+ } else { -+ ecn_inflight_lo = ~0U; -+ } -+ -+ /* Loss response. */ -+ if (bbr->loss_in_round) { -+ /* Reduce bw and inflight to (1 - beta). */ -+ if (bbr->bw_lo == ~0U) -+ bbr->bw_lo = bbr_max_bw(sk); -+ if (bbr->inflight_lo == ~0U) -+ bbr->inflight_lo = tp->snd_cwnd; -+ beta = bbr->params.beta; -+ bbr->bw_lo = -+ max_t(u32, bbr->bw_latest, -+ (u64)bbr->bw_lo * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ bbr->inflight_lo = -+ max_t(u32, bbr->inflight_latest, -+ (u64)bbr->inflight_lo * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ } -+ -+ /* Adjust to the lower of the levels implied by loss or ECN. */ -+ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); -+} -+ -+/* Reset any short-term lower-bound adaptation to congestion, so that we can -+ * push our inflight up. -+ */ -+static void bbr2_reset_lower_bounds(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->bw_lo = ~0U; -+ bbr->inflight_lo = ~0U; -+} -+ -+/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state -+ * machine phase where we adapt our lower bound based on congestion signals. -+ */ -+static void bbr2_reset_congestion_signals(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->loss_in_round = 0; -+ bbr->ecn_in_round = 0; -+ bbr->loss_in_cycle = 0; -+ bbr->ecn_in_cycle = 0; -+ bbr->bw_latest = 0; -+ bbr->inflight_latest = 0; -+} -+ -+/* Update (most of) our congestion signals: track the recent rate and volume of -+ * delivered data, presence of loss, and EWMA degree of ECN marking. -+ */ -+static void bbr2_update_congestion_signals( -+ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw; -+ -+ bbr->loss_round_start = 0; -+ if (rs->interval_us <= 0 || !rs->acked_sacked) -+ return; /* Not a valid observation */ -+ bw = ctx->sample_bw; -+ -+ if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) -+ bbr2_take_bw_hi_sample(sk, bw); -+ -+ bbr->loss_in_round |= (rs->losses > 0); -+ -+ /* Update rate and volume of delivered data from latest round trip: */ -+ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); -+ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); -+ -+ if (before(rs->prior_delivered, bbr->loss_round_delivered)) -+ return; /* skip the per-round-trip updates */ -+ /* Now do per-round-trip updates. */ -+ bbr->loss_round_delivered = tp->delivered; /* mark round trip */ -+ bbr->loss_round_start = 1; -+ bbr2_adapt_lower_bounds(sk); -+ -+ /* Update windowed "latest" (single-round-trip) filters. */ -+ bbr->loss_in_round = 0; -+ bbr->ecn_in_round = 0; -+ bbr->bw_latest = ctx->sample_bw; -+ bbr->inflight_latest = rs->delivered; -+} -+ -+/* Bandwidth probing can cause loss. To help coexistence with loss-based -+ * congestion control we spread out our probing in a Reno-conscious way. Due to -+ * the shape of the Reno sawtooth, the time required between loss epochs for an -+ * idealized Reno flow is a number of round trips that is the BDP of that -+ * flow. We count packet-timed round trips directly, since measured RTT can -+ * vary widely, and Reno is driven by packet-timed round trips. -+ */ -+static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 inflight, rounds, reno_gain, reno_rounds; -+ -+ /* Random loss can shave some small percentage off of our inflight -+ * in each round. To survive this, flows need robust periodic probes. -+ */ -+ rounds = bbr->params.bw_probe_max_rounds; -+ -+ reno_gain = bbr->params.bw_probe_reno_gain; -+ if (reno_gain) { -+ inflight = bbr2_target_inflight(sk); -+ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; -+ rounds = min(rounds, reno_rounds); -+ } -+ return bbr->rounds_since_probe >= rounds; -+} -+ -+/* How long do we want to wait before probing for bandwidth (and risking -+ * loss)? We randomize the wait, for better mixing and fairness convergence. -+ * -+ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. -+ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, -+ * (eg 4K video to a broadband user): -+ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets -+ * -+ * We bound the BBR-native inter-bw-probe wall clock time to be: -+ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time -+ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must -+ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs -+ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable -+ * amount of time to discover unutilized bw on human-scale interactive -+ * time-scales (e.g. perhaps traffic from a web page download that we -+ * were competing with is now complete). -+ */ -+static void bbr2_pick_probe_wait(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ /* Decide the random round-trip bound for wait until probe: */ -+ bbr->rounds_since_probe = -+ prandom_u32_max(bbr->params.bw_probe_rand_rounds); -+ /* Decide the random wall clock bound for wait until probe: */ -+ bbr->probe_wait_us = bbr->params.bw_probe_base_us + -+ prandom_u32_max(bbr->params.bw_probe_rand_us); -+} -+ -+static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->cycle_idx = cycle_idx; -+ /* New phase, so need to update cwnd and pacing rate. */ -+ bbr->try_fast_path = 0; -+} -+ -+/* Send at estimated bw to fill the pipe, but not queue. We need this phase -+ * before PROBE_UP, because as soon as we send faster than the available bw -+ * we will start building a queue, and if the buffer is shallow we can cause -+ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and -+ * inflight_hi estimates will underestimate. -+ */ -+static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr2_reset_lower_bounds(sk); -+ if (bbr->inflight_hi != ~0U) -+ bbr->inflight_hi += bbr->params.refill_add_inc; -+ bbr->bw_probe_up_rounds = bw_probe_up_rounds; -+ bbr->bw_probe_up_acks = 0; -+ bbr->stopped_risky_probe = 0; -+ bbr->ack_phase = BBR_ACKS_REFILLING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); -+} -+ -+/* Now probe max deliverable data rate and volume. */ -+static void bbr2_start_bw_probe_up(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->ack_phase = BBR_ACKS_PROBE_STARTING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr->cycle_mstamp = tp->tcp_mstamp; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); -+ bbr2_raise_inflight_hi_slope(sk); -+} -+ -+/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall -+ * clock time at which to probe beyond an inflight that we think to be -+ * safe. This will knowingly risk packet loss, so we want to do this rarely, to -+ * keep packet loss rates low. Also start a round-trip counter, to probe faster -+ * if we estimate a Reno flow at our BDP would probe faster. -+ */ -+static void bbr2_start_bw_probe_down(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr2_reset_congestion_signals(sk); -+ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ -+ bbr2_pick_probe_wait(sk); -+ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ -+ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); -+} -+ -+/* Cruise: maintain what we estimate to be a neutral, conservative -+ * operating point, without attempting to probe up for bandwidth or down for -+ * RTT, and only reducing inflight in response to loss/ECN signals. -+ */ -+static void bbr2_start_bw_probe_cruise(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr->inflight_lo != ~0U) -+ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); -+ -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); -+} -+ -+/* Loss and/or ECN rate is too high while probing. -+ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. -+ */ -+static void bbr2_handle_inflight_too_high(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ const u32 beta = bbr->params.beta; -+ -+ bbr->prev_probe_too_high = 1; -+ bbr->bw_probe_samples = 0; /* only react once per probe */ -+ bbr->debug.event = 'L'; /* Loss/ECN too high */ -+ /* If we are app-limited then we are not robustly -+ * probing the max volume of inflight data we think -+ * might be safe (analogous to how app-limited bw -+ * samples are not known to be robustly probing bw). -+ */ -+ if (!rs->is_app_limited) -+ bbr->inflight_hi = max_t(u32, rs->tx_in_flight, -+ (u64)bbr2_target_inflight(sk) * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) -+ bbr2_start_bw_probe_down(sk); -+} -+ -+/* If we're seeing bw and loss samples reflecting our bw probing, adapt -+ * using the signals we see. If loss or ECN mark rate gets too high, then adapt -+ * inflight_hi downward. If we're able to push inflight higher without such -+ * signals, push higher: adapt inflight_hi upward. -+ */ -+static bool bbr2_adapt_upper_bounds(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ /* Track when we'll see bw/loss samples resulting from our bw probes. */ -+ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) -+ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; -+ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { -+ /* End of samples from bw probing phase. */ -+ bbr->bw_probe_samples = 0; -+ bbr->ack_phase = BBR_ACKS_INIT; -+ /* At this point in the cycle, our current bw sample is also -+ * our best recent chance at finding the highest available bw -+ * for this flow. So now is the best time to forget the bw -+ * samples from the previous cycle, by advancing the window. -+ */ -+ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) -+ bbr2_advance_bw_hi_filter(sk); -+ /* If we had an inflight_hi, then probed and pushed inflight all -+ * the way up to hit that inflight_hi without seeing any -+ * high loss/ECN in all the resulting ACKs from that probing, -+ * then probe up again, this time letting inflight persist at -+ * inflight_hi for a round trip, then accelerating beyond. -+ */ -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { -+ bbr->debug.event = 'R'; /* reprobe */ -+ bbr2_start_bw_probe_refill(sk, 0); -+ return true; /* yes, decided state transition */ -+ } -+ } -+ -+ if (bbr2_is_inflight_too_high(sk, rs)) { -+ if (bbr->bw_probe_samples) /* sample is from bw probing? */ -+ bbr2_handle_inflight_too_high(sk, rs); -+ } else { -+ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ -+ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ -+ return false; -+ -+ /* To be resilient to random loss, we must raise inflight_hi -+ * if we observe in any phase that a higher level is safe. -+ */ -+ if (rs->tx_in_flight > bbr->inflight_hi) { -+ bbr->inflight_hi = rs->tx_in_flight; -+ bbr->debug.event = 'U'; /* raise up inflight_hi */ -+ } -+ -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx == BBR_BW_PROBE_UP) -+ bbr2_probe_inflight_hi_upward(sk, rs); -+ } -+ -+ return false; -+} -+ -+/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ -+static bool bbr2_check_time_to_probe_bw(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 n; -+ -+ /* If we seem to be at an operating point where we are not seeing loss -+ * but we are seeing ECN marks, then when the ECN marks cease we reprobe -+ * quickly (in case a burst of cross-traffic has ceased and freed up bw, -+ * or in case we are sharing with multiplicatively probing traffic). -+ */ -+ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && -+ bbr->ecn_in_cycle && !bbr->loss_in_cycle && -+ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { -+ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ -+ /* Calculate n so that when bbr2_raise_inflight_hi_slope() -+ * computes growth_this_round as 2^n it will be roughly the -+ * desired volume of data (inflight_hi*ecn_reprobe_gain). -+ */ -+ n = ilog2((((u64)bbr->inflight_hi * -+ bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); -+ bbr2_start_bw_probe_refill(sk, n); -+ return true; -+ } -+ -+ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || -+ bbr2_is_reno_coexistence_probe_time(sk)) { -+ bbr2_start_bw_probe_refill(sk, 0); -+ return true; -+ } -+ return false; -+} -+ -+/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ -+static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool is_under_bdp, is_long_enough; -+ -+ /* Always need to pull inflight down to leave headroom in queue. */ -+ if (inflight > bbr2_inflight_with_headroom(sk)) -+ return false; -+ -+ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); -+ if (bbr->params.drain_to_target) -+ return is_under_bdp; -+ -+ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); -+ return is_under_bdp || is_long_enough; -+} -+ -+/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ -+static void bbr2_update_cycle_phase(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool is_risky = false, is_queuing = false; -+ u32 inflight, bw; -+ -+ if (!bbr_full_bw_reached(sk)) -+ return; -+ -+ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ -+ if (bbr2_adapt_upper_bounds(sk, rs)) -+ return; /* already decided state transition */ -+ -+ if (bbr->mode != BBR_PROBE_BW) -+ return; -+ -+ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); -+ bw = bbr_max_bw(sk); -+ -+ switch (bbr->cycle_idx) { -+ /* First we spend most of our time cruising with a pacing_gain of 1.0, -+ * which paces at the estimated bw, to try to fully use the pipe -+ * without building queue. If we encounter loss/ECN marks, we adapt -+ * by slowing down. -+ */ -+ case BBR_BW_PROBE_CRUISE: -+ if (bbr2_check_time_to_probe_bw(sk)) -+ return; /* already decided state transition */ -+ break; -+ -+ /* After cruising, when it's time to probe, we first "refill": we send -+ * at the estimated bw to fill the pipe, before probing higher and -+ * knowingly risking overflowing the bottleneck buffer (causing loss). -+ */ -+ case BBR_BW_PROBE_REFILL: -+ if (bbr->round_start) { -+ /* After one full round trip of sending in REFILL, we -+ * start to see bw samples reflecting our REFILL, which -+ * may be putting too much data in flight. -+ */ -+ bbr->bw_probe_samples = 1; -+ bbr2_start_bw_probe_up(sk); -+ } -+ break; -+ -+ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to -+ * probe for bw. If we have not seen loss/ECN, we try to raise inflight -+ * to at least pacing_gain*BDP; note that this may take more than -+ * min_rtt if min_rtt is small (e.g. on a LAN). -+ * -+ * We terminate PROBE_UP bandwidth probing upon any of the following: -+ * -+ * (1) We've pushed inflight up to hit the inflight_hi target set in the -+ * most recent previous bw probe phase. Thus we want to start -+ * draining the queue immediately because it's very likely the most -+ * recently sent packets will fill the queue and cause drops. -+ * (checked here) -+ * (2) We have probed for at least 1*min_rtt_us, and the -+ * estimated queue is high enough (inflight > 1.25 * estimated_bdp). -+ * (checked here) -+ * (3) Loss filter says loss rate is "too high". -+ * (checked in bbr_is_inflight_too_high()) -+ * (4) ECN filter says ECN mark rate is "too high". -+ * (checked in bbr_is_inflight_too_high()) -+ */ -+ case BBR_BW_PROBE_UP: -+ if (bbr->prev_probe_too_high && -+ inflight >= bbr->inflight_hi) { -+ bbr->stopped_risky_probe = 1; -+ is_risky = true; -+ bbr->debug.event = 'D'; /* D for danger */ -+ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && -+ inflight >= -+ bbr_inflight(sk, bw, -+ bbr->params.bw_probe_pif_gain)) { -+ is_queuing = true; -+ bbr->debug.event = 'Q'; /* building Queue */ -+ } -+ if (is_risky || is_queuing) { -+ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ -+ bbr2_start_bw_probe_down(sk); /* restart w/ down */ -+ } -+ break; -+ -+ /* After probing in PROBE_UP, we have usually accumulated some data in -+ * the bottleneck buffer (if bw probing didn't find more bw). We next -+ * enter PROBE_DOWN to try to drain any excess data from the queue. To -+ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until -+ * our inflight is less then that target cruising point, which is the -+ * minimum of (a) the amount needed to leave headroom, and (b) the -+ * estimated BDP. Once inflight falls to match the target, we estimate -+ * the queue is drained; persisting would underutilize the pipe. -+ */ -+ case BBR_BW_PROBE_DOWN: -+ if (bbr2_check_time_to_probe_bw(sk)) -+ return; /* already decided state transition */ -+ if (bbr2_check_time_to_cruise(sk, inflight, bw)) -+ bbr2_start_bw_probe_cruise(sk); -+ break; -+ -+ default: -+ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); -+ } -+} -+ -+/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ -+static void bbr2_exit_probe_rtt(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr2_reset_lower_bounds(sk); -+ if (bbr_full_bw_reached(sk)) { -+ bbr->mode = BBR_PROBE_BW; -+ /* Raising inflight after PROBE_RTT may cause loss, so reset -+ * the PROBE_BW clock and schedule the next bandwidth probe for -+ * a friendly and randomized future point in time. -+ */ -+ bbr2_start_bw_probe_down(sk); -+ /* Since we are exiting PROBE_RTT, we know inflight is -+ * below our estimated BDP, so it is reasonable to cruise. -+ */ -+ bbr2_start_bw_probe_cruise(sk); -+ } else { -+ bbr->mode = BBR_STARTUP; -+ } -+} -+ -+/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until -+ * the end of the round in recovery to get a good estimate of how many packets -+ * have been lost, and how many we need to drain with a low pacing rate. -+ */ -+static void bbr2_check_loss_too_high_in_startup(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr_full_bw_reached(sk)) -+ return; -+ -+ /* For STARTUP exit, check the loss rate at the end of each round trip -+ * of Recovery episodes in STARTUP. We check the loss rate at the end -+ * of the round trip to filter out noisy/low loss and have a better -+ * sense of inflight (extent of loss), so we can drain more accurately. -+ */ -+ if (rs->losses && bbr->loss_events_in_round < 0xf) -+ bbr->loss_events_in_round++; /* update saturating counter */ -+ if (bbr->params.full_loss_cnt && bbr->loss_round_start && -+ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && -+ bbr->loss_events_in_round >= bbr->params.full_loss_cnt && -+ bbr2_is_inflight_too_high(sk, rs)) { -+ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ -+ bbr2_handle_queue_too_high_in_startup(sk); -+ return; -+ } -+ if (bbr->loss_round_start) -+ bbr->loss_events_in_round = 0; -+} -+ -+/* If we are done draining, advance into steady state operation in PROBE_BW. */ -+static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (bbr_check_drain(sk, rs, ctx)) { -+ bbr->mode = BBR_PROBE_BW; -+ bbr2_start_bw_probe_down(sk); -+ } -+} -+ -+static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ bbr2_update_congestion_signals(sk, rs, ctx); -+ bbr_update_ack_aggregation(sk, rs); -+ bbr2_check_loss_too_high_in_startup(sk, rs); -+ bbr_check_full_bw_reached(sk, rs); -+ bbr2_check_drain(sk, rs, ctx); -+ bbr2_update_cycle_phase(sk, rs); -+ bbr_update_min_rtt(sk, rs); -+} -+ -+/* Fast path for app-limited case. -+ * -+ * On each ack, we execute bbr state machine, which primarily consists of: -+ * 1) update model based on new rate sample, and -+ * 2) update control based on updated model or state change. -+ * -+ * There are certain workload/scenarios, e.g. app-limited case, where -+ * either we can skip updating model or we can skip update of both model -+ * as well as control. This provides signifcant softirq cpu savings for -+ * processing incoming acks. -+ * -+ * In case of app-limited, if there is no congestion (loss/ecn) and -+ * if observed bw sample is less than current estimated bw, then we can -+ * skip some of the computation in bbr state processing: -+ * -+ * - if there is no rtt/mode/phase change: In this case, since all the -+ * parameters of the network model are constant, we can skip model -+ * as well control update. -+ * -+ * - else we can skip rest of the model update. But we still need to -+ * update the control to account for the new rtt/mode/phase. -+ * -+ * Returns whether we can take fast path or not. -+ */ -+static bool bbr2_fast_path(struct sock *sk, bool *update_model, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 prev_min_rtt_us, prev_mode; -+ -+ if (bbr->params.fast_path && bbr->try_fast_path && -+ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && -+ !bbr->loss_in_round && !bbr->ecn_in_round) { -+ prev_mode = bbr->mode; -+ prev_min_rtt_us = bbr->min_rtt_us; -+ bbr2_check_drain(sk, rs, ctx); -+ bbr2_update_cycle_phase(sk, rs); -+ bbr_update_min_rtt(sk, rs); -+ -+ if (bbr->mode == prev_mode && -+ bbr->min_rtt_us == prev_min_rtt_us && -+ bbr->try_fast_path) -+ return true; -+ -+ /* Skip model update, but control still needs to be updated */ -+ *update_model = false; -+ } -+ return false; -+} -+ -+static void bbr2_main(struct sock *sk, const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct bbr_context ctx = { 0 }; -+ bool update_model = true; -+ u32 bw; -+ int ce_ratio = -1; -+ -+ bbr->debug.event = '.'; /* init to default NOP (no event yet) */ -+ -+ bbr_update_round_start(sk, rs, &ctx); -+ if (bbr->round_start) { -+ bbr->rounds_since_probe = -+ min_t(s32, bbr->rounds_since_probe + 1, 0xFF); -+ ce_ratio = bbr2_update_ecn_alpha(sk); -+ tcp_plb_update_state(sk, &bbr->plb, ce_ratio); -+ tcp_plb_check_rehash(sk, &bbr->plb); -+ } -+ -+ bbr->ecn_in_round |= rs->is_ece; -+ bbr_calculate_bw_sample(sk, rs, &ctx); -+ -+ if (bbr2_fast_path(sk, &update_model, rs, &ctx)) -+ goto out; -+ -+ if (update_model) -+ bbr2_update_model(sk, rs, &ctx); -+ -+ bbr_update_gains(sk); -+ bw = bbr_bw(sk); -+ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); -+ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, -+ tp->snd_cwnd, &ctx); -+ bbr2_bound_cwnd_for_inflight_model(sk); -+ -+out: -+ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; -+ bbr->loss_in_cycle |= rs->lost > 0; -+ bbr->ecn_in_cycle |= rs->delivered_ce > 0; -+ -+ bbr_debug(sk, rs->acked_sacked, rs, &ctx); -+} -+ -+/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared -+ * down here, so that the algorithm functions that use the parameters must use -+ * the per-socket parameters; if they accidentally use the global version -+ * then there will be a compile error. -+ * TODO(ncardwell): move all per-socket parameters down to this section. -+ */ -+ -+/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. -+ * No loss response when 0. Max allwed value is 255. -+ */ -+static u32 bbr_beta = BBR_UNIT * 30 / 100; -+ -+/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. -+ * Max allowed value is 255. -+ */ -+static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ -+ -+/* The initial value for the ecn_alpha state variable. Default and max -+ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly -+ * to congestion if the bottleneck is congested when the flow starts up. -+ */ -+static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ -+ -+/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. -+ * No ECN based bounding when 0. Max allwed value is 255. -+ */ -+static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ -+ -+/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. -+ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. -+ */ -+static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ -+ -+/* Max RTT (in usec) at which to use sender-side ECN logic. -+ * Disabled when 0 (ECN allowed at any RTT). -+ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. -+ */ -+static u32 bbr_ecn_max_rtt_us = 5000; -+ -+/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN -+ * clears then use a multiplicative increase to quickly reprobe bw by -+ * starting inflight probing at the given multiple of inflight_hi. -+ * Default for this experimental knob is 0 (disabled). -+ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. -+ */ -+static u32 bbr_ecn_reprobe_gain; -+ -+/* Estimate bw probing has gone too far if loss rate exceeds this level. */ -+static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ -+ -+/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, -+ * and loss rate is higher than bbr_loss_thresh. -+ * Disabled if 0. Max allowed value is 15 (0xF). -+ */ -+static u32 bbr_full_loss_cnt = 8; -+ -+/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh -+ * meets this count. Max allowed value is 3. -+ */ -+static u32 bbr_full_ecn_cnt = 2; -+ -+/* Fraction of unutilized headroom to try to leave in path upon high loss. */ -+static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; -+ -+/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. -+ * Default is 1.25x, as in BBR v1. Max allowed is 511. -+ */ -+static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; -+ -+/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. -+ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. -+ * Max allowed is 511. -+ */ -+static u32 bbr_bw_probe_reno_gain = BBR_UNIT; -+ -+/* Max number of packet-timed rounds to wait before probing for bandwidth. If -+ * we want to tolerate 1% random loss per round, and not have this cut our -+ * inflight too much, we must probe for bw periodically on roughly this scale. -+ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. -+ * We aim to be fair with Reno/CUBIC up to a BDP of at least: -+ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets -+ */ -+static u32 bbr_bw_probe_max_rounds = 63; -+ -+/* Max amount of randomness to inject in round counting for Reno-coexistence. -+ * Max value is 15. -+ */ -+static u32 bbr_bw_probe_rand_rounds = 2; -+ -+/* Use BBR-native probe time scale starting at this many usec. -+ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: -+ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs -+ */ -+static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ -+ -+/* Use BBR-native probes spread over this many usec: */ -+static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ -+ -+/* Undo the model changes made in loss recovery if recovery was spurious? */ -+static bool bbr_undo = true; -+ -+/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ -+static bool bbr_fast_path = true; /* default: enabled */ -+ -+/* Use fast ack mode ? */ -+static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ -+ -+/* How much to additively increase inflight_hi when entering REFILL? */ -+static u32 bbr_refill_add_inc; /* default: disabled */ -+ -+module_param_named(beta, bbr_beta, uint, 0644); -+module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); -+module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); -+module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); -+module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); -+module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); -+module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); -+module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); -+module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664); -+module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664); -+module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664); -+module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664); -+module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664); -+module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664); -+module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664); -+module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664); -+module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664); -+module_param_named(undo, bbr_undo, bool, 0664); -+module_param_named(fast_path, bbr_fast_path, bool, 0664); -+module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664); -+module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664); -+ -+static void bbr2_init(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ const struct net *net = sock_net(sk); -+ -+ bbr_init(sk); /* run shared init code for v1 and v2 */ -+ -+ /* BBR v2 parameters: */ -+ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); -+ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); -+ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); -+ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); -+ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); -+ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); -+ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); -+ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); -+ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); -+ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); -+ bbr->params.inflight_headroom = -+ min_t(u32, 0xFFU, bbr_inflight_headroom); -+ bbr->params.bw_probe_pif_gain = -+ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); -+ bbr->params.bw_probe_reno_gain = -+ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); -+ bbr->params.bw_probe_max_rounds = -+ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); -+ bbr->params.bw_probe_rand_rounds = -+ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); -+ bbr->params.bw_probe_base_us = -+ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); -+ bbr->params.bw_probe_rand_us = -+ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); -+ bbr->params.undo = bbr_undo; -+ bbr->params.fast_path = bbr_fast_path ? 1 : 0; -+ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); -+ -+ /* BBR v2 state: */ -+ bbr->initialized = 1; -+ /* Start sampling ECN mark rate after first full flight is ACKed: */ -+ bbr->loss_round_delivered = tp->delivered + 1; -+ bbr->loss_round_start = 0; -+ bbr->undo_bw_lo = 0; -+ bbr->undo_inflight_lo = 0; -+ bbr->undo_inflight_hi = 0; -+ bbr->loss_events_in_round = 0; -+ bbr->startup_ecn_rounds = 0; -+ bbr2_reset_congestion_signals(sk); -+ bbr->bw_lo = ~0U; -+ bbr->bw_hi[0] = 0; -+ bbr->bw_hi[1] = 0; -+ bbr->inflight_lo = ~0U; -+ bbr->inflight_hi = ~0U; -+ bbr->bw_probe_up_cnt = ~0U; -+ bbr->bw_probe_up_acks = 0; -+ bbr->bw_probe_up_rounds = 0; -+ bbr->probe_wait_us = 0; -+ bbr->stopped_risky_probe = 0; -+ bbr->ack_phase = BBR_ACKS_INIT; -+ bbr->rounds_since_probe = 0; -+ bbr->bw_probe_samples = 0; -+ bbr->prev_probe_too_high = 0; -+ bbr->ecn_eligible = 0; -+ bbr->ecn_alpha = bbr->params.ecn_alpha_init; -+ bbr->alpha_last_delivered = 0; -+ bbr->alpha_last_delivered_ce = 0; -+ -+ bbr->plb.enabled = 0; -+ bbr->plb.consec_cong_rounds = 0; -+ bbr->plb.pause_until = 0; -+ if ((tp->ecn_flags & TCP_ECN_OK) && -+ net->ipv4.sysctl_tcp_plb_enabled) -+ bbr->plb.enabled = 1; -+ -+ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); -+ -+ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable) -+ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; -+} -+ -+/* Core TCP stack informs us that the given skb was just marked lost. */ -+static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct tcp_skb_cb *scb = TCP_SKB_CB(skb); -+ struct rate_sample rs; -+ -+ /* Capture "current" data over the full round trip of loss, -+ * to have a better chance to see the full capacity of the path. -+ */ -+ if (!bbr->loss_in_round) /* first loss in this round trip? */ -+ bbr->loss_round_delivered = tp->delivered; /* set round trip */ -+ bbr->loss_in_round = 1; -+ bbr->loss_in_cycle = 1; -+ -+ if (!bbr->bw_probe_samples) -+ return; /* not an skb sent while probing for bandwidth */ -+ if (unlikely(!scb->tx.delivered_mstamp)) -+ return; /* skb was SACKed, reneged, marked lost; ignore it */ -+ /* We are probing for bandwidth. Construct a rate sample that -+ * estimates what happened in the flight leading up to this lost skb, -+ * then see if the loss rate went too high, and if so at which packet. -+ */ -+ memset(&rs, 0, sizeof(rs)); -+ rs.tx_in_flight = scb->tx.in_flight; -+ rs.lost = tp->lost - scb->tx.lost; -+ rs.is_app_limited = scb->tx.is_app_limited; -+ if (bbr2_is_inflight_too_high(sk, &rs)) { -+ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); -+ bbr2_handle_inflight_too_high(sk, &rs); -+ } -+} -+ -+/* Revert short-term model if current loss recovery event was spurious. */ -+static u32 bbr2_undo_cwnd(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->debug.undo = 1; -+ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ -+ bbr->full_bw_cnt = 0; -+ bbr->loss_in_round = 0; -+ -+ if (!bbr->params.undo) -+ return tp->snd_cwnd; -+ -+ /* Revert to cwnd and other state saved before loss episode. */ -+ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); -+ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); -+ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); -+ return bbr->prior_cwnd; -+} -+ -+/* Entering loss recovery, so save state for when we undo recovery. */ -+static u32 bbr2_ssthresh(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr_save_cwnd(sk); -+ /* For undo, save state that adapts based on loss signal. */ -+ bbr->undo_bw_lo = bbr->bw_lo; -+ bbr->undo_inflight_lo = bbr->inflight_lo; -+ bbr->undo_inflight_hi = bbr->inflight_hi; -+ return tcp_sk(sk)->snd_ssthresh; -+} -+ -+static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) -+{ -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ return BBR2_PHASE_STARTUP; -+ case BBR_DRAIN: -+ return BBR2_PHASE_DRAIN; -+ case BBR_PROBE_BW: -+ break; -+ case BBR_PROBE_RTT: -+ return BBR2_PHASE_PROBE_RTT; -+ default: -+ return BBR2_PHASE_INVALID; -+ } -+ switch (bbr->cycle_idx) { -+ case BBR_BW_PROBE_UP: -+ return BBR2_PHASE_PROBE_BW_UP; -+ case BBR_BW_PROBE_DOWN: -+ return BBR2_PHASE_PROBE_BW_DOWN; -+ case BBR_BW_PROBE_CRUISE: -+ return BBR2_PHASE_PROBE_BW_CRUISE; -+ case BBR_BW_PROBE_REFILL: -+ return BBR2_PHASE_PROBE_BW_REFILL; -+ default: -+ return BBR2_PHASE_INVALID; -+ } -+} -+ -+static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, -+ union tcp_cc_info *info) -+{ -+ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || -+ ext & (1 << (INET_DIAG_VEGASINFO - 1))) { -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); -+ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); -+ u64 bw_lo = bbr->bw_lo == ~0U ? -+ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); -+ -+ memset(&info->bbr2, 0, sizeof(info->bbr2)); -+ info->bbr2.bbr_bw_lsb = (u32)bw; -+ info->bbr2.bbr_bw_msb = (u32)(bw >> 32); -+ info->bbr2.bbr_min_rtt = bbr->min_rtt_us; -+ info->bbr2.bbr_pacing_gain = bbr->pacing_gain; -+ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; -+ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; -+ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); -+ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; -+ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); -+ info->bbr2.bbr_mode = bbr->mode; -+ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); -+ info->bbr2.bbr_version = (__u8)2; -+ info->bbr2.bbr_inflight_lo = bbr->inflight_lo; -+ info->bbr2.bbr_inflight_hi = bbr->inflight_hi; -+ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); -+ *attr = INET_DIAG_BBRINFO; -+ return sizeof(info->bbr2); -+ } -+ return 0; -+} -+ -+static void bbr2_set_state(struct sock *sk, u8 new_state) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (new_state == TCP_CA_Loss) { -+ struct rate_sample rs = { .losses = 1 }; -+ struct bbr_context ctx = { 0 }; -+ -+ tcp_plb_update_state_upon_rto(sk, &bbr->plb); -+ bbr->prev_ca_state = TCP_CA_Loss; -+ bbr->full_bw = 0; -+ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { -+ /* bbr_adapt_lower_bounds() needs cwnd before -+ * we suffered an RTO, to update inflight_lo: -+ */ -+ bbr->inflight_lo = -+ max(tp->snd_cwnd, bbr->prior_cwnd); -+ } -+ bbr_debug(sk, 0, &rs, &ctx); -+ } else if (bbr->prev_ca_state == TCP_CA_Loss && -+ new_state != TCP_CA_Loss) { -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr->try_fast_path = 0; /* bound cwnd using latest model */ -+ } -+} -+ -+static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { -+ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, -+ .name = "bbr2", -+ .owner = THIS_MODULE, -+ .init = bbr2_init, -+ .cong_control = bbr2_main, -+ .sndbuf_expand = bbr_sndbuf_expand, -+ .skb_marked_lost = bbr2_skb_marked_lost, -+ .undo_cwnd = bbr2_undo_cwnd, -+ .cwnd_event = bbr_cwnd_event, -+ .ssthresh = bbr2_ssthresh, -+ .tso_segs = bbr_tso_segs, -+ .get_info = bbr2_get_info, -+ .set_state = bbr2_set_state, -+}; -+ -+static int __init bbr_register(void) -+{ -+ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); -+ return tcp_register_congestion_control(&tcp_bbr2_cong_ops); -+} -+ -+static void __exit bbr_unregister(void) -+{ -+ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); -+} -+ -+module_init(bbr_register); -+module_exit(bbr_unregister); -+ -+MODULE_AUTHOR("Van Jacobson "); -+MODULE_AUTHOR("Neal Cardwell "); -+MODULE_AUTHOR("Yuchung Cheng "); -+MODULE_AUTHOR("Soheil Hassas Yeganeh "); -+MODULE_AUTHOR("Priyaranjan Jha "); -+MODULE_AUTHOR("Yousuk Seung "); -+MODULE_AUTHOR("Kevin Yang "); -+MODULE_AUTHOR("Arjun Roy "); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); -diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c -index d3cae40749e8..0f268f2ff2e9 100644 ---- a/net/ipv4/tcp_cong.c -+++ b/net/ipv4/tcp_cong.c -@@ -189,6 +189,7 @@ void tcp_init_congestion_control(struct sock *sk) - struct inet_connection_sock *icsk = inet_csk(sk); - - tcp_sk(sk)->prior_ssthresh = 0; -+ tcp_sk(sk)->fast_ack_mode = 0; - if (icsk->icsk_ca_ops->init) - icsk->icsk_ca_ops->init(sk); - if (tcp_ca_needs_ecn(sk)) -diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c -index bc2ea12221f9..fa5c3e5d9c85 100644 ---- a/net/ipv4/tcp_input.c -+++ b/net/ipv4/tcp_input.c -@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) - tcp_enter_quickack_mode(sk, 2); - break; - case INET_ECN_CE: -- if (tcp_ca_needs_ecn(sk)) -+ if (tcp_ca_wants_ce_events(sk)) - tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); - - if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { -@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) - tp->ecn_flags |= TCP_ECN_SEEN; - break; - default: -- if (tcp_ca_needs_ecn(sk)) -+ if (tcp_ca_wants_ce_events(sk)) - tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); - tp->ecn_flags |= TCP_ECN_SEEN; - break; -@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) - */ - static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) - { -+ struct sock *sk = (struct sock *)tp; -+ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; -+ - tp->lost += tcp_skb_pcount(skb); -+ if (ca_ops->skb_marked_lost) -+ ca_ops->skb_marked_lost(sk, skb); - } - - void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) -@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, - WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); - tcp_skb_pcount_add(skb, -pcount); - -+ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ -+ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, -+ "prev in_flight: %u skb in_flight: %u pcount: %u", -+ TCP_SKB_CB(prev)->tx.in_flight, -+ TCP_SKB_CB(skb)->tx.in_flight, -+ pcount)) -+ TCP_SKB_CB(skb)->tx.in_flight = 0; -+ else -+ TCP_SKB_CB(skb)->tx.in_flight -= pcount; -+ TCP_SKB_CB(prev)->tx.in_flight += pcount; -+ - /* When we're adding to gso_segs == 1, gso_size will be zero, - * in theory this shouldn't be necessary but as long as DSACK - * code can come after this skb later on it's better to keep -@@ -3811,6 +3827,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) - - prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; - rs.prior_in_flight = tcp_packets_in_flight(tp); -+ tcp_rate_check_app_limited(sk); - - /* ts_recent update must be made after we are sure that the packet - * is in window. -@@ -3909,6 +3926,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) - delivered = tcp_newly_delivered(sk, delivered, flag); - lost = tp->lost - lost; /* freshly marked lost */ - rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); -+ rs.is_ece = !!(flag & FLAG_ECE); - tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); - tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); - tcp_xmit_recovery(sk, rexmit); -@@ -5508,13 +5526,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) - - /* More than one full frame received... */ - if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && -+ (tp->fast_ack_mode == 1 || - /* ... and right edge of window advances far enough. - * (tcp_recvmsg() will send ACK otherwise). - * If application uses SO_RCVLOWAT, we want send ack now if - * we have not received enough bytes to satisfy the condition. - */ -- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || -- __tcp_select_window(sk) >= tp->rcv_wnd)) || -+ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || -+ __tcp_select_window(sk) >= tp->rcv_wnd))) || - /* We ACK each frame or... */ - tcp_in_quickack_mode(sk) || - /* Protocol state mandates a one-time immediate ACK */ -diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c -index 5b019ba2b9d2..5884f4388c61 100644 ---- a/net/ipv4/tcp_ipv4.c -+++ b/net/ipv4/tcp_ipv4.c -@@ -3165,6 +3165,13 @@ static int __net_init tcp_sk_init(struct net *net) - net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0; - atomic_set(&net->ipv4.tfo_active_disable_times, 0); - -+ /* Set default values for PLB */ -+ net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */ -+ net->ipv4.sysctl_tcp_plb_cong_thresh = 128; /* 50% congestion */ -+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3; -+ net->ipv4.sysctl_tcp_plb_rehash_rounds = 12; -+ net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60; -+ - /* Reno is always built in */ - if (!net_eq(net, &init_net) && - bpf_try_module_get(init_net.ipv4.tcp_congestion_control, -diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c -index 290019de766d..15e89addae08 100644 ---- a/net/ipv4/tcp_output.c -+++ b/net/ipv4/tcp_output.c -@@ -375,7 +375,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, - th->cwr = 1; - skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; - } -- } else if (!tcp_ca_needs_ecn(sk)) { -+ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && -+ !tcp_ca_needs_ecn(sk)) { - /* ACK or retransmitted segment: clear ECT|CE */ - INET_ECN_dontxmit(sk); - } -@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, - { - struct tcp_sock *tp = tcp_sk(sk); - struct sk_buff *buff; -- int nsize, old_factor; -+ int nsize, old_factor, inflight_prev; - long limit; - int nlen; - u8 flags; -@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, - - if (diff) - tcp_adjust_pcount(sk, skb, diff); -+ -+ /* Set buff tx.in_flight as if buff were sent by itself. */ -+ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; -+ if (WARN_ONCE(inflight_prev < 0, -+ "inconsistent: tx.in_flight: %u old_factor: %d", -+ TCP_SKB_CB(skb)->tx.in_flight, old_factor)) -+ inflight_prev = 0; -+ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + -+ tcp_skb_pcount(buff); - } - - /* Link BUFF into the send queue. */ -@@ -1988,13 +1998,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, - static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) - { - const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; -- u32 min_tso, tso_segs; -- -- min_tso = ca_ops->min_tso_segs ? -- ca_ops->min_tso_segs(sk) : -- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); -+ u32 tso_segs; - -- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); -+ tso_segs = ca_ops->tso_segs ? -+ ca_ops->tso_segs(sk, mss_now) : -+ tcp_tso_autosize(sk, mss_now, -+ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs)); - return min_t(u32, tso_segs, sk->sk_gso_max_segs); - } - -@@ -2630,6 +2639,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, - skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); - list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); - tcp_init_tso_segs(skb, mss_now); -+ tcp_set_tx_in_flight(sk, skb); - goto repair; /* Skip network transmission */ - } - -diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c -new file mode 100644 -index 000000000000..71b02c0404ce ---- /dev/null -+++ b/net/ipv4/tcp_plb.c -@@ -0,0 +1,100 @@ -+/* Protective Load Balancing (PLB) -+ * -+ * PLB was designed to reduce link load imbalance across datacenter -+ * switches. PLB is a host-based optimization; it leverages congestion -+ * signals from the transport layer to randomly change the path of the -+ * connection experiencing sustained congestion. PLB prefers to repath -+ * after idle periods to minimize packet reordering. It repaths by -+ * changing the IPv6 Flow Label on the packets of a connection, which -+ * datacenter switches include as part of ECMP/WCMP hashing. -+ * -+ * PLB is described in detail in: -+ * -+ * Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu, -+ * Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson, -+ * David Wetherall,Abdul Kabbani: -+ * "PLB: Congestion Signals are Simple and Effective for -+ * Network Load Balancing" -+ * In ACM SIGCOMM 2022, Amsterdam Netherlands. -+ * -+ */ -+ -+#include -+ -+/* Called once per round-trip to update PLB state for a connection. */ -+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, -+ const int cong_ratio) -+{ -+ struct net *net = sock_net(sk); -+ -+ if (!plb->enabled) -+ return; -+ -+ if (cong_ratio >= 0) { -+ if (cong_ratio < net->ipv4.sysctl_tcp_plb_cong_thresh) -+ plb->consec_cong_rounds = 0; -+ else if (plb->consec_cong_rounds < -+ net->ipv4.sysctl_tcp_plb_rehash_rounds) -+ plb->consec_cong_rounds++; -+ } -+} -+EXPORT_SYMBOL_GPL(tcp_plb_update_state); -+ -+/* Check whether recent congestion has been persistent enough to warrant -+ * a load balancing decision that switches the connection to another path. -+ */ -+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) -+{ -+ struct net *net = sock_net(sk); -+ bool can_idle_rehash, can_force_rehash; -+ -+ if (!plb->enabled) -+ return; -+ -+ /* Note that tcp_jiffies32 can wrap, so we clear pause_until -+ * to 0 to indicate there is no recent RTO event that constrains -+ * PLB rehashing. -+ */ -+ if (plb->pause_until && -+ !before(tcp_jiffies32, plb->pause_until)) -+ plb->pause_until = 0; -+ -+ can_idle_rehash = net->ipv4.sysctl_tcp_plb_idle_rehash_rounds && -+ !tcp_sk(sk)->packets_out && -+ plb->consec_cong_rounds >= -+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds; -+ can_force_rehash = plb->consec_cong_rounds >= -+ net->ipv4.sysctl_tcp_plb_rehash_rounds; -+ -+ if (!plb->pause_until && (can_idle_rehash || can_force_rehash)) { -+ sk_rethink_txhash(sk); -+ plb->consec_cong_rounds = 0; -+ tcp_sk(sk)->ecn_rehash++; -+ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPECNREHASH); -+ } -+} -+EXPORT_SYMBOL_GPL(tcp_plb_check_rehash); -+ -+/* Upon RTO, disallow load balancing for a while, to avoid having load -+ * balancing decisions switch traffic to a black-holed path that was -+ * previously avoided with a sk_rethink_txhash() call at RTO time. -+ */ -+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb) -+{ -+ struct net *net = sock_net(sk); -+ u32 pause; -+ -+ if (!plb->enabled) -+ return; -+ -+ pause = net->ipv4.sysctl_tcp_plb_suspend_rto_sec * HZ; -+ pause += prandom_u32_max(pause); -+ plb->pause_until = tcp_jiffies32 + pause; -+ -+ /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call -+ * that may switch this connection to a path with completely different -+ * congestion characteristics. -+ */ -+ plb->consec_cong_rounds = 0; -+} -+EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto); -diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c -index a8f6d9d06f2e..a8b4c9504570 100644 ---- a/net/ipv4/tcp_rate.c -+++ b/net/ipv4/tcp_rate.c -@@ -34,6 +34,24 @@ - * ready to send in the write queue. - */ - -+void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ u32 in_flight; -+ -+ /* Check, sanitize, and record packets in flight after skb was sent. */ -+ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); -+ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, -+ "insane in_flight %u cc %s mss %u " -+ "cwnd %u pif %u %u %u %u\n", -+ in_flight, inet_csk(sk)->icsk_ca_ops->name, -+ tp->mss_cache, tp->snd_cwnd, -+ tp->packets_out, tp->retrans_out, -+ tp->sacked_out, tp->lost_out)) -+ in_flight = TCPCB_IN_FLIGHT_MAX; -+ TCP_SKB_CB(skb)->tx.in_flight = in_flight; -+} -+ - /* Snapshot the current delivery information in the skb, to generate - * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). - */ -@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) - TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; - TCP_SKB_CB(skb)->tx.delivered = tp->delivered; - TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; -+ TCP_SKB_CB(skb)->tx.lost = tp->lost; - TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; -+ tcp_set_tx_in_flight(sk, skb); - } - - /* When an skb is sacked or acked, we fill in the rate sample with the (prior) -@@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - if (!rs->prior_delivered || - tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, - scb->end_seq, rs->last_end_seq)) { -+ rs->prior_lost = scb->tx.lost; - rs->prior_delivered_ce = scb->tx.delivered_ce; - rs->prior_delivered = scb->tx.delivered; - rs->prior_mstamp = scb->tx.delivered_mstamp; - rs->is_app_limited = scb->tx.is_app_limited; - rs->is_retrans = scb->sacked & TCPCB_RETRANS; - rs->last_end_seq = scb->end_seq; -+ rs->tx_in_flight = scb->tx.in_flight; - - /* Record send time of most recently ACKed packet: */ - tp->first_tx_mstamp = tx_tstamp; - /* Find the duration of the "send phase" of this window: */ -- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, -- scb->tx.first_tx_mstamp); -+ rs->interval_us = tcp_stamp32_us_delta( -+ tp->first_tx_mstamp, -+ scb->tx.first_tx_mstamp); - - } - /* Mark off the skb delivered once it's sacked to avoid being -@@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - return; - } - rs->delivered = tp->delivered - rs->prior_delivered; -+ rs->lost = tp->lost - rs->prior_lost; - - rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; - /* delivered_ce occupies less than 32 bits in the skb control block */ -@@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - * longer phase. - */ - snd_us = rs->interval_us; /* send phase */ -- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, -+ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, - rs->prior_mstamp); /* ack phase */ - rs->interval_us = max(snd_us, ack_us); - -diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c -index b4dfb82d6ecb..bb613fc8948a 100644 ---- a/net/ipv4/tcp_timer.c -+++ b/net/ipv4/tcp_timer.c -@@ -605,6 +605,7 @@ void tcp_write_timer_handler(struct sock *sk) - return; - } - -+ tcp_rate_check_app_limited(sk); - tcp_mstamp_refresh(tcp_sk(sk)); - event = icsk->icsk_pending; - --- -2.38.0.rc1.8.g2a7d63a245 - -From 98c01e8963ef2954a163a27812234015f3bf036e Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 25 Sep 2022 23:49:46 +0200 -Subject: [PATCH 03/13] futex-winesync - -Signed-off-by: Peter Jung ---- - Documentation/admin-guide/devices.txt | 3 +- - Documentation/userspace-api/index.rst | 1 + - .../userspace-api/ioctl/ioctl-number.rst | 2 + - Documentation/userspace-api/winesync.rst | 444 +++++ - MAINTAINERS | 9 + - drivers/misc/Kconfig | 11 + - drivers/misc/Makefile | 1 + - drivers/misc/winesync.c | 1212 ++++++++++++++ - include/linux/miscdevice.h | 1 + - include/uapi/linux/winesync.h | 71 + - tools/testing/selftests/Makefile | 1 + - .../selftests/drivers/winesync/Makefile | 8 + - .../testing/selftests/drivers/winesync/config | 1 + - .../selftests/drivers/winesync/winesync.c | 1479 +++++++++++++++++ - 14 files changed, 3243 insertions(+), 1 deletion(-) - create mode 100644 Documentation/userspace-api/winesync.rst - create mode 100644 drivers/misc/winesync.c - create mode 100644 include/uapi/linux/winesync.h - create mode 100644 tools/testing/selftests/drivers/winesync/Makefile - create mode 100644 tools/testing/selftests/drivers/winesync/config - create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c - -diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt -index 9764d6edb189..a4696d3b4a5a 100644 ---- a/Documentation/admin-guide/devices.txt -+++ b/Documentation/admin-guide/devices.txt -@@ -376,8 +376,9 @@ - 240 = /dev/userio Serio driver testing device - 241 = /dev/vhost-vsock Host kernel driver for virtio vsock - 242 = /dev/rfkill Turning off radio transmissions (rfkill) -+ 243 = /dev/winesync Wine synchronization primitive device - -- 243-254 Reserved for local use -+ 244-254 Reserved for local use - 255 Reserved for MISC_DYNAMIC_MINOR - - 11 char Raw keyboard device (Linux/SPARC only) -diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst -index a61eac0c73f8..0bf697ddcb09 100644 ---- a/Documentation/userspace-api/index.rst -+++ b/Documentation/userspace-api/index.rst -@@ -29,6 +29,7 @@ place where this information is gathered. - sysfs-platform_profile - vduse - futex2 -+ winesync - - .. only:: subproject and html - -diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst -index 3b985b19f39d..3f313fd4338c 100644 ---- a/Documentation/userspace-api/ioctl/ioctl-number.rst -+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst -@@ -375,6 +375,8 @@ Code Seq# Include File Comments - - 0xF6 all LTTng Linux Trace Toolkit Next Generation - -+0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives -+ - 0xF8 all arch/x86/include/uapi/asm/amd_hsmp.h AMD HSMP EPYC system management interface driver - - 0xFD all linux/dm-ioctl.h -diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst -new file mode 100644 -index 000000000000..f0110d2744c7 ---- /dev/null -+++ b/Documentation/userspace-api/winesync.rst -@@ -0,0 +1,444 @@ -+===================================== -+Wine synchronization primitive driver -+===================================== -+ -+This page documents the user-space API for the winesync driver. -+ -+winesync is a support driver for emulation of NT synchronization -+primitives by the Wine project or other NT emulators. It exists -+because implementation in user-space, using existing tools, cannot -+simultaneously satisfy performance, correctness, and security -+constraints. It is implemented entirely in software, and does not -+drive any hardware device. -+ -+This interface is meant as a compatibility tool only, and should not -+be used for general synchronization. Instead use generic, versatile -+interfaces such as futex(2) and poll(2). -+ -+Synchronization primitives -+========================== -+ -+The winesync driver exposes three types of synchronization primitives: -+semaphores, mutexes, and events. -+ -+A semaphore holds a single volatile 32-bit counter, and a static -+32-bit integer denoting the maximum value. It is considered signaled -+when the counter is nonzero. The counter is decremented by one when a -+wait is satisfied. Both the initial and maximum count are established -+when the semaphore is created. -+ -+A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit -+identifier denoting its owner. A mutex is considered signaled when its -+owner is zero (indicating that it is not owned). The recursion count -+is incremented when a wait is satisfied, and ownership is set to the -+given identifier. -+ -+A mutex also holds an internal flag denoting whether its previous -+owner has died; such a mutex is said to be inconsistent. Owner death -+is not tracked automatically based on thread death, but rather must be -+communicated using ``WINESYNC_IOC_KILL_OWNER``. An inconsistent mutex -+is inherently considered unowned. -+ -+Except for the "unowned" semantics of zero, the actual value of the -+owner identifier is not interpreted by the winesync driver at all. The -+intended use is to store a thread identifier; however, the winesync -+driver does not actually validate that a calling thread provides -+consistent or unique identifiers. -+ -+An event holds a volatile boolean state denoting whether it is -+signaled or not. There are two types of events, auto-reset and -+manual-reset. An auto-reset event is designaled when a wait is -+satisfied; a manual-reset event is not. The event type is specified -+when the event is created. -+ -+Unless specified otherwise, all operations on an object are atomic and -+totally ordered with respect to other operations on the same object. -+ -+Objects are represented by unsigned 32-bit integers. -+ -+Char device -+=========== -+ -+The winesync driver creates a single char device /dev/winesync. Each -+file description opened on the device represents a unique namespace. -+That is, objects created on one open file description are shared -+across all its individual descriptors, but are not shared with other -+open() calls on the same device. The same file description may be -+shared across multiple processes. -+ -+ioctl reference -+=============== -+ -+All operations on the device are done through ioctls. There are three -+structures used in ioctl calls:: -+ -+ struct winesync_sem_args { -+ __u32 sem; -+ __u32 count; -+ __u32 max; -+ }; -+ -+ struct winesync_mutex_args { -+ __u32 mutex; -+ __u32 owner; -+ __u32 count; -+ }; -+ -+ struct winesync_event_args { -+ __u32 event; -+ __u32 signaled; -+ __u32 manual; -+ }; -+ -+ struct winesync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 pad; -+ }; -+ -+Depending on the ioctl, members of the structure may be used as input, -+output, or not at all. All ioctls return 0 on success. -+ -+The ioctls are as follows: -+ -+.. c:macro:: WINESYNC_IOC_CREATE_SEM -+ -+ Create a semaphore object. Takes a pointer to struct -+ :c:type:`winesync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``sem`` -+ - On output, contains the identifier of the created semaphore. -+ * - ``count`` -+ - Initial count of the semaphore. -+ * - ``max`` -+ - Maximum count of the semaphore. -+ -+ Fails with ``EINVAL`` if ``count`` is greater than ``max``. -+ -+.. c:macro:: WINESYNC_IOC_CREATE_MUTEX -+ -+ Create a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``mutex`` -+ - On output, contains the identifier of the created mutex. -+ * - ``count`` -+ - Initial recursion count of the mutex. -+ * - ``owner`` -+ - Initial owner of the mutex. -+ -+ If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is -+ zero and ``count`` is nonzero, the function fails with ``EINVAL``. -+ -+.. c:macro:: WINESYNC_IOC_CREATE_EVENT -+ -+ Create an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - On output, contains the identifier of the created event. -+ * - ``signaled`` -+ - If nonzero, the event is initially signaled, otherwise -+ nonsignaled. -+ * - ``manual`` -+ - If nonzero, the event is a manual-reset event, otherwise -+ auto-reset. -+ -+.. c:macro:: WINESYNC_IOC_DELETE -+ -+ Delete an object of any type. Takes an input-only pointer to a -+ 32-bit integer denoting the object to delete. -+ -+ Wait ioctls currently in progress are not interrupted, and behave as -+ if the object remains valid. -+ -+.. c:macro:: WINESYNC_IOC_PUT_SEM -+ -+ Post to a semaphore object. Takes a pointer to struct -+ :c:type:`winesync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``sem`` -+ - Semaphore object to post to. -+ * - ``count`` -+ - Count to add to the semaphore. On output, contains the -+ previous count of the semaphore. -+ * - ``max`` -+ - Not used. -+ -+ If adding ``count`` to the semaphore's current count would raise the -+ latter past the semaphore's maximum count, the ioctl fails with -+ ``EOVERFLOW`` and the semaphore is not affected. If raising the -+ semaphore's count causes it to become signaled, eligible threads -+ waiting on this semaphore will be woken and the semaphore's count -+ decremented appropriately. -+ -+.. c:macro:: WINESYNC_IOC_PUT_MUTEX -+ -+ Release a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``mutex`` -+ - Mutex object to release. -+ * - ``owner`` -+ - Mutex owner identifier. -+ * - ``count`` -+ - On output, contains the previous recursion count. -+ -+ If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` -+ is not the current owner of the mutex, the ioctl fails with -+ ``EPERM``. -+ -+ The mutex's count will be decremented by one. If decrementing the -+ mutex's count causes it to become zero, the mutex is marked as -+ unowned and signaled, and eligible threads waiting on it will be -+ woken as appropriate. -+ -+.. c:macro:: WINESYNC_IOC_SET_EVENT -+ -+ Signal an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object to set. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. -+ -+ Eligible threads will be woken, and auto-reset events will be -+ designaled appropriately. -+ -+.. c:macro:: WINESYNC_IOC_RESET_EVENT -+ -+ Designal an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object to reset. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. -+ -+.. c:macro:: WINESYNC_IOC_PULSE_EVENT -+ -+ Wake threads waiting on an event object without leaving it in a -+ signaled state. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object to pulse. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. -+ -+ A pulse operation can be thought of as a set followed by a reset, -+ performed as a single atomic operation. If two threads are waiting -+ on an auto-reset event which is pulsed, only one will be woken. If -+ two threads are waiting a manual-reset event which is pulsed, both -+ will be woken. However, in both cases, the event will be unsignaled -+ afterwards, and a simultaneous read operation will always report the -+ event as unsignaled. -+ -+.. c:macro:: WINESYNC_IOC_READ_SEM -+ -+ Read the current state of a semaphore object. Takes a pointer to -+ struct :c:type:`winesync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``sem`` -+ - Semaphore object to read. -+ * - ``count`` -+ - On output, contains the current count of the semaphore. -+ * - ``max`` -+ - On output, contains the maximum count of the semaphore. -+ -+.. c:macro:: WINESYNC_IOC_READ_MUTEX -+ -+ Read the current state of a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``mutex`` -+ - Mutex object to read. -+ * - ``owner`` -+ - On output, contains the current owner of the mutex, or zero -+ if the mutex is not currently owned. -+ * - ``count`` -+ - On output, contains the current recursion count of the mutex. -+ -+ If the mutex is marked as inconsistent, the function fails with -+ ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to -+ zero. -+ -+.. c:macro:: WINESYNC_IOC_READ_EVENT -+ -+ Read the current state of an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object. -+ * - ``signaled`` -+ - On output, contains the current state of the event. -+ * - ``manual`` -+ - On output, contains 1 if the event is a manual-reset event, -+ and 0 otherwise. -+ -+.. c:macro:: WINESYNC_IOC_KILL_OWNER -+ -+ Mark any mutexes owned by the given owner as unowned and -+ inconsistent. Takes an input-only pointer to a 32-bit integer -+ denoting the owner. If the owner is zero, the ioctl fails with -+ ``EINVAL``. -+ -+ For each mutex currently owned by the given owner, eligible threads -+ waiting on said mutex will be woken as appropriate (and such waits -+ will fail with ``EOWNERDEAD``, as described below). -+ -+ The operation as a whole is not atomic; however, the modification of -+ each mutex is atomic and totally ordered with respect to other -+ operations on the same mutex. -+ -+.. c:macro:: WINESYNC_IOC_WAIT_ANY -+ -+ Poll on any of a list of objects, atomically acquiring at most one. -+ Takes a pointer to struct :c:type:`winesync_wait_args`, which is -+ used as follows: -+ -+ .. list-table:: -+ -+ * - ``timeout`` -+ - Optional pointer to a 64-bit struct :c:type:`timespec` -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). The timeout is specified in -+ absolute format, as measured against the MONOTONIC clock. If -+ the timeout is equal to or earlier than the current time, the -+ function returns immediately without sleeping. If ``timeout`` -+ is zero, i.e. NULL, the function will sleep until an object -+ is signaled, and will not fail with ``ETIMEDOUT``. -+ * - ``objs`` -+ - Pointer to an array of ``count`` 32-bit object identifiers -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). If any identifier is -+ invalid, the function fails with ``EINVAL``. -+ * - ``count`` -+ - Number of object identifiers specified in the ``objs`` array. -+ * - ``owner`` -+ - Mutex owner identifier. If any object in ``objs`` is a mutex, -+ the ioctl will attempt to acquire that mutex on behalf of -+ ``owner``. If ``owner`` is zero, the ioctl fails with -+ ``EINVAL``. -+ * - ``index`` -+ - On success, contains the index (into ``objs``) of the object -+ which was signaled. If ``alert`` was signaled instead, -+ this contains ``count``. -+ * - ``alert`` -+ - Optional event object identifier. If nonzero, this specifies -+ an "alert" event object which, if signaled, will terminate -+ the wait. If nonzero, the identifier must point to a valid -+ event. -+ -+ This function attempts to acquire one of the given objects. If -+ unable to do so, it sleeps until an object becomes signaled, -+ subsequently acquiring it, or the timeout expires. In the latter -+ case the ioctl fails with ``ETIMEDOUT``. The function only acquires -+ one object, even if multiple objects are signaled. -+ -+ A semaphore is considered to be signaled if its count is nonzero, -+ and is acquired by decrementing its count by one. A mutex is -+ considered to be signaled if it is unowned or if its owner matches -+ the ``owner`` argument, and is acquired by incrementing its -+ recursion count by one and setting its owner to the ``owner`` -+ argument. An auto-reset event is acquired by designaling it; a -+ manual-reset event is not affected by acquisition. -+ -+ Acquisition is atomic and totally ordered with respect to other -+ operations on the same object. If two wait operations (with -+ different ``owner`` identifiers) are queued on the same mutex, only -+ one is signaled. If two wait operations are queued on the same -+ semaphore, and a value of one is posted to it, only one is signaled. -+ The order in which threads are signaled is not specified. -+ -+ If an inconsistent mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Although this is a failure return, the function may -+ otherwise be considered successful. The mutex is marked as owned by -+ the given owner (with a recursion count of 1) and as no longer -+ inconsistent, and ``index`` is still set to the index of the mutex. -+ -+ The ``alert`` argument is an "extra" event which can terminate the -+ wait, independently of all other objects. If members of ``objs`` and -+ ``alert`` are both simultaneously signaled, a member of ``objs`` -+ will always be given priority and acquired first. Aside from this, -+ for "any" waits, there is no difference between passing an event as -+ this parameter, and passing it as an additional object at the end of -+ the ``objs`` array. For "all" waits, there is an additional -+ difference, as described below. -+ -+ It is valid to pass the same object more than once, including by -+ passing the same event in the ``objs`` array and in ``alert``. If a -+ wakeup occurs due to that object being signaled, ``index`` is set to -+ the lowest index corresponding to that object. -+ -+ The function may fail with ``EINTR`` if a signal is received. -+ -+.. c:macro:: WINESYNC_IOC_WAIT_ALL -+ -+ Poll on a list of objects, atomically acquiring all of them. Takes a -+ pointer to struct :c:type:`winesync_wait_args`, which is used -+ identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is -+ always filled with zero on success if not woken via alert. -+ -+ This function attempts to simultaneously acquire all of the given -+ objects. If unable to do so, it sleeps until all objects become -+ simultaneously signaled, subsequently acquiring them, or the timeout -+ expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and -+ no objects are modified. -+ -+ Objects may become signaled and subsequently designaled (through -+ acquisition by other threads) while this thread is sleeping. Only -+ once all objects are simultaneously signaled does the ioctl acquire -+ them and return. The entire acquisition is atomic and totally -+ ordered with respect to other operations on any of the given -+ objects. -+ -+ If an inconsistent mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects -+ are nevertheless marked as acquired. Note that if multiple mutex -+ objects are specified, there is no way to know which were marked as -+ inconsistent. -+ -+ As with "any" waits, the ``alert`` argument is an "extra" event -+ which can terminate the wait. Critically, however, an "all" wait -+ will succeed if all members in ``objs`` are signaled, *or* if -+ ``alert`` is signaled. In the latter case ``index`` will be set to -+ ``count``. As with "any" waits, if both conditions are filled, the -+ former takes priority, and objects in ``objs`` will be acquired. -+ -+ Unlike ``WINESYNC_IOC_WAIT_ANY``, it is not valid to pass the same -+ object more than once, nor is it valid to pass the same object in -+ ``objs`` and in ``alert`` If this is attempted, the function fails -+ with ``EINVAL``. -diff --git a/MAINTAINERS b/MAINTAINERS -index f5ca4aefd184..31a7aa60cdc3 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -21921,6 +21921,15 @@ M: David Härdeman - S: Maintained - F: drivers/media/rc/winbond-cir.c - -+WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER -+M: Zebediah Figura -+L: wine-devel@winehq.org -+S: Supported -+F: Documentation/userspace-api/winesync.rst -+F: drivers/misc/winesync.c -+F: include/uapi/linux/winesync.h -+F: tools/testing/selftests/drivers/winesync/ -+ - WINSYSTEMS EBC-C384 WATCHDOG DRIVER - M: William Breathitt Gray - L: linux-watchdog@vger.kernel.org -diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig -index 94e9fb4cdd76..4f9e3d80a6e8 100644 ---- a/drivers/misc/Kconfig -+++ b/drivers/misc/Kconfig -@@ -496,6 +496,17 @@ config VCPU_STALL_DETECTOR - - If you do not intend to run this kernel as a guest, say N. - -+config WINESYNC -+ tristate "Synchronization primitives for Wine" -+ help -+ This module provides kernel support for synchronization primitives -+ used by Wine. It is not a hardware driver. -+ -+ To compile this driver as a module, choose M here: the -+ module will be called winesync. -+ -+ If unsure, say N. -+ - source "drivers/misc/c2port/Kconfig" - source "drivers/misc/eeprom/Kconfig" - source "drivers/misc/cb710/Kconfig" -diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile -index 2be8542616dd..d061fe45407b 100644 ---- a/drivers/misc/Makefile -+++ b/drivers/misc/Makefile -@@ -58,6 +58,7 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ - obj-$(CONFIG_UACCE) += uacce/ - obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o - obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o -+obj-$(CONFIG_WINESYNC) += winesync.o - obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o - obj-$(CONFIG_OPEN_DICE) += open-dice.o - obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o -\ No newline at end of file -diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c -new file mode 100644 -index 000000000000..7a28f58dbbf2 ---- /dev/null -+++ b/drivers/misc/winesync.c -@@ -0,0 +1,1212 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * winesync.c - Kernel driver for Wine synchronization primitives -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define WINESYNC_NAME "winesync" -+ -+enum winesync_type { -+ WINESYNC_TYPE_SEM, -+ WINESYNC_TYPE_MUTEX, -+ WINESYNC_TYPE_EVENT, -+}; -+ -+struct winesync_obj { -+ struct rcu_head rhead; -+ struct kref refcount; -+ spinlock_t lock; -+ -+ /* -+ * any_waiters is protected by the object lock, but all_waiters is -+ * protected by the device wait_all_lock. -+ */ -+ struct list_head any_waiters; -+ struct list_head all_waiters; -+ -+ /* -+ * Hint describing how many tasks are queued on this object in a -+ * wait-all operation. -+ * -+ * Any time we do a wake, we may need to wake "all" waiters as well as -+ * "any" waiters. In order to atomically wake "all" waiters, we must -+ * lock all of the objects, and that means grabbing the wait_all_lock -+ * below (and, due to lock ordering rules, before locking this object). -+ * However, wait-all is a rare operation, and grabbing the wait-all -+ * lock for every wake would create unnecessary contention. Therefore we -+ * first check whether all_hint is zero, and, if it is, we skip trying -+ * to wake "all" waiters. -+ * -+ * This hint isn't protected by any lock. It might change during the -+ * course of a wake, but there's no meaningful race there; it's only a -+ * hint. -+ * -+ * Since wait requests must originate from user-space threads, we're -+ * limited here by PID_MAX_LIMIT, so there's no risk of saturation. -+ */ -+ atomic_t all_hint; -+ -+ enum winesync_type type; -+ -+ /* The following fields are protected by the object lock. */ -+ union { -+ struct { -+ __u32 count; -+ __u32 max; -+ } sem; -+ struct { -+ __u32 count; -+ __u32 owner; -+ bool ownerdead; -+ } mutex; -+ struct { -+ bool manual; -+ bool signaled; -+ } event; -+ } u; -+}; -+ -+struct winesync_q_entry { -+ struct list_head node; -+ struct winesync_q *q; -+ struct winesync_obj *obj; -+ __u32 index; -+}; -+ -+struct winesync_q { -+ struct task_struct *task; -+ __u32 owner; -+ -+ /* -+ * Protected via atomic_cmpxchg(). Only the thread that wins the -+ * compare-and-swap may actually change object states and wake this -+ * task. -+ */ -+ atomic_t signaled; -+ -+ bool all; -+ bool ownerdead; -+ __u32 count; -+ struct winesync_q_entry entries[]; -+}; -+ -+struct winesync_device { -+ /* -+ * Wait-all operations must atomically grab all objects, and be totally -+ * ordered with respect to each other and wait-any operations. If one -+ * thread is trying to acquire several objects, another thread cannot -+ * touch the object at the same time. -+ * -+ * We achieve this by grabbing multiple object locks at the same time. -+ * However, this creates a lock ordering problem. To solve that problem, -+ * wait_all_lock is taken first whenever multiple objects must be locked -+ * at the same time. -+ */ -+ spinlock_t wait_all_lock; -+ -+ struct xarray objects; -+}; -+ -+static struct winesync_obj *get_obj(struct winesync_device *dev, __u32 id) -+{ -+ struct winesync_obj *obj; -+ -+ rcu_read_lock(); -+ obj = xa_load(&dev->objects, id); -+ if (obj && !kref_get_unless_zero(&obj->refcount)) -+ obj = NULL; -+ rcu_read_unlock(); -+ -+ return obj; -+} -+ -+static void destroy_obj(struct kref *ref) -+{ -+ struct winesync_obj *obj = container_of(ref, struct winesync_obj, refcount); -+ -+ kfree_rcu(obj, rhead); -+} -+ -+static void put_obj(struct winesync_obj *obj) -+{ -+ kref_put(&obj->refcount, destroy_obj); -+} -+ -+static struct winesync_obj *get_obj_typed(struct winesync_device *dev, __u32 id, -+ enum winesync_type type) -+{ -+ struct winesync_obj *obj = get_obj(dev, id); -+ -+ if (obj && obj->type != type) { -+ put_obj(obj); -+ return NULL; -+ } -+ return obj; -+} -+ -+static int winesync_char_open(struct inode *inode, struct file *file) -+{ -+ struct winesync_device *dev; -+ -+ dev = kzalloc(sizeof(*dev), GFP_KERNEL); -+ if (!dev) -+ return -ENOMEM; -+ -+ spin_lock_init(&dev->wait_all_lock); -+ -+ xa_init_flags(&dev->objects, XA_FLAGS_ALLOC); -+ -+ file->private_data = dev; -+ return nonseekable_open(inode, file); -+} -+ -+static int winesync_char_release(struct inode *inode, struct file *file) -+{ -+ struct winesync_device *dev = file->private_data; -+ struct winesync_obj *obj; -+ unsigned long id; -+ -+ xa_for_each(&dev->objects, id, obj) -+ put_obj(obj); -+ -+ xa_destroy(&dev->objects); -+ -+ kfree(dev); -+ -+ return 0; -+} -+ -+static void init_obj(struct winesync_obj *obj) -+{ -+ kref_init(&obj->refcount); -+ atomic_set(&obj->all_hint, 0); -+ spin_lock_init(&obj->lock); -+ INIT_LIST_HEAD(&obj->any_waiters); -+ INIT_LIST_HEAD(&obj->all_waiters); -+} -+ -+static bool is_signaled(struct winesync_obj *obj, __u32 owner) -+{ -+ lockdep_assert_held(&obj->lock); -+ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ return !!obj->u.sem.count; -+ case WINESYNC_TYPE_MUTEX: -+ if (obj->u.mutex.owner && obj->u.mutex.owner != owner) -+ return false; -+ return obj->u.mutex.count < UINT_MAX; -+ case WINESYNC_TYPE_EVENT: -+ return obj->u.event.signaled; -+ } -+ -+ WARN(1, "bad object type %#x\n", obj->type); -+ return false; -+} -+ -+/* -+ * "locked_obj" is an optional pointer to an object which is already locked and -+ * should not be locked again. This is necessary so that changing an object's -+ * state and waking it can be a single atomic operation. -+ */ -+static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, -+ struct winesync_obj *locked_obj) -+{ -+ __u32 count = q->count; -+ bool can_wake = true; -+ __u32 i; -+ -+ lockdep_assert_held(&dev->wait_all_lock); -+ if (locked_obj) -+ lockdep_assert_held(&locked_obj->lock); -+ -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ spin_lock(&q->entries[i].obj->lock); -+ } -+ -+ for (i = 0; i < count; i++) { -+ if (!is_signaled(q->entries[i].obj, q->owner)) { -+ can_wake = false; -+ break; -+ } -+ } -+ -+ if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { -+ for (i = 0; i < count; i++) { -+ struct winesync_obj *obj = q->entries[i].obj; -+ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ obj->u.sem.count--; -+ break; -+ case WINESYNC_TYPE_MUTEX: -+ if (obj->u.mutex.ownerdead) -+ q->ownerdead = true; -+ obj->u.mutex.ownerdead = false; -+ obj->u.mutex.count++; -+ obj->u.mutex.owner = q->owner; -+ break; -+ case WINESYNC_TYPE_EVENT: -+ if (!obj->u.event.manual) -+ obj->u.event.signaled = false; -+ break; -+ } -+ } -+ wake_up_process(q->task); -+ } -+ -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ spin_unlock(&q->entries[i].obj->lock); -+ } -+} -+ -+static void try_wake_all_obj(struct winesync_device *dev, -+ struct winesync_obj *obj) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&dev->wait_all_lock); -+ lockdep_assert_held(&obj->lock); -+ -+ list_for_each_entry(entry, &obj->all_waiters, node) -+ try_wake_all(dev, entry->q, obj); -+} -+ -+static void try_wake_any_sem(struct winesync_obj *sem) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&sem->lock); -+ -+ list_for_each_entry(entry, &sem->any_waiters, node) { -+ struct winesync_q *q = entry->q; -+ -+ if (!sem->u.sem.count) -+ break; -+ -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ sem->u.sem.count--; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static void try_wake_any_mutex(struct winesync_obj *mutex) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&mutex->lock); -+ -+ list_for_each_entry(entry, &mutex->any_waiters, node) { -+ struct winesync_q *q = entry->q; -+ -+ if (mutex->u.mutex.count == UINT_MAX) -+ break; -+ if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) -+ continue; -+ -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ if (mutex->u.mutex.ownerdead) -+ q->ownerdead = true; -+ mutex->u.mutex.ownerdead = false; -+ mutex->u.mutex.count++; -+ mutex->u.mutex.owner = q->owner; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static void try_wake_any_event(struct winesync_obj *event) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&event->lock); -+ -+ list_for_each_entry(entry, &event->any_waiters, node) { -+ struct winesync_q *q = entry->q; -+ -+ if (!event->u.event.signaled) -+ break; -+ -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ if (!event->u.event.manual) -+ event->u.event.signaled = false; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static int winesync_create_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ if (args.count > args.max) -+ return -EINVAL; -+ -+ sem = kzalloc(sizeof(*sem), GFP_KERNEL); -+ if (!sem) -+ return -ENOMEM; -+ -+ init_obj(sem); -+ sem->type = WINESYNC_TYPE_SEM; -+ sem->u.sem.count = args.count; -+ sem->u.sem.max = args.max; -+ -+ ret = xa_alloc(&dev->objects, &id, sem, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(sem); -+ return ret; -+ } -+ -+ return put_user(id, &user_args->sem); -+} -+ -+static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ if (!args.owner != !args.count) -+ return -EINVAL; -+ -+ mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); -+ if (!mutex) -+ return -ENOMEM; -+ -+ init_obj(mutex); -+ mutex->type = WINESYNC_TYPE_MUTEX; -+ mutex->u.mutex.count = args.count; -+ mutex->u.mutex.owner = args.owner; -+ -+ ret = xa_alloc(&dev->objects, &id, mutex, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(mutex); -+ return ret; -+ } -+ -+ return put_user(id, &user_args->mutex); -+} -+ -+static int winesync_create_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = kzalloc(sizeof(*event), GFP_KERNEL); -+ if (!event) -+ return -ENOMEM; -+ -+ init_obj(event); -+ event->type = WINESYNC_TYPE_EVENT; -+ event->u.event.manual = args.manual; -+ event->u.event.signaled = args.signaled; -+ -+ ret = xa_alloc(&dev->objects, &id, event, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(event); -+ return ret; -+ } -+ -+ return put_user(id, &user_args->event); -+} -+ -+static int winesync_delete(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_obj *obj; -+ __u32 id; -+ -+ if (get_user(id, (__u32 __user *)argp)) -+ return -EFAULT; -+ -+ obj = xa_erase(&dev->objects, id); -+ if (!obj) -+ return -EINVAL; -+ -+ put_obj(obj); -+ return 0; -+} -+ -+/* -+ * Actually change the semaphore state, returning -EOVERFLOW if it is made -+ * invalid. -+ */ -+static int put_sem_state(struct winesync_obj *sem, __u32 count) -+{ -+ lockdep_assert_held(&sem->lock); -+ -+ if (sem->u.sem.count + count < sem->u.sem.count || -+ sem->u.sem.count + count > sem->u.sem.max) -+ return -EOVERFLOW; -+ -+ sem->u.sem.count += count; -+ return 0; -+} -+ -+static int winesync_put_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 prev_count; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ sem = get_obj_typed(dev, args.sem, WINESYNC_TYPE_SEM); -+ if (!sem) -+ return -EINVAL; -+ -+ if (atomic_read(&sem->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&sem->lock); -+ -+ prev_count = sem->u.sem.count; -+ ret = put_sem_state(sem, args.count); -+ if (!ret) { -+ try_wake_all_obj(dev, sem); -+ try_wake_any_sem(sem); -+ } -+ -+ spin_unlock(&sem->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&sem->lock); -+ -+ prev_count = sem->u.sem.count; -+ ret = put_sem_state(sem, args.count); -+ if (!ret) -+ try_wake_any_sem(sem); -+ -+ spin_unlock(&sem->lock); -+ } -+ -+ put_obj(sem); -+ -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; -+ -+ return ret; -+} -+ -+/* -+ * Actually change the mutex state, returning -EPERM if not the owner. -+ */ -+static int put_mutex_state(struct winesync_obj *mutex, -+ const struct winesync_mutex_args *args) -+{ -+ lockdep_assert_held(&mutex->lock); -+ -+ if (mutex->u.mutex.owner != args->owner) -+ return -EPERM; -+ -+ if (!--mutex->u.mutex.count) -+ mutex->u.mutex.owner = 0; -+ return 0; -+} -+ -+static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 prev_count; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ if (!args.owner) -+ return -EINVAL; -+ -+ mutex = get_obj_typed(dev, args.mutex, WINESYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -EINVAL; -+ -+ if (atomic_read(&mutex->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&mutex->lock); -+ -+ prev_count = mutex->u.mutex.count; -+ ret = put_mutex_state(mutex, &args); -+ if (!ret) { -+ try_wake_all_obj(dev, mutex); -+ try_wake_any_mutex(mutex); -+ } -+ -+ spin_unlock(&mutex->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&mutex->lock); -+ -+ prev_count = mutex->u.mutex.count; -+ ret = put_mutex_state(mutex, &args); -+ if (!ret) -+ try_wake_any_mutex(mutex); -+ -+ spin_unlock(&mutex->lock); -+ } -+ -+ put_obj(mutex); -+ -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; -+ -+ return ret; -+} -+ -+static int winesync_read_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 id; -+ -+ if (get_user(id, &user_args->sem)) -+ return -EFAULT; -+ -+ sem = get_obj_typed(dev, id, WINESYNC_TYPE_SEM); -+ if (!sem) -+ return -EINVAL; -+ -+ args.sem = id; -+ spin_lock(&sem->lock); -+ args.count = sem->u.sem.count; -+ args.max = sem->u.sem.max; -+ spin_unlock(&sem->lock); -+ -+ put_obj(sem); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} -+ -+static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 id; -+ int ret; -+ -+ if (get_user(id, &user_args->mutex)) -+ return -EFAULT; -+ -+ mutex = get_obj_typed(dev, id, WINESYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -EINVAL; -+ -+ args.mutex = id; -+ spin_lock(&mutex->lock); -+ args.count = mutex->u.mutex.count; -+ args.owner = mutex->u.mutex.owner; -+ ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; -+ spin_unlock(&mutex->lock); -+ -+ put_obj(mutex); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return ret; -+} -+ -+static int winesync_read_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ __u32 id; -+ -+ if (get_user(id, &user_args->event)) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, id, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ args.event = id; -+ spin_lock(&event->lock); -+ args.manual = event->u.event.manual; -+ args.signaled = event->u.event.signaled; -+ spin_unlock(&event->lock); -+ -+ put_obj(event); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} -+ -+/* -+ * Actually change the mutex state to mark its owner as dead. -+ */ -+static void put_mutex_ownerdead_state(struct winesync_obj *mutex) -+{ -+ lockdep_assert_held(&mutex->lock); -+ -+ mutex->u.mutex.ownerdead = true; -+ mutex->u.mutex.owner = 0; -+ mutex->u.mutex.count = 0; -+} -+ -+static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_obj *obj; -+ unsigned long id; -+ __u32 owner; -+ -+ if (get_user(owner, (__u32 __user *)argp)) -+ return -EFAULT; -+ if (!owner) -+ return -EINVAL; -+ -+ rcu_read_lock(); -+ -+ xa_for_each(&dev->objects, id, obj) { -+ if (!kref_get_unless_zero(&obj->refcount)) -+ continue; -+ -+ if (obj->type != WINESYNC_TYPE_MUTEX) { -+ put_obj(obj); -+ continue; -+ } -+ -+ if (atomic_read(&obj->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&obj->lock); -+ -+ if (obj->u.mutex.owner == owner) { -+ put_mutex_ownerdead_state(obj); -+ try_wake_all_obj(dev, obj); -+ try_wake_any_mutex(obj); -+ } -+ -+ spin_unlock(&obj->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&obj->lock); -+ -+ if (obj->u.mutex.owner == owner) { -+ put_mutex_ownerdead_state(obj); -+ try_wake_any_mutex(obj); -+ } -+ -+ spin_unlock(&obj->lock); -+ } -+ -+ put_obj(obj); -+ } -+ -+ rcu_read_unlock(); -+ -+ return 0; -+} -+ -+static int winesync_set_event(struct winesync_device *dev, void __user *argp, -+ bool pulse) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ bool prev_state; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ if (atomic_read(&event->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ try_wake_all_obj(dev, event); -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ } -+ -+ put_obj(event); -+ -+ if (put_user(prev_state, &user_args->signaled)) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+static int winesync_reset_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ bool prev_state; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ -+ put_obj(event); -+ -+ if (put_user(prev_state, &user_args->signaled)) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) -+{ -+ int ret = 0; -+ -+ do { -+ if (signal_pending(current)) { -+ ret = -ERESTARTSYS; -+ break; -+ } -+ -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (atomic_read(&q->signaled) != -1) { -+ ret = 0; -+ break; -+ } -+ ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); -+ } while (ret < 0); -+ __set_current_state(TASK_RUNNING); -+ -+ return ret; -+} -+ -+/* -+ * Allocate and initialize the winesync_q structure, but do not queue us yet. -+ * Also, calculate the relative timeout. -+ */ -+static int setup_wait(struct winesync_device *dev, -+ const struct winesync_wait_args *args, bool all, -+ ktime_t *ret_timeout, struct winesync_q **ret_q) -+{ -+ const __u32 count = args->count; -+ struct winesync_q *q; -+ ktime_t timeout = 0; -+ __u32 total_count; -+ __u32 *ids; -+ __u32 i, j; -+ -+ if (!args->owner) -+ return -EINVAL; -+ -+ if (args->timeout) { -+ struct timespec64 to; -+ -+ if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) -+ return -EFAULT; -+ if (!timespec64_valid(&to)) -+ return -EINVAL; -+ -+ timeout = timespec64_to_ns(&to); -+ } -+ -+ total_count = count; -+ if (args->alert) -+ total_count++; -+ -+ ids = kmalloc_array(total_count, sizeof(*ids), GFP_KERNEL); -+ if (!ids) -+ return -ENOMEM; -+ if (copy_from_user(ids, u64_to_user_ptr(args->objs), -+ array_size(count, sizeof(*ids)))) { -+ kfree(ids); -+ return -EFAULT; -+ } -+ if (args->alert) -+ ids[count] = args->alert; -+ -+ q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); -+ if (!q) { -+ kfree(ids); -+ return -ENOMEM; -+ } -+ q->task = current; -+ q->owner = args->owner; -+ atomic_set(&q->signaled, -1); -+ q->all = all; -+ q->ownerdead = false; -+ q->count = count; -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = get_obj(dev, ids[i]); -+ -+ if (!obj) -+ goto err; -+ -+ if (all) { -+ /* Check that the objects are all distinct. */ -+ for (j = 0; j < i; j++) { -+ if (obj == q->entries[j].obj) { -+ put_obj(obj); -+ goto err; -+ } -+ } -+ } -+ -+ entry->obj = obj; -+ entry->q = q; -+ entry->index = i; -+ } -+ -+ kfree(ids); -+ -+ *ret_q = q; -+ *ret_timeout = timeout; -+ return 0; -+ -+err: -+ for (j = 0; j < i; j++) -+ put_obj(q->entries[j].obj); -+ kfree(ids); -+ kfree(q); -+ return -EINVAL; -+} -+ -+static void try_wake_any_obj(struct winesync_obj *obj) -+{ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ try_wake_any_sem(obj); -+ break; -+ case WINESYNC_TYPE_MUTEX: -+ try_wake_any_mutex(obj); -+ break; -+ case WINESYNC_TYPE_EVENT: -+ try_wake_any_event(obj); -+ break; -+ } -+} -+ -+static int winesync_wait_any(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_wait_args args; -+ struct winesync_q *q; -+ __u32 i, total_count; -+ ktime_t timeout; -+ int signaled; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, false, &timeout, &q); -+ if (ret < 0) -+ return ret; -+ -+ total_count = args.count; -+ if (args.alert) -+ total_count++; -+ -+ /* queue ourselves */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* -+ * Check if we are already signaled. -+ * -+ * Note that the API requires that normal objects are checked before -+ * the alert event. Hence we queue the alert event last, and check -+ * objects in order. -+ */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_obj *obj = q->entries[i].obj; -+ -+ if (atomic_read(&q->signaled) != -1) -+ break; -+ -+ spin_lock(&obj->lock); -+ try_wake_any_obj(obj); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* sleep */ -+ -+ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); -+ -+ /* and finally, unqueue */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_del(&entry->node); -+ spin_unlock(&obj->lock); -+ -+ put_obj(obj); -+ } -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct winesync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; -+} -+ -+static int winesync_wait_all(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_wait_args args; -+ struct winesync_q *q; -+ ktime_t timeout; -+ int signaled; -+ __u32 i; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, true, &timeout, &q); -+ if (ret < 0) -+ return ret; -+ -+ /* queue ourselves */ -+ -+ spin_lock(&dev->wait_all_lock); -+ -+ for (i = 0; i < args.count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ atomic_inc(&obj->all_hint); -+ -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. -+ */ -+ list_add_tail(&entry->node, &obj->all_waiters); -+ } -+ if (args.alert) { -+ struct winesync_q_entry *entry = &q->entries[args.count]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* check if we are already signaled */ -+ -+ try_wake_all(dev, q, NULL); -+ -+ spin_unlock(&dev->wait_all_lock); -+ -+ /* -+ * Check if the alert event is signaled, making sure to do so only -+ * after checking if the other objects are signaled. -+ */ -+ -+ if (args.alert) { -+ struct winesync_obj *obj = q->entries[args.count].obj; -+ -+ if (atomic_read(&q->signaled) == -1) { -+ spin_lock(&obj->lock); -+ try_wake_any_obj(obj); -+ spin_unlock(&obj->lock); -+ } -+ } -+ -+ /* sleep */ -+ -+ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); -+ -+ /* and finally, unqueue */ -+ -+ spin_lock(&dev->wait_all_lock); -+ -+ for (i = 0; i < args.count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. -+ */ -+ list_del(&entry->node); -+ -+ atomic_dec(&obj->all_hint); -+ -+ put_obj(obj); -+ } -+ if (args.alert) { -+ struct winesync_q_entry *entry = &q->entries[args.count]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_del(&entry->node); -+ spin_unlock(&obj->lock); -+ -+ put_obj(obj); -+ } -+ -+ spin_unlock(&dev->wait_all_lock); -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct winesync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; -+} -+ -+static long winesync_char_ioctl(struct file *file, unsigned int cmd, -+ unsigned long parm) -+{ -+ struct winesync_device *dev = file->private_data; -+ void __user *argp = (void __user *)parm; -+ -+ switch (cmd) { -+ case WINESYNC_IOC_CREATE_EVENT: -+ return winesync_create_event(dev, argp); -+ case WINESYNC_IOC_CREATE_MUTEX: -+ return winesync_create_mutex(dev, argp); -+ case WINESYNC_IOC_CREATE_SEM: -+ return winesync_create_sem(dev, argp); -+ case WINESYNC_IOC_DELETE: -+ return winesync_delete(dev, argp); -+ case WINESYNC_IOC_KILL_OWNER: -+ return winesync_kill_owner(dev, argp); -+ case WINESYNC_IOC_PULSE_EVENT: -+ return winesync_set_event(dev, argp, true); -+ case WINESYNC_IOC_PUT_MUTEX: -+ return winesync_put_mutex(dev, argp); -+ case WINESYNC_IOC_PUT_SEM: -+ return winesync_put_sem(dev, argp); -+ case WINESYNC_IOC_READ_EVENT: -+ return winesync_read_event(dev, argp); -+ case WINESYNC_IOC_READ_MUTEX: -+ return winesync_read_mutex(dev, argp); -+ case WINESYNC_IOC_READ_SEM: -+ return winesync_read_sem(dev, argp); -+ case WINESYNC_IOC_RESET_EVENT: -+ return winesync_reset_event(dev, argp); -+ case WINESYNC_IOC_SET_EVENT: -+ return winesync_set_event(dev, argp, false); -+ case WINESYNC_IOC_WAIT_ALL: -+ return winesync_wait_all(dev, argp); -+ case WINESYNC_IOC_WAIT_ANY: -+ return winesync_wait_any(dev, argp); -+ default: -+ return -ENOSYS; -+ } -+} -+ -+static const struct file_operations winesync_fops = { -+ .owner = THIS_MODULE, -+ .open = winesync_char_open, -+ .release = winesync_char_release, -+ .unlocked_ioctl = winesync_char_ioctl, -+ .compat_ioctl = winesync_char_ioctl, -+ .llseek = no_llseek, -+}; -+ -+static struct miscdevice winesync_misc = { -+ .minor = WINESYNC_MINOR, -+ .name = WINESYNC_NAME, -+ .fops = &winesync_fops, -+}; -+ -+static int __init winesync_init(void) -+{ -+ return misc_register(&winesync_misc); -+} -+ -+static void __exit winesync_exit(void) -+{ -+ misc_deregister(&winesync_misc); -+} -+ -+module_init(winesync_init); -+module_exit(winesync_exit); -+ -+MODULE_AUTHOR("Zebediah Figura"); -+MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); -+MODULE_LICENSE("GPL"); -+MODULE_ALIAS("devname:" WINESYNC_NAME); -+MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); -diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h -index c0fea6ca5076..36fc5d5315a4 100644 ---- a/include/linux/miscdevice.h -+++ b/include/linux/miscdevice.h -@@ -71,6 +71,7 @@ - #define USERIO_MINOR 240 - #define VHOST_VSOCK_MINOR 241 - #define RFKILL_MINOR 242 -+#define WINESYNC_MINOR 243 - #define MISC_DYNAMIC_MINOR 255 - - struct device; -diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h -new file mode 100644 -index 000000000000..5b4e369f7469 ---- /dev/null -+++ b/include/uapi/linux/winesync.h -@@ -0,0 +1,71 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * Kernel support for Wine synchronization primitives -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ -+ -+#ifndef __LINUX_WINESYNC_H -+#define __LINUX_WINESYNC_H -+ -+#include -+ -+struct winesync_sem_args { -+ __u32 sem; -+ __u32 count; -+ __u32 max; -+}; -+ -+struct winesync_mutex_args { -+ __u32 mutex; -+ __u32 owner; -+ __u32 count; -+}; -+ -+struct winesync_event_args { -+ __u32 event; -+ __u32 manual; -+ __u32 signaled; -+}; -+ -+struct winesync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 alert; -+}; -+ -+#define WINESYNC_IOC_BASE 0xf7 -+ -+#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 1, __u32) -+#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 2, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 3, \ -+ struct winesync_wait_args) -+#define WINESYNC_IOC_WAIT_ALL _IOWR(WINESYNC_IOC_BASE, 4, \ -+ struct winesync_wait_args) -+#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 5, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 7, __u32) -+#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 8, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 9, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_CREATE_EVENT _IOWR(WINESYNC_IOC_BASE, 10, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_SET_EVENT _IOWR(WINESYNC_IOC_BASE, 11, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_RESET_EVENT _IOWR(WINESYNC_IOC_BASE, 12, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_PULSE_EVENT _IOWR(WINESYNC_IOC_BASE, 13, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_READ_EVENT _IOWR(WINESYNC_IOC_BASE, 14, \ -+ struct winesync_event_args) -+ -+#endif -diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile -index 1fc89b8ef433..c7d3d9f5e0a0 100644 ---- a/tools/testing/selftests/Makefile -+++ b/tools/testing/selftests/Makefile -@@ -14,6 +14,7 @@ TARGETS += drivers/dma-buf - TARGETS += drivers/s390x/uvdevice - TARGETS += drivers/net/bonding - TARGETS += drivers/net/team -+TARGETS += drivers/winesync - TARGETS += efivarfs - TARGETS += exec - TARGETS += filesystems -diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile -new file mode 100644 -index 000000000000..43b39fdeea10 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/Makefile -@@ -0,0 +1,8 @@ -+# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only -+TEST_GEN_PROGS := winesync -+ -+top_srcdir =../../../../.. -+CFLAGS += -I$(top_srcdir)/usr/include -+LDLIBS += -lpthread -+ -+include ../../lib.mk -diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config -new file mode 100644 -index 000000000000..60539c826d06 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/config -@@ -0,0 +1 @@ -+CONFIG_WINESYNC=y -diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c -new file mode 100644 -index 000000000000..169e922484b0 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/winesync.c -@@ -0,0 +1,1479 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Various unit tests for the "winesync" synchronization primitive driver. -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ -+ -+#define _GNU_SOURCE -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../kselftest_harness.h" -+ -+static int read_sem_state(int fd, __u32 sem, __u32 *count, __u32 *max) -+{ -+ struct winesync_sem_args args; -+ int ret; -+ -+ args.sem = sem; -+ args.count = 0xdeadbeef; -+ args.max = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &args); -+ *count = args.count; -+ *max = args.max; -+ return ret; -+} -+ -+#define check_sem_state(fd, sem, count, max) \ -+ ({ \ -+ __u32 __count, __max; \ -+ int ret = read_sem_state((fd), (sem), &__count, &__max); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((max), __max); \ -+ }) -+ -+static int put_sem(int fd, __u32 sem, __u32 *count) -+{ -+ struct winesync_sem_args args; -+ int ret; -+ -+ args.sem = sem; -+ args.count = *count; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &args); -+ *count = args.count; -+ return ret; -+} -+ -+static int read_mutex_state(int fd, __u32 mutex, __u32 *count, __u32 *owner) -+{ -+ struct winesync_mutex_args args; -+ int ret; -+ -+ args.mutex = mutex; -+ args.count = 0xdeadbeef; -+ args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &args); -+ *count = args.count; -+ *owner = args.owner; -+ return ret; -+} -+ -+#define check_mutex_state(fd, mutex, count, owner) \ -+ ({ \ -+ __u32 __count, __owner; \ -+ int ret = read_mutex_state((fd), (mutex), &__count, &__owner); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((owner), __owner); \ -+ }) -+ -+static int put_mutex(int fd, __u32 mutex, __u32 owner, __u32 *count) -+{ -+ struct winesync_mutex_args args; -+ int ret; -+ -+ args.mutex = mutex; -+ args.owner = owner; -+ args.count = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &args); -+ *count = args.count; -+ return ret; -+} -+ -+static int read_event_state(int fd, __u32 event, __u32 *signaled, __u32 *manual) -+{ -+ struct winesync_event_args args; -+ int ret; -+ -+ args.event = event; -+ args.signaled = 0xdeadbeef; -+ args.manual = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &args); -+ *signaled = args.signaled; -+ *manual = args.manual; -+ return ret; -+} -+ -+#define check_event_state(fd, event, signaled, manual) \ -+ ({ \ -+ __u32 __signaled, __manual; \ -+ int ret = read_event_state((fd), (event), \ -+ &__signaled, &__manual); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((signaled), __signaled); \ -+ EXPECT_EQ((manual), __manual); \ -+ }) -+ -+static int wait_objs(int fd, unsigned long request, __u32 count, -+ const __u32 *objs, __u32 owner, __u32 alert, __u32 *index) -+{ -+ struct winesync_wait_args args = {0}; -+ struct timespec timeout; -+ int ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ args.timeout = (uintptr_t)&timeout; -+ args.count = count; -+ args.objs = (uintptr_t)objs; -+ args.owner = owner; -+ args.index = 0xdeadbeef; -+ args.alert = alert; -+ ret = ioctl(fd, request, &args); -+ *index = args.index; -+ return ret; -+} -+ -+static int wait_any(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, -+ count, objs, owner, 0, index); -+} -+ -+static int wait_all(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, -+ count, objs, owner, 0, index); -+} -+ -+static int wait_any_alert(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 alert, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, -+ count, objs, owner, alert, index); -+} -+ -+static int wait_all_alert(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 alert, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, -+ count, objs, owner, alert, index); -+} -+ -+TEST(semaphore_state) -+{ -+ struct winesync_sem_args sem_args; -+ struct timespec timeout; -+ __u32 sem, count, index; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 3; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ check_sem_state(fd, sem, 2, 2); -+ -+ count = 0; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_sem_state(fd, sem, 2, 2); -+ -+ count = 1; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(fd, sem, 2, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem, 1, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem, 0, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ count = 3; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(fd, sem, 0, 2); -+ -+ count = 2; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(fd, sem, 2, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ -+ count = 1; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(fd, sem, 1, 2); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(mutex_state) -+{ -+ struct winesync_mutex_args mutex_args; -+ __u32 mutex, owner, count, index; -+ struct timespec timeout; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 0; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 2; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ mutex = mutex_args.mutex; -+ check_mutex_state(fd, mutex, 2, 123); -+ -+ ret = put_mutex(fd, mutex, 0, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = put_mutex(fd, mutex, 456, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ check_mutex_state(fd, mutex, 2, 123); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ check_mutex_state(fd, mutex, 0, 0); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 2, 456); -+ -+ ret = put_mutex(fd, mutex, 456, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 0; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ owner = 456; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ mutex = mutex_args.mutex; -+ check_mutex_state(fd, mutex, 0, 0); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(manual_event_state) -+{ -+ struct winesync_event_args event_args; -+ __u32 index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 1; -+ event_args.signaled = 0; -+ event_args.event = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, event_args.event); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(auto_event_state) -+{ -+ struct winesync_event_args event_args; -+ __u32 index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 0; -+ event_args.signaled = 1; -+ event_args.event = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, event_args.event); -+ -+ check_event_state(fd, event_args.event, 1, 0); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 0); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(test_wait_any) -+{ -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2], owner, index; -+ struct timespec timeout; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ /* test waiting on the same object twice */ -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ objs[0] = objs[1] = sem_args.sem; -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, wait_args.index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ -+ ret = wait_any(fd, 0, NULL, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(test_wait_all) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2], owner, index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ ret = wait_all(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); -+ -+ sem_args.count = 3; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 2, 3); -+ check_mutex_state(fd, mutex_args.mutex, 3, 123); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = event_args.event; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ /* test waiting on the same object twice */ -+ objs[0] = objs[1] = sem_args.sem; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(invalid_objects) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2] = {0}; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ sem_args.max = 1; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.mutex = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ event_args.event = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = sem_args.sem + 1; -+ wait_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ objs[0] = sem_args.sem + 1; -+ objs[1] = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ -+ sem_args.sem = mutex_args.mutex; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+struct wake_args -+{ -+ int fd; -+ __u32 obj; -+}; -+ -+struct wait_args -+{ -+ int fd; -+ unsigned long request; -+ struct winesync_wait_args *args; -+ int ret; -+ int err; -+}; -+ -+static void *wait_thread(void *arg) -+{ -+ struct wait_args *args = arg; -+ -+ args->ret = ioctl(args->fd, args->request, args->args); -+ args->err = errno; -+ return NULL; -+} -+ -+static void get_abs_timeout(struct timespec *timeout, clockid_t clock, -+ unsigned int ms) -+{ -+ clock_gettime(clock, timeout); -+ timeout->tv_nsec += ms * 1000000; -+ timeout->tv_sec += (timeout->tv_nsec / 1000000000); -+ timeout->tv_nsec %= 1000000000; -+} -+ -+static int wait_for_thread(pthread_t thread, unsigned int ms) -+{ -+ struct timespec timeout; -+ get_abs_timeout(&timeout, CLOCK_REALTIME, ms); -+ return pthread_timedjoin_np(thread, NULL, &timeout); -+} -+ -+TEST(wake_any) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 objs[2], count, index; -+ struct timespec timeout; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ -+ /* test waking the semaphore */ -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 456; -+ wait_args.index = 0xdeadbeef; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(0, wait_args.index); -+ -+ /* test waking the mutex */ -+ -+ /* first grab it again for owner 123 */ -+ ret = wait_any(fd, 1, &mutex_args.mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.owner = 456; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, mutex_args.count); -+ check_mutex_state(fd, mutex_args.mutex, 1, 456); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ /* test waking events */ -+ -+ event_args.manual = false; -+ event_args.signaled = false; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[1] = event_args.event; -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ event_args.manual = true; -+ event_args.signaled = false; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[1] = event_args.event; -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ /* delete an object while it's being waited on */ -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); -+ wait_args.owner = 123; -+ objs[1] = mutex_args.mutex; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); -+ -+ close(fd); -+} -+ -+TEST(wake_all) -+{ -+ struct winesync_event_args manual_event_args = {0}; -+ struct winesync_event_args auto_event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 objs[4], count, index; -+ struct timespec timeout; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ manual_event_args.manual = true; -+ manual_event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ -+ auto_event_args.manual = false; -+ auto_event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ objs[2] = manual_event_args.event; -+ objs[3] = auto_event_args.event; -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 4; -+ wait_args.owner = 456; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ -+ ret = wait_any(fd, 1, &sem_args.sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, manual_event_args.signaled); -+ -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ check_sem_state(fd, sem_args.sem, 2, 3); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, auto_event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, manual_event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, auto_event_args.signaled); -+ -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 456); -+ check_event_state(fd, manual_event_args.event, 1, 1); -+ check_event_state(fd, auto_event_args.event, 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ -+ /* delete an object while it's being waited on */ -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); -+ wait_args.owner = 123; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &manual_event_args.event); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &auto_event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); -+ -+ close(fd); -+} -+ -+TEST(alert_any) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ struct timespec timeout; -+ __u32 objs[2], index; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[0] = sem_args.sem; -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[1] = sem_args.sem; -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ /* test wakeup via alert */ -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event_args.event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ /* test with an auto-reset event */ -+ -+ event_args.manual = false; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ sem_args.sem = objs[0]; -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(alert_all) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ struct timespec timeout; -+ __u32 objs[2], index; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[0] = sem_args.sem; -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[1] = sem_args.sem; -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ /* test wakeup via alert */ -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event_args.event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ /* test with an auto-reset event */ -+ -+ event_args.manual = false; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ sem_args.sem = objs[1]; -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST_HARNESS_MAIN --- -2.38.0.rc1.8.g2a7d63a245 - -From e28474f737770960d42ba4e4bddc8dd6eb52590b Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Fri, 5 Aug 2022 19:33:47 +0200 -Subject: [PATCH 04/13] Introducing-OpenVPN-Data-Channel-Offload - -Signed-off-by: Peter Jung ---- - MAINTAINERS | 8 + - drivers/net/Kconfig | 19 + - drivers/net/Makefile | 1 + - drivers/net/ovpn-dco/Makefile | 21 + - drivers/net/ovpn-dco/addr.h | 41 + - drivers/net/ovpn-dco/bind.c | 62 ++ - drivers/net/ovpn-dco/bind.h | 67 ++ - drivers/net/ovpn-dco/crypto.c | 154 ++++ - drivers/net/ovpn-dco/crypto.h | 144 ++++ - drivers/net/ovpn-dco/crypto_aead.c | 367 +++++++++ - drivers/net/ovpn-dco/crypto_aead.h | 27 + - drivers/net/ovpn-dco/main.c | 271 +++++++ - drivers/net/ovpn-dco/main.h | 32 + - drivers/net/ovpn-dco/netlink.c | 1143 ++++++++++++++++++++++++++++ - drivers/net/ovpn-dco/netlink.h | 22 + - drivers/net/ovpn-dco/ovpn.c | 600 +++++++++++++++ - drivers/net/ovpn-dco/ovpn.h | 43 ++ - drivers/net/ovpn-dco/ovpnstruct.h | 59 ++ - drivers/net/ovpn-dco/peer.c | 906 ++++++++++++++++++++++ - drivers/net/ovpn-dco/peer.h | 168 ++++ - drivers/net/ovpn-dco/pktid.c | 127 ++++ - drivers/net/ovpn-dco/pktid.h | 116 +++ - drivers/net/ovpn-dco/proto.h | 101 +++ - drivers/net/ovpn-dco/rcu.h | 21 + - drivers/net/ovpn-dco/skb.h | 54 ++ - drivers/net/ovpn-dco/sock.c | 134 ++++ - drivers/net/ovpn-dco/sock.h | 54 ++ - drivers/net/ovpn-dco/stats.c | 20 + - drivers/net/ovpn-dco/stats.h | 67 ++ - drivers/net/ovpn-dco/tcp.c | 326 ++++++++ - drivers/net/ovpn-dco/tcp.h | 38 + - drivers/net/ovpn-dco/udp.c | 343 +++++++++ - drivers/net/ovpn-dco/udp.h | 25 + - include/net/netlink.h | 1 + - include/uapi/linux/ovpn_dco.h | 265 +++++++ - include/uapi/linux/udp.h | 1 + - 36 files changed, 5848 insertions(+) - create mode 100644 drivers/net/ovpn-dco/Makefile - create mode 100644 drivers/net/ovpn-dco/addr.h - create mode 100644 drivers/net/ovpn-dco/bind.c - create mode 100644 drivers/net/ovpn-dco/bind.h - create mode 100644 drivers/net/ovpn-dco/crypto.c - create mode 100644 drivers/net/ovpn-dco/crypto.h - create mode 100644 drivers/net/ovpn-dco/crypto_aead.c - create mode 100644 drivers/net/ovpn-dco/crypto_aead.h - create mode 100644 drivers/net/ovpn-dco/main.c - create mode 100644 drivers/net/ovpn-dco/main.h - create mode 100644 drivers/net/ovpn-dco/netlink.c - create mode 100644 drivers/net/ovpn-dco/netlink.h - create mode 100644 drivers/net/ovpn-dco/ovpn.c - create mode 100644 drivers/net/ovpn-dco/ovpn.h - create mode 100644 drivers/net/ovpn-dco/ovpnstruct.h - create mode 100644 drivers/net/ovpn-dco/peer.c - create mode 100644 drivers/net/ovpn-dco/peer.h - create mode 100644 drivers/net/ovpn-dco/pktid.c - create mode 100644 drivers/net/ovpn-dco/pktid.h - create mode 100644 drivers/net/ovpn-dco/proto.h - create mode 100644 drivers/net/ovpn-dco/rcu.h - create mode 100644 drivers/net/ovpn-dco/skb.h - create mode 100644 drivers/net/ovpn-dco/sock.c - create mode 100644 drivers/net/ovpn-dco/sock.h - create mode 100644 drivers/net/ovpn-dco/stats.c - create mode 100644 drivers/net/ovpn-dco/stats.h - create mode 100644 drivers/net/ovpn-dco/tcp.c - create mode 100644 drivers/net/ovpn-dco/tcp.h - create mode 100644 drivers/net/ovpn-dco/udp.c - create mode 100644 drivers/net/ovpn-dco/udp.h - create mode 100644 include/uapi/linux/ovpn_dco.h - -diff --git a/MAINTAINERS b/MAINTAINERS -index 31a7aa60cdc3..a29c9731350c 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -15320,6 +15320,14 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git - F: Documentation/filesystems/overlayfs.rst - F: fs/overlayfs/ - -+OVPN-DCO NETWORK DRIVER -+M: Antonio Quartulli -+L: openvpn-devel@lists.sourceforge.net (moderated for non-subscribers) -+L: netdev@vger.kernel.org -+S: Maintained -+F: drivers/net/ovpn-dco/ -+F: include/uapi/linux/ovpn_dco.h -+ - P54 WIRELESS DRIVER - M: Christian Lamparter - L: linux-wireless@vger.kernel.org -diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig -index 94c889802566..349866bd4448 100644 ---- a/drivers/net/Kconfig -+++ b/drivers/net/Kconfig -@@ -116,6 +116,25 @@ config WIREGUARD_DEBUG - - Say N here unless you know what you're doing. - -+config OVPN_DCO -+ tristate "OpenVPN data channel offload" -+ depends on NET && INET -+ select NET_UDP_TUNNEL -+ select DST_CACHE -+ select CRYPTO -+ select CRYPTO_AES -+ select CRYPTO_GCM -+ select CRYPTO_CHACHA20POLY1305 -+ help -+ This module enhances the performance of an OpenVPN connection by -+ allowing the user to offload the data channel processing to -+ kernelspace. -+ Connection handshake, parameters negotiation and other non-data -+ related mechanisms are still performed in userspace. -+ -+ The OpenVPN userspace software at version 2.6 or higher is required -+ to use this functionality. -+ - config EQUALIZER - tristate "EQL (serial line load balancing) support" - help -diff --git a/drivers/net/Makefile b/drivers/net/Makefile -index 3f1192d3c52d..8ed151e8d233 100644 ---- a/drivers/net/Makefile -+++ b/drivers/net/Makefile -@@ -11,6 +11,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/ - obj-$(CONFIG_IPVTAP) += ipvlan/ - obj-$(CONFIG_DUMMY) += dummy.o - obj-$(CONFIG_WIREGUARD) += wireguard/ -+obj-$(CONFIG_OVPN_DCO) += ovpn-dco/ - obj-$(CONFIG_EQUALIZER) += eql.o - obj-$(CONFIG_IFB) += ifb.o - obj-$(CONFIG_MACSEC) += macsec.o -diff --git a/drivers/net/ovpn-dco/Makefile b/drivers/net/ovpn-dco/Makefile -new file mode 100644 -index 000000000000..7efefe8f13a9 ---- /dev/null -+++ b/drivers/net/ovpn-dco/Makefile -@@ -0,0 +1,21 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# ovpn-dco -- OpenVPN data channel offload -+# -+# Copyright (C) 2020-2022 OpenVPN, Inc. -+# -+# Author: Antonio Quartulli -+ -+obj-$(CONFIG_OVPN_DCO) += ovpn-dco.o -+ovpn-dco-y += main.o -+ovpn-dco-y += bind.o -+ovpn-dco-y += crypto.o -+ovpn-dco-y += ovpn.o -+ovpn-dco-y += peer.o -+ovpn-dco-y += sock.o -+ovpn-dco-y += stats.o -+ovpn-dco-y += netlink.o -+ovpn-dco-y += crypto_aead.o -+ovpn-dco-y += pktid.o -+ovpn-dco-y += tcp.o -+ovpn-dco-y += udp.o -diff --git a/drivers/net/ovpn-dco/addr.h b/drivers/net/ovpn-dco/addr.h -new file mode 100644 -index 000000000000..3d6ad0fc15af ---- /dev/null -+++ b/drivers/net/ovpn-dco/addr.h -@@ -0,0 +1,41 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNADDR_H_ -+#define _NET_OVPN_DCO_OVPNADDR_H_ -+ -+#include "crypto.h" -+ -+#include -+#include -+#include -+#include -+ -+/* our basic transport layer address */ -+struct ovpn_sockaddr { -+ union { -+ struct sockaddr_in in4; -+ struct sockaddr_in6 in6; -+ }; -+}; -+ -+/* Translate skb->protocol value to AF_INET or AF_INET6 */ -+static inline unsigned short skb_protocol_to_family(const struct sk_buff *skb) -+{ -+ switch (skb->protocol) { -+ case htons(ETH_P_IP): -+ return AF_INET; -+ case htons(ETH_P_IPV6): -+ return AF_INET6; -+ default: -+ return 0; -+ } -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNADDR_H_ */ -diff --git a/drivers/net/ovpn-dco/bind.c b/drivers/net/ovpn-dco/bind.c -new file mode 100644 -index 000000000000..107697ea983e ---- /dev/null -+++ b/drivers/net/ovpn-dco/bind.c -@@ -0,0 +1,62 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2012-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "ovpn.h" -+#include "bind.h" -+#include "peer.h" -+ -+#include -+#include -+#include -+#include -+ -+/* Given a remote sockaddr, compute the skb hash -+ * and get a dst_entry so we can send packets to the remote. -+ * Called from process context or softirq (must be indicated with -+ * process_context bool). -+ */ -+struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *ss) -+{ -+ struct ovpn_bind *bind; -+ size_t sa_len; -+ -+ if (ss->ss_family == AF_INET) -+ sa_len = sizeof(struct sockaddr_in); -+ else if (ss->ss_family == AF_INET6) -+ sa_len = sizeof(struct sockaddr_in6); -+ else -+ return ERR_PTR(-EAFNOSUPPORT); -+ -+ bind = kzalloc(sizeof(*bind), GFP_ATOMIC); -+ if (unlikely(!bind)) -+ return ERR_PTR(-ENOMEM); -+ -+ memcpy(&bind->sa, ss, sa_len); -+ -+ return bind; -+} -+ -+static void ovpn_bind_release_rcu(struct rcu_head *head) -+{ -+ struct ovpn_bind *bind = container_of(head, struct ovpn_bind, rcu); -+ -+ kfree(bind); -+} -+ -+void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *new) -+{ -+ struct ovpn_bind *old; -+ -+ spin_lock_bh(&peer->lock); -+ old = rcu_replace_pointer(peer->bind, new, true); -+ spin_unlock_bh(&peer->lock); -+ -+ if (old) -+ call_rcu(&old->rcu, ovpn_bind_release_rcu); -+} -diff --git a/drivers/net/ovpn-dco/bind.h b/drivers/net/ovpn-dco/bind.h -new file mode 100644 -index 000000000000..a562e471acae ---- /dev/null -+++ b/drivers/net/ovpn-dco/bind.h -@@ -0,0 +1,67 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OVPN -- OpenVPN protocol accelerator for Linux -+ * Copyright (C) 2012-2022 OpenVPN, Inc. -+ * All rights reserved. -+ * Author: James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNBIND_H_ -+#define _NET_OVPN_DCO_OVPNBIND_H_ -+ -+#include "addr.h" -+#include "rcu.h" -+ -+#include -+#include -+#include -+ -+struct ovpn_peer; -+ -+struct ovpn_bind { -+ struct ovpn_sockaddr sa; /* remote sockaddr */ -+ -+ union { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } local; -+ -+ struct rcu_head rcu; -+}; -+ -+static inline bool ovpn_bind_skb_src_match(const struct ovpn_bind *bind, struct sk_buff *skb) -+{ -+ const unsigned short family = skb_protocol_to_family(skb); -+ const struct ovpn_sockaddr *sa = &bind->sa; -+ -+ if (unlikely(!bind)) -+ return false; -+ -+ if (unlikely(sa->in4.sin_family != family)) -+ return false; -+ -+ switch (family) { -+ case AF_INET: -+ if (unlikely(sa->in4.sin_addr.s_addr != ip_hdr(skb)->saddr)) -+ return false; -+ -+ if (unlikely(sa->in4.sin_port != udp_hdr(skb)->source)) -+ return false; -+ break; -+ case AF_INET6: -+ if (unlikely(!ipv6_addr_equal(&sa->in6.sin6_addr, &ipv6_hdr(skb)->saddr))) -+ return false; -+ -+ if (unlikely(sa->in6.sin6_port != udp_hdr(skb)->source)) -+ return false; -+ break; -+ default: -+ return false; -+ } -+ -+ return true; -+} -+ -+struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *sa); -+void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *bind); -+ -+#endif /* _NET_OVPN_DCO_OVPNBIND_H_ */ -diff --git a/drivers/net/ovpn-dco/crypto.c b/drivers/net/ovpn-dco/crypto.c -new file mode 100644 -index 000000000000..fcc3a351ba9d ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto.c -@@ -0,0 +1,154 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "crypto_aead.h" -+#include "crypto.h" -+ -+#include -+ -+static void ovpn_ks_destroy_rcu(struct rcu_head *head) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ ks = container_of(head, struct ovpn_crypto_key_slot, rcu); -+ ovpn_aead_crypto_key_slot_destroy(ks); -+} -+ -+void ovpn_crypto_key_slot_release(struct kref *kref) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ ks = container_of(kref, struct ovpn_crypto_key_slot, refcount); -+ call_rcu(&ks->rcu, ovpn_ks_destroy_rcu); -+} -+ -+/* can only be invoked when all peer references have been dropped (i.e. RCU -+ * release routine) -+ */ -+void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ ks = rcu_access_pointer(cs->primary); -+ if (ks) { -+ RCU_INIT_POINTER(cs->primary, NULL); -+ ovpn_crypto_key_slot_put(ks); -+ } -+ -+ ks = rcu_access_pointer(cs->secondary); -+ if (ks) { -+ RCU_INIT_POINTER(cs->secondary, NULL); -+ ovpn_crypto_key_slot_put(ks); -+ } -+ -+ mutex_destroy(&cs->mutex); -+} -+ -+/* removes the primary key from the crypto context */ -+void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ mutex_lock(&cs->mutex); -+ ks = rcu_replace_pointer(cs->primary, NULL, lockdep_is_held(&cs->mutex)); -+ ovpn_crypto_key_slot_put(ks); -+ mutex_unlock(&cs->mutex); -+} -+ -+/* Reset the ovpn_crypto_state object in a way that is atomic -+ * to RCU readers. -+ */ -+int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, -+ const struct ovpn_peer_key_reset *pkr) -+ __must_hold(cs->mutex) -+{ -+ struct ovpn_crypto_key_slot *old = NULL; -+ struct ovpn_crypto_key_slot *new; -+ -+ lockdep_assert_held(&cs->mutex); -+ -+ new = ovpn_aead_crypto_key_slot_new(&pkr->key); -+ if (IS_ERR(new)) -+ return PTR_ERR(new); -+ -+ switch (pkr->slot) { -+ case OVPN_KEY_SLOT_PRIMARY: -+ old = rcu_replace_pointer(cs->primary, new, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ case OVPN_KEY_SLOT_SECONDARY: -+ old = rcu_replace_pointer(cs->secondary, new, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ default: -+ goto free_key; -+ } -+ -+ if (old) -+ ovpn_crypto_key_slot_put(old); -+ -+ return 0; -+free_key: -+ ovpn_crypto_key_slot_put(new); -+ return -EINVAL; -+} -+ -+void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, -+ enum ovpn_key_slot slot) -+{ -+ struct ovpn_crypto_key_slot *ks = NULL; -+ -+ mutex_lock(&cs->mutex); -+ switch (slot) { -+ case OVPN_KEY_SLOT_PRIMARY: -+ ks = rcu_replace_pointer(cs->primary, NULL, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ case OVPN_KEY_SLOT_SECONDARY: -+ ks = rcu_replace_pointer(cs->secondary, NULL, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ default: -+ pr_warn("Invalid slot to release: %u\n", slot); -+ break; -+ } -+ mutex_unlock(&cs->mutex); -+ -+ if (!ks) { -+ pr_debug("Key slot already released: %u\n", slot); -+ return; -+ } -+ pr_debug("deleting key slot %u, key_id=%u\n", slot, ks->key_id); -+ -+ ovpn_crypto_key_slot_put(ks); -+} -+ -+/* this swap is not atomic, but there will be a very short time frame where the -+ * old_secondary key won't be available. This should not be a big deal as most -+ * likely both peers are already using the new primary at this point. -+ */ -+void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs) -+{ -+ const struct ovpn_crypto_key_slot *old_primary, *old_secondary; -+ -+ mutex_lock(&cs->mutex); -+ -+ old_secondary = rcu_dereference_protected(cs->secondary, -+ lockdep_is_held(&cs->mutex)); -+ old_primary = rcu_replace_pointer(cs->primary, old_secondary, -+ lockdep_is_held(&cs->mutex)); -+ rcu_assign_pointer(cs->secondary, old_primary); -+ -+ pr_debug("key swapped: %u <-> %u\n", -+ old_primary ? old_primary->key_id : 0, -+ old_secondary ? old_secondary->key_id : 0); -+ -+ mutex_unlock(&cs->mutex); -+} -diff --git a/drivers/net/ovpn-dco/crypto.h b/drivers/net/ovpn-dco/crypto.h -new file mode 100644 -index 000000000000..79f580e54a63 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto.h -@@ -0,0 +1,144 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNCRYPTO_H_ -+#define _NET_OVPN_DCO_OVPNCRYPTO_H_ -+ -+#include "main.h" -+#include "pktid.h" -+ -+#include -+#include -+ -+struct ovpn_peer; -+struct ovpn_crypto_key_slot; -+ -+/* info needed for both encrypt and decrypt directions */ -+struct ovpn_key_direction { -+ const u8 *cipher_key; -+ size_t cipher_key_size; -+ const u8 *nonce_tail; /* only needed for GCM modes */ -+ size_t nonce_tail_size; /* only needed for GCM modes */ -+}; -+ -+/* all info for a particular symmetric key (primary or secondary) */ -+struct ovpn_key_config { -+ enum ovpn_cipher_alg cipher_alg; -+ u8 key_id; -+ struct ovpn_key_direction encrypt; -+ struct ovpn_key_direction decrypt; -+}; -+ -+/* used to pass settings from netlink to the crypto engine */ -+struct ovpn_peer_key_reset { -+ enum ovpn_key_slot slot; -+ struct ovpn_key_config key; -+}; -+ -+struct ovpn_crypto_key_slot { -+ u8 key_id; -+ -+ struct crypto_aead *encrypt; -+ struct crypto_aead *decrypt; -+ struct ovpn_nonce_tail nonce_tail_xmit; -+ struct ovpn_nonce_tail nonce_tail_recv; -+ -+ struct ovpn_pktid_recv pid_recv ____cacheline_aligned_in_smp; -+ struct ovpn_pktid_xmit pid_xmit ____cacheline_aligned_in_smp; -+ struct kref refcount; -+ struct rcu_head rcu; -+}; -+ -+struct ovpn_crypto_state { -+ struct ovpn_crypto_key_slot __rcu *primary; -+ struct ovpn_crypto_key_slot __rcu *secondary; -+ -+ /* protects primary and secondary slots */ -+ struct mutex mutex; -+}; -+ -+static inline bool ovpn_crypto_key_slot_hold(struct ovpn_crypto_key_slot *ks) -+{ -+ return kref_get_unless_zero(&ks->refcount); -+} -+ -+static inline void ovpn_crypto_state_init(struct ovpn_crypto_state *cs) -+{ -+ RCU_INIT_POINTER(cs->primary, NULL); -+ RCU_INIT_POINTER(cs->secondary, NULL); -+ mutex_init(&cs->mutex); -+} -+ -+static inline struct ovpn_crypto_key_slot * -+ovpn_crypto_key_id_to_slot(const struct ovpn_crypto_state *cs, u8 key_id) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ if (unlikely(!cs)) -+ return NULL; -+ -+ rcu_read_lock(); -+ ks = rcu_dereference(cs->primary); -+ if (ks && ks->key_id == key_id) { -+ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ goto out; -+ } -+ -+ ks = rcu_dereference(cs->secondary); -+ if (ks && ks->key_id == key_id) { -+ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ goto out; -+ } -+ -+ /* when both key slots are occupied but no matching key ID is found, ks has to be reset to -+ * NULL to avoid carrying a stale pointer -+ */ -+ ks = NULL; -+out: -+ rcu_read_unlock(); -+ -+ return ks; -+} -+ -+static inline struct ovpn_crypto_key_slot * -+ovpn_crypto_key_slot_primary(const struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ rcu_read_lock(); -+ ks = rcu_dereference(cs->primary); -+ if (unlikely(ks && !ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ rcu_read_unlock(); -+ -+ return ks; -+} -+ -+void ovpn_crypto_key_slot_release(struct kref *kref); -+ -+static inline void ovpn_crypto_key_slot_put(struct ovpn_crypto_key_slot *ks) -+{ -+ kref_put(&ks->refcount, ovpn_crypto_key_slot_release); -+} -+ -+int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, -+ const struct ovpn_peer_key_reset *pkr); -+ -+void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, -+ enum ovpn_key_slot slot); -+ -+void ovpn_crypto_state_release(struct ovpn_crypto_state *cs); -+ -+void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs); -+ -+void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs); -+ -+#endif /* _NET_OVPN_DCO_OVPNCRYPTO_H_ */ -diff --git a/drivers/net/ovpn-dco/crypto_aead.c b/drivers/net/ovpn-dco/crypto_aead.c -new file mode 100644 -index 000000000000..c21bff90d748 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto_aead.c -@@ -0,0 +1,367 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "crypto_aead.h" -+#include "crypto.h" -+#include "pktid.h" -+#include "proto.h" -+#include "skb.h" -+ -+#include -+#include -+#include -+ -+#define AUTH_TAG_SIZE 16 -+ -+static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) -+{ -+ return OVPN_OP_SIZE_V2 + /* OP header size */ -+ 4 + /* Packet ID */ -+ crypto_aead_authsize(ks->encrypt); /* Auth Tag */ -+} -+ -+int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id) -+{ -+ const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); -+ const unsigned int head_size = ovpn_aead_encap_overhead(ks); -+ struct scatterlist sg[MAX_SKB_FRAGS + 2]; -+ DECLARE_CRYPTO_WAIT(wait); -+ struct aead_request *req; -+ struct sk_buff *trailer; -+ u8 iv[NONCE_SIZE]; -+ int nfrags, ret; -+ u32 pktid, op; -+ -+ /* Sample AEAD header format: -+ * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... -+ * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] -+ * [4-byte -+ * IV head] -+ */ -+ -+ /* check that there's enough headroom in the skb for packet -+ * encapsulation, after adding network header and encryption overhead -+ */ -+ if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM + head_size))) -+ return -ENOBUFS; -+ -+ /* get number of skb frags and ensure that packet data is writable */ -+ nfrags = skb_cow_data(skb, 0, &trailer); -+ if (unlikely(nfrags < 0)) -+ return nfrags; -+ -+ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) -+ return -ENOSPC; -+ -+ req = aead_request_alloc(ks->encrypt, GFP_KERNEL); -+ if (unlikely(!req)) -+ return -ENOMEM; -+ -+ /* sg table: -+ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), -+ * 1, 2, 3, ..., n: payload, -+ * n+1: auth_tag (len=tag_size) -+ */ -+ sg_init_table(sg, nfrags + 2); -+ -+ /* build scatterlist to encrypt packet payload */ -+ ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); -+ if (unlikely(nfrags != ret)) { -+ ret = -EINVAL; -+ goto free_req; -+ } -+ -+ /* append auth_tag onto scatterlist */ -+ __skb_push(skb, tag_size); -+ sg_set_buf(sg + nfrags + 1, skb->data, tag_size); -+ -+ /* obtain packet ID, which is used both as a first -+ * 4 bytes of nonce and last 4 bytes of associated data. -+ */ -+ ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); -+ if (unlikely(ret < 0)) -+ goto free_req; -+ -+ /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes nonce */ -+ ovpn_pktid_aead_write(pktid, &ks->nonce_tail_xmit, iv); -+ -+ /* make space for packet id and push it to the front */ -+ __skb_push(skb, NONCE_WIRE_SIZE); -+ memcpy(skb->data, iv, NONCE_WIRE_SIZE); -+ -+ /* add packet op as head of additional data */ -+ op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer_id); -+ __skb_push(skb, OVPN_OP_SIZE_V2); -+ BUILD_BUG_ON(sizeof(op) != OVPN_OP_SIZE_V2); -+ *((__force __be32 *)skb->data) = htonl(op); -+ -+ /* AEAD Additional data */ -+ sg_set_buf(sg, skb->data, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); -+ -+ /* setup async crypto operation */ -+ aead_request_set_tfm(req, ks->encrypt); -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP, -+ crypto_req_done, &wait); -+ aead_request_set_crypt(req, sg, sg, skb->len - head_size, iv); -+ aead_request_set_ad(req, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); -+ -+ /* encrypt it */ -+ ret = crypto_wait_req(crypto_aead_encrypt(req), &wait); -+ if (ret < 0) -+ net_err_ratelimited("%s: encrypt failed: %d\n", __func__, ret); -+ -+free_req: -+ aead_request_free(req); -+ return ret; -+} -+ -+int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) -+{ -+ const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); -+ struct scatterlist sg[MAX_SKB_FRAGS + 2]; -+ int ret, payload_len, nfrags; -+ u8 *sg_data, iv[NONCE_SIZE]; -+ unsigned int payload_offset; -+ DECLARE_CRYPTO_WAIT(wait); -+ struct aead_request *req; -+ struct sk_buff *trailer; -+ unsigned int sg_len; -+ __be32 *pid; -+ -+ payload_offset = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE + tag_size; -+ payload_len = skb->len - payload_offset; -+ -+ /* sanity check on packet size, payload size must be >= 0 */ -+ if (unlikely(payload_len < 0)) -+ return -EINVAL; -+ -+ /* Prepare the skb data buffer to be accessed up until the auth tag. -+ * This is required because this area is directly mapped into the sg list. -+ */ -+ if (unlikely(!pskb_may_pull(skb, payload_offset))) -+ return -ENODATA; -+ -+ /* get number of skb frags and ensure that packet data is writable */ -+ nfrags = skb_cow_data(skb, 0, &trailer); -+ if (unlikely(nfrags < 0)) -+ return nfrags; -+ -+ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) -+ return -ENOSPC; -+ -+ req = aead_request_alloc(ks->decrypt, GFP_KERNEL); -+ if (unlikely(!req)) -+ return -ENOMEM; -+ -+ /* sg table: -+ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), -+ * 1, 2, 3, ..., n: payload, -+ * n+1: auth_tag (len=tag_size) -+ */ -+ sg_init_table(sg, nfrags + 2); -+ -+ /* packet op is head of additional data */ -+ sg_data = skb->data; -+ sg_len = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE; -+ sg_set_buf(sg, sg_data, sg_len); -+ -+ /* build scatterlist to decrypt packet payload */ -+ ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); -+ if (unlikely(nfrags != ret)) { -+ ret = -EINVAL; -+ goto free_req; -+ } -+ -+ /* append auth_tag onto scatterlist */ -+ sg_set_buf(sg + nfrags + 1, skb->data + sg_len, tag_size); -+ -+ /* copy nonce into IV buffer */ -+ memcpy(iv, skb->data + OVPN_OP_SIZE_V2, NONCE_WIRE_SIZE); -+ memcpy(iv + NONCE_WIRE_SIZE, ks->nonce_tail_recv.u8, -+ sizeof(struct ovpn_nonce_tail)); -+ -+ /* setup async crypto operation */ -+ aead_request_set_tfm(req, ks->decrypt); -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP, -+ crypto_req_done, &wait); -+ aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); -+ -+ aead_request_set_ad(req, NONCE_WIRE_SIZE + OVPN_OP_SIZE_V2); -+ -+ /* decrypt it */ -+ ret = crypto_wait_req(crypto_aead_decrypt(req), &wait); -+ if (ret < 0) { -+ net_err_ratelimited("%s: decrypt failed: %d\n", __func__, ret); -+ goto free_req; -+ } -+ -+ /* PID sits after the op */ -+ pid = (__force __be32 *)(skb->data + OVPN_OP_SIZE_V2); -+ ret = ovpn_pktid_recv(&ks->pid_recv, ntohl(*pid), 0); -+ if (unlikely(ret < 0)) -+ goto free_req; -+ -+ /* point to encapsulated IP packet */ -+ __skb_pull(skb, payload_offset); -+ -+free_req: -+ aead_request_free(req); -+ return ret; -+} -+ -+/* Initialize a struct crypto_aead object */ -+struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, -+ const unsigned char *key, unsigned int keylen) -+{ -+ struct crypto_aead *aead; -+ int ret; -+ -+ aead = crypto_alloc_aead(alg_name, 0, 0); -+ if (IS_ERR(aead)) { -+ ret = PTR_ERR(aead); -+ pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); -+ aead = NULL; -+ goto error; -+ } -+ -+ ret = crypto_aead_setkey(aead, key, keylen); -+ if (ret) { -+ pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, keylen, ret); -+ goto error; -+ } -+ -+ ret = crypto_aead_setauthsize(aead, AUTH_TAG_SIZE); -+ if (ret) { -+ pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, ret); -+ goto error; -+ } -+ -+ /* basic AEAD assumption */ -+ if (crypto_aead_ivsize(aead) != NONCE_SIZE) { -+ pr_err("%s IV size must be %d\n", title, NONCE_SIZE); -+ ret = -EINVAL; -+ goto error; -+ } -+ -+ pr_debug("********* Cipher %s (%s)\n", alg_name, title); -+ pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); -+ pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); -+ pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); -+ pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); -+ pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); -+ -+ return aead; -+ -+error: -+ crypto_free_aead(aead); -+ return ERR_PTR(ret); -+} -+ -+void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks) -+{ -+ if (!ks) -+ return; -+ -+ crypto_free_aead(ks->encrypt); -+ crypto_free_aead(ks->decrypt); -+ kfree(ks); -+} -+ -+static struct ovpn_crypto_key_slot * -+ovpn_aead_crypto_key_slot_init(enum ovpn_cipher_alg alg, -+ const unsigned char *encrypt_key, -+ unsigned int encrypt_keylen, -+ const unsigned char *decrypt_key, -+ unsigned int decrypt_keylen, -+ const unsigned char *encrypt_nonce_tail, -+ unsigned int encrypt_nonce_tail_len, -+ const unsigned char *decrypt_nonce_tail, -+ unsigned int decrypt_nonce_tail_len, -+ u16 key_id) -+{ -+ struct ovpn_crypto_key_slot *ks = NULL; -+ const char *alg_name; -+ int ret; -+ -+ /* validate crypto alg */ -+ switch (alg) { -+ case OVPN_CIPHER_ALG_AES_GCM: -+ alg_name = "gcm(aes)"; -+ break; -+ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: -+ alg_name = "rfc7539(chacha20,poly1305)"; -+ break; -+ default: -+ return ERR_PTR(-EOPNOTSUPP); -+ } -+ -+ /* build the key slot */ -+ ks = kmalloc(sizeof(*ks), GFP_KERNEL); -+ if (!ks) -+ return ERR_PTR(-ENOMEM); -+ -+ ks->encrypt = NULL; -+ ks->decrypt = NULL; -+ kref_init(&ks->refcount); -+ ks->key_id = key_id; -+ -+ ks->encrypt = ovpn_aead_init("encrypt", alg_name, encrypt_key, -+ encrypt_keylen); -+ if (IS_ERR(ks->encrypt)) { -+ ret = PTR_ERR(ks->encrypt); -+ ks->encrypt = NULL; -+ goto destroy_ks; -+ } -+ -+ ks->decrypt = ovpn_aead_init("decrypt", alg_name, decrypt_key, -+ decrypt_keylen); -+ if (IS_ERR(ks->decrypt)) { -+ ret = PTR_ERR(ks->decrypt); -+ ks->decrypt = NULL; -+ goto destroy_ks; -+ } -+ -+ if (sizeof(struct ovpn_nonce_tail) != encrypt_nonce_tail_len || -+ sizeof(struct ovpn_nonce_tail) != decrypt_nonce_tail_len) { -+ ret = -EINVAL; -+ goto destroy_ks; -+ } -+ -+ memcpy(ks->nonce_tail_xmit.u8, encrypt_nonce_tail, -+ sizeof(struct ovpn_nonce_tail)); -+ memcpy(ks->nonce_tail_recv.u8, decrypt_nonce_tail, -+ sizeof(struct ovpn_nonce_tail)); -+ -+ /* init packet ID generation/validation */ -+ ovpn_pktid_xmit_init(&ks->pid_xmit); -+ ovpn_pktid_recv_init(&ks->pid_recv); -+ -+ return ks; -+ -+destroy_ks: -+ ovpn_aead_crypto_key_slot_destroy(ks); -+ return ERR_PTR(ret); -+} -+ -+struct ovpn_crypto_key_slot * -+ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) -+{ -+ return ovpn_aead_crypto_key_slot_init(kc->cipher_alg, -+ kc->encrypt.cipher_key, -+ kc->encrypt.cipher_key_size, -+ kc->decrypt.cipher_key, -+ kc->decrypt.cipher_key_size, -+ kc->encrypt.nonce_tail, -+ kc->encrypt.nonce_tail_size, -+ kc->decrypt.nonce_tail, -+ kc->decrypt.nonce_tail_size, -+ kc->key_id); -+} -diff --git a/drivers/net/ovpn-dco/crypto_aead.h b/drivers/net/ovpn-dco/crypto_aead.h -new file mode 100644 -index 000000000000..1e3054e7d5a4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto_aead.h -@@ -0,0 +1,27 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNAEAD_H_ -+#define _NET_OVPN_DCO_OVPNAEAD_H_ -+ -+#include "crypto.h" -+ -+#include -+#include -+ -+struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, -+ const unsigned char *key, unsigned int keylen); -+ -+int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id); -+int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); -+ -+struct ovpn_crypto_key_slot *ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc); -+void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks); -+ -+#endif /* _NET_OVPN_DCO_OVPNAEAD_H_ */ -diff --git a/drivers/net/ovpn-dco/main.c b/drivers/net/ovpn-dco/main.c -new file mode 100644 -index 000000000000..4eb90ea7a500 ---- /dev/null -+++ b/drivers/net/ovpn-dco/main.c -@@ -0,0 +1,271 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#include "main.h" -+ -+#include "ovpn.h" -+#include "ovpnstruct.h" -+#include "netlink.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+/* Driver info */ -+#define DRV_NAME "ovpn-dco" -+#define DRV_VERSION OVPN_DCO_VERSION -+#define DRV_DESCRIPTION "OpenVPN data channel offload (ovpn-dco)" -+#define DRV_COPYRIGHT "(C) 2020-2022 OpenVPN, Inc." -+ -+static void ovpn_struct_free(struct net_device *net) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(net); -+ -+ security_tun_dev_free_security(ovpn->security); -+ free_percpu(net->tstats); -+ flush_workqueue(ovpn->crypto_wq); -+ flush_workqueue(ovpn->events_wq); -+ destroy_workqueue(ovpn->crypto_wq); -+ destroy_workqueue(ovpn->events_wq); -+ rcu_barrier(); -+} -+ -+/* Net device open */ -+static int ovpn_net_open(struct net_device *dev) -+{ -+ struct in_device *dev_v4 = __in_dev_get_rtnl(dev); -+ -+ if (dev_v4) { -+ /* disable redirects as Linux gets confused by ovpn-dco handling same-LAN routing */ -+ IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); -+ IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; -+ } -+ -+ netif_tx_start_all_queues(dev); -+ return 0; -+} -+ -+/* Net device stop -- called prior to device unload */ -+static int ovpn_net_stop(struct net_device *dev) -+{ -+ netif_tx_stop_all_queues(dev); -+ return 0; -+} -+ -+/******************************************* -+ * ovpn ethtool ops -+ *******************************************/ -+ -+static int ovpn_get_link_ksettings(struct net_device *dev, -+ struct ethtool_link_ksettings *cmd) -+{ -+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 0); -+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, 0); -+ cmd->base.speed = SPEED_1000; -+ cmd->base.duplex = DUPLEX_FULL; -+ cmd->base.port = PORT_TP; -+ cmd->base.phy_address = 0; -+ cmd->base.transceiver = XCVR_INTERNAL; -+ cmd->base.autoneg = AUTONEG_DISABLE; -+ -+ return 0; -+} -+ -+static void ovpn_get_drvinfo(struct net_device *dev, -+ struct ethtool_drvinfo *info) -+{ -+ strscpy(info->driver, DRV_NAME, sizeof(info->driver)); -+ strscpy(info->version, DRV_VERSION, sizeof(info->version)); -+ strscpy(info->bus_info, "ovpn", sizeof(info->bus_info)); -+} -+ -+bool ovpn_dev_is_valid(const struct net_device *dev) -+{ -+ return dev->netdev_ops->ndo_start_xmit == ovpn_net_xmit; -+} -+ -+/******************************************* -+ * ovpn exported methods -+ *******************************************/ -+ -+static const struct net_device_ops ovpn_netdev_ops = { -+ .ndo_open = ovpn_net_open, -+ .ndo_stop = ovpn_net_stop, -+ .ndo_start_xmit = ovpn_net_xmit, -+ .ndo_get_stats64 = dev_get_tstats64, -+}; -+ -+static const struct ethtool_ops ovpn_ethtool_ops = { -+ .get_link_ksettings = ovpn_get_link_ksettings, -+ .get_drvinfo = ovpn_get_drvinfo, -+ .get_link = ethtool_op_get_link, -+ .get_ts_info = ethtool_op_get_ts_info, -+}; -+ -+static void ovpn_setup(struct net_device *dev) -+{ -+ /* compute the overhead considering AEAD encryption */ -+ const int overhead = sizeof(u32) + NONCE_WIRE_SIZE + 16 + sizeof(struct udphdr) + -+ max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); -+ -+ netdev_features_t feat = NETIF_F_SG | NETIF_F_LLTX | -+ NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_GSO | -+ NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA; -+ -+ dev->ethtool_ops = &ovpn_ethtool_ops; -+ dev->needs_free_netdev = true; -+ -+ dev->netdev_ops = &ovpn_netdev_ops; -+ -+ dev->priv_destructor = ovpn_struct_free; -+ -+ /* Point-to-Point TUN Device */ -+ dev->hard_header_len = 0; -+ dev->addr_len = 0; -+ dev->mtu = ETH_DATA_LEN - overhead; -+ dev->min_mtu = IPV4_MIN_MTU; -+ dev->max_mtu = IP_MAX_MTU - overhead; -+ -+ /* Zero header length */ -+ dev->type = ARPHRD_NONE; -+ dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; -+ -+ dev->features |= feat; -+ dev->hw_features |= feat; -+ dev->hw_enc_features |= feat; -+ -+ dev->needed_headroom = OVPN_HEAD_ROOM; -+ dev->needed_tailroom = OVPN_MAX_PADDING; -+} -+ -+static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = { -+ [IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, __OVPN_MODE_FIRST, -+ __OVPN_MODE_AFTER_LAST - 1), -+}; -+ -+static int ovpn_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], -+ struct nlattr *data[], struct netlink_ext_ack *extack) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ int ret; -+ -+ ret = security_tun_dev_create(); -+ if (ret < 0) -+ return ret; -+ -+ ret = ovpn_struct_init(dev); -+ if (ret < 0) -+ return ret; -+ -+ ovpn->mode = OVPN_MODE_P2P; -+ if (data && data[IFLA_OVPN_MODE]) { -+ ovpn->mode = nla_get_u8(data[IFLA_OVPN_MODE]); -+ netdev_dbg(dev, "%s: setting device (%s) mode: %u\n", __func__, dev->name, -+ ovpn->mode); -+ } -+ -+ return register_netdevice(dev); -+} -+ -+static void ovpn_dellink(struct net_device *dev, struct list_head *head) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ -+ switch (ovpn->mode) { -+ case OVPN_MODE_P2P: -+ ovpn_peer_release_p2p(ovpn); -+ break; -+ default: -+ ovpn_peers_free(ovpn); -+ break; -+ } -+ -+ unregister_netdevice_queue(dev, head); -+} -+ -+/** -+ * ovpn_num_queues - define number of queues to allocate per device -+ * -+ * The value returned by this function is used to decide how many RX and TX -+ * queues to allocate when creating the netdev object -+ * -+ * Return the number of queues to allocate -+ */ -+static unsigned int ovpn_num_queues(void) -+{ -+ return num_online_cpus(); -+} -+ -+static struct rtnl_link_ops ovpn_link_ops __read_mostly = { -+ .kind = DRV_NAME, -+ .priv_size = sizeof(struct ovpn_struct), -+ .setup = ovpn_setup, -+ .policy = ovpn_policy, -+ .maxtype = IFLA_OVPN_MAX, -+ .newlink = ovpn_newlink, -+ .dellink = ovpn_dellink, -+ .get_num_tx_queues = ovpn_num_queues, -+ .get_num_rx_queues = ovpn_num_queues, -+}; -+ -+static int __init ovpn_init(void) -+{ -+ int err = 0; -+ -+ pr_info("%s %s -- %s\n", DRV_DESCRIPTION, DRV_VERSION, DRV_COPYRIGHT); -+ -+ /* init RTNL link ops */ -+ err = rtnl_link_register(&ovpn_link_ops); -+ if (err) { -+ pr_err("ovpn: can't register RTNL link ops\n"); -+ goto err; -+ } -+ -+ err = ovpn_netlink_register(); -+ if (err) { -+ pr_err("ovpn: can't register netlink family\n"); -+ goto err_rtnl_unregister; -+ } -+ -+ return 0; -+ -+err_rtnl_unregister: -+ rtnl_link_unregister(&ovpn_link_ops); -+err: -+ pr_err("ovpn: initialization failed, error status=%d\n", err); -+ return err; -+} -+ -+static __exit void ovpn_cleanup(void) -+{ -+ rtnl_link_unregister(&ovpn_link_ops); -+ ovpn_netlink_unregister(); -+ rcu_barrier(); /* because we use call_rcu */ -+} -+ -+module_init(ovpn_init); -+module_exit(ovpn_cleanup); -+ -+MODULE_DESCRIPTION(DRV_DESCRIPTION); -+MODULE_AUTHOR(DRV_COPYRIGHT); -+MODULE_LICENSE("GPL"); -+MODULE_VERSION(DRV_VERSION); -+MODULE_ALIAS_RTNL_LINK(DRV_NAME); -+MODULE_ALIAS_GENL_FAMILY(OVPN_NL_NAME); -diff --git a/drivers/net/ovpn-dco/main.h b/drivers/net/ovpn-dco/main.h -new file mode 100644 -index 000000000000..c4ef200b30f4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/main.h -@@ -0,0 +1,32 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_MAIN_H_ -+#define _NET_OVPN_DCO_MAIN_H_ -+ -+#include -+#include -+#include -+#include -+ -+#define OVPN_DCO_VERSION "2.0.0" -+ -+struct net_device; -+bool ovpn_dev_is_valid(const struct net_device *dev); -+ -+#define SKB_HEADER_LEN \ -+ (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ -+ sizeof(struct udphdr) + NET_SKB_PAD) -+ -+#define OVPN_HEAD_ROOM ALIGN(16 + SKB_HEADER_LEN, 4) -+#define OVPN_MAX_PADDING 16 -+#define OVPN_QUEUE_LEN 1024 -+#define OVPN_MAX_TUN_QUEUE_LEN 0x10000 -+ -+#endif /* _NET_OVPN_DCO_OVPN_DCO_H_ */ -diff --git a/drivers/net/ovpn-dco/netlink.c b/drivers/net/ovpn-dco/netlink.c -new file mode 100644 -index 000000000000..ee5c943e7db4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/netlink.c -@@ -0,0 +1,1143 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "proto.h" -+#include "netlink.h" -+#include "ovpnstruct.h" -+#include "udp.h" -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/** The ovpn-dco netlink family */ -+static struct genl_family ovpn_netlink_family; -+ -+enum ovpn_netlink_multicast_groups { -+ OVPN_MCGRP_PEERS, -+}; -+ -+static const struct genl_multicast_group ovpn_netlink_mcgrps[] = { -+ [OVPN_MCGRP_PEERS] = { .name = OVPN_NL_MULTICAST_GROUP_PEERS }, -+}; -+ -+/** Key direction policy. Can be used for configuring an encryption and a decryption key */ -+static const struct nla_policy ovpn_netlink_policy_key_dir[OVPN_KEY_DIR_ATTR_MAX + 1] = { -+ [OVPN_KEY_DIR_ATTR_CIPHER_KEY] = NLA_POLICY_MAX_LEN(U8_MAX), -+ [OVPN_KEY_DIR_ATTR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(NONCE_TAIL_SIZE), -+}; -+ -+/** CMD_NEW_KEY policy */ -+static const struct nla_policy ovpn_netlink_policy_new_key[OVPN_NEW_KEY_ATTR_MAX + 1] = { -+ [OVPN_NEW_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_NEW_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, -+ __OVPN_KEY_SLOT_AFTER_LAST - 1), -+ [OVPN_NEW_KEY_ATTR_KEY_ID] = { .type = NLA_U8 }, -+ [OVPN_NEW_KEY_ATTR_CIPHER_ALG] = { .type = NLA_U16 }, -+ [OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), -+ [OVPN_NEW_KEY_ATTR_DECRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), -+}; -+ -+/** CMD_DEL_KEY policy */ -+static const struct nla_policy ovpn_netlink_policy_del_key[OVPN_DEL_KEY_ATTR_MAX + 1] = { -+ [OVPN_DEL_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_DEL_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, -+ __OVPN_KEY_SLOT_AFTER_LAST - 1), -+}; -+ -+/** CMD_SWAP_KEYS policy */ -+static const struct nla_policy ovpn_netlink_policy_swap_keys[OVPN_SWAP_KEYS_ATTR_MAX + 1] = { -+ [OVPN_SWAP_KEYS_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_NEW_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_new_peer[OVPN_NEW_PEER_ATTR_MAX + 1] = { -+ [OVPN_NEW_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE] = NLA_POLICY_MIN_LEN(sizeof(struct sockaddr)), -+ [OVPN_NEW_PEER_ATTR_SOCKET] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_IPV4] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_IPV6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), -+ [OVPN_NEW_PEER_ATTR_LOCAL_IP] = NLA_POLICY_MAX_LEN(sizeof(struct in6_addr)), -+}; -+ -+/** CMD_SET_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_set_peer[OVPN_SET_PEER_ATTR_MAX + 1] = { -+ [OVPN_SET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, -+ [OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_DEL_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_del_peer[OVPN_DEL_PEER_ATTR_MAX + 1] = { -+ [OVPN_DEL_PEER_ATTR_REASON] = NLA_POLICY_RANGE(NLA_U8, __OVPN_DEL_PEER_REASON_FIRST, -+ __OVPN_DEL_PEER_REASON_AFTER_LAST - 1), -+ [OVPN_DEL_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_GET_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_get_peer[OVPN_GET_PEER_ATTR_MAX + 1] = { -+ [OVPN_GET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_PACKET polocy */ -+static const struct nla_policy ovpn_netlink_policy_packet[OVPN_PACKET_ATTR_MAX + 1] = { -+ [OVPN_PACKET_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_PACKET_ATTR_PACKET] = NLA_POLICY_MAX_LEN(U16_MAX), -+}; -+ -+/** Generic message container policy */ -+static const struct nla_policy ovpn_netlink_policy[OVPN_ATTR_MAX + 1] = { -+ [OVPN_ATTR_IFINDEX] = { .type = NLA_U32 }, -+ [OVPN_ATTR_NEW_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_peer), -+ [OVPN_ATTR_SET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_set_peer), -+ [OVPN_ATTR_DEL_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_peer), -+ [OVPN_ATTR_GET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_get_peer), -+ [OVPN_ATTR_NEW_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_key), -+ [OVPN_ATTR_SWAP_KEYS] = NLA_POLICY_NESTED(ovpn_netlink_policy_swap_keys), -+ [OVPN_ATTR_DEL_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_key), -+ [OVPN_ATTR_PACKET] = NLA_POLICY_NESTED(ovpn_netlink_policy_packet), -+}; -+ -+static struct net_device * -+ovpn_get_dev_from_attrs(struct net *net, struct nlattr **attrs) -+{ -+ struct net_device *dev; -+ int ifindex; -+ -+ if (!attrs[OVPN_ATTR_IFINDEX]) -+ return ERR_PTR(-EINVAL); -+ -+ ifindex = nla_get_u32(attrs[OVPN_ATTR_IFINDEX]); -+ -+ dev = dev_get_by_index(net, ifindex); -+ if (!dev) -+ return ERR_PTR(-ENODEV); -+ -+ if (!ovpn_dev_is_valid(dev)) -+ goto err_put_dev; -+ -+ return dev; -+ -+err_put_dev: -+ dev_put(dev); -+ -+ return ERR_PTR(-EINVAL); -+} -+ -+/** -+ * ovpn_pre_doit() - Prepare ovpn genl doit request -+ * @ops: requested netlink operation -+ * @skb: Netlink message with request data -+ * @info: receiver information -+ * -+ * Return: 0 on success or negative error number in case of failure -+ */ -+static int ovpn_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct net *net = genl_info_net(info); -+ struct net_device *dev; -+ -+ dev = ovpn_get_dev_from_attrs(net, info->attrs); -+ if (IS_ERR(dev)) -+ return PTR_ERR(dev); -+ -+ info->user_ptr[0] = netdev_priv(dev); -+ -+ return 0; -+} -+ -+/** -+ * ovpn_post_doit() - complete ovpn genl doit request -+ * @ops: requested netlink operation -+ * @skb: Netlink message with request data -+ * @info: receiver information -+ */ -+static void ovpn_post_doit(const struct genl_ops *ops, struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct ovpn_struct *ovpn; -+ -+ ovpn = info->user_ptr[0]; -+ dev_put(ovpn->dev); -+} -+ -+static int ovpn_netlink_get_key_dir(struct genl_info *info, struct nlattr *key, -+ enum ovpn_cipher_alg cipher, -+ struct ovpn_key_direction *dir) -+{ -+ struct nlattr *attr, *attrs[OVPN_KEY_DIR_ATTR_MAX + 1]; -+ int ret; -+ -+ ret = nla_parse_nested(attrs, OVPN_KEY_DIR_ATTR_MAX, key, NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ switch (cipher) { -+ case OVPN_CIPHER_ALG_AES_GCM: -+ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: -+ attr = attrs[OVPN_KEY_DIR_ATTR_CIPHER_KEY]; -+ if (!attr) -+ return -EINVAL; -+ -+ dir->cipher_key = nla_data(attr); -+ dir->cipher_key_size = nla_len(attr); -+ -+ attr = attrs[OVPN_KEY_DIR_ATTR_NONCE_TAIL]; -+ /* These algorithms require a 96bit nonce, -+ * Construct it by combining 4-bytes packet id and -+ * 8-bytes nonce-tail from userspace -+ */ -+ if (!attr) -+ return -EINVAL; -+ -+ dir->nonce_tail = nla_data(attr); -+ dir->nonce_tail_size = nla_len(attr); -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+static int ovpn_netlink_new_key(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_NEW_KEY_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer_key_reset pkr; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_NEW_KEY]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_NEW_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_KEY], -+ NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_NEW_KEY_ATTR_PEER_ID] || -+ !attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT] || -+ !attrs[OVPN_NEW_KEY_ATTR_KEY_ID] || -+ !attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG] || -+ !attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] || -+ !attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_NEW_KEY_ATTR_PEER_ID]); -+ pkr.slot = nla_get_u8(attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT]); -+ pkr.key.key_id = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_KEY_ID]); -+ -+ pkr.key.cipher_alg = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG]); -+ -+ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY], -+ pkr.key.cipher_alg, &pkr.key.encrypt); -+ if (ret < 0) -+ return ret; -+ -+ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY], -+ pkr.key.cipher_alg, &pkr.key.decrypt); -+ if (ret < 0) -+ return ret; -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) { -+ netdev_dbg(ovpn->dev, "%s: no peer with id %u to set key for\n", __func__, peer_id); -+ return -ENOENT; -+ } -+ -+ mutex_lock(&peer->crypto.mutex); -+ ret = ovpn_crypto_state_reset(&peer->crypto, &pkr); -+ if (ret < 0) { -+ netdev_dbg(ovpn->dev, "%s: cannot install new key for peer %u\n", __func__, -+ peer_id); -+ goto unlock; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: new key installed (id=%u) for peer %u\n", __func__, -+ pkr.key.key_id, peer_id); -+unlock: -+ mutex_unlock(&peer->crypto.mutex); -+ ovpn_peer_put(peer); -+ return ret; -+} -+ -+static int ovpn_netlink_del_key(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_DEL_KEY_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ enum ovpn_key_slot slot; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_DEL_KEY]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_DEL_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_KEY], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_DEL_KEY_ATTR_PEER_ID] || !attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_DEL_KEY_ATTR_PEER_ID]); -+ slot = nla_get_u8(attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ ovpn_crypto_key_slot_delete(&peer->crypto, slot); -+ ovpn_peer_put(peer); -+ -+ return 0; -+} -+ -+static int ovpn_netlink_swap_keys(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SWAP_KEYS_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_SWAP_KEYS]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_SWAP_KEYS_ATTR_MAX, info->attrs[OVPN_ATTR_SWAP_KEYS], -+ NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ ovpn_crypto_key_slots_swap(&peer->crypto); -+ ovpn_peer_put(peer); -+ -+ return 0; -+} -+ -+static int ovpn_netlink_new_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_NEW_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct sockaddr_storage *ss = NULL; -+ struct sockaddr_in mapped; -+ struct sockaddr_in6 *in6; -+ struct ovpn_peer *peer; -+ size_t sa_len, ip_len; -+ struct socket *sock; -+ u8 *local_ip = NULL; -+ u32 sockfd, id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_NEW_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_NEW_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_NEW_PEER_ATTR_PEER_ID] || !attrs[OVPN_NEW_PEER_ATTR_SOCKET] || -+ (!attrs[OVPN_NEW_PEER_ATTR_IPV4] && !attrs[OVPN_NEW_PEER_ATTR_IPV6])) { -+ netdev_dbg(ovpn->dev, "%s: basic attributes missing\n", __func__); -+ return -EINVAL; -+ } -+ -+ /* lookup the fd in the kernel table and extract the socket object */ -+ sockfd = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_SOCKET]); -+ /* sockfd_lookup() increases sock's refcounter */ -+ sock = sockfd_lookup(sockfd, &ret); -+ if (!sock) { -+ netdev_dbg(ovpn->dev, "%s: cannot lookup peer socket (fd=%u): %d\n", __func__, -+ sockfd, ret); -+ return -ENOTSOCK; -+ } -+ -+ /* Only when using UDP as transport protocol the remote endpoint must be configured -+ * so that ovpn-dco knows where to send packets to. -+ * -+ * In case of TCP, the socket is connected to the peer and ovpn-dco will just send bytes -+ * over it, without the need to specify a destination. -+ */ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) { -+ ret = -EINVAL; -+ -+ if (!attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]) { -+ netdev_err(ovpn->dev, "%s: cannot add UDP peer with no remote endpoint\n", -+ __func__); -+ goto sockfd_release; -+ } -+ -+ ss = nla_data(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); -+ sa_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); -+ switch (sa_len) { -+ case sizeof(struct sockaddr_in): -+ if (ss->ss_family == AF_INET) -+ /* valid sockaddr */ -+ break; -+ -+ netdev_err(ovpn->dev, "%s: remote sockaddr_in has invalid family\n", -+ __func__); -+ goto sockfd_release; -+ case sizeof(struct sockaddr_in6): -+ if (ss->ss_family == AF_INET6) -+ /* valid sockaddr */ -+ break; -+ -+ netdev_err(ovpn->dev, "%s: remote sockaddr_in6 has invalid family\n", -+ __func__); -+ goto sockfd_release; -+ default: -+ netdev_err(ovpn->dev, "%s: invalid size for sockaddr\n", __func__); -+ goto sockfd_release; -+ } -+ -+ if (ss->ss_family == AF_INET6) { -+ in6 = (struct sockaddr_in6 *)ss; -+ -+ if (ipv6_addr_type(&in6->sin6_addr) & IPV6_ADDR_MAPPED) { -+ mapped.sin_family = AF_INET; -+ mapped.sin_addr.s_addr = in6->sin6_addr.s6_addr32[3]; -+ mapped.sin_port = in6->sin6_port; -+ ss = (struct sockaddr_storage *)&mapped; -+ } -+ } -+ -+ /* When using UDP we may be talking over socket bound to 0.0.0.0/::. -+ * In this case, if the host has multiple IPs, we need to make sure -+ * that outgoing traffic has as source IP the same address that the -+ * peer is using to reach us. -+ * -+ * Since early control packets were all forwarded to userspace, we -+ * need the latter to tell us what IP has to be used. -+ */ -+ if (attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]) { -+ ip_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); -+ local_ip = nla_data(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); -+ -+ if (ip_len == sizeof(struct in_addr)) { -+ if (ss->ss_family != AF_INET) { -+ netdev_dbg(ovpn->dev, -+ "%s: the specified local IP is IPv4, but the peer endpoint is not\n", -+ __func__); -+ goto sockfd_release; -+ } -+ } else if (ip_len == sizeof(struct in6_addr)) { -+ bool is_mapped = ipv6_addr_type((struct in6_addr *)local_ip) & -+ IPV6_ADDR_MAPPED; -+ -+ if (ss->ss_family != AF_INET6 && !is_mapped) { -+ netdev_dbg(ovpn->dev, -+ "%s: the specified local IP is IPv6, but the peer endpoint is not\n", -+ __func__); -+ goto sockfd_release; -+ } -+ -+ if (is_mapped) -+ /* this is an IPv6-mapped IPv4 address, therefore extract -+ * the actual v4 address from the last 4 bytes -+ */ -+ local_ip += 12; -+ } else { -+ netdev_dbg(ovpn->dev, -+ "%s: invalid length %zu for local IP\n", __func__, -+ ip_len); -+ goto sockfd_release; -+ } -+ } -+ -+ /* sanity checks passed */ -+ ret = 0; -+ } -+ -+ id = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_PEER_ID]); -+ peer = ovpn_peer_new(ovpn, ss, sock, id, local_ip); -+ if (IS_ERR(peer)) { -+ netdev_err(ovpn->dev, "%s: cannot create new peer object for peer %u %pIScp\n", -+ __func__, id, ss); -+ ret = PTR_ERR(peer); -+ goto sockfd_release; -+ } -+ -+ if (attrs[OVPN_NEW_PEER_ATTR_IPV4]) { -+ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV4]) != sizeof(struct in_addr)) { -+ ret = -EINVAL; -+ goto peer_release; -+ } -+ -+ peer->vpn_addrs.ipv4.s_addr = nla_get_be32(attrs[OVPN_NEW_PEER_ATTR_IPV4]); -+ } -+ -+ if (attrs[OVPN_NEW_PEER_ATTR_IPV6]) { -+ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV6]) != sizeof(struct in6_addr)) { -+ ret = -EINVAL; -+ goto peer_release; -+ } -+ -+ memcpy(&peer->vpn_addrs.ipv6, nla_data(attrs[OVPN_NEW_PEER_ATTR_IPV6]), -+ sizeof(struct in6_addr)); -+ } -+ -+ netdev_dbg(ovpn->dev, -+ "%s: adding peer with endpoint=%pIScp/%s id=%u VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n", -+ __func__, ss, sock->sk->sk_prot_creator->name, peer->id, -+ &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6); -+ -+ ret = ovpn_peer_add(ovpn, peer); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot add new peer (id=%u) to hashtable: %d\n", -+ __func__, peer->id, ret); -+ goto peer_release; -+ } -+ -+ return 0; -+ -+peer_release: -+ /* release right away because peer is not really used in any context */ -+ ovpn_peer_release(peer); -+ return ret; -+ -+sockfd_release: -+ sockfd_put(sock); -+ return ret; -+} -+ -+static int ovpn_netlink_set_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ u32 peer_id, interv, timeout; -+ bool keepalive_set = false; -+ struct ovpn_peer *peer; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_SET_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_SET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_SET_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_SET_PEER_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_PEER_ID]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ /* when setting the keepalive, both parameters have to be configured */ -+ if (attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] && -+ attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]) { -+ keepalive_set = true; -+ interv = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL]); -+ timeout = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]); -+ } -+ -+ if (keepalive_set) -+ ovpn_peer_keepalive_set(peer, interv, timeout); -+ -+ ovpn_peer_put(peer); -+ return 0; -+} -+ -+static int ovpn_netlink_send_peer(struct sk_buff *skb, const struct ovpn_peer *peer, u32 portid, -+ u32 seq, int flags) -+{ -+ const struct ovpn_bind *bind; -+ struct nlattr *attr; -+ void *hdr; -+ -+ hdr = genlmsg_put(skb, portid, seq, &ovpn_netlink_family, flags, OVPN_CMD_GET_PEER); -+ if (!hdr) { -+ netdev_dbg(peer->ovpn->dev, "%s: cannot create message header\n", __func__); -+ return -EMSGSIZE; -+ } -+ -+ attr = nla_nest_start(skb, OVPN_ATTR_GET_PEER); -+ if (!attr) { -+ netdev_dbg(peer->ovpn->dev, "%s: cannot create submessage\n", __func__); -+ goto err; -+ } -+ -+ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_PEER_ID, peer->id)) -+ goto err; -+ -+ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV4, sizeof(peer->vpn_addrs.ipv4), -+ &peer->vpn_addrs.ipv4)) -+ goto err; -+ -+ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV6, sizeof(peer->vpn_addrs.ipv6), -+ &peer->vpn_addrs.ipv6)) -+ goto err; -+ -+ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, -+ peer->keepalive_interval) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, -+ peer->keepalive_timeout)) -+ goto err; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (bind) { -+ if (bind->sa.in4.sin_family == AF_INET) { -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ sizeof(bind->sa.in4), &bind->sa.in4) || -+ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ sizeof(bind->local.ipv4), &bind->local.ipv4)) -+ goto err_unlock; -+ } else if (bind->sa.in4.sin_family == AF_INET6) { -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ sizeof(bind->sa.in6), &bind->sa.in6) || -+ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ sizeof(bind->local.ipv6), &bind->local.ipv6)) -+ goto err_unlock; -+ } -+ } -+ rcu_read_unlock(); -+ -+ if (nla_put_net16(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, -+ inet_sk(peer->sock->sock->sk)->inet_sport) || -+ /* RX stats */ -+ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_RX_BYTES, -+ atomic64_read(&peer->stats.rx.bytes), -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, -+ atomic_read(&peer->stats.rx.packets)) || -+ /* TX stats */ -+ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_TX_BYTES, -+ atomic64_read(&peer->stats.tx.bytes), -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, -+ atomic_read(&peer->stats.tx.packets))) -+ goto err; -+ -+ nla_nest_end(skb, attr); -+ genlmsg_end(skb, hdr); -+ -+ return 0; -+err_unlock: -+ rcu_read_unlock(); -+err: -+ genlmsg_cancel(skb, hdr); -+ return -EMSGSIZE; -+} -+ -+static int ovpn_netlink_get_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ struct sk_buff *msg; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_GET_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_GET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_GET_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_GET_PEER_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_GET_PEER_ATTR_PEER_ID]); -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); -+ if (!msg) -+ return -ENOMEM; -+ -+ ret = ovpn_netlink_send_peer(msg, peer, info->snd_portid, info->snd_seq, 0); -+ if (ret < 0) { -+ nlmsg_free(msg); -+ goto err; -+ } -+ -+ ret = genlmsg_reply(msg, info); -+err: -+ ovpn_peer_put(peer); -+ return ret; -+} -+ -+static int ovpn_netlink_dump_done(struct netlink_callback *cb) -+{ -+ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; -+ -+ dev_put(ovpn->dev); -+ return 0; -+} -+ -+static int ovpn_netlink_dump_prepare(struct netlink_callback *cb) -+{ -+ struct net *netns = sock_net(cb->skb->sk); -+ struct nlattr **attrbuf; -+ struct net_device *dev; -+ int ret; -+ -+ attrbuf = kcalloc(OVPN_ATTR_MAX + 1, sizeof(*attrbuf), GFP_KERNEL); -+ if (!attrbuf) -+ return -ENOMEM; -+ -+ ret = nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrbuf, OVPN_ATTR_MAX, -+ ovpn_netlink_policy, NULL); -+ if (ret < 0) -+ goto err; -+ -+ dev = ovpn_get_dev_from_attrs(netns, attrbuf); -+ if (IS_ERR(dev)) { -+ ret = PTR_ERR(dev); -+ goto err; -+ } -+ -+ cb->args[0] = (long)netdev_priv(dev); -+ ret = 0; -+err: -+ kfree(attrbuf); -+ return ret; -+} -+ -+static int ovpn_netlink_dump_peers(struct sk_buff *skb, struct netlink_callback *cb) -+{ -+ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; -+ int ret, bkt, last_idx = cb->args[1], dumped = 0; -+ struct ovpn_peer *peer; -+ -+ if (!ovpn) { -+ ret = ovpn_netlink_dump_prepare(cb); -+ if (ret < 0) { -+ netdev_dbg(ovpn->dev, "%s: cannot prepare for dump: %d\n", __func__, ret); -+ return ret; -+ } -+ -+ ovpn = (struct ovpn_struct *)cb->args[0]; -+ } -+ -+ rcu_read_lock(); -+ hash_for_each_rcu(ovpn->peers.by_id, bkt, peer, hash_entry_id) { -+ /* skip already dumped peers that were dumped by previous invocations */ -+ if (last_idx > 0) { -+ last_idx--; -+ continue; -+ } -+ -+ if (ovpn_netlink_send_peer(skb, peer, NETLINK_CB(cb->skb).portid, -+ cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) -+ break; -+ -+ /* count peers being dumped during this invocation */ -+ dumped++; -+ } -+ rcu_read_unlock(); -+ -+ /* sum up peers dumped in this message, so that at the next invocation -+ * we can continue from where we left -+ */ -+ cb->args[1] += dumped; -+ -+ return skb->len; -+} -+ -+static int ovpn_netlink_del_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_DEL_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_DEL_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_DEL_PEER_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_DEL_PEER_ATTR_PEER_ID]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ netdev_dbg(ovpn->dev, "%s: peer id=%u\n", __func__, peer->id); -+ ret = ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_USERSPACE); -+ ovpn_peer_put(peer); -+ -+ return ret; -+} -+ -+static int ovpn_netlink_register_packet(struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ -+ /* only one registered process per interface is allowed for now */ -+ if (ovpn->registered_nl_portid_set) { -+ netdev_dbg(ovpn->dev, "%s: userspace listener already registered\n", __func__); -+ return -EBUSY; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: registering userspace at %u\n", __func__, info->snd_portid); -+ -+ ovpn->registered_nl_portid = info->snd_portid; -+ ovpn->registered_nl_portid_set = true; -+ -+ return 0; -+} -+ -+static int ovpn_netlink_packet(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_PACKET_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ const u8 *packet; -+ u32 peer_id; -+ size_t len; -+ u8 opcode; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_PACKET]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_PACKET_ATTR_MAX, info->attrs[OVPN_ATTR_PACKET], -+ NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_PACKET_ATTR_PACKET] || !attrs[OVPN_PACKET_ATTR_PEER_ID]) { -+ netdev_dbg(ovpn->dev, "received netlink packet with no payload\n"); -+ return -EINVAL; -+ } -+ -+ peer_id = nla_get_u32(attrs[OVPN_PACKET_ATTR_PEER_ID]); -+ -+ len = nla_len(attrs[OVPN_PACKET_ATTR_PACKET]); -+ -+ if (len < 4 || len > ovpn->dev->mtu) { -+ netdev_dbg(ovpn->dev, "%s: invalid packet size %zu (min is 4, max is MTU: %u)\n", -+ __func__, len, ovpn->dev->mtu); -+ return -EINVAL; -+ } -+ -+ packet = nla_data(attrs[OVPN_PACKET_ATTR_PACKET]); -+ opcode = ovpn_opcode_from_byte(packet[0]); -+ -+ /* reject data packets from userspace as they could lead to IV reuse */ -+ if (opcode == OVPN_DATA_V1 || opcode == OVPN_DATA_V2) { -+ netdev_dbg(ovpn->dev, "%s: rejecting data packet from userspace (opcode=%u)\n", -+ __func__, opcode); -+ return -EINVAL; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: sending userspace packet to peer %u...\n", __func__, peer_id); -+ -+ return ovpn_send_data(ovpn, peer_id, packet, len); -+} -+ -+static const struct genl_ops ovpn_netlink_ops[] = { -+ { -+ .cmd = OVPN_CMD_NEW_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_new_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_SET_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_set_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_DEL_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_del_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_GET_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_get_peer, -+ .dumpit = ovpn_netlink_dump_peers, -+ .done = ovpn_netlink_dump_done, -+ }, -+ { -+ .cmd = OVPN_CMD_NEW_KEY, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_new_key, -+ }, -+ { -+ .cmd = OVPN_CMD_DEL_KEY, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_del_key, -+ }, -+ { -+ .cmd = OVPN_CMD_SWAP_KEYS, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_swap_keys, -+ }, -+ { -+ .cmd = OVPN_CMD_REGISTER_PACKET, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_register_packet, -+ }, -+ { -+ .cmd = OVPN_CMD_PACKET, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_packet, -+ }, -+}; -+ -+static struct genl_family ovpn_netlink_family __ro_after_init = { -+ .hdrsize = 0, -+ .name = OVPN_NL_NAME, -+ .version = 1, -+ .maxattr = OVPN_ATTR_MAX, -+ .policy = ovpn_netlink_policy, -+ .netnsok = true, -+ .pre_doit = ovpn_pre_doit, -+ .post_doit = ovpn_post_doit, -+ .module = THIS_MODULE, -+ .ops = ovpn_netlink_ops, -+ .n_ops = ARRAY_SIZE(ovpn_netlink_ops), -+ .mcgrps = ovpn_netlink_mcgrps, -+ .n_mcgrps = ARRAY_SIZE(ovpn_netlink_mcgrps), -+}; -+ -+int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer) -+{ -+ struct sk_buff *msg; -+ struct nlattr *attr; -+ void *hdr; -+ int ret; -+ -+ netdev_info(peer->ovpn->dev, "%s: deleting peer with id %u, reason %d\n", -+ peer->ovpn->dev->name, peer->id, peer->delete_reason); -+ -+ msg = nlmsg_new(100, GFP_KERNEL); -+ if (!msg) -+ return -ENOMEM; -+ -+ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, -+ OVPN_CMD_DEL_PEER); -+ if (!hdr) { -+ ret = -ENOBUFS; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, peer->ovpn->dev->ifindex)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ attr = nla_nest_start(msg, OVPN_ATTR_DEL_PEER); -+ if (!attr) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u8(msg, OVPN_DEL_PEER_ATTR_REASON, peer->delete_reason)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_DEL_PEER_ATTR_PEER_ID, peer->id)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ nla_nest_end(msg, attr); -+ -+ genlmsg_end(msg, hdr); -+ -+ genlmsg_multicast_netns(&ovpn_netlink_family, dev_net(peer->ovpn->dev), -+ msg, 0, OVPN_MCGRP_PEERS, GFP_KERNEL); -+ -+ return 0; -+ -+err_free_msg: -+ nlmsg_free(msg); -+ return ret; -+} -+ -+int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ const u8 *buf, size_t len) -+{ -+ struct nlattr *attr; -+ struct sk_buff *msg; -+ void *hdr; -+ int ret; -+ -+ if (!ovpn->registered_nl_portid_set) { -+ net_warn_ratelimited("%s: no userspace listener\n", __func__); -+ return 0; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: sending packet to userspace, len: %zd\n", __func__, len); -+ -+ msg = nlmsg_new(100 + len, GFP_ATOMIC); -+ if (!msg) -+ return -ENOMEM; -+ -+ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, -+ OVPN_CMD_PACKET); -+ if (!hdr) { -+ ret = -ENOBUFS; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, ovpn->dev->ifindex)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ attr = nla_nest_start(msg, OVPN_ATTR_PACKET); -+ if (!attr) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put(msg, OVPN_PACKET_ATTR_PACKET, len, buf)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_PACKET_ATTR_PEER_ID, peer->id)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ nla_nest_end(msg, attr); -+ -+ genlmsg_end(msg, hdr); -+ -+ return genlmsg_unicast(dev_net(ovpn->dev), msg, -+ ovpn->registered_nl_portid); -+ -+err_free_msg: -+ nlmsg_free(msg); -+ return ret; -+} -+ -+static int ovpn_netlink_notify(struct notifier_block *nb, unsigned long state, -+ void *_notify) -+{ -+ struct netlink_notify *notify = _notify; -+ struct ovpn_struct *ovpn; -+ struct net_device *dev; -+ struct net *netns; -+ bool found = false; -+ -+ if (state != NETLINK_URELEASE || notify->protocol != NETLINK_GENERIC) -+ return NOTIFY_DONE; -+ -+ rcu_read_lock(); -+ for_each_net_rcu(netns) { -+ for_each_netdev_rcu(netns, dev) { -+ if (!ovpn_dev_is_valid(dev)) -+ continue; -+ -+ ovpn = netdev_priv(dev); -+ if (notify->portid != ovpn->registered_nl_portid) -+ continue; -+ -+ found = true; -+ netdev_dbg(ovpn->dev, "%s: deregistering userspace listener\n", __func__); -+ ovpn->registered_nl_portid_set = false; -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ -+ /* if no interface matched our purposes, pass the notification along */ -+ if (!found) -+ return NOTIFY_DONE; -+ -+ return NOTIFY_OK; -+} -+ -+static struct notifier_block ovpn_netlink_notifier = { -+ .notifier_call = ovpn_netlink_notify, -+}; -+ -+int ovpn_netlink_init(struct ovpn_struct *ovpn) -+{ -+ ovpn->registered_nl_portid_set = false; -+ -+ return 0; -+} -+ -+/** -+ * ovpn_netlink_register() - register the ovpn genl netlink family -+ */ -+int __init ovpn_netlink_register(void) -+{ -+ int ret; -+ -+ ret = genl_register_family(&ovpn_netlink_family); -+ if (ret) -+ return ret; -+ -+ ret = netlink_register_notifier(&ovpn_netlink_notifier); -+ if (ret) -+ goto err; -+ -+ return 0; -+err: -+ genl_unregister_family(&ovpn_netlink_family); -+ return ret; -+} -+ -+/** -+ * ovpn_netlink_unregister() - unregister the ovpn genl netlink family -+ */ -+void __exit ovpn_netlink_unregister(void) -+{ -+ netlink_unregister_notifier(&ovpn_netlink_notifier); -+ genl_unregister_family(&ovpn_netlink_family); -+} -diff --git a/drivers/net/ovpn-dco/netlink.h b/drivers/net/ovpn-dco/netlink.h -new file mode 100644 -index 000000000000..843daf052c03 ---- /dev/null -+++ b/drivers/net/ovpn-dco/netlink.h -@@ -0,0 +1,22 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_NETLINK_H_ -+#define _NET_OVPN_DCO_NETLINK_H_ -+ -+struct ovpn_struct; -+struct ovpn_peer; -+ -+int ovpn_netlink_init(struct ovpn_struct *ovpn); -+int ovpn_netlink_register(void); -+void ovpn_netlink_unregister(void); -+int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ const u8 *buf, size_t len); -+int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer); -+ -+#endif /* _NET_OVPN_DCO_NETLINK_H_ */ -diff --git a/drivers/net/ovpn-dco/ovpn.c b/drivers/net/ovpn-dco/ovpn.c -new file mode 100644 -index 000000000000..66c019174f5e ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpn.c -@@ -0,0 +1,600 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "bind.h" -+#include "netlink.h" -+#include "ovpn.h" -+#include "sock.h" -+#include "peer.h" -+#include "stats.h" -+#include "proto.h" -+#include "crypto.h" -+#include "crypto_aead.h" -+#include "skb.h" -+#include "tcp.h" -+#include "udp.h" -+ -+#include -+#include -+ -+static const unsigned char ovpn_keepalive_message[] = { -+ 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, -+ 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 -+}; -+ -+static const unsigned char ovpn_explicit_exit_notify_message[] = { -+ 0x28, 0x7f, 0x34, 0x6b, 0xd4, 0xef, 0x7a, 0x81, -+ 0x2d, 0x56, 0xb8, 0xd3, 0xaf, 0xc5, 0x45, 0x9c, -+ 6 // OCC_EXIT -+}; -+ -+/* Is keepalive message? -+ * Assumes that single byte at skb->data is defined. -+ */ -+static bool ovpn_is_keepalive(struct sk_buff *skb) -+{ -+ if (*skb->data != OVPN_KEEPALIVE_FIRST_BYTE) -+ return false; -+ -+ if (!pskb_may_pull(skb, sizeof(ovpn_keepalive_message))) -+ return false; -+ -+ return !memcmp(skb->data, ovpn_keepalive_message, -+ sizeof(ovpn_keepalive_message)); -+} -+ -+int ovpn_struct_init(struct net_device *dev) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ int err; -+ -+ memset(ovpn, 0, sizeof(*ovpn)); -+ -+ ovpn->dev = dev; -+ -+ err = ovpn_netlink_init(ovpn); -+ if (err < 0) -+ return err; -+ -+ spin_lock_init(&ovpn->lock); -+ spin_lock_init(&ovpn->peers.lock); -+ -+ ovpn->crypto_wq = alloc_workqueue("ovpn-crypto-wq-%s", -+ WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, -+ dev->name); -+ if (!ovpn->crypto_wq) -+ return -ENOMEM; -+ -+ ovpn->events_wq = alloc_workqueue("ovpn-event-wq-%s", WQ_MEM_RECLAIM, 0, dev->name); -+ if (!ovpn->events_wq) -+ return -ENOMEM; -+ -+ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); -+ if (!dev->tstats) -+ return -ENOMEM; -+ -+ err = security_tun_dev_alloc_security(&ovpn->security); -+ if (err < 0) -+ return err; -+ -+ /* kernel -> userspace tun queue length */ -+ ovpn->max_tun_queue_len = OVPN_MAX_TUN_QUEUE_LEN; -+ -+ return 0; -+} -+ -+/* Called after decrypt to write IP packet to tun netdev. -+ * This method is expected to manage/free skb. -+ */ -+static void tun_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ /* packet integrity was verified on the VPN layer - no need to perform -+ * any additional check along the stack -+ */ -+ skb->ip_summed = CHECKSUM_UNNECESSARY; -+ skb->csum_level = ~0; -+ -+ /* skb hash for transport packet no longer valid after decapsulation */ -+ skb_clear_hash(skb); -+ -+ /* post-decrypt scrub -- prepare to inject encapsulated packet onto tun -+ * interface, based on __skb_tunnel_rx() in dst.h -+ */ -+ skb->dev = peer->ovpn->dev; -+ skb_set_queue_mapping(skb, 0); -+ skb_scrub_packet(skb, true); -+ -+ skb_reset_network_header(skb); -+ skb_reset_transport_header(skb); -+ skb_probe_transport_header(skb); -+ skb_reset_inner_headers(skb); -+ -+ /* update per-cpu RX stats with the stored size of encrypted packet */ -+ -+ /* we are in softirq context - hence no locking nor disable preemption needed */ -+ dev_sw_netstats_rx_add(peer->ovpn->dev, OVPN_SKB_CB(skb)->rx_stats_size); -+ -+ /* cause packet to be "received" by tun interface */ -+ napi_gro_receive(&peer->napi, skb); -+} -+ -+int ovpn_napi_poll(struct napi_struct *napi, int budget) -+{ -+ struct ovpn_peer *peer = container_of(napi, struct ovpn_peer, napi); -+ struct sk_buff *skb; -+ int work_done = 0; -+ -+ if (unlikely(budget <= 0)) -+ return 0; -+ /* this function should schedule at most 'budget' number of -+ * packets for delivery to the tun interface. -+ * If in the queue we have more packets than what allowed by the -+ * budget, the next polling will take care of those -+ */ -+ while ((work_done < budget) && -+ (skb = ptr_ring_consume_bh(&peer->netif_rx_ring))) { -+ tun_netdev_write(peer, skb); -+ work_done++; -+ } -+ -+ if (work_done < budget) -+ napi_complete_done(napi, work_done); -+ -+ return work_done; -+} -+ -+static int ovpn_transport_to_userspace(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ struct sk_buff *skb) -+{ -+ int ret; -+ -+ ret = skb_linearize(skb); -+ if (ret < 0) -+ return ret; -+ -+ ret = ovpn_netlink_send_packet(ovpn, peer, skb->data, skb->len); -+ if (ret < 0) -+ return ret; -+ -+ consume_skb(skb); -+ return 0; -+} -+ -+/* Entry point for processing an incoming packet (in skb form) -+ * -+ * Enqueue the packet and schedule RX consumer. -+ * Reference to peer is dropped only in case of success. -+ * -+ * Return 0 if the packet was handled (and consumed) -+ * Return <0 in case of error (return value is error code) -+ */ -+int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ int ret; -+ -+ /* At this point we know the packet is from a configured peer. -+ * DATA_V2 packets are handled in kernel space, the rest goes to user space. -+ * -+ * Packets are sent to userspace via netlink API in order to be consistenbt across -+ * UDP and TCP. -+ */ -+ if (unlikely(ovpn_opcode_from_skb(skb, 0) != OVPN_DATA_V2)) { -+ ret = ovpn_transport_to_userspace(ovpn, peer, skb); -+ if (ret < 0) -+ return ret; -+ -+ ovpn_peer_put(peer); -+ return 0; -+ } -+ -+ ret = ptr_ring_produce_bh(&peer->rx_ring, skb); -+ if (unlikely(ret < 0)) -+ return -ENOSPC; -+ -+ if (!queue_work(ovpn->crypto_wq, &peer->decrypt_work)) -+ ovpn_peer_put(peer); -+ -+ return 0; -+} -+ -+static int ovpn_decrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_peer *allowed_peer = NULL; -+ struct ovpn_crypto_key_slot *ks; -+ unsigned int rx_stats_size; -+ __be16 proto; -+ int ret = -1; -+ u8 key_id; -+ -+ /* save original packet size for stats accounting */ -+ OVPN_SKB_CB(skb)->rx_stats_size = skb->len; -+ -+ /* get the key slot matching the key Id in the received packet */ -+ key_id = ovpn_key_id_from_skb(skb); -+ ks = ovpn_crypto_key_id_to_slot(&peer->crypto, key_id); -+ if (unlikely(!ks)) { -+ net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n", __func__, -+ peer->id, key_id); -+ goto drop; -+ } -+ -+ /* decrypt */ -+ ret = ovpn_aead_decrypt(ks, skb); -+ -+ ovpn_crypto_key_slot_put(ks); -+ -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: error during decryption for peer %u, key-id %u: %d\n", -+ __func__, peer->id, key_id, ret); -+ goto drop; -+ } -+ -+ /* note event of authenticated packet received for keepalive */ -+ ovpn_peer_keepalive_recv_reset(peer); -+ -+ /* update source and destination endpoint for this peer */ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_UDP) -+ ovpn_peer_update_local_endpoint(peer, skb); -+ -+ /* increment RX stats */ -+ rx_stats_size = OVPN_SKB_CB(skb)->rx_stats_size; -+ ovpn_peer_stats_increment_rx(&peer->stats, rx_stats_size); -+ -+ /* check if this is a valid datapacket that has to be delivered to the -+ * tun interface -+ */ -+ skb_reset_network_header(skb); -+ proto = ovpn_ip_check_protocol(skb); -+ if (unlikely(!proto)) { -+ /* check if null packet */ -+ if (unlikely(!pskb_may_pull(skb, 1))) { -+ ret = -EINVAL; -+ goto drop; -+ } -+ -+ /* check if special OpenVPN message */ -+ if (ovpn_is_keepalive(skb)) { -+ netdev_dbg(peer->ovpn->dev, "%s: ping received from peer with id %u\n", -+ __func__, peer->id); -+ /* not an error */ -+ consume_skb(skb); -+ /* inform the caller that NAPI should not be scheduled -+ * for this packet -+ */ -+ return -1; -+ } -+ -+ ret = -EPROTONOSUPPORT; -+ goto drop; -+ } -+ skb->protocol = proto; -+ -+ /* perform Reverse Path Filtering (RPF) */ -+ allowed_peer = ovpn_peer_lookup_vpn_addr(peer->ovpn, skb, true); -+ if (unlikely(allowed_peer != peer)) { -+ ret = -EPERM; -+ goto drop; -+ } -+ -+ ret = ptr_ring_produce_bh(&peer->netif_rx_ring, skb); -+drop: -+ if (likely(allowed_peer)) -+ ovpn_peer_put(allowed_peer); -+ -+ if (unlikely(ret < 0)) -+ kfree_skb(skb); -+ -+ return ret; -+} -+ -+/* pick packet from RX queue, decrypt and forward it to the tun device */ -+void ovpn_decrypt_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ -+ peer = container_of(work, struct ovpn_peer, decrypt_work); -+ while ((skb = ptr_ring_consume_bh(&peer->rx_ring))) { -+ if (likely(ovpn_decrypt_one(peer, skb) == 0)) { -+ /* if a packet has been enqueued for NAPI, signal -+ * availability to the networking stack -+ */ -+ local_bh_disable(); -+ napi_schedule(&peer->napi); -+ local_bh_enable(); -+ } -+ -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+ ovpn_peer_put(peer); -+} -+ -+static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ bool success = false; -+ int ret; -+ -+ /* get primary key to be used for encrypting data */ -+ ks = ovpn_crypto_key_slot_primary(&peer->crypto); -+ if (unlikely(!ks)) { -+ net_info_ratelimited("%s: error while retrieving primary key slot\n", __func__); -+ return false; -+ } -+ -+ if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && -+ skb_checksum_help(skb))) { -+ net_err_ratelimited("%s: cannot compute checksum for outgoing packet\n", __func__); -+ goto err; -+ } -+ -+ ovpn_peer_stats_increment_tx(&peer->stats, skb->len); -+ -+ /* encrypt */ -+ ret = ovpn_aead_encrypt(ks, skb, peer->id); -+ if (unlikely(ret < 0)) { -+ /* if we ran out of IVs we must kill the key as it can't be used anymore */ -+ if (ret == -ERANGE) { -+ netdev_warn(peer->ovpn->dev, -+ "%s: killing primary key as we ran out of IVs\n", __func__); -+ ovpn_crypto_kill_primary(&peer->crypto); -+ goto err; -+ } -+ net_err_ratelimited("%s: error during encryption for peer %u, key-id %u: %d\n", -+ __func__, peer->id, ks->key_id, ret); -+ goto err; -+ } -+ -+ success = true; -+err: -+ ovpn_crypto_key_slot_put(ks); -+ return success; -+} -+ -+/* Process packets in TX queue in a transport-specific way. -+ * -+ * UDP transport - encrypt and send across the tunnel. -+ * TCP transport - encrypt and put into TCP TX queue. -+ */ -+void ovpn_encrypt_work(struct work_struct *work) -+{ -+ struct sk_buff *skb, *curr, *next; -+ struct ovpn_peer *peer; -+ -+ peer = container_of(work, struct ovpn_peer, encrypt_work); -+ while ((skb = ptr_ring_consume_bh(&peer->tx_ring))) { -+ /* this might be a GSO-segmented skb list: process each skb -+ * independently -+ */ -+ skb_list_walk_safe(skb, curr, next) { -+ /* if one segment fails encryption, we drop the entire -+ * packet, because it does not really make sense to send -+ * only part of it at this point -+ */ -+ if (unlikely(!ovpn_encrypt_one(peer, curr))) { -+ kfree_skb_list(skb); -+ skb = NULL; -+ break; -+ } -+ } -+ -+ /* successful encryption */ -+ if (skb) { -+ skb_list_walk_safe(skb, curr, next) { -+ skb_mark_not_on_list(curr); -+ -+ switch (peer->sock->sock->sk->sk_protocol) { -+ case IPPROTO_UDP: -+ ovpn_udp_send_skb(peer->ovpn, peer, curr); -+ break; -+ case IPPROTO_TCP: -+ ovpn_tcp_send_skb(peer, curr); -+ break; -+ default: -+ /* no transport configured yet */ -+ consume_skb(skb); -+ break; -+ } -+ } -+ -+ /* note event of authenticated packet xmit for keepalive */ -+ ovpn_peer_keepalive_xmit_reset(peer); -+ } -+ -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+ ovpn_peer_put(peer); -+} -+ -+/* Put skb into TX queue and schedule a consumer */ -+static void ovpn_queue_skb(struct ovpn_struct *ovpn, struct sk_buff *skb, struct ovpn_peer *peer) -+{ -+ int ret; -+ -+ if (likely(!peer)) -+ peer = ovpn_peer_lookup_vpn_addr(ovpn, skb, false); -+ if (unlikely(!peer)) { -+ net_dbg_ratelimited("%s: no peer to send data to\n", ovpn->dev->name); -+ goto drop; -+ } -+ -+ ret = ptr_ring_produce_bh(&peer->tx_ring, skb); -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: cannot queue packet to TX ring\n", __func__); -+ goto drop; -+ } -+ -+ if (!queue_work(ovpn->crypto_wq, &peer->encrypt_work)) -+ ovpn_peer_put(peer); -+ -+ return; -+drop: -+ if (peer) -+ ovpn_peer_put(peer); -+ kfree_skb_list(skb); -+} -+ -+/* Net device start xmit -+ */ -+netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ struct sk_buff *segments, *tmp, *curr, *next; -+ struct sk_buff_head skb_list; -+ __be16 proto; -+ int ret; -+ -+ /* reset netfilter state */ -+ nf_reset_ct(skb); -+ -+ /* verify IP header size in network packet */ -+ proto = ovpn_ip_check_protocol(skb); -+ if (unlikely(!proto || skb->protocol != proto)) { -+ net_dbg_ratelimited("%s: dropping malformed payload packet\n", -+ dev->name); -+ goto drop; -+ } -+ -+ if (skb_is_gso(skb)) { -+ segments = skb_gso_segment(skb, 0); -+ if (IS_ERR(segments)) { -+ ret = PTR_ERR(segments); -+ net_dbg_ratelimited("%s: cannot segment packet: %d\n", dev->name, ret); -+ goto drop; -+ } -+ -+ consume_skb(skb); -+ skb = segments; -+ } -+ -+ /* from this moment on, "skb" might be a list */ -+ -+ __skb_queue_head_init(&skb_list); -+ skb_list_walk_safe(skb, curr, next) { -+ skb_mark_not_on_list(curr); -+ -+ tmp = skb_share_check(curr, GFP_ATOMIC); -+ if (unlikely(!tmp)) { -+ kfree_skb_list(next); -+ net_dbg_ratelimited("%s: skb_share_check failed\n", dev->name); -+ goto drop_list; -+ } -+ -+ __skb_queue_tail(&skb_list, tmp); -+ } -+ skb_list.prev->next = NULL; -+ -+ ovpn_queue_skb(ovpn, skb_list.next, NULL); -+ -+ return NETDEV_TX_OK; -+ -+drop_list: -+ skb_queue_walk_safe(&skb_list, curr, next) -+ kfree_skb(curr); -+drop: -+ skb_tx_error(skb); -+ kfree_skb_list(skb); -+ return NET_XMIT_DROP; -+} -+ -+/* Encrypt and transmit a special message to peer, such as keepalive -+ * or explicit-exit-notify. Called from softirq context. -+ * Assumes that caller holds a reference to peer. -+ */ -+static void ovpn_xmit_special(struct ovpn_peer *peer, const void *data, -+ const unsigned int len) -+{ -+ struct ovpn_struct *ovpn; -+ struct sk_buff *skb; -+ -+ ovpn = peer->ovpn; -+ if (unlikely(!ovpn)) -+ return; -+ -+ skb = alloc_skb(256 + len, GFP_ATOMIC); -+ if (unlikely(!skb)) -+ return; -+ -+ skb_reserve(skb, 128); -+ skb->priority = TC_PRIO_BESTEFFORT; -+ memcpy(__skb_put(skb, len), data, len); -+ -+ /* increase reference counter when passing peer to sending queue */ -+ if (!ovpn_peer_hold(peer)) { -+ netdev_dbg(ovpn->dev, "%s: cannot hold peer reference for sending special packet\n", -+ __func__); -+ kfree_skb(skb); -+ return; -+ } -+ -+ ovpn_queue_skb(ovpn, skb, peer); -+} -+ -+void ovpn_keepalive_xmit(struct ovpn_peer *peer) -+{ -+ ovpn_xmit_special(peer, ovpn_keepalive_message, -+ sizeof(ovpn_keepalive_message)); -+} -+ -+/* Transmit explicit exit notification. -+ * Called from process context. -+ */ -+void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer) -+{ -+ ovpn_xmit_special(peer, ovpn_explicit_exit_notify_message, -+ sizeof(ovpn_explicit_exit_notify_message)); -+} -+ -+/* Copy buffer into skb and send it across the tunnel. -+ * -+ * For UDP transport: just sent the skb to peer -+ * For TCP transport: put skb into TX queue -+ */ -+int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len) -+{ -+ u16 skb_len = SKB_HEADER_LEN + len; -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ bool tcp = false; -+ int ret = 0; -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (unlikely(!peer)) { -+ netdev_dbg(ovpn->dev, "no peer to send data to\n"); -+ return -EHOSTUNREACH; -+ } -+ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) { -+ skb_len += sizeof(u16); -+ tcp = true; -+ } -+ -+ skb = alloc_skb(skb_len, GFP_ATOMIC); -+ if (unlikely(!skb)) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ skb_reserve(skb, SKB_HEADER_LEN); -+ skb_put_data(skb, data, len); -+ -+ /* prepend TCP packet with size, as required by OpenVPN protocol */ -+ if (tcp) { -+ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); -+ ovpn_queue_tcp_skb(peer, skb); -+ } else { -+ ovpn_udp_send_skb(ovpn, peer, skb); -+ } -+out: -+ ovpn_peer_put(peer); -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/ovpn.h b/drivers/net/ovpn-dco/ovpn.h -new file mode 100644 -index 000000000000..9364fd5dd309 ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpn.h -@@ -0,0 +1,43 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPN_H_ -+#define _NET_OVPN_DCO_OVPN_H_ -+ -+#include "main.h" -+#include "peer.h" -+#include "sock.h" -+#include "ovpnstruct.h" -+ -+#include -+#include -+#include -+ -+struct ovpn_struct; -+struct net_device; -+ -+int ovpn_struct_init(struct net_device *dev); -+ -+u16 ovpn_select_queue(struct net_device *dev, struct sk_buff *skb, -+ struct net_device *sb_dev); -+ -+void ovpn_keepalive_xmit(struct ovpn_peer *peer); -+void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer); -+ -+netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev); -+ -+int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb); -+ -+void ovpn_encrypt_work(struct work_struct *work); -+void ovpn_decrypt_work(struct work_struct *work); -+int ovpn_napi_poll(struct napi_struct *napi, int budget); -+ -+int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len); -+ -+#endif /* _NET_OVPN_DCO_OVPN_H_ */ -diff --git a/drivers/net/ovpn-dco/ovpnstruct.h b/drivers/net/ovpn-dco/ovpnstruct.h -new file mode 100644 -index 000000000000..f9bc559609cd ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpnstruct.h -@@ -0,0 +1,59 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNSTRUCT_H_ -+#define _NET_OVPN_DCO_OVPNSTRUCT_H_ -+ -+#include "peer.h" -+ -+#include -+#include -+#include -+ -+/* Our state per ovpn interface */ -+struct ovpn_struct { -+ /* read-mostly objects in this section */ -+ struct net_device *dev; -+ -+ /* device operation mode (i.e. P2P, MP) */ -+ enum ovpn_mode mode; -+ -+ /* protect writing to the ovpn_struct object */ -+ spinlock_t lock; -+ -+ /* workqueue used to schedule crypto work that may sleep */ -+ struct workqueue_struct *crypto_wq; -+ /* workqueue used to schedule generic event that may sleep or that need -+ * to be performed out of softirq context -+ */ -+ struct workqueue_struct *events_wq; -+ -+ /* list of known peers */ -+ struct { -+ DECLARE_HASHTABLE(by_id, 12); -+ DECLARE_HASHTABLE(by_transp_addr, 12); -+ DECLARE_HASHTABLE(by_vpn_addr, 12); -+ /* protects write access to any of the hashtables above */ -+ spinlock_t lock; -+ } peers; -+ -+ /* for p2p mode */ -+ struct ovpn_peer __rcu *peer; -+ -+ unsigned int max_tun_queue_len; -+ -+ netdev_features_t set_features; -+ -+ void *security; -+ -+ u32 registered_nl_portid; -+ bool registered_nl_portid_set; -+}; -+ -+#endif /* _NET_OVPN_DCO_OVPNSTRUCT_H_ */ -diff --git a/drivers/net/ovpn-dco/peer.c b/drivers/net/ovpn-dco/peer.c -new file mode 100644 -index 000000000000..87d3f1b34c4d ---- /dev/null -+++ b/drivers/net/ovpn-dco/peer.c -@@ -0,0 +1,906 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "ovpn.h" -+#include "bind.h" -+#include "crypto.h" -+#include "peer.h" -+#include "netlink.h" -+#include "tcp.h" -+ -+#include -+#include -+#include -+#include -+#include -+ -+static void ovpn_peer_ping(struct timer_list *t) -+{ -+ struct ovpn_peer *peer = from_timer(peer, t, keepalive_xmit); -+ -+ netdev_dbg(peer->ovpn->dev, "%s: sending ping to peer %u\n", __func__, peer->id); -+ ovpn_keepalive_xmit(peer); -+} -+ -+static void ovpn_peer_expire(struct timer_list *t) -+{ -+ struct ovpn_peer *peer = from_timer(peer, t, keepalive_recv); -+ -+ netdev_dbg(peer->ovpn->dev, "%s: peer %u expired\n", __func__, peer->id); -+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_EXPIRED); -+} -+ -+/* Construct a new peer */ -+static struct ovpn_peer *ovpn_peer_create(struct ovpn_struct *ovpn, u32 id) -+{ -+ struct ovpn_peer *peer; -+ int ret; -+ -+ /* alloc and init peer object */ -+ peer = kzalloc(sizeof(*peer), GFP_KERNEL); -+ if (!peer) -+ return ERR_PTR(-ENOMEM); -+ -+ peer->id = id; -+ peer->halt = false; -+ peer->ovpn = ovpn; -+ -+ peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY); -+ peer->vpn_addrs.ipv6 = in6addr_any; -+ -+ RCU_INIT_POINTER(peer->bind, NULL); -+ ovpn_crypto_state_init(&peer->crypto); -+ spin_lock_init(&peer->lock); -+ kref_init(&peer->refcount); -+ ovpn_peer_stats_init(&peer->stats); -+ -+ INIT_WORK(&peer->encrypt_work, ovpn_encrypt_work); -+ INIT_WORK(&peer->decrypt_work, ovpn_decrypt_work); -+ -+ ret = dst_cache_init(&peer->dst_cache, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot initialize dst cache\n", __func__); -+ goto err; -+ } -+ -+ ret = ptr_ring_init(&peer->tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate TX ring\n", __func__); -+ goto err_dst_cache; -+ } -+ -+ ret = ptr_ring_init(&peer->rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate RX ring\n", __func__); -+ goto err_tx_ring; -+ } -+ -+ ret = ptr_ring_init(&peer->netif_rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate NETIF RX ring\n", __func__); -+ goto err_rx_ring; -+ } -+ -+ /* configure and start NAPI */ -+ netif_tx_napi_add(ovpn->dev, &peer->napi, ovpn_napi_poll, -+ NAPI_POLL_WEIGHT); -+ napi_enable(&peer->napi); -+ -+ dev_hold(ovpn->dev); -+ -+ timer_setup(&peer->keepalive_xmit, ovpn_peer_ping, 0); -+ timer_setup(&peer->keepalive_recv, ovpn_peer_expire, 0); -+ -+ return peer; -+err_rx_ring: -+ ptr_ring_cleanup(&peer->rx_ring, NULL); -+err_tx_ring: -+ ptr_ring_cleanup(&peer->tx_ring, NULL); -+err_dst_cache: -+ dst_cache_destroy(&peer->dst_cache); -+err: -+ kfree(peer); -+ return ERR_PTR(ret); -+} -+ -+/* Reset the ovpn_sockaddr associated with a peer */ -+static int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, const struct sockaddr_storage *ss, -+ const u8 *local_ip) -+{ -+ struct ovpn_bind *bind; -+ size_t ip_len; -+ -+ /* create new ovpn_bind object */ -+ bind = ovpn_bind_from_sockaddr(ss); -+ if (IS_ERR(bind)) -+ return PTR_ERR(bind); -+ -+ if (local_ip) { -+ if (ss->ss_family == AF_INET) { -+ ip_len = sizeof(struct in_addr); -+ } else if (ss->ss_family == AF_INET6) { -+ ip_len = sizeof(struct in6_addr); -+ } else { -+ netdev_dbg(peer->ovpn->dev, "%s: invalid family for remote endpoint\n", -+ __func__); -+ kfree(bind); -+ return -EINVAL; -+ } -+ -+ memcpy(&bind->local, local_ip, ip_len); -+ } -+ -+ /* set binding */ -+ ovpn_bind_reset(peer, bind); -+ -+ return 0; -+} -+ -+void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct sockaddr_storage ss; -+ const u8 *local_ip = NULL; -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa; -+ struct ovpn_bind *bind; -+ sa_family_t family; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) -+ goto unlock; -+ -+ if (likely(ovpn_bind_skb_src_match(bind, skb))) -+ goto unlock; -+ -+ family = skb_protocol_to_family(skb); -+ -+ if (bind->sa.in4.sin_family == family) -+ local_ip = (u8 *)&bind->local; -+ -+ switch (family) { -+ case AF_INET: -+ sa = (struct sockaddr_in *)&ss; -+ sa->sin_family = AF_INET; -+ sa->sin_addr.s_addr = ip_hdr(skb)->saddr; -+ sa->sin_port = udp_hdr(skb)->source; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)&ss; -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = ipv6_hdr(skb)->saddr; -+ sa6->sin6_port = udp_hdr(skb)->source; -+ sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); -+ break; -+ default: -+ goto unlock; -+ } -+ -+ netdev_dbg(peer->ovpn->dev, "%s: peer %d floated to %pIScp", __func__, peer->id, &ss); -+ ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip); -+unlock: -+ rcu_read_unlock(); -+} -+ -+static void ovpn_peer_timer_delete_all(struct ovpn_peer *peer) -+{ -+ del_timer_sync(&peer->keepalive_xmit); -+ del_timer_sync(&peer->keepalive_recv); -+} -+ -+static void ovpn_peer_free(struct ovpn_peer *peer) -+{ -+ ovpn_bind_reset(peer, NULL); -+ ovpn_peer_timer_delete_all(peer); -+ -+ WARN_ON(!__ptr_ring_empty(&peer->tx_ring)); -+ ptr_ring_cleanup(&peer->tx_ring, NULL); -+ WARN_ON(!__ptr_ring_empty(&peer->rx_ring)); -+ ptr_ring_cleanup(&peer->rx_ring, NULL); -+ WARN_ON(!__ptr_ring_empty(&peer->netif_rx_ring)); -+ ptr_ring_cleanup(&peer->netif_rx_ring, NULL); -+ -+ dst_cache_destroy(&peer->dst_cache); -+ -+ dev_put(peer->ovpn->dev); -+ -+ kfree(peer); -+} -+ -+static void ovpn_peer_release_rcu(struct rcu_head *head) -+{ -+ struct ovpn_peer *peer = container_of(head, struct ovpn_peer, rcu); -+ -+ ovpn_crypto_state_release(&peer->crypto); -+ ovpn_peer_free(peer); -+} -+ -+void ovpn_peer_release(struct ovpn_peer *peer) -+{ -+ napi_disable(&peer->napi); -+ netif_napi_del(&peer->napi); -+ -+ if (peer->sock) -+ ovpn_socket_put(peer->sock); -+ -+ call_rcu(&peer->rcu, ovpn_peer_release_rcu); -+} -+ -+static void ovpn_peer_delete_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, -+ delete_work); -+ ovpn_peer_release(peer); -+ ovpn_netlink_notify_del_peer(peer); -+} -+ -+/* Use with kref_put calls, when releasing refcount -+ * on ovpn_peer objects. This method should only -+ * be called from process context with config_mutex held. -+ */ -+void ovpn_peer_release_kref(struct kref *kref) -+{ -+ struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount); -+ -+ INIT_WORK(&peer->delete_work, ovpn_peer_delete_work); -+ queue_work(peer->ovpn->events_wq, &peer->delete_work); -+} -+ -+struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, -+ struct socket *sock, u32 id, uint8_t *local_ip) -+{ -+ struct ovpn_peer *peer; -+ int ret; -+ -+ /* create new peer */ -+ peer = ovpn_peer_create(ovpn, id); -+ if (IS_ERR(peer)) -+ return peer; -+ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) { -+ /* a UDP peer must have a remote endpoint */ -+ if (!sa) { -+ ovpn_peer_release(peer); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ /* set peer sockaddr */ -+ ret = ovpn_peer_reset_sockaddr(peer, sa, local_ip); -+ if (ret < 0) { -+ ovpn_peer_release(peer); -+ return ERR_PTR(ret); -+ } -+ } -+ -+ peer->sock = ovpn_socket_new(sock, peer); -+ if (IS_ERR(peer->sock)) { -+ peer->sock = NULL; -+ ovpn_peer_release(peer); -+ return ERR_PTR(-ENOTSOCK); -+ } -+ -+ /* schedule initial TCP RX work only after having assigned peer->sock */ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) -+ queue_work(peer->ovpn->events_wq, &peer->tcp.rx_work); -+ -+ return peer; -+} -+ -+/* Configure keepalive parameters */ -+void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) -+{ -+ u32 delta; -+ -+ netdev_dbg(peer->ovpn->dev, -+ "%s: scheduling keepalive for peer %u: interval=%u timeout=%u\n", __func__, -+ peer->id, interval, timeout); -+ -+ peer->keepalive_interval = interval; -+ if (interval > 0) { -+ delta = msecs_to_jiffies(interval * MSEC_PER_SEC); -+ mod_timer(&peer->keepalive_xmit, jiffies + delta); -+ } else { -+ del_timer(&peer->keepalive_xmit); -+ } -+ -+ peer->keepalive_timeout = timeout; -+ if (timeout) { -+ delta = msecs_to_jiffies(timeout * MSEC_PER_SEC); -+ mod_timer(&peer->keepalive_recv, jiffies + delta); -+ } else { -+ del_timer(&peer->keepalive_recv); -+ } -+} -+ -+#define ovpn_peer_index(_tbl, _key, _key_len) \ -+ (jhash(_key, _key_len, 0) % HASH_SIZE(_tbl)) \ -+ -+static struct ovpn_peer *ovpn_peer_lookup_vpn_addr4(struct hlist_head *head, __be32 *addr) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr4) { -+ if (*addr != tmp->vpn_addrs.ipv4.s_addr) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+static struct ovpn_peer *ovpn_peer_lookup_vpn_addr6(struct hlist_head *head, struct in6_addr *addr) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ int i; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr6) { -+ for (i = 0; i < 4; i++) { -+ if (addr->s6_addr32[i] != tmp->vpn_addrs.ipv6.s6_addr32[i]) -+ continue; -+ } -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+/** -+ * ovpn_nexthop4() - looks up the IP of the nexthop for the given destination -+ * -+ * Looks up in the IPv4 system routing table the IO of the nexthop to be used -+ * to reach the destination passed as argument. IF no nexthop can be found, the -+ * destination itself is returned as it probably has to be used as nexthop. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @dst: the destination to be looked up -+ * -+ * Return the IP of the next hop if found or the dst itself otherwise -+ */ -+static __be32 ovpn_nexthop4(struct ovpn_struct *ovpn, __be32 dst) -+{ -+ struct rtable *rt; -+ struct flowi4 fl = { -+ .daddr = dst -+ }; -+ -+ rt = ip_route_output_flow(dev_net(ovpn->dev), &fl, NULL); -+ if (IS_ERR(rt)) { -+ net_dbg_ratelimited("%s: no route to host %pI4\n", __func__, &dst); -+ /* if we end up here this packet is probably going to be thrown away later */ -+ return dst; -+ } -+ -+ if (!rt->rt_uses_gateway) -+ goto out; -+ -+ dst = rt->rt_gw4; -+out: -+ ip_rt_put(rt); -+ return dst; -+} -+ -+/** -+ * ovpn_nexthop6() - looks up the IPv6 of the nexthop for the given destination -+ * -+ * Looks up in the IPv6 system routing table the IO of the nexthop to be used -+ * to reach the destination passed as argument. IF no nexthop can be found, the -+ * destination itself is returned as it probably has to be used as nexthop. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @dst: the destination to be looked up -+ * -+ * Return the IP of the next hop if found or the dst itself otherwise -+ */ -+static struct in6_addr ovpn_nexthop6(struct ovpn_struct *ovpn, struct in6_addr dst) -+{ -+#if IS_ENABLED(CONFIG_IPV6) -+ struct rt6_info *rt; -+ struct flowi6 fl = { -+ .daddr = dst, -+ }; -+ -+ rt = (struct rt6_info *)ipv6_stub->ipv6_dst_lookup_flow(dev_net(ovpn->dev), NULL, &fl, -+ NULL); -+ if (IS_ERR(rt)) { -+ net_dbg_ratelimited("%s: no route to host %pI6\n", __func__, &dst); -+ /* if we end up here this packet is probably going to be thrown away later */ -+ return dst; -+ } -+ -+ if (!(rt->rt6i_flags & RTF_GATEWAY)) -+ goto out; -+ -+ dst = rt->rt6i_gateway; -+out: -+ dst_release((struct dst_entry *)rt); -+#endif -+ return dst; -+} -+ -+/** -+ * ovpn_peer_lookup_vpn_addr() - Lookup peer to send skb to -+ * -+ * This function takes a tunnel packet and looks up the peer to send it to -+ * after encapsulation. The skb is expected to be the in-tunnel packet, without -+ * any OpenVPN related header. -+ * -+ * Assume that the IP header is accessible in the skb data. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @skb: the skb to extract the destination address from -+ * -+ * Return the peer if found or NULL otherwise. -+ */ -+struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, -+ bool use_src) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ struct hlist_head *head; -+ struct rt6_info *rt6i = NULL; -+ struct rtable *rt = NULL; -+ sa_family_t sa_fam; -+ struct in6_addr addr6; -+ __be32 addr4; -+ u32 index; -+ -+ /* in P2P mode, no matter the destination, packets are always sent to the single peer -+ * listening on the other side -+ */ -+ if (ovpn->mode == OVPN_MODE_P2P) { -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ return peer; -+ } -+ -+ sa_fam = skb_protocol_to_family(skb); -+ -+ switch (sa_fam) { -+ case AF_INET: -+ if (use_src) -+ addr4 = ip_hdr(skb)->saddr; -+ else -+ addr4 = ip_hdr(skb)->daddr; -+ addr4 = ovpn_nexthop4(ovpn, addr4); -+ -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr4, sizeof(addr4)); -+ head = &ovpn->peers.by_vpn_addr[index]; -+ -+ peer = ovpn_peer_lookup_vpn_addr4(head, &addr4); -+ break; -+ case AF_INET6: -+ if (use_src) -+ addr6 = ipv6_hdr(skb)->saddr; -+ else -+ addr6 = ipv6_hdr(skb)->daddr; -+ addr6 = ovpn_nexthop6(ovpn, addr6); -+ -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr6, sizeof(addr6)); -+ head = &ovpn->peers.by_vpn_addr[index]; -+ -+ peer = ovpn_peer_lookup_vpn_addr6(head, &addr6); -+ break; -+ } -+ -+ if (rt) -+ ip_rt_put(rt); -+ if (rt6i) -+ dst_release((struct dst_entry *)rt6i); -+ -+ return peer; -+} -+ -+static bool ovpn_peer_transp_match(struct ovpn_peer *peer, struct sockaddr_storage *ss) -+{ -+ struct ovpn_bind *bind = rcu_dereference(peer->bind); -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ -+ if (unlikely(!bind)) -+ return false; -+ -+ if (ss->ss_family != bind->sa.in4.sin_family) -+ return false; -+ -+ switch (ss->ss_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)ss; -+ if (sa4->sin_addr.s_addr != bind->sa.in4.sin_addr.s_addr) -+ return false; -+ if (sa4->sin_port != bind->sa.in4.sin_port) -+ return false; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)ss; -+ if (memcmp(&sa6->sin6_addr, &bind->sa.in6.sin6_addr, sizeof(struct in6_addr))) -+ return false; -+ if (sa6->sin6_port != bind->sa.in6.sin6_port) -+ return false; -+ break; -+ default: -+ return false; -+ } -+ -+ return true; -+} -+ -+static bool ovpn_peer_skb_to_sockaddr(struct sk_buff *skb, struct sockaddr_storage *ss) -+{ -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ -+ ss->ss_family = skb_protocol_to_family(skb); -+ switch (ss->ss_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)ss; -+ sa4->sin_family = AF_INET; -+ sa4->sin_addr.s_addr = ip_hdr(skb)->saddr; -+ sa4->sin_port = udp_hdr(skb)->source; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)ss; -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = ipv6_hdr(skb)->saddr; -+ sa6->sin6_port = udp_hdr(skb)->source; -+ break; -+ default: -+ return false; -+ } -+ -+ return true; -+} -+ -+static struct ovpn_peer *ovpn_peer_lookup_transp_addr_p2p(struct ovpn_struct *ovpn, -+ struct sockaddr_storage *ss) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && ovpn_peer_transp_match(tmp, ss) && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb) -+{ -+ struct ovpn_peer *peer = NULL, *tmp; -+ struct sockaddr_storage ss = { 0 }; -+ struct hlist_head *head; -+ size_t sa_len; -+ bool found; -+ u32 index; -+ -+ if (unlikely(!ovpn_peer_skb_to_sockaddr(skb, &ss))) -+ return NULL; -+ -+ if (ovpn->mode == OVPN_MODE_P2P) -+ return ovpn_peer_lookup_transp_addr_p2p(ovpn, &ss); -+ -+ switch (ss.ss_family) { -+ case AF_INET: -+ sa_len = sizeof(struct sockaddr_in); -+ break; -+ case AF_INET6: -+ sa_len = sizeof(struct sockaddr_in6); -+ break; -+ default: -+ return NULL; -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &ss, sa_len); -+ head = &ovpn->peers.by_transp_addr[index]; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_transp_addr) { -+ found = ovpn_peer_transp_match(tmp, &ss); -+ if (!found) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+static struct ovpn_peer *ovpn_peer_lookup_id_p2p(struct ovpn_struct *ovpn, u32 peer_id) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ struct hlist_head *head; -+ u32 index; -+ -+ if (ovpn->mode == OVPN_MODE_P2P) -+ return ovpn_peer_lookup_id_p2p(ovpn, peer_id); -+ -+ index = ovpn_peer_index(ovpn->peers.by_id, &peer_id, sizeof(peer_id)); -+ head = &ovpn->peers.by_id[index]; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_id) { -+ if (tmp->id != peer_id) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_bind *bind; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) -+ goto unlock; -+ -+ switch (skb_protocol_to_family(skb)) { -+ case AF_INET: -+ if (unlikely(bind->local.ipv4.s_addr != ip_hdr(skb)->daddr)) { -+ netdev_dbg(peer->ovpn->dev, -+ "%s: learning local IPv4 for peer %d (%pI4 -> %pI4)\n", __func__, -+ peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); -+ bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; -+ } -+ break; -+ case AF_INET6: -+ if (unlikely(memcmp(&bind->local.ipv6, &ipv6_hdr(skb)->daddr, -+ sizeof(bind->local.ipv6)))) { -+ netdev_dbg(peer->ovpn->dev, -+ "%s: learning local IPv6 for peer %d (%pI6c -> %pI6c\n", -+ __func__, peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); -+ bind->local.ipv6 = ipv6_hdr(skb)->daddr; -+ } -+ break; -+ default: -+ break; -+ } -+unlock: -+ rcu_read_unlock(); -+} -+ -+static int ovpn_peer_add_mp(struct ovpn_struct *ovpn, struct ovpn_peer *peer) -+{ -+ struct sockaddr_storage sa = { 0 }; -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ struct ovpn_bind *bind; -+ struct ovpn_peer *tmp; -+ size_t salen; -+ int ret = 0; -+ u32 index; -+ -+ spin_lock_bh(&ovpn->peers.lock); -+ /* do not add duplicates */ -+ tmp = ovpn_peer_lookup_id(ovpn, peer->id); -+ if (tmp) { -+ ovpn_peer_put(tmp); -+ ret = -EEXIST; -+ goto unlock; -+ } -+ -+ hlist_del_init_rcu(&peer->hash_entry_transp_addr); -+ bind = rcu_dereference_protected(peer->bind, true); -+ /* peers connected via UDP have bind == NULL */ -+ if (bind) { -+ switch (bind->sa.in4.sin_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)&sa; -+ -+ sa4->sin_family = AF_INET; -+ sa4->sin_addr.s_addr = bind->sa.in4.sin_addr.s_addr; -+ sa4->sin_port = bind->sa.in4.sin_port; -+ salen = sizeof(*sa4); -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)&sa; -+ -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = bind->sa.in6.sin6_addr; -+ sa6->sin6_port = bind->sa.in6.sin6_port; -+ salen = sizeof(*sa6); -+ break; -+ default: -+ ret = -EPROTONOSUPPORT; -+ goto unlock; -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &sa, salen); -+ hlist_add_head_rcu(&peer->hash_entry_transp_addr, -+ &ovpn->peers.by_transp_addr[index]); -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_id, &peer->id, sizeof(peer->id)); -+ hlist_add_head_rcu(&peer->hash_entry_id, &ovpn->peers.by_id[index]); -+ -+ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv4, -+ sizeof(peer->vpn_addrs.ipv4)); -+ hlist_add_head_rcu(&peer->hash_entry_addr4, &ovpn->peers.by_vpn_addr[index]); -+ } -+ -+ hlist_del_init_rcu(&peer->hash_entry_addr6); -+ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) { -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv6, -+ sizeof(peer->vpn_addrs.ipv6)); -+ hlist_add_head_rcu(&peer->hash_entry_addr6, &ovpn->peers.by_vpn_addr[index]); -+ } -+ -+unlock: -+ spin_unlock_bh(&ovpn->peers.lock); -+ -+ return ret; -+} -+ -+static int ovpn_peer_add_p2p(struct ovpn_struct *ovpn, struct ovpn_peer *peer) -+{ -+ struct ovpn_peer *tmp; -+ -+ spin_lock_bh(&ovpn->lock); -+ /* in p2p mode it is possible to have a single peer only, therefore the -+ * old one is released and substituted by the new one -+ */ -+ tmp = rcu_dereference(ovpn->peer); -+ if (tmp) { -+ tmp->delete_reason = OVPN_DEL_PEER_REASON_TEARDOWN; -+ ovpn_peer_put(tmp); -+ } -+ -+ rcu_assign_pointer(ovpn->peer, peer); -+ spin_unlock_bh(&ovpn->lock); -+ -+ return 0; -+} -+ -+/* assume refcounter was increased by caller */ -+int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer) -+{ -+ switch (ovpn->mode) { -+ case OVPN_MODE_MP: -+ return ovpn_peer_add_mp(ovpn, peer); -+ case OVPN_MODE_P2P: -+ return ovpn_peer_add_p2p(ovpn, peer); -+ default: -+ return -EOPNOTSUPP; -+ } -+} -+ -+static void ovpn_peer_unhash(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ hlist_del_init_rcu(&peer->hash_entry_id); -+ hlist_del_init_rcu(&peer->hash_entry_addr4); -+ hlist_del_init_rcu(&peer->hash_entry_addr6); -+ hlist_del_init_rcu(&peer->hash_entry_transp_addr); -+ -+ ovpn_peer_put(peer); -+ peer->delete_reason = reason; -+} -+ -+static int ovpn_peer_del_mp(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ struct ovpn_peer *tmp; -+ int ret = 0; -+ -+ spin_lock_bh(&peer->ovpn->peers.lock); -+ tmp = ovpn_peer_lookup_id(peer->ovpn, peer->id); -+ if (tmp != peer) { -+ ret = -ENOENT; -+ goto unlock; -+ } -+ ovpn_peer_unhash(peer, reason); -+ -+unlock: -+ spin_unlock_bh(&peer->ovpn->peers.lock); -+ -+ if (tmp) -+ ovpn_peer_put(tmp); -+ -+ return ret; -+} -+ -+static int ovpn_peer_del_p2p(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ struct ovpn_peer *tmp; -+ int ret = -ENOENT; -+ -+ spin_lock_bh(&peer->ovpn->lock); -+ tmp = rcu_dereference(peer->ovpn->peer); -+ if (tmp != peer) -+ goto unlock; -+ -+ ovpn_peer_put(tmp); -+ tmp->delete_reason = reason; -+ RCU_INIT_POINTER(peer->ovpn->peer, NULL); -+ ret = 0; -+ -+unlock: -+ spin_unlock_bh(&peer->ovpn->lock); -+ -+ return ret; -+} -+ -+void ovpn_peer_release_p2p(struct ovpn_struct *ovpn) -+{ -+ struct ovpn_peer *tmp; -+ -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (!tmp) -+ goto unlock; -+ -+ ovpn_peer_del_p2p(tmp, OVPN_DEL_PEER_REASON_TEARDOWN); -+unlock: -+ rcu_read_unlock(); -+} -+ -+int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ switch (peer->ovpn->mode) { -+ case OVPN_MODE_MP: -+ return ovpn_peer_del_mp(peer, reason); -+ case OVPN_MODE_P2P: -+ return ovpn_peer_del_p2p(peer, reason); -+ default: -+ return -EOPNOTSUPP; -+ } -+} -+ -+void ovpn_peers_free(struct ovpn_struct *ovpn) -+{ -+ struct hlist_node *tmp; -+ struct ovpn_peer *peer; -+ int bkt; -+ -+ spin_lock_bh(&ovpn->peers.lock); -+ hash_for_each_safe(ovpn->peers.by_id, bkt, tmp, peer, hash_entry_id) -+ ovpn_peer_unhash(peer, OVPN_DEL_PEER_REASON_TEARDOWN); -+ spin_unlock_bh(&ovpn->peers.lock); -+} -diff --git a/drivers/net/ovpn-dco/peer.h b/drivers/net/ovpn-dco/peer.h -new file mode 100644 -index 000000000000..b759c2c9da48 ---- /dev/null -+++ b/drivers/net/ovpn-dco/peer.h -@@ -0,0 +1,168 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNPEER_H_ -+#define _NET_OVPN_DCO_OVPNPEER_H_ -+ -+#include "addr.h" -+#include "bind.h" -+#include "sock.h" -+#include "stats.h" -+ -+#include -+#include -+#include -+ -+struct ovpn_peer { -+ struct ovpn_struct *ovpn; -+ -+ u32 id; -+ -+ struct { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } vpn_addrs; -+ -+ struct hlist_node hash_entry_id; -+ struct hlist_node hash_entry_addr4; -+ struct hlist_node hash_entry_addr6; -+ struct hlist_node hash_entry_transp_addr; -+ -+ /* work objects to handle encryption/decryption of packets. -+ * these works are queued on the ovpn->crypt_wq workqueue. -+ */ -+ struct work_struct encrypt_work; -+ struct work_struct decrypt_work; -+ -+ struct ptr_ring tx_ring; -+ struct ptr_ring rx_ring; -+ struct ptr_ring netif_rx_ring; -+ -+ struct napi_struct napi; -+ -+ struct ovpn_socket *sock; -+ -+ /* state of the TCP reading. Needed to keep track of how much of a single packet has already -+ * been read from the stream and how much is missing -+ */ -+ struct { -+ struct ptr_ring tx_ring; -+ struct work_struct tx_work; -+ struct work_struct rx_work; -+ -+ u8 raw_len[sizeof(u16)]; -+ struct sk_buff *skb; -+ u16 offset; -+ u16 data_len; -+ struct { -+ void (*sk_state_change)(struct sock *sk); -+ void (*sk_data_ready)(struct sock *sk); -+ void (*sk_write_space)(struct sock *sk); -+ } sk_cb; -+ } tcp; -+ -+ struct dst_cache dst_cache; -+ -+ /* our crypto state */ -+ struct ovpn_crypto_state crypto; -+ -+ /* our binding to peer, protected by spinlock */ -+ struct ovpn_bind __rcu *bind; -+ -+ /* timer used to send periodic ping messages to the other peer, if no -+ * other data was sent within the past keepalive_interval seconds -+ */ -+ struct timer_list keepalive_xmit; -+ /* keepalive interval in seconds */ -+ unsigned long keepalive_interval; -+ -+ /* timer used to mark a peer as expired when no data is received for -+ * keepalive_timeout seconds -+ */ -+ struct timer_list keepalive_recv; -+ /* keepalive timeout in seconds */ -+ unsigned long keepalive_timeout; -+ -+ /* true if ovpn_peer_mark_delete was called */ -+ bool halt; -+ -+ /* per-peer rx/tx stats */ -+ struct ovpn_peer_stats stats; -+ -+ /* why peer was deleted - keepalive timeout, module removed etc */ -+ enum ovpn_del_peer_reason delete_reason; -+ -+ /* protects binding to peer (bind) and timers -+ * (keepalive_xmit, keepalive_expire) -+ */ -+ spinlock_t lock; -+ -+ /* needed because crypto methods can go async */ -+ struct kref refcount; -+ -+ /* needed to free a peer in an RCU safe way */ -+ struct rcu_head rcu; -+ -+ /* needed to notify userspace about deletion */ -+ struct work_struct delete_work; -+}; -+ -+void ovpn_peer_release_kref(struct kref *kref); -+void ovpn_peer_release(struct ovpn_peer *peer); -+ -+static inline bool ovpn_peer_hold(struct ovpn_peer *peer) -+{ -+ return kref_get_unless_zero(&peer->refcount); -+} -+ -+static inline void ovpn_peer_put(struct ovpn_peer *peer) -+{ -+ kref_put(&peer->refcount, ovpn_peer_release_kref); -+} -+ -+static inline void ovpn_peer_keepalive_recv_reset(struct ovpn_peer *peer) -+{ -+ u32 delta = msecs_to_jiffies(peer->keepalive_timeout * MSEC_PER_SEC); -+ -+ if (unlikely(!delta)) -+ return; -+ -+ mod_timer(&peer->keepalive_recv, jiffies + delta); -+} -+ -+static inline void ovpn_peer_keepalive_xmit_reset(struct ovpn_peer *peer) -+{ -+ u32 delta = msecs_to_jiffies(peer->keepalive_interval * MSEC_PER_SEC); -+ -+ if (unlikely(!delta)) -+ return; -+ -+ mod_timer(&peer->keepalive_xmit, jiffies + delta); -+} -+ -+struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, -+ struct socket *sock, u32 id, uint8_t *local_ip); -+ -+void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout); -+ -+int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer); -+int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason); -+struct ovpn_peer *ovpn_peer_find(struct ovpn_struct *ovpn, u32 peer_id); -+void ovpn_peer_release_p2p(struct ovpn_struct *ovpn); -+void ovpn_peers_free(struct ovpn_struct *ovpn); -+ -+struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb); -+struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, -+ bool use_src); -+struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id); -+ -+void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb); -+void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb); -+ -+#endif /* _NET_OVPN_DCO_OVPNPEER_H_ */ -diff --git a/drivers/net/ovpn-dco/pktid.c b/drivers/net/ovpn-dco/pktid.c -new file mode 100644 -index 000000000000..fcde8fba5156 ---- /dev/null -+++ b/drivers/net/ovpn-dco/pktid.c -@@ -0,0 +1,127 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#include "pktid.h" -+ -+#include -+#include -+ -+void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid) -+{ -+ atomic64_set(&pid->seq_num, 1); -+ pid->tcp_linear = NULL; -+} -+ -+void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr) -+{ -+ memset(pr, 0, sizeof(*pr)); -+ spin_lock_init(&pr->lock); -+} -+ -+/* Packet replay detection. -+ * Allows ID backtrack of up to REPLAY_WINDOW_SIZE - 1. -+ */ -+int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time) -+{ -+ const unsigned long now = jiffies; -+ int ret; -+ -+ spin_lock(&pr->lock); -+ -+ /* expire backtracks at or below pr->id after PKTID_RECV_EXPIRE time */ -+ if (unlikely(time_after_eq(now, pr->expire))) -+ pr->id_floor = pr->id; -+ -+ /* ID must not be zero */ -+ if (unlikely(pkt_id == 0)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ /* time changed? */ -+ if (unlikely(pkt_time != pr->time)) { -+ if (pkt_time > pr->time) { -+ /* time moved forward, accept */ -+ pr->base = 0; -+ pr->extent = 0; -+ pr->id = 0; -+ pr->time = pkt_time; -+ pr->id_floor = 0; -+ } else { -+ /* time moved backward, reject */ -+ ret = -ETIME; -+ goto out; -+ } -+ } -+ -+ if (likely(pkt_id == pr->id + 1)) { -+ /* well-formed ID sequence (incremented by 1) */ -+ pr->base = REPLAY_INDEX(pr->base, -1); -+ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); -+ if (pr->extent < REPLAY_WINDOW_SIZE) -+ ++pr->extent; -+ pr->id = pkt_id; -+ } else if (pkt_id > pr->id) { -+ /* ID jumped forward by more than one */ -+ const unsigned int delta = pkt_id - pr->id; -+ -+ if (delta < REPLAY_WINDOW_SIZE) { -+ unsigned int i; -+ -+ pr->base = REPLAY_INDEX(pr->base, -delta); -+ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); -+ pr->extent += delta; -+ if (pr->extent > REPLAY_WINDOW_SIZE) -+ pr->extent = REPLAY_WINDOW_SIZE; -+ for (i = 1; i < delta; ++i) { -+ unsigned int newb = REPLAY_INDEX(pr->base, i); -+ -+ pr->history[newb / 8] &= ~BIT(newb % 8); -+ } -+ } else { -+ pr->base = 0; -+ pr->extent = REPLAY_WINDOW_SIZE; -+ memset(pr->history, 0, sizeof(pr->history)); -+ pr->history[0] = 1; -+ } -+ pr->id = pkt_id; -+ } else { -+ /* ID backtrack */ -+ const unsigned int delta = pr->id - pkt_id; -+ -+ if (delta > pr->max_backtrack) -+ pr->max_backtrack = delta; -+ if (delta < pr->extent) { -+ if (pkt_id > pr->id_floor) { -+ const unsigned int ri = REPLAY_INDEX(pr->base, -+ delta); -+ u8 *p = &pr->history[ri / 8]; -+ const u8 mask = (1 << (ri % 8)); -+ -+ if (*p & mask) { -+ ret = -EINVAL; -+ goto out; -+ } -+ *p |= mask; -+ } else { -+ ret = -EINVAL; -+ goto out; -+ } -+ } else { -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ -+ pr->expire = now + PKTID_RECV_EXPIRE; -+ ret = 0; -+out: -+ spin_unlock(&pr->lock); -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/pktid.h b/drivers/net/ovpn-dco/pktid.h -new file mode 100644 -index 000000000000..2447bb37ba55 ---- /dev/null -+++ b/drivers/net/ovpn-dco/pktid.h -@@ -0,0 +1,116 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNPKTID_H_ -+#define _NET_OVPN_DCO_OVPNPKTID_H_ -+ -+#include "main.h" -+ -+/* When the OpenVPN protocol is run in AEAD mode, use -+ * the OpenVPN packet ID as the AEAD nonce: -+ * -+ * 00000005 521c3b01 4308c041 -+ * [seq # ] [ nonce_tail ] -+ * [ 12-byte full IV ] -> NONCE_SIZE -+ * [4-bytes -> NONCE_WIRE_SIZE -+ * on wire] -+ */ -+ -+/* OpenVPN nonce size */ -+#define NONCE_SIZE 12 -+/* amount of bytes of the nonce received from user space */ -+#define NONCE_TAIL_SIZE 8 -+ -+/* OpenVPN nonce size reduced by 8-byte nonce tail -- this is the -+ * size of the AEAD Associated Data (AD) sent over the wire -+ * and is normally the head of the IV -+ */ -+#define NONCE_WIRE_SIZE (NONCE_SIZE - sizeof(struct ovpn_nonce_tail)) -+ -+/* If no packets received for this length of time, set a backtrack floor -+ * at highest received packet ID thus far. -+ */ -+#define PKTID_RECV_EXPIRE (30 * HZ) -+ -+/* Last 8 bytes of AEAD nonce -+ * Provided by userspace and usually derived from -+ * key material generated during TLS handshake -+ */ -+struct ovpn_nonce_tail { -+ u8 u8[NONCE_TAIL_SIZE]; -+}; -+ -+/* Packet-ID state for transmitter */ -+struct ovpn_pktid_xmit { -+ atomic64_t seq_num; -+ struct ovpn_tcp_linear *tcp_linear; -+}; -+ -+/* replay window sizing in bytes = 2^REPLAY_WINDOW_ORDER */ -+#define REPLAY_WINDOW_ORDER 8 -+ -+#define REPLAY_WINDOW_BYTES BIT(REPLAY_WINDOW_ORDER) -+#define REPLAY_WINDOW_SIZE (REPLAY_WINDOW_BYTES * 8) -+#define REPLAY_INDEX(base, i) (((base) + (i)) & (REPLAY_WINDOW_SIZE - 1)) -+ -+/* Packet-ID state for receiver. -+ * Other than lock member, can be zeroed to initialize. -+ */ -+struct ovpn_pktid_recv { -+ /* "sliding window" bitmask of recent packet IDs received */ -+ u8 history[REPLAY_WINDOW_BYTES]; -+ /* bit position of deque base in history */ -+ unsigned int base; -+ /* extent (in bits) of deque in history */ -+ unsigned int extent; -+ /* expiration of history in jiffies */ -+ unsigned long expire; -+ /* highest sequence number received */ -+ u32 id; -+ /* highest time stamp received */ -+ u32 time; -+ /* we will only accept backtrack IDs > id_floor */ -+ u32 id_floor; -+ unsigned int max_backtrack; -+ /* protects entire pktd ID state */ -+ spinlock_t lock; -+}; -+ -+/* Get the next packet ID for xmit */ -+static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) -+{ -+ const s64 seq_num = atomic64_fetch_add_unless(&pid->seq_num, 1, -+ 0x100000000LL); -+ /* when the 32bit space is over, we return an error because the packet ID is used to create -+ * the cipher IV and we do not want to re-use the same value more than once -+ */ -+ if (unlikely(seq_num == 0x100000000LL)) -+ return -ERANGE; -+ -+ *pktid = (u32)seq_num; -+ -+ return 0; -+} -+ -+/* Write 12-byte AEAD IV to dest */ -+static inline void ovpn_pktid_aead_write(const u32 pktid, -+ const struct ovpn_nonce_tail *nt, -+ unsigned char *dest) -+{ -+ *(__force __be32 *)(dest) = htonl(pktid); -+ BUILD_BUG_ON(4 + sizeof(struct ovpn_nonce_tail) != NONCE_SIZE); -+ memcpy(dest + 4, nt->u8, sizeof(struct ovpn_nonce_tail)); -+} -+ -+void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid); -+void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr); -+ -+int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time); -+ -+#endif /* _NET_OVPN_DCO_OVPNPKTID_H_ */ -diff --git a/drivers/net/ovpn-dco/proto.h b/drivers/net/ovpn-dco/proto.h -new file mode 100644 -index 000000000000..875529021c1b ---- /dev/null -+++ b/drivers/net/ovpn-dco/proto.h -@@ -0,0 +1,101 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNPROTO_H_ -+#define _NET_OVPN_DCO_OVPNPROTO_H_ -+ -+#include "main.h" -+ -+#include -+ -+/* Methods for operating on the initial command -+ * byte of the OpenVPN protocol. -+ */ -+ -+/* packet opcode (high 5 bits) and key-id (low 3 bits) are combined in -+ * one byte -+ */ -+#define OVPN_KEY_ID_MASK 0x07 -+#define OVPN_OPCODE_SHIFT 3 -+#define OVPN_OPCODE_MASK 0x1F -+/* upper bounds on opcode and key ID */ -+#define OVPN_KEY_ID_MAX (OVPN_KEY_ID_MASK + 1) -+#define OVPN_OPCODE_MAX (OVPN_OPCODE_MASK + 1) -+/* packet opcodes of interest to us */ -+#define OVPN_DATA_V1 6 /* data channel V1 packet */ -+#define OVPN_DATA_V2 9 /* data channel V2 packet */ -+/* size of initial packet opcode */ -+#define OVPN_OP_SIZE_V1 1 -+#define OVPN_OP_SIZE_V2 4 -+#define OVPN_PEER_ID_MASK 0x00FFFFFF -+#define OVPN_PEER_ID_UNDEF 0x00FFFFFF -+/* first byte of keepalive message */ -+#define OVPN_KEEPALIVE_FIRST_BYTE 0x2a -+/* first byte of exit message */ -+#define OVPN_EXPLICIT_EXIT_NOTIFY_FIRST_BYTE 0x28 -+ -+/** -+ * Extract the OP code from the specified byte -+ * -+ * Return the OP code -+ */ -+static inline u8 ovpn_opcode_from_byte(u8 byte) -+{ -+ return byte >> OVPN_OPCODE_SHIFT; -+} -+ -+/** -+ * Extract the OP code from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first byte. -+ * -+ * Return the OP code -+ */ -+static inline u8 ovpn_opcode_from_skb(const struct sk_buff *skb, u16 offset) -+{ -+ return ovpn_opcode_from_byte(*(skb->data + offset)); -+} -+ -+/** -+ * Extract the key ID from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first byte. -+ * -+ * Return the key ID -+ */ -+ -+static inline u8 ovpn_key_id_from_skb(const struct sk_buff *skb) -+{ -+ return *skb->data & OVPN_KEY_ID_MASK; -+} -+ -+/** -+ * Extract the peer ID from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first 4 bytes. -+ * -+ * Return the peer ID. -+ */ -+ -+static inline u32 ovpn_peer_id_from_skb(const struct sk_buff *skb, u16 offset) -+{ -+ return ntohl(*(__be32 *)(skb->data + offset)) & OVPN_PEER_ID_MASK; -+} -+ -+static inline u32 ovpn_opcode_compose(u8 opcode, u8 key_id, u32 peer_id) -+{ -+ const u8 op = (opcode << OVPN_OPCODE_SHIFT) | (key_id & OVPN_KEY_ID_MASK); -+ -+ return (op << 24) | (peer_id & OVPN_PEER_ID_MASK); -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNPROTO_H_ */ -diff --git a/drivers/net/ovpn-dco/rcu.h b/drivers/net/ovpn-dco/rcu.h -new file mode 100644 -index 000000000000..02a50f49ba2e ---- /dev/null -+++ b/drivers/net/ovpn-dco/rcu.h -@@ -0,0 +1,21 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNRCU_H_ -+#define _NET_OVPN_DCO_OVPNRCU_H_ -+ -+static inline void ovpn_rcu_lockdep_assert_held(void) -+{ -+#ifdef CONFIG_PROVE_RCU -+ RCU_LOCKDEP_WARN(!rcu_read_lock_held(), -+ "ovpn-dco RCU read lock not held"); -+#endif -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNRCU_H_ */ -diff --git a/drivers/net/ovpn-dco/skb.h b/drivers/net/ovpn-dco/skb.h -new file mode 100644 -index 000000000000..d38dc2da01df ---- /dev/null -+++ b/drivers/net/ovpn-dco/skb.h -@@ -0,0 +1,54 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_SKB_H_ -+#define _NET_OVPN_DCO_SKB_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define OVPN_SKB_CB(skb) ((struct ovpn_skb_cb *)&((skb)->cb)) -+ -+struct ovpn_skb_cb { -+ /* original recv packet size for stats accounting */ -+ unsigned int rx_stats_size; -+ -+ union { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } local; -+ sa_family_t sa_fam; -+}; -+ -+/* Return IP protocol version from skb header. -+ * Return 0 if protocol is not IPv4/IPv6 or cannot be read. -+ */ -+static inline __be16 ovpn_ip_check_protocol(struct sk_buff *skb) -+{ -+ __be16 proto = 0; -+ -+ /* skb could be non-linear, -+ * make sure IP header is in non-fragmented part -+ */ -+ if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) -+ return 0; -+ -+ if (ip_hdr(skb)->version == 4) -+ proto = htons(ETH_P_IP); -+ else if (ip_hdr(skb)->version == 6) -+ proto = htons(ETH_P_IPV6); -+ -+ return proto; -+} -+ -+#endif /* _NET_OVPN_DCO_SKB_H_ */ -diff --git a/drivers/net/ovpn-dco/sock.c b/drivers/net/ovpn-dco/sock.c -new file mode 100644 -index 000000000000..e92a4a9b952e ---- /dev/null -+++ b/drivers/net/ovpn-dco/sock.c -@@ -0,0 +1,134 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "sock.h" -+#include "rcu.h" -+#include "tcp.h" -+#include "udp.h" -+ -+#include -+#include -+ -+/* Finalize release of socket, called after RCU grace period */ -+static void ovpn_socket_detach(struct socket *sock) -+{ -+ if (!sock) -+ return; -+ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) -+ ovpn_udp_socket_detach(sock); -+ else if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ovpn_tcp_socket_detach(sock); -+ -+ sockfd_put(sock); -+} -+ -+void ovpn_socket_release_kref(struct kref *kref) -+{ -+ struct ovpn_socket *sock = container_of(kref, struct ovpn_socket, refcount); -+ -+ ovpn_socket_detach(sock->sock); -+ kfree_rcu(sock, rcu); -+} -+ -+static bool ovpn_socket_hold(struct ovpn_socket *sock) -+{ -+ return kref_get_unless_zero(&sock->refcount); -+} -+ -+static struct ovpn_socket *ovpn_socket_get(struct socket *sock) -+{ -+ struct ovpn_socket *ovpn_sock; -+ -+ rcu_read_lock(); -+ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); -+ if (!ovpn_socket_hold(ovpn_sock)) { -+ pr_warn("%s: found ovpn_socket with ref = 0\n", __func__); -+ ovpn_sock = NULL; -+ } -+ rcu_read_unlock(); -+ -+ return ovpn_sock; -+} -+ -+/* Finalize release of socket, called after RCU grace period */ -+static int ovpn_socket_attach(struct socket *sock, struct ovpn_peer *peer) -+{ -+ int ret = -EOPNOTSUPP; -+ -+ if (!sock || !peer) -+ return -EINVAL; -+ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) -+ ret = ovpn_udp_socket_attach(sock, peer->ovpn); -+ else if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ret = ovpn_tcp_socket_attach(sock, peer); -+ -+ return ret; -+} -+ -+struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk) -+{ -+ struct ovpn_socket *ovpn_sock; -+ -+ ovpn_rcu_lockdep_assert_held(); -+ -+ if (unlikely(READ_ONCE(udp_sk(sk)->encap_type) != UDP_ENCAP_OVPNINUDP)) -+ return NULL; -+ -+ ovpn_sock = rcu_dereference_sk_user_data(sk); -+ if (unlikely(!ovpn_sock)) -+ return NULL; -+ -+ /* make sure that sk matches our stored transport socket */ -+ if (unlikely(!ovpn_sock->sock || sk != ovpn_sock->sock->sk)) -+ return NULL; -+ -+ return ovpn_sock->ovpn; -+} -+ -+struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) -+{ -+ struct ovpn_socket *ovpn_sock; -+ int ret; -+ -+ ret = ovpn_socket_attach(sock, peer); -+ if (ret < 0 && ret != -EALREADY) -+ return ERR_PTR(ret); -+ -+ /* if this socket is already owned by this interface, just increase the refcounter */ -+ if (ret == -EALREADY) { -+ /* caller is expected to increase the sock refcounter before passing it to this -+ * function. For this reason we drop it if not needed, like when this socket is -+ * already owned. -+ */ -+ ovpn_sock = ovpn_socket_get(sock); -+ sockfd_put(sock); -+ return ovpn_sock; -+ } -+ -+ ovpn_sock = kzalloc(sizeof(*ovpn_sock), GFP_KERNEL); -+ if (!ovpn_sock) -+ return ERR_PTR(-ENOMEM); -+ -+ ovpn_sock->ovpn = peer->ovpn; -+ ovpn_sock->sock = sock; -+ kref_init(&ovpn_sock->refcount); -+ -+ /* TCP sockets are per-peer, therefore they are linked to their unique peer */ -+ if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ovpn_sock->peer = peer; -+ -+ rcu_assign_sk_user_data(sock->sk, ovpn_sock); -+ -+ return ovpn_sock; -+} -diff --git a/drivers/net/ovpn-dco/sock.h b/drivers/net/ovpn-dco/sock.h -new file mode 100644 -index 000000000000..9e79c1b5fe04 ---- /dev/null -+++ b/drivers/net/ovpn-dco/sock.h -@@ -0,0 +1,54 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_SOCK_H_ -+#define _NET_OVPN_DCO_SOCK_H_ -+ -+#include -+#include -+#include -+ -+#include "peer.h" -+ -+struct ovpn_struct; -+ -+/** -+ * struct ovpn_socket - a kernel socket referenced in the ovpn-dco code -+ */ -+struct ovpn_socket { -+ union { -+ /** @ovpn: the VPN session object owning this socket (UDP only) */ -+ struct ovpn_struct *ovpn; -+ -+ /** @peer: the unique peer transmitting over this socket (TCP only) */ -+ struct ovpn_peer *peer; -+ }; -+ -+ /** @sock: the kernel socket */ -+ struct socket *sock; -+ -+ /** @refcount: amount of contexts currently referencing this object */ -+ struct kref refcount; -+ -+ /** @rcu: member used to schedule RCU destructor callback */ -+ struct rcu_head rcu; -+}; -+ -+struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk); -+ -+void ovpn_socket_release_kref(struct kref *kref); -+ -+static inline void ovpn_socket_put(struct ovpn_socket *sock) -+{ -+ kref_put(&sock->refcount, ovpn_socket_release_kref); -+} -+ -+struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer); -+ -+#endif /* _NET_OVPN_DCO_SOCK_H_ */ -diff --git a/drivers/net/ovpn-dco/stats.c b/drivers/net/ovpn-dco/stats.c -new file mode 100644 -index 000000000000..ee000b2a2177 ---- /dev/null -+++ b/drivers/net/ovpn-dco/stats.c -@@ -0,0 +1,20 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "stats.h" -+ -+void ovpn_peer_stats_init(struct ovpn_peer_stats *ps) -+{ -+ atomic64_set(&ps->rx.bytes, 0); -+ atomic_set(&ps->rx.packets, 0); -+ -+ atomic64_set(&ps->tx.bytes, 0); -+ atomic_set(&ps->tx.packets, 0); -+} -diff --git a/drivers/net/ovpn-dco/stats.h b/drivers/net/ovpn-dco/stats.h -new file mode 100644 -index 000000000000..3aa6bdc049c6 ---- /dev/null -+++ b/drivers/net/ovpn-dco/stats.h -@@ -0,0 +1,67 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ * Lev Stipakov -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNSTATS_H_ -+#define _NET_OVPN_DCO_OVPNSTATS_H_ -+ -+#include -+#include -+ -+struct ovpn_struct; -+ -+/* per-peer stats, measured on transport layer */ -+ -+/* one stat */ -+struct ovpn_peer_stat { -+ atomic64_t bytes; -+ atomic_t packets; -+}; -+ -+/* rx and tx stats, enabled by notify_per != 0 or period != 0 */ -+struct ovpn_peer_stats { -+ struct ovpn_peer_stat rx; -+ struct ovpn_peer_stat tx; -+}; -+ -+/* struct for OVPN_ERR_STATS */ -+ -+struct ovpn_err_stat { -+ unsigned int category; -+ int errcode; -+ u64 count; -+}; -+ -+struct ovpn_err_stats { -+ /* total stats, returned by kovpn */ -+ unsigned int total_stats; -+ /* number of stats dimensioned below */ -+ unsigned int n_stats; -+ struct ovpn_err_stat stats[]; -+}; -+ -+void ovpn_peer_stats_init(struct ovpn_peer_stats *ps); -+ -+static inline void ovpn_peer_stats_increment(struct ovpn_peer_stat *stat, const unsigned int n) -+{ -+ atomic64_add(n, &stat->bytes); -+ atomic_inc(&stat->packets); -+} -+ -+static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats, const unsigned int n) -+{ -+ ovpn_peer_stats_increment(&stats->rx, n); -+} -+ -+static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats, const unsigned int n) -+{ -+ ovpn_peer_stats_increment(&stats->tx, n); -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNSTATS_H_ */ -diff --git a/drivers/net/ovpn-dco/tcp.c b/drivers/net/ovpn-dco/tcp.c -new file mode 100644 -index 000000000000..7e6690fee6e7 ---- /dev/null -+++ b/drivers/net/ovpn-dco/tcp.c -@@ -0,0 +1,326 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "ovpnstruct.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "skb.h" -+#include "tcp.h" -+ -+#include -+#include -+#include -+ -+static void ovpn_tcp_state_change(struct sock *sk) -+{ -+} -+ -+static void ovpn_tcp_data_ready(struct sock *sk) -+{ -+ struct ovpn_socket *sock; -+ -+ rcu_read_lock(); -+ sock = rcu_dereference_sk_user_data(sk); -+ rcu_read_unlock(); -+ -+ if (!sock || !sock->peer) -+ return; -+ -+ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.rx_work); -+} -+ -+static void ovpn_tcp_write_space(struct sock *sk) -+{ -+ struct ovpn_socket *sock; -+ -+ rcu_read_lock(); -+ sock = rcu_dereference_sk_user_data(sk); -+ rcu_read_unlock(); -+ -+ if (!sock || !sock->peer) -+ return; -+ -+ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.tx_work); -+} -+ -+static void ovpn_destroy_skb(void *skb) -+{ -+ consume_skb(skb); -+} -+ -+void ovpn_tcp_socket_detach(struct socket *sock) -+{ -+ struct ovpn_socket *ovpn_sock; -+ struct ovpn_peer *peer; -+ -+ if (!sock) -+ return; -+ -+ rcu_read_lock(); -+ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ -+ if (!ovpn_sock->peer) -+ return; -+ -+ peer = ovpn_sock->peer; -+ -+ /* restore CBs that were saved in ovpn_sock_set_tcp_cb() */ -+ write_lock_bh(&sock->sk->sk_callback_lock); -+ sock->sk->sk_state_change = peer->tcp.sk_cb.sk_state_change; -+ sock->sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; -+ sock->sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; -+ rcu_assign_sk_user_data(sock->sk, NULL); -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ -+ /* cancel any ongoing work. Done after removing the CBs so that these workers cannot be -+ * re-armed -+ */ -+ cancel_work_sync(&peer->tcp.tx_work); -+ cancel_work_sync(&peer->tcp.rx_work); -+ -+ ptr_ring_cleanup(&peer->tcp.tx_ring, ovpn_destroy_skb); -+} -+ -+/* Try to send one skb (or part of it) over the TCP stream. -+ * -+ * Return 0 on success or a negative error code otherwise. -+ * -+ * Note that the skb is modified by putting away the data being sent, therefore -+ * the caller should check if skb->len is zero to understand if the full skb was -+ * sent or not. -+ */ -+static int ovpn_tcp_send_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; -+ struct kvec iv = { 0 }; -+ int ret; -+ -+ if (skb_linearize(skb) < 0) { -+ net_err_ratelimited("%s: can't linearize packet\n", __func__); -+ return -ENOMEM; -+ } -+ -+ /* initialize iv structure now as skb_linearize() may have changed skb->data */ -+ iv.iov_base = skb->data; -+ iv.iov_len = skb->len; -+ -+ ret = kernel_sendmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len); -+ if (ret > 0) { -+ __skb_pull(skb, ret); -+ -+ /* since we update per-cpu stats in process context, -+ * we need to disable softirqs -+ */ -+ local_bh_disable(); -+ dev_sw_netstats_tx_add(peer->ovpn->dev, 1, ret); -+ local_bh_enable(); -+ -+ return 0; -+ } -+ -+ return ret; -+} -+ -+/* Process packets in TCP TX queue */ -+static void ovpn_tcp_tx_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ int ret; -+ -+ peer = container_of(work, struct ovpn_peer, tcp.tx_work); -+ while ((skb = __ptr_ring_peek(&peer->tcp.tx_ring))) { -+ ret = ovpn_tcp_send_one(peer, skb); -+ if (ret < 0 && ret != -EAGAIN) { -+ net_warn_ratelimited("%s: cannot send TCP packet to peer %u: %d\n", __func__, -+ peer->id, ret); -+ /* in case of TCP error stop sending loop and delete peer */ -+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_ERROR); -+ break; -+ } else if (!skb->len) { -+ /* skb was entirely consumed and can now be removed from the ring */ -+ __ptr_ring_discard_one(&peer->tcp.tx_ring); -+ consume_skb(skb); -+ } -+ -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+} -+ -+static int ovpn_tcp_rx_one(struct ovpn_peer *peer) -+{ -+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; -+ struct ovpn_skb_cb *cb; -+ int status, ret; -+ -+ /* no skb allocated means that we have to read (or finish reading) the 2 bytes prefix -+ * containing the actual packet size. -+ */ -+ if (!peer->tcp.skb) { -+ struct kvec iv = { -+ .iov_base = peer->tcp.raw_len + peer->tcp.offset, -+ .iov_len = sizeof(u16) - peer->tcp.offset, -+ }; -+ -+ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); -+ if (ret <= 0) -+ return ret; -+ -+ peer->tcp.offset += ret; -+ /* the entire packet size was read, prepare skb for reading data */ -+ if (peer->tcp.offset == sizeof(u16)) { -+ u16 len = ntohs(*(__be16 *)peer->tcp.raw_len); -+ /* invalid packet length: this is a fatal TCP error */ -+ if (!len) { -+ netdev_err(peer->ovpn->dev, "%s: received invalid packet length\n", -+ __func__); -+ return -EINVAL; -+ } -+ -+ peer->tcp.skb = netdev_alloc_skb_ip_align(peer->ovpn->dev, len); -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = len; -+ } -+ } else { -+ struct kvec iv = { -+ .iov_base = peer->tcp.skb->data + peer->tcp.offset, -+ .iov_len = peer->tcp.data_len - peer->tcp.offset, -+ }; -+ -+ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); -+ if (ret <= 0) -+ return ret; -+ -+ peer->tcp.offset += ret; -+ /* full packet received, send it up for processing */ -+ if (peer->tcp.offset == peer->tcp.data_len) { -+ /* update the skb data structure with the amount of data written by -+ * kernel_recvmsg() -+ */ -+ skb_put(peer->tcp.skb, peer->tcp.data_len); -+ -+ /* do not perform IP caching for TCP connections */ -+ cb = OVPN_SKB_CB(peer->tcp.skb); -+ cb->sa_fam = AF_UNSPEC; -+ -+ /* hold reference to peer as requird by ovpn_recv() */ -+ ovpn_peer_hold(peer); -+ status = ovpn_recv(peer->ovpn, peer, peer->tcp.skb); -+ /* skb not consumed - free it now */ -+ if (unlikely(status < 0)) -+ kfree_skb(peer->tcp.skb); -+ -+ peer->tcp.skb = NULL; -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = 0; -+ } -+ } -+ -+ return ret; -+} -+ -+static void ovpn_tcp_rx_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, tcp.rx_work); -+ int ret; -+ -+ while (true) { -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ -+ ret = ovpn_tcp_rx_one(peer); -+ if (ret <= 0) -+ break; -+ } -+ -+ if (ret < 0 && ret != -EAGAIN) -+ netdev_err(peer->ovpn->dev, "%s: TCP socket error: %d\n", __func__, ret); -+} -+ -+/* Put packet into TCP TX queue and schedule a consumer */ -+void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ int ret; -+ -+ ret = ptr_ring_produce_bh(&peer->tcp.tx_ring, skb); -+ if (ret < 0) { -+ kfree_skb_list(skb); -+ return; -+ } -+ -+ queue_work(peer->ovpn->events_wq, &peer->tcp.tx_work); -+} -+ -+/* Set TCP encapsulation callbacks */ -+int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer) -+{ -+ void *old_data; -+ int ret; -+ -+ INIT_WORK(&peer->tcp.tx_work, ovpn_tcp_tx_work); -+ INIT_WORK(&peer->tcp.rx_work, ovpn_tcp_rx_work); -+ -+ ret = ptr_ring_init(&peer->tcp.tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(peer->ovpn->dev, "cannot allocate TCP TX ring\n"); -+ return ret; -+ } -+ -+ peer->tcp.skb = NULL; -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = 0; -+ -+ write_lock_bh(&sock->sk->sk_callback_lock); -+ -+ /* make sure no pre-existing encapsulation handler exists */ -+ rcu_read_lock(); -+ old_data = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ if (old_data) { -+ netdev_err(peer->ovpn->dev, "provided socket already taken by other user\n"); -+ ret = -EBUSY; -+ goto err; -+ } -+ -+ /* sanity check */ -+ if (sock->sk->sk_protocol != IPPROTO_TCP) { -+ netdev_err(peer->ovpn->dev, "expected TCP socket\n"); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ /* only a fully connected socket are expected. Connection should be handled in userspace */ -+ if (sock->sk->sk_state != TCP_ESTABLISHED) { -+ netdev_err(peer->ovpn->dev, "unexpected state for TCP socket: %d\n", -+ sock->sk->sk_state); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ /* save current CBs so that they can be restored upon socket release */ -+ peer->tcp.sk_cb.sk_state_change = sock->sk->sk_state_change; -+ peer->tcp.sk_cb.sk_data_ready = sock->sk->sk_data_ready; -+ peer->tcp.sk_cb.sk_write_space = sock->sk->sk_write_space; -+ -+ /* assign our static CBs */ -+ sock->sk->sk_state_change = ovpn_tcp_state_change; -+ sock->sk->sk_data_ready = ovpn_tcp_data_ready; -+ sock->sk->sk_write_space = ovpn_tcp_write_space; -+ -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ -+ return 0; -+err: -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ ptr_ring_cleanup(&peer->tcp.tx_ring, NULL); -+ -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/tcp.h b/drivers/net/ovpn-dco/tcp.h -new file mode 100644 -index 000000000000..d243a8e1c34e ---- /dev/null -+++ b/drivers/net/ovpn-dco/tcp.h -@@ -0,0 +1,38 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_TCP_H_ -+#define _NET_OVPN_DCO_TCP_H_ -+ -+#include "peer.h" -+ -+#include -+#include -+#include -+#include -+ -+void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb); -+ -+int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer); -+void ovpn_tcp_socket_detach(struct socket *sock); -+ -+/* Prepare skb and enqueue it for sending to peer. -+ * -+ * Preparation consist in prepending the skb payload with its size. -+ * Required by the OpenVPN protocol in order to extract packets from -+ * the TCP stream on the receiver side. -+ */ -+static inline void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ u16 len = skb->len; -+ -+ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); -+ ovpn_queue_tcp_skb(peer, skb); -+} -+ -+#endif /* _NET_OVPN_DCO_TCP_H_ */ -diff --git a/drivers/net/ovpn-dco/udp.c b/drivers/net/ovpn-dco/udp.c -new file mode 100644 -index 000000000000..afa236d1f15c ---- /dev/null -+++ b/drivers/net/ovpn-dco/udp.c -@@ -0,0 +1,343 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "bind.h" -+#include "ovpn.h" -+#include "ovpnstruct.h" -+#include "peer.h" -+#include "proto.h" -+#include "skb.h" -+#include "udp.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/** -+ * ovpn_udp_encap_recv() - Start processing a received UDP packet. -+ * If the first byte of the payload is DATA_V2, the packet is further processed, -+ * otherwise it is forwarded to the UDP stack for delivery to user space. -+ * -+ * @sk: the socket the packet was received on -+ * @skb: the sk_buff containing the actual packet -+ * -+ * Return codes: -+ * 0 : we consumed or dropped packet -+ * >0 : skb should be passed up to userspace as UDP (packet not consumed) -+ * <0 : skb should be resubmitted as proto -N (packet not consumed) -+ */ -+static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) -+{ -+ struct ovpn_peer *peer = NULL; -+ struct ovpn_struct *ovpn; -+ u32 peer_id; -+ u8 opcode; -+ int ret; -+ -+ ovpn = ovpn_from_udp_sock(sk); -+ if (unlikely(!ovpn)) { -+ net_err_ratelimited("%s: cannot obtain ovpn object from UDP socket\n", __func__); -+ goto drop; -+ } -+ -+ /* Make sure the first 4 bytes of the skb data buffer after the UDP header are accessible. -+ * They are required to fetch the OP code, the key ID and the peer ID. -+ */ -+ if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) + 4))) { -+ net_dbg_ratelimited("%s: packet too small\n", __func__); -+ goto drop; -+ } -+ -+ opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr)); -+ if (likely(opcode == OVPN_DATA_V2)) { -+ peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr)); -+ /* some OpenVPN server implementations send data packets with the peer-id set to -+ * undef. In this case we skip the peer lookup by peer-id and we try with the -+ * transport address -+ */ -+ if (peer_id != OVPN_PEER_ID_UNDEF) { -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) { -+ net_err_ratelimited("%s: received data from unknown peer (id: %d)\n", -+ __func__, peer_id); -+ goto drop; -+ } -+ -+ /* check if this peer changed it's IP address and update state */ -+ ovpn_peer_float(peer, skb); -+ } -+ } -+ -+ if (!peer) { -+ /* might be a control packet or a data packet with undef peer-id */ -+ peer = ovpn_peer_lookup_transp_addr(ovpn, skb); -+ if (unlikely(!peer)) { -+ if (opcode != OVPN_DATA_V2) { -+ netdev_dbg(ovpn->dev, -+ "%s: control packet from unknown peer, sending to userspace", -+ __func__); -+ return 1; -+ } -+ -+ netdev_dbg(ovpn->dev, -+ "%s: received data with undef peer-id from unknown source\n", -+ __func__); -+ goto drop; -+ } -+ } -+ -+ /* pop off outer UDP header */ -+ __skb_pull(skb, sizeof(struct udphdr)); -+ -+ ret = ovpn_recv(ovpn, peer, skb); -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: cannot handle incoming packet: %d\n", __func__, ret); -+ goto drop; -+ } -+ -+ /* should this be a non DATA_V2 packet, ret will be >0 and this will instruct the UDP -+ * stack to continue processing this packet as usual (i.e. deliver to user space) -+ */ -+ return ret; -+ -+drop: -+ if (peer) -+ ovpn_peer_put(peer); -+ kfree_skb(skb); -+ return 0; -+} -+ -+static int ovpn_udp4_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ struct rtable *rt; -+ struct flowi4 fl = { -+ .saddr = bind->local.ipv4.s_addr, -+ .daddr = bind->sa.in4.sin_addr.s_addr, -+ .fl4_sport = inet_sk(sk)->inet_sport, -+ .fl4_dport = bind->sa.in4.sin_port, -+ .flowi4_proto = sk->sk_protocol, -+ .flowi4_mark = sk->sk_mark, -+ }; -+ int ret; -+ -+ local_bh_disable(); -+ rt = dst_cache_get_ip4(cache, &fl.saddr); -+ if (rt) -+ goto transmit; -+ -+ if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { -+ /* we may end up here when the cached address is not usable anymore. -+ * In this case we reset address/cache and perform a new look up -+ */ -+ fl.saddr = 0; -+ bind->local.ipv4.s_addr = 0; -+ dst_cache_reset(cache); -+ } -+ -+ rt = ip_route_output_flow(sock_net(sk), &fl, sk); -+ if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { -+ fl.saddr = 0; -+ bind->local.ipv4.s_addr = 0; -+ dst_cache_reset(cache); -+ -+ rt = ip_route_output_flow(sock_net(sk), &fl, sk); -+ } -+ -+ if (IS_ERR(rt)) { -+ ret = PTR_ERR(rt); -+ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, -+ &bind->sa.in4, ret); -+ goto err; -+ } -+ dst_cache_set_ip4(cache, &rt->dst, fl.saddr); -+ -+transmit: -+ udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, -+ ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, -+ fl.fl4_dport, false, sk->sk_no_check_tx); -+ ret = 0; -+err: -+ local_bh_enable(); -+ return ret; -+} -+ -+#if IS_ENABLED(CONFIG_IPV6) -+static int ovpn_udp6_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ struct dst_entry *dst; -+ int ret; -+ -+ struct flowi6 fl = { -+ .saddr = bind->local.ipv6, -+ .daddr = bind->sa.in6.sin6_addr, -+ .fl6_sport = inet_sk(sk)->inet_sport, -+ .fl6_dport = bind->sa.in6.sin6_port, -+ .flowi6_proto = sk->sk_protocol, -+ .flowi6_mark = sk->sk_mark, -+ .flowi6_oif = bind->sa.in6.sin6_scope_id, -+ }; -+ -+ local_bh_disable(); -+ dst = dst_cache_get_ip6(cache, &fl.saddr); -+ if (dst) -+ goto transmit; -+ -+ if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { -+ /* we may end up here when the cached address is not usable anymore. -+ * In this case we reset address/cache and perform a new look up -+ */ -+ fl.saddr = in6addr_any; -+ bind->local.ipv6 = in6addr_any; -+ dst_cache_reset(cache); -+ } -+ -+ dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); -+ if (IS_ERR(dst)) { -+ ret = PTR_ERR(dst); -+ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, -+ &bind->sa.in6, ret); -+ goto err; -+ } -+ dst_cache_set_ip6(cache, dst, &fl.saddr); -+ -+transmit: -+ udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0, -+ ip6_dst_hoplimit(dst), 0, fl.fl6_sport, -+ fl.fl6_dport, udp_get_no_check6_tx(sk)); -+ ret = 0; -+err: -+ local_bh_enable(); -+ return ret; -+} -+#endif -+ -+/* Transmit skb utilizing kernel-provided UDP tunneling framework. -+ * -+ * rcu_read_lock should be held on entry. -+ * On return, the skb is consumed. -+ */ -+static int ovpn_udp_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ int ret; -+ -+ ovpn_rcu_lockdep_assert_held(); -+ -+ /* set sk to null if skb is already orphaned */ -+ if (!skb->destructor) -+ skb->sk = NULL; -+ -+ switch (bind->sa.in4.sin_family) { -+ case AF_INET: -+ ret = ovpn_udp4_output(ovpn, bind, cache, sk, skb); -+ break; -+#if IS_ENABLED(CONFIG_IPV6) -+ case AF_INET6: -+ ret = ovpn_udp6_output(ovpn, bind, cache, sk, skb); -+ break; -+#endif -+ default: -+ ret = -EAFNOSUPPORT; -+ break; -+ } -+ -+ return ret; -+} -+ -+void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, -+ struct sk_buff *skb) -+{ -+ struct ovpn_bind *bind; -+ struct socket *sock; -+ int ret = -1; -+ -+ skb->dev = ovpn->dev; -+ /* no checksum performed at this layer */ -+ skb->ip_summed = CHECKSUM_NONE; -+ -+ /* get socket info */ -+ sock = peer->sock->sock; -+ if (unlikely(!sock)) { -+ net_dbg_ratelimited("%s: no sock for remote peer\n", __func__); -+ goto out; -+ } -+ -+ rcu_read_lock(); -+ /* get binding */ -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) { -+ net_dbg_ratelimited("%s: no bind for remote peer\n", __func__); -+ goto out_unlock; -+ } -+ -+ /* crypto layer -> transport (UDP) */ -+ ret = ovpn_udp_output(ovpn, bind, &peer->dst_cache, sock->sk, skb); -+ -+out_unlock: -+ rcu_read_unlock(); -+out: -+ if (ret < 0) -+ kfree_skb(skb); -+} -+ -+/* Set UDP encapsulation callbacks */ -+int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn) -+{ -+ struct udp_tunnel_sock_cfg cfg = { -+ .sk_user_data = ovpn, -+ .encap_type = UDP_ENCAP_OVPNINUDP, -+ .encap_rcv = ovpn_udp_encap_recv, -+ }; -+ struct ovpn_socket *old_data; -+ -+ /* sanity check */ -+ if (sock->sk->sk_protocol != IPPROTO_UDP) { -+ netdev_err(ovpn->dev, "%s: expected UDP socket\n", __func__); -+ return -EINVAL; -+ } -+ -+ /* make sure no pre-existing encapsulation handler exists */ -+ rcu_read_lock(); -+ old_data = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ if (old_data) { -+ if (old_data->ovpn == ovpn) { -+ netdev_dbg(ovpn->dev, -+ "%s: provided socket already owned by this interface\n", -+ __func__); -+ return -EALREADY; -+ } -+ -+ netdev_err(ovpn->dev, "%s: provided socket already taken by other user\n", -+ __func__); -+ return -EBUSY; -+ } -+ -+ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); -+ -+ return 0; -+} -+ -+/* Detach socket from encapsulation handler and/or other callbacks */ -+void ovpn_udp_socket_detach(struct socket *sock) -+{ -+ struct udp_tunnel_sock_cfg cfg = { }; -+ -+ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); -+} -diff --git a/drivers/net/ovpn-dco/udp.h b/drivers/net/ovpn-dco/udp.h -new file mode 100644 -index 000000000000..be94fb74669b ---- /dev/null -+++ b/drivers/net/ovpn-dco/udp.h -@@ -0,0 +1,25 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_UDP_H_ -+#define _NET_OVPN_DCO_UDP_H_ -+ -+#include "peer.h" -+#include "ovpnstruct.h" -+ -+#include -+#include -+#include -+#include -+ -+int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn); -+void ovpn_udp_socket_detach(struct socket *sock); -+void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, -+ struct sk_buff *skb); -+ -+#endif /* _NET_OVPN_DCO_UDP_H_ */ -diff --git a/include/net/netlink.h b/include/net/netlink.h -index 7a2a9d3144ba..335f44871529 100644 ---- a/include/net/netlink.h -+++ b/include/net/netlink.h -@@ -441,6 +441,7 @@ struct nla_policy { - .max = _len \ - } - #define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len) -+#define NLA_POLICY_MAX_LEN(_len) NLA_POLICY_MAX(NLA_BINARY, _len) - - /** - * struct nl_info - netlink source information -diff --git a/include/uapi/linux/ovpn_dco.h b/include/uapi/linux/ovpn_dco.h -new file mode 100644 -index 000000000000..6afee8b3fedd ---- /dev/null -+++ b/include/uapi/linux/ovpn_dco.h -@@ -0,0 +1,265 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _UAPI_LINUX_OVPN_DCO_H_ -+#define _UAPI_LINUX_OVPN_DCO_H_ -+ -+#define OVPN_NL_NAME "ovpn-dco" -+ -+#define OVPN_NL_MULTICAST_GROUP_PEERS "peers" -+ -+/** -+ * enum ovpn_nl_commands - supported netlink commands -+ */ -+enum ovpn_nl_commands { -+ /** -+ * @OVPN_CMD_UNSPEC: unspecified command to catch errors -+ */ -+ OVPN_CMD_UNSPEC = 0, -+ -+ /** -+ * @OVPN_CMD_NEW_PEER: Configure peer with its crypto keys -+ */ -+ OVPN_CMD_NEW_PEER, -+ -+ /** -+ * @OVPN_CMD_SET_PEER: Tweak parameters for an existing peer -+ */ -+ OVPN_CMD_SET_PEER, -+ -+ /** -+ * @OVPN_CMD_DEL_PEER: Remove peer from internal table -+ */ -+ OVPN_CMD_DEL_PEER, -+ -+ OVPN_CMD_NEW_KEY, -+ -+ OVPN_CMD_SWAP_KEYS, -+ -+ OVPN_CMD_DEL_KEY, -+ -+ /** -+ * @OVPN_CMD_REGISTER_PACKET: Register for specific packet types to be -+ * forwarded to userspace -+ */ -+ OVPN_CMD_REGISTER_PACKET, -+ -+ /** -+ * @OVPN_CMD_PACKET: Send a packet from userspace to kernelspace. Also -+ * used to send to userspace packets for which a process had registered -+ * with OVPN_CMD_REGISTER_PACKET -+ */ -+ OVPN_CMD_PACKET, -+ -+ /** -+ * @OVPN_CMD_GET_PEER: Retrieve the status of a peer or all peers -+ */ -+ OVPN_CMD_GET_PEER, -+}; -+ -+enum ovpn_cipher_alg { -+ /** -+ * @OVPN_CIPHER_ALG_NONE: No encryption - reserved for debugging only -+ */ -+ OVPN_CIPHER_ALG_NONE = 0, -+ /** -+ * @OVPN_CIPHER_ALG_AES_GCM: AES-GCM AEAD cipher with any allowed key size -+ */ -+ OVPN_CIPHER_ALG_AES_GCM, -+ /** -+ * @OVPN_CIPHER_ALG_CHACHA20_POLY1305: ChaCha20Poly1305 AEAD cipher -+ */ -+ OVPN_CIPHER_ALG_CHACHA20_POLY1305, -+}; -+ -+enum ovpn_del_peer_reason { -+ __OVPN_DEL_PEER_REASON_FIRST, -+ OVPN_DEL_PEER_REASON_TEARDOWN = __OVPN_DEL_PEER_REASON_FIRST, -+ OVPN_DEL_PEER_REASON_USERSPACE, -+ OVPN_DEL_PEER_REASON_EXPIRED, -+ OVPN_DEL_PEER_REASON_TRANSPORT_ERROR, -+ __OVPN_DEL_PEER_REASON_AFTER_LAST -+}; -+ -+enum ovpn_key_slot { -+ __OVPN_KEY_SLOT_FIRST, -+ OVPN_KEY_SLOT_PRIMARY = __OVPN_KEY_SLOT_FIRST, -+ OVPN_KEY_SLOT_SECONDARY, -+ __OVPN_KEY_SLOT_AFTER_LAST, -+}; -+ -+enum ovpn_netlink_attrs { -+ OVPN_ATTR_UNSPEC = 0, -+ OVPN_ATTR_IFINDEX, -+ OVPN_ATTR_NEW_PEER, -+ OVPN_ATTR_SET_PEER, -+ OVPN_ATTR_DEL_PEER, -+ OVPN_ATTR_NEW_KEY, -+ OVPN_ATTR_SWAP_KEYS, -+ OVPN_ATTR_DEL_KEY, -+ OVPN_ATTR_PACKET, -+ OVPN_ATTR_GET_PEER, -+ -+ __OVPN_ATTR_AFTER_LAST, -+ OVPN_ATTR_MAX = __OVPN_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_key_dir_attrs { -+ OVPN_KEY_DIR_ATTR_UNSPEC = 0, -+ OVPN_KEY_DIR_ATTR_CIPHER_KEY, -+ OVPN_KEY_DIR_ATTR_NONCE_TAIL, -+ -+ __OVPN_KEY_DIR_ATTR_AFTER_LAST, -+ OVPN_KEY_DIR_ATTR_MAX = __OVPN_KEY_DIR_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_new_key_attrs { -+ OVPN_NEW_KEY_ATTR_UNSPEC = 0, -+ OVPN_NEW_KEY_ATTR_PEER_ID, -+ OVPN_NEW_KEY_ATTR_KEY_SLOT, -+ OVPN_NEW_KEY_ATTR_KEY_ID, -+ OVPN_NEW_KEY_ATTR_CIPHER_ALG, -+ OVPN_NEW_KEY_ATTR_ENCRYPT_KEY, -+ OVPN_NEW_KEY_ATTR_DECRYPT_KEY, -+ -+ __OVPN_NEW_KEY_ATTR_AFTER_LAST, -+ OVPN_NEW_KEY_ATTR_MAX = __OVPN_NEW_KEY_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_del_key_attrs { -+ OVPN_DEL_KEY_ATTR_UNSPEC = 0, -+ OVPN_DEL_KEY_ATTR_PEER_ID, -+ OVPN_DEL_KEY_ATTR_KEY_SLOT, -+ -+ __OVPN_DEL_KEY_ATTR_AFTER_LAST, -+ OVPN_DEL_KEY_ATTR_MAX = __OVPN_DEL_KEY_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_swap_keys_attrs { -+ OVPN_SWAP_KEYS_ATTR_UNSPEC = 0, -+ OVPN_SWAP_KEYS_ATTR_PEER_ID, -+ -+ __OVPN_SWAP_KEYS_ATTR_AFTER_LAST, -+ OVPN_SWAP_KEYS_ATTR_MAX = __OVPN_SWAP_KEYS_ATTR_AFTER_LAST - 1, -+ -+}; -+ -+enum ovpn_netlink_new_peer_attrs { -+ OVPN_NEW_PEER_ATTR_UNSPEC = 0, -+ OVPN_NEW_PEER_ATTR_PEER_ID, -+ OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE, -+ OVPN_NEW_PEER_ATTR_SOCKET, -+ OVPN_NEW_PEER_ATTR_IPV4, -+ OVPN_NEW_PEER_ATTR_IPV6, -+ OVPN_NEW_PEER_ATTR_LOCAL_IP, -+ -+ __OVPN_NEW_PEER_ATTR_AFTER_LAST, -+ OVPN_NEW_PEER_ATTR_MAX = __OVPN_NEW_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_set_peer_attrs { -+ OVPN_SET_PEER_ATTR_UNSPEC = 0, -+ OVPN_SET_PEER_ATTR_PEER_ID, -+ OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT, -+ -+ __OVPN_SET_PEER_ATTR_AFTER_LAST, -+ OVPN_SET_PEER_ATTR_MAX = __OVPN_SET_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_del_peer_attrs { -+ OVPN_DEL_PEER_ATTR_UNSPEC = 0, -+ OVPN_DEL_PEER_ATTR_REASON, -+ OVPN_DEL_PEER_ATTR_PEER_ID, -+ -+ __OVPN_DEL_PEER_ATTR_AFTER_LAST, -+ OVPN_DEL_PEER_ATTR_MAX = __OVPN_DEL_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_get_peer_attrs { -+ OVPN_GET_PEER_ATTR_UNSPEC = 0, -+ OVPN_GET_PEER_ATTR_PEER_ID, -+ -+ __OVPN_GET_PEER_ATTR_AFTER_LAST, -+ OVPN_GET_PEER_ATTR_MAX = __OVPN_GET_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_get_peer_response_attrs { -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC = 0, -+ OVPN_GET_PEER_RESP_ATTR_PEER_ID, -+ OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ OVPN_GET_PEER_RESP_ATTR_IPV4, -+ OVPN_GET_PEER_RESP_ATTR_IPV6, -+ OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, -+ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, -+ OVPN_GET_PEER_RESP_ATTR_RX_BYTES, -+ OVPN_GET_PEER_RESP_ATTR_TX_BYTES, -+ OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, -+ OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, -+ -+ __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST, -+ OVPN_GET_PEER_RESP_ATTR_MAX = __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_peer_stats_attrs { -+ OVPN_PEER_STATS_ATTR_UNSPEC = 0, -+ OVPN_PEER_STATS_BYTES, -+ OVPN_PEER_STATS_PACKETS, -+ -+ __OVPN_PEER_STATS_ATTR_AFTER_LAST, -+ OVPN_PEER_STATS_ATTR_MAX = __OVPN_PEER_STATS_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_peer_attrs { -+ OVPN_PEER_ATTR_UNSPEC = 0, -+ OVPN_PEER_ATTR_PEER_ID, -+ OVPN_PEER_ATTR_SOCKADDR_REMOTE, -+ OVPN_PEER_ATTR_IPV4, -+ OVPN_PEER_ATTR_IPV6, -+ OVPN_PEER_ATTR_LOCAL_IP, -+ OVPN_PEER_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_PEER_ATTR_KEEPALIVE_TIMEOUT, -+ OVPN_PEER_ATTR_ENCRYPT_KEY, -+ OVPN_PEER_ATTR_DECRYPT_KEY, -+ OVPN_PEER_ATTR_RX_STATS, -+ OVPN_PEER_ATTR_TX_STATS, -+ -+ __OVPN_PEER_ATTR_AFTER_LAST, -+ OVPN_PEER_ATTR_MAX = __OVPN_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_packet_attrs { -+ OVPN_PACKET_ATTR_UNSPEC = 0, -+ OVPN_PACKET_ATTR_PACKET, -+ OVPN_PACKET_ATTR_PEER_ID, -+ -+ __OVPN_PACKET_ATTR_AFTER_LAST, -+ OVPN_PACKET_ATTR_MAX = __OVPN_PACKET_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_ifla_attrs { -+ IFLA_OVPN_UNSPEC = 0, -+ IFLA_OVPN_MODE, -+ -+ __IFLA_OVPN_AFTER_LAST, -+ IFLA_OVPN_MAX = __IFLA_OVPN_AFTER_LAST - 1, -+}; -+ -+enum ovpn_mode { -+ __OVPN_MODE_FIRST = 0, -+ OVPN_MODE_P2P = __OVPN_MODE_FIRST, -+ OVPN_MODE_MP, -+ -+ __OVPN_MODE_AFTER_LAST, -+}; -+ -+#endif /* _UAPI_LINUX_OVPN_DCO_H_ */ -diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h -index 4828794efcf8..8008c762e6b8 100644 ---- a/include/uapi/linux/udp.h -+++ b/include/uapi/linux/udp.h -@@ -43,5 +43,6 @@ struct udphdr { - #define UDP_ENCAP_GTP1U 5 /* 3GPP TS 29.060 */ - #define UDP_ENCAP_RXRPC 6 - #define TCP_ENCAP_ESPINTCP 7 /* Yikes, this is really xfrm encap types. */ -+#define UDP_ENCAP_OVPNINUDP 8 /* OpenVPN over UDP connection */ - - #endif /* _UAPI_LINUX_UDP_H */ --- -2.38.0.rc1.8.g2a7d63a245 - -From c8708680c88426cc0d0f2769502a803364b1e120 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Tue, 27 Sep 2022 18:01:21 +0200 -Subject: [PATCH 05/13] maple-lru-khugepage - -Signed-off-by: Peter Jung ---- - Documentation/admin-guide/mm/index.rst | 1 + - Documentation/admin-guide/mm/multigen_lru.rst | 162 + - Documentation/core-api/index.rst | 1 + - Documentation/core-api/maple_tree.rst | 217 + - Documentation/mm/index.rst | 1 + - Documentation/mm/multigen_lru.rst | 159 + - MAINTAINERS | 12 + - arch/Kconfig | 8 + - arch/alpha/include/uapi/asm/mman.h | 2 + - arch/arm64/include/asm/pgtable.h | 15 +- - arch/arm64/kernel/elfcore.c | 16 +- - arch/arm64/kernel/vdso.c | 3 +- - arch/mips/include/uapi/asm/mman.h | 2 + - arch/parisc/include/uapi/asm/mman.h | 2 + - arch/parisc/kernel/cache.c | 9 +- - arch/powerpc/kernel/vdso.c | 6 +- - arch/powerpc/mm/book3s32/tlb.c | 11 +- - arch/powerpc/mm/book3s64/subpage_prot.c | 13 +- - arch/riscv/kernel/vdso.c | 3 +- - arch/s390/kernel/vdso.c | 3 +- - arch/s390/mm/gmap.c | 6 +- - arch/um/kernel/tlb.c | 14 +- - arch/x86/Kconfig | 1 + - arch/x86/entry/vdso/vma.c | 9 +- - arch/x86/include/asm/pgtable.h | 9 +- - arch/x86/kernel/tboot.c | 2 +- - arch/x86/mm/pgtable.c | 5 +- - arch/xtensa/include/uapi/asm/mman.h | 2 + - arch/xtensa/kernel/syscall.c | 18 +- - drivers/firmware/efi/efi.c | 2 +- - drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 14 +- - drivers/misc/cxl/fault.c | 45 +- - drivers/tee/optee/call.c | 18 +- - drivers/xen/privcmd.c | 2 +- - fs/coredump.c | 34 +- - fs/exec.c | 14 +- - fs/fuse/dev.c | 3 +- - fs/proc/base.c | 5 +- - fs/proc/internal.h | 2 +- - fs/proc/task_mmu.c | 76 +- - fs/proc/task_nommu.c | 45 +- - fs/userfaultfd.c | 62 +- - include/linux/cgroup.h | 15 +- - include/linux/huge_mm.h | 23 +- - include/linux/maple_tree.h | 685 + - include/linux/memcontrol.h | 36 + - include/linux/mm.h | 83 +- - include/linux/mm_inline.h | 231 +- - include/linux/mm_types.h | 119 +- - include/linux/mm_types_task.h | 12 - - include/linux/mmzone.h | 216 + - include/linux/nodemask.h | 1 + - include/linux/page-flags-layout.h | 16 +- - include/linux/page-flags.h | 4 +- - include/linux/pgtable.h | 17 +- - include/linux/sched.h | 5 +- - include/linux/swap.h | 4 + - include/linux/userfaultfd_k.h | 7 +- - include/linux/vm_event_item.h | 4 - - include/linux/vmacache.h | 28 - - include/linux/vmstat.h | 6 - - include/trace/events/huge_memory.h | 1 + - include/trace/events/maple_tree.h | 123 + - include/trace/events/mmap.h | 73 + - include/uapi/asm-generic/mman-common.h | 2 + - init/main.c | 2 + - ipc/shm.c | 21 +- - kernel/acct.c | 11 +- - kernel/bounds.c | 7 + - kernel/bpf/task_iter.c | 10 +- - kernel/cgroup/cgroup-internal.h | 1 - - kernel/debug/debug_core.c | 12 - - kernel/events/core.c | 3 +- - kernel/events/uprobes.c | 9 +- - kernel/exit.c | 1 + - kernel/fork.c | 71 +- - kernel/sched/core.c | 1 + - kernel/sched/fair.c | 10 +- - lib/Kconfig.debug | 17 +- - lib/Makefile | 2 +- - lib/maple_tree.c | 7130 +++ - lib/test_maple_tree.c | 38307 ++++++++++++++++ - mm/Kconfig | 26 + - mm/Makefile | 2 +- - mm/damon/vaddr-test.h | 36 +- - mm/damon/vaddr.c | 53 +- - mm/debug.c | 14 +- - mm/gup.c | 7 +- - mm/huge_memory.c | 39 +- - mm/init-mm.c | 4 +- - mm/internal.h | 11 +- - mm/khugepaged.c | 772 +- - mm/ksm.c | 28 +- - mm/madvise.c | 11 +- - mm/memcontrol.c | 34 +- - mm/memory.c | 77 +- - mm/mempolicy.c | 56 +- - mm/mlock.c | 37 +- - mm/mm_init.c | 6 +- - mm/mmap.c | 2154 +- - mm/mmzone.c | 2 + - mm/mprotect.c | 8 +- - mm/mremap.c | 22 +- - mm/msync.c | 2 +- - mm/nommu.c | 260 +- - mm/oom_kill.c | 3 +- - mm/pagewalk.c | 2 +- - mm/rmap.c | 21 +- - mm/swap.c | 54 +- - mm/swapfile.c | 4 +- - mm/util.c | 32 - - mm/vmacache.c | 117 - - mm/vmscan.c | 3247 +- - mm/vmstat.c | 4 - - mm/workingset.c | 110 +- - tools/include/linux/slab.h | 4 + - tools/include/uapi/asm-generic/mman-common.h | 2 + - tools/testing/radix-tree/.gitignore | 2 + - tools/testing/radix-tree/Makefile | 9 +- - tools/testing/radix-tree/generated/autoconf.h | 1 + - tools/testing/radix-tree/linux.c | 160 +- - tools/testing/radix-tree/linux/kernel.h | 1 + - tools/testing/radix-tree/linux/lockdep.h | 2 + - tools/testing/radix-tree/linux/maple_tree.h | 7 + - tools/testing/radix-tree/maple.c | 59 + - .../radix-tree/trace/events/maple_tree.h | 5 + - tools/testing/selftests/vm/khugepaged.c | 563 +- - 127 files changed, 53649 insertions(+), 2781 deletions(-) - create mode 100644 Documentation/admin-guide/mm/multigen_lru.rst - create mode 100644 Documentation/core-api/maple_tree.rst - create mode 100644 Documentation/mm/multigen_lru.rst - create mode 100644 include/linux/maple_tree.h - delete mode 100644 include/linux/vmacache.h - create mode 100644 include/trace/events/maple_tree.h - create mode 100644 lib/maple_tree.c - create mode 100644 lib/test_maple_tree.c - delete mode 100644 mm/vmacache.c - create mode 100644 tools/testing/radix-tree/linux/maple_tree.h - create mode 100644 tools/testing/radix-tree/maple.c - create mode 100644 tools/testing/radix-tree/trace/events/maple_tree.h - -diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst -index 1bd11118dfb1..d1064e0ba34a 100644 ---- a/Documentation/admin-guide/mm/index.rst -+++ b/Documentation/admin-guide/mm/index.rst -@@ -32,6 +32,7 @@ the Linux memory management. - idle_page_tracking - ksm - memory-hotplug -+ multigen_lru - nommu-mmap - numa_memory_policy - numaperf -diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst -new file mode 100644 -index 000000000000..33e068830497 ---- /dev/null -+++ b/Documentation/admin-guide/mm/multigen_lru.rst -@@ -0,0 +1,162 @@ -+.. SPDX-License-Identifier: GPL-2.0 -+ -+============= -+Multi-Gen LRU -+============= -+The multi-gen LRU is an alternative LRU implementation that optimizes -+page reclaim and improves performance under memory pressure. Page -+reclaim decides the kernel's caching policy and ability to overcommit -+memory. It directly impacts the kswapd CPU usage and RAM efficiency. -+ -+Quick start -+=========== -+Build the kernel with the following configurations. -+ -+* ``CONFIG_LRU_GEN=y`` -+* ``CONFIG_LRU_GEN_ENABLED=y`` -+ -+All set! -+ -+Runtime options -+=============== -+``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the -+following subsections. -+ -+Kill switch -+----------- -+``enabled`` accepts different values to enable or disable the -+following components. Its default value depends on -+``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled -+unless some of them have unforeseen side effects. Writing to -+``enabled`` has no effect when a component is not supported by the -+hardware, and valid values will be accepted even when the main switch -+is off. -+ -+====== =============================================================== -+Values Components -+====== =============================================================== -+0x0001 The main switch for the multi-gen LRU. -+0x0002 Clearing the accessed bit in leaf page table entries in large -+ batches, when MMU sets it (e.g., on x86). This behavior can -+ theoretically worsen lock contention (mmap_lock). If it is -+ disabled, the multi-gen LRU will suffer a minor performance -+ degradation for workloads that contiguously map hot pages, -+ whose accessed bits can be otherwise cleared by fewer larger -+ batches. -+0x0004 Clearing the accessed bit in non-leaf page table entries as -+ well, when MMU sets it (e.g., on x86). This behavior was not -+ verified on x86 varieties other than Intel and AMD. If it is -+ disabled, the multi-gen LRU will suffer a negligible -+ performance degradation. -+[yYnN] Apply to all the components above. -+====== =============================================================== -+ -+E.g., -+:: -+ -+ echo y >/sys/kernel/mm/lru_gen/enabled -+ cat /sys/kernel/mm/lru_gen/enabled -+ 0x0007 -+ echo 5 >/sys/kernel/mm/lru_gen/enabled -+ cat /sys/kernel/mm/lru_gen/enabled -+ 0x0005 -+ -+Thrashing prevention -+-------------------- -+Personal computers are more sensitive to thrashing because it can -+cause janks (lags when rendering UI) and negatively impact user -+experience. The multi-gen LRU offers thrashing prevention to the -+majority of laptop and desktop users who do not have ``oomd``. -+ -+Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of -+``N`` milliseconds from getting evicted. The OOM killer is triggered -+if this working set cannot be kept in memory. In other words, this -+option works as an adjustable pressure relief valve, and when open, it -+terminates applications that are hopefully not being used. -+ -+Based on the average human detectable lag (~100ms), ``N=1000`` usually -+eliminates intolerable janks due to thrashing. Larger values like -+``N=3000`` make janks less noticeable at the risk of premature OOM -+kills. -+ -+The default value ``0`` means disabled. -+ -+Experimental features -+===================== -+``/sys/kernel/debug/lru_gen`` accepts commands described in the -+following subsections. Multiple command lines are supported, so does -+concatenation with delimiters ``,`` and ``;``. -+ -+``/sys/kernel/debug/lru_gen_full`` provides additional stats for -+debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from -+evicted generations in this file. -+ -+Working set estimation -+---------------------- -+Working set estimation measures how much memory an application needs -+in a given time interval, and it is usually done with little impact on -+the performance of the application. E.g., data centers want to -+optimize job scheduling (bin packing) to improve memory utilizations. -+When a new job comes in, the job scheduler needs to find out whether -+each server it manages can allocate a certain amount of memory for -+this new job before it can pick a candidate. To do so, the job -+scheduler needs to estimate the working sets of the existing jobs. -+ -+When it is read, ``lru_gen`` returns a histogram of numbers of pages -+accessed over different time intervals for each memcg and node. -+``MAX_NR_GENS`` decides the number of bins for each histogram. The -+histograms are noncumulative. -+:: -+ -+ memcg memcg_id memcg_path -+ node node_id -+ min_gen_nr age_in_ms nr_anon_pages nr_file_pages -+ ... -+ max_gen_nr age_in_ms nr_anon_pages nr_file_pages -+ -+Each bin contains an estimated number of pages that have been accessed -+within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages -+and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of -+the former is the largest and that of the latter is the smallest. -+ -+Users can write the following command to ``lru_gen`` to create a new -+generation ``max_gen_nr+1``: -+ -+ ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` -+ -+``can_swap`` defaults to the swap setting and, if it is set to ``1``, -+it forces the scan of anon pages when swap is off, and vice versa. -+``force_scan`` defaults to ``1`` and, if it is set to ``0``, it -+employs heuristics to reduce the overhead, which is likely to reduce -+the coverage as well. -+ -+A typical use case is that a job scheduler runs this command at a -+certain time interval to create new generations, and it ranks the -+servers it manages based on the sizes of their cold pages defined by -+this time interval. -+ -+Proactive reclaim -+----------------- -+Proactive reclaim induces page reclaim when there is no memory -+pressure. It usually targets cold pages only. E.g., when a new job -+comes in, the job scheduler wants to proactively reclaim cold pages on -+the server it selected, to improve the chance of successfully landing -+this new job. -+ -+Users can write the following command to ``lru_gen`` to evict -+generations less than or equal to ``min_gen_nr``. -+ -+ ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` -+ -+``min_gen_nr`` should be less than ``max_gen_nr-1``, since -+``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to -+the active list) and therefore cannot be evicted. ``swappiness`` -+overrides the default value in ``/proc/sys/vm/swappiness``. -+``nr_to_reclaim`` limits the number of pages to evict. -+ -+A typical use case is that a job scheduler runs this command before it -+tries to land a new job on a server. If it fails to materialize enough -+cold pages because of the overestimation, it retries on the next -+server according to the ranking result obtained from the working set -+estimation step. This less forceful approach limits the impacts on the -+existing jobs. -diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst -index dc95df462eea..1da6a4fac664 100644 ---- a/Documentation/core-api/index.rst -+++ b/Documentation/core-api/index.rst -@@ -36,6 +36,7 @@ Library functionality that is used throughout the kernel. - kref - assoc_array - xarray -+ maple_tree - idr - circular-buffers - rbtree -diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst -new file mode 100644 -index 000000000000..45defcf15da7 ---- /dev/null -+++ b/Documentation/core-api/maple_tree.rst -@@ -0,0 +1,217 @@ -+.. SPDX-License-Identifier: GPL-2.0+ -+ -+ -+========== -+Maple Tree -+========== -+ -+:Author: Liam R. Howlett -+ -+Overview -+======== -+ -+The Maple Tree is a B-Tree data type which is optimized for storing -+non-overlapping ranges, including ranges of size 1. The tree was designed to -+be simple to use and does not require a user written search method. It -+supports iterating over a range of entries and going to the previous or next -+entry in a cache-efficient manner. The tree can also be put into an RCU-safe -+mode of operation which allows reading and writing concurrently. Writers must -+synchronize on a lock, which can be the default spinlock, or the user can set -+the lock to an external lock of a different type. -+ -+The Maple Tree maintains a small memory footprint and was designed to use -+modern processor cache efficiently. The majority of the users will be able to -+use the normal API. An :ref:`maple-tree-advanced-api` exists for more complex -+scenarios. The most important usage of the Maple Tree is the tracking of the -+virtual memory areas. -+ -+The Maple Tree can store values between ``0`` and ``ULONG_MAX``. The Maple -+Tree reserves values with the bottom two bits set to '10' which are below 4096 -+(ie 2, 6, 10 .. 4094) for internal use. If the entries may use reserved -+entries then the users can convert the entries using xa_mk_value() and convert -+them back by calling xa_to_value(). If the user needs to use a reserved -+value, then the user can convert the value when using the -+:ref:`maple-tree-advanced-api`, but are blocked by the normal API. -+ -+The Maple Tree can also be configured to support searching for a gap of a given -+size (or larger). -+ -+Pre-allocating of nodes is also supported using the -+:ref:`maple-tree-advanced-api`. This is useful for users who must guarantee a -+successful store operation within a given -+code segment when allocating cannot be done. Allocations of nodes are -+relatively small at around 256 bytes. -+ -+.. _maple-tree-normal-api: -+ -+Normal API -+========== -+ -+Start by initialising a maple tree, either with DEFINE_MTREE() for statically -+allocated maple trees or mt_init() for dynamically allocated ones. A -+freshly-initialised maple tree contains a ``NULL`` pointer for the range ``0`` -+- ``ULONG_MAX``. There are currently two types of maple trees supported: the -+allocation tree and the regular tree. The regular tree has a higher branching -+factor for internal nodes. The allocation tree has a lower branching factor -+but allows the user to search for a gap of a given size or larger from either -+``0`` upwards or ``ULONG_MAX`` down. An allocation tree can be used by -+passing in the ``MT_FLAGS_ALLOC_RANGE`` flag when initialising the tree. -+ -+You can then set entries using mtree_store() or mtree_store_range(). -+mtree_store() will overwrite any entry with the new entry and return 0 on -+success or an error code otherwise. mtree_store_range() works in the same way -+but takes a range. mtree_load() is used to retrieve the entry stored at a -+given index. You can use mtree_erase() to erase an entire range by only -+knowing one value within that range, or mtree_store() call with an entry of -+NULL may be used to partially erase a range or many ranges at once. -+ -+If you want to only store a new entry to a range (or index) if that range is -+currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which -+return -EEXIST if the range is not empty. -+ -+You can search for an entry from an index upwards by using mt_find(). -+ -+You can walk each entry within a range by calling mt_for_each(). You must -+provide a temporary variable to store a cursor. If you want to walk each -+element of the tree then ``0`` and ``ULONG_MAX`` may be used as the range. If -+the caller is going to hold the lock for the duration of the walk then it is -+worth looking at the mas_for_each() API in the :ref:`maple-tree-advanced-api` -+section. -+ -+Sometimes it is necessary to ensure the next call to store to a maple tree does -+not allocate memory, please see :ref:`maple-tree-advanced-api` for this use case. -+ -+Finally, you can remove all entries from a maple tree by calling -+mtree_destroy(). If the maple tree entries are pointers, you may wish to free -+the entries first. -+ -+Allocating Nodes -+---------------- -+ -+The allocations are handled by the internal tree code. See -+:ref:`maple-tree-advanced-alloc` for other options. -+ -+Locking -+------- -+ -+You do not have to worry about locking. See :ref:`maple-tree-advanced-locks` -+for other options. -+ -+The Maple Tree uses RCU and an internal spinlock to synchronise access: -+ -+Takes RCU read lock: -+ * mtree_load() -+ * mt_find() -+ * mt_for_each() -+ * mt_next() -+ * mt_prev() -+ -+Takes ma_lock internally: -+ * mtree_store() -+ * mtree_store_range() -+ * mtree_insert() -+ * mtree_insert_range() -+ * mtree_erase() -+ * mtree_destroy() -+ * mt_set_in_rcu() -+ * mt_clear_in_rcu() -+ -+If you want to take advantage of the internal lock to protect the data -+structures that you are storing in the Maple Tree, you can call mtree_lock() -+before calling mtree_load(), then take a reference count on the object you -+have found before calling mtree_unlock(). This will prevent stores from -+removing the object from the tree between looking up the object and -+incrementing the refcount. You can also use RCU to avoid dereferencing -+freed memory, but an explanation of that is beyond the scope of this -+document. -+ -+.. _maple-tree-advanced-api: -+ -+Advanced API -+============ -+ -+The advanced API offers more flexibility and better performance at the -+cost of an interface which can be harder to use and has fewer safeguards. -+You must take care of your own locking while using the advanced API. -+You can use the ma_lock, RCU or an external lock for protection. -+You can mix advanced and normal operations on the same array, as long -+as the locking is compatible. The :ref:`maple-tree-normal-api` is implemented -+in terms of the advanced API. -+ -+The advanced API is based around the ma_state, this is where the 'mas' -+prefix originates. The ma_state struct keeps track of tree operations to make -+life easier for both internal and external tree users. -+ -+Initialising the maple tree is the same as in the :ref:`maple-tree-normal-api`. -+Please see above. -+ -+The maple state keeps track of the range start and end in mas->index and -+mas->last, respectively. -+ -+mas_walk() will walk the tree to the location of mas->index and set the -+mas->index and mas->last according to the range for the entry. -+ -+You can set entries using mas_store(). mas_store() will overwrite any entry -+with the new entry and return the first existing entry that is overwritten. -+The range is passed in as members of the maple state: index and last. -+ -+You can use mas_erase() to erase an entire range by setting index and -+last of the maple state to the desired range to erase. This will erase -+the first range that is found in that range, set the maple state index -+and last as the range that was erased and return the entry that existed -+at that location. -+ -+You can walk each entry within a range by using mas_for_each(). If you want -+to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as -+the range. If the lock needs to be periodically dropped, see the locking -+section mas_pause(). -+ -+Using a maple state allows mas_next() and mas_prev() to function as if the -+tree was a linked list. With such a high branching factor the amortized -+performance penalty is outweighed by cache optimization. mas_next() will -+return the next entry which occurs after the entry at index. mas_prev() -+will return the previous entry which occurs before the entry at index. -+ -+mas_find() will find the first entry which exists at or above index on -+the first call, and the next entry from every subsequent calls. -+ -+mas_find_rev() will find the fist entry which exists at or below the last on -+the first call, and the previous entry from every subsequent calls. -+ -+If the user needs to yield the lock during an operation, then the maple state -+must be paused using mas_pause(). -+ -+There are a few extra interfaces provided when using an allocation tree. -+If you wish to search for a gap within a range, then mas_empty_area() -+or mas_empty_area_rev() can be used. mas_empty_area() searches for a gap -+starting at the lowest index given up to the maximum of the range. -+mas_empty_area_rev() searches for a gap starting at the highest index given -+and continues downward to the lower bound of the range. -+ -+.. _maple-tree-advanced-alloc: -+ -+Advanced Allocating Nodes -+------------------------- -+ -+Allocations are usually handled internally to the tree, however if allocations -+need to occur before a write occurs then calling mas_expected_entries() will -+allocate the worst-case number of needed nodes to insert the provided number of -+ranges. This also causes the tree to enter mass insertion mode. Once -+insertions are complete calling mas_destroy() on the maple state will free the -+unused allocations. -+ -+.. _maple-tree-advanced-locks: -+ -+Advanced Locking -+---------------- -+ -+The maple tree uses a spinlock by default, but external locks can be used for -+tree updates as well. To use an external lock, the tree must be initialized -+with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the -+MTREE_INIT_EXT() #define, which takes an external lock as an argument. -+ -+Functions and structures -+======================== -+ -+.. kernel-doc:: include/linux/maple_tree.h -+.. kernel-doc:: lib/maple_tree.c -diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst -index 575ccd40e30c..4aa12b8be278 100644 ---- a/Documentation/mm/index.rst -+++ b/Documentation/mm/index.rst -@@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. - ksm - memory-model - mmu_notifier -+ multigen_lru - numa - overcommit-accounting - page_migration -diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst -new file mode 100644 -index 000000000000..d7062c6a8946 ---- /dev/null -+++ b/Documentation/mm/multigen_lru.rst -@@ -0,0 +1,159 @@ -+.. SPDX-License-Identifier: GPL-2.0 -+ -+============= -+Multi-Gen LRU -+============= -+The multi-gen LRU is an alternative LRU implementation that optimizes -+page reclaim and improves performance under memory pressure. Page -+reclaim decides the kernel's caching policy and ability to overcommit -+memory. It directly impacts the kswapd CPU usage and RAM efficiency. -+ -+Design overview -+=============== -+Objectives -+---------- -+The design objectives are: -+ -+* Good representation of access recency -+* Try to profit from spatial locality -+* Fast paths to make obvious choices -+* Simple self-correcting heuristics -+ -+The representation of access recency is at the core of all LRU -+implementations. In the multi-gen LRU, each generation represents a -+group of pages with similar access recency. Generations establish a -+(time-based) common frame of reference and therefore help make better -+choices, e.g., between different memcgs on a computer or different -+computers in a data center (for job scheduling). -+ -+Exploiting spatial locality improves efficiency when gathering the -+accessed bit. A rmap walk targets a single page and does not try to -+profit from discovering a young PTE. A page table walk can sweep all -+the young PTEs in an address space, but the address space can be too -+sparse to make a profit. The key is to optimize both methods and use -+them in combination. -+ -+Fast paths reduce code complexity and runtime overhead. Unmapped pages -+do not require TLB flushes; clean pages do not require writeback. -+These facts are only helpful when other conditions, e.g., access -+recency, are similar. With generations as a common frame of reference, -+additional factors stand out. But obvious choices might not be good -+choices; thus self-correction is necessary. -+ -+The benefits of simple self-correcting heuristics are self-evident. -+Again, with generations as a common frame of reference, this becomes -+attainable. Specifically, pages in the same generation can be -+categorized based on additional factors, and a feedback loop can -+statistically compare the refault percentages across those categories -+and infer which of them are better choices. -+ -+Assumptions -+----------- -+The protection of hot pages and the selection of cold pages are based -+on page access channels and patterns. There are two access channels: -+ -+* Accesses through page tables -+* Accesses through file descriptors -+ -+The protection of the former channel is by design stronger because: -+ -+1. The uncertainty in determining the access patterns of the former -+ channel is higher due to the approximation of the accessed bit. -+2. The cost of evicting the former channel is higher due to the TLB -+ flushes required and the likelihood of encountering the dirty bit. -+3. The penalty of underprotecting the former channel is higher because -+ applications usually do not prepare themselves for major page -+ faults like they do for blocked I/O. E.g., GUI applications -+ commonly use dedicated I/O threads to avoid blocking rendering -+ threads. -+ -+There are also two access patterns: -+ -+* Accesses exhibiting temporal locality -+* Accesses not exhibiting temporal locality -+ -+For the reasons listed above, the former channel is assumed to follow -+the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is -+present, and the latter channel is assumed to follow the latter -+pattern unless outlying refaults have been observed. -+ -+Workflow overview -+================= -+Evictable pages are divided into multiple generations for each -+``lruvec``. The youngest generation number is stored in -+``lrugen->max_seq`` for both anon and file types as they are aged on -+an equal footing. The oldest generation numbers are stored in -+``lrugen->min_seq[]`` separately for anon and file types as clean file -+pages can be evicted regardless of swap constraints. These three -+variables are monotonically increasing. -+ -+Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` -+bits in order to fit into the gen counter in ``folio->flags``. Each -+truncated generation number is an index to ``lrugen->lists[]``. The -+sliding window technique is used to track at least ``MIN_NR_GENS`` and -+at most ``MAX_NR_GENS`` generations. The gen counter stores a value -+within ``[1, MAX_NR_GENS]`` while a page is on one of -+``lrugen->lists[]``; otherwise it stores zero. -+ -+Each generation is divided into multiple tiers. A page accessed ``N`` -+times through file descriptors is in tier ``order_base_2(N)``. Unlike -+generations, tiers do not have dedicated ``lrugen->lists[]``. In -+contrast to moving across generations, which requires the LRU lock, -+moving across tiers only involves atomic operations on -+``folio->flags`` and therefore has a negligible cost. A feedback loop -+modeled after the PID controller monitors refaults over all the tiers -+from anon and file types and decides which tiers from which types to -+evict or protect. -+ -+There are two conceptually independent procedures: the aging and the -+eviction. They form a closed-loop system, i.e., the page reclaim. -+ -+Aging -+----- -+The aging produces young generations. Given an ``lruvec``, it -+increments ``max_seq`` when ``max_seq-min_seq+1`` approaches -+``MIN_NR_GENS``. The aging promotes hot pages to the youngest -+generation when it finds them accessed through page tables; the -+demotion of cold pages happens consequently when it increments -+``max_seq``. The aging uses page table walks and rmap walks to find -+young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` -+and calls ``walk_page_range()`` with each ``mm_struct`` on this list -+to scan PTEs, and after each iteration, it increments ``max_seq``. For -+the latter, when the eviction walks the rmap and finds a young PTE, -+the aging scans the adjacent PTEs. For both, on finding a young PTE, -+the aging clears the accessed bit and updates the gen counter of the -+page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. -+ -+Eviction -+-------- -+The eviction consumes old generations. Given an ``lruvec``, it -+increments ``min_seq`` when ``lrugen->lists[]`` indexed by -+``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to -+evict from, it first compares ``min_seq[]`` to select the older type. -+If both types are equally old, it selects the one whose first tier has -+a lower refault percentage. The first tier contains single-use -+unmapped clean pages, which are the best bet. The eviction sorts a -+page according to its gen counter if the aging has found this page -+accessed through page tables and updated its gen counter. It also -+moves a page to the next generation, i.e., ``min_seq+1``, if this page -+was accessed multiple times through file descriptors and the feedback -+loop has detected outlying refaults from the tier this page is in. To -+this end, the feedback loop uses the first tier as the baseline, for -+the reason stated earlier. -+ -+Summary -+------- -+The multi-gen LRU can be disassembled into the following parts: -+ -+* Generations -+* Rmap walks -+* Page table walks -+* Bloom filters -+* PID controller -+ -+The aging and the eviction form a producer-consumer model; -+specifically, the latter drives the former by the sliding window over -+generations. Within the aging, rmap walks drive page table walks by -+inserting hot densely populated page tables to the Bloom filters. -+Within the eviction, the PID controller uses refaults as the feedback -+to select types to evict and tiers to protect. -diff --git a/MAINTAINERS b/MAINTAINERS -index a29c9731350c..96a09757feb3 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -12094,6 +12094,18 @@ L: linux-man@vger.kernel.org - S: Maintained - W: http://www.kernel.org/doc/man-pages - -+MAPLE TREE -+M: Liam R. Howlett -+L: linux-mm@kvack.org -+S: Supported -+F: Documentation/core-api/maple_tree.rst -+F: include/linux/maple_tree.h -+F: include/trace/events/maple_tree.h -+F: lib/maple_tree.c -+F: lib/test_maple_tree.c -+F: tools/testing/radix-tree/linux/maple_tree.h -+F: tools/testing/radix-tree/maple.c -+ - MARDUK (CREATOR CI40) DEVICE TREE SUPPORT - M: Rahul Bedarkar - L: linux-mips@vger.kernel.org -diff --git a/arch/Kconfig b/arch/Kconfig -index 8b311e400ec1..bf19a84fffa2 100644 ---- a/arch/Kconfig -+++ b/arch/Kconfig -@@ -1418,6 +1418,14 @@ config DYNAMIC_SIGFRAME - config HAVE_ARCH_NODE_DEV_GROUP - bool - -+config ARCH_HAS_NONLEAF_PMD_YOUNG -+ bool -+ help -+ Architectures that select this option are capable of setting the -+ accessed bit in non-leaf PMD entries when using them as part of linear -+ address translations. Page table walkers that clear the accessed bit -+ may use this capability to reduce their search space. -+ - source "kernel/gcov/Kconfig" - - source "scripts/gcc-plugins/Kconfig" -diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h -index 4aa996423b0d..763929e814e9 100644 ---- a/arch/alpha/include/uapi/asm/mman.h -+++ b/arch/alpha/include/uapi/asm/mman.h -@@ -76,6 +76,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h -index b5df82aa99e6..71a1af42f0e8 100644 ---- a/arch/arm64/include/asm/pgtable.h -+++ b/arch/arm64/include/asm/pgtable.h -@@ -1082,24 +1082,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, - * page after fork() + CoW for pfn mappings. We don't always have a - * hardware-managed access flag on arm64. - */ --static inline bool arch_faults_on_old_pte(void) --{ -- /* The register read below requires a stable CPU to make any sense */ -- cant_migrate(); -- -- return !cpu_has_hw_af(); --} --#define arch_faults_on_old_pte arch_faults_on_old_pte -+#define arch_has_hw_pte_young cpu_has_hw_af - - /* - * Experimentally, it's cheap to set the access flag in hardware and we - * benefit from prefaulting mappings as 'old' to start with. - */ --static inline bool arch_wants_old_prefaulted_pte(void) --{ -- return !arch_faults_on_old_pte(); --} --#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte -+#define arch_wants_old_prefaulted_pte cpu_has_hw_af - - static inline bool pud_sect_supported(void) - { -diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c -index 98d67444a5b6..27ef7ad3ffd2 100644 ---- a/arch/arm64/kernel/elfcore.c -+++ b/arch/arm64/kernel/elfcore.c -@@ -8,9 +8,9 @@ - #include - #include - --#define for_each_mte_vma(tsk, vma) \ -+#define for_each_mte_vma(vmi, vma) \ - if (system_supports_mte()) \ -- for (vma = tsk->mm->mmap; vma; vma = vma->vm_next) \ -+ for_each_vma(vmi, vma) \ - if (vma->vm_flags & VM_MTE) - - static unsigned long mte_vma_tag_dump_size(struct vm_area_struct *vma) -@@ -81,8 +81,9 @@ Elf_Half elf_core_extra_phdrs(void) - { - struct vm_area_struct *vma; - int vma_count = 0; -+ VMA_ITERATOR(vmi, current->mm, 0); - -- for_each_mte_vma(current, vma) -+ for_each_mte_vma(vmi, vma) - vma_count++; - - return vma_count; -@@ -91,8 +92,9 @@ Elf_Half elf_core_extra_phdrs(void) - int elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, current->mm, 0); - -- for_each_mte_vma(current, vma) { -+ for_each_mte_vma(vmi, vma) { - struct elf_phdr phdr; - - phdr.p_type = PT_AARCH64_MEMTAG_MTE; -@@ -116,8 +118,9 @@ size_t elf_core_extra_data_size(void) - { - struct vm_area_struct *vma; - size_t data_size = 0; -+ VMA_ITERATOR(vmi, current->mm, 0); - -- for_each_mte_vma(current, vma) -+ for_each_mte_vma(vmi, vma) - data_size += mte_vma_tag_dump_size(vma); - - return data_size; -@@ -126,8 +129,9 @@ size_t elf_core_extra_data_size(void) - int elf_core_write_extra_data(struct coredump_params *cprm) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, current->mm, 0); - -- for_each_mte_vma(current, vma) { -+ for_each_mte_vma(vmi, vma) { - if (vma->vm_flags & VM_DONTDUMP) - continue; - -diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c -index a61fc4f989b3..a8388af62b99 100644 ---- a/arch/arm64/kernel/vdso.c -+++ b/arch/arm64/kernel/vdso.c -@@ -136,10 +136,11 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) - { - struct mm_struct *mm = task->mm; - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - - mmap_read_lock(mm); - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - unsigned long size = vma->vm_end - vma->vm_start; - - if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) -diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h -index 1be428663c10..c6e1fc77c996 100644 ---- a/arch/mips/include/uapi/asm/mman.h -+++ b/arch/mips/include/uapi/asm/mman.h -@@ -103,6 +103,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h -index a7ea3204a5fa..22133a6a506e 100644 ---- a/arch/parisc/include/uapi/asm/mman.h -+++ b/arch/parisc/include/uapi/asm/mman.h -@@ -70,6 +70,8 @@ - #define MADV_WIPEONFORK 71 /* Zero memory on fork, child only */ - #define MADV_KEEPONFORK 72 /* Undo MADV_WIPEONFORK */ - -+#define MADV_COLLAPSE 73 /* Synchronous hugepage collapse */ -+ - #define MADV_HWPOISON 100 /* poison a page for testing */ - #define MADV_SOFT_OFFLINE 101 /* soft offline page for testing */ - -diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c -index 3feb7694e0ca..1d3b8bc8a623 100644 ---- a/arch/parisc/kernel/cache.c -+++ b/arch/parisc/kernel/cache.c -@@ -657,15 +657,20 @@ static inline unsigned long mm_total_size(struct mm_struct *mm) - { - struct vm_area_struct *vma; - unsigned long usize = 0; -+ VMA_ITERATOR(vmi, mm, 0); - -- for (vma = mm->mmap; vma && usize < parisc_cache_flush_threshold; vma = vma->vm_next) -+ for_each_vma(vmi, vma) { -+ if (usize >= parisc_cache_flush_threshold) -+ break; - usize += vma->vm_end - vma->vm_start; -+ } - return usize; - } - - void flush_cache_mm(struct mm_struct *mm) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - - /* - * Flushing the whole cache on each cpu takes forever on -@@ -685,7 +690,7 @@ void flush_cache_mm(struct mm_struct *mm) - } - - /* Flush mm */ -- for (vma = mm->mmap; vma; vma = vma->vm_next) -+ for_each_vma(vmi, vma) - flush_cache_pages(vma, vma->vm_start, vma->vm_end); - } - -diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c -index 0da287544054..94a8fa5017c3 100644 ---- a/arch/powerpc/kernel/vdso.c -+++ b/arch/powerpc/kernel/vdso.c -@@ -113,18 +113,18 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page) - int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) - { - struct mm_struct *mm = task->mm; -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - - mmap_read_lock(mm); -- -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - unsigned long size = vma->vm_end - vma->vm_start; - - if (vma_is_special_mapping(vma, &vvar_spec)) - zap_page_range(vma, vma->vm_start, size); - } -- - mmap_read_unlock(mm); -+ - return 0; - } - -diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c -index 19f0ef950d77..9ad6b56bfec9 100644 ---- a/arch/powerpc/mm/book3s32/tlb.c -+++ b/arch/powerpc/mm/book3s32/tlb.c -@@ -81,14 +81,15 @@ EXPORT_SYMBOL(hash__flush_range); - void hash__flush_tlb_mm(struct mm_struct *mm) - { - struct vm_area_struct *mp; -+ VMA_ITERATOR(vmi, mm, 0); - - /* -- * It is safe to go down the mm's list of vmas when called -- * from dup_mmap, holding mmap_lock. It would also be safe from -- * unmap_region or exit_mmap, but not from vmtruncate on SMP - -- * but it seems dup_mmap is the only SMP case which gets here. -+ * It is safe to iterate the vmas when called from dup_mmap, -+ * holding mmap_lock. It would also be safe from unmap_region -+ * or exit_mmap, but not from vmtruncate on SMP - but it seems -+ * dup_mmap is the only SMP case which gets here. - */ -- for (mp = mm->mmap; mp != NULL; mp = mp->vm_next) -+ for_each_vma(vmi, mp) - hash__flush_range(mp->vm_mm, mp->vm_start, mp->vm_end); - } - EXPORT_SYMBOL(hash__flush_tlb_mm); -diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c -index 60c6ea16a972..d73b3b4176e8 100644 ---- a/arch/powerpc/mm/book3s64/subpage_prot.c -+++ b/arch/powerpc/mm/book3s64/subpage_prot.c -@@ -149,24 +149,15 @@ static void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, - unsigned long len) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, addr); - - /* - * We don't try too hard, we just mark all the vma in that range - * VM_NOHUGEPAGE and split them. - */ -- vma = find_vma(mm, addr); -- /* -- * If the range is in unmapped range, just return -- */ -- if (vma && ((addr + len) <= vma->vm_start)) -- return; -- -- while (vma) { -- if (vma->vm_start >= (addr + len)) -- break; -+ for_each_vma_range(vmi, vma, addr + len) { - vma->vm_flags |= VM_NOHUGEPAGE; - walk_page_vma(vma, &subpage_walk_ops, NULL); -- vma = vma->vm_next; - } - } - #else -diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c -index 69b05b6c181b..692e7ae3dcb8 100644 ---- a/arch/riscv/kernel/vdso.c -+++ b/arch/riscv/kernel/vdso.c -@@ -114,11 +114,12 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) - { - struct mm_struct *mm = task->mm; - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - struct __vdso_info *vdso_info = mm->context.vdso_info; - - mmap_read_lock(mm); - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - unsigned long size = vma->vm_end - vma->vm_start; - - if (vma_is_special_mapping(vma, vdso_info->dm)) -diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c -index 5075cde77b29..535099f2736d 100644 ---- a/arch/s390/kernel/vdso.c -+++ b/arch/s390/kernel/vdso.c -@@ -69,10 +69,11 @@ static struct page *find_timens_vvar_page(struct vm_area_struct *vma) - int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) - { - struct mm_struct *mm = task->mm; -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - - mmap_read_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - unsigned long size = vma->vm_end - vma->vm_start; - - if (!vma_is_special_mapping(vma, &vvar_mapping)) -diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c -index 62758cb5872f..02d15c8dc92e 100644 ---- a/arch/s390/mm/gmap.c -+++ b/arch/s390/mm/gmap.c -@@ -2515,8 +2515,9 @@ static const struct mm_walk_ops thp_split_walk_ops = { - static inline void thp_split_mm(struct mm_struct *mm) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - -- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - vma->vm_flags &= ~VM_HUGEPAGE; - vma->vm_flags |= VM_NOHUGEPAGE; - walk_page_vma(vma, &thp_split_walk_ops, NULL); -@@ -2584,8 +2585,9 @@ int gmap_mark_unmergeable(void) - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - int ret; -+ VMA_ITERATOR(vmi, mm, 0); - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, - MADV_UNMERGEABLE, &vma->vm_flags); - if (ret) -diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c -index bc38f79ca3a3..ad449173a1a1 100644 ---- a/arch/um/kernel/tlb.c -+++ b/arch/um/kernel/tlb.c -@@ -584,21 +584,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, - - void flush_tlb_mm(struct mm_struct *mm) - { -- struct vm_area_struct *vma = mm->mmap; -+ struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - -- while (vma != NULL) { -+ for_each_vma(vmi, vma) - fix_range(mm, vma->vm_start, vma->vm_end, 0); -- vma = vma->vm_next; -- } - } - - void force_flush_all(void) - { - struct mm_struct *mm = current->mm; -- struct vm_area_struct *vma = mm->mmap; -+ struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - -- while (vma != NULL) { -+ for_each_vma(vmi, vma) - fix_range(mm, vma->vm_start, vma->vm_end, 1); -- vma = vma->vm_next; -- } - } -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index f9920f1341c8..674d694a665e 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -85,6 +85,7 @@ config X86 - select ARCH_HAS_PMEM_API if X86_64 - select ARCH_HAS_PTE_DEVMAP if X86_64 - select ARCH_HAS_PTE_SPECIAL -+ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 - select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 - select ARCH_HAS_COPY_MC if X86_64 - select ARCH_HAS_SET_MEMORY -diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c -index 1000d457c332..6292b960037b 100644 ---- a/arch/x86/entry/vdso/vma.c -+++ b/arch/x86/entry/vdso/vma.c -@@ -127,17 +127,17 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) - { - struct mm_struct *mm = task->mm; - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - - mmap_read_lock(mm); -- -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - unsigned long size = vma->vm_end - vma->vm_start; - - if (vma_is_special_mapping(vma, &vvar_mapping)) - zap_page_range(vma, vma->vm_start, size); - } -- - mmap_read_unlock(mm); -+ - return 0; - } - #else -@@ -354,6 +354,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) - { - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - - mmap_write_lock(mm); - /* -@@ -363,7 +364,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) - * We could search vma near context.vdso, but it's a slowpath, - * so let's explicitly check all VMAs to be completely sure. - */ -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (vma_is_special_mapping(vma, &vdso_mapping) || - vma_is_special_mapping(vma, &vvar_mapping)) { - mmap_write_unlock(mm); -diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h -index 44e2d6f1dbaa..5059799bebe3 100644 ---- a/arch/x86/include/asm/pgtable.h -+++ b/arch/x86/include/asm/pgtable.h -@@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) - - static inline int pmd_bad(pmd_t pmd) - { -- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; -+ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != -+ (_KERNPG_TABLE & ~_PAGE_ACCESSED); - } - - static inline unsigned long pages_to_mb(unsigned long npg) -@@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) - return boot_cpu_has_bug(X86_BUG_L1TF); - } - --#define arch_faults_on_old_pte arch_faults_on_old_pte --static inline bool arch_faults_on_old_pte(void) -+#define arch_has_hw_pte_young arch_has_hw_pte_young -+static inline bool arch_has_hw_pte_young(void) - { -- return false; -+ return true; - } - - #ifdef CONFIG_PAGE_TABLE_CHECK -diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c -index 3bacd935f840..4c1bcb6053fc 100644 ---- a/arch/x86/kernel/tboot.c -+++ b/arch/x86/kernel/tboot.c -@@ -95,7 +95,7 @@ void __init tboot_probe(void) - - static pgd_t *tboot_pg_dir; - static struct mm_struct tboot_mm = { -- .mm_rb = RB_ROOT, -+ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, tboot_mm.mmap_lock), - .pgd = swapper_pg_dir, - .mm_users = ATOMIC_INIT(2), - .mm_count = ATOMIC_INIT(1), -diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c -index a932d7712d85..8525f2876fb4 100644 ---- a/arch/x86/mm/pgtable.c -+++ b/arch/x86/mm/pgtable.c -@@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, - return ret; - } - --#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) - int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmdp) - { -@@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, - - return ret; - } -+#endif -+ -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE - int pudp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pud_t *pudp) - { -diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h -index 7966a58af472..1ff0c858544f 100644 ---- a/arch/xtensa/include/uapi/asm/mman.h -+++ b/arch/xtensa/include/uapi/asm/mman.h -@@ -111,6 +111,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/arch/xtensa/kernel/syscall.c b/arch/xtensa/kernel/syscall.c -index 201356faa7e6..b3c2450d6f23 100644 ---- a/arch/xtensa/kernel/syscall.c -+++ b/arch/xtensa/kernel/syscall.c -@@ -58,6 +58,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, unsigned long flags) - { - struct vm_area_struct *vmm; -+ struct vma_iterator vmi; - - if (flags & MAP_FIXED) { - /* We do not accept a shared mapping if it would violate -@@ -79,15 +80,20 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, - else - addr = PAGE_ALIGN(addr); - -- for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { -- /* At this point: (!vmm || addr < vmm->vm_end). */ -- if (TASK_SIZE - len < addr) -- return -ENOMEM; -- if (!vmm || addr + len <= vm_start_gap(vmm)) -- return addr; -+ vma_iter_init(&vmi, current->mm, addr); -+ for_each_vma(vmi, vmm) { -+ /* At this point: (addr < vmm->vm_end). */ -+ if (addr + len <= vm_start_gap(vmm)) -+ break; -+ - addr = vmm->vm_end; - if (flags & MAP_SHARED) - addr = COLOUR_ALIGN(addr, pgoff); - } -+ -+ if (TASK_SIZE - len < addr) -+ return -ENOMEM; -+ -+ return addr; - } - #endif -diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c -index e4080ad96089..042a3ef4db1c 100644 ---- a/drivers/firmware/efi/efi.c -+++ b/drivers/firmware/efi/efi.c -@@ -57,7 +57,7 @@ static unsigned long __initdata mem_reserve = EFI_INVALID_TABLE_ADDR; - static unsigned long __initdata rt_prop = EFI_INVALID_TABLE_ADDR; - - struct mm_struct efi_mm = { -- .mm_rb = RB_ROOT, -+ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, efi_mm.mmap_lock), - .mm_users = ATOMIC_INIT(2), - .mm_count = ATOMIC_INIT(1), - .write_protect_seq = SEQCNT_ZERO(efi_mm.write_protect_seq), -diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c -index 8423df021b71..d4398948f016 100644 ---- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c -+++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c -@@ -426,12 +426,11 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { - static int - probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) - { -- const unsigned long end = addr + len; -+ VMA_ITERATOR(vmi, mm, addr); - struct vm_area_struct *vma; -- int ret = -EFAULT; - - mmap_read_lock(mm); -- for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { -+ for_each_vma_range(vmi, vma, addr + len) { - /* Check for holes, note that we also update the addr below */ - if (vma->vm_start > addr) - break; -@@ -439,16 +438,13 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) - if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) - break; - -- if (vma->vm_end >= end) { -- ret = 0; -- break; -- } -- - addr = vma->vm_end; - } - mmap_read_unlock(mm); - -- return ret; -+ if (vma) -+ return -EFAULT; -+ return 0; - } - - /* -diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c -index 60c829113299..2c64f55cf01f 100644 ---- a/drivers/misc/cxl/fault.c -+++ b/drivers/misc/cxl/fault.c -@@ -280,22 +280,6 @@ void cxl_handle_fault(struct work_struct *fault_work) - mmput(mm); - } - --static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) --{ -- struct mm_struct *mm; -- -- mm = get_mem_context(ctx); -- if (mm == NULL) { -- pr_devel("cxl_prefault_one unable to get mm %i\n", -- pid_nr(ctx->pid)); -- return; -- } -- -- cxl_fault_segment(ctx, mm, ea); -- -- mmput(mm); --} -- - static u64 next_segment(u64 ea, u64 vsid) - { - if (vsid & SLB_VSID_B_1T) -@@ -306,23 +290,16 @@ static u64 next_segment(u64 ea, u64 vsid) - return ea + 1; - } - --static void cxl_prefault_vma(struct cxl_context *ctx) -+static void cxl_prefault_vma(struct cxl_context *ctx, struct mm_struct *mm) - { - u64 ea, last_esid = 0; - struct copro_slb slb; -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - int rc; -- struct mm_struct *mm; -- -- mm = get_mem_context(ctx); -- if (mm == NULL) { -- pr_devel("cxl_prefault_vm unable to get mm %i\n", -- pid_nr(ctx->pid)); -- return; -- } - - mmap_read_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - for (ea = vma->vm_start; ea < vma->vm_end; - ea = next_segment(ea, slb.vsid)) { - rc = copro_calculate_slb(mm, ea, &slb); -@@ -337,20 +314,28 @@ static void cxl_prefault_vma(struct cxl_context *ctx) - } - } - mmap_read_unlock(mm); -- -- mmput(mm); - } - - void cxl_prefault(struct cxl_context *ctx, u64 wed) - { -+ struct mm_struct *mm = get_mem_context(ctx); -+ -+ if (mm == NULL) { -+ pr_devel("cxl_prefault unable to get mm %i\n", -+ pid_nr(ctx->pid)); -+ return; -+ } -+ - switch (ctx->afu->prefault_mode) { - case CXL_PREFAULT_WED: -- cxl_prefault_one(ctx, wed); -+ cxl_fault_segment(ctx, mm, wed); - break; - case CXL_PREFAULT_ALL: -- cxl_prefault_vma(ctx); -+ cxl_prefault_vma(ctx, mm); - break; - default: - break; - } -+ -+ mmput(mm); - } -diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c -index 28f87cd8b3ed..290b1bb0e9cd 100644 ---- a/drivers/tee/optee/call.c -+++ b/drivers/tee/optee/call.c -@@ -492,15 +492,18 @@ static bool is_normal_memory(pgprot_t p) - #endif - } - --static int __check_mem_type(struct vm_area_struct *vma, unsigned long end) -+static int __check_mem_type(struct mm_struct *mm, unsigned long start, -+ unsigned long end) - { -- while (vma && is_normal_memory(vma->vm_page_prot)) { -- if (vma->vm_end >= end) -- return 0; -- vma = vma->vm_next; -+ struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, start); -+ -+ for_each_vma_range(vmi, vma, end) { -+ if (!is_normal_memory(vma->vm_page_prot)) -+ return -EINVAL; - } - -- return -EINVAL; -+ return 0; - } - - int optee_check_mem_type(unsigned long start, size_t num_pages) -@@ -516,8 +519,7 @@ int optee_check_mem_type(unsigned long start, size_t num_pages) - return 0; - - mmap_read_lock(mm); -- rc = __check_mem_type(find_vma(mm, start), -- start + num_pages * PAGE_SIZE); -+ rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE); - mmap_read_unlock(mm); - - return rc; -diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c -index e88e8f6f0a33..fae50a24630b 100644 ---- a/drivers/xen/privcmd.c -+++ b/drivers/xen/privcmd.c -@@ -282,7 +282,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata) - struct page, lru); - struct privcmd_mmap_entry *msg = page_address(page); - -- vma = find_vma(mm, msg->va); -+ vma = vma_lookup(mm, msg->va); - rc = -EINVAL; - - if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data) -diff --git a/fs/coredump.c b/fs/coredump.c -index 9f4aae202109..35f2af85b9bc 100644 ---- a/fs/coredump.c -+++ b/fs/coredump.c -@@ -1072,30 +1072,20 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, - return vma->vm_end - vma->vm_start; - } - --static struct vm_area_struct *first_vma(struct task_struct *tsk, -- struct vm_area_struct *gate_vma) --{ -- struct vm_area_struct *ret = tsk->mm->mmap; -- -- if (ret) -- return ret; -- return gate_vma; --} -- - /* - * Helper function for iterating across a vma list. It ensures that the caller - * will visit `gate_vma' prior to terminating the search. - */ --static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, -+static struct vm_area_struct *coredump_next_vma(struct ma_state *mas, -+ struct vm_area_struct *vma, - struct vm_area_struct *gate_vma) - { -- struct vm_area_struct *ret; -- -- ret = this_vma->vm_next; -- if (ret) -- return ret; -- if (this_vma == gate_vma) -+ if (gate_vma && (vma == gate_vma)) - return NULL; -+ -+ vma = mas_next(mas, ULONG_MAX); -+ if (vma) -+ return vma; - return gate_vma; - } - -@@ -1119,9 +1109,10 @@ static void free_vma_snapshot(struct coredump_params *cprm) - */ - static bool dump_vma_snapshot(struct coredump_params *cprm) - { -- struct vm_area_struct *vma, *gate_vma; -+ struct vm_area_struct *gate_vma, *vma = NULL; - struct mm_struct *mm = current->mm; -- int i; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); -+ int i = 0; - - /* - * Once the stack expansion code is fixed to not change VMA bounds -@@ -1141,8 +1132,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) - return false; - } - -- for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; -- vma = next_vma(vma, gate_vma), i++) { -+ while ((vma = coredump_next_vma(&mas, vma, gate_vma)) != NULL) { - struct core_vma_metadata *m = cprm->vma_meta + i; - - m->start = vma->vm_start; -@@ -1150,10 +1140,10 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) - m->flags = vma->vm_flags; - m->dump_size = vma_dump_size(vma, cprm->mm_flags); - m->pgoff = vma->vm_pgoff; -- - m->file = vma->vm_file; - if (m->file) - get_file(m->file); -+ i++; - } - - mmap_write_unlock(mm); -diff --git a/fs/exec.c b/fs/exec.c -index d046dbb9cbd0..3a7bae4e16e5 100644 ---- a/fs/exec.c -+++ b/fs/exec.c -@@ -28,7 +28,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -683,6 +682,8 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) - unsigned long length = old_end - old_start; - unsigned long new_start = old_start - shift; - unsigned long new_end = old_end - shift; -+ VMA_ITERATOR(vmi, mm, new_start); -+ struct vm_area_struct *next; - struct mmu_gather tlb; - - BUG_ON(new_start > new_end); -@@ -691,7 +692,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) - * ensure there are no vmas between where we want to go - * and where we are - */ -- if (vma != find_vma(mm, new_start)) -+ if (vma != vma_next(&vmi)) - return -EFAULT; - - /* -@@ -710,12 +711,13 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) - - lru_add_drain(); - tlb_gather_mmu(&tlb, mm); -+ next = vma_next(&vmi); - if (new_end > old_start) { - /* - * when the old and new regions overlap clear from new_end. - */ - free_pgd_range(&tlb, new_end, old_end, new_end, -- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); -+ next ? next->vm_start : USER_PGTABLES_CEILING); - } else { - /* - * otherwise, clean from old_start; this is done to not touch -@@ -724,7 +726,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) - * for the others its just a little faster. - */ - free_pgd_range(&tlb, old_start, old_end, new_end, -- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); -+ next ? next->vm_start : USER_PGTABLES_CEILING); - } - tlb_finish_mmu(&tlb); - -@@ -1011,6 +1013,7 @@ static int exec_mmap(struct mm_struct *mm) - active_mm = tsk->active_mm; - tsk->active_mm = mm; - tsk->mm = mm; -+ lru_gen_add_mm(mm); - /* - * This prevents preemption while active_mm is being loaded and - * it and mm are being updated, which could cause problems for -@@ -1023,9 +1026,8 @@ static int exec_mmap(struct mm_struct *mm) - activate_mm(active_mm, mm); - if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) - local_irq_enable(); -- tsk->mm->vmacache_seqnum = 0; -- vmacache_flush(tsk); - task_unlock(tsk); -+ lru_gen_use_mm(mm); - if (old_mm) { - mmap_read_unlock(old_mm); - BUG_ON(active_mm != old_mm); -diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c -index 51897427a534..b4a6e0a1b945 100644 ---- a/fs/fuse/dev.c -+++ b/fs/fuse/dev.c -@@ -776,7 +776,8 @@ static int fuse_check_page(struct page *page) - 1 << PG_active | - 1 << PG_workingset | - 1 << PG_reclaim | -- 1 << PG_waiters))) { -+ 1 << PG_waiters | -+ LRU_GEN_MASK | LRU_REFS_MASK))) { - dump_page(page, "fuse: trying to steal weird page"); - return 1; - } -diff --git a/fs/proc/base.c b/fs/proc/base.c -index 93f7e3d971e4..12885a75913f 100644 ---- a/fs/proc/base.c -+++ b/fs/proc/base.c -@@ -2350,6 +2350,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) - GENRADIX(struct map_files_info) fa; - struct map_files_info *p; - int ret; -+ struct vma_iterator vmi; - - genradix_init(&fa); - -@@ -2388,7 +2389,9 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) - * routine might require mmap_lock taken in might_fault(). - */ - -- for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { -+ pos = 2; -+ vma_iter_init(&vmi, mm, 0); -+ for_each_vma(vmi, vma) { - if (!vma->vm_file) - continue; - if (++pos <= ctx->pos) -diff --git a/fs/proc/internal.h b/fs/proc/internal.h -index 06a80f78433d..f03000764ce5 100644 ---- a/fs/proc/internal.h -+++ b/fs/proc/internal.h -@@ -285,7 +285,7 @@ struct proc_maps_private { - struct task_struct *task; - struct mm_struct *mm; - #ifdef CONFIG_MMU -- struct vm_area_struct *tail_vma; -+ struct vma_iterator iter; - #endif - #ifdef CONFIG_NUMA - struct mempolicy *task_mempolicy; -diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c -index 4e0023643f8b..7a10fe08beb3 100644 ---- a/fs/proc/task_mmu.c -+++ b/fs/proc/task_mmu.c -@@ -1,6 +1,5 @@ - // SPDX-License-Identifier: GPL-2.0 - #include --#include - #include - #include - #include -@@ -124,12 +123,26 @@ static void release_task_mempolicy(struct proc_maps_private *priv) - } - #endif - -+static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv, -+ loff_t *ppos) -+{ -+ struct vm_area_struct *vma = vma_next(&priv->iter); -+ -+ if (vma) { -+ *ppos = vma->vm_start; -+ } else { -+ *ppos = -2UL; -+ vma = get_gate_vma(priv->mm); -+ } -+ -+ return vma; -+} -+ - static void *m_start(struct seq_file *m, loff_t *ppos) - { - struct proc_maps_private *priv = m->private; - unsigned long last_addr = *ppos; - struct mm_struct *mm; -- struct vm_area_struct *vma; - - /* See m_next(). Zero at the start or after lseek. */ - if (last_addr == -1UL) -@@ -153,31 +166,21 @@ static void *m_start(struct seq_file *m, loff_t *ppos) - return ERR_PTR(-EINTR); - } - -+ vma_iter_init(&priv->iter, mm, last_addr); - hold_task_mempolicy(priv); -- priv->tail_vma = get_gate_vma(mm); -- -- vma = find_vma(mm, last_addr); -- if (vma) -- return vma; -+ if (last_addr == -2UL) -+ return get_gate_vma(mm); - -- return priv->tail_vma; -+ return proc_get_vma(priv, ppos); - } - - static void *m_next(struct seq_file *m, void *v, loff_t *ppos) - { -- struct proc_maps_private *priv = m->private; -- struct vm_area_struct *next, *vma = v; -- -- if (vma == priv->tail_vma) -- next = NULL; -- else if (vma->vm_next) -- next = vma->vm_next; -- else -- next = priv->tail_vma; -- -- *ppos = next ? next->vm_start : -1UL; -- -- return next; -+ if (*ppos == -2UL) { -+ *ppos = -1UL; -+ return NULL; -+ } -+ return proc_get_vma(m->private, ppos); - } - - static void m_stop(struct seq_file *m, void *v) -@@ -864,7 +867,7 @@ static int show_smap(struct seq_file *m, void *v) - __show_smap(m, &mss, false); - - seq_printf(m, "THPeligible: %d\n", -- hugepage_vma_check(vma, vma->vm_flags, true, false)); -+ hugepage_vma_check(vma, vma->vm_flags, true, false, true)); - - if (arch_pkeys_enabled()) - seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); -@@ -877,16 +880,16 @@ static int show_smaps_rollup(struct seq_file *m, void *v) - { - struct proc_maps_private *priv = m->private; - struct mem_size_stats mss; -- struct mm_struct *mm; -+ struct mm_struct *mm = priv->mm; - struct vm_area_struct *vma; -- unsigned long last_vma_end = 0; -+ unsigned long vma_start = 0, last_vma_end = 0; - int ret = 0; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - priv->task = get_proc_task(priv->inode); - if (!priv->task) - return -ESRCH; - -- mm = priv->mm; - if (!mm || !mmget_not_zero(mm)) { - ret = -ESRCH; - goto out_put_task; -@@ -899,8 +902,13 @@ static int show_smaps_rollup(struct seq_file *m, void *v) - goto out_put_mm; - - hold_task_mempolicy(priv); -+ vma = mas_find(&mas, 0); -+ -+ if (unlikely(!vma)) -+ goto empty_set; - -- for (vma = priv->mm->mmap; vma;) { -+ vma_start = vma->vm_start; -+ do { - smap_gather_stats(vma, &mss, 0); - last_vma_end = vma->vm_end; - -@@ -909,6 +917,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) - * access it for write request. - */ - if (mmap_lock_is_contended(mm)) { -+ mas_pause(&mas); - mmap_read_unlock(mm); - ret = mmap_read_lock_killable(mm); - if (ret) { -@@ -952,7 +961,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) - * contains last_vma_end. - * Iterate VMA' from last_vma_end. - */ -- vma = find_vma(mm, last_vma_end - 1); -+ vma = mas_find(&mas, ULONG_MAX); - /* Case 3 above */ - if (!vma) - break; -@@ -966,11 +975,10 @@ static int show_smaps_rollup(struct seq_file *m, void *v) - smap_gather_stats(vma, &mss, last_vma_end); - } - /* Case 2 above */ -- vma = vma->vm_next; -- } -+ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); - -- show_vma_header_prefix(m, priv->mm->mmap->vm_start, -- last_vma_end, 0, 0, 0, 0); -+empty_set: -+ show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); - seq_pad(m, ' '); - seq_puts(m, "[rollup]\n"); - -@@ -1263,6 +1271,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, - return -ESRCH; - mm = get_task_mm(task); - if (mm) { -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - struct mmu_notifier_range range; - struct clear_refs_private cp = { - .type = type, -@@ -1282,7 +1291,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, - } - - if (type == CLEAR_REFS_SOFT_DIRTY) { -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ mas_for_each(&mas, vma, ULONG_MAX) { - if (!(vma->vm_flags & VM_SOFTDIRTY)) - continue; - vma->vm_flags &= ~VM_SOFTDIRTY; -@@ -1294,8 +1303,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, - 0, NULL, mm, 0, -1UL); - mmu_notifier_invalidate_range_start(&range); - } -- walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, -- &cp); -+ walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); - if (type == CLEAR_REFS_SOFT_DIRTY) { - mmu_notifier_invalidate_range_end(&range); - flush_tlb_mm(mm); -diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c -index a6d21fc0033c..2fd06f52b6a4 100644 ---- a/fs/proc/task_nommu.c -+++ b/fs/proc/task_nommu.c -@@ -20,15 +20,13 @@ - */ - void task_mem(struct seq_file *m, struct mm_struct *mm) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - struct vm_region *region; -- struct rb_node *p; - unsigned long bytes = 0, sbytes = 0, slack = 0, size; -- -- mmap_read_lock(mm); -- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { -- vma = rb_entry(p, struct vm_area_struct, vm_rb); - -+ mmap_read_lock(mm); -+ for_each_vma(vmi, vma) { - bytes += kobjsize(vma); - - region = vma->vm_region; -@@ -82,15 +80,13 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) - - unsigned long task_vsize(struct mm_struct *mm) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; -- struct rb_node *p; - unsigned long vsize = 0; - - mmap_read_lock(mm); -- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { -- vma = rb_entry(p, struct vm_area_struct, vm_rb); -+ for_each_vma(vmi, vma) - vsize += vma->vm_end - vma->vm_start; -- } - mmap_read_unlock(mm); - return vsize; - } -@@ -99,14 +95,13 @@ unsigned long task_statm(struct mm_struct *mm, - unsigned long *shared, unsigned long *text, - unsigned long *data, unsigned long *resident) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - struct vm_region *region; -- struct rb_node *p; - unsigned long size = kobjsize(mm); - - mmap_read_lock(mm); -- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { -- vma = rb_entry(p, struct vm_area_struct, vm_rb); -+ for_each_vma(vmi, vma) { - size += kobjsize(vma); - region = vma->vm_region; - if (region) { -@@ -190,17 +185,19 @@ static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) - */ - static int show_map(struct seq_file *m, void *_p) - { -- struct rb_node *p = _p; -- -- return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb)); -+ return nommu_vma_show(m, _p); - } - - static void *m_start(struct seq_file *m, loff_t *pos) - { - struct proc_maps_private *priv = m->private; - struct mm_struct *mm; -- struct rb_node *p; -- loff_t n = *pos; -+ struct vm_area_struct *vma; -+ unsigned long addr = *pos; -+ -+ /* See m_next(). Zero at the start or after lseek. */ -+ if (addr == -1UL) -+ return NULL; - - /* pin the task and mm whilst we play with them */ - priv->task = get_proc_task(priv->inode); -@@ -216,10 +213,10 @@ static void *m_start(struct seq_file *m, loff_t *pos) - return ERR_PTR(-EINTR); - } - -- /* start from the Nth VMA */ -- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) -- if (n-- == 0) -- return p; -+ /* start the next element from addr */ -+ vma = find_vma(mm, addr); -+ if (vma) -+ return vma; - - mmap_read_unlock(mm); - mmput(mm); -@@ -242,10 +239,10 @@ static void m_stop(struct seq_file *m, void *_vml) - - static void *m_next(struct seq_file *m, void *_p, loff_t *pos) - { -- struct rb_node *p = _p; -+ struct vm_area_struct *vma = _p; - -- (*pos)++; -- return p ? rb_next(p) : NULL; -+ *pos = vma->vm_end; -+ return find_vma(vma->vm_mm, vma->vm_end); - } - - static const struct seq_operations proc_pid_maps_ops = { -diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c -index 175de70e3adf..f662cede4488 100644 ---- a/fs/userfaultfd.c -+++ b/fs/userfaultfd.c -@@ -615,14 +615,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, - if (release_new_ctx) { - struct vm_area_struct *vma; - struct mm_struct *mm = release_new_ctx->mm; -+ VMA_ITERATOR(vmi, mm, 0); - - /* the various vma->vm_userfaultfd_ctx still points to it */ - mmap_write_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) -+ for_each_vma(vmi, vma) { - if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - } -+ } - mmap_write_unlock(mm); - - userfaultfd_ctx_put(release_new_ctx); -@@ -803,11 +805,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, - return false; - } - --int userfaultfd_unmap_prep(struct vm_area_struct *vma, -- unsigned long start, unsigned long end, -- struct list_head *unmaps) -+int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, -+ unsigned long end, struct list_head *unmaps) - { -- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { -+ VMA_ITERATOR(vmi, mm, start); -+ struct vm_area_struct *vma; -+ -+ for_each_vma_range(vmi, vma, end) { - struct userfaultfd_unmap_ctx *unmap_ctx; - struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; - -@@ -857,6 +861,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) - /* len == 0 means wake all */ - struct userfaultfd_wake_range range = { .len = 0, }; - unsigned long new_flags; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - WRITE_ONCE(ctx->released, true); - -@@ -873,7 +878,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) - */ - mmap_write_lock(mm); - prev = NULL; -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ mas_for_each(&mas, vma, ULONG_MAX) { - cond_resched(); - BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ - !!(vma->vm_flags & __VM_UFFD_FLAGS)); -@@ -887,10 +892,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file) - vma->vm_file, vma->vm_pgoff, - vma_policy(vma), - NULL_VM_UFFD_CTX, anon_vma_name(vma)); -- if (prev) -+ if (prev) { -+ mas_pause(&mas); - vma = prev; -- else -+ } else { - prev = vma; -+ } -+ - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - } -@@ -1272,6 +1280,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - bool found; - bool basic_ioctls; - unsigned long start, end, vma_end; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - user_uffdio_register = (struct uffdio_register __user *) arg; - -@@ -1314,7 +1323,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - goto out; - - mmap_write_lock(mm); -- vma = find_vma_prev(mm, start, &prev); -+ mas_set(&mas, start); -+ vma = mas_find(&mas, ULONG_MAX); - if (!vma) - goto out_unlock; - -@@ -1339,7 +1349,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - */ - found = false; - basic_ioctls = false; -- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { -+ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { - cond_resched(); - - BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ -@@ -1399,8 +1409,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - } - BUG_ON(!found); - -- if (vma->vm_start < start) -- prev = vma; -+ mas_set(&mas, start); -+ prev = mas_prev(&mas, 0); -+ if (prev != vma) -+ mas_next(&mas, ULONG_MAX); - - ret = 0; - do { -@@ -1430,6 +1442,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - ((struct vm_userfaultfd_ctx){ ctx }), - anon_vma_name(vma)); - if (prev) { -+ /* vma_merge() invalidated the mas */ -+ mas_pause(&mas); - vma = prev; - goto next; - } -@@ -1437,11 +1451,15 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - ret = split_vma(mm, vma, start, 1); - if (ret) - break; -+ /* split_vma() invalidated the mas */ -+ mas_pause(&mas); - } - if (vma->vm_end > end) { - ret = split_vma(mm, vma, end, 0); - if (ret) - break; -+ /* split_vma() invalidated the mas */ -+ mas_pause(&mas); - } - next: - /* -@@ -1458,8 +1476,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - skip: - prev = vma; - start = vma->vm_end; -- vma = vma->vm_next; -- } while (vma && vma->vm_start < end); -+ vma = mas_next(&mas, end - 1); -+ } while (vma); - out_unlock: - mmap_write_unlock(mm); - mmput(mm); -@@ -1503,6 +1521,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - bool found; - unsigned long start, end, vma_end; - const void __user *buf = (void __user *)arg; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - ret = -EFAULT; - if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) -@@ -1521,7 +1540,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - goto out; - - mmap_write_lock(mm); -- vma = find_vma_prev(mm, start, &prev); -+ mas_set(&mas, start); -+ vma = mas_find(&mas, ULONG_MAX); - if (!vma) - goto out_unlock; - -@@ -1546,7 +1566,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - */ - found = false; - ret = -EINVAL; -- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { -+ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { - cond_resched(); - - BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ -@@ -1566,8 +1586,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - } - BUG_ON(!found); - -- if (vma->vm_start < start) -- prev = vma; -+ mas_set(&mas, start); -+ prev = mas_prev(&mas, 0); -+ if (prev != vma) -+ mas_next(&mas, ULONG_MAX); - - ret = 0; - do { -@@ -1636,8 +1658,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - skip: - prev = vma; - start = vma->vm_end; -- vma = vma->vm_next; -- } while (vma && vma->vm_start < end); -+ vma = mas_next(&mas, end - 1); -+ } while (vma); - out_unlock: - mmap_write_unlock(mm); - mmput(mm); -diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h -index ac5d0515680e..9179463c3c9f 100644 ---- a/include/linux/cgroup.h -+++ b/include/linux/cgroup.h -@@ -432,6 +432,18 @@ static inline void cgroup_put(struct cgroup *cgrp) - css_put(&cgrp->self); - } - -+extern struct mutex cgroup_mutex; -+ -+static inline void cgroup_lock(void) -+{ -+ mutex_lock(&cgroup_mutex); -+} -+ -+static inline void cgroup_unlock(void) -+{ -+ mutex_unlock(&cgroup_mutex); -+} -+ - /** - * task_css_set_check - obtain a task's css_set with extra access conditions - * @task: the task to obtain css_set for -@@ -446,7 +458,6 @@ static inline void cgroup_put(struct cgroup *cgrp) - * as locks used during the cgroup_subsys::attach() methods. - */ - #ifdef CONFIG_PROVE_RCU --extern struct mutex cgroup_mutex; - extern spinlock_t css_set_lock; - #define task_css_set_check(task, __c) \ - rcu_dereference_check((task)->cgroups, \ -@@ -708,6 +719,8 @@ struct cgroup; - static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } - static inline void css_get(struct cgroup_subsys_state *css) {} - static inline void css_put(struct cgroup_subsys_state *css) {} -+static inline void cgroup_lock(void) {} -+static inline void cgroup_unlock(void) {} - static inline int cgroup_attach_task_all(struct task_struct *from, - struct task_struct *t) { return 0; } - static inline int cgroupstats_build(struct cgroupstats *stats, -diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h -index 768e5261fdae..38265f9f782e 100644 ---- a/include/linux/huge_mm.h -+++ b/include/linux/huge_mm.h -@@ -168,9 +168,8 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) - !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); - } - --bool hugepage_vma_check(struct vm_area_struct *vma, -- unsigned long vm_flags, -- bool smaps, bool in_pf); -+bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, -+ bool smaps, bool in_pf, bool enforce_sysfs); - - #define transparent_hugepage_use_zero_page() \ - (transparent_hugepage_flags & \ -@@ -219,6 +218,9 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, - - int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, - int advice); -+int madvise_collapse(struct vm_area_struct *vma, -+ struct vm_area_struct **prev, -+ unsigned long start, unsigned long end); - void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, - unsigned long end, long adjust_next); - spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma); -@@ -321,8 +323,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, - } - - static inline bool hugepage_vma_check(struct vm_area_struct *vma, -- unsigned long vm_flags, -- bool smaps, bool in_pf) -+ unsigned long vm_flags, bool smaps, -+ bool in_pf, bool enforce_sysfs) - { - return false; - } -@@ -362,9 +364,16 @@ static inline void split_huge_pmd_address(struct vm_area_struct *vma, - static inline int hugepage_madvise(struct vm_area_struct *vma, - unsigned long *vm_flags, int advice) - { -- BUG(); -- return 0; -+ return -EINVAL; - } -+ -+static inline int madvise_collapse(struct vm_area_struct *vma, -+ struct vm_area_struct **prev, -+ unsigned long start, unsigned long end) -+{ -+ return -EINVAL; -+} -+ - static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, - unsigned long start, - unsigned long end, -diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h -new file mode 100644 -index 000000000000..2effab72add1 ---- /dev/null -+++ b/include/linux/maple_tree.h -@@ -0,0 +1,685 @@ -+/* SPDX-License-Identifier: GPL-2.0+ */ -+#ifndef _LINUX_MAPLE_TREE_H -+#define _LINUX_MAPLE_TREE_H -+/* -+ * Maple Tree - An RCU-safe adaptive tree for storing ranges -+ * Copyright (c) 2018-2022 Oracle -+ * Authors: Liam R. Howlett -+ * Matthew Wilcox -+ */ -+ -+#include -+#include -+#include -+/* #define CONFIG_MAPLE_RCU_DISABLED */ -+/* #define CONFIG_DEBUG_MAPLE_TREE_VERBOSE */ -+ -+/* -+ * Allocated nodes are mutable until they have been inserted into the tree, -+ * at which time they cannot change their type until they have been removed -+ * from the tree and an RCU grace period has passed. -+ * -+ * Removed nodes have their ->parent set to point to themselves. RCU readers -+ * check ->parent before relying on the value that they loaded from the -+ * slots array. This lets us reuse the slots array for the RCU head. -+ * -+ * Nodes in the tree point to their parent unless bit 0 is set. -+ */ -+#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) -+/* 64bit sizes */ -+#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ -+#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ -+#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ -+#define MAPLE_ARANGE64_META_MAX 15 /* Out of range for metadata */ -+#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) -+#else -+/* 32bit sizes */ -+#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ -+#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ -+#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ -+#define MAPLE_ARANGE64_META_MAX 31 /* Out of range for metadata */ -+#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) -+#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ -+ -+#define MAPLE_NODE_MASK 255UL -+ -+/* -+ * The node->parent of the root node has bit 0 set and the rest of the pointer -+ * is a pointer to the tree itself. No more bits are available in this pointer -+ * (on m68k, the data structure may only be 2-byte aligned). -+ * -+ * Internal non-root nodes can only have maple_range_* nodes as parents. The -+ * parent pointer is 256B aligned like all other tree nodes. When storing a 32 -+ * or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an -+ * extra bit to store the offset. This extra bit comes from a reuse of the last -+ * bit in the node type. This is possible by using bit 1 to indicate if bit 2 -+ * is part of the type or the slot. -+ * -+ * Once the type is decided, the decision of an allocation range type or a range -+ * type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE -+ * flag. -+ * -+ * Node types: -+ * 0x??1 = Root -+ * 0x?00 = 16 bit nodes -+ * 0x010 = 32 bit nodes -+ * 0x110 = 64 bit nodes -+ * -+ * Slot size and location in the parent pointer: -+ * type : slot location -+ * 0x??1 : Root -+ * 0x?00 : 16 bit values, type in 0-1, slot in 2-6 -+ * 0x010 : 32 bit values, type in 0-2, slot in 3-6 -+ * 0x110 : 64 bit values, type in 0-2, slot in 3-6 -+ */ -+ -+/* -+ * This metadata is used to optimize the gap updating code and in reverse -+ * searching for gaps or any other code that needs to find the end of the data. -+ */ -+struct maple_metadata { -+ unsigned char end; -+ unsigned char gap; -+}; -+ -+/* -+ * Leaf nodes do not store pointers to nodes, they store user data. Users may -+ * store almost any bit pattern. As noted above, the optimisation of storing an -+ * entry at 0 in the root pointer cannot be done for data which have the bottom -+ * two bits set to '10'. We also reserve values with the bottom two bits set to -+ * '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs -+ * return errnos as a negative errno shifted right by two bits and the bottom -+ * two bits set to '10', and while choosing to store these values in the array -+ * is not an error, it may lead to confusion if you're testing for an error with -+ * mas_is_err(). -+ * -+ * Non-leaf nodes store the type of the node pointed to (enum maple_type in bits -+ * 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now. -+ * -+ * In regular B-Tree terms, pivots are called keys. The term pivot is used to -+ * indicate that the tree is specifying ranges, Pivots may appear in the -+ * subtree with an entry attached to the value whereas keys are unique to a -+ * specific position of a B-tree. Pivot values are inclusive of the slot with -+ * the same index. -+ */ -+ -+struct maple_range_64 { -+ struct maple_pnode *parent; -+ unsigned long pivot[MAPLE_RANGE64_SLOTS - 1]; -+ union { -+ void __rcu *slot[MAPLE_RANGE64_SLOTS]; -+ struct { -+ void __rcu *pad[MAPLE_RANGE64_SLOTS - 1]; -+ struct maple_metadata meta; -+ }; -+ }; -+}; -+ -+/* -+ * At tree creation time, the user can specify that they're willing to trade off -+ * storing fewer entries in a tree in return for storing more information in -+ * each node. -+ * -+ * The maple tree supports recording the largest range of NULL entries available -+ * in this node, also called gaps. This optimises the tree for allocating a -+ * range. -+ */ -+struct maple_arange_64 { -+ struct maple_pnode *parent; -+ unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1]; -+ void __rcu *slot[MAPLE_ARANGE64_SLOTS]; -+ unsigned long gap[MAPLE_ARANGE64_SLOTS]; -+ struct maple_metadata meta; -+}; -+ -+struct maple_alloc { -+ unsigned long total; -+ unsigned char node_count; -+ unsigned int request_count; -+ struct maple_alloc *slot[MAPLE_ALLOC_SLOTS]; -+}; -+ -+struct maple_topiary { -+ struct maple_pnode *parent; -+ struct maple_enode *next; /* Overlaps the pivot */ -+}; -+ -+enum maple_type { -+ maple_dense, -+ maple_leaf_64, -+ maple_range_64, -+ maple_arange_64, -+}; -+ -+ -+/** -+ * DOC: Maple tree flags -+ * -+ * * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree -+ * * MT_FLAGS_USE_RCU - Operate in RCU mode -+ * * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags -+ * * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value -+ * * MT_FLAGS_LOCK_MASK - How the mt_lock is used -+ * * MT_FLAGS_LOCK_IRQ - Acquired irq-safe -+ * * MT_FLAGS_LOCK_BH - Acquired bh-safe -+ * * MT_FLAGS_LOCK_EXTERN - mt_lock is not used -+ * -+ * MAPLE_HEIGHT_MAX The largest height that can be stored -+ */ -+#define MT_FLAGS_ALLOC_RANGE 0x01 -+#define MT_FLAGS_USE_RCU 0x02 -+#define MT_FLAGS_HEIGHT_OFFSET 0x02 -+#define MT_FLAGS_HEIGHT_MASK 0x7C -+#define MT_FLAGS_LOCK_MASK 0x300 -+#define MT_FLAGS_LOCK_IRQ 0x100 -+#define MT_FLAGS_LOCK_BH 0x200 -+#define MT_FLAGS_LOCK_EXTERN 0x300 -+ -+#define MAPLE_HEIGHT_MAX 31 -+ -+ -+#define MAPLE_NODE_TYPE_MASK 0x0F -+#define MAPLE_NODE_TYPE_SHIFT 0x03 -+ -+#define MAPLE_RESERVED_RANGE 4096 -+ -+#ifdef CONFIG_LOCKDEP -+typedef struct lockdep_map *lockdep_map_p; -+#define mt_lock_is_held(mt) lock_is_held(mt->ma_external_lock) -+#define mt_set_external_lock(mt, lock) \ -+ (mt)->ma_external_lock = &(lock)->dep_map -+#else -+typedef struct { /* nothing */ } lockdep_map_p; -+#define mt_lock_is_held(mt) 1 -+#define mt_set_external_lock(mt, lock) do { } while (0) -+#endif -+ -+/* -+ * If the tree contains a single entry at index 0, it is usually stored in -+ * tree->ma_root. To optimise for the page cache, an entry which ends in '00', -+ * '01' or '11' is stored in the root, but an entry which ends in '10' will be -+ * stored in a node. Bits 3-6 are used to store enum maple_type. -+ * -+ * The flags are used both to store some immutable information about this tree -+ * (set at tree creation time) and dynamic information set under the spinlock. -+ * -+ * Another use of flags are to indicate global states of the tree. This is the -+ * case with the MAPLE_USE_RCU flag, which indicates the tree is currently in -+ * RCU mode. This mode was added to allow the tree to reuse nodes instead of -+ * re-allocating and RCU freeing nodes when there is a single user. -+ */ -+struct maple_tree { -+ union { -+ spinlock_t ma_lock; -+ lockdep_map_p ma_external_lock; -+ }; -+ void __rcu *ma_root; -+ unsigned int ma_flags; -+}; -+ -+/** -+ * MTREE_INIT() - Initialize a maple tree -+ * @name: The maple tree name -+ * @__flags: The maple tree flags -+ * -+ */ -+#define MTREE_INIT(name, __flags) { \ -+ .ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), \ -+ .ma_flags = __flags, \ -+ .ma_root = NULL, \ -+} -+ -+/** -+ * MTREE_INIT_EXT() - Initialize a maple tree with an external lock. -+ * @name: The tree name -+ * @__flags: The maple tree flags -+ * @__lock: The external lock -+ */ -+#ifdef CONFIG_LOCKDEP -+#define MTREE_INIT_EXT(name, __flags, __lock) { \ -+ .ma_external_lock = &(__lock).dep_map, \ -+ .ma_flags = (__flags), \ -+ .ma_root = NULL, \ -+} -+#else -+#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags) -+#endif -+ -+#define DEFINE_MTREE(name) \ -+ struct maple_tree name = MTREE_INIT(name, 0) -+ -+#define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) -+#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock)) -+ -+/* -+ * The Maple Tree squeezes various bits in at various points which aren't -+ * necessarily obvious. Usually, this is done by observing that pointers are -+ * N-byte aligned and thus the bottom log_2(N) bits are available for use. We -+ * don't use the high bits of pointers to store additional information because -+ * we don't know what bits are unused on any given architecture. -+ * -+ * Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8 -+ * low bits for our own purposes. Nodes are currently of 4 types: -+ * 1. Single pointer (Range is 0-0) -+ * 2. Non-leaf Allocation Range nodes -+ * 3. Non-leaf Range nodes -+ * 4. Leaf Range nodes All nodes consist of a number of node slots, -+ * pivots, and a parent pointer. -+ */ -+ -+struct maple_node { -+ union { -+ struct { -+ struct maple_pnode *parent; -+ void __rcu *slot[MAPLE_NODE_SLOTS]; -+ }; -+ struct { -+ void *pad; -+ struct rcu_head rcu; -+ struct maple_enode *piv_parent; -+ unsigned char parent_slot; -+ enum maple_type type; -+ unsigned char slot_len; -+ unsigned int ma_flags; -+ }; -+ struct maple_range_64 mr64; -+ struct maple_arange_64 ma64; -+ struct maple_alloc alloc; -+ }; -+}; -+ -+/* -+ * More complicated stores can cause two nodes to become one or three and -+ * potentially alter the height of the tree. Either half of the tree may need -+ * to be rebalanced against the other. The ma_topiary struct is used to track -+ * which nodes have been 'cut' from the tree so that the change can be done -+ * safely at a later date. This is done to support RCU. -+ */ -+struct ma_topiary { -+ struct maple_enode *head; -+ struct maple_enode *tail; -+ struct maple_tree *mtree; -+}; -+ -+void *mtree_load(struct maple_tree *mt, unsigned long index); -+ -+int mtree_insert(struct maple_tree *mt, unsigned long index, -+ void *entry, gfp_t gfp); -+int mtree_insert_range(struct maple_tree *mt, unsigned long first, -+ unsigned long last, void *entry, gfp_t gfp); -+int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, -+ void *entry, unsigned long size, unsigned long min, -+ unsigned long max, gfp_t gfp); -+int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, -+ void *entry, unsigned long size, unsigned long min, -+ unsigned long max, gfp_t gfp); -+ -+int mtree_store_range(struct maple_tree *mt, unsigned long first, -+ unsigned long last, void *entry, gfp_t gfp); -+int mtree_store(struct maple_tree *mt, unsigned long index, -+ void *entry, gfp_t gfp); -+void *mtree_erase(struct maple_tree *mt, unsigned long index); -+ -+void mtree_destroy(struct maple_tree *mt); -+void __mt_destroy(struct maple_tree *mt); -+ -+/** -+ * mtree_empty() - Determine if a tree has any present entries. -+ * @mt: Maple Tree. -+ * -+ * Context: Any context. -+ * Return: %true if the tree contains only NULL pointers. -+ */ -+static inline bool mtree_empty(const struct maple_tree *mt) -+{ -+ return mt->ma_root == NULL; -+} -+ -+/* Advanced API */ -+ -+/* -+ * The maple state is defined in the struct ma_state and is used to keep track -+ * of information during operations, and even between operations when using the -+ * advanced API. -+ * -+ * If state->node has bit 0 set then it references a tree location which is not -+ * a node (eg the root). If bit 1 is set, the rest of the bits are a negative -+ * errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the -+ * node type. -+ * -+ * state->alloc either has a request number of nodes or an allocated node. If -+ * stat->alloc has a requested number of nodes, the first bit will be set (0x1) -+ * and the remaining bits are the value. If state->alloc is a node, then the -+ * node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for -+ * storing more allocated nodes, a total number of nodes allocated, and the -+ * node_count in this node. node_count is the number of allocated nodes in this -+ * node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further -+ * nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc -+ * by removing a node from the state->alloc node until state->alloc->node_count -+ * is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted -+ * to state->alloc. Nodes are pushed onto state->alloc by putting the current -+ * state->alloc into the pushed node's slot[0]. -+ * -+ * The state also contains the implied min/max of the state->node, the depth of -+ * this search, and the offset. The implied min/max are either from the parent -+ * node or are 0-oo for the root node. The depth is incremented or decremented -+ * every time a node is walked down or up. The offset is the slot/pivot of -+ * interest in the node - either for reading or writing. -+ * -+ * When returning a value the maple state index and last respectively contain -+ * the start and end of the range for the entry. Ranges are inclusive in the -+ * Maple Tree. -+ */ -+struct ma_state { -+ struct maple_tree *tree; /* The tree we're operating in */ -+ unsigned long index; /* The index we're operating on - range start */ -+ unsigned long last; /* The last index we're operating on - range end */ -+ struct maple_enode *node; /* The node containing this entry */ -+ unsigned long min; /* The minimum index of this node - implied pivot min */ -+ unsigned long max; /* The maximum index of this node - implied pivot max */ -+ struct maple_alloc *alloc; /* Allocated nodes for this operation */ -+ unsigned char depth; /* depth of tree descent during write */ -+ unsigned char offset; -+ unsigned char mas_flags; -+}; -+ -+struct ma_wr_state { -+ struct ma_state *mas; -+ struct maple_node *node; /* Decoded mas->node */ -+ unsigned long r_min; /* range min */ -+ unsigned long r_max; /* range max */ -+ enum maple_type type; /* mas->node type */ -+ unsigned char offset_end; /* The offset where the write ends */ -+ unsigned char node_end; /* mas->node end */ -+ unsigned long *pivots; /* mas->node->pivots pointer */ -+ unsigned long end_piv; /* The pivot at the offset end */ -+ void __rcu **slots; /* mas->node->slots pointer */ -+ void *entry; /* The entry to write */ -+ void *content; /* The existing entry that is being overwritten */ -+}; -+ -+#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock)) -+#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock)) -+ -+ -+/* -+ * Special values for ma_state.node. -+ * MAS_START means we have not searched the tree. -+ * MAS_ROOT means we have searched the tree and the entry we found lives in -+ * the root of the tree (ie it has index 0, length 1 and is the only entry in -+ * the tree). -+ * MAS_NONE means we have searched the tree and there is no node in the -+ * tree for this entry. For example, we searched for index 1 in an empty -+ * tree. Or we have a tree which points to a full leaf node and we -+ * searched for an entry which is larger than can be contained in that -+ * leaf node. -+ * MA_ERROR represents an errno. After dropping the lock and attempting -+ * to resolve the error, the walk would have to be restarted from the -+ * top of the tree as the tree may have been modified. -+ */ -+#define MAS_START ((struct maple_enode *)1UL) -+#define MAS_ROOT ((struct maple_enode *)5UL) -+#define MAS_NONE ((struct maple_enode *)9UL) -+#define MAS_PAUSE ((struct maple_enode *)17UL) -+#define MA_ERROR(err) \ -+ ((struct maple_enode *)(((unsigned long)err << 2) | 2UL)) -+ -+#define MA_STATE(name, mt, first, end) \ -+ struct ma_state name = { \ -+ .tree = mt, \ -+ .index = first, \ -+ .last = end, \ -+ .node = MAS_START, \ -+ .min = 0, \ -+ .max = ULONG_MAX, \ -+ .alloc = NULL, \ -+ } -+ -+#define MA_WR_STATE(name, ma_state, wr_entry) \ -+ struct ma_wr_state name = { \ -+ .mas = ma_state, \ -+ .content = NULL, \ -+ .entry = wr_entry, \ -+ } -+ -+#define MA_TOPIARY(name, tree) \ -+ struct ma_topiary name = { \ -+ .head = NULL, \ -+ .tail = NULL, \ -+ .mtree = tree, \ -+ } -+ -+void *mas_walk(struct ma_state *mas); -+void *mas_store(struct ma_state *mas, void *entry); -+void *mas_erase(struct ma_state *mas); -+int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp); -+void mas_store_prealloc(struct ma_state *mas, void *entry); -+void *mas_find(struct ma_state *mas, unsigned long max); -+void *mas_find_rev(struct ma_state *mas, unsigned long min); -+int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp); -+bool mas_is_err(struct ma_state *mas); -+ -+bool mas_nomem(struct ma_state *mas, gfp_t gfp); -+void mas_pause(struct ma_state *mas); -+void maple_tree_init(void); -+void mas_destroy(struct ma_state *mas); -+int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); -+ -+void *mas_prev(struct ma_state *mas, unsigned long min); -+void *mas_next(struct ma_state *mas, unsigned long max); -+ -+int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, -+ unsigned long size); -+ -+/* Checks if a mas has not found anything */ -+static inline bool mas_is_none(struct ma_state *mas) -+{ -+ return mas->node == MAS_NONE; -+} -+ -+/* Checks if a mas has been paused */ -+static inline bool mas_is_paused(struct ma_state *mas) -+{ -+ return mas->node == MAS_PAUSE; -+} -+ -+void mas_dup_tree(struct ma_state *oldmas, struct ma_state *mas); -+void mas_dup_store(struct ma_state *mas, void *entry); -+ -+/* -+ * This finds an empty area from the highest address to the lowest. -+ * AKA "Topdown" version, -+ */ -+int mas_empty_area_rev(struct ma_state *mas, unsigned long min, -+ unsigned long max, unsigned long size); -+/** -+ * mas_reset() - Reset a Maple Tree operation state. -+ * @mas: Maple Tree operation state. -+ * -+ * Resets the error or walk state of the @mas so future walks of the -+ * array will start from the root. Use this if you have dropped the -+ * lock and want to reuse the ma_state. -+ * -+ * Context: Any context. -+ */ -+static inline void mas_reset(struct ma_state *mas) -+{ -+ mas->node = MAS_START; -+} -+ -+/** -+ * mas_for_each() - Iterate over a range of the maple tree. -+ * @__mas: Maple Tree operation state (maple_state) -+ * @__entry: Entry retrieved from the tree -+ * @__max: maximum index to retrieve from the tree -+ * -+ * When returned, mas->index and mas->last will hold the entire range for the -+ * entry. -+ * -+ * Note: may return the zero entry. -+ * -+ */ -+#define mas_for_each(__mas, __entry, __max) \ -+ while (((__entry) = mas_find((__mas), (__max))) != NULL) -+ -+ -+/** -+ * mas_set_range() - Set up Maple Tree operation state for a different index. -+ * @mas: Maple Tree operation state. -+ * @start: New start of range in the Maple Tree. -+ * @last: New end of range in the Maple Tree. -+ * -+ * Move the operation state to refer to a different range. This will -+ * have the effect of starting a walk from the top; see mas_next() -+ * to move to an adjacent index. -+ */ -+static inline -+void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last) -+{ -+ mas->index = start; -+ mas->last = last; -+ mas->node = MAS_START; -+} -+ -+/** -+ * mas_set() - Set up Maple Tree operation state for a different index. -+ * @mas: Maple Tree operation state. -+ * @index: New index into the Maple Tree. -+ * -+ * Move the operation state to refer to a different index. This will -+ * have the effect of starting a walk from the top; see mas_next() -+ * to move to an adjacent index. -+ */ -+static inline void mas_set(struct ma_state *mas, unsigned long index) -+{ -+ -+ mas_set_range(mas, index, index); -+} -+ -+static inline bool mt_external_lock(const struct maple_tree *mt) -+{ -+ return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN; -+} -+ -+/** -+ * mt_init_flags() - Initialise an empty maple tree with flags. -+ * @mt: Maple Tree -+ * @flags: maple tree flags. -+ * -+ * If you need to initialise a Maple Tree with special flags (eg, an -+ * allocation tree), use this function. -+ * -+ * Context: Any context. -+ */ -+static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags) -+{ -+ mt->ma_flags = flags; -+ if (!mt_external_lock(mt)) -+ spin_lock_init(&mt->ma_lock); -+ rcu_assign_pointer(mt->ma_root, NULL); -+} -+ -+/** -+ * mt_init() - Initialise an empty maple tree. -+ * @mt: Maple Tree -+ * -+ * An empty Maple Tree. -+ * -+ * Context: Any context. -+ */ -+static inline void mt_init(struct maple_tree *mt) -+{ -+ mt_init_flags(mt, 0); -+} -+ -+static inline bool mt_in_rcu(struct maple_tree *mt) -+{ -+#ifdef CONFIG_MAPLE_RCU_DISABLED -+ return false; -+#endif -+ return mt->ma_flags & MT_FLAGS_USE_RCU; -+} -+ -+/** -+ * mt_clear_in_rcu() - Switch the tree to non-RCU mode. -+ * @mt: The Maple Tree -+ */ -+static inline void mt_clear_in_rcu(struct maple_tree *mt) -+{ -+ if (!mt_in_rcu(mt)) -+ return; -+ -+ if (mt_external_lock(mt)) { -+ BUG_ON(!mt_lock_is_held(mt)); -+ mt->ma_flags &= ~MT_FLAGS_USE_RCU; -+ } else { -+ mtree_lock(mt); -+ mt->ma_flags &= ~MT_FLAGS_USE_RCU; -+ mtree_unlock(mt); -+ } -+} -+ -+/** -+ * mt_set_in_rcu() - Switch the tree to RCU safe mode. -+ * @mt: The Maple Tree -+ */ -+static inline void mt_set_in_rcu(struct maple_tree *mt) -+{ -+ if (mt_in_rcu(mt)) -+ return; -+ -+ if (mt_external_lock(mt)) { -+ BUG_ON(!mt_lock_is_held(mt)); -+ mt->ma_flags |= MT_FLAGS_USE_RCU; -+ } else { -+ mtree_lock(mt); -+ mt->ma_flags |= MT_FLAGS_USE_RCU; -+ mtree_unlock(mt); -+ } -+} -+ -+void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max); -+void *mt_find_after(struct maple_tree *mt, unsigned long *index, -+ unsigned long max); -+void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min); -+void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max); -+ -+/** -+ * mt_for_each - Iterate over each entry starting at index until max. -+ * @__tree: The Maple Tree -+ * @__entry: The current entry -+ * @__index: The index to update to track the location in the tree -+ * @__max: The maximum limit for @index -+ * -+ * Note: Will not return the zero entry. -+ */ -+#define mt_for_each(__tree, __entry, __index, __max) \ -+ for (__entry = mt_find(__tree, &(__index), __max); \ -+ __entry; __entry = mt_find_after(__tree, &(__index), __max)) -+ -+ -+#ifdef CONFIG_DEBUG_MAPLE_TREE -+extern atomic_t maple_tree_tests_run; -+extern atomic_t maple_tree_tests_passed; -+ -+void mt_dump(const struct maple_tree *mt); -+void mt_validate(struct maple_tree *mt); -+#define MT_BUG_ON(__tree, __x) do { \ -+ atomic_inc(&maple_tree_tests_run); \ -+ if (__x) { \ -+ pr_info("BUG at %s:%d (%u)\n", \ -+ __func__, __LINE__, __x); \ -+ mt_dump(__tree); \ -+ pr_info("Pass: %u Run:%u\n", \ -+ atomic_read(&maple_tree_tests_passed), \ -+ atomic_read(&maple_tree_tests_run)); \ -+ dump_stack(); \ -+ } else { \ -+ atomic_inc(&maple_tree_tests_passed); \ -+ } \ -+} while (0) -+#else -+#define MT_BUG_ON(__tree, __x) BUG_ON(__x) -+#endif /* CONFIG_DEBUG_MAPLE_TREE */ -+ -+#endif /*_LINUX_MAPLE_TREE_H */ -diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h -index 6257867fbf95..207cfd3b42e5 100644 ---- a/include/linux/memcontrol.h -+++ b/include/linux/memcontrol.h -@@ -350,6 +350,11 @@ struct mem_cgroup { - struct deferred_split deferred_split_queue; - #endif - -+#ifdef CONFIG_LRU_GEN -+ /* per-memcg mm_struct list */ -+ struct lru_gen_mm_list mm_list; -+#endif -+ - struct mem_cgroup_per_node *nodeinfo[]; - }; - -@@ -444,6 +449,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - * - * For a kmem folio a caller should hold an rcu read lock to protect memcg - * associated with a kmem folio from being released. -@@ -505,6 +511,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - * - * For a kmem page a caller should hold an rcu read lock to protect memcg - * associated with a kmem page from being released. -@@ -959,6 +966,23 @@ void unlock_page_memcg(struct page *page); - - void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); - -+/* try to stablize folio_memcg() for all the pages in a memcg */ -+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) -+{ -+ rcu_read_lock(); -+ -+ if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) -+ return true; -+ -+ rcu_read_unlock(); -+ return false; -+} -+ -+static inline void mem_cgroup_unlock_pages(void) -+{ -+ rcu_read_unlock(); -+} -+ - /* idx can be of type enum memcg_stat_item or node_stat_item */ - static inline void mod_memcg_state(struct mem_cgroup *memcg, - int idx, int val) -@@ -1433,6 +1457,18 @@ static inline void folio_memcg_unlock(struct folio *folio) - { - } - -+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) -+{ -+ /* to match folio_memcg_rcu() */ -+ rcu_read_lock(); -+ return true; -+} -+ -+static inline void mem_cgroup_unlock_pages(void) -+{ -+ rcu_read_unlock(); -+} -+ - static inline void mem_cgroup_handle_over_high(void) - { - } -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 21f8b27bd9fd..9ac0e02e2238 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -661,6 +661,38 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) - return vma->vm_flags & VM_ACCESS_FLAGS; - } - -+static inline -+struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) -+{ -+ return mas_find(&vmi->mas, max); -+} -+ -+static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) -+{ -+ /* -+ * Uses vma_find() to get the first VMA when the iterator starts. -+ * Calling mas_next() could skip the first entry. -+ */ -+ return vma_find(vmi, ULONG_MAX); -+} -+ -+static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) -+{ -+ return mas_prev(&vmi->mas, 0); -+} -+ -+static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) -+{ -+ return vmi->mas.index; -+} -+ -+#define for_each_vma(__vmi, __vma) \ -+ while (((__vma) = vma_next(&(__vmi))) != NULL) -+ -+/* The MM code likes to work with exclusive end addresses */ -+#define for_each_vma_range(__vmi, __vma, __end) \ -+ while (((__vma) = vma_find(&(__vmi), (__end) - 1)) != NULL) -+ - #ifdef CONFIG_SHMEM - /* - * The vma_is_shmem is not inline because it is used only by slow -@@ -1465,6 +1497,11 @@ static inline unsigned long folio_pfn(struct folio *folio) - return page_to_pfn(&folio->page); - } - -+static inline struct folio *pfn_folio(unsigned long pfn) -+{ -+ return page_folio(pfn_to_page(pfn)); -+} -+ - static inline atomic_t *folio_pincount_ptr(struct folio *folio) - { - return &folio_page(folio, 1)->compound_pincount; -@@ -1795,8 +1832,9 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, - unsigned long size); - void zap_page_range(struct vm_area_struct *vma, unsigned long address, - unsigned long size); --void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, -- unsigned long start, unsigned long end); -+void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, -+ struct vm_area_struct *start_vma, unsigned long start, -+ unsigned long end); - - struct mmu_notifier_range; - -@@ -2593,14 +2631,15 @@ extern int __split_vma(struct mm_struct *, struct vm_area_struct *, - extern int split_vma(struct mm_struct *, struct vm_area_struct *, - unsigned long addr, int new_below); - extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); --extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, -- struct rb_node **, struct rb_node *); - extern void unlink_file_vma(struct vm_area_struct *); - extern struct vm_area_struct *copy_vma(struct vm_area_struct **, - unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); - extern void exit_mmap(struct mm_struct *); - -+void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas); -+void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas); -+ - static inline int check_data_rlimit(unsigned long rlim, - unsigned long new, - unsigned long start, -@@ -2648,8 +2687,9 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, - extern unsigned long do_mmap(struct file *file, unsigned long addr, - unsigned long len, unsigned long prot, unsigned long flags, - unsigned long pgoff, unsigned long *populate, struct list_head *uf); --extern int __do_munmap(struct mm_struct *, unsigned long, size_t, -- struct list_head *uf, bool downgrade); -+extern int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, -+ unsigned long start, size_t len, struct list_head *uf, -+ bool downgrade); - extern int do_munmap(struct mm_struct *, unsigned long, size_t, - struct list_head *uf); - extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); -@@ -2716,26 +2756,12 @@ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long add - extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, - struct vm_area_struct **pprev); - --/** -- * find_vma_intersection() - Look up the first VMA which intersects the interval -- * @mm: The process address space. -- * @start_addr: The inclusive start user address. -- * @end_addr: The exclusive end user address. -- * -- * Returns: The first VMA within the provided range, %NULL otherwise. Assumes -- * start_addr < end_addr. -+/* -+ * Look up the first VMA which intersects the interval [start_addr, end_addr) -+ * NULL if none. Assume start_addr < end_addr. - */ --static inline - struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, -- unsigned long start_addr, -- unsigned long end_addr) --{ -- struct vm_area_struct *vma = find_vma(mm, start_addr); -- -- if (vma && end_addr <= vma->vm_start) -- vma = NULL; -- return vma; --} -+ unsigned long start_addr, unsigned long end_addr); - - /** - * vma_lookup() - Find a VMA at a specific address -@@ -2747,12 +2773,7 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, - static inline - struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) - { -- struct vm_area_struct *vma = find_vma(mm, addr); -- -- if (vma && addr < vma->vm_start) -- vma = NULL; -- -- return vma; -+ return mtree_load(&mm->mm_mt, addr); - } - - static inline unsigned long vm_start_gap(struct vm_area_struct *vma) -@@ -2788,7 +2809,7 @@ static inline unsigned long vma_pages(struct vm_area_struct *vma) - static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, - unsigned long vm_start, unsigned long vm_end) - { -- struct vm_area_struct *vma = find_vma(mm, vm_start); -+ struct vm_area_struct *vma = vma_lookup(mm, vm_start); - - if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) - vma = NULL; -diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h -index 7b25b53c474a..4949eda9a9a2 100644 ---- a/include/linux/mm_inline.h -+++ b/include/linux/mm_inline.h -@@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) - return folio_is_file_lru(page_folio(page)); - } - --static __always_inline void update_lru_size(struct lruvec *lruvec, -+static __always_inline void __update_lru_size(struct lruvec *lruvec, - enum lru_list lru, enum zone_type zid, - long nr_pages) - { - struct pglist_data *pgdat = lruvec_pgdat(lruvec); - -+ lockdep_assert_held(&lruvec->lru_lock); -+ WARN_ON_ONCE(nr_pages != (int)nr_pages); -+ - __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); - __mod_zone_page_state(&pgdat->node_zones[zid], - NR_ZONE_LRU_BASE + lru, nr_pages); -+} -+ -+static __always_inline void update_lru_size(struct lruvec *lruvec, -+ enum lru_list lru, enum zone_type zid, -+ long nr_pages) -+{ -+ __update_lru_size(lruvec, lru, zid, nr_pages); - #ifdef CONFIG_MEMCG - mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); - #endif -@@ -94,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) - return lru; - } - -+#ifdef CONFIG_LRU_GEN -+ -+#ifdef CONFIG_LRU_GEN_ENABLED -+static inline bool lru_gen_enabled(void) -+{ -+ DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); -+ -+ return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); -+} -+#else -+static inline bool lru_gen_enabled(void) -+{ -+ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); -+ -+ return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); -+} -+#endif -+ -+static inline bool lru_gen_in_fault(void) -+{ -+ return current->in_lru_fault; -+} -+ -+static inline int lru_gen_from_seq(unsigned long seq) -+{ -+ return seq % MAX_NR_GENS; -+} -+ -+static inline int lru_hist_from_seq(unsigned long seq) -+{ -+ return seq % NR_HIST_GENS; -+} -+ -+static inline int lru_tier_from_refs(int refs) -+{ -+ VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); -+ -+ /* see the comment in folio_lru_refs() */ -+ return order_base_2(refs + 1); -+} -+ -+static inline int folio_lru_refs(struct folio *folio) -+{ -+ unsigned long flags = READ_ONCE(folio->flags); -+ bool workingset = flags & BIT(PG_workingset); -+ -+ /* -+ * Return the number of accesses beyond PG_referenced, i.e., N-1 if the -+ * total number of accesses is N>1, since N=0,1 both map to the first -+ * tier. lru_tier_from_refs() will account for this off-by-one. Also see -+ * the comment on MAX_NR_TIERS. -+ */ -+ return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; -+} -+ -+static inline int folio_lru_gen(struct folio *folio) -+{ -+ unsigned long flags = READ_ONCE(folio->flags); -+ -+ return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+} -+ -+static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) -+{ -+ unsigned long max_seq = lruvec->lrugen.max_seq; -+ -+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); -+ -+ /* see the comment on MIN_NR_GENS */ -+ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); -+} -+ -+static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, -+ int old_gen, int new_gen) -+{ -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); -+ -+ if (old_gen >= 0) -+ WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], -+ lrugen->nr_pages[old_gen][type][zone] - delta); -+ if (new_gen >= 0) -+ WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], -+ lrugen->nr_pages[new_gen][type][zone] + delta); -+ -+ /* addition */ -+ if (old_gen < 0) { -+ if (lru_gen_is_active(lruvec, new_gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, delta); -+ return; -+ } -+ -+ /* deletion */ -+ if (new_gen < 0) { -+ if (lru_gen_is_active(lruvec, old_gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, -delta); -+ return; -+ } -+ -+ /* promotion */ -+ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { -+ __update_lru_size(lruvec, lru, zone, -delta); -+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); -+ } -+ -+ /* demotion requires isolation, e.g., lru_deactivate_fn() */ -+ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); -+} -+ -+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ unsigned long seq; -+ unsigned long flags; -+ int gen = folio_lru_gen(folio); -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); -+ -+ if (folio_test_unevictable(folio) || !lrugen->enabled) -+ return false; -+ /* -+ * There are three common cases for this page: -+ * 1. If it's hot, e.g., freshly faulted in or previously hot and -+ * migrated, add it to the youngest generation. -+ * 2. If it's cold but can't be evicted immediately, i.e., an anon page -+ * not in swapcache or a dirty page pending writeback, add it to the -+ * second oldest generation. -+ * 3. Everything else (clean, cold) is added to the oldest generation. -+ */ -+ if (folio_test_active(folio)) -+ seq = lrugen->max_seq; -+ else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || -+ (folio_test_reclaim(folio) && -+ (folio_test_dirty(folio) || folio_test_writeback(folio)))) -+ seq = lrugen->min_seq[type] + 1; -+ else -+ seq = lrugen->min_seq[type]; -+ -+ gen = lru_gen_from_seq(seq); -+ flags = (gen + 1UL) << LRU_GEN_PGOFF; -+ /* see the comment on MIN_NR_GENS about PG_active */ -+ set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); -+ -+ lru_gen_update_size(lruvec, folio, -1, gen); -+ /* for folio_rotate_reclaimable() */ -+ if (reclaiming) -+ list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); -+ else -+ list_add(&folio->lru, &lrugen->lists[gen][type][zone]); -+ -+ return true; -+} -+ -+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ unsigned long flags; -+ int gen = folio_lru_gen(folio); -+ -+ if (gen < 0) -+ return false; -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ -+ /* for folio_migrate_flags() */ -+ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; -+ flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); -+ gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+ -+ lru_gen_update_size(lruvec, folio, gen, -1); -+ list_del(&folio->lru); -+ -+ return true; -+} -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline bool lru_gen_enabled(void) -+{ -+ return false; -+} -+ -+static inline bool lru_gen_in_fault(void) -+{ -+ return false; -+} -+ -+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ return false; -+} -+ -+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ return false; -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ - static __always_inline - void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_add_folio(lruvec, folio, false)) -+ return; -+ - update_lru_size(lruvec, lru, folio_zonenum(folio), - folio_nr_pages(folio)); - if (lru != LRU_UNEVICTABLE) -@@ -116,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_add_folio(lruvec, folio, true)) -+ return; -+ - update_lru_size(lruvec, lru, folio_zonenum(folio), - folio_nr_pages(folio)); - /* This is not expected to be used on LRU_UNEVICTABLE */ -@@ -133,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_del_folio(lruvec, folio, false)) -+ return; -+ - if (lru != LRU_UNEVICTABLE) - list_del(&folio->lru); - update_lru_size(lruvec, lru, folio_zonenum(folio), -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index cf97f3884fda..5e32211cb5a9 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -9,6 +9,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -407,21 +408,6 @@ struct vm_area_struct { - unsigned long vm_end; /* The first byte after our end address - within vm_mm. */ - -- /* linked list of VM areas per task, sorted by address */ -- struct vm_area_struct *vm_next, *vm_prev; -- -- struct rb_node vm_rb; -- -- /* -- * Largest free memory gap in bytes to the left of this VMA. -- * Either between this VMA and vma->vm_prev, or between one of the -- * VMAs below us in the VMA rbtree and its ->vm_prev. This helps -- * get_unmapped_area find a free area of the right size. -- */ -- unsigned long rb_subtree_gap; -- -- /* Second cache line starts here. */ -- - struct mm_struct *vm_mm; /* The address space we belong to. */ - - /* -@@ -485,9 +471,7 @@ struct vm_area_struct { - struct kioctx_table; - struct mm_struct { - struct { -- struct vm_area_struct *mmap; /* list of VMAs */ -- struct rb_root mm_rb; -- u64 vmacache_seqnum; /* per-thread vmacache */ -+ struct maple_tree mm_mt; - #ifdef CONFIG_MMU - unsigned long (*get_unmapped_area) (struct file *filp, - unsigned long addr, unsigned long len, -@@ -501,7 +485,6 @@ struct mm_struct { - unsigned long mmap_compat_legacy_base; - #endif - unsigned long task_size; /* size of task vm space */ -- unsigned long highest_vm_end; /* highest vma end address */ - pgd_t * pgd; - - #ifdef CONFIG_MEMBARRIER -@@ -672,6 +655,22 @@ struct mm_struct { - */ - unsigned long ksm_merging_pages; - #endif -+#ifdef CONFIG_LRU_GEN -+ struct { -+ /* this mm_struct is on lru_gen_mm_list */ -+ struct list_head list; -+ /* -+ * Set when switching to this mm_struct, as a hint of -+ * whether it has been used since the last time per-node -+ * page table walkers cleared the corresponding bits. -+ */ -+ unsigned long bitmap; -+#ifdef CONFIG_MEMCG -+ /* points to the memcg of "owner" above */ -+ struct mem_cgroup *memcg; -+#endif -+ } lru_gen; -+#endif /* CONFIG_LRU_GEN */ - } __randomize_layout; - - /* -@@ -681,6 +680,7 @@ struct mm_struct { - unsigned long cpu_bitmap[]; - }; - -+#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN) - extern struct mm_struct init_mm; - - /* Pointer magic because the dynamic array size confuses some compilers. */ -@@ -698,6 +698,87 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) - return (struct cpumask *)&mm->cpu_bitmap; - } - -+#ifdef CONFIG_LRU_GEN -+ -+struct lru_gen_mm_list { -+ /* mm_struct list for page table walkers */ -+ struct list_head fifo; -+ /* protects the list above */ -+ spinlock_t lock; -+}; -+ -+void lru_gen_add_mm(struct mm_struct *mm); -+void lru_gen_del_mm(struct mm_struct *mm); -+#ifdef CONFIG_MEMCG -+void lru_gen_migrate_mm(struct mm_struct *mm); -+#endif -+ -+static inline void lru_gen_init_mm(struct mm_struct *mm) -+{ -+ INIT_LIST_HEAD(&mm->lru_gen.list); -+ mm->lru_gen.bitmap = 0; -+#ifdef CONFIG_MEMCG -+ mm->lru_gen.memcg = NULL; -+#endif -+} -+ -+static inline void lru_gen_use_mm(struct mm_struct *mm) -+{ -+ /* -+ * When the bitmap is set, page reclaim knows this mm_struct has been -+ * used since the last time it cleared the bitmap. So it might be worth -+ * walking the page tables of this mm_struct to clear the accessed bit. -+ */ -+ WRITE_ONCE(mm->lru_gen.bitmap, -1); -+} -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline void lru_gen_add_mm(struct mm_struct *mm) -+{ -+} -+ -+static inline void lru_gen_del_mm(struct mm_struct *mm) -+{ -+} -+ -+#ifdef CONFIG_MEMCG -+static inline void lru_gen_migrate_mm(struct mm_struct *mm) -+{ -+} -+#endif -+ -+static inline void lru_gen_init_mm(struct mm_struct *mm) -+{ -+} -+ -+static inline void lru_gen_use_mm(struct mm_struct *mm) -+{ -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ -+struct vma_iterator { -+ struct ma_state mas; -+}; -+ -+#define VMA_ITERATOR(name, __mm, __addr) \ -+ struct vma_iterator name = { \ -+ .mas = { \ -+ .tree = &(__mm)->mm_mt, \ -+ .index = __addr, \ -+ .node = MAS_START, \ -+ }, \ -+ } -+ -+static inline void vma_iter_init(struct vma_iterator *vmi, -+ struct mm_struct *mm, unsigned long addr) -+{ -+ vmi->mas.tree = &mm->mm_mt; -+ vmi->mas.index = addr; -+ vmi->mas.node = MAS_START; -+} -+ - struct mmu_gather; - extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); - extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); -diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h -index c1bc6731125c..0bb4b6da9993 100644 ---- a/include/linux/mm_types_task.h -+++ b/include/linux/mm_types_task.h -@@ -24,18 +24,6 @@ - IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) - #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) - --/* -- * The per task VMA cache array: -- */ --#define VMACACHE_BITS 2 --#define VMACACHE_SIZE (1U << VMACACHE_BITS) --#define VMACACHE_MASK (VMACACHE_SIZE - 1) -- --struct vmacache { -- u64 seqnum; -- struct vm_area_struct *vmas[VMACACHE_SIZE]; --}; -- - /* - * When updating this, please also update struct resident_page_types[] in - * kernel/fork.c -diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index e24b40c52468..1543001feba9 100644 ---- a/include/linux/mmzone.h -+++ b/include/linux/mmzone.h -@@ -306,6 +306,8 @@ static inline bool is_active_lru(enum lru_list lru) - return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); - } - -+#define WORKINGSET_ANON 0 -+#define WORKINGSET_FILE 1 - #define ANON_AND_FILE 2 - - enum lruvec_flags { -@@ -314,6 +316,207 @@ enum lruvec_flags { - */ - }; - -+#endif /* !__GENERATING_BOUNDS_H */ -+ -+/* -+ * Evictable pages are divided into multiple generations. The youngest and the -+ * oldest generation numbers, max_seq and min_seq, are monotonically increasing. -+ * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An -+ * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the -+ * corresponding generation. The gen counter in folio->flags stores gen+1 while -+ * a page is on one of lrugen->lists[]. Otherwise it stores 0. -+ * -+ * A page is added to the youngest generation on faulting. The aging needs to -+ * check the accessed bit at least twice before handing this page over to the -+ * eviction. The first check takes care of the accessed bit set on the initial -+ * fault; the second check makes sure this page hasn't been used since then. -+ * This process, AKA second chance, requires a minimum of two generations, -+ * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive -+ * LRU, e.g., /proc/vmstat, these two generations are considered active; the -+ * rest of generations, if they exist, are considered inactive. See -+ * lru_gen_is_active(). -+ * -+ * PG_active is always cleared while a page is on one of lrugen->lists[] so that -+ * the aging needs not to worry about it. And it's set again when a page -+ * considered active is isolated for non-reclaiming purposes, e.g., migration. -+ * See lru_gen_add_folio() and lru_gen_del_folio(). -+ * -+ * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the -+ * number of categories of the active/inactive LRU when keeping track of -+ * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits -+ * in folio->flags. -+ */ -+#define MIN_NR_GENS 2U -+#define MAX_NR_GENS 4U -+ -+/* -+ * Each generation is divided into multiple tiers. A page accessed N times -+ * through file descriptors is in tier order_base_2(N). A page in the first tier -+ * (N=0,1) is marked by PG_referenced unless it was faulted in through page -+ * tables or read ahead. A page in any other tier (N>1) is marked by -+ * PG_referenced and PG_workingset. This implies a minimum of two tiers is -+ * supported without using additional bits in folio->flags. -+ * -+ * In contrast to moving across generations which requires the LRU lock, moving -+ * across tiers only involves atomic operations on folio->flags and therefore -+ * has a negligible cost in the buffered access path. In the eviction path, -+ * comparisons of refaulted/(evicted+protected) from the first tier and the -+ * rest infer whether pages accessed multiple times through file descriptors -+ * are statistically hot and thus worth protecting. -+ * -+ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the -+ * number of categories of the active/inactive LRU when keeping track of -+ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in -+ * folio->flags. -+ */ -+#define MAX_NR_TIERS 4U -+ -+#ifndef __GENERATING_BOUNDS_H -+ -+struct lruvec; -+struct page_vma_mapped_walk; -+ -+#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) -+#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) -+ -+#ifdef CONFIG_LRU_GEN -+ -+enum { -+ LRU_GEN_ANON, -+ LRU_GEN_FILE, -+}; -+ -+enum { -+ LRU_GEN_CORE, -+ LRU_GEN_MM_WALK, -+ LRU_GEN_NONLEAF_YOUNG, -+ NR_LRU_GEN_CAPS -+}; -+ -+#define MIN_LRU_BATCH BITS_PER_LONG -+#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) -+ -+/* whether to keep historical stats from evicted generations */ -+#ifdef CONFIG_LRU_GEN_STATS -+#define NR_HIST_GENS MAX_NR_GENS -+#else -+#define NR_HIST_GENS 1U -+#endif -+ -+/* -+ * The youngest generation number is stored in max_seq for both anon and file -+ * types as they are aged on an equal footing. The oldest generation numbers are -+ * stored in min_seq[] separately for anon and file types as clean file pages -+ * can be evicted regardless of swap constraints. -+ * -+ * Normally anon and file min_seq are in sync. But if swapping is constrained, -+ * e.g., out of swap space, file min_seq is allowed to advance and leave anon -+ * min_seq behind. -+ * -+ * The number of pages in each generation is eventually consistent and therefore -+ * can be transiently negative when reset_batch_size() is pending. -+ */ -+struct lru_gen_struct { -+ /* the aging increments the youngest generation number */ -+ unsigned long max_seq; -+ /* the eviction increments the oldest generation numbers */ -+ unsigned long min_seq[ANON_AND_FILE]; -+ /* the birth time of each generation in jiffies */ -+ unsigned long timestamps[MAX_NR_GENS]; -+ /* the multi-gen LRU lists, lazily sorted on eviction */ -+ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* the multi-gen LRU sizes, eventually consistent */ -+ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* the exponential moving average of refaulted */ -+ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; -+ /* the exponential moving average of evicted+protected */ -+ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; -+ /* the first tier doesn't need protection, hence the minus one */ -+ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; -+ /* can be modified without holding the LRU lock */ -+ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; -+ atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; -+ /* whether the multi-gen LRU is enabled */ -+ bool enabled; -+}; -+ -+enum { -+ MM_LEAF_TOTAL, /* total leaf entries */ -+ MM_LEAF_OLD, /* old leaf entries */ -+ MM_LEAF_YOUNG, /* young leaf entries */ -+ MM_NONLEAF_TOTAL, /* total non-leaf entries */ -+ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ -+ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ -+ NR_MM_STATS -+}; -+ -+/* double-buffering Bloom filters */ -+#define NR_BLOOM_FILTERS 2 -+ -+struct lru_gen_mm_state { -+ /* set to max_seq after each iteration */ -+ unsigned long seq; -+ /* where the current iteration continues (inclusive) */ -+ struct list_head *head; -+ /* where the last iteration ended (exclusive) */ -+ struct list_head *tail; -+ /* to wait for the last page table walker to finish */ -+ struct wait_queue_head wait; -+ /* Bloom filters flip after each iteration */ -+ unsigned long *filters[NR_BLOOM_FILTERS]; -+ /* the mm stats for debugging */ -+ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; -+ /* the number of concurrent page table walkers */ -+ int nr_walkers; -+}; -+ -+struct lru_gen_mm_walk { -+ /* the lruvec under reclaim */ -+ struct lruvec *lruvec; -+ /* unstable max_seq from lru_gen_struct */ -+ unsigned long max_seq; -+ /* the next address within an mm to scan */ -+ unsigned long next_addr; -+ /* to batch promoted pages */ -+ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* to batch the mm stats */ -+ int mm_stats[NR_MM_STATS]; -+ /* total batched items */ -+ int batched; -+ bool can_swap; -+ bool force_scan; -+}; -+ -+void lru_gen_init_lruvec(struct lruvec *lruvec); -+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); -+ -+#ifdef CONFIG_MEMCG -+void lru_gen_init_memcg(struct mem_cgroup *memcg); -+void lru_gen_exit_memcg(struct mem_cgroup *memcg); -+#endif -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline void lru_gen_init_lruvec(struct lruvec *lruvec) -+{ -+} -+ -+static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) -+{ -+} -+ -+#ifdef CONFIG_MEMCG -+static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) -+{ -+} -+ -+static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) -+{ -+} -+#endif -+ -+#endif /* CONFIG_LRU_GEN */ -+ - struct lruvec { - struct list_head lists[NR_LRU_LISTS]; - /* per lruvec lru_lock for memcg */ -@@ -331,6 +534,12 @@ struct lruvec { - unsigned long refaults[ANON_AND_FILE]; - /* Various lruvec state flags (enum lruvec_flags) */ - unsigned long flags; -+#ifdef CONFIG_LRU_GEN -+ /* evictable pages divided into generations */ -+ struct lru_gen_struct lrugen; -+ /* to concurrently iterate lru_gen_mm_list */ -+ struct lru_gen_mm_state mm_state; -+#endif - #ifdef CONFIG_MEMCG - struct pglist_data *pgdat; - #endif -@@ -746,6 +955,8 @@ static inline bool zone_is_empty(struct zone *zone) - #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) - #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) - #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) -+#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) -+#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) - - /* - * Define the bit shifts to access each section. For non-existent -@@ -1007,6 +1218,11 @@ typedef struct pglist_data { - - unsigned long flags; - -+#ifdef CONFIG_LRU_GEN -+ /* kswap mm walk data */ -+ struct lru_gen_mm_walk mm_walk; -+#endif -+ - ZONE_PADDING(_pad2_) - - /* Per-node vmstats */ -diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h -index 4b71a96190a8..3a0eec9f2faa 100644 ---- a/include/linux/nodemask.h -+++ b/include/linux/nodemask.h -@@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) - #define first_online_node 0 - #define first_memory_node 0 - #define next_online_node(nid) (MAX_NUMNODES) -+#define next_memory_node(nid) (MAX_NUMNODES) - #define nr_node_ids 1U - #define nr_online_nodes 1U - -diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h -index ef1e3e736e14..7d79818dc065 100644 ---- a/include/linux/page-flags-layout.h -+++ b/include/linux/page-flags-layout.h -@@ -55,7 +55,8 @@ - #define SECTIONS_WIDTH 0 - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ -+ <= BITS_PER_LONG - NR_PAGEFLAGS - #define NODES_WIDTH NODES_SHIFT - #elif defined(CONFIG_SPARSEMEM_VMEMMAP) - #error "Vmemmap: No space for nodes field in page flags" -@@ -89,8 +90,8 @@ - #define LAST_CPUPID_SHIFT 0 - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ -- <= BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ -+ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS - #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT - #else - #define LAST_CPUPID_WIDTH 0 -@@ -100,10 +101,15 @@ - #define LAST_CPUPID_NOT_IN_PAGE_FLAGS - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ -- > BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ -+ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS - #error "Not enough bits in page flags" - #endif - -+/* see the comment on MAX_NR_TIERS */ -+#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ -+ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ -+ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) -+ - #endif - #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ -diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h -index 465ff35a8c00..0b0ae5084e60 100644 ---- a/include/linux/page-flags.h -+++ b/include/linux/page-flags.h -@@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) - 1UL << PG_private | 1UL << PG_private_2 | \ - 1UL << PG_writeback | 1UL << PG_reserved | \ - 1UL << PG_slab | 1UL << PG_active | \ -- 1UL << PG_unevictable | __PG_MLOCKED) -+ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) - - /* - * Flags checked when a page is prepped for return by the page allocator. -@@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) - * alloc-free cycle to prevent from reusing the page. - */ - #define PAGE_FLAGS_CHECK_AT_PREP \ -- (PAGEFLAGS_MASK & ~__PG_HWPOISON) -+ ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) - - #define PAGE_FLAGS_PRIVATE \ - (1UL << PG_private | 1UL << PG_private_2) -diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h -index 014ee8f0fbaa..d9095251bffd 100644 ---- a/include/linux/pgtable.h -+++ b/include/linux/pgtable.h -@@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - #endif - - #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG --#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) - static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) -@@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - BUILD_BUG(); - return 0; - } --#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -+#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ - #endif - - #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -@@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, - #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - #endif - -+#ifndef arch_has_hw_pte_young -+/* -+ * Return whether the accessed bit is supported on the local CPU. -+ * -+ * This stub assumes accessing through an old PTE triggers a page fault. -+ * Architectures that automatically set the access bit should overwrite it. -+ */ -+static inline bool arch_has_hw_pte_young(void) -+{ -+ return false; -+} -+#endif -+ - #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR - static inline pte_t ptep_get_and_clear(struct mm_struct *mm, - unsigned long address, -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 367b30ed77cb..2cc9429d3585 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -861,7 +861,6 @@ struct task_struct { - struct mm_struct *active_mm; - - /* Per-thread vma caching: */ -- struct vmacache vmacache; - - #ifdef SPLIT_RSS_COUNTING - struct task_rss_stat rss_stat; -@@ -914,6 +913,10 @@ struct task_struct { - #ifdef CONFIG_MEMCG - unsigned in_user_fault:1; - #endif -+#ifdef CONFIG_LRU_GEN -+ /* whether the LRU algorithm may apply to this access */ -+ unsigned in_lru_fault:1; -+#endif - #ifdef CONFIG_COMPAT_BRK - unsigned brk_randomized:1; - #endif -diff --git a/include/linux/swap.h b/include/linux/swap.h -index 43150b9bbc5c..6308150b234a 100644 ---- a/include/linux/swap.h -+++ b/include/linux/swap.h -@@ -162,6 +162,10 @@ union swap_header { - */ - struct reclaim_state { - unsigned long reclaimed_slab; -+#ifdef CONFIG_LRU_GEN -+ /* per-thread mm walk data */ -+ struct lru_gen_mm_walk *mm_walk; -+#endif - }; - - #ifdef __KERNEL__ -diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h -index e1b8a915e9e9..f07e6998bb68 100644 ---- a/include/linux/userfaultfd_k.h -+++ b/include/linux/userfaultfd_k.h -@@ -175,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma, - unsigned long start, - unsigned long end); - --extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, -- unsigned long start, unsigned long end, -- struct list_head *uf); -+extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, -+ unsigned long end, struct list_head *uf); - extern void userfaultfd_unmap_complete(struct mm_struct *mm, - struct list_head *uf); - -@@ -258,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma, - return true; - } - --static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, -+static inline int userfaultfd_unmap_prep(struct mm_struct *mm, - unsigned long start, unsigned long end, - struct list_head *uf) - { -diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h -index f3fc36cd2276..3518dba1e02f 100644 ---- a/include/linux/vm_event_item.h -+++ b/include/linux/vm_event_item.h -@@ -129,10 +129,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, - NR_TLB_LOCAL_FLUSH_ALL, - NR_TLB_LOCAL_FLUSH_ONE, - #endif /* CONFIG_DEBUG_TLBFLUSH */ --#ifdef CONFIG_DEBUG_VM_VMACACHE -- VMACACHE_FIND_CALLS, -- VMACACHE_FIND_HITS, --#endif - #ifdef CONFIG_SWAP - SWAP_RA, - SWAP_RA_HIT, -diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h -deleted file mode 100644 -index 6fce268a4588..000000000000 ---- a/include/linux/vmacache.h -+++ /dev/null -@@ -1,28 +0,0 @@ --/* SPDX-License-Identifier: GPL-2.0 */ --#ifndef __LINUX_VMACACHE_H --#define __LINUX_VMACACHE_H -- --#include --#include -- --static inline void vmacache_flush(struct task_struct *tsk) --{ -- memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas)); --} -- --extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma); --extern struct vm_area_struct *vmacache_find(struct mm_struct *mm, -- unsigned long addr); -- --#ifndef CONFIG_MMU --extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, -- unsigned long start, -- unsigned long end); --#endif -- --static inline void vmacache_invalidate(struct mm_struct *mm) --{ -- mm->vmacache_seqnum++; --} -- --#endif /* __LINUX_VMACACHE_H */ -diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h -index bfe38869498d..19cf5b6892ce 100644 ---- a/include/linux/vmstat.h -+++ b/include/linux/vmstat.h -@@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu) - #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) - #endif - --#ifdef CONFIG_DEBUG_VM_VMACACHE --#define count_vm_vmacache_event(x) count_vm_event(x) --#else --#define count_vm_vmacache_event(x) do {} while (0) --#endif -- - #define __count_zid_vm_events(item, zid, delta) \ - __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) - -diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h -index d651f3437367..55392bf30a03 100644 ---- a/include/trace/events/huge_memory.h -+++ b/include/trace/events/huge_memory.h -@@ -11,6 +11,7 @@ - EM( SCAN_FAIL, "failed") \ - EM( SCAN_SUCCEED, "succeeded") \ - EM( SCAN_PMD_NULL, "pmd_null") \ -+ EM( SCAN_PMD_MAPPED, "page_pmd_mapped") \ - EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \ - EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ - EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ -diff --git a/include/trace/events/maple_tree.h b/include/trace/events/maple_tree.h -new file mode 100644 -index 000000000000..2be403bdc2bd ---- /dev/null -+++ b/include/trace/events/maple_tree.h -@@ -0,0 +1,123 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM maple_tree -+ -+#if !defined(_TRACE_MM_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_MM_H -+ -+ -+#include -+ -+struct ma_state; -+ -+TRACE_EVENT(ma_op, -+ -+ TP_PROTO(const char *fn, struct ma_state *mas), -+ -+ TP_ARGS(fn, mas), -+ -+ TP_STRUCT__entry( -+ __field(const char *, fn) -+ __field(unsigned long, min) -+ __field(unsigned long, max) -+ __field(unsigned long, index) -+ __field(unsigned long, last) -+ __field(void *, node) -+ ), -+ -+ TP_fast_assign( -+ __entry->fn = fn; -+ __entry->min = mas->min; -+ __entry->max = mas->max; -+ __entry->index = mas->index; -+ __entry->last = mas->last; -+ __entry->node = mas->node; -+ ), -+ -+ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", -+ __entry->fn, -+ (void *) __entry->node, -+ (unsigned long) __entry->min, -+ (unsigned long) __entry->max, -+ (unsigned long) __entry->index, -+ (unsigned long) __entry->last -+ ) -+) -+TRACE_EVENT(ma_read, -+ -+ TP_PROTO(const char *fn, struct ma_state *mas), -+ -+ TP_ARGS(fn, mas), -+ -+ TP_STRUCT__entry( -+ __field(const char *, fn) -+ __field(unsigned long, min) -+ __field(unsigned long, max) -+ __field(unsigned long, index) -+ __field(unsigned long, last) -+ __field(void *, node) -+ ), -+ -+ TP_fast_assign( -+ __entry->fn = fn; -+ __entry->min = mas->min; -+ __entry->max = mas->max; -+ __entry->index = mas->index; -+ __entry->last = mas->last; -+ __entry->node = mas->node; -+ ), -+ -+ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", -+ __entry->fn, -+ (void *) __entry->node, -+ (unsigned long) __entry->min, -+ (unsigned long) __entry->max, -+ (unsigned long) __entry->index, -+ (unsigned long) __entry->last -+ ) -+) -+ -+TRACE_EVENT(ma_write, -+ -+ TP_PROTO(const char *fn, struct ma_state *mas, unsigned long piv, -+ void *val), -+ -+ TP_ARGS(fn, mas, piv, val), -+ -+ TP_STRUCT__entry( -+ __field(const char *, fn) -+ __field(unsigned long, min) -+ __field(unsigned long, max) -+ __field(unsigned long, index) -+ __field(unsigned long, last) -+ __field(unsigned long, piv) -+ __field(void *, val) -+ __field(void *, node) -+ ), -+ -+ TP_fast_assign( -+ __entry->fn = fn; -+ __entry->min = mas->min; -+ __entry->max = mas->max; -+ __entry->index = mas->index; -+ __entry->last = mas->last; -+ __entry->piv = piv; -+ __entry->val = val; -+ __entry->node = mas->node; -+ ), -+ -+ TP_printk("%s\tNode %p (%lu %lu) range:%lu-%lu piv (%lu) val %p", -+ __entry->fn, -+ (void *) __entry->node, -+ (unsigned long) __entry->min, -+ (unsigned long) __entry->max, -+ (unsigned long) __entry->index, -+ (unsigned long) __entry->last, -+ (unsigned long) __entry->piv, -+ (void *) __entry->val -+ ) -+) -+#endif /* _TRACE_MM_H */ -+ -+/* This part must be outside protection */ -+#include -diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h -index 4661f7ba07c0..216de5f03621 100644 ---- a/include/trace/events/mmap.h -+++ b/include/trace/events/mmap.h -@@ -42,6 +42,79 @@ TRACE_EVENT(vm_unmapped_area, - __entry->low_limit, __entry->high_limit, __entry->align_mask, - __entry->align_offset) - ); -+ -+TRACE_EVENT(vma_mas_szero, -+ TP_PROTO(struct maple_tree *mt, unsigned long start, -+ unsigned long end), -+ -+ TP_ARGS(mt, start, end), -+ -+ TP_STRUCT__entry( -+ __field(struct maple_tree *, mt) -+ __field(unsigned long, start) -+ __field(unsigned long, end) -+ ), -+ -+ TP_fast_assign( -+ __entry->mt = mt; -+ __entry->start = start; -+ __entry->end = end; -+ ), -+ -+ TP_printk("mt_mod %p, (NULL), SNULL, %lu, %lu,", -+ __entry->mt, -+ (unsigned long) __entry->start, -+ (unsigned long) __entry->end -+ ) -+); -+ -+TRACE_EVENT(vma_store, -+ TP_PROTO(struct maple_tree *mt, struct vm_area_struct *vma), -+ -+ TP_ARGS(mt, vma), -+ -+ TP_STRUCT__entry( -+ __field(struct maple_tree *, mt) -+ __field(struct vm_area_struct *, vma) -+ __field(unsigned long, vm_start) -+ __field(unsigned long, vm_end) -+ ), -+ -+ TP_fast_assign( -+ __entry->mt = mt; -+ __entry->vma = vma; -+ __entry->vm_start = vma->vm_start; -+ __entry->vm_end = vma->vm_end - 1; -+ ), -+ -+ TP_printk("mt_mod %p, (%p), STORE, %lu, %lu,", -+ __entry->mt, __entry->vma, -+ (unsigned long) __entry->vm_start, -+ (unsigned long) __entry->vm_end -+ ) -+); -+ -+ -+TRACE_EVENT(exit_mmap, -+ TP_PROTO(struct mm_struct *mm), -+ -+ TP_ARGS(mm), -+ -+ TP_STRUCT__entry( -+ __field(struct mm_struct *, mm) -+ __field(struct maple_tree *, mt) -+ ), -+ -+ TP_fast_assign( -+ __entry->mm = mm; -+ __entry->mt = &mm->mm_mt; -+ ), -+ -+ TP_printk("mt_mod %p, DESTROY\n", -+ __entry->mt -+ ) -+); -+ - #endif - - /* This part must be outside protection */ -diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h -index 6c1aa92a92e4..6ce1f1ceb432 100644 ---- a/include/uapi/asm-generic/mman-common.h -+++ b/include/uapi/asm-generic/mman-common.h -@@ -77,6 +77,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/init/main.c b/init/main.c -index 1fe7942f5d4a..df800fc61b2a 100644 ---- a/init/main.c -+++ b/init/main.c -@@ -117,6 +117,7 @@ static int kernel_init(void *); - - extern void init_IRQ(void); - extern void radix_tree_init(void); -+extern void maple_tree_init(void); - - /* - * Debug helper: via this flag we know that we are in 'early bootup code' -@@ -1002,6 +1003,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) - "Interrupts were enabled *very* early, fixing it\n")) - local_irq_disable(); - radix_tree_init(); -+ maple_tree_init(); - - /* - * Set up housekeeping before setting up workqueues to allow the unbound -diff --git a/ipc/shm.c b/ipc/shm.c -index b3048ebd5c31..7d86f058fb86 100644 ---- a/ipc/shm.c -+++ b/ipc/shm.c -@@ -1721,7 +1721,7 @@ long ksys_shmdt(char __user *shmaddr) - #ifdef CONFIG_MMU - loff_t size = 0; - struct file *file; -- struct vm_area_struct *next; -+ VMA_ITERATOR(vmi, mm, addr); - #endif - - if (addr & ~PAGE_MASK) -@@ -1751,12 +1751,9 @@ long ksys_shmdt(char __user *shmaddr) - * match the usual checks anyway. So assume all vma's are - * above the starting address given. - */ -- vma = find_vma(mm, addr); - - #ifdef CONFIG_MMU -- while (vma) { -- next = vma->vm_next; -- -+ for_each_vma(vmi, vma) { - /* - * Check if the starting address would match, i.e. it's - * a fragment created by mprotect() and/or munmap(), or it -@@ -1774,6 +1771,7 @@ long ksys_shmdt(char __user *shmaddr) - file = vma->vm_file; - size = i_size_read(file_inode(vma->vm_file)); - do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); -+ mas_pause(&vmi.mas); - /* - * We discovered the size of the shm segment, so - * break out of here and fall through to the next -@@ -1781,10 +1779,9 @@ long ksys_shmdt(char __user *shmaddr) - * searching for matching vma's. - */ - retval = 0; -- vma = next; -+ vma = vma_next(&vmi); - break; - } -- vma = next; - } - - /* -@@ -1794,17 +1791,19 @@ long ksys_shmdt(char __user *shmaddr) - */ - size = PAGE_ALIGN(size); - while (vma && (loff_t)(vma->vm_end - addr) <= size) { -- next = vma->vm_next; -- - /* finding a matching vma now does not alter retval */ - if ((vma->vm_ops == &shm_vm_ops) && - ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && -- (vma->vm_file == file)) -+ (vma->vm_file == file)) { - do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); -- vma = next; -+ mas_pause(&vmi.mas); -+ } -+ -+ vma = vma_next(&vmi); - } - - #else /* CONFIG_MMU */ -+ vma = vma_lookup(mm, addr); - /* under NOMMU conditions, the exact address to be destroyed must be - * given - */ -diff --git a/kernel/acct.c b/kernel/acct.c -index 13706356ec54..62200d799b9b 100644 ---- a/kernel/acct.c -+++ b/kernel/acct.c -@@ -555,15 +555,14 @@ void acct_collect(long exitcode, int group_dead) - unsigned long vsize = 0; - - if (group_dead && current->mm) { -+ struct mm_struct *mm = current->mm; -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - -- mmap_read_lock(current->mm); -- vma = current->mm->mmap; -- while (vma) { -+ mmap_read_lock(mm); -+ for_each_vma(vmi, vma) - vsize += vma->vm_end - vma->vm_start; -- vma = vma->vm_next; -- } -- mmap_read_unlock(current->mm); -+ mmap_read_unlock(mm); - } - - spin_lock_irq(¤t->sighand->siglock); -diff --git a/kernel/bounds.c b/kernel/bounds.c -index 9795d75b09b2..b529182e8b04 100644 ---- a/kernel/bounds.c -+++ b/kernel/bounds.c -@@ -22,6 +22,13 @@ int main(void) - DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); - #endif - DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); -+#ifdef CONFIG_LRU_GEN -+ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); -+ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); -+#else -+ DEFINE(LRU_GEN_WIDTH, 0); -+ DEFINE(__LRU_REFS_WIDTH, 0); -+#endif - /* End of constants */ - - return 0; -diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c -index 8c921799def4..1c8debd42dc9 100644 ---- a/kernel/bpf/task_iter.c -+++ b/kernel/bpf/task_iter.c -@@ -299,8 +299,8 @@ struct bpf_iter_seq_task_vma_info { - }; - - enum bpf_task_vma_iter_find_op { -- task_vma_iter_first_vma, /* use mm->mmap */ -- task_vma_iter_next_vma, /* use curr_vma->vm_next */ -+ task_vma_iter_first_vma, /* use find_vma() with addr 0 */ -+ task_vma_iter_next_vma, /* use vma_next() with curr_vma */ - task_vma_iter_find_vma, /* use find_vma() to find next vma */ - }; - -@@ -400,10 +400,10 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) - - switch (op) { - case task_vma_iter_first_vma: -- curr_vma = curr_task->mm->mmap; -+ curr_vma = find_vma(curr_task->mm, 0); - break; - case task_vma_iter_next_vma: -- curr_vma = curr_vma->vm_next; -+ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); - break; - case task_vma_iter_find_vma: - /* We dropped mmap_lock so it is necessary to use find_vma -@@ -417,7 +417,7 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) - if (curr_vma && - curr_vma->vm_start == info->prev_vm_start && - curr_vma->vm_end == info->prev_vm_end) -- curr_vma = curr_vma->vm_next; -+ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); - break; - } - if (!curr_vma) { -diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h -index 36b740cb3d59..63dc3e82be4f 100644 ---- a/kernel/cgroup/cgroup-internal.h -+++ b/kernel/cgroup/cgroup-internal.h -@@ -164,7 +164,6 @@ struct cgroup_mgctx { - #define DEFINE_CGROUP_MGCTX(name) \ - struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) - --extern struct mutex cgroup_mutex; - extern spinlock_t css_set_lock; - extern struct cgroup_subsys *cgroup_subsys[]; - extern struct list_head cgroup_roots; -diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c -index 7beceb447211..d5e9ccde3ab8 100644 ---- a/kernel/debug/debug_core.c -+++ b/kernel/debug/debug_core.c -@@ -50,7 +50,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -283,17 +282,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr) - if (!CACHE_FLUSH_IS_SAFE) - return; - -- if (current->mm) { -- int i; -- -- for (i = 0; i < VMACACHE_SIZE; i++) { -- if (!current->vmacache.vmas[i]) -- continue; -- flush_cache_range(current->vmacache.vmas[i], -- addr, addr + BREAK_INSTR_SIZE); -- } -- } -- - /* Force flush instruction cache if it was outside the mm */ - flush_icache_range(addr, addr + BREAK_INSTR_SIZE); - } -diff --git a/kernel/events/core.c b/kernel/events/core.c -index 2621fd24ad26..101c5912c3fc 100644 ---- a/kernel/events/core.c -+++ b/kernel/events/core.c -@@ -10229,8 +10229,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, - struct perf_addr_filter_range *fr) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (!vma->vm_file) - continue; - -diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c -index 2eaa327f8158..401bc2d24ce0 100644 ---- a/kernel/events/uprobes.c -+++ b/kernel/events/uprobes.c -@@ -349,9 +349,10 @@ static bool valid_ref_ctr_vma(struct uprobe *uprobe, - static struct vm_area_struct * - find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *tmp; - -- for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) -+ for_each_vma(vmi, tmp) - if (valid_ref_ctr_vma(uprobe, tmp)) - return tmp; - -@@ -1231,11 +1232,12 @@ int uprobe_apply(struct inode *inode, loff_t offset, - - static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - int err = 0; - - mmap_read_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - unsigned long vaddr; - loff_t offset; - -@@ -1983,9 +1985,10 @@ bool uprobe_deny_signal(void) - - static void mmf_recalc_uprobes(struct mm_struct *mm) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (!valid_vma(vma, false)) - continue; - /* -diff --git a/kernel/exit.c b/kernel/exit.c -index 84021b24f79e..98a33bd7c25c 100644 ---- a/kernel/exit.c -+++ b/kernel/exit.c -@@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm) - goto retry; - } - WRITE_ONCE(mm->owner, c); -+ lru_gen_migrate_mm(mm); - task_unlock(c); - put_task_struct(c); - } -diff --git a/kernel/fork.c b/kernel/fork.c -index 704fe6bc9cb4..1add76edca04 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -43,7 +43,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -479,7 +478,6 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) - */ - *new = data_race(*orig); - INIT_LIST_HEAD(&new->anon_vma_chain); -- new->vm_next = new->vm_prev = NULL; - dup_anon_vma_name(orig, new); - } - return new; -@@ -584,11 +582,12 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) - static __latent_entropy int dup_mmap(struct mm_struct *mm, - struct mm_struct *oldmm) - { -- struct vm_area_struct *mpnt, *tmp, *prev, **pprev; -- struct rb_node **rb_link, *rb_parent; -+ struct vm_area_struct *mpnt, *tmp; - int retval; -- unsigned long charge; -+ unsigned long charge = 0; - LIST_HEAD(uf); -+ MA_STATE(old_mas, &oldmm->mm_mt, 0, 0); -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - uprobe_start_dup_mmap(); - if (mmap_write_lock_killable(oldmm)) { -@@ -610,16 +609,16 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - mm->exec_vm = oldmm->exec_vm; - mm->stack_vm = oldmm->stack_vm; - -- rb_link = &mm->mm_rb.rb_node; -- rb_parent = NULL; -- pprev = &mm->mmap; - retval = ksm_fork(mm, oldmm); - if (retval) - goto out; - khugepaged_fork(mm, oldmm); - -- prev = NULL; -- for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { -+ retval = mas_expected_entries(&mas, oldmm->map_count); -+ if (retval) -+ goto out; -+ -+ mas_for_each(&old_mas, mpnt, ULONG_MAX) { - struct file *file; - - if (mpnt->vm_flags & VM_DONTCOPY) { -@@ -633,7 +632,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - */ - if (fatal_signal_pending(current)) { - retval = -EINTR; -- goto out; -+ goto loop_out; - } - if (mpnt->vm_flags & VM_ACCOUNT) { - unsigned long len = vma_pages(mpnt); -@@ -686,17 +685,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - if (is_vm_hugetlb_page(tmp)) - reset_vma_resv_huge_pages(tmp); - -- /* -- * Link in the new vma and copy the page table entries. -- */ -- *pprev = tmp; -- pprev = &tmp->vm_next; -- tmp->vm_prev = prev; -- prev = tmp; -- -- __vma_link_rb(mm, tmp, rb_link, rb_parent); -- rb_link = &tmp->vm_rb.rb_right; -- rb_parent = &tmp->vm_rb; -+ /* Link the vma into the MT */ -+ mas.index = tmp->vm_start; -+ mas.last = tmp->vm_end - 1; -+ mas_store(&mas, tmp); -+ if (mas_is_err(&mas)) -+ goto fail_nomem_mas_store; - - mm->map_count++; - if (!(tmp->vm_flags & VM_WIPEONFORK)) -@@ -706,10 +700,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - tmp->vm_ops->open(tmp); - - if (retval) -- goto out; -+ goto loop_out; - } - /* a new mm has just been created */ - retval = arch_dup_mmap(oldmm, mm); -+loop_out: -+ mas_destroy(&mas); - out: - mmap_write_unlock(mm); - flush_tlb_mm(oldmm); -@@ -718,6 +714,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - fail_uprobe_end: - uprobe_end_dup_mmap(); - return retval; -+ -+fail_nomem_mas_store: -+ unlink_anon_vmas(tmp); - fail_nomem_anon_vma_fork: - mpol_put(vma_policy(tmp)); - fail_nomem_policy: -@@ -725,7 +724,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - fail_nomem: - retval = -ENOMEM; - vm_unacct_memory(charge); -- goto out; -+ goto loop_out; - } - - static inline int mm_alloc_pgd(struct mm_struct *mm) -@@ -1113,9 +1112,8 @@ static void mm_init_uprobes_state(struct mm_struct *mm) - static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - struct user_namespace *user_ns) - { -- mm->mmap = NULL; -- mm->mm_rb = RB_ROOT; -- mm->vmacache_seqnum = 0; -+ mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); -+ mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); - atomic_set(&mm->mm_users, 1); - atomic_set(&mm->mm_count, 1); - seqcount_init(&mm->write_protect_seq); -@@ -1156,6 +1154,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - goto fail_nocontext; - - mm->user_ns = get_user_ns(user_ns); -+ lru_gen_init_mm(mm); - return mm; - - fail_nocontext: -@@ -1198,6 +1197,7 @@ static inline void __mmput(struct mm_struct *mm) - } - if (mm->binfmt) - module_put(mm->binfmt->module); -+ lru_gen_del_mm(mm); - mmdrop(mm); - } - -@@ -1289,13 +1289,16 @@ int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) - /* Forbid mm->exe_file change if old file still mapped. */ - old_exe_file = get_mm_exe_file(mm); - if (old_exe_file) { -+ VMA_ITERATOR(vmi, mm, 0); - mmap_read_lock(mm); -- for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (!vma->vm_file) - continue; - if (path_equal(&vma->vm_file->f_path, -- &old_exe_file->f_path)) -+ &old_exe_file->f_path)) { - ret = -EBUSY; -+ break; -+ } - } - mmap_read_unlock(mm); - fput(old_exe_file); -@@ -1571,9 +1574,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) - if (!oldmm) - return 0; - -- /* initialize the new vmacache entries */ -- vmacache_flush(tsk); -- - if (clone_flags & CLONE_VM) { - mmget(oldmm); - mm = oldmm; -@@ -2700,6 +2700,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) - get_task_struct(p); - } - -+ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { -+ /* lock the task to synchronize with memcg migration */ -+ task_lock(p); -+ lru_gen_add_mm(p->mm); -+ task_unlock(p); -+ } -+ - wake_up_new_task(p); - - /* forking complete and child started to run, tell ptracer */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index f2534e712a89..265944f14948 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -5178,6 +5178,7 @@ context_switch(struct rq *rq, struct task_struct *prev, - * finish_task_switch()'s mmdrop(). - */ - switch_mm_irqs_off(prev->active_mm, next->mm, next); -+ lru_gen_use_mm(next->mm); - - if (!prev->mm) { // from kernel - /* will mmdrop() in finish_task_switch(). */ -diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 74d773040b54..b544cd26794a 100644 ---- a/kernel/sched/fair.c -+++ b/kernel/sched/fair.c -@@ -2783,6 +2783,7 @@ static void task_numa_work(struct callback_head *work) - struct task_struct *p = current; - struct mm_struct *mm = p->mm; - u64 runtime = p->se.sum_exec_runtime; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - struct vm_area_struct *vma; - unsigned long start, end; - unsigned long nr_pte_updates = 0; -@@ -2839,13 +2840,16 @@ static void task_numa_work(struct callback_head *work) - - if (!mmap_read_trylock(mm)) - return; -- vma = find_vma(mm, start); -+ mas_set(&mas, start); -+ vma = mas_find(&mas, ULONG_MAX); - if (!vma) { - reset_ptenuma_scan(p); - start = 0; -- vma = mm->mmap; -+ mas_set(&mas, start); -+ vma = mas_find(&mas, ULONG_MAX); - } -- for (; vma; vma = vma->vm_next) { -+ -+ for (; vma; vma = mas_find(&mas, ULONG_MAX)) { - if (!vma_migratable(vma) || !vma_policy_mof(vma) || - is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { - continue; -diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug -index d3e5f36bb01e..72cef567aae6 100644 ---- a/lib/Kconfig.debug -+++ b/lib/Kconfig.debug -@@ -814,13 +814,12 @@ config DEBUG_VM - - If unsure, say N. - --config DEBUG_VM_VMACACHE -- bool "Debug VMA caching" -+config DEBUG_VM_MAPLE_TREE -+ bool "Debug VM maple trees" - depends on DEBUG_VM -+ select DEBUG_MAPLE_TREE - help -- Enable this to turn on VMA caching debug information. Doing so -- can cause significant overhead, so only enable it in non-production -- environments. -+ Enable VM maple tree debugging information and extra validations. - - If unsure, say N. - -@@ -1637,6 +1636,14 @@ config BUG_ON_DATA_CORRUPTION - - If unsure, say N. - -+config DEBUG_MAPLE_TREE -+ bool "Debug maple trees" -+ depends on DEBUG_KERNEL -+ help -+ Enable maple tree debugging information and extra validations. -+ -+ If unsure, say N. -+ - endmenu - - config DEBUG_CREDENTIALS -diff --git a/lib/Makefile b/lib/Makefile -index ffabc30a27d4..6dc0d6f8e57d 100644 ---- a/lib/Makefile -+++ b/lib/Makefile -@@ -29,7 +29,7 @@ endif - - lib-y := ctype.o string.o vsprintf.o cmdline.o \ - rbtree.o radix-tree.o timerqueue.o xarray.o \ -- idr.o extable.o irq_regs.o argv_split.o \ -+ maple_tree.o idr.o extable.o irq_regs.o argv_split.o \ - flex_proportions.o ratelimit.o show_mem.o \ - is_single_threaded.o plist.o decompress.o kobject_uevent.o \ - earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ -diff --git a/lib/maple_tree.c b/lib/maple_tree.c -new file mode 100644 -index 000000000000..e1743803c851 ---- /dev/null -+++ b/lib/maple_tree.c -@@ -0,0 +1,7130 @@ -+// SPDX-License-Identifier: GPL-2.0+ -+/* -+ * Maple Tree implementation -+ * Copyright (c) 2018-2022 Oracle Corporation -+ * Authors: Liam R. Howlett -+ * Matthew Wilcox -+ */ -+ -+/* -+ * DOC: Interesting implementation details of the Maple Tree -+ * -+ * Each node type has a number of slots for entries and a number of slots for -+ * pivots. In the case of dense nodes, the pivots are implied by the position -+ * and are simply the slot index + the minimum of the node. -+ * -+ * In regular B-Tree terms, pivots are called keys. The term pivot is used to -+ * indicate that the tree is specifying ranges, Pivots may appear in the -+ * subtree with an entry attached to the value where as keys are unique to a -+ * specific position of a B-tree. Pivot values are inclusive of the slot with -+ * the same index. -+ * -+ * -+ * The following illustrates the layout of a range64 nodes slots and pivots. -+ * -+ * -+ * Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 | -+ * ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ -+ * │ │ │ │ │ │ │ │ └─ Implied maximum -+ * │ │ │ │ │ │ │ └─ Pivot 14 -+ * │ │ │ │ │ │ └─ Pivot 13 -+ * │ │ │ │ │ └─ Pivot 12 -+ * │ │ │ │ └─ Pivot 11 -+ * │ │ │ └─ Pivot 2 -+ * │ │ └─ Pivot 1 -+ * │ └─ Pivot 0 -+ * └─ Implied minimum -+ * -+ * Slot contents: -+ * Internal (non-leaf) nodes contain pointers to other nodes. -+ * Leaf nodes contain entries. -+ * -+ * The location of interest is often referred to as an offset. All offsets have -+ * a slot, but the last offset has an implied pivot from the node above (or -+ * UINT_MAX for the root node. -+ * -+ * Ranges complicate certain write activities. When modifying any of -+ * the B-tree variants, it is known that one entry will either be added or -+ * deleted. When modifying the Maple Tree, one store operation may overwrite -+ * the entire data set, or one half of the tree, or the middle half of the tree. -+ * -+ */ -+ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define CREATE_TRACE_POINTS -+#include -+ -+#define MA_ROOT_PARENT 1 -+ -+/* -+ * Maple state flags -+ * * MA_STATE_BULK - Bulk insert mode -+ * * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert -+ * * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation -+ */ -+#define MA_STATE_BULK 1 -+#define MA_STATE_REBALANCE 2 -+#define MA_STATE_PREALLOC 4 -+ -+#define ma_parent_ptr(x) ((struct maple_pnode *)(x)) -+#define ma_mnode_ptr(x) ((struct maple_node *)(x)) -+#define ma_enode_ptr(x) ((struct maple_enode *)(x)) -+static struct kmem_cache *maple_node_cache; -+ -+#ifdef CONFIG_DEBUG_MAPLE_TREE -+static const unsigned long mt_max[] = { -+ [maple_dense] = MAPLE_NODE_SLOTS, -+ [maple_leaf_64] = ULONG_MAX, -+ [maple_range_64] = ULONG_MAX, -+ [maple_arange_64] = ULONG_MAX, -+}; -+#define mt_node_max(x) mt_max[mte_node_type(x)] -+#endif -+ -+static const unsigned char mt_slots[] = { -+ [maple_dense] = MAPLE_NODE_SLOTS, -+ [maple_leaf_64] = MAPLE_RANGE64_SLOTS, -+ [maple_range_64] = MAPLE_RANGE64_SLOTS, -+ [maple_arange_64] = MAPLE_ARANGE64_SLOTS, -+}; -+#define mt_slot_count(x) mt_slots[mte_node_type(x)] -+ -+static const unsigned char mt_pivots[] = { -+ [maple_dense] = 0, -+ [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1, -+ [maple_range_64] = MAPLE_RANGE64_SLOTS - 1, -+ [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1, -+}; -+#define mt_pivot_count(x) mt_pivots[mte_node_type(x)] -+ -+static const unsigned char mt_min_slots[] = { -+ [maple_dense] = MAPLE_NODE_SLOTS / 2, -+ [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, -+ [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, -+ [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1, -+}; -+#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] -+ -+#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) -+#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) -+ -+struct maple_big_node { -+ struct maple_pnode *parent; -+ unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; -+ union { -+ struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; -+ struct { -+ unsigned long padding[MAPLE_BIG_NODE_GAPS]; -+ unsigned long gap[MAPLE_BIG_NODE_GAPS]; -+ }; -+ }; -+ unsigned char b_end; -+ enum maple_type type; -+}; -+ -+/* -+ * The maple_subtree_state is used to build a tree to replace a segment of an -+ * existing tree in a more atomic way. Any walkers of the older tree will hit a -+ * dead node and restart on updates. -+ */ -+struct maple_subtree_state { -+ struct ma_state *orig_l; /* Original left side of subtree */ -+ struct ma_state *orig_r; /* Original right side of subtree */ -+ struct ma_state *l; /* New left side of subtree */ -+ struct ma_state *m; /* New middle of subtree (rare) */ -+ struct ma_state *r; /* New right side of subtree */ -+ struct ma_topiary *free; /* nodes to be freed */ -+ struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ -+ struct maple_big_node *bn; -+}; -+ -+/* Functions */ -+static inline struct maple_node *mt_alloc_one(gfp_t gfp) -+{ -+ return kmem_cache_alloc(maple_node_cache, gfp | __GFP_ZERO); -+} -+ -+static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes) -+{ -+ return kmem_cache_alloc_bulk(maple_node_cache, gfp | __GFP_ZERO, size, -+ nodes); -+} -+ -+static inline void mt_free_bulk(size_t size, void __rcu **nodes) -+{ -+ kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes); -+} -+ -+static void mt_free_rcu(struct rcu_head *head) -+{ -+ struct maple_node *node = container_of(head, struct maple_node, rcu); -+ -+ kmem_cache_free(maple_node_cache, node); -+} -+ -+/* -+ * ma_free_rcu() - Use rcu callback to free a maple node -+ * @node: The node to free -+ * -+ * The maple tree uses the parent pointer to indicate this node is no longer in -+ * use and will be freed. -+ */ -+static void ma_free_rcu(struct maple_node *node) -+{ -+ node->parent = ma_parent_ptr(node); -+ call_rcu(&node->rcu, mt_free_rcu); -+} -+ -+static unsigned int mt_height(const struct maple_tree *mt) -+{ -+ return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; -+} -+ -+static void mas_set_height(struct ma_state *mas) -+{ -+ unsigned int new_flags = mas->tree->ma_flags; -+ -+ new_flags &= ~MT_FLAGS_HEIGHT_MASK; -+ BUG_ON(mas->depth > MAPLE_HEIGHT_MAX); -+ new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET; -+ mas->tree->ma_flags = new_flags; -+} -+ -+static unsigned int mas_mt_height(struct ma_state *mas) -+{ -+ return mt_height(mas->tree); -+} -+ -+static inline enum maple_type mte_node_type(const struct maple_enode *entry) -+{ -+ return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) & -+ MAPLE_NODE_TYPE_MASK; -+} -+ -+static inline bool ma_is_dense(const enum maple_type type) -+{ -+ return type < maple_leaf_64; -+} -+ -+static inline bool ma_is_leaf(const enum maple_type type) -+{ -+ return type < maple_range_64; -+} -+ -+static inline bool mte_is_leaf(const struct maple_enode *entry) -+{ -+ return ma_is_leaf(mte_node_type(entry)); -+} -+ -+/* -+ * We also reserve values with the bottom two bits set to '10' which are -+ * below 4096 -+ */ -+static inline bool mt_is_reserved(const void *entry) -+{ -+ return ((unsigned long)entry < MAPLE_RESERVED_RANGE) && -+ xa_is_internal(entry); -+} -+ -+static inline void mas_set_err(struct ma_state *mas, long err) -+{ -+ mas->node = MA_ERROR(err); -+} -+ -+static inline bool mas_is_ptr(struct ma_state *mas) -+{ -+ return mas->node == MAS_ROOT; -+} -+ -+static inline bool mas_is_start(struct ma_state *mas) -+{ -+ return mas->node == MAS_START; -+} -+ -+bool mas_is_err(struct ma_state *mas) -+{ -+ return xa_is_err(mas->node); -+} -+ -+static inline bool mas_searchable(struct ma_state *mas) -+{ -+ if (mas_is_none(mas)) -+ return false; -+ -+ if (mas_is_ptr(mas)) -+ return false; -+ -+ return true; -+} -+ -+static inline struct maple_node *mte_to_node(const struct maple_enode *entry) -+{ -+ return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK); -+} -+ -+/* -+ * mte_to_mat() - Convert a maple encoded node to a maple topiary node. -+ * @entry: The maple encoded node -+ * -+ * Return: a maple topiary pointer -+ */ -+static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) -+{ -+ return (struct maple_topiary *) -+ ((unsigned long)entry & ~MAPLE_NODE_MASK); -+} -+ -+/* -+ * mas_mn() - Get the maple state node. -+ * @mas: The maple state -+ * -+ * Return: the maple node (not encoded - bare pointer). -+ */ -+static inline struct maple_node *mas_mn(const struct ma_state *mas) -+{ -+ return mte_to_node(mas->node); -+} -+ -+/* -+ * mte_set_node_dead() - Set a maple encoded node as dead. -+ * @mn: The maple encoded node. -+ */ -+static inline void mte_set_node_dead(struct maple_enode *mn) -+{ -+ mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn)); -+ smp_wmb(); /* Needed for RCU */ -+} -+ -+/* Bit 1 indicates the root is a node */ -+#define MAPLE_ROOT_NODE 0x02 -+/* maple_type stored bit 3-6 */ -+#define MAPLE_ENODE_TYPE_SHIFT 0x03 -+/* Bit 2 means a NULL somewhere below */ -+#define MAPLE_ENODE_NULL 0x04 -+ -+static inline struct maple_enode *mt_mk_node(const struct maple_node *node, -+ enum maple_type type) -+{ -+ return (void *)((unsigned long)node | -+ (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); -+} -+ -+static inline void *mte_mk_root(const struct maple_enode *node) -+{ -+ return (void *)((unsigned long)node | MAPLE_ROOT_NODE); -+} -+ -+static inline void *mte_safe_root(const struct maple_enode *node) -+{ -+ return (void *)((unsigned long)node & ~MAPLE_ROOT_NODE); -+} -+ -+static inline void mte_set_full(const struct maple_enode *node) -+{ -+ node = (void *)((unsigned long)node & ~MAPLE_ENODE_NULL); -+} -+ -+static inline void mte_clear_full(const struct maple_enode *node) -+{ -+ node = (void *)((unsigned long)node | MAPLE_ENODE_NULL); -+} -+ -+static inline bool ma_is_root(struct maple_node *node) -+{ -+ return ((unsigned long)node->parent & MA_ROOT_PARENT); -+} -+ -+static inline bool mte_is_root(const struct maple_enode *node) -+{ -+ return ma_is_root(mte_to_node(node)); -+} -+ -+static inline bool mas_is_root_limits(const struct ma_state *mas) -+{ -+ return !mas->min && mas->max == ULONG_MAX; -+} -+ -+static inline bool mt_is_alloc(struct maple_tree *mt) -+{ -+ return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); -+} -+ -+/* -+ * The Parent Pointer -+ * Excluding root, the parent pointer is 256B aligned like all other tree nodes. -+ * When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16 -+ * bit values need an extra bit to store the offset. This extra bit comes from -+ * a reuse of the last bit in the node type. This is possible by using bit 1 to -+ * indicate if bit 2 is part of the type or the slot. -+ * -+ * Note types: -+ * 0x??1 = Root -+ * 0x?00 = 16 bit nodes -+ * 0x010 = 32 bit nodes -+ * 0x110 = 64 bit nodes -+ * -+ * Slot size and alignment -+ * 0b??1 : Root -+ * 0b?00 : 16 bit values, type in 0-1, slot in 2-7 -+ * 0b010 : 32 bit values, type in 0-2, slot in 3-7 -+ * 0b110 : 64 bit values, type in 0-2, slot in 3-7 -+ */ -+ -+#define MAPLE_PARENT_ROOT 0x01 -+ -+#define MAPLE_PARENT_SLOT_SHIFT 0x03 -+#define MAPLE_PARENT_SLOT_MASK 0xF8 -+ -+#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02 -+#define MAPLE_PARENT_16B_SLOT_MASK 0xFC -+ -+#define MAPLE_PARENT_RANGE64 0x06 -+#define MAPLE_PARENT_RANGE32 0x04 -+#define MAPLE_PARENT_NOT_RANGE16 0x02 -+ -+/* -+ * mte_parent_shift() - Get the parent shift for the slot storage. -+ * @parent: The parent pointer cast as an unsigned long -+ * Return: The shift into that pointer to the star to of the slot -+ */ -+static inline unsigned long mte_parent_shift(unsigned long parent) -+{ -+ /* Note bit 1 == 0 means 16B */ -+ if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) -+ return MAPLE_PARENT_SLOT_SHIFT; -+ -+ return MAPLE_PARENT_16B_SLOT_SHIFT; -+} -+ -+/* -+ * mte_parent_slot_mask() - Get the slot mask for the parent. -+ * @parent: The parent pointer cast as an unsigned long. -+ * Return: The slot mask for that parent. -+ */ -+static inline unsigned long mte_parent_slot_mask(unsigned long parent) -+{ -+ /* Note bit 1 == 0 means 16B */ -+ if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) -+ return MAPLE_PARENT_SLOT_MASK; -+ -+ return MAPLE_PARENT_16B_SLOT_MASK; -+} -+ -+/* -+ * mas_parent_enum() - Return the maple_type of the parent from the stored -+ * parent type. -+ * @mas: The maple state -+ * @node: The maple_enode to extract the parent's enum -+ * Return: The node->parent maple_type -+ */ -+static inline -+enum maple_type mte_parent_enum(struct maple_enode *p_enode, -+ struct maple_tree *mt) -+{ -+ unsigned long p_type; -+ -+ p_type = (unsigned long)p_enode; -+ if (p_type & MAPLE_PARENT_ROOT) -+ return 0; /* Validated in the caller. */ -+ -+ p_type &= MAPLE_NODE_MASK; -+ p_type = p_type & ~(MAPLE_PARENT_ROOT | mte_parent_slot_mask(p_type)); -+ -+ switch (p_type) { -+ case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */ -+ if (mt_is_alloc(mt)) -+ return maple_arange_64; -+ return maple_range_64; -+ } -+ -+ return 0; -+} -+ -+static inline -+enum maple_type mas_parent_enum(struct ma_state *mas, struct maple_enode *enode) -+{ -+ return mte_parent_enum(ma_enode_ptr(mte_to_node(enode)->parent), mas->tree); -+} -+ -+/* -+ * mte_set_parent() - Set the parent node and encode the slot -+ * @enode: The encoded maple node. -+ * @parent: The encoded maple node that is the parent of @enode. -+ * @slot: The slot that @enode resides in @parent. -+ * -+ * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the -+ * parent type. -+ */ -+static inline -+void mte_set_parent(struct maple_enode *enode, const struct maple_enode *parent, -+ unsigned char slot) -+{ -+ unsigned long val = (unsigned long) parent; -+ unsigned long shift; -+ unsigned long type; -+ enum maple_type p_type = mte_node_type(parent); -+ -+ BUG_ON(p_type == maple_dense); -+ BUG_ON(p_type == maple_leaf_64); -+ -+ switch (p_type) { -+ case maple_range_64: -+ case maple_arange_64: -+ shift = MAPLE_PARENT_SLOT_SHIFT; -+ type = MAPLE_PARENT_RANGE64; -+ break; -+ default: -+ case maple_dense: -+ case maple_leaf_64: -+ shift = type = 0; -+ break; -+ } -+ -+ val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ -+ val |= (slot << shift) | type; -+ mte_to_node(enode)->parent = ma_parent_ptr(val); -+} -+ -+/* -+ * mte_parent_slot() - get the parent slot of @enode. -+ * @enode: The encoded maple node. -+ * -+ * Return: The slot in the parent node where @enode resides. -+ */ -+static inline unsigned int mte_parent_slot(const struct maple_enode *enode) -+{ -+ unsigned long val = (unsigned long) mte_to_node(enode)->parent; -+ -+ /* Root. */ -+ if (val & 1) -+ return 0; -+ -+ /* -+ * Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost -+ * by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT -+ */ -+ return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val); -+} -+ -+/* -+ * mte_parent() - Get the parent of @node. -+ * @node: The encoded maple node. -+ * -+ * Return: The parent maple node. -+ */ -+static inline struct maple_node *mte_parent(const struct maple_enode *enode) -+{ -+ return (void *)((unsigned long) -+ (mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK); -+} -+ -+/* -+ * ma_dead_node() - check if the @enode is dead. -+ * @enode: The encoded maple node -+ * -+ * Return: true if dead, false otherwise. -+ */ -+static inline bool ma_dead_node(const struct maple_node *node) -+{ -+ struct maple_node *parent = (void *)((unsigned long) -+ node->parent & ~MAPLE_NODE_MASK); -+ -+ return (parent == node); -+} -+/* -+ * mte_dead_node() - check if the @enode is dead. -+ * @enode: The encoded maple node -+ * -+ * Return: true if dead, false otherwise. -+ */ -+static inline bool mte_dead_node(const struct maple_enode *enode) -+{ -+ struct maple_node *parent, *node; -+ -+ node = mte_to_node(enode); -+ parent = mte_parent(enode); -+ return (parent == node); -+} -+ -+/* -+ * mas_allocated() - Get the number of nodes allocated in a maple state. -+ * @mas: The maple state -+ * -+ * The ma_state alloc member is overloaded to hold a pointer to the first -+ * allocated node or to the number of requested nodes to allocate. If bit 0 is -+ * set, then the alloc contains the number of requested nodes. If there is an -+ * allocated node, then the total allocated nodes is in that node. -+ * -+ * Return: The total number of nodes allocated -+ */ -+static inline unsigned long mas_allocated(const struct ma_state *mas) -+{ -+ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) -+ return 0; -+ -+ return mas->alloc->total; -+} -+ -+/* -+ * mas_set_alloc_req() - Set the requested number of allocations. -+ * @mas: the maple state -+ * @count: the number of allocations. -+ * -+ * The requested number of allocations is either in the first allocated node, -+ * located in @mas->alloc->request_count, or directly in @mas->alloc if there is -+ * no allocated node. Set the request either in the node or do the necessary -+ * encoding to store in @mas->alloc directly. -+ */ -+static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) -+{ -+ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) { -+ if (!count) -+ mas->alloc = NULL; -+ else -+ mas->alloc = (struct maple_alloc *)(((count) << 1U) | 1U); -+ return; -+ } -+ -+ mas->alloc->request_count = count; -+} -+ -+/* -+ * mas_alloc_req() - get the requested number of allocations. -+ * @mas: The maple state -+ * -+ * The alloc count is either stored directly in @mas, or in -+ * @mas->alloc->request_count if there is at least one node allocated. Decode -+ * the request count if it's stored directly in @mas->alloc. -+ * -+ * Return: The allocation request count. -+ */ -+static inline unsigned int mas_alloc_req(const struct ma_state *mas) -+{ -+ if ((unsigned long)mas->alloc & 0x1) -+ return (unsigned long)(mas->alloc) >> 1; -+ else if (mas->alloc) -+ return mas->alloc->request_count; -+ return 0; -+} -+ -+/* -+ * ma_pivots() - Get a pointer to the maple node pivots. -+ * @node - the maple node -+ * @type - the node type -+ * -+ * Return: A pointer to the maple node pivots -+ */ -+static inline unsigned long *ma_pivots(struct maple_node *node, -+ enum maple_type type) -+{ -+ switch (type) { -+ case maple_arange_64: -+ return node->ma64.pivot; -+ case maple_range_64: -+ case maple_leaf_64: -+ return node->mr64.pivot; -+ case maple_dense: -+ return NULL; -+ } -+ return NULL; -+} -+ -+/* -+ * ma_gaps() - Get a pointer to the maple node gaps. -+ * @node - the maple node -+ * @type - the node type -+ * -+ * Return: A pointer to the maple node gaps -+ */ -+static inline unsigned long *ma_gaps(struct maple_node *node, -+ enum maple_type type) -+{ -+ switch (type) { -+ case maple_arange_64: -+ return node->ma64.gap; -+ case maple_range_64: -+ case maple_leaf_64: -+ case maple_dense: -+ return NULL; -+ } -+ return NULL; -+} -+ -+/* -+ * mte_pivot() - Get the pivot at @piv of the maple encoded node. -+ * @mn: The maple encoded node. -+ * @piv: The pivot. -+ * -+ * Return: the pivot at @piv of @mn. -+ */ -+static inline unsigned long mte_pivot(const struct maple_enode *mn, -+ unsigned char piv) -+{ -+ struct maple_node *node = mte_to_node(mn); -+ -+ if (piv >= mt_pivots[piv]) { -+ WARN_ON(1); -+ return 0; -+ } -+ switch (mte_node_type(mn)) { -+ case maple_arange_64: -+ return node->ma64.pivot[piv]; -+ case maple_range_64: -+ case maple_leaf_64: -+ return node->mr64.pivot[piv]; -+ case maple_dense: -+ return 0; -+ } -+ return 0; -+} -+ -+/* -+ * mas_safe_pivot() - get the pivot at @piv or mas->max. -+ * @mas: The maple state -+ * @pivots: The pointer to the maple node pivots -+ * @piv: The pivot to fetch -+ * @type: The maple node type -+ * -+ * Return: The pivot at @piv within the limit of the @pivots array, @mas->max -+ * otherwise. -+ */ -+static inline unsigned long -+mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, -+ unsigned char piv, enum maple_type type) -+{ -+ if (piv >= mt_pivots[type]) -+ return mas->max; -+ -+ return pivots[piv]; -+} -+ -+/* -+ * mas_safe_min() - Return the minimum for a given offset. -+ * @mas: The maple state -+ * @pivots: The pointer to the maple node pivots -+ * @offset: The offset into the pivot array -+ * -+ * Return: The minimum range value that is contained in @offset. -+ */ -+static inline unsigned long -+mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset) -+{ -+ if (likely(offset)) -+ return pivots[offset - 1] + 1; -+ -+ return mas->min; -+} -+ -+/* -+ * mas_logical_pivot() - Get the logical pivot of a given offset. -+ * @mas: The maple state -+ * @pivots: The pointer to the maple node pivots -+ * @offset: The offset into the pivot array -+ * @type: The maple node type -+ * -+ * When there is no value at a pivot (beyond the end of the data), then the -+ * pivot is actually @mas->max. -+ * -+ * Return: the logical pivot of a given @offset. -+ */ -+static inline unsigned long -+mas_logical_pivot(struct ma_state *mas, unsigned long *pivots, -+ unsigned char offset, enum maple_type type) -+{ -+ unsigned long lpiv = mas_safe_pivot(mas, pivots, offset, type); -+ -+ if (likely(lpiv)) -+ return lpiv; -+ -+ if (likely(offset)) -+ return mas->max; -+ -+ return lpiv; -+} -+ -+/* -+ * mte_set_pivot() - Set a pivot to a value in an encoded maple node. -+ * @mn: The encoded maple node -+ * @piv: The pivot offset -+ * @val: The value of the pivot -+ */ -+static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, -+ unsigned long val) -+{ -+ struct maple_node *node = mte_to_node(mn); -+ enum maple_type type = mte_node_type(mn); -+ -+ BUG_ON(piv >= mt_pivots[type]); -+ switch (type) { -+ default: -+ case maple_range_64: -+ case maple_leaf_64: -+ node->mr64.pivot[piv] = val; -+ break; -+ case maple_arange_64: -+ node->ma64.pivot[piv] = val; -+ break; -+ case maple_dense: -+ break; -+ } -+ -+} -+ -+/* -+ * ma_slots() - Get a pointer to the maple node slots. -+ * @mn: The maple node -+ * @mt: The maple node type -+ * -+ * Return: A pointer to the maple node slots -+ */ -+static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) -+{ -+ switch (mt) { -+ default: -+ case maple_arange_64: -+ return mn->ma64.slot; -+ case maple_range_64: -+ case maple_leaf_64: -+ return mn->mr64.slot; -+ case maple_dense: -+ return mn->slot; -+ } -+} -+ -+static inline bool mt_locked(const struct maple_tree *mt) -+{ -+ return mt_external_lock(mt) ? mt_lock_is_held(mt) : -+ lockdep_is_held(&mt->ma_lock); -+} -+ -+static inline void *mt_slot(const struct maple_tree *mt, -+ void __rcu **slots, unsigned char offset) -+{ -+ return rcu_dereference_check(slots[offset], mt_locked(mt)); -+} -+ -+/* -+ * mas_slot_locked() - Get the slot value when holding the maple tree lock. -+ * @mas: The maple state -+ * @slots: The pointer to the slots -+ * @offset: The offset into the slots array to fetch -+ * -+ * Return: The entry stored in @slots at the @offset. -+ */ -+static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots, -+ unsigned char offset) -+{ -+ return rcu_dereference_protected(slots[offset], mt_locked(mas->tree)); -+} -+ -+/* -+ * mas_slot() - Get the slot value when not holding the maple tree lock. -+ * @mas: The maple state -+ * @slots: The pointer to the slots -+ * @offset: The offset into the slots array to fetch -+ * -+ * Return: The entry stored in @slots at the @offset -+ */ -+static inline void *mas_slot(struct ma_state *mas, void __rcu **slots, -+ unsigned char offset) -+{ -+ return mt_slot(mas->tree, slots, offset); -+} -+ -+/* -+ * mas_root() - Get the maple tree root. -+ * @mas: The maple state. -+ * -+ * Return: The pointer to the root of the tree -+ */ -+static inline void *mas_root(struct ma_state *mas) -+{ -+ return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree)); -+} -+ -+static inline void *mt_root_locked(struct maple_tree *mt) -+{ -+ return rcu_dereference_protected(mt->ma_root, mt_locked(mt)); -+} -+ -+/* -+ * mas_root_locked() - Get the maple tree root when holding the maple tree lock. -+ * @mas: The maple state. -+ * -+ * Return: The pointer to the root of the tree -+ */ -+static inline void *mas_root_locked(struct ma_state *mas) -+{ -+ return mt_root_locked(mas->tree); -+} -+ -+static inline struct maple_metadata *ma_meta(struct maple_node *mn, -+ enum maple_type mt) -+{ -+ switch (mt) { -+ case maple_arange_64: -+ return &mn->ma64.meta; -+ default: -+ return &mn->mr64.meta; -+ } -+} -+ -+/* -+ * ma_set_meta() - Set the metadata information of a node. -+ * @mn: The maple node -+ * @mt: The maple node type -+ * @offset: The offset of the highest sub-gap in this node. -+ * @end: The end of the data in this node. -+ */ -+static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt, -+ unsigned char offset, unsigned char end) -+{ -+ struct maple_metadata *meta = ma_meta(mn, mt); -+ -+ meta->gap = offset; -+ meta->end = end; -+} -+ -+/* -+ * ma_meta_end() - Get the data end of a node from the metadata -+ * @mn: The maple node -+ * @mt: The maple node type -+ */ -+static inline unsigned char ma_meta_end(struct maple_node *mn, -+ enum maple_type mt) -+{ -+ struct maple_metadata *meta = ma_meta(mn, mt); -+ -+ return meta->end; -+} -+ -+/* -+ * ma_meta_gap() - Get the largest gap location of a node from the metadata -+ * @mn: The maple node -+ * @mt: The maple node type -+ */ -+static inline unsigned char ma_meta_gap(struct maple_node *mn, -+ enum maple_type mt) -+{ -+ BUG_ON(mt != maple_arange_64); -+ -+ return mn->ma64.meta.gap; -+} -+ -+/* -+ * ma_set_meta_gap() - Set the largest gap location in a nodes metadata -+ * @mn: The maple node -+ * @mn: The maple node type -+ * @offset: The location of the largest gap. -+ */ -+static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, -+ unsigned char offset) -+{ -+ -+ struct maple_metadata *meta = ma_meta(mn, mt); -+ -+ meta->gap = offset; -+} -+ -+/* -+ * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. -+ * @mat - the ma_topiary, a linked list of dead nodes. -+ * @dead_enode - the node to be marked as dead and added to the tail of the list -+ * -+ * Add the @dead_enode to the linked list in @mat. -+ */ -+static inline void mat_add(struct ma_topiary *mat, -+ struct maple_enode *dead_enode) -+{ -+ mte_set_node_dead(dead_enode); -+ mte_to_mat(dead_enode)->next = NULL; -+ if (!mat->tail) { -+ mat->tail = mat->head = dead_enode; -+ return; -+ } -+ -+ mte_to_mat(mat->tail)->next = dead_enode; -+ mat->tail = dead_enode; -+} -+ -+static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); -+static inline void mas_free(struct ma_state *mas, struct maple_enode *used); -+ -+/* -+ * mas_mat_free() - Free all nodes in a dead list. -+ * @mas - the maple state -+ * @mat - the ma_topiary linked list of dead nodes to free. -+ * -+ * Free walk a dead list. -+ */ -+static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat) -+{ -+ struct maple_enode *next; -+ -+ while (mat->head) { -+ next = mte_to_mat(mat->head)->next; -+ mas_free(mas, mat->head); -+ mat->head = next; -+ } -+} -+ -+/* -+ * mas_mat_destroy() - Free all nodes and subtrees in a dead list. -+ * @mas - the maple state -+ * @mat - the ma_topiary linked list of dead nodes to free. -+ * -+ * Destroy walk a dead list. -+ */ -+static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat) -+{ -+ struct maple_enode *next; -+ -+ while (mat->head) { -+ next = mte_to_mat(mat->head)->next; -+ mte_destroy_walk(mat->head, mat->mtree); -+ mat->head = next; -+ } -+} -+/* -+ * mas_descend() - Descend into the slot stored in the ma_state. -+ * @mas - the maple state. -+ * -+ * Note: Not RCU safe, only use in write side or debug code. -+ */ -+static inline void mas_descend(struct ma_state *mas) -+{ -+ enum maple_type type; -+ unsigned long *pivots; -+ struct maple_node *node; -+ void __rcu **slots; -+ -+ node = mas_mn(mas); -+ type = mte_node_type(mas->node); -+ pivots = ma_pivots(node, type); -+ slots = ma_slots(node, type); -+ -+ if (mas->offset) -+ mas->min = pivots[mas->offset - 1] + 1; -+ mas->max = mas_safe_pivot(mas, pivots, mas->offset, type); -+ mas->node = mas_slot(mas, slots, mas->offset); -+} -+ -+/* -+ * mte_set_gap() - Set a maple node gap. -+ * @mn: The encoded maple node -+ * @gap: The offset of the gap to set -+ * @val: The gap value -+ */ -+static inline void mte_set_gap(const struct maple_enode *mn, -+ unsigned char gap, unsigned long val) -+{ -+ switch (mte_node_type(mn)) { -+ default: -+ break; -+ case maple_arange_64: -+ mte_to_node(mn)->ma64.gap[gap] = val; -+ break; -+ } -+} -+ -+/* -+ * mas_ascend() - Walk up a level of the tree. -+ * @mas: The maple state -+ * -+ * Sets the @mas->max and @mas->min to the correct values when walking up. This -+ * may cause several levels of walking up to find the correct min and max. -+ * May find a dead node which will cause a premature return. -+ * Return: 1 on dead node, 0 otherwise -+ */ -+static int mas_ascend(struct ma_state *mas) -+{ -+ struct maple_enode *p_enode; /* parent enode. */ -+ struct maple_enode *a_enode; /* ancestor enode. */ -+ struct maple_node *a_node; /* ancestor node. */ -+ struct maple_node *p_node; /* parent node. */ -+ unsigned char a_slot; -+ enum maple_type a_type; -+ unsigned long min, max; -+ unsigned long *pivots; -+ unsigned char offset; -+ bool set_max = false, set_min = false; -+ -+ a_node = mas_mn(mas); -+ if (ma_is_root(a_node)) { -+ mas->offset = 0; -+ return 0; -+ } -+ -+ p_node = mte_parent(mas->node); -+ if (unlikely(a_node == p_node)) -+ return 1; -+ a_type = mas_parent_enum(mas, mas->node); -+ offset = mte_parent_slot(mas->node); -+ a_enode = mt_mk_node(p_node, a_type); -+ -+ /* Check to make sure all parent information is still accurate */ -+ if (p_node != mte_parent(mas->node)) -+ return 1; -+ -+ mas->node = a_enode; -+ mas->offset = offset; -+ -+ if (mte_is_root(a_enode)) { -+ mas->max = ULONG_MAX; -+ mas->min = 0; -+ return 0; -+ } -+ -+ min = 0; -+ max = ULONG_MAX; -+ do { -+ p_enode = a_enode; -+ a_type = mas_parent_enum(mas, p_enode); -+ a_node = mte_parent(p_enode); -+ a_slot = mte_parent_slot(p_enode); -+ pivots = ma_pivots(a_node, a_type); -+ a_enode = mt_mk_node(a_node, a_type); -+ -+ if (!set_min && a_slot) { -+ set_min = true; -+ min = pivots[a_slot - 1] + 1; -+ } -+ -+ if (!set_max && a_slot < mt_pivots[a_type]) { -+ set_max = true; -+ max = pivots[a_slot]; -+ } -+ -+ if (unlikely(ma_dead_node(a_node))) -+ return 1; -+ -+ if (unlikely(ma_is_root(a_node))) -+ break; -+ -+ } while (!set_min || !set_max); -+ -+ mas->max = max; -+ mas->min = min; -+ return 0; -+} -+ -+/* -+ * mas_pop_node() - Get a previously allocated maple node from the maple state. -+ * @mas: The maple state -+ * -+ * Return: A pointer to a maple node. -+ */ -+static inline struct maple_node *mas_pop_node(struct ma_state *mas) -+{ -+ struct maple_alloc *ret, *node = mas->alloc; -+ unsigned long total = mas_allocated(mas); -+ -+ /* nothing or a request pending. */ -+ if (unlikely(!total)) -+ return NULL; -+ -+ if (total == 1) { -+ /* single allocation in this ma_state */ -+ mas->alloc = NULL; -+ ret = node; -+ goto single_node; -+ } -+ -+ if (!node->node_count) { -+ /* Single allocation in this node. */ -+ mas->alloc = node->slot[0]; -+ node->slot[0] = NULL; -+ mas->alloc->total = node->total - 1; -+ ret = node; -+ goto new_head; -+ } -+ -+ node->total--; -+ ret = node->slot[node->node_count]; -+ node->slot[node->node_count--] = NULL; -+ -+single_node: -+new_head: -+ ret->total = 0; -+ ret->node_count = 0; -+ if (ret->request_count) { -+ mas_set_alloc_req(mas, ret->request_count + 1); -+ ret->request_count = 0; -+ } -+ return (struct maple_node *)ret; -+} -+ -+/* -+ * mas_push_node() - Push a node back on the maple state allocation. -+ * @mas: The maple state -+ * @used: The used maple node -+ * -+ * Stores the maple node back into @mas->alloc for reuse. Updates allocated and -+ * requested node count as necessary. -+ */ -+static inline void mas_push_node(struct ma_state *mas, struct maple_node *used) -+{ -+ struct maple_alloc *reuse = (struct maple_alloc *)used; -+ struct maple_alloc *head = mas->alloc; -+ unsigned long count; -+ unsigned int requested = mas_alloc_req(mas); -+ -+ memset(reuse, 0, sizeof(*reuse)); -+ count = mas_allocated(mas); -+ -+ if (count && (head->node_count < MAPLE_ALLOC_SLOTS - 1)) { -+ if (head->slot[0]) -+ head->node_count++; -+ head->slot[head->node_count] = reuse; -+ head->total++; -+ goto done; -+ } -+ -+ reuse->total = 1; -+ if ((head) && !((unsigned long)head & 0x1)) { -+ head->request_count = 0; -+ reuse->slot[0] = head; -+ reuse->total += head->total; -+ } -+ -+ mas->alloc = reuse; -+done: -+ if (requested > 1) -+ mas_set_alloc_req(mas, requested - 1); -+} -+ -+/* -+ * mas_alloc_nodes() - Allocate nodes into a maple state -+ * @mas: The maple state -+ * @gfp: The GFP Flags -+ */ -+static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) -+{ -+ struct maple_alloc *node; -+ struct maple_alloc **nodep = &mas->alloc; -+ unsigned long allocated = mas_allocated(mas); -+ unsigned long success = allocated; -+ unsigned int requested = mas_alloc_req(mas); -+ unsigned int count; -+ void **slots = NULL; -+ unsigned int max_req = 0; -+ -+ if (!requested) -+ return; -+ -+ mas_set_alloc_req(mas, 0); -+ if (mas->mas_flags & MA_STATE_PREALLOC) { -+ if (allocated) -+ return; -+ WARN_ON(!allocated); -+ } -+ -+ if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS - 1) { -+ node = (struct maple_alloc *)mt_alloc_one(gfp); -+ if (!node) -+ goto nomem_one; -+ -+ if (allocated) -+ node->slot[0] = mas->alloc; -+ -+ success++; -+ mas->alloc = node; -+ requested--; -+ } -+ -+ node = mas->alloc; -+ while (requested) { -+ max_req = MAPLE_ALLOC_SLOTS; -+ if (node->slot[0]) { -+ unsigned int offset = node->node_count + 1; -+ -+ slots = (void **)&node->slot[offset]; -+ max_req -= offset; -+ } else { -+ slots = (void **)&node->slot; -+ } -+ -+ max_req = min(requested, max_req); -+ count = mt_alloc_bulk(gfp, max_req, slots); -+ if (!count) -+ goto nomem_bulk; -+ -+ node->node_count += count; -+ /* zero indexed. */ -+ if (slots == (void **)&node->slot) -+ node->node_count--; -+ -+ success += count; -+ nodep = &node->slot[0]; -+ node = *nodep; -+ requested -= count; -+ } -+ mas->alloc->total = success; -+ return; -+ -+nomem_bulk: -+ /* Clean up potential freed allocations on bulk failure */ -+ memset(slots, 0, max_req * sizeof(unsigned long)); -+nomem_one: -+ mas_set_alloc_req(mas, requested); -+ if (mas->alloc && !(((unsigned long)mas->alloc & 0x1))) -+ mas->alloc->total = success; -+ mas_set_err(mas, -ENOMEM); -+ return; -+ -+} -+ -+/* -+ * mas_free() - Free an encoded maple node -+ * @mas: The maple state -+ * @used: The encoded maple node to free. -+ * -+ * Uses rcu free if necessary, pushes @used back on the maple state allocations -+ * otherwise. -+ */ -+static inline void mas_free(struct ma_state *mas, struct maple_enode *used) -+{ -+ struct maple_node *tmp = mte_to_node(used); -+ -+ if (mt_in_rcu(mas->tree)) -+ ma_free_rcu(tmp); -+ else -+ mas_push_node(mas, tmp); -+} -+ -+/* -+ * mas_node_count() - Check if enough nodes are allocated and request more if -+ * there is not enough nodes. -+ * @mas: The maple state -+ * @count: The number of nodes needed -+ * @gfp: the gfp flags -+ */ -+static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp) -+{ -+ unsigned long allocated = mas_allocated(mas); -+ -+ if (allocated < count) { -+ mas_set_alloc_req(mas, count - allocated); -+ mas_alloc_nodes(mas, gfp); -+ } -+} -+ -+/* -+ * mas_node_count() - Check if enough nodes are allocated and request more if -+ * there is not enough nodes. -+ * @mas: The maple state -+ * @count: The number of nodes needed -+ * -+ * Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags. -+ */ -+static void mas_node_count(struct ma_state *mas, int count) -+{ -+ return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN); -+} -+ -+/* -+ * mas_start() - Sets up maple state for operations. -+ * @mas: The maple state. -+ * -+ * If mas->node == MAS_START, then set the min, max, depth, and offset to -+ * defaults. -+ * -+ * Return: -+ * - If mas->node is an error or not MAS_START, return NULL. -+ * - If it's an empty tree: NULL & mas->node == MAS_NONE -+ * - If it's a single entry: The entry & mas->node == MAS_ROOT -+ * - If it's a tree: NULL & mas->node == safe root node. -+ */ -+static inline struct maple_enode *mas_start(struct ma_state *mas) -+{ -+ if (likely(mas_is_start(mas))) { -+ struct maple_enode *root; -+ -+ mas->node = MAS_NONE; -+ mas->min = 0; -+ mas->max = ULONG_MAX; -+ mas->depth = 0; -+ mas->offset = 0; -+ -+ root = mas_root(mas); -+ /* Tree with nodes */ -+ if (likely(xa_is_node(root))) { -+ mas->node = mte_safe_root(root); -+ return NULL; -+ } -+ -+ /* empty tree */ -+ if (unlikely(!root)) { -+ mas->offset = MAPLE_NODE_SLOTS; -+ return NULL; -+ } -+ -+ /* Single entry tree */ -+ mas->node = MAS_ROOT; -+ mas->offset = MAPLE_NODE_SLOTS; -+ -+ /* Single entry tree. */ -+ if (mas->index > 0) -+ return NULL; -+ -+ return root; -+ } -+ -+ return NULL; -+} -+ -+/* -+ * ma_data_end() - Find the end of the data in a node. -+ * @node: The maple node -+ * @type: The maple node type -+ * @pivots: The array of pivots in the node -+ * @max: The maximum value in the node -+ * -+ * Uses metadata to find the end of the data when possible. -+ * Return: The zero indexed last slot with data (may be null). -+ */ -+static inline unsigned char ma_data_end(struct maple_node *node, -+ enum maple_type type, -+ unsigned long *pivots, -+ unsigned long max) -+{ -+ unsigned char offset; -+ -+ if (type == maple_arange_64) -+ return ma_meta_end(node, type); -+ -+ offset = mt_pivots[type] - 1; -+ if (likely(!pivots[offset])) -+ return ma_meta_end(node, type); -+ -+ if (likely(pivots[offset] == max)) -+ return offset; -+ -+ return mt_pivots[type]; -+} -+ -+/* -+ * mas_data_end() - Find the end of the data (slot). -+ * @mas: the maple state -+ * -+ * This method is optimized to check the metadata of a node if the node type -+ * supports data end metadata. -+ * -+ * Return: The zero indexed last slot with data (may be null). -+ */ -+static inline unsigned char mas_data_end(struct ma_state *mas) -+{ -+ enum maple_type type; -+ struct maple_node *node; -+ unsigned char offset; -+ unsigned long *pivots; -+ -+ type = mte_node_type(mas->node); -+ node = mas_mn(mas); -+ if (type == maple_arange_64) -+ return ma_meta_end(node, type); -+ -+ pivots = ma_pivots(node, type); -+ offset = mt_pivots[type] - 1; -+ if (likely(!pivots[offset])) -+ return ma_meta_end(node, type); -+ -+ if (likely(pivots[offset] == mas->max)) -+ return offset; -+ -+ return mt_pivots[type]; -+} -+ -+/* -+ * mas_leaf_max_gap() - Returns the largest gap in a leaf node -+ * @mas - the maple state -+ * -+ * Return: The maximum gap in the leaf. -+ */ -+static unsigned long mas_leaf_max_gap(struct ma_state *mas) -+{ -+ enum maple_type mt; -+ unsigned long pstart, gap, max_gap; -+ struct maple_node *mn; -+ unsigned long *pivots; -+ void __rcu **slots; -+ unsigned char i; -+ unsigned char max_piv; -+ -+ mt = mte_node_type(mas->node); -+ mn = mas_mn(mas); -+ slots = ma_slots(mn, mt); -+ max_gap = 0; -+ if (unlikely(ma_is_dense(mt))) { -+ gap = 0; -+ for (i = 0; i < mt_slots[mt]; i++) { -+ if (slots[i]) { -+ if (gap > max_gap) -+ max_gap = gap; -+ gap = 0; -+ } else { -+ gap++; -+ } -+ } -+ if (gap > max_gap) -+ max_gap = gap; -+ return max_gap; -+ } -+ -+ /* -+ * Check the first implied pivot optimizes the loop below and slot 1 may -+ * be skipped if there is a gap in slot 0. -+ */ -+ pivots = ma_pivots(mn, mt); -+ if (likely(!slots[0])) { -+ max_gap = pivots[0] - mas->min + 1; -+ i = 2; -+ } else { -+ i = 1; -+ } -+ -+ /* reduce max_piv as the special case is checked before the loop */ -+ max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1; -+ /* -+ * Check end implied pivot which can only be a gap on the right most -+ * node. -+ */ -+ if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) { -+ gap = ULONG_MAX - pivots[max_piv]; -+ if (gap > max_gap) -+ max_gap = gap; -+ } -+ -+ for (; i <= max_piv; i++) { -+ /* data == no gap. */ -+ if (likely(slots[i])) -+ continue; -+ -+ pstart = pivots[i - 1]; -+ gap = pivots[i] - pstart; -+ if (gap > max_gap) -+ max_gap = gap; -+ -+ /* There cannot be two gaps in a row. */ -+ i++; -+ } -+ return max_gap; -+} -+ -+/* -+ * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) -+ * @node: The maple node -+ * @gaps: The pointer to the gaps -+ * @mt: The maple node type -+ * @*off: Pointer to store the offset location of the gap. -+ * -+ * Uses the metadata data end to scan backwards across set gaps. -+ * -+ * Return: The maximum gap value -+ */ -+static inline unsigned long -+ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, -+ unsigned char *off) -+{ -+ unsigned char offset, i; -+ unsigned long max_gap = 0; -+ -+ i = offset = ma_meta_end(node, mt); -+ do { -+ if (gaps[i] > max_gap) { -+ max_gap = gaps[i]; -+ offset = i; -+ } -+ } while (i--); -+ -+ *off = offset; -+ return max_gap; -+} -+ -+/* -+ * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. -+ * @mas: The maple state. -+ * -+ * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap. -+ * -+ * Return: The gap value. -+ */ -+static inline unsigned long mas_max_gap(struct ma_state *mas) -+{ -+ unsigned long *gaps; -+ unsigned char offset; -+ enum maple_type mt; -+ struct maple_node *node; -+ -+ mt = mte_node_type(mas->node); -+ if (ma_is_leaf(mt)) -+ return mas_leaf_max_gap(mas); -+ -+ node = mas_mn(mas); -+ offset = ma_meta_gap(node, mt); -+ if (offset == MAPLE_ARANGE64_META_MAX) -+ return 0; -+ -+ gaps = ma_gaps(node, mt); -+ return gaps[offset]; -+} -+ -+/* -+ * mas_parent_gap() - Set the parent gap and any gaps above, as needed -+ * @mas: The maple state -+ * @offset: The gap offset in the parent to set -+ * @new: The new gap value. -+ * -+ * Set the parent gap then continue to set the gap upwards, using the metadata -+ * of the parent to see if it is necessary to check the node above. -+ */ -+static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, -+ unsigned long new) -+{ -+ unsigned long meta_gap = 0; -+ struct maple_node *pnode; -+ struct maple_enode *penode; -+ unsigned long *pgaps; -+ unsigned char meta_offset; -+ enum maple_type pmt; -+ -+ pnode = mte_parent(mas->node); -+ pmt = mas_parent_enum(mas, mas->node); -+ penode = mt_mk_node(pnode, pmt); -+ pgaps = ma_gaps(pnode, pmt); -+ -+ascend: -+ meta_offset = ma_meta_gap(pnode, pmt); -+ if (meta_offset == MAPLE_ARANGE64_META_MAX) -+ meta_gap = 0; -+ else -+ meta_gap = pgaps[meta_offset]; -+ -+ pgaps[offset] = new; -+ -+ if (meta_gap == new) -+ return; -+ -+ if (offset != meta_offset) { -+ if (meta_gap > new) -+ return; -+ -+ ma_set_meta_gap(pnode, pmt, offset); -+ } else if (new < meta_gap) { -+ meta_offset = 15; -+ new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); -+ ma_set_meta_gap(pnode, pmt, meta_offset); -+ } -+ -+ if (ma_is_root(pnode)) -+ return; -+ -+ /* Go to the parent node. */ -+ pnode = mte_parent(penode); -+ pmt = mas_parent_enum(mas, penode); -+ pgaps = ma_gaps(pnode, pmt); -+ offset = mte_parent_slot(penode); -+ penode = mt_mk_node(pnode, pmt); -+ goto ascend; -+} -+ -+/* -+ * mas_update_gap() - Update a nodes gaps and propagate up if necessary. -+ * @mas - the maple state. -+ */ -+static inline void mas_update_gap(struct ma_state *mas) -+{ -+ unsigned char pslot; -+ unsigned long p_gap; -+ unsigned long max_gap; -+ -+ if (!mt_is_alloc(mas->tree)) -+ return; -+ -+ if (mte_is_root(mas->node)) -+ return; -+ -+ max_gap = mas_max_gap(mas); -+ -+ pslot = mte_parent_slot(mas->node); -+ p_gap = ma_gaps(mte_parent(mas->node), -+ mas_parent_enum(mas, mas->node))[pslot]; -+ -+ if (p_gap != max_gap) -+ mas_parent_gap(mas, pslot, max_gap); -+} -+ -+/* -+ * mas_adopt_children() - Set the parent pointer of all nodes in @parent to -+ * @parent with the slot encoded. -+ * @mas - the maple state (for the tree) -+ * @parent - the maple encoded node containing the children. -+ */ -+static inline void mas_adopt_children(struct ma_state *mas, -+ struct maple_enode *parent) -+{ -+ enum maple_type type = mte_node_type(parent); -+ struct maple_node *node = mas_mn(mas); -+ void __rcu **slots = ma_slots(node, type); -+ unsigned long *pivots = ma_pivots(node, type); -+ struct maple_enode *child; -+ unsigned char offset; -+ -+ offset = ma_data_end(node, type, pivots, mas->max); -+ do { -+ child = mas_slot_locked(mas, slots, offset); -+ mte_set_parent(child, parent, offset); -+ } while (offset--); -+} -+ -+/* -+ * mas_replace() - Replace a maple node in the tree with mas->node. Uses the -+ * parent encoding to locate the maple node in the tree. -+ * @mas - the ma_state to use for operations. -+ * @advanced - boolean to adopt the child nodes and free the old node (false) or -+ * leave the node (true) and handle the adoption and free elsewhere. -+ */ -+static inline void mas_replace(struct ma_state *mas, bool advanced) -+ __must_hold(mas->tree->lock) -+{ -+ struct maple_node *mn = mas_mn(mas); -+ struct maple_enode *old_enode; -+ unsigned char offset = 0; -+ void __rcu **slots = NULL; -+ -+ if (ma_is_root(mn)) { -+ old_enode = mas_root_locked(mas); -+ } else { -+ offset = mte_parent_slot(mas->node); -+ slots = ma_slots(mte_parent(mas->node), -+ mas_parent_enum(mas, mas->node)); -+ old_enode = mas_slot_locked(mas, slots, offset); -+ } -+ -+ if (!advanced && !mte_is_leaf(mas->node)) -+ mas_adopt_children(mas, mas->node); -+ -+ if (mte_is_root(mas->node)) { -+ mn->parent = ma_parent_ptr( -+ ((unsigned long)mas->tree | MA_ROOT_PARENT)); -+ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); -+ mas_set_height(mas); -+ } else { -+ rcu_assign_pointer(slots[offset], mas->node); -+ } -+ -+ if (!advanced) -+ mas_free(mas, old_enode); -+} -+ -+/* -+ * mas_new_child() - Find the new child of a node. -+ * @mas: the maple state -+ * @child: the maple state to store the child. -+ */ -+static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) -+ __must_hold(mas->tree->lock) -+{ -+ enum maple_type mt; -+ unsigned char offset; -+ unsigned char end; -+ unsigned long *pivots; -+ struct maple_enode *entry; -+ struct maple_node *node; -+ void __rcu **slots; -+ -+ mt = mte_node_type(mas->node); -+ node = mas_mn(mas); -+ slots = ma_slots(node, mt); -+ pivots = ma_pivots(node, mt); -+ end = ma_data_end(node, mt, pivots, mas->max); -+ for (offset = mas->offset; offset <= end; offset++) { -+ entry = mas_slot_locked(mas, slots, offset); -+ if (mte_parent(entry) == node) { -+ *child = *mas; -+ mas->offset = offset + 1; -+ child->offset = offset; -+ mas_descend(child); -+ child->offset = 0; -+ return true; -+ } -+ } -+ return false; -+} -+ -+/* -+ * mab_shift_right() - Shift the data in mab right. Note, does not clean out the -+ * old data or set b_node->b_end. -+ * @b_node: the maple_big_node -+ * @shift: the shift count -+ */ -+static inline void mab_shift_right(struct maple_big_node *b_node, -+ unsigned char shift) -+{ -+ unsigned long size = b_node->b_end * sizeof(unsigned long); -+ -+ memmove(b_node->pivot + shift, b_node->pivot, size); -+ memmove(b_node->slot + shift, b_node->slot, size); -+ if (b_node->type == maple_arange_64) -+ memmove(b_node->gap + shift, b_node->gap, size); -+} -+ -+/* -+ * mab_middle_node() - Check if a middle node is needed (unlikely) -+ * @b_node: the maple_big_node that contains the data. -+ * @size: the amount of data in the b_node -+ * @split: the potential split location -+ * @slot_count: the size that can be stored in a single node being considered. -+ * -+ * Return: true if a middle node is required. -+ */ -+static inline bool mab_middle_node(struct maple_big_node *b_node, int split, -+ unsigned char slot_count) -+{ -+ unsigned char size = b_node->b_end; -+ -+ if (size >= 2 * slot_count) -+ return true; -+ -+ if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) -+ return true; -+ -+ return false; -+} -+ -+/* -+ * mab_no_null_split() - ensure the split doesn't fall on a NULL -+ * @b_node: the maple_big_node with the data -+ * @split: the suggested split location -+ * @slot_count: the number of slots in the node being considered. -+ * -+ * Return: the split location. -+ */ -+static inline int mab_no_null_split(struct maple_big_node *b_node, -+ unsigned char split, unsigned char slot_count) -+{ -+ if (!b_node->slot[split]) { -+ /* -+ * If the split is less than the max slot && the right side will -+ * still be sufficient, then increment the split on NULL. -+ */ -+ if ((split < slot_count - 1) && -+ (b_node->b_end - split) > (mt_min_slots[b_node->type])) -+ split++; -+ else -+ split--; -+ } -+ return split; -+} -+ -+/* -+ * mab_calc_split() - Calculate the split location and if there needs to be two -+ * splits. -+ * @bn: The maple_big_node with the data -+ * @mid_split: The second split, if required. 0 otherwise. -+ * -+ * Return: The first split location. The middle split is set in @mid_split. -+ */ -+static inline int mab_calc_split(struct ma_state *mas, -+ struct maple_big_node *bn, unsigned char *mid_split, unsigned long min) -+{ -+ unsigned char b_end = bn->b_end; -+ int split = b_end / 2; /* Assume equal split. */ -+ unsigned char slot_min, slot_count = mt_slots[bn->type]; -+ -+ /* -+ * To support gap tracking, all NULL entries are kept together and a node cannot -+ * end on a NULL entry, with the exception of the left-most leaf. The -+ * limitation means that the split of a node must be checked for this condition -+ * and be able to put more data in one direction or the other. -+ */ -+ if (unlikely((mas->mas_flags & MA_STATE_BULK))) { -+ *mid_split = 0; -+ split = b_end - mt_min_slots[bn->type]; -+ -+ if (!ma_is_leaf(bn->type)) -+ return split; -+ -+ mas->mas_flags |= MA_STATE_REBALANCE; -+ if (!bn->slot[split]) -+ split--; -+ return split; -+ } -+ -+ /* -+ * Although extremely rare, it is possible to enter what is known as the 3-way -+ * split scenario. The 3-way split comes about by means of a store of a range -+ * that overwrites the end and beginning of two full nodes. The result is a set -+ * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can -+ * also be located in different parent nodes which are also full. This can -+ * carry upwards all the way to the root in the worst case. -+ */ -+ if (unlikely(mab_middle_node(bn, split, slot_count))) { -+ split = b_end / 3; -+ *mid_split = split * 2; -+ } else { -+ slot_min = mt_min_slots[bn->type]; -+ -+ *mid_split = 0; -+ /* -+ * Avoid having a range less than the slot count unless it -+ * causes one node to be deficient. -+ * NOTE: mt_min_slots is 1 based, b_end and split are zero. -+ */ -+ while (((bn->pivot[split] - min) < slot_count - 1) && -+ (split < slot_count - 1) && (b_end - split > slot_min)) -+ split++; -+ } -+ -+ /* Avoid ending a node on a NULL entry */ -+ split = mab_no_null_split(bn, split, slot_count); -+ if (!(*mid_split)) -+ return split; -+ -+ *mid_split = mab_no_null_split(bn, *mid_split, slot_count); -+ -+ return split; -+} -+ -+/* -+ * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node -+ * and set @b_node->b_end to the next free slot. -+ * @mas: The maple state -+ * @mas_start: The starting slot to copy -+ * @mas_end: The end slot to copy (inclusively) -+ * @b_node: The maple_big_node to place the data -+ * @mab_start: The starting location in maple_big_node to store the data. -+ */ -+static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, -+ unsigned char mas_end, struct maple_big_node *b_node, -+ unsigned char mab_start) -+{ -+ enum maple_type mt; -+ struct maple_node *node; -+ void __rcu **slots; -+ unsigned long *pivots, *gaps; -+ int i = mas_start, j = mab_start; -+ unsigned char piv_end; -+ -+ node = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ pivots = ma_pivots(node, mt); -+ if (!i) { -+ b_node->pivot[j] = pivots[i++]; -+ if (unlikely(i > mas_end)) -+ goto complete; -+ j++; -+ } -+ -+ piv_end = min(mas_end, mt_pivots[mt]); -+ for (; i < piv_end; i++, j++) { -+ b_node->pivot[j] = pivots[i]; -+ if (unlikely(!b_node->pivot[j])) -+ break; -+ -+ if (unlikely(mas->max == b_node->pivot[j])) -+ goto complete; -+ } -+ -+ if (likely(i <= mas_end)) -+ b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); -+ -+complete: -+ b_node->b_end = ++j; -+ j -= mab_start; -+ slots = ma_slots(node, mt); -+ memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); -+ if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) { -+ gaps = ma_gaps(node, mt); -+ memcpy(b_node->gap + mab_start, gaps + mas_start, -+ sizeof(unsigned long) * j); -+ } -+} -+ -+/* -+ * mas_leaf_set_meta() - Set the metadata of a leaf if possible. -+ * @mas: The maple state -+ * @node: The maple node -+ * @pivots: pointer to the maple node pivots -+ * @mt: The maple type -+ * @end: The assumed end -+ * -+ * Note, end may be incremented within this function but not modified at the -+ * source. This is fine since the metadata is the last thing to be stored in a -+ * node during a write. -+ */ -+static inline void mas_leaf_set_meta(struct ma_state *mas, -+ struct maple_node *node, unsigned long *pivots, -+ enum maple_type mt, unsigned char end) -+{ -+ /* There is no room for metadata already */ -+ if (mt_pivots[mt] <= end) -+ return; -+ -+ if (pivots[end] && pivots[end] < mas->max) -+ end++; -+ -+ if (end < mt_slots[mt] - 1) -+ ma_set_meta(node, mt, 0, end); -+} -+ -+/* -+ * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. -+ * @b_node: the maple_big_node that has the data -+ * @mab_start: the start location in @b_node. -+ * @mab_end: The end location in @b_node (inclusively) -+ * @mas: The maple state with the maple encoded node. -+ */ -+static inline void mab_mas_cp(struct maple_big_node *b_node, -+ unsigned char mab_start, unsigned char mab_end, -+ struct ma_state *mas, bool new_max) -+{ -+ int i, j = 0; -+ enum maple_type mt = mte_node_type(mas->node); -+ struct maple_node *node = mte_to_node(mas->node); -+ void __rcu **slots = ma_slots(node, mt); -+ unsigned long *pivots = ma_pivots(node, mt); -+ unsigned long *gaps = NULL; -+ unsigned char end; -+ -+ if (mab_end - mab_start > mt_pivots[mt]) -+ mab_end--; -+ -+ if (!pivots[mt_pivots[mt] - 1]) -+ slots[mt_pivots[mt]] = NULL; -+ -+ i = mab_start; -+ do { -+ pivots[j++] = b_node->pivot[i++]; -+ } while (i <= mab_end && likely(b_node->pivot[i])); -+ -+ memcpy(slots, b_node->slot + mab_start, -+ sizeof(void *) * (i - mab_start)); -+ -+ if (new_max) -+ mas->max = b_node->pivot[i - 1]; -+ -+ end = j - 1; -+ if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { -+ unsigned long max_gap = 0; -+ unsigned char offset = 15; -+ -+ gaps = ma_gaps(node, mt); -+ do { -+ gaps[--j] = b_node->gap[--i]; -+ if (gaps[j] > max_gap) { -+ offset = j; -+ max_gap = gaps[j]; -+ } -+ } while (j); -+ -+ ma_set_meta(node, mt, offset, end); -+ } else { -+ mas_leaf_set_meta(mas, node, pivots, mt, end); -+ } -+} -+ -+/* -+ * mas_descend_adopt() - Descend through a sub-tree and adopt children. -+ * @mas: the maple state with the maple encoded node of the sub-tree. -+ * -+ * Descend through a sub-tree and adopt children who do not have the correct -+ * parents set. Follow the parents which have the correct parents as they are -+ * the new entries which need to be followed to find other incorrectly set -+ * parents. -+ */ -+static inline void mas_descend_adopt(struct ma_state *mas) -+{ -+ struct ma_state list[3], next[3]; -+ int i, n; -+ -+ /* -+ * At each level there may be up to 3 correct parent pointers which indicates -+ * the new nodes which need to be walked to find any new nodes at a lower level. -+ */ -+ -+ for (i = 0; i < 3; i++) { -+ list[i] = *mas; -+ list[i].offset = 0; -+ next[i].offset = 0; -+ } -+ next[0] = *mas; -+ -+ while (!mte_is_leaf(list[0].node)) { -+ n = 0; -+ for (i = 0; i < 3; i++) { -+ if (mas_is_none(&list[i])) -+ continue; -+ -+ if (i && list[i-1].node == list[i].node) -+ continue; -+ -+ while ((n < 3) && (mas_new_child(&list[i], &next[n]))) -+ n++; -+ -+ mas_adopt_children(&list[i], list[i].node); -+ } -+ -+ while (n < 3) -+ next[n++].node = MAS_NONE; -+ -+ /* descend by setting the list to the children */ -+ for (i = 0; i < 3; i++) -+ list[i] = next[i]; -+ } -+} -+ -+/* -+ * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. -+ * @mas: The maple state -+ * @end: The maple node end -+ * @mt: The maple node type -+ */ -+static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, -+ enum maple_type mt) -+{ -+ if (!(mas->mas_flags & MA_STATE_BULK)) -+ return; -+ -+ if (mte_is_root(mas->node)) -+ return; -+ -+ if (end > mt_min_slots[mt]) { -+ mas->mas_flags &= ~MA_STATE_REBALANCE; -+ return; -+ } -+} -+ -+/* -+ * mas_store_b_node() - Store an @entry into the b_node while also copying the -+ * data from a maple encoded node. -+ * @wr_mas: the maple write state -+ * @b_node: the maple_big_node to fill with data -+ * @offset_end: the offset to end copying -+ * -+ * Return: The actual end of the data stored in @b_node -+ */ -+static inline void mas_store_b_node(struct ma_wr_state *wr_mas, -+ struct maple_big_node *b_node, unsigned char offset_end) -+{ -+ unsigned char slot; -+ unsigned char b_end; -+ /* Possible underflow of piv will wrap back to 0 before use. */ -+ unsigned long piv; -+ struct ma_state *mas = wr_mas->mas; -+ -+ b_node->type = wr_mas->type; -+ b_end = 0; -+ slot = mas->offset; -+ if (slot) { -+ /* Copy start data up to insert. */ -+ mas_mab_cp(mas, 0, slot - 1, b_node, 0); -+ b_end = b_node->b_end; -+ piv = b_node->pivot[b_end - 1]; -+ } else -+ piv = mas->min - 1; -+ -+ if (piv + 1 < mas->index) { -+ /* Handle range starting after old range */ -+ b_node->slot[b_end] = wr_mas->content; -+ if (!wr_mas->content) -+ b_node->gap[b_end] = mas->index - 1 - piv; -+ b_node->pivot[b_end++] = mas->index - 1; -+ } -+ -+ /* Store the new entry. */ -+ mas->offset = b_end; -+ b_node->slot[b_end] = wr_mas->entry; -+ b_node->pivot[b_end] = mas->last; -+ -+ /* Appended. */ -+ if (mas->last >= mas->max) -+ goto b_end; -+ -+ /* Handle new range ending before old range ends */ -+ piv = mas_logical_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); -+ if (piv > mas->last) { -+ if (piv == ULONG_MAX) -+ mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type); -+ -+ if (offset_end != slot) -+ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, -+ offset_end); -+ -+ b_node->slot[++b_end] = wr_mas->content; -+ if (!wr_mas->content) -+ b_node->gap[b_end] = piv - mas->last + 1; -+ b_node->pivot[b_end] = piv; -+ } -+ -+ slot = offset_end + 1; -+ if (slot > wr_mas->node_end) -+ goto b_end; -+ -+ /* Copy end data to the end of the node. */ -+ mas_mab_cp(mas, slot, wr_mas->node_end + 1, b_node, ++b_end); -+ b_node->b_end--; -+ return; -+ -+b_end: -+ b_node->b_end = b_end; -+} -+ -+/* -+ * mas_prev_sibling() - Find the previous node with the same parent. -+ * @mas: the maple state -+ * -+ * Return: True if there is a previous sibling, false otherwise. -+ */ -+static inline bool mas_prev_sibling(struct ma_state *mas) -+{ -+ unsigned int p_slot = mte_parent_slot(mas->node); -+ -+ if (mte_is_root(mas->node)) -+ return false; -+ -+ if (!p_slot) -+ return false; -+ -+ mas_ascend(mas); -+ mas->offset = p_slot - 1; -+ mas_descend(mas); -+ return true; -+} -+ -+/* -+ * mas_next_sibling() - Find the next node with the same parent. -+ * @mas: the maple state -+ * -+ * Return: true if there is a next sibling, false otherwise. -+ */ -+static inline bool mas_next_sibling(struct ma_state *mas) -+{ -+ MA_STATE(parent, mas->tree, mas->index, mas->last); -+ -+ if (mte_is_root(mas->node)) -+ return false; -+ -+ parent = *mas; -+ mas_ascend(&parent); -+ parent.offset = mte_parent_slot(mas->node) + 1; -+ if (parent.offset > mas_data_end(&parent)) -+ return false; -+ -+ *mas = parent; -+ mas_descend(mas); -+ return true; -+} -+ -+/* -+ * mte_node_or_node() - Return the encoded node or MAS_NONE. -+ * @enode: The encoded maple node. -+ * -+ * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state. -+ * -+ * Return: @enode or MAS_NONE -+ */ -+static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) -+{ -+ if (enode) -+ return enode; -+ -+ return ma_enode_ptr(MAS_NONE); -+} -+ -+/* -+ * mas_wr_node_walk() - Find the correct offset for the index in the @mas. -+ * @wr_mas: The maple write state -+ * -+ * Uses mas_slot_locked() and does not need to worry about dead nodes. -+ */ -+static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ unsigned char count; -+ unsigned char offset; -+ unsigned long index, min, max; -+ -+ if (unlikely(ma_is_dense(wr_mas->type))) { -+ wr_mas->r_max = wr_mas->r_min = mas->index; -+ mas->offset = mas->index = mas->min; -+ return; -+ } -+ -+ wr_mas->node = mas_mn(wr_mas->mas); -+ wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); -+ count = wr_mas->node_end = ma_data_end(wr_mas->node, wr_mas->type, -+ wr_mas->pivots, mas->max); -+ offset = mas->offset; -+ min = mas_safe_min(mas, wr_mas->pivots, offset); -+ if (unlikely(offset == count)) -+ goto max; -+ -+ max = wr_mas->pivots[offset]; -+ index = mas->index; -+ if (unlikely(index <= max)) -+ goto done; -+ -+ if (unlikely(!max && offset)) -+ goto max; -+ -+ min = max + 1; -+ while (++offset < count) { -+ max = wr_mas->pivots[offset]; -+ if (index <= max) -+ goto done; -+ else if (unlikely(!max)) -+ break; -+ -+ min = max + 1; -+ } -+ -+max: -+ max = mas->max; -+done: -+ wr_mas->r_max = max; -+ wr_mas->r_min = min; -+ wr_mas->offset_end = mas->offset = offset; -+} -+ -+/* -+ * mas_topiary_range() - Add a range of slots to the topiary. -+ * @mas: The maple state -+ * @destroy: The topiary to add the slots (usually destroy) -+ * @start: The starting slot inclusively -+ * @end: The end slot inclusively -+ */ -+static inline void mas_topiary_range(struct ma_state *mas, -+ struct ma_topiary *destroy, unsigned char start, unsigned char end) -+{ -+ void __rcu **slots; -+ unsigned char offset; -+ -+ MT_BUG_ON(mas->tree, mte_is_leaf(mas->node)); -+ slots = ma_slots(mas_mn(mas), mte_node_type(mas->node)); -+ for (offset = start; offset <= end; offset++) { -+ struct maple_enode *enode = mas_slot_locked(mas, slots, offset); -+ -+ if (mte_dead_node(enode)) -+ continue; -+ -+ mat_add(destroy, enode); -+ } -+} -+ -+/* -+ * mast_topiary() - Add the portions of the tree to the removal list; either to -+ * be freed or discarded (destroy walk). -+ * @mast: The maple_subtree_state. -+ */ -+static inline void mast_topiary(struct maple_subtree_state *mast) -+{ -+ MA_WR_STATE(wr_mas, mast->orig_l, NULL); -+ unsigned char r_start, r_end; -+ unsigned char l_start, l_end; -+ void __rcu **l_slots, **r_slots; -+ -+ wr_mas.type = mte_node_type(mast->orig_l->node); -+ mast->orig_l->index = mast->orig_l->last; -+ mas_wr_node_walk(&wr_mas); -+ l_start = mast->orig_l->offset + 1; -+ l_end = mas_data_end(mast->orig_l); -+ r_start = 0; -+ r_end = mast->orig_r->offset; -+ -+ if (r_end) -+ r_end--; -+ -+ l_slots = ma_slots(mas_mn(mast->orig_l), -+ mte_node_type(mast->orig_l->node)); -+ -+ r_slots = ma_slots(mas_mn(mast->orig_r), -+ mte_node_type(mast->orig_r->node)); -+ -+ if ((l_start < l_end) && -+ mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_start))) { -+ l_start++; -+ } -+ -+ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_end))) { -+ if (r_end) -+ r_end--; -+ } -+ -+ if ((l_start > r_end) && (mast->orig_l->node == mast->orig_r->node)) -+ return; -+ -+ /* At the node where left and right sides meet, add the parts between */ -+ if (mast->orig_l->node == mast->orig_r->node) { -+ return mas_topiary_range(mast->orig_l, mast->destroy, -+ l_start, r_end); -+ } -+ -+ /* mast->orig_r is different and consumed. */ -+ if (mte_is_leaf(mast->orig_r->node)) -+ return; -+ -+ if (mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_end))) -+ l_end--; -+ -+ -+ if (l_start <= l_end) -+ mas_topiary_range(mast->orig_l, mast->destroy, l_start, l_end); -+ -+ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_start))) -+ r_start++; -+ -+ if (r_start <= r_end) -+ mas_topiary_range(mast->orig_r, mast->destroy, 0, r_end); -+} -+ -+/* -+ * mast_rebalance_next() - Rebalance against the next node -+ * @mast: The maple subtree state -+ * @old_r: The encoded maple node to the right (next node). -+ */ -+static inline void mast_rebalance_next(struct maple_subtree_state *mast) -+{ -+ unsigned char b_end = mast->bn->b_end; -+ -+ mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), -+ mast->bn, b_end); -+ mast->orig_r->last = mast->orig_r->max; -+} -+ -+/* -+ * mast_rebalance_prev() - Rebalance against the previous node -+ * @mast: The maple subtree state -+ * @old_l: The encoded maple node to the left (previous node) -+ */ -+static inline void mast_rebalance_prev(struct maple_subtree_state *mast) -+{ -+ unsigned char end = mas_data_end(mast->orig_l) + 1; -+ unsigned char b_end = mast->bn->b_end; -+ -+ mab_shift_right(mast->bn, end); -+ mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); -+ mast->l->min = mast->orig_l->min; -+ mast->orig_l->index = mast->orig_l->min; -+ mast->bn->b_end = end + b_end; -+ mast->l->offset += end; -+} -+ -+/* -+ * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring -+ * the node to the right. Checking the nodes to the right then the left at each -+ * level upwards until root is reached. Free and destroy as needed. -+ * Data is copied into the @mast->bn. -+ * @mast: The maple_subtree_state. -+ */ -+static inline -+bool mast_spanning_rebalance(struct maple_subtree_state *mast) -+{ -+ struct ma_state r_tmp = *mast->orig_r; -+ struct ma_state l_tmp = *mast->orig_l; -+ struct maple_enode *ancestor = NULL; -+ unsigned char start, end; -+ unsigned char depth = 0; -+ -+ r_tmp = *mast->orig_r; -+ l_tmp = *mast->orig_l; -+ do { -+ mas_ascend(mast->orig_r); -+ mas_ascend(mast->orig_l); -+ depth++; -+ if (!ancestor && -+ (mast->orig_r->node == mast->orig_l->node)) { -+ ancestor = mast->orig_r->node; -+ end = mast->orig_r->offset - 1; -+ start = mast->orig_l->offset + 1; -+ } -+ -+ if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { -+ if (!ancestor) { -+ ancestor = mast->orig_r->node; -+ start = 0; -+ } -+ -+ mast->orig_r->offset++; -+ do { -+ mas_descend(mast->orig_r); -+ mast->orig_r->offset = 0; -+ depth--; -+ } while (depth); -+ -+ mast_rebalance_next(mast); -+ do { -+ unsigned char l_off = 0; -+ struct maple_enode *child = r_tmp.node; -+ -+ mas_ascend(&r_tmp); -+ if (ancestor == r_tmp.node) -+ l_off = start; -+ -+ if (r_tmp.offset) -+ r_tmp.offset--; -+ -+ if (l_off < r_tmp.offset) -+ mas_topiary_range(&r_tmp, mast->destroy, -+ l_off, r_tmp.offset); -+ -+ if (l_tmp.node != child) -+ mat_add(mast->free, child); -+ -+ } while (r_tmp.node != ancestor); -+ -+ *mast->orig_l = l_tmp; -+ return true; -+ -+ } else if (mast->orig_l->offset != 0) { -+ if (!ancestor) { -+ ancestor = mast->orig_l->node; -+ end = mas_data_end(mast->orig_l); -+ } -+ -+ mast->orig_l->offset--; -+ do { -+ mas_descend(mast->orig_l); -+ mast->orig_l->offset = -+ mas_data_end(mast->orig_l); -+ depth--; -+ } while (depth); -+ -+ mast_rebalance_prev(mast); -+ do { -+ unsigned char r_off; -+ struct maple_enode *child = l_tmp.node; -+ -+ mas_ascend(&l_tmp); -+ if (ancestor == l_tmp.node) -+ r_off = end; -+ else -+ r_off = mas_data_end(&l_tmp); -+ -+ if (l_tmp.offset < r_off) -+ l_tmp.offset++; -+ -+ if (l_tmp.offset < r_off) -+ mas_topiary_range(&l_tmp, mast->destroy, -+ l_tmp.offset, r_off); -+ -+ if (r_tmp.node != child) -+ mat_add(mast->free, child); -+ -+ } while (l_tmp.node != ancestor); -+ -+ *mast->orig_r = r_tmp; -+ return true; -+ } -+ } while (!mte_is_root(mast->orig_r->node)); -+ -+ *mast->orig_r = r_tmp; -+ *mast->orig_l = l_tmp; -+ return false; -+} -+ -+/* -+ * mast_ascend_free() - Add current original maple state nodes to the free list -+ * and ascend. -+ * @mast: the maple subtree state. -+ * -+ * Ascend the original left and right sides and add the previous nodes to the -+ * free list. Set the slots to point to the correct location in the new nodes. -+ */ -+static inline void -+mast_ascend_free(struct maple_subtree_state *mast) -+{ -+ MA_WR_STATE(wr_mas, mast->orig_r, NULL); -+ struct maple_enode *left = mast->orig_l->node; -+ struct maple_enode *right = mast->orig_r->node; -+ -+ mas_ascend(mast->orig_l); -+ mas_ascend(mast->orig_r); -+ mat_add(mast->free, left); -+ -+ if (left != right) -+ mat_add(mast->free, right); -+ -+ mast->orig_r->offset = 0; -+ mast->orig_r->index = mast->r->max; -+ /* last should be larger than or equal to index */ -+ if (mast->orig_r->last < mast->orig_r->index) -+ mast->orig_r->last = mast->orig_r->index; -+ /* -+ * The node may not contain the value so set slot to ensure all -+ * of the nodes contents are freed or destroyed. -+ */ -+ wr_mas.type = mte_node_type(mast->orig_r->node); -+ mas_wr_node_walk(&wr_mas); -+ /* Set up the left side of things */ -+ mast->orig_l->offset = 0; -+ mast->orig_l->index = mast->l->min; -+ wr_mas.mas = mast->orig_l; -+ wr_mas.type = mte_node_type(mast->orig_l->node); -+ mas_wr_node_walk(&wr_mas); -+ -+ mast->bn->type = wr_mas.type; -+} -+ -+/* -+ * mas_new_ma_node() - Create and return a new maple node. Helper function. -+ * @mas: the maple state with the allocations. -+ * @b_node: the maple_big_node with the type encoding. -+ * -+ * Use the node type from the maple_big_node to allocate a new node from the -+ * ma_state. This function exists mainly for code readability. -+ * -+ * Return: A new maple encoded node -+ */ -+static inline struct maple_enode -+*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) -+{ -+ return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); -+} -+ -+/* -+ * mas_mab_to_node() - Set up right and middle nodes -+ * -+ * @mas: the maple state that contains the allocations. -+ * @b_node: the node which contains the data. -+ * @left: The pointer which will have the left node -+ * @right: The pointer which may have the right node -+ * @middle: the pointer which may have the middle node (rare) -+ * @mid_split: the split location for the middle node -+ * -+ * Return: the split of left. -+ */ -+static inline unsigned char mas_mab_to_node(struct ma_state *mas, -+ struct maple_big_node *b_node, struct maple_enode **left, -+ struct maple_enode **right, struct maple_enode **middle, -+ unsigned char *mid_split, unsigned long min) -+{ -+ unsigned char split = 0; -+ unsigned char slot_count = mt_slots[b_node->type]; -+ -+ *left = mas_new_ma_node(mas, b_node); -+ *right = NULL; -+ *middle = NULL; -+ *mid_split = 0; -+ -+ if (b_node->b_end < slot_count) { -+ split = b_node->b_end; -+ } else { -+ split = mab_calc_split(mas, b_node, mid_split, min); -+ *right = mas_new_ma_node(mas, b_node); -+ } -+ -+ if (*mid_split) -+ *middle = mas_new_ma_node(mas, b_node); -+ -+ return split; -+ -+} -+ -+/* -+ * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end -+ * pointer. -+ * @b_node - the big node to add the entry -+ * @mas - the maple state to get the pivot (mas->max) -+ * @entry - the entry to add, if NULL nothing happens. -+ */ -+static inline void mab_set_b_end(struct maple_big_node *b_node, -+ struct ma_state *mas, -+ void *entry) -+{ -+ if (!entry) -+ return; -+ -+ b_node->slot[b_node->b_end] = entry; -+ if (mt_is_alloc(mas->tree)) -+ b_node->gap[b_node->b_end] = mas_max_gap(mas); -+ b_node->pivot[b_node->b_end++] = mas->max; -+} -+ -+/* -+ * mas_set_split_parent() - combine_then_separate helper function. Sets the parent -+ * of @mas->node to either @left or @right, depending on @slot and @split -+ * -+ * @mas - the maple state with the node that needs a parent -+ * @left - possible parent 1 -+ * @right - possible parent 2 -+ * @slot - the slot the mas->node was placed -+ * @split - the split location between @left and @right -+ */ -+static inline void mas_set_split_parent(struct ma_state *mas, -+ struct maple_enode *left, -+ struct maple_enode *right, -+ unsigned char *slot, unsigned char split) -+{ -+ if (mas_is_none(mas)) -+ return; -+ -+ if ((*slot) <= split) -+ mte_set_parent(mas->node, left, *slot); -+ else if (right) -+ mte_set_parent(mas->node, right, (*slot) - split - 1); -+ -+ (*slot)++; -+} -+ -+/* -+ * mte_mid_split_check() - Check if the next node passes the mid-split -+ * @**l: Pointer to left encoded maple node. -+ * @**m: Pointer to middle encoded maple node. -+ * @**r: Pointer to right encoded maple node. -+ * @slot: The offset -+ * @*split: The split location. -+ * @mid_split: The middle split. -+ */ -+static inline void mte_mid_split_check(struct maple_enode **l, -+ struct maple_enode **r, -+ struct maple_enode *right, -+ unsigned char slot, -+ unsigned char *split, -+ unsigned char mid_split) -+{ -+ if (*r == right) -+ return; -+ -+ if (slot < mid_split) -+ return; -+ -+ *l = *r; -+ *r = right; -+ *split = mid_split; -+} -+ -+/* -+ * mast_set_split_parents() - Helper function to set three nodes parents. Slot -+ * is taken from @mast->l. -+ * @mast - the maple subtree state -+ * @left - the left node -+ * @right - the right node -+ * @split - the split location. -+ */ -+static inline void mast_set_split_parents(struct maple_subtree_state *mast, -+ struct maple_enode *left, -+ struct maple_enode *middle, -+ struct maple_enode *right, -+ unsigned char split, -+ unsigned char mid_split) -+{ -+ unsigned char slot; -+ struct maple_enode *l = left; -+ struct maple_enode *r = right; -+ -+ if (mas_is_none(mast->l)) -+ return; -+ -+ if (middle) -+ r = middle; -+ -+ slot = mast->l->offset; -+ -+ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); -+ mas_set_split_parent(mast->l, l, r, &slot, split); -+ -+ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); -+ mas_set_split_parent(mast->m, l, r, &slot, split); -+ -+ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); -+ mas_set_split_parent(mast->r, l, r, &slot, split); -+} -+ -+/* -+ * mas_wmb_replace() - Write memory barrier and replace -+ * @mas: The maple state -+ * @free: the maple topiary list of nodes to free -+ * @destroy: The maple topiary list of nodes to destroy (walk and free) -+ * -+ * Updates gap as necessary. -+ */ -+static inline void mas_wmb_replace(struct ma_state *mas, -+ struct ma_topiary *free, -+ struct ma_topiary *destroy) -+{ -+ /* All nodes must see old data as dead prior to replacing that data */ -+ smp_wmb(); /* Needed for RCU */ -+ -+ /* Insert the new data in the tree */ -+ mas_replace(mas, true); -+ -+ if (!mte_is_leaf(mas->node)) -+ mas_descend_adopt(mas); -+ -+ mas_mat_free(mas, free); -+ -+ if (destroy) -+ mas_mat_destroy(mas, destroy); -+ -+ if (mte_is_leaf(mas->node)) -+ return; -+ -+ mas_update_gap(mas); -+} -+ -+/* -+ * mast_new_root() - Set a new tree root during subtree creation -+ * @mast: The maple subtree state -+ * @mas: The maple state -+ */ -+static inline void mast_new_root(struct maple_subtree_state *mast, -+ struct ma_state *mas) -+{ -+ mas_mn(mast->l)->parent = -+ ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT)); -+ if (!mte_dead_node(mast->orig_l->node) && -+ !mte_is_root(mast->orig_l->node)) { -+ do { -+ mast_ascend_free(mast); -+ mast_topiary(mast); -+ } while (!mte_is_root(mast->orig_l->node)); -+ } -+ if ((mast->orig_l->node != mas->node) && -+ (mast->l->depth > mas_mt_height(mas))) { -+ mat_add(mast->free, mas->node); -+ } -+} -+ -+/* -+ * mast_cp_to_nodes() - Copy data out to nodes. -+ * @mast: The maple subtree state -+ * @left: The left encoded maple node -+ * @middle: The middle encoded maple node -+ * @right: The right encoded maple node -+ * @split: The location to split between left and (middle ? middle : right) -+ * @mid_split: The location to split between middle and right. -+ */ -+static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, -+ struct maple_enode *left, struct maple_enode *middle, -+ struct maple_enode *right, unsigned char split, unsigned char mid_split) -+{ -+ bool new_lmax = true; -+ -+ mast->l->node = mte_node_or_none(left); -+ mast->m->node = mte_node_or_none(middle); -+ mast->r->node = mte_node_or_none(right); -+ -+ mast->l->min = mast->orig_l->min; -+ if (split == mast->bn->b_end) { -+ mast->l->max = mast->orig_r->max; -+ new_lmax = false; -+ } -+ -+ mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); -+ -+ if (middle) { -+ mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); -+ mast->m->min = mast->bn->pivot[split] + 1; -+ split = mid_split; -+ } -+ -+ mast->r->max = mast->orig_r->max; -+ if (right) { -+ mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); -+ mast->r->min = mast->bn->pivot[split] + 1; -+ } -+} -+ -+/* -+ * mast_combine_cp_left - Copy in the original left side of the tree into the -+ * combined data set in the maple subtree state big node. -+ * @mast: The maple subtree state -+ */ -+static inline void mast_combine_cp_left(struct maple_subtree_state *mast) -+{ -+ unsigned char l_slot = mast->orig_l->offset; -+ -+ if (!l_slot) -+ return; -+ -+ mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); -+} -+ -+/* -+ * mast_combine_cp_right: Copy in the original right side of the tree into the -+ * combined data set in the maple subtree state big node. -+ * @mast: The maple subtree state -+ */ -+static inline void mast_combine_cp_right(struct maple_subtree_state *mast) -+{ -+ if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) -+ return; -+ -+ mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, -+ mt_slot_count(mast->orig_r->node), mast->bn, -+ mast->bn->b_end); -+ mast->orig_r->last = mast->orig_r->max; -+} -+ -+/* -+ * mast_sufficient: Check if the maple subtree state has enough data in the big -+ * node to create at least one sufficient node -+ * @mast: the maple subtree state -+ */ -+static inline bool mast_sufficient(struct maple_subtree_state *mast) -+{ -+ if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) -+ return true; -+ -+ return false; -+} -+ -+/* -+ * mast_overflow: Check if there is too much data in the subtree state for a -+ * single node. -+ * @mast: The maple subtree state -+ */ -+static inline bool mast_overflow(struct maple_subtree_state *mast) -+{ -+ if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node)) -+ return true; -+ -+ return false; -+} -+ -+static inline void *mtree_range_walk(struct ma_state *mas) -+{ -+ unsigned long *pivots; -+ unsigned char offset; -+ struct maple_node *node; -+ struct maple_enode *next, *last; -+ enum maple_type type; -+ void __rcu **slots; -+ unsigned char end; -+ unsigned long max, min; -+ unsigned long prev_max, prev_min; -+ -+ last = next = mas->node; -+ prev_min = min = mas->min; -+ max = mas->max; -+ do { -+ offset = 0; -+ last = next; -+ node = mte_to_node(next); -+ type = mte_node_type(next); -+ pivots = ma_pivots(node, type); -+ end = ma_data_end(node, type, pivots, max); -+ if (unlikely(ma_dead_node(node))) -+ goto dead_node; -+ -+ if (pivots[offset] >= mas->index) { -+ prev_max = max; -+ prev_min = min; -+ max = pivots[offset]; -+ goto next; -+ } -+ -+ do { -+ offset++; -+ } while ((offset < end) && (pivots[offset] < mas->index)); -+ -+ prev_min = min; -+ min = pivots[offset - 1] + 1; -+ prev_max = max; -+ if (likely(offset < end && pivots[offset])) -+ max = pivots[offset]; -+ -+next: -+ slots = ma_slots(node, type); -+ next = mt_slot(mas->tree, slots, offset); -+ if (unlikely(ma_dead_node(node))) -+ goto dead_node; -+ } while (!ma_is_leaf(type)); -+ -+ mas->offset = offset; -+ mas->index = min; -+ mas->last = max; -+ mas->min = prev_min; -+ mas->max = prev_max; -+ mas->node = last; -+ return (void *) next; -+ -+dead_node: -+ mas_reset(mas); -+ return NULL; -+} -+ -+/* -+ * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. -+ * @mas: The starting maple state -+ * @mast: The maple_subtree_state, keeps track of 4 maple states. -+ * @count: The estimated count of iterations needed. -+ * -+ * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root -+ * is hit. First @b_node is split into two entries which are inserted into the -+ * next iteration of the loop. @b_node is returned populated with the final -+ * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the -+ * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last -+ * to account of what has been copied into the new sub-tree. The update of -+ * orig_l_mas->last is used in mas_consume to find the slots that will need to -+ * be either freed or destroyed. orig_l_mas->depth keeps track of the height of -+ * the new sub-tree in case the sub-tree becomes the full tree. -+ * -+ * Return: the number of elements in b_node during the last loop. -+ */ -+static int mas_spanning_rebalance(struct ma_state *mas, -+ struct maple_subtree_state *mast, unsigned char count) -+{ -+ unsigned char split, mid_split; -+ unsigned char slot = 0; -+ struct maple_enode *left = NULL, *middle = NULL, *right = NULL; -+ -+ MA_STATE(l_mas, mas->tree, mas->index, mas->index); -+ MA_STATE(r_mas, mas->tree, mas->index, mas->last); -+ MA_STATE(m_mas, mas->tree, mas->index, mas->index); -+ MA_TOPIARY(free, mas->tree); -+ MA_TOPIARY(destroy, mas->tree); -+ -+ /* -+ * The tree needs to be rebalanced and leaves need to be kept at the same level. -+ * Rebalancing is done by use of the ``struct maple_topiary``. -+ */ -+ mast->l = &l_mas; -+ mast->m = &m_mas; -+ mast->r = &r_mas; -+ mast->free = &free; -+ mast->destroy = &destroy; -+ l_mas.node = r_mas.node = m_mas.node = MAS_NONE; -+ if (!(mast->orig_l->min && mast->orig_r->max == ULONG_MAX) && -+ unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) -+ mast_spanning_rebalance(mast); -+ -+ mast->orig_l->depth = 0; -+ -+ /* -+ * Each level of the tree is examined and balanced, pushing data to the left or -+ * right, or rebalancing against left or right nodes is employed to avoid -+ * rippling up the tree to limit the amount of churn. Once a new sub-section of -+ * the tree is created, there may be a mix of new and old nodes. The old nodes -+ * will have the incorrect parent pointers and currently be in two trees: the -+ * original tree and the partially new tree. To remedy the parent pointers in -+ * the old tree, the new data is swapped into the active tree and a walk down -+ * the tree is performed and the parent pointers are updated. -+ * See mas_descend_adopt() for more information.. -+ */ -+ while (count--) { -+ mast->bn->b_end--; -+ mast->bn->type = mte_node_type(mast->orig_l->node); -+ split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle, -+ &mid_split, mast->orig_l->min); -+ mast_set_split_parents(mast, left, middle, right, split, -+ mid_split); -+ mast_cp_to_nodes(mast, left, middle, right, split, mid_split); -+ -+ /* -+ * Copy data from next level in the tree to mast->bn from next -+ * iteration -+ */ -+ memset(mast->bn, 0, sizeof(struct maple_big_node)); -+ mast->bn->type = mte_node_type(left); -+ mast->orig_l->depth++; -+ -+ /* Root already stored in l->node. */ -+ if (mas_is_root_limits(mast->l)) -+ goto new_root; -+ -+ mast_ascend_free(mast); -+ mast_combine_cp_left(mast); -+ l_mas.offset = mast->bn->b_end; -+ mab_set_b_end(mast->bn, &l_mas, left); -+ mab_set_b_end(mast->bn, &m_mas, middle); -+ mab_set_b_end(mast->bn, &r_mas, right); -+ -+ /* Copy anything necessary out of the right node. */ -+ mast_combine_cp_right(mast); -+ mast_topiary(mast); -+ mast->orig_l->last = mast->orig_l->max; -+ -+ if (mast_sufficient(mast)) -+ continue; -+ -+ if (mast_overflow(mast)) -+ continue; -+ -+ /* May be a new root stored in mast->bn */ -+ if (mas_is_root_limits(mast->orig_l)) -+ break; -+ -+ mast_spanning_rebalance(mast); -+ -+ /* rebalancing from other nodes may require another loop. */ -+ if (!count) -+ count++; -+ } -+ -+ l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), -+ mte_node_type(mast->orig_l->node)); -+ mast->orig_l->depth++; -+ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); -+ mte_set_parent(left, l_mas.node, slot); -+ if (middle) -+ mte_set_parent(middle, l_mas.node, ++slot); -+ -+ if (right) -+ mte_set_parent(right, l_mas.node, ++slot); -+ -+ if (mas_is_root_limits(mast->l)) { -+new_root: -+ mast_new_root(mast, mas); -+ } else { -+ mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; -+ } -+ -+ if (!mte_dead_node(mast->orig_l->node)) -+ mat_add(&free, mast->orig_l->node); -+ -+ mas->depth = mast->orig_l->depth; -+ *mast->orig_l = l_mas; -+ mte_set_node_dead(mas->node); -+ -+ /* Set up mas for insertion. */ -+ mast->orig_l->depth = mas->depth; -+ mast->orig_l->alloc = mas->alloc; -+ *mas = *mast->orig_l; -+ mas_wmb_replace(mas, &free, &destroy); -+ mtree_range_walk(mas); -+ return mast->bn->b_end; -+} -+ -+/* -+ * mas_rebalance() - Rebalance a given node. -+ * @mas: The maple state -+ * @b_node: The big maple node. -+ * -+ * Rebalance two nodes into a single node or two new nodes that are sufficient. -+ * Continue upwards until tree is sufficient. -+ * -+ * Return: the number of elements in b_node during the last loop. -+ */ -+static inline int mas_rebalance(struct ma_state *mas, -+ struct maple_big_node *b_node) -+{ -+ char empty_count = mas_mt_height(mas); -+ struct maple_subtree_state mast; -+ unsigned char shift, b_end = ++b_node->b_end; -+ -+ MA_STATE(l_mas, mas->tree, mas->index, mas->last); -+ MA_STATE(r_mas, mas->tree, mas->index, mas->last); -+ -+ trace_ma_op(__func__, mas); -+ -+ /* -+ * Rebalancing occurs if a node is insufficient. Data is rebalanced -+ * against the node to the right if it exists, otherwise the node to the -+ * left of this node is rebalanced against this node. If rebalancing -+ * causes just one node to be produced instead of two, then the parent -+ * is also examined and rebalanced if it is insufficient. Every level -+ * tries to combine the data in the same way. If one node contains the -+ * entire range of the tree, then that node is used as a new root node. -+ */ -+ mas_node_count(mas, 1 + empty_count * 3); -+ if (mas_is_err(mas)) -+ return 0; -+ -+ mast.orig_l = &l_mas; -+ mast.orig_r = &r_mas; -+ mast.bn = b_node; -+ mast.bn->type = mte_node_type(mas->node); -+ -+ l_mas = r_mas = *mas; -+ -+ if (mas_next_sibling(&r_mas)) { -+ mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); -+ r_mas.last = r_mas.index = r_mas.max; -+ } else { -+ mas_prev_sibling(&l_mas); -+ shift = mas_data_end(&l_mas) + 1; -+ mab_shift_right(b_node, shift); -+ mas->offset += shift; -+ mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); -+ b_node->b_end = shift + b_end; -+ l_mas.index = l_mas.last = l_mas.min; -+ } -+ -+ return mas_spanning_rebalance(mas, &mast, empty_count); -+} -+ -+/* -+ * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple -+ * state. -+ * @mas: The maple state -+ * @end: The end of the left-most node. -+ * -+ * During a mass-insert event (such as forking), it may be necessary to -+ * rebalance the left-most node when it is not sufficient. -+ */ -+static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end) -+{ -+ enum maple_type mt = mte_node_type(mas->node); -+ struct maple_node reuse, *newnode, *parent, *new_left, *left, *node; -+ struct maple_enode *eparent; -+ unsigned char offset, tmp, split = mt_slots[mt] / 2; -+ void __rcu **l_slots, **slots; -+ unsigned long *l_pivs, *pivs, gap; -+ bool in_rcu = mt_in_rcu(mas->tree); -+ -+ MA_STATE(l_mas, mas->tree, mas->index, mas->last); -+ -+ l_mas = *mas; -+ mas_prev_sibling(&l_mas); -+ -+ /* set up node. */ -+ if (in_rcu) { -+ /* Allocate for both left and right as well as parent. */ -+ mas_node_count(mas, 3); -+ if (mas_is_err(mas)) -+ return; -+ -+ newnode = mas_pop_node(mas); -+ } else { -+ newnode = &reuse; -+ } -+ -+ node = mas_mn(mas); -+ newnode->parent = node->parent; -+ slots = ma_slots(newnode, mt); -+ pivs = ma_pivots(newnode, mt); -+ left = mas_mn(&l_mas); -+ l_slots = ma_slots(left, mt); -+ l_pivs = ma_pivots(left, mt); -+ if (!l_slots[split]) -+ split++; -+ tmp = mas_data_end(&l_mas) - split; -+ -+ memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp); -+ memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp); -+ pivs[tmp] = l_mas.max; -+ memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end); -+ memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end); -+ -+ l_mas.max = l_pivs[split]; -+ mas->min = l_mas.max + 1; -+ eparent = mt_mk_node(mte_parent(l_mas.node), -+ mas_parent_enum(&l_mas, l_mas.node)); -+ tmp += end; -+ if (!in_rcu) { -+ unsigned char max_p = mt_pivots[mt]; -+ unsigned char max_s = mt_slots[mt]; -+ -+ if (tmp < max_p) -+ memset(pivs + tmp, 0, -+ sizeof(unsigned long *) * (max_p - tmp)); -+ -+ if (tmp < mt_slots[mt]) -+ memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp)); -+ -+ memcpy(node, newnode, sizeof(struct maple_node)); -+ ma_set_meta(node, mt, 0, tmp - 1); -+ mte_set_pivot(eparent, mte_parent_slot(l_mas.node), -+ l_pivs[split]); -+ -+ /* Remove data from l_pivs. */ -+ tmp = split + 1; -+ memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); -+ memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); -+ ma_set_meta(left, mt, 0, split); -+ -+ goto done; -+ } -+ -+ /* RCU requires replacing both l_mas, mas, and parent. */ -+ mas->node = mt_mk_node(newnode, mt); -+ ma_set_meta(newnode, mt, 0, tmp); -+ -+ new_left = mas_pop_node(mas); -+ new_left->parent = left->parent; -+ mt = mte_node_type(l_mas.node); -+ slots = ma_slots(new_left, mt); -+ pivs = ma_pivots(new_left, mt); -+ memcpy(slots, l_slots, sizeof(void *) * split); -+ memcpy(pivs, l_pivs, sizeof(unsigned long) * split); -+ ma_set_meta(new_left, mt, 0, split); -+ l_mas.node = mt_mk_node(new_left, mt); -+ -+ /* replace parent. */ -+ offset = mte_parent_slot(mas->node); -+ mt = mas_parent_enum(&l_mas, l_mas.node); -+ parent = mas_pop_node(mas); -+ slots = ma_slots(parent, mt); -+ pivs = ma_pivots(parent, mt); -+ memcpy(parent, mte_to_node(eparent), sizeof(struct maple_node)); -+ rcu_assign_pointer(slots[offset], mas->node); -+ rcu_assign_pointer(slots[offset - 1], l_mas.node); -+ pivs[offset - 1] = l_mas.max; -+ eparent = mt_mk_node(parent, mt); -+done: -+ gap = mas_leaf_max_gap(mas); -+ mte_set_gap(eparent, mte_parent_slot(mas->node), gap); -+ gap = mas_leaf_max_gap(&l_mas); -+ mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap); -+ mas_ascend(mas); -+ -+ if (in_rcu) -+ mas_replace(mas, false); -+ -+ mas_update_gap(mas); -+} -+ -+/* -+ * mas_split_final_node() - Split the final node in a subtree operation. -+ * @mast: the maple subtree state -+ * @mas: The maple state -+ * @height: The height of the tree in case it's a new root. -+ */ -+static inline bool mas_split_final_node(struct maple_subtree_state *mast, -+ struct ma_state *mas, int height) -+{ -+ struct maple_enode *ancestor; -+ -+ if (mte_is_root(mas->node)) { -+ if (mt_is_alloc(mas->tree)) -+ mast->bn->type = maple_arange_64; -+ else -+ mast->bn->type = maple_range_64; -+ mas->depth = height; -+ } -+ /* -+ * Only a single node is used here, could be root. -+ * The Big_node data should just fit in a single node. -+ */ -+ ancestor = mas_new_ma_node(mas, mast->bn); -+ mte_set_parent(mast->l->node, ancestor, mast->l->offset); -+ mte_set_parent(mast->r->node, ancestor, mast->r->offset); -+ mte_to_node(ancestor)->parent = mas_mn(mas)->parent; -+ -+ mast->l->node = ancestor; -+ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); -+ mas->offset = mast->bn->b_end - 1; -+ return true; -+} -+ -+/* -+ * mast_fill_bnode() - Copy data into the big node in the subtree state -+ * @mast: The maple subtree state -+ * @mas: the maple state -+ * @skip: The number of entries to skip for new nodes insertion. -+ */ -+static inline void mast_fill_bnode(struct maple_subtree_state *mast, -+ struct ma_state *mas, -+ unsigned char skip) -+{ -+ bool cp = true; -+ struct maple_enode *old = mas->node; -+ unsigned char split; -+ -+ memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap)); -+ memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot)); -+ memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot)); -+ mast->bn->b_end = 0; -+ -+ if (mte_is_root(mas->node)) { -+ cp = false; -+ } else { -+ mas_ascend(mas); -+ mat_add(mast->free, old); -+ mas->offset = mte_parent_slot(mas->node); -+ } -+ -+ if (cp && mast->l->offset) -+ mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); -+ -+ split = mast->bn->b_end; -+ mab_set_b_end(mast->bn, mast->l, mast->l->node); -+ mast->r->offset = mast->bn->b_end; -+ mab_set_b_end(mast->bn, mast->r, mast->r->node); -+ if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) -+ cp = false; -+ -+ if (cp) -+ mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, -+ mast->bn, mast->bn->b_end); -+ -+ mast->bn->b_end--; -+ mast->bn->type = mte_node_type(mas->node); -+} -+ -+/* -+ * mast_split_data() - Split the data in the subtree state big node into regular -+ * nodes. -+ * @mast: The maple subtree state -+ * @mas: The maple state -+ * @split: The location to split the big node -+ */ -+static inline void mast_split_data(struct maple_subtree_state *mast, -+ struct ma_state *mas, unsigned char split) -+{ -+ unsigned char p_slot; -+ -+ mab_mas_cp(mast->bn, 0, split, mast->l, true); -+ mte_set_pivot(mast->r->node, 0, mast->r->max); -+ mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); -+ mast->l->offset = mte_parent_slot(mas->node); -+ mast->l->max = mast->bn->pivot[split]; -+ mast->r->min = mast->l->max + 1; -+ if (mte_is_leaf(mas->node)) -+ return; -+ -+ p_slot = mast->orig_l->offset; -+ mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, -+ &p_slot, split); -+ mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, -+ &p_slot, split); -+} -+ -+/* -+ * mas_push_data() - Instead of splitting a node, it is beneficial to push the -+ * data to the right or left node if there is room. -+ * @mas: The maple state -+ * @height: The current height of the maple state -+ * @mast: The maple subtree state -+ * @left: Push left or not. -+ * -+ * Keeping the height of the tree low means faster lookups. -+ * -+ * Return: True if pushed, false otherwise. -+ */ -+static inline bool mas_push_data(struct ma_state *mas, int height, -+ struct maple_subtree_state *mast, bool left) -+{ -+ unsigned char slot_total = mast->bn->b_end; -+ unsigned char end, space, split; -+ -+ MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); -+ tmp_mas = *mas; -+ tmp_mas.depth = mast->l->depth; -+ -+ if (left && !mas_prev_sibling(&tmp_mas)) -+ return false; -+ else if (!left && !mas_next_sibling(&tmp_mas)) -+ return false; -+ -+ end = mas_data_end(&tmp_mas); -+ slot_total += end; -+ space = 2 * mt_slot_count(mas->node) - 2; -+ /* -2 instead of -1 to ensure there isn't a triple split */ -+ if (ma_is_leaf(mast->bn->type)) -+ space--; -+ -+ if (mas->max == ULONG_MAX) -+ space--; -+ -+ if (slot_total >= space) -+ return false; -+ -+ /* Get the data; Fill mast->bn */ -+ mast->bn->b_end++; -+ if (left) { -+ mab_shift_right(mast->bn, end + 1); -+ mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); -+ mast->bn->b_end = slot_total + 1; -+ } else { -+ mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); -+ } -+ -+ /* Configure mast for splitting of mast->bn */ -+ split = mt_slots[mast->bn->type] - 2; -+ if (left) { -+ /* Switch mas to prev node */ -+ mat_add(mast->free, mas->node); -+ *mas = tmp_mas; -+ /* Start using mast->l for the left side. */ -+ tmp_mas.node = mast->l->node; -+ *mast->l = tmp_mas; -+ } else { -+ mat_add(mast->free, tmp_mas.node); -+ tmp_mas.node = mast->r->node; -+ *mast->r = tmp_mas; -+ split = slot_total - split; -+ } -+ split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); -+ /* Update parent slot for split calculation. */ -+ if (left) -+ mast->orig_l->offset += end + 1; -+ -+ mast_split_data(mast, mas, split); -+ mast_fill_bnode(mast, mas, 2); -+ mas_split_final_node(mast, mas, height + 1); -+ return true; -+} -+ -+/* -+ * mas_split() - Split data that is too big for one node into two. -+ * @mas: The maple state -+ * @b_node: The maple big node -+ * Return: 1 on success, 0 on failure. -+ */ -+static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) -+{ -+ -+ struct maple_subtree_state mast; -+ int height = 0; -+ unsigned char mid_split, split = 0; -+ -+ /* -+ * Splitting is handled differently from any other B-tree; the Maple -+ * Tree splits upwards. Splitting up means that the split operation -+ * occurs when the walk of the tree hits the leaves and not on the way -+ * down. The reason for splitting up is that it is impossible to know -+ * how much space will be needed until the leaf is (or leaves are) -+ * reached. Since overwriting data is allowed and a range could -+ * overwrite more than one range or result in changing one entry into 3 -+ * entries, it is impossible to know if a split is required until the -+ * data is examined. -+ * -+ * Splitting is a balancing act between keeping allocations to a minimum -+ * and avoiding a 'jitter' event where a tree is expanded to make room -+ * for an entry followed by a contraction when the entry is removed. To -+ * accomplish the balance, there are empty slots remaining in both left -+ * and right nodes after a split. -+ */ -+ MA_STATE(l_mas, mas->tree, mas->index, mas->last); -+ MA_STATE(r_mas, mas->tree, mas->index, mas->last); -+ MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); -+ MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); -+ MA_TOPIARY(mat, mas->tree); -+ -+ trace_ma_op(__func__, mas); -+ mas->depth = mas_mt_height(mas); -+ /* Allocation failures will happen early. */ -+ mas_node_count(mas, 1 + mas->depth * 2); -+ if (mas_is_err(mas)) -+ return 0; -+ -+ mast.l = &l_mas; -+ mast.r = &r_mas; -+ mast.orig_l = &prev_l_mas; -+ mast.orig_r = &prev_r_mas; -+ mast.free = &mat; -+ mast.bn = b_node; -+ -+ while (height++ <= mas->depth) { -+ if (mt_slots[b_node->type] > b_node->b_end) { -+ mas_split_final_node(&mast, mas, height); -+ break; -+ } -+ -+ l_mas = r_mas = *mas; -+ l_mas.node = mas_new_ma_node(mas, b_node); -+ r_mas.node = mas_new_ma_node(mas, b_node); -+ /* -+ * Another way that 'jitter' is avoided is to terminate a split up early if the -+ * left or right node has space to spare. This is referred to as "pushing left" -+ * or "pushing right" and is similar to the B* tree, except the nodes left or -+ * right can rarely be reused due to RCU, but the ripple upwards is halted which -+ * is a significant savings. -+ */ -+ /* Try to push left. */ -+ if (mas_push_data(mas, height, &mast, true)) -+ break; -+ -+ /* Try to push right. */ -+ if (mas_push_data(mas, height, &mast, false)) -+ break; -+ -+ split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min); -+ mast_split_data(&mast, mas, split); -+ /* -+ * Usually correct, mab_mas_cp in the above call overwrites -+ * r->max. -+ */ -+ mast.r->max = mas->max; -+ mast_fill_bnode(&mast, mas, 1); -+ prev_l_mas = *mast.l; -+ prev_r_mas = *mast.r; -+ } -+ -+ /* Set the original node as dead */ -+ mat_add(mast.free, mas->node); -+ mas->node = l_mas.node; -+ mas_wmb_replace(mas, mast.free, NULL); -+ mtree_range_walk(mas); -+ return 1; -+} -+ -+/* -+ * mas_reuse_node() - Reuse the node to store the data. -+ * @wr_mas: The maple write state -+ * @bn: The maple big node -+ * @end: The end of the data. -+ * -+ * Will always return false in RCU mode. -+ * -+ * Return: True if node was reused, false otherwise. -+ */ -+static inline bool mas_reuse_node(struct ma_wr_state *wr_mas, -+ struct maple_big_node *bn, unsigned char end) -+{ -+ /* Need to be rcu safe. */ -+ if (mt_in_rcu(wr_mas->mas->tree)) -+ return false; -+ -+ if (end > bn->b_end) { -+ int clear = mt_slots[wr_mas->type] - bn->b_end; -+ -+ memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--); -+ memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear); -+ } -+ mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false); -+ return true; -+} -+ -+/* -+ * mas_commit_b_node() - Commit the big node into the tree. -+ * @wr_mas: The maple write state -+ * @b_node: The maple big node -+ * @end: The end of the data. -+ */ -+static inline int mas_commit_b_node(struct ma_wr_state *wr_mas, -+ struct maple_big_node *b_node, unsigned char end) -+{ -+ struct maple_node *node; -+ unsigned char b_end = b_node->b_end; -+ enum maple_type b_type = b_node->type; -+ -+ if ((b_end < mt_min_slots[b_type]) && -+ (!mte_is_root(wr_mas->mas->node)) && -+ (mas_mt_height(wr_mas->mas) > 1)) -+ return mas_rebalance(wr_mas->mas, b_node); -+ -+ if (b_end >= mt_slots[b_type]) -+ return mas_split(wr_mas->mas, b_node); -+ -+ if (mas_reuse_node(wr_mas, b_node, end)) -+ goto reuse_node; -+ -+ mas_node_count(wr_mas->mas, 1); -+ if (mas_is_err(wr_mas->mas)) -+ return 0; -+ -+ node = mas_pop_node(wr_mas->mas); -+ node->parent = mas_mn(wr_mas->mas)->parent; -+ wr_mas->mas->node = mt_mk_node(node, b_type); -+ mab_mas_cp(b_node, 0, b_end, wr_mas->mas, true); -+ -+ mas_replace(wr_mas->mas, false); -+reuse_node: -+ mas_update_gap(wr_mas->mas); -+ return 1; -+} -+ -+/* -+ * mas_root_expand() - Expand a root to a node -+ * @mas: The maple state -+ * @entry: The entry to store into the tree -+ */ -+static inline int mas_root_expand(struct ma_state *mas, void *entry) -+{ -+ void *contents = mas_root_locked(mas); -+ enum maple_type type = maple_leaf_64; -+ struct maple_node *node; -+ void __rcu **slots; -+ unsigned long *pivots; -+ int slot = 0; -+ -+ mas_node_count(mas, 1); -+ if (unlikely(mas_is_err(mas))) -+ return 0; -+ -+ node = mas_pop_node(mas); -+ pivots = ma_pivots(node, type); -+ slots = ma_slots(node, type); -+ node->parent = ma_parent_ptr( -+ ((unsigned long)mas->tree | MA_ROOT_PARENT)); -+ mas->node = mt_mk_node(node, type); -+ -+ if (mas->index) { -+ if (contents) { -+ rcu_assign_pointer(slots[slot], contents); -+ if (likely(mas->index > 1)) -+ slot++; -+ } -+ pivots[slot++] = mas->index - 1; -+ } -+ -+ rcu_assign_pointer(slots[slot], entry); -+ mas->offset = slot; -+ pivots[slot] = mas->last; -+ if (mas->last != ULONG_MAX) -+ slot++; -+ mas->depth = 1; -+ mas_set_height(mas); -+ -+ /* swap the new root into the tree */ -+ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); -+ ma_set_meta(node, maple_leaf_64, 0, slot); -+ return slot; -+} -+ -+static inline void mas_store_root(struct ma_state *mas, void *entry) -+{ -+ if (likely((mas->last != 0) || (mas->index != 0))) -+ mas_root_expand(mas, entry); -+ else if (((unsigned long) (entry) & 3) == 2) -+ mas_root_expand(mas, entry); -+ else { -+ rcu_assign_pointer(mas->tree->ma_root, entry); -+ mas->node = MAS_START; -+ } -+} -+ -+/* -+ * mas_is_span_wr() - Check if the write needs to be treated as a write that -+ * spans the node. -+ * @mas: The maple state -+ * @piv: The pivot value being written -+ * @type: The maple node type -+ * @entry: The data to write -+ * -+ * Spanning writes are writes that start in one node and end in another OR if -+ * the write of a %NULL will cause the node to end with a %NULL. -+ * -+ * Return: True if this is a spanning write, false otherwise. -+ */ -+static bool mas_is_span_wr(struct ma_wr_state *wr_mas) -+{ -+ unsigned long max; -+ unsigned long last = wr_mas->mas->last; -+ unsigned long piv = wr_mas->r_max; -+ enum maple_type type = wr_mas->type; -+ void *entry = wr_mas->entry; -+ -+ /* Contained in this pivot */ -+ if (piv > last) -+ return false; -+ -+ max = wr_mas->mas->max; -+ if (unlikely(ma_is_leaf(type))) { -+ /* Fits in the node, but may span slots. */ -+ if (last < max) -+ return false; -+ -+ /* Writes to the end of the node but not null. */ -+ if ((last == max) && entry) -+ return false; -+ -+ /* -+ * Writing ULONG_MAX is not a spanning write regardless of the -+ * value being written as long as the range fits in the node. -+ */ -+ if ((last == ULONG_MAX) && (last == max)) -+ return false; -+ } else if (piv == last) { -+ if (entry) -+ return false; -+ -+ /* Detect spanning store wr walk */ -+ if (last == ULONG_MAX) -+ return false; -+ } -+ -+ trace_ma_write(__func__, wr_mas->mas, piv, entry); -+ -+ return true; -+} -+ -+static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas) -+{ -+ wr_mas->mas->depth++; -+ wr_mas->type = mte_node_type(wr_mas->mas->node); -+ mas_wr_node_walk(wr_mas); -+ wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); -+} -+ -+static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas) -+{ -+ wr_mas->mas->max = wr_mas->r_max; -+ wr_mas->mas->min = wr_mas->r_min; -+ wr_mas->mas->node = wr_mas->content; -+ wr_mas->mas->offset = 0; -+} -+/* -+ * mas_wr_walk() - Walk the tree for a write. -+ * @wr_mas: The maple write state -+ * -+ * Uses mas_slot_locked() and does not need to worry about dead nodes. -+ * -+ * Return: True if it's contained in a node, false on spanning write. -+ */ -+static bool mas_wr_walk(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ -+ while (true) { -+ mas_wr_walk_descend(wr_mas); -+ if (unlikely(mas_is_span_wr(wr_mas))) -+ return false; -+ -+ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, -+ mas->offset); -+ if (ma_is_leaf(wr_mas->type)) -+ return true; -+ -+ mas_wr_walk_traverse(wr_mas); -+ } -+ -+ return true; -+} -+ -+static bool mas_wr_walk_index(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ -+ while (true) { -+ mas_wr_walk_descend(wr_mas); -+ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, -+ mas->offset); -+ if (ma_is_leaf(wr_mas->type)) -+ return true; -+ mas_wr_walk_traverse(wr_mas); -+ -+ } -+ return true; -+} -+/* -+ * mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs. -+ * @l_wr_mas: The left maple write state -+ * @r_wr_mas: The right maple write state -+ */ -+static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, -+ struct ma_wr_state *r_wr_mas) -+{ -+ struct ma_state *r_mas = r_wr_mas->mas; -+ struct ma_state *l_mas = l_wr_mas->mas; -+ unsigned char l_slot; -+ -+ l_slot = l_mas->offset; -+ if (!l_wr_mas->content) -+ l_mas->index = l_wr_mas->r_min; -+ -+ if ((l_mas->index == l_wr_mas->r_min) && -+ (l_slot && -+ !mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) { -+ if (l_slot > 1) -+ l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1; -+ else -+ l_mas->index = l_mas->min; -+ -+ l_mas->offset = l_slot - 1; -+ } -+ -+ if (!r_wr_mas->content) { -+ if (r_mas->last < r_wr_mas->r_max) -+ r_mas->last = r_wr_mas->r_max; -+ r_mas->offset++; -+ } else if ((r_mas->last == r_wr_mas->r_max) && -+ (r_mas->last < r_mas->max) && -+ !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { -+ r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, -+ r_wr_mas->type, r_mas->offset + 1); -+ r_mas->offset++; -+ } -+} -+ -+static inline void *mas_state_walk(struct ma_state *mas) -+{ -+ void *entry; -+ -+ entry = mas_start(mas); -+ if (mas_is_none(mas)) -+ return NULL; -+ -+ if (mas_is_ptr(mas)) -+ return entry; -+ -+ return mtree_range_walk(mas); -+} -+ -+/* -+ * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up -+ * to date. -+ * -+ * @mas: The maple state. -+ * -+ * Note: Leaves mas in undesirable state. -+ * Return: The entry for @mas->index or %NULL on dead node. -+ */ -+static inline void *mtree_lookup_walk(struct ma_state *mas) -+{ -+ unsigned long *pivots; -+ unsigned char offset; -+ struct maple_node *node; -+ struct maple_enode *next; -+ enum maple_type type; -+ void __rcu **slots; -+ unsigned char end; -+ unsigned long max; -+ -+ next = mas->node; -+ max = ULONG_MAX; -+ do { -+ offset = 0; -+ node = mte_to_node(next); -+ type = mte_node_type(next); -+ pivots = ma_pivots(node, type); -+ end = ma_data_end(node, type, pivots, max); -+ if (unlikely(ma_dead_node(node))) -+ goto dead_node; -+ -+ if (pivots[offset] >= mas->index) -+ goto next; -+ -+ do { -+ offset++; -+ } while ((offset < end) && (pivots[offset] < mas->index)); -+ -+ if (likely(offset > end)) -+ max = pivots[offset]; -+ -+next: -+ slots = ma_slots(node, type); -+ next = mt_slot(mas->tree, slots, offset); -+ if (unlikely(ma_dead_node(node))) -+ goto dead_node; -+ } while (!ma_is_leaf(type)); -+ -+ return (void *) next; -+ -+dead_node: -+ mas_reset(mas); -+ return NULL; -+} -+ -+/* -+ * mas_new_root() - Create a new root node that only contains the entry passed -+ * in. -+ * @mas: The maple state -+ * @entry: The entry to store. -+ * -+ * Only valid when the index == 0 and the last == ULONG_MAX -+ * -+ * Return 0 on error, 1 on success. -+ */ -+static inline int mas_new_root(struct ma_state *mas, void *entry) -+{ -+ struct maple_enode *root = mas_root_locked(mas); -+ enum maple_type type = maple_leaf_64; -+ struct maple_node *node; -+ void __rcu **slots; -+ unsigned long *pivots; -+ -+ if (!entry && !mas->index && mas->last == ULONG_MAX) { -+ mas->depth = 0; -+ mas_set_height(mas); -+ rcu_assign_pointer(mas->tree->ma_root, entry); -+ mas->node = MAS_START; -+ goto done; -+ } -+ -+ mas_node_count(mas, 1); -+ if (mas_is_err(mas)) -+ return 0; -+ -+ node = mas_pop_node(mas); -+ pivots = ma_pivots(node, type); -+ slots = ma_slots(node, type); -+ node->parent = ma_parent_ptr( -+ ((unsigned long)mas->tree | MA_ROOT_PARENT)); -+ mas->node = mt_mk_node(node, type); -+ rcu_assign_pointer(slots[0], entry); -+ pivots[0] = mas->last; -+ mas->depth = 1; -+ mas_set_height(mas); -+ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); -+ -+done: -+ if (xa_is_node(root)) -+ mte_destroy_walk(root, mas->tree); -+ -+ return 1; -+} -+/* -+ * mas_wr_spanning_store() - Create a subtree with the store operation completed -+ * and new nodes where necessary, then place the sub-tree in the actual tree. -+ * Note that mas is expected to point to the node which caused the store to -+ * span. -+ * @wr_mas: The maple write state -+ * -+ * Return: 0 on error, positive on success. -+ */ -+static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) -+{ -+ struct maple_subtree_state mast; -+ struct maple_big_node b_node; -+ struct ma_state *mas; -+ unsigned char height; -+ -+ /* Left and Right side of spanning store */ -+ MA_STATE(l_mas, NULL, 0, 0); -+ MA_STATE(r_mas, NULL, 0, 0); -+ -+ MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); -+ MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); -+ -+ /* -+ * A store operation that spans multiple nodes is called a spanning -+ * store and is handled early in the store call stack by the function -+ * mas_is_span_wr(). When a spanning store is identified, the maple -+ * state is duplicated. The first maple state walks the left tree path -+ * to ``index``, the duplicate walks the right tree path to ``last``. -+ * The data in the two nodes are combined into a single node, two nodes, -+ * or possibly three nodes (see the 3-way split above). A ``NULL`` -+ * written to the last entry of a node is considered a spanning store as -+ * a rebalance is required for the operation to complete and an overflow -+ * of data may happen. -+ */ -+ mas = wr_mas->mas; -+ trace_ma_op(__func__, mas); -+ -+ if (unlikely(!mas->index && mas->last == ULONG_MAX)) -+ return mas_new_root(mas, wr_mas->entry); -+ /* -+ * Node rebalancing may occur due to this store, so there may be three new -+ * entries per level plus a new root. -+ */ -+ height = mas_mt_height(mas); -+ mas_node_count(mas, 1 + height * 3); -+ if (mas_is_err(mas)) -+ return 0; -+ -+ /* -+ * Set up right side. Need to get to the next offset after the spanning -+ * store to ensure it's not NULL and to combine both the next node and -+ * the node with the start together. -+ */ -+ r_mas = *mas; -+ /* Avoid overflow, walk to next slot in the tree. */ -+ if (r_mas.last + 1) -+ r_mas.last++; -+ -+ r_mas.index = r_mas.last; -+ mas_wr_walk_index(&r_wr_mas); -+ r_mas.last = r_mas.index = mas->last; -+ -+ /* Set up left side. */ -+ l_mas = *mas; -+ mas_wr_walk_index(&l_wr_mas); -+ -+ if (!wr_mas->entry) { -+ mas_extend_spanning_null(&l_wr_mas, &r_wr_mas); -+ mas->offset = l_mas.offset; -+ mas->index = l_mas.index; -+ mas->last = l_mas.last = r_mas.last; -+ } -+ -+ /* expanding NULLs may make this cover the entire range */ -+ if (!l_mas.index && r_mas.last == ULONG_MAX) { -+ mas_set_range(mas, 0, ULONG_MAX); -+ return mas_new_root(mas, wr_mas->entry); -+ } -+ -+ memset(&b_node, 0, sizeof(struct maple_big_node)); -+ /* Copy l_mas and store the value in b_node. */ -+ mas_store_b_node(&l_wr_mas, &b_node, l_wr_mas.node_end); -+ /* Copy r_mas into b_node. */ -+ if (r_mas.offset <= r_wr_mas.node_end) -+ mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end, -+ &b_node, b_node.b_end + 1); -+ else -+ b_node.b_end++; -+ -+ /* Stop spanning searches by searching for just index. */ -+ l_mas.index = l_mas.last = mas->index; -+ -+ mast.bn = &b_node; -+ mast.orig_l = &l_mas; -+ mast.orig_r = &r_mas; -+ /* Combine l_mas and r_mas and split them up evenly again. */ -+ return mas_spanning_rebalance(mas, &mast, height + 1); -+} -+ -+/* -+ * mas_wr_node_store() - Attempt to store the value in a node -+ * @wr_mas: The maple write state -+ * -+ * Attempts to reuse the node, but may allocate. -+ * -+ * Return: True if stored, false otherwise -+ */ -+static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ void __rcu **dst_slots; -+ unsigned long *dst_pivots; -+ unsigned char dst_offset; -+ unsigned char new_end = wr_mas->node_end; -+ unsigned char offset; -+ unsigned char node_slots = mt_slots[wr_mas->type]; -+ struct maple_node reuse, *newnode; -+ unsigned char copy_size, max_piv = mt_pivots[wr_mas->type]; -+ bool in_rcu = mt_in_rcu(mas->tree); -+ -+ offset = mas->offset; -+ if (mas->last == wr_mas->r_max) { -+ /* runs right to the end of the node */ -+ if (mas->last == mas->max) -+ new_end = offset; -+ /* don't copy this offset */ -+ wr_mas->offset_end++; -+ } else if (mas->last < wr_mas->r_max) { -+ /* new range ends in this range */ -+ if (unlikely(wr_mas->r_max == ULONG_MAX)) -+ mas_bulk_rebalance(mas, wr_mas->node_end, wr_mas->type); -+ -+ new_end++; -+ } else { -+ if (wr_mas->end_piv == mas->last) -+ wr_mas->offset_end++; -+ -+ new_end -= wr_mas->offset_end - offset - 1; -+ } -+ -+ /* new range starts within a range */ -+ if (wr_mas->r_min < mas->index) -+ new_end++; -+ -+ /* Not enough room */ -+ if (new_end >= node_slots) -+ return false; -+ -+ /* Not enough data. */ -+ if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) && -+ !(mas->mas_flags & MA_STATE_BULK)) -+ return false; -+ -+ /* set up node. */ -+ if (in_rcu) { -+ mas_node_count(mas, 1); -+ if (mas_is_err(mas)) -+ return false; -+ -+ newnode = mas_pop_node(mas); -+ } else { -+ memset(&reuse, 0, sizeof(struct maple_node)); -+ newnode = &reuse; -+ } -+ -+ newnode->parent = mas_mn(mas)->parent; -+ dst_pivots = ma_pivots(newnode, wr_mas->type); -+ dst_slots = ma_slots(newnode, wr_mas->type); -+ /* Copy from start to insert point */ -+ memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * (offset + 1)); -+ memcpy(dst_slots, wr_mas->slots, sizeof(void *) * (offset + 1)); -+ dst_offset = offset; -+ -+ /* Handle insert of new range starting after old range */ -+ if (wr_mas->r_min < mas->index) { -+ mas->offset++; -+ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->content); -+ dst_pivots[dst_offset++] = mas->index - 1; -+ } -+ -+ /* Store the new entry and range end. */ -+ if (dst_offset < max_piv) -+ dst_pivots[dst_offset] = mas->last; -+ mas->offset = dst_offset; -+ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->entry); -+ -+ /* -+ * this range wrote to the end of the node or it overwrote the rest of -+ * the data -+ */ -+ if (wr_mas->offset_end > wr_mas->node_end || mas->last >= mas->max) { -+ new_end = dst_offset; -+ goto done; -+ } -+ -+ dst_offset++; -+ /* Copy to the end of node if necessary. */ -+ copy_size = wr_mas->node_end - wr_mas->offset_end + 1; -+ memcpy(dst_slots + dst_offset, wr_mas->slots + wr_mas->offset_end, -+ sizeof(void *) * copy_size); -+ if (dst_offset < max_piv) { -+ if (copy_size > max_piv - dst_offset) -+ copy_size = max_piv - dst_offset; -+ -+ memcpy(dst_pivots + dst_offset, -+ wr_mas->pivots + wr_mas->offset_end, -+ sizeof(unsigned long) * copy_size); -+ } -+ -+ if ((wr_mas->node_end == node_slots - 1) && (new_end < node_slots - 1)) -+ dst_pivots[new_end] = mas->max; -+ -+done: -+ mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end); -+ if (in_rcu) { -+ mas->node = mt_mk_node(newnode, wr_mas->type); -+ mas_replace(mas, false); -+ } else { -+ memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); -+ } -+ trace_ma_write(__func__, mas, 0, wr_mas->entry); -+ mas_update_gap(mas); -+ return true; -+} -+ -+/* -+ * mas_wr_slot_store: Attempt to store a value in a slot. -+ * @wr_mas: the maple write state -+ * -+ * Return: True if stored, false otherwise -+ */ -+static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ unsigned long lmax; /* Logical max. */ -+ unsigned char offset = mas->offset; -+ -+ if ((wr_mas->r_max > mas->last) && ((wr_mas->r_min != mas->index) || -+ (offset != wr_mas->node_end))) -+ return false; -+ -+ if (offset == wr_mas->node_end - 1) -+ lmax = mas->max; -+ else -+ lmax = wr_mas->pivots[offset + 1]; -+ -+ /* going to overwrite too many slots. */ -+ if (lmax < mas->last) -+ return false; -+ -+ if (wr_mas->r_min == mas->index) { -+ /* overwriting two or more ranges with one. */ -+ if (lmax == mas->last) -+ return false; -+ -+ /* Overwriting all of offset and a portion of offset + 1. */ -+ rcu_assign_pointer(wr_mas->slots[offset], wr_mas->entry); -+ wr_mas->pivots[offset] = mas->last; -+ goto done; -+ } -+ -+ /* Doesn't end on the next range end. */ -+ if (lmax != mas->last) -+ return false; -+ -+ /* Overwriting a portion of offset and all of offset + 1 */ -+ if ((offset + 1 < mt_pivots[wr_mas->type]) && -+ (wr_mas->entry || wr_mas->pivots[offset + 1])) -+ wr_mas->pivots[offset + 1] = mas->last; -+ -+ rcu_assign_pointer(wr_mas->slots[offset + 1], wr_mas->entry); -+ wr_mas->pivots[offset] = mas->index - 1; -+ mas->offset++; /* Keep mas accurate. */ -+ -+done: -+ trace_ma_write(__func__, mas, 0, wr_mas->entry); -+ mas_update_gap(mas); -+ return true; -+} -+ -+static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) -+{ -+ while ((wr_mas->mas->last > wr_mas->end_piv) && -+ (wr_mas->offset_end < wr_mas->node_end)) -+ wr_mas->end_piv = wr_mas->pivots[++wr_mas->offset_end]; -+ -+ if (wr_mas->mas->last > wr_mas->end_piv) -+ wr_mas->end_piv = wr_mas->mas->max; -+} -+ -+static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ -+ if (mas->last < wr_mas->end_piv && !wr_mas->slots[wr_mas->offset_end]) -+ mas->last = wr_mas->end_piv; -+ -+ /* Check next slot(s) if we are overwriting the end */ -+ if ((mas->last == wr_mas->end_piv) && -+ (wr_mas->node_end != wr_mas->offset_end) && -+ !wr_mas->slots[wr_mas->offset_end + 1]) { -+ wr_mas->offset_end++; -+ if (wr_mas->offset_end == wr_mas->node_end) -+ mas->last = mas->max; -+ else -+ mas->last = wr_mas->pivots[wr_mas->offset_end]; -+ wr_mas->end_piv = mas->last; -+ } -+ -+ if (!wr_mas->content) { -+ /* If this one is null, the next and prev are not */ -+ mas->index = wr_mas->r_min; -+ } else { -+ /* Check prev slot if we are overwriting the start */ -+ if (mas->index == wr_mas->r_min && mas->offset && -+ !wr_mas->slots[mas->offset - 1]) { -+ mas->offset--; -+ wr_mas->r_min = mas->index = -+ mas_safe_min(mas, wr_mas->pivots, mas->offset); -+ wr_mas->r_max = wr_mas->pivots[mas->offset]; -+ } -+ } -+} -+ -+static inline bool mas_wr_append(struct ma_wr_state *wr_mas) -+{ -+ unsigned char end = wr_mas->node_end; -+ unsigned char new_end = end + 1; -+ struct ma_state *mas = wr_mas->mas; -+ unsigned char node_pivots = mt_pivots[wr_mas->type]; -+ -+ if ((mas->index != wr_mas->r_min) && (mas->last == wr_mas->r_max)) { -+ if (new_end < node_pivots) -+ wr_mas->pivots[new_end] = wr_mas->pivots[end]; -+ -+ if (new_end < node_pivots) -+ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); -+ -+ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->entry); -+ mas->offset = new_end; -+ wr_mas->pivots[end] = mas->index - 1; -+ -+ return true; -+ } -+ -+ if ((mas->index == wr_mas->r_min) && (mas->last < wr_mas->r_max)) { -+ if (new_end < node_pivots) -+ wr_mas->pivots[new_end] = wr_mas->pivots[end]; -+ -+ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->content); -+ if (new_end < node_pivots) -+ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); -+ -+ wr_mas->pivots[end] = mas->last; -+ rcu_assign_pointer(wr_mas->slots[end], wr_mas->entry); -+ return true; -+ } -+ -+ return false; -+} -+ -+/* -+ * mas_wr_bnode() - Slow path for a modification. -+ * @wr_mas: The write maple state -+ * -+ * This is where split, rebalance end up. -+ */ -+static void mas_wr_bnode(struct ma_wr_state *wr_mas) -+{ -+ struct maple_big_node b_node; -+ -+ trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry); -+ memset(&b_node, 0, sizeof(struct maple_big_node)); -+ mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); -+ mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end); -+} -+ -+static inline void mas_wr_modify(struct ma_wr_state *wr_mas) -+{ -+ unsigned char node_slots; -+ unsigned char node_size; -+ struct ma_state *mas = wr_mas->mas; -+ -+ /* Direct replacement */ -+ if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) { -+ rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry); -+ if (!!wr_mas->entry ^ !!wr_mas->content) -+ mas_update_gap(mas); -+ return; -+ } -+ -+ /* Attempt to append */ -+ node_slots = mt_slots[wr_mas->type]; -+ node_size = wr_mas->node_end - wr_mas->offset_end + mas->offset + 2; -+ if (mas->max == ULONG_MAX) -+ node_size++; -+ -+ /* slot and node store will not fit, go to the slow path */ -+ if (unlikely(node_size >= node_slots)) -+ goto slow_path; -+ -+ if (wr_mas->entry && (wr_mas->node_end < node_slots - 1) && -+ (mas->offset == wr_mas->node_end) && mas_wr_append(wr_mas)) { -+ if (!wr_mas->content || !wr_mas->entry) -+ mas_update_gap(mas); -+ return; -+ } -+ -+ if ((wr_mas->offset_end - mas->offset <= 1) && mas_wr_slot_store(wr_mas)) -+ return; -+ else if (mas_wr_node_store(wr_mas)) -+ return; -+ -+ if (mas_is_err(mas)) -+ return; -+ -+slow_path: -+ mas_wr_bnode(wr_mas); -+} -+ -+/* -+ * mas_wr_store_entry() - Internal call to store a value -+ * @mas: The maple state -+ * @entry: The entry to store. -+ * -+ * Return: The contents that was stored at the index. -+ */ -+static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas) -+{ -+ struct ma_state *mas = wr_mas->mas; -+ -+ wr_mas->content = mas_start(mas); -+ if (mas_is_none(mas) || mas_is_ptr(mas)) { -+ mas_store_root(mas, wr_mas->entry); -+ return wr_mas->content; -+ } -+ -+ if (unlikely(!mas_wr_walk(wr_mas))) { -+ mas_wr_spanning_store(wr_mas); -+ return wr_mas->content; -+ } -+ -+ /* At this point, we are at the leaf node that needs to be altered. */ -+ wr_mas->end_piv = wr_mas->r_max; -+ mas_wr_end_piv(wr_mas); -+ -+ if (!wr_mas->entry) -+ mas_wr_extend_null(wr_mas); -+ -+ /* New root for a single pointer */ -+ if (unlikely(!mas->index && mas->last == ULONG_MAX)) { -+ mas_new_root(mas, wr_mas->entry); -+ return wr_mas->content; -+ } -+ -+ mas_wr_modify(wr_mas); -+ return wr_mas->content; -+} -+ -+/** -+ * mas_insert() - Internal call to insert a value -+ * @mas: The maple state -+ * @entry: The entry to store -+ * -+ * Return: %NULL or the contents that already exists at the requested index -+ * otherwise. The maple state needs to be checked for error conditions. -+ */ -+static inline void *mas_insert(struct ma_state *mas, void *entry) -+{ -+ MA_WR_STATE(wr_mas, mas, entry); -+ -+ /* -+ * Inserting a new range inserts either 0, 1, or 2 pivots within the -+ * tree. If the insert fits exactly into an existing gap with a value -+ * of NULL, then the slot only needs to be written with the new value. -+ * If the range being inserted is adjacent to another range, then only a -+ * single pivot needs to be inserted (as well as writing the entry). If -+ * the new range is within a gap but does not touch any other ranges, -+ * then two pivots need to be inserted: the start - 1, and the end. As -+ * usual, the entry must be written. Most operations require a new node -+ * to be allocated and replace an existing node to ensure RCU safety, -+ * when in RCU mode. The exception to requiring a newly allocated node -+ * is when inserting at the end of a node (appending). When done -+ * carefully, appending can reuse the node in place. -+ */ -+ wr_mas.content = mas_start(mas); -+ if (wr_mas.content) -+ goto exists; -+ -+ if (mas_is_none(mas) || mas_is_ptr(mas)) { -+ mas_store_root(mas, entry); -+ return NULL; -+ } -+ -+ /* spanning writes always overwrite something */ -+ if (!mas_wr_walk(&wr_mas)) -+ goto exists; -+ -+ /* At this point, we are at the leaf node that needs to be altered. */ -+ wr_mas.offset_end = mas->offset; -+ wr_mas.end_piv = wr_mas.r_max; -+ -+ if (wr_mas.content || (mas->last > wr_mas.r_max)) -+ goto exists; -+ -+ if (!entry) -+ return NULL; -+ -+ mas_wr_modify(&wr_mas); -+ return wr_mas.content; -+ -+exists: -+ mas_set_err(mas, -EEXIST); -+ return wr_mas.content; -+ -+} -+ -+/* -+ * mas_prev_node() - Find the prev non-null entry at the same level in the -+ * tree. The prev value will be mas->node[mas->offset] or MAS_NONE. -+ * @mas: The maple state -+ * @min: The lower limit to search -+ * -+ * The prev node value will be mas->node[mas->offset] or MAS_NONE. -+ * Return: 1 if the node is dead, 0 otherwise. -+ */ -+static inline int mas_prev_node(struct ma_state *mas, unsigned long min) -+{ -+ enum maple_type mt; -+ int offset, level; -+ void __rcu **slots; -+ struct maple_node *node; -+ struct maple_enode *enode; -+ unsigned long *pivots; -+ -+ if (mas_is_none(mas)) -+ return 0; -+ -+ level = 0; -+ do { -+ node = mas_mn(mas); -+ if (ma_is_root(node)) -+ goto no_entry; -+ -+ /* Walk up. */ -+ if (unlikely(mas_ascend(mas))) -+ return 1; -+ offset = mas->offset; -+ level++; -+ } while (!offset); -+ -+ offset--; -+ mt = mte_node_type(mas->node); -+ node = mas_mn(mas); -+ slots = ma_slots(node, mt); -+ pivots = ma_pivots(node, mt); -+ mas->max = pivots[offset]; -+ if (offset) -+ mas->min = pivots[offset - 1] + 1; -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ if (mas->max < min) -+ goto no_entry_min; -+ -+ while (level > 1) { -+ level--; -+ enode = mas_slot(mas, slots, offset); -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ mas->node = enode; -+ mt = mte_node_type(mas->node); -+ node = mas_mn(mas); -+ slots = ma_slots(node, mt); -+ pivots = ma_pivots(node, mt); -+ offset = ma_data_end(node, mt, pivots, mas->max); -+ if (offset) -+ mas->min = pivots[offset - 1] + 1; -+ -+ if (offset < mt_pivots[mt]) -+ mas->max = pivots[offset]; -+ -+ if (mas->max < min) -+ goto no_entry; -+ } -+ -+ mas->node = mas_slot(mas, slots, offset); -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ mas->offset = mas_data_end(mas); -+ if (unlikely(mte_dead_node(mas->node))) -+ return 1; -+ -+ return 0; -+ -+no_entry_min: -+ mas->offset = offset; -+ if (offset) -+ mas->min = pivots[offset - 1] + 1; -+no_entry: -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ mas->node = MAS_NONE; -+ return 0; -+} -+ -+/* -+ * mas_next_node() - Get the next node at the same level in the tree. -+ * @mas: The maple state -+ * @max: The maximum pivot value to check. -+ * -+ * The next value will be mas->node[mas->offset] or MAS_NONE. -+ * Return: 1 on dead node, 0 otherwise. -+ */ -+static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, -+ unsigned long max) -+{ -+ unsigned long min, pivot; -+ unsigned long *pivots; -+ struct maple_enode *enode; -+ int level = 0; -+ unsigned char offset; -+ enum maple_type mt; -+ void __rcu **slots; -+ -+ if (mas->max >= max) -+ goto no_entry; -+ -+ level = 0; -+ do { -+ if (ma_is_root(node)) -+ goto no_entry; -+ -+ min = mas->max + 1; -+ if (min > max) -+ goto no_entry; -+ -+ if (unlikely(mas_ascend(mas))) -+ return 1; -+ -+ offset = mas->offset; -+ level++; -+ node = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ pivots = ma_pivots(node, mt); -+ } while (unlikely(offset == ma_data_end(node, mt, pivots, mas->max))); -+ -+ slots = ma_slots(node, mt); -+ pivot = mas_safe_pivot(mas, pivots, ++offset, mt); -+ while (unlikely(level > 1)) { -+ /* Descend, if necessary */ -+ enode = mas_slot(mas, slots, offset); -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ mas->node = enode; -+ level--; -+ node = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ slots = ma_slots(node, mt); -+ pivots = ma_pivots(node, mt); -+ offset = 0; -+ pivot = pivots[0]; -+ } -+ -+ enode = mas_slot(mas, slots, offset); -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ mas->node = enode; -+ mas->min = min; -+ mas->max = pivot; -+ return 0; -+ -+no_entry: -+ if (unlikely(ma_dead_node(node))) -+ return 1; -+ -+ mas->node = MAS_NONE; -+ return 0; -+} -+ -+/* -+ * mas_next_nentry() - Get the next node entry -+ * @mas: The maple state -+ * @max: The maximum value to check -+ * @*range_start: Pointer to store the start of the range. -+ * -+ * Sets @mas->offset to the offset of the next node entry, @mas->last to the -+ * pivot of the entry. -+ * -+ * Return: The next entry, %NULL otherwise -+ */ -+static inline void *mas_next_nentry(struct ma_state *mas, -+ struct maple_node *node, unsigned long max, enum maple_type type) -+{ -+ unsigned char count; -+ unsigned long pivot; -+ unsigned long *pivots; -+ void __rcu **slots; -+ void *entry; -+ -+ if (mas->last == mas->max) { -+ mas->index = mas->max; -+ return NULL; -+ } -+ -+ pivots = ma_pivots(node, type); -+ slots = ma_slots(node, type); -+ mas->index = mas_safe_min(mas, pivots, mas->offset); -+ if (ma_dead_node(node)) -+ return NULL; -+ -+ if (mas->index > max) -+ return NULL; -+ -+ count = ma_data_end(node, type, pivots, mas->max); -+ if (mas->offset > count) -+ return NULL; -+ -+ while (mas->offset < count) { -+ pivot = pivots[mas->offset]; -+ entry = mas_slot(mas, slots, mas->offset); -+ if (ma_dead_node(node)) -+ return NULL; -+ -+ if (entry) -+ goto found; -+ -+ if (pivot >= max) -+ return NULL; -+ -+ mas->index = pivot + 1; -+ mas->offset++; -+ } -+ -+ if (mas->index > mas->max) { -+ mas->index = mas->last; -+ return NULL; -+ } -+ -+ pivot = mas_safe_pivot(mas, pivots, mas->offset, type); -+ entry = mas_slot(mas, slots, mas->offset); -+ if (ma_dead_node(node)) -+ return NULL; -+ -+ if (!pivot) -+ return NULL; -+ -+ if (!entry) -+ return NULL; -+ -+found: -+ mas->last = pivot; -+ return entry; -+} -+ -+static inline void mas_rewalk(struct ma_state *mas, unsigned long index) -+{ -+ -+retry: -+ mas_set(mas, index); -+ mas_state_walk(mas); -+ if (mas_is_start(mas)) -+ goto retry; -+ -+ return; -+ -+} -+ -+/* -+ * mas_next_entry() - Internal function to get the next entry. -+ * @mas: The maple state -+ * @limit: The maximum range start. -+ * -+ * Set the @mas->node to the next entry and the range_start to -+ * the beginning value for the entry. Does not check beyond @limit. -+ * Sets @mas->index and @mas->last to the limit if it is hit. -+ * Restarts on dead nodes. -+ * -+ * Return: the next entry or %NULL. -+ */ -+static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) -+{ -+ void *entry = NULL; -+ struct maple_enode *prev_node; -+ struct maple_node *node; -+ unsigned char offset; -+ unsigned long last; -+ enum maple_type mt; -+ -+ last = mas->last; -+retry: -+ offset = mas->offset; -+ prev_node = mas->node; -+ node = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ mas->offset++; -+ if (unlikely(mas->offset >= mt_slots[mt])) { -+ mas->offset = mt_slots[mt] - 1; -+ goto next_node; -+ } -+ -+ while (!mas_is_none(mas)) { -+ entry = mas_next_nentry(mas, node, limit, mt); -+ if (unlikely(ma_dead_node(node))) { -+ mas_rewalk(mas, last); -+ goto retry; -+ } -+ -+ if (likely(entry)) -+ return entry; -+ -+ if (unlikely((mas->index > limit))) -+ break; -+ -+next_node: -+ prev_node = mas->node; -+ offset = mas->offset; -+ if (unlikely(mas_next_node(mas, node, limit))) { -+ mas_rewalk(mas, last); -+ goto retry; -+ } -+ mas->offset = 0; -+ node = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ } -+ -+ mas->index = mas->last = limit; -+ mas->offset = offset; -+ mas->node = prev_node; -+ return NULL; -+} -+ -+/* -+ * mas_prev_nentry() - Get the previous node entry. -+ * @mas: The maple state. -+ * @limit: The lower limit to check for a value. -+ * -+ * Return: the entry, %NULL otherwise. -+ */ -+static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit, -+ unsigned long index) -+{ -+ unsigned long pivot, min; -+ unsigned char offset; -+ struct maple_node *mn; -+ enum maple_type mt; -+ unsigned long *pivots; -+ void __rcu **slots; -+ void *entry; -+ -+retry: -+ if (!mas->offset) -+ return NULL; -+ -+ mn = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ offset = mas->offset - 1; -+ if (offset >= mt_slots[mt]) -+ offset = mt_slots[mt] - 1; -+ -+ slots = ma_slots(mn, mt); -+ pivots = ma_pivots(mn, mt); -+ if (offset == mt_pivots[mt]) -+ pivot = mas->max; -+ else -+ pivot = pivots[offset]; -+ -+ if (unlikely(ma_dead_node(mn))) { -+ mas_rewalk(mas, index); -+ goto retry; -+ } -+ -+ while (offset && ((!mas_slot(mas, slots, offset) && pivot >= limit) || -+ !pivot)) -+ pivot = pivots[--offset]; -+ -+ min = mas_safe_min(mas, pivots, offset); -+ entry = mas_slot(mas, slots, offset); -+ if (unlikely(ma_dead_node(mn))) { -+ mas_rewalk(mas, index); -+ goto retry; -+ } -+ -+ if (likely(entry)) { -+ mas->offset = offset; -+ mas->last = pivot; -+ mas->index = min; -+ } -+ return entry; -+} -+ -+static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min) -+{ -+ void *entry; -+ -+retry: -+ while (likely(!mas_is_none(mas))) { -+ entry = mas_prev_nentry(mas, min, mas->index); -+ if (unlikely(mas->last < min)) -+ goto not_found; -+ -+ if (likely(entry)) -+ return entry; -+ -+ if (unlikely(mas_prev_node(mas, min))) { -+ mas_rewalk(mas, mas->index); -+ goto retry; -+ } -+ -+ mas->offset++; -+ } -+ -+ mas->offset--; -+not_found: -+ mas->index = mas->last = min; -+ return NULL; -+} -+ -+/* -+ * mas_rev_awalk() - Internal function. Reverse allocation walk. Find the -+ * highest gap address of a given size in a given node and descend. -+ * @mas: The maple state -+ * @size: The needed size. -+ * -+ * Return: True if found in a leaf, false otherwise. -+ * -+ */ -+static bool mas_rev_awalk(struct ma_state *mas, unsigned long size) -+{ -+ enum maple_type type = mte_node_type(mas->node); -+ struct maple_node *node = mas_mn(mas); -+ unsigned long *pivots, *gaps; -+ void __rcu **slots; -+ unsigned long gap = 0; -+ unsigned long max, min, index; -+ unsigned char offset; -+ -+ if (unlikely(mas_is_err(mas))) -+ return true; -+ -+ if (ma_is_dense(type)) { -+ /* dense nodes. */ -+ mas->offset = (unsigned char)(mas->index - mas->min); -+ return true; -+ } -+ -+ pivots = ma_pivots(node, type); -+ slots = ma_slots(node, type); -+ gaps = ma_gaps(node, type); -+ offset = mas->offset; -+ min = mas_safe_min(mas, pivots, offset); -+ /* Skip out of bounds. */ -+ while (mas->last < min) -+ min = mas_safe_min(mas, pivots, --offset); -+ -+ max = mas_safe_pivot(mas, pivots, offset, type); -+ index = mas->index; -+ while (index <= max) { -+ gap = 0; -+ if (gaps) -+ gap = gaps[offset]; -+ else if (!mas_slot(mas, slots, offset)) -+ gap = max - min + 1; -+ -+ if (gap) { -+ if ((size <= gap) && (size <= mas->last - min + 1)) -+ break; -+ -+ if (!gaps) { -+ /* Skip the next slot, it cannot be a gap. */ -+ if (offset < 2) -+ goto ascend; -+ -+ offset -= 2; -+ max = pivots[offset]; -+ min = mas_safe_min(mas, pivots, offset); -+ continue; -+ } -+ } -+ -+ if (!offset) -+ goto ascend; -+ -+ offset--; -+ max = min - 1; -+ min = mas_safe_min(mas, pivots, offset); -+ } -+ -+ if (unlikely(index > max)) { -+ mas_set_err(mas, -EBUSY); -+ return false; -+ } -+ -+ if (unlikely(ma_is_leaf(type))) { -+ mas->offset = offset; -+ mas->min = min; -+ mas->max = min + gap - 1; -+ return true; -+ } -+ -+ /* descend, only happens under lock. */ -+ mas->node = mas_slot(mas, slots, offset); -+ mas->min = min; -+ mas->max = max; -+ mas->offset = mas_data_end(mas); -+ return false; -+ -+ascend: -+ if (mte_is_root(mas->node)) -+ mas_set_err(mas, -EBUSY); -+ -+ return false; -+} -+ -+static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) -+{ -+ enum maple_type type = mte_node_type(mas->node); -+ unsigned long pivot, min, gap = 0; -+ unsigned char count, offset; -+ unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type); -+ void __rcu **slots = ma_slots(mas_mn(mas), type); -+ bool found = false; -+ -+ if (ma_is_dense(type)) { -+ mas->offset = (unsigned char)(mas->index - mas->min); -+ return true; -+ } -+ -+ gaps = ma_gaps(mte_to_node(mas->node), type); -+ offset = mas->offset; -+ count = mt_slots[type]; -+ min = mas_safe_min(mas, pivots, offset); -+ for (; offset < count; offset++) { -+ pivot = mas_safe_pivot(mas, pivots, offset, type); -+ if (offset && !pivot) -+ break; -+ -+ /* Not within lower bounds */ -+ if (mas->index > pivot) -+ goto next_slot; -+ -+ if (gaps) -+ gap = gaps[offset]; -+ else if (!mas_slot(mas, slots, offset)) -+ gap = min(pivot, mas->last) - max(mas->index, min) + 1; -+ else -+ goto next_slot; -+ -+ if (gap >= size) { -+ if (ma_is_leaf(type)) { -+ found = true; -+ goto done; -+ } -+ if (mas->index <= pivot) { -+ mas->node = mas_slot(mas, slots, offset); -+ mas->min = min; -+ mas->max = pivot; -+ offset = 0; -+ type = mte_node_type(mas->node); -+ count = mt_slots[type]; -+ break; -+ } -+ } -+next_slot: -+ min = pivot + 1; -+ if (mas->last <= pivot) { -+ mas_set_err(mas, -EBUSY); -+ return true; -+ } -+ } -+ -+ if (mte_is_root(mas->node)) -+ found = true; -+done: -+ mas->offset = offset; -+ return found; -+} -+ -+/** -+ * mas_walk() - Search for @mas->index in the tree. -+ * @mas: The maple state. -+ * -+ * mas->index and mas->last will be set to the range if there is a value. If -+ * mas->node is MAS_NONE, reset to MAS_START. -+ * -+ * Return: the entry at the location or %NULL. -+ */ -+void *mas_walk(struct ma_state *mas) -+{ -+ void *entry; -+ -+retry: -+ entry = mas_state_walk(mas); -+ if (mas_is_start(mas)) -+ goto retry; -+ -+ if (mas_is_ptr(mas)) { -+ if (!mas->index) { -+ mas->last = 0; -+ } else { -+ mas->index = 1; -+ mas->last = ULONG_MAX; -+ } -+ return entry; -+ } -+ -+ if (mas_is_none(mas)) { -+ mas->index = 0; -+ mas->last = ULONG_MAX; -+ } -+ -+ return entry; -+} -+ -+static inline bool mas_rewind_node(struct ma_state *mas) -+{ -+ unsigned char slot; -+ -+ do { -+ if (mte_is_root(mas->node)) { -+ slot = mas->offset; -+ if (!slot) -+ return false; -+ } else { -+ mas_ascend(mas); -+ slot = mas->offset; -+ } -+ } while (!slot); -+ -+ mas->offset = --slot; -+ return true; -+} -+ -+/* -+ * mas_skip_node() - Internal function. Skip over a node. -+ * @mas: The maple state. -+ * -+ * Return: true if there is another node, false otherwise. -+ */ -+static inline bool mas_skip_node(struct ma_state *mas) -+{ -+ unsigned char slot, slot_count; -+ unsigned long *pivots; -+ enum maple_type mt; -+ -+ mt = mte_node_type(mas->node); -+ slot_count = mt_slots[mt] - 1; -+ do { -+ if (mte_is_root(mas->node)) { -+ slot = mas->offset; -+ if (slot > slot_count) { -+ mas_set_err(mas, -EBUSY); -+ return false; -+ } -+ } else { -+ mas_ascend(mas); -+ slot = mas->offset; -+ mt = mte_node_type(mas->node); -+ slot_count = mt_slots[mt] - 1; -+ } -+ } while (slot > slot_count); -+ -+ mas->offset = ++slot; -+ pivots = ma_pivots(mas_mn(mas), mt); -+ if (slot > 0) -+ mas->min = pivots[slot - 1] + 1; -+ -+ if (slot <= slot_count) -+ mas->max = pivots[slot]; -+ -+ return true; -+} -+ -+/* -+ * mas_awalk() - Allocation walk. Search from low address to high, for a gap of -+ * @size -+ * @mas: The maple state -+ * @size: The size of the gap required -+ * -+ * Search between @mas->index and @mas->last for a gap of @size. -+ */ -+static inline void mas_awalk(struct ma_state *mas, unsigned long size) -+{ -+ struct maple_enode *last = NULL; -+ -+ /* -+ * There are 4 options: -+ * go to child (descend) -+ * go back to parent (ascend) -+ * no gap found. (return, slot == MAPLE_NODE_SLOTS) -+ * found the gap. (return, slot != MAPLE_NODE_SLOTS) -+ */ -+ while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) { -+ if (last == mas->node) -+ mas_skip_node(mas); -+ else -+ last = mas->node; -+ } -+} -+ -+/* -+ * mas_fill_gap() - Fill a located gap with @entry. -+ * @mas: The maple state -+ * @entry: The value to store -+ * @slot: The offset into the node to store the @entry -+ * @size: The size of the entry -+ * @index: The start location -+ */ -+static inline void mas_fill_gap(struct ma_state *mas, void *entry, -+ unsigned char slot, unsigned long size, unsigned long *index) -+{ -+ MA_WR_STATE(wr_mas, mas, entry); -+ unsigned char pslot = mte_parent_slot(mas->node); -+ struct maple_enode *mn = mas->node; -+ unsigned long *pivots; -+ enum maple_type ptype; -+ /* -+ * mas->index is the start address for the search -+ * which may no longer be needed. -+ * mas->last is the end address for the search -+ */ -+ -+ *index = mas->index; -+ mas->last = mas->index + size - 1; -+ -+ /* -+ * It is possible that using mas->max and mas->min to correctly -+ * calculate the index and last will cause an issue in the gap -+ * calculation, so fix the ma_state here -+ */ -+ mas_ascend(mas); -+ ptype = mte_node_type(mas->node); -+ pivots = ma_pivots(mas_mn(mas), ptype); -+ mas->max = mas_safe_pivot(mas, pivots, pslot, ptype); -+ mas->min = mas_safe_min(mas, pivots, pslot); -+ mas->node = mn; -+ mas->offset = slot; -+ mas_wr_store_entry(&wr_mas); -+} -+ -+/* -+ * mas_sparse_area() - Internal function. Return upper or lower limit when -+ * searching for a gap in an empty tree. -+ * @mas: The maple state -+ * @min: the minimum range -+ * @max: The maximum range -+ * @size: The size of the gap -+ * @fwd: Searching forward or back -+ */ -+static inline void mas_sparse_area(struct ma_state *mas, unsigned long min, -+ unsigned long max, unsigned long size, bool fwd) -+{ -+ unsigned long start = 0; -+ -+ if (!unlikely(mas_is_none(mas))) -+ start++; -+ /* mas_is_ptr */ -+ -+ if (start < min) -+ start = min; -+ -+ if (fwd) { -+ mas->index = start; -+ mas->last = start + size - 1; -+ return; -+ } -+ -+ mas->index = max; -+} -+ -+/* -+ * mas_empty_area() - Get the lowest address within the range that is -+ * sufficient for the size requested. -+ * @mas: The maple state -+ * @min: The lowest value of the range -+ * @max: The highest value of the range -+ * @size: The size needed -+ */ -+int mas_empty_area(struct ma_state *mas, unsigned long min, -+ unsigned long max, unsigned long size) -+{ -+ unsigned char offset; -+ unsigned long *pivots; -+ enum maple_type mt; -+ -+ if (mas_is_start(mas)) -+ mas_start(mas); -+ else if (mas->offset >= 2) -+ mas->offset -= 2; -+ else if (!mas_skip_node(mas)) -+ return -EBUSY; -+ -+ /* Empty set */ -+ if (mas_is_none(mas) || mas_is_ptr(mas)) { -+ mas_sparse_area(mas, min, max, size, true); -+ return 0; -+ } -+ -+ /* The start of the window can only be within these values */ -+ mas->index = min; -+ mas->last = max; -+ mas_awalk(mas, size); -+ -+ if (unlikely(mas_is_err(mas))) -+ return xa_err(mas->node); -+ -+ offset = mas->offset; -+ if (unlikely(offset == MAPLE_NODE_SLOTS)) -+ return -EBUSY; -+ -+ mt = mte_node_type(mas->node); -+ pivots = ma_pivots(mas_mn(mas), mt); -+ if (offset) -+ mas->min = pivots[offset - 1] + 1; -+ -+ if (offset < mt_pivots[mt]) -+ mas->max = pivots[offset]; -+ -+ if (mas->index < mas->min) -+ mas->index = mas->min; -+ -+ mas->last = mas->index + size - 1; -+ return 0; -+} -+ -+/* -+ * mas_empty_area_rev() - Get the highest address within the range that is -+ * sufficient for the size requested. -+ * @mas: The maple state -+ * @min: The lowest value of the range -+ * @max: The highest value of the range -+ * @size: The size needed -+ */ -+int mas_empty_area_rev(struct ma_state *mas, unsigned long min, -+ unsigned long max, unsigned long size) -+{ -+ struct maple_enode *last = mas->node; -+ -+ if (mas_is_start(mas)) { -+ mas_start(mas); -+ mas->offset = mas_data_end(mas); -+ } else if (mas->offset >= 2) { -+ mas->offset -= 2; -+ } else if (!mas_rewind_node(mas)) { -+ return -EBUSY; -+ } -+ -+ /* Empty set. */ -+ if (mas_is_none(mas) || mas_is_ptr(mas)) { -+ mas_sparse_area(mas, min, max, size, false); -+ return 0; -+ } -+ -+ /* The start of the window can only be within these values. */ -+ mas->index = min; -+ mas->last = max; -+ -+ while (!mas_rev_awalk(mas, size)) { -+ if (last == mas->node) { -+ if (!mas_rewind_node(mas)) -+ return -EBUSY; -+ } else { -+ last = mas->node; -+ } -+ } -+ -+ if (mas_is_err(mas)) -+ return xa_err(mas->node); -+ -+ if (unlikely(mas->offset == MAPLE_NODE_SLOTS)) -+ return -EBUSY; -+ -+ /* -+ * mas_rev_awalk() has set mas->min and mas->max to the gap values. If -+ * the maximum is outside the window we are searching, then use the last -+ * location in the search. -+ * mas->max and mas->min is the range of the gap. -+ * mas->index and mas->last are currently set to the search range. -+ */ -+ -+ /* Trim the upper limit to the max. */ -+ if (mas->max <= mas->last) -+ mas->last = mas->max; -+ -+ mas->index = mas->last - size + 1; -+ return 0; -+} -+ -+static inline int mas_alloc(struct ma_state *mas, void *entry, -+ unsigned long size, unsigned long *index) -+{ -+ unsigned long min; -+ -+ mas_start(mas); -+ if (mas_is_none(mas) || mas_is_ptr(mas)) { -+ mas_root_expand(mas, entry); -+ if (mas_is_err(mas)) -+ return xa_err(mas->node); -+ -+ if (!mas->index) -+ return mte_pivot(mas->node, 0); -+ return mte_pivot(mas->node, 1); -+ } -+ -+ /* Must be walking a tree. */ -+ mas_awalk(mas, size); -+ if (mas_is_err(mas)) -+ return xa_err(mas->node); -+ -+ if (mas->offset == MAPLE_NODE_SLOTS) -+ goto no_gap; -+ -+ /* -+ * At this point, mas->node points to the right node and we have an -+ * offset that has a sufficient gap. -+ */ -+ min = mas->min; -+ if (mas->offset) -+ min = mte_pivot(mas->node, mas->offset - 1) + 1; -+ -+ if (mas->index < min) -+ mas->index = min; -+ -+ mas_fill_gap(mas, entry, mas->offset, size, index); -+ return 0; -+ -+no_gap: -+ return -EBUSY; -+} -+ -+static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min, -+ unsigned long max, void *entry, -+ unsigned long size, unsigned long *index) -+{ -+ int ret = 0; -+ -+ ret = mas_empty_area_rev(mas, min, max, size); -+ if (ret) -+ return ret; -+ -+ if (mas_is_err(mas)) -+ return xa_err(mas->node); -+ -+ if (mas->offset == MAPLE_NODE_SLOTS) -+ goto no_gap; -+ -+ mas_fill_gap(mas, entry, mas->offset, size, index); -+ return 0; -+ -+no_gap: -+ return -EBUSY; -+} -+ -+/* -+ * mas_dead_leaves() - Mark all leaves of a node as dead. -+ * @mas: The maple state -+ * @slots: Pointer to the slot array -+ * -+ * Must hold the write lock. -+ * -+ * Return: The number of leaves marked as dead. -+ */ -+static inline -+unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots) -+{ -+ struct maple_node *node; -+ enum maple_type type; -+ void *entry; -+ int offset; -+ -+ for (offset = 0; offset < mt_slot_count(mas->node); offset++) { -+ entry = mas_slot_locked(mas, slots, offset); -+ type = mte_node_type(entry); -+ node = mte_to_node(entry); -+ /* Use both node and type to catch LE & BE metadata */ -+ if (!node || !type) -+ break; -+ -+ mte_set_node_dead(entry); -+ smp_wmb(); /* Needed for RCU */ -+ node->type = type; -+ rcu_assign_pointer(slots[offset], node); -+ } -+ -+ return offset; -+} -+ -+static void __rcu **mas_dead_walk(struct ma_state *mas, unsigned char offset) -+{ -+ struct maple_node *node, *next; -+ void __rcu **slots = NULL; -+ -+ next = mas_mn(mas); -+ do { -+ mas->node = ma_enode_ptr(next); -+ node = mas_mn(mas); -+ slots = ma_slots(node, node->type); -+ next = mas_slot_locked(mas, slots, offset); -+ offset = 0; -+ } while (!ma_is_leaf(next->type)); -+ -+ return slots; -+} -+ -+static void mt_free_walk(struct rcu_head *head) -+{ -+ void __rcu **slots; -+ struct maple_node *node, *start; -+ struct maple_tree mt; -+ unsigned char offset; -+ enum maple_type type; -+ MA_STATE(mas, &mt, 0, 0); -+ -+ node = container_of(head, struct maple_node, rcu); -+ -+ if (ma_is_leaf(node->type)) -+ goto free_leaf; -+ -+ mt_init_flags(&mt, node->ma_flags); -+ mas_lock(&mas); -+ start = node; -+ mas.node = mt_mk_node(node, node->type); -+ slots = mas_dead_walk(&mas, 0); -+ node = mas_mn(&mas); -+ do { -+ mt_free_bulk(node->slot_len, slots); -+ offset = node->parent_slot + 1; -+ mas.node = node->piv_parent; -+ if (mas_mn(&mas) == node) -+ goto start_slots_free; -+ -+ type = mte_node_type(mas.node); -+ slots = ma_slots(mte_to_node(mas.node), type); -+ if ((offset < mt_slots[type]) && (slots[offset])) -+ slots = mas_dead_walk(&mas, offset); -+ -+ node = mas_mn(&mas); -+ } while ((node != start) || (node->slot_len < offset)); -+ -+ slots = ma_slots(node, node->type); -+ mt_free_bulk(node->slot_len, slots); -+ -+start_slots_free: -+ mas_unlock(&mas); -+free_leaf: -+ mt_free_rcu(&node->rcu); -+} -+ -+static inline void __rcu **mas_destroy_descend(struct ma_state *mas, -+ struct maple_enode *prev, unsigned char offset) -+{ -+ struct maple_node *node; -+ struct maple_enode *next = mas->node; -+ void __rcu **slots = NULL; -+ -+ do { -+ mas->node = next; -+ node = mas_mn(mas); -+ slots = ma_slots(node, mte_node_type(mas->node)); -+ next = mas_slot_locked(mas, slots, 0); -+ if ((mte_dead_node(next))) -+ next = mas_slot_locked(mas, slots, 1); -+ -+ mte_set_node_dead(mas->node); -+ node->type = mte_node_type(mas->node); -+ node->piv_parent = prev; -+ node->parent_slot = offset; -+ offset = 0; -+ prev = mas->node; -+ } while (!mte_is_leaf(next)); -+ -+ return slots; -+} -+ -+static void mt_destroy_walk(struct maple_enode *enode, unsigned char ma_flags, -+ bool free) -+{ -+ void __rcu **slots; -+ struct maple_node *node = mte_to_node(enode); -+ struct maple_enode *start; -+ struct maple_tree mt; -+ -+ MA_STATE(mas, &mt, 0, 0); -+ -+ if (mte_is_leaf(enode)) -+ goto free_leaf; -+ -+ mt_init_flags(&mt, ma_flags); -+ mas_lock(&mas); -+ -+ mas.node = start = enode; -+ slots = mas_destroy_descend(&mas, start, 0); -+ node = mas_mn(&mas); -+ do { -+ enum maple_type type; -+ unsigned char offset; -+ struct maple_enode *parent, *tmp; -+ -+ node->slot_len = mas_dead_leaves(&mas, slots); -+ if (free) -+ mt_free_bulk(node->slot_len, slots); -+ offset = node->parent_slot + 1; -+ mas.node = node->piv_parent; -+ if (mas_mn(&mas) == node) -+ goto start_slots_free; -+ -+ type = mte_node_type(mas.node); -+ slots = ma_slots(mte_to_node(mas.node), type); -+ if (offset >= mt_slots[type]) -+ goto next; -+ -+ tmp = mas_slot_locked(&mas, slots, offset); -+ if (mte_node_type(tmp) && mte_to_node(tmp)) { -+ parent = mas.node; -+ mas.node = tmp; -+ slots = mas_destroy_descend(&mas, parent, offset); -+ } -+next: -+ node = mas_mn(&mas); -+ } while (start != mas.node); -+ -+ node = mas_mn(&mas); -+ node->slot_len = mas_dead_leaves(&mas, slots); -+ if (free) -+ mt_free_bulk(node->slot_len, slots); -+ -+start_slots_free: -+ mas_unlock(&mas); -+ -+free_leaf: -+ if (free) -+ mt_free_rcu(&node->rcu); -+} -+ -+/* -+ * mte_destroy_walk() - Free a tree or sub-tree. -+ * @enode - the encoded maple node (maple_enode) to start -+ * @mn - the tree to free - needed for node types. -+ * -+ * Must hold the write lock. -+ */ -+static inline void mte_destroy_walk(struct maple_enode *enode, -+ struct maple_tree *mt) -+{ -+ struct maple_node *node = mte_to_node(enode); -+ -+ if (mt_in_rcu(mt)) { -+ mt_destroy_walk(enode, mt->ma_flags, false); -+ call_rcu(&node->rcu, mt_free_walk); -+ } else { -+ mt_destroy_walk(enode, mt->ma_flags, true); -+ } -+} -+ -+static void mas_wr_store_setup(struct ma_wr_state *wr_mas) -+{ -+ if (!mas_is_start(wr_mas->mas)) { -+ if (mas_is_none(wr_mas->mas)) { -+ mas_reset(wr_mas->mas); -+ } else { -+ wr_mas->r_max = wr_mas->mas->max; -+ wr_mas->type = mte_node_type(wr_mas->mas->node); -+ if (mas_is_span_wr(wr_mas)) -+ mas_reset(wr_mas->mas); -+ } -+ } -+ -+} -+ -+/* Interface */ -+ -+/** -+ * mas_store() - Store an @entry. -+ * @mas: The maple state. -+ * @entry: The entry to store. -+ * -+ * The @mas->index and @mas->last is used to set the range for the @entry. -+ * Note: The @mas should have pre-allocated entries to ensure there is memory to -+ * store the entry. Please see mas_expected_entries()/mas_destroy() for more details. -+ * -+ * Return: the first entry between mas->index and mas->last or %NULL. -+ */ -+void *mas_store(struct ma_state *mas, void *entry) -+{ -+ MA_WR_STATE(wr_mas, mas, entry); -+ -+ trace_ma_write(__func__, mas, 0, entry); -+#ifdef CONFIG_DEBUG_MAPLE_TREE -+ if (mas->index > mas->last) -+ pr_err("Error %lu > %lu %p\n", mas->index, mas->last, entry); -+ MT_BUG_ON(mas->tree, mas->index > mas->last); -+ if (mas->index > mas->last) { -+ mas_set_err(mas, -EINVAL); -+ return NULL; -+ } -+ -+#endif -+ -+ /* -+ * Storing is the same operation as insert with the added caveat that it -+ * can overwrite entries. Although this seems simple enough, one may -+ * want to examine what happens if a single store operation was to -+ * overwrite multiple entries within a self-balancing B-Tree. -+ */ -+ mas_wr_store_setup(&wr_mas); -+ mas_wr_store_entry(&wr_mas); -+ return wr_mas.content; -+} -+ -+/** -+ * mas_store_gfp() - Store a value into the tree. -+ * @mas: The maple state -+ * @entry: The entry to store -+ * @gfp: The GFP_FLAGS to use for allocations if necessary. -+ * -+ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not -+ * be allocated. -+ */ -+int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) -+{ -+ MA_WR_STATE(wr_mas, mas, entry); -+ -+ mas_wr_store_setup(&wr_mas); -+ trace_ma_write(__func__, mas, 0, entry); -+retry: -+ mas_wr_store_entry(&wr_mas); -+ if (unlikely(mas_nomem(mas, gfp))) -+ goto retry; -+ -+ if (unlikely(mas_is_err(mas))) -+ return xa_err(mas->node); -+ -+ return 0; -+} -+ -+/** -+ * mas_store_prealloc() - Store a value into the tree using memory -+ * preallocated in the maple state. -+ * @mas: The maple state -+ * @entry: The entry to store. -+ */ -+void mas_store_prealloc(struct ma_state *mas, void *entry) -+{ -+ MA_WR_STATE(wr_mas, mas, entry); -+ -+ mas_wr_store_setup(&wr_mas); -+ trace_ma_write(__func__, mas, 0, entry); -+ mas_wr_store_entry(&wr_mas); -+ BUG_ON(mas_is_err(mas)); -+ mas_destroy(mas); -+} -+ -+/** -+ * mas_preallocate() - Preallocate enough nodes for a store operation -+ * @mas: The maple state -+ * @entry: The entry that will be stored -+ * @gfp: The GFP_FLAGS to use for allocations. -+ * -+ * Return: 0 on success, -ENOMEM if memory could not be allocated. -+ */ -+int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) -+{ -+ int ret; -+ -+ mas_node_count_gfp(mas, 1 + mas_mt_height(mas) * 3, gfp); -+ mas->mas_flags |= MA_STATE_PREALLOC; -+ if (likely(!mas_is_err(mas))) -+ return 0; -+ -+ mas_set_alloc_req(mas, 0); -+ ret = xa_err(mas->node); -+ mas_reset(mas); -+ mas_destroy(mas); -+ mas_reset(mas); -+ return ret; -+} -+ -+/* -+ * mas_destroy() - destroy a maple state. -+ * @mas: The maple state -+ * -+ * Upon completion, check the left-most node and rebalance against the node to -+ * the right if necessary. Frees any allocated nodes associated with this maple -+ * state. -+ */ -+void mas_destroy(struct ma_state *mas) -+{ -+ struct maple_alloc *node; -+ -+ /* -+ * When using mas_for_each() to insert an expected number of elements, -+ * it is possible that the number inserted is less than the expected -+ * number. To fix an invalid final node, a check is performed here to -+ * rebalance the previous node with the final node. -+ */ -+ if (mas->mas_flags & MA_STATE_REBALANCE) { -+ unsigned char end; -+ -+ if (mas_is_start(mas)) -+ mas_start(mas); -+ -+ mtree_range_walk(mas); -+ end = mas_data_end(mas) + 1; -+ if (end < mt_min_slot_count(mas->node) - 1) -+ mas_destroy_rebalance(mas, end); -+ -+ mas->mas_flags &= ~MA_STATE_REBALANCE; -+ } -+ mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC); -+ -+ while (mas->alloc && !((unsigned long)mas->alloc & 0x1)) { -+ node = mas->alloc; -+ mas->alloc = node->slot[0]; -+ if (node->node_count > 0) -+ mt_free_bulk(node->node_count, -+ (void __rcu **)&node->slot[1]); -+ kmem_cache_free(maple_node_cache, node); -+ } -+ mas->alloc = NULL; -+} -+ -+/* -+ * mas_expected_entries() - Set the expected number of entries that will be inserted. -+ * @mas: The maple state -+ * @nr_entries: The number of expected entries. -+ * -+ * This will attempt to pre-allocate enough nodes to store the expected number -+ * of entries. The allocations will occur using the bulk allocator interface -+ * for speed. Please call mas_destroy() on the @mas after inserting the entries -+ * to ensure any unused nodes are freed. -+ * -+ * Return: 0 on success, -ENOMEM if memory could not be allocated. -+ */ -+int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) -+{ -+ int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2; -+ struct maple_enode *enode = mas->node; -+ int nr_nodes; -+ int ret; -+ -+ /* -+ * Sometimes it is necessary to duplicate a tree to a new tree, such as -+ * forking a process and duplicating the VMAs from one tree to a new -+ * tree. When such a situation arises, it is known that the new tree is -+ * not going to be used until the entire tree is populated. For -+ * performance reasons, it is best to use a bulk load with RCU disabled. -+ * This allows for optimistic splitting that favours the left and reuse -+ * of nodes during the operation. -+ */ -+ -+ /* Optimize splitting for bulk insert in-order */ -+ mas->mas_flags |= MA_STATE_BULK; -+ -+ /* -+ * Avoid overflow, assume a gap between each entry and a trailing null. -+ * If this is wrong, it just means allocation can happen during -+ * insertion of entries. -+ */ -+ nr_nodes = max(nr_entries, nr_entries * 2 + 1); -+ if (!mt_is_alloc(mas->tree)) -+ nonleaf_cap = MAPLE_RANGE64_SLOTS - 2; -+ -+ /* Leaves; reduce slots to keep space for expansion */ -+ nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2); -+ /* Internal nodes */ -+ nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap); -+ /* Add working room for split (2 nodes) + new parents */ -+ mas_node_count(mas, nr_nodes + 3); -+ -+ /* Detect if allocations run out */ -+ mas->mas_flags |= MA_STATE_PREALLOC; -+ -+ if (!mas_is_err(mas)) -+ return 0; -+ -+ ret = xa_err(mas->node); -+ mas->node = enode; -+ mas_destroy(mas); -+ return ret; -+ -+} -+ -+/** -+ * mas_next() - Get the next entry. -+ * @mas: The maple state -+ * @max: The maximum index to check. -+ * -+ * Returns the next entry after @mas->index. -+ * Must hold rcu_read_lock or the write lock. -+ * Can return the zero entry. -+ * -+ * Return: The next entry or %NULL -+ */ -+void *mas_next(struct ma_state *mas, unsigned long max) -+{ -+ if (mas_is_none(mas) || mas_is_paused(mas)) -+ mas->node = MAS_START; -+ -+ if (mas_is_start(mas)) -+ mas_walk(mas); /* Retries on dead nodes handled by mas_walk */ -+ -+ if (mas_is_ptr(mas)) { -+ if (!mas->index) { -+ mas->index = 1; -+ mas->last = ULONG_MAX; -+ } -+ return NULL; -+ } -+ -+ if (mas->last == ULONG_MAX) -+ return NULL; -+ -+ /* Retries on dead nodes handled by mas_next_entry */ -+ return mas_next_entry(mas, max); -+} -+EXPORT_SYMBOL_GPL(mas_next); -+ -+/** -+ * mt_next() - get the next value in the maple tree -+ * @mt: The maple tree -+ * @index: The start index -+ * @max: The maximum index to check -+ * -+ * Return: The entry at @index or higher, or %NULL if nothing is found. -+ */ -+void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max) -+{ -+ void *entry = NULL; -+ MA_STATE(mas, mt, index, index); -+ -+ rcu_read_lock(); -+ entry = mas_next(&mas, max); -+ rcu_read_unlock(); -+ return entry; -+} -+EXPORT_SYMBOL_GPL(mt_next); -+ -+/** -+ * mas_prev() - Get the previous entry -+ * @mas: The maple state -+ * @min: The minimum value to check. -+ * -+ * Must hold rcu_read_lock or the write lock. -+ * Will reset mas to MAS_START if the node is MAS_NONE. Will stop on not -+ * searchable nodes. -+ * -+ * Return: the previous value or %NULL. -+ */ -+void *mas_prev(struct ma_state *mas, unsigned long min) -+{ -+ if (!mas->index) { -+ /* Nothing comes before 0 */ -+ mas->last = 0; -+ return NULL; -+ } -+ -+ if (unlikely(mas_is_ptr(mas))) -+ return NULL; -+ -+ if (mas_is_none(mas) || mas_is_paused(mas)) -+ mas->node = MAS_START; -+ -+ if (mas_is_start(mas)) { -+ mas_walk(mas); -+ if (!mas->index) -+ return NULL; -+ } -+ -+ if (mas_is_ptr(mas)) { -+ if (!mas->index) { -+ mas->last = 0; -+ return NULL; -+ } -+ -+ mas->index = mas->last = 0; -+ return mas_root_locked(mas); -+ } -+ return mas_prev_entry(mas, min); -+} -+EXPORT_SYMBOL_GPL(mas_prev); -+ -+/** -+ * mt_prev() - get the previous value in the maple tree -+ * @mt: The maple tree -+ * @index: The start index -+ * @min: The minimum index to check -+ * -+ * Return: The entry at @index or lower, or %NULL if nothing is found. -+ */ -+void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min) -+{ -+ void *entry = NULL; -+ MA_STATE(mas, mt, index, index); -+ -+ rcu_read_lock(); -+ entry = mas_prev(&mas, min); -+ rcu_read_unlock(); -+ return entry; -+} -+EXPORT_SYMBOL_GPL(mt_prev); -+ -+/** -+ * mas_pause() - Pause a mas_find/mas_for_each to drop the lock. -+ * @mas: The maple state to pause -+ * -+ * Some users need to pause a walk and drop the lock they're holding in -+ * order to yield to a higher priority thread or carry out an operation -+ * on an entry. Those users should call this function before they drop -+ * the lock. It resets the @mas to be suitable for the next iteration -+ * of the loop after the user has reacquired the lock. If most entries -+ * found during a walk require you to call mas_pause(), the mt_for_each() -+ * iterator may be more appropriate. -+ * -+ */ -+void mas_pause(struct ma_state *mas) -+{ -+ mas->node = MAS_PAUSE; -+} -+EXPORT_SYMBOL_GPL(mas_pause); -+ -+/** -+ * mas_find() - On the first call, find the entry at or after mas->index up to -+ * %max. Otherwise, find the entry after mas->index. -+ * @mas: The maple state -+ * @max: The maximum value to check. -+ * -+ * Must hold rcu_read_lock or the write lock. -+ * If an entry exists, last and index are updated accordingly. -+ * May set @mas->node to MAS_NONE. -+ * -+ * Return: The entry or %NULL. -+ */ -+void *mas_find(struct ma_state *mas, unsigned long max) -+{ -+ if (unlikely(mas_is_paused(mas))) { -+ if (unlikely(mas->last == ULONG_MAX)) { -+ mas->node = MAS_NONE; -+ return NULL; -+ } -+ mas->node = MAS_START; -+ mas->index = ++mas->last; -+ } -+ -+ if (unlikely(mas_is_start(mas))) { -+ /* First run or continue */ -+ void *entry; -+ -+ if (mas->index > max) -+ return NULL; -+ -+ entry = mas_walk(mas); -+ if (entry) -+ return entry; -+ } -+ -+ if (unlikely(!mas_searchable(mas))) -+ return NULL; -+ -+ /* Retries on dead nodes handled by mas_next_entry */ -+ return mas_next_entry(mas, max); -+} -+ -+/** -+ * mas_find_rev: On the first call, find the first non-null entry at or below -+ * mas->index down to %min. Otherwise find the first non-null entry below -+ * mas->index down to %min. -+ * @mas: The maple state -+ * @min: The minimum value to check. -+ * -+ * Must hold rcu_read_lock or the write lock. -+ * If an entry exists, last and index are updated accordingly. -+ * May set @mas->node to MAS_NONE. -+ * -+ * Return: The entry or %NULL. -+ */ -+void *mas_find_rev(struct ma_state *mas, unsigned long min) -+{ -+ if (unlikely(mas_is_paused(mas))) { -+ if (unlikely(mas->last == ULONG_MAX)) { -+ mas->node = MAS_NONE; -+ return NULL; -+ } -+ mas->node = MAS_START; -+ mas->last = --mas->index; -+ } -+ -+ if (unlikely(mas_is_start(mas))) { -+ /* First run or continue */ -+ void *entry; -+ -+ if (mas->index < min) -+ return NULL; -+ -+ entry = mas_walk(mas); -+ if (entry) -+ return entry; -+ } -+ -+ if (unlikely(!mas_searchable(mas))) -+ return NULL; -+ -+ if (mas->index < min) -+ return NULL; -+ -+ /* Retries on dead nodes handled by mas_next_entry */ -+ return mas_prev_entry(mas, min); -+} -+EXPORT_SYMBOL_GPL(mas_find); -+ -+/** -+ * mas_erase() - Find the range in which index resides and erase the entire -+ * range. -+ * @mas: The maple state -+ * -+ * Must hold the write lock. -+ * Searches for @mas->index, sets @mas->index and @mas->last to the range and -+ * erases that range. -+ * -+ * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. -+ */ -+void *mas_erase(struct ma_state *mas) -+{ -+ void *entry; -+ MA_WR_STATE(wr_mas, mas, NULL); -+ -+ if (mas_is_none(mas) || mas_is_paused(mas)) -+ mas->node = MAS_START; -+ -+ /* Retry unnecessary when holding the write lock. */ -+ entry = mas_state_walk(mas); -+ if (!entry) -+ return NULL; -+ -+write_retry: -+ /* Must reset to ensure spanning writes of last slot are detected */ -+ mas_reset(mas); -+ mas_wr_store_setup(&wr_mas); -+ mas_wr_store_entry(&wr_mas); -+ if (mas_nomem(mas, GFP_KERNEL)) -+ goto write_retry; -+ -+ return entry; -+} -+EXPORT_SYMBOL_GPL(mas_erase); -+ -+/** -+ * mas_nomem() - Check if there was an error allocating and do the allocation -+ * if necessary If there are allocations, then free them. -+ * @mas: The maple state -+ * @gfp: The GFP_FLAGS to use for allocations -+ * Return: true on allocation, false otherwise. -+ */ -+bool mas_nomem(struct ma_state *mas, gfp_t gfp) -+ __must_hold(mas->tree->lock) -+{ -+ if (likely(mas->node != MA_ERROR(-ENOMEM))) { -+ mas_destroy(mas); -+ return false; -+ } -+ -+ if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { -+ mtree_unlock(mas->tree); -+ mas_alloc_nodes(mas, gfp); -+ mtree_lock(mas->tree); -+ } else { -+ mas_alloc_nodes(mas, gfp); -+ } -+ -+ if (!mas_allocated(mas)) -+ return false; -+ -+ mas->node = MAS_START; -+ return true; -+} -+ -+void __init maple_tree_init(void) -+{ -+ maple_node_cache = kmem_cache_create("maple_node", -+ sizeof(struct maple_node), sizeof(struct maple_node), -+ SLAB_PANIC, NULL); -+} -+ -+/** -+ * mtree_load() - Load a value stored in a maple tree -+ * @mt: The maple tree -+ * @index: The index to load -+ * -+ * Return: the entry or %NULL -+ */ -+void *mtree_load(struct maple_tree *mt, unsigned long index) -+{ -+ MA_STATE(mas, mt, index, index); -+ void *entry; -+ -+ trace_ma_read(__func__, &mas); -+ rcu_read_lock(); -+retry: -+ entry = mas_start(&mas); -+ if (unlikely(mas_is_none(&mas))) -+ goto unlock; -+ -+ if (unlikely(mas_is_ptr(&mas))) { -+ if (index) -+ entry = NULL; -+ -+ goto unlock; -+ } -+ -+ entry = mtree_lookup_walk(&mas); -+ if (!entry && unlikely(mas_is_start(&mas))) -+ goto retry; -+unlock: -+ rcu_read_unlock(); -+ if (xa_is_zero(entry)) -+ return NULL; -+ -+ return entry; -+} -+EXPORT_SYMBOL(mtree_load); -+ -+/** -+ * mtree_store_range() - Store an entry at a given range. -+ * @mt: The maple tree -+ * @index: The start of the range -+ * @last: The end of the range -+ * @entry: The entry to store -+ * @gfp: The GFP_FLAGS to use for allocations -+ * -+ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not -+ * be allocated. -+ */ -+int mtree_store_range(struct maple_tree *mt, unsigned long index, -+ unsigned long last, void *entry, gfp_t gfp) -+{ -+ MA_STATE(mas, mt, index, last); -+ MA_WR_STATE(wr_mas, &mas, entry); -+ -+ trace_ma_write(__func__, &mas, 0, entry); -+ if (WARN_ON_ONCE(xa_is_advanced(entry))) -+ return -EINVAL; -+ -+ if (index > last) -+ return -EINVAL; -+ -+ mtree_lock(mt); -+retry: -+ mas_wr_store_entry(&wr_mas); -+ if (mas_nomem(&mas, gfp)) -+ goto retry; -+ -+ mtree_unlock(mt); -+ if (mas_is_err(&mas)) -+ return xa_err(mas.node); -+ -+ return 0; -+} -+EXPORT_SYMBOL(mtree_store_range); -+ -+/** -+ * mtree_store() - Store an entry at a given index. -+ * @mt: The maple tree -+ * @index: The index to store the value -+ * @entry: The entry to store -+ * @gfp: The GFP_FLAGS to use for allocations -+ * -+ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not -+ * be allocated. -+ */ -+int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, -+ gfp_t gfp) -+{ -+ return mtree_store_range(mt, index, index, entry, gfp); -+} -+EXPORT_SYMBOL(mtree_store); -+ -+/** -+ * mtree_insert_range() - Insert an entry at a give range if there is no value. -+ * @mt: The maple tree -+ * @first: The start of the range -+ * @last: The end of the range -+ * @entry: The entry to store -+ * @gfp: The GFP_FLAGS to use for allocations. -+ * -+ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid -+ * request, -ENOMEM if memory could not be allocated. -+ */ -+int mtree_insert_range(struct maple_tree *mt, unsigned long first, -+ unsigned long last, void *entry, gfp_t gfp) -+{ -+ MA_STATE(ms, mt, first, last); -+ -+ if (WARN_ON_ONCE(xa_is_advanced(entry))) -+ return -EINVAL; -+ -+ if (first > last) -+ return -EINVAL; -+ -+ mtree_lock(mt); -+retry: -+ mas_insert(&ms, entry); -+ if (mas_nomem(&ms, gfp)) -+ goto retry; -+ -+ mtree_unlock(mt); -+ if (mas_is_err(&ms)) -+ return xa_err(ms.node); -+ -+ return 0; -+} -+EXPORT_SYMBOL(mtree_insert_range); -+ -+/** -+ * mtree_insert() - Insert an entry at a give index if there is no value. -+ * @mt: The maple tree -+ * @index : The index to store the value -+ * @entry: The entry to store -+ * @gfp: The FGP_FLAGS to use for allocations. -+ * -+ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid -+ * request, -ENOMEM if memory could not be allocated. -+ */ -+int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, -+ gfp_t gfp) -+{ -+ return mtree_insert_range(mt, index, index, entry, gfp); -+} -+EXPORT_SYMBOL(mtree_insert); -+ -+int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, -+ void *entry, unsigned long size, unsigned long min, -+ unsigned long max, gfp_t gfp) -+{ -+ int ret = 0; -+ -+ MA_STATE(mas, mt, min, max - size); -+ if (!mt_is_alloc(mt)) -+ return -EINVAL; -+ -+ if (WARN_ON_ONCE(mt_is_reserved(entry))) -+ return -EINVAL; -+ -+ if (min > max) -+ return -EINVAL; -+ -+ if (max < size) -+ return -EINVAL; -+ -+ if (!size) -+ return -EINVAL; -+ -+ mtree_lock(mt); -+retry: -+ mas.offset = 0; -+ mas.index = min; -+ mas.last = max - size; -+ ret = mas_alloc(&mas, entry, size, startp); -+ if (mas_nomem(&mas, gfp)) -+ goto retry; -+ -+ mtree_unlock(mt); -+ return ret; -+} -+EXPORT_SYMBOL(mtree_alloc_range); -+ -+int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, -+ void *entry, unsigned long size, unsigned long min, -+ unsigned long max, gfp_t gfp) -+{ -+ int ret = 0; -+ -+ MA_STATE(mas, mt, min, max - size); -+ if (!mt_is_alloc(mt)) -+ return -EINVAL; -+ -+ if (WARN_ON_ONCE(mt_is_reserved(entry))) -+ return -EINVAL; -+ -+ if (min >= max) -+ return -EINVAL; -+ -+ if (max < size - 1) -+ return -EINVAL; -+ -+ if (!size) -+ return -EINVAL; -+ -+ mtree_lock(mt); -+retry: -+ ret = mas_rev_alloc(&mas, min, max, entry, size, startp); -+ if (mas_nomem(&mas, gfp)) -+ goto retry; -+ -+ mtree_unlock(mt); -+ return ret; -+} -+EXPORT_SYMBOL(mtree_alloc_rrange); -+ -+/** -+ * mtree_erase() - Find an index and erase the entire range. -+ * @mt: The maple tree -+ * @index: The index to erase -+ * -+ * Erasing is the same as a walk to an entry then a store of a NULL to that -+ * ENTIRE range. In fact, it is implemented as such using the advanced API. -+ * -+ * Return: The entry stored at the @index or %NULL -+ */ -+void *mtree_erase(struct maple_tree *mt, unsigned long index) -+{ -+ void *entry = NULL; -+ -+ MA_STATE(mas, mt, index, index); -+ trace_ma_op(__func__, &mas); -+ -+ mtree_lock(mt); -+ entry = mas_erase(&mas); -+ mtree_unlock(mt); -+ -+ return entry; -+} -+EXPORT_SYMBOL(mtree_erase); -+ -+/** -+ * __mt_destroy() - Walk and free all nodes of a locked maple tree. -+ * @mt: The maple tree -+ * -+ * Note: Does not handle locking. -+ */ -+void __mt_destroy(struct maple_tree *mt) -+{ -+ void *root = mt_root_locked(mt); -+ -+ rcu_assign_pointer(mt->ma_root, NULL); -+ if (xa_is_node(root)) -+ mte_destroy_walk(root, mt); -+ -+ mt->ma_flags = 0; -+} -+EXPORT_SYMBOL_GPL(__mt_destroy); -+ -+/** -+ * mtree_destroy() - Destroy a maple tree -+ * @mt: The maple tree -+ * -+ * Frees all resources used by the tree. Handles locking. -+ */ -+void mtree_destroy(struct maple_tree *mt) -+{ -+ mtree_lock(mt); -+ __mt_destroy(mt); -+ mtree_unlock(mt); -+} -+EXPORT_SYMBOL(mtree_destroy); -+ -+/** -+ * mt_find() - Search from the start up until an entry is found. -+ * @mt: The maple tree -+ * @index: Pointer which contains the start location of the search -+ * @max: The maximum value to check -+ * -+ * Handles locking. @index will be incremented to one beyond the range. -+ * -+ * Return: The entry at or after the @index or %NULL -+ */ -+void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) -+{ -+ MA_STATE(mas, mt, *index, *index); -+ void *entry; -+#ifdef CONFIG_DEBUG_MAPLE_TREE -+ unsigned long copy = *index; -+#endif -+ -+ trace_ma_read(__func__, &mas); -+ -+ if ((*index) > max) -+ return NULL; -+ -+ rcu_read_lock(); -+retry: -+ entry = mas_state_walk(&mas); -+ if (mas_is_start(&mas)) -+ goto retry; -+ -+ if (unlikely(xa_is_zero(entry))) -+ entry = NULL; -+ -+ if (entry) -+ goto unlock; -+ -+ while (mas_searchable(&mas) && (mas.index < max)) { -+ entry = mas_next_entry(&mas, max); -+ if (likely(entry && !xa_is_zero(entry))) -+ break; -+ } -+ -+ if (unlikely(xa_is_zero(entry))) -+ entry = NULL; -+unlock: -+ rcu_read_unlock(); -+ if (likely(entry)) { -+ *index = mas.last + 1; -+#ifdef CONFIG_DEBUG_MAPLE_TREE -+ if ((*index) && (*index) <= copy) -+ pr_err("index not increased! %lx <= %lx\n", -+ *index, copy); -+ MT_BUG_ON(mt, (*index) && ((*index) <= copy)); -+#endif -+ } -+ -+ return entry; -+} -+EXPORT_SYMBOL(mt_find); -+ -+/** -+ * mt_find_after() - Search from the start up until an entry is found. -+ * @mt: The maple tree -+ * @index: Pointer which contains the start location of the search -+ * @max: The maximum value to check -+ * -+ * Handles locking, detects wrapping on index == 0 -+ * -+ * Return: The entry at or after the @index or %NULL -+ */ -+void *mt_find_after(struct maple_tree *mt, unsigned long *index, -+ unsigned long max) -+{ -+ if (!(*index)) -+ return NULL; -+ -+ return mt_find(mt, index, max); -+} -+EXPORT_SYMBOL(mt_find_after); -+ -+#ifdef CONFIG_DEBUG_MAPLE_TREE -+atomic_t maple_tree_tests_run; -+EXPORT_SYMBOL_GPL(maple_tree_tests_run); -+atomic_t maple_tree_tests_passed; -+EXPORT_SYMBOL_GPL(maple_tree_tests_passed); -+ -+#ifndef __KERNEL__ -+extern void kmem_cache_set_non_kernel(struct kmem_cache *, unsigned int); -+void mt_set_non_kernel(unsigned int val) -+{ -+ kmem_cache_set_non_kernel(maple_node_cache, val); -+} -+ -+extern unsigned long kmem_cache_get_alloc(struct kmem_cache *); -+unsigned long mt_get_alloc_size(void) -+{ -+ return kmem_cache_get_alloc(maple_node_cache); -+} -+ -+extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *); -+void mt_zero_nr_tallocated(void) -+{ -+ kmem_cache_zero_nr_tallocated(maple_node_cache); -+} -+ -+extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *); -+unsigned int mt_nr_tallocated(void) -+{ -+ return kmem_cache_nr_tallocated(maple_node_cache); -+} -+ -+extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *); -+unsigned int mt_nr_allocated(void) -+{ -+ return kmem_cache_nr_allocated(maple_node_cache); -+} -+ -+/* -+ * mas_dead_node() - Check if the maple state is pointing to a dead node. -+ * @mas: The maple state -+ * @index: The index to restore in @mas. -+ * -+ * Used in test code. -+ * Return: 1 if @mas has been reset to MAS_START, 0 otherwise. -+ */ -+static inline int mas_dead_node(struct ma_state *mas, unsigned long index) -+{ -+ if (unlikely(!mas_searchable(mas) || mas_is_start(mas))) -+ return 0; -+ -+ if (likely(!mte_dead_node(mas->node))) -+ return 0; -+ -+ mas_rewalk(mas, index); -+ return 1; -+} -+#endif /* not defined __KERNEL__ */ -+ -+/* -+ * mas_get_slot() - Get the entry in the maple state node stored at @offset. -+ * @mas: The maple state -+ * @offset: The offset into the slot array to fetch. -+ * -+ * Return: The entry stored at @offset. -+ */ -+static inline struct maple_enode *mas_get_slot(struct ma_state *mas, -+ unsigned char offset) -+{ -+ return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)), -+ offset); -+} -+ -+ -+/* -+ * mas_first_entry() - Go the first leaf and find the first entry. -+ * @mas: the maple state. -+ * @limit: the maximum index to check. -+ * @*r_start: Pointer to set to the range start. -+ * -+ * Sets mas->offset to the offset of the entry, r_start to the range minimum. -+ * -+ * Return: The first entry or MAS_NONE. -+ */ -+static inline void *mas_first_entry(struct ma_state *mas, struct maple_node *mn, -+ unsigned long limit, enum maple_type mt) -+ -+{ -+ unsigned long max; -+ unsigned long *pivots; -+ void __rcu **slots; -+ void *entry = NULL; -+ -+ mas->index = mas->min; -+ if (mas->index > limit) -+ goto none; -+ -+ max = mas->max; -+ mas->offset = 0; -+ while (likely(!ma_is_leaf(mt))) { -+ MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); -+ slots = ma_slots(mn, mt); -+ pivots = ma_pivots(mn, mt); -+ max = pivots[0]; -+ entry = mas_slot(mas, slots, 0); -+ if (unlikely(ma_dead_node(mn))) -+ return NULL; -+ mas->node = entry; -+ mn = mas_mn(mas); -+ mt = mte_node_type(mas->node); -+ } -+ MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); -+ -+ mas->max = max; -+ slots = ma_slots(mn, mt); -+ entry = mas_slot(mas, slots, 0); -+ if (unlikely(ma_dead_node(mn))) -+ return NULL; -+ -+ /* Slot 0 or 1 must be set */ -+ if (mas->index > limit) -+ goto none; -+ -+ if (likely(entry)) -+ return entry; -+ -+ pivots = ma_pivots(mn, mt); -+ mas->index = pivots[0] + 1; -+ mas->offset = 1; -+ entry = mas_slot(mas, slots, 1); -+ if (unlikely(ma_dead_node(mn))) -+ return NULL; -+ -+ if (mas->index > limit) -+ goto none; -+ -+ if (likely(entry)) -+ return entry; -+ -+none: -+ if (likely(!ma_dead_node(mn))) -+ mas->node = MAS_NONE; -+ return NULL; -+} -+ -+/* Depth first search, post-order */ -+static void mas_dfs_postorder(struct ma_state *mas, unsigned long max) -+{ -+ -+ struct maple_enode *p = MAS_NONE, *mn = mas->node; -+ unsigned long p_min, p_max; -+ -+ mas_next_node(mas, mas_mn(mas), max); -+ if (!mas_is_none(mas)) -+ return; -+ -+ if (mte_is_root(mn)) -+ return; -+ -+ mas->node = mn; -+ mas_ascend(mas); -+ while (mas->node != MAS_NONE) { -+ p = mas->node; -+ p_min = mas->min; -+ p_max = mas->max; -+ mas_prev_node(mas, 0); -+ } -+ -+ if (p == MAS_NONE) -+ return; -+ -+ mas->node = p; -+ mas->max = p_max; -+ mas->min = p_min; -+} -+ -+/* Tree validations */ -+static void mt_dump_node(const struct maple_tree *mt, void *entry, -+ unsigned long min, unsigned long max, unsigned int depth); -+static void mt_dump_range(unsigned long min, unsigned long max, -+ unsigned int depth) -+{ -+ static const char spaces[] = " "; -+ -+ if (min == max) -+ pr_info("%.*s%lu: ", depth * 2, spaces, min); -+ else -+ pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max); -+} -+ -+static void mt_dump_entry(void *entry, unsigned long min, unsigned long max, -+ unsigned int depth) -+{ -+ mt_dump_range(min, max, depth); -+ -+ if (xa_is_value(entry)) -+ pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry), -+ xa_to_value(entry), entry); -+ else if (xa_is_zero(entry)) -+ pr_cont("zero (%ld)\n", xa_to_internal(entry)); -+ else if (mt_is_reserved(entry)) -+ pr_cont("UNKNOWN ENTRY (%p)\n", entry); -+ else -+ pr_cont("%p\n", entry); -+} -+ -+static void mt_dump_range64(const struct maple_tree *mt, void *entry, -+ unsigned long min, unsigned long max, unsigned int depth) -+{ -+ struct maple_range_64 *node = &mte_to_node(entry)->mr64; -+ bool leaf = mte_is_leaf(entry); -+ unsigned long first = min; -+ int i; -+ -+ pr_cont(" contents: "); -+ for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) -+ pr_cont("%p %lu ", node->slot[i], node->pivot[i]); -+ pr_cont("%p\n", node->slot[i]); -+ for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) { -+ unsigned long last = max; -+ -+ if (i < (MAPLE_RANGE64_SLOTS - 1)) -+ last = node->pivot[i]; -+ else if (!node->slot[i] && max != mt_max[mte_node_type(entry)]) -+ break; -+ if (last == 0 && i > 0) -+ break; -+ if (leaf) -+ mt_dump_entry(mt_slot(mt, node->slot, i), -+ first, last, depth + 1); -+ else if (node->slot[i]) -+ mt_dump_node(mt, mt_slot(mt, node->slot, i), -+ first, last, depth + 1); -+ -+ if (last == max) -+ break; -+ if (last > max) { -+ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", -+ node, last, max, i); -+ break; -+ } -+ first = last + 1; -+ } -+} -+ -+static void mt_dump_arange64(const struct maple_tree *mt, void *entry, -+ unsigned long min, unsigned long max, unsigned int depth) -+{ -+ struct maple_arange_64 *node = &mte_to_node(entry)->ma64; -+ bool leaf = mte_is_leaf(entry); -+ unsigned long first = min; -+ int i; -+ -+ pr_cont(" contents: "); -+ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) -+ pr_cont("%lu ", node->gap[i]); -+ pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap); -+ for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) -+ pr_cont("%p %lu ", node->slot[i], node->pivot[i]); -+ pr_cont("%p\n", node->slot[i]); -+ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) { -+ unsigned long last = max; -+ -+ if (i < (MAPLE_ARANGE64_SLOTS - 1)) -+ last = node->pivot[i]; -+ else if (!node->slot[i]) -+ break; -+ if (last == 0 && i > 0) -+ break; -+ if (leaf) -+ mt_dump_entry(mt_slot(mt, node->slot, i), -+ first, last, depth + 1); -+ else if (node->slot[i]) -+ mt_dump_node(mt, mt_slot(mt, node->slot, i), -+ first, last, depth + 1); -+ -+ if (last == max) -+ break; -+ if (last > max) { -+ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", -+ node, last, max, i); -+ break; -+ } -+ first = last + 1; -+ } -+} -+ -+static void mt_dump_node(const struct maple_tree *mt, void *entry, -+ unsigned long min, unsigned long max, unsigned int depth) -+{ -+ struct maple_node *node = mte_to_node(entry); -+ unsigned int type = mte_node_type(entry); -+ unsigned int i; -+ -+ mt_dump_range(min, max, depth); -+ -+ pr_cont("node %p depth %d type %d parent %p", node, depth, type, -+ node ? node->parent : NULL); -+ switch (type) { -+ case maple_dense: -+ pr_cont("\n"); -+ for (i = 0; i < MAPLE_NODE_SLOTS; i++) { -+ if (min + i > max) -+ pr_cont("OUT OF RANGE: "); -+ mt_dump_entry(mt_slot(mt, node->slot, i), -+ min + i, min + i, depth); -+ } -+ break; -+ case maple_leaf_64: -+ case maple_range_64: -+ mt_dump_range64(mt, entry, min, max, depth); -+ break; -+ case maple_arange_64: -+ mt_dump_arange64(mt, entry, min, max, depth); -+ break; -+ -+ default: -+ pr_cont(" UNKNOWN TYPE\n"); -+ } -+} -+ -+void mt_dump(const struct maple_tree *mt) -+{ -+ void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt)); -+ -+ pr_info("maple_tree(%p) flags %X, height %u root %p\n", -+ mt, mt->ma_flags, mt_height(mt), entry); -+ if (!xa_is_node(entry)) -+ mt_dump_entry(entry, 0, 0, 0); -+ else if (entry) -+ mt_dump_node(mt, entry, 0, mt_max[mte_node_type(entry)], 0); -+} -+ -+/* -+ * Calculate the maximum gap in a node and check if that's what is reported in -+ * the parent (unless root). -+ */ -+static void mas_validate_gaps(struct ma_state *mas) -+{ -+ struct maple_enode *mte = mas->node; -+ struct maple_node *p_mn; -+ unsigned long gap = 0, max_gap = 0; -+ unsigned long p_end, p_start = mas->min; -+ unsigned char p_slot; -+ unsigned long *gaps = NULL; -+ unsigned long *pivots = ma_pivots(mte_to_node(mte), mte_node_type(mte)); -+ int i; -+ -+ if (ma_is_dense(mte_node_type(mte))) { -+ for (i = 0; i < mt_slot_count(mte); i++) { -+ if (mas_get_slot(mas, i)) { -+ if (gap > max_gap) -+ max_gap = gap; -+ gap = 0; -+ continue; -+ } -+ gap++; -+ } -+ goto counted; -+ } -+ -+ gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte)); -+ for (i = 0; i < mt_slot_count(mte); i++) { -+ p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte)); -+ -+ if (!gaps) { -+ if (mas_get_slot(mas, i)) { -+ gap = 0; -+ goto not_empty; -+ } -+ -+ gap += p_end - p_start + 1; -+ } else { -+ void *entry = mas_get_slot(mas, i); -+ -+ gap = gaps[i]; -+ if (!entry) { -+ if (gap != p_end - p_start + 1) { -+ pr_err("%p[%u] -> %p %lu != %lu - %lu + 1\n", -+ mas_mn(mas), i, -+ mas_get_slot(mas, i), gap, -+ p_end, p_start); -+ mt_dump(mas->tree); -+ -+ MT_BUG_ON(mas->tree, -+ gap != p_end - p_start + 1); -+ } -+ } else { -+ if (gap > p_end - p_start + 1) { -+ pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", -+ mas_mn(mas), i, gap, p_end, p_start, -+ p_end - p_start + 1); -+ MT_BUG_ON(mas->tree, -+ gap > p_end - p_start + 1); -+ } -+ } -+ } -+ -+ if (gap > max_gap) -+ max_gap = gap; -+not_empty: -+ p_start = p_end + 1; -+ if (p_end >= mas->max) -+ break; -+ } -+ -+counted: -+ if (mte_is_root(mte)) -+ return; -+ -+ p_slot = mte_parent_slot(mas->node); -+ p_mn = mte_parent(mte); -+ MT_BUG_ON(mas->tree, max_gap > mas->max); -+ if (ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap) { -+ pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap); -+ mt_dump(mas->tree); -+ } -+ -+ MT_BUG_ON(mas->tree, -+ ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap); -+} -+ -+static void mas_validate_parent_slot(struct ma_state *mas) -+{ -+ struct maple_node *parent; -+ struct maple_enode *node; -+ enum maple_type p_type = mas_parent_enum(mas, mas->node); -+ unsigned char p_slot = mte_parent_slot(mas->node); -+ void __rcu **slots; -+ int i; -+ -+ if (mte_is_root(mas->node)) -+ return; -+ -+ parent = mte_parent(mas->node); -+ slots = ma_slots(parent, p_type); -+ MT_BUG_ON(mas->tree, mas_mn(mas) == parent); -+ -+ /* Check prev/next parent slot for duplicate node entry */ -+ -+ for (i = 0; i < mt_slots[p_type]; i++) { -+ node = mas_slot(mas, slots, i); -+ if (i == p_slot) { -+ if (node != mas->node) -+ pr_err("parent %p[%u] does not have %p\n", -+ parent, i, mas_mn(mas)); -+ MT_BUG_ON(mas->tree, node != mas->node); -+ } else if (node == mas->node) { -+ pr_err("Invalid child %p at parent %p[%u] p_slot %u\n", -+ mas_mn(mas), parent, i, p_slot); -+ MT_BUG_ON(mas->tree, node == mas->node); -+ } -+ } -+} -+ -+static void mas_validate_child_slot(struct ma_state *mas) -+{ -+ enum maple_type type = mte_node_type(mas->node); -+ void __rcu **slots = ma_slots(mte_to_node(mas->node), type); -+ unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type); -+ struct maple_enode *child; -+ unsigned char i; -+ -+ if (mte_is_leaf(mas->node)) -+ return; -+ -+ for (i = 0; i < mt_slots[type]; i++) { -+ child = mas_slot(mas, slots, i); -+ if (!pivots[i] || pivots[i] == mas->max) -+ break; -+ -+ if (!child) -+ break; -+ -+ if (mte_parent_slot(child) != i) { -+ pr_err("Slot error at %p[%u]: child %p has pslot %u\n", -+ mas_mn(mas), i, mte_to_node(child), -+ mte_parent_slot(child)); -+ MT_BUG_ON(mas->tree, 1); -+ } -+ -+ if (mte_parent(child) != mte_to_node(mas->node)) { -+ pr_err("child %p has parent %p not %p\n", -+ mte_to_node(child), mte_parent(child), -+ mte_to_node(mas->node)); -+ MT_BUG_ON(mas->tree, 1); -+ } -+ } -+} -+ -+/* -+ * Validate all pivots are within mas->min and mas->max. -+ */ -+static void mas_validate_limits(struct ma_state *mas) -+{ -+ int i; -+ unsigned long prev_piv = 0; -+ enum maple_type type = mte_node_type(mas->node); -+ void __rcu **slots = ma_slots(mte_to_node(mas->node), type); -+ unsigned long *pivots = ma_pivots(mas_mn(mas), type); -+ -+ /* all limits are fine here. */ -+ if (mte_is_root(mas->node)) -+ return; -+ -+ for (i = 0; i < mt_slots[type]; i++) { -+ unsigned long piv; -+ -+ piv = mas_safe_pivot(mas, pivots, i, type); -+ -+ if (!piv && (i != 0)) -+ break; -+ -+ if (!mte_is_leaf(mas->node)) { -+ void *entry = mas_slot(mas, slots, i); -+ -+ if (!entry) -+ pr_err("%p[%u] cannot be null\n", -+ mas_mn(mas), i); -+ -+ MT_BUG_ON(mas->tree, !entry); -+ } -+ -+ if (prev_piv > piv) { -+ pr_err("%p[%u] piv %lu < prev_piv %lu\n", -+ mas_mn(mas), i, piv, prev_piv); -+ MT_BUG_ON(mas->tree, piv < prev_piv); -+ } -+ -+ if (piv < mas->min) { -+ pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i, -+ piv, mas->min); -+ MT_BUG_ON(mas->tree, piv < mas->min); -+ } -+ if (piv > mas->max) { -+ pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i, -+ piv, mas->max); -+ MT_BUG_ON(mas->tree, piv > mas->max); -+ } -+ prev_piv = piv; -+ if (piv == mas->max) -+ break; -+ } -+ for (i += 1; i < mt_slots[type]; i++) { -+ void *entry = mas_slot(mas, slots, i); -+ -+ if (entry && (i != mt_slots[type] - 1)) { -+ pr_err("%p[%u] should not have entry %p\n", mas_mn(mas), -+ i, entry); -+ MT_BUG_ON(mas->tree, entry != NULL); -+ } -+ -+ if (i < mt_pivots[type]) { -+ unsigned long piv = pivots[i]; -+ -+ if (!piv) -+ continue; -+ -+ pr_err("%p[%u] should not have piv %lu\n", -+ mas_mn(mas), i, piv); -+ MT_BUG_ON(mas->tree, i < mt_pivots[type] - 1); -+ } -+ } -+} -+ -+static void mt_validate_nulls(struct maple_tree *mt) -+{ -+ void *entry, *last = (void *)1; -+ unsigned char offset = 0; -+ void __rcu **slots; -+ MA_STATE(mas, mt, 0, 0); -+ -+ mas_start(&mas); -+ if (mas_is_none(&mas) || (mas.node == MAS_ROOT)) -+ return; -+ -+ while (!mte_is_leaf(mas.node)) -+ mas_descend(&mas); -+ -+ slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); -+ do { -+ entry = mas_slot(&mas, slots, offset); -+ if (!last && !entry) { -+ pr_err("Sequential nulls end at %p[%u]\n", -+ mas_mn(&mas), offset); -+ } -+ MT_BUG_ON(mt, !last && !entry); -+ last = entry; -+ if (offset == mas_data_end(&mas)) { -+ mas_next_node(&mas, mas_mn(&mas), ULONG_MAX); -+ if (mas_is_none(&mas)) -+ return; -+ offset = 0; -+ slots = ma_slots(mte_to_node(mas.node), -+ mte_node_type(mas.node)); -+ } else { -+ offset++; -+ } -+ -+ } while (!mas_is_none(&mas)); -+} -+ -+/* -+ * validate a maple tree by checking: -+ * 1. The limits (pivots are within mas->min to mas->max) -+ * 2. The gap is correctly set in the parents -+ */ -+void mt_validate(struct maple_tree *mt) -+{ -+ unsigned char end; -+ -+ MA_STATE(mas, mt, 0, 0); -+ rcu_read_lock(); -+ mas_start(&mas); -+ if (!mas_searchable(&mas)) -+ goto done; -+ -+ mas_first_entry(&mas, mas_mn(&mas), ULONG_MAX, mte_node_type(mas.node)); -+ while (!mas_is_none(&mas)) { -+ MT_BUG_ON(mas.tree, mte_dead_node(mas.node)); -+ if (!mte_is_root(mas.node)) { -+ end = mas_data_end(&mas); -+ if ((end < mt_min_slot_count(mas.node)) && -+ (mas.max != ULONG_MAX)) { -+ pr_err("Invalid size %u of %p\n", end, -+ mas_mn(&mas)); -+ MT_BUG_ON(mas.tree, 1); -+ } -+ -+ } -+ mas_validate_parent_slot(&mas); -+ mas_validate_child_slot(&mas); -+ mas_validate_limits(&mas); -+ if (mt_is_alloc(mt)) -+ mas_validate_gaps(&mas); -+ mas_dfs_postorder(&mas, ULONG_MAX); -+ } -+ mt_validate_nulls(mt); -+done: -+ rcu_read_unlock(); -+ -+} -+ -+#endif /* CONFIG_DEBUG_MAPLE_TREE */ -diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c -new file mode 100644 -index 000000000000..4f69e009a015 ---- /dev/null -+++ b/lib/test_maple_tree.c -@@ -0,0 +1,38307 @@ -+// SPDX-License-Identifier: GPL-2.0+ -+/* -+ * test_maple_tree.c: Test the maple tree API -+ * Copyright (c) 2018 Liam R. Howlett -+ * Author: Liam R. Howlett -+ */ -+ -+#include -+#include -+#include -+#include -+ -+#define MTREE_ALLOC_MAX 0x2000000000000Ul -+#define CONFIG_DEBUG_MAPLE_TREE -+#define CONFIG_MAPLE_SEARCH -+/* #define BENCH_SLOT_STORE */ -+/* #define BENCH_NODE_STORE */ -+/* #define BENCH_AWALK */ -+/* #define BENCH_WALK */ -+/* #define BENCH_MT_FOR_EACH */ -+/* #define BENCH_FORK */ -+static -+int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp) -+{ -+ return mtree_insert(mt, index, xa_mk_value(index & LONG_MAX), gfp); -+} -+ -+static void mtree_erase_index(struct maple_tree *mt, unsigned long index) -+{ -+ MT_BUG_ON(mt, mtree_erase(mt, index) != xa_mk_value(index & LONG_MAX)); -+ MT_BUG_ON(mt, mtree_load(mt, index) != NULL); -+} -+ -+static int mtree_test_insert(struct maple_tree *mt, unsigned long index, -+ void *ptr) -+{ -+ return mtree_insert(mt, index, ptr, GFP_KERNEL); -+} -+ -+static int mtree_test_store_range(struct maple_tree *mt, unsigned long start, -+ unsigned long end, void *ptr) -+{ -+ return mtree_store_range(mt, start, end, ptr, GFP_KERNEL); -+} -+ -+static int mtree_test_store(struct maple_tree *mt, unsigned long start, -+ void *ptr) -+{ -+ return mtree_test_store_range(mt, start, start, ptr); -+} -+ -+static int mtree_test_insert_range(struct maple_tree *mt, unsigned long start, -+ unsigned long end, void *ptr) -+{ -+ return mtree_insert_range(mt, start, end, ptr, GFP_KERNEL); -+} -+ -+static void *mtree_test_load(struct maple_tree *mt, unsigned long index) -+{ -+ return mtree_load(mt, index); -+} -+ -+static void *mtree_test_erase(struct maple_tree *mt, unsigned long index) -+{ -+ return mtree_erase(mt, index); -+} -+ -+static noinline void check_mtree_alloc_range(struct maple_tree *mt, -+ unsigned long start, unsigned long end, unsigned long size, -+ unsigned long expected, int eret, void *ptr) -+{ -+ -+ unsigned long result = expected + 1; -+ int ret; -+ -+ ret = mtree_alloc_range(mt, &result, ptr, size, start, end, -+ GFP_KERNEL); -+ MT_BUG_ON(mt, ret != eret); -+ if (ret) -+ return; -+ -+ MT_BUG_ON(mt, result != expected); -+} -+ -+static noinline void check_mtree_alloc_rrange(struct maple_tree *mt, -+ unsigned long start, unsigned long end, unsigned long size, -+ unsigned long expected, int eret, void *ptr) -+{ -+ -+ unsigned long result = expected + 1; -+ int ret; -+ -+ ret = mtree_alloc_rrange(mt, &result, ptr, size, start, end - 1, -+ GFP_KERNEL); -+ MT_BUG_ON(mt, ret != eret); -+ if (ret) -+ return; -+ -+ MT_BUG_ON(mt, result != expected); -+} -+ -+static noinline void check_load(struct maple_tree *mt, unsigned long index, -+ void *ptr) -+{ -+ void *ret = mtree_test_load(mt, index); -+ -+ if (ret != ptr) -+ pr_err("Load %lu returned %p expect %p\n", index, ret, ptr); -+ MT_BUG_ON(mt, ret != ptr); -+} -+ -+static noinline void check_store_range(struct maple_tree *mt, -+ unsigned long start, unsigned long end, void *ptr, int expected) -+{ -+ int ret = -EINVAL; -+ unsigned long i; -+ -+ ret = mtree_test_store_range(mt, start, end, ptr); -+ MT_BUG_ON(mt, ret != expected); -+ -+ if (ret) -+ return; -+ -+ for (i = start; i <= end; i++) -+ check_load(mt, i, ptr); -+} -+ -+static noinline void check_insert_range(struct maple_tree *mt, -+ unsigned long start, unsigned long end, void *ptr, int expected) -+{ -+ int ret = -EINVAL; -+ unsigned long i; -+ -+ ret = mtree_test_insert_range(mt, start, end, ptr); -+ MT_BUG_ON(mt, ret != expected); -+ -+ if (ret) -+ return; -+ -+ for (i = start; i <= end; i++) -+ check_load(mt, i, ptr); -+} -+ -+static noinline void check_insert(struct maple_tree *mt, unsigned long index, -+ void *ptr) -+{ -+ int ret = -EINVAL; -+ -+ ret = mtree_test_insert(mt, index, ptr); -+ MT_BUG_ON(mt, ret != 0); -+} -+ -+static noinline void check_erase(struct maple_tree *mt, unsigned long index, -+ void *ptr) -+{ -+ MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr); -+} -+ -+static noinline void check_dup_insert(struct maple_tree *mt, -+ unsigned long index, void *ptr) -+{ -+ int ret = -EINVAL; -+ -+ ret = mtree_test_insert(mt, index, ptr); -+ MT_BUG_ON(mt, ret != -EEXIST); -+} -+ -+ -+static noinline -+void check_index_load(struct maple_tree *mt, unsigned long index) -+{ -+ return check_load(mt, index, xa_mk_value(index & LONG_MAX)); -+} -+ -+static noinline void check_nomem(struct maple_tree *mt) -+{ -+ MA_STATE(ms, mt, 1, 1); -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ /* Ensure no bypassing of allocation failures */ -+ mt_set_non_kernel(0); -+ -+ /* Storing something at 1 requires memory allocation */ -+ MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM); -+ /* Storing something at 0 does not */ -+ MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0); -+ -+ /* -+ * Simulate two threads racing; the first one fails to allocate -+ * memory to insert an entry at 1, then the second one succeeds -+ * in allocating memory to insert an entry at 2. The first one -+ * then needs to free the node it allocated. LeakSanitizer will -+ * notice this, as will the 'nr_allocated' debugging aid in the -+ * userspace test suite. -+ */ -+ mtree_lock(mt); -+ mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */ -+ MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM)); -+ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ -+ MT_BUG_ON(mt, ms.node != MAS_START); -+ mtree_unlock(mt); -+ MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0); -+ mtree_lock(mt); -+ mas_store(&ms, &ms); /* insert 1 -> &ms */ -+ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+} -+ -+static inline int not_empty(struct maple_node *node) -+{ -+ int i; -+ -+ if (node->parent) -+ return 1; -+ -+ for (i = 0; i < ARRAY_SIZE(node->slot); i++) -+ if (node->slot[i]) -+ return 1; -+ -+ return 0; -+} -+ -+static noinline void check_new_node(struct maple_tree *mt) -+{ -+ -+ struct maple_node *mn, *mn2, *mn3; -+ struct maple_alloc *smn; -+ struct maple_node *nodes[100]; -+ int i, j, total; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ /* Try allocating 3 nodes */ -+ mtree_lock(mt); -+ /* request 3 nodes to be allocated. */ -+ mas_node_count(&mas, 3); -+ /* Allocation request of 3. */ -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); -+ /* Allocate failed. */ -+ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ -+ MT_BUG_ON(mt, mas_allocated(&mas) != 3); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, mas.alloc == NULL); -+ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); -+ mas_push_node(&mas, mn); -+ mas_nomem(&mas, GFP_KERNEL); /* free */ -+ mtree_unlock(mt); -+ -+ -+ /* Try allocating 1 node, then 2 more */ -+ mtree_lock(mt); -+ /* Set allocation request to 1. */ -+ mas_set_alloc_req(&mas, 1); -+ /* Check Allocation request of 1. */ -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); -+ mas_set_err(&mas, -ENOMEM); -+ /* Validate allocation request. */ -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ /* Eat the requested node. */ -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, mn->slot[0] != NULL); -+ MT_BUG_ON(mt, mn->slot[1] != NULL); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ -+ ma_free_rcu(mn); -+ mas.node = MAS_START; -+ mas_nomem(&mas, GFP_KERNEL); -+ /* Allocate 3 nodes, will fail. */ -+ mas_node_count(&mas, 3); -+ /* Drop the lock and allocate 3 nodes. */ -+ mas_nomem(&mas, GFP_KERNEL); -+ /* Ensure 3 are allocated. */ -+ MT_BUG_ON(mt, mas_allocated(&mas) != 3); -+ /* Allocation request of 0. */ -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); -+ -+ MT_BUG_ON(mt, mas.alloc == NULL); -+ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); -+ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); -+ /* Ensure we counted 3. */ -+ MT_BUG_ON(mt, mas_allocated(&mas) != 3); -+ /* Free. */ -+ mas_nomem(&mas, GFP_KERNEL); -+ -+ /* Set allocation request to 1. */ -+ mas_set_alloc_req(&mas, 1); -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); -+ mas_set_err(&mas, -ENOMEM); -+ /* Validate allocation request. */ -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 1); -+ /* Check the node is only one node. */ -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, mn->slot[0] != NULL); -+ MT_BUG_ON(mt, mn->slot[1] != NULL); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ mas_push_node(&mas, mn); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 1); -+ MT_BUG_ON(mt, mas.alloc->node_count); -+ -+ mas_set_alloc_req(&mas, 2); /* request 2 more. */ -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); -+ mas_set_err(&mas, -ENOMEM); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 3); -+ MT_BUG_ON(mt, mas.alloc == NULL); -+ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); -+ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); -+ for (i = 2; i >= 0; i--) { -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, mas_allocated(&mas) != i); -+ MT_BUG_ON(mt, !mn); -+ MT_BUG_ON(mt, not_empty(mn)); -+ ma_free_rcu(mn); -+ } -+ -+ total = 64; -+ mas_set_alloc_req(&mas, total); /* request 2 more. */ -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != total); -+ mas_set_err(&mas, -ENOMEM); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ for (i = total; i > 0; i--) { -+ unsigned int e = 0; /* expected node_count */ -+ -+ if (i >= 35) -+ e = i - 35; -+ else if (i >= 5) -+ e = i - 5; -+ else if (i >= 2) -+ e = i - 2; -+ MT_BUG_ON(mt, mas.alloc->node_count != e); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); -+ MT_BUG_ON(mt, !mn); -+ ma_free_rcu(mn); -+ } -+ -+ total = 100; -+ for (i = 1; i < total; i++) { -+ mas_set_alloc_req(&mas, i); -+ mas_set_err(&mas, -ENOMEM); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ for (j = i; j > 0; j--) { -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); -+ MT_BUG_ON(mt, !mn); -+ MT_BUG_ON(mt, not_empty(mn)); -+ mas_push_node(&mas, mn); -+ MT_BUG_ON(mt, mas_allocated(&mas) != j); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); -+ ma_free_rcu(mn); -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ -+ mas_set_alloc_req(&mas, i); -+ mas_set_err(&mas, -ENOMEM); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ for (j = 0; j <= i/2; j++) { -+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); -+ nodes[j] = mas_pop_node(&mas); -+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); -+ } -+ -+ while (j) { -+ j--; -+ mas_push_node(&mas, nodes[j]); -+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas) != i); -+ for (j = 0; j <= i/2; j++) { -+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ ma_free_rcu(mn); -+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); -+ } -+ MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); -+ -+ } -+ -+ /* Set allocation request. */ -+ total = 500; -+ mas_node_count(&mas, total); -+ /* Drop the lock and allocate the nodes. */ -+ mas_nomem(&mas, GFP_KERNEL); -+ MT_BUG_ON(mt, !mas.alloc); -+ i = 1; -+ smn = mas.alloc; -+ while (i < total) { -+ for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) { -+ i++; -+ MT_BUG_ON(mt, !smn->slot[j]); -+ if (i == total) -+ break; -+ } -+ smn = smn->slot[0]; /* next. */ -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas) != total); -+ mas_nomem(&mas, GFP_KERNEL); /* Free. */ -+ -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ for (i = 1; i < 128; i++) { -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ -+ for (j = i; j > 0; j--) { /*Free the requests */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, not_empty(mn)); -+ ma_free_rcu(mn); -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ } -+ -+ for (i = 1; i < MAPLE_NODE_MASK + 1; i++) { -+ MA_STATE(mas2, mt, 0, 0); -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ -+ for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, not_empty(mn)); -+ mas_push_node(&mas2, mn); -+ MT_BUG_ON(mt, mas_allocated(&mas2) != j); -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ MT_BUG_ON(mt, mas_allocated(&mas2) != i); -+ -+ for (j = i; j > 0; j--) { /*Free the requests */ -+ MT_BUG_ON(mt, mas_allocated(&mas2) != j); -+ mn = mas_pop_node(&mas2); /* get the next node. */ -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, not_empty(mn)); -+ ma_free_rcu(mn); -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas2) != 0); -+ } -+ -+ -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ -+ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); -+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); -+ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ MT_BUG_ON(mt, mn == NULL); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); -+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2); -+ -+ mas_push_node(&mas, mn); -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); -+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); -+ -+ /* Check the limit of pop/push/pop */ -+ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ -+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); -+ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); -+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); -+ MT_BUG_ON(mt, mas_alloc_req(&mas)); -+ MT_BUG_ON(mt, mas.alloc->node_count); -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); -+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); -+ mas_push_node(&mas, mn); -+ MT_BUG_ON(mt, mas.alloc->node_count); -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ ma_free_rcu(mn); -+ for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) { -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, not_empty(mn)); -+ ma_free_rcu(mn); -+ } -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ -+ -+ for (i = 3; i < MAPLE_NODE_MASK * 3; i++) { -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ mas_push_node(&mas, mn); /* put it back */ -+ mas_destroy(&mas); -+ -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ mn2 = mas_pop_node(&mas); /* get the next node. */ -+ mas_push_node(&mas, mn); /* put them back */ -+ mas_push_node(&mas, mn2); -+ mas_destroy(&mas); -+ -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ mn2 = mas_pop_node(&mas); /* get the next node. */ -+ mn3 = mas_pop_node(&mas); /* get the next node. */ -+ mas_push_node(&mas, mn); /* put them back */ -+ mas_push_node(&mas, mn2); -+ mas_push_node(&mas, mn3); -+ mas_destroy(&mas); -+ -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ ma_free_rcu(mn); -+ mas_destroy(&mas); -+ -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, i); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ ma_free_rcu(mn); -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ ma_free_rcu(mn); -+ mn = mas_pop_node(&mas); /* get the next node. */ -+ ma_free_rcu(mn); -+ mas_destroy(&mas); -+ } -+ -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, 5); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ MT_BUG_ON(mt, mas_allocated(&mas) != 5); -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, 10); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mas.node = MAS_START; -+ MT_BUG_ON(mt, mas_allocated(&mas) != 10); -+ mas_destroy(&mas); -+ -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); -+ mas.node = MA_ERROR(-ENOMEM); -+ mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ -+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ -+ mas.node = MAS_START; -+ MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); -+ mas_destroy(&mas); -+ -+ mtree_unlock(mt); -+} -+ -+static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, -+ bool verbose) -+{ -+ unsigned long i = max, j; -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ -+ mt_zero_nr_tallocated(); -+ while (i) { -+ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); -+ for (j = i; j <= max; j++) -+ check_index_load(mt, j); -+ -+ check_load(mt, i - 1, NULL); -+ mt_set_in_rcu(mt); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mt_clear_in_rcu(mt); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ i--; -+ } -+ check_load(mt, max + 1, NULL); -+ -+ if (verbose) { -+ rcu_barrier(); -+ mt_dump(mt); -+ pr_info(" %s test of 0-%lu %luK in %d active (%d total)\n", -+ __func__, max, mt_get_alloc_size()/1024, mt_nr_allocated(), -+ mt_nr_tallocated()); -+ } -+} -+ -+static noinline void check_seq(struct maple_tree *mt, unsigned long max, -+ bool verbose) -+{ -+ unsigned long i, j; -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ -+ mt_zero_nr_tallocated(); -+ for (i = 0; i <= max; i++) { -+ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); -+ for (j = 0; j <= i; j++) -+ check_index_load(mt, j); -+ -+ if (i) -+ MT_BUG_ON(mt, !mt_height(mt)); -+ check_load(mt, i + 1, NULL); -+ } -+ if (verbose) { -+ rcu_barrier(); -+ mt_dump(mt); -+ pr_info(" seq test of 0-%lu %luK in %d active (%d total)\n", -+ max, mt_get_alloc_size()/1024, mt_nr_allocated(), -+ mt_nr_tallocated()); -+ } -+} -+ -+static noinline void check_lb_not_empty(struct maple_tree *mt) -+{ -+ unsigned long i, j; -+ unsigned long huge = 4000UL * 1000 * 1000; -+ -+ -+ i = huge; -+ while (i > 4096) { -+ check_insert(mt, i, (void *) i); -+ for (j = huge; j >= i; j /= 2) { -+ check_load(mt, j-1, NULL); -+ check_load(mt, j, (void *) j); -+ check_load(mt, j+1, NULL); -+ } -+ i /= 2; -+ } -+ mtree_destroy(mt); -+} -+ -+static noinline void check_lower_bound_split(struct maple_tree *mt) -+{ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ check_lb_not_empty(mt); -+} -+ -+static noinline void check_upper_bound_split(struct maple_tree *mt) -+{ -+ unsigned long i, j; -+ unsigned long huge = 4000UL * 1000 * 1000; -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ -+ i = 4096; -+ while (i < huge) { -+ check_insert(mt, i, (void *) i); -+ for (j = i; j >= huge; j *= 2) { -+ check_load(mt, j-1, NULL); -+ check_load(mt, j, (void *) j); -+ check_load(mt, j+1, NULL); -+ } -+ i *= 2; -+ } -+ mtree_destroy(mt); -+} -+ -+static noinline void check_mid_split(struct maple_tree *mt) -+{ -+ unsigned long huge = 8000UL * 1000 * 1000; -+ -+ check_insert(mt, huge, (void *) huge); -+ check_insert(mt, 0, xa_mk_value(0)); -+ check_lb_not_empty(mt); -+} -+ -+static noinline void check_rev_find(struct maple_tree *mt) -+{ -+ int i, nr_entries = 200; -+ void *val; -+ MA_STATE(mas, mt, 0, 0); -+ -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ mas_set(&mas, 1000); -+ val = mas_find_rev(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(100)); -+ val = mas_find_rev(&mas, 1000); -+ MT_BUG_ON(mt, val != NULL); -+ -+ mas_set(&mas, 999); -+ val = mas_find_rev(&mas, 997); -+ MT_BUG_ON(mt, val != NULL); -+ -+ mas_set(&mas, 1000); -+ val = mas_find_rev(&mas, 900); -+ MT_BUG_ON(mt, val != xa_mk_value(100)); -+ val = mas_find_rev(&mas, 900); -+ MT_BUG_ON(mt, val != xa_mk_value(99)); -+ -+ mas_set(&mas, 20); -+ val = mas_find_rev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(2)); -+ val = mas_find_rev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(1)); -+ val = mas_find_rev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(0)); -+ val = mas_find_rev(&mas, 0); -+ MT_BUG_ON(mt, val != NULL); -+} -+ -+static noinline void check_find(struct maple_tree *mt) -+{ -+ unsigned long val = 0; -+ unsigned long count = 20; -+ unsigned long max; -+ unsigned long last = 0, index = 0; -+ void *entry, *entry2; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ /* Insert 0. */ -+ MT_BUG_ON(mt, mtree_insert_index(mt, val++, GFP_KERNEL)); -+ -+ for (int i = 0; i <= count; i++) { -+ if (val != 64) -+ MT_BUG_ON(mt, mtree_insert_index(mt, val, GFP_KERNEL)); -+ else -+ MT_BUG_ON(mt, mtree_insert(mt, val, -+ XA_ZERO_ENTRY, GFP_KERNEL)); -+ -+ val <<= 2; -+ } -+ -+ val = 0; -+ mas_set(&mas, val); -+ mas_lock(&mas); -+ while ((entry = mas_find(&mas, 268435456)) != NULL) { -+ if (val != 64) -+ MT_BUG_ON(mt, xa_mk_value(val) != entry); -+ else -+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); -+ -+ val <<= 2; -+ /* For zero check. */ -+ if (!val) -+ val = 1; -+ } -+ mas_unlock(&mas); -+ -+ val = 0; -+ mas_set(&mas, val); -+ mas_lock(&mas); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ if (val != 64) -+ MT_BUG_ON(mt, xa_mk_value(val) != entry); -+ else -+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); -+ val <<= 2; -+ /* For zero check. */ -+ if (!val) -+ val = 1; -+ } -+ mas_unlock(&mas); -+ -+ /* Test mas_pause */ -+ val = 0; -+ mas_set(&mas, val); -+ mas_lock(&mas); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ if (val != 64) -+ MT_BUG_ON(mt, xa_mk_value(val) != entry); -+ else -+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); -+ val <<= 2; -+ /* For zero check. */ -+ if (!val) -+ val = 1; -+ -+ mas_pause(&mas); -+ mas_unlock(&mas); -+ mas_lock(&mas); -+ } -+ mas_unlock(&mas); -+ -+ val = 0; -+ max = 300; /* A value big enough to include XA_ZERO_ENTRY at 64. */ -+ mt_for_each(mt, entry, index, max) { -+ MT_BUG_ON(mt, xa_mk_value(val) != entry); -+ val <<= 2; -+ if (val == 64) /* Skip zero entry. */ -+ val <<= 2; -+ /* For zero check. */ -+ if (!val) -+ val = 1; -+ } -+ -+ val = 0; -+ max = 0; -+ index = 0; -+ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); -+ mt_for_each(mt, entry, index, ULONG_MAX) { -+ if (val == 4398046511104) -+ MT_BUG_ON(mt, entry != -+ xa_mk_value(ULONG_MAX & LONG_MAX)); -+ else -+ MT_BUG_ON(mt, xa_mk_value(val) != entry); -+ val <<= 2; -+ if (val == 64) /* Skip zero entry. */ -+ val <<= 2; -+ /* For zero check. */ -+ if (!val) -+ val = 1; -+ max++; -+ MT_BUG_ON(mt, max > 25); -+ } -+ mtree_erase_index(mt, ULONG_MAX); -+ -+ mas_reset(&mas); -+ index = 17; -+ entry = mt_find(mt, &index, 512); -+ MT_BUG_ON(mt, xa_mk_value(256) != entry); -+ -+ mas_reset(&mas); -+ index = 17; -+ entry = mt_find(mt, &index, 20); -+ MT_BUG_ON(mt, entry != NULL); -+ -+ -+ /* Range check.. */ -+ /* Insert ULONG_MAX */ -+ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); -+ -+ val = 0; -+ mas_set(&mas, 0); -+ mas_lock(&mas); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ if (val == 64) -+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); -+ else if (val == 4398046511104) -+ MT_BUG_ON(mt, entry != xa_mk_value(ULONG_MAX & LONG_MAX)); -+ else -+ MT_BUG_ON(mt, xa_mk_value(val) != entry); -+ val <<= 2; -+ -+ /* For zero check. */ -+ if (!val) -+ val = 1; -+ mas_pause(&mas); -+ mas_unlock(&mas); -+ mas_lock(&mas); -+ } -+ mas_unlock(&mas); -+ -+ mas_set(&mas, 1048576); -+ mas_lock(&mas); -+ entry = mas_find(&mas, 1048576); -+ mas_unlock(&mas); -+ MT_BUG_ON(mas.tree, entry == NULL); -+ -+ /* -+ * Find last value. -+ * 1. get the expected value, leveraging the existence of an end entry -+ * 2. delete end entry -+ * 3. find the last value but searching for ULONG_MAX and then using -+ * prev -+ */ -+ /* First, get the expected result. */ -+ mas_lock(&mas); -+ mas_reset(&mas); -+ mas.index = ULONG_MAX; /* start at max.. */ -+ entry = mas_find(&mas, ULONG_MAX); -+ entry = mas_prev(&mas, 0); -+ index = mas.index; -+ last = mas.last; -+ -+ /* Erase the last entry. */ -+ mas_reset(&mas); -+ mas.index = ULONG_MAX; -+ mas.last = ULONG_MAX; -+ mas_erase(&mas); -+ -+ /* Get the previous value from MAS_START */ -+ mas_reset(&mas); -+ entry2 = mas_prev(&mas, 0); -+ -+ /* Check results. */ -+ MT_BUG_ON(mt, entry != entry2); -+ MT_BUG_ON(mt, index != mas.index); -+ MT_BUG_ON(mt, last != mas.last); -+ -+ -+ mas.node = MAS_NONE; -+ mas.index = ULONG_MAX; -+ mas.last = ULONG_MAX; -+ entry2 = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, entry != entry2); -+ -+ mas_set(&mas, 0); -+ MT_BUG_ON(mt, mas_prev(&mas, 0) != NULL); -+ -+ mas_unlock(&mas); -+ mtree_destroy(mt); -+} -+ -+static noinline void check_find_2(struct maple_tree *mt) -+{ -+ unsigned long i, j; -+ void *entry; -+ -+ MA_STATE(mas, mt, 0, 0); -+ rcu_read_lock(); -+ mas_for_each(&mas, entry, ULONG_MAX) -+ MT_BUG_ON(mt, true); -+ rcu_read_unlock(); -+ -+ for (i = 0; i < 256; i++) { -+ mtree_insert_index(mt, i, GFP_KERNEL); -+ j = 0; -+ mas_set(&mas, 0); -+ rcu_read_lock(); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ MT_BUG_ON(mt, entry != xa_mk_value(j)); -+ j++; -+ } -+ rcu_read_unlock(); -+ MT_BUG_ON(mt, j != i + 1); -+ } -+ -+ for (i = 0; i < 256; i++) { -+ mtree_erase_index(mt, i); -+ j = i + 1; -+ mas_set(&mas, 0); -+ rcu_read_lock(); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ if (xa_is_zero(entry)) -+ continue; -+ -+ MT_BUG_ON(mt, entry != xa_mk_value(j)); -+ j++; -+ } -+ rcu_read_unlock(); -+ MT_BUG_ON(mt, j != 256); -+ } -+ -+ /*MT_BUG_ON(mt, !mtree_empty(mt)); */ -+} -+ -+#define erase_ptr(i) entry[i%2] -+#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2]) -+#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2]) -+#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2]) -+ -+static noinline void check_erase_testset(struct maple_tree *mt) -+{ -+ unsigned long set[] = { 5015, 5014, 5017, 25, 1000, -+ 1001, 1002, 1003, 1005, 0, -+ 6003, 6002, 6008, 6012, 6015, -+ 7003, 7002, 7008, 7012, 7015, -+ 8003, 8002, 8008, 8012, 8015, -+ 9003, 9002, 9008, 9012, 9015, -+ 10003, 10002, 10008, 10012, 10015, -+ 11003, 11002, 11008, 11012, 11015, -+ 12003, 12002, 12008, 12012, 12015, -+ 13003, 13002, 13008, 13012, 13015, -+ 14003, 14002, 14008, 14012, 14015, -+ 15003, 15002, 15008, 15012, 15015, -+ }; -+ -+ -+ void *ptr = &set; -+ void *entry[2] = { ptr, mt }; -+ void *root_node; -+ -+ -+ rcu_register_thread(); -+ mt_set_in_rcu(mt); -+ for (int i = 0; i < 4; i++) -+ erase_check_insert(mt, i); -+ for (int i = 0; i < 4; i++) -+ erase_check_load(mt, i); -+ -+ mt_set_non_kernel(2); -+ erase_check_erase(mt, 1); -+ erase_check_load(mt, 0); -+ check_load(mt, set[1], NULL); -+ for (int i = 2; i < 4; i++) -+ erase_check_load(mt, i); -+ -+ -+ erase_check_erase(mt, 2); -+ erase_check_load(mt, 0); -+ check_load(mt, set[1], NULL); -+ check_load(mt, set[2], NULL); -+ -+ erase_check_insert(mt, 1); -+ erase_check_insert(mt, 2); -+ -+ for (int i = 0; i < 4; i++) -+ erase_check_load(mt, i); -+ -+ /* Check erase and load without an allocation. */ -+ erase_check_load(mt, 3); -+ erase_check_erase(mt, 1); -+ erase_check_load(mt, 0); -+ check_load(mt, set[1], NULL); -+ for (int i = 2; i < 4; i++) -+ erase_check_load(mt, i); -+ -+ /* -+ * Set the newly erased node. This will produce a different allocated -+ * node to avoid busy slots. -+ */ -+ root_node = mt->ma_root; -+ erase_check_insert(mt, 1); -+ -+ erase_check_load(mt, 0); -+ check_load(mt, 5016, NULL); -+ erase_check_load(mt, 1); -+ check_load(mt, 5013, NULL); -+ erase_check_load(mt, 2); -+ check_load(mt, 5018, NULL); -+ erase_check_load(mt, 3); -+ -+ erase_check_erase(mt, 2); /* erase 5017 to check append */ -+ erase_check_load(mt, 0); -+ check_load(mt, 5016, NULL); -+ erase_check_load(mt, 1); -+ check_load(mt, 5013, NULL); -+ check_load(mt, set[2], NULL); -+ check_load(mt, 5018, NULL); -+ -+ erase_check_load(mt, 3); -+ -+ root_node = mt->ma_root; -+ erase_check_insert(mt, 2); -+ -+ erase_check_load(mt, 0); -+ check_load(mt, 5016, NULL); -+ erase_check_load(mt, 1); -+ check_load(mt, 5013, NULL); -+ erase_check_load(mt, 2); -+ check_load(mt, 5018, NULL); -+ erase_check_load(mt, 3); -+ -+ mt_set_non_kernel(1); -+ erase_check_erase(mt, 2); /* erase 5017 to check append */ -+ erase_check_load(mt, 0); -+ check_load(mt, 5016, NULL); -+ check_load(mt, set[2], NULL); -+ erase_check_erase(mt, 0); /* erase 5015 to check append */ -+ check_load(mt, set[0], NULL); -+ check_load(mt, 5016, NULL); -+ erase_check_insert(mt, 4); /* 1000 < Should not split. */ -+ check_load(mt, set[0], NULL); -+ check_load(mt, 5016, NULL); -+ erase_check_load(mt, 1); -+ check_load(mt, 5013, NULL); -+ check_load(mt, set[2], NULL); -+ check_load(mt, 5018, NULL); -+ erase_check_load(mt, 4); -+ check_load(mt, 999, NULL); -+ check_load(mt, 1001, NULL); -+ erase_check_load(mt, 4); -+ if (mt_in_rcu(mt)) -+ MT_BUG_ON(mt, root_node == mt->ma_root); -+ else -+ MT_BUG_ON(mt, root_node != mt->ma_root); -+ -+ /* Should not have split. */ -+ MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root)); -+ -+ -+ /* Coalesce testing */ -+ erase_check_insert(mt, 0); -+ erase_check_insert(mt, 2); -+ -+ for (int i = 5; i < 25; i++) { -+ erase_check_insert(mt, i); -+ for (int j = i; j >= 0; j--) -+ erase_check_load(mt, j); -+ } -+ -+ erase_check_erase(mt, 14); /*6015 */ -+ for (int i = 0; i < 25; i++) { -+ if (i == 14) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ erase_check_erase(mt, 16); /*7002 */ -+ for (int i = 0; i < 25; i++) { -+ if (i == 16 || i == 14) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ -+ mt_set_non_kernel(1); -+ erase_check_erase(mt, 13); /*6012 */ -+ for (int i = 0; i < 25; i++) { -+ if (i == 16 || i == 14 || i == 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ erase_check_erase(mt, 15); /*7003 */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 16 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ mt_set_non_kernel(2); -+ erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 17 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ erase_check_erase(mt, 18); /*7012 */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 18 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ mt_set_non_kernel(2); -+ erase_check_erase(mt, 19); /*7015 */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 19 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ erase_check_erase(mt, 20); /*8003 */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 20 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ erase_check_erase(mt, 21); /*8002 */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 21 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ mt_set_non_kernel(2); -+ erase_check_erase(mt, 22); /*8008 */ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 22 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ for (int i = 23; i < 25; i++) -+ erase_check_erase(mt, i); -+ -+ for (int i = 0; i < 25; i++) { -+ if (i <= 25 && i >= 13) -+ check_load(mt, set[i], NULL); -+ else -+ erase_check_load(mt, i); -+ } -+ -+ /* Shrinking tree test. */ -+ -+ for (int i = 13; i < ARRAY_SIZE(set); i++) -+ erase_check_insert(mt, i); -+ -+ mt_set_non_kernel(99); -+ for (int i = 18; i < ARRAY_SIZE(set); i++) { -+ erase_check_erase(mt, i); -+ for (int j = 0; j < ARRAY_SIZE(set); j++) { -+ if (j < 18 || j > i) -+ erase_check_load(mt, j); -+ else -+ check_load(mt, set[j], NULL); -+ } -+ } -+ mt_set_non_kernel(35); -+ for (int i = 0; i < 18; i++) { -+ erase_check_erase(mt, i); -+ for (int j = 0; j < ARRAY_SIZE(set); j++) { -+ if (j < 18 && j > i) -+ erase_check_load(mt, j); -+ else -+ check_load(mt, set[j], NULL); -+ } -+ } -+ erase_check_insert(mt, 8); -+ erase_check_insert(mt, 9); -+ erase_check_erase(mt, 8); -+ rcu_unregister_thread(); -+} -+ -+#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \ -+ a[(i)], a[(i + 1)], ptr) -+#define STORE 1 -+#define SNULL 2 -+#define ERASE 3 -+#define ec_type_str(x) \ -+ (((x) == STORE) ? \ -+ "STORE" : \ -+ (((x) == SNULL) ? \ -+ "SNULL" : "ERASE") \ -+ ) -+#define check_erase2_debug 0 -+void *mas_next(struct ma_state *mas, unsigned long max); -+ -+/* Calculate the overwritten entries. */ -+int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end, -+ void *s_entry, unsigned long s_min, -+ void *e_entry, unsigned long e_max, -+ unsigned long *set, int i, bool null_entry) -+{ -+ int count = 0, span = 0; -+ unsigned long retry = 0; -+ void *entry; -+ struct ma_state tmp; -+ -+ -+ /* count slots */ -+ memcpy(&tmp, mas_start, sizeof(tmp)); -+ entry = mas_next(&tmp, mas_end->last); -+ while (entry) { -+ BUG_ON(retry > 50); /* stop infinite retry on testing. */ -+ if (xa_is_zero(s_entry)) { -+ retry++; -+ continue; -+ } -+ count++; -+ span++; -+ entry = mas_next(&tmp, mas_end->last); -+ } -+ -+ if (null_entry) { -+ /* Check splitting end. */ -+ if (e_entry && (e_max > mas_end->last)) -+ count--; -+ -+ /* check overwrite of entire start */ -+ if (s_entry && (s_min == mas_start->index)) -+ count++; -+ } else { /* !null_entry (store) */ -+ bool esplit = e_max > mas_end->last; -+ bool ssplit = s_min != mas_start->index; -+ -+ if (s_entry && e_entry) { -+ if (esplit && ssplit) -+ count--; -+ else if (ssplit) -+ count--; -+ else if (esplit) { -+ if (span) -+ count--; -+ } -+ } else if (s_entry && !e_entry) { -+ if (ssplit) -+ count--; -+ } else if (!s_entry && e_entry) { -+ if (esplit) -+ count--; -+ count--; -+ } else { -+ count--; -+ } -+ } -+ return count; -+} -+ -+/* -+ * mas_node_walk() - Walk a maple node to offset of the index. -+ * @mas: The maple state -+ * @type: The maple node type -+ * @*range_min: Pointer to store the minimum range of the offset -+ * @*range_max: Pointer to store the maximum range of the offset -+ * -+ * The offset will be stored in the maple state. -+ * -+ */ -+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, -+ enum maple_type type, unsigned long *range_min, -+ unsigned long *range_max) -+ -+{ -+ unsigned long *pivots; -+ unsigned char count; -+ unsigned long prev, max; -+ unsigned char offset; -+ unsigned long index; -+ -+ if (unlikely(ma_is_dense(type))) { -+ (*range_max) = (*range_min) = mas->index; -+ if (unlikely(ma_dead_node(node))) -+ return; -+ -+ mas->offset = mas->index = mas->min; -+ return; -+ } -+ -+ pivots = ma_pivots(node, type); -+ max = pivots[0]; -+ if (unlikely(ma_dead_node(node))) -+ return; -+ -+ offset = 0; -+ prev = mas->min; -+ index = mas->index; -+ if (unlikely(index <= max)) -+ goto offset_zero; -+ -+ count = mt_pivots[type]; -+ while (++offset < count) { -+ prev = max; -+ max = pivots[offset]; -+ if (unlikely(ma_dead_node(node))) -+ return; -+ -+ if (index <= max) -+ goto offset_found; -+ else if (unlikely(!max)) -+ goto mas_max; -+ } -+ -+ prev = max; -+mas_max: -+ max = mas->max; -+offset_found: -+ prev++; -+offset_zero: -+ mas->offset = offset; -+ if (ma_is_leaf(type)) { -+ *range_max = max; -+ *range_min = prev; -+ } else { -+ mas->max = max; -+ mas->min = prev; -+ } -+} -+ -+/* -+ * mas_descend_walk(): Locates a value and sets the mas->node and slot -+ * accordingly. range_min and range_max are set to the range which the entry is -+ * valid. -+ * @mas: The maple state -+ * @*range_min: A pointer to store the minimum of the range -+ * @*range_max: A pointer to store the maximum of the range -+ * -+ * Check mas->node is still valid on return of any value. -+ * -+ * Return: true if pointing to a valid node and offset. False otherwise. -+ */ -+static inline bool mas_descend_walk(struct ma_state *mas, -+ unsigned long *range_min, unsigned long *range_max) -+{ -+ struct maple_enode *next; -+ struct maple_node *node; -+ enum maple_type type; -+ -+ next = mas->node; -+ while (true) { -+ node = mte_to_node(next); -+ type = mte_node_type(next); -+ mas_node_walk(mas, node, type, range_min, range_max); -+ next = mas_slot(mas, ma_slots(node, type), mas->offset); -+ if (unlikely(ma_dead_node(node))) -+ return false; -+ -+ if (unlikely(ma_is_leaf(type))) -+ return true; -+ -+ /* Descend. */ -+ mas->node = next; -+ } -+ return false; -+} -+ -+/* -+ * mas_tree_walk() - Walk to @mas->index and set the range values. -+ * @mas: The maple state. -+ * @*range_min: The minimum range to be set. -+ * @*range_max: The maximum range to be set. -+ * -+ * Ranges are only valid if there is a valid entry at @mas->index. -+ * -+ * Return: True if a value exists, false otherwise. -+ */ -+static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, -+ unsigned long *range_max) -+{ -+ bool ret; -+ -+retry: -+ ret = false; -+ mas_start(mas); -+ if (mas_is_none(mas)) -+ goto not_found; -+ -+ if (mas_is_ptr(mas)) { -+ *range_min = *range_max = 0; -+ if (!mas->index) -+ return true; -+ -+ goto not_found; -+ } -+ -+ ret = mas_descend_walk(mas, range_min, range_max); -+ if (unlikely(mte_dead_node(mas->node))) { -+ mas->node = MAS_START; -+ goto retry; -+ } -+ -+ return ret; -+ -+not_found: -+ mas->offset = MAPLE_NODE_SLOTS; -+ return false; -+} -+ -+static inline void *mas_range_load(struct ma_state *mas, -+ unsigned long *range_min, unsigned long *range_max) -+ -+{ -+ void *entry = NULL; -+ unsigned long index = mas->index; -+ -+ if (mas_is_none(mas) || mas_is_paused(mas)) -+ mas->node = MAS_START; -+retry: -+ if (mas_tree_walk(mas, range_min, range_max)) -+ if (unlikely(mas->node == MAS_ROOT)) -+ return mas_root(mas); -+ -+ if (likely(mas->offset != MAPLE_NODE_SLOTS)) -+ entry = mas_get_slot(mas, mas->offset); -+ -+ if (mas_dead_node(mas, index)) -+ goto retry; -+ -+ return entry; -+} -+static noinline void check_erase2_testset(struct maple_tree *mt, -+ unsigned long *set, unsigned long size) -+{ -+ int entry_count = 0; -+ int check = 0; -+ void *foo; -+ unsigned long addr = 0; -+ void *s_entry = NULL, *e_entry = NULL; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ for (int i = 0; i < size; i += 3) { -+ unsigned long s_min, s_max; -+ unsigned long e_min, e_max; -+ void *value = NULL; -+ -+ MA_STATE(mas_start, mt, set[i+1], set[i+1]); -+ MA_STATE(mas_end, mt, set[i+2], set[i+2]); -+ mt_set_non_kernel(127); -+#if check_erase2_debug -+ pr_err("%s: %d %s %lu - %lu\n", __func__, i, -+ ec_type_str(set[i]), -+ set[i+1], set[i+2]); -+#endif -+ s_entry = mas_range_load(&mas_start, &s_min, &s_max); -+ e_entry = mas_range_load(&mas_end, &e_min, &e_max); -+ -+ switch (set[i]) { -+ case SNULL: -+ if ((s_min == set[i+1]) && (s_max == set[i+2])) { -+ if (s_entry) -+ entry_count--; -+ } else if ((s_min != set[i+1]) && (s_max != set[i+2])) { -+ entry_count++; -+ } else if ((mas_start.node != mas_end.node) || -+ (mas_start.offset != mas_end.offset)) { -+ entry_count -= -+ mas_ce2_over_count(&mas_start, &mas_end, -+ s_entry, s_min, -+ e_entry, e_max, set, i, -+ true); -+ } -+ -+ -+ erase_check_store_range(mt, set, i + 1, value); -+ break; -+ case STORE: -+ value = xa_mk_value(set[i + 1]); -+ if (mas_start.offset > mt_slot_count(mas_start.node)) { -+ entry_count++; /* appending an entry. */ -+ } else if ((s_min == e_min) && (s_max == e_max)) { -+ if (!entry_count) -+ entry_count++; -+ -+ else if (s_entry) { -+ if (e_max > mas_end.last) -+ entry_count++; -+ -+ if (s_min < mas_start.index) -+ entry_count++; -+ -+ } else { -+ entry_count++; -+ } -+ } else { -+ entry_count -= -+ mas_ce2_over_count(&mas_start, &mas_end, -+ s_entry, s_min, -+ e_entry, e_max, set, i, -+ false); -+ } -+ -+ erase_check_store_range(mt, set, i + 1, value); -+ break; -+ case ERASE: -+ if (!s_entry) -+ break; -+ check_erase(mt, set[i+1], xa_mk_value(set[i+1])); -+ entry_count--; -+ break; -+ } -+ mt_validate(mt); -+ if (entry_count) -+ MT_BUG_ON(mt, !mt_height(mt)); -+#if check_erase2_debug > 1 -+ mt_dump(mt); -+#endif -+#if check_erase2_debug -+ pr_err("Done\n"); -+#endif -+ -+ check = 0; -+ addr = 0; -+ mt_for_each(mt, foo, addr, ULONG_MAX) { -+ check++; -+#if check_erase2_debug > 2 -+ pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check); -+#endif -+ if (check > entry_count) -+ break; -+ } -+ -+#if check_erase2_debug > 2 -+ pr_err("mt_for_each %d and count %d\n", check, entry_count); -+#endif -+ -+ MT_BUG_ON(mt, check != entry_count); -+ -+ check = 0; -+ addr = 0; -+ mas_reset(&mas); -+ mas.index = 0; -+ rcu_read_lock(); -+ mas_for_each(&mas, foo, ULONG_MAX) { -+ if (xa_is_zero(foo)) { -+ if (addr == mas.index) { -+ mt_dump(mas.tree); -+ pr_err("retry failed %lu - %lu\n", -+ mas.index, mas.last); -+ MT_BUG_ON(mt, 1); -+ } -+ addr = mas.index; -+ continue; -+ } -+#if check_erase2_debug > 2 -+ pr_err("mas: %lu -> %p\n", mas.index, foo); -+#endif -+ check++; -+ if (check > entry_count) -+ break; -+ } -+ rcu_read_unlock(); -+#if check_erase2_debug > 2 -+ pr_err("mas_for_each %d and count %d\n", check, entry_count); -+ mt_validate(mt); -+#endif -+ -+ MT_BUG_ON(mt, check != entry_count); -+ -+ MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); -+ } -+} -+ -+ -+/* These tests were pulled from kvm tests. */ -+static noinline void check_erase2_sets(struct maple_tree *mt) -+{ -+ void *entry; -+ unsigned long start = 0; -+ unsigned long set[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140721266458624, 140737488351231, -+ERASE, 140721266458624, 140737488351231, -+STORE, 140721266458624, 140721266462719, -+STORE, 94735788949504, 94735789121535, -+ERASE, 94735788949504, 94735789121535, -+STORE, 94735788949504, 94735788965887, -+STORE, 94735788965888, 94735789121535, -+ERASE, 94735788965888, 94735789121535, -+STORE, 94735788965888, 94735789068287, -+STORE, 94735789068288, 94735789109247, -+STORE, 94735789109248, 94735789121535, -+STORE, 140253902692352, 140253902864383, -+ERASE, 140253902692352, 140253902864383, -+STORE, 140253902692352, 140253902696447, -+STORE, 140253902696448, 140253902864383, -+ }; -+ unsigned long set2[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140735933583360, 140737488351231, -+ERASE, 140735933583360, 140737488351231, -+STORE, 140735933583360, 140735933587455, -+STORE, 94811003260928, 94811003432959, -+ERASE, 94811003260928, 94811003432959, -+STORE, 94811003260928, 94811003277311, -+STORE, 94811003277312, 94811003432959, -+ERASE, 94811003277312, 94811003432959, -+STORE, 94811003277312, 94811003379711, -+STORE, 94811003379712, 94811003420671, -+STORE, 94811003420672, 94811003432959, -+STORE, 140277094653952, 140277094825983, -+ERASE, 140277094653952, 140277094825983, -+STORE, 140277094653952, 140277094658047, -+STORE, 140277094658048, 140277094825983, -+ERASE, 140277094658048, 140277094825983, -+STORE, 140277094658048, 140277094780927, -+STORE, 140277094780928, 140277094813695, -+STORE, 140277094813696, 140277094821887, -+STORE, 140277094821888, 140277094825983, -+STORE, 140735933906944, 140735933911039, -+ }; -+ unsigned long set3[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140735790264320, 140737488351231, -+ERASE, 140735790264320, 140737488351231, -+STORE, 140735790264320, 140735790268415, -+STORE, 94016597282816, 94016597454847, -+ERASE, 94016597282816, 94016597454847, -+STORE, 94016597282816, 94016597299199, -+STORE, 94016597299200, 94016597454847, -+ERASE, 94016597299200, 94016597454847, -+STORE, 94016597299200, 94016597401599, -+STORE, 94016597401600, 94016597442559, -+STORE, 94016597442560, 94016597454847, -+STORE, 140496959283200, 140496959455231, -+ERASE, 140496959283200, 140496959455231, -+STORE, 140496959283200, 140496959287295, -+STORE, 140496959287296, 140496959455231, -+ERASE, 140496959287296, 140496959455231, -+STORE, 140496959287296, 140496959410175, -+STORE, 140496959410176, 140496959442943, -+STORE, 140496959442944, 140496959451135, -+STORE, 140496959451136, 140496959455231, -+STORE, 140735791718400, 140735791722495, -+STORE, 140735791706112, 140735791718399, -+STORE, 47135835713536, 47135835721727, -+STORE, 47135835721728, 47135835729919, -+STORE, 47135835729920, 47135835893759, -+ERASE, 47135835729920, 47135835893759, -+STORE, 47135835729920, 47135835742207, -+STORE, 47135835742208, 47135835893759, -+STORE, 47135835840512, 47135835893759, -+STORE, 47135835742208, 47135835840511, -+ERASE, 47135835742208, 47135835840511, -+STORE, 47135835742208, 47135835840511, -+STORE, 47135835885568, 47135835893759, -+STORE, 47135835840512, 47135835885567, -+ERASE, 47135835840512, 47135835885567, -+STORE, 47135835840512, 47135835893759, -+ERASE, 47135835840512, 47135835893759, -+STORE, 47135835840512, 47135835885567, -+STORE, 47135835885568, 47135835893759, -+ }; -+ -+ unsigned long set4[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140728251703296, 140737488351231, -+ERASE, 140728251703296, 140737488351231, -+STORE, 140728251703296, 140728251707391, -+STORE, 94668429205504, 94668429377535, -+ERASE, 94668429205504, 94668429377535, -+STORE, 94668429205504, 94668429221887, -+STORE, 94668429221888, 94668429377535, -+ERASE, 94668429221888, 94668429377535, -+STORE, 94668429221888, 94668429324287, -+STORE, 94668429324288, 94668429365247, -+STORE, 94668429365248, 94668429377535, -+STORE, 47646523273216, 47646523445247, -+ERASE, 47646523273216, 47646523445247, -+STORE, 47646523273216, 47646523277311, -+STORE, 47646523277312, 47646523445247, -+ERASE, 47646523277312, 47646523445247, -+STORE, 47646523277312, 47646523400191, -+ }; -+ -+ unsigned long set5[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140726874062848, 140737488351231, -+ERASE, 140726874062848, 140737488351231, -+STORE, 140726874062848, 140726874066943, -+STORE, 94248892870656, 94248893042687, -+ERASE, 94248892870656, 94248893042687, -+STORE, 94248892870656, 94248892887039, -+STORE, 94248892887040, 94248893042687, -+ERASE, 94248892887040, 94248893042687, -+STORE, 94248892887040, 94248892989439, -+STORE, 94248892989440, 94248893030399, -+STORE, 94248893030400, 94248893042687, -+STORE, 47884786266112, 47884786438143, -+ERASE, 47884786266112, 47884786438143, -+STORE, 47884786266112, 47884786270207, -+STORE, 47884786270208, 47884786438143, -+ERASE, 47884786270208, 47884786438143, -+STORE, 47884786270208, 47884786393087, -+STORE, 47884786393088, 47884786425855, -+STORE, 47884786425856, 47884786434047, -+STORE, 47884786434048, 47884786438143, -+STORE, 140726874513408, 140726874517503, -+STORE, 140726874501120, 140726874513407, -+STORE, 47884786438144, 47884786446335, -+STORE, 47884786446336, 47884786454527, -+STORE, 47884786454528, 47884786618367, -+ERASE, 47884786454528, 47884786618367, -+STORE, 47884786454528, 47884786466815, -+STORE, 47884786466816, 47884786618367, -+STORE, 47884786565120, 47884786618367, -+STORE, 47884786466816, 47884786565119, -+ERASE, 47884786466816, 47884786565119, -+STORE, 47884786466816, 47884786565119, -+STORE, 47884786610176, 47884786618367, -+STORE, 47884786565120, 47884786610175, -+ERASE, 47884786565120, 47884786610175, -+STORE, 47884786565120, 47884786618367, -+ERASE, 47884786565120, 47884786618367, -+STORE, 47884786565120, 47884786610175, -+STORE, 47884786610176, 47884786618367, -+ERASE, 47884786610176, 47884786618367, -+STORE, 47884786610176, 47884786618367, -+STORE, 47884786618368, 47884789669887, -+STORE, 47884787163136, 47884789669887, -+STORE, 47884786618368, 47884787163135, -+ERASE, 47884787163136, 47884789669887, -+STORE, 47884787163136, 47884789448703, -+STORE, 47884789448704, 47884789669887, -+STORE, 47884788858880, 47884789448703, -+STORE, 47884787163136, 47884788858879, -+ERASE, 47884787163136, 47884788858879, -+STORE, 47884787163136, 47884788858879, -+STORE, 47884789444608, 47884789448703, -+STORE, 47884788858880, 47884789444607, -+ERASE, 47884788858880, 47884789444607, -+STORE, 47884788858880, 47884789444607, -+STORE, 47884789653504, 47884789669887, -+STORE, 47884789448704, 47884789653503, -+ERASE, 47884789448704, 47884789653503, -+STORE, 47884789448704, 47884789653503, -+ERASE, 47884789653504, 47884789669887, -+STORE, 47884789653504, 47884789669887, -+STORE, 47884789669888, 47884791508991, -+STORE, 47884789809152, 47884791508991, -+STORE, 47884789669888, 47884789809151, -+ERASE, 47884789809152, 47884791508991, -+STORE, 47884789809152, 47884791468031, -+STORE, 47884791468032, 47884791508991, -+STORE, 47884791152640, 47884791468031, -+STORE, 47884789809152, 47884791152639, -+ERASE, 47884789809152, 47884791152639, -+STORE, 47884789809152, 47884791152639, -+STORE, 47884791463936, 47884791468031, -+STORE, 47884791152640, 47884791463935, -+ERASE, 47884791152640, 47884791463935, -+STORE, 47884791152640, 47884791463935, -+STORE, 47884791492608, 47884791508991, -+STORE, 47884791468032, 47884791492607, -+ERASE, 47884791468032, 47884791492607, -+STORE, 47884791468032, 47884791492607, -+ERASE, 47884791492608, 47884791508991, -+STORE, 47884791492608, 47884791508991, -+STORE, 47884791508992, 47884791644159, -+ERASE, 47884791508992, 47884791644159, -+STORE, 47884791508992, 47884791533567, -+STORE, 47884791533568, 47884791644159, -+STORE, 47884791595008, 47884791644159, -+STORE, 47884791533568, 47884791595007, -+ERASE, 47884791533568, 47884791595007, -+STORE, 47884791533568, 47884791595007, -+STORE, 47884791619584, 47884791644159, -+STORE, 47884791595008, 47884791619583, -+ERASE, 47884791595008, 47884791619583, -+STORE, 47884791595008, 47884791644159, -+ERASE, 47884791595008, 47884791644159, -+STORE, 47884791595008, 47884791619583, -+STORE, 47884791619584, 47884791644159, -+STORE, 47884791627776, 47884791644159, -+STORE, 47884791619584, 47884791627775, -+ERASE, 47884791619584, 47884791627775, -+STORE, 47884791619584, 47884791627775, -+ERASE, 47884791627776, 47884791644159, -+STORE, 47884791627776, 47884791644159, -+STORE, 47884791644160, 47884791664639, -+ERASE, 47884791644160, 47884791664639, -+STORE, 47884791644160, 47884791648255, -+STORE, 47884791648256, 47884791664639, -+STORE, 47884791652352, 47884791664639, -+STORE, 47884791648256, 47884791652351, -+ERASE, 47884791648256, 47884791652351, -+STORE, 47884791648256, 47884791652351, -+STORE, 47884791656448, 47884791664639, -+STORE, 47884791652352, 47884791656447, -+ERASE, 47884791652352, 47884791656447, -+STORE, 47884791652352, 47884791664639, -+ERASE, 47884791652352, 47884791664639, -+STORE, 47884791652352, 47884791656447, -+STORE, 47884791656448, 47884791664639, -+ERASE, 47884791656448, 47884791664639, -+STORE, 47884791656448, 47884791664639, -+STORE, 47884791664640, 47884791672831, -+ERASE, 47884791468032, 47884791492607, -+STORE, 47884791468032, 47884791484415, -+STORE, 47884791484416, 47884791492607, -+ERASE, 47884791656448, 47884791664639, -+STORE, 47884791656448, 47884791660543, -+STORE, 47884791660544, 47884791664639, -+ERASE, 47884791619584, 47884791627775, -+STORE, 47884791619584, 47884791623679, -+STORE, 47884791623680, 47884791627775, -+ }; -+ -+ unsigned long set6[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140722999021568, 140737488351231, -+ERASE, 140722999021568, 140737488351231, -+STORE, 140722999021568, 140722999025663, -+STORE, 94901500268544, 94901500440575, -+ERASE, 94901500268544, 94901500440575, -+STORE, 94901500268544, 94901500284927, -+STORE, 94901500284928, 94901500440575, -+ERASE, 94901500284928, 94901500440575, -+STORE, 94901500284928, 94901500387327, -+STORE, 94901500387328, 94901500428287, -+STORE, 94901500428288, 94901500440575, -+STORE, 47430426660864, 47430426832895, -+ERASE, 47430426660864, 47430426832895, -+STORE, 47430426660864, 47430426664959, -+STORE, 47430426664960, 47430426832895, -+ERASE, 47430426664960, 47430426832895, -+STORE, 47430426664960, 47430426787839, -+STORE, 47430426787840, 47430426820607, -+STORE, 47430426820608, 47430426828799, -+STORE, 47430426828800, 47430426832895, -+STORE, 140722999115776, 140722999119871, -+STORE, 140722999103488, 140722999115775, -+STORE, 47430426832896, 47430426841087, -+STORE, 47430426841088, 47430426849279, -+STORE, 47430426849280, 47430427013119, -+ERASE, 47430426849280, 47430427013119, -+STORE, 47430426849280, 47430426861567, -+STORE, 47430426861568, 47430427013119, -+STORE, 47430426959872, 47430427013119, -+STORE, 47430426861568, 47430426959871, -+ERASE, 47430426861568, 47430426959871, -+STORE, 47430426861568, 47430426959871, -+STORE, 47430427004928, 47430427013119, -+STORE, 47430426959872, 47430427004927, -+ERASE, 47430426959872, 47430427004927, -+STORE, 47430426959872, 47430427013119, -+ERASE, 47430426959872, 47430427013119, -+STORE, 47430426959872, 47430427004927, -+STORE, 47430427004928, 47430427013119, -+ERASE, 47430427004928, 47430427013119, -+STORE, 47430427004928, 47430427013119, -+STORE, 47430427013120, 47430430064639, -+STORE, 47430427557888, 47430430064639, -+STORE, 47430427013120, 47430427557887, -+ERASE, 47430427557888, 47430430064639, -+STORE, 47430427557888, 47430429843455, -+STORE, 47430429843456, 47430430064639, -+STORE, 47430429253632, 47430429843455, -+STORE, 47430427557888, 47430429253631, -+ERASE, 47430427557888, 47430429253631, -+STORE, 47430427557888, 47430429253631, -+STORE, 47430429839360, 47430429843455, -+STORE, 47430429253632, 47430429839359, -+ERASE, 47430429253632, 47430429839359, -+STORE, 47430429253632, 47430429839359, -+STORE, 47430430048256, 47430430064639, -+STORE, 47430429843456, 47430430048255, -+ERASE, 47430429843456, 47430430048255, -+STORE, 47430429843456, 47430430048255, -+ERASE, 47430430048256, 47430430064639, -+STORE, 47430430048256, 47430430064639, -+STORE, 47430430064640, 47430431903743, -+STORE, 47430430203904, 47430431903743, -+STORE, 47430430064640, 47430430203903, -+ERASE, 47430430203904, 47430431903743, -+STORE, 47430430203904, 47430431862783, -+STORE, 47430431862784, 47430431903743, -+STORE, 47430431547392, 47430431862783, -+STORE, 47430430203904, 47430431547391, -+ERASE, 47430430203904, 47430431547391, -+STORE, 47430430203904, 47430431547391, -+STORE, 47430431858688, 47430431862783, -+STORE, 47430431547392, 47430431858687, -+ERASE, 47430431547392, 47430431858687, -+STORE, 47430431547392, 47430431858687, -+STORE, 47430431887360, 47430431903743, -+STORE, 47430431862784, 47430431887359, -+ERASE, 47430431862784, 47430431887359, -+STORE, 47430431862784, 47430431887359, -+ERASE, 47430431887360, 47430431903743, -+STORE, 47430431887360, 47430431903743, -+STORE, 47430431903744, 47430432038911, -+ERASE, 47430431903744, 47430432038911, -+STORE, 47430431903744, 47430431928319, -+STORE, 47430431928320, 47430432038911, -+STORE, 47430431989760, 47430432038911, -+STORE, 47430431928320, 47430431989759, -+ERASE, 47430431928320, 47430431989759, -+STORE, 47430431928320, 47430431989759, -+STORE, 47430432014336, 47430432038911, -+STORE, 47430431989760, 47430432014335, -+ERASE, 47430431989760, 47430432014335, -+STORE, 47430431989760, 47430432038911, -+ERASE, 47430431989760, 47430432038911, -+STORE, 47430431989760, 47430432014335, -+STORE, 47430432014336, 47430432038911, -+STORE, 47430432022528, 47430432038911, -+STORE, 47430432014336, 47430432022527, -+ERASE, 47430432014336, 47430432022527, -+STORE, 47430432014336, 47430432022527, -+ERASE, 47430432022528, 47430432038911, -+STORE, 47430432022528, 47430432038911, -+STORE, 47430432038912, 47430432059391, -+ERASE, 47430432038912, 47430432059391, -+STORE, 47430432038912, 47430432043007, -+STORE, 47430432043008, 47430432059391, -+STORE, 47430432047104, 47430432059391, -+STORE, 47430432043008, 47430432047103, -+ERASE, 47430432043008, 47430432047103, -+STORE, 47430432043008, 47430432047103, -+STORE, 47430432051200, 47430432059391, -+STORE, 47430432047104, 47430432051199, -+ERASE, 47430432047104, 47430432051199, -+STORE, 47430432047104, 47430432059391, -+ERASE, 47430432047104, 47430432059391, -+STORE, 47430432047104, 47430432051199, -+STORE, 47430432051200, 47430432059391, -+ERASE, 47430432051200, 47430432059391, -+STORE, 47430432051200, 47430432059391, -+STORE, 47430432059392, 47430432067583, -+ERASE, 47430431862784, 47430431887359, -+STORE, 47430431862784, 47430431879167, -+STORE, 47430431879168, 47430431887359, -+ERASE, 47430432051200, 47430432059391, -+STORE, 47430432051200, 47430432055295, -+STORE, 47430432055296, 47430432059391, -+ERASE, 47430432014336, 47430432022527, -+STORE, 47430432014336, 47430432018431, -+STORE, 47430432018432, 47430432022527, -+ }; -+ unsigned long set7[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140729808330752, 140737488351231, -+ERASE, 140729808330752, 140737488351231, -+STORE, 140729808330752, 140729808334847, -+STORE, 94629632020480, 94629632192511, -+ERASE, 94629632020480, 94629632192511, -+STORE, 94629632020480, 94629632036863, -+STORE, 94629632036864, 94629632192511, -+ERASE, 94629632036864, 94629632192511, -+STORE, 94629632036864, 94629632139263, -+STORE, 94629632139264, 94629632180223, -+STORE, 94629632180224, 94629632192511, -+STORE, 47439981776896, 47439981948927, -+ERASE, 47439981776896, 47439981948927, -+STORE, 47439981776896, 47439981780991, -+STORE, 47439981780992, 47439981948927, -+ERASE, 47439981780992, 47439981948927, -+STORE, 47439981780992, 47439981903871, -+STORE, 47439981903872, 47439981936639, -+STORE, 47439981936640, 47439981944831, -+STORE, 47439981944832, 47439981948927, -+STORE, 140729808474112, 140729808478207, -+STORE, 140729808461824, 140729808474111, -+STORE, 47439981948928, 47439981957119, -+STORE, 47439981957120, 47439981965311, -+STORE, 47439981965312, 47439982129151, -+ERASE, 47439981965312, 47439982129151, -+STORE, 47439981965312, 47439981977599, -+STORE, 47439981977600, 47439982129151, -+STORE, 47439982075904, 47439982129151, -+STORE, 47439981977600, 47439982075903, -+ERASE, 47439981977600, 47439982075903, -+STORE, 47439981977600, 47439982075903, -+STORE, 47439982120960, 47439982129151, -+STORE, 47439982075904, 47439982120959, -+ERASE, 47439982075904, 47439982120959, -+STORE, 47439982075904, 47439982129151, -+ERASE, 47439982075904, 47439982129151, -+STORE, 47439982075904, 47439982120959, -+STORE, 47439982120960, 47439982129151, -+ERASE, 47439982120960, 47439982129151, -+STORE, 47439982120960, 47439982129151, -+STORE, 47439982129152, 47439985180671, -+STORE, 47439982673920, 47439985180671, -+STORE, 47439982129152, 47439982673919, -+ERASE, 47439982673920, 47439985180671, -+STORE, 47439982673920, 47439984959487, -+STORE, 47439984959488, 47439985180671, -+STORE, 47439984369664, 47439984959487, -+STORE, 47439982673920, 47439984369663, -+ERASE, 47439982673920, 47439984369663, -+STORE, 47439982673920, 47439984369663, -+STORE, 47439984955392, 47439984959487, -+STORE, 47439984369664, 47439984955391, -+ERASE, 47439984369664, 47439984955391, -+STORE, 47439984369664, 47439984955391, -+STORE, 47439985164288, 47439985180671, -+STORE, 47439984959488, 47439985164287, -+ERASE, 47439984959488, 47439985164287, -+STORE, 47439984959488, 47439985164287, -+ERASE, 47439985164288, 47439985180671, -+STORE, 47439985164288, 47439985180671, -+STORE, 47439985180672, 47439987019775, -+STORE, 47439985319936, 47439987019775, -+STORE, 47439985180672, 47439985319935, -+ERASE, 47439985319936, 47439987019775, -+STORE, 47439985319936, 47439986978815, -+STORE, 47439986978816, 47439987019775, -+STORE, 47439986663424, 47439986978815, -+STORE, 47439985319936, 47439986663423, -+ERASE, 47439985319936, 47439986663423, -+STORE, 47439985319936, 47439986663423, -+STORE, 47439986974720, 47439986978815, -+STORE, 47439986663424, 47439986974719, -+ERASE, 47439986663424, 47439986974719, -+STORE, 47439986663424, 47439986974719, -+STORE, 47439987003392, 47439987019775, -+STORE, 47439986978816, 47439987003391, -+ERASE, 47439986978816, 47439987003391, -+STORE, 47439986978816, 47439987003391, -+ERASE, 47439987003392, 47439987019775, -+STORE, 47439987003392, 47439987019775, -+STORE, 47439987019776, 47439987154943, -+ERASE, 47439987019776, 47439987154943, -+STORE, 47439987019776, 47439987044351, -+STORE, 47439987044352, 47439987154943, -+STORE, 47439987105792, 47439987154943, -+STORE, 47439987044352, 47439987105791, -+ERASE, 47439987044352, 47439987105791, -+STORE, 47439987044352, 47439987105791, -+STORE, 47439987130368, 47439987154943, -+STORE, 47439987105792, 47439987130367, -+ERASE, 47439987105792, 47439987130367, -+STORE, 47439987105792, 47439987154943, -+ERASE, 47439987105792, 47439987154943, -+STORE, 47439987105792, 47439987130367, -+STORE, 47439987130368, 47439987154943, -+STORE, 47439987138560, 47439987154943, -+STORE, 47439987130368, 47439987138559, -+ERASE, 47439987130368, 47439987138559, -+STORE, 47439987130368, 47439987138559, -+ERASE, 47439987138560, 47439987154943, -+STORE, 47439987138560, 47439987154943, -+STORE, 47439987154944, 47439987175423, -+ERASE, 47439987154944, 47439987175423, -+STORE, 47439987154944, 47439987159039, -+STORE, 47439987159040, 47439987175423, -+STORE, 47439987163136, 47439987175423, -+STORE, 47439987159040, 47439987163135, -+ERASE, 47439987159040, 47439987163135, -+STORE, 47439987159040, 47439987163135, -+STORE, 47439987167232, 47439987175423, -+STORE, 47439987163136, 47439987167231, -+ERASE, 47439987163136, 47439987167231, -+STORE, 47439987163136, 47439987175423, -+ERASE, 47439987163136, 47439987175423, -+STORE, 47439987163136, 47439987167231, -+STORE, 47439987167232, 47439987175423, -+ERASE, 47439987167232, 47439987175423, -+STORE, 47439987167232, 47439987175423, -+STORE, 47439987175424, 47439987183615, -+ERASE, 47439986978816, 47439987003391, -+STORE, 47439986978816, 47439986995199, -+STORE, 47439986995200, 47439987003391, -+ERASE, 47439987167232, 47439987175423, -+STORE, 47439987167232, 47439987171327, -+STORE, 47439987171328, 47439987175423, -+ERASE, 47439987130368, 47439987138559, -+STORE, 47439987130368, 47439987134463, -+STORE, 47439987134464, 47439987138559, -+ }; -+ unsigned long set8[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140722482974720, 140737488351231, -+ERASE, 140722482974720, 140737488351231, -+STORE, 140722482974720, 140722482978815, -+STORE, 94121505034240, 94121505206271, -+ERASE, 94121505034240, 94121505206271, -+STORE, 94121505034240, 94121505050623, -+STORE, 94121505050624, 94121505206271, -+ERASE, 94121505050624, 94121505206271, -+STORE, 94121505050624, 94121505153023, -+STORE, 94121505153024, 94121505193983, -+STORE, 94121505193984, 94121505206271, -+STORE, 47708483284992, 47708483457023, -+ERASE, 47708483284992, 47708483457023, -+STORE, 47708483284992, 47708483289087, -+STORE, 47708483289088, 47708483457023, -+ERASE, 47708483289088, 47708483457023, -+STORE, 47708483289088, 47708483411967, -+STORE, 47708483411968, 47708483444735, -+STORE, 47708483444736, 47708483452927, -+STORE, 47708483452928, 47708483457023, -+STORE, 140722483142656, 140722483146751, -+STORE, 140722483130368, 140722483142655, -+STORE, 47708483457024, 47708483465215, -+STORE, 47708483465216, 47708483473407, -+STORE, 47708483473408, 47708483637247, -+ERASE, 47708483473408, 47708483637247, -+STORE, 47708483473408, 47708483485695, -+STORE, 47708483485696, 47708483637247, -+STORE, 47708483584000, 47708483637247, -+STORE, 47708483485696, 47708483583999, -+ERASE, 47708483485696, 47708483583999, -+STORE, 47708483485696, 47708483583999, -+STORE, 47708483629056, 47708483637247, -+STORE, 47708483584000, 47708483629055, -+ERASE, 47708483584000, 47708483629055, -+STORE, 47708483584000, 47708483637247, -+ERASE, 47708483584000, 47708483637247, -+STORE, 47708483584000, 47708483629055, -+STORE, 47708483629056, 47708483637247, -+ERASE, 47708483629056, 47708483637247, -+STORE, 47708483629056, 47708483637247, -+STORE, 47708483637248, 47708486688767, -+STORE, 47708484182016, 47708486688767, -+STORE, 47708483637248, 47708484182015, -+ERASE, 47708484182016, 47708486688767, -+STORE, 47708484182016, 47708486467583, -+STORE, 47708486467584, 47708486688767, -+STORE, 47708485877760, 47708486467583, -+STORE, 47708484182016, 47708485877759, -+ERASE, 47708484182016, 47708485877759, -+STORE, 47708484182016, 47708485877759, -+STORE, 47708486463488, 47708486467583, -+STORE, 47708485877760, 47708486463487, -+ERASE, 47708485877760, 47708486463487, -+STORE, 47708485877760, 47708486463487, -+STORE, 47708486672384, 47708486688767, -+STORE, 47708486467584, 47708486672383, -+ERASE, 47708486467584, 47708486672383, -+STORE, 47708486467584, 47708486672383, -+ERASE, 47708486672384, 47708486688767, -+STORE, 47708486672384, 47708486688767, -+STORE, 47708486688768, 47708488527871, -+STORE, 47708486828032, 47708488527871, -+STORE, 47708486688768, 47708486828031, -+ERASE, 47708486828032, 47708488527871, -+STORE, 47708486828032, 47708488486911, -+STORE, 47708488486912, 47708488527871, -+STORE, 47708488171520, 47708488486911, -+STORE, 47708486828032, 47708488171519, -+ERASE, 47708486828032, 47708488171519, -+STORE, 47708486828032, 47708488171519, -+STORE, 47708488482816, 47708488486911, -+STORE, 47708488171520, 47708488482815, -+ERASE, 47708488171520, 47708488482815, -+STORE, 47708488171520, 47708488482815, -+STORE, 47708488511488, 47708488527871, -+STORE, 47708488486912, 47708488511487, -+ERASE, 47708488486912, 47708488511487, -+STORE, 47708488486912, 47708488511487, -+ERASE, 47708488511488, 47708488527871, -+STORE, 47708488511488, 47708488527871, -+STORE, 47708488527872, 47708488663039, -+ERASE, 47708488527872, 47708488663039, -+STORE, 47708488527872, 47708488552447, -+STORE, 47708488552448, 47708488663039, -+STORE, 47708488613888, 47708488663039, -+STORE, 47708488552448, 47708488613887, -+ERASE, 47708488552448, 47708488613887, -+STORE, 47708488552448, 47708488613887, -+STORE, 47708488638464, 47708488663039, -+STORE, 47708488613888, 47708488638463, -+ERASE, 47708488613888, 47708488638463, -+STORE, 47708488613888, 47708488663039, -+ERASE, 47708488613888, 47708488663039, -+STORE, 47708488613888, 47708488638463, -+STORE, 47708488638464, 47708488663039, -+STORE, 47708488646656, 47708488663039, -+STORE, 47708488638464, 47708488646655, -+ERASE, 47708488638464, 47708488646655, -+STORE, 47708488638464, 47708488646655, -+ERASE, 47708488646656, 47708488663039, -+STORE, 47708488646656, 47708488663039, -+STORE, 47708488663040, 47708488683519, -+ERASE, 47708488663040, 47708488683519, -+STORE, 47708488663040, 47708488667135, -+STORE, 47708488667136, 47708488683519, -+STORE, 47708488671232, 47708488683519, -+STORE, 47708488667136, 47708488671231, -+ERASE, 47708488667136, 47708488671231, -+STORE, 47708488667136, 47708488671231, -+STORE, 47708488675328, 47708488683519, -+STORE, 47708488671232, 47708488675327, -+ERASE, 47708488671232, 47708488675327, -+STORE, 47708488671232, 47708488683519, -+ERASE, 47708488671232, 47708488683519, -+STORE, 47708488671232, 47708488675327, -+STORE, 47708488675328, 47708488683519, -+ERASE, 47708488675328, 47708488683519, -+STORE, 47708488675328, 47708488683519, -+STORE, 47708488683520, 47708488691711, -+ERASE, 47708488486912, 47708488511487, -+STORE, 47708488486912, 47708488503295, -+STORE, 47708488503296, 47708488511487, -+ERASE, 47708488675328, 47708488683519, -+STORE, 47708488675328, 47708488679423, -+STORE, 47708488679424, 47708488683519, -+ERASE, 47708488638464, 47708488646655, -+STORE, 47708488638464, 47708488642559, -+STORE, 47708488642560, 47708488646655, -+ }; -+ -+ unsigned long set9[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140736427839488, 140737488351231, -+ERASE, 140736427839488, 140736427839488, -+STORE, 140736427839488, 140736427843583, -+STORE, 94071213395968, 94071213567999, -+ERASE, 94071213395968, 94071213395968, -+STORE, 94071213395968, 94071213412351, -+STORE, 94071213412352, 94071213567999, -+ERASE, 94071213412352, 94071213412352, -+STORE, 94071213412352, 94071213514751, -+STORE, 94071213514752, 94071213555711, -+STORE, 94071213555712, 94071213567999, -+STORE, 139968410644480, 139968410816511, -+ERASE, 139968410644480, 139968410644480, -+STORE, 139968410644480, 139968410648575, -+STORE, 139968410648576, 139968410816511, -+ERASE, 139968410648576, 139968410648576, -+STORE, 139968410648576, 139968410771455, -+STORE, 139968410771456, 139968410804223, -+STORE, 139968410804224, 139968410812415, -+STORE, 139968410812416, 139968410816511, -+STORE, 140736429277184, 140736429281279, -+STORE, 140736429264896, 140736429277183, -+STORE, 47664384352256, 47664384360447, -+STORE, 47664384360448, 47664384368639, -+STORE, 47664384368640, 47664384532479, -+ERASE, 47664384368640, 47664384368640, -+STORE, 47664384368640, 47664384380927, -+STORE, 47664384380928, 47664384532479, -+STORE, 47664384479232, 47664384532479, -+STORE, 47664384380928, 47664384479231, -+ERASE, 47664384380928, 47664384380928, -+STORE, 47664384380928, 47664384479231, -+STORE, 47664384524288, 47664384532479, -+STORE, 47664384479232, 47664384524287, -+ERASE, 47664384479232, 47664384479232, -+STORE, 47664384479232, 47664384532479, -+ERASE, 47664384479232, 47664384479232, -+STORE, 47664384479232, 47664384524287, -+STORE, 47664384524288, 47664384532479, -+ERASE, 47664384524288, 47664384524288, -+STORE, 47664384524288, 47664384532479, -+STORE, 47664384532480, 47664387583999, -+STORE, 47664385077248, 47664387583999, -+STORE, 47664384532480, 47664385077247, -+ERASE, 47664385077248, 47664385077248, -+STORE, 47664385077248, 47664387362815, -+STORE, 47664387362816, 47664387583999, -+STORE, 47664386772992, 47664387362815, -+STORE, 47664385077248, 47664386772991, -+ERASE, 47664385077248, 47664385077248, -+STORE, 47664385077248, 47664386772991, -+STORE, 47664387358720, 47664387362815, -+STORE, 47664386772992, 47664387358719, -+ERASE, 47664386772992, 47664386772992, -+STORE, 47664386772992, 47664387358719, -+STORE, 47664387567616, 47664387583999, -+STORE, 47664387362816, 47664387567615, -+ERASE, 47664387362816, 47664387362816, -+STORE, 47664387362816, 47664387567615, -+ERASE, 47664387567616, 47664387567616, -+STORE, 47664387567616, 47664387583999, -+STORE, 47664387584000, 47664389423103, -+STORE, 47664387723264, 47664389423103, -+STORE, 47664387584000, 47664387723263, -+ERASE, 47664387723264, 47664387723264, -+STORE, 47664387723264, 47664389382143, -+STORE, 47664389382144, 47664389423103, -+STORE, 47664389066752, 47664389382143, -+STORE, 47664387723264, 47664389066751, -+ERASE, 47664387723264, 47664387723264, -+STORE, 47664387723264, 47664389066751, -+STORE, 47664389378048, 47664389382143, -+STORE, 47664389066752, 47664389378047, -+ERASE, 47664389066752, 47664389066752, -+STORE, 47664389066752, 47664389378047, -+STORE, 47664389406720, 47664389423103, -+STORE, 47664389382144, 47664389406719, -+ERASE, 47664389382144, 47664389382144, -+STORE, 47664389382144, 47664389406719, -+ERASE, 47664389406720, 47664389406720, -+STORE, 47664389406720, 47664389423103, -+STORE, 47664389423104, 47664389558271, -+ERASE, 47664389423104, 47664389423104, -+STORE, 47664389423104, 47664389447679, -+STORE, 47664389447680, 47664389558271, -+STORE, 47664389509120, 47664389558271, -+STORE, 47664389447680, 47664389509119, -+ERASE, 47664389447680, 47664389447680, -+STORE, 47664389447680, 47664389509119, -+STORE, 47664389533696, 47664389558271, -+STORE, 47664389509120, 47664389533695, -+ERASE, 47664389509120, 47664389509120, -+STORE, 47664389509120, 47664389558271, -+ERASE, 47664389509120, 47664389509120, -+STORE, 47664389509120, 47664389533695, -+STORE, 47664389533696, 47664389558271, -+STORE, 47664389541888, 47664389558271, -+STORE, 47664389533696, 47664389541887, -+ERASE, 47664389533696, 47664389533696, -+STORE, 47664389533696, 47664389541887, -+ERASE, 47664389541888, 47664389541888, -+STORE, 47664389541888, 47664389558271, -+STORE, 47664389558272, 47664389578751, -+ERASE, 47664389558272, 47664389558272, -+STORE, 47664389558272, 47664389562367, -+STORE, 47664389562368, 47664389578751, -+STORE, 47664389566464, 47664389578751, -+STORE, 47664389562368, 47664389566463, -+ERASE, 47664389562368, 47664389562368, -+STORE, 47664389562368, 47664389566463, -+STORE, 47664389570560, 47664389578751, -+STORE, 47664389566464, 47664389570559, -+ERASE, 47664389566464, 47664389566464, -+STORE, 47664389566464, 47664389578751, -+ERASE, 47664389566464, 47664389566464, -+STORE, 47664389566464, 47664389570559, -+STORE, 47664389570560, 47664389578751, -+ERASE, 47664389570560, 47664389570560, -+STORE, 47664389570560, 47664389578751, -+STORE, 47664389578752, 47664389586943, -+ERASE, 47664389382144, 47664389382144, -+STORE, 47664389382144, 47664389398527, -+STORE, 47664389398528, 47664389406719, -+ERASE, 47664389570560, 47664389570560, -+STORE, 47664389570560, 47664389574655, -+STORE, 47664389574656, 47664389578751, -+ERASE, 47664389533696, 47664389533696, -+STORE, 47664389533696, 47664389537791, -+STORE, 47664389537792, 47664389541887, -+ERASE, 47664387362816, 47664387362816, -+STORE, 47664387362816, 47664387559423, -+STORE, 47664387559424, 47664387567615, -+ERASE, 47664384524288, 47664384524288, -+STORE, 47664384524288, 47664384528383, -+STORE, 47664384528384, 47664384532479, -+ERASE, 94071213555712, 94071213555712, -+STORE, 94071213555712, 94071213563903, -+STORE, 94071213563904, 94071213567999, -+ERASE, 139968410804224, 139968410804224, -+STORE, 139968410804224, 139968410808319, -+STORE, 139968410808320, 139968410812415, -+ERASE, 47664384352256, 47664384352256, -+STORE, 94071244402688, 94071244537855, -+STORE, 140737488347136, 140737488351231, -+STORE, 140728271503360, 140737488351231, -+ERASE, 140728271503360, 140728271503360, -+STORE, 140728271503360, 140728271507455, -+STORE, 94410361982976, 94410362155007, -+ERASE, 94410361982976, 94410361982976, -+STORE, 94410361982976, 94410361999359, -+STORE, 94410361999360, 94410362155007, -+ERASE, 94410361999360, 94410361999360, -+STORE, 94410361999360, 94410362101759, -+STORE, 94410362101760, 94410362142719, -+STORE, 94410362142720, 94410362155007, -+STORE, 140351953997824, 140351954169855, -+ERASE, 140351953997824, 140351953997824, -+STORE, 140351953997824, 140351954001919, -+STORE, 140351954001920, 140351954169855, -+ERASE, 140351954001920, 140351954001920, -+STORE, 140351954001920, 140351954124799, -+STORE, 140351954124800, 140351954157567, -+STORE, 140351954157568, 140351954165759, -+STORE, 140351954165760, 140351954169855, -+STORE, 140728272429056, 140728272433151, -+STORE, 140728272416768, 140728272429055, -+STORE, 47280840998912, 47280841007103, -+STORE, 47280841007104, 47280841015295, -+STORE, 47280841015296, 47280841179135, -+ERASE, 47280841015296, 47280841015296, -+STORE, 47280841015296, 47280841027583, -+STORE, 47280841027584, 47280841179135, -+STORE, 47280841125888, 47280841179135, -+STORE, 47280841027584, 47280841125887, -+ERASE, 47280841027584, 47280841027584, -+STORE, 47280841027584, 47280841125887, -+STORE, 47280841170944, 47280841179135, -+STORE, 47280841125888, 47280841170943, -+ERASE, 47280841125888, 47280841125888, -+STORE, 47280841125888, 47280841179135, -+ERASE, 47280841125888, 47280841125888, -+STORE, 47280841125888, 47280841170943, -+STORE, 47280841170944, 47280841179135, -+ERASE, 47280841170944, 47280841170944, -+STORE, 47280841170944, 47280841179135, -+STORE, 47280841179136, 47280844230655, -+STORE, 47280841723904, 47280844230655, -+STORE, 47280841179136, 47280841723903, -+ERASE, 47280841723904, 47280841723904, -+STORE, 47280841723904, 47280844009471, -+STORE, 47280844009472, 47280844230655, -+STORE, 47280843419648, 47280844009471, -+STORE, 47280841723904, 47280843419647, -+ERASE, 47280841723904, 47280841723904, -+STORE, 47280841723904, 47280843419647, -+STORE, 47280844005376, 47280844009471, -+STORE, 47280843419648, 47280844005375, -+ERASE, 47280843419648, 47280843419648, -+STORE, 47280843419648, 47280844005375, -+STORE, 47280844214272, 47280844230655, -+STORE, 47280844009472, 47280844214271, -+ERASE, 47280844009472, 47280844009472, -+STORE, 47280844009472, 47280844214271, -+ERASE, 47280844214272, 47280844214272, -+STORE, 47280844214272, 47280844230655, -+STORE, 47280844230656, 47280846069759, -+STORE, 47280844369920, 47280846069759, -+STORE, 47280844230656, 47280844369919, -+ERASE, 47280844369920, 47280844369920, -+STORE, 47280844369920, 47280846028799, -+STORE, 47280846028800, 47280846069759, -+STORE, 47280845713408, 47280846028799, -+STORE, 47280844369920, 47280845713407, -+ERASE, 47280844369920, 47280844369920, -+STORE, 47280844369920, 47280845713407, -+STORE, 47280846024704, 47280846028799, -+STORE, 47280845713408, 47280846024703, -+ERASE, 47280845713408, 47280845713408, -+STORE, 47280845713408, 47280846024703, -+STORE, 47280846053376, 47280846069759, -+STORE, 47280846028800, 47280846053375, -+ERASE, 47280846028800, 47280846028800, -+STORE, 47280846028800, 47280846053375, -+ERASE, 47280846053376, 47280846053376, -+STORE, 47280846053376, 47280846069759, -+STORE, 47280846069760, 47280846204927, -+ERASE, 47280846069760, 47280846069760, -+STORE, 47280846069760, 47280846094335, -+STORE, 47280846094336, 47280846204927, -+STORE, 47280846155776, 47280846204927, -+STORE, 47280846094336, 47280846155775, -+ERASE, 47280846094336, 47280846094336, -+STORE, 47280846094336, 47280846155775, -+STORE, 47280846180352, 47280846204927, -+STORE, 47280846155776, 47280846180351, -+ERASE, 47280846155776, 47280846155776, -+STORE, 47280846155776, 47280846204927, -+ERASE, 47280846155776, 47280846155776, -+STORE, 47280846155776, 47280846180351, -+STORE, 47280846180352, 47280846204927, -+STORE, 47280846188544, 47280846204927, -+STORE, 47280846180352, 47280846188543, -+ERASE, 47280846180352, 47280846180352, -+STORE, 47280846180352, 47280846188543, -+ERASE, 47280846188544, 47280846188544, -+STORE, 47280846188544, 47280846204927, -+STORE, 47280846204928, 47280846225407, -+ERASE, 47280846204928, 47280846204928, -+STORE, 47280846204928, 47280846209023, -+STORE, 47280846209024, 47280846225407, -+STORE, 47280846213120, 47280846225407, -+STORE, 47280846209024, 47280846213119, -+ERASE, 47280846209024, 47280846209024, -+STORE, 47280846209024, 47280846213119, -+STORE, 47280846217216, 47280846225407, -+STORE, 47280846213120, 47280846217215, -+ERASE, 47280846213120, 47280846213120, -+STORE, 47280846213120, 47280846225407, -+ERASE, 47280846213120, 47280846213120, -+STORE, 47280846213120, 47280846217215, -+STORE, 47280846217216, 47280846225407, -+ERASE, 47280846217216, 47280846217216, -+STORE, 47280846217216, 47280846225407, -+STORE, 47280846225408, 47280846233599, -+ERASE, 47280846028800, 47280846028800, -+STORE, 47280846028800, 47280846045183, -+STORE, 47280846045184, 47280846053375, -+ERASE, 47280846217216, 47280846217216, -+STORE, 47280846217216, 47280846221311, -+STORE, 47280846221312, 47280846225407, -+ERASE, 47280846180352, 47280846180352, -+STORE, 47280846180352, 47280846184447, -+STORE, 47280846184448, 47280846188543, -+ERASE, 47280844009472, 47280844009472, -+STORE, 47280844009472, 47280844206079, -+STORE, 47280844206080, 47280844214271, -+ERASE, 47280841170944, 47280841170944, -+STORE, 47280841170944, 47280841175039, -+STORE, 47280841175040, 47280841179135, -+ERASE, 94410362142720, 94410362142720, -+STORE, 94410362142720, 94410362150911, -+STORE, 94410362150912, 94410362155007, -+ERASE, 140351954157568, 140351954157568, -+STORE, 140351954157568, 140351954161663, -+STORE, 140351954161664, 140351954165759, -+ERASE, 47280840998912, 47280840998912, -+STORE, 94410379456512, 94410379591679, -+STORE, 140737488347136, 140737488351231, -+STORE, 140732946362368, 140737488351231, -+ERASE, 140732946362368, 140732946362368, -+STORE, 140732946362368, 140732946366463, -+STORE, 94352937934848, 94352938106879, -+ERASE, 94352937934848, 94352937934848, -+STORE, 94352937934848, 94352937951231, -+STORE, 94352937951232, 94352938106879, -+ERASE, 94352937951232, 94352937951232, -+STORE, 94352937951232, 94352938053631, -+STORE, 94352938053632, 94352938094591, -+STORE, 94352938094592, 94352938106879, -+STORE, 140595518742528, 140595518914559, -+ERASE, 140595518742528, 140595518742528, -+STORE, 140595518742528, 140595518746623, -+STORE, 140595518746624, 140595518914559, -+ERASE, 140595518746624, 140595518746624, -+STORE, 140595518746624, 140595518869503, -+STORE, 140595518869504, 140595518902271, -+STORE, 140595518902272, 140595518910463, -+STORE, 140595518910464, 140595518914559, -+STORE, 140732947468288, 140732947472383, -+STORE, 140732947456000, 140732947468287, -+STORE, 47037276254208, 47037276262399, -+STORE, 47037276262400, 47037276270591, -+STORE, 47037276270592, 47037276434431, -+ERASE, 47037276270592, 47037276270592, -+STORE, 47037276270592, 47037276282879, -+STORE, 47037276282880, 47037276434431, -+STORE, 47037276381184, 47037276434431, -+STORE, 47037276282880, 47037276381183, -+ERASE, 47037276282880, 47037276282880, -+STORE, 47037276282880, 47037276381183, -+STORE, 47037276426240, 47037276434431, -+STORE, 47037276381184, 47037276426239, -+ERASE, 47037276381184, 47037276381184, -+STORE, 47037276381184, 47037276434431, -+ERASE, 47037276381184, 47037276381184, -+STORE, 47037276381184, 47037276426239, -+STORE, 47037276426240, 47037276434431, -+ERASE, 47037276426240, 47037276426240, -+STORE, 47037276426240, 47037276434431, -+STORE, 47037276434432, 47037279485951, -+STORE, 47037276979200, 47037279485951, -+STORE, 47037276434432, 47037276979199, -+ERASE, 47037276979200, 47037276979200, -+STORE, 47037276979200, 47037279264767, -+STORE, 47037279264768, 47037279485951, -+STORE, 47037278674944, 47037279264767, -+STORE, 47037276979200, 47037278674943, -+ERASE, 47037276979200, 47037276979200, -+STORE, 47037276979200, 47037278674943, -+STORE, 47037279260672, 47037279264767, -+STORE, 47037278674944, 47037279260671, -+ERASE, 47037278674944, 47037278674944, -+STORE, 47037278674944, 47037279260671, -+STORE, 47037279469568, 47037279485951, -+STORE, 47037279264768, 47037279469567, -+ERASE, 47037279264768, 47037279264768, -+STORE, 47037279264768, 47037279469567, -+ERASE, 47037279469568, 47037279469568, -+STORE, 47037279469568, 47037279485951, -+STORE, 47037279485952, 47037281325055, -+STORE, 47037279625216, 47037281325055, -+STORE, 47037279485952, 47037279625215, -+ERASE, 47037279625216, 47037279625216, -+STORE, 47037279625216, 47037281284095, -+STORE, 47037281284096, 47037281325055, -+STORE, 47037280968704, 47037281284095, -+STORE, 47037279625216, 47037280968703, -+ERASE, 47037279625216, 47037279625216, -+STORE, 47037279625216, 47037280968703, -+STORE, 47037281280000, 47037281284095, -+STORE, 47037280968704, 47037281279999, -+ERASE, 47037280968704, 47037280968704, -+STORE, 47037280968704, 47037281279999, -+STORE, 47037281308672, 47037281325055, -+STORE, 47037281284096, 47037281308671, -+ERASE, 47037281284096, 47037281284096, -+STORE, 47037281284096, 47037281308671, -+ERASE, 47037281308672, 47037281308672, -+STORE, 47037281308672, 47037281325055, -+STORE, 47037281325056, 47037281460223, -+ERASE, 47037281325056, 47037281325056, -+STORE, 47037281325056, 47037281349631, -+STORE, 47037281349632, 47037281460223, -+STORE, 47037281411072, 47037281460223, -+STORE, 47037281349632, 47037281411071, -+ERASE, 47037281349632, 47037281349632, -+STORE, 47037281349632, 47037281411071, -+STORE, 47037281435648, 47037281460223, -+STORE, 47037281411072, 47037281435647, -+ERASE, 47037281411072, 47037281411072, -+STORE, 47037281411072, 47037281460223, -+ERASE, 47037281411072, 47037281411072, -+STORE, 47037281411072, 47037281435647, -+STORE, 47037281435648, 47037281460223, -+STORE, 47037281443840, 47037281460223, -+STORE, 47037281435648, 47037281443839, -+ERASE, 47037281435648, 47037281435648, -+STORE, 47037281435648, 47037281443839, -+ERASE, 47037281443840, 47037281443840, -+STORE, 47037281443840, 47037281460223, -+STORE, 47037281460224, 47037281480703, -+ERASE, 47037281460224, 47037281460224, -+STORE, 47037281460224, 47037281464319, -+STORE, 47037281464320, 47037281480703, -+STORE, 47037281468416, 47037281480703, -+STORE, 47037281464320, 47037281468415, -+ERASE, 47037281464320, 47037281464320, -+STORE, 47037281464320, 47037281468415, -+STORE, 47037281472512, 47037281480703, -+STORE, 47037281468416, 47037281472511, -+ERASE, 47037281468416, 47037281468416, -+STORE, 47037281468416, 47037281480703, -+ERASE, 47037281468416, 47037281468416, -+STORE, 47037281468416, 47037281472511, -+STORE, 47037281472512, 47037281480703, -+ERASE, 47037281472512, 47037281472512, -+STORE, 47037281472512, 47037281480703, -+STORE, 47037281480704, 47037281488895, -+ERASE, 47037281284096, 47037281284096, -+STORE, 47037281284096, 47037281300479, -+STORE, 47037281300480, 47037281308671, -+ERASE, 47037281472512, 47037281472512, -+STORE, 47037281472512, 47037281476607, -+STORE, 47037281476608, 47037281480703, -+ERASE, 47037281435648, 47037281435648, -+STORE, 47037281435648, 47037281439743, -+STORE, 47037281439744, 47037281443839, -+ERASE, 47037279264768, 47037279264768, -+STORE, 47037279264768, 47037279461375, -+STORE, 47037279461376, 47037279469567, -+ERASE, 47037276426240, 47037276426240, -+STORE, 47037276426240, 47037276430335, -+STORE, 47037276430336, 47037276434431, -+ERASE, 94352938094592, 94352938094592, -+STORE, 94352938094592, 94352938102783, -+STORE, 94352938102784, 94352938106879, -+ERASE, 140595518902272, 140595518902272, -+STORE, 140595518902272, 140595518906367, -+STORE, 140595518906368, 140595518910463, -+ERASE, 47037276254208, 47037276254208, -+STORE, 94352938438656, 94352938573823, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733506027520, 140737488351231, -+ERASE, 140733506027520, 140733506027520, -+STORE, 140733506027520, 140733506031615, -+STORE, 94150123073536, 94150123245567, -+ERASE, 94150123073536, 94150123073536, -+STORE, 94150123073536, 94150123089919, -+STORE, 94150123089920, 94150123245567, -+ERASE, 94150123089920, 94150123089920, -+STORE, 94150123089920, 94150123192319, -+STORE, 94150123192320, 94150123233279, -+STORE, 94150123233280, 94150123245567, -+STORE, 140081290375168, 140081290547199, -+ERASE, 140081290375168, 140081290375168, -+STORE, 140081290375168, 140081290379263, -+STORE, 140081290379264, 140081290547199, -+ERASE, 140081290379264, 140081290379264, -+STORE, 140081290379264, 140081290502143, -+STORE, 140081290502144, 140081290534911, -+STORE, 140081290534912, 140081290543103, -+STORE, 140081290543104, 140081290547199, -+STORE, 140733506707456, 140733506711551, -+STORE, 140733506695168, 140733506707455, -+STORE, 47551504621568, 47551504629759, -+STORE, 47551504629760, 47551504637951, -+STORE, 47551504637952, 47551504801791, -+ERASE, 47551504637952, 47551504637952, -+STORE, 47551504637952, 47551504650239, -+STORE, 47551504650240, 47551504801791, -+STORE, 47551504748544, 47551504801791, -+STORE, 47551504650240, 47551504748543, -+ERASE, 47551504650240, 47551504650240, -+STORE, 47551504650240, 47551504748543, -+STORE, 47551504793600, 47551504801791, -+STORE, 47551504748544, 47551504793599, -+ERASE, 47551504748544, 47551504748544, -+STORE, 47551504748544, 47551504801791, -+ERASE, 47551504748544, 47551504748544, -+STORE, 47551504748544, 47551504793599, -+STORE, 47551504793600, 47551504801791, -+ERASE, 47551504793600, 47551504793600, -+STORE, 47551504793600, 47551504801791, -+STORE, 47551504801792, 47551507853311, -+STORE, 47551505346560, 47551507853311, -+STORE, 47551504801792, 47551505346559, -+ERASE, 47551505346560, 47551505346560, -+STORE, 47551505346560, 47551507632127, -+STORE, 47551507632128, 47551507853311, -+STORE, 47551507042304, 47551507632127, -+STORE, 47551505346560, 47551507042303, -+ERASE, 47551505346560, 47551505346560, -+STORE, 47551505346560, 47551507042303, -+STORE, 47551507628032, 47551507632127, -+STORE, 47551507042304, 47551507628031, -+ERASE, 47551507042304, 47551507042304, -+STORE, 47551507042304, 47551507628031, -+STORE, 47551507836928, 47551507853311, -+STORE, 47551507632128, 47551507836927, -+ERASE, 47551507632128, 47551507632128, -+STORE, 47551507632128, 47551507836927, -+ERASE, 47551507836928, 47551507836928, -+STORE, 47551507836928, 47551507853311, -+STORE, 47551507853312, 47551509692415, -+STORE, 47551507992576, 47551509692415, -+STORE, 47551507853312, 47551507992575, -+ERASE, 47551507992576, 47551507992576, -+STORE, 47551507992576, 47551509651455, -+STORE, 47551509651456, 47551509692415, -+STORE, 47551509336064, 47551509651455, -+STORE, 47551507992576, 47551509336063, -+ERASE, 47551507992576, 47551507992576, -+STORE, 47551507992576, 47551509336063, -+STORE, 47551509647360, 47551509651455, -+STORE, 47551509336064, 47551509647359, -+ERASE, 47551509336064, 47551509336064, -+STORE, 47551509336064, 47551509647359, -+STORE, 47551509676032, 47551509692415, -+STORE, 47551509651456, 47551509676031, -+ERASE, 47551509651456, 47551509651456, -+STORE, 47551509651456, 47551509676031, -+ERASE, 47551509676032, 47551509676032, -+STORE, 47551509676032, 47551509692415, -+STORE, 47551509692416, 47551509827583, -+ERASE, 47551509692416, 47551509692416, -+STORE, 47551509692416, 47551509716991, -+STORE, 47551509716992, 47551509827583, -+STORE, 47551509778432, 47551509827583, -+STORE, 47551509716992, 47551509778431, -+ERASE, 47551509716992, 47551509716992, -+STORE, 47551509716992, 47551509778431, -+STORE, 47551509803008, 47551509827583, -+STORE, 47551509778432, 47551509803007, -+ERASE, 47551509778432, 47551509778432, -+STORE, 47551509778432, 47551509827583, -+ERASE, 47551509778432, 47551509778432, -+STORE, 47551509778432, 47551509803007, -+STORE, 47551509803008, 47551509827583, -+STORE, 47551509811200, 47551509827583, -+STORE, 47551509803008, 47551509811199, -+ERASE, 47551509803008, 47551509803008, -+STORE, 47551509803008, 47551509811199, -+ERASE, 47551509811200, 47551509811200, -+STORE, 47551509811200, 47551509827583, -+STORE, 47551509827584, 47551509848063, -+ERASE, 47551509827584, 47551509827584, -+STORE, 47551509827584, 47551509831679, -+STORE, 47551509831680, 47551509848063, -+STORE, 47551509835776, 47551509848063, -+STORE, 47551509831680, 47551509835775, -+ERASE, 47551509831680, 47551509831680, -+STORE, 47551509831680, 47551509835775, -+STORE, 47551509839872, 47551509848063, -+STORE, 47551509835776, 47551509839871, -+ERASE, 47551509835776, 47551509835776, -+STORE, 47551509835776, 47551509848063, -+ERASE, 47551509835776, 47551509835776, -+STORE, 47551509835776, 47551509839871, -+STORE, 47551509839872, 47551509848063, -+ERASE, 47551509839872, 47551509839872, -+STORE, 47551509839872, 47551509848063, -+STORE, 47551509848064, 47551509856255, -+ERASE, 47551509651456, 47551509651456, -+STORE, 47551509651456, 47551509667839, -+STORE, 47551509667840, 47551509676031, -+ERASE, 47551509839872, 47551509839872, -+STORE, 47551509839872, 47551509843967, -+STORE, 47551509843968, 47551509848063, -+ERASE, 47551509803008, 47551509803008, -+STORE, 47551509803008, 47551509807103, -+STORE, 47551509807104, 47551509811199, -+ERASE, 47551507632128, 47551507632128, -+STORE, 47551507632128, 47551507828735, -+STORE, 47551507828736, 47551507836927, -+ERASE, 47551504793600, 47551504793600, -+STORE, 47551504793600, 47551504797695, -+STORE, 47551504797696, 47551504801791, -+ERASE, 94150123233280, 94150123233280, -+STORE, 94150123233280, 94150123241471, -+STORE, 94150123241472, 94150123245567, -+ERASE, 140081290534912, 140081290534912, -+STORE, 140081290534912, 140081290539007, -+STORE, 140081290539008, 140081290543103, -+ERASE, 47551504621568, 47551504621568, -+STORE, 94150148112384, 94150148247551, -+STORE, 140737488347136, 140737488351231, -+STORE, 140734389334016, 140737488351231, -+ERASE, 140734389334016, 140734389334016, -+STORE, 140734389334016, 140734389338111, -+STORE, 94844636606464, 94844636778495, -+ERASE, 94844636606464, 94844636606464, -+STORE, 94844636606464, 94844636622847, -+STORE, 94844636622848, 94844636778495, -+ERASE, 94844636622848, 94844636622848, -+STORE, 94844636622848, 94844636725247, -+STORE, 94844636725248, 94844636766207, -+STORE, 94844636766208, 94844636778495, -+STORE, 139922765217792, 139922765389823, -+ERASE, 139922765217792, 139922765217792, -+STORE, 139922765217792, 139922765221887, -+STORE, 139922765221888, 139922765389823, -+ERASE, 139922765221888, 139922765221888, -+STORE, 139922765221888, 139922765344767, -+STORE, 139922765344768, 139922765377535, -+STORE, 139922765377536, 139922765385727, -+STORE, 139922765385728, 139922765389823, -+STORE, 140734389678080, 140734389682175, -+STORE, 140734389665792, 140734389678079, -+STORE, 47710029778944, 47710029787135, -+STORE, 47710029787136, 47710029795327, -+STORE, 47710029795328, 47710029959167, -+ERASE, 47710029795328, 47710029795328, -+STORE, 47710029795328, 47710029807615, -+STORE, 47710029807616, 47710029959167, -+STORE, 47710029905920, 47710029959167, -+STORE, 47710029807616, 47710029905919, -+ERASE, 47710029807616, 47710029807616, -+STORE, 47710029807616, 47710029905919, -+STORE, 47710029950976, 47710029959167, -+STORE, 47710029905920, 47710029950975, -+ERASE, 47710029905920, 47710029905920, -+STORE, 47710029905920, 47710029959167, -+ERASE, 47710029905920, 47710029905920, -+STORE, 47710029905920, 47710029950975, -+STORE, 47710029950976, 47710029959167, -+ERASE, 47710029950976, 47710029950976, -+STORE, 47710029950976, 47710029959167, -+STORE, 47710029959168, 47710033010687, -+STORE, 47710030503936, 47710033010687, -+STORE, 47710029959168, 47710030503935, -+ERASE, 47710030503936, 47710030503936, -+STORE, 47710030503936, 47710032789503, -+STORE, 47710032789504, 47710033010687, -+STORE, 47710032199680, 47710032789503, -+STORE, 47710030503936, 47710032199679, -+ERASE, 47710030503936, 47710030503936, -+STORE, 47710030503936, 47710032199679, -+STORE, 47710032785408, 47710032789503, -+STORE, 47710032199680, 47710032785407, -+ERASE, 47710032199680, 47710032199680, -+STORE, 47710032199680, 47710032785407, -+STORE, 47710032994304, 47710033010687, -+STORE, 47710032789504, 47710032994303, -+ERASE, 47710032789504, 47710032789504, -+STORE, 47710032789504, 47710032994303, -+ERASE, 47710032994304, 47710032994304, -+STORE, 47710032994304, 47710033010687, -+STORE, 47710033010688, 47710034849791, -+STORE, 47710033149952, 47710034849791, -+STORE, 47710033010688, 47710033149951, -+ERASE, 47710033149952, 47710033149952, -+STORE, 47710033149952, 47710034808831, -+STORE, 47710034808832, 47710034849791, -+STORE, 47710034493440, 47710034808831, -+STORE, 47710033149952, 47710034493439, -+ERASE, 47710033149952, 47710033149952, -+STORE, 47710033149952, 47710034493439, -+STORE, 47710034804736, 47710034808831, -+STORE, 47710034493440, 47710034804735, -+ERASE, 47710034493440, 47710034493440, -+STORE, 47710034493440, 47710034804735, -+STORE, 47710034833408, 47710034849791, -+STORE, 47710034808832, 47710034833407, -+ERASE, 47710034808832, 47710034808832, -+STORE, 47710034808832, 47710034833407, -+ERASE, 47710034833408, 47710034833408, -+STORE, 47710034833408, 47710034849791, -+STORE, 47710034849792, 47710034984959, -+ERASE, 47710034849792, 47710034849792, -+STORE, 47710034849792, 47710034874367, -+STORE, 47710034874368, 47710034984959, -+STORE, 47710034935808, 47710034984959, -+STORE, 47710034874368, 47710034935807, -+ERASE, 47710034874368, 47710034874368, -+STORE, 47710034874368, 47710034935807, -+STORE, 47710034960384, 47710034984959, -+STORE, 47710034935808, 47710034960383, -+ERASE, 47710034935808, 47710034935808, -+STORE, 47710034935808, 47710034984959, -+ERASE, 47710034935808, 47710034935808, -+STORE, 47710034935808, 47710034960383, -+STORE, 47710034960384, 47710034984959, -+STORE, 47710034968576, 47710034984959, -+STORE, 47710034960384, 47710034968575, -+ERASE, 47710034960384, 47710034960384, -+STORE, 47710034960384, 47710034968575, -+ERASE, 47710034968576, 47710034968576, -+STORE, 47710034968576, 47710034984959, -+STORE, 47710034984960, 47710035005439, -+ERASE, 47710034984960, 47710034984960, -+STORE, 47710034984960, 47710034989055, -+STORE, 47710034989056, 47710035005439, -+STORE, 47710034993152, 47710035005439, -+STORE, 47710034989056, 47710034993151, -+ERASE, 47710034989056, 47710034989056, -+STORE, 47710034989056, 47710034993151, -+STORE, 47710034997248, 47710035005439, -+STORE, 47710034993152, 47710034997247, -+ERASE, 47710034993152, 47710034993152, -+STORE, 47710034993152, 47710035005439, -+ERASE, 47710034993152, 47710034993152, -+STORE, 47710034993152, 47710034997247, -+STORE, 47710034997248, 47710035005439, -+ERASE, 47710034997248, 47710034997248, -+STORE, 47710034997248, 47710035005439, -+STORE, 47710035005440, 47710035013631, -+ERASE, 47710034808832, 47710034808832, -+STORE, 47710034808832, 47710034825215, -+STORE, 47710034825216, 47710034833407, -+ERASE, 47710034997248, 47710034997248, -+STORE, 47710034997248, 47710035001343, -+STORE, 47710035001344, 47710035005439, -+ERASE, 47710034960384, 47710034960384, -+STORE, 47710034960384, 47710034964479, -+STORE, 47710034964480, 47710034968575, -+ERASE, 47710032789504, 47710032789504, -+STORE, 47710032789504, 47710032986111, -+STORE, 47710032986112, 47710032994303, -+ERASE, 47710029950976, 47710029950976, -+STORE, 47710029950976, 47710029955071, -+STORE, 47710029955072, 47710029959167, -+ERASE, 94844636766208, 94844636766208, -+STORE, 94844636766208, 94844636774399, -+STORE, 94844636774400, 94844636778495, -+ERASE, 139922765377536, 139922765377536, -+STORE, 139922765377536, 139922765381631, -+STORE, 139922765381632, 139922765385727, -+ERASE, 47710029778944, 47710029778944, -+STORE, 94844641775616, 94844641910783, -+STORE, 140737488347136, 140737488351231, -+STORE, 140732213886976, 140737488351231, -+ERASE, 140732213886976, 140732213886976, -+STORE, 140732213886976, 140732213891071, -+STORE, 94240508887040, 94240509059071, -+ERASE, 94240508887040, 94240508887040, -+STORE, 94240508887040, 94240508903423, -+STORE, 94240508903424, 94240509059071, -+ERASE, 94240508903424, 94240508903424, -+STORE, 94240508903424, 94240509005823, -+STORE, 94240509005824, 94240509046783, -+STORE, 94240509046784, 94240509059071, -+STORE, 140275106516992, 140275106689023, -+ERASE, 140275106516992, 140275106516992, -+STORE, 140275106516992, 140275106521087, -+STORE, 140275106521088, 140275106689023, -+ERASE, 140275106521088, 140275106521088, -+STORE, 140275106521088, 140275106643967, -+STORE, 140275106643968, 140275106676735, -+STORE, 140275106676736, 140275106684927, -+STORE, 140275106684928, 140275106689023, -+STORE, 140732213977088, 140732213981183, -+STORE, 140732213964800, 140732213977087, -+STORE, 47357688479744, 47357688487935, -+STORE, 47357688487936, 47357688496127, -+STORE, 47357688496128, 47357688659967, -+ERASE, 47357688496128, 47357688496128, -+STORE, 47357688496128, 47357688508415, -+STORE, 47357688508416, 47357688659967, -+STORE, 47357688606720, 47357688659967, -+STORE, 47357688508416, 47357688606719, -+ERASE, 47357688508416, 47357688508416, -+STORE, 47357688508416, 47357688606719, -+STORE, 47357688651776, 47357688659967, -+STORE, 47357688606720, 47357688651775, -+ERASE, 47357688606720, 47357688606720, -+STORE, 47357688606720, 47357688659967, -+ERASE, 47357688606720, 47357688606720, -+STORE, 47357688606720, 47357688651775, -+STORE, 47357688651776, 47357688659967, -+ERASE, 47357688651776, 47357688651776, -+STORE, 47357688651776, 47357688659967, -+STORE, 47357688659968, 47357691711487, -+STORE, 47357689204736, 47357691711487, -+STORE, 47357688659968, 47357689204735, -+ERASE, 47357689204736, 47357689204736, -+STORE, 47357689204736, 47357691490303, -+STORE, 47357691490304, 47357691711487, -+STORE, 47357690900480, 47357691490303, -+STORE, 47357689204736, 47357690900479, -+ERASE, 47357689204736, 47357689204736, -+STORE, 47357689204736, 47357690900479, -+STORE, 47357691486208, 47357691490303, -+STORE, 47357690900480, 47357691486207, -+ERASE, 47357690900480, 47357690900480, -+STORE, 47357690900480, 47357691486207, -+STORE, 47357691695104, 47357691711487, -+STORE, 47357691490304, 47357691695103, -+ERASE, 47357691490304, 47357691490304, -+STORE, 47357691490304, 47357691695103, -+ERASE, 47357691695104, 47357691695104, -+STORE, 47357691695104, 47357691711487, -+STORE, 47357691711488, 47357693550591, -+STORE, 47357691850752, 47357693550591, -+STORE, 47357691711488, 47357691850751, -+ERASE, 47357691850752, 47357691850752, -+STORE, 47357691850752, 47357693509631, -+STORE, 47357693509632, 47357693550591, -+STORE, 47357693194240, 47357693509631, -+STORE, 47357691850752, 47357693194239, -+ERASE, 47357691850752, 47357691850752, -+STORE, 47357691850752, 47357693194239, -+STORE, 47357693505536, 47357693509631, -+STORE, 47357693194240, 47357693505535, -+ERASE, 47357693194240, 47357693194240, -+STORE, 47357693194240, 47357693505535, -+STORE, 47357693534208, 47357693550591, -+STORE, 47357693509632, 47357693534207, -+ERASE, 47357693509632, 47357693509632, -+STORE, 47357693509632, 47357693534207, -+ERASE, 47357693534208, 47357693534208, -+STORE, 47357693534208, 47357693550591, -+STORE, 47357693550592, 47357693685759, -+ERASE, 47357693550592, 47357693550592, -+STORE, 47357693550592, 47357693575167, -+STORE, 47357693575168, 47357693685759, -+STORE, 47357693636608, 47357693685759, -+STORE, 47357693575168, 47357693636607, -+ERASE, 47357693575168, 47357693575168, -+STORE, 47357693575168, 47357693636607, -+STORE, 47357693661184, 47357693685759, -+STORE, 47357693636608, 47357693661183, -+ERASE, 47357693636608, 47357693636608, -+STORE, 47357693636608, 47357693685759, -+ERASE, 47357693636608, 47357693636608, -+STORE, 47357693636608, 47357693661183, -+STORE, 47357693661184, 47357693685759, -+STORE, 47357693669376, 47357693685759, -+STORE, 47357693661184, 47357693669375, -+ERASE, 47357693661184, 47357693661184, -+STORE, 47357693661184, 47357693669375, -+ERASE, 47357693669376, 47357693669376, -+STORE, 47357693669376, 47357693685759, -+STORE, 47357693685760, 47357693706239, -+ERASE, 47357693685760, 47357693685760, -+STORE, 47357693685760, 47357693689855, -+STORE, 47357693689856, 47357693706239, -+STORE, 47357693693952, 47357693706239, -+STORE, 47357693689856, 47357693693951, -+ERASE, 47357693689856, 47357693689856, -+STORE, 47357693689856, 47357693693951, -+STORE, 47357693698048, 47357693706239, -+STORE, 47357693693952, 47357693698047, -+ERASE, 47357693693952, 47357693693952, -+STORE, 47357693693952, 47357693706239, -+ERASE, 47357693693952, 47357693693952, -+STORE, 47357693693952, 47357693698047, -+STORE, 47357693698048, 47357693706239, -+ERASE, 47357693698048, 47357693698048, -+STORE, 47357693698048, 47357693706239, -+STORE, 47357693706240, 47357693714431, -+ERASE, 47357693509632, 47357693509632, -+STORE, 47357693509632, 47357693526015, -+STORE, 47357693526016, 47357693534207, -+ERASE, 47357693698048, 47357693698048, -+STORE, 47357693698048, 47357693702143, -+STORE, 47357693702144, 47357693706239, -+ERASE, 47357693661184, 47357693661184, -+STORE, 47357693661184, 47357693665279, -+STORE, 47357693665280, 47357693669375, -+ERASE, 47357691490304, 47357691490304, -+STORE, 47357691490304, 47357691686911, -+STORE, 47357691686912, 47357691695103, -+ERASE, 47357688651776, 47357688651776, -+STORE, 47357688651776, 47357688655871, -+STORE, 47357688655872, 47357688659967, -+ERASE, 94240509046784, 94240509046784, -+STORE, 94240509046784, 94240509054975, -+STORE, 94240509054976, 94240509059071, -+ERASE, 140275106676736, 140275106676736, -+STORE, 140275106676736, 140275106680831, -+STORE, 140275106680832, 140275106684927, -+ERASE, 47357688479744, 47357688479744, -+STORE, 94240518361088, 94240518496255, -+STORE, 140737488347136, 140737488351231, -+STORE, 140732688277504, 140737488351231, -+ERASE, 140732688277504, 140732688277504, -+STORE, 140732688277504, 140732688281599, -+STORE, 94629171351552, 94629172064255, -+ERASE, 94629171351552, 94629171351552, -+STORE, 94629171351552, 94629171400703, -+STORE, 94629171400704, 94629172064255, -+ERASE, 94629171400704, 94629171400704, -+STORE, 94629171400704, 94629171945471, -+STORE, 94629171945472, 94629172043775, -+STORE, 94629172043776, 94629172064255, -+STORE, 139770707644416, 139770707816447, -+ERASE, 139770707644416, 139770707644416, -+STORE, 139770707644416, 139770707648511, -+STORE, 139770707648512, 139770707816447, -+ERASE, 139770707648512, 139770707648512, -+STORE, 139770707648512, 139770707771391, -+STORE, 139770707771392, 139770707804159, -+STORE, 139770707804160, 139770707812351, -+STORE, 139770707812352, 139770707816447, -+STORE, 140732689121280, 140732689125375, -+STORE, 140732689108992, 140732689121279, -+STORE, 47862087352320, 47862087360511, -+STORE, 47862087360512, 47862087368703, -+STORE, 47862087368704, 47862087475199, -+STORE, 47862087385088, 47862087475199, -+STORE, 47862087368704, 47862087385087, -+ERASE, 47862087385088, 47862087385088, -+STORE, 47862087385088, 47862087458815, -+STORE, 47862087458816, 47862087475199, -+STORE, 47862087438336, 47862087458815, -+STORE, 47862087385088, 47862087438335, -+ERASE, 47862087385088, 47862087385088, -+STORE, 47862087385088, 47862087438335, -+STORE, 47862087454720, 47862087458815, -+STORE, 47862087438336, 47862087454719, -+ERASE, 47862087438336, 47862087438336, -+STORE, 47862087438336, 47862087454719, -+STORE, 47862087467008, 47862087475199, -+STORE, 47862087458816, 47862087467007, -+ERASE, 47862087458816, 47862087458816, -+STORE, 47862087458816, 47862087467007, -+ERASE, 47862087467008, 47862087467008, -+STORE, 47862087467008, 47862087475199, -+STORE, 47862087475200, 47862089314303, -+STORE, 47862087614464, 47862089314303, -+STORE, 47862087475200, 47862087614463, -+ERASE, 47862087614464, 47862087614464, -+STORE, 47862087614464, 47862089273343, -+STORE, 47862089273344, 47862089314303, -+STORE, 47862088957952, 47862089273343, -+STORE, 47862087614464, 47862088957951, -+ERASE, 47862087614464, 47862087614464, -+STORE, 47862087614464, 47862088957951, -+STORE, 47862089269248, 47862089273343, -+STORE, 47862088957952, 47862089269247, -+ERASE, 47862088957952, 47862088957952, -+STORE, 47862088957952, 47862089269247, -+STORE, 47862089297920, 47862089314303, -+STORE, 47862089273344, 47862089297919, -+ERASE, 47862089273344, 47862089273344, -+STORE, 47862089273344, 47862089297919, -+ERASE, 47862089297920, 47862089297920, -+STORE, 47862089297920, 47862089314303, -+STORE, 47862089297920, 47862089326591, -+ERASE, 47862089273344, 47862089273344, -+STORE, 47862089273344, 47862089289727, -+STORE, 47862089289728, 47862089297919, -+ERASE, 47862087458816, 47862087458816, -+STORE, 47862087458816, 47862087462911, -+STORE, 47862087462912, 47862087467007, -+ERASE, 94629172043776, 94629172043776, -+STORE, 94629172043776, 94629172060159, -+STORE, 94629172060160, 94629172064255, -+ERASE, 139770707804160, 139770707804160, -+STORE, 139770707804160, 139770707808255, -+STORE, 139770707808256, 139770707812351, -+ERASE, 47862087352320, 47862087352320, -+STORE, 94629197533184, 94629197668351, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727540711424, 140737488351231, -+ERASE, 140727540711424, 140727540711424, -+STORE, 140727540711424, 140727540715519, -+STORE, 94299865313280, 94299866025983, -+ERASE, 94299865313280, 94299865313280, -+STORE, 94299865313280, 94299865362431, -+STORE, 94299865362432, 94299866025983, -+ERASE, 94299865362432, 94299865362432, -+STORE, 94299865362432, 94299865907199, -+STORE, 94299865907200, 94299866005503, -+STORE, 94299866005504, 94299866025983, -+STORE, 140680268763136, 140680268935167, -+ERASE, 140680268763136, 140680268763136, -+STORE, 140680268763136, 140680268767231, -+STORE, 140680268767232, 140680268935167, -+ERASE, 140680268767232, 140680268767232, -+STORE, 140680268767232, 140680268890111, -+STORE, 140680268890112, 140680268922879, -+STORE, 140680268922880, 140680268931071, -+STORE, 140680268931072, 140680268935167, -+STORE, 140727541424128, 140727541428223, -+STORE, 140727541411840, 140727541424127, -+STORE, 46952526233600, 46952526241791, -+STORE, 46952526241792, 46952526249983, -+STORE, 46952526249984, 46952526356479, -+STORE, 46952526266368, 46952526356479, -+STORE, 46952526249984, 46952526266367, -+ERASE, 46952526266368, 46952526266368, -+STORE, 46952526266368, 46952526340095, -+STORE, 46952526340096, 46952526356479, -+STORE, 46952526319616, 46952526340095, -+STORE, 46952526266368, 46952526319615, -+ERASE, 46952526266368, 46952526266368, -+STORE, 46952526266368, 46952526319615, -+STORE, 46952526336000, 46952526340095, -+STORE, 46952526319616, 46952526335999, -+ERASE, 46952526319616, 46952526319616, -+STORE, 46952526319616, 46952526335999, -+STORE, 46952526348288, 46952526356479, -+STORE, 46952526340096, 46952526348287, -+ERASE, 46952526340096, 46952526340096, -+STORE, 46952526340096, 46952526348287, -+ERASE, 46952526348288, 46952526348288, -+STORE, 46952526348288, 46952526356479, -+STORE, 46952526356480, 46952528195583, -+STORE, 46952526495744, 46952528195583, -+STORE, 46952526356480, 46952526495743, -+ERASE, 46952526495744, 46952526495744, -+STORE, 46952526495744, 46952528154623, -+STORE, 46952528154624, 46952528195583, -+STORE, 46952527839232, 46952528154623, -+STORE, 46952526495744, 46952527839231, -+ERASE, 46952526495744, 46952526495744, -+STORE, 46952526495744, 46952527839231, -+STORE, 46952528150528, 46952528154623, -+STORE, 46952527839232, 46952528150527, -+ERASE, 46952527839232, 46952527839232, -+STORE, 46952527839232, 46952528150527, -+STORE, 46952528179200, 46952528195583, -+STORE, 46952528154624, 46952528179199, -+ERASE, 46952528154624, 46952528154624, -+STORE, 46952528154624, 46952528179199, -+ERASE, 46952528179200, 46952528179200, -+STORE, 46952528179200, 46952528195583, -+STORE, 46952528179200, 46952528207871, -+ERASE, 46952528154624, 46952528154624, -+STORE, 46952528154624, 46952528171007, -+STORE, 46952528171008, 46952528179199, -+ERASE, 46952526340096, 46952526340096, -+STORE, 46952526340096, 46952526344191, -+STORE, 46952526344192, 46952526348287, -+ERASE, 94299866005504, 94299866005504, -+STORE, 94299866005504, 94299866021887, -+STORE, 94299866021888, 94299866025983, -+ERASE, 140680268922880, 140680268922880, -+STORE, 140680268922880, 140680268926975, -+STORE, 140680268926976, 140680268931071, -+ERASE, 46952526233600, 46952526233600, -+STORE, 140737488347136, 140737488351231, -+STORE, 140722874793984, 140737488351231, -+ERASE, 140722874793984, 140722874793984, -+STORE, 140722874793984, 140722874798079, -+STORE, 94448916213760, 94448916926463, -+ERASE, 94448916213760, 94448916213760, -+STORE, 94448916213760, 94448916262911, -+STORE, 94448916262912, 94448916926463, -+ERASE, 94448916262912, 94448916262912, -+STORE, 94448916262912, 94448916807679, -+STORE, 94448916807680, 94448916905983, -+STORE, 94448916905984, 94448916926463, -+STORE, 140389117046784, 140389117218815, -+ERASE, 140389117046784, 140389117046784, -+STORE, 140389117046784, 140389117050879, -+STORE, 140389117050880, 140389117218815, -+ERASE, 140389117050880, 140389117050880, -+STORE, 140389117050880, 140389117173759, -+STORE, 140389117173760, 140389117206527, -+STORE, 140389117206528, 140389117214719, -+STORE, 140389117214720, 140389117218815, -+STORE, 140722875297792, 140722875301887, -+STORE, 140722875285504, 140722875297791, -+STORE, 47243677949952, 47243677958143, -+STORE, 47243677958144, 47243677966335, -+STORE, 47243677966336, 47243678072831, -+STORE, 47243677982720, 47243678072831, -+STORE, 47243677966336, 47243677982719, -+ERASE, 47243677982720, 47243677982720, -+STORE, 47243677982720, 47243678056447, -+STORE, 47243678056448, 47243678072831, -+STORE, 47243678035968, 47243678056447, -+STORE, 47243677982720, 47243678035967, -+ERASE, 47243677982720, 47243677982720, -+STORE, 47243677982720, 47243678035967, -+STORE, 47243678052352, 47243678056447, -+STORE, 47243678035968, 47243678052351, -+ERASE, 47243678035968, 47243678035968, -+STORE, 47243678035968, 47243678052351, -+STORE, 47243678064640, 47243678072831, -+STORE, 47243678056448, 47243678064639, -+ERASE, 47243678056448, 47243678056448, -+STORE, 47243678056448, 47243678064639, -+ERASE, 47243678064640, 47243678064640, -+STORE, 47243678064640, 47243678072831, -+STORE, 47243678072832, 47243679911935, -+STORE, 47243678212096, 47243679911935, -+STORE, 47243678072832, 47243678212095, -+ERASE, 47243678212096, 47243678212096, -+STORE, 47243678212096, 47243679870975, -+STORE, 47243679870976, 47243679911935, -+STORE, 47243679555584, 47243679870975, -+STORE, 47243678212096, 47243679555583, -+ERASE, 47243678212096, 47243678212096, -+STORE, 47243678212096, 47243679555583, -+STORE, 47243679866880, 47243679870975, -+STORE, 47243679555584, 47243679866879, -+ERASE, 47243679555584, 47243679555584, -+STORE, 47243679555584, 47243679866879, -+STORE, 47243679895552, 47243679911935, -+STORE, 47243679870976, 47243679895551, -+ERASE, 47243679870976, 47243679870976, -+STORE, 47243679870976, 47243679895551, -+ERASE, 47243679895552, 47243679895552, -+STORE, 47243679895552, 47243679911935, -+STORE, 47243679895552, 47243679924223, -+ERASE, 47243679870976, 47243679870976, -+STORE, 47243679870976, 47243679887359, -+STORE, 47243679887360, 47243679895551, -+ERASE, 47243678056448, 47243678056448, -+STORE, 47243678056448, 47243678060543, -+STORE, 47243678060544, 47243678064639, -+ERASE, 94448916905984, 94448916905984, -+STORE, 94448916905984, 94448916922367, -+STORE, 94448916922368, 94448916926463, -+ERASE, 140389117206528, 140389117206528, -+STORE, 140389117206528, 140389117210623, -+STORE, 140389117210624, 140389117214719, -+ERASE, 47243677949952, 47243677949952, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733068505088, 140737488351231, -+ERASE, 140733068505088, 140733068505088, -+STORE, 140733068505088, 140733068509183, -+STORE, 94207145750528, 94207146463231, -+ERASE, 94207145750528, 94207145750528, -+STORE, 94207145750528, 94207145799679, -+STORE, 94207145799680, 94207146463231, -+ERASE, 94207145799680, 94207145799680, -+STORE, 94207145799680, 94207146344447, -+STORE, 94207146344448, 94207146442751, -+STORE, 94207146442752, 94207146463231, -+STORE, 140684504911872, 140684505083903, -+ERASE, 140684504911872, 140684504911872, -+STORE, 140684504911872, 140684504915967, -+STORE, 140684504915968, 140684505083903, -+ERASE, 140684504915968, 140684504915968, -+STORE, 140684504915968, 140684505038847, -+STORE, 140684505038848, 140684505071615, -+STORE, 140684505071616, 140684505079807, -+STORE, 140684505079808, 140684505083903, -+STORE, 140733068607488, 140733068611583, -+STORE, 140733068595200, 140733068607487, -+STORE, 46948290084864, 46948290093055, -+STORE, 46948290093056, 46948290101247, -+STORE, 46948290101248, 46948290207743, -+STORE, 46948290117632, 46948290207743, -+STORE, 46948290101248, 46948290117631, -+ERASE, 46948290117632, 46948290117632, -+STORE, 46948290117632, 46948290191359, -+STORE, 46948290191360, 46948290207743, -+STORE, 46948290170880, 46948290191359, -+STORE, 46948290117632, 46948290170879, -+ERASE, 46948290117632, 46948290117632, -+STORE, 46948290117632, 46948290170879, -+STORE, 46948290187264, 46948290191359, -+STORE, 46948290170880, 46948290187263, -+ERASE, 46948290170880, 46948290170880, -+STORE, 46948290170880, 46948290187263, -+STORE, 46948290199552, 46948290207743, -+STORE, 46948290191360, 46948290199551, -+ERASE, 46948290191360, 46948290191360, -+STORE, 46948290191360, 46948290199551, -+ERASE, 46948290199552, 46948290199552, -+STORE, 46948290199552, 46948290207743, -+STORE, 46948290207744, 46948292046847, -+STORE, 46948290347008, 46948292046847, -+STORE, 46948290207744, 46948290347007, -+ERASE, 46948290347008, 46948290347008, -+STORE, 46948290347008, 46948292005887, -+STORE, 46948292005888, 46948292046847, -+STORE, 46948291690496, 46948292005887, -+STORE, 46948290347008, 46948291690495, -+ERASE, 46948290347008, 46948290347008, -+STORE, 46948290347008, 46948291690495, -+STORE, 46948292001792, 46948292005887, -+STORE, 46948291690496, 46948292001791, -+ERASE, 46948291690496, 46948291690496, -+STORE, 46948291690496, 46948292001791, -+STORE, 46948292030464, 46948292046847, -+STORE, 46948292005888, 46948292030463, -+ERASE, 46948292005888, 46948292005888, -+STORE, 46948292005888, 46948292030463, -+ERASE, 46948292030464, 46948292030464, -+STORE, 46948292030464, 46948292046847, -+STORE, 46948292030464, 46948292059135, -+ERASE, 46948292005888, 46948292005888, -+STORE, 46948292005888, 46948292022271, -+STORE, 46948292022272, 46948292030463, -+ERASE, 46948290191360, 46948290191360, -+STORE, 46948290191360, 46948290195455, -+STORE, 46948290195456, 46948290199551, -+ERASE, 94207146442752, 94207146442752, -+STORE, 94207146442752, 94207146459135, -+STORE, 94207146459136, 94207146463231, -+ERASE, 140684505071616, 140684505071616, -+STORE, 140684505071616, 140684505075711, -+STORE, 140684505075712, 140684505079807, -+ERASE, 46948290084864, 46948290084864, -+STORE, 140737488347136, 140737488351231, -+STORE, 140726367158272, 140737488351231, -+ERASE, 140726367158272, 140726367158272, -+STORE, 140726367158272, 140726367162367, -+STORE, 94436124106752, 94436124819455, -+ERASE, 94436124106752, 94436124106752, -+STORE, 94436124106752, 94436124155903, -+STORE, 94436124155904, 94436124819455, -+ERASE, 94436124155904, 94436124155904, -+STORE, 94436124155904, 94436124700671, -+STORE, 94436124700672, 94436124798975, -+STORE, 94436124798976, 94436124819455, -+STORE, 140049025044480, 140049025216511, -+ERASE, 140049025044480, 140049025044480, -+STORE, 140049025044480, 140049025048575, -+STORE, 140049025048576, 140049025216511, -+ERASE, 140049025048576, 140049025048576, -+STORE, 140049025048576, 140049025171455, -+STORE, 140049025171456, 140049025204223, -+STORE, 140049025204224, 140049025212415, -+STORE, 140049025212416, 140049025216511, -+STORE, 140726367256576, 140726367260671, -+STORE, 140726367244288, 140726367256575, -+STORE, 47583769952256, 47583769960447, -+STORE, 47583769960448, 47583769968639, -+STORE, 47583769968640, 47583770075135, -+STORE, 47583769985024, 47583770075135, -+STORE, 47583769968640, 47583769985023, -+ERASE, 47583769985024, 47583769985024, -+STORE, 47583769985024, 47583770058751, -+STORE, 47583770058752, 47583770075135, -+STORE, 47583770038272, 47583770058751, -+STORE, 47583769985024, 47583770038271, -+ERASE, 47583769985024, 47583769985024, -+STORE, 47583769985024, 47583770038271, -+STORE, 47583770054656, 47583770058751, -+STORE, 47583770038272, 47583770054655, -+ERASE, 47583770038272, 47583770038272, -+STORE, 47583770038272, 47583770054655, -+STORE, 47583770066944, 47583770075135, -+STORE, 47583770058752, 47583770066943, -+ERASE, 47583770058752, 47583770058752, -+STORE, 47583770058752, 47583770066943, -+ERASE, 47583770066944, 47583770066944, -+STORE, 47583770066944, 47583770075135, -+STORE, 47583770075136, 47583771914239, -+STORE, 47583770214400, 47583771914239, -+STORE, 47583770075136, 47583770214399, -+ERASE, 47583770214400, 47583770214400, -+STORE, 47583770214400, 47583771873279, -+STORE, 47583771873280, 47583771914239, -+STORE, 47583771557888, 47583771873279, -+STORE, 47583770214400, 47583771557887, -+ERASE, 47583770214400, 47583770214400, -+STORE, 47583770214400, 47583771557887, -+STORE, 47583771869184, 47583771873279, -+STORE, 47583771557888, 47583771869183, -+ERASE, 47583771557888, 47583771557888, -+STORE, 47583771557888, 47583771869183, -+STORE, 47583771897856, 47583771914239, -+STORE, 47583771873280, 47583771897855, -+ERASE, 47583771873280, 47583771873280, -+STORE, 47583771873280, 47583771897855, -+ERASE, 47583771897856, 47583771897856, -+STORE, 47583771897856, 47583771914239, -+STORE, 47583771897856, 47583771926527, -+ERASE, 47583771873280, 47583771873280, -+STORE, 47583771873280, 47583771889663, -+STORE, 47583771889664, 47583771897855, -+ERASE, 47583770058752, 47583770058752, -+STORE, 47583770058752, 47583770062847, -+STORE, 47583770062848, 47583770066943, -+ERASE, 94436124798976, 94436124798976, -+STORE, 94436124798976, 94436124815359, -+STORE, 94436124815360, 94436124819455, -+ERASE, 140049025204224, 140049025204224, -+STORE, 140049025204224, 140049025208319, -+STORE, 140049025208320, 140049025212415, -+ERASE, 47583769952256, 47583769952256, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727116099584, 140737488351231, -+ERASE, 140727116099584, 140727116099584, -+STORE, 140727116099584, 140727116103679, -+STORE, 94166319734784, 94166320447487, -+ERASE, 94166319734784, 94166319734784, -+STORE, 94166319734784, 94166319783935, -+STORE, 94166319783936, 94166320447487, -+ERASE, 94166319783936, 94166319783936, -+STORE, 94166319783936, 94166320328703, -+STORE, 94166320328704, 94166320427007, -+STORE, 94166320427008, 94166320447487, -+STORE, 139976559542272, 139976559714303, -+ERASE, 139976559542272, 139976559542272, -+STORE, 139976559542272, 139976559546367, -+STORE, 139976559546368, 139976559714303, -+ERASE, 139976559546368, 139976559546368, -+STORE, 139976559546368, 139976559669247, -+STORE, 139976559669248, 139976559702015, -+STORE, 139976559702016, 139976559710207, -+STORE, 139976559710208, 139976559714303, -+STORE, 140727116222464, 140727116226559, -+STORE, 140727116210176, 140727116222463, -+STORE, 47656235454464, 47656235462655, -+STORE, 47656235462656, 47656235470847, -+STORE, 47656235470848, 47656235577343, -+STORE, 47656235487232, 47656235577343, -+STORE, 47656235470848, 47656235487231, -+ERASE, 47656235487232, 47656235487232, -+STORE, 47656235487232, 47656235560959, -+STORE, 47656235560960, 47656235577343, -+STORE, 47656235540480, 47656235560959, -+STORE, 47656235487232, 47656235540479, -+ERASE, 47656235487232, 47656235487232, -+STORE, 47656235487232, 47656235540479, -+STORE, 47656235556864, 47656235560959, -+STORE, 47656235540480, 47656235556863, -+ERASE, 47656235540480, 47656235540480, -+STORE, 47656235540480, 47656235556863, -+STORE, 47656235569152, 47656235577343, -+STORE, 47656235560960, 47656235569151, -+ERASE, 47656235560960, 47656235560960, -+STORE, 47656235560960, 47656235569151, -+ERASE, 47656235569152, 47656235569152, -+STORE, 47656235569152, 47656235577343, -+STORE, 47656235577344, 47656237416447, -+STORE, 47656235716608, 47656237416447, -+STORE, 47656235577344, 47656235716607, -+ERASE, 47656235716608, 47656235716608, -+STORE, 47656235716608, 47656237375487, -+STORE, 47656237375488, 47656237416447, -+STORE, 47656237060096, 47656237375487, -+STORE, 47656235716608, 47656237060095, -+ERASE, 47656235716608, 47656235716608, -+STORE, 47656235716608, 47656237060095, -+STORE, 47656237371392, 47656237375487, -+STORE, 47656237060096, 47656237371391, -+ERASE, 47656237060096, 47656237060096, -+STORE, 47656237060096, 47656237371391, -+STORE, 47656237400064, 47656237416447, -+STORE, 47656237375488, 47656237400063, -+ERASE, 47656237375488, 47656237375488, -+STORE, 47656237375488, 47656237400063, -+ERASE, 47656237400064, 47656237400064, -+STORE, 47656237400064, 47656237416447, -+STORE, 47656237400064, 47656237428735, -+ERASE, 47656237375488, 47656237375488, -+STORE, 47656237375488, 47656237391871, -+STORE, 47656237391872, 47656237400063, -+ERASE, 47656235560960, 47656235560960, -+STORE, 47656235560960, 47656235565055, -+STORE, 47656235565056, 47656235569151, -+ERASE, 94166320427008, 94166320427008, -+STORE, 94166320427008, 94166320443391, -+STORE, 94166320443392, 94166320447487, -+ERASE, 139976559702016, 139976559702016, -+STORE, 139976559702016, 139976559706111, -+STORE, 139976559706112, 139976559710207, -+ERASE, 47656235454464, 47656235454464, -+STORE, 94166332153856, 94166332289023, -+STORE, 140737488347136, 140737488351231, -+STORE, 140726412816384, 140737488351231, -+ERASE, 140726412816384, 140726412816384, -+STORE, 140726412816384, 140726412820479, -+STORE, 94094884507648, 94094885220351, -+ERASE, 94094884507648, 94094884507648, -+STORE, 94094884507648, 94094884556799, -+STORE, 94094884556800, 94094885220351, -+ERASE, 94094884556800, 94094884556800, -+STORE, 94094884556800, 94094885101567, -+STORE, 94094885101568, 94094885199871, -+STORE, 94094885199872, 94094885220351, -+STORE, 139773773938688, 139773774110719, -+ERASE, 139773773938688, 139773773938688, -+STORE, 139773773938688, 139773773942783, -+STORE, 139773773942784, 139773774110719, -+ERASE, 139773773942784, 139773773942784, -+STORE, 139773773942784, 139773774065663, -+STORE, 139773774065664, 139773774098431, -+STORE, 139773774098432, 139773774106623, -+STORE, 139773774106624, 139773774110719, -+STORE, 140726412963840, 140726412967935, -+STORE, 140726412951552, 140726412963839, -+STORE, 47859021058048, 47859021066239, -+STORE, 47859021066240, 47859021074431, -+STORE, 47859021074432, 47859021180927, -+STORE, 47859021090816, 47859021180927, -+STORE, 47859021074432, 47859021090815, -+ERASE, 47859021090816, 47859021090816, -+STORE, 47859021090816, 47859021164543, -+STORE, 47859021164544, 47859021180927, -+STORE, 47859021144064, 47859021164543, -+STORE, 47859021090816, 47859021144063, -+ERASE, 47859021090816, 47859021090816, -+STORE, 47859021090816, 47859021144063, -+STORE, 47859021160448, 47859021164543, -+STORE, 47859021144064, 47859021160447, -+ERASE, 47859021144064, 47859021144064, -+STORE, 47859021144064, 47859021160447, -+STORE, 47859021172736, 47859021180927, -+STORE, 47859021164544, 47859021172735, -+ERASE, 47859021164544, 47859021164544, -+STORE, 47859021164544, 47859021172735, -+ERASE, 47859021172736, 47859021172736, -+STORE, 47859021172736, 47859021180927, -+STORE, 47859021180928, 47859023020031, -+STORE, 47859021320192, 47859023020031, -+STORE, 47859021180928, 47859021320191, -+ERASE, 47859021320192, 47859021320192, -+STORE, 47859021320192, 47859022979071, -+STORE, 47859022979072, 47859023020031, -+STORE, 47859022663680, 47859022979071, -+STORE, 47859021320192, 47859022663679, -+ERASE, 47859021320192, 47859021320192, -+STORE, 47859021320192, 47859022663679, -+STORE, 47859022974976, 47859022979071, -+STORE, 47859022663680, 47859022974975, -+ERASE, 47859022663680, 47859022663680, -+STORE, 47859022663680, 47859022974975, -+STORE, 47859023003648, 47859023020031, -+STORE, 47859022979072, 47859023003647, -+ERASE, 47859022979072, 47859022979072, -+STORE, 47859022979072, 47859023003647, -+ERASE, 47859023003648, 47859023003648, -+STORE, 47859023003648, 47859023020031, -+STORE, 47859023003648, 47859023032319, -+ERASE, 47859022979072, 47859022979072, -+STORE, 47859022979072, 47859022995455, -+STORE, 47859022995456, 47859023003647, -+ERASE, 47859021164544, 47859021164544, -+STORE, 47859021164544, 47859021168639, -+STORE, 47859021168640, 47859021172735, -+ERASE, 94094885199872, 94094885199872, -+STORE, 94094885199872, 94094885216255, -+STORE, 94094885216256, 94094885220351, -+ERASE, 139773774098432, 139773774098432, -+STORE, 139773774098432, 139773774102527, -+STORE, 139773774102528, 139773774106623, -+ERASE, 47859021058048, 47859021058048, -+STORE, 94094901108736, 94094901243903, -+STORE, 140737488347136, 140737488351231, -+STORE, 140736567963648, 140737488351231, -+ERASE, 140736567963648, 140736567963648, -+STORE, 140736567963648, 140736567967743, -+STORE, 94924425748480, 94924426461183, -+ERASE, 94924425748480, 94924425748480, -+STORE, 94924425748480, 94924425797631, -+STORE, 94924425797632, 94924426461183, -+ERASE, 94924425797632, 94924425797632, -+STORE, 94924425797632, 94924426342399, -+STORE, 94924426342400, 94924426440703, -+STORE, 94924426440704, 94924426461183, -+STORE, 140042126319616, 140042126491647, -+ERASE, 140042126319616, 140042126319616, -+STORE, 140042126319616, 140042126323711, -+STORE, 140042126323712, 140042126491647, -+ERASE, 140042126323712, 140042126323712, -+STORE, 140042126323712, 140042126446591, -+STORE, 140042126446592, 140042126479359, -+STORE, 140042126479360, 140042126487551, -+STORE, 140042126487552, 140042126491647, -+STORE, 140736568672256, 140736568676351, -+STORE, 140736568659968, 140736568672255, -+STORE, 47590668677120, 47590668685311, -+STORE, 47590668685312, 47590668693503, -+STORE, 47590668693504, 47590668799999, -+STORE, 47590668709888, 47590668799999, -+STORE, 47590668693504, 47590668709887, -+ERASE, 47590668709888, 47590668709888, -+STORE, 47590668709888, 47590668783615, -+STORE, 47590668783616, 47590668799999, -+STORE, 47590668763136, 47590668783615, -+STORE, 47590668709888, 47590668763135, -+ERASE, 47590668709888, 47590668709888, -+STORE, 47590668709888, 47590668763135, -+STORE, 47590668779520, 47590668783615, -+STORE, 47590668763136, 47590668779519, -+ERASE, 47590668763136, 47590668763136, -+STORE, 47590668763136, 47590668779519, -+STORE, 47590668791808, 47590668799999, -+STORE, 47590668783616, 47590668791807, -+ERASE, 47590668783616, 47590668783616, -+STORE, 47590668783616, 47590668791807, -+ERASE, 47590668791808, 47590668791808, -+STORE, 47590668791808, 47590668799999, -+STORE, 47590668800000, 47590670639103, -+STORE, 47590668939264, 47590670639103, -+STORE, 47590668800000, 47590668939263, -+ERASE, 47590668939264, 47590668939264, -+STORE, 47590668939264, 47590670598143, -+STORE, 47590670598144, 47590670639103, -+STORE, 47590670282752, 47590670598143, -+STORE, 47590668939264, 47590670282751, -+ERASE, 47590668939264, 47590668939264, -+STORE, 47590668939264, 47590670282751, -+STORE, 47590670594048, 47590670598143, -+STORE, 47590670282752, 47590670594047, -+ERASE, 47590670282752, 47590670282752, -+STORE, 47590670282752, 47590670594047, -+STORE, 47590670622720, 47590670639103, -+STORE, 47590670598144, 47590670622719, -+ERASE, 47590670598144, 47590670598144, -+STORE, 47590670598144, 47590670622719, -+ERASE, 47590670622720, 47590670622720, -+STORE, 47590670622720, 47590670639103, -+STORE, 47590670622720, 47590670651391, -+ERASE, 47590670598144, 47590670598144, -+STORE, 47590670598144, 47590670614527, -+STORE, 47590670614528, 47590670622719, -+ERASE, 47590668783616, 47590668783616, -+STORE, 47590668783616, 47590668787711, -+STORE, 47590668787712, 47590668791807, -+ERASE, 94924426440704, 94924426440704, -+STORE, 94924426440704, 94924426457087, -+STORE, 94924426457088, 94924426461183, -+ERASE, 140042126479360, 140042126479360, -+STORE, 140042126479360, 140042126483455, -+STORE, 140042126483456, 140042126487551, -+ERASE, 47590668677120, 47590668677120, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733281439744, 140737488351231, -+ERASE, 140733281439744, 140733281439744, -+STORE, 140733281439744, 140733281443839, -+STORE, 94490667069440, 94490667782143, -+ERASE, 94490667069440, 94490667069440, -+STORE, 94490667069440, 94490667118591, -+STORE, 94490667118592, 94490667782143, -+ERASE, 94490667118592, 94490667118592, -+STORE, 94490667118592, 94490667663359, -+STORE, 94490667663360, 94490667761663, -+STORE, 94490667761664, 94490667782143, -+STORE, 139878215118848, 139878215290879, -+ERASE, 139878215118848, 139878215118848, -+STORE, 139878215118848, 139878215122943, -+STORE, 139878215122944, 139878215290879, -+ERASE, 139878215122944, 139878215122944, -+STORE, 139878215122944, 139878215245823, -+STORE, 139878215245824, 139878215278591, -+STORE, 139878215278592, 139878215286783, -+STORE, 139878215286784, 139878215290879, -+STORE, 140733281464320, 140733281468415, -+STORE, 140733281452032, 140733281464319, -+STORE, 47754579877888, 47754579886079, -+STORE, 47754579886080, 47754579894271, -+STORE, 47754579894272, 47754580000767, -+STORE, 47754579910656, 47754580000767, -+STORE, 47754579894272, 47754579910655, -+ERASE, 47754579910656, 47754579910656, -+STORE, 47754579910656, 47754579984383, -+STORE, 47754579984384, 47754580000767, -+STORE, 47754579963904, 47754579984383, -+STORE, 47754579910656, 47754579963903, -+ERASE, 47754579910656, 47754579910656, -+STORE, 47754579910656, 47754579963903, -+STORE, 47754579980288, 47754579984383, -+STORE, 47754579963904, 47754579980287, -+ERASE, 47754579963904, 47754579963904, -+STORE, 47754579963904, 47754579980287, -+STORE, 47754579992576, 47754580000767, -+STORE, 47754579984384, 47754579992575, -+ERASE, 47754579984384, 47754579984384, -+STORE, 47754579984384, 47754579992575, -+ERASE, 47754579992576, 47754579992576, -+STORE, 47754579992576, 47754580000767, -+STORE, 47754580000768, 47754581839871, -+STORE, 47754580140032, 47754581839871, -+STORE, 47754580000768, 47754580140031, -+ERASE, 47754580140032, 47754580140032, -+STORE, 47754580140032, 47754581798911, -+STORE, 47754581798912, 47754581839871, -+STORE, 47754581483520, 47754581798911, -+STORE, 47754580140032, 47754581483519, -+ERASE, 47754580140032, 47754580140032, -+STORE, 47754580140032, 47754581483519, -+STORE, 47754581794816, 47754581798911, -+STORE, 47754581483520, 47754581794815, -+ERASE, 47754581483520, 47754581483520, -+STORE, 47754581483520, 47754581794815, -+STORE, 47754581823488, 47754581839871, -+STORE, 47754581798912, 47754581823487, -+ERASE, 47754581798912, 47754581798912, -+STORE, 47754581798912, 47754581823487, -+ERASE, 47754581823488, 47754581823488, -+STORE, 47754581823488, 47754581839871, -+STORE, 47754581823488, 47754581852159, -+ERASE, 47754581798912, 47754581798912, -+STORE, 47754581798912, 47754581815295, -+STORE, 47754581815296, 47754581823487, -+ERASE, 47754579984384, 47754579984384, -+STORE, 47754579984384, 47754579988479, -+STORE, 47754579988480, 47754579992575, -+ERASE, 94490667761664, 94490667761664, -+STORE, 94490667761664, 94490667778047, -+STORE, 94490667778048, 94490667782143, -+ERASE, 139878215278592, 139878215278592, -+STORE, 139878215278592, 139878215282687, -+STORE, 139878215282688, 139878215286783, -+ERASE, 47754579877888, 47754579877888, -+STORE, 94490669649920, 94490669785087, -+STORE, 140737488347136, 140737488351231, -+STORE, 140735382188032, 140737488351231, -+ERASE, 140735382188032, 140735382188032, -+STORE, 140735382188032, 140735382192127, -+STORE, 94150181302272, 94150182014975, -+ERASE, 94150181302272, 94150181302272, -+STORE, 94150181302272, 94150181351423, -+STORE, 94150181351424, 94150182014975, -+ERASE, 94150181351424, 94150181351424, -+STORE, 94150181351424, 94150181896191, -+STORE, 94150181896192, 94150181994495, -+STORE, 94150181994496, 94150182014975, -+STORE, 139679752458240, 139679752630271, -+ERASE, 139679752458240, 139679752458240, -+STORE, 139679752458240, 139679752462335, -+STORE, 139679752462336, 139679752630271, -+ERASE, 139679752462336, 139679752462336, -+STORE, 139679752462336, 139679752585215, -+STORE, 139679752585216, 139679752617983, -+STORE, 139679752617984, 139679752626175, -+STORE, 139679752626176, 139679752630271, -+STORE, 140735382536192, 140735382540287, -+STORE, 140735382523904, 140735382536191, -+STORE, 47953042538496, 47953042546687, -+STORE, 47953042546688, 47953042554879, -+STORE, 47953042554880, 47953042661375, -+STORE, 47953042571264, 47953042661375, -+STORE, 47953042554880, 47953042571263, -+ERASE, 47953042571264, 47953042571264, -+STORE, 47953042571264, 47953042644991, -+STORE, 47953042644992, 47953042661375, -+STORE, 47953042624512, 47953042644991, -+STORE, 47953042571264, 47953042624511, -+ERASE, 47953042571264, 47953042571264, -+STORE, 47953042571264, 47953042624511, -+STORE, 47953042640896, 47953042644991, -+STORE, 47953042624512, 47953042640895, -+ERASE, 47953042624512, 47953042624512, -+STORE, 47953042624512, 47953042640895, -+STORE, 47953042653184, 47953042661375, -+STORE, 47953042644992, 47953042653183, -+ERASE, 47953042644992, 47953042644992, -+STORE, 47953042644992, 47953042653183, -+ERASE, 47953042653184, 47953042653184, -+STORE, 47953042653184, 47953042661375, -+STORE, 47953042661376, 47953044500479, -+STORE, 47953042800640, 47953044500479, -+STORE, 47953042661376, 47953042800639, -+ERASE, 47953042800640, 47953042800640, -+STORE, 47953042800640, 47953044459519, -+STORE, 47953044459520, 47953044500479, -+STORE, 47953044144128, 47953044459519, -+STORE, 47953042800640, 47953044144127, -+ERASE, 47953042800640, 47953042800640, -+STORE, 47953042800640, 47953044144127, -+STORE, 47953044455424, 47953044459519, -+STORE, 47953044144128, 47953044455423, -+ERASE, 47953044144128, 47953044144128, -+STORE, 47953044144128, 47953044455423, -+STORE, 47953044484096, 47953044500479, -+STORE, 47953044459520, 47953044484095, -+ERASE, 47953044459520, 47953044459520, -+STORE, 47953044459520, 47953044484095, -+ERASE, 47953044484096, 47953044484096, -+STORE, 47953044484096, 47953044500479, -+STORE, 47953044484096, 47953044512767, -+ERASE, 47953044459520, 47953044459520, -+STORE, 47953044459520, 47953044475903, -+STORE, 47953044475904, 47953044484095, -+ERASE, 47953042644992, 47953042644992, -+STORE, 47953042644992, 47953042649087, -+STORE, 47953042649088, 47953042653183, -+ERASE, 94150181994496, 94150181994496, -+STORE, 94150181994496, 94150182010879, -+STORE, 94150182010880, 94150182014975, -+ERASE, 139679752617984, 139679752617984, -+STORE, 139679752617984, 139679752622079, -+STORE, 139679752622080, 139679752626175, -+ERASE, 47953042538496, 47953042538496, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737044123648, 140737488351231, -+ERASE, 140737044123648, 140737044123648, -+STORE, 140737044123648, 140737044127743, -+STORE, 94425324294144, 94425325006847, -+ERASE, 94425324294144, 94425324294144, -+STORE, 94425324294144, 94425324343295, -+STORE, 94425324343296, 94425325006847, -+ERASE, 94425324343296, 94425324343296, -+STORE, 94425324343296, 94425324888063, -+STORE, 94425324888064, 94425324986367, -+STORE, 94425324986368, 94425325006847, -+STORE, 140382015016960, 140382015188991, -+ERASE, 140382015016960, 140382015016960, -+STORE, 140382015016960, 140382015021055, -+STORE, 140382015021056, 140382015188991, -+ERASE, 140382015021056, 140382015021056, -+STORE, 140382015021056, 140382015143935, -+STORE, 140382015143936, 140382015176703, -+STORE, 140382015176704, 140382015184895, -+STORE, 140382015184896, 140382015188991, -+STORE, 140737045585920, 140737045590015, -+STORE, 140737045573632, 140737045585919, -+STORE, 47250779979776, 47250779987967, -+STORE, 47250779987968, 47250779996159, -+STORE, 47250779996160, 47250780102655, -+STORE, 47250780012544, 47250780102655, -+STORE, 47250779996160, 47250780012543, -+ERASE, 47250780012544, 47250780012544, -+STORE, 47250780012544, 47250780086271, -+STORE, 47250780086272, 47250780102655, -+STORE, 47250780065792, 47250780086271, -+STORE, 47250780012544, 47250780065791, -+ERASE, 47250780012544, 47250780012544, -+STORE, 47250780012544, 47250780065791, -+STORE, 47250780082176, 47250780086271, -+STORE, 47250780065792, 47250780082175, -+ERASE, 47250780065792, 47250780065792, -+STORE, 47250780065792, 47250780082175, -+STORE, 47250780094464, 47250780102655, -+STORE, 47250780086272, 47250780094463, -+ERASE, 47250780086272, 47250780086272, -+STORE, 47250780086272, 47250780094463, -+ERASE, 47250780094464, 47250780094464, -+STORE, 47250780094464, 47250780102655, -+STORE, 47250780102656, 47250781941759, -+STORE, 47250780241920, 47250781941759, -+STORE, 47250780102656, 47250780241919, -+ERASE, 47250780241920, 47250780241920, -+STORE, 47250780241920, 47250781900799, -+STORE, 47250781900800, 47250781941759, -+STORE, 47250781585408, 47250781900799, -+STORE, 47250780241920, 47250781585407, -+ERASE, 47250780241920, 47250780241920, -+STORE, 47250780241920, 47250781585407, -+STORE, 47250781896704, 47250781900799, -+STORE, 47250781585408, 47250781896703, -+ERASE, 47250781585408, 47250781585408, -+STORE, 47250781585408, 47250781896703, -+STORE, 47250781925376, 47250781941759, -+STORE, 47250781900800, 47250781925375, -+ERASE, 47250781900800, 47250781900800, -+STORE, 47250781900800, 47250781925375, -+ERASE, 47250781925376, 47250781925376, -+STORE, 47250781925376, 47250781941759, -+STORE, 47250781925376, 47250781954047, -+ERASE, 47250781900800, 47250781900800, -+STORE, 47250781900800, 47250781917183, -+STORE, 47250781917184, 47250781925375, -+ERASE, 47250780086272, 47250780086272, -+STORE, 47250780086272, 47250780090367, -+STORE, 47250780090368, 47250780094463, -+ERASE, 94425324986368, 94425324986368, -+STORE, 94425324986368, 94425325002751, -+STORE, 94425325002752, 94425325006847, -+ERASE, 140382015176704, 140382015176704, -+STORE, 140382015176704, 140382015180799, -+STORE, 140382015180800, 140382015184895, -+ERASE, 47250779979776, 47250779979776, -+STORE, 94425351438336, 94425351573503, -+STORE, 140737488347136, 140737488351231, -+STORE, 140736801144832, 140737488351231, -+ERASE, 140736801144832, 140736801144832, -+STORE, 140736801144832, 140736801148927, -+STORE, 94629429358592, 94629430071295, -+ERASE, 94629429358592, 94629429358592, -+STORE, 94629429358592, 94629429407743, -+STORE, 94629429407744, 94629430071295, -+ERASE, 94629429407744, 94629429407744, -+STORE, 94629429407744, 94629429952511, -+STORE, 94629429952512, 94629430050815, -+STORE, 94629430050816, 94629430071295, -+STORE, 139801685483520, 139801685655551, -+ERASE, 139801685483520, 139801685483520, -+STORE, 139801685483520, 139801685487615, -+STORE, 139801685487616, 139801685655551, -+ERASE, 139801685487616, 139801685487616, -+STORE, 139801685487616, 139801685610495, -+STORE, 139801685610496, 139801685643263, -+STORE, 139801685643264, 139801685651455, -+STORE, 139801685651456, 139801685655551, -+STORE, 140736801198080, 140736801202175, -+STORE, 140736801185792, 140736801198079, -+STORE, 47831109513216, 47831109521407, -+STORE, 47831109521408, 47831109529599, -+STORE, 47831109529600, 47831109636095, -+STORE, 47831109545984, 47831109636095, -+STORE, 47831109529600, 47831109545983, -+ERASE, 47831109545984, 47831109545984, -+STORE, 47831109545984, 47831109619711, -+STORE, 47831109619712, 47831109636095, -+STORE, 47831109599232, 47831109619711, -+STORE, 47831109545984, 47831109599231, -+ERASE, 47831109545984, 47831109545984, -+STORE, 47831109545984, 47831109599231, -+STORE, 47831109615616, 47831109619711, -+STORE, 47831109599232, 47831109615615, -+ERASE, 47831109599232, 47831109599232, -+STORE, 47831109599232, 47831109615615, -+STORE, 47831109627904, 47831109636095, -+STORE, 47831109619712, 47831109627903, -+ERASE, 47831109619712, 47831109619712, -+STORE, 47831109619712, 47831109627903, -+ERASE, 47831109627904, 47831109627904, -+STORE, 47831109627904, 47831109636095, -+STORE, 47831109636096, 47831111475199, -+STORE, 47831109775360, 47831111475199, -+STORE, 47831109636096, 47831109775359, -+ERASE, 47831109775360, 47831109775360, -+STORE, 47831109775360, 47831111434239, -+STORE, 47831111434240, 47831111475199, -+STORE, 47831111118848, 47831111434239, -+STORE, 47831109775360, 47831111118847, -+ERASE, 47831109775360, 47831109775360, -+STORE, 47831109775360, 47831111118847, -+STORE, 47831111430144, 47831111434239, -+STORE, 47831111118848, 47831111430143, -+ERASE, 47831111118848, 47831111118848, -+STORE, 47831111118848, 47831111430143, -+STORE, 47831111458816, 47831111475199, -+STORE, 47831111434240, 47831111458815, -+ERASE, 47831111434240, 47831111434240, -+STORE, 47831111434240, 47831111458815, -+ERASE, 47831111458816, 47831111458816, -+STORE, 47831111458816, 47831111475199, -+STORE, 47831111458816, 47831111487487, -+ERASE, 47831111434240, 47831111434240, -+STORE, 47831111434240, 47831111450623, -+STORE, 47831111450624, 47831111458815, -+ERASE, 47831109619712, 47831109619712, -+STORE, 47831109619712, 47831109623807, -+STORE, 47831109623808, 47831109627903, -+ERASE, 94629430050816, 94629430050816, -+STORE, 94629430050816, 94629430067199, -+STORE, 94629430067200, 94629430071295, -+ERASE, 139801685643264, 139801685643264, -+STORE, 139801685643264, 139801685647359, -+STORE, 139801685647360, 139801685651455, -+ERASE, 47831109513216, 47831109513216, -+STORE, 140737488347136, 140737488351231, -+STORE, 140729419612160, 140737488351231, -+ERASE, 140729419612160, 140729419612160, -+STORE, 140729419612160, 140729419616255, -+STORE, 94443354148864, 94443354861567, -+ERASE, 94443354148864, 94443354148864, -+STORE, 94443354148864, 94443354198015, -+STORE, 94443354198016, 94443354861567, -+ERASE, 94443354198016, 94443354198016, -+STORE, 94443354198016, 94443354742783, -+STORE, 94443354742784, 94443354841087, -+STORE, 94443354841088, 94443354861567, -+STORE, 139741700038656, 139741700210687, -+ERASE, 139741700038656, 139741700038656, -+STORE, 139741700038656, 139741700042751, -+STORE, 139741700042752, 139741700210687, -+ERASE, 139741700042752, 139741700042752, -+STORE, 139741700042752, 139741700165631, -+STORE, 139741700165632, 139741700198399, -+STORE, 139741700198400, 139741700206591, -+STORE, 139741700206592, 139741700210687, -+STORE, 140729420574720, 140729420578815, -+STORE, 140729420562432, 140729420574719, -+STORE, 47891094958080, 47891094966271, -+STORE, 47891094966272, 47891094974463, -+STORE, 47891094974464, 47891095080959, -+STORE, 47891094990848, 47891095080959, -+STORE, 47891094974464, 47891094990847, -+ERASE, 47891094990848, 47891094990848, -+STORE, 47891094990848, 47891095064575, -+STORE, 47891095064576, 47891095080959, -+STORE, 47891095044096, 47891095064575, -+STORE, 47891094990848, 47891095044095, -+ERASE, 47891094990848, 47891094990848, -+STORE, 47891094990848, 47891095044095, -+STORE, 47891095060480, 47891095064575, -+STORE, 47891095044096, 47891095060479, -+ERASE, 47891095044096, 47891095044096, -+STORE, 47891095044096, 47891095060479, -+STORE, 47891095072768, 47891095080959, -+STORE, 47891095064576, 47891095072767, -+ERASE, 47891095064576, 47891095064576, -+STORE, 47891095064576, 47891095072767, -+ERASE, 47891095072768, 47891095072768, -+STORE, 47891095072768, 47891095080959, -+STORE, 47891095080960, 47891096920063, -+STORE, 47891095220224, 47891096920063, -+STORE, 47891095080960, 47891095220223, -+ERASE, 47891095220224, 47891095220224, -+STORE, 47891095220224, 47891096879103, -+STORE, 47891096879104, 47891096920063, -+STORE, 47891096563712, 47891096879103, -+STORE, 47891095220224, 47891096563711, -+ERASE, 47891095220224, 47891095220224, -+STORE, 47891095220224, 47891096563711, -+STORE, 47891096875008, 47891096879103, -+STORE, 47891096563712, 47891096875007, -+ERASE, 47891096563712, 47891096563712, -+STORE, 47891096563712, 47891096875007, -+STORE, 47891096903680, 47891096920063, -+STORE, 47891096879104, 47891096903679, -+ERASE, 47891096879104, 47891096879104, -+STORE, 47891096879104, 47891096903679, -+ERASE, 47891096903680, 47891096903680, -+STORE, 47891096903680, 47891096920063, -+STORE, 47891096903680, 47891096932351, -+ERASE, 47891096879104, 47891096879104, -+STORE, 47891096879104, 47891096895487, -+STORE, 47891096895488, 47891096903679, -+ERASE, 47891095064576, 47891095064576, -+STORE, 47891095064576, 47891095068671, -+STORE, 47891095068672, 47891095072767, -+ERASE, 94443354841088, 94443354841088, -+STORE, 94443354841088, 94443354857471, -+STORE, 94443354857472, 94443354861567, -+ERASE, 139741700198400, 139741700198400, -+STORE, 139741700198400, 139741700202495, -+STORE, 139741700202496, 139741700206591, -+ERASE, 47891094958080, 47891094958080, -+STORE, 94443360825344, 94443360960511, -+STORE, 140737488347136, 140737488351231, -+STORE, 140722961661952, 140737488351231, -+ERASE, 140722961661952, 140722961661952, -+STORE, 140722961661952, 140722961666047, -+STORE, 94878388944896, 94878389657599, -+ERASE, 94878388944896, 94878388944896, -+STORE, 94878388944896, 94878388994047, -+STORE, 94878388994048, 94878389657599, -+ERASE, 94878388994048, 94878388994048, -+STORE, 94878388994048, 94878389538815, -+STORE, 94878389538816, 94878389637119, -+STORE, 94878389637120, 94878389657599, -+STORE, 140210690056192, 140210690228223, -+ERASE, 140210690056192, 140210690056192, -+STORE, 140210690056192, 140210690060287, -+STORE, 140210690060288, 140210690228223, -+ERASE, 140210690060288, 140210690060288, -+STORE, 140210690060288, 140210690183167, -+STORE, 140210690183168, 140210690215935, -+STORE, 140210690215936, 140210690224127, -+STORE, 140210690224128, 140210690228223, -+STORE, 140722963148800, 140722963152895, -+STORE, 140722963136512, 140722963148799, -+STORE, 47422104940544, 47422104948735, -+STORE, 47422104948736, 47422104956927, -+STORE, 47422104956928, 47422105063423, -+STORE, 47422104973312, 47422105063423, -+STORE, 47422104956928, 47422104973311, -+ERASE, 47422104973312, 47422104973312, -+STORE, 47422104973312, 47422105047039, -+STORE, 47422105047040, 47422105063423, -+STORE, 47422105026560, 47422105047039, -+STORE, 47422104973312, 47422105026559, -+ERASE, 47422104973312, 47422104973312, -+STORE, 47422104973312, 47422105026559, -+STORE, 47422105042944, 47422105047039, -+STORE, 47422105026560, 47422105042943, -+ERASE, 47422105026560, 47422105026560, -+STORE, 47422105026560, 47422105042943, -+STORE, 47422105055232, 47422105063423, -+STORE, 47422105047040, 47422105055231, -+ERASE, 47422105047040, 47422105047040, -+STORE, 47422105047040, 47422105055231, -+ERASE, 47422105055232, 47422105055232, -+STORE, 47422105055232, 47422105063423, -+STORE, 47422105063424, 47422106902527, -+STORE, 47422105202688, 47422106902527, -+STORE, 47422105063424, 47422105202687, -+ERASE, 47422105202688, 47422105202688, -+STORE, 47422105202688, 47422106861567, -+STORE, 47422106861568, 47422106902527, -+STORE, 47422106546176, 47422106861567, -+STORE, 47422105202688, 47422106546175, -+ERASE, 47422105202688, 47422105202688, -+STORE, 47422105202688, 47422106546175, -+STORE, 47422106857472, 47422106861567, -+STORE, 47422106546176, 47422106857471, -+ERASE, 47422106546176, 47422106546176, -+STORE, 47422106546176, 47422106857471, -+STORE, 47422106886144, 47422106902527, -+STORE, 47422106861568, 47422106886143, -+ERASE, 47422106861568, 47422106861568, -+STORE, 47422106861568, 47422106886143, -+ERASE, 47422106886144, 47422106886144, -+STORE, 47422106886144, 47422106902527, -+STORE, 47422106886144, 47422106914815, -+ERASE, 47422106861568, 47422106861568, -+STORE, 47422106861568, 47422106877951, -+STORE, 47422106877952, 47422106886143, -+ERASE, 47422105047040, 47422105047040, -+STORE, 47422105047040, 47422105051135, -+STORE, 47422105051136, 47422105055231, -+ERASE, 94878389637120, 94878389637120, -+STORE, 94878389637120, 94878389653503, -+STORE, 94878389653504, 94878389657599, -+ERASE, 140210690215936, 140210690215936, -+STORE, 140210690215936, 140210690220031, -+STORE, 140210690220032, 140210690224127, -+ERASE, 47422104940544, 47422104940544, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727690309632, 140737488351231, -+ERASE, 140727690309632, 140727690309632, -+STORE, 140727690309632, 140727690313727, -+STORE, 94121892208640, 94121892921343, -+ERASE, 94121892208640, 94121892208640, -+STORE, 94121892208640, 94121892257791, -+STORE, 94121892257792, 94121892921343, -+ERASE, 94121892257792, 94121892257792, -+STORE, 94121892257792, 94121892802559, -+STORE, 94121892802560, 94121892900863, -+STORE, 94121892900864, 94121892921343, -+STORE, 140662438326272, 140662438498303, -+ERASE, 140662438326272, 140662438326272, -+STORE, 140662438326272, 140662438330367, -+STORE, 140662438330368, 140662438498303, -+ERASE, 140662438330368, 140662438330368, -+STORE, 140662438330368, 140662438453247, -+STORE, 140662438453248, 140662438486015, -+STORE, 140662438486016, 140662438494207, -+STORE, 140662438494208, 140662438498303, -+STORE, 140727690379264, 140727690383359, -+STORE, 140727690366976, 140727690379263, -+STORE, 46970356670464, 46970356678655, -+STORE, 46970356678656, 46970356686847, -+STORE, 46970356686848, 46970356793343, -+STORE, 46970356703232, 46970356793343, -+STORE, 46970356686848, 46970356703231, -+ERASE, 46970356703232, 46970356703232, -+STORE, 46970356703232, 46970356776959, -+STORE, 46970356776960, 46970356793343, -+STORE, 46970356756480, 46970356776959, -+STORE, 46970356703232, 46970356756479, -+ERASE, 46970356703232, 46970356703232, -+STORE, 46970356703232, 46970356756479, -+STORE, 46970356772864, 46970356776959, -+STORE, 46970356756480, 46970356772863, -+ERASE, 46970356756480, 46970356756480, -+STORE, 46970356756480, 46970356772863, -+STORE, 46970356785152, 46970356793343, -+STORE, 46970356776960, 46970356785151, -+ERASE, 46970356776960, 46970356776960, -+STORE, 46970356776960, 46970356785151, -+ERASE, 46970356785152, 46970356785152, -+STORE, 46970356785152, 46970356793343, -+STORE, 46970356793344, 46970358632447, -+STORE, 46970356932608, 46970358632447, -+STORE, 46970356793344, 46970356932607, -+ERASE, 46970356932608, 46970356932608, -+STORE, 46970356932608, 46970358591487, -+STORE, 46970358591488, 46970358632447, -+STORE, 46970358276096, 46970358591487, -+STORE, 46970356932608, 46970358276095, -+ERASE, 46970356932608, 46970356932608, -+STORE, 46970356932608, 46970358276095, -+STORE, 46970358587392, 46970358591487, -+STORE, 46970358276096, 46970358587391, -+ERASE, 46970358276096, 46970358276096, -+STORE, 46970358276096, 46970358587391, -+STORE, 46970358616064, 46970358632447, -+STORE, 46970358591488, 46970358616063, -+ERASE, 46970358591488, 46970358591488, -+STORE, 46970358591488, 46970358616063, -+ERASE, 46970358616064, 46970358616064, -+STORE, 46970358616064, 46970358632447, -+STORE, 46970358616064, 46970358644735, -+ERASE, 46970358591488, 46970358591488, -+STORE, 46970358591488, 46970358607871, -+STORE, 46970358607872, 46970358616063, -+ERASE, 46970356776960, 46970356776960, -+STORE, 46970356776960, 46970356781055, -+STORE, 46970356781056, 46970356785151, -+ERASE, 94121892900864, 94121892900864, -+STORE, 94121892900864, 94121892917247, -+STORE, 94121892917248, 94121892921343, -+ERASE, 140662438486016, 140662438486016, -+STORE, 140662438486016, 140662438490111, -+STORE, 140662438490112, 140662438494207, -+ERASE, 46970356670464, 46970356670464, -+STORE, 94121898610688, 94121898745855, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737189351424, 140737488351231, -+ERASE, 140737189351424, 140737189351424, -+STORE, 140737189351424, 140737189355519, -+STORE, 93847948832768, 93847949545471, -+ERASE, 93847948832768, 93847948832768, -+STORE, 93847948832768, 93847948881919, -+STORE, 93847948881920, 93847949545471, -+ERASE, 93847948881920, 93847948881920, -+STORE, 93847948881920, 93847949426687, -+STORE, 93847949426688, 93847949524991, -+STORE, 93847949524992, 93847949545471, -+STORE, 139698989985792, 139698990157823, -+ERASE, 139698989985792, 139698989985792, -+STORE, 139698989985792, 139698989989887, -+STORE, 139698989989888, 139698990157823, -+ERASE, 139698989989888, 139698989989888, -+STORE, 139698989989888, 139698990112767, -+STORE, 139698990112768, 139698990145535, -+STORE, 139698990145536, 139698990153727, -+STORE, 139698990153728, 139698990157823, -+STORE, 140737189744640, 140737189748735, -+STORE, 140737189732352, 140737189744639, -+STORE, 47933805010944, 47933805019135, -+STORE, 47933805019136, 47933805027327, -+STORE, 47933805027328, 47933805133823, -+STORE, 47933805043712, 47933805133823, -+STORE, 47933805027328, 47933805043711, -+ERASE, 47933805043712, 47933805043712, -+STORE, 47933805043712, 47933805117439, -+STORE, 47933805117440, 47933805133823, -+STORE, 47933805096960, 47933805117439, -+STORE, 47933805043712, 47933805096959, -+ERASE, 47933805043712, 47933805043712, -+STORE, 47933805043712, 47933805096959, -+STORE, 47933805113344, 47933805117439, -+STORE, 47933805096960, 47933805113343, -+ERASE, 47933805096960, 47933805096960, -+STORE, 47933805096960, 47933805113343, -+STORE, 47933805125632, 47933805133823, -+STORE, 47933805117440, 47933805125631, -+ERASE, 47933805117440, 47933805117440, -+STORE, 47933805117440, 47933805125631, -+ERASE, 47933805125632, 47933805125632, -+STORE, 47933805125632, 47933805133823, -+STORE, 47933805133824, 47933806972927, -+STORE, 47933805273088, 47933806972927, -+STORE, 47933805133824, 47933805273087, -+ERASE, 47933805273088, 47933805273088, -+STORE, 47933805273088, 47933806931967, -+STORE, 47933806931968, 47933806972927, -+STORE, 47933806616576, 47933806931967, -+STORE, 47933805273088, 47933806616575, -+ERASE, 47933805273088, 47933805273088, -+STORE, 47933805273088, 47933806616575, -+STORE, 47933806927872, 47933806931967, -+STORE, 47933806616576, 47933806927871, -+ERASE, 47933806616576, 47933806616576, -+STORE, 47933806616576, 47933806927871, -+STORE, 47933806956544, 47933806972927, -+STORE, 47933806931968, 47933806956543, -+ERASE, 47933806931968, 47933806931968, -+STORE, 47933806931968, 47933806956543, -+ERASE, 47933806956544, 47933806956544, -+STORE, 47933806956544, 47933806972927, -+STORE, 47933806956544, 47933806985215, -+ERASE, 47933806931968, 47933806931968, -+STORE, 47933806931968, 47933806948351, -+STORE, 47933806948352, 47933806956543, -+ERASE, 47933805117440, 47933805117440, -+STORE, 47933805117440, 47933805121535, -+STORE, 47933805121536, 47933805125631, -+ERASE, 93847949524992, 93847949524992, -+STORE, 93847949524992, 93847949541375, -+STORE, 93847949541376, 93847949545471, -+ERASE, 139698990145536, 139698990145536, -+STORE, 139698990145536, 139698990149631, -+STORE, 139698990149632, 139698990153727, -+ERASE, 47933805010944, 47933805010944, -+STORE, 140737488347136, 140737488351231, -+STORE, 140725553991680, 140737488351231, -+ERASE, 140725553991680, 140725553991680, -+STORE, 140725553991680, 140725553995775, -+STORE, 93980056248320, 93980056961023, -+ERASE, 93980056248320, 93980056248320, -+STORE, 93980056248320, 93980056297471, -+STORE, 93980056297472, 93980056961023, -+ERASE, 93980056297472, 93980056297472, -+STORE, 93980056297472, 93980056842239, -+STORE, 93980056842240, 93980056940543, -+STORE, 93980056940544, 93980056961023, -+STORE, 140146588971008, 140146589143039, -+ERASE, 140146588971008, 140146588971008, -+STORE, 140146588971008, 140146588975103, -+STORE, 140146588975104, 140146589143039, -+ERASE, 140146588975104, 140146588975104, -+STORE, 140146588975104, 140146589097983, -+STORE, 140146589097984, 140146589130751, -+STORE, 140146589130752, 140146589138943, -+STORE, 140146589138944, 140146589143039, -+STORE, 140725554860032, 140725554864127, -+STORE, 140725554847744, 140725554860031, -+STORE, 47486206025728, 47486206033919, -+STORE, 47486206033920, 47486206042111, -+STORE, 47486206042112, 47486206148607, -+STORE, 47486206058496, 47486206148607, -+STORE, 47486206042112, 47486206058495, -+ERASE, 47486206058496, 47486206058496, -+STORE, 47486206058496, 47486206132223, -+STORE, 47486206132224, 47486206148607, -+STORE, 47486206111744, 47486206132223, -+STORE, 47486206058496, 47486206111743, -+ERASE, 47486206058496, 47486206058496, -+STORE, 47486206058496, 47486206111743, -+STORE, 47486206128128, 47486206132223, -+STORE, 47486206111744, 47486206128127, -+ERASE, 47486206111744, 47486206111744, -+STORE, 47486206111744, 47486206128127, -+STORE, 47486206140416, 47486206148607, -+STORE, 47486206132224, 47486206140415, -+ERASE, 47486206132224, 47486206132224, -+STORE, 47486206132224, 47486206140415, -+ERASE, 47486206140416, 47486206140416, -+STORE, 47486206140416, 47486206148607, -+STORE, 47486206148608, 47486207987711, -+STORE, 47486206287872, 47486207987711, -+STORE, 47486206148608, 47486206287871, -+ERASE, 47486206287872, 47486206287872, -+STORE, 47486206287872, 47486207946751, -+STORE, 47486207946752, 47486207987711, -+STORE, 47486207631360, 47486207946751, -+STORE, 47486206287872, 47486207631359, -+ERASE, 47486206287872, 47486206287872, -+STORE, 47486206287872, 47486207631359, -+STORE, 47486207942656, 47486207946751, -+STORE, 47486207631360, 47486207942655, -+ERASE, 47486207631360, 47486207631360, -+STORE, 47486207631360, 47486207942655, -+STORE, 47486207971328, 47486207987711, -+STORE, 47486207946752, 47486207971327, -+ERASE, 47486207946752, 47486207946752, -+STORE, 47486207946752, 47486207971327, -+ERASE, 47486207971328, 47486207971328, -+STORE, 47486207971328, 47486207987711, -+STORE, 47486207971328, 47486207999999, -+ERASE, 47486207946752, 47486207946752, -+STORE, 47486207946752, 47486207963135, -+STORE, 47486207963136, 47486207971327, -+ERASE, 47486206132224, 47486206132224, -+STORE, 47486206132224, 47486206136319, -+STORE, 47486206136320, 47486206140415, -+ERASE, 93980056940544, 93980056940544, -+STORE, 93980056940544, 93980056956927, -+STORE, 93980056956928, 93980056961023, -+ERASE, 140146589130752, 140146589130752, -+STORE, 140146589130752, 140146589134847, -+STORE, 140146589134848, 140146589138943, -+ERASE, 47486206025728, 47486206025728, -+STORE, 93980070006784, 93980070141951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727334776832, 140737488351231, -+ERASE, 140727334776832, 140727334776832, -+STORE, 140727334776832, 140727334780927, -+STORE, 94049747247104, 94049747959807, -+ERASE, 94049747247104, 94049747247104, -+STORE, 94049747247104, 94049747296255, -+STORE, 94049747296256, 94049747959807, -+ERASE, 94049747296256, 94049747296256, -+STORE, 94049747296256, 94049747841023, -+STORE, 94049747841024, 94049747939327, -+STORE, 94049747939328, 94049747959807, -+STORE, 140227307216896, 140227307388927, -+ERASE, 140227307216896, 140227307216896, -+STORE, 140227307216896, 140227307220991, -+STORE, 140227307220992, 140227307388927, -+ERASE, 140227307220992, 140227307220992, -+STORE, 140227307220992, 140227307343871, -+STORE, 140227307343872, 140227307376639, -+STORE, 140227307376640, 140227307384831, -+STORE, 140227307384832, 140227307388927, -+STORE, 140727335337984, 140727335342079, -+STORE, 140727335325696, 140727335337983, -+STORE, 47405487779840, 47405487788031, -+STORE, 47405487788032, 47405487796223, -+STORE, 47405487796224, 47405487902719, -+STORE, 47405487812608, 47405487902719, -+STORE, 47405487796224, 47405487812607, -+ERASE, 47405487812608, 47405487812608, -+STORE, 47405487812608, 47405487886335, -+STORE, 47405487886336, 47405487902719, -+STORE, 47405487865856, 47405487886335, -+STORE, 47405487812608, 47405487865855, -+ERASE, 47405487812608, 47405487812608, -+STORE, 47405487812608, 47405487865855, -+STORE, 47405487882240, 47405487886335, -+STORE, 47405487865856, 47405487882239, -+ERASE, 47405487865856, 47405487865856, -+STORE, 47405487865856, 47405487882239, -+STORE, 47405487894528, 47405487902719, -+STORE, 47405487886336, 47405487894527, -+ERASE, 47405487886336, 47405487886336, -+STORE, 47405487886336, 47405487894527, -+ERASE, 47405487894528, 47405487894528, -+STORE, 47405487894528, 47405487902719, -+STORE, 47405487902720, 47405489741823, -+STORE, 47405488041984, 47405489741823, -+STORE, 47405487902720, 47405488041983, -+ERASE, 47405488041984, 47405488041984, -+STORE, 47405488041984, 47405489700863, -+STORE, 47405489700864, 47405489741823, -+STORE, 47405489385472, 47405489700863, -+STORE, 47405488041984, 47405489385471, -+ERASE, 47405488041984, 47405488041984, -+STORE, 47405488041984, 47405489385471, -+STORE, 47405489696768, 47405489700863, -+STORE, 47405489385472, 47405489696767, -+ERASE, 47405489385472, 47405489385472, -+STORE, 47405489385472, 47405489696767, -+STORE, 47405489725440, 47405489741823, -+STORE, 47405489700864, 47405489725439, -+ERASE, 47405489700864, 47405489700864, -+STORE, 47405489700864, 47405489725439, -+ERASE, 47405489725440, 47405489725440, -+STORE, 47405489725440, 47405489741823, -+STORE, 47405489725440, 47405489754111, -+ERASE, 47405489700864, 47405489700864, -+STORE, 47405489700864, 47405489717247, -+STORE, 47405489717248, 47405489725439, -+ERASE, 47405487886336, 47405487886336, -+STORE, 47405487886336, 47405487890431, -+STORE, 47405487890432, 47405487894527, -+ERASE, 94049747939328, 94049747939328, -+STORE, 94049747939328, 94049747955711, -+STORE, 94049747955712, 94049747959807, -+ERASE, 140227307376640, 140227307376640, -+STORE, 140227307376640, 140227307380735, -+STORE, 140227307380736, 140227307384831, -+ERASE, 47405487779840, 47405487779840, -+STORE, 94049758810112, 94049758945279, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727079718912, 140737488351231, -+ERASE, 140727079718912, 140727079718912, -+STORE, 140727079718912, 140727079723007, -+STORE, 94250996527104, 94250997239807, -+ERASE, 94250996527104, 94250996527104, -+STORE, 94250996527104, 94250996576255, -+STORE, 94250996576256, 94250997239807, -+ERASE, 94250996576256, 94250996576256, -+STORE, 94250996576256, 94250997121023, -+STORE, 94250997121024, 94250997219327, -+STORE, 94250997219328, 94250997239807, -+STORE, 140060022587392, 140060022759423, -+ERASE, 140060022587392, 140060022587392, -+STORE, 140060022587392, 140060022591487, -+STORE, 140060022591488, 140060022759423, -+ERASE, 140060022591488, 140060022591488, -+STORE, 140060022591488, 140060022714367, -+STORE, 140060022714368, 140060022747135, -+STORE, 140060022747136, 140060022755327, -+STORE, 140060022755328, 140060022759423, -+STORE, 140727079788544, 140727079792639, -+STORE, 140727079776256, 140727079788543, -+/* this next one caused issues when lowering the efficiency */ -+STORE, 47572772409344, 47572772417535, -+STORE, 47572772417536, 47572772425727, -+STORE, 47572772425728, 47572772532223, -+STORE, 47572772442112, 47572772532223, -+STORE, 47572772425728, 47572772442111, -+ERASE, 47572772442112, 47572772442112, -+STORE, 47572772442112, 47572772515839, -+STORE, 47572772515840, 47572772532223, -+STORE, 47572772495360, 47572772515839, -+STORE, 47572772442112, 47572772495359, -+ERASE, 47572772442112, 47572772442112, -+STORE, 47572772442112, 47572772495359, -+STORE, 47572772511744, 47572772515839, -+STORE, 47572772495360, 47572772511743, -+ERASE, 47572772495360, 47572772495360, -+STORE, 47572772495360, 47572772511743, -+STORE, 47572772524032, 47572772532223, -+STORE, 47572772515840, 47572772524031, -+ERASE, 47572772515840, 47572772515840, -+STORE, 47572772515840, 47572772524031, -+ERASE, 47572772524032, 47572772524032, -+STORE, 47572772524032, 47572772532223, -+STORE, 47572772532224, 47572774371327, -+STORE, 47572772671488, 47572774371327, -+STORE, 47572772532224, 47572772671487, -+ERASE, 47572772671488, 47572772671488, -+STORE, 47572772671488, 47572774330367, -+STORE, 47572774330368, 47572774371327, -+STORE, 47572774014976, 47572774330367, -+STORE, 47572772671488, 47572774014975, -+ERASE, 47572772671488, 47572772671488, -+STORE, 47572772671488, 47572774014975, -+STORE, 47572774326272, 47572774330367, -+STORE, 47572774014976, 47572774326271, -+ERASE, 47572774014976, 47572774014976, -+STORE, 47572774014976, 47572774326271, -+STORE, 47572774354944, 47572774371327, -+STORE, 47572774330368, 47572774354943, -+ERASE, 47572774330368, 47572774330368, -+STORE, 47572774330368, 47572774354943, -+ERASE, 47572774354944, 47572774354944, -+STORE, 47572774354944, 47572774371327, -+STORE, 47572774354944, 47572774383615, -+ERASE, 47572774330368, 47572774330368, -+STORE, 47572774330368, 47572774346751, -+STORE, 47572774346752, 47572774354943, -+ERASE, 47572772515840, 47572772515840, -+STORE, 47572772515840, 47572772519935, -+STORE, 47572772519936, 47572772524031, -+ERASE, 94250997219328, 94250997219328, -+STORE, 94250997219328, 94250997235711, -+STORE, 94250997235712, 94250997239807, -+ERASE, 140060022747136, 140060022747136, -+STORE, 140060022747136, 140060022751231, -+STORE, 140060022751232, 140060022755327, -+ERASE, 47572772409344, 47572772409344, -+STORE, 94251018305536, 94251018440703, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730012389376, 140737488351231, -+ERASE, 140730012389376, 140730012389376, -+STORE, 140730012389376, 140730012393471, -+STORE, 94382607675392, 94382607695871, -+ERASE, 94382607675392, 94382607675392, -+STORE, 94382607675392, 94382607679487, -+STORE, 94382607679488, 94382607695871, -+ERASE, 94382607679488, 94382607679488, -+STORE, 94382607679488, 94382607683583, -+STORE, 94382607683584, 94382607687679, -+STORE, 94382607687680, 94382607695871, -+STORE, 140252451454976, 140252451627007, -+ERASE, 140252451454976, 140252451454976, -+STORE, 140252451454976, 140252451459071, -+STORE, 140252451459072, 140252451627007, -+ERASE, 140252451459072, 140252451459072, -+STORE, 140252451459072, 140252451581951, -+STORE, 140252451581952, 140252451614719, -+STORE, 140252451614720, 140252451622911, -+STORE, 140252451622912, 140252451627007, -+STORE, 140730013548544, 140730013552639, -+STORE, 140730013536256, 140730013548543, -+STORE, 47380343541760, 47380343549951, -+STORE, 47380343549952, 47380343558143, -+STORE, 47380343558144, 47380345397247, -+STORE, 47380343697408, 47380345397247, -+STORE, 47380343558144, 47380343697407, -+ERASE, 47380343697408, 47380343697408, -+STORE, 47380343697408, 47380345356287, -+STORE, 47380345356288, 47380345397247, -+STORE, 47380345040896, 47380345356287, -+STORE, 47380343697408, 47380345040895, -+ERASE, 47380343697408, 47380343697408, -+STORE, 47380343697408, 47380345040895, -+STORE, 47380345352192, 47380345356287, -+STORE, 47380345040896, 47380345352191, -+ERASE, 47380345040896, 47380345040896, -+STORE, 47380345040896, 47380345352191, -+STORE, 47380345380864, 47380345397247, -+STORE, 47380345356288, 47380345380863, -+ERASE, 47380345356288, 47380345356288, -+STORE, 47380345356288, 47380345380863, -+ERASE, 47380345380864, 47380345380864, -+STORE, 47380345380864, 47380345397247, -+ERASE, 47380345356288, 47380345356288, -+STORE, 47380345356288, 47380345372671, -+STORE, 47380345372672, 47380345380863, -+ERASE, 94382607687680, 94382607687680, -+STORE, 94382607687680, 94382607691775, -+STORE, 94382607691776, 94382607695871, -+ERASE, 140252451614720, 140252451614720, -+STORE, 140252451614720, 140252451618815, -+STORE, 140252451618816, 140252451622911, -+ERASE, 47380343541760, 47380343541760, -+STORE, 94382626803712, 94382626938879, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730900271104, 140737488351231, -+ERASE, 140730900271104, 140730900271104, -+STORE, 140730900271104, 140730900275199, -+STORE, 93855478120448, 93855478337535, -+ERASE, 93855478120448, 93855478120448, -+STORE, 93855478120448, 93855478198271, -+STORE, 93855478198272, 93855478337535, -+ERASE, 93855478198272, 93855478198272, -+STORE, 93855478198272, 93855478243327, -+STORE, 93855478243328, 93855478288383, -+STORE, 93855478288384, 93855478337535, -+STORE, 140092686573568, 140092686745599, -+ERASE, 140092686573568, 140092686573568, -+STORE, 140092686573568, 140092686577663, -+STORE, 140092686577664, 140092686745599, -+ERASE, 140092686577664, 140092686577664, -+STORE, 140092686577664, 140092686700543, -+STORE, 140092686700544, 140092686733311, -+STORE, 140092686733312, 140092686741503, -+STORE, 140092686741504, 140092686745599, -+STORE, 140730900537344, 140730900541439, -+STORE, 140730900525056, 140730900537343, -+STORE, 47540108423168, 47540108431359, -+STORE, 47540108431360, 47540108439551, -+STORE, 47540108439552, 47540110278655, -+STORE, 47540108578816, 47540110278655, -+STORE, 47540108439552, 47540108578815, -+ERASE, 47540108578816, 47540108578816, -+STORE, 47540108578816, 47540110237695, -+STORE, 47540110237696, 47540110278655, -+STORE, 47540109922304, 47540110237695, -+STORE, 47540108578816, 47540109922303, -+ERASE, 47540108578816, 47540108578816, -+STORE, 47540108578816, 47540109922303, -+STORE, 47540110233600, 47540110237695, -+STORE, 47540109922304, 47540110233599, -+ERASE, 47540109922304, 47540109922304, -+STORE, 47540109922304, 47540110233599, -+STORE, 47540110262272, 47540110278655, -+STORE, 47540110237696, 47540110262271, -+ERASE, 47540110237696, 47540110237696, -+STORE, 47540110237696, 47540110262271, -+ERASE, 47540110262272, 47540110262272, -+STORE, 47540110262272, 47540110278655, -+ERASE, 47540110237696, 47540110237696, -+STORE, 47540110237696, 47540110254079, -+STORE, 47540110254080, 47540110262271, -+ERASE, 93855478288384, 93855478288384, -+STORE, 93855478288384, 93855478333439, -+STORE, 93855478333440, 93855478337535, -+ERASE, 140092686733312, 140092686733312, -+STORE, 140092686733312, 140092686737407, -+STORE, 140092686737408, 140092686741503, -+ERASE, 47540108423168, 47540108423168, -+STORE, 93855492222976, 93855492358143, -+STORE, 93855492222976, 93855492493311, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733498146816, 140737488351231, -+ERASE, 140733498146816, 140733498146816, -+STORE, 140733498146816, 140733498150911, -+STORE, 94170739654656, 94170740367359, -+ERASE, 94170739654656, 94170739654656, -+STORE, 94170739654656, 94170739703807, -+STORE, 94170739703808, 94170740367359, -+ERASE, 94170739703808, 94170739703808, -+STORE, 94170739703808, 94170740248575, -+STORE, 94170740248576, 94170740346879, -+STORE, 94170740346880, 94170740367359, -+STORE, 140024788877312, 140024789049343, -+ERASE, 140024788877312, 140024788877312, -+STORE, 140024788877312, 140024788881407, -+STORE, 140024788881408, 140024789049343, -+ERASE, 140024788881408, 140024788881408, -+STORE, 140024788881408, 140024789004287, -+STORE, 140024789004288, 140024789037055, -+STORE, 140024789037056, 140024789045247, -+STORE, 140024789045248, 140024789049343, -+STORE, 140733499023360, 140733499027455, -+STORE, 140733499011072, 140733499023359, -+STORE, 47608006119424, 47608006127615, -+STORE, 47608006127616, 47608006135807, -+STORE, 47608006135808, 47608006242303, -+STORE, 47608006152192, 47608006242303, -+STORE, 47608006135808, 47608006152191, -+ERASE, 47608006152192, 47608006152192, -+STORE, 47608006152192, 47608006225919, -+STORE, 47608006225920, 47608006242303, -+STORE, 47608006205440, 47608006225919, -+STORE, 47608006152192, 47608006205439, -+ERASE, 47608006152192, 47608006152192, -+STORE, 47608006152192, 47608006205439, -+STORE, 47608006221824, 47608006225919, -+STORE, 47608006205440, 47608006221823, -+ERASE, 47608006205440, 47608006205440, -+STORE, 47608006205440, 47608006221823, -+STORE, 47608006234112, 47608006242303, -+STORE, 47608006225920, 47608006234111, -+ERASE, 47608006225920, 47608006225920, -+STORE, 47608006225920, 47608006234111, -+ERASE, 47608006234112, 47608006234112, -+STORE, 47608006234112, 47608006242303, -+STORE, 47608006242304, 47608008081407, -+STORE, 47608006381568, 47608008081407, -+STORE, 47608006242304, 47608006381567, -+ERASE, 47608006381568, 47608006381568, -+STORE, 47608006381568, 47608008040447, -+STORE, 47608008040448, 47608008081407, -+STORE, 47608007725056, 47608008040447, -+STORE, 47608006381568, 47608007725055, -+ERASE, 47608006381568, 47608006381568, -+STORE, 47608006381568, 47608007725055, -+STORE, 47608008036352, 47608008040447, -+STORE, 47608007725056, 47608008036351, -+ERASE, 47608007725056, 47608007725056, -+STORE, 47608007725056, 47608008036351, -+STORE, 47608008065024, 47608008081407, -+STORE, 47608008040448, 47608008065023, -+ERASE, 47608008040448, 47608008040448, -+STORE, 47608008040448, 47608008065023, -+ERASE, 47608008065024, 47608008065024, -+STORE, 47608008065024, 47608008081407, -+STORE, 47608008065024, 47608008093695, -+ERASE, 47608008040448, 47608008040448, -+STORE, 47608008040448, 47608008056831, -+STORE, 47608008056832, 47608008065023, -+ERASE, 47608006225920, 47608006225920, -+STORE, 47608006225920, 47608006230015, -+STORE, 47608006230016, 47608006234111, -+ERASE, 94170740346880, 94170740346880, -+STORE, 94170740346880, 94170740363263, -+STORE, 94170740363264, 94170740367359, -+ERASE, 140024789037056, 140024789037056, -+STORE, 140024789037056, 140024789041151, -+STORE, 140024789041152, 140024789045247, -+ERASE, 47608006119424, 47608006119424, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730264326144, 140737488351231, -+ERASE, 140730264326144, 140730264326144, -+STORE, 140730264326144, 140730264330239, -+STORE, 94653216407552, 94653217120255, -+ERASE, 94653216407552, 94653216407552, -+STORE, 94653216407552, 94653216456703, -+STORE, 94653216456704, 94653217120255, -+ERASE, 94653216456704, 94653216456704, -+STORE, 94653216456704, 94653217001471, -+STORE, 94653217001472, 94653217099775, -+STORE, 94653217099776, 94653217120255, -+STORE, 140103617011712, 140103617183743, -+ERASE, 140103617011712, 140103617011712, -+STORE, 140103617011712, 140103617015807, -+STORE, 140103617015808, 140103617183743, -+ERASE, 140103617015808, 140103617015808, -+STORE, 140103617015808, 140103617138687, -+STORE, 140103617138688, 140103617171455, -+STORE, 140103617171456, 140103617179647, -+STORE, 140103617179648, 140103617183743, -+STORE, 140730265427968, 140730265432063, -+STORE, 140730265415680, 140730265427967, -+STORE, 47529177985024, 47529177993215, -+STORE, 47529177993216, 47529178001407, -+STORE, 47529178001408, 47529178107903, -+STORE, 47529178017792, 47529178107903, -+STORE, 47529178001408, 47529178017791, -+ERASE, 47529178017792, 47529178017792, -+STORE, 47529178017792, 47529178091519, -+STORE, 47529178091520, 47529178107903, -+STORE, 47529178071040, 47529178091519, -+STORE, 47529178017792, 47529178071039, -+ERASE, 47529178017792, 47529178017792, -+STORE, 47529178017792, 47529178071039, -+STORE, 47529178087424, 47529178091519, -+STORE, 47529178071040, 47529178087423, -+ERASE, 47529178071040, 47529178071040, -+STORE, 47529178071040, 47529178087423, -+STORE, 47529178099712, 47529178107903, -+STORE, 47529178091520, 47529178099711, -+ERASE, 47529178091520, 47529178091520, -+STORE, 47529178091520, 47529178099711, -+ERASE, 47529178099712, 47529178099712, -+STORE, 47529178099712, 47529178107903, -+STORE, 47529178107904, 47529179947007, -+STORE, 47529178247168, 47529179947007, -+STORE, 47529178107904, 47529178247167, -+ERASE, 47529178247168, 47529178247168, -+STORE, 47529178247168, 47529179906047, -+STORE, 47529179906048, 47529179947007, -+STORE, 47529179590656, 47529179906047, -+STORE, 47529178247168, 47529179590655, -+ERASE, 47529178247168, 47529178247168, -+STORE, 47529178247168, 47529179590655, -+STORE, 47529179901952, 47529179906047, -+STORE, 47529179590656, 47529179901951, -+ERASE, 47529179590656, 47529179590656, -+STORE, 47529179590656, 47529179901951, -+STORE, 47529179930624, 47529179947007, -+STORE, 47529179906048, 47529179930623, -+ERASE, 47529179906048, 47529179906048, -+STORE, 47529179906048, 47529179930623, -+ERASE, 47529179930624, 47529179930624, -+STORE, 47529179930624, 47529179947007, -+STORE, 47529179930624, 47529179959295, -+ERASE, 47529179906048, 47529179906048, -+STORE, 47529179906048, 47529179922431, -+STORE, 47529179922432, 47529179930623, -+ERASE, 47529178091520, 47529178091520, -+STORE, 47529178091520, 47529178095615, -+STORE, 47529178095616, 47529178099711, -+ERASE, 94653217099776, 94653217099776, -+STORE, 94653217099776, 94653217116159, -+STORE, 94653217116160, 94653217120255, -+ERASE, 140103617171456, 140103617171456, -+STORE, 140103617171456, 140103617175551, -+STORE, 140103617175552, 140103617179647, -+ERASE, 47529177985024, 47529177985024, -+STORE, 94653241135104, 94653241270271, -+STORE, 140737488347136, 140737488351231, -+STORE, 140736284549120, 140737488351231, -+ERASE, 140736284549120, 140736284549120, -+STORE, 140736284549120, 140736284553215, -+STORE, 93963663822848, 93963664506879, -+ERASE, 93963663822848, 93963663822848, -+STORE, 93963663822848, 93963663884287, -+STORE, 93963663884288, 93963664506879, -+ERASE, 93963663884288, 93963663884288, -+STORE, 93963663884288, 93963664240639, -+STORE, 93963664240640, 93963664379903, -+STORE, 93963664379904, 93963664506879, -+STORE, 140450188439552, 140450188611583, -+ERASE, 140450188439552, 140450188439552, -+STORE, 140450188439552, 140450188443647, -+STORE, 140450188443648, 140450188611583, -+ERASE, 140450188443648, 140450188443648, -+STORE, 140450188443648, 140450188566527, -+STORE, 140450188566528, 140450188599295, -+STORE, 140450188599296, 140450188607487, -+STORE, 140450188607488, 140450188611583, -+STORE, 140736284577792, 140736284581887, -+STORE, 140736284565504, 140736284577791, -+STORE, 47182606557184, 47182606565375, -+STORE, 47182606565376, 47182606573567, -+STORE, 47182606573568, 47182608412671, -+STORE, 47182606712832, 47182608412671, -+STORE, 47182606573568, 47182606712831, -+ERASE, 47182606712832, 47182606712832, -+STORE, 47182606712832, 47182608371711, -+STORE, 47182608371712, 47182608412671, -+STORE, 47182608056320, 47182608371711, -+STORE, 47182606712832, 47182608056319, -+ERASE, 47182606712832, 47182606712832, -+STORE, 47182606712832, 47182608056319, -+STORE, 47182608367616, 47182608371711, -+STORE, 47182608056320, 47182608367615, -+ERASE, 47182608056320, 47182608056320, -+STORE, 47182608056320, 47182608367615, -+STORE, 47182608396288, 47182608412671, -+STORE, 47182608371712, 47182608396287, -+ERASE, 47182608371712, 47182608371712, -+STORE, 47182608371712, 47182608396287, -+ERASE, 47182608396288, 47182608396288, -+STORE, 47182608396288, 47182608412671, -+STORE, 47182608412672, 47182608523263, -+STORE, 47182608429056, 47182608523263, -+STORE, 47182608412672, 47182608429055, -+ERASE, 47182608429056, 47182608429056, -+STORE, 47182608429056, 47182608515071, -+STORE, 47182608515072, 47182608523263, -+STORE, 47182608490496, 47182608515071, -+STORE, 47182608429056, 47182608490495, -+ERASE, 47182608429056, 47182608429056, -+STORE, 47182608429056, 47182608490495, -+STORE, 47182608510976, 47182608515071, -+STORE, 47182608490496, 47182608510975, -+ERASE, 47182608490496, 47182608490496, -+STORE, 47182608490496, 47182608510975, -+ERASE, 47182608515072, 47182608515072, -+STORE, 47182608515072, 47182608523263, -+STORE, 47182608523264, 47182608568319, -+ERASE, 47182608523264, 47182608523264, -+STORE, 47182608523264, 47182608531455, -+STORE, 47182608531456, 47182608568319, -+STORE, 47182608551936, 47182608568319, -+STORE, 47182608531456, 47182608551935, -+ERASE, 47182608531456, 47182608531456, -+STORE, 47182608531456, 47182608551935, -+STORE, 47182608560128, 47182608568319, -+STORE, 47182608551936, 47182608560127, -+ERASE, 47182608551936, 47182608551936, -+STORE, 47182608551936, 47182608568319, -+ERASE, 47182608551936, 47182608551936, -+STORE, 47182608551936, 47182608560127, -+STORE, 47182608560128, 47182608568319, -+ERASE, 47182608560128, 47182608560128, -+STORE, 47182608560128, 47182608568319, -+STORE, 47182608568320, 47182608916479, -+STORE, 47182608609280, 47182608916479, -+STORE, 47182608568320, 47182608609279, -+ERASE, 47182608609280, 47182608609280, -+STORE, 47182608609280, 47182608891903, -+STORE, 47182608891904, 47182608916479, -+STORE, 47182608822272, 47182608891903, -+STORE, 47182608609280, 47182608822271, -+ERASE, 47182608609280, 47182608609280, -+STORE, 47182608609280, 47182608822271, -+STORE, 47182608887808, 47182608891903, -+STORE, 47182608822272, 47182608887807, -+ERASE, 47182608822272, 47182608822272, -+STORE, 47182608822272, 47182608887807, -+ERASE, 47182608891904, 47182608891904, -+STORE, 47182608891904, 47182608916479, -+STORE, 47182608916480, 47182611177471, -+STORE, 47182609068032, 47182611177471, -+STORE, 47182608916480, 47182609068031, -+ERASE, 47182609068032, 47182609068032, -+STORE, 47182609068032, 47182611161087, -+STORE, 47182611161088, 47182611177471, -+STORE, 47182611169280, 47182611177471, -+STORE, 47182611161088, 47182611169279, -+ERASE, 47182611161088, 47182611161088, -+STORE, 47182611161088, 47182611169279, -+ERASE, 47182611169280, 47182611169280, -+STORE, 47182611169280, 47182611177471, -+STORE, 47182611177472, 47182611312639, -+ERASE, 47182611177472, 47182611177472, -+STORE, 47182611177472, 47182611202047, -+STORE, 47182611202048, 47182611312639, -+STORE, 47182611263488, 47182611312639, -+STORE, 47182611202048, 47182611263487, -+ERASE, 47182611202048, 47182611202048, -+STORE, 47182611202048, 47182611263487, -+STORE, 47182611288064, 47182611312639, -+STORE, 47182611263488, 47182611288063, -+ERASE, 47182611263488, 47182611263488, -+STORE, 47182611263488, 47182611312639, -+ERASE, 47182611263488, 47182611263488, -+STORE, 47182611263488, 47182611288063, -+STORE, 47182611288064, 47182611312639, -+STORE, 47182611296256, 47182611312639, -+STORE, 47182611288064, 47182611296255, -+ERASE, 47182611288064, 47182611288064, -+STORE, 47182611288064, 47182611296255, -+ERASE, 47182611296256, 47182611296256, -+STORE, 47182611296256, 47182611312639, -+STORE, 47182611296256, 47182611320831, -+STORE, 47182611320832, 47182611484671, -+ERASE, 47182611320832, 47182611320832, -+STORE, 47182611320832, 47182611333119, -+STORE, 47182611333120, 47182611484671, -+STORE, 47182611431424, 47182611484671, -+STORE, 47182611333120, 47182611431423, -+ERASE, 47182611333120, 47182611333120, -+STORE, 47182611333120, 47182611431423, -+STORE, 47182611476480, 47182611484671, -+STORE, 47182611431424, 47182611476479, -+ERASE, 47182611431424, 47182611431424, -+STORE, 47182611431424, 47182611484671, -+ERASE, 47182611431424, 47182611431424, -+STORE, 47182611431424, 47182611476479, -+STORE, 47182611476480, 47182611484671, -+ERASE, 47182611476480, 47182611476480, -+STORE, 47182611476480, 47182611484671, -+STORE, 47182611484672, 47182612082687, -+STORE, 47182611603456, 47182612082687, -+STORE, 47182611484672, 47182611603455, -+ERASE, 47182611603456, 47182611603456, -+STORE, 47182611603456, 47182612029439, -+STORE, 47182612029440, 47182612082687, -+STORE, 47182611918848, 47182612029439, -+STORE, 47182611603456, 47182611918847, -+ERASE, 47182611603456, 47182611603456, -+STORE, 47182611603456, 47182611918847, -+STORE, 47182612025344, 47182612029439, -+STORE, 47182611918848, 47182612025343, -+ERASE, 47182611918848, 47182611918848, -+STORE, 47182611918848, 47182612025343, -+ERASE, 47182612029440, 47182612029440, -+STORE, 47182612029440, 47182612082687, -+STORE, 47182612082688, 47182615134207, -+STORE, 47182612627456, 47182615134207, -+STORE, 47182612082688, 47182612627455, -+ERASE, 47182612627456, 47182612627456, -+STORE, 47182612627456, 47182614913023, -+STORE, 47182614913024, 47182615134207, -+STORE, 47182614323200, 47182614913023, -+STORE, 47182612627456, 47182614323199, -+ERASE, 47182612627456, 47182612627456, -+STORE, 47182612627456, 47182614323199, -+STORE, 47182614908928, 47182614913023, -+STORE, 47182614323200, 47182614908927, -+ERASE, 47182614323200, 47182614323200, -+STORE, 47182614323200, 47182614908927, -+STORE, 47182615117824, 47182615134207, -+STORE, 47182614913024, 47182615117823, -+ERASE, 47182614913024, 47182614913024, -+STORE, 47182614913024, 47182615117823, -+ERASE, 47182615117824, 47182615117824, -+STORE, 47182615117824, 47182615134207, -+STORE, 47182615134208, 47182615166975, -+ERASE, 47182615134208, 47182615134208, -+STORE, 47182615134208, 47182615142399, -+STORE, 47182615142400, 47182615166975, -+STORE, 47182615154688, 47182615166975, -+STORE, 47182615142400, 47182615154687, -+ERASE, 47182615142400, 47182615142400, -+STORE, 47182615142400, 47182615154687, -+STORE, 47182615158784, 47182615166975, -+STORE, 47182615154688, 47182615158783, -+ERASE, 47182615154688, 47182615154688, -+STORE, 47182615154688, 47182615166975, -+ERASE, 47182615154688, 47182615154688, -+STORE, 47182615154688, 47182615158783, -+STORE, 47182615158784, 47182615166975, -+ERASE, 47182615158784, 47182615158784, -+STORE, 47182615158784, 47182615166975, -+STORE, 47182615166976, 47182615203839, -+ERASE, 47182615166976, 47182615166976, -+STORE, 47182615166976, 47182615175167, -+STORE, 47182615175168, 47182615203839, -+STORE, 47182615191552, 47182615203839, -+STORE, 47182615175168, 47182615191551, -+ERASE, 47182615175168, 47182615175168, -+STORE, 47182615175168, 47182615191551, -+STORE, 47182615195648, 47182615203839, -+STORE, 47182615191552, 47182615195647, -+ERASE, 47182615191552, 47182615191552, -+STORE, 47182615191552, 47182615203839, -+ERASE, 47182615191552, 47182615191552, -+STORE, 47182615191552, 47182615195647, -+STORE, 47182615195648, 47182615203839, -+ERASE, 47182615195648, 47182615195648, -+STORE, 47182615195648, 47182615203839, -+STORE, 47182615203840, 47182615678975, -+ERASE, 47182615203840, 47182615203840, -+STORE, 47182615203840, 47182615212031, -+STORE, 47182615212032, 47182615678975, -+STORE, 47182615547904, 47182615678975, -+STORE, 47182615212032, 47182615547903, -+ERASE, 47182615212032, 47182615212032, -+STORE, 47182615212032, 47182615547903, -+STORE, 47182615670784, 47182615678975, -+STORE, 47182615547904, 47182615670783, -+ERASE, 47182615547904, 47182615547904, -+STORE, 47182615547904, 47182615678975, -+ERASE, 47182615547904, 47182615547904, -+STORE, 47182615547904, 47182615670783, -+STORE, 47182615670784, 47182615678975, -+ERASE, 47182615670784, 47182615670784, -+STORE, 47182615670784, 47182615678975, -+STORE, 47182615678976, 47182615687167, -+STORE, 47182615687168, 47182615707647, -+ERASE, 47182615687168, 47182615687168, -+STORE, 47182615687168, 47182615691263, -+STORE, 47182615691264, 47182615707647, -+STORE, 47182615695360, 47182615707647, -+STORE, 47182615691264, 47182615695359, -+ERASE, 47182615691264, 47182615691264, -+STORE, 47182615691264, 47182615695359, -+STORE, 47182615699456, 47182615707647, -+STORE, 47182615695360, 47182615699455, -+ERASE, 47182615695360, 47182615695360, -+STORE, 47182615695360, 47182615707647, -+ERASE, 47182615695360, 47182615695360, -+STORE, 47182615695360, 47182615699455, -+STORE, 47182615699456, 47182615707647, -+ERASE, 47182615699456, 47182615699456, -+STORE, 47182615699456, 47182615707647, -+STORE, 47182615707648, 47182615715839, -+ERASE, 47182608371712, 47182608371712, -+STORE, 47182608371712, 47182608388095, -+STORE, 47182608388096, 47182608396287, -+ERASE, 47182615699456, 47182615699456, -+STORE, 47182615699456, 47182615703551, -+STORE, 47182615703552, 47182615707647, -+ERASE, 47182611288064, 47182611288064, -+STORE, 47182611288064, 47182611292159, -+STORE, 47182611292160, 47182611296255, -+ERASE, 47182615670784, 47182615670784, -+STORE, 47182615670784, 47182615674879, -+STORE, 47182615674880, 47182615678975, -+ERASE, 47182615195648, 47182615195648, -+STORE, 47182615195648, 47182615199743, -+STORE, 47182615199744, 47182615203839, -+ERASE, 47182615158784, 47182615158784, -+STORE, 47182615158784, 47182615162879, -+STORE, 47182615162880, 47182615166975, -+ERASE, 47182614913024, 47182614913024, -+STORE, 47182614913024, 47182615109631, -+STORE, 47182615109632, 47182615117823, -+ERASE, 47182612029440, 47182612029440, -+STORE, 47182612029440, 47182612066303, -+STORE, 47182612066304, 47182612082687, -+ERASE, 47182611476480, 47182611476480, -+STORE, 47182611476480, 47182611480575, -+STORE, 47182611480576, 47182611484671, -+ERASE, 47182611161088, 47182611161088, -+STORE, 47182611161088, 47182611165183, -+STORE, 47182611165184, 47182611169279, -+ERASE, 47182608891904, 47182608891904, -+STORE, 47182608891904, 47182608912383, -+STORE, 47182608912384, 47182608916479, -+ERASE, 47182608560128, 47182608560128, -+STORE, 47182608560128, 47182608564223, -+STORE, 47182608564224, 47182608568319, -+ERASE, 47182608515072, 47182608515072, -+STORE, 47182608515072, 47182608519167, -+STORE, 47182608519168, 47182608523263, -+ERASE, 93963664379904, 93963664379904, -+STORE, 93963664379904, 93963664502783, -+STORE, 93963664502784, 93963664506879, -+ERASE, 140450188599296, 140450188599296, -+STORE, 140450188599296, 140450188603391, -+STORE, 140450188603392, 140450188607487, -+ERASE, 47182606557184, 47182606557184, -+STORE, 93963694723072, 93963694858239, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730313261056, 140737488351231, -+ERASE, 140730313261056, 140730313261056, -+STORE, 140730313261056, 140730313265151, -+STORE, 94386579017728, 94386579697663, -+ERASE, 94386579017728, 94386579017728, -+STORE, 94386579017728, 94386579083263, -+STORE, 94386579083264, 94386579697663, -+ERASE, 94386579083264, 94386579083264, -+STORE, 94386579083264, 94386579431423, -+STORE, 94386579431424, 94386579570687, -+STORE, 94386579570688, 94386579697663, -+STORE, 140124810838016, 140124811010047, -+ERASE, 140124810838016, 140124810838016, -+STORE, 140124810838016, 140124810842111, -+STORE, 140124810842112, 140124811010047, -+ERASE, 140124810842112, 140124810842112, -+STORE, 140124810842112, 140124810964991, -+STORE, 140124810964992, 140124810997759, -+STORE, 140124810997760, 140124811005951, -+STORE, 140124811005952, 140124811010047, -+STORE, 140730313601024, 140730313605119, -+STORE, 140730313588736, 140730313601023, -+STORE, 47507984158720, 47507984166911, -+STORE, 47507984166912, 47507984175103, -+STORE, 47507984175104, 47507986014207, -+STORE, 47507984314368, 47507986014207, -+STORE, 47507984175104, 47507984314367, -+ERASE, 47507984314368, 47507984314368, -+STORE, 47507984314368, 47507985973247, -+STORE, 47507985973248, 47507986014207, -+STORE, 47507985657856, 47507985973247, -+STORE, 47507984314368, 47507985657855, -+ERASE, 47507984314368, 47507984314368, -+STORE, 47507984314368, 47507985657855, -+STORE, 47507985969152, 47507985973247, -+STORE, 47507985657856, 47507985969151, -+ERASE, 47507985657856, 47507985657856, -+STORE, 47507985657856, 47507985969151, -+STORE, 47507985997824, 47507986014207, -+STORE, 47507985973248, 47507985997823, -+ERASE, 47507985973248, 47507985973248, -+STORE, 47507985973248, 47507985997823, -+ERASE, 47507985997824, 47507985997824, -+STORE, 47507985997824, 47507986014207, -+STORE, 47507986014208, 47507986124799, -+STORE, 47507986030592, 47507986124799, -+STORE, 47507986014208, 47507986030591, -+ERASE, 47507986030592, 47507986030592, -+STORE, 47507986030592, 47507986116607, -+STORE, 47507986116608, 47507986124799, -+STORE, 47507986092032, 47507986116607, -+STORE, 47507986030592, 47507986092031, -+ERASE, 47507986030592, 47507986030592, -+STORE, 47507986030592, 47507986092031, -+STORE, 47507986112512, 47507986116607, -+STORE, 47507986092032, 47507986112511, -+ERASE, 47507986092032, 47507986092032, -+STORE, 47507986092032, 47507986112511, -+ERASE, 47507986116608, 47507986116608, -+STORE, 47507986116608, 47507986124799, -+STORE, 47507986124800, 47507986169855, -+ERASE, 47507986124800, 47507986124800, -+STORE, 47507986124800, 47507986132991, -+STORE, 47507986132992, 47507986169855, -+STORE, 47507986153472, 47507986169855, -+STORE, 47507986132992, 47507986153471, -+ERASE, 47507986132992, 47507986132992, -+STORE, 47507986132992, 47507986153471, -+STORE, 47507986161664, 47507986169855, -+STORE, 47507986153472, 47507986161663, -+ERASE, 47507986153472, 47507986153472, -+STORE, 47507986153472, 47507986169855, -+ERASE, 47507986153472, 47507986153472, -+STORE, 47507986153472, 47507986161663, -+STORE, 47507986161664, 47507986169855, -+ERASE, 47507986161664, 47507986161664, -+STORE, 47507986161664, 47507986169855, -+STORE, 47507986169856, 47507986518015, -+STORE, 47507986210816, 47507986518015, -+STORE, 47507986169856, 47507986210815, -+ERASE, 47507986210816, 47507986210816, -+STORE, 47507986210816, 47507986493439, -+STORE, 47507986493440, 47507986518015, -+STORE, 47507986423808, 47507986493439, -+STORE, 47507986210816, 47507986423807, -+ERASE, 47507986210816, 47507986210816, -+STORE, 47507986210816, 47507986423807, -+STORE, 47507986489344, 47507986493439, -+STORE, 47507986423808, 47507986489343, -+ERASE, 47507986423808, 47507986423808, -+STORE, 47507986423808, 47507986489343, -+ERASE, 47507986493440, 47507986493440, -+STORE, 47507986493440, 47507986518015, -+STORE, 47507986518016, 47507988779007, -+STORE, 47507986669568, 47507988779007, -+STORE, 47507986518016, 47507986669567, -+ERASE, 47507986669568, 47507986669568, -+STORE, 47507986669568, 47507988762623, -+STORE, 47507988762624, 47507988779007, -+STORE, 47507988770816, 47507988779007, -+STORE, 47507988762624, 47507988770815, -+ERASE, 47507988762624, 47507988762624, -+STORE, 47507988762624, 47507988770815, -+ERASE, 47507988770816, 47507988770816, -+STORE, 47507988770816, 47507988779007, -+STORE, 47507988779008, 47507988914175, -+ERASE, 47507988779008, 47507988779008, -+STORE, 47507988779008, 47507988803583, -+STORE, 47507988803584, 47507988914175, -+STORE, 47507988865024, 47507988914175, -+STORE, 47507988803584, 47507988865023, -+ERASE, 47507988803584, 47507988803584, -+STORE, 47507988803584, 47507988865023, -+STORE, 47507988889600, 47507988914175, -+STORE, 47507988865024, 47507988889599, -+ERASE, 47507988865024, 47507988865024, -+STORE, 47507988865024, 47507988914175, -+ERASE, 47507988865024, 47507988865024, -+STORE, 47507988865024, 47507988889599, -+STORE, 47507988889600, 47507988914175, -+STORE, 47507988897792, 47507988914175, -+STORE, 47507988889600, 47507988897791, -+ERASE, 47507988889600, 47507988889600, -+STORE, 47507988889600, 47507988897791, -+ERASE, 47507988897792, 47507988897792, -+STORE, 47507988897792, 47507988914175, -+STORE, 47507988897792, 47507988922367, -+STORE, 47507988922368, 47507989086207, -+ERASE, 47507988922368, 47507988922368, -+STORE, 47507988922368, 47507988934655, -+STORE, 47507988934656, 47507989086207, -+STORE, 47507989032960, 47507989086207, -+STORE, 47507988934656, 47507989032959, -+ERASE, 47507988934656, 47507988934656, -+STORE, 47507988934656, 47507989032959, -+STORE, 47507989078016, 47507989086207, -+STORE, 47507989032960, 47507989078015, -+ERASE, 47507989032960, 47507989032960, -+STORE, 47507989032960, 47507989086207, -+ERASE, 47507989032960, 47507989032960, -+STORE, 47507989032960, 47507989078015, -+STORE, 47507989078016, 47507989086207, -+ERASE, 47507989078016, 47507989078016, -+STORE, 47507989078016, 47507989086207, -+STORE, 47507989086208, 47507989684223, -+STORE, 47507989204992, 47507989684223, -+STORE, 47507989086208, 47507989204991, -+ERASE, 47507989204992, 47507989204992, -+STORE, 47507989204992, 47507989630975, -+STORE, 47507989630976, 47507989684223, -+STORE, 47507989520384, 47507989630975, -+STORE, 47507989204992, 47507989520383, -+ERASE, 47507989204992, 47507989204992, -+STORE, 47507989204992, 47507989520383, -+STORE, 47507989626880, 47507989630975, -+STORE, 47507989520384, 47507989626879, -+ERASE, 47507989520384, 47507989520384, -+STORE, 47507989520384, 47507989626879, -+ERASE, 47507989630976, 47507989630976, -+STORE, 47507989630976, 47507989684223, -+STORE, 47507989684224, 47507992735743, -+STORE, 47507990228992, 47507992735743, -+STORE, 47507989684224, 47507990228991, -+ERASE, 47507990228992, 47507990228992, -+STORE, 47507990228992, 47507992514559, -+STORE, 47507992514560, 47507992735743, -+STORE, 47507991924736, 47507992514559, -+STORE, 47507990228992, 47507991924735, -+ERASE, 47507990228992, 47507990228992, -+STORE, 47507990228992, 47507991924735, -+STORE, 47507992510464, 47507992514559, -+STORE, 47507991924736, 47507992510463, -+ERASE, 47507991924736, 47507991924736, -+STORE, 47507991924736, 47507992510463, -+STORE, 47507992719360, 47507992735743, -+STORE, 47507992514560, 47507992719359, -+ERASE, 47507992514560, 47507992514560, -+STORE, 47507992514560, 47507992719359, -+ERASE, 47507992719360, 47507992719360, -+STORE, 47507992719360, 47507992735743, -+STORE, 47507992735744, 47507992768511, -+ERASE, 47507992735744, 47507992735744, -+STORE, 47507992735744, 47507992743935, -+STORE, 47507992743936, 47507992768511, -+STORE, 47507992756224, 47507992768511, -+STORE, 47507992743936, 47507992756223, -+ERASE, 47507992743936, 47507992743936, -+STORE, 47507992743936, 47507992756223, -+STORE, 47507992760320, 47507992768511, -+STORE, 47507992756224, 47507992760319, -+ERASE, 47507992756224, 47507992756224, -+STORE, 47507992756224, 47507992768511, -+ERASE, 47507992756224, 47507992756224, -+STORE, 47507992756224, 47507992760319, -+STORE, 47507992760320, 47507992768511, -+ERASE, 47507992760320, 47507992760320, -+STORE, 47507992760320, 47507992768511, -+STORE, 47507992768512, 47507992805375, -+ERASE, 47507992768512, 47507992768512, -+STORE, 47507992768512, 47507992776703, -+STORE, 47507992776704, 47507992805375, -+STORE, 47507992793088, 47507992805375, -+STORE, 47507992776704, 47507992793087, -+ERASE, 47507992776704, 47507992776704, -+STORE, 47507992776704, 47507992793087, -+STORE, 47507992797184, 47507992805375, -+STORE, 47507992793088, 47507992797183, -+ERASE, 47507992793088, 47507992793088, -+STORE, 47507992793088, 47507992805375, -+ERASE, 47507992793088, 47507992793088, -+STORE, 47507992793088, 47507992797183, -+STORE, 47507992797184, 47507992805375, -+ERASE, 47507992797184, 47507992797184, -+STORE, 47507992797184, 47507992805375, -+STORE, 47507992805376, 47507993280511, -+ERASE, 47507992805376, 47507992805376, -+STORE, 47507992805376, 47507992813567, -+STORE, 47507992813568, 47507993280511, -+STORE, 47507993149440, 47507993280511, -+STORE, 47507992813568, 47507993149439, -+ERASE, 47507992813568, 47507992813568, -+STORE, 47507992813568, 47507993149439, -+STORE, 47507993272320, 47507993280511, -+STORE, 47507993149440, 47507993272319, -+ERASE, 47507993149440, 47507993149440, -+STORE, 47507993149440, 47507993280511, -+ERASE, 47507993149440, 47507993149440, -+STORE, 47507993149440, 47507993272319, -+STORE, 47507993272320, 47507993280511, -+ERASE, 47507993272320, 47507993272320, -+STORE, 47507993272320, 47507993280511, -+STORE, 47507993280512, 47507993288703, -+STORE, 47507993288704, 47507993309183, -+ERASE, 47507993288704, 47507993288704, -+STORE, 47507993288704, 47507993292799, -+STORE, 47507993292800, 47507993309183, -+STORE, 47507993296896, 47507993309183, -+STORE, 47507993292800, 47507993296895, -+ERASE, 47507993292800, 47507993292800, -+STORE, 47507993292800, 47507993296895, -+STORE, 47507993300992, 47507993309183, -+STORE, 47507993296896, 47507993300991, -+ERASE, 47507993296896, 47507993296896, -+STORE, 47507993296896, 47507993309183, -+ERASE, 47507993296896, 47507993296896, -+STORE, 47507993296896, 47507993300991, -+STORE, 47507993300992, 47507993309183, -+ERASE, 47507993300992, 47507993300992, -+STORE, 47507993300992, 47507993309183, -+STORE, 47507993309184, 47507993317375, -+ERASE, 47507985973248, 47507985973248, -+STORE, 47507985973248, 47507985989631, -+STORE, 47507985989632, 47507985997823, -+ERASE, 47507993300992, 47507993300992, -+STORE, 47507993300992, 47507993305087, -+STORE, 47507993305088, 47507993309183, -+ERASE, 47507988889600, 47507988889600, -+STORE, 47507988889600, 47507988893695, -+STORE, 47507988893696, 47507988897791, -+ERASE, 47507993272320, 47507993272320, -+STORE, 47507993272320, 47507993276415, -+STORE, 47507993276416, 47507993280511, -+ERASE, 47507992797184, 47507992797184, -+STORE, 47507992797184, 47507992801279, -+STORE, 47507992801280, 47507992805375, -+ERASE, 47507992760320, 47507992760320, -+STORE, 47507992760320, 47507992764415, -+STORE, 47507992764416, 47507992768511, -+ERASE, 47507992514560, 47507992514560, -+STORE, 47507992514560, 47507992711167, -+STORE, 47507992711168, 47507992719359, -+ERASE, 47507989630976, 47507989630976, -+STORE, 47507989630976, 47507989667839, -+STORE, 47507989667840, 47507989684223, -+ERASE, 47507989078016, 47507989078016, -+STORE, 47507989078016, 47507989082111, -+STORE, 47507989082112, 47507989086207, -+ERASE, 47507988762624, 47507988762624, -+STORE, 47507988762624, 47507988766719, -+STORE, 47507988766720, 47507988770815, -+ERASE, 47507986493440, 47507986493440, -+STORE, 47507986493440, 47507986513919, -+STORE, 47507986513920, 47507986518015, -+ERASE, 47507986161664, 47507986161664, -+STORE, 47507986161664, 47507986165759, -+STORE, 47507986165760, 47507986169855, -+ERASE, 47507986116608, 47507986116608, -+STORE, 47507986116608, 47507986120703, -+STORE, 47507986120704, 47507986124799, -+ERASE, 94386579570688, 94386579570688, -+STORE, 94386579570688, 94386579693567, -+STORE, 94386579693568, 94386579697663, -+ERASE, 140124810997760, 140124810997760, -+STORE, 140124810997760, 140124811001855, -+STORE, 140124811001856, 140124811005951, -+ERASE, 47507984158720, 47507984158720, -+STORE, 94386583982080, 94386584117247, -+STORE, 94386583982080, 94386584256511, -+ERASE, 94386583982080, 94386583982080, -+STORE, 94386583982080, 94386584223743, -+STORE, 94386584223744, 94386584256511, -+ERASE, 94386584223744, 94386584223744, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733763395584, 140737488351231, -+ERASE, 140733763395584, 140733763395584, -+STORE, 140733763395584, 140733763399679, -+STORE, 94011546472448, 94011547152383, -+ERASE, 94011546472448, 94011546472448, -+STORE, 94011546472448, 94011546537983, -+STORE, 94011546537984, 94011547152383, -+ERASE, 94011546537984, 94011546537984, -+STORE, 94011546537984, 94011546886143, -+STORE, 94011546886144, 94011547025407, -+STORE, 94011547025408, 94011547152383, -+STORE, 139757597949952, 139757598121983, -+ERASE, 139757597949952, 139757597949952, -+STORE, 139757597949952, 139757597954047, -+STORE, 139757597954048, 139757598121983, -+ERASE, 139757597954048, 139757597954048, -+STORE, 139757597954048, 139757598076927, -+STORE, 139757598076928, 139757598109695, -+STORE, 139757598109696, 139757598117887, -+STORE, 139757598117888, 139757598121983, -+STORE, 140733763596288, 140733763600383, -+STORE, 140733763584000, 140733763596287, -+STORE, 47875197046784, 47875197054975, -+STORE, 47875197054976, 47875197063167, -+STORE, 47875197063168, 47875198902271, -+STORE, 47875197202432, 47875198902271, -+STORE, 47875197063168, 47875197202431, -+ERASE, 47875197202432, 47875197202432, -+STORE, 47875197202432, 47875198861311, -+STORE, 47875198861312, 47875198902271, -+STORE, 47875198545920, 47875198861311, -+STORE, 47875197202432, 47875198545919, -+ERASE, 47875197202432, 47875197202432, -+STORE, 47875197202432, 47875198545919, -+STORE, 47875198857216, 47875198861311, -+STORE, 47875198545920, 47875198857215, -+ERASE, 47875198545920, 47875198545920, -+STORE, 47875198545920, 47875198857215, -+STORE, 47875198885888, 47875198902271, -+STORE, 47875198861312, 47875198885887, -+ERASE, 47875198861312, 47875198861312, -+STORE, 47875198861312, 47875198885887, -+ERASE, 47875198885888, 47875198885888, -+STORE, 47875198885888, 47875198902271, -+STORE, 47875198902272, 47875199012863, -+STORE, 47875198918656, 47875199012863, -+STORE, 47875198902272, 47875198918655, -+ERASE, 47875198918656, 47875198918656, -+STORE, 47875198918656, 47875199004671, -+STORE, 47875199004672, 47875199012863, -+STORE, 47875198980096, 47875199004671, -+STORE, 47875198918656, 47875198980095, -+ERASE, 47875198918656, 47875198918656, -+STORE, 47875198918656, 47875198980095, -+STORE, 47875199000576, 47875199004671, -+STORE, 47875198980096, 47875199000575, -+ERASE, 47875198980096, 47875198980096, -+STORE, 47875198980096, 47875199000575, -+ERASE, 47875199004672, 47875199004672, -+STORE, 47875199004672, 47875199012863, -+STORE, 47875199012864, 47875199057919, -+ERASE, 47875199012864, 47875199012864, -+STORE, 47875199012864, 47875199021055, -+STORE, 47875199021056, 47875199057919, -+STORE, 47875199041536, 47875199057919, -+STORE, 47875199021056, 47875199041535, -+ERASE, 47875199021056, 47875199021056, -+STORE, 47875199021056, 47875199041535, -+STORE, 47875199049728, 47875199057919, -+STORE, 47875199041536, 47875199049727, -+ERASE, 47875199041536, 47875199041536, -+STORE, 47875199041536, 47875199057919, -+ERASE, 47875199041536, 47875199041536, -+STORE, 47875199041536, 47875199049727, -+STORE, 47875199049728, 47875199057919, -+ERASE, 47875199049728, 47875199049728, -+STORE, 47875199049728, 47875199057919, -+STORE, 47875199057920, 47875199406079, -+STORE, 47875199098880, 47875199406079, -+STORE, 47875199057920, 47875199098879, -+ERASE, 47875199098880, 47875199098880, -+STORE, 47875199098880, 47875199381503, -+STORE, 47875199381504, 47875199406079, -+STORE, 47875199311872, 47875199381503, -+STORE, 47875199098880, 47875199311871, -+ERASE, 47875199098880, 47875199098880, -+STORE, 47875199098880, 47875199311871, -+STORE, 47875199377408, 47875199381503, -+STORE, 47875199311872, 47875199377407, -+ERASE, 47875199311872, 47875199311872, -+STORE, 47875199311872, 47875199377407, -+ERASE, 47875199381504, 47875199381504, -+STORE, 47875199381504, 47875199406079, -+STORE, 47875199406080, 47875201667071, -+STORE, 47875199557632, 47875201667071, -+STORE, 47875199406080, 47875199557631, -+ERASE, 47875199557632, 47875199557632, -+STORE, 47875199557632, 47875201650687, -+STORE, 47875201650688, 47875201667071, -+STORE, 47875201658880, 47875201667071, -+STORE, 47875201650688, 47875201658879, -+ERASE, 47875201650688, 47875201650688, -+STORE, 47875201650688, 47875201658879, -+ERASE, 47875201658880, 47875201658880, -+STORE, 47875201658880, 47875201667071, -+STORE, 47875201667072, 47875201802239, -+ERASE, 47875201667072, 47875201667072, -+STORE, 47875201667072, 47875201691647, -+STORE, 47875201691648, 47875201802239, -+STORE, 47875201753088, 47875201802239, -+STORE, 47875201691648, 47875201753087, -+ERASE, 47875201691648, 47875201691648, -+STORE, 47875201691648, 47875201753087, -+STORE, 47875201777664, 47875201802239, -+STORE, 47875201753088, 47875201777663, -+ERASE, 47875201753088, 47875201753088, -+STORE, 47875201753088, 47875201802239, -+ERASE, 47875201753088, 47875201753088, -+STORE, 47875201753088, 47875201777663, -+STORE, 47875201777664, 47875201802239, -+STORE, 47875201785856, 47875201802239, -+STORE, 47875201777664, 47875201785855, -+ERASE, 47875201777664, 47875201777664, -+STORE, 47875201777664, 47875201785855, -+ERASE, 47875201785856, 47875201785856, -+STORE, 47875201785856, 47875201802239, -+STORE, 47875201785856, 47875201810431, -+STORE, 47875201810432, 47875201974271, -+ERASE, 47875201810432, 47875201810432, -+STORE, 47875201810432, 47875201822719, -+STORE, 47875201822720, 47875201974271, -+STORE, 47875201921024, 47875201974271, -+STORE, 47875201822720, 47875201921023, -+ERASE, 47875201822720, 47875201822720, -+STORE, 47875201822720, 47875201921023, -+STORE, 47875201966080, 47875201974271, -+STORE, 47875201921024, 47875201966079, -+ERASE, 47875201921024, 47875201921024, -+STORE, 47875201921024, 47875201974271, -+ERASE, 47875201921024, 47875201921024, -+STORE, 47875201921024, 47875201966079, -+STORE, 47875201966080, 47875201974271, -+ERASE, 47875201966080, 47875201966080, -+STORE, 47875201966080, 47875201974271, -+STORE, 47875201974272, 47875202572287, -+STORE, 47875202093056, 47875202572287, -+STORE, 47875201974272, 47875202093055, -+ERASE, 47875202093056, 47875202093056, -+STORE, 47875202093056, 47875202519039, -+STORE, 47875202519040, 47875202572287, -+STORE, 47875202408448, 47875202519039, -+STORE, 47875202093056, 47875202408447, -+ERASE, 47875202093056, 47875202093056, -+STORE, 47875202093056, 47875202408447, -+STORE, 47875202514944, 47875202519039, -+STORE, 47875202408448, 47875202514943, -+ERASE, 47875202408448, 47875202408448, -+STORE, 47875202408448, 47875202514943, -+ERASE, 47875202519040, 47875202519040, -+STORE, 47875202519040, 47875202572287, -+STORE, 47875202572288, 47875205623807, -+STORE, 47875203117056, 47875205623807, -+STORE, 47875202572288, 47875203117055, -+ERASE, 47875203117056, 47875203117056, -+STORE, 47875203117056, 47875205402623, -+STORE, 47875205402624, 47875205623807, -+STORE, 47875204812800, 47875205402623, -+STORE, 47875203117056, 47875204812799, -+ERASE, 47875203117056, 47875203117056, -+STORE, 47875203117056, 47875204812799, -+STORE, 47875205398528, 47875205402623, -+STORE, 47875204812800, 47875205398527, -+ERASE, 47875204812800, 47875204812800, -+STORE, 47875204812800, 47875205398527, -+STORE, 47875205607424, 47875205623807, -+STORE, 47875205402624, 47875205607423, -+ERASE, 47875205402624, 47875205402624, -+STORE, 47875205402624, 47875205607423, -+ERASE, 47875205607424, 47875205607424, -+STORE, 47875205607424, 47875205623807, -+STORE, 47875205623808, 47875205656575, -+ERASE, 47875205623808, 47875205623808, -+STORE, 47875205623808, 47875205631999, -+STORE, 47875205632000, 47875205656575, -+STORE, 47875205644288, 47875205656575, -+STORE, 47875205632000, 47875205644287, -+ERASE, 47875205632000, 47875205632000, -+STORE, 47875205632000, 47875205644287, -+STORE, 47875205648384, 47875205656575, -+STORE, 47875205644288, 47875205648383, -+ERASE, 47875205644288, 47875205644288, -+STORE, 47875205644288, 47875205656575, -+ERASE, 47875205644288, 47875205644288, -+STORE, 47875205644288, 47875205648383, -+STORE, 47875205648384, 47875205656575, -+ERASE, 47875205648384, 47875205648384, -+STORE, 47875205648384, 47875205656575, -+STORE, 47875205656576, 47875205693439, -+ERASE, 47875205656576, 47875205656576, -+STORE, 47875205656576, 47875205664767, -+STORE, 47875205664768, 47875205693439, -+STORE, 47875205681152, 47875205693439, -+STORE, 47875205664768, 47875205681151, -+ERASE, 47875205664768, 47875205664768, -+STORE, 47875205664768, 47875205681151, -+STORE, 47875205685248, 47875205693439, -+STORE, 47875205681152, 47875205685247, -+ERASE, 47875205681152, 47875205681152, -+STORE, 47875205681152, 47875205693439, -+ERASE, 47875205681152, 47875205681152, -+STORE, 47875205681152, 47875205685247, -+STORE, 47875205685248, 47875205693439, -+ERASE, 47875205685248, 47875205685248, -+STORE, 47875205685248, 47875205693439, -+STORE, 47875205693440, 47875206168575, -+ERASE, 47875205693440, 47875205693440, -+STORE, 47875205693440, 47875205701631, -+STORE, 47875205701632, 47875206168575, -+STORE, 47875206037504, 47875206168575, -+STORE, 47875205701632, 47875206037503, -+ERASE, 47875205701632, 47875205701632, -+STORE, 47875205701632, 47875206037503, -+STORE, 47875206160384, 47875206168575, -+STORE, 47875206037504, 47875206160383, -+ERASE, 47875206037504, 47875206037504, -+STORE, 47875206037504, 47875206168575, -+ERASE, 47875206037504, 47875206037504, -+STORE, 47875206037504, 47875206160383, -+STORE, 47875206160384, 47875206168575, -+ERASE, 47875206160384, 47875206160384, -+STORE, 47875206160384, 47875206168575, -+STORE, 47875206168576, 47875206176767, -+STORE, 47875206176768, 47875206197247, -+ERASE, 47875206176768, 47875206176768, -+STORE, 47875206176768, 47875206180863, -+STORE, 47875206180864, 47875206197247, -+STORE, 47875206184960, 47875206197247, -+STORE, 47875206180864, 47875206184959, -+ERASE, 47875206180864, 47875206180864, -+STORE, 47875206180864, 47875206184959, -+STORE, 47875206189056, 47875206197247, -+STORE, 47875206184960, 47875206189055, -+ERASE, 47875206184960, 47875206184960, -+STORE, 47875206184960, 47875206197247, -+ERASE, 47875206184960, 47875206184960, -+STORE, 47875206184960, 47875206189055, -+STORE, 47875206189056, 47875206197247, -+ERASE, 47875206189056, 47875206189056, -+STORE, 47875206189056, 47875206197247, -+STORE, 47875206197248, 47875206205439, -+ERASE, 47875198861312, 47875198861312, -+STORE, 47875198861312, 47875198877695, -+STORE, 47875198877696, 47875198885887, -+ERASE, 47875206189056, 47875206189056, -+STORE, 47875206189056, 47875206193151, -+STORE, 47875206193152, 47875206197247, -+ERASE, 47875201777664, 47875201777664, -+STORE, 47875201777664, 47875201781759, -+STORE, 47875201781760, 47875201785855, -+ERASE, 47875206160384, 47875206160384, -+STORE, 47875206160384, 47875206164479, -+STORE, 47875206164480, 47875206168575, -+ERASE, 47875205685248, 47875205685248, -+STORE, 47875205685248, 47875205689343, -+STORE, 47875205689344, 47875205693439, -+ERASE, 47875205648384, 47875205648384, -+STORE, 47875205648384, 47875205652479, -+STORE, 47875205652480, 47875205656575, -+ERASE, 47875205402624, 47875205402624, -+STORE, 47875205402624, 47875205599231, -+STORE, 47875205599232, 47875205607423, -+ERASE, 47875202519040, 47875202519040, -+STORE, 47875202519040, 47875202555903, -+STORE, 47875202555904, 47875202572287, -+ERASE, 47875201966080, 47875201966080, -+STORE, 47875201966080, 47875201970175, -+STORE, 47875201970176, 47875201974271, -+ERASE, 47875201650688, 47875201650688, -+STORE, 47875201650688, 47875201654783, -+STORE, 47875201654784, 47875201658879, -+ERASE, 47875199381504, 47875199381504, -+STORE, 47875199381504, 47875199401983, -+STORE, 47875199401984, 47875199406079, -+ERASE, 47875199049728, 47875199049728, -+STORE, 47875199049728, 47875199053823, -+STORE, 47875199053824, 47875199057919, -+ERASE, 47875199004672, 47875199004672, -+STORE, 47875199004672, 47875199008767, -+STORE, 47875199008768, 47875199012863, -+ERASE, 94011547025408, 94011547025408, -+STORE, 94011547025408, 94011547148287, -+STORE, 94011547148288, 94011547152383, -+ERASE, 139757598109696, 139757598109696, -+STORE, 139757598109696, 139757598113791, -+STORE, 139757598113792, 139757598117887, -+ERASE, 47875197046784, 47875197046784, -+STORE, 94011557584896, 94011557720063, -+STORE, 94011557584896, 94011557855231, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557851135, -+STORE, 94011557851136, 94011557855231, -+ERASE, 94011557851136, 94011557851136, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557847039, -+STORE, 94011557847040, 94011557851135, -+ERASE, 94011557847040, 94011557847040, -+STORE, 94011557584896, 94011557982207, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557978111, -+STORE, 94011557978112, 94011557982207, -+ERASE, 94011557978112, 94011557978112, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557974015, -+STORE, 94011557974016, 94011557978111, -+ERASE, 94011557974016, 94011557974016, -+STORE, 140737488347136, 140737488351231, -+STORE, 140734130360320, 140737488351231, -+ERASE, 140734130360320, 140734130360320, -+STORE, 140734130360320, 140734130364415, -+STORE, 94641232105472, 94641232785407, -+ERASE, 94641232105472, 94641232105472, -+STORE, 94641232105472, 94641232171007, -+STORE, 94641232171008, 94641232785407, -+ERASE, 94641232171008, 94641232171008, -+STORE, 94641232171008, 94641232519167, -+STORE, 94641232519168, 94641232658431, -+STORE, 94641232658432, 94641232785407, -+STORE, 139726599516160, 139726599688191, -+ERASE, 139726599516160, 139726599516160, -+STORE, 139726599516160, 139726599520255, -+STORE, 139726599520256, 139726599688191, -+ERASE, 139726599520256, 139726599520256, -+STORE, 139726599520256, 139726599643135, -+STORE, 139726599643136, 139726599675903, -+STORE, 139726599675904, 139726599684095, -+STORE, 139726599684096, 139726599688191, -+STORE, 140734130446336, 140734130450431, -+STORE, 140734130434048, 140734130446335, -+STORE, 47906195480576, 47906195488767, -+STORE, 47906195488768, 47906195496959, -+STORE, 47906195496960, 47906197336063, -+STORE, 47906195636224, 47906197336063, -+STORE, 47906195496960, 47906195636223, -+ERASE, 47906195636224, 47906195636224, -+STORE, 47906195636224, 47906197295103, -+STORE, 47906197295104, 47906197336063, -+STORE, 47906196979712, 47906197295103, -+STORE, 47906195636224, 47906196979711, -+ERASE, 47906195636224, 47906195636224, -+STORE, 47906195636224, 47906196979711, -+STORE, 47906197291008, 47906197295103, -+STORE, 47906196979712, 47906197291007, -+ERASE, 47906196979712, 47906196979712, -+STORE, 47906196979712, 47906197291007, -+STORE, 47906197319680, 47906197336063, -+STORE, 47906197295104, 47906197319679, -+ERASE, 47906197295104, 47906197295104, -+STORE, 47906197295104, 47906197319679, -+ERASE, 47906197319680, 47906197319680, -+STORE, 47906197319680, 47906197336063, -+STORE, 47906197336064, 47906197446655, -+STORE, 47906197352448, 47906197446655, -+STORE, 47906197336064, 47906197352447, -+ERASE, 47906197352448, 47906197352448, -+STORE, 47906197352448, 47906197438463, -+STORE, 47906197438464, 47906197446655, -+STORE, 47906197413888, 47906197438463, -+STORE, 47906197352448, 47906197413887, -+ERASE, 47906197352448, 47906197352448, -+STORE, 47906197352448, 47906197413887, -+STORE, 47906197434368, 47906197438463, -+STORE, 47906197413888, 47906197434367, -+ERASE, 47906197413888, 47906197413888, -+STORE, 47906197413888, 47906197434367, -+ERASE, 47906197438464, 47906197438464, -+STORE, 47906197438464, 47906197446655, -+STORE, 47906197446656, 47906197491711, -+ERASE, 47906197446656, 47906197446656, -+STORE, 47906197446656, 47906197454847, -+STORE, 47906197454848, 47906197491711, -+STORE, 47906197475328, 47906197491711, -+STORE, 47906197454848, 47906197475327, -+ERASE, 47906197454848, 47906197454848, -+STORE, 47906197454848, 47906197475327, -+STORE, 47906197483520, 47906197491711, -+STORE, 47906197475328, 47906197483519, -+ERASE, 47906197475328, 47906197475328, -+STORE, 47906197475328, 47906197491711, -+ERASE, 47906197475328, 47906197475328, -+STORE, 47906197475328, 47906197483519, -+STORE, 47906197483520, 47906197491711, -+ERASE, 47906197483520, 47906197483520, -+STORE, 47906197483520, 47906197491711, -+STORE, 47906197491712, 47906197839871, -+STORE, 47906197532672, 47906197839871, -+STORE, 47906197491712, 47906197532671, -+ERASE, 47906197532672, 47906197532672, -+STORE, 47906197532672, 47906197815295, -+STORE, 47906197815296, 47906197839871, -+STORE, 47906197745664, 47906197815295, -+STORE, 47906197532672, 47906197745663, -+ERASE, 47906197532672, 47906197532672, -+STORE, 47906197532672, 47906197745663, -+STORE, 47906197811200, 47906197815295, -+STORE, 47906197745664, 47906197811199, -+ERASE, 47906197745664, 47906197745664, -+STORE, 47906197745664, 47906197811199, -+ERASE, 47906197815296, 47906197815296, -+STORE, 47906197815296, 47906197839871, -+STORE, 47906197839872, 47906200100863, -+STORE, 47906197991424, 47906200100863, -+STORE, 47906197839872, 47906197991423, -+ERASE, 47906197991424, 47906197991424, -+STORE, 47906197991424, 47906200084479, -+STORE, 47906200084480, 47906200100863, -+STORE, 47906200092672, 47906200100863, -+STORE, 47906200084480, 47906200092671, -+ERASE, 47906200084480, 47906200084480, -+STORE, 47906200084480, 47906200092671, -+ERASE, 47906200092672, 47906200092672, -+STORE, 47906200092672, 47906200100863, -+STORE, 47906200100864, 47906200236031, -+ERASE, 47906200100864, 47906200100864, -+STORE, 47906200100864, 47906200125439, -+STORE, 47906200125440, 47906200236031, -+STORE, 47906200186880, 47906200236031, -+STORE, 47906200125440, 47906200186879, -+ERASE, 47906200125440, 47906200125440, -+STORE, 47906200125440, 47906200186879, -+STORE, 47906200211456, 47906200236031, -+STORE, 47906200186880, 47906200211455, -+ERASE, 47906200186880, 47906200186880, -+STORE, 47906200186880, 47906200236031, -+ERASE, 47906200186880, 47906200186880, -+STORE, 47906200186880, 47906200211455, -+STORE, 47906200211456, 47906200236031, -+STORE, 47906200219648, 47906200236031, -+STORE, 47906200211456, 47906200219647, -+ERASE, 47906200211456, 47906200211456, -+STORE, 47906200211456, 47906200219647, -+ERASE, 47906200219648, 47906200219648, -+STORE, 47906200219648, 47906200236031, -+STORE, 47906200219648, 47906200244223, -+STORE, 47906200244224, 47906200408063, -+ERASE, 47906200244224, 47906200244224, -+STORE, 47906200244224, 47906200256511, -+STORE, 47906200256512, 47906200408063, -+STORE, 47906200354816, 47906200408063, -+STORE, 47906200256512, 47906200354815, -+ERASE, 47906200256512, 47906200256512, -+STORE, 47906200256512, 47906200354815, -+STORE, 47906200399872, 47906200408063, -+STORE, 47906200354816, 47906200399871, -+ERASE, 47906200354816, 47906200354816, -+STORE, 47906200354816, 47906200408063, -+ERASE, 47906200354816, 47906200354816, -+STORE, 47906200354816, 47906200399871, -+STORE, 47906200399872, 47906200408063, -+ERASE, 47906200399872, 47906200399872, -+STORE, 47906200399872, 47906200408063, -+STORE, 47906200408064, 47906201006079, -+STORE, 47906200526848, 47906201006079, -+STORE, 47906200408064, 47906200526847, -+ERASE, 47906200526848, 47906200526848, -+STORE, 47906200526848, 47906200952831, -+STORE, 47906200952832, 47906201006079, -+STORE, 47906200842240, 47906200952831, -+STORE, 47906200526848, 47906200842239, -+ERASE, 47906200526848, 47906200526848, -+STORE, 47906200526848, 47906200842239, -+STORE, 47906200948736, 47906200952831, -+STORE, 47906200842240, 47906200948735, -+ERASE, 47906200842240, 47906200842240, -+STORE, 47906200842240, 47906200948735, -+ERASE, 47906200952832, 47906200952832, -+STORE, 47906200952832, 47906201006079, -+STORE, 47906201006080, 47906204057599, -+STORE, 47906201550848, 47906204057599, -+STORE, 47906201006080, 47906201550847, -+ERASE, 47906201550848, 47906201550848, -+STORE, 47906201550848, 47906203836415, -+STORE, 47906203836416, 47906204057599, -+STORE, 47906203246592, 47906203836415, -+STORE, 47906201550848, 47906203246591, -+ERASE, 47906201550848, 47906201550848, -+STORE, 47906201550848, 47906203246591, -+STORE, 47906203832320, 47906203836415, -+STORE, 47906203246592, 47906203832319, -+ERASE, 47906203246592, 47906203246592, -+STORE, 47906203246592, 47906203832319, -+STORE, 47906204041216, 47906204057599, -+STORE, 47906203836416, 47906204041215, -+ERASE, 47906203836416, 47906203836416, -+STORE, 47906203836416, 47906204041215, -+ERASE, 47906204041216, 47906204041216, -+STORE, 47906204041216, 47906204057599, -+STORE, 47906204057600, 47906204090367, -+ERASE, 47906204057600, 47906204057600, -+STORE, 47906204057600, 47906204065791, -+STORE, 47906204065792, 47906204090367, -+STORE, 47906204078080, 47906204090367, -+STORE, 47906204065792, 47906204078079, -+ERASE, 47906204065792, 47906204065792, -+STORE, 47906204065792, 47906204078079, -+STORE, 47906204082176, 47906204090367, -+STORE, 47906204078080, 47906204082175, -+ERASE, 47906204078080, 47906204078080, -+STORE, 47906204078080, 47906204090367, -+ERASE, 47906204078080, 47906204078080, -+STORE, 47906204078080, 47906204082175, -+STORE, 47906204082176, 47906204090367, -+ERASE, 47906204082176, 47906204082176, -+STORE, 47906204082176, 47906204090367, -+STORE, 47906204090368, 47906204127231, -+ERASE, 47906204090368, 47906204090368, -+STORE, 47906204090368, 47906204098559, -+STORE, 47906204098560, 47906204127231, -+STORE, 47906204114944, 47906204127231, -+STORE, 47906204098560, 47906204114943, -+ERASE, 47906204098560, 47906204098560, -+STORE, 47906204098560, 47906204114943, -+STORE, 47906204119040, 47906204127231, -+STORE, 47906204114944, 47906204119039, -+ERASE, 47906204114944, 47906204114944, -+STORE, 47906204114944, 47906204127231, -+ERASE, 47906204114944, 47906204114944, -+STORE, 47906204114944, 47906204119039, -+STORE, 47906204119040, 47906204127231, -+ERASE, 47906204119040, 47906204119040, -+STORE, 47906204119040, 47906204127231, -+STORE, 47906204127232, 47906204602367, -+ERASE, 47906204127232, 47906204127232, -+STORE, 47906204127232, 47906204135423, -+STORE, 47906204135424, 47906204602367, -+STORE, 47906204471296, 47906204602367, -+STORE, 47906204135424, 47906204471295, -+ERASE, 47906204135424, 47906204135424, -+STORE, 47906204135424, 47906204471295, -+STORE, 47906204594176, 47906204602367, -+STORE, 47906204471296, 47906204594175, -+ERASE, 47906204471296, 47906204471296, -+STORE, 47906204471296, 47906204602367, -+ERASE, 47906204471296, 47906204471296, -+STORE, 47906204471296, 47906204594175, -+STORE, 47906204594176, 47906204602367, -+ERASE, 47906204594176, 47906204594176, -+STORE, 47906204594176, 47906204602367, -+STORE, 47906204602368, 47906204610559, -+STORE, 47906204610560, 47906204631039, -+ERASE, 47906204610560, 47906204610560, -+STORE, 47906204610560, 47906204614655, -+STORE, 47906204614656, 47906204631039, -+STORE, 47906204618752, 47906204631039, -+STORE, 47906204614656, 47906204618751, -+ERASE, 47906204614656, 47906204614656, -+STORE, 47906204614656, 47906204618751, -+STORE, 47906204622848, 47906204631039, -+STORE, 47906204618752, 47906204622847, -+ERASE, 47906204618752, 47906204618752, -+STORE, 47906204618752, 47906204631039, -+ERASE, 47906204618752, 47906204618752, -+STORE, 47906204618752, 47906204622847, -+STORE, 47906204622848, 47906204631039, -+ERASE, 47906204622848, 47906204622848, -+STORE, 47906204622848, 47906204631039, -+STORE, 47906204631040, 47906204639231, -+ERASE, 47906197295104, 47906197295104, -+STORE, 47906197295104, 47906197311487, -+STORE, 47906197311488, 47906197319679, -+ERASE, 47906204622848, 47906204622848, -+STORE, 47906204622848, 47906204626943, -+STORE, 47906204626944, 47906204631039, -+ERASE, 47906200211456, 47906200211456, -+STORE, 47906200211456, 47906200215551, -+STORE, 47906200215552, 47906200219647, -+ERASE, 47906204594176, 47906204594176, -+STORE, 47906204594176, 47906204598271, -+STORE, 47906204598272, 47906204602367, -+ERASE, 47906204119040, 47906204119040, -+STORE, 47906204119040, 47906204123135, -+STORE, 47906204123136, 47906204127231, -+ERASE, 47906204082176, 47906204082176, -+STORE, 47906204082176, 47906204086271, -+STORE, 47906204086272, 47906204090367, -+ERASE, 47906203836416, 47906203836416, -+STORE, 47906203836416, 47906204033023, -+STORE, 47906204033024, 47906204041215, -+ERASE, 47906200952832, 47906200952832, -+STORE, 47906200952832, 47906200989695, -+STORE, 47906200989696, 47906201006079, -+ERASE, 47906200399872, 47906200399872, -+STORE, 47906200399872, 47906200403967, -+STORE, 47906200403968, 47906200408063, -+ERASE, 47906200084480, 47906200084480, -+STORE, 47906200084480, 47906200088575, -+STORE, 47906200088576, 47906200092671, -+ERASE, 47906197815296, 47906197815296, -+STORE, 47906197815296, 47906197835775, -+STORE, 47906197835776, 47906197839871, -+ERASE, 47906197483520, 47906197483520, -+STORE, 47906197483520, 47906197487615, -+STORE, 47906197487616, 47906197491711, -+ERASE, 47906197438464, 47906197438464, -+STORE, 47906197438464, 47906197442559, -+STORE, 47906197442560, 47906197446655, -+ERASE, 94641232658432, 94641232658432, -+STORE, 94641232658432, 94641232781311, -+STORE, 94641232781312, 94641232785407, -+ERASE, 139726599675904, 139726599675904, -+STORE, 139726599675904, 139726599679999, -+STORE, 139726599680000, 139726599684095, -+ERASE, 47906195480576, 47906195480576, -+STORE, 94641242615808, 94641242750975, -+ }; -+ -+ unsigned long set10[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140736427839488, 140737488351231, -+ERASE, 140736427839488, 140736427839488, -+STORE, 140736427839488, 140736427843583, -+STORE, 94071213395968, 94071213567999, -+ERASE, 94071213395968, 94071213395968, -+STORE, 94071213395968, 94071213412351, -+STORE, 94071213412352, 94071213567999, -+ERASE, 94071213412352, 94071213412352, -+STORE, 94071213412352, 94071213514751, -+STORE, 94071213514752, 94071213555711, -+STORE, 94071213555712, 94071213567999, -+STORE, 139968410644480, 139968410816511, -+ERASE, 139968410644480, 139968410644480, -+STORE, 139968410644480, 139968410648575, -+STORE, 139968410648576, 139968410816511, -+ERASE, 139968410648576, 139968410648576, -+STORE, 139968410648576, 139968410771455, -+STORE, 139968410771456, 139968410804223, -+STORE, 139968410804224, 139968410812415, -+STORE, 139968410812416, 139968410816511, -+STORE, 140736429277184, 140736429281279, -+STORE, 140736429264896, 140736429277183, -+STORE, 47664384352256, 47664384360447, -+STORE, 47664384360448, 47664384368639, -+STORE, 47664384368640, 47664384532479, -+ERASE, 47664384368640, 47664384368640, -+STORE, 47664384368640, 47664384380927, -+STORE, 47664384380928, 47664384532479, -+STORE, 47664384479232, 47664384532479, -+STORE, 47664384380928, 47664384479231, -+ERASE, 47664384380928, 47664384380928, -+STORE, 47664384380928, 47664384479231, -+STORE, 47664384524288, 47664384532479, -+STORE, 47664384479232, 47664384524287, -+ERASE, 47664384479232, 47664384479232, -+STORE, 47664384479232, 47664384532479, -+ERASE, 47664384479232, 47664384479232, -+STORE, 47664384479232, 47664384524287, -+STORE, 47664384524288, 47664384532479, -+ERASE, 47664384524288, 47664384524288, -+STORE, 47664384524288, 47664384532479, -+STORE, 47664384532480, 47664387583999, -+STORE, 47664385077248, 47664387583999, -+STORE, 47664384532480, 47664385077247, -+ERASE, 47664385077248, 47664385077248, -+STORE, 47664385077248, 47664387362815, -+STORE, 47664387362816, 47664387583999, -+STORE, 47664386772992, 47664387362815, -+STORE, 47664385077248, 47664386772991, -+ERASE, 47664385077248, 47664385077248, -+STORE, 47664385077248, 47664386772991, -+STORE, 47664387358720, 47664387362815, -+STORE, 47664386772992, 47664387358719, -+ERASE, 47664386772992, 47664386772992, -+STORE, 47664386772992, 47664387358719, -+STORE, 47664387567616, 47664387583999, -+STORE, 47664387362816, 47664387567615, -+ERASE, 47664387362816, 47664387362816, -+STORE, 47664387362816, 47664387567615, -+ERASE, 47664387567616, 47664387567616, -+STORE, 47664387567616, 47664387583999, -+STORE, 47664387584000, 47664389423103, -+STORE, 47664387723264, 47664389423103, -+STORE, 47664387584000, 47664387723263, -+ERASE, 47664387723264, 47664387723264, -+STORE, 47664387723264, 47664389382143, -+STORE, 47664389382144, 47664389423103, -+STORE, 47664389066752, 47664389382143, -+STORE, 47664387723264, 47664389066751, -+ERASE, 47664387723264, 47664387723264, -+STORE, 47664387723264, 47664389066751, -+STORE, 47664389378048, 47664389382143, -+STORE, 47664389066752, 47664389378047, -+ERASE, 47664389066752, 47664389066752, -+STORE, 47664389066752, 47664389378047, -+STORE, 47664389406720, 47664389423103, -+STORE, 47664389382144, 47664389406719, -+ERASE, 47664389382144, 47664389382144, -+STORE, 47664389382144, 47664389406719, -+ERASE, 47664389406720, 47664389406720, -+STORE, 47664389406720, 47664389423103, -+STORE, 47664389423104, 47664389558271, -+ERASE, 47664389423104, 47664389423104, -+STORE, 47664389423104, 47664389447679, -+STORE, 47664389447680, 47664389558271, -+STORE, 47664389509120, 47664389558271, -+STORE, 47664389447680, 47664389509119, -+ERASE, 47664389447680, 47664389447680, -+STORE, 47664389447680, 47664389509119, -+STORE, 47664389533696, 47664389558271, -+STORE, 47664389509120, 47664389533695, -+ERASE, 47664389509120, 47664389509120, -+STORE, 47664389509120, 47664389558271, -+ERASE, 47664389509120, 47664389509120, -+STORE, 47664389509120, 47664389533695, -+STORE, 47664389533696, 47664389558271, -+STORE, 47664389541888, 47664389558271, -+STORE, 47664389533696, 47664389541887, -+ERASE, 47664389533696, 47664389533696, -+STORE, 47664389533696, 47664389541887, -+ERASE, 47664389541888, 47664389541888, -+STORE, 47664389541888, 47664389558271, -+STORE, 47664389558272, 47664389578751, -+ERASE, 47664389558272, 47664389558272, -+STORE, 47664389558272, 47664389562367, -+STORE, 47664389562368, 47664389578751, -+STORE, 47664389566464, 47664389578751, -+STORE, 47664389562368, 47664389566463, -+ERASE, 47664389562368, 47664389562368, -+STORE, 47664389562368, 47664389566463, -+STORE, 47664389570560, 47664389578751, -+STORE, 47664389566464, 47664389570559, -+ERASE, 47664389566464, 47664389566464, -+STORE, 47664389566464, 47664389578751, -+ERASE, 47664389566464, 47664389566464, -+STORE, 47664389566464, 47664389570559, -+STORE, 47664389570560, 47664389578751, -+ERASE, 47664389570560, 47664389570560, -+STORE, 47664389570560, 47664389578751, -+STORE, 47664389578752, 47664389586943, -+ERASE, 47664389382144, 47664389382144, -+STORE, 47664389382144, 47664389398527, -+STORE, 47664389398528, 47664389406719, -+ERASE, 47664389570560, 47664389570560, -+STORE, 47664389570560, 47664389574655, -+STORE, 47664389574656, 47664389578751, -+ERASE, 47664389533696, 47664389533696, -+STORE, 47664389533696, 47664389537791, -+STORE, 47664389537792, 47664389541887, -+ERASE, 47664387362816, 47664387362816, -+STORE, 47664387362816, 47664387559423, -+STORE, 47664387559424, 47664387567615, -+ERASE, 47664384524288, 47664384524288, -+STORE, 47664384524288, 47664384528383, -+STORE, 47664384528384, 47664384532479, -+ERASE, 94071213555712, 94071213555712, -+STORE, 94071213555712, 94071213563903, -+STORE, 94071213563904, 94071213567999, -+ERASE, 139968410804224, 139968410804224, -+STORE, 139968410804224, 139968410808319, -+STORE, 139968410808320, 139968410812415, -+ERASE, 47664384352256, 47664384352256, -+STORE, 94071244402688, 94071244537855, -+STORE, 140737488347136, 140737488351231, -+STORE, 140728271503360, 140737488351231, -+ERASE, 140728271503360, 140728271503360, -+STORE, 140728271503360, 140728271507455, -+STORE, 94410361982976, 94410362155007, -+ERASE, 94410361982976, 94410361982976, -+STORE, 94410361982976, 94410361999359, -+STORE, 94410361999360, 94410362155007, -+ERASE, 94410361999360, 94410361999360, -+STORE, 94410361999360, 94410362101759, -+STORE, 94410362101760, 94410362142719, -+STORE, 94410362142720, 94410362155007, -+STORE, 140351953997824, 140351954169855, -+ERASE, 140351953997824, 140351953997824, -+STORE, 140351953997824, 140351954001919, -+STORE, 140351954001920, 140351954169855, -+ERASE, 140351954001920, 140351954001920, -+STORE, 140351954001920, 140351954124799, -+STORE, 140351954124800, 140351954157567, -+STORE, 140351954157568, 140351954165759, -+STORE, 140351954165760, 140351954169855, -+STORE, 140728272429056, 140728272433151, -+STORE, 140728272416768, 140728272429055, -+STORE, 47280840998912, 47280841007103, -+STORE, 47280841007104, 47280841015295, -+STORE, 47280841015296, 47280841179135, -+ERASE, 47280841015296, 47280841015296, -+STORE, 47280841015296, 47280841027583, -+STORE, 47280841027584, 47280841179135, -+STORE, 47280841125888, 47280841179135, -+STORE, 47280841027584, 47280841125887, -+ERASE, 47280841027584, 47280841027584, -+STORE, 47280841027584, 47280841125887, -+STORE, 47280841170944, 47280841179135, -+STORE, 47280841125888, 47280841170943, -+ERASE, 47280841125888, 47280841125888, -+STORE, 47280841125888, 47280841179135, -+ERASE, 47280841125888, 47280841125888, -+STORE, 47280841125888, 47280841170943, -+STORE, 47280841170944, 47280841179135, -+ERASE, 47280841170944, 47280841170944, -+STORE, 47280841170944, 47280841179135, -+STORE, 47280841179136, 47280844230655, -+STORE, 47280841723904, 47280844230655, -+STORE, 47280841179136, 47280841723903, -+ERASE, 47280841723904, 47280841723904, -+STORE, 47280841723904, 47280844009471, -+STORE, 47280844009472, 47280844230655, -+STORE, 47280843419648, 47280844009471, -+STORE, 47280841723904, 47280843419647, -+ERASE, 47280841723904, 47280841723904, -+STORE, 47280841723904, 47280843419647, -+STORE, 47280844005376, 47280844009471, -+STORE, 47280843419648, 47280844005375, -+ERASE, 47280843419648, 47280843419648, -+STORE, 47280843419648, 47280844005375, -+STORE, 47280844214272, 47280844230655, -+STORE, 47280844009472, 47280844214271, -+ERASE, 47280844009472, 47280844009472, -+STORE, 47280844009472, 47280844214271, -+ERASE, 47280844214272, 47280844214272, -+STORE, 47280844214272, 47280844230655, -+STORE, 47280844230656, 47280846069759, -+STORE, 47280844369920, 47280846069759, -+STORE, 47280844230656, 47280844369919, -+ERASE, 47280844369920, 47280844369920, -+STORE, 47280844369920, 47280846028799, -+STORE, 47280846028800, 47280846069759, -+STORE, 47280845713408, 47280846028799, -+STORE, 47280844369920, 47280845713407, -+ERASE, 47280844369920, 47280844369920, -+STORE, 47280844369920, 47280845713407, -+STORE, 47280846024704, 47280846028799, -+STORE, 47280845713408, 47280846024703, -+ERASE, 47280845713408, 47280845713408, -+STORE, 47280845713408, 47280846024703, -+STORE, 47280846053376, 47280846069759, -+STORE, 47280846028800, 47280846053375, -+ERASE, 47280846028800, 47280846028800, -+STORE, 47280846028800, 47280846053375, -+ERASE, 47280846053376, 47280846053376, -+STORE, 47280846053376, 47280846069759, -+STORE, 47280846069760, 47280846204927, -+ERASE, 47280846069760, 47280846069760, -+STORE, 47280846069760, 47280846094335, -+STORE, 47280846094336, 47280846204927, -+STORE, 47280846155776, 47280846204927, -+STORE, 47280846094336, 47280846155775, -+ERASE, 47280846094336, 47280846094336, -+STORE, 47280846094336, 47280846155775, -+STORE, 47280846180352, 47280846204927, -+STORE, 47280846155776, 47280846180351, -+ERASE, 47280846155776, 47280846155776, -+STORE, 47280846155776, 47280846204927, -+ERASE, 47280846155776, 47280846155776, -+STORE, 47280846155776, 47280846180351, -+STORE, 47280846180352, 47280846204927, -+STORE, 47280846188544, 47280846204927, -+STORE, 47280846180352, 47280846188543, -+ERASE, 47280846180352, 47280846180352, -+STORE, 47280846180352, 47280846188543, -+ERASE, 47280846188544, 47280846188544, -+STORE, 47280846188544, 47280846204927, -+STORE, 47280846204928, 47280846225407, -+ERASE, 47280846204928, 47280846204928, -+STORE, 47280846204928, 47280846209023, -+STORE, 47280846209024, 47280846225407, -+STORE, 47280846213120, 47280846225407, -+STORE, 47280846209024, 47280846213119, -+ERASE, 47280846209024, 47280846209024, -+STORE, 47280846209024, 47280846213119, -+STORE, 47280846217216, 47280846225407, -+STORE, 47280846213120, 47280846217215, -+ERASE, 47280846213120, 47280846213120, -+STORE, 47280846213120, 47280846225407, -+ERASE, 47280846213120, 47280846213120, -+STORE, 47280846213120, 47280846217215, -+STORE, 47280846217216, 47280846225407, -+ERASE, 47280846217216, 47280846217216, -+STORE, 47280846217216, 47280846225407, -+STORE, 47280846225408, 47280846233599, -+ERASE, 47280846028800, 47280846028800, -+STORE, 47280846028800, 47280846045183, -+STORE, 47280846045184, 47280846053375, -+ERASE, 47280846217216, 47280846217216, -+STORE, 47280846217216, 47280846221311, -+STORE, 47280846221312, 47280846225407, -+ERASE, 47280846180352, 47280846180352, -+STORE, 47280846180352, 47280846184447, -+STORE, 47280846184448, 47280846188543, -+ERASE, 47280844009472, 47280844009472, -+STORE, 47280844009472, 47280844206079, -+STORE, 47280844206080, 47280844214271, -+ERASE, 47280841170944, 47280841170944, -+STORE, 47280841170944, 47280841175039, -+STORE, 47280841175040, 47280841179135, -+ERASE, 94410362142720, 94410362142720, -+STORE, 94410362142720, 94410362150911, -+STORE, 94410362150912, 94410362155007, -+ERASE, 140351954157568, 140351954157568, -+STORE, 140351954157568, 140351954161663, -+STORE, 140351954161664, 140351954165759, -+ERASE, 47280840998912, 47280840998912, -+STORE, 94410379456512, 94410379591679, -+STORE, 140737488347136, 140737488351231, -+STORE, 140732946362368, 140737488351231, -+ERASE, 140732946362368, 140732946362368, -+STORE, 140732946362368, 140732946366463, -+STORE, 94352937934848, 94352938106879, -+ERASE, 94352937934848, 94352937934848, -+STORE, 94352937934848, 94352937951231, -+STORE, 94352937951232, 94352938106879, -+ERASE, 94352937951232, 94352937951232, -+STORE, 94352937951232, 94352938053631, -+STORE, 94352938053632, 94352938094591, -+STORE, 94352938094592, 94352938106879, -+STORE, 140595518742528, 140595518914559, -+ERASE, 140595518742528, 140595518742528, -+STORE, 140595518742528, 140595518746623, -+STORE, 140595518746624, 140595518914559, -+ERASE, 140595518746624, 140595518746624, -+STORE, 140595518746624, 140595518869503, -+STORE, 140595518869504, 140595518902271, -+STORE, 140595518902272, 140595518910463, -+STORE, 140595518910464, 140595518914559, -+STORE, 140732947468288, 140732947472383, -+STORE, 140732947456000, 140732947468287, -+STORE, 47037276254208, 47037276262399, -+STORE, 47037276262400, 47037276270591, -+STORE, 47037276270592, 47037276434431, -+ERASE, 47037276270592, 47037276270592, -+STORE, 47037276270592, 47037276282879, -+STORE, 47037276282880, 47037276434431, -+STORE, 47037276381184, 47037276434431, -+STORE, 47037276282880, 47037276381183, -+ERASE, 47037276282880, 47037276282880, -+STORE, 47037276282880, 47037276381183, -+STORE, 47037276426240, 47037276434431, -+STORE, 47037276381184, 47037276426239, -+ERASE, 47037276381184, 47037276381184, -+STORE, 47037276381184, 47037276434431, -+ERASE, 47037276381184, 47037276381184, -+STORE, 47037276381184, 47037276426239, -+STORE, 47037276426240, 47037276434431, -+ERASE, 47037276426240, 47037276426240, -+STORE, 47037276426240, 47037276434431, -+STORE, 47037276434432, 47037279485951, -+STORE, 47037276979200, 47037279485951, -+STORE, 47037276434432, 47037276979199, -+ERASE, 47037276979200, 47037276979200, -+STORE, 47037276979200, 47037279264767, -+STORE, 47037279264768, 47037279485951, -+STORE, 47037278674944, 47037279264767, -+STORE, 47037276979200, 47037278674943, -+ERASE, 47037276979200, 47037276979200, -+STORE, 47037276979200, 47037278674943, -+STORE, 47037279260672, 47037279264767, -+STORE, 47037278674944, 47037279260671, -+ERASE, 47037278674944, 47037278674944, -+STORE, 47037278674944, 47037279260671, -+STORE, 47037279469568, 47037279485951, -+STORE, 47037279264768, 47037279469567, -+ERASE, 47037279264768, 47037279264768, -+STORE, 47037279264768, 47037279469567, -+ERASE, 47037279469568, 47037279469568, -+STORE, 47037279469568, 47037279485951, -+STORE, 47037279485952, 47037281325055, -+STORE, 47037279625216, 47037281325055, -+STORE, 47037279485952, 47037279625215, -+ERASE, 47037279625216, 47037279625216, -+STORE, 47037279625216, 47037281284095, -+STORE, 47037281284096, 47037281325055, -+STORE, 47037280968704, 47037281284095, -+STORE, 47037279625216, 47037280968703, -+ERASE, 47037279625216, 47037279625216, -+STORE, 47037279625216, 47037280968703, -+STORE, 47037281280000, 47037281284095, -+STORE, 47037280968704, 47037281279999, -+ERASE, 47037280968704, 47037280968704, -+STORE, 47037280968704, 47037281279999, -+STORE, 47037281308672, 47037281325055, -+STORE, 47037281284096, 47037281308671, -+ERASE, 47037281284096, 47037281284096, -+STORE, 47037281284096, 47037281308671, -+ERASE, 47037281308672, 47037281308672, -+STORE, 47037281308672, 47037281325055, -+STORE, 47037281325056, 47037281460223, -+ERASE, 47037281325056, 47037281325056, -+STORE, 47037281325056, 47037281349631, -+STORE, 47037281349632, 47037281460223, -+STORE, 47037281411072, 47037281460223, -+STORE, 47037281349632, 47037281411071, -+ERASE, 47037281349632, 47037281349632, -+STORE, 47037281349632, 47037281411071, -+STORE, 47037281435648, 47037281460223, -+STORE, 47037281411072, 47037281435647, -+ERASE, 47037281411072, 47037281411072, -+STORE, 47037281411072, 47037281460223, -+ERASE, 47037281411072, 47037281411072, -+STORE, 47037281411072, 47037281435647, -+STORE, 47037281435648, 47037281460223, -+STORE, 47037281443840, 47037281460223, -+STORE, 47037281435648, 47037281443839, -+ERASE, 47037281435648, 47037281435648, -+STORE, 47037281435648, 47037281443839, -+ERASE, 47037281443840, 47037281443840, -+STORE, 47037281443840, 47037281460223, -+STORE, 47037281460224, 47037281480703, -+ERASE, 47037281460224, 47037281460224, -+STORE, 47037281460224, 47037281464319, -+STORE, 47037281464320, 47037281480703, -+STORE, 47037281468416, 47037281480703, -+STORE, 47037281464320, 47037281468415, -+ERASE, 47037281464320, 47037281464320, -+STORE, 47037281464320, 47037281468415, -+STORE, 47037281472512, 47037281480703, -+STORE, 47037281468416, 47037281472511, -+ERASE, 47037281468416, 47037281468416, -+STORE, 47037281468416, 47037281480703, -+ERASE, 47037281468416, 47037281468416, -+STORE, 47037281468416, 47037281472511, -+STORE, 47037281472512, 47037281480703, -+ERASE, 47037281472512, 47037281472512, -+STORE, 47037281472512, 47037281480703, -+STORE, 47037281480704, 47037281488895, -+ERASE, 47037281284096, 47037281284096, -+STORE, 47037281284096, 47037281300479, -+STORE, 47037281300480, 47037281308671, -+ERASE, 47037281472512, 47037281472512, -+STORE, 47037281472512, 47037281476607, -+STORE, 47037281476608, 47037281480703, -+ERASE, 47037281435648, 47037281435648, -+STORE, 47037281435648, 47037281439743, -+STORE, 47037281439744, 47037281443839, -+ERASE, 47037279264768, 47037279264768, -+STORE, 47037279264768, 47037279461375, -+STORE, 47037279461376, 47037279469567, -+ERASE, 47037276426240, 47037276426240, -+STORE, 47037276426240, 47037276430335, -+STORE, 47037276430336, 47037276434431, -+ERASE, 94352938094592, 94352938094592, -+STORE, 94352938094592, 94352938102783, -+STORE, 94352938102784, 94352938106879, -+ERASE, 140595518902272, 140595518902272, -+STORE, 140595518902272, 140595518906367, -+STORE, 140595518906368, 140595518910463, -+ERASE, 47037276254208, 47037276254208, -+STORE, 94352938438656, 94352938573823, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733506027520, 140737488351231, -+ERASE, 140733506027520, 140733506027520, -+STORE, 140733506027520, 140733506031615, -+STORE, 94150123073536, 94150123245567, -+ERASE, 94150123073536, 94150123073536, -+STORE, 94150123073536, 94150123089919, -+STORE, 94150123089920, 94150123245567, -+ERASE, 94150123089920, 94150123089920, -+STORE, 94150123089920, 94150123192319, -+STORE, 94150123192320, 94150123233279, -+STORE, 94150123233280, 94150123245567, -+STORE, 140081290375168, 140081290547199, -+ERASE, 140081290375168, 140081290375168, -+STORE, 140081290375168, 140081290379263, -+STORE, 140081290379264, 140081290547199, -+ERASE, 140081290379264, 140081290379264, -+STORE, 140081290379264, 140081290502143, -+STORE, 140081290502144, 140081290534911, -+STORE, 140081290534912, 140081290543103, -+STORE, 140081290543104, 140081290547199, -+STORE, 140733506707456, 140733506711551, -+STORE, 140733506695168, 140733506707455, -+STORE, 47551504621568, 47551504629759, -+STORE, 47551504629760, 47551504637951, -+STORE, 47551504637952, 47551504801791, -+ERASE, 47551504637952, 47551504637952, -+STORE, 47551504637952, 47551504650239, -+STORE, 47551504650240, 47551504801791, -+STORE, 47551504748544, 47551504801791, -+STORE, 47551504650240, 47551504748543, -+ERASE, 47551504650240, 47551504650240, -+STORE, 47551504650240, 47551504748543, -+STORE, 47551504793600, 47551504801791, -+STORE, 47551504748544, 47551504793599, -+ERASE, 47551504748544, 47551504748544, -+STORE, 47551504748544, 47551504801791, -+ERASE, 47551504748544, 47551504748544, -+STORE, 47551504748544, 47551504793599, -+STORE, 47551504793600, 47551504801791, -+ERASE, 47551504793600, 47551504793600, -+STORE, 47551504793600, 47551504801791, -+STORE, 47551504801792, 47551507853311, -+STORE, 47551505346560, 47551507853311, -+STORE, 47551504801792, 47551505346559, -+ERASE, 47551505346560, 47551505346560, -+STORE, 47551505346560, 47551507632127, -+STORE, 47551507632128, 47551507853311, -+STORE, 47551507042304, 47551507632127, -+STORE, 47551505346560, 47551507042303, -+ERASE, 47551505346560, 47551505346560, -+STORE, 47551505346560, 47551507042303, -+STORE, 47551507628032, 47551507632127, -+STORE, 47551507042304, 47551507628031, -+ERASE, 47551507042304, 47551507042304, -+STORE, 47551507042304, 47551507628031, -+STORE, 47551507836928, 47551507853311, -+STORE, 47551507632128, 47551507836927, -+ERASE, 47551507632128, 47551507632128, -+STORE, 47551507632128, 47551507836927, -+ERASE, 47551507836928, 47551507836928, -+STORE, 47551507836928, 47551507853311, -+STORE, 47551507853312, 47551509692415, -+STORE, 47551507992576, 47551509692415, -+STORE, 47551507853312, 47551507992575, -+ERASE, 47551507992576, 47551507992576, -+STORE, 47551507992576, 47551509651455, -+STORE, 47551509651456, 47551509692415, -+STORE, 47551509336064, 47551509651455, -+STORE, 47551507992576, 47551509336063, -+ERASE, 47551507992576, 47551507992576, -+STORE, 47551507992576, 47551509336063, -+STORE, 47551509647360, 47551509651455, -+STORE, 47551509336064, 47551509647359, -+ERASE, 47551509336064, 47551509336064, -+STORE, 47551509336064, 47551509647359, -+STORE, 47551509676032, 47551509692415, -+STORE, 47551509651456, 47551509676031, -+ERASE, 47551509651456, 47551509651456, -+STORE, 47551509651456, 47551509676031, -+ERASE, 47551509676032, 47551509676032, -+STORE, 47551509676032, 47551509692415, -+STORE, 47551509692416, 47551509827583, -+ERASE, 47551509692416, 47551509692416, -+STORE, 47551509692416, 47551509716991, -+STORE, 47551509716992, 47551509827583, -+STORE, 47551509778432, 47551509827583, -+STORE, 47551509716992, 47551509778431, -+ERASE, 47551509716992, 47551509716992, -+STORE, 47551509716992, 47551509778431, -+STORE, 47551509803008, 47551509827583, -+STORE, 47551509778432, 47551509803007, -+ERASE, 47551509778432, 47551509778432, -+STORE, 47551509778432, 47551509827583, -+ERASE, 47551509778432, 47551509778432, -+STORE, 47551509778432, 47551509803007, -+STORE, 47551509803008, 47551509827583, -+STORE, 47551509811200, 47551509827583, -+STORE, 47551509803008, 47551509811199, -+ERASE, 47551509803008, 47551509803008, -+STORE, 47551509803008, 47551509811199, -+ERASE, 47551509811200, 47551509811200, -+STORE, 47551509811200, 47551509827583, -+STORE, 47551509827584, 47551509848063, -+ERASE, 47551509827584, 47551509827584, -+STORE, 47551509827584, 47551509831679, -+STORE, 47551509831680, 47551509848063, -+STORE, 47551509835776, 47551509848063, -+STORE, 47551509831680, 47551509835775, -+ERASE, 47551509831680, 47551509831680, -+STORE, 47551509831680, 47551509835775, -+STORE, 47551509839872, 47551509848063, -+STORE, 47551509835776, 47551509839871, -+ERASE, 47551509835776, 47551509835776, -+STORE, 47551509835776, 47551509848063, -+ERASE, 47551509835776, 47551509835776, -+STORE, 47551509835776, 47551509839871, -+STORE, 47551509839872, 47551509848063, -+ERASE, 47551509839872, 47551509839872, -+STORE, 47551509839872, 47551509848063, -+STORE, 47551509848064, 47551509856255, -+ERASE, 47551509651456, 47551509651456, -+STORE, 47551509651456, 47551509667839, -+STORE, 47551509667840, 47551509676031, -+ERASE, 47551509839872, 47551509839872, -+STORE, 47551509839872, 47551509843967, -+STORE, 47551509843968, 47551509848063, -+ERASE, 47551509803008, 47551509803008, -+STORE, 47551509803008, 47551509807103, -+STORE, 47551509807104, 47551509811199, -+ERASE, 47551507632128, 47551507632128, -+STORE, 47551507632128, 47551507828735, -+STORE, 47551507828736, 47551507836927, -+ERASE, 47551504793600, 47551504793600, -+STORE, 47551504793600, 47551504797695, -+STORE, 47551504797696, 47551504801791, -+ERASE, 94150123233280, 94150123233280, -+STORE, 94150123233280, 94150123241471, -+STORE, 94150123241472, 94150123245567, -+ERASE, 140081290534912, 140081290534912, -+STORE, 140081290534912, 140081290539007, -+STORE, 140081290539008, 140081290543103, -+ERASE, 47551504621568, 47551504621568, -+STORE, 94150148112384, 94150148247551, -+STORE, 140737488347136, 140737488351231, -+STORE, 140734389334016, 140737488351231, -+ERASE, 140734389334016, 140734389334016, -+STORE, 140734389334016, 140734389338111, -+STORE, 94844636606464, 94844636778495, -+ERASE, 94844636606464, 94844636606464, -+STORE, 94844636606464, 94844636622847, -+STORE, 94844636622848, 94844636778495, -+ERASE, 94844636622848, 94844636622848, -+STORE, 94844636622848, 94844636725247, -+STORE, 94844636725248, 94844636766207, -+STORE, 94844636766208, 94844636778495, -+STORE, 139922765217792, 139922765389823, -+ERASE, 139922765217792, 139922765217792, -+STORE, 139922765217792, 139922765221887, -+STORE, 139922765221888, 139922765389823, -+ERASE, 139922765221888, 139922765221888, -+STORE, 139922765221888, 139922765344767, -+STORE, 139922765344768, 139922765377535, -+STORE, 139922765377536, 139922765385727, -+STORE, 139922765385728, 139922765389823, -+STORE, 140734389678080, 140734389682175, -+STORE, 140734389665792, 140734389678079, -+STORE, 47710029778944, 47710029787135, -+STORE, 47710029787136, 47710029795327, -+STORE, 47710029795328, 47710029959167, -+ERASE, 47710029795328, 47710029795328, -+STORE, 47710029795328, 47710029807615, -+STORE, 47710029807616, 47710029959167, -+STORE, 47710029905920, 47710029959167, -+STORE, 47710029807616, 47710029905919, -+ERASE, 47710029807616, 47710029807616, -+STORE, 47710029807616, 47710029905919, -+STORE, 47710029950976, 47710029959167, -+STORE, 47710029905920, 47710029950975, -+ERASE, 47710029905920, 47710029905920, -+STORE, 47710029905920, 47710029959167, -+ERASE, 47710029905920, 47710029905920, -+STORE, 47710029905920, 47710029950975, -+STORE, 47710029950976, 47710029959167, -+ERASE, 47710029950976, 47710029950976, -+STORE, 47710029950976, 47710029959167, -+STORE, 47710029959168, 47710033010687, -+STORE, 47710030503936, 47710033010687, -+STORE, 47710029959168, 47710030503935, -+ERASE, 47710030503936, 47710030503936, -+STORE, 47710030503936, 47710032789503, -+STORE, 47710032789504, 47710033010687, -+STORE, 47710032199680, 47710032789503, -+STORE, 47710030503936, 47710032199679, -+ERASE, 47710030503936, 47710030503936, -+STORE, 47710030503936, 47710032199679, -+STORE, 47710032785408, 47710032789503, -+STORE, 47710032199680, 47710032785407, -+ERASE, 47710032199680, 47710032199680, -+STORE, 47710032199680, 47710032785407, -+STORE, 47710032994304, 47710033010687, -+STORE, 47710032789504, 47710032994303, -+ERASE, 47710032789504, 47710032789504, -+STORE, 47710032789504, 47710032994303, -+ERASE, 47710032994304, 47710032994304, -+STORE, 47710032994304, 47710033010687, -+STORE, 47710033010688, 47710034849791, -+STORE, 47710033149952, 47710034849791, -+STORE, 47710033010688, 47710033149951, -+ERASE, 47710033149952, 47710033149952, -+STORE, 47710033149952, 47710034808831, -+STORE, 47710034808832, 47710034849791, -+STORE, 47710034493440, 47710034808831, -+STORE, 47710033149952, 47710034493439, -+ERASE, 47710033149952, 47710033149952, -+STORE, 47710033149952, 47710034493439, -+STORE, 47710034804736, 47710034808831, -+STORE, 47710034493440, 47710034804735, -+ERASE, 47710034493440, 47710034493440, -+STORE, 47710034493440, 47710034804735, -+STORE, 47710034833408, 47710034849791, -+STORE, 47710034808832, 47710034833407, -+ERASE, 47710034808832, 47710034808832, -+STORE, 47710034808832, 47710034833407, -+ERASE, 47710034833408, 47710034833408, -+STORE, 47710034833408, 47710034849791, -+STORE, 47710034849792, 47710034984959, -+ERASE, 47710034849792, 47710034849792, -+STORE, 47710034849792, 47710034874367, -+STORE, 47710034874368, 47710034984959, -+STORE, 47710034935808, 47710034984959, -+STORE, 47710034874368, 47710034935807, -+ERASE, 47710034874368, 47710034874368, -+STORE, 47710034874368, 47710034935807, -+STORE, 47710034960384, 47710034984959, -+STORE, 47710034935808, 47710034960383, -+ERASE, 47710034935808, 47710034935808, -+STORE, 47710034935808, 47710034984959, -+ERASE, 47710034935808, 47710034935808, -+STORE, 47710034935808, 47710034960383, -+STORE, 47710034960384, 47710034984959, -+STORE, 47710034968576, 47710034984959, -+STORE, 47710034960384, 47710034968575, -+ERASE, 47710034960384, 47710034960384, -+STORE, 47710034960384, 47710034968575, -+ERASE, 47710034968576, 47710034968576, -+STORE, 47710034968576, 47710034984959, -+STORE, 47710034984960, 47710035005439, -+ERASE, 47710034984960, 47710034984960, -+STORE, 47710034984960, 47710034989055, -+STORE, 47710034989056, 47710035005439, -+STORE, 47710034993152, 47710035005439, -+STORE, 47710034989056, 47710034993151, -+ERASE, 47710034989056, 47710034989056, -+STORE, 47710034989056, 47710034993151, -+STORE, 47710034997248, 47710035005439, -+STORE, 47710034993152, 47710034997247, -+ERASE, 47710034993152, 47710034993152, -+STORE, 47710034993152, 47710035005439, -+ERASE, 47710034993152, 47710034993152, -+STORE, 47710034993152, 47710034997247, -+STORE, 47710034997248, 47710035005439, -+ERASE, 47710034997248, 47710034997248, -+STORE, 47710034997248, 47710035005439, -+STORE, 47710035005440, 47710035013631, -+ERASE, 47710034808832, 47710034808832, -+STORE, 47710034808832, 47710034825215, -+STORE, 47710034825216, 47710034833407, -+ERASE, 47710034997248, 47710034997248, -+STORE, 47710034997248, 47710035001343, -+STORE, 47710035001344, 47710035005439, -+ERASE, 47710034960384, 47710034960384, -+STORE, 47710034960384, 47710034964479, -+STORE, 47710034964480, 47710034968575, -+ERASE, 47710032789504, 47710032789504, -+STORE, 47710032789504, 47710032986111, -+STORE, 47710032986112, 47710032994303, -+ERASE, 47710029950976, 47710029950976, -+STORE, 47710029950976, 47710029955071, -+STORE, 47710029955072, 47710029959167, -+ERASE, 94844636766208, 94844636766208, -+STORE, 94844636766208, 94844636774399, -+STORE, 94844636774400, 94844636778495, -+ERASE, 139922765377536, 139922765377536, -+STORE, 139922765377536, 139922765381631, -+STORE, 139922765381632, 139922765385727, -+ERASE, 47710029778944, 47710029778944, -+STORE, 94844641775616, 94844641910783, -+STORE, 140737488347136, 140737488351231, -+STORE, 140732213886976, 140737488351231, -+ERASE, 140732213886976, 140732213886976, -+STORE, 140732213886976, 140732213891071, -+STORE, 94240508887040, 94240509059071, -+ERASE, 94240508887040, 94240508887040, -+STORE, 94240508887040, 94240508903423, -+STORE, 94240508903424, 94240509059071, -+ERASE, 94240508903424, 94240508903424, -+STORE, 94240508903424, 94240509005823, -+STORE, 94240509005824, 94240509046783, -+STORE, 94240509046784, 94240509059071, -+STORE, 140275106516992, 140275106689023, -+ERASE, 140275106516992, 140275106516992, -+STORE, 140275106516992, 140275106521087, -+STORE, 140275106521088, 140275106689023, -+ERASE, 140275106521088, 140275106521088, -+STORE, 140275106521088, 140275106643967, -+STORE, 140275106643968, 140275106676735, -+STORE, 140275106676736, 140275106684927, -+STORE, 140275106684928, 140275106689023, -+STORE, 140732213977088, 140732213981183, -+STORE, 140732213964800, 140732213977087, -+STORE, 47357688479744, 47357688487935, -+STORE, 47357688487936, 47357688496127, -+STORE, 47357688496128, 47357688659967, -+ERASE, 47357688496128, 47357688496128, -+STORE, 47357688496128, 47357688508415, -+STORE, 47357688508416, 47357688659967, -+STORE, 47357688606720, 47357688659967, -+STORE, 47357688508416, 47357688606719, -+ERASE, 47357688508416, 47357688508416, -+STORE, 47357688508416, 47357688606719, -+STORE, 47357688651776, 47357688659967, -+STORE, 47357688606720, 47357688651775, -+ERASE, 47357688606720, 47357688606720, -+STORE, 47357688606720, 47357688659967, -+ERASE, 47357688606720, 47357688606720, -+STORE, 47357688606720, 47357688651775, -+STORE, 47357688651776, 47357688659967, -+ERASE, 47357688651776, 47357688651776, -+STORE, 47357688651776, 47357688659967, -+STORE, 47357688659968, 47357691711487, -+STORE, 47357689204736, 47357691711487, -+STORE, 47357688659968, 47357689204735, -+ERASE, 47357689204736, 47357689204736, -+STORE, 47357689204736, 47357691490303, -+STORE, 47357691490304, 47357691711487, -+STORE, 47357690900480, 47357691490303, -+STORE, 47357689204736, 47357690900479, -+ERASE, 47357689204736, 47357689204736, -+STORE, 47357689204736, 47357690900479, -+STORE, 47357691486208, 47357691490303, -+STORE, 47357690900480, 47357691486207, -+ERASE, 47357690900480, 47357690900480, -+STORE, 47357690900480, 47357691486207, -+STORE, 47357691695104, 47357691711487, -+STORE, 47357691490304, 47357691695103, -+ERASE, 47357691490304, 47357691490304, -+STORE, 47357691490304, 47357691695103, -+ERASE, 47357691695104, 47357691695104, -+STORE, 47357691695104, 47357691711487, -+STORE, 47357691711488, 47357693550591, -+STORE, 47357691850752, 47357693550591, -+STORE, 47357691711488, 47357691850751, -+ERASE, 47357691850752, 47357691850752, -+STORE, 47357691850752, 47357693509631, -+STORE, 47357693509632, 47357693550591, -+STORE, 47357693194240, 47357693509631, -+STORE, 47357691850752, 47357693194239, -+ERASE, 47357691850752, 47357691850752, -+STORE, 47357691850752, 47357693194239, -+STORE, 47357693505536, 47357693509631, -+STORE, 47357693194240, 47357693505535, -+ERASE, 47357693194240, 47357693194240, -+STORE, 47357693194240, 47357693505535, -+STORE, 47357693534208, 47357693550591, -+STORE, 47357693509632, 47357693534207, -+ERASE, 47357693509632, 47357693509632, -+STORE, 47357693509632, 47357693534207, -+ERASE, 47357693534208, 47357693534208, -+STORE, 47357693534208, 47357693550591, -+STORE, 47357693550592, 47357693685759, -+ERASE, 47357693550592, 47357693550592, -+STORE, 47357693550592, 47357693575167, -+STORE, 47357693575168, 47357693685759, -+STORE, 47357693636608, 47357693685759, -+STORE, 47357693575168, 47357693636607, -+ERASE, 47357693575168, 47357693575168, -+STORE, 47357693575168, 47357693636607, -+STORE, 47357693661184, 47357693685759, -+STORE, 47357693636608, 47357693661183, -+ERASE, 47357693636608, 47357693636608, -+STORE, 47357693636608, 47357693685759, -+ERASE, 47357693636608, 47357693636608, -+STORE, 47357693636608, 47357693661183, -+STORE, 47357693661184, 47357693685759, -+STORE, 47357693669376, 47357693685759, -+STORE, 47357693661184, 47357693669375, -+ERASE, 47357693661184, 47357693661184, -+STORE, 47357693661184, 47357693669375, -+ERASE, 47357693669376, 47357693669376, -+STORE, 47357693669376, 47357693685759, -+STORE, 47357693685760, 47357693706239, -+ERASE, 47357693685760, 47357693685760, -+STORE, 47357693685760, 47357693689855, -+STORE, 47357693689856, 47357693706239, -+STORE, 47357693693952, 47357693706239, -+STORE, 47357693689856, 47357693693951, -+ERASE, 47357693689856, 47357693689856, -+STORE, 47357693689856, 47357693693951, -+STORE, 47357693698048, 47357693706239, -+STORE, 47357693693952, 47357693698047, -+ERASE, 47357693693952, 47357693693952, -+STORE, 47357693693952, 47357693706239, -+ERASE, 47357693693952, 47357693693952, -+STORE, 47357693693952, 47357693698047, -+STORE, 47357693698048, 47357693706239, -+ERASE, 47357693698048, 47357693698048, -+STORE, 47357693698048, 47357693706239, -+STORE, 47357693706240, 47357693714431, -+ERASE, 47357693509632, 47357693509632, -+STORE, 47357693509632, 47357693526015, -+STORE, 47357693526016, 47357693534207, -+ERASE, 47357693698048, 47357693698048, -+STORE, 47357693698048, 47357693702143, -+STORE, 47357693702144, 47357693706239, -+ERASE, 47357693661184, 47357693661184, -+STORE, 47357693661184, 47357693665279, -+STORE, 47357693665280, 47357693669375, -+ERASE, 47357691490304, 47357691490304, -+STORE, 47357691490304, 47357691686911, -+STORE, 47357691686912, 47357691695103, -+ERASE, 47357688651776, 47357688651776, -+STORE, 47357688651776, 47357688655871, -+STORE, 47357688655872, 47357688659967, -+ERASE, 94240509046784, 94240509046784, -+STORE, 94240509046784, 94240509054975, -+STORE, 94240509054976, 94240509059071, -+ERASE, 140275106676736, 140275106676736, -+STORE, 140275106676736, 140275106680831, -+STORE, 140275106680832, 140275106684927, -+ERASE, 47357688479744, 47357688479744, -+STORE, 94240518361088, 94240518496255, -+STORE, 140737488347136, 140737488351231, -+STORE, 140732688277504, 140737488351231, -+ERASE, 140732688277504, 140732688277504, -+STORE, 140732688277504, 140732688281599, -+STORE, 94629171351552, 94629172064255, -+ERASE, 94629171351552, 94629171351552, -+STORE, 94629171351552, 94629171400703, -+STORE, 94629171400704, 94629172064255, -+ERASE, 94629171400704, 94629171400704, -+STORE, 94629171400704, 94629171945471, -+STORE, 94629171945472, 94629172043775, -+STORE, 94629172043776, 94629172064255, -+STORE, 139770707644416, 139770707816447, -+ERASE, 139770707644416, 139770707644416, -+STORE, 139770707644416, 139770707648511, -+STORE, 139770707648512, 139770707816447, -+ERASE, 139770707648512, 139770707648512, -+STORE, 139770707648512, 139770707771391, -+STORE, 139770707771392, 139770707804159, -+STORE, 139770707804160, 139770707812351, -+STORE, 139770707812352, 139770707816447, -+STORE, 140732689121280, 140732689125375, -+STORE, 140732689108992, 140732689121279, -+STORE, 47862087352320, 47862087360511, -+STORE, 47862087360512, 47862087368703, -+STORE, 47862087368704, 47862087475199, -+STORE, 47862087385088, 47862087475199, -+STORE, 47862087368704, 47862087385087, -+ERASE, 47862087385088, 47862087385088, -+STORE, 47862087385088, 47862087458815, -+STORE, 47862087458816, 47862087475199, -+STORE, 47862087438336, 47862087458815, -+STORE, 47862087385088, 47862087438335, -+ERASE, 47862087385088, 47862087385088, -+STORE, 47862087385088, 47862087438335, -+STORE, 47862087454720, 47862087458815, -+STORE, 47862087438336, 47862087454719, -+ERASE, 47862087438336, 47862087438336, -+STORE, 47862087438336, 47862087454719, -+STORE, 47862087467008, 47862087475199, -+STORE, 47862087458816, 47862087467007, -+ERASE, 47862087458816, 47862087458816, -+STORE, 47862087458816, 47862087467007, -+ERASE, 47862087467008, 47862087467008, -+STORE, 47862087467008, 47862087475199, -+STORE, 47862087475200, 47862089314303, -+STORE, 47862087614464, 47862089314303, -+STORE, 47862087475200, 47862087614463, -+ERASE, 47862087614464, 47862087614464, -+STORE, 47862087614464, 47862089273343, -+STORE, 47862089273344, 47862089314303, -+STORE, 47862088957952, 47862089273343, -+STORE, 47862087614464, 47862088957951, -+ERASE, 47862087614464, 47862087614464, -+STORE, 47862087614464, 47862088957951, -+STORE, 47862089269248, 47862089273343, -+STORE, 47862088957952, 47862089269247, -+ERASE, 47862088957952, 47862088957952, -+STORE, 47862088957952, 47862089269247, -+STORE, 47862089297920, 47862089314303, -+STORE, 47862089273344, 47862089297919, -+ERASE, 47862089273344, 47862089273344, -+STORE, 47862089273344, 47862089297919, -+ERASE, 47862089297920, 47862089297920, -+STORE, 47862089297920, 47862089314303, -+STORE, 47862089297920, 47862089326591, -+ERASE, 47862089273344, 47862089273344, -+STORE, 47862089273344, 47862089289727, -+STORE, 47862089289728, 47862089297919, -+ERASE, 47862087458816, 47862087458816, -+STORE, 47862087458816, 47862087462911, -+STORE, 47862087462912, 47862087467007, -+ERASE, 94629172043776, 94629172043776, -+STORE, 94629172043776, 94629172060159, -+STORE, 94629172060160, 94629172064255, -+ERASE, 139770707804160, 139770707804160, -+STORE, 139770707804160, 139770707808255, -+STORE, 139770707808256, 139770707812351, -+ERASE, 47862087352320, 47862087352320, -+STORE, 94629197533184, 94629197668351, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727540711424, 140737488351231, -+ERASE, 140727540711424, 140727540711424, -+STORE, 140727540711424, 140727540715519, -+STORE, 94299865313280, 94299866025983, -+ERASE, 94299865313280, 94299865313280, -+STORE, 94299865313280, 94299865362431, -+STORE, 94299865362432, 94299866025983, -+ERASE, 94299865362432, 94299865362432, -+STORE, 94299865362432, 94299865907199, -+STORE, 94299865907200, 94299866005503, -+STORE, 94299866005504, 94299866025983, -+STORE, 140680268763136, 140680268935167, -+ERASE, 140680268763136, 140680268763136, -+STORE, 140680268763136, 140680268767231, -+STORE, 140680268767232, 140680268935167, -+ERASE, 140680268767232, 140680268767232, -+STORE, 140680268767232, 140680268890111, -+STORE, 140680268890112, 140680268922879, -+STORE, 140680268922880, 140680268931071, -+STORE, 140680268931072, 140680268935167, -+STORE, 140727541424128, 140727541428223, -+STORE, 140727541411840, 140727541424127, -+STORE, 46952526233600, 46952526241791, -+STORE, 46952526241792, 46952526249983, -+STORE, 46952526249984, 46952526356479, -+STORE, 46952526266368, 46952526356479, -+STORE, 46952526249984, 46952526266367, -+ERASE, 46952526266368, 46952526266368, -+STORE, 46952526266368, 46952526340095, -+STORE, 46952526340096, 46952526356479, -+STORE, 46952526319616, 46952526340095, -+STORE, 46952526266368, 46952526319615, -+ERASE, 46952526266368, 46952526266368, -+STORE, 46952526266368, 46952526319615, -+STORE, 46952526336000, 46952526340095, -+STORE, 46952526319616, 46952526335999, -+ERASE, 46952526319616, 46952526319616, -+STORE, 46952526319616, 46952526335999, -+STORE, 46952526348288, 46952526356479, -+STORE, 46952526340096, 46952526348287, -+ERASE, 46952526340096, 46952526340096, -+STORE, 46952526340096, 46952526348287, -+ERASE, 46952526348288, 46952526348288, -+STORE, 46952526348288, 46952526356479, -+STORE, 46952526356480, 46952528195583, -+STORE, 46952526495744, 46952528195583, -+STORE, 46952526356480, 46952526495743, -+ERASE, 46952526495744, 46952526495744, -+STORE, 46952526495744, 46952528154623, -+STORE, 46952528154624, 46952528195583, -+STORE, 46952527839232, 46952528154623, -+STORE, 46952526495744, 46952527839231, -+ERASE, 46952526495744, 46952526495744, -+STORE, 46952526495744, 46952527839231, -+STORE, 46952528150528, 46952528154623, -+STORE, 46952527839232, 46952528150527, -+ERASE, 46952527839232, 46952527839232, -+STORE, 46952527839232, 46952528150527, -+STORE, 46952528179200, 46952528195583, -+STORE, 46952528154624, 46952528179199, -+ERASE, 46952528154624, 46952528154624, -+STORE, 46952528154624, 46952528179199, -+ERASE, 46952528179200, 46952528179200, -+STORE, 46952528179200, 46952528195583, -+STORE, 46952528179200, 46952528207871, -+ERASE, 46952528154624, 46952528154624, -+STORE, 46952528154624, 46952528171007, -+STORE, 46952528171008, 46952528179199, -+ERASE, 46952526340096, 46952526340096, -+STORE, 46952526340096, 46952526344191, -+STORE, 46952526344192, 46952526348287, -+ERASE, 94299866005504, 94299866005504, -+STORE, 94299866005504, 94299866021887, -+STORE, 94299866021888, 94299866025983, -+ERASE, 140680268922880, 140680268922880, -+STORE, 140680268922880, 140680268926975, -+STORE, 140680268926976, 140680268931071, -+ERASE, 46952526233600, 46952526233600, -+STORE, 140737488347136, 140737488351231, -+STORE, 140722874793984, 140737488351231, -+ERASE, 140722874793984, 140722874793984, -+STORE, 140722874793984, 140722874798079, -+STORE, 94448916213760, 94448916926463, -+ERASE, 94448916213760, 94448916213760, -+STORE, 94448916213760, 94448916262911, -+STORE, 94448916262912, 94448916926463, -+ERASE, 94448916262912, 94448916262912, -+STORE, 94448916262912, 94448916807679, -+STORE, 94448916807680, 94448916905983, -+STORE, 94448916905984, 94448916926463, -+STORE, 140389117046784, 140389117218815, -+ERASE, 140389117046784, 140389117046784, -+STORE, 140389117046784, 140389117050879, -+STORE, 140389117050880, 140389117218815, -+ERASE, 140389117050880, 140389117050880, -+STORE, 140389117050880, 140389117173759, -+STORE, 140389117173760, 140389117206527, -+STORE, 140389117206528, 140389117214719, -+STORE, 140389117214720, 140389117218815, -+STORE, 140722875297792, 140722875301887, -+STORE, 140722875285504, 140722875297791, -+STORE, 47243677949952, 47243677958143, -+STORE, 47243677958144, 47243677966335, -+STORE, 47243677966336, 47243678072831, -+STORE, 47243677982720, 47243678072831, -+STORE, 47243677966336, 47243677982719, -+ERASE, 47243677982720, 47243677982720, -+STORE, 47243677982720, 47243678056447, -+STORE, 47243678056448, 47243678072831, -+STORE, 47243678035968, 47243678056447, -+STORE, 47243677982720, 47243678035967, -+ERASE, 47243677982720, 47243677982720, -+STORE, 47243677982720, 47243678035967, -+STORE, 47243678052352, 47243678056447, -+STORE, 47243678035968, 47243678052351, -+ERASE, 47243678035968, 47243678035968, -+STORE, 47243678035968, 47243678052351, -+STORE, 47243678064640, 47243678072831, -+STORE, 47243678056448, 47243678064639, -+ERASE, 47243678056448, 47243678056448, -+STORE, 47243678056448, 47243678064639, -+ERASE, 47243678064640, 47243678064640, -+STORE, 47243678064640, 47243678072831, -+STORE, 47243678072832, 47243679911935, -+STORE, 47243678212096, 47243679911935, -+STORE, 47243678072832, 47243678212095, -+ERASE, 47243678212096, 47243678212096, -+STORE, 47243678212096, 47243679870975, -+STORE, 47243679870976, 47243679911935, -+STORE, 47243679555584, 47243679870975, -+STORE, 47243678212096, 47243679555583, -+ERASE, 47243678212096, 47243678212096, -+STORE, 47243678212096, 47243679555583, -+STORE, 47243679866880, 47243679870975, -+STORE, 47243679555584, 47243679866879, -+ERASE, 47243679555584, 47243679555584, -+STORE, 47243679555584, 47243679866879, -+STORE, 47243679895552, 47243679911935, -+STORE, 47243679870976, 47243679895551, -+ERASE, 47243679870976, 47243679870976, -+STORE, 47243679870976, 47243679895551, -+ERASE, 47243679895552, 47243679895552, -+STORE, 47243679895552, 47243679911935, -+STORE, 47243679895552, 47243679924223, -+ERASE, 47243679870976, 47243679870976, -+STORE, 47243679870976, 47243679887359, -+STORE, 47243679887360, 47243679895551, -+ERASE, 47243678056448, 47243678056448, -+STORE, 47243678056448, 47243678060543, -+STORE, 47243678060544, 47243678064639, -+ERASE, 94448916905984, 94448916905984, -+STORE, 94448916905984, 94448916922367, -+STORE, 94448916922368, 94448916926463, -+ERASE, 140389117206528, 140389117206528, -+STORE, 140389117206528, 140389117210623, -+STORE, 140389117210624, 140389117214719, -+ERASE, 47243677949952, 47243677949952, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733068505088, 140737488351231, -+ERASE, 140733068505088, 140733068505088, -+STORE, 140733068505088, 140733068509183, -+STORE, 94207145750528, 94207146463231, -+ERASE, 94207145750528, 94207145750528, -+STORE, 94207145750528, 94207145799679, -+STORE, 94207145799680, 94207146463231, -+ERASE, 94207145799680, 94207145799680, -+STORE, 94207145799680, 94207146344447, -+STORE, 94207146344448, 94207146442751, -+STORE, 94207146442752, 94207146463231, -+STORE, 140684504911872, 140684505083903, -+ERASE, 140684504911872, 140684504911872, -+STORE, 140684504911872, 140684504915967, -+STORE, 140684504915968, 140684505083903, -+ERASE, 140684504915968, 140684504915968, -+STORE, 140684504915968, 140684505038847, -+STORE, 140684505038848, 140684505071615, -+STORE, 140684505071616, 140684505079807, -+STORE, 140684505079808, 140684505083903, -+STORE, 140733068607488, 140733068611583, -+STORE, 140733068595200, 140733068607487, -+STORE, 46948290084864, 46948290093055, -+STORE, 46948290093056, 46948290101247, -+STORE, 46948290101248, 46948290207743, -+STORE, 46948290117632, 46948290207743, -+STORE, 46948290101248, 46948290117631, -+ERASE, 46948290117632, 46948290117632, -+STORE, 46948290117632, 46948290191359, -+STORE, 46948290191360, 46948290207743, -+STORE, 46948290170880, 46948290191359, -+STORE, 46948290117632, 46948290170879, -+ERASE, 46948290117632, 46948290117632, -+STORE, 46948290117632, 46948290170879, -+STORE, 46948290187264, 46948290191359, -+STORE, 46948290170880, 46948290187263, -+ERASE, 46948290170880, 46948290170880, -+STORE, 46948290170880, 46948290187263, -+STORE, 46948290199552, 46948290207743, -+STORE, 46948290191360, 46948290199551, -+ERASE, 46948290191360, 46948290191360, -+STORE, 46948290191360, 46948290199551, -+ERASE, 46948290199552, 46948290199552, -+STORE, 46948290199552, 46948290207743, -+STORE, 46948290207744, 46948292046847, -+STORE, 46948290347008, 46948292046847, -+STORE, 46948290207744, 46948290347007, -+ERASE, 46948290347008, 46948290347008, -+STORE, 46948290347008, 46948292005887, -+STORE, 46948292005888, 46948292046847, -+STORE, 46948291690496, 46948292005887, -+STORE, 46948290347008, 46948291690495, -+ERASE, 46948290347008, 46948290347008, -+STORE, 46948290347008, 46948291690495, -+STORE, 46948292001792, 46948292005887, -+STORE, 46948291690496, 46948292001791, -+ERASE, 46948291690496, 46948291690496, -+STORE, 46948291690496, 46948292001791, -+STORE, 46948292030464, 46948292046847, -+STORE, 46948292005888, 46948292030463, -+ERASE, 46948292005888, 46948292005888, -+STORE, 46948292005888, 46948292030463, -+ERASE, 46948292030464, 46948292030464, -+STORE, 46948292030464, 46948292046847, -+STORE, 46948292030464, 46948292059135, -+ERASE, 46948292005888, 46948292005888, -+STORE, 46948292005888, 46948292022271, -+STORE, 46948292022272, 46948292030463, -+ERASE, 46948290191360, 46948290191360, -+STORE, 46948290191360, 46948290195455, -+STORE, 46948290195456, 46948290199551, -+ERASE, 94207146442752, 94207146442752, -+STORE, 94207146442752, 94207146459135, -+STORE, 94207146459136, 94207146463231, -+ERASE, 140684505071616, 140684505071616, -+STORE, 140684505071616, 140684505075711, -+STORE, 140684505075712, 140684505079807, -+ERASE, 46948290084864, 46948290084864, -+STORE, 140737488347136, 140737488351231, -+STORE, 140726367158272, 140737488351231, -+ERASE, 140726367158272, 140726367158272, -+STORE, 140726367158272, 140726367162367, -+STORE, 94436124106752, 94436124819455, -+ERASE, 94436124106752, 94436124106752, -+STORE, 94436124106752, 94436124155903, -+STORE, 94436124155904, 94436124819455, -+ERASE, 94436124155904, 94436124155904, -+STORE, 94436124155904, 94436124700671, -+STORE, 94436124700672, 94436124798975, -+STORE, 94436124798976, 94436124819455, -+STORE, 140049025044480, 140049025216511, -+ERASE, 140049025044480, 140049025044480, -+STORE, 140049025044480, 140049025048575, -+STORE, 140049025048576, 140049025216511, -+ERASE, 140049025048576, 140049025048576, -+STORE, 140049025048576, 140049025171455, -+STORE, 140049025171456, 140049025204223, -+STORE, 140049025204224, 140049025212415, -+STORE, 140049025212416, 140049025216511, -+STORE, 140726367256576, 140726367260671, -+STORE, 140726367244288, 140726367256575, -+STORE, 47583769952256, 47583769960447, -+STORE, 47583769960448, 47583769968639, -+STORE, 47583769968640, 47583770075135, -+STORE, 47583769985024, 47583770075135, -+STORE, 47583769968640, 47583769985023, -+ERASE, 47583769985024, 47583769985024, -+STORE, 47583769985024, 47583770058751, -+STORE, 47583770058752, 47583770075135, -+STORE, 47583770038272, 47583770058751, -+STORE, 47583769985024, 47583770038271, -+ERASE, 47583769985024, 47583769985024, -+STORE, 47583769985024, 47583770038271, -+STORE, 47583770054656, 47583770058751, -+STORE, 47583770038272, 47583770054655, -+ERASE, 47583770038272, 47583770038272, -+STORE, 47583770038272, 47583770054655, -+STORE, 47583770066944, 47583770075135, -+STORE, 47583770058752, 47583770066943, -+ERASE, 47583770058752, 47583770058752, -+STORE, 47583770058752, 47583770066943, -+ERASE, 47583770066944, 47583770066944, -+STORE, 47583770066944, 47583770075135, -+STORE, 47583770075136, 47583771914239, -+STORE, 47583770214400, 47583771914239, -+STORE, 47583770075136, 47583770214399, -+ERASE, 47583770214400, 47583770214400, -+STORE, 47583770214400, 47583771873279, -+STORE, 47583771873280, 47583771914239, -+STORE, 47583771557888, 47583771873279, -+STORE, 47583770214400, 47583771557887, -+ERASE, 47583770214400, 47583770214400, -+STORE, 47583770214400, 47583771557887, -+STORE, 47583771869184, 47583771873279, -+STORE, 47583771557888, 47583771869183, -+ERASE, 47583771557888, 47583771557888, -+STORE, 47583771557888, 47583771869183, -+STORE, 47583771897856, 47583771914239, -+STORE, 47583771873280, 47583771897855, -+ERASE, 47583771873280, 47583771873280, -+STORE, 47583771873280, 47583771897855, -+ERASE, 47583771897856, 47583771897856, -+STORE, 47583771897856, 47583771914239, -+STORE, 47583771897856, 47583771926527, -+ERASE, 47583771873280, 47583771873280, -+STORE, 47583771873280, 47583771889663, -+STORE, 47583771889664, 47583771897855, -+ERASE, 47583770058752, 47583770058752, -+STORE, 47583770058752, 47583770062847, -+STORE, 47583770062848, 47583770066943, -+ERASE, 94436124798976, 94436124798976, -+STORE, 94436124798976, 94436124815359, -+STORE, 94436124815360, 94436124819455, -+ERASE, 140049025204224, 140049025204224, -+STORE, 140049025204224, 140049025208319, -+STORE, 140049025208320, 140049025212415, -+ERASE, 47583769952256, 47583769952256, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727116099584, 140737488351231, -+ERASE, 140727116099584, 140727116099584, -+STORE, 140727116099584, 140727116103679, -+STORE, 94166319734784, 94166320447487, -+ERASE, 94166319734784, 94166319734784, -+STORE, 94166319734784, 94166319783935, -+STORE, 94166319783936, 94166320447487, -+ERASE, 94166319783936, 94166319783936, -+STORE, 94166319783936, 94166320328703, -+STORE, 94166320328704, 94166320427007, -+STORE, 94166320427008, 94166320447487, -+STORE, 139976559542272, 139976559714303, -+ERASE, 139976559542272, 139976559542272, -+STORE, 139976559542272, 139976559546367, -+STORE, 139976559546368, 139976559714303, -+ERASE, 139976559546368, 139976559546368, -+STORE, 139976559546368, 139976559669247, -+STORE, 139976559669248, 139976559702015, -+STORE, 139976559702016, 139976559710207, -+STORE, 139976559710208, 139976559714303, -+STORE, 140727116222464, 140727116226559, -+STORE, 140727116210176, 140727116222463, -+STORE, 47656235454464, 47656235462655, -+STORE, 47656235462656, 47656235470847, -+STORE, 47656235470848, 47656235577343, -+STORE, 47656235487232, 47656235577343, -+STORE, 47656235470848, 47656235487231, -+ERASE, 47656235487232, 47656235487232, -+STORE, 47656235487232, 47656235560959, -+STORE, 47656235560960, 47656235577343, -+STORE, 47656235540480, 47656235560959, -+STORE, 47656235487232, 47656235540479, -+ERASE, 47656235487232, 47656235487232, -+STORE, 47656235487232, 47656235540479, -+STORE, 47656235556864, 47656235560959, -+STORE, 47656235540480, 47656235556863, -+ERASE, 47656235540480, 47656235540480, -+STORE, 47656235540480, 47656235556863, -+STORE, 47656235569152, 47656235577343, -+STORE, 47656235560960, 47656235569151, -+ERASE, 47656235560960, 47656235560960, -+STORE, 47656235560960, 47656235569151, -+ERASE, 47656235569152, 47656235569152, -+STORE, 47656235569152, 47656235577343, -+STORE, 47656235577344, 47656237416447, -+STORE, 47656235716608, 47656237416447, -+STORE, 47656235577344, 47656235716607, -+ERASE, 47656235716608, 47656235716608, -+STORE, 47656235716608, 47656237375487, -+STORE, 47656237375488, 47656237416447, -+STORE, 47656237060096, 47656237375487, -+STORE, 47656235716608, 47656237060095, -+ERASE, 47656235716608, 47656235716608, -+STORE, 47656235716608, 47656237060095, -+STORE, 47656237371392, 47656237375487, -+STORE, 47656237060096, 47656237371391, -+ERASE, 47656237060096, 47656237060096, -+STORE, 47656237060096, 47656237371391, -+STORE, 47656237400064, 47656237416447, -+STORE, 47656237375488, 47656237400063, -+ERASE, 47656237375488, 47656237375488, -+STORE, 47656237375488, 47656237400063, -+ERASE, 47656237400064, 47656237400064, -+STORE, 47656237400064, 47656237416447, -+STORE, 47656237400064, 47656237428735, -+ERASE, 47656237375488, 47656237375488, -+STORE, 47656237375488, 47656237391871, -+STORE, 47656237391872, 47656237400063, -+ERASE, 47656235560960, 47656235560960, -+STORE, 47656235560960, 47656235565055, -+STORE, 47656235565056, 47656235569151, -+ERASE, 94166320427008, 94166320427008, -+STORE, 94166320427008, 94166320443391, -+STORE, 94166320443392, 94166320447487, -+ERASE, 139976559702016, 139976559702016, -+STORE, 139976559702016, 139976559706111, -+STORE, 139976559706112, 139976559710207, -+ERASE, 47656235454464, 47656235454464, -+STORE, 94166332153856, 94166332289023, -+STORE, 140737488347136, 140737488351231, -+STORE, 140726412816384, 140737488351231, -+ERASE, 140726412816384, 140726412816384, -+STORE, 140726412816384, 140726412820479, -+STORE, 94094884507648, 94094885220351, -+ERASE, 94094884507648, 94094884507648, -+STORE, 94094884507648, 94094884556799, -+STORE, 94094884556800, 94094885220351, -+ERASE, 94094884556800, 94094884556800, -+STORE, 94094884556800, 94094885101567, -+STORE, 94094885101568, 94094885199871, -+STORE, 94094885199872, 94094885220351, -+STORE, 139773773938688, 139773774110719, -+ERASE, 139773773938688, 139773773938688, -+STORE, 139773773938688, 139773773942783, -+STORE, 139773773942784, 139773774110719, -+ERASE, 139773773942784, 139773773942784, -+STORE, 139773773942784, 139773774065663, -+STORE, 139773774065664, 139773774098431, -+STORE, 139773774098432, 139773774106623, -+STORE, 139773774106624, 139773774110719, -+STORE, 140726412963840, 140726412967935, -+STORE, 140726412951552, 140726412963839, -+STORE, 47859021058048, 47859021066239, -+STORE, 47859021066240, 47859021074431, -+STORE, 47859021074432, 47859021180927, -+STORE, 47859021090816, 47859021180927, -+STORE, 47859021074432, 47859021090815, -+ERASE, 47859021090816, 47859021090816, -+STORE, 47859021090816, 47859021164543, -+STORE, 47859021164544, 47859021180927, -+STORE, 47859021144064, 47859021164543, -+STORE, 47859021090816, 47859021144063, -+ERASE, 47859021090816, 47859021090816, -+STORE, 47859021090816, 47859021144063, -+STORE, 47859021160448, 47859021164543, -+STORE, 47859021144064, 47859021160447, -+ERASE, 47859021144064, 47859021144064, -+STORE, 47859021144064, 47859021160447, -+STORE, 47859021172736, 47859021180927, -+STORE, 47859021164544, 47859021172735, -+ERASE, 47859021164544, 47859021164544, -+STORE, 47859021164544, 47859021172735, -+ERASE, 47859021172736, 47859021172736, -+STORE, 47859021172736, 47859021180927, -+STORE, 47859021180928, 47859023020031, -+STORE, 47859021320192, 47859023020031, -+STORE, 47859021180928, 47859021320191, -+ERASE, 47859021320192, 47859021320192, -+STORE, 47859021320192, 47859022979071, -+STORE, 47859022979072, 47859023020031, -+STORE, 47859022663680, 47859022979071, -+STORE, 47859021320192, 47859022663679, -+ERASE, 47859021320192, 47859021320192, -+STORE, 47859021320192, 47859022663679, -+STORE, 47859022974976, 47859022979071, -+STORE, 47859022663680, 47859022974975, -+ERASE, 47859022663680, 47859022663680, -+STORE, 47859022663680, 47859022974975, -+STORE, 47859023003648, 47859023020031, -+STORE, 47859022979072, 47859023003647, -+ERASE, 47859022979072, 47859022979072, -+STORE, 47859022979072, 47859023003647, -+ERASE, 47859023003648, 47859023003648, -+STORE, 47859023003648, 47859023020031, -+STORE, 47859023003648, 47859023032319, -+ERASE, 47859022979072, 47859022979072, -+STORE, 47859022979072, 47859022995455, -+STORE, 47859022995456, 47859023003647, -+ERASE, 47859021164544, 47859021164544, -+STORE, 47859021164544, 47859021168639, -+STORE, 47859021168640, 47859021172735, -+ERASE, 94094885199872, 94094885199872, -+STORE, 94094885199872, 94094885216255, -+STORE, 94094885216256, 94094885220351, -+ERASE, 139773774098432, 139773774098432, -+STORE, 139773774098432, 139773774102527, -+STORE, 139773774102528, 139773774106623, -+ERASE, 47859021058048, 47859021058048, -+STORE, 94094901108736, 94094901243903, -+STORE, 140737488347136, 140737488351231, -+STORE, 140736567963648, 140737488351231, -+ERASE, 140736567963648, 140736567963648, -+STORE, 140736567963648, 140736567967743, -+STORE, 94924425748480, 94924426461183, -+ERASE, 94924425748480, 94924425748480, -+STORE, 94924425748480, 94924425797631, -+STORE, 94924425797632, 94924426461183, -+ERASE, 94924425797632, 94924425797632, -+STORE, 94924425797632, 94924426342399, -+STORE, 94924426342400, 94924426440703, -+STORE, 94924426440704, 94924426461183, -+STORE, 140042126319616, 140042126491647, -+ERASE, 140042126319616, 140042126319616, -+STORE, 140042126319616, 140042126323711, -+STORE, 140042126323712, 140042126491647, -+ERASE, 140042126323712, 140042126323712, -+STORE, 140042126323712, 140042126446591, -+STORE, 140042126446592, 140042126479359, -+STORE, 140042126479360, 140042126487551, -+STORE, 140042126487552, 140042126491647, -+STORE, 140736568672256, 140736568676351, -+STORE, 140736568659968, 140736568672255, -+STORE, 47590668677120, 47590668685311, -+STORE, 47590668685312, 47590668693503, -+STORE, 47590668693504, 47590668799999, -+STORE, 47590668709888, 47590668799999, -+STORE, 47590668693504, 47590668709887, -+ERASE, 47590668709888, 47590668709888, -+STORE, 47590668709888, 47590668783615, -+STORE, 47590668783616, 47590668799999, -+STORE, 47590668763136, 47590668783615, -+STORE, 47590668709888, 47590668763135, -+ERASE, 47590668709888, 47590668709888, -+STORE, 47590668709888, 47590668763135, -+STORE, 47590668779520, 47590668783615, -+STORE, 47590668763136, 47590668779519, -+ERASE, 47590668763136, 47590668763136, -+STORE, 47590668763136, 47590668779519, -+STORE, 47590668791808, 47590668799999, -+STORE, 47590668783616, 47590668791807, -+ERASE, 47590668783616, 47590668783616, -+STORE, 47590668783616, 47590668791807, -+ERASE, 47590668791808, 47590668791808, -+STORE, 47590668791808, 47590668799999, -+STORE, 47590668800000, 47590670639103, -+STORE, 47590668939264, 47590670639103, -+STORE, 47590668800000, 47590668939263, -+ERASE, 47590668939264, 47590668939264, -+STORE, 47590668939264, 47590670598143, -+STORE, 47590670598144, 47590670639103, -+STORE, 47590670282752, 47590670598143, -+STORE, 47590668939264, 47590670282751, -+ERASE, 47590668939264, 47590668939264, -+STORE, 47590668939264, 47590670282751, -+STORE, 47590670594048, 47590670598143, -+STORE, 47590670282752, 47590670594047, -+ERASE, 47590670282752, 47590670282752, -+STORE, 47590670282752, 47590670594047, -+STORE, 47590670622720, 47590670639103, -+STORE, 47590670598144, 47590670622719, -+ERASE, 47590670598144, 47590670598144, -+STORE, 47590670598144, 47590670622719, -+ERASE, 47590670622720, 47590670622720, -+STORE, 47590670622720, 47590670639103, -+STORE, 47590670622720, 47590670651391, -+ERASE, 47590670598144, 47590670598144, -+STORE, 47590670598144, 47590670614527, -+STORE, 47590670614528, 47590670622719, -+ERASE, 47590668783616, 47590668783616, -+STORE, 47590668783616, 47590668787711, -+STORE, 47590668787712, 47590668791807, -+ERASE, 94924426440704, 94924426440704, -+STORE, 94924426440704, 94924426457087, -+STORE, 94924426457088, 94924426461183, -+ERASE, 140042126479360, 140042126479360, -+STORE, 140042126479360, 140042126483455, -+STORE, 140042126483456, 140042126487551, -+ERASE, 47590668677120, 47590668677120, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733281439744, 140737488351231, -+ERASE, 140733281439744, 140733281439744, -+STORE, 140733281439744, 140733281443839, -+STORE, 94490667069440, 94490667782143, -+ERASE, 94490667069440, 94490667069440, -+STORE, 94490667069440, 94490667118591, -+STORE, 94490667118592, 94490667782143, -+ERASE, 94490667118592, 94490667118592, -+STORE, 94490667118592, 94490667663359, -+STORE, 94490667663360, 94490667761663, -+STORE, 94490667761664, 94490667782143, -+STORE, 139878215118848, 139878215290879, -+ERASE, 139878215118848, 139878215118848, -+STORE, 139878215118848, 139878215122943, -+STORE, 139878215122944, 139878215290879, -+ERASE, 139878215122944, 139878215122944, -+STORE, 139878215122944, 139878215245823, -+STORE, 139878215245824, 139878215278591, -+STORE, 139878215278592, 139878215286783, -+STORE, 139878215286784, 139878215290879, -+STORE, 140733281464320, 140733281468415, -+STORE, 140733281452032, 140733281464319, -+STORE, 47754579877888, 47754579886079, -+STORE, 47754579886080, 47754579894271, -+STORE, 47754579894272, 47754580000767, -+STORE, 47754579910656, 47754580000767, -+STORE, 47754579894272, 47754579910655, -+ERASE, 47754579910656, 47754579910656, -+STORE, 47754579910656, 47754579984383, -+STORE, 47754579984384, 47754580000767, -+STORE, 47754579963904, 47754579984383, -+STORE, 47754579910656, 47754579963903, -+ERASE, 47754579910656, 47754579910656, -+STORE, 47754579910656, 47754579963903, -+STORE, 47754579980288, 47754579984383, -+STORE, 47754579963904, 47754579980287, -+ERASE, 47754579963904, 47754579963904, -+STORE, 47754579963904, 47754579980287, -+STORE, 47754579992576, 47754580000767, -+STORE, 47754579984384, 47754579992575, -+ERASE, 47754579984384, 47754579984384, -+STORE, 47754579984384, 47754579992575, -+ERASE, 47754579992576, 47754579992576, -+STORE, 47754579992576, 47754580000767, -+STORE, 47754580000768, 47754581839871, -+STORE, 47754580140032, 47754581839871, -+STORE, 47754580000768, 47754580140031, -+ERASE, 47754580140032, 47754580140032, -+STORE, 47754580140032, 47754581798911, -+STORE, 47754581798912, 47754581839871, -+STORE, 47754581483520, 47754581798911, -+STORE, 47754580140032, 47754581483519, -+ERASE, 47754580140032, 47754580140032, -+STORE, 47754580140032, 47754581483519, -+STORE, 47754581794816, 47754581798911, -+STORE, 47754581483520, 47754581794815, -+ERASE, 47754581483520, 47754581483520, -+STORE, 47754581483520, 47754581794815, -+STORE, 47754581823488, 47754581839871, -+STORE, 47754581798912, 47754581823487, -+ERASE, 47754581798912, 47754581798912, -+STORE, 47754581798912, 47754581823487, -+ERASE, 47754581823488, 47754581823488, -+STORE, 47754581823488, 47754581839871, -+STORE, 47754581823488, 47754581852159, -+ERASE, 47754581798912, 47754581798912, -+STORE, 47754581798912, 47754581815295, -+STORE, 47754581815296, 47754581823487, -+ERASE, 47754579984384, 47754579984384, -+STORE, 47754579984384, 47754579988479, -+STORE, 47754579988480, 47754579992575, -+ERASE, 94490667761664, 94490667761664, -+STORE, 94490667761664, 94490667778047, -+STORE, 94490667778048, 94490667782143, -+ERASE, 139878215278592, 139878215278592, -+STORE, 139878215278592, 139878215282687, -+STORE, 139878215282688, 139878215286783, -+ERASE, 47754579877888, 47754579877888, -+STORE, 94490669649920, 94490669785087, -+STORE, 140737488347136, 140737488351231, -+STORE, 140735382188032, 140737488351231, -+ERASE, 140735382188032, 140735382188032, -+STORE, 140735382188032, 140735382192127, -+STORE, 94150181302272, 94150182014975, -+ERASE, 94150181302272, 94150181302272, -+STORE, 94150181302272, 94150181351423, -+STORE, 94150181351424, 94150182014975, -+ERASE, 94150181351424, 94150181351424, -+STORE, 94150181351424, 94150181896191, -+STORE, 94150181896192, 94150181994495, -+STORE, 94150181994496, 94150182014975, -+STORE, 139679752458240, 139679752630271, -+ERASE, 139679752458240, 139679752458240, -+STORE, 139679752458240, 139679752462335, -+STORE, 139679752462336, 139679752630271, -+ERASE, 139679752462336, 139679752462336, -+STORE, 139679752462336, 139679752585215, -+STORE, 139679752585216, 139679752617983, -+STORE, 139679752617984, 139679752626175, -+STORE, 139679752626176, 139679752630271, -+STORE, 140735382536192, 140735382540287, -+STORE, 140735382523904, 140735382536191, -+STORE, 47953042538496, 47953042546687, -+STORE, 47953042546688, 47953042554879, -+STORE, 47953042554880, 47953042661375, -+STORE, 47953042571264, 47953042661375, -+STORE, 47953042554880, 47953042571263, -+ERASE, 47953042571264, 47953042571264, -+STORE, 47953042571264, 47953042644991, -+STORE, 47953042644992, 47953042661375, -+STORE, 47953042624512, 47953042644991, -+STORE, 47953042571264, 47953042624511, -+ERASE, 47953042571264, 47953042571264, -+STORE, 47953042571264, 47953042624511, -+STORE, 47953042640896, 47953042644991, -+STORE, 47953042624512, 47953042640895, -+ERASE, 47953042624512, 47953042624512, -+STORE, 47953042624512, 47953042640895, -+STORE, 47953042653184, 47953042661375, -+STORE, 47953042644992, 47953042653183, -+ERASE, 47953042644992, 47953042644992, -+STORE, 47953042644992, 47953042653183, -+ERASE, 47953042653184, 47953042653184, -+STORE, 47953042653184, 47953042661375, -+STORE, 47953042661376, 47953044500479, -+STORE, 47953042800640, 47953044500479, -+STORE, 47953042661376, 47953042800639, -+ERASE, 47953042800640, 47953042800640, -+STORE, 47953042800640, 47953044459519, -+STORE, 47953044459520, 47953044500479, -+STORE, 47953044144128, 47953044459519, -+STORE, 47953042800640, 47953044144127, -+ERASE, 47953042800640, 47953042800640, -+STORE, 47953042800640, 47953044144127, -+STORE, 47953044455424, 47953044459519, -+STORE, 47953044144128, 47953044455423, -+ERASE, 47953044144128, 47953044144128, -+STORE, 47953044144128, 47953044455423, -+STORE, 47953044484096, 47953044500479, -+STORE, 47953044459520, 47953044484095, -+ERASE, 47953044459520, 47953044459520, -+STORE, 47953044459520, 47953044484095, -+ERASE, 47953044484096, 47953044484096, -+STORE, 47953044484096, 47953044500479, -+STORE, 47953044484096, 47953044512767, -+ERASE, 47953044459520, 47953044459520, -+STORE, 47953044459520, 47953044475903, -+STORE, 47953044475904, 47953044484095, -+ERASE, 47953042644992, 47953042644992, -+STORE, 47953042644992, 47953042649087, -+STORE, 47953042649088, 47953042653183, -+ERASE, 94150181994496, 94150181994496, -+STORE, 94150181994496, 94150182010879, -+STORE, 94150182010880, 94150182014975, -+ERASE, 139679752617984, 139679752617984, -+STORE, 139679752617984, 139679752622079, -+STORE, 139679752622080, 139679752626175, -+ERASE, 47953042538496, 47953042538496, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737044123648, 140737488351231, -+ERASE, 140737044123648, 140737044123648, -+STORE, 140737044123648, 140737044127743, -+STORE, 94425324294144, 94425325006847, -+ERASE, 94425324294144, 94425324294144, -+STORE, 94425324294144, 94425324343295, -+STORE, 94425324343296, 94425325006847, -+ERASE, 94425324343296, 94425324343296, -+STORE, 94425324343296, 94425324888063, -+STORE, 94425324888064, 94425324986367, -+STORE, 94425324986368, 94425325006847, -+STORE, 140382015016960, 140382015188991, -+ERASE, 140382015016960, 140382015016960, -+STORE, 140382015016960, 140382015021055, -+STORE, 140382015021056, 140382015188991, -+ERASE, 140382015021056, 140382015021056, -+STORE, 140382015021056, 140382015143935, -+STORE, 140382015143936, 140382015176703, -+STORE, 140382015176704, 140382015184895, -+STORE, 140382015184896, 140382015188991, -+STORE, 140737045585920, 140737045590015, -+STORE, 140737045573632, 140737045585919, -+STORE, 47250779979776, 47250779987967, -+STORE, 47250779987968, 47250779996159, -+STORE, 47250779996160, 47250780102655, -+STORE, 47250780012544, 47250780102655, -+STORE, 47250779996160, 47250780012543, -+ERASE, 47250780012544, 47250780012544, -+STORE, 47250780012544, 47250780086271, -+STORE, 47250780086272, 47250780102655, -+STORE, 47250780065792, 47250780086271, -+STORE, 47250780012544, 47250780065791, -+ERASE, 47250780012544, 47250780012544, -+STORE, 47250780012544, 47250780065791, -+STORE, 47250780082176, 47250780086271, -+STORE, 47250780065792, 47250780082175, -+ERASE, 47250780065792, 47250780065792, -+STORE, 47250780065792, 47250780082175, -+STORE, 47250780094464, 47250780102655, -+STORE, 47250780086272, 47250780094463, -+ERASE, 47250780086272, 47250780086272, -+STORE, 47250780086272, 47250780094463, -+ERASE, 47250780094464, 47250780094464, -+STORE, 47250780094464, 47250780102655, -+STORE, 47250780102656, 47250781941759, -+STORE, 47250780241920, 47250781941759, -+STORE, 47250780102656, 47250780241919, -+ERASE, 47250780241920, 47250780241920, -+STORE, 47250780241920, 47250781900799, -+STORE, 47250781900800, 47250781941759, -+STORE, 47250781585408, 47250781900799, -+STORE, 47250780241920, 47250781585407, -+ERASE, 47250780241920, 47250780241920, -+STORE, 47250780241920, 47250781585407, -+STORE, 47250781896704, 47250781900799, -+STORE, 47250781585408, 47250781896703, -+ERASE, 47250781585408, 47250781585408, -+STORE, 47250781585408, 47250781896703, -+STORE, 47250781925376, 47250781941759, -+STORE, 47250781900800, 47250781925375, -+ERASE, 47250781900800, 47250781900800, -+STORE, 47250781900800, 47250781925375, -+ERASE, 47250781925376, 47250781925376, -+STORE, 47250781925376, 47250781941759, -+STORE, 47250781925376, 47250781954047, -+ERASE, 47250781900800, 47250781900800, -+STORE, 47250781900800, 47250781917183, -+STORE, 47250781917184, 47250781925375, -+ERASE, 47250780086272, 47250780086272, -+STORE, 47250780086272, 47250780090367, -+STORE, 47250780090368, 47250780094463, -+ERASE, 94425324986368, 94425324986368, -+STORE, 94425324986368, 94425325002751, -+STORE, 94425325002752, 94425325006847, -+ERASE, 140382015176704, 140382015176704, -+STORE, 140382015176704, 140382015180799, -+STORE, 140382015180800, 140382015184895, -+ERASE, 47250779979776, 47250779979776, -+STORE, 94425351438336, 94425351573503, -+STORE, 140737488347136, 140737488351231, -+STORE, 140736801144832, 140737488351231, -+ERASE, 140736801144832, 140736801144832, -+STORE, 140736801144832, 140736801148927, -+STORE, 94629429358592, 94629430071295, -+ERASE, 94629429358592, 94629429358592, -+STORE, 94629429358592, 94629429407743, -+STORE, 94629429407744, 94629430071295, -+ERASE, 94629429407744, 94629429407744, -+STORE, 94629429407744, 94629429952511, -+STORE, 94629429952512, 94629430050815, -+STORE, 94629430050816, 94629430071295, -+STORE, 139801685483520, 139801685655551, -+ERASE, 139801685483520, 139801685483520, -+STORE, 139801685483520, 139801685487615, -+STORE, 139801685487616, 139801685655551, -+ERASE, 139801685487616, 139801685487616, -+STORE, 139801685487616, 139801685610495, -+STORE, 139801685610496, 139801685643263, -+STORE, 139801685643264, 139801685651455, -+STORE, 139801685651456, 139801685655551, -+STORE, 140736801198080, 140736801202175, -+STORE, 140736801185792, 140736801198079, -+STORE, 47831109513216, 47831109521407, -+STORE, 47831109521408, 47831109529599, -+STORE, 47831109529600, 47831109636095, -+STORE, 47831109545984, 47831109636095, -+STORE, 47831109529600, 47831109545983, -+ERASE, 47831109545984, 47831109545984, -+STORE, 47831109545984, 47831109619711, -+STORE, 47831109619712, 47831109636095, -+STORE, 47831109599232, 47831109619711, -+STORE, 47831109545984, 47831109599231, -+ERASE, 47831109545984, 47831109545984, -+STORE, 47831109545984, 47831109599231, -+STORE, 47831109615616, 47831109619711, -+STORE, 47831109599232, 47831109615615, -+ERASE, 47831109599232, 47831109599232, -+STORE, 47831109599232, 47831109615615, -+STORE, 47831109627904, 47831109636095, -+STORE, 47831109619712, 47831109627903, -+ERASE, 47831109619712, 47831109619712, -+STORE, 47831109619712, 47831109627903, -+ERASE, 47831109627904, 47831109627904, -+STORE, 47831109627904, 47831109636095, -+STORE, 47831109636096, 47831111475199, -+STORE, 47831109775360, 47831111475199, -+STORE, 47831109636096, 47831109775359, -+ERASE, 47831109775360, 47831109775360, -+STORE, 47831109775360, 47831111434239, -+STORE, 47831111434240, 47831111475199, -+STORE, 47831111118848, 47831111434239, -+STORE, 47831109775360, 47831111118847, -+ERASE, 47831109775360, 47831109775360, -+STORE, 47831109775360, 47831111118847, -+STORE, 47831111430144, 47831111434239, -+STORE, 47831111118848, 47831111430143, -+ERASE, 47831111118848, 47831111118848, -+STORE, 47831111118848, 47831111430143, -+STORE, 47831111458816, 47831111475199, -+STORE, 47831111434240, 47831111458815, -+ERASE, 47831111434240, 47831111434240, -+STORE, 47831111434240, 47831111458815, -+ERASE, 47831111458816, 47831111458816, -+STORE, 47831111458816, 47831111475199, -+STORE, 47831111458816, 47831111487487, -+ERASE, 47831111434240, 47831111434240, -+STORE, 47831111434240, 47831111450623, -+STORE, 47831111450624, 47831111458815, -+ERASE, 47831109619712, 47831109619712, -+STORE, 47831109619712, 47831109623807, -+STORE, 47831109623808, 47831109627903, -+ERASE, 94629430050816, 94629430050816, -+STORE, 94629430050816, 94629430067199, -+STORE, 94629430067200, 94629430071295, -+ERASE, 139801685643264, 139801685643264, -+STORE, 139801685643264, 139801685647359, -+STORE, 139801685647360, 139801685651455, -+ERASE, 47831109513216, 47831109513216, -+STORE, 140737488347136, 140737488351231, -+STORE, 140729419612160, 140737488351231, -+ERASE, 140729419612160, 140729419612160, -+STORE, 140729419612160, 140729419616255, -+STORE, 94443354148864, 94443354861567, -+ERASE, 94443354148864, 94443354148864, -+STORE, 94443354148864, 94443354198015, -+STORE, 94443354198016, 94443354861567, -+ERASE, 94443354198016, 94443354198016, -+STORE, 94443354198016, 94443354742783, -+STORE, 94443354742784, 94443354841087, -+STORE, 94443354841088, 94443354861567, -+STORE, 139741700038656, 139741700210687, -+ERASE, 139741700038656, 139741700038656, -+STORE, 139741700038656, 139741700042751, -+STORE, 139741700042752, 139741700210687, -+ERASE, 139741700042752, 139741700042752, -+STORE, 139741700042752, 139741700165631, -+STORE, 139741700165632, 139741700198399, -+STORE, 139741700198400, 139741700206591, -+STORE, 139741700206592, 139741700210687, -+STORE, 140729420574720, 140729420578815, -+STORE, 140729420562432, 140729420574719, -+STORE, 47891094958080, 47891094966271, -+STORE, 47891094966272, 47891094974463, -+STORE, 47891094974464, 47891095080959, -+STORE, 47891094990848, 47891095080959, -+STORE, 47891094974464, 47891094990847, -+ERASE, 47891094990848, 47891094990848, -+STORE, 47891094990848, 47891095064575, -+STORE, 47891095064576, 47891095080959, -+STORE, 47891095044096, 47891095064575, -+STORE, 47891094990848, 47891095044095, -+ERASE, 47891094990848, 47891094990848, -+STORE, 47891094990848, 47891095044095, -+STORE, 47891095060480, 47891095064575, -+STORE, 47891095044096, 47891095060479, -+ERASE, 47891095044096, 47891095044096, -+STORE, 47891095044096, 47891095060479, -+STORE, 47891095072768, 47891095080959, -+STORE, 47891095064576, 47891095072767, -+ERASE, 47891095064576, 47891095064576, -+STORE, 47891095064576, 47891095072767, -+ERASE, 47891095072768, 47891095072768, -+STORE, 47891095072768, 47891095080959, -+STORE, 47891095080960, 47891096920063, -+STORE, 47891095220224, 47891096920063, -+STORE, 47891095080960, 47891095220223, -+ERASE, 47891095220224, 47891095220224, -+STORE, 47891095220224, 47891096879103, -+STORE, 47891096879104, 47891096920063, -+STORE, 47891096563712, 47891096879103, -+STORE, 47891095220224, 47891096563711, -+ERASE, 47891095220224, 47891095220224, -+STORE, 47891095220224, 47891096563711, -+STORE, 47891096875008, 47891096879103, -+STORE, 47891096563712, 47891096875007, -+ERASE, 47891096563712, 47891096563712, -+STORE, 47891096563712, 47891096875007, -+STORE, 47891096903680, 47891096920063, -+STORE, 47891096879104, 47891096903679, -+ERASE, 47891096879104, 47891096879104, -+STORE, 47891096879104, 47891096903679, -+ERASE, 47891096903680, 47891096903680, -+STORE, 47891096903680, 47891096920063, -+STORE, 47891096903680, 47891096932351, -+ERASE, 47891096879104, 47891096879104, -+STORE, 47891096879104, 47891096895487, -+STORE, 47891096895488, 47891096903679, -+ERASE, 47891095064576, 47891095064576, -+STORE, 47891095064576, 47891095068671, -+STORE, 47891095068672, 47891095072767, -+ERASE, 94443354841088, 94443354841088, -+STORE, 94443354841088, 94443354857471, -+STORE, 94443354857472, 94443354861567, -+ERASE, 139741700198400, 139741700198400, -+STORE, 139741700198400, 139741700202495, -+STORE, 139741700202496, 139741700206591, -+ERASE, 47891094958080, 47891094958080, -+STORE, 94443360825344, 94443360960511, -+STORE, 140737488347136, 140737488351231, -+STORE, 140722961661952, 140737488351231, -+ERASE, 140722961661952, 140722961661952, -+STORE, 140722961661952, 140722961666047, -+STORE, 94878388944896, 94878389657599, -+ERASE, 94878388944896, 94878388944896, -+STORE, 94878388944896, 94878388994047, -+STORE, 94878388994048, 94878389657599, -+ERASE, 94878388994048, 94878388994048, -+STORE, 94878388994048, 94878389538815, -+STORE, 94878389538816, 94878389637119, -+STORE, 94878389637120, 94878389657599, -+STORE, 140210690056192, 140210690228223, -+ERASE, 140210690056192, 140210690056192, -+STORE, 140210690056192, 140210690060287, -+STORE, 140210690060288, 140210690228223, -+ERASE, 140210690060288, 140210690060288, -+STORE, 140210690060288, 140210690183167, -+STORE, 140210690183168, 140210690215935, -+STORE, 140210690215936, 140210690224127, -+STORE, 140210690224128, 140210690228223, -+STORE, 140722963148800, 140722963152895, -+STORE, 140722963136512, 140722963148799, -+STORE, 47422104940544, 47422104948735, -+STORE, 47422104948736, 47422104956927, -+STORE, 47422104956928, 47422105063423, -+STORE, 47422104973312, 47422105063423, -+STORE, 47422104956928, 47422104973311, -+ERASE, 47422104973312, 47422104973312, -+STORE, 47422104973312, 47422105047039, -+STORE, 47422105047040, 47422105063423, -+STORE, 47422105026560, 47422105047039, -+STORE, 47422104973312, 47422105026559, -+ERASE, 47422104973312, 47422104973312, -+STORE, 47422104973312, 47422105026559, -+STORE, 47422105042944, 47422105047039, -+STORE, 47422105026560, 47422105042943, -+ERASE, 47422105026560, 47422105026560, -+STORE, 47422105026560, 47422105042943, -+STORE, 47422105055232, 47422105063423, -+STORE, 47422105047040, 47422105055231, -+ERASE, 47422105047040, 47422105047040, -+STORE, 47422105047040, 47422105055231, -+ERASE, 47422105055232, 47422105055232, -+STORE, 47422105055232, 47422105063423, -+STORE, 47422105063424, 47422106902527, -+STORE, 47422105202688, 47422106902527, -+STORE, 47422105063424, 47422105202687, -+ERASE, 47422105202688, 47422105202688, -+STORE, 47422105202688, 47422106861567, -+STORE, 47422106861568, 47422106902527, -+STORE, 47422106546176, 47422106861567, -+STORE, 47422105202688, 47422106546175, -+ERASE, 47422105202688, 47422105202688, -+STORE, 47422105202688, 47422106546175, -+STORE, 47422106857472, 47422106861567, -+STORE, 47422106546176, 47422106857471, -+ERASE, 47422106546176, 47422106546176, -+STORE, 47422106546176, 47422106857471, -+STORE, 47422106886144, 47422106902527, -+STORE, 47422106861568, 47422106886143, -+ERASE, 47422106861568, 47422106861568, -+STORE, 47422106861568, 47422106886143, -+ERASE, 47422106886144, 47422106886144, -+STORE, 47422106886144, 47422106902527, -+STORE, 47422106886144, 47422106914815, -+ERASE, 47422106861568, 47422106861568, -+STORE, 47422106861568, 47422106877951, -+STORE, 47422106877952, 47422106886143, -+ERASE, 47422105047040, 47422105047040, -+STORE, 47422105047040, 47422105051135, -+STORE, 47422105051136, 47422105055231, -+ERASE, 94878389637120, 94878389637120, -+STORE, 94878389637120, 94878389653503, -+STORE, 94878389653504, 94878389657599, -+ERASE, 140210690215936, 140210690215936, -+STORE, 140210690215936, 140210690220031, -+STORE, 140210690220032, 140210690224127, -+ERASE, 47422104940544, 47422104940544, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727690309632, 140737488351231, -+ERASE, 140727690309632, 140727690309632, -+STORE, 140727690309632, 140727690313727, -+STORE, 94121892208640, 94121892921343, -+ERASE, 94121892208640, 94121892208640, -+STORE, 94121892208640, 94121892257791, -+STORE, 94121892257792, 94121892921343, -+ERASE, 94121892257792, 94121892257792, -+STORE, 94121892257792, 94121892802559, -+STORE, 94121892802560, 94121892900863, -+STORE, 94121892900864, 94121892921343, -+STORE, 140662438326272, 140662438498303, -+ERASE, 140662438326272, 140662438326272, -+STORE, 140662438326272, 140662438330367, -+STORE, 140662438330368, 140662438498303, -+ERASE, 140662438330368, 140662438330368, -+STORE, 140662438330368, 140662438453247, -+STORE, 140662438453248, 140662438486015, -+STORE, 140662438486016, 140662438494207, -+STORE, 140662438494208, 140662438498303, -+STORE, 140727690379264, 140727690383359, -+STORE, 140727690366976, 140727690379263, -+STORE, 46970356670464, 46970356678655, -+STORE, 46970356678656, 46970356686847, -+STORE, 46970356686848, 46970356793343, -+STORE, 46970356703232, 46970356793343, -+STORE, 46970356686848, 46970356703231, -+ERASE, 46970356703232, 46970356703232, -+STORE, 46970356703232, 46970356776959, -+STORE, 46970356776960, 46970356793343, -+STORE, 46970356756480, 46970356776959, -+STORE, 46970356703232, 46970356756479, -+ERASE, 46970356703232, 46970356703232, -+STORE, 46970356703232, 46970356756479, -+STORE, 46970356772864, 46970356776959, -+STORE, 46970356756480, 46970356772863, -+ERASE, 46970356756480, 46970356756480, -+STORE, 46970356756480, 46970356772863, -+STORE, 46970356785152, 46970356793343, -+STORE, 46970356776960, 46970356785151, -+ERASE, 46970356776960, 46970356776960, -+STORE, 46970356776960, 46970356785151, -+ERASE, 46970356785152, 46970356785152, -+STORE, 46970356785152, 46970356793343, -+STORE, 46970356793344, 46970358632447, -+STORE, 46970356932608, 46970358632447, -+STORE, 46970356793344, 46970356932607, -+ERASE, 46970356932608, 46970356932608, -+STORE, 46970356932608, 46970358591487, -+STORE, 46970358591488, 46970358632447, -+STORE, 46970358276096, 46970358591487, -+STORE, 46970356932608, 46970358276095, -+ERASE, 46970356932608, 46970356932608, -+STORE, 46970356932608, 46970358276095, -+STORE, 46970358587392, 46970358591487, -+STORE, 46970358276096, 46970358587391, -+ERASE, 46970358276096, 46970358276096, -+STORE, 46970358276096, 46970358587391, -+STORE, 46970358616064, 46970358632447, -+STORE, 46970358591488, 46970358616063, -+ERASE, 46970358591488, 46970358591488, -+STORE, 46970358591488, 46970358616063, -+ERASE, 46970358616064, 46970358616064, -+STORE, 46970358616064, 46970358632447, -+STORE, 46970358616064, 46970358644735, -+ERASE, 46970358591488, 46970358591488, -+STORE, 46970358591488, 46970358607871, -+STORE, 46970358607872, 46970358616063, -+ERASE, 46970356776960, 46970356776960, -+STORE, 46970356776960, 46970356781055, -+STORE, 46970356781056, 46970356785151, -+ERASE, 94121892900864, 94121892900864, -+STORE, 94121892900864, 94121892917247, -+STORE, 94121892917248, 94121892921343, -+ERASE, 140662438486016, 140662438486016, -+STORE, 140662438486016, 140662438490111, -+STORE, 140662438490112, 140662438494207, -+ERASE, 46970356670464, 46970356670464, -+STORE, 94121898610688, 94121898745855, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737189351424, 140737488351231, -+ERASE, 140737189351424, 140737189351424, -+STORE, 140737189351424, 140737189355519, -+STORE, 93847948832768, 93847949545471, -+ERASE, 93847948832768, 93847948832768, -+STORE, 93847948832768, 93847948881919, -+STORE, 93847948881920, 93847949545471, -+ERASE, 93847948881920, 93847948881920, -+STORE, 93847948881920, 93847949426687, -+STORE, 93847949426688, 93847949524991, -+STORE, 93847949524992, 93847949545471, -+STORE, 139698989985792, 139698990157823, -+ERASE, 139698989985792, 139698989985792, -+STORE, 139698989985792, 139698989989887, -+STORE, 139698989989888, 139698990157823, -+ERASE, 139698989989888, 139698989989888, -+STORE, 139698989989888, 139698990112767, -+STORE, 139698990112768, 139698990145535, -+STORE, 139698990145536, 139698990153727, -+STORE, 139698990153728, 139698990157823, -+STORE, 140737189744640, 140737189748735, -+STORE, 140737189732352, 140737189744639, -+STORE, 47933805010944, 47933805019135, -+STORE, 47933805019136, 47933805027327, -+STORE, 47933805027328, 47933805133823, -+STORE, 47933805043712, 47933805133823, -+STORE, 47933805027328, 47933805043711, -+ERASE, 47933805043712, 47933805043712, -+STORE, 47933805043712, 47933805117439, -+STORE, 47933805117440, 47933805133823, -+STORE, 47933805096960, 47933805117439, -+STORE, 47933805043712, 47933805096959, -+ERASE, 47933805043712, 47933805043712, -+STORE, 47933805043712, 47933805096959, -+STORE, 47933805113344, 47933805117439, -+STORE, 47933805096960, 47933805113343, -+ERASE, 47933805096960, 47933805096960, -+STORE, 47933805096960, 47933805113343, -+STORE, 47933805125632, 47933805133823, -+STORE, 47933805117440, 47933805125631, -+ERASE, 47933805117440, 47933805117440, -+STORE, 47933805117440, 47933805125631, -+ERASE, 47933805125632, 47933805125632, -+STORE, 47933805125632, 47933805133823, -+STORE, 47933805133824, 47933806972927, -+STORE, 47933805273088, 47933806972927, -+STORE, 47933805133824, 47933805273087, -+ERASE, 47933805273088, 47933805273088, -+STORE, 47933805273088, 47933806931967, -+STORE, 47933806931968, 47933806972927, -+STORE, 47933806616576, 47933806931967, -+STORE, 47933805273088, 47933806616575, -+ERASE, 47933805273088, 47933805273088, -+STORE, 47933805273088, 47933806616575, -+STORE, 47933806927872, 47933806931967, -+STORE, 47933806616576, 47933806927871, -+ERASE, 47933806616576, 47933806616576, -+STORE, 47933806616576, 47933806927871, -+STORE, 47933806956544, 47933806972927, -+STORE, 47933806931968, 47933806956543, -+ERASE, 47933806931968, 47933806931968, -+STORE, 47933806931968, 47933806956543, -+ERASE, 47933806956544, 47933806956544, -+STORE, 47933806956544, 47933806972927, -+STORE, 47933806956544, 47933806985215, -+ERASE, 47933806931968, 47933806931968, -+STORE, 47933806931968, 47933806948351, -+STORE, 47933806948352, 47933806956543, -+ERASE, 47933805117440, 47933805117440, -+STORE, 47933805117440, 47933805121535, -+STORE, 47933805121536, 47933805125631, -+ERASE, 93847949524992, 93847949524992, -+STORE, 93847949524992, 93847949541375, -+STORE, 93847949541376, 93847949545471, -+ERASE, 139698990145536, 139698990145536, -+STORE, 139698990145536, 139698990149631, -+STORE, 139698990149632, 139698990153727, -+ERASE, 47933805010944, 47933805010944, -+STORE, 140737488347136, 140737488351231, -+STORE, 140725553991680, 140737488351231, -+ERASE, 140725553991680, 140725553991680, -+STORE, 140725553991680, 140725553995775, -+STORE, 93980056248320, 93980056961023, -+ERASE, 93980056248320, 93980056248320, -+STORE, 93980056248320, 93980056297471, -+STORE, 93980056297472, 93980056961023, -+ERASE, 93980056297472, 93980056297472, -+STORE, 93980056297472, 93980056842239, -+STORE, 93980056842240, 93980056940543, -+STORE, 93980056940544, 93980056961023, -+STORE, 140146588971008, 140146589143039, -+ERASE, 140146588971008, 140146588971008, -+STORE, 140146588971008, 140146588975103, -+STORE, 140146588975104, 140146589143039, -+ERASE, 140146588975104, 140146588975104, -+STORE, 140146588975104, 140146589097983, -+STORE, 140146589097984, 140146589130751, -+STORE, 140146589130752, 140146589138943, -+STORE, 140146589138944, 140146589143039, -+STORE, 140725554860032, 140725554864127, -+STORE, 140725554847744, 140725554860031, -+STORE, 47486206025728, 47486206033919, -+STORE, 47486206033920, 47486206042111, -+STORE, 47486206042112, 47486206148607, -+STORE, 47486206058496, 47486206148607, -+STORE, 47486206042112, 47486206058495, -+ERASE, 47486206058496, 47486206058496, -+STORE, 47486206058496, 47486206132223, -+STORE, 47486206132224, 47486206148607, -+STORE, 47486206111744, 47486206132223, -+STORE, 47486206058496, 47486206111743, -+ERASE, 47486206058496, 47486206058496, -+STORE, 47486206058496, 47486206111743, -+STORE, 47486206128128, 47486206132223, -+STORE, 47486206111744, 47486206128127, -+ERASE, 47486206111744, 47486206111744, -+STORE, 47486206111744, 47486206128127, -+STORE, 47486206140416, 47486206148607, -+STORE, 47486206132224, 47486206140415, -+ERASE, 47486206132224, 47486206132224, -+STORE, 47486206132224, 47486206140415, -+ERASE, 47486206140416, 47486206140416, -+STORE, 47486206140416, 47486206148607, -+STORE, 47486206148608, 47486207987711, -+STORE, 47486206287872, 47486207987711, -+STORE, 47486206148608, 47486206287871, -+ERASE, 47486206287872, 47486206287872, -+STORE, 47486206287872, 47486207946751, -+STORE, 47486207946752, 47486207987711, -+STORE, 47486207631360, 47486207946751, -+STORE, 47486206287872, 47486207631359, -+ERASE, 47486206287872, 47486206287872, -+STORE, 47486206287872, 47486207631359, -+STORE, 47486207942656, 47486207946751, -+STORE, 47486207631360, 47486207942655, -+ERASE, 47486207631360, 47486207631360, -+STORE, 47486207631360, 47486207942655, -+STORE, 47486207971328, 47486207987711, -+STORE, 47486207946752, 47486207971327, -+ERASE, 47486207946752, 47486207946752, -+STORE, 47486207946752, 47486207971327, -+ERASE, 47486207971328, 47486207971328, -+STORE, 47486207971328, 47486207987711, -+STORE, 47486207971328, 47486207999999, -+ERASE, 47486207946752, 47486207946752, -+STORE, 47486207946752, 47486207963135, -+STORE, 47486207963136, 47486207971327, -+ERASE, 47486206132224, 47486206132224, -+STORE, 47486206132224, 47486206136319, -+STORE, 47486206136320, 47486206140415, -+ERASE, 93980056940544, 93980056940544, -+STORE, 93980056940544, 93980056956927, -+STORE, 93980056956928, 93980056961023, -+ERASE, 140146589130752, 140146589130752, -+STORE, 140146589130752, 140146589134847, -+STORE, 140146589134848, 140146589138943, -+ERASE, 47486206025728, 47486206025728, -+STORE, 93980070006784, 93980070141951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727334776832, 140737488351231, -+ERASE, 140727334776832, 140727334776832, -+STORE, 140727334776832, 140727334780927, -+STORE, 94049747247104, 94049747959807, -+ERASE, 94049747247104, 94049747247104, -+STORE, 94049747247104, 94049747296255, -+STORE, 94049747296256, 94049747959807, -+ERASE, 94049747296256, 94049747296256, -+STORE, 94049747296256, 94049747841023, -+STORE, 94049747841024, 94049747939327, -+STORE, 94049747939328, 94049747959807, -+STORE, 140227307216896, 140227307388927, -+ERASE, 140227307216896, 140227307216896, -+STORE, 140227307216896, 140227307220991, -+STORE, 140227307220992, 140227307388927, -+ERASE, 140227307220992, 140227307220992, -+STORE, 140227307220992, 140227307343871, -+STORE, 140227307343872, 140227307376639, -+STORE, 140227307376640, 140227307384831, -+STORE, 140227307384832, 140227307388927, -+STORE, 140727335337984, 140727335342079, -+STORE, 140727335325696, 140727335337983, -+STORE, 47405487779840, 47405487788031, -+STORE, 47405487788032, 47405487796223, -+STORE, 47405487796224, 47405487902719, -+STORE, 47405487812608, 47405487902719, -+STORE, 47405487796224, 47405487812607, -+ERASE, 47405487812608, 47405487812608, -+STORE, 47405487812608, 47405487886335, -+STORE, 47405487886336, 47405487902719, -+STORE, 47405487865856, 47405487886335, -+STORE, 47405487812608, 47405487865855, -+ERASE, 47405487812608, 47405487812608, -+STORE, 47405487812608, 47405487865855, -+STORE, 47405487882240, 47405487886335, -+STORE, 47405487865856, 47405487882239, -+ERASE, 47405487865856, 47405487865856, -+STORE, 47405487865856, 47405487882239, -+STORE, 47405487894528, 47405487902719, -+STORE, 47405487886336, 47405487894527, -+ERASE, 47405487886336, 47405487886336, -+STORE, 47405487886336, 47405487894527, -+ERASE, 47405487894528, 47405487894528, -+STORE, 47405487894528, 47405487902719, -+STORE, 47405487902720, 47405489741823, -+STORE, 47405488041984, 47405489741823, -+STORE, 47405487902720, 47405488041983, -+ERASE, 47405488041984, 47405488041984, -+STORE, 47405488041984, 47405489700863, -+STORE, 47405489700864, 47405489741823, -+STORE, 47405489385472, 47405489700863, -+STORE, 47405488041984, 47405489385471, -+ERASE, 47405488041984, 47405488041984, -+STORE, 47405488041984, 47405489385471, -+STORE, 47405489696768, 47405489700863, -+STORE, 47405489385472, 47405489696767, -+ERASE, 47405489385472, 47405489385472, -+STORE, 47405489385472, 47405489696767, -+STORE, 47405489725440, 47405489741823, -+STORE, 47405489700864, 47405489725439, -+ERASE, 47405489700864, 47405489700864, -+STORE, 47405489700864, 47405489725439, -+ERASE, 47405489725440, 47405489725440, -+STORE, 47405489725440, 47405489741823, -+STORE, 47405489725440, 47405489754111, -+ERASE, 47405489700864, 47405489700864, -+STORE, 47405489700864, 47405489717247, -+STORE, 47405489717248, 47405489725439, -+ERASE, 47405487886336, 47405487886336, -+STORE, 47405487886336, 47405487890431, -+STORE, 47405487890432, 47405487894527, -+ERASE, 94049747939328, 94049747939328, -+STORE, 94049747939328, 94049747955711, -+STORE, 94049747955712, 94049747959807, -+ERASE, 140227307376640, 140227307376640, -+STORE, 140227307376640, 140227307380735, -+STORE, 140227307380736, 140227307384831, -+ERASE, 47405487779840, 47405487779840, -+STORE, 94049758810112, 94049758945279, -+STORE, 140737488347136, 140737488351231, -+STORE, 140727079718912, 140737488351231, -+ERASE, 140727079718912, 140727079718912, -+STORE, 140727079718912, 140727079723007, -+STORE, 94250996527104, 94250997239807, -+ERASE, 94250996527104, 94250996527104, -+STORE, 94250996527104, 94250996576255, -+STORE, 94250996576256, 94250997239807, -+ERASE, 94250996576256, 94250996576256, -+STORE, 94250996576256, 94250997121023, -+STORE, 94250997121024, 94250997219327, -+STORE, 94250997219328, 94250997239807, -+STORE, 140060022587392, 140060022759423, -+ERASE, 140060022587392, 140060022587392, -+STORE, 140060022587392, 140060022591487, -+STORE, 140060022591488, 140060022759423, -+ERASE, 140060022591488, 140060022591488, -+STORE, 140060022591488, 140060022714367, -+STORE, 140060022714368, 140060022747135, -+STORE, 140060022747136, 140060022755327, -+STORE, 140060022755328, 140060022759423, -+STORE, 140727079788544, 140727079792639, -+STORE, 140727079776256, 140727079788543, -+STORE, 47572772409344, 47572772417535, -+STORE, 47572772417536, 47572772425727, -+STORE, 47572772425728, 47572772532223, -+STORE, 47572772442112, 47572772532223, -+STORE, 47572772425728, 47572772442111, -+ERASE, 47572772442112, 47572772442112, -+STORE, 47572772442112, 47572772515839, -+STORE, 47572772515840, 47572772532223, -+STORE, 47572772495360, 47572772515839, -+STORE, 47572772442112, 47572772495359, -+ERASE, 47572772442112, 47572772442112, -+STORE, 47572772442112, 47572772495359, -+STORE, 47572772511744, 47572772515839, -+STORE, 47572772495360, 47572772511743, -+ERASE, 47572772495360, 47572772495360, -+STORE, 47572772495360, 47572772511743, -+STORE, 47572772524032, 47572772532223, -+STORE, 47572772515840, 47572772524031, -+ERASE, 47572772515840, 47572772515840, -+STORE, 47572772515840, 47572772524031, -+ERASE, 47572772524032, 47572772524032, -+STORE, 47572772524032, 47572772532223, -+STORE, 47572772532224, 47572774371327, -+STORE, 47572772671488, 47572774371327, -+STORE, 47572772532224, 47572772671487, -+ERASE, 47572772671488, 47572772671488, -+STORE, 47572772671488, 47572774330367, -+STORE, 47572774330368, 47572774371327, -+STORE, 47572774014976, 47572774330367, -+STORE, 47572772671488, 47572774014975, -+ERASE, 47572772671488, 47572772671488, -+STORE, 47572772671488, 47572774014975, -+STORE, 47572774326272, 47572774330367, -+STORE, 47572774014976, 47572774326271, -+ERASE, 47572774014976, 47572774014976, -+STORE, 47572774014976, 47572774326271, -+STORE, 47572774354944, 47572774371327, -+STORE, 47572774330368, 47572774354943, -+ERASE, 47572774330368, 47572774330368, -+STORE, 47572774330368, 47572774354943, -+ERASE, 47572774354944, 47572774354944, -+STORE, 47572774354944, 47572774371327, -+STORE, 47572774354944, 47572774383615, -+ERASE, 47572774330368, 47572774330368, -+STORE, 47572774330368, 47572774346751, -+STORE, 47572774346752, 47572774354943, -+ERASE, 47572772515840, 47572772515840, -+STORE, 47572772515840, 47572772519935, -+STORE, 47572772519936, 47572772524031, -+ERASE, 94250997219328, 94250997219328, -+STORE, 94250997219328, 94250997235711, -+STORE, 94250997235712, 94250997239807, -+ERASE, 140060022747136, 140060022747136, -+STORE, 140060022747136, 140060022751231, -+STORE, 140060022751232, 140060022755327, -+ERASE, 47572772409344, 47572772409344, -+STORE, 94251018305536, 94251018440703, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730012389376, 140737488351231, -+ERASE, 140730012389376, 140730012389376, -+STORE, 140730012389376, 140730012393471, -+STORE, 94382607675392, 94382607695871, -+ERASE, 94382607675392, 94382607675392, -+STORE, 94382607675392, 94382607679487, -+STORE, 94382607679488, 94382607695871, -+ERASE, 94382607679488, 94382607679488, -+STORE, 94382607679488, 94382607683583, -+STORE, 94382607683584, 94382607687679, -+STORE, 94382607687680, 94382607695871, -+STORE, 140252451454976, 140252451627007, -+ERASE, 140252451454976, 140252451454976, -+STORE, 140252451454976, 140252451459071, -+STORE, 140252451459072, 140252451627007, -+ERASE, 140252451459072, 140252451459072, -+STORE, 140252451459072, 140252451581951, -+STORE, 140252451581952, 140252451614719, -+STORE, 140252451614720, 140252451622911, -+STORE, 140252451622912, 140252451627007, -+STORE, 140730013548544, 140730013552639, -+STORE, 140730013536256, 140730013548543, -+STORE, 47380343541760, 47380343549951, -+STORE, 47380343549952, 47380343558143, -+STORE, 47380343558144, 47380345397247, -+STORE, 47380343697408, 47380345397247, -+STORE, 47380343558144, 47380343697407, -+ERASE, 47380343697408, 47380343697408, -+STORE, 47380343697408, 47380345356287, -+STORE, 47380345356288, 47380345397247, -+STORE, 47380345040896, 47380345356287, -+STORE, 47380343697408, 47380345040895, -+ERASE, 47380343697408, 47380343697408, -+STORE, 47380343697408, 47380345040895, -+STORE, 47380345352192, 47380345356287, -+STORE, 47380345040896, 47380345352191, -+ERASE, 47380345040896, 47380345040896, -+STORE, 47380345040896, 47380345352191, -+STORE, 47380345380864, 47380345397247, -+STORE, 47380345356288, 47380345380863, -+ERASE, 47380345356288, 47380345356288, -+STORE, 47380345356288, 47380345380863, -+ERASE, 47380345380864, 47380345380864, -+STORE, 47380345380864, 47380345397247, -+ERASE, 47380345356288, 47380345356288, -+STORE, 47380345356288, 47380345372671, -+STORE, 47380345372672, 47380345380863, -+ERASE, 94382607687680, 94382607687680, -+STORE, 94382607687680, 94382607691775, -+STORE, 94382607691776, 94382607695871, -+ERASE, 140252451614720, 140252451614720, -+STORE, 140252451614720, 140252451618815, -+STORE, 140252451618816, 140252451622911, -+ERASE, 47380343541760, 47380343541760, -+STORE, 94382626803712, 94382626938879, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730900271104, 140737488351231, -+ERASE, 140730900271104, 140730900271104, -+STORE, 140730900271104, 140730900275199, -+STORE, 93855478120448, 93855478337535, -+ERASE, 93855478120448, 93855478120448, -+STORE, 93855478120448, 93855478198271, -+STORE, 93855478198272, 93855478337535, -+ERASE, 93855478198272, 93855478198272, -+STORE, 93855478198272, 93855478243327, -+STORE, 93855478243328, 93855478288383, -+STORE, 93855478288384, 93855478337535, -+STORE, 140092686573568, 140092686745599, -+ERASE, 140092686573568, 140092686573568, -+STORE, 140092686573568, 140092686577663, -+STORE, 140092686577664, 140092686745599, -+ERASE, 140092686577664, 140092686577664, -+STORE, 140092686577664, 140092686700543, -+STORE, 140092686700544, 140092686733311, -+STORE, 140092686733312, 140092686741503, -+STORE, 140092686741504, 140092686745599, -+STORE, 140730900537344, 140730900541439, -+STORE, 140730900525056, 140730900537343, -+STORE, 47540108423168, 47540108431359, -+STORE, 47540108431360, 47540108439551, -+STORE, 47540108439552, 47540110278655, -+STORE, 47540108578816, 47540110278655, -+STORE, 47540108439552, 47540108578815, -+ERASE, 47540108578816, 47540108578816, -+STORE, 47540108578816, 47540110237695, -+STORE, 47540110237696, 47540110278655, -+STORE, 47540109922304, 47540110237695, -+STORE, 47540108578816, 47540109922303, -+ERASE, 47540108578816, 47540108578816, -+STORE, 47540108578816, 47540109922303, -+STORE, 47540110233600, 47540110237695, -+STORE, 47540109922304, 47540110233599, -+ERASE, 47540109922304, 47540109922304, -+STORE, 47540109922304, 47540110233599, -+STORE, 47540110262272, 47540110278655, -+STORE, 47540110237696, 47540110262271, -+ERASE, 47540110237696, 47540110237696, -+STORE, 47540110237696, 47540110262271, -+ERASE, 47540110262272, 47540110262272, -+STORE, 47540110262272, 47540110278655, -+ERASE, 47540110237696, 47540110237696, -+STORE, 47540110237696, 47540110254079, -+STORE, 47540110254080, 47540110262271, -+ERASE, 93855478288384, 93855478288384, -+STORE, 93855478288384, 93855478333439, -+STORE, 93855478333440, 93855478337535, -+ERASE, 140092686733312, 140092686733312, -+STORE, 140092686733312, 140092686737407, -+STORE, 140092686737408, 140092686741503, -+ERASE, 47540108423168, 47540108423168, -+STORE, 93855492222976, 93855492358143, -+STORE, 93855492222976, 93855492493311, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733498146816, 140737488351231, -+ERASE, 140733498146816, 140733498146816, -+STORE, 140733498146816, 140733498150911, -+STORE, 94170739654656, 94170740367359, -+ERASE, 94170739654656, 94170739654656, -+STORE, 94170739654656, 94170739703807, -+STORE, 94170739703808, 94170740367359, -+ERASE, 94170739703808, 94170739703808, -+STORE, 94170739703808, 94170740248575, -+STORE, 94170740248576, 94170740346879, -+STORE, 94170740346880, 94170740367359, -+STORE, 140024788877312, 140024789049343, -+ERASE, 140024788877312, 140024788877312, -+STORE, 140024788877312, 140024788881407, -+STORE, 140024788881408, 140024789049343, -+ERASE, 140024788881408, 140024788881408, -+STORE, 140024788881408, 140024789004287, -+STORE, 140024789004288, 140024789037055, -+STORE, 140024789037056, 140024789045247, -+STORE, 140024789045248, 140024789049343, -+STORE, 140733499023360, 140733499027455, -+STORE, 140733499011072, 140733499023359, -+STORE, 47608006119424, 47608006127615, -+STORE, 47608006127616, 47608006135807, -+STORE, 47608006135808, 47608006242303, -+STORE, 47608006152192, 47608006242303, -+STORE, 47608006135808, 47608006152191, -+ERASE, 47608006152192, 47608006152192, -+STORE, 47608006152192, 47608006225919, -+STORE, 47608006225920, 47608006242303, -+STORE, 47608006205440, 47608006225919, -+STORE, 47608006152192, 47608006205439, -+ERASE, 47608006152192, 47608006152192, -+STORE, 47608006152192, 47608006205439, -+STORE, 47608006221824, 47608006225919, -+STORE, 47608006205440, 47608006221823, -+ERASE, 47608006205440, 47608006205440, -+STORE, 47608006205440, 47608006221823, -+STORE, 47608006234112, 47608006242303, -+STORE, 47608006225920, 47608006234111, -+ERASE, 47608006225920, 47608006225920, -+STORE, 47608006225920, 47608006234111, -+ERASE, 47608006234112, 47608006234112, -+STORE, 47608006234112, 47608006242303, -+STORE, 47608006242304, 47608008081407, -+STORE, 47608006381568, 47608008081407, -+STORE, 47608006242304, 47608006381567, -+ERASE, 47608006381568, 47608006381568, -+STORE, 47608006381568, 47608008040447, -+STORE, 47608008040448, 47608008081407, -+STORE, 47608007725056, 47608008040447, -+STORE, 47608006381568, 47608007725055, -+ERASE, 47608006381568, 47608006381568, -+STORE, 47608006381568, 47608007725055, -+STORE, 47608008036352, 47608008040447, -+STORE, 47608007725056, 47608008036351, -+ERASE, 47608007725056, 47608007725056, -+STORE, 47608007725056, 47608008036351, -+STORE, 47608008065024, 47608008081407, -+STORE, 47608008040448, 47608008065023, -+ERASE, 47608008040448, 47608008040448, -+STORE, 47608008040448, 47608008065023, -+ERASE, 47608008065024, 47608008065024, -+STORE, 47608008065024, 47608008081407, -+STORE, 47608008065024, 47608008093695, -+ERASE, 47608008040448, 47608008040448, -+STORE, 47608008040448, 47608008056831, -+STORE, 47608008056832, 47608008065023, -+ERASE, 47608006225920, 47608006225920, -+STORE, 47608006225920, 47608006230015, -+STORE, 47608006230016, 47608006234111, -+ERASE, 94170740346880, 94170740346880, -+STORE, 94170740346880, 94170740363263, -+STORE, 94170740363264, 94170740367359, -+ERASE, 140024789037056, 140024789037056, -+STORE, 140024789037056, 140024789041151, -+STORE, 140024789041152, 140024789045247, -+ERASE, 47608006119424, 47608006119424, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730264326144, 140737488351231, -+ERASE, 140730264326144, 140730264326144, -+STORE, 140730264326144, 140730264330239, -+STORE, 94653216407552, 94653217120255, -+ERASE, 94653216407552, 94653216407552, -+STORE, 94653216407552, 94653216456703, -+STORE, 94653216456704, 94653217120255, -+ERASE, 94653216456704, 94653216456704, -+STORE, 94653216456704, 94653217001471, -+STORE, 94653217001472, 94653217099775, -+STORE, 94653217099776, 94653217120255, -+STORE, 140103617011712, 140103617183743, -+ERASE, 140103617011712, 140103617011712, -+STORE, 140103617011712, 140103617015807, -+STORE, 140103617015808, 140103617183743, -+ERASE, 140103617015808, 140103617015808, -+STORE, 140103617015808, 140103617138687, -+STORE, 140103617138688, 140103617171455, -+STORE, 140103617171456, 140103617179647, -+STORE, 140103617179648, 140103617183743, -+STORE, 140730265427968, 140730265432063, -+STORE, 140730265415680, 140730265427967, -+STORE, 47529177985024, 47529177993215, -+STORE, 47529177993216, 47529178001407, -+STORE, 47529178001408, 47529178107903, -+STORE, 47529178017792, 47529178107903, -+STORE, 47529178001408, 47529178017791, -+ERASE, 47529178017792, 47529178017792, -+STORE, 47529178017792, 47529178091519, -+STORE, 47529178091520, 47529178107903, -+STORE, 47529178071040, 47529178091519, -+STORE, 47529178017792, 47529178071039, -+ERASE, 47529178017792, 47529178017792, -+STORE, 47529178017792, 47529178071039, -+STORE, 47529178087424, 47529178091519, -+STORE, 47529178071040, 47529178087423, -+ERASE, 47529178071040, 47529178071040, -+STORE, 47529178071040, 47529178087423, -+STORE, 47529178099712, 47529178107903, -+STORE, 47529178091520, 47529178099711, -+ERASE, 47529178091520, 47529178091520, -+STORE, 47529178091520, 47529178099711, -+ERASE, 47529178099712, 47529178099712, -+STORE, 47529178099712, 47529178107903, -+STORE, 47529178107904, 47529179947007, -+STORE, 47529178247168, 47529179947007, -+STORE, 47529178107904, 47529178247167, -+ERASE, 47529178247168, 47529178247168, -+STORE, 47529178247168, 47529179906047, -+STORE, 47529179906048, 47529179947007, -+STORE, 47529179590656, 47529179906047, -+STORE, 47529178247168, 47529179590655, -+ERASE, 47529178247168, 47529178247168, -+STORE, 47529178247168, 47529179590655, -+STORE, 47529179901952, 47529179906047, -+STORE, 47529179590656, 47529179901951, -+ERASE, 47529179590656, 47529179590656, -+STORE, 47529179590656, 47529179901951, -+STORE, 47529179930624, 47529179947007, -+STORE, 47529179906048, 47529179930623, -+ERASE, 47529179906048, 47529179906048, -+STORE, 47529179906048, 47529179930623, -+ERASE, 47529179930624, 47529179930624, -+STORE, 47529179930624, 47529179947007, -+STORE, 47529179930624, 47529179959295, -+ERASE, 47529179906048, 47529179906048, -+STORE, 47529179906048, 47529179922431, -+STORE, 47529179922432, 47529179930623, -+ERASE, 47529178091520, 47529178091520, -+STORE, 47529178091520, 47529178095615, -+STORE, 47529178095616, 47529178099711, -+ERASE, 94653217099776, 94653217099776, -+STORE, 94653217099776, 94653217116159, -+STORE, 94653217116160, 94653217120255, -+ERASE, 140103617171456, 140103617171456, -+STORE, 140103617171456, 140103617175551, -+STORE, 140103617175552, 140103617179647, -+ERASE, 47529177985024, 47529177985024, -+STORE, 94653241135104, 94653241270271, -+STORE, 140737488347136, 140737488351231, -+STORE, 140736284549120, 140737488351231, -+ERASE, 140736284549120, 140736284549120, -+STORE, 140736284549120, 140736284553215, -+STORE, 93963663822848, 93963664506879, -+ERASE, 93963663822848, 93963663822848, -+STORE, 93963663822848, 93963663884287, -+STORE, 93963663884288, 93963664506879, -+ERASE, 93963663884288, 93963663884288, -+STORE, 93963663884288, 93963664240639, -+STORE, 93963664240640, 93963664379903, -+STORE, 93963664379904, 93963664506879, -+STORE, 140450188439552, 140450188611583, -+ERASE, 140450188439552, 140450188439552, -+STORE, 140450188439552, 140450188443647, -+STORE, 140450188443648, 140450188611583, -+ERASE, 140450188443648, 140450188443648, -+STORE, 140450188443648, 140450188566527, -+STORE, 140450188566528, 140450188599295, -+STORE, 140450188599296, 140450188607487, -+STORE, 140450188607488, 140450188611583, -+STORE, 140736284577792, 140736284581887, -+STORE, 140736284565504, 140736284577791, -+STORE, 47182606557184, 47182606565375, -+STORE, 47182606565376, 47182606573567, -+STORE, 47182606573568, 47182608412671, -+STORE, 47182606712832, 47182608412671, -+STORE, 47182606573568, 47182606712831, -+ERASE, 47182606712832, 47182606712832, -+STORE, 47182606712832, 47182608371711, -+STORE, 47182608371712, 47182608412671, -+STORE, 47182608056320, 47182608371711, -+STORE, 47182606712832, 47182608056319, -+ERASE, 47182606712832, 47182606712832, -+STORE, 47182606712832, 47182608056319, -+STORE, 47182608367616, 47182608371711, -+STORE, 47182608056320, 47182608367615, -+ERASE, 47182608056320, 47182608056320, -+STORE, 47182608056320, 47182608367615, -+STORE, 47182608396288, 47182608412671, -+STORE, 47182608371712, 47182608396287, -+ERASE, 47182608371712, 47182608371712, -+STORE, 47182608371712, 47182608396287, -+ERASE, 47182608396288, 47182608396288, -+STORE, 47182608396288, 47182608412671, -+STORE, 47182608412672, 47182608523263, -+STORE, 47182608429056, 47182608523263, -+STORE, 47182608412672, 47182608429055, -+ERASE, 47182608429056, 47182608429056, -+STORE, 47182608429056, 47182608515071, -+STORE, 47182608515072, 47182608523263, -+STORE, 47182608490496, 47182608515071, -+STORE, 47182608429056, 47182608490495, -+ERASE, 47182608429056, 47182608429056, -+STORE, 47182608429056, 47182608490495, -+STORE, 47182608510976, 47182608515071, -+STORE, 47182608490496, 47182608510975, -+ERASE, 47182608490496, 47182608490496, -+STORE, 47182608490496, 47182608510975, -+ERASE, 47182608515072, 47182608515072, -+STORE, 47182608515072, 47182608523263, -+STORE, 47182608523264, 47182608568319, -+ERASE, 47182608523264, 47182608523264, -+STORE, 47182608523264, 47182608531455, -+STORE, 47182608531456, 47182608568319, -+STORE, 47182608551936, 47182608568319, -+STORE, 47182608531456, 47182608551935, -+ERASE, 47182608531456, 47182608531456, -+STORE, 47182608531456, 47182608551935, -+STORE, 47182608560128, 47182608568319, -+STORE, 47182608551936, 47182608560127, -+ERASE, 47182608551936, 47182608551936, -+STORE, 47182608551936, 47182608568319, -+ERASE, 47182608551936, 47182608551936, -+STORE, 47182608551936, 47182608560127, -+STORE, 47182608560128, 47182608568319, -+ERASE, 47182608560128, 47182608560128, -+STORE, 47182608560128, 47182608568319, -+STORE, 47182608568320, 47182608916479, -+STORE, 47182608609280, 47182608916479, -+STORE, 47182608568320, 47182608609279, -+ERASE, 47182608609280, 47182608609280, -+STORE, 47182608609280, 47182608891903, -+STORE, 47182608891904, 47182608916479, -+STORE, 47182608822272, 47182608891903, -+STORE, 47182608609280, 47182608822271, -+ERASE, 47182608609280, 47182608609280, -+STORE, 47182608609280, 47182608822271, -+STORE, 47182608887808, 47182608891903, -+STORE, 47182608822272, 47182608887807, -+ERASE, 47182608822272, 47182608822272, -+STORE, 47182608822272, 47182608887807, -+ERASE, 47182608891904, 47182608891904, -+STORE, 47182608891904, 47182608916479, -+STORE, 47182608916480, 47182611177471, -+STORE, 47182609068032, 47182611177471, -+STORE, 47182608916480, 47182609068031, -+ERASE, 47182609068032, 47182609068032, -+STORE, 47182609068032, 47182611161087, -+STORE, 47182611161088, 47182611177471, -+STORE, 47182611169280, 47182611177471, -+STORE, 47182611161088, 47182611169279, -+ERASE, 47182611161088, 47182611161088, -+STORE, 47182611161088, 47182611169279, -+ERASE, 47182611169280, 47182611169280, -+STORE, 47182611169280, 47182611177471, -+STORE, 47182611177472, 47182611312639, -+ERASE, 47182611177472, 47182611177472, -+STORE, 47182611177472, 47182611202047, -+STORE, 47182611202048, 47182611312639, -+STORE, 47182611263488, 47182611312639, -+STORE, 47182611202048, 47182611263487, -+ERASE, 47182611202048, 47182611202048, -+STORE, 47182611202048, 47182611263487, -+STORE, 47182611288064, 47182611312639, -+STORE, 47182611263488, 47182611288063, -+ERASE, 47182611263488, 47182611263488, -+STORE, 47182611263488, 47182611312639, -+ERASE, 47182611263488, 47182611263488, -+STORE, 47182611263488, 47182611288063, -+STORE, 47182611288064, 47182611312639, -+STORE, 47182611296256, 47182611312639, -+STORE, 47182611288064, 47182611296255, -+ERASE, 47182611288064, 47182611288064, -+STORE, 47182611288064, 47182611296255, -+ERASE, 47182611296256, 47182611296256, -+STORE, 47182611296256, 47182611312639, -+STORE, 47182611296256, 47182611320831, -+STORE, 47182611320832, 47182611484671, -+ERASE, 47182611320832, 47182611320832, -+STORE, 47182611320832, 47182611333119, -+STORE, 47182611333120, 47182611484671, -+STORE, 47182611431424, 47182611484671, -+STORE, 47182611333120, 47182611431423, -+ERASE, 47182611333120, 47182611333120, -+STORE, 47182611333120, 47182611431423, -+STORE, 47182611476480, 47182611484671, -+STORE, 47182611431424, 47182611476479, -+ERASE, 47182611431424, 47182611431424, -+STORE, 47182611431424, 47182611484671, -+ERASE, 47182611431424, 47182611431424, -+STORE, 47182611431424, 47182611476479, -+STORE, 47182611476480, 47182611484671, -+ERASE, 47182611476480, 47182611476480, -+STORE, 47182611476480, 47182611484671, -+STORE, 47182611484672, 47182612082687, -+STORE, 47182611603456, 47182612082687, -+STORE, 47182611484672, 47182611603455, -+ERASE, 47182611603456, 47182611603456, -+STORE, 47182611603456, 47182612029439, -+STORE, 47182612029440, 47182612082687, -+STORE, 47182611918848, 47182612029439, -+STORE, 47182611603456, 47182611918847, -+ERASE, 47182611603456, 47182611603456, -+STORE, 47182611603456, 47182611918847, -+STORE, 47182612025344, 47182612029439, -+STORE, 47182611918848, 47182612025343, -+ERASE, 47182611918848, 47182611918848, -+STORE, 47182611918848, 47182612025343, -+ERASE, 47182612029440, 47182612029440, -+STORE, 47182612029440, 47182612082687, -+STORE, 47182612082688, 47182615134207, -+STORE, 47182612627456, 47182615134207, -+STORE, 47182612082688, 47182612627455, -+ERASE, 47182612627456, 47182612627456, -+STORE, 47182612627456, 47182614913023, -+STORE, 47182614913024, 47182615134207, -+STORE, 47182614323200, 47182614913023, -+STORE, 47182612627456, 47182614323199, -+ERASE, 47182612627456, 47182612627456, -+STORE, 47182612627456, 47182614323199, -+STORE, 47182614908928, 47182614913023, -+STORE, 47182614323200, 47182614908927, -+ERASE, 47182614323200, 47182614323200, -+STORE, 47182614323200, 47182614908927, -+STORE, 47182615117824, 47182615134207, -+STORE, 47182614913024, 47182615117823, -+ERASE, 47182614913024, 47182614913024, -+STORE, 47182614913024, 47182615117823, -+ERASE, 47182615117824, 47182615117824, -+STORE, 47182615117824, 47182615134207, -+STORE, 47182615134208, 47182615166975, -+ERASE, 47182615134208, 47182615134208, -+STORE, 47182615134208, 47182615142399, -+STORE, 47182615142400, 47182615166975, -+STORE, 47182615154688, 47182615166975, -+STORE, 47182615142400, 47182615154687, -+ERASE, 47182615142400, 47182615142400, -+STORE, 47182615142400, 47182615154687, -+STORE, 47182615158784, 47182615166975, -+STORE, 47182615154688, 47182615158783, -+ERASE, 47182615154688, 47182615154688, -+STORE, 47182615154688, 47182615166975, -+ERASE, 47182615154688, 47182615154688, -+STORE, 47182615154688, 47182615158783, -+STORE, 47182615158784, 47182615166975, -+ERASE, 47182615158784, 47182615158784, -+STORE, 47182615158784, 47182615166975, -+STORE, 47182615166976, 47182615203839, -+ERASE, 47182615166976, 47182615166976, -+STORE, 47182615166976, 47182615175167, -+STORE, 47182615175168, 47182615203839, -+STORE, 47182615191552, 47182615203839, -+STORE, 47182615175168, 47182615191551, -+ERASE, 47182615175168, 47182615175168, -+STORE, 47182615175168, 47182615191551, -+STORE, 47182615195648, 47182615203839, -+STORE, 47182615191552, 47182615195647, -+ERASE, 47182615191552, 47182615191552, -+STORE, 47182615191552, 47182615203839, -+ERASE, 47182615191552, 47182615191552, -+STORE, 47182615191552, 47182615195647, -+STORE, 47182615195648, 47182615203839, -+ERASE, 47182615195648, 47182615195648, -+STORE, 47182615195648, 47182615203839, -+STORE, 47182615203840, 47182615678975, -+ERASE, 47182615203840, 47182615203840, -+STORE, 47182615203840, 47182615212031, -+STORE, 47182615212032, 47182615678975, -+STORE, 47182615547904, 47182615678975, -+STORE, 47182615212032, 47182615547903, -+ERASE, 47182615212032, 47182615212032, -+STORE, 47182615212032, 47182615547903, -+STORE, 47182615670784, 47182615678975, -+STORE, 47182615547904, 47182615670783, -+ERASE, 47182615547904, 47182615547904, -+STORE, 47182615547904, 47182615678975, -+ERASE, 47182615547904, 47182615547904, -+STORE, 47182615547904, 47182615670783, -+STORE, 47182615670784, 47182615678975, -+ERASE, 47182615670784, 47182615670784, -+STORE, 47182615670784, 47182615678975, -+STORE, 47182615678976, 47182615687167, -+STORE, 47182615687168, 47182615707647, -+ERASE, 47182615687168, 47182615687168, -+STORE, 47182615687168, 47182615691263, -+STORE, 47182615691264, 47182615707647, -+STORE, 47182615695360, 47182615707647, -+STORE, 47182615691264, 47182615695359, -+ERASE, 47182615691264, 47182615691264, -+STORE, 47182615691264, 47182615695359, -+STORE, 47182615699456, 47182615707647, -+STORE, 47182615695360, 47182615699455, -+ERASE, 47182615695360, 47182615695360, -+STORE, 47182615695360, 47182615707647, -+ERASE, 47182615695360, 47182615695360, -+STORE, 47182615695360, 47182615699455, -+STORE, 47182615699456, 47182615707647, -+ERASE, 47182615699456, 47182615699456, -+STORE, 47182615699456, 47182615707647, -+STORE, 47182615707648, 47182615715839, -+ERASE, 47182608371712, 47182608371712, -+STORE, 47182608371712, 47182608388095, -+STORE, 47182608388096, 47182608396287, -+ERASE, 47182615699456, 47182615699456, -+STORE, 47182615699456, 47182615703551, -+STORE, 47182615703552, 47182615707647, -+ERASE, 47182611288064, 47182611288064, -+STORE, 47182611288064, 47182611292159, -+STORE, 47182611292160, 47182611296255, -+ERASE, 47182615670784, 47182615670784, -+STORE, 47182615670784, 47182615674879, -+STORE, 47182615674880, 47182615678975, -+ERASE, 47182615195648, 47182615195648, -+STORE, 47182615195648, 47182615199743, -+STORE, 47182615199744, 47182615203839, -+ERASE, 47182615158784, 47182615158784, -+STORE, 47182615158784, 47182615162879, -+STORE, 47182615162880, 47182615166975, -+ERASE, 47182614913024, 47182614913024, -+STORE, 47182614913024, 47182615109631, -+STORE, 47182615109632, 47182615117823, -+ERASE, 47182612029440, 47182612029440, -+STORE, 47182612029440, 47182612066303, -+STORE, 47182612066304, 47182612082687, -+ERASE, 47182611476480, 47182611476480, -+STORE, 47182611476480, 47182611480575, -+STORE, 47182611480576, 47182611484671, -+ERASE, 47182611161088, 47182611161088, -+STORE, 47182611161088, 47182611165183, -+STORE, 47182611165184, 47182611169279, -+ERASE, 47182608891904, 47182608891904, -+STORE, 47182608891904, 47182608912383, -+STORE, 47182608912384, 47182608916479, -+ERASE, 47182608560128, 47182608560128, -+STORE, 47182608560128, 47182608564223, -+STORE, 47182608564224, 47182608568319, -+ERASE, 47182608515072, 47182608515072, -+STORE, 47182608515072, 47182608519167, -+STORE, 47182608519168, 47182608523263, -+ERASE, 93963664379904, 93963664379904, -+STORE, 93963664379904, 93963664502783, -+STORE, 93963664502784, 93963664506879, -+ERASE, 140450188599296, 140450188599296, -+STORE, 140450188599296, 140450188603391, -+STORE, 140450188603392, 140450188607487, -+ERASE, 47182606557184, 47182606557184, -+STORE, 93963694723072, 93963694858239, -+STORE, 140737488347136, 140737488351231, -+STORE, 140730313261056, 140737488351231, -+ERASE, 140730313261056, 140730313261056, -+STORE, 140730313261056, 140730313265151, -+STORE, 94386579017728, 94386579697663, -+ERASE, 94386579017728, 94386579017728, -+STORE, 94386579017728, 94386579083263, -+STORE, 94386579083264, 94386579697663, -+ERASE, 94386579083264, 94386579083264, -+STORE, 94386579083264, 94386579431423, -+STORE, 94386579431424, 94386579570687, -+STORE, 94386579570688, 94386579697663, -+STORE, 140124810838016, 140124811010047, -+ERASE, 140124810838016, 140124810838016, -+STORE, 140124810838016, 140124810842111, -+STORE, 140124810842112, 140124811010047, -+ERASE, 140124810842112, 140124810842112, -+STORE, 140124810842112, 140124810964991, -+STORE, 140124810964992, 140124810997759, -+STORE, 140124810997760, 140124811005951, -+STORE, 140124811005952, 140124811010047, -+STORE, 140730313601024, 140730313605119, -+STORE, 140730313588736, 140730313601023, -+STORE, 47507984158720, 47507984166911, -+STORE, 47507984166912, 47507984175103, -+STORE, 47507984175104, 47507986014207, -+STORE, 47507984314368, 47507986014207, -+STORE, 47507984175104, 47507984314367, -+ERASE, 47507984314368, 47507984314368, -+STORE, 47507984314368, 47507985973247, -+STORE, 47507985973248, 47507986014207, -+STORE, 47507985657856, 47507985973247, -+STORE, 47507984314368, 47507985657855, -+ERASE, 47507984314368, 47507984314368, -+STORE, 47507984314368, 47507985657855, -+STORE, 47507985969152, 47507985973247, -+STORE, 47507985657856, 47507985969151, -+ERASE, 47507985657856, 47507985657856, -+STORE, 47507985657856, 47507985969151, -+STORE, 47507985997824, 47507986014207, -+STORE, 47507985973248, 47507985997823, -+ERASE, 47507985973248, 47507985973248, -+STORE, 47507985973248, 47507985997823, -+ERASE, 47507985997824, 47507985997824, -+STORE, 47507985997824, 47507986014207, -+STORE, 47507986014208, 47507986124799, -+STORE, 47507986030592, 47507986124799, -+STORE, 47507986014208, 47507986030591, -+ERASE, 47507986030592, 47507986030592, -+STORE, 47507986030592, 47507986116607, -+STORE, 47507986116608, 47507986124799, -+STORE, 47507986092032, 47507986116607, -+STORE, 47507986030592, 47507986092031, -+ERASE, 47507986030592, 47507986030592, -+STORE, 47507986030592, 47507986092031, -+STORE, 47507986112512, 47507986116607, -+STORE, 47507986092032, 47507986112511, -+ERASE, 47507986092032, 47507986092032, -+STORE, 47507986092032, 47507986112511, -+ERASE, 47507986116608, 47507986116608, -+STORE, 47507986116608, 47507986124799, -+STORE, 47507986124800, 47507986169855, -+ERASE, 47507986124800, 47507986124800, -+STORE, 47507986124800, 47507986132991, -+STORE, 47507986132992, 47507986169855, -+STORE, 47507986153472, 47507986169855, -+STORE, 47507986132992, 47507986153471, -+ERASE, 47507986132992, 47507986132992, -+STORE, 47507986132992, 47507986153471, -+STORE, 47507986161664, 47507986169855, -+STORE, 47507986153472, 47507986161663, -+ERASE, 47507986153472, 47507986153472, -+STORE, 47507986153472, 47507986169855, -+ERASE, 47507986153472, 47507986153472, -+STORE, 47507986153472, 47507986161663, -+STORE, 47507986161664, 47507986169855, -+ERASE, 47507986161664, 47507986161664, -+STORE, 47507986161664, 47507986169855, -+STORE, 47507986169856, 47507986518015, -+STORE, 47507986210816, 47507986518015, -+STORE, 47507986169856, 47507986210815, -+ERASE, 47507986210816, 47507986210816, -+STORE, 47507986210816, 47507986493439, -+STORE, 47507986493440, 47507986518015, -+STORE, 47507986423808, 47507986493439, -+STORE, 47507986210816, 47507986423807, -+ERASE, 47507986210816, 47507986210816, -+STORE, 47507986210816, 47507986423807, -+STORE, 47507986489344, 47507986493439, -+STORE, 47507986423808, 47507986489343, -+ERASE, 47507986423808, 47507986423808, -+STORE, 47507986423808, 47507986489343, -+ERASE, 47507986493440, 47507986493440, -+STORE, 47507986493440, 47507986518015, -+STORE, 47507986518016, 47507988779007, -+STORE, 47507986669568, 47507988779007, -+STORE, 47507986518016, 47507986669567, -+ERASE, 47507986669568, 47507986669568, -+STORE, 47507986669568, 47507988762623, -+STORE, 47507988762624, 47507988779007, -+STORE, 47507988770816, 47507988779007, -+STORE, 47507988762624, 47507988770815, -+ERASE, 47507988762624, 47507988762624, -+STORE, 47507988762624, 47507988770815, -+ERASE, 47507988770816, 47507988770816, -+STORE, 47507988770816, 47507988779007, -+STORE, 47507988779008, 47507988914175, -+ERASE, 47507988779008, 47507988779008, -+STORE, 47507988779008, 47507988803583, -+STORE, 47507988803584, 47507988914175, -+STORE, 47507988865024, 47507988914175, -+STORE, 47507988803584, 47507988865023, -+ERASE, 47507988803584, 47507988803584, -+STORE, 47507988803584, 47507988865023, -+STORE, 47507988889600, 47507988914175, -+STORE, 47507988865024, 47507988889599, -+ERASE, 47507988865024, 47507988865024, -+STORE, 47507988865024, 47507988914175, -+ERASE, 47507988865024, 47507988865024, -+STORE, 47507988865024, 47507988889599, -+STORE, 47507988889600, 47507988914175, -+STORE, 47507988897792, 47507988914175, -+STORE, 47507988889600, 47507988897791, -+ERASE, 47507988889600, 47507988889600, -+STORE, 47507988889600, 47507988897791, -+ERASE, 47507988897792, 47507988897792, -+STORE, 47507988897792, 47507988914175, -+STORE, 47507988897792, 47507988922367, -+STORE, 47507988922368, 47507989086207, -+ERASE, 47507988922368, 47507988922368, -+STORE, 47507988922368, 47507988934655, -+STORE, 47507988934656, 47507989086207, -+STORE, 47507989032960, 47507989086207, -+STORE, 47507988934656, 47507989032959, -+ERASE, 47507988934656, 47507988934656, -+STORE, 47507988934656, 47507989032959, -+STORE, 47507989078016, 47507989086207, -+STORE, 47507989032960, 47507989078015, -+ERASE, 47507989032960, 47507989032960, -+STORE, 47507989032960, 47507989086207, -+ERASE, 47507989032960, 47507989032960, -+STORE, 47507989032960, 47507989078015, -+STORE, 47507989078016, 47507989086207, -+ERASE, 47507989078016, 47507989078016, -+STORE, 47507989078016, 47507989086207, -+STORE, 47507989086208, 47507989684223, -+STORE, 47507989204992, 47507989684223, -+STORE, 47507989086208, 47507989204991, -+ERASE, 47507989204992, 47507989204992, -+STORE, 47507989204992, 47507989630975, -+STORE, 47507989630976, 47507989684223, -+STORE, 47507989520384, 47507989630975, -+STORE, 47507989204992, 47507989520383, -+ERASE, 47507989204992, 47507989204992, -+STORE, 47507989204992, 47507989520383, -+STORE, 47507989626880, 47507989630975, -+STORE, 47507989520384, 47507989626879, -+ERASE, 47507989520384, 47507989520384, -+STORE, 47507989520384, 47507989626879, -+ERASE, 47507989630976, 47507989630976, -+STORE, 47507989630976, 47507989684223, -+STORE, 47507989684224, 47507992735743, -+STORE, 47507990228992, 47507992735743, -+STORE, 47507989684224, 47507990228991, -+ERASE, 47507990228992, 47507990228992, -+STORE, 47507990228992, 47507992514559, -+STORE, 47507992514560, 47507992735743, -+STORE, 47507991924736, 47507992514559, -+STORE, 47507990228992, 47507991924735, -+ERASE, 47507990228992, 47507990228992, -+STORE, 47507990228992, 47507991924735, -+STORE, 47507992510464, 47507992514559, -+STORE, 47507991924736, 47507992510463, -+ERASE, 47507991924736, 47507991924736, -+STORE, 47507991924736, 47507992510463, -+STORE, 47507992719360, 47507992735743, -+STORE, 47507992514560, 47507992719359, -+ERASE, 47507992514560, 47507992514560, -+STORE, 47507992514560, 47507992719359, -+ERASE, 47507992719360, 47507992719360, -+STORE, 47507992719360, 47507992735743, -+STORE, 47507992735744, 47507992768511, -+ERASE, 47507992735744, 47507992735744, -+STORE, 47507992735744, 47507992743935, -+STORE, 47507992743936, 47507992768511, -+STORE, 47507992756224, 47507992768511, -+STORE, 47507992743936, 47507992756223, -+ERASE, 47507992743936, 47507992743936, -+STORE, 47507992743936, 47507992756223, -+STORE, 47507992760320, 47507992768511, -+STORE, 47507992756224, 47507992760319, -+ERASE, 47507992756224, 47507992756224, -+STORE, 47507992756224, 47507992768511, -+ERASE, 47507992756224, 47507992756224, -+STORE, 47507992756224, 47507992760319, -+STORE, 47507992760320, 47507992768511, -+ERASE, 47507992760320, 47507992760320, -+STORE, 47507992760320, 47507992768511, -+STORE, 47507992768512, 47507992805375, -+ERASE, 47507992768512, 47507992768512, -+STORE, 47507992768512, 47507992776703, -+STORE, 47507992776704, 47507992805375, -+STORE, 47507992793088, 47507992805375, -+STORE, 47507992776704, 47507992793087, -+ERASE, 47507992776704, 47507992776704, -+STORE, 47507992776704, 47507992793087, -+STORE, 47507992797184, 47507992805375, -+STORE, 47507992793088, 47507992797183, -+ERASE, 47507992793088, 47507992793088, -+STORE, 47507992793088, 47507992805375, -+ERASE, 47507992793088, 47507992793088, -+STORE, 47507992793088, 47507992797183, -+STORE, 47507992797184, 47507992805375, -+ERASE, 47507992797184, 47507992797184, -+STORE, 47507992797184, 47507992805375, -+STORE, 47507992805376, 47507993280511, -+ERASE, 47507992805376, 47507992805376, -+STORE, 47507992805376, 47507992813567, -+STORE, 47507992813568, 47507993280511, -+STORE, 47507993149440, 47507993280511, -+STORE, 47507992813568, 47507993149439, -+ERASE, 47507992813568, 47507992813568, -+STORE, 47507992813568, 47507993149439, -+STORE, 47507993272320, 47507993280511, -+STORE, 47507993149440, 47507993272319, -+ERASE, 47507993149440, 47507993149440, -+STORE, 47507993149440, 47507993280511, -+ERASE, 47507993149440, 47507993149440, -+STORE, 47507993149440, 47507993272319, -+STORE, 47507993272320, 47507993280511, -+ERASE, 47507993272320, 47507993272320, -+STORE, 47507993272320, 47507993280511, -+STORE, 47507993280512, 47507993288703, -+STORE, 47507993288704, 47507993309183, -+ERASE, 47507993288704, 47507993288704, -+STORE, 47507993288704, 47507993292799, -+STORE, 47507993292800, 47507993309183, -+STORE, 47507993296896, 47507993309183, -+STORE, 47507993292800, 47507993296895, -+ERASE, 47507993292800, 47507993292800, -+STORE, 47507993292800, 47507993296895, -+STORE, 47507993300992, 47507993309183, -+STORE, 47507993296896, 47507993300991, -+ERASE, 47507993296896, 47507993296896, -+STORE, 47507993296896, 47507993309183, -+ERASE, 47507993296896, 47507993296896, -+STORE, 47507993296896, 47507993300991, -+STORE, 47507993300992, 47507993309183, -+ERASE, 47507993300992, 47507993300992, -+STORE, 47507993300992, 47507993309183, -+STORE, 47507993309184, 47507993317375, -+ERASE, 47507985973248, 47507985973248, -+STORE, 47507985973248, 47507985989631, -+STORE, 47507985989632, 47507985997823, -+ERASE, 47507993300992, 47507993300992, -+STORE, 47507993300992, 47507993305087, -+STORE, 47507993305088, 47507993309183, -+ERASE, 47507988889600, 47507988889600, -+STORE, 47507988889600, 47507988893695, -+STORE, 47507988893696, 47507988897791, -+ERASE, 47507993272320, 47507993272320, -+STORE, 47507993272320, 47507993276415, -+STORE, 47507993276416, 47507993280511, -+ERASE, 47507992797184, 47507992797184, -+STORE, 47507992797184, 47507992801279, -+STORE, 47507992801280, 47507992805375, -+ERASE, 47507992760320, 47507992760320, -+STORE, 47507992760320, 47507992764415, -+STORE, 47507992764416, 47507992768511, -+ERASE, 47507992514560, 47507992514560, -+STORE, 47507992514560, 47507992711167, -+STORE, 47507992711168, 47507992719359, -+ERASE, 47507989630976, 47507989630976, -+STORE, 47507989630976, 47507989667839, -+STORE, 47507989667840, 47507989684223, -+ERASE, 47507989078016, 47507989078016, -+STORE, 47507989078016, 47507989082111, -+STORE, 47507989082112, 47507989086207, -+ERASE, 47507988762624, 47507988762624, -+STORE, 47507988762624, 47507988766719, -+STORE, 47507988766720, 47507988770815, -+ERASE, 47507986493440, 47507986493440, -+STORE, 47507986493440, 47507986513919, -+STORE, 47507986513920, 47507986518015, -+ERASE, 47507986161664, 47507986161664, -+STORE, 47507986161664, 47507986165759, -+STORE, 47507986165760, 47507986169855, -+ERASE, 47507986116608, 47507986116608, -+STORE, 47507986116608, 47507986120703, -+STORE, 47507986120704, 47507986124799, -+ERASE, 94386579570688, 94386579570688, -+STORE, 94386579570688, 94386579693567, -+STORE, 94386579693568, 94386579697663, -+ERASE, 140124810997760, 140124810997760, -+STORE, 140124810997760, 140124811001855, -+STORE, 140124811001856, 140124811005951, -+ERASE, 47507984158720, 47507984158720, -+STORE, 94386583982080, 94386584117247, -+STORE, 94386583982080, 94386584256511, -+ERASE, 94386583982080, 94386583982080, -+STORE, 94386583982080, 94386584223743, -+STORE, 94386584223744, 94386584256511, -+ERASE, 94386584223744, 94386584223744, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733763395584, 140737488351231, -+ERASE, 140733763395584, 140733763395584, -+STORE, 140733763395584, 140733763399679, -+STORE, 94011546472448, 94011547152383, -+ERASE, 94011546472448, 94011546472448, -+STORE, 94011546472448, 94011546537983, -+STORE, 94011546537984, 94011547152383, -+ERASE, 94011546537984, 94011546537984, -+STORE, 94011546537984, 94011546886143, -+STORE, 94011546886144, 94011547025407, -+STORE, 94011547025408, 94011547152383, -+STORE, 139757597949952, 139757598121983, -+ERASE, 139757597949952, 139757597949952, -+STORE, 139757597949952, 139757597954047, -+STORE, 139757597954048, 139757598121983, -+ERASE, 139757597954048, 139757597954048, -+STORE, 139757597954048, 139757598076927, -+STORE, 139757598076928, 139757598109695, -+STORE, 139757598109696, 139757598117887, -+STORE, 139757598117888, 139757598121983, -+STORE, 140733763596288, 140733763600383, -+STORE, 140733763584000, 140733763596287, -+STORE, 47875197046784, 47875197054975, -+STORE, 47875197054976, 47875197063167, -+STORE, 47875197063168, 47875198902271, -+STORE, 47875197202432, 47875198902271, -+STORE, 47875197063168, 47875197202431, -+ERASE, 47875197202432, 47875197202432, -+STORE, 47875197202432, 47875198861311, -+STORE, 47875198861312, 47875198902271, -+STORE, 47875198545920, 47875198861311, -+STORE, 47875197202432, 47875198545919, -+ERASE, 47875197202432, 47875197202432, -+STORE, 47875197202432, 47875198545919, -+STORE, 47875198857216, 47875198861311, -+STORE, 47875198545920, 47875198857215, -+ERASE, 47875198545920, 47875198545920, -+STORE, 47875198545920, 47875198857215, -+STORE, 47875198885888, 47875198902271, -+STORE, 47875198861312, 47875198885887, -+ERASE, 47875198861312, 47875198861312, -+STORE, 47875198861312, 47875198885887, -+ERASE, 47875198885888, 47875198885888, -+STORE, 47875198885888, 47875198902271, -+STORE, 47875198902272, 47875199012863, -+STORE, 47875198918656, 47875199012863, -+STORE, 47875198902272, 47875198918655, -+ERASE, 47875198918656, 47875198918656, -+STORE, 47875198918656, 47875199004671, -+STORE, 47875199004672, 47875199012863, -+STORE, 47875198980096, 47875199004671, -+STORE, 47875198918656, 47875198980095, -+ERASE, 47875198918656, 47875198918656, -+STORE, 47875198918656, 47875198980095, -+STORE, 47875199000576, 47875199004671, -+STORE, 47875198980096, 47875199000575, -+ERASE, 47875198980096, 47875198980096, -+STORE, 47875198980096, 47875199000575, -+ERASE, 47875199004672, 47875199004672, -+STORE, 47875199004672, 47875199012863, -+STORE, 47875199012864, 47875199057919, -+ERASE, 47875199012864, 47875199012864, -+STORE, 47875199012864, 47875199021055, -+STORE, 47875199021056, 47875199057919, -+STORE, 47875199041536, 47875199057919, -+STORE, 47875199021056, 47875199041535, -+ERASE, 47875199021056, 47875199021056, -+STORE, 47875199021056, 47875199041535, -+STORE, 47875199049728, 47875199057919, -+STORE, 47875199041536, 47875199049727, -+ERASE, 47875199041536, 47875199041536, -+STORE, 47875199041536, 47875199057919, -+ERASE, 47875199041536, 47875199041536, -+STORE, 47875199041536, 47875199049727, -+STORE, 47875199049728, 47875199057919, -+ERASE, 47875199049728, 47875199049728, -+STORE, 47875199049728, 47875199057919, -+STORE, 47875199057920, 47875199406079, -+STORE, 47875199098880, 47875199406079, -+STORE, 47875199057920, 47875199098879, -+ERASE, 47875199098880, 47875199098880, -+STORE, 47875199098880, 47875199381503, -+STORE, 47875199381504, 47875199406079, -+STORE, 47875199311872, 47875199381503, -+STORE, 47875199098880, 47875199311871, -+ERASE, 47875199098880, 47875199098880, -+STORE, 47875199098880, 47875199311871, -+STORE, 47875199377408, 47875199381503, -+STORE, 47875199311872, 47875199377407, -+ERASE, 47875199311872, 47875199311872, -+STORE, 47875199311872, 47875199377407, -+ERASE, 47875199381504, 47875199381504, -+STORE, 47875199381504, 47875199406079, -+STORE, 47875199406080, 47875201667071, -+STORE, 47875199557632, 47875201667071, -+STORE, 47875199406080, 47875199557631, -+ERASE, 47875199557632, 47875199557632, -+STORE, 47875199557632, 47875201650687, -+STORE, 47875201650688, 47875201667071, -+STORE, 47875201658880, 47875201667071, -+STORE, 47875201650688, 47875201658879, -+ERASE, 47875201650688, 47875201650688, -+STORE, 47875201650688, 47875201658879, -+ERASE, 47875201658880, 47875201658880, -+STORE, 47875201658880, 47875201667071, -+STORE, 47875201667072, 47875201802239, -+ERASE, 47875201667072, 47875201667072, -+STORE, 47875201667072, 47875201691647, -+STORE, 47875201691648, 47875201802239, -+STORE, 47875201753088, 47875201802239, -+STORE, 47875201691648, 47875201753087, -+ERASE, 47875201691648, 47875201691648, -+STORE, 47875201691648, 47875201753087, -+STORE, 47875201777664, 47875201802239, -+STORE, 47875201753088, 47875201777663, -+ERASE, 47875201753088, 47875201753088, -+STORE, 47875201753088, 47875201802239, -+ERASE, 47875201753088, 47875201753088, -+STORE, 47875201753088, 47875201777663, -+STORE, 47875201777664, 47875201802239, -+STORE, 47875201785856, 47875201802239, -+STORE, 47875201777664, 47875201785855, -+ERASE, 47875201777664, 47875201777664, -+STORE, 47875201777664, 47875201785855, -+ERASE, 47875201785856, 47875201785856, -+STORE, 47875201785856, 47875201802239, -+STORE, 47875201785856, 47875201810431, -+STORE, 47875201810432, 47875201974271, -+ERASE, 47875201810432, 47875201810432, -+STORE, 47875201810432, 47875201822719, -+STORE, 47875201822720, 47875201974271, -+STORE, 47875201921024, 47875201974271, -+STORE, 47875201822720, 47875201921023, -+ERASE, 47875201822720, 47875201822720, -+STORE, 47875201822720, 47875201921023, -+STORE, 47875201966080, 47875201974271, -+STORE, 47875201921024, 47875201966079, -+ERASE, 47875201921024, 47875201921024, -+STORE, 47875201921024, 47875201974271, -+ERASE, 47875201921024, 47875201921024, -+STORE, 47875201921024, 47875201966079, -+STORE, 47875201966080, 47875201974271, -+ERASE, 47875201966080, 47875201966080, -+STORE, 47875201966080, 47875201974271, -+STORE, 47875201974272, 47875202572287, -+STORE, 47875202093056, 47875202572287, -+STORE, 47875201974272, 47875202093055, -+ERASE, 47875202093056, 47875202093056, -+STORE, 47875202093056, 47875202519039, -+STORE, 47875202519040, 47875202572287, -+STORE, 47875202408448, 47875202519039, -+STORE, 47875202093056, 47875202408447, -+ERASE, 47875202093056, 47875202093056, -+STORE, 47875202093056, 47875202408447, -+STORE, 47875202514944, 47875202519039, -+STORE, 47875202408448, 47875202514943, -+ERASE, 47875202408448, 47875202408448, -+STORE, 47875202408448, 47875202514943, -+ERASE, 47875202519040, 47875202519040, -+STORE, 47875202519040, 47875202572287, -+STORE, 47875202572288, 47875205623807, -+STORE, 47875203117056, 47875205623807, -+STORE, 47875202572288, 47875203117055, -+ERASE, 47875203117056, 47875203117056, -+STORE, 47875203117056, 47875205402623, -+STORE, 47875205402624, 47875205623807, -+STORE, 47875204812800, 47875205402623, -+STORE, 47875203117056, 47875204812799, -+ERASE, 47875203117056, 47875203117056, -+STORE, 47875203117056, 47875204812799, -+STORE, 47875205398528, 47875205402623, -+STORE, 47875204812800, 47875205398527, -+ERASE, 47875204812800, 47875204812800, -+STORE, 47875204812800, 47875205398527, -+STORE, 47875205607424, 47875205623807, -+STORE, 47875205402624, 47875205607423, -+ERASE, 47875205402624, 47875205402624, -+STORE, 47875205402624, 47875205607423, -+ERASE, 47875205607424, 47875205607424, -+STORE, 47875205607424, 47875205623807, -+STORE, 47875205623808, 47875205656575, -+ERASE, 47875205623808, 47875205623808, -+STORE, 47875205623808, 47875205631999, -+STORE, 47875205632000, 47875205656575, -+STORE, 47875205644288, 47875205656575, -+STORE, 47875205632000, 47875205644287, -+ERASE, 47875205632000, 47875205632000, -+STORE, 47875205632000, 47875205644287, -+STORE, 47875205648384, 47875205656575, -+STORE, 47875205644288, 47875205648383, -+ERASE, 47875205644288, 47875205644288, -+STORE, 47875205644288, 47875205656575, -+ERASE, 47875205644288, 47875205644288, -+STORE, 47875205644288, 47875205648383, -+STORE, 47875205648384, 47875205656575, -+ERASE, 47875205648384, 47875205648384, -+STORE, 47875205648384, 47875205656575, -+STORE, 47875205656576, 47875205693439, -+ERASE, 47875205656576, 47875205656576, -+STORE, 47875205656576, 47875205664767, -+STORE, 47875205664768, 47875205693439, -+STORE, 47875205681152, 47875205693439, -+STORE, 47875205664768, 47875205681151, -+ERASE, 47875205664768, 47875205664768, -+STORE, 47875205664768, 47875205681151, -+STORE, 47875205685248, 47875205693439, -+STORE, 47875205681152, 47875205685247, -+ERASE, 47875205681152, 47875205681152, -+STORE, 47875205681152, 47875205693439, -+ERASE, 47875205681152, 47875205681152, -+STORE, 47875205681152, 47875205685247, -+STORE, 47875205685248, 47875205693439, -+ERASE, 47875205685248, 47875205685248, -+STORE, 47875205685248, 47875205693439, -+STORE, 47875205693440, 47875206168575, -+ERASE, 47875205693440, 47875205693440, -+STORE, 47875205693440, 47875205701631, -+STORE, 47875205701632, 47875206168575, -+STORE, 47875206037504, 47875206168575, -+STORE, 47875205701632, 47875206037503, -+ERASE, 47875205701632, 47875205701632, -+STORE, 47875205701632, 47875206037503, -+STORE, 47875206160384, 47875206168575, -+STORE, 47875206037504, 47875206160383, -+ERASE, 47875206037504, 47875206037504, -+STORE, 47875206037504, 47875206168575, -+ERASE, 47875206037504, 47875206037504, -+STORE, 47875206037504, 47875206160383, -+STORE, 47875206160384, 47875206168575, -+ERASE, 47875206160384, 47875206160384, -+STORE, 47875206160384, 47875206168575, -+STORE, 47875206168576, 47875206176767, -+STORE, 47875206176768, 47875206197247, -+ERASE, 47875206176768, 47875206176768, -+STORE, 47875206176768, 47875206180863, -+STORE, 47875206180864, 47875206197247, -+STORE, 47875206184960, 47875206197247, -+STORE, 47875206180864, 47875206184959, -+ERASE, 47875206180864, 47875206180864, -+STORE, 47875206180864, 47875206184959, -+STORE, 47875206189056, 47875206197247, -+STORE, 47875206184960, 47875206189055, -+ERASE, 47875206184960, 47875206184960, -+STORE, 47875206184960, 47875206197247, -+ERASE, 47875206184960, 47875206184960, -+STORE, 47875206184960, 47875206189055, -+STORE, 47875206189056, 47875206197247, -+ERASE, 47875206189056, 47875206189056, -+STORE, 47875206189056, 47875206197247, -+STORE, 47875206197248, 47875206205439, -+ERASE, 47875198861312, 47875198861312, -+STORE, 47875198861312, 47875198877695, -+STORE, 47875198877696, 47875198885887, -+ERASE, 47875206189056, 47875206189056, -+STORE, 47875206189056, 47875206193151, -+STORE, 47875206193152, 47875206197247, -+ERASE, 47875201777664, 47875201777664, -+STORE, 47875201777664, 47875201781759, -+STORE, 47875201781760, 47875201785855, -+ERASE, 47875206160384, 47875206160384, -+STORE, 47875206160384, 47875206164479, -+STORE, 47875206164480, 47875206168575, -+ERASE, 47875205685248, 47875205685248, -+STORE, 47875205685248, 47875205689343, -+STORE, 47875205689344, 47875205693439, -+ERASE, 47875205648384, 47875205648384, -+STORE, 47875205648384, 47875205652479, -+STORE, 47875205652480, 47875205656575, -+ERASE, 47875205402624, 47875205402624, -+STORE, 47875205402624, 47875205599231, -+STORE, 47875205599232, 47875205607423, -+ERASE, 47875202519040, 47875202519040, -+STORE, 47875202519040, 47875202555903, -+STORE, 47875202555904, 47875202572287, -+ERASE, 47875201966080, 47875201966080, -+STORE, 47875201966080, 47875201970175, -+STORE, 47875201970176, 47875201974271, -+ERASE, 47875201650688, 47875201650688, -+STORE, 47875201650688, 47875201654783, -+STORE, 47875201654784, 47875201658879, -+ERASE, 47875199381504, 47875199381504, -+STORE, 47875199381504, 47875199401983, -+STORE, 47875199401984, 47875199406079, -+ERASE, 47875199049728, 47875199049728, -+STORE, 47875199049728, 47875199053823, -+STORE, 47875199053824, 47875199057919, -+ERASE, 47875199004672, 47875199004672, -+STORE, 47875199004672, 47875199008767, -+STORE, 47875199008768, 47875199012863, -+ERASE, 94011547025408, 94011547025408, -+STORE, 94011547025408, 94011547148287, -+STORE, 94011547148288, 94011547152383, -+ERASE, 139757598109696, 139757598109696, -+STORE, 139757598109696, 139757598113791, -+STORE, 139757598113792, 139757598117887, -+ERASE, 47875197046784, 47875197046784, -+STORE, 94011557584896, 94011557720063, -+STORE, 94011557584896, 94011557855231, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557851135, -+STORE, 94011557851136, 94011557855231, -+ERASE, 94011557851136, 94011557851136, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557847039, -+STORE, 94011557847040, 94011557851135, -+ERASE, 94011557847040, 94011557847040, -+STORE, 94011557584896, 94011557982207, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557978111, -+STORE, 94011557978112, 94011557982207, -+ERASE, 94011557978112, 94011557978112, -+ERASE, 94011557584896, 94011557584896, -+STORE, 94011557584896, 94011557974015, -+STORE, 94011557974016, 94011557978111, -+ERASE, 94011557974016, 94011557974016, -+STORE, 140737488347136, 140737488351231, -+STORE, 140734130360320, 140737488351231, -+ERASE, 140734130360320, 140734130360320, -+STORE, 140734130360320, 140734130364415, -+STORE, 94641232105472, 94641232785407, -+ERASE, 94641232105472, 94641232105472, -+STORE, 94641232105472, 94641232171007, -+STORE, 94641232171008, 94641232785407, -+ERASE, 94641232171008, 94641232171008, -+STORE, 94641232171008, 94641232519167, -+STORE, 94641232519168, 94641232658431, -+STORE, 94641232658432, 94641232785407, -+STORE, 139726599516160, 139726599688191, -+ERASE, 139726599516160, 139726599516160, -+STORE, 139726599516160, 139726599520255, -+STORE, 139726599520256, 139726599688191, -+ERASE, 139726599520256, 139726599520256, -+STORE, 139726599520256, 139726599643135, -+STORE, 139726599643136, 139726599675903, -+STORE, 139726599675904, 139726599684095, -+STORE, 139726599684096, 139726599688191, -+STORE, 140734130446336, 140734130450431, -+STORE, 140734130434048, 140734130446335, -+STORE, 47906195480576, 47906195488767, -+STORE, 47906195488768, 47906195496959, -+STORE, 47906195496960, 47906197336063, -+STORE, 47906195636224, 47906197336063, -+STORE, 47906195496960, 47906195636223, -+ERASE, 47906195636224, 47906195636224, -+STORE, 47906195636224, 47906197295103, -+STORE, 47906197295104, 47906197336063, -+STORE, 47906196979712, 47906197295103, -+STORE, 47906195636224, 47906196979711, -+ERASE, 47906195636224, 47906195636224, -+STORE, 47906195636224, 47906196979711, -+STORE, 47906197291008, 47906197295103, -+STORE, 47906196979712, 47906197291007, -+ERASE, 47906196979712, 47906196979712, -+STORE, 47906196979712, 47906197291007, -+STORE, 47906197319680, 47906197336063, -+STORE, 47906197295104, 47906197319679, -+ERASE, 47906197295104, 47906197295104, -+STORE, 47906197295104, 47906197319679, -+ERASE, 47906197319680, 47906197319680, -+STORE, 47906197319680, 47906197336063, -+STORE, 47906197336064, 47906197446655, -+STORE, 47906197352448, 47906197446655, -+STORE, 47906197336064, 47906197352447, -+ERASE, 47906197352448, 47906197352448, -+STORE, 47906197352448, 47906197438463, -+STORE, 47906197438464, 47906197446655, -+STORE, 47906197413888, 47906197438463, -+STORE, 47906197352448, 47906197413887, -+ERASE, 47906197352448, 47906197352448, -+STORE, 47906197352448, 47906197413887, -+STORE, 47906197434368, 47906197438463, -+STORE, 47906197413888, 47906197434367, -+ERASE, 47906197413888, 47906197413888, -+STORE, 47906197413888, 47906197434367, -+ERASE, 47906197438464, 47906197438464, -+STORE, 47906197438464, 47906197446655, -+STORE, 47906197446656, 47906197491711, -+ERASE, 47906197446656, 47906197446656, -+STORE, 47906197446656, 47906197454847, -+STORE, 47906197454848, 47906197491711, -+STORE, 47906197475328, 47906197491711, -+STORE, 47906197454848, 47906197475327, -+ERASE, 47906197454848, 47906197454848, -+STORE, 47906197454848, 47906197475327, -+STORE, 47906197483520, 47906197491711, -+STORE, 47906197475328, 47906197483519, -+ERASE, 47906197475328, 47906197475328, -+STORE, 47906197475328, 47906197491711, -+ERASE, 47906197475328, 47906197475328, -+STORE, 47906197475328, 47906197483519, -+STORE, 47906197483520, 47906197491711, -+ERASE, 47906197483520, 47906197483520, -+STORE, 47906197483520, 47906197491711, -+STORE, 47906197491712, 47906197839871, -+STORE, 47906197532672, 47906197839871, -+STORE, 47906197491712, 47906197532671, -+ERASE, 47906197532672, 47906197532672, -+STORE, 47906197532672, 47906197815295, -+STORE, 47906197815296, 47906197839871, -+STORE, 47906197745664, 47906197815295, -+STORE, 47906197532672, 47906197745663, -+ERASE, 47906197532672, 47906197532672, -+STORE, 47906197532672, 47906197745663, -+STORE, 47906197811200, 47906197815295, -+STORE, 47906197745664, 47906197811199, -+ERASE, 47906197745664, 47906197745664, -+STORE, 47906197745664, 47906197811199, -+ERASE, 47906197815296, 47906197815296, -+STORE, 47906197815296, 47906197839871, -+STORE, 47906197839872, 47906200100863, -+STORE, 47906197991424, 47906200100863, -+STORE, 47906197839872, 47906197991423, -+ERASE, 47906197991424, 47906197991424, -+STORE, 47906197991424, 47906200084479, -+STORE, 47906200084480, 47906200100863, -+STORE, 47906200092672, 47906200100863, -+STORE, 47906200084480, 47906200092671, -+ERASE, 47906200084480, 47906200084480, -+STORE, 47906200084480, 47906200092671, -+ERASE, 47906200092672, 47906200092672, -+STORE, 47906200092672, 47906200100863, -+STORE, 47906200100864, 47906200236031, -+ERASE, 47906200100864, 47906200100864, -+STORE, 47906200100864, 47906200125439, -+STORE, 47906200125440, 47906200236031, -+STORE, 47906200186880, 47906200236031, -+STORE, 47906200125440, 47906200186879, -+ERASE, 47906200125440, 47906200125440, -+STORE, 47906200125440, 47906200186879, -+STORE, 47906200211456, 47906200236031, -+STORE, 47906200186880, 47906200211455, -+ERASE, 47906200186880, 47906200186880, -+STORE, 47906200186880, 47906200236031, -+ERASE, 47906200186880, 47906200186880, -+STORE, 47906200186880, 47906200211455, -+STORE, 47906200211456, 47906200236031, -+STORE, 47906200219648, 47906200236031, -+STORE, 47906200211456, 47906200219647, -+ERASE, 47906200211456, 47906200211456, -+STORE, 47906200211456, 47906200219647, -+ERASE, 47906200219648, 47906200219648, -+STORE, 47906200219648, 47906200236031, -+STORE, 47906200219648, 47906200244223, -+STORE, 47906200244224, 47906200408063, -+ERASE, 47906200244224, 47906200244224, -+STORE, 47906200244224, 47906200256511, -+STORE, 47906200256512, 47906200408063, -+STORE, 47906200354816, 47906200408063, -+STORE, 47906200256512, 47906200354815, -+ERASE, 47906200256512, 47906200256512, -+STORE, 47906200256512, 47906200354815, -+STORE, 47906200399872, 47906200408063, -+STORE, 47906200354816, 47906200399871, -+ERASE, 47906200354816, 47906200354816, -+STORE, 47906200354816, 47906200408063, -+ERASE, 47906200354816, 47906200354816, -+STORE, 47906200354816, 47906200399871, -+STORE, 47906200399872, 47906200408063, -+ERASE, 47906200399872, 47906200399872, -+STORE, 47906200399872, 47906200408063, -+STORE, 47906200408064, 47906201006079, -+STORE, 47906200526848, 47906201006079, -+STORE, 47906200408064, 47906200526847, -+ERASE, 47906200526848, 47906200526848, -+STORE, 47906200526848, 47906200952831, -+STORE, 47906200952832, 47906201006079, -+STORE, 47906200842240, 47906200952831, -+STORE, 47906200526848, 47906200842239, -+ERASE, 47906200526848, 47906200526848, -+STORE, 47906200526848, 47906200842239, -+STORE, 47906200948736, 47906200952831, -+STORE, 47906200842240, 47906200948735, -+ERASE, 47906200842240, 47906200842240, -+STORE, 47906200842240, 47906200948735, -+ERASE, 47906200952832, 47906200952832, -+STORE, 47906200952832, 47906201006079, -+STORE, 47906201006080, 47906204057599, -+STORE, 47906201550848, 47906204057599, -+STORE, 47906201006080, 47906201550847, -+ERASE, 47906201550848, 47906201550848, -+STORE, 47906201550848, 47906203836415, -+STORE, 47906203836416, 47906204057599, -+STORE, 47906203246592, 47906203836415, -+STORE, 47906201550848, 47906203246591, -+ERASE, 47906201550848, 47906201550848, -+STORE, 47906201550848, 47906203246591, -+STORE, 47906203832320, 47906203836415, -+STORE, 47906203246592, 47906203832319, -+ERASE, 47906203246592, 47906203246592, -+STORE, 47906203246592, 47906203832319, -+STORE, 47906204041216, 47906204057599, -+STORE, 47906203836416, 47906204041215, -+ERASE, 47906203836416, 47906203836416, -+STORE, 47906203836416, 47906204041215, -+ERASE, 47906204041216, 47906204041216, -+STORE, 47906204041216, 47906204057599, -+STORE, 47906204057600, 47906204090367, -+ERASE, 47906204057600, 47906204057600, -+STORE, 47906204057600, 47906204065791, -+STORE, 47906204065792, 47906204090367, -+STORE, 47906204078080, 47906204090367, -+STORE, 47906204065792, 47906204078079, -+ERASE, 47906204065792, 47906204065792, -+STORE, 47906204065792, 47906204078079, -+STORE, 47906204082176, 47906204090367, -+STORE, 47906204078080, 47906204082175, -+ERASE, 47906204078080, 47906204078080, -+STORE, 47906204078080, 47906204090367, -+ERASE, 47906204078080, 47906204078080, -+STORE, 47906204078080, 47906204082175, -+STORE, 47906204082176, 47906204090367, -+ERASE, 47906204082176, 47906204082176, -+STORE, 47906204082176, 47906204090367, -+STORE, 47906204090368, 47906204127231, -+ERASE, 47906204090368, 47906204090368, -+STORE, 47906204090368, 47906204098559, -+STORE, 47906204098560, 47906204127231, -+STORE, 47906204114944, 47906204127231, -+STORE, 47906204098560, 47906204114943, -+ERASE, 47906204098560, 47906204098560, -+STORE, 47906204098560, 47906204114943, -+STORE, 47906204119040, 47906204127231, -+STORE, 47906204114944, 47906204119039, -+ERASE, 47906204114944, 47906204114944, -+STORE, 47906204114944, 47906204127231, -+ERASE, 47906204114944, 47906204114944, -+STORE, 47906204114944, 47906204119039, -+STORE, 47906204119040, 47906204127231, -+ERASE, 47906204119040, 47906204119040, -+STORE, 47906204119040, 47906204127231, -+STORE, 47906204127232, 47906204602367, -+ERASE, 47906204127232, 47906204127232, -+STORE, 47906204127232, 47906204135423, -+STORE, 47906204135424, 47906204602367, -+STORE, 47906204471296, 47906204602367, -+STORE, 47906204135424, 47906204471295, -+ERASE, 47906204135424, 47906204135424, -+STORE, 47906204135424, 47906204471295, -+STORE, 47906204594176, 47906204602367, -+STORE, 47906204471296, 47906204594175, -+ERASE, 47906204471296, 47906204471296, -+STORE, 47906204471296, 47906204602367, -+ERASE, 47906204471296, 47906204471296, -+STORE, 47906204471296, 47906204594175, -+STORE, 47906204594176, 47906204602367, -+ERASE, 47906204594176, 47906204594176, -+STORE, 47906204594176, 47906204602367, -+STORE, 47906204602368, 47906204610559, -+STORE, 47906204610560, 47906204631039, -+ERASE, 47906204610560, 47906204610560, -+STORE, 47906204610560, 47906204614655, -+STORE, 47906204614656, 47906204631039, -+STORE, 47906204618752, 47906204631039, -+STORE, 47906204614656, 47906204618751, -+ERASE, 47906204614656, 47906204614656, -+STORE, 47906204614656, 47906204618751, -+STORE, 47906204622848, 47906204631039, -+STORE, 47906204618752, 47906204622847, -+ERASE, 47906204618752, 47906204618752, -+STORE, 47906204618752, 47906204631039, -+ERASE, 47906204618752, 47906204618752, -+STORE, 47906204618752, 47906204622847, -+STORE, 47906204622848, 47906204631039, -+ERASE, 47906204622848, 47906204622848, -+STORE, 47906204622848, 47906204631039, -+STORE, 47906204631040, 47906204639231, -+ERASE, 47906197295104, 47906197295104, -+STORE, 47906197295104, 47906197311487, -+STORE, 47906197311488, 47906197319679, -+ERASE, 47906204622848, 47906204622848, -+STORE, 47906204622848, 47906204626943, -+STORE, 47906204626944, 47906204631039, -+ERASE, 47906200211456, 47906200211456, -+STORE, 47906200211456, 47906200215551, -+STORE, 47906200215552, 47906200219647, -+ERASE, 47906204594176, 47906204594176, -+STORE, 47906204594176, 47906204598271, -+STORE, 47906204598272, 47906204602367, -+ERASE, 47906204119040, 47906204119040, -+STORE, 47906204119040, 47906204123135, -+STORE, 47906204123136, 47906204127231, -+ERASE, 47906204082176, 47906204082176, -+STORE, 47906204082176, 47906204086271, -+STORE, 47906204086272, 47906204090367, -+ERASE, 47906203836416, 47906203836416, -+STORE, 47906203836416, 47906204033023, -+STORE, 47906204033024, 47906204041215, -+ERASE, 47906200952832, 47906200952832, -+STORE, 47906200952832, 47906200989695, -+STORE, 47906200989696, 47906201006079, -+ERASE, 47906200399872, 47906200399872, -+STORE, 47906200399872, 47906200403967, -+STORE, 47906200403968, 47906200408063, -+ERASE, 47906200084480, 47906200084480, -+STORE, 47906200084480, 47906200088575, -+STORE, 47906200088576, 47906200092671, -+ERASE, 47906197815296, 47906197815296, -+STORE, 47906197815296, 47906197835775, -+STORE, 47906197835776, 47906197839871, -+ERASE, 47906197483520, 47906197483520, -+STORE, 47906197483520, 47906197487615, -+STORE, 47906197487616, 47906197491711, -+ERASE, 47906197438464, 47906197438464, -+STORE, 47906197438464, 47906197442559, -+STORE, 47906197442560, 47906197446655, -+ERASE, 94641232658432, 94641232658432, -+STORE, 94641232658432, 94641232781311, -+STORE, 94641232781312, 94641232785407, -+ERASE, 139726599675904, 139726599675904, -+STORE, 139726599675904, 139726599679999, -+STORE, 139726599680000, 139726599684095, -+ERASE, 47906195480576, 47906195480576, -+STORE, 94641242615808, 94641242750975, -+ }; -+ unsigned long set11[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140732658499584, 140737488351231, -+ERASE, 140732658499584, 140732658499584, -+STORE, 140732658499584, 140732658503679, -+STORE, 94029856579584, 94029856751615, -+ERASE, 94029856579584, 94029856579584, -+STORE, 94029856579584, 94029856595967, -+STORE, 94029856595968, 94029856751615, -+ERASE, 94029856595968, 94029856595968, -+STORE, 94029856595968, 94029856698367, -+STORE, 94029856698368, 94029856739327, -+STORE, 94029856739328, 94029856751615, -+STORE, 140014592573440, 140014592745471, -+ERASE, 140014592573440, 140014592573440, -+STORE, 140014592573440, 140014592577535, -+STORE, 140014592577536, 140014592745471, -+ERASE, 140014592577536, 140014592577536, -+STORE, 140014592577536, 140014592700415, -+STORE, 140014592700416, 140014592733183, -+STORE, 140014592733184, 140014592741375, -+STORE, 140014592741376, 140014592745471, -+STORE, 140732658565120, 140732658569215, -+STORE, 140732658552832, 140732658565119, -+ }; -+ -+ unsigned long set12[] = { /* contains 12 values. */ -+STORE, 140737488347136, 140737488351231, -+STORE, 140732658499584, 140737488351231, -+ERASE, 140732658499584, 140732658499584, -+STORE, 140732658499584, 140732658503679, -+STORE, 94029856579584, 94029856751615, -+ERASE, 94029856579584, 94029856579584, -+STORE, 94029856579584, 94029856595967, -+STORE, 94029856595968, 94029856751615, -+ERASE, 94029856595968, 94029856595968, -+STORE, 94029856595968, 94029856698367, -+STORE, 94029856698368, 94029856739327, -+STORE, 94029856739328, 94029856751615, -+STORE, 140014592573440, 140014592745471, -+ERASE, 140014592573440, 140014592573440, -+STORE, 140014592573440, 140014592577535, -+STORE, 140014592577536, 140014592745471, -+ERASE, 140014592577536, 140014592577536, -+STORE, 140014592577536, 140014592700415, -+STORE, 140014592700416, 140014592733183, -+STORE, 140014592733184, 140014592741375, -+STORE, 140014592741376, 140014592745471, -+STORE, 140732658565120, 140732658569215, -+STORE, 140732658552832, 140732658565119, -+STORE, 140014592741375, 140014592741375, /* contrived */ -+STORE, 140014592733184, 140014592741376, /* creates first entry retry. */ -+ }; -+ unsigned long set13[] = { -+STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */ -+STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */ -+STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */ -+STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */ -+STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */ -+STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */ -+STORE, 140373518663680, 140373518684159,/*: deleted (257) */ -+STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */ -+STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */ -+STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */ -+STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */ -+ }; -+ unsigned long set14[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140731667996672, 140737488351231, -+SNULL, 140731668000767, 140737488351231, -+STORE, 140731667996672, 140731668000767, -+STORE, 140731667865600, 140731668000767, -+STORE, 94077521272832, 94077521313791, -+SNULL, 94077521301503, 94077521313791, -+STORE, 94077521272832, 94077521301503, -+STORE, 94077521301504, 94077521313791, -+ERASE, 94077521301504, 94077521313791, -+STORE, 94077521305600, 94077521313791, -+STORE, 139826134630400, 139826136883199, -+SNULL, 139826134773759, 139826136883199, -+STORE, 139826134630400, 139826134773759, -+STORE, 139826134773760, 139826136883199, -+ERASE, 139826134773760, 139826136883199, -+STORE, 139826136870912, 139826136879103, -+STORE, 139826136879104, 139826136883199, -+STORE, 140731668013056, 140731668017151, -+STORE, 140731668000768, 140731668013055, -+STORE, 139826136862720, 139826136870911, -+STORE, 139826132406272, 139826134630399, -+SNULL, 139826134056959, 139826134630399, -+STORE, 139826132406272, 139826134056959, -+STORE, 139826134056960, 139826134630399, -+SNULL, 139826134056960, 139826134626303, -+STORE, 139826134626304, 139826134630399, -+STORE, 139826134056960, 139826134626303, -+ERASE, 139826134056960, 139826134626303, -+STORE, 139826134056960, 139826134626303, -+ERASE, 139826134626304, 139826134630399, -+STORE, 139826134626304, 139826134630399, -+STORE, 139826136842240, 139826136862719, -+STORE, 139826130022400, 139826132406271, -+SNULL, 139826130022400, 139826130288639, -+STORE, 139826130288640, 139826132406271, -+STORE, 139826130022400, 139826130288639, -+SNULL, 139826132381695, 139826132406271, -+STORE, 139826130288640, 139826132381695, -+STORE, 139826132381696, 139826132406271, -+SNULL, 139826132381696, 139826132402175, -+STORE, 139826132402176, 139826132406271, -+STORE, 139826132381696, 139826132402175, -+ERASE, 139826132381696, 139826132402175, -+STORE, 139826132381696, 139826132402175, -+ERASE, 139826132402176, 139826132406271, -+STORE, 139826132402176, 139826132406271, -+STORE, 139826127806464, 139826130022399, -+SNULL, 139826127806464, 139826127904767, -+STORE, 139826127904768, 139826130022399, -+STORE, 139826127806464, 139826127904767, -+SNULL, 139826129997823, 139826130022399, -+STORE, 139826127904768, 139826129997823, -+STORE, 139826129997824, 139826130022399, -+SNULL, 139826129997824, 139826130006015, -+STORE, 139826130006016, 139826130022399, -+STORE, 139826129997824, 139826130006015, -+ERASE, 139826129997824, 139826130006015, -+STORE, 139826129997824, 139826130006015, -+ERASE, 139826130006016, 139826130022399, -+STORE, 139826130006016, 139826130022399, -+STORE, 139826124009472, 139826127806463, -+SNULL, 139826124009472, 139826125668351, -+STORE, 139826125668352, 139826127806463, -+STORE, 139826124009472, 139826125668351, -+SNULL, 139826127765503, 139826127806463, -+STORE, 139826125668352, 139826127765503, -+STORE, 139826127765504, 139826127806463, -+SNULL, 139826127765504, 139826127790079, -+STORE, 139826127790080, 139826127806463, -+STORE, 139826127765504, 139826127790079, -+ERASE, 139826127765504, 139826127790079, -+STORE, 139826127765504, 139826127790079, -+ERASE, 139826127790080, 139826127806463, -+STORE, 139826127790080, 139826127806463, -+STORE, 139826121748480, 139826124009471, -+SNULL, 139826121748480, 139826121900031, -+STORE, 139826121900032, 139826124009471, -+STORE, 139826121748480, 139826121900031, -+SNULL, 139826123993087, 139826124009471, -+STORE, 139826121900032, 139826123993087, -+STORE, 139826123993088, 139826124009471, -+SNULL, 139826123993088, 139826124001279, -+STORE, 139826124001280, 139826124009471, -+STORE, 139826123993088, 139826124001279, -+ERASE, 139826123993088, 139826124001279, -+STORE, 139826123993088, 139826124001279, -+ERASE, 139826124001280, 139826124009471, -+STORE, 139826124001280, 139826124009471, -+STORE, 139826119626752, 139826121748479, -+SNULL, 139826119626752, 139826119643135, -+STORE, 139826119643136, 139826121748479, -+STORE, 139826119626752, 139826119643135, -+SNULL, 139826121740287, 139826121748479, -+STORE, 139826119643136, 139826121740287, -+STORE, 139826121740288, 139826121748479, -+ERASE, 139826121740288, 139826121748479, -+STORE, 139826121740288, 139826121748479, -+STORE, 139826136834048, 139826136842239, -+STORE, 139826117496832, 139826119626751, -+SNULL, 139826117496832, 139826117525503, -+STORE, 139826117525504, 139826119626751, -+STORE, 139826117496832, 139826117525503, -+SNULL, 139826119618559, 139826119626751, -+STORE, 139826117525504, 139826119618559, -+STORE, 139826119618560, 139826119626751, -+ERASE, 139826119618560, 139826119626751, -+STORE, 139826119618560, 139826119626751, -+STORE, 139826115244032, 139826117496831, -+SNULL, 139826115244032, 139826115395583, -+STORE, 139826115395584, 139826117496831, -+STORE, 139826115244032, 139826115395583, -+SNULL, 139826117488639, 139826117496831, -+STORE, 139826115395584, 139826117488639, -+STORE, 139826117488640, 139826117496831, -+ERASE, 139826117488640, 139826117496831, -+STORE, 139826117488640, 139826117496831, -+STORE, 139826113073152, 139826115244031, -+SNULL, 139826113073152, 139826113142783, -+STORE, 139826113142784, 139826115244031, -+STORE, 139826113073152, 139826113142783, -+SNULL, 139826115235839, 139826115244031, -+STORE, 139826113142784, 139826115235839, -+STORE, 139826115235840, 139826115244031, -+ERASE, 139826115235840, 139826115244031, -+STORE, 139826115235840, 139826115244031, -+STORE, 139826109861888, 139826113073151, -+SNULL, 139826109861888, 139826110939135, -+STORE, 139826110939136, 139826113073151, -+STORE, 139826109861888, 139826110939135, -+SNULL, 139826113036287, 139826113073151, -+STORE, 139826110939136, 139826113036287, -+STORE, 139826113036288, 139826113073151, -+ERASE, 139826113036288, 139826113073151, -+STORE, 139826113036288, 139826113073151, -+STORE, 139826107727872, 139826109861887, -+SNULL, 139826107727872, 139826107756543, -+STORE, 139826107756544, 139826109861887, -+STORE, 139826107727872, 139826107756543, -+SNULL, 139826109853695, 139826109861887, -+STORE, 139826107756544, 139826109853695, -+STORE, 139826109853696, 139826109861887, -+ERASE, 139826109853696, 139826109861887, -+STORE, 139826109853696, 139826109861887, -+STORE, 139826105417728, 139826107727871, -+SNULL, 139826105417728, 139826105622527, -+STORE, 139826105622528, 139826107727871, -+STORE, 139826105417728, 139826105622527, -+SNULL, 139826107719679, 139826107727871, -+STORE, 139826105622528, 139826107719679, -+STORE, 139826107719680, 139826107727871, -+ERASE, 139826107719680, 139826107727871, -+STORE, 139826107719680, 139826107727871, -+STORE, 139826136825856, 139826136842239, -+STORE, 139826103033856, 139826105417727, -+SNULL, 139826103033856, 139826103226367, -+STORE, 139826103226368, 139826105417727, -+STORE, 139826103033856, 139826103226367, -+SNULL, 139826105319423, 139826105417727, -+STORE, 139826103226368, 139826105319423, -+STORE, 139826105319424, 139826105417727, -+ERASE, 139826105319424, 139826105417727, -+STORE, 139826105319424, 139826105417727, -+STORE, 139826100916224, 139826103033855, -+SNULL, 139826100916224, 139826100932607, -+STORE, 139826100932608, 139826103033855, -+STORE, 139826100916224, 139826100932607, -+SNULL, 139826103025663, 139826103033855, -+STORE, 139826100932608, 139826103025663, -+STORE, 139826103025664, 139826103033855, -+ERASE, 139826103025664, 139826103033855, -+STORE, 139826103025664, 139826103033855, -+STORE, 139826098348032, 139826100916223, -+SNULL, 139826098348032, 139826098814975, -+STORE, 139826098814976, 139826100916223, -+STORE, 139826098348032, 139826098814975, -+SNULL, 139826100908031, 139826100916223, -+STORE, 139826098814976, 139826100908031, -+STORE, 139826100908032, 139826100916223, -+ERASE, 139826100908032, 139826100916223, -+STORE, 139826100908032, 139826100916223, -+STORE, 139826096234496, 139826098348031, -+SNULL, 139826096234496, 139826096246783, -+STORE, 139826096246784, 139826098348031, -+STORE, 139826096234496, 139826096246783, -+SNULL, 139826098339839, 139826098348031, -+STORE, 139826096246784, 139826098339839, -+STORE, 139826098339840, 139826098348031, -+ERASE, 139826098339840, 139826098348031, -+STORE, 139826098339840, 139826098348031, -+STORE, 139826094055424, 139826096234495, -+SNULL, 139826094055424, 139826094133247, -+STORE, 139826094133248, 139826096234495, -+STORE, 139826094055424, 139826094133247, -+SNULL, 139826096226303, 139826096234495, -+STORE, 139826094133248, 139826096226303, -+STORE, 139826096226304, 139826096234495, -+ERASE, 139826096226304, 139826096234495, -+STORE, 139826096226304, 139826096234495, -+STORE, 139826136817664, 139826136842239, -+STORE, 139826091937792, 139826094055423, -+SNULL, 139826091937792, 139826091954175, -+STORE, 139826091954176, 139826094055423, -+STORE, 139826091937792, 139826091954175, -+SNULL, 139826094047231, 139826094055423, -+STORE, 139826091954176, 139826094047231, -+STORE, 139826094047232, 139826094055423, -+ERASE, 139826094047232, 139826094055423, -+STORE, 139826094047232, 139826094055423, -+STORE, 139826136809472, 139826136842239, -+SNULL, 139826127781887, 139826127790079, -+STORE, 139826127765504, 139826127781887, -+STORE, 139826127781888, 139826127790079, -+SNULL, 139826094051327, 139826094055423, -+STORE, 139826094047232, 139826094051327, -+STORE, 139826094051328, 139826094055423, -+SNULL, 139826096230399, 139826096234495, -+STORE, 139826096226304, 139826096230399, -+STORE, 139826096230400, 139826096234495, -+SNULL, 139826098343935, 139826098348031, -+STORE, 139826098339840, 139826098343935, -+STORE, 139826098343936, 139826098348031, -+SNULL, 139826130001919, 139826130006015, -+STORE, 139826129997824, 139826130001919, -+STORE, 139826130001920, 139826130006015, -+SNULL, 139826100912127, 139826100916223, -+STORE, 139826100908032, 139826100912127, -+STORE, 139826100912128, 139826100916223, -+SNULL, 139826103029759, 139826103033855, -+STORE, 139826103025664, 139826103029759, -+STORE, 139826103029760, 139826103033855, -+SNULL, 139826105413631, 139826105417727, -+STORE, 139826105319424, 139826105413631, -+STORE, 139826105413632, 139826105417727, -+SNULL, 139826107723775, 139826107727871, -+STORE, 139826107719680, 139826107723775, -+STORE, 139826107723776, 139826107727871, -+SNULL, 139826109857791, 139826109861887, -+STORE, 139826109853696, 139826109857791, -+STORE, 139826109857792, 139826109861887, -+SNULL, 139826113044479, 139826113073151, -+STORE, 139826113036288, 139826113044479, -+STORE, 139826113044480, 139826113073151, -+SNULL, 139826115239935, 139826115244031, -+STORE, 139826115235840, 139826115239935, -+STORE, 139826115239936, 139826115244031, -+SNULL, 139826117492735, 139826117496831, -+STORE, 139826117488640, 139826117492735, -+STORE, 139826117492736, 139826117496831, -+SNULL, 139826119622655, 139826119626751, -+STORE, 139826119618560, 139826119622655, -+STORE, 139826119622656, 139826119626751, -+SNULL, 139826121744383, 139826121748479, -+STORE, 139826121740288, 139826121744383, -+STORE, 139826121744384, 139826121748479, -+SNULL, 139826123997183, 139826124001279, -+STORE, 139826123993088, 139826123997183, -+STORE, 139826123997184, 139826124001279, -+SNULL, 139826132398079, 139826132402175, -+STORE, 139826132381696, 139826132398079, -+STORE, 139826132398080, 139826132402175, -+SNULL, 139826134622207, 139826134626303, -+STORE, 139826134056960, 139826134622207, -+STORE, 139826134622208, 139826134626303, -+SNULL, 94077521309695, 94077521313791, -+STORE, 94077521305600, 94077521309695, -+STORE, 94077521309696, 94077521313791, -+SNULL, 139826136875007, 139826136879103, -+STORE, 139826136870912, 139826136875007, -+STORE, 139826136875008, 139826136879103, -+ERASE, 139826136842240, 139826136862719, -+STORE, 94077554049024, 94077554184191, -+STORE, 139826136543232, 139826136842239, -+STORE, 139826136276992, 139826136842239, -+STORE, 139826136010752, 139826136842239, -+STORE, 139826135744512, 139826136842239, -+SNULL, 139826136543231, 139826136842239, -+STORE, 139826135744512, 139826136543231, -+STORE, 139826136543232, 139826136842239, -+SNULL, 139826136543232, 139826136809471, -+STORE, 139826136809472, 139826136842239, -+STORE, 139826136543232, 139826136809471, -+ }; -+ unsigned long set15[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140722061451264, 140737488351231, -+SNULL, 140722061455359, 140737488351231, -+STORE, 140722061451264, 140722061455359, -+STORE, 140722061320192, 140722061455359, -+STORE, 94728600248320, 94728600289279, -+SNULL, 94728600276991, 94728600289279, -+STORE, 94728600248320, 94728600276991, -+STORE, 94728600276992, 94728600289279, -+ERASE, 94728600276992, 94728600289279, -+STORE, 94728600281088, 94728600289279, -+STORE, 139906806779904, 139906809032703, -+SNULL, 139906806923263, 139906809032703, -+STORE, 139906806779904, 139906806923263, -+STORE, 139906806923264, 139906809032703, -+ERASE, 139906806923264, 139906809032703, -+STORE, 139906809020416, 139906809028607, -+STORE, 139906809028608, 139906809032703, -+STORE, 140722061692928, 140722061697023, -+STORE, 140722061680640, 140722061692927, -+STORE, 139906809012224, 139906809020415, -+STORE, 139906804555776, 139906806779903, -+SNULL, 139906806206463, 139906806779903, -+STORE, 139906804555776, 139906806206463, -+STORE, 139906806206464, 139906806779903, -+SNULL, 139906806206464, 139906806775807, -+STORE, 139906806775808, 139906806779903, -+STORE, 139906806206464, 139906806775807, -+ERASE, 139906806206464, 139906806775807, -+STORE, 139906806206464, 139906806775807, -+ERASE, 139906806775808, 139906806779903, -+STORE, 139906806775808, 139906806779903, -+STORE, 139906808991744, 139906809012223, -+STORE, 139906802171904, 139906804555775, -+SNULL, 139906802171904, 139906802438143, -+STORE, 139906802438144, 139906804555775, -+STORE, 139906802171904, 139906802438143, -+SNULL, 139906804531199, 139906804555775, -+STORE, 139906802438144, 139906804531199, -+STORE, 139906804531200, 139906804555775, -+SNULL, 139906804531200, 139906804551679, -+STORE, 139906804551680, 139906804555775, -+STORE, 139906804531200, 139906804551679, -+ERASE, 139906804531200, 139906804551679, -+STORE, 139906804531200, 139906804551679, -+ERASE, 139906804551680, 139906804555775, -+STORE, 139906804551680, 139906804555775, -+STORE, 139906799955968, 139906802171903, -+SNULL, 139906799955968, 139906800054271, -+STORE, 139906800054272, 139906802171903, -+STORE, 139906799955968, 139906800054271, -+SNULL, 139906802147327, 139906802171903, -+STORE, 139906800054272, 139906802147327, -+STORE, 139906802147328, 139906802171903, -+SNULL, 139906802147328, 139906802155519, -+STORE, 139906802155520, 139906802171903, -+STORE, 139906802147328, 139906802155519, -+ERASE, 139906802147328, 139906802155519, -+STORE, 139906802147328, 139906802155519, -+ERASE, 139906802155520, 139906802171903, -+STORE, 139906802155520, 139906802171903, -+STORE, 139906796158976, 139906799955967, -+SNULL, 139906796158976, 139906797817855, -+STORE, 139906797817856, 139906799955967, -+STORE, 139906796158976, 139906797817855, -+SNULL, 139906799915007, 139906799955967, -+STORE, 139906797817856, 139906799915007, -+STORE, 139906799915008, 139906799955967, -+SNULL, 139906799915008, 139906799939583, -+STORE, 139906799939584, 139906799955967, -+STORE, 139906799915008, 139906799939583, -+ERASE, 139906799915008, 139906799939583, -+STORE, 139906799915008, 139906799939583, -+ERASE, 139906799939584, 139906799955967, -+STORE, 139906799939584, 139906799955967, -+STORE, 139906793897984, 139906796158975, -+SNULL, 139906793897984, 139906794049535, -+STORE, 139906794049536, 139906796158975, -+STORE, 139906793897984, 139906794049535, -+SNULL, 139906796142591, 139906796158975, -+STORE, 139906794049536, 139906796142591, -+STORE, 139906796142592, 139906796158975, -+SNULL, 139906796142592, 139906796150783, -+STORE, 139906796150784, 139906796158975, -+STORE, 139906796142592, 139906796150783, -+ERASE, 139906796142592, 139906796150783, -+STORE, 139906796142592, 139906796150783, -+ERASE, 139906796150784, 139906796158975, -+STORE, 139906796150784, 139906796158975, -+STORE, 139906791776256, 139906793897983, -+SNULL, 139906791776256, 139906791792639, -+STORE, 139906791792640, 139906793897983, -+STORE, 139906791776256, 139906791792639, -+SNULL, 139906793889791, 139906793897983, -+STORE, 139906791792640, 139906793889791, -+STORE, 139906793889792, 139906793897983, -+ERASE, 139906793889792, 139906793897983, -+STORE, 139906793889792, 139906793897983, -+STORE, 139906808983552, 139906808991743, -+STORE, 139906789646336, 139906791776255, -+SNULL, 139906789646336, 139906789675007, -+STORE, 139906789675008, 139906791776255, -+STORE, 139906789646336, 139906789675007, -+SNULL, 139906791768063, 139906791776255, -+STORE, 139906789675008, 139906791768063, -+STORE, 139906791768064, 139906791776255, -+ERASE, 139906791768064, 139906791776255, -+STORE, 139906791768064, 139906791776255, -+STORE, 139906787393536, 139906789646335, -+SNULL, 139906787393536, 139906787545087, -+STORE, 139906787545088, 139906789646335, -+STORE, 139906787393536, 139906787545087, -+SNULL, 139906789638143, 139906789646335, -+STORE, 139906787545088, 139906789638143, -+STORE, 139906789638144, 139906789646335, -+ERASE, 139906789638144, 139906789646335, -+STORE, 139906789638144, 139906789646335, -+STORE, 139906785222656, 139906787393535, -+SNULL, 139906785222656, 139906785292287, -+STORE, 139906785292288, 139906787393535, -+STORE, 139906785222656, 139906785292287, -+SNULL, 139906787385343, 139906787393535, -+STORE, 139906785292288, 139906787385343, -+STORE, 139906787385344, 139906787393535, -+ERASE, 139906787385344, 139906787393535, -+STORE, 139906787385344, 139906787393535, -+STORE, 139906782011392, 139906785222655, -+SNULL, 139906782011392, 139906783088639, -+STORE, 139906783088640, 139906785222655, -+STORE, 139906782011392, 139906783088639, -+SNULL, 139906785185791, 139906785222655, -+STORE, 139906783088640, 139906785185791, -+STORE, 139906785185792, 139906785222655, -+ERASE, 139906785185792, 139906785222655, -+STORE, 139906785185792, 139906785222655, -+STORE, 139906779877376, 139906782011391, -+SNULL, 139906779877376, 139906779906047, -+STORE, 139906779906048, 139906782011391, -+STORE, 139906779877376, 139906779906047, -+SNULL, 139906782003199, 139906782011391, -+STORE, 139906779906048, 139906782003199, -+STORE, 139906782003200, 139906782011391, -+ERASE, 139906782003200, 139906782011391, -+STORE, 139906782003200, 139906782011391, -+STORE, 139906777567232, 139906779877375, -+SNULL, 139906777567232, 139906777772031, -+STORE, 139906777772032, 139906779877375, -+STORE, 139906777567232, 139906777772031, -+SNULL, 139906779869183, 139906779877375, -+STORE, 139906777772032, 139906779869183, -+STORE, 139906779869184, 139906779877375, -+ERASE, 139906779869184, 139906779877375, -+STORE, 139906779869184, 139906779877375, -+STORE, 139906808975360, 139906808991743, -+STORE, 139906775183360, 139906777567231, -+SNULL, 139906775183360, 139906775375871, -+STORE, 139906775375872, 139906777567231, -+STORE, 139906775183360, 139906775375871, -+SNULL, 139906777468927, 139906777567231, -+STORE, 139906775375872, 139906777468927, -+STORE, 139906777468928, 139906777567231, -+ERASE, 139906777468928, 139906777567231, -+STORE, 139906777468928, 139906777567231, -+STORE, 139906773065728, 139906775183359, -+SNULL, 139906773065728, 139906773082111, -+STORE, 139906773082112, 139906775183359, -+STORE, 139906773065728, 139906773082111, -+SNULL, 139906775175167, 139906775183359, -+STORE, 139906773082112, 139906775175167, -+STORE, 139906775175168, 139906775183359, -+ERASE, 139906775175168, 139906775183359, -+STORE, 139906775175168, 139906775183359, -+STORE, 139906770497536, 139906773065727, -+SNULL, 139906770497536, 139906770964479, -+STORE, 139906770964480, 139906773065727, -+STORE, 139906770497536, 139906770964479, -+SNULL, 139906773057535, 139906773065727, -+STORE, 139906770964480, 139906773057535, -+STORE, 139906773057536, 139906773065727, -+ERASE, 139906773057536, 139906773065727, -+STORE, 139906773057536, 139906773065727, -+STORE, 139906768384000, 139906770497535, -+SNULL, 139906768384000, 139906768396287, -+STORE, 139906768396288, 139906770497535, -+STORE, 139906768384000, 139906768396287, -+SNULL, 139906770489343, 139906770497535, -+STORE, 139906768396288, 139906770489343, -+STORE, 139906770489344, 139906770497535, -+ERASE, 139906770489344, 139906770497535, -+STORE, 139906770489344, 139906770497535, -+STORE, 139906766204928, 139906768383999, -+SNULL, 139906766204928, 139906766282751, -+STORE, 139906766282752, 139906768383999, -+STORE, 139906766204928, 139906766282751, -+SNULL, 139906768375807, 139906768383999, -+STORE, 139906766282752, 139906768375807, -+STORE, 139906768375808, 139906768383999, -+ERASE, 139906768375808, 139906768383999, -+STORE, 139906768375808, 139906768383999, -+STORE, 139906808967168, 139906808991743, -+STORE, 139906764087296, 139906766204927, -+SNULL, 139906764087296, 139906764103679, -+STORE, 139906764103680, 139906766204927, -+STORE, 139906764087296, 139906764103679, -+SNULL, 139906766196735, 139906766204927, -+STORE, 139906764103680, 139906766196735, -+STORE, 139906766196736, 139906766204927, -+ERASE, 139906766196736, 139906766204927, -+STORE, 139906766196736, 139906766204927, -+STORE, 139906808958976, 139906808991743, -+SNULL, 139906799931391, 139906799939583, -+STORE, 139906799915008, 139906799931391, -+STORE, 139906799931392, 139906799939583, -+SNULL, 139906766200831, 139906766204927, -+STORE, 139906766196736, 139906766200831, -+STORE, 139906766200832, 139906766204927, -+SNULL, 139906768379903, 139906768383999, -+STORE, 139906768375808, 139906768379903, -+STORE, 139906768379904, 139906768383999, -+SNULL, 139906770493439, 139906770497535, -+STORE, 139906770489344, 139906770493439, -+STORE, 139906770493440, 139906770497535, -+SNULL, 139906802151423, 139906802155519, -+STORE, 139906802147328, 139906802151423, -+STORE, 139906802151424, 139906802155519, -+SNULL, 139906773061631, 139906773065727, -+STORE, 139906773057536, 139906773061631, -+STORE, 139906773061632, 139906773065727, -+SNULL, 139906775179263, 139906775183359, -+STORE, 139906775175168, 139906775179263, -+STORE, 139906775179264, 139906775183359, -+SNULL, 139906777563135, 139906777567231, -+STORE, 139906777468928, 139906777563135, -+STORE, 139906777563136, 139906777567231, -+SNULL, 139906779873279, 139906779877375, -+STORE, 139906779869184, 139906779873279, -+STORE, 139906779873280, 139906779877375, -+SNULL, 139906782007295, 139906782011391, -+STORE, 139906782003200, 139906782007295, -+STORE, 139906782007296, 139906782011391, -+SNULL, 139906785193983, 139906785222655, -+STORE, 139906785185792, 139906785193983, -+STORE, 139906785193984, 139906785222655, -+SNULL, 139906787389439, 139906787393535, -+STORE, 139906787385344, 139906787389439, -+STORE, 139906787389440, 139906787393535, -+SNULL, 139906789642239, 139906789646335, -+STORE, 139906789638144, 139906789642239, -+STORE, 139906789642240, 139906789646335, -+SNULL, 139906791772159, 139906791776255, -+STORE, 139906791768064, 139906791772159, -+STORE, 139906791772160, 139906791776255, -+SNULL, 139906793893887, 139906793897983, -+STORE, 139906793889792, 139906793893887, -+STORE, 139906793893888, 139906793897983, -+SNULL, 139906796146687, 139906796150783, -+STORE, 139906796142592, 139906796146687, -+STORE, 139906796146688, 139906796150783, -+SNULL, 139906804547583, 139906804551679, -+STORE, 139906804531200, 139906804547583, -+STORE, 139906804547584, 139906804551679, -+SNULL, 139906806771711, 139906806775807, -+STORE, 139906806206464, 139906806771711, -+STORE, 139906806771712, 139906806775807, -+SNULL, 94728600285183, 94728600289279, -+STORE, 94728600281088, 94728600285183, -+STORE, 94728600285184, 94728600289279, -+SNULL, 139906809024511, 139906809028607, -+STORE, 139906809020416, 139906809024511, -+STORE, 139906809024512, 139906809028607, -+ERASE, 139906808991744, 139906809012223, -+STORE, 94728620138496, 94728620273663, -+STORE, 139906808692736, 139906808991743, -+STORE, 139906808426496, 139906808991743, -+STORE, 139906808160256, 139906808991743, -+STORE, 139906807894016, 139906808991743, -+SNULL, 139906808692735, 139906808991743, -+STORE, 139906807894016, 139906808692735, -+STORE, 139906808692736, 139906808991743, -+SNULL, 139906808692736, 139906808958975, -+STORE, 139906808958976, 139906808991743, -+STORE, 139906808692736, 139906808958975, -+ }; -+ -+ unsigned long set16[] = { -+STORE, 94174808662016, 94174809321471, -+STORE, 94174811414528, 94174811426815, -+STORE, 94174811426816, 94174811430911, -+STORE, 94174811430912, 94174811443199, -+STORE, 94174841700352, 94174841835519, -+STORE, 140173257838592, 140173259497471, -+STORE, 140173259497472, 140173261594623, -+STORE, 140173261594624, 140173261611007, -+STORE, 140173261611008, 140173261619199, -+STORE, 140173261619200, 140173261635583, -+STORE, 140173261635584, 140173261778943, -+STORE, 140173263863808, 140173263871999, -+STORE, 140173263876096, 140173263880191, -+STORE, 140173263880192, 140173263884287, -+STORE, 140173263884288, 140173263888383, -+STORE, 140729801007104, 140729801142271, -+STORE, 140729801617408, 140729801629695, -+STORE, 140729801629696, 140729801633791, -+STORE, 140737488347136, 140737488351231, -+STORE, 140728166858752, 140737488351231, -+SNULL, 140728166862847, 140737488351231, -+STORE, 140728166858752, 140728166862847, -+STORE, 140728166727680, 140728166862847, -+STORE, 93912949866496, 93912950337535, -+SNULL, 93912950288383, 93912950337535, -+STORE, 93912949866496, 93912950288383, -+STORE, 93912950288384, 93912950337535, -+ERASE, 93912950288384, 93912950337535, -+STORE, 93912950292480, 93912950337535, -+STORE, 139921863385088, 139921865637887, -+SNULL, 139921863528447, 139921865637887, -+STORE, 139921863385088, 139921863528447, -+STORE, 139921863528448, 139921865637887, -+ERASE, 139921863528448, 139921865637887, -+STORE, 139921865625600, 139921865633791, -+STORE, 139921865633792, 139921865637887, -+STORE, 140728167899136, 140728167903231, -+STORE, 140728167886848, 140728167899135, -+STORE, 139921865601024, 139921865625599, -+STORE, 139921865592832, 139921865601023, -+STORE, 139921861251072, 139921863385087, -+SNULL, 139921861251072, 139921861279743, -+STORE, 139921861279744, 139921863385087, -+STORE, 139921861251072, 139921861279743, -+SNULL, 139921863376895, 139921863385087, -+STORE, 139921861279744, 139921863376895, -+STORE, 139921863376896, 139921863385087, -+ERASE, 139921863376896, 139921863385087, -+STORE, 139921863376896, 139921863385087, -+STORE, 139921858867200, 139921861251071, -+SNULL, 139921858867200, 139921859133439, -+STORE, 139921859133440, 139921861251071, -+STORE, 139921858867200, 139921859133439, -+SNULL, 139921861226495, 139921861251071, -+STORE, 139921859133440, 139921861226495, -+STORE, 139921861226496, 139921861251071, -+SNULL, 139921861226496, 139921861246975, -+STORE, 139921861246976, 139921861251071, -+STORE, 139921861226496, 139921861246975, -+ERASE, 139921861226496, 139921861246975, -+STORE, 139921861226496, 139921861246975, -+ERASE, 139921861246976, 139921861251071, -+STORE, 139921861246976, 139921861251071, -+STORE, 139921856675840, 139921858867199, -+SNULL, 139921856675840, 139921856765951, -+STORE, 139921856765952, 139921858867199, -+STORE, 139921856675840, 139921856765951, -+SNULL, 139921858859007, 139921858867199, -+STORE, 139921856765952, 139921858859007, -+STORE, 139921858859008, 139921858867199, -+ERASE, 139921858859008, 139921858867199, -+STORE, 139921858859008, 139921858867199, -+STORE, 139921854414848, 139921856675839, -+SNULL, 139921854414848, 139921854566399, -+STORE, 139921854566400, 139921856675839, -+STORE, 139921854414848, 139921854566399, -+SNULL, 139921856659455, 139921856675839, -+STORE, 139921854566400, 139921856659455, -+STORE, 139921856659456, 139921856675839, -+SNULL, 139921856659456, 139921856667647, -+STORE, 139921856667648, 139921856675839, -+STORE, 139921856659456, 139921856667647, -+ERASE, 139921856659456, 139921856667647, -+STORE, 139921856659456, 139921856667647, -+ERASE, 139921856667648, 139921856675839, -+STORE, 139921856667648, 139921856675839, -+STORE, 139921852284928, 139921854414847, -+SNULL, 139921852284928, 139921852313599, -+STORE, 139921852313600, 139921854414847, -+STORE, 139921852284928, 139921852313599, -+SNULL, 139921854406655, 139921854414847, -+STORE, 139921852313600, 139921854406655, -+STORE, 139921854406656, 139921854414847, -+ERASE, 139921854406656, 139921854414847, -+STORE, 139921854406656, 139921854414847, -+STORE, 139921850068992, 139921852284927, -+SNULL, 139921850068992, 139921850167295, -+STORE, 139921850167296, 139921852284927, -+STORE, 139921850068992, 139921850167295, -+SNULL, 139921852260351, 139921852284927, -+STORE, 139921850167296, 139921852260351, -+STORE, 139921852260352, 139921852284927, -+SNULL, 139921852260352, 139921852268543, -+STORE, 139921852268544, 139921852284927, -+STORE, 139921852260352, 139921852268543, -+ERASE, 139921852260352, 139921852268543, -+STORE, 139921852260352, 139921852268543, -+ERASE, 139921852268544, 139921852284927, -+STORE, 139921852268544, 139921852284927, -+STORE, 139921865584640, 139921865601023, -+STORE, 139921846272000, 139921850068991, -+SNULL, 139921846272000, 139921847930879, -+STORE, 139921847930880, 139921850068991, -+STORE, 139921846272000, 139921847930879, -+SNULL, 139921850028031, 139921850068991, -+STORE, 139921847930880, 139921850028031, -+STORE, 139921850028032, 139921850068991, -+SNULL, 139921850028032, 139921850052607, -+STORE, 139921850052608, 139921850068991, -+STORE, 139921850028032, 139921850052607, -+ERASE, 139921850028032, 139921850052607, -+STORE, 139921850028032, 139921850052607, -+ERASE, 139921850052608, 139921850068991, -+STORE, 139921850052608, 139921850068991, -+STORE, 139921844154368, 139921846271999, -+SNULL, 139921844154368, 139921844170751, -+STORE, 139921844170752, 139921846271999, -+STORE, 139921844154368, 139921844170751, -+SNULL, 139921846263807, 139921846271999, -+STORE, 139921844170752, 139921846263807, -+STORE, 139921846263808, 139921846271999, -+ERASE, 139921846263808, 139921846271999, -+STORE, 139921846263808, 139921846271999, -+STORE, 139921842036736, 139921844154367, -+SNULL, 139921842036736, 139921842053119, -+STORE, 139921842053120, 139921844154367, -+STORE, 139921842036736, 139921842053119, -+SNULL, 139921844146175, 139921844154367, -+STORE, 139921842053120, 139921844146175, -+STORE, 139921844146176, 139921844154367, -+ERASE, 139921844146176, 139921844154367, -+STORE, 139921844146176, 139921844154367, -+STORE, 139921839468544, 139921842036735, -+SNULL, 139921839468544, 139921839935487, -+STORE, 139921839935488, 139921842036735, -+STORE, 139921839468544, 139921839935487, -+SNULL, 139921842028543, 139921842036735, -+STORE, 139921839935488, 139921842028543, -+STORE, 139921842028544, 139921842036735, -+ERASE, 139921842028544, 139921842036735, -+STORE, 139921842028544, 139921842036735, -+STORE, 139921837355008, 139921839468543, -+SNULL, 139921837355008, 139921837367295, -+STORE, 139921837367296, 139921839468543, -+STORE, 139921837355008, 139921837367295, -+SNULL, 139921839460351, 139921839468543, -+STORE, 139921837367296, 139921839460351, -+STORE, 139921839460352, 139921839468543, -+ERASE, 139921839460352, 139921839468543, -+STORE, 139921839460352, 139921839468543, -+STORE, 139921865576448, 139921865601023, -+STORE, 139921865564160, 139921865601023, -+SNULL, 139921850044415, 139921850052607, -+STORE, 139921850028032, 139921850044415, -+STORE, 139921850044416, 139921850052607, -+SNULL, 139921839464447, 139921839468543, -+STORE, 139921839460352, 139921839464447, -+STORE, 139921839464448, 139921839468543, -+SNULL, 139921852264447, 139921852268543, -+STORE, 139921852260352, 139921852264447, -+STORE, 139921852264448, 139921852268543, -+SNULL, 139921842032639, 139921842036735, -+STORE, 139921842028544, 139921842032639, -+STORE, 139921842032640, 139921842036735, -+SNULL, 139921844150271, 139921844154367, -+STORE, 139921844146176, 139921844150271, -+STORE, 139921844150272, 139921844154367, -+SNULL, 139921846267903, 139921846271999, -+STORE, 139921846263808, 139921846267903, -+STORE, 139921846267904, 139921846271999, -+SNULL, 139921854410751, 139921854414847, -+STORE, 139921854406656, 139921854410751, -+STORE, 139921854410752, 139921854414847, -+SNULL, 139921856663551, 139921856667647, -+STORE, 139921856659456, 139921856663551, -+STORE, 139921856663552, 139921856667647, -+SNULL, 139921858863103, 139921858867199, -+STORE, 139921858859008, 139921858863103, -+STORE, 139921858863104, 139921858867199, -+SNULL, 139921861242879, 139921861246975, -+STORE, 139921861226496, 139921861242879, -+STORE, 139921861242880, 139921861246975, -+SNULL, 139921863380991, 139921863385087, -+STORE, 139921863376896, 139921863380991, -+STORE, 139921863380992, 139921863385087, -+SNULL, 93912950333439, 93912950337535, -+STORE, 93912950292480, 93912950333439, -+STORE, 93912950333440, 93912950337535, -+SNULL, 139921865629695, 139921865633791, -+STORE, 139921865625600, 139921865629695, -+STORE, 139921865629696, 139921865633791, -+ERASE, 139921865601024, 139921865625599, -+STORE, 93912968110080, 93912968245247, -+STORE, 139921828913152, 139921837355007, -+STORE, 139921865621504, 139921865625599, -+STORE, 139921865617408, 139921865621503, -+STORE, 139921865613312, 139921865617407, -+STORE, 139921865547776, 139921865564159, -+ }; -+ -+ unsigned long set17[] = { -+STORE, 94397057224704, 94397057646591, -+STORE, 94397057650688, 94397057691647, -+STORE, 94397057691648, 94397057695743, -+STORE, 94397075271680, 94397075406847, -+STORE, 139953169051648, 139953169063935, -+STORE, 139953169063936, 139953171156991, -+STORE, 139953171156992, 139953171161087, -+STORE, 139953171161088, 139953171165183, -+STORE, 139953171165184, 139953171632127, -+STORE, 139953171632128, 139953173725183, -+STORE, 139953173725184, 139953173729279, -+STORE, 139953173729280, 139953173733375, -+STORE, 139953173733376, 139953173749759, -+STORE, 139953173749760, 139953175842815, -+STORE, 139953175842816, 139953175846911, -+STORE, 139953175846912, 139953175851007, -+STORE, 139953175851008, 139953175867391, -+STORE, 139953175867392, 139953177960447, -+STORE, 139953177960448, 139953177964543, -+STORE, 139953177964544, 139953177968639, -+STORE, 139953177968640, 139953179627519, -+STORE, 139953179627520, 139953181724671, -+STORE, 139953181724672, 139953181741055, -+STORE, 139953181741056, 139953181749247, -+STORE, 139953181749248, 139953181765631, -+STORE, 139953181765632, 139953181863935, -+STORE, 139953181863936, 139953183956991, -+STORE, 139953183956992, 139953183961087, -+STORE, 139953183961088, 139953183965183, -+STORE, 139953183965184, 139953183981567, -+STORE, 139953183981568, 139953184010239, -+STORE, 139953184010240, 139953186103295, -+STORE, 139953186103296, 139953186107391, -+STORE, 139953186107392, 139953186111487, -+STORE, 139953186111488, 139953186263039, -+STORE, 139953186263040, 139953188356095, -+STORE, 139953188356096, 139953188360191, -+STORE, 139953188360192, 139953188364287, -+STORE, 139953188364288, 139953188372479, -+STORE, 139953188372480, 139953188462591, -+STORE, 139953188462592, 139953190555647, -+STORE, 139953190555648, 139953190559743, -+STORE, 139953190559744, 139953190563839, -+STORE, 139953190563840, 139953190830079, -+STORE, 139953190830080, 139953192923135, -+STORE, 139953192923136, 139953192939519, -+STORE, 139953192939520, 139953192943615, -+STORE, 139953192943616, 139953192947711, -+STORE, 139953192947712, 139953192976383, -+STORE, 139953192976384, 139953195073535, -+STORE, 139953195073536, 139953195077631, -+STORE, 139953195077632, 139953195081727, -+STORE, 139953195081728, 139953195225087, -+STORE, 139953197281280, 139953197318143, -+STORE, 139953197322240, 139953197326335, -+STORE, 139953197326336, 139953197330431, -+STORE, 139953197330432, 139953197334527, -+STORE, 140720477511680, 140720477646847, -+STORE, 140720478302208, 140720478314495, -+STORE, 140720478314496, 140720478318591, -+ }; -+ unsigned long set18[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140724953673728, 140737488351231, -+SNULL, 140724953677823, 140737488351231, -+STORE, 140724953673728, 140724953677823, -+STORE, 140724953542656, 140724953677823, -+STORE, 94675199266816, 94675199311871, -+SNULL, 94675199303679, 94675199311871, -+STORE, 94675199266816, 94675199303679, -+STORE, 94675199303680, 94675199311871, -+ERASE, 94675199303680, 94675199311871, -+STORE, 94675199303680, 94675199311871, -+STORE, 140222970605568, 140222972858367, -+SNULL, 140222970748927, 140222972858367, -+STORE, 140222970605568, 140222970748927, -+STORE, 140222970748928, 140222972858367, -+ERASE, 140222970748928, 140222972858367, -+STORE, 140222972846080, 140222972854271, -+STORE, 140222972854272, 140222972858367, -+STORE, 140724954365952, 140724954370047, -+STORE, 140724954353664, 140724954365951, -+STORE, 140222972841984, 140222972846079, -+STORE, 140222972833792, 140222972841983, -+STORE, 140222968475648, 140222970605567, -+SNULL, 140222968475648, 140222968504319, -+STORE, 140222968504320, 140222970605567, -+STORE, 140222968475648, 140222968504319, -+SNULL, 140222970597375, 140222970605567, -+STORE, 140222968504320, 140222970597375, -+STORE, 140222970597376, 140222970605567, -+ERASE, 140222970597376, 140222970605567, -+STORE, 140222970597376, 140222970605567, -+ }; -+ unsigned long set19[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140725182459904, 140737488351231, -+SNULL, 140725182463999, 140737488351231, -+STORE, 140725182459904, 140725182463999, -+STORE, 140725182328832, 140725182463999, -+STORE, 94730166636544, 94730166763519, -+SNULL, 94730166747135, 94730166763519, -+STORE, 94730166636544, 94730166747135, -+STORE, 94730166747136, 94730166763519, -+ERASE, 94730166747136, 94730166763519, -+STORE, 94730166751232, 94730166763519, -+STORE, 140656834555904, 140656836808703, -+SNULL, 140656834699263, 140656836808703, -+STORE, 140656834555904, 140656834699263, -+STORE, 140656834699264, 140656836808703, -+ERASE, 140656834699264, 140656836808703, -+STORE, 140656836796416, 140656836804607, -+STORE, 140656836804608, 140656836808703, -+STORE, 140725183389696, 140725183393791, -+STORE, 140725183377408, 140725183389695, -+STORE, 140656836788224, 140656836796415, -+STORE, 140656832331776, 140656834555903, -+SNULL, 140656833982463, 140656834555903, -+STORE, 140656832331776, 140656833982463, -+STORE, 140656833982464, 140656834555903, -+SNULL, 140656833982464, 140656834551807, -+STORE, 140656834551808, 140656834555903, -+STORE, 140656833982464, 140656834551807, -+ERASE, 140656833982464, 140656834551807, -+STORE, 140656833982464, 140656834551807, -+ERASE, 140656834551808, 140656834555903, -+STORE, 140656834551808, 140656834555903, -+STORE, 140656836763648, 140656836788223, -+STORE, 140656830070784, 140656832331775, -+SNULL, 140656830070784, 140656830222335, -+STORE, 140656830222336, 140656832331775, -+STORE, 140656830070784, 140656830222335, -+SNULL, 140656832315391, 140656832331775, -+STORE, 140656830222336, 140656832315391, -+STORE, 140656832315392, 140656832331775, -+SNULL, 140656832315392, 140656832323583, -+STORE, 140656832323584, 140656832331775, -+STORE, 140656832315392, 140656832323583, -+ERASE, 140656832315392, 140656832323583, -+STORE, 140656832315392, 140656832323583, -+ERASE, 140656832323584, 140656832331775, -+STORE, 140656832323584, 140656832331775, -+STORE, 140656827940864, 140656830070783, -+SNULL, 140656827940864, 140656827969535, -+STORE, 140656827969536, 140656830070783, -+STORE, 140656827940864, 140656827969535, -+SNULL, 140656830062591, 140656830070783, -+STORE, 140656827969536, 140656830062591, -+STORE, 140656830062592, 140656830070783, -+ERASE, 140656830062592, 140656830070783, -+STORE, 140656830062592, 140656830070783, -+STORE, 140656825724928, 140656827940863, -+SNULL, 140656825724928, 140656825823231, -+STORE, 140656825823232, 140656827940863, -+STORE, 140656825724928, 140656825823231, -+SNULL, 140656827916287, 140656827940863, -+STORE, 140656825823232, 140656827916287, -+STORE, 140656827916288, 140656827940863, -+SNULL, 140656827916288, 140656827924479, -+STORE, 140656827924480, 140656827940863, -+STORE, 140656827916288, 140656827924479, -+ERASE, 140656827916288, 140656827924479, -+STORE, 140656827916288, 140656827924479, -+ERASE, 140656827924480, 140656827940863, -+STORE, 140656827924480, 140656827940863, -+STORE, 140656821927936, 140656825724927, -+SNULL, 140656821927936, 140656823586815, -+STORE, 140656823586816, 140656825724927, -+STORE, 140656821927936, 140656823586815, -+SNULL, 140656825683967, 140656825724927, -+STORE, 140656823586816, 140656825683967, -+STORE, 140656825683968, 140656825724927, -+SNULL, 140656825683968, 140656825708543, -+STORE, 140656825708544, 140656825724927, -+STORE, 140656825683968, 140656825708543, -+ERASE, 140656825683968, 140656825708543, -+STORE, 140656825683968, 140656825708543, -+ERASE, 140656825708544, 140656825724927, -+STORE, 140656825708544, 140656825724927, -+STORE, 140656819806208, 140656821927935, -+SNULL, 140656819806208, 140656819822591, -+STORE, 140656819822592, 140656821927935, -+STORE, 140656819806208, 140656819822591, -+SNULL, 140656821919743, 140656821927935, -+STORE, 140656819822592, 140656821919743, -+STORE, 140656821919744, 140656821927935, -+ERASE, 140656821919744, 140656821927935, -+STORE, 140656821919744, 140656821927935, -+STORE, 140656836755456, 140656836763647, -+STORE, 140656817553408, 140656819806207, -+SNULL, 140656817553408, 140656817704959, -+STORE, 140656817704960, 140656819806207, -+STORE, 140656817553408, 140656817704959, -+SNULL, 140656819798015, 140656819806207, -+STORE, 140656817704960, 140656819798015, -+STORE, 140656819798016, 140656819806207, -+ERASE, 140656819798016, 140656819806207, -+STORE, 140656819798016, 140656819806207, -+STORE, 140656815382528, 140656817553407, -+SNULL, 140656815382528, 140656815452159, -+STORE, 140656815452160, 140656817553407, -+STORE, 140656815382528, 140656815452159, -+SNULL, 140656817545215, 140656817553407, -+STORE, 140656815452160, 140656817545215, -+STORE, 140656817545216, 140656817553407, -+ERASE, 140656817545216, 140656817553407, -+STORE, 140656817545216, 140656817553407, -+STORE, 140656812171264, 140656815382527, -+SNULL, 140656812171264, 140656813248511, -+STORE, 140656813248512, 140656815382527, -+STORE, 140656812171264, 140656813248511, -+SNULL, 140656815345663, 140656815382527, -+STORE, 140656813248512, 140656815345663, -+STORE, 140656815345664, 140656815382527, -+ERASE, 140656815345664, 140656815382527, -+STORE, 140656815345664, 140656815382527, -+STORE, 140656810037248, 140656812171263, -+SNULL, 140656810037248, 140656810065919, -+STORE, 140656810065920, 140656812171263, -+STORE, 140656810037248, 140656810065919, -+SNULL, 140656812163071, 140656812171263, -+STORE, 140656810065920, 140656812163071, -+STORE, 140656812163072, 140656812171263, -+ERASE, 140656812163072, 140656812171263, -+STORE, 140656812163072, 140656812171263, -+STORE, 140656807727104, 140656810037247, -+SNULL, 140656807727104, 140656807931903, -+STORE, 140656807931904, 140656810037247, -+STORE, 140656807727104, 140656807931903, -+SNULL, 140656810029055, 140656810037247, -+STORE, 140656807931904, 140656810029055, -+STORE, 140656810029056, 140656810037247, -+ERASE, 140656810029056, 140656810037247, -+STORE, 140656810029056, 140656810037247, -+STORE, 140656805343232, 140656807727103, -+SNULL, 140656805343232, 140656805535743, -+STORE, 140656805535744, 140656807727103, -+STORE, 140656805343232, 140656805535743, -+SNULL, 140656807628799, 140656807727103, -+STORE, 140656805535744, 140656807628799, -+STORE, 140656807628800, 140656807727103, -+ERASE, 140656807628800, 140656807727103, -+STORE, 140656807628800, 140656807727103, -+STORE, 140656836747264, 140656836763647, -+STORE, 140656802775040, 140656805343231, -+SNULL, 140656802775040, 140656803241983, -+STORE, 140656803241984, 140656805343231, -+STORE, 140656802775040, 140656803241983, -+SNULL, 140656805335039, 140656805343231, -+STORE, 140656803241984, 140656805335039, -+STORE, 140656805335040, 140656805343231, -+ERASE, 140656805335040, 140656805343231, -+STORE, 140656805335040, 140656805343231, -+STORE, 140656800661504, 140656802775039, -+SNULL, 140656800661504, 140656800673791, -+STORE, 140656800673792, 140656802775039, -+STORE, 140656800661504, 140656800673791, -+SNULL, 140656802766847, 140656802775039, -+STORE, 140656800673792, 140656802766847, -+STORE, 140656802766848, 140656802775039, -+ERASE, 140656802766848, 140656802775039, -+STORE, 140656802766848, 140656802775039, -+STORE, 140656798482432, 140656800661503, -+SNULL, 140656798482432, 140656798560255, -+STORE, 140656798560256, 140656800661503, -+STORE, 140656798482432, 140656798560255, -+SNULL, 140656800653311, 140656800661503, -+STORE, 140656798560256, 140656800653311, -+STORE, 140656800653312, 140656800661503, -+ERASE, 140656800653312, 140656800661503, -+STORE, 140656800653312, 140656800661503, -+STORE, 140656796364800, 140656798482431, -+SNULL, 140656796364800, 140656796381183, -+STORE, 140656796381184, 140656798482431, -+STORE, 140656796364800, 140656796381183, -+SNULL, 140656798474239, 140656798482431, -+STORE, 140656796381184, 140656798474239, -+STORE, 140656798474240, 140656798482431, -+ERASE, 140656798474240, 140656798482431, -+STORE, 140656798474240, 140656798482431, -+STORE, 140656836739072, 140656836763647, -+STORE, 140656836726784, 140656836763647, -+SNULL, 140656825700351, 140656825708543, -+STORE, 140656825683968, 140656825700351, -+STORE, 140656825700352, 140656825708543, -+SNULL, 140656798478335, 140656798482431, -+STORE, 140656798474240, 140656798478335, -+STORE, 140656798478336, 140656798482431, -+SNULL, 140656800657407, 140656800661503, -+STORE, 140656800653312, 140656800657407, -+STORE, 140656800657408, 140656800661503, -+SNULL, 140656802770943, 140656802775039, -+STORE, 140656802766848, 140656802770943, -+STORE, 140656802770944, 140656802775039, -+SNULL, 140656827920383, 140656827924479, -+STORE, 140656827916288, 140656827920383, -+STORE, 140656827920384, 140656827924479, -+SNULL, 140656805339135, 140656805343231, -+STORE, 140656805335040, 140656805339135, -+STORE, 140656805339136, 140656805343231, -+SNULL, 140656807723007, 140656807727103, -+STORE, 140656807628800, 140656807723007, -+STORE, 140656807723008, 140656807727103, -+SNULL, 140656810033151, 140656810037247, -+STORE, 140656810029056, 140656810033151, -+STORE, 140656810033152, 140656810037247, -+SNULL, 140656812167167, 140656812171263, -+STORE, 140656812163072, 140656812167167, -+STORE, 140656812167168, 140656812171263, -+SNULL, 140656815353855, 140656815382527, -+STORE, 140656815345664, 140656815353855, -+STORE, 140656815353856, 140656815382527, -+SNULL, 140656817549311, 140656817553407, -+STORE, 140656817545216, 140656817549311, -+STORE, 140656817549312, 140656817553407, -+SNULL, 140656819802111, 140656819806207, -+STORE, 140656819798016, 140656819802111, -+STORE, 140656819802112, 140656819806207, -+SNULL, 140656821923839, 140656821927935, -+STORE, 140656821919744, 140656821923839, -+STORE, 140656821923840, 140656821927935, -+SNULL, 140656830066687, 140656830070783, -+STORE, 140656830062592, 140656830066687, -+STORE, 140656830066688, 140656830070783, -+SNULL, 140656832319487, 140656832323583, -+STORE, 140656832315392, 140656832319487, -+STORE, 140656832319488, 140656832323583, -+SNULL, 140656834547711, 140656834551807, -+STORE, 140656833982464, 140656834547711, -+STORE, 140656834547712, 140656834551807, -+SNULL, 94730166759423, 94730166763519, -+STORE, 94730166751232, 94730166759423, -+STORE, 94730166759424, 94730166763519, -+SNULL, 140656836800511, 140656836804607, -+STORE, 140656836796416, 140656836800511, -+STORE, 140656836800512, 140656836804607, -+ERASE, 140656836763648, 140656836788223, -+STORE, 94730171318272, 94730171453439, -+STORE, 140656836784128, 140656836788223, -+STORE, 140656836780032, 140656836784127, -+STORE, 140656791920640, 140656796364799, -+STORE, 140656836775936, 140656836780031, -+STORE, 140656787476480, 140656791920639, -+STORE, 140656779083776, 140656787476479, -+SNULL, 140656779087871, 140656787476479, -+STORE, 140656779083776, 140656779087871, -+STORE, 140656779087872, 140656787476479, -+STORE, 140656836771840, 140656836775935, -+STORE, 140656774639616, 140656779083775, -+STORE, 140656766246912, 140656774639615, -+SNULL, 140656766251007, 140656774639615, -+STORE, 140656766246912, 140656766251007, -+STORE, 140656766251008, 140656774639615, -+ERASE, 140656791920640, 140656796364799, -+ERASE, 140656836780032, 140656836784127, -+ERASE, 140656787476480, 140656791920639, -+ERASE, 140656836775936, 140656836780031, -+STORE, 140656836780032, 140656836784127, -+STORE, 140656791920640, 140656796364799, -+STORE, 140656836775936, 140656836780031, -+STORE, 140656787476480, 140656791920639, -+ERASE, 140656774639616, 140656779083775, -+ }; -+ unsigned long set20[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140735952392192, 140737488351231, -+SNULL, 140735952396287, 140737488351231, -+STORE, 140735952392192, 140735952396287, -+STORE, 140735952261120, 140735952396287, -+STORE, 94849008947200, 94849009414143, -+SNULL, 94849009364991, 94849009414143, -+STORE, 94849008947200, 94849009364991, -+STORE, 94849009364992, 94849009414143, -+ERASE, 94849009364992, 94849009414143, -+STORE, 94849009364992, 94849009414143, -+STORE, 140590397943808, 140590400196607, -+SNULL, 140590398087167, 140590400196607, -+STORE, 140590397943808, 140590398087167, -+STORE, 140590398087168, 140590400196607, -+ERASE, 140590398087168, 140590400196607, -+STORE, 140590400184320, 140590400192511, -+STORE, 140590400192512, 140590400196607, -+STORE, 140735952850944, 140735952855039, -+STORE, 140735952838656, 140735952850943, -+STORE, 140590400180224, 140590400184319, -+STORE, 140590400172032, 140590400180223, -+STORE, 140590395809792, 140590397943807, -+SNULL, 140590395809792, 140590395838463, -+STORE, 140590395838464, 140590397943807, -+STORE, 140590395809792, 140590395838463, -+SNULL, 140590397935615, 140590397943807, -+STORE, 140590395838464, 140590397935615, -+STORE, 140590397935616, 140590397943807, -+ERASE, 140590397935616, 140590397943807, -+STORE, 140590397935616, 140590397943807, -+STORE, 140590393425920, 140590395809791, -+SNULL, 140590393425920, 140590393692159, -+STORE, 140590393692160, 140590395809791, -+STORE, 140590393425920, 140590393692159, -+SNULL, 140590395785215, 140590395809791, -+STORE, 140590393692160, 140590395785215, -+STORE, 140590395785216, 140590395809791, -+SNULL, 140590395785216, 140590395805695, -+STORE, 140590395805696, 140590395809791, -+STORE, 140590395785216, 140590395805695, -+ERASE, 140590395785216, 140590395805695, -+STORE, 140590395785216, 140590395805695, -+ERASE, 140590395805696, 140590395809791, -+STORE, 140590395805696, 140590395809791, -+STORE, 140590391234560, 140590393425919, -+SNULL, 140590391234560, 140590391324671, -+STORE, 140590391324672, 140590393425919, -+STORE, 140590391234560, 140590391324671, -+SNULL, 140590393417727, 140590393425919, -+STORE, 140590391324672, 140590393417727, -+STORE, 140590393417728, 140590393425919, -+ERASE, 140590393417728, 140590393425919, -+STORE, 140590393417728, 140590393425919, -+STORE, 140590388973568, 140590391234559, -+SNULL, 140590388973568, 140590389125119, -+STORE, 140590389125120, 140590391234559, -+STORE, 140590388973568, 140590389125119, -+SNULL, 140590391218175, 140590391234559, -+STORE, 140590389125120, 140590391218175, -+STORE, 140590391218176, 140590391234559, -+SNULL, 140590391218176, 140590391226367, -+STORE, 140590391226368, 140590391234559, -+STORE, 140590391218176, 140590391226367, -+ERASE, 140590391218176, 140590391226367, -+STORE, 140590391218176, 140590391226367, -+ERASE, 140590391226368, 140590391234559, -+STORE, 140590391226368, 140590391234559, -+STORE, 140590386843648, 140590388973567, -+SNULL, 140590386843648, 140590386872319, -+STORE, 140590386872320, 140590388973567, -+STORE, 140590386843648, 140590386872319, -+SNULL, 140590388965375, 140590388973567, -+STORE, 140590386872320, 140590388965375, -+STORE, 140590388965376, 140590388973567, -+ERASE, 140590388965376, 140590388973567, -+STORE, 140590388965376, 140590388973567, -+STORE, 140590384627712, 140590386843647, -+SNULL, 140590384627712, 140590384726015, -+STORE, 140590384726016, 140590386843647, -+STORE, 140590384627712, 140590384726015, -+SNULL, 140590386819071, 140590386843647, -+STORE, 140590384726016, 140590386819071, -+STORE, 140590386819072, 140590386843647, -+SNULL, 140590386819072, 140590386827263, -+STORE, 140590386827264, 140590386843647, -+STORE, 140590386819072, 140590386827263, -+ERASE, 140590386819072, 140590386827263, -+STORE, 140590386819072, 140590386827263, -+ERASE, 140590386827264, 140590386843647, -+STORE, 140590386827264, 140590386843647, -+STORE, 140590400163840, 140590400180223, -+STORE, 140590380830720, 140590384627711, -+SNULL, 140590380830720, 140590382489599, -+STORE, 140590382489600, 140590384627711, -+STORE, 140590380830720, 140590382489599, -+SNULL, 140590384586751, 140590384627711, -+STORE, 140590382489600, 140590384586751, -+STORE, 140590384586752, 140590384627711, -+SNULL, 140590384586752, 140590384611327, -+STORE, 140590384611328, 140590384627711, -+STORE, 140590384586752, 140590384611327, -+ERASE, 140590384586752, 140590384611327, -+STORE, 140590384586752, 140590384611327, -+ERASE, 140590384611328, 140590384627711, -+STORE, 140590384611328, 140590384627711, -+STORE, 140590378713088, 140590380830719, -+SNULL, 140590378713088, 140590378729471, -+STORE, 140590378729472, 140590380830719, -+STORE, 140590378713088, 140590378729471, -+SNULL, 140590380822527, 140590380830719, -+STORE, 140590378729472, 140590380822527, -+STORE, 140590380822528, 140590380830719, -+ERASE, 140590380822528, 140590380830719, -+STORE, 140590380822528, 140590380830719, -+STORE, 140590376595456, 140590378713087, -+SNULL, 140590376595456, 140590376611839, -+STORE, 140590376611840, 140590378713087, -+STORE, 140590376595456, 140590376611839, -+SNULL, 140590378704895, 140590378713087, -+STORE, 140590376611840, 140590378704895, -+STORE, 140590378704896, 140590378713087, -+ERASE, 140590378704896, 140590378713087, -+STORE, 140590378704896, 140590378713087, -+STORE, 140590374027264, 140590376595455, -+SNULL, 140590374027264, 140590374494207, -+STORE, 140590374494208, 140590376595455, -+STORE, 140590374027264, 140590374494207, -+SNULL, 140590376587263, 140590376595455, -+STORE, 140590374494208, 140590376587263, -+STORE, 140590376587264, 140590376595455, -+ERASE, 140590376587264, 140590376595455, -+STORE, 140590376587264, 140590376595455, -+STORE, 140590371913728, 140590374027263, -+SNULL, 140590371913728, 140590371926015, -+STORE, 140590371926016, 140590374027263, -+STORE, 140590371913728, 140590371926015, -+SNULL, 140590374019071, 140590374027263, -+STORE, 140590371926016, 140590374019071, -+STORE, 140590374019072, 140590374027263, -+ERASE, 140590374019072, 140590374027263, -+STORE, 140590374019072, 140590374027263, -+STORE, 140590400155648, 140590400180223, -+STORE, 140590400143360, 140590400180223, -+SNULL, 140590384603135, 140590384611327, -+STORE, 140590384586752, 140590384603135, -+STORE, 140590384603136, 140590384611327, -+SNULL, 140590374023167, 140590374027263, -+STORE, 140590374019072, 140590374023167, -+STORE, 140590374023168, 140590374027263, -+SNULL, 140590386823167, 140590386827263, -+STORE, 140590386819072, 140590386823167, -+STORE, 140590386823168, 140590386827263, -+SNULL, 140590376591359, 140590376595455, -+ }; -+ unsigned long set21[] = { -+STORE, 93874710941696, 93874711363583, -+STORE, 93874711367680, 93874711408639, -+STORE, 93874711408640, 93874711412735, -+STORE, 93874720989184, 93874721124351, -+STORE, 140708365086720, 140708365099007, -+STORE, 140708365099008, 140708367192063, -+STORE, 140708367192064, 140708367196159, -+STORE, 140708367196160, 140708367200255, -+STORE, 140708367200256, 140708367667199, -+STORE, 140708367667200, 140708369760255, -+STORE, 140708369760256, 140708369764351, -+STORE, 140708369764352, 140708369768447, -+STORE, 140708369768448, 140708369784831, -+STORE, 140708369784832, 140708371877887, -+STORE, 140708371877888, 140708371881983, -+STORE, 140708371881984, 140708371886079, -+STORE, 140708371886080, 140708371902463, -+STORE, 140708371902464, 140708373995519, -+STORE, 140708373995520, 140708373999615, -+STORE, 140708373999616, 140708374003711, -+STORE, 140708374003712, 140708375662591, -+STORE, 140708375662592, 140708377759743, -+STORE, 140708377759744, 140708377776127, -+STORE, 140708377776128, 140708377784319, -+STORE, 140708377784320, 140708377800703, -+STORE, 140708377800704, 140708377899007, -+STORE, 140708377899008, 140708379992063, -+STORE, 140708379992064, 140708379996159, -+STORE, 140708379996160, 140708380000255, -+STORE, 140708380000256, 140708380016639, -+STORE, 140708380016640, 140708380045311, -+STORE, 140708380045312, 140708382138367, -+STORE, 140708382138368, 140708382142463, -+STORE, 140708382142464, 140708382146559, -+STORE, 140708382146560, 140708382298111, -+STORE, 140708382298112, 140708384391167, -+STORE, 140708384391168, 140708384395263, -+STORE, 140708384395264, 140708384399359, -+STORE, 140708384399360, 140708384407551, -+STORE, 140708384407552, 140708384497663, -+STORE, 140708384497664, 140708386590719, -+STORE, 140708386590720, 140708386594815, -+STORE, 140708386594816, 140708386598911, -+STORE, 140708386598912, 140708386865151, -+STORE, 140708386865152, 140708388958207, -+STORE, 140708388958208, 140708388974591, -+STORE, 140708388974592, 140708388978687, -+STORE, 140708388978688, 140708388982783, -+STORE, 140708388982784, 140708389011455, -+STORE, 140708389011456, 140708391108607, -+STORE, 140708391108608, 140708391112703, -+STORE, 140708391112704, 140708391116799, -+STORE, 140708391116800, 140708391260159, -+STORE, 140708393291776, 140708393308159, -+STORE, 140708393308160, 140708393312255, -+STORE, 140708393312256, 140708393316351, -+STORE, 140708393316352, 140708393353215, -+STORE, 140708393353216, 140708393357311, -+STORE, 140708393357312, 140708393361407, -+STORE, 140708393361408, 140708393365503, -+STORE, 140708393365504, 140708393369599, -+STORE, 140730557042688, 140730557177855, -+STORE, 140730557235200, 140730557247487, -+STORE, 140730557247488, 140730557251583, -+ERASE, 140708393353216, 140708393357311, -+ERASE, 140708393312256, 140708393316351, -+ERASE, 140708393308160, 140708393312255, -+ERASE, 140708393291776, 140708393308159, -+ }; -+ unsigned long set22[] = { -+STORE, 93951397134336, 93951397183487, -+STORE, 93951397183488, 93951397728255, -+STORE, 93951397728256, 93951397826559, -+STORE, 93951397826560, 93951397842943, -+STORE, 93951397842944, 93951397847039, -+STORE, 93951425974272, 93951426109439, -+STORE, 140685152665600, 140685152677887, -+STORE, 140685152677888, 140685152829439, -+STORE, 140685152829440, 140685154181119, -+STORE, 140685154181120, 140685154484223, -+STORE, 140685154484224, 140685154496511, -+STORE, 140685154496512, 140685154508799, -+STORE, 140685154508800, 140685154525183, -+STORE, 140685154525184, 140685154541567, -+STORE, 140685154541568, 140685154590719, -+STORE, 140685154590720, 140685154603007, -+STORE, 140685154603008, 140685154607103, -+STORE, 140685154607104, 140685154611199, -+STORE, 140685154611200, 140685154615295, -+STORE, 140685154615296, 140685154631679, -+STORE, 140685154639872, 140685154643967, -+STORE, 140685154643968, 140685154766847, -+STORE, 140685154766848, 140685154799615, -+STORE, 140685154803712, 140685154807807, -+STORE, 140685154807808, 140685154811903, -+STORE, 140685154811904, 140685154815999, -+STORE, 140722188902400, 140722189037567, -+STORE, 140722189512704, 140722189524991, -+STORE, 140722189524992, 140722189529087, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733429354496, 140737488351231, -+SNULL, 140733429358591, 140737488351231, -+STORE, 140733429354496, 140733429358591, -+STORE, 140733429223424, 140733429358591, -+STORE, 94526683537408, 94526683660287, -+SNULL, 94526683553791, 94526683660287, -+STORE, 94526683537408, 94526683553791, -+STORE, 94526683553792, 94526683660287, -+ERASE, 94526683553792, 94526683660287, -+STORE, 94526683553792, 94526683623423, -+STORE, 94526683623424, 94526683647999, -+STORE, 94526683652096, 94526683660287, -+STORE, 140551363747840, 140551363923967, -+SNULL, 140551363751935, 140551363923967, -+STORE, 140551363747840, 140551363751935, -+STORE, 140551363751936, 140551363923967, -+ERASE, 140551363751936, 140551363923967, -+STORE, 140551363751936, 140551363874815, -+STORE, 140551363874816, 140551363907583, -+STORE, 140551363911680, 140551363919871, -+STORE, 140551363919872, 140551363923967, -+STORE, 140733429690368, 140733429694463, -+STORE, 140733429678080, 140733429690367, -+STORE, 140551363739648, 140551363747839, -+STORE, 140551363731456, 140551363739647, -+STORE, 140551363379200, 140551363731455, -+SNULL, 140551363379200, 140551363420159, -+STORE, 140551363420160, 140551363731455, -+STORE, 140551363379200, 140551363420159, -+SNULL, 140551363706879, 140551363731455, -+STORE, 140551363420160, 140551363706879, -+STORE, 140551363706880, 140551363731455, -+SNULL, 140551363420160, 140551363637247, -+STORE, 140551363637248, 140551363706879, -+STORE, 140551363420160, 140551363637247, -+ERASE, 140551363420160, 140551363637247, -+STORE, 140551363420160, 140551363637247, -+SNULL, 140551363637248, 140551363702783, -+STORE, 140551363702784, 140551363706879, -+STORE, 140551363637248, 140551363702783, -+ERASE, 140551363637248, 140551363702783, -+STORE, 140551363637248, 140551363702783, -+ERASE, 140551363706880, 140551363731455, -+STORE, 140551363706880, 140551363731455, -+STORE, 140551361531904, 140551363379199, -+SNULL, 140551361683455, 140551363379199, -+STORE, 140551361531904, 140551361683455, -+STORE, 140551361683456, 140551363379199, -+SNULL, 140551361683456, 140551363035135, -+STORE, 140551363035136, 140551363379199, -+STORE, 140551361683456, 140551363035135, -+ERASE, 140551361683456, 140551363035135, -+STORE, 140551361683456, 140551363035135, -+SNULL, 140551363035136, 140551363338239, -+STORE, 140551363338240, 140551363379199, -+STORE, 140551363035136, 140551363338239, -+ERASE, 140551363035136, 140551363338239, -+STORE, 140551363035136, 140551363379199, -+SNULL, 140551363338239, 140551363379199, -+STORE, 140551363035136, 140551363338239, -+STORE, 140551363338240, 140551363379199, -+SNULL, 140551363338240, 140551363362815, -+STORE, 140551363362816, 140551363379199, -+STORE, 140551363338240, 140551363362815, -+ERASE, 140551363338240, 140551363362815, -+STORE, 140551363338240, 140551363362815, -+ERASE, 140551363362816, 140551363379199, -+STORE, 140551363362816, 140551363379199, -+STORE, 140551361519616, 140551361531903, -+SNULL, 140551363350527, 140551363362815, -+STORE, 140551363338240, 140551363350527, -+STORE, 140551363350528, 140551363362815, -+SNULL, 140551363727359, 140551363731455, -+STORE, 140551363706880, 140551363727359, -+STORE, 140551363727360, 140551363731455, -+SNULL, 94526683656191, 94526683660287, -+STORE, 94526683652096, 94526683656191, -+STORE, 94526683656192, 94526683660287, -+SNULL, 140551363915775, 140551363919871, -+STORE, 140551363911680, 140551363915775, -+STORE, 140551363915776, 140551363919871, -+ERASE, 140551363739648, 140551363747839, -+STORE, 94526715490304, 94526715625471, -+STORE, 140551361253376, 140551361531903, -+STORE, 140551360987136, 140551361531903, -+STORE, 140551360720896, 140551361531903, -+STORE, 140551360454656, 140551361531903, -+SNULL, 140551361253375, 140551361531903, -+STORE, 140551360454656, 140551361253375, -+STORE, 140551361253376, 140551361531903, -+SNULL, 140551361253376, 140551361519615, -+STORE, 140551361519616, 140551361531903, -+STORE, 140551361253376, 140551361519615, -+ERASE, 140551361253376, 140551361519615, -+ }; -+ -+ unsigned long set23[] = { -+STORE, 94014447943680, 94014448156671, -+STORE, 94014450253824, 94014450257919, -+STORE, 94014450257920, 94014450266111, -+STORE, 94014450266112, 94014450278399, -+STORE, 94014464225280, 94014464630783, -+STORE, 139761764306944, 139761765965823, -+STORE, 139761765965824, 139761768062975, -+STORE, 139761768062976, 139761768079359, -+STORE, 139761768079360, 139761768087551, -+STORE, 139761768087552, 139761768103935, -+STORE, 139761768103936, 139761768116223, -+STORE, 139761768116224, 139761770209279, -+STORE, 139761770209280, 139761770213375, -+STORE, 139761770213376, 139761770217471, -+STORE, 139761770217472, 139761770360831, -+STORE, 139761770729472, 139761772412927, -+STORE, 139761772412928, 139761772429311, -+STORE, 139761772457984, 139761772462079, -+STORE, 139761772462080, 139761772466175, -+STORE, 139761772466176, 139761772470271, -+STORE, 140724336517120, 140724336652287, -+STORE, 140724336955392, 140724336967679, -+STORE, 140724336967680, 140724336971775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140721840295936, 140737488351231, -+SNULL, 140721840300031, 140737488351231, -+STORE, 140721840295936, 140721840300031, -+STORE, 140721840164864, 140721840300031, -+STORE, 93937913667584, 93937915830271, -+SNULL, 93937913729023, 93937915830271, -+STORE, 93937913667584, 93937913729023, -+STORE, 93937913729024, 93937915830271, -+ERASE, 93937913729024, 93937915830271, -+STORE, 93937915822080, 93937915830271, -+STORE, 140598835335168, 140598837587967, -+SNULL, 140598835478527, 140598837587967, -+STORE, 140598835335168, 140598835478527, -+STORE, 140598835478528, 140598837587967, -+ERASE, 140598835478528, 140598837587967, -+STORE, 140598837575680, 140598837583871, -+STORE, 140598837583872, 140598837587967, -+STORE, 140721841086464, 140721841090559, -+STORE, 140721841074176, 140721841086463, -+STORE, 140598837547008, 140598837575679, -+STORE, 140598837538816, 140598837547007, -+STORE, 140598831538176, 140598835335167, -+SNULL, 140598831538176, 140598833197055, -+STORE, 140598833197056, 140598835335167, -+STORE, 140598831538176, 140598833197055, -+SNULL, 140598835294207, 140598835335167, -+STORE, 140598833197056, 140598835294207, -+STORE, 140598835294208, 140598835335167, -+SNULL, 140598835294208, 140598835318783, -+STORE, 140598835318784, 140598835335167, -+STORE, 140598835294208, 140598835318783, -+ERASE, 140598835294208, 140598835318783, -+STORE, 140598835294208, 140598835318783, -+ERASE, 140598835318784, 140598835335167, -+STORE, 140598835318784, 140598835335167, -+SNULL, 140598835310591, 140598835318783, -+STORE, 140598835294208, 140598835310591, -+STORE, 140598835310592, 140598835318783, -+SNULL, 93937915826175, 93937915830271, -+STORE, 93937915822080, 93937915826175, -+STORE, 93937915826176, 93937915830271, -+SNULL, 140598837579775, 140598837583871, -+STORE, 140598837575680, 140598837579775, -+STORE, 140598837579776, 140598837583871, -+ERASE, 140598837547008, 140598837575679, -+STORE, 93937929179136, 93937929314303, -+STORE, 140598835855360, 140598837538815, -+STORE, 140737488347136, 140737488351231, -+STORE, 140728187723776, 140737488351231, -+SNULL, 140728187727871, 140737488351231, -+STORE, 140728187723776, 140728187727871, -+STORE, 140728187592704, 140728187727871, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 140583951437824, 140583953690623, -+SNULL, 140583951581183, 140583953690623, -+STORE, 140583951437824, 140583951581183, -+STORE, 140583951581184, 140583953690623, -+ERASE, 140583951581184, 140583953690623, -+STORE, 140583953678336, 140583953686527, -+STORE, 140583953686528, 140583953690623, -+STORE, 140728189116416, 140728189120511, -+STORE, 140728189104128, 140728189116415, -+STORE, 140583953649664, 140583953678335, -+STORE, 140583953641472, 140583953649663, -+STORE, 140583948275712, 140583951437823, -+SNULL, 140583948275712, 140583949336575, -+STORE, 140583949336576, 140583951437823, -+STORE, 140583948275712, 140583949336575, -+SNULL, 140583951429631, 140583951437823, -+STORE, 140583949336576, 140583951429631, -+STORE, 140583951429632, 140583951437823, -+ERASE, 140583951429632, 140583951437823, -+STORE, 140583951429632, 140583951437823, -+STORE, 140583944478720, 140583948275711, -+SNULL, 140583944478720, 140583946137599, -+STORE, 140583946137600, 140583948275711, -+STORE, 140583944478720, 140583946137599, -+SNULL, 140583948234751, 140583948275711, -+STORE, 140583946137600, 140583948234751, -+STORE, 140583948234752, 140583948275711, -+SNULL, 140583948234752, 140583948259327, -+STORE, 140583948259328, 140583948275711, -+STORE, 140583948234752, 140583948259327, -+ERASE, 140583948234752, 140583948259327, -+STORE, 140583948234752, 140583948259327, -+ERASE, 140583948259328, 140583948275711, -+STORE, 140583948259328, 140583948275711, -+STORE, 140583953629184, 140583953649663, -+SNULL, 140583948251135, 140583948259327, -+STORE, 140583948234752, 140583948251135, -+STORE, 140583948251136, 140583948259327, -+SNULL, 140583951433727, 140583951437823, -+STORE, 140583951429632, 140583951433727, -+STORE, 140583951433728, 140583951437823, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 140583953682431, 140583953686527, -+STORE, 140583953678336, 140583953682431, -+STORE, 140583953682432, 140583953686527, -+ERASE, 140583953649664, 140583953678335, -+STORE, 17821696, 17956863, -+STORE, 17821696, 18104319, -+STORE, 140583951945728, 140583953629183, -+STORE, 94014447943680, 94014448156671, -+STORE, 94014450253824, 94014450257919, -+STORE, 94014450257920, 94014450266111, -+STORE, 94014450266112, 94014450278399, -+STORE, 94014464225280, 94014465196031, -+STORE, 139761764306944, 139761765965823, -+STORE, 139761765965824, 139761768062975, -+STORE, 139761768062976, 139761768079359, -+STORE, 139761768079360, 139761768087551, -+STORE, 139761768087552, 139761768103935, -+STORE, 139761768103936, 139761768116223, -+STORE, 139761768116224, 139761770209279, -+STORE, 139761770209280, 139761770213375, -+STORE, 139761770213376, 139761770217471, -+STORE, 139761770217472, 139761770360831, -+STORE, 139761770729472, 139761772412927, -+STORE, 139761772412928, 139761772429311, -+STORE, 139761772457984, 139761772462079, -+STORE, 139761772462080, 139761772466175, -+STORE, 139761772466176, 139761772470271, -+STORE, 140724336517120, 140724336652287, -+STORE, 140724336955392, 140724336967679, -+STORE, 140724336967680, 140724336971775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140726063296512, 140737488351231, -+SNULL, 140726063300607, 140737488351231, -+STORE, 140726063296512, 140726063300607, -+STORE, 140726063165440, 140726063300607, -+STORE, 94016795934720, 94016798158847, -+SNULL, 94016796045311, 94016798158847, -+STORE, 94016795934720, 94016796045311, -+STORE, 94016796045312, 94016798158847, -+ERASE, 94016796045312, 94016798158847, -+STORE, 94016798138368, 94016798150655, -+STORE, 94016798150656, 94016798158847, -+STORE, 139975915966464, 139975918219263, -+SNULL, 139975916109823, 139975918219263, -+STORE, 139975915966464, 139975916109823, -+STORE, 139975916109824, 139975918219263, -+ERASE, 139975916109824, 139975918219263, -+STORE, 139975918206976, 139975918215167, -+STORE, 139975918215168, 139975918219263, -+STORE, 140726064541696, 140726064545791, -+STORE, 140726064529408, 140726064541695, -+STORE, 139975918178304, 139975918206975, -+STORE, 139975918170112, 139975918178303, -+STORE, 139975912169472, 139975915966463, -+SNULL, 139975912169472, 139975913828351, -+STORE, 139975913828352, 139975915966463, -+STORE, 139975912169472, 139975913828351, -+SNULL, 139975915925503, 139975915966463, -+STORE, 139975913828352, 139975915925503, -+STORE, 139975915925504, 139975915966463, -+SNULL, 139975915925504, 139975915950079, -+STORE, 139975915950080, 139975915966463, -+STORE, 139975915925504, 139975915950079, -+ERASE, 139975915925504, 139975915950079, -+STORE, 139975915925504, 139975915950079, -+ERASE, 139975915950080, 139975915966463, -+STORE, 139975915950080, 139975915966463, -+SNULL, 139975915941887, 139975915950079, -+STORE, 139975915925504, 139975915941887, -+STORE, 139975915941888, 139975915950079, -+SNULL, 94016798146559, 94016798150655, -+STORE, 94016798138368, 94016798146559, -+STORE, 94016798146560, 94016798150655, -+SNULL, 139975918211071, 139975918215167, -+STORE, 139975918206976, 139975918211071, -+STORE, 139975918211072, 139975918215167, -+ERASE, 139975918178304, 139975918206975, -+STORE, 94016804925440, 94016805060607, -+STORE, 94596177661952, 94596177772543, -+STORE, 94596179865600, 94596179873791, -+STORE, 94596179873792, 94596179877887, -+STORE, 94596179877888, 94596179886079, -+STORE, 94596211597312, 94596211863551, -+STORE, 140127351840768, 140127353499647, -+STORE, 140127353499648, 140127355596799, -+STORE, 140127355596800, 140127355613183, -+STORE, 140127355613184, 140127355621375, -+STORE, 140127355621376, 140127355637759, -+STORE, 140127355637760, 140127355781119, -+STORE, 140127357841408, 140127357849599, -+STORE, 140127357878272, 140127357882367, -+STORE, 140127357882368, 140127357886463, -+STORE, 140127357886464, 140127357890559, -+STORE, 140726167252992, 140726167392255, -+STORE, 140726167838720, 140726167851007, -+STORE, 140726167851008, 140726167855103, -+STORE, 140737488347136, 140737488351231, -+STORE, 140731874017280, 140737488351231, -+SNULL, 140731874021375, 140737488351231, -+STORE, 140731874017280, 140731874021375, -+STORE, 140731873886208, 140731874021375, -+STORE, 94178682265600, 94178684489727, -+SNULL, 94178682376191, 94178684489727, -+STORE, 94178682265600, 94178682376191, -+STORE, 94178682376192, 94178684489727, -+ERASE, 94178682376192, 94178684489727, -+STORE, 94178684469248, 94178684481535, -+STORE, 94178684481536, 94178684489727, -+STORE, 140460853403648, 140460855656447, -+SNULL, 140460853547007, 140460855656447, -+STORE, 140460853403648, 140460853547007, -+STORE, 140460853547008, 140460855656447, -+ERASE, 140460853547008, 140460855656447, -+STORE, 140460855644160, 140460855652351, -+STORE, 140460855652352, 140460855656447, -+STORE, 140731874103296, 140731874107391, -+STORE, 140731874091008, 140731874103295, -+STORE, 140460855615488, 140460855644159, -+STORE, 140460855607296, 140460855615487, -+STORE, 140460849606656, 140460853403647, -+SNULL, 140460849606656, 140460851265535, -+STORE, 140460851265536, 140460853403647, -+STORE, 140460849606656, 140460851265535, -+SNULL, 140460853362687, 140460853403647, -+STORE, 140460851265536, 140460853362687, -+STORE, 140460853362688, 140460853403647, -+SNULL, 140460853362688, 140460853387263, -+STORE, 140460853387264, 140460853403647, -+STORE, 140460853362688, 140460853387263, -+ERASE, 140460853362688, 140460853387263, -+STORE, 140460853362688, 140460853387263, -+ERASE, 140460853387264, 140460853403647, -+STORE, 140460853387264, 140460853403647, -+SNULL, 140460853379071, 140460853387263, -+STORE, 140460853362688, 140460853379071, -+STORE, 140460853379072, 140460853387263, -+SNULL, 94178684477439, 94178684481535, -+STORE, 94178684469248, 94178684477439, -+STORE, 94178684477440, 94178684481535, -+SNULL, 140460855648255, 140460855652351, -+STORE, 140460855644160, 140460855648255, -+STORE, 140460855648256, 140460855652351, -+ERASE, 140460855615488, 140460855644159, -+STORE, 94178692063232, 94178692198399, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733096603648, 140737488351231, -+SNULL, 140733096611839, 140737488351231, -+STORE, 140733096603648, 140733096611839, -+STORE, 140733096472576, 140733096611839, -+STORE, 94796716122112, 94796718325759, -+SNULL, 94796716224511, 94796718325759, -+STORE, 94796716122112, 94796716224511, -+STORE, 94796716224512, 94796718325759, -+ERASE, 94796716224512, 94796718325759, -+STORE, 94796718317568, 94796718325759, -+STORE, 139667892793344, 139667895046143, -+SNULL, 139667892936703, 139667895046143, -+STORE, 139667892793344, 139667892936703, -+STORE, 139667892936704, 139667895046143, -+ERASE, 139667892936704, 139667895046143, -+STORE, 139667895033856, 139667895042047, -+STORE, 139667895042048, 139667895046143, -+STORE, 140733096857600, 140733096861695, -+STORE, 140733096845312, 140733096857599, -+STORE, 139667895005184, 139667895033855, -+STORE, 139667894996992, 139667895005183, -+STORE, 139667890532352, 139667892793343, -+SNULL, 139667890532352, 139667890683903, -+STORE, 139667890683904, 139667892793343, -+STORE, 139667890532352, 139667890683903, -+SNULL, 139667892776959, 139667892793343, -+STORE, 139667890683904, 139667892776959, -+STORE, 139667892776960, 139667892793343, -+SNULL, 139667892776960, 139667892785151, -+STORE, 139667892785152, 139667892793343, -+STORE, 139667892776960, 139667892785151, -+ERASE, 139667892776960, 139667892785151, -+STORE, 139667892776960, 139667892785151, -+ERASE, 139667892785152, 139667892793343, -+STORE, 139667892785152, 139667892793343, -+STORE, 139667886735360, 139667890532351, -+SNULL, 139667886735360, 139667888394239, -+STORE, 139667888394240, 139667890532351, -+STORE, 139667886735360, 139667888394239, -+SNULL, 139667890491391, 139667890532351, -+STORE, 139667888394240, 139667890491391, -+STORE, 139667890491392, 139667890532351, -+SNULL, 139667890491392, 139667890515967, -+STORE, 139667890515968, 139667890532351, -+STORE, 139667890491392, 139667890515967, -+ERASE, 139667890491392, 139667890515967, -+STORE, 139667890491392, 139667890515967, -+ERASE, 139667890515968, 139667890532351, -+STORE, 139667890515968, 139667890532351, -+STORE, 139667884167168, 139667886735359, -+SNULL, 139667884167168, 139667884634111, -+STORE, 139667884634112, 139667886735359, -+STORE, 139667884167168, 139667884634111, -+SNULL, 139667886727167, 139667886735359, -+STORE, 139667884634112, 139667886727167, -+STORE, 139667886727168, 139667886735359, -+ERASE, 139667886727168, 139667886735359, -+STORE, 139667886727168, 139667886735359, -+STORE, 139667882053632, 139667884167167, -+SNULL, 139667882053632, 139667882065919, -+STORE, 139667882065920, 139667884167167, -+STORE, 139667882053632, 139667882065919, -+SNULL, 139667884158975, 139667884167167, -+STORE, 139667882065920, 139667884158975, -+STORE, 139667884158976, 139667884167167, -+ERASE, 139667884158976, 139667884167167, -+STORE, 139667884158976, 139667884167167, -+STORE, 139667879837696, 139667882053631, -+SNULL, 139667879837696, 139667879935999, -+STORE, 139667879936000, 139667882053631, -+STORE, 139667879837696, 139667879935999, -+SNULL, 139667882029055, 139667882053631, -+STORE, 139667879936000, 139667882029055, -+STORE, 139667882029056, 139667882053631, -+SNULL, 139667882029056, 139667882037247, -+STORE, 139667882037248, 139667882053631, -+STORE, 139667882029056, 139667882037247, -+ERASE, 139667882029056, 139667882037247, -+STORE, 139667882029056, 139667882037247, -+ERASE, 139667882037248, 139667882053631, -+STORE, 139667882037248, 139667882053631, -+STORE, 139667894988800, 139667895005183, -+SNULL, 139667890507775, 139667890515967, -+STORE, 139667890491392, 139667890507775, -+STORE, 139667890507776, 139667890515967, -+SNULL, 139667882033151, 139667882037247, -+STORE, 139667882029056, 139667882033151, -+STORE, 139667882033152, 139667882037247, -+SNULL, 139667884163071, 139667884167167, -+STORE, 139667884158976, 139667884163071, -+STORE, 139667884163072, 139667884167167, -+SNULL, 139667886731263, 139667886735359, -+STORE, 139667886727168, 139667886731263, -+STORE, 139667886731264, 139667886735359, -+SNULL, 139667892781055, 139667892785151, -+STORE, 139667892776960, 139667892781055, -+STORE, 139667892781056, 139667892785151, -+SNULL, 94796718321663, 94796718325759, -+STORE, 94796718317568, 94796718321663, -+STORE, 94796718321664, 94796718325759, -+SNULL, 139667895037951, 139667895042047, -+STORE, 139667895033856, 139667895037951, -+STORE, 139667895037952, 139667895042047, -+ERASE, 139667895005184, 139667895033855, -+STORE, 94796726063104, 94796726198271, -+STORE, 139667893305344, 139667894988799, -+STORE, 139667895005184, 139667895033855, -+STORE, 94796726063104, 94796726333439, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722489507840, 140737488351231, -+SNULL, 140722489516031, 140737488351231, -+STORE, 140722489507840, 140722489516031, -+STORE, 140722489376768, 140722489516031, -+STORE, 93980993265664, 93980995489791, -+SNULL, 93980993376255, 93980995489791, -+STORE, 93980993265664, 93980993376255, -+STORE, 93980993376256, 93980995489791, -+ERASE, 93980993376256, 93980995489791, -+STORE, 93980995469312, 93980995481599, -+STORE, 93980995481600, 93980995489791, -+STORE, 140261313593344, 140261315846143, -+SNULL, 140261313736703, 140261315846143, -+STORE, 140261313593344, 140261313736703, -+STORE, 140261313736704, 140261315846143, -+ERASE, 140261313736704, 140261315846143, -+STORE, 140261315833856, 140261315842047, -+STORE, 140261315842048, 140261315846143, -+STORE, 140722489675776, 140722489679871, -+STORE, 140722489663488, 140722489675775, -+STORE, 140261315805184, 140261315833855, -+STORE, 140261315796992, 140261315805183, -+STORE, 140261309796352, 140261313593343, -+SNULL, 140261309796352, 140261311455231, -+STORE, 140261311455232, 140261313593343, -+STORE, 140261309796352, 140261311455231, -+SNULL, 140261313552383, 140261313593343, -+STORE, 140261311455232, 140261313552383, -+STORE, 140261313552384, 140261313593343, -+SNULL, 140261313552384, 140261313576959, -+STORE, 140261313576960, 140261313593343, -+STORE, 140261313552384, 140261313576959, -+ERASE, 140261313552384, 140261313576959, -+STORE, 140261313552384, 140261313576959, -+ERASE, 140261313576960, 140261313593343, -+STORE, 140261313576960, 140261313593343, -+SNULL, 140261313568767, 140261313576959, -+STORE, 140261313552384, 140261313568767, -+STORE, 140261313568768, 140261313576959, -+SNULL, 93980995477503, 93980995481599, -+STORE, 93980995469312, 93980995477503, -+STORE, 93980995477504, 93980995481599, -+SNULL, 140261315837951, 140261315842047, -+STORE, 140261315833856, 140261315837951, -+STORE, 140261315837952, 140261315842047, -+ERASE, 140261315805184, 140261315833855, -+STORE, 93980997443584, 93980997578751, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140737488338944, 140737488351231, -+STORE, 140734059450368, 140737488351231, -+SNULL, 140734059462655, 140737488351231, -+STORE, 140734059450368, 140734059462655, -+STORE, 140734059319296, 140734059462655, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 140307554983936, 140307557236735, -+SNULL, 140307555127295, 140307557236735, -+STORE, 140307554983936, 140307555127295, -+STORE, 140307555127296, 140307557236735, -+ERASE, 140307555127296, 140307557236735, -+STORE, 140307557224448, 140307557232639, -+STORE, 140307557232640, 140307557236735, -+STORE, 140734059483136, 140734059487231, -+STORE, 140734059470848, 140734059483135, -+STORE, 140307557195776, 140307557224447, -+STORE, 140307557187584, 140307557195775, -+STORE, 140307551821824, 140307554983935, -+SNULL, 140307551821824, 140307552882687, -+STORE, 140307552882688, 140307554983935, -+STORE, 140307551821824, 140307552882687, -+SNULL, 140307554975743, 140307554983935, -+STORE, 140307552882688, 140307554975743, -+STORE, 140307554975744, 140307554983935, -+ERASE, 140307554975744, 140307554983935, -+STORE, 140307554975744, 140307554983935, -+STORE, 140307548024832, 140307551821823, -+SNULL, 140307548024832, 140307549683711, -+STORE, 140307549683712, 140307551821823, -+STORE, 140307548024832, 140307549683711, -+SNULL, 140307551780863, 140307551821823, -+STORE, 140307549683712, 140307551780863, -+STORE, 140307551780864, 140307551821823, -+SNULL, 140307551780864, 140307551805439, -+STORE, 140307551805440, 140307551821823, -+STORE, 140307551780864, 140307551805439, -+ERASE, 140307551780864, 140307551805439, -+STORE, 140307551780864, 140307551805439, -+ERASE, 140307551805440, 140307551821823, -+STORE, 140307551805440, 140307551821823, -+STORE, 140307557175296, 140307557195775, -+SNULL, 140307551797247, 140307551805439, -+STORE, 140307551780864, 140307551797247, -+STORE, 140307551797248, 140307551805439, -+SNULL, 140307554979839, 140307554983935, -+STORE, 140307554975744, 140307554979839, -+STORE, 140307554979840, 140307554983935, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 140307557228543, 140307557232639, -+STORE, 140307557224448, 140307557228543, -+STORE, 140307557228544, 140307557232639, -+ERASE, 140307557195776, 140307557224447, -+STORE, 39698432, 39833599, -+STORE, 39698432, 39981055, -+STORE, 94306485321728, 94306485432319, -+STORE, 94306487525376, 94306487533567, -+STORE, 94306487533568, 94306487537663, -+STORE, 94306487537664, 94306487545855, -+STORE, 94306488868864, 94306489004031, -+STORE, 140497673998336, 140497675657215, -+STORE, 140497675657216, 140497677754367, -+STORE, 140497677754368, 140497677770751, -+STORE, 140497677770752, 140497677778943, -+STORE, 140497677778944, 140497677795327, -+STORE, 140497677795328, 140497677938687, -+STORE, 140497679998976, 140497680007167, -+STORE, 140497680035840, 140497680039935, -+STORE, 140497680039936, 140497680044031, -+STORE, 140497680044032, 140497680048127, -+STORE, 140732780462080, 140732780601343, -+STORE, 140732782239744, 140732782252031, -+STORE, 140732782252032, 140732782256127, -+STORE, 94236915900416, 94236916011007, -+STORE, 94236918104064, 94236918112255, -+STORE, 94236918112256, 94236918116351, -+STORE, 94236918116352, 94236918124543, -+STORE, 94236939489280, 94236939624447, -+STORE, 140046091743232, 140046093402111, -+STORE, 140046093402112, 140046095499263, -+STORE, 140046095499264, 140046095515647, -+STORE, 140046095515648, 140046095523839, -+STORE, 140046095523840, 140046095540223, -+STORE, 140046095540224, 140046095683583, -+STORE, 140046097743872, 140046097752063, -+STORE, 140046097780736, 140046097784831, -+STORE, 140046097784832, 140046097788927, -+STORE, 140046097788928, 140046097793023, -+STORE, 140726694449152, 140726694588415, -+STORE, 140726695313408, 140726695325695, -+STORE, 140726695325696, 140726695329791, -+STORE, 94894582779904, 94894582992895, -+STORE, 94894585090048, 94894585094143, -+STORE, 94894585094144, 94894585102335, -+STORE, 94894585102336, 94894585114623, -+STORE, 94894592868352, 94894594293759, -+STORE, 139733563842560, 139733565501439, -+STORE, 139733565501440, 139733567598591, -+STORE, 139733567598592, 139733567614975, -+STORE, 139733567614976, 139733567623167, -+STORE, 139733567623168, 139733567639551, -+STORE, 139733567639552, 139733567651839, -+STORE, 139733567651840, 139733569744895, -+STORE, 139733569744896, 139733569748991, -+STORE, 139733569748992, 139733569753087, -+STORE, 139733569753088, 139733569896447, -+STORE, 139733570265088, 139733571948543, -+STORE, 139733571948544, 139733571964927, -+STORE, 139733571993600, 139733571997695, -+STORE, 139733571997696, 139733572001791, -+STORE, 139733572001792, 139733572005887, -+STORE, 140726369255424, 140726369394687, -+STORE, 140726370402304, 140726370414591, -+STORE, 140726370414592, 140726370418687, -+STORE, 94899236483072, 94899236696063, -+STORE, 94899238793216, 94899238797311, -+STORE, 94899238797312, 94899238805503, -+STORE, 94899238805504, 94899238817791, -+STORE, 94899263045632, 94899263979519, -+STORE, 140040959893504, 140040961552383, -+STORE, 140040961552384, 140040963649535, -+STORE, 140040963649536, 140040963665919, -+STORE, 140040963665920, 140040963674111, -+STORE, 140040963674112, 140040963690495, -+STORE, 140040963690496, 140040963702783, -+STORE, 140040963702784, 140040965795839, -+STORE, 140040965795840, 140040965799935, -+STORE, 140040965799936, 140040965804031, -+STORE, 140040965804032, 140040965947391, -+STORE, 140040966316032, 140040967999487, -+STORE, 140040967999488, 140040968015871, -+STORE, 140040968044544, 140040968048639, -+STORE, 140040968048640, 140040968052735, -+STORE, 140040968052736, 140040968056831, -+STORE, 140729921359872, 140729921499135, -+STORE, 140729921613824, 140729921626111, -+STORE, 140729921626112, 140729921630207, -+STORE, 94818265190400, 94818265403391, -+STORE, 94818267500544, 94818267504639, -+STORE, 94818267504640, 94818267512831, -+STORE, 94818267512832, 94818267525119, -+STORE, 94818283372544, 94818285858815, -+STORE, 139818425675776, 139818427334655, -+STORE, 139818427334656, 139818429431807, -+STORE, 139818429431808, 139818429448191, -+STORE, 139818429448192, 139818429456383, -+STORE, 139818429456384, 139818429472767, -+STORE, 139818429472768, 139818429485055, -+STORE, 139818429485056, 139818431578111, -+STORE, 139818431578112, 139818431582207, -+STORE, 139818431582208, 139818431586303, -+STORE, 139818431586304, 139818431729663, -+STORE, 139818432098304, 139818433781759, -+STORE, 139818433781760, 139818433798143, -+STORE, 139818433826816, 139818433830911, -+STORE, 139818433830912, 139818433835007, -+STORE, 139818433835008, 139818433839103, -+STORE, 140726170509312, 140726170648575, -+STORE, 140726171824128, 140726171836415, -+STORE, 140726171836416, 140726171840511, -+STORE, 94611513188352, 94611513401343, -+STORE, 94611515498496, 94611515502591, -+STORE, 94611515502592, 94611515510783, -+STORE, 94611515510784, 94611515523071, -+STORE, 94611516502016, 94611516907519, -+STORE, 140596246388736, 140596248047615, -+STORE, 140596248047616, 140596250144767, -+STORE, 140596250144768, 140596250161151, -+STORE, 140596250161152, 140596250169343, -+STORE, 140596250169344, 140596250185727, -+STORE, 140596250185728, 140596250198015, -+STORE, 140596250198016, 140596252291071, -+STORE, 140596252291072, 140596252295167, -+STORE, 140596252295168, 140596252299263, -+STORE, 140596252299264, 140596252442623, -+STORE, 140596252811264, 140596254494719, -+STORE, 140596254494720, 140596254511103, -+STORE, 140596254539776, 140596254543871, -+STORE, 140596254543872, 140596254547967, -+STORE, 140596254547968, 140596254552063, -+STORE, 140731551338496, 140731551477759, -+STORE, 140731551780864, 140731551793151, -+STORE, 140731551793152, 140731551797247, -+STORE, 94313835851776, 94313836064767, -+STORE, 94313838161920, 94313838166015, -+STORE, 94313838166016, 94313838174207, -+STORE, 94313838174208, 94313838186495, -+STORE, 94313858416640, 94313861906431, -+STORE, 140693503918080, 140693505576959, -+STORE, 140693505576960, 140693507674111, -+STORE, 140693507674112, 140693507690495, -+STORE, 140693507690496, 140693507698687, -+STORE, 140693507698688, 140693507715071, -+STORE, 140693507715072, 140693507727359, -+STORE, 140693507727360, 140693509820415, -+STORE, 140693509820416, 140693509824511, -+STORE, 140693509824512, 140693509828607, -+STORE, 140693509828608, 140693509971967, -+STORE, 140693510340608, 140693512024063, -+STORE, 140693512024064, 140693512040447, -+STORE, 140693512069120, 140693512073215, -+STORE, 140693512073216, 140693512077311, -+STORE, 140693512077312, 140693512081407, -+STORE, 140721116065792, 140721116205055, -+STORE, 140721117831168, 140721117843455, -+STORE, 140721117843456, 140721117847551, -+STORE, 94843650150400, 94843650363391, -+STORE, 94843652460544, 94843652464639, -+STORE, 94843652464640, 94843652472831, -+STORE, 94843652472832, 94843652485119, -+STORE, 94843685388288, 94843686281215, -+STORE, 140484193681408, 140484195340287, -+STORE, 140484195340288, 140484197437439, -+STORE, 140484197437440, 140484197453823, -+STORE, 140484197453824, 140484197462015, -+STORE, 140484197462016, 140484197478399, -+STORE, 140484197478400, 140484197490687, -+STORE, 140484197490688, 140484199583743, -+STORE, 140484199583744, 140484199587839, -+STORE, 140484199587840, 140484199591935, -+STORE, 140484199591936, 140484199735295, -+STORE, 140484200103936, 140484201787391, -+STORE, 140484201787392, 140484201803775, -+STORE, 140484201832448, 140484201836543, -+STORE, 140484201836544, 140484201840639, -+STORE, 140484201840640, 140484201844735, -+STORE, 140726294315008, 140726294454271, -+STORE, 140726295646208, 140726295658495, -+STORE, 140726295658496, 140726295662591, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140720422371328, 140737488351231, -+SNULL, 140720422379519, 140737488351231, -+STORE, 140720422371328, 140720422379519, -+STORE, 140720422240256, 140720422379519, -+STORE, 94417967845376, 94417970180095, -+SNULL, 94417968058367, 94417970180095, -+STORE, 94417967845376, 94417968058367, -+STORE, 94417968058368, 94417970180095, -+ERASE, 94417968058368, 94417970180095, -+STORE, 94417970155520, 94417970167807, -+STORE, 94417970167808, 94417970180095, -+STORE, 140252450045952, 140252452298751, -+SNULL, 140252450189311, 140252452298751, -+STORE, 140252450045952, 140252450189311, -+STORE, 140252450189312, 140252452298751, -+ERASE, 140252450189312, 140252452298751, -+STORE, 140252452286464, 140252452294655, -+STORE, 140252452294656, 140252452298751, -+STORE, 140720422416384, 140720422420479, -+STORE, 140720422404096, 140720422416383, -+STORE, 140252452257792, 140252452286463, -+STORE, 140252452249600, 140252452257791, -+STORE, 140252447932416, 140252450045951, -+SNULL, 140252447932416, 140252447944703, -+STORE, 140252447944704, 140252450045951, -+STORE, 140252447932416, 140252447944703, -+SNULL, 140252450037759, 140252450045951, -+STORE, 140252447944704, 140252450037759, -+STORE, 140252450037760, 140252450045951, -+ERASE, 140252450037760, 140252450045951, -+STORE, 140252450037760, 140252450045951, -+STORE, 140252444135424, 140252447932415, -+SNULL, 140252444135424, 140252445794303, -+STORE, 140252445794304, 140252447932415, -+STORE, 140252444135424, 140252445794303, -+SNULL, 140252447891455, 140252447932415, -+STORE, 140252445794304, 140252447891455, -+STORE, 140252447891456, 140252447932415, -+SNULL, 140252447891456, 140252447916031, -+STORE, 140252447916032, 140252447932415, -+STORE, 140252447891456, 140252447916031, -+ERASE, 140252447891456, 140252447916031, -+STORE, 140252447891456, 140252447916031, -+ERASE, 140252447916032, 140252447932415, -+STORE, 140252447916032, 140252447932415, -+STORE, 140252452241408, 140252452257791, -+SNULL, 140252447907839, 140252447916031, -+STORE, 140252447891456, 140252447907839, -+STORE, 140252447907840, 140252447916031, -+SNULL, 140252450041855, 140252450045951, -+STORE, 140252450037760, 140252450041855, -+STORE, 140252450041856, 140252450045951, -+SNULL, 94417970159615, 94417970167807, -+STORE, 94417970155520, 94417970159615, -+STORE, 94417970159616, 94417970167807, -+SNULL, 140252452290559, 140252452294655, -+STORE, 140252452286464, 140252452290559, -+STORE, 140252452290560, 140252452294655, -+ERASE, 140252452257792, 140252452286463, -+STORE, 94417996333056, 94417996468223, -+STORE, 140252450557952, 140252452241407, -+STORE, 94417996333056, 94417996603391, -+STORE, 94417996333056, 94417996738559, -+STORE, 94417996333056, 94417996910591, -+SNULL, 94417996881919, 94417996910591, -+STORE, 94417996333056, 94417996881919, -+STORE, 94417996881920, 94417996910591, -+ERASE, 94417996881920, 94417996910591, -+STORE, 94417996333056, 94417997017087, -+STORE, 94417996333056, 94417997152255, -+SNULL, 94417997135871, 94417997152255, -+STORE, 94417996333056, 94417997135871, -+STORE, 94417997135872, 94417997152255, -+ERASE, 94417997135872, 94417997152255, -+STORE, 94417996333056, 94417997291519, -+SNULL, 94417997271039, 94417997291519, -+STORE, 94417996333056, 94417997271039, -+STORE, 94417997271040, 94417997291519, -+ERASE, 94417997271040, 94417997291519, -+STORE, 94417996333056, 94417997406207, -+SNULL, 94417997381631, 94417997406207, -+STORE, 94417996333056, 94417997381631, -+STORE, 94417997381632, 94417997406207, -+ERASE, 94417997381632, 94417997406207, -+STORE, 94417996333056, 94417997516799, -+SNULL, 94417997488127, 94417997516799, -+STORE, 94417996333056, 94417997488127, -+STORE, 94417997488128, 94417997516799, -+ERASE, 94417997488128, 94417997516799, -+STORE, 94417996333056, 94417997643775, -+SNULL, 94417997631487, 94417997643775, -+STORE, 94417996333056, 94417997631487, -+STORE, 94417997631488, 94417997643775, -+ERASE, 94417997631488, 94417997643775, -+SNULL, 94417997590527, 94417997631487, -+STORE, 94417996333056, 94417997590527, -+STORE, 94417997590528, 94417997631487, -+ERASE, 94417997590528, 94417997631487, -+STORE, 94417996333056, 94417997733887, -+STORE, 94417996333056, 94417997869055, -+STORE, 94417996333056, 94417998004223, -+SNULL, 94417998000127, 94417998004223, -+STORE, 94417996333056, 94417998000127, -+STORE, 94417998000128, 94417998004223, -+ERASE, 94417998000128, 94417998004223, -+STORE, 94049170993152, 94049171206143, -+STORE, 94049173303296, 94049173307391, -+STORE, 94049173307392, 94049173315583, -+STORE, 94049173315584, 94049173327871, -+STORE, 94049176236032, 94049183645695, -+STORE, 139807795544064, 139807797202943, -+STORE, 139807797202944, 139807799300095, -+STORE, 139807799300096, 139807799316479, -+STORE, 139807799316480, 139807799324671, -+STORE, 139807799324672, 139807799341055, -+STORE, 139807799341056, 139807799353343, -+STORE, 139807799353344, 139807801446399, -+STORE, 139807801446400, 139807801450495, -+STORE, 139807801450496, 139807801454591, -+STORE, 139807801454592, 139807801597951, -+STORE, 139807801966592, 139807803650047, -+STORE, 139807803650048, 139807803666431, -+STORE, 139807803695104, 139807803699199, -+STORE, 139807803699200, 139807803703295, -+STORE, 139807803703296, 139807803707391, -+STORE, 140727555538944, 140727555678207, -+STORE, 140727555940352, 140727555952639, -+STORE, 140727555952640, 140727555956735, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722483441664, 140737488351231, -+SNULL, 140722483449855, 140737488351231, -+STORE, 140722483441664, 140722483449855, -+STORE, 140722483310592, 140722483449855, -+STORE, 94416704921600, 94416707145727, -+SNULL, 94416705032191, 94416707145727, -+STORE, 94416704921600, 94416705032191, -+STORE, 94416705032192, 94416707145727, -+ERASE, 94416705032192, 94416707145727, -+STORE, 94416707125248, 94416707137535, -+STORE, 94416707137536, 94416707145727, -+STORE, 140555439296512, 140555441549311, -+SNULL, 140555439439871, 140555441549311, -+STORE, 140555439296512, 140555439439871, -+STORE, 140555439439872, 140555441549311, -+ERASE, 140555439439872, 140555441549311, -+STORE, 140555441537024, 140555441545215, -+STORE, 140555441545216, 140555441549311, -+STORE, 140722484781056, 140722484785151, -+STORE, 140722484768768, 140722484781055, -+STORE, 140555441508352, 140555441537023, -+STORE, 140555441500160, 140555441508351, -+STORE, 140555435499520, 140555439296511, -+SNULL, 140555435499520, 140555437158399, -+STORE, 140555437158400, 140555439296511, -+STORE, 140555435499520, 140555437158399, -+SNULL, 140555439255551, 140555439296511, -+STORE, 140555437158400, 140555439255551, -+STORE, 140555439255552, 140555439296511, -+SNULL, 140555439255552, 140555439280127, -+STORE, 140555439280128, 140555439296511, -+STORE, 140555439255552, 140555439280127, -+ERASE, 140555439255552, 140555439280127, -+STORE, 140555439255552, 140555439280127, -+ERASE, 140555439280128, 140555439296511, -+STORE, 140555439280128, 140555439296511, -+SNULL, 140555439271935, 140555439280127, -+STORE, 140555439255552, 140555439271935, -+STORE, 140555439271936, 140555439280127, -+SNULL, 94416707133439, 94416707137535, -+STORE, 94416707125248, 94416707133439, -+STORE, 94416707133440, 94416707137535, -+SNULL, 140555441541119, 140555441545215, -+STORE, 140555441537024, 140555441541119, -+STORE, 140555441541120, 140555441545215, -+ERASE, 140555441508352, 140555441537023, -+STORE, 94416724672512, 94416724807679, -+STORE, 94686636953600, 94686637166591, -+STORE, 94686639263744, 94686639267839, -+STORE, 94686639267840, 94686639276031, -+STORE, 94686639276032, 94686639288319, -+STORE, 94686662193152, 94686663163903, -+STORE, 140312944431104, 140312946089983, -+STORE, 140312946089984, 140312948187135, -+STORE, 140312948187136, 140312948203519, -+STORE, 140312948203520, 140312948211711, -+STORE, 140312948211712, 140312948228095, -+STORE, 140312948228096, 140312948240383, -+STORE, 140312948240384, 140312950333439, -+STORE, 140312950333440, 140312950337535, -+STORE, 140312950337536, 140312950341631, -+STORE, 140312950341632, 140312950484991, -+STORE, 140312950853632, 140312952537087, -+STORE, 140312952537088, 140312952553471, -+STORE, 140312952582144, 140312952586239, -+STORE, 140312952586240, 140312952590335, -+STORE, 140312952590336, 140312952594431, -+STORE, 140730598920192, 140730599059455, -+STORE, 140730599108608, 140730599120895, -+STORE, 140730599120896, 140730599124991, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140726234079232, 140737488351231, -+SNULL, 140726234087423, 140737488351231, -+STORE, 140726234079232, 140726234087423, -+STORE, 140726233948160, 140726234087423, -+STORE, 94589467578368, 94589469802495, -+SNULL, 94589467688959, 94589469802495, -+STORE, 94589467578368, 94589467688959, -+STORE, 94589467688960, 94589469802495, -+ERASE, 94589467688960, 94589469802495, -+STORE, 94589469782016, 94589469794303, -+STORE, 94589469794304, 94589469802495, -+STORE, 140587082842112, 140587085094911, -+SNULL, 140587082985471, 140587085094911, -+STORE, 140587082842112, 140587082985471, -+STORE, 140587082985472, 140587085094911, -+ERASE, 140587082985472, 140587085094911, -+STORE, 140587085082624, 140587085090815, -+STORE, 140587085090816, 140587085094911, -+STORE, 140726234103808, 140726234107903, -+STORE, 140726234091520, 140726234103807, -+STORE, 140587085053952, 140587085082623, -+STORE, 140587085045760, 140587085053951, -+STORE, 140587079045120, 140587082842111, -+SNULL, 140587079045120, 140587080703999, -+STORE, 140587080704000, 140587082842111, -+STORE, 140587079045120, 140587080703999, -+SNULL, 140587082801151, 140587082842111, -+STORE, 140587080704000, 140587082801151, -+STORE, 140587082801152, 140587082842111, -+SNULL, 140587082801152, 140587082825727, -+STORE, 140587082825728, 140587082842111, -+STORE, 140587082801152, 140587082825727, -+ERASE, 140587082801152, 140587082825727, -+STORE, 140587082801152, 140587082825727, -+ERASE, 140587082825728, 140587082842111, -+STORE, 140587082825728, 140587082842111, -+SNULL, 140587082817535, 140587082825727, -+STORE, 140587082801152, 140587082817535, -+STORE, 140587082817536, 140587082825727, -+SNULL, 94589469790207, 94589469794303, -+STORE, 94589469782016, 94589469790207, -+STORE, 94589469790208, 94589469794303, -+SNULL, 140587085086719, 140587085090815, -+STORE, 140587085082624, 140587085086719, -+STORE, 140587085086720, 140587085090815, -+ERASE, 140587085053952, 140587085082623, -+STORE, 94589477507072, 94589477642239, -+STORE, 94225448325120, 94225448538111, -+STORE, 94225450635264, 94225450639359, -+STORE, 94225450639360, 94225450647551, -+STORE, 94225450647552, 94225450659839, -+STORE, 94225470246912, 94225473548287, -+STORE, 140199245496320, 140199247155199, -+STORE, 140199247155200, 140199249252351, -+STORE, 140199249252352, 140199249268735, -+STORE, 140199249268736, 140199249276927, -+STORE, 140199249276928, 140199249293311, -+STORE, 140199249293312, 140199249305599, -+STORE, 140199249305600, 140199251398655, -+STORE, 140199251398656, 140199251402751, -+STORE, 140199251402752, 140199251406847, -+STORE, 140199251406848, 140199251550207, -+STORE, 140199251918848, 140199253602303, -+STORE, 140199253602304, 140199253618687, -+STORE, 140199253647360, 140199253651455, -+STORE, 140199253651456, 140199253655551, -+STORE, 140199253655552, 140199253659647, -+STORE, 140726264414208, 140726264553471, -+STORE, 140726265843712, 140726265855999, -+STORE, 140726265856000, 140726265860095, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733508358144, 140737488351231, -+SNULL, 140733508366335, 140737488351231, -+STORE, 140733508358144, 140733508366335, -+STORE, 140733508227072, 140733508366335, -+STORE, 94766263947264, 94766266171391, -+SNULL, 94766264057855, 94766266171391, -+STORE, 94766263947264, 94766264057855, -+STORE, 94766264057856, 94766266171391, -+ERASE, 94766264057856, 94766266171391, -+STORE, 94766266150912, 94766266163199, -+STORE, 94766266163200, 94766266171391, -+STORE, 140693985132544, 140693987385343, -+SNULL, 140693985275903, 140693987385343, -+STORE, 140693985132544, 140693985275903, -+STORE, 140693985275904, 140693987385343, -+ERASE, 140693985275904, 140693987385343, -+STORE, 140693987373056, 140693987381247, -+STORE, 140693987381248, 140693987385343, -+STORE, 140733509939200, 140733509943295, -+STORE, 140733509926912, 140733509939199, -+STORE, 140693987344384, 140693987373055, -+STORE, 140693987336192, 140693987344383, -+STORE, 140693981335552, 140693985132543, -+SNULL, 140693981335552, 140693982994431, -+STORE, 140693982994432, 140693985132543, -+STORE, 140693981335552, 140693982994431, -+SNULL, 140693985091583, 140693985132543, -+STORE, 140693982994432, 140693985091583, -+STORE, 140693985091584, 140693985132543, -+SNULL, 140693985091584, 140693985116159, -+STORE, 140693985116160, 140693985132543, -+STORE, 140693985091584, 140693985116159, -+ERASE, 140693985091584, 140693985116159, -+STORE, 140693985091584, 140693985116159, -+ERASE, 140693985116160, 140693985132543, -+STORE, 140693985116160, 140693985132543, -+SNULL, 140693985107967, 140693985116159, -+STORE, 140693985091584, 140693985107967, -+STORE, 140693985107968, 140693985116159, -+SNULL, 94766266159103, 94766266163199, -+STORE, 94766266150912, 94766266159103, -+STORE, 94766266159104, 94766266163199, -+SNULL, 140693987377151, 140693987381247, -+STORE, 140693987373056, 140693987377151, -+STORE, 140693987377152, 140693987381247, -+ERASE, 140693987344384, 140693987373055, -+STORE, 94766282035200, 94766282170367, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724769353728, 140737488351231, -+SNULL, 140724769361919, 140737488351231, -+STORE, 140724769353728, 140724769361919, -+STORE, 140724769222656, 140724769361919, -+STORE, 94710460526592, 94710462750719, -+SNULL, 94710460637183, 94710462750719, -+STORE, 94710460526592, 94710460637183, -+STORE, 94710460637184, 94710462750719, -+ERASE, 94710460637184, 94710462750719, -+STORE, 94710462730240, 94710462742527, -+STORE, 94710462742528, 94710462750719, -+STORE, 140469764395008, 140469766647807, -+SNULL, 140469764538367, 140469766647807, -+STORE, 140469764395008, 140469764538367, -+STORE, 140469764538368, 140469766647807, -+ERASE, 140469764538368, 140469766647807, -+STORE, 140469766635520, 140469766643711, -+STORE, 140469766643712, 140469766647807, -+STORE, 140724770877440, 140724770881535, -+STORE, 140724770865152, 140724770877439, -+STORE, 140469766606848, 140469766635519, -+STORE, 140469766598656, 140469766606847, -+STORE, 140469760598016, 140469764395007, -+SNULL, 140469760598016, 140469762256895, -+STORE, 140469762256896, 140469764395007, -+STORE, 140469760598016, 140469762256895, -+SNULL, 140469764354047, 140469764395007, -+STORE, 140469762256896, 140469764354047, -+STORE, 140469764354048, 140469764395007, -+SNULL, 140469764354048, 140469764378623, -+STORE, 140469764378624, 140469764395007, -+STORE, 140469764354048, 140469764378623, -+ERASE, 140469764354048, 140469764378623, -+STORE, 140469764354048, 140469764378623, -+ERASE, 140469764378624, 140469764395007, -+STORE, 140469764378624, 140469764395007, -+SNULL, 140469764370431, 140469764378623, -+STORE, 140469764354048, 140469764370431, -+STORE, 140469764370432, 140469764378623, -+SNULL, 94710462738431, 94710462742527, -+STORE, 94710462730240, 94710462738431, -+STORE, 94710462738432, 94710462742527, -+SNULL, 140469766639615, 140469766643711, -+STORE, 140469766635520, 140469766639615, -+STORE, 140469766639616, 140469766643711, -+ERASE, 140469766606848, 140469766635519, -+STORE, 94710485581824, 94710485716991, -+STORE, 94105755795456, 94105756008447, -+STORE, 94105758105600, 94105758109695, -+STORE, 94105758109696, 94105758117887, -+STORE, 94105758117888, 94105758130175, -+STORE, 94105788981248, 94105794871295, -+STORE, 140641190031360, 140641191690239, -+STORE, 140641191690240, 140641193787391, -+STORE, 140641193787392, 140641193803775, -+STORE, 140641193803776, 140641193811967, -+STORE, 140641193811968, 140641193828351, -+STORE, 140641193828352, 140641193840639, -+STORE, 140641193840640, 140641195933695, -+STORE, 140641195933696, 140641195937791, -+STORE, 140641195937792, 140641195941887, -+STORE, 140641195941888, 140641196085247, -+STORE, 140641196453888, 140641198137343, -+STORE, 140641198137344, 140641198153727, -+STORE, 140641198182400, 140641198186495, -+STORE, 140641198186496, 140641198190591, -+STORE, 140641198190592, 140641198194687, -+STORE, 140731980034048, 140731980173311, -+STORE, 140731981078528, 140731981090815, -+STORE, 140731981090816, 140731981094911, -+STORE, 93828086431744, 93828086644735, -+STORE, 93828088741888, 93828088745983, -+STORE, 93828088745984, 93828088754175, -+STORE, 93828088754176, 93828088766463, -+STORE, 93828094193664, 93828096831487, -+STORE, 139844717334528, 139844718993407, -+STORE, 139844718993408, 139844721090559, -+STORE, 139844721090560, 139844721106943, -+STORE, 139844721106944, 139844721115135, -+STORE, 139844721115136, 139844721131519, -+STORE, 139844721131520, 139844721143807, -+STORE, 139844721143808, 139844723236863, -+STORE, 139844723236864, 139844723240959, -+STORE, 139844723240960, 139844723245055, -+STORE, 139844723245056, 139844723388415, -+STORE, 139844723757056, 139844725440511, -+STORE, 139844725440512, 139844725456895, -+STORE, 139844725485568, 139844725489663, -+STORE, 139844725489664, 139844725493759, -+STORE, 139844725493760, 139844725497855, -+STORE, 140729996185600, 140729996324863, -+STORE, 140729996828672, 140729996840959, -+STORE, 140729996840960, 140729996845055, -+STORE, 140737488347136, 140737488351231, -+STORE, 140722494771200, 140737488351231, -+SNULL, 140722494775295, 140737488351231, -+STORE, 140722494771200, 140722494775295, -+STORE, 140722494640128, 140722494775295, -+STORE, 94324011311104, 94324013535231, -+SNULL, 94324011421695, 94324013535231, -+STORE, 94324011311104, 94324011421695, -+STORE, 94324011421696, 94324013535231, -+ERASE, 94324011421696, 94324013535231, -+STORE, 94324013514752, 94324013527039, -+STORE, 94324013527040, 94324013535231, -+STORE, 140151462309888, 140151464562687, -+SNULL, 140151462453247, 140151464562687, -+STORE, 140151462309888, 140151462453247, -+STORE, 140151462453248, 140151464562687, -+ERASE, 140151462453248, 140151464562687, -+STORE, 140151464550400, 140151464558591, -+STORE, 140151464558592, 140151464562687, -+STORE, 140722495467520, 140722495471615, -+STORE, 140722495455232, 140722495467519, -+STORE, 140151464521728, 140151464550399, -+STORE, 140151464513536, 140151464521727, -+STORE, 140151458512896, 140151462309887, -+SNULL, 140151458512896, 140151460171775, -+STORE, 140151460171776, 140151462309887, -+STORE, 140151458512896, 140151460171775, -+SNULL, 140151462268927, 140151462309887, -+STORE, 140151460171776, 140151462268927, -+STORE, 140151462268928, 140151462309887, -+SNULL, 140151462268928, 140151462293503, -+STORE, 140151462293504, 140151462309887, -+STORE, 140151462268928, 140151462293503, -+ERASE, 140151462268928, 140151462293503, -+STORE, 140151462268928, 140151462293503, -+ERASE, 140151462293504, 140151462309887, -+STORE, 140151462293504, 140151462309887, -+SNULL, 140151462285311, 140151462293503, -+STORE, 140151462268928, 140151462285311, -+STORE, 140151462285312, 140151462293503, -+SNULL, 94324013522943, 94324013527039, -+STORE, 94324013514752, 94324013522943, -+STORE, 94324013522944, 94324013527039, -+SNULL, 140151464554495, 140151464558591, -+STORE, 140151464550400, 140151464554495, -+STORE, 140151464554496, 140151464558591, -+ERASE, 140151464521728, 140151464550399, -+STORE, 94324024778752, 94324024913919, -+STORE, 94899262967808, 94899263180799, -+STORE, 94899265277952, 94899265282047, -+STORE, 94899265282048, 94899265290239, -+STORE, 94899265290240, 94899265302527, -+STORE, 94899295469568, 94899298689023, -+STORE, 140434388418560, 140434390077439, -+STORE, 140434390077440, 140434392174591, -+STORE, 140434392174592, 140434392190975, -+STORE, 140434392190976, 140434392199167, -+STORE, 140434392199168, 140434392215551, -+STORE, 140434392215552, 140434392227839, -+STORE, 140434392227840, 140434394320895, -+STORE, 140434394320896, 140434394324991, -+STORE, 140434394324992, 140434394329087, -+STORE, 140434394329088, 140434394472447, -+STORE, 140434394841088, 140434396524543, -+STORE, 140434396524544, 140434396540927, -+STORE, 140434396569600, 140434396573695, -+STORE, 140434396573696, 140434396577791, -+STORE, 140434396577792, 140434396581887, -+STORE, 140720618135552, 140720618274815, -+STORE, 140720618418176, 140720618430463, -+STORE, 140720618430464, 140720618434559, -+STORE, 94425529798656, 94425530011647, -+STORE, 94425532108800, 94425532112895, -+STORE, 94425532112896, 94425532121087, -+STORE, 94425532121088, 94425532133375, -+STORE, 94425557753856, 94425566576639, -+STORE, 140600528470016, 140600530128895, -+STORE, 140600530128896, 140600532226047, -+STORE, 140600532226048, 140600532242431, -+STORE, 140600532242432, 140600532250623, -+STORE, 140600532250624, 140600532267007, -+STORE, 140600532267008, 140600532279295, -+STORE, 140600532279296, 140600534372351, -+STORE, 140600534372352, 140600534376447, -+STORE, 140600534376448, 140600534380543, -+STORE, 140600534380544, 140600534523903, -+STORE, 140600534892544, 140600536575999, -+STORE, 140600536576000, 140600536592383, -+STORE, 140600536621056, 140600536625151, -+STORE, 140600536625152, 140600536629247, -+STORE, 140600536629248, 140600536633343, -+STORE, 140721857785856, 140721857925119, -+STORE, 140721858068480, 140721858080767, -+STORE, 140721858080768, 140721858084863, -+STORE, 94425529798656, 94425530011647, -+STORE, 94425532108800, 94425532112895, -+STORE, 94425532112896, 94425532121087, -+STORE, 94425532121088, 94425532133375, -+STORE, 94425557753856, 94425568772095, -+STORE, 140600528470016, 140600530128895, -+STORE, 140600530128896, 140600532226047, -+STORE, 140600532226048, 140600532242431, -+STORE, 140600532242432, 140600532250623, -+STORE, 140600532250624, 140600532267007, -+STORE, 140600532267008, 140600532279295, -+STORE, 140600532279296, 140600534372351, -+STORE, 140600534372352, 140600534376447, -+STORE, 140600534376448, 140600534380543, -+STORE, 140600534380544, 140600534523903, -+STORE, 140600534892544, 140600536575999, -+STORE, 140600536576000, 140600536592383, -+STORE, 140600536621056, 140600536625151, -+STORE, 140600536625152, 140600536629247, -+STORE, 140600536629248, 140600536633343, -+STORE, 140721857785856, 140721857925119, -+STORE, 140721858068480, 140721858080767, -+STORE, 140721858080768, 140721858084863, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735611645952, 140737488351231, -+SNULL, 140735611654143, 140737488351231, -+STORE, 140735611645952, 140735611654143, -+STORE, 140735611514880, 140735611654143, -+STORE, 94592137641984, 94592139866111, -+SNULL, 94592137752575, 94592139866111, -+STORE, 94592137641984, 94592137752575, -+STORE, 94592137752576, 94592139866111, -+ERASE, 94592137752576, 94592139866111, -+STORE, 94592139845632, 94592139857919, -+STORE, 94592139857920, 94592139866111, -+STORE, 140350425030656, 140350427283455, -+SNULL, 140350425174015, 140350427283455, -+STORE, 140350425030656, 140350425174015, -+STORE, 140350425174016, 140350427283455, -+ERASE, 140350425174016, 140350427283455, -+STORE, 140350427271168, 140350427279359, -+STORE, 140350427279360, 140350427283455, -+STORE, 140735612043264, 140735612047359, -+STORE, 140735612030976, 140735612043263, -+STORE, 140350427242496, 140350427271167, -+STORE, 140350427234304, 140350427242495, -+STORE, 140350421233664, 140350425030655, -+SNULL, 140350421233664, 140350422892543, -+STORE, 140350422892544, 140350425030655, -+STORE, 140350421233664, 140350422892543, -+SNULL, 140350424989695, 140350425030655, -+STORE, 140350422892544, 140350424989695, -+STORE, 140350424989696, 140350425030655, -+SNULL, 140350424989696, 140350425014271, -+STORE, 140350425014272, 140350425030655, -+STORE, 140350424989696, 140350425014271, -+ERASE, 140350424989696, 140350425014271, -+STORE, 140350424989696, 140350425014271, -+ERASE, 140350425014272, 140350425030655, -+STORE, 140350425014272, 140350425030655, -+SNULL, 140350425006079, 140350425014271, -+STORE, 140350424989696, 140350425006079, -+STORE, 140350425006080, 140350425014271, -+SNULL, 94592139853823, 94592139857919, -+STORE, 94592139845632, 94592139853823, -+STORE, 94592139853824, 94592139857919, -+SNULL, 140350427275263, 140350427279359, -+STORE, 140350427271168, 140350427275263, -+STORE, 140350427275264, 140350427279359, -+ERASE, 140350427242496, 140350427271167, -+STORE, 94592164823040, 94592164958207, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723500535808, 140737488351231, -+SNULL, 140723500543999, 140737488351231, -+STORE, 140723500535808, 140723500543999, -+STORE, 140723500404736, 140723500543999, -+STORE, 94458379010048, 94458381234175, -+SNULL, 94458379120639, 94458381234175, -+STORE, 94458379010048, 94458379120639, -+STORE, 94458379120640, 94458381234175, -+ERASE, 94458379120640, 94458381234175, -+STORE, 94458381213696, 94458381225983, -+STORE, 94458381225984, 94458381234175, -+STORE, 139771674230784, 139771676483583, -+SNULL, 139771674374143, 139771676483583, -+STORE, 139771674230784, 139771674374143, -+STORE, 139771674374144, 139771676483583, -+ERASE, 139771674374144, 139771676483583, -+STORE, 139771676471296, 139771676479487, -+STORE, 139771676479488, 139771676483583, -+STORE, 140723500769280, 140723500773375, -+STORE, 140723500756992, 140723500769279, -+STORE, 139771676442624, 139771676471295, -+STORE, 139771676434432, 139771676442623, -+STORE, 139771670433792, 139771674230783, -+SNULL, 139771670433792, 139771672092671, -+STORE, 139771672092672, 139771674230783, -+STORE, 139771670433792, 139771672092671, -+SNULL, 139771674189823, 139771674230783, -+STORE, 139771672092672, 139771674189823, -+STORE, 139771674189824, 139771674230783, -+SNULL, 139771674189824, 139771674214399, -+STORE, 139771674214400, 139771674230783, -+STORE, 139771674189824, 139771674214399, -+ERASE, 139771674189824, 139771674214399, -+STORE, 139771674189824, 139771674214399, -+ERASE, 139771674214400, 139771674230783, -+STORE, 139771674214400, 139771674230783, -+SNULL, 139771674206207, 139771674214399, -+STORE, 139771674189824, 139771674206207, -+STORE, 139771674206208, 139771674214399, -+SNULL, 94458381221887, 94458381225983, -+STORE, 94458381213696, 94458381221887, -+STORE, 94458381221888, 94458381225983, -+SNULL, 139771676475391, 139771676479487, -+STORE, 139771676471296, 139771676475391, -+STORE, 139771676475392, 139771676479487, -+ERASE, 139771676442624, 139771676471295, -+STORE, 94458401873920, 94458402009087, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731316264960, 140737488351231, -+SNULL, 140731316273151, 140737488351231, -+STORE, 140731316264960, 140731316273151, -+STORE, 140731316133888, 140731316273151, -+STORE, 94437830881280, 94437833215999, -+SNULL, 94437831094271, 94437833215999, -+STORE, 94437830881280, 94437831094271, -+STORE, 94437831094272, 94437833215999, -+ERASE, 94437831094272, 94437833215999, -+STORE, 94437833191424, 94437833203711, -+STORE, 94437833203712, 94437833215999, -+STORE, 140265986031616, 140265988284415, -+SNULL, 140265986174975, 140265988284415, -+STORE, 140265986031616, 140265986174975, -+STORE, 140265986174976, 140265988284415, -+ERASE, 140265986174976, 140265988284415, -+STORE, 140265988272128, 140265988280319, -+STORE, 140265988280320, 140265988284415, -+STORE, 140731316318208, 140731316322303, -+STORE, 140731316305920, 140731316318207, -+STORE, 140265988243456, 140265988272127, -+STORE, 140265988235264, 140265988243455, -+STORE, 140265983918080, 140265986031615, -+SNULL, 140265983918080, 140265983930367, -+STORE, 140265983930368, 140265986031615, -+STORE, 140265983918080, 140265983930367, -+SNULL, 140265986023423, 140265986031615, -+STORE, 140265983930368, 140265986023423, -+STORE, 140265986023424, 140265986031615, -+ERASE, 140265986023424, 140265986031615, -+STORE, 140265986023424, 140265986031615, -+STORE, 140265980121088, 140265983918079, -+SNULL, 140265980121088, 140265981779967, -+STORE, 140265981779968, 140265983918079, -+STORE, 140265980121088, 140265981779967, -+SNULL, 140265983877119, 140265983918079, -+STORE, 140265981779968, 140265983877119, -+STORE, 140265983877120, 140265983918079, -+SNULL, 140265983877120, 140265983901695, -+STORE, 140265983901696, 140265983918079, -+STORE, 140265983877120, 140265983901695, -+ERASE, 140265983877120, 140265983901695, -+STORE, 140265983877120, 140265983901695, -+ERASE, 140265983901696, 140265983918079, -+STORE, 140265983901696, 140265983918079, -+STORE, 140265988227072, 140265988243455, -+SNULL, 140265983893503, 140265983901695, -+STORE, 140265983877120, 140265983893503, -+STORE, 140265983893504, 140265983901695, -+SNULL, 140265986027519, 140265986031615, -+STORE, 140265986023424, 140265986027519, -+STORE, 140265986027520, 140265986031615, -+SNULL, 94437833195519, 94437833203711, -+STORE, 94437833191424, 94437833195519, -+STORE, 94437833195520, 94437833203711, -+SNULL, 140265988276223, 140265988280319, -+STORE, 140265988272128, 140265988276223, -+STORE, 140265988276224, 140265988280319, -+ERASE, 140265988243456, 140265988272127, -+STORE, 94437847638016, 94437847773183, -+STORE, 140265986543616, 140265988227071, -+STORE, 94437847638016, 94437847908351, -+STORE, 94437847638016, 94437848043519, -+STORE, 94437847638016, 94437848190975, -+SNULL, 94437848178687, 94437848190975, -+STORE, 94437847638016, 94437848178687, -+STORE, 94437848178688, 94437848190975, -+ERASE, 94437848178688, 94437848190975, -+STORE, 94437847638016, 94437848330239, -+STORE, 94437847638016, 94437848465407, -+SNULL, 94437848444927, 94437848465407, -+STORE, 94437847638016, 94437848444927, -+STORE, 94437848444928, 94437848465407, -+ERASE, 94437848444928, 94437848465407, -+STORE, 94437847638016, 94437848584191, -+STORE, 94437847638016, 94437848719359, -+SNULL, 94437848678399, 94437848719359, -+STORE, 94437847638016, 94437848678399, -+STORE, 94437848678400, 94437848719359, -+ERASE, 94437848678400, 94437848719359, -+STORE, 94437847638016, 94437848842239, -+SNULL, 94437848825855, 94437848842239, -+STORE, 94437847638016, 94437848825855, -+STORE, 94437848825856, 94437848842239, -+ERASE, 94437848825856, 94437848842239, -+STORE, 94437847638016, 94437848961023, -+STORE, 94437847638016, 94437849096191, -+STORE, 94661814710272, 94661814923263, -+STORE, 94661817020416, 94661817024511, -+STORE, 94661817024512, 94661817032703, -+STORE, 94661817032704, 94661817044991, -+STORE, 94661840424960, 94661841240063, -+STORE, 140582259814400, 140582261473279, -+STORE, 140582261473280, 140582263570431, -+STORE, 140582263570432, 140582263586815, -+STORE, 140582263586816, 140582263595007, -+STORE, 140582263595008, 140582263611391, -+STORE, 140582263611392, 140582263623679, -+STORE, 140582263623680, 140582265716735, -+STORE, 140582265716736, 140582265720831, -+STORE, 140582265720832, 140582265724927, -+STORE, 140582265724928, 140582265868287, -+STORE, 140582266236928, 140582267920383, -+STORE, 140582267920384, 140582267936767, -+STORE, 140582267965440, 140582267969535, -+STORE, 140582267969536, 140582267973631, -+STORE, 140582267973632, 140582267977727, -+STORE, 140735472508928, 140735472648191, -+STORE, 140735472672768, 140735472685055, -+STORE, 140735472685056, 140735472689151, -+STORE, 94440069140480, 94440069353471, -+STORE, 94440071450624, 94440071454719, -+STORE, 94440071454720, 94440071462911, -+STORE, 94440071462912, 94440071475199, -+STORE, 94440072122368, 94440079048703, -+STORE, 140112218095616, 140112219754495, -+STORE, 140112219754496, 140112221851647, -+STORE, 140112221851648, 140112221868031, -+STORE, 140112221868032, 140112221876223, -+STORE, 140112221876224, 140112221892607, -+STORE, 140112221892608, 140112221904895, -+STORE, 140112221904896, 140112223997951, -+STORE, 140112223997952, 140112224002047, -+STORE, 140112224002048, 140112224006143, -+STORE, 140112224006144, 140112224149503, -+STORE, 140112224518144, 140112226201599, -+STORE, 140112226201600, 140112226217983, -+STORE, 140112226246656, 140112226250751, -+STORE, 140112226250752, 140112226254847, -+STORE, 140112226254848, 140112226258943, -+STORE, 140737460969472, 140737461108735, -+STORE, 140737462083584, 140737462095871, -+STORE, 140737462095872, 140737462099967, -+STORE, 94257654345728, 94257654390783, -+STORE, 94257656483840, 94257656487935, -+STORE, 94257656487936, 94257656492031, -+STORE, 94257656492032, 94257656496127, -+STORE, 94257665859584, 94257665994751, -+STORE, 140507070345216, 140507070386175, -+STORE, 140507070386176, 140507072483327, -+STORE, 140507072483328, 140507072487423, -+STORE, 140507072487424, 140507072491519, -+STORE, 140507072491520, 140507072516095, -+STORE, 140507072516096, 140507072561151, -+STORE, 140507072561152, 140507074654207, -+STORE, 140507074654208, 140507074658303, -+STORE, 140507074658304, 140507074662399, -+STORE, 140507074662400, 140507074744319, -+STORE, 140507074744320, 140507076841471, -+STORE, 140507076841472, 140507076845567, -+STORE, 140507076845568, 140507076849663, -+STORE, 140507076849664, 140507076857855, -+STORE, 140507076857856, 140507076886527, -+STORE, 140507076886528, 140507078979583, -+STORE, 140507078979584, 140507078983679, -+STORE, 140507078983680, 140507078987775, -+STORE, 140507078987776, 140507079086079, -+STORE, 140507079086080, 140507081179135, -+STORE, 140507081179136, 140507081183231, -+STORE, 140507081183232, 140507081187327, -+STORE, 140507081187328, 140507081203711, -+STORE, 140507081203712, 140507081220095, -+STORE, 140507081220096, 140507083317247, -+STORE, 140507083317248, 140507083321343, -+STORE, 140507083321344, 140507083325439, -+STORE, 140507083325440, 140507083792383, -+STORE, 140507083792384, 140507085885439, -+STORE, 140507085885440, 140507085889535, -+STORE, 140507085889536, 140507085893631, -+STORE, 140507085893632, 140507085905919, -+STORE, 140507085905920, 140507087998975, -+STORE, 140507087998976, 140507088003071, -+STORE, 140507088003072, 140507088007167, -+STORE, 140507088007168, 140507088125951, -+STORE, 140507088125952, 140507090219007, -+STORE, 140507090219008, 140507090223103, -+STORE, 140507090223104, 140507090227199, -+STORE, 140507090227200, 140507090268159, -+STORE, 140507090268160, 140507091927039, -+STORE, 140507091927040, 140507094024191, -+STORE, 140507094024192, 140507094040575, -+STORE, 140507094040576, 140507094048767, -+STORE, 140507094048768, 140507094065151, -+STORE, 140507094065152, 140507094216703, -+STORE, 140507094216704, 140507096309759, -+STORE, 140507096309760, 140507096313855, -+STORE, 140507096313856, 140507096317951, -+STORE, 140507096317952, 140507096326143, -+STORE, 140507096326144, 140507096379391, -+STORE, 140507096379392, 140507098472447, -+STORE, 140507098472448, 140507098476543, -+STORE, 140507098476544, 140507098480639, -+STORE, 140507098480640, 140507098623999, -+STORE, 140507098980352, 140507100663807, -+STORE, 140507100663808, 140507100692479, -+STORE, 140507100721152, 140507100725247, -+STORE, 140507100725248, 140507100729343, -+STORE, 140507100729344, 140507100733439, -+STORE, 140728152780800, 140728152915967, -+STORE, 140728153698304, 140728153710591, -+STORE, 140728153710592, 140728153714687, -+STORE, 140507068137472, 140507070345215, -+SNULL, 140507068137472, 140507068190719, -+STORE, 140507068190720, 140507070345215, -+STORE, 140507068137472, 140507068190719, -+SNULL, 140507070287871, 140507070345215, -+STORE, 140507068190720, 140507070287871, -+STORE, 140507070287872, 140507070345215, -+SNULL, 140507070287872, 140507070296063, -+STORE, 140507070296064, 140507070345215, -+STORE, 140507070287872, 140507070296063, -+ERASE, 140507070287872, 140507070296063, -+STORE, 140507070287872, 140507070296063, -+ERASE, 140507070296064, 140507070345215, -+STORE, 140507070296064, 140507070345215, -+STORE, 140507100692480, 140507100721151, -+STORE, 140507065810944, 140507068137471, -+SNULL, 140507065810944, 140507065843711, -+STORE, 140507065843712, 140507068137471, -+STORE, 140507065810944, 140507065843711, -+SNULL, 140507067940863, 140507068137471, -+STORE, 140507065843712, 140507067940863, -+STORE, 140507067940864, 140507068137471, -+SNULL, 140507067940864, 140507067949055, -+STORE, 140507067949056, 140507068137471, -+STORE, 140507067940864, 140507067949055, -+ERASE, 140507067940864, 140507067949055, -+STORE, 140507067940864, 140507067949055, -+ERASE, 140507067949056, 140507068137471, -+STORE, 140507067949056, 140507068137471, -+SNULL, 140507067944959, 140507067949055, -+STORE, 140507067940864, 140507067944959, -+STORE, 140507067944960, 140507067949055, -+SNULL, 140507070291967, 140507070296063, -+STORE, 140507070287872, 140507070291967, -+STORE, 140507070291968, 140507070296063, -+ERASE, 140507100692480, 140507100721151, -+STORE, 140507063705600, 140507065810943, -+SNULL, 140507063705600, 140507063709695, -+STORE, 140507063709696, 140507065810943, -+STORE, 140507063705600, 140507063709695, -+SNULL, 140507065802751, 140507065810943, -+STORE, 140507063709696, 140507065802751, -+STORE, 140507065802752, 140507065810943, -+ERASE, 140507065802752, 140507065810943, -+STORE, 140507065802752, 140507065810943, -+SNULL, 140507065806847, 140507065810943, -+STORE, 140507065802752, 140507065806847, -+STORE, 140507065806848, 140507065810943, -+STORE, 140507061600256, 140507063705599, -+SNULL, 140507061600256, 140507061604351, -+STORE, 140507061604352, 140507063705599, -+STORE, 140507061600256, 140507061604351, -+SNULL, 140507063697407, 140507063705599, -+STORE, 140507061604352, 140507063697407, -+STORE, 140507063697408, 140507063705599, -+ERASE, 140507063697408, 140507063705599, -+STORE, 140507063697408, 140507063705599, -+SNULL, 140507063701503, 140507063705599, -+STORE, 140507063697408, 140507063701503, -+STORE, 140507063701504, 140507063705599, -+STORE, 140507059490816, 140507061600255, -+SNULL, 140507059490816, 140507059499007, -+STORE, 140507059499008, 140507061600255, -+STORE, 140507059490816, 140507059499007, -+SNULL, 140507061592063, 140507061600255, -+STORE, 140507059499008, 140507061592063, -+STORE, 140507061592064, 140507061600255, -+ERASE, 140507061592064, 140507061600255, -+STORE, 140507061592064, 140507061600255, -+SNULL, 140507061596159, 140507061600255, -+STORE, 140507061592064, 140507061596159, -+STORE, 140507061596160, 140507061600255, -+STORE, 140507057377280, 140507059490815, -+SNULL, 140507057377280, 140507057389567, -+STORE, 140507057389568, 140507059490815, -+STORE, 140507057377280, 140507057389567, -+SNULL, 140507059482623, 140507059490815, -+STORE, 140507057389568, 140507059482623, -+STORE, 140507059482624, 140507059490815, -+ERASE, 140507059482624, 140507059490815, -+STORE, 140507059482624, 140507059490815, -+SNULL, 140507059486719, 140507059490815, -+STORE, 140507059482624, 140507059486719, -+STORE, 140507059486720, 140507059490815, -+STORE, 140507055255552, 140507057377279, -+SNULL, 140507055255552, 140507055276031, -+STORE, 140507055276032, 140507057377279, -+STORE, 140507055255552, 140507055276031, -+SNULL, 140507057369087, 140507057377279, -+STORE, 140507055276032, 140507057369087, -+STORE, 140507057369088, 140507057377279, -+ERASE, 140507057369088, 140507057377279, -+STORE, 140507057369088, 140507057377279, -+SNULL, 140507057373183, 140507057377279, -+STORE, 140507057369088, 140507057373183, -+STORE, 140507057373184, 140507057377279, -+STORE, 140507098693632, 140507098980351, -+SNULL, 140507098959871, 140507098980351, -+STORE, 140507098693632, 140507098959871, -+STORE, 140507098959872, 140507098980351, -+SNULL, 140507098959872, 140507098976255, -+STORE, 140507098976256, 140507098980351, -+STORE, 140507098959872, 140507098976255, -+ERASE, 140507098959872, 140507098976255, -+STORE, 140507098959872, 140507098976255, -+ERASE, 140507098976256, 140507098980351, -+STORE, 140507098976256, 140507098980351, -+STORE, 140507100692480, 140507100721151, -+STORE, 140507053125632, 140507055255551, -+SNULL, 140507053125632, 140507053154303, -+STORE, 140507053154304, 140507055255551, -+STORE, 140507053125632, 140507053154303, -+SNULL, 140507055247359, 140507055255551, -+STORE, 140507053154304, 140507055247359, -+STORE, 140507055247360, 140507055255551, -+ERASE, 140507055247360, 140507055255551, -+STORE, 140507055247360, 140507055255551, -+STORE, 140507051012096, 140507053125631, -+SNULL, 140507051012096, 140507051024383, -+STORE, 140507051024384, 140507053125631, -+STORE, 140507051012096, 140507051024383, -+SNULL, 140507053117439, 140507053125631, -+STORE, 140507051024384, 140507053117439, -+STORE, 140507053117440, 140507053125631, -+ERASE, 140507053117440, 140507053125631, -+STORE, 140507053117440, 140507053125631, -+SNULL, 140507053121535, 140507053125631, -+STORE, 140507053117440, 140507053121535, -+STORE, 140507053121536, 140507053125631, -+SNULL, 140507055251455, 140507055255551, -+STORE, 140507055247360, 140507055251455, -+STORE, 140507055251456, 140507055255551, -+SNULL, 140507098972159, 140507098976255, -+STORE, 140507098959872, 140507098972159, -+STORE, 140507098972160, 140507098976255, -+ERASE, 140507100692480, 140507100721151, -+STORE, 140507100717056, 140507100721151, -+ERASE, 140507100717056, 140507100721151, -+STORE, 140507100717056, 140507100721151, -+ERASE, 140507100717056, 140507100721151, -+STORE, 140507100717056, 140507100721151, -+ERASE, 140507100717056, 140507100721151, -+STORE, 140507100717056, 140507100721151, -+ERASE, 140507100717056, 140507100721151, -+STORE, 140507100692480, 140507100721151, -+ERASE, 140507068137472, 140507068190719, -+ERASE, 140507068190720, 140507070287871, -+ERASE, 140507070287872, 140507070291967, -+ERASE, 140507070291968, 140507070296063, -+ERASE, 140507070296064, 140507070345215, -+ERASE, 140507065810944, 140507065843711, -+ERASE, 140507065843712, 140507067940863, -+ERASE, 140507067940864, 140507067944959, -+ERASE, 140507067944960, 140507067949055, -+ERASE, 140507067949056, 140507068137471, -+ERASE, 140507063705600, 140507063709695, -+ERASE, 140507063709696, 140507065802751, -+ERASE, 140507065802752, 140507065806847, -+ERASE, 140507065806848, 140507065810943, -+ERASE, 140507061600256, 140507061604351, -+ERASE, 140507061604352, 140507063697407, -+ERASE, 140507063697408, 140507063701503, -+ERASE, 140507063701504, 140507063705599, -+ERASE, 140507059490816, 140507059499007, -+ERASE, 140507059499008, 140507061592063, -+ERASE, 140507061592064, 140507061596159, -+ERASE, 140507061596160, 140507061600255, -+ERASE, 140507057377280, 140507057389567, -+ERASE, 140507057389568, 140507059482623, -+ERASE, 140507059482624, 140507059486719, -+ERASE, 140507059486720, 140507059490815, -+ERASE, 140507055255552, 140507055276031, -+ERASE, 140507055276032, 140507057369087, -+ERASE, 140507057369088, 140507057373183, -+ERASE, 140507057373184, 140507057377279, -+ERASE, 140507098693632, 140507098959871, -+ERASE, 140507098959872, 140507098972159, -+ERASE, 140507098972160, 140507098976255, -+ERASE, 140507098976256, 140507098980351, -+ERASE, 140507051012096, 140507051024383, -+ERASE, 140507051024384, 140507053117439, -+ERASE, 140507053117440, 140507053121535, -+ERASE, 140507053121536, 140507053125631, -+STORE, 94036448296960, 94036448509951, -+STORE, 94036450607104, 94036450611199, -+STORE, 94036450611200, 94036450619391, -+STORE, 94036450619392, 94036450631679, -+STORE, 94036482445312, 94036502376447, -+STORE, 140469487013888, 140469488672767, -+STORE, 140469488672768, 140469490769919, -+STORE, 140469490769920, 140469490786303, -+STORE, 140469490786304, 140469490794495, -+STORE, 140469490794496, 140469490810879, -+STORE, 140469490810880, 140469490823167, -+STORE, 140469490823168, 140469492916223, -+STORE, 140469492916224, 140469492920319, -+STORE, 140469492920320, 140469492924415, -+STORE, 140469492924416, 140469493067775, -+STORE, 140469493436416, 140469495119871, -+STORE, 140469495119872, 140469495136255, -+STORE, 140469495164928, 140469495169023, -+STORE, 140469495169024, 140469495173119, -+STORE, 140469495173120, 140469495177215, -+STORE, 140732281446400, 140732281585663, -+STORE, 140732282736640, 140732282748927, -+STORE, 140732282748928, 140732282753023, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723411931136, 140737488351231, -+SNULL, 140723411939327, 140737488351231, -+STORE, 140723411931136, 140723411939327, -+STORE, 140723411800064, 140723411939327, -+STORE, 93993768685568, 93993770909695, -+SNULL, 93993768796159, 93993770909695, -+STORE, 93993768685568, 93993768796159, -+STORE, 93993768796160, 93993770909695, -+ERASE, 93993768796160, 93993770909695, -+STORE, 93993770889216, 93993770901503, -+STORE, 93993770901504, 93993770909695, -+STORE, 140508681740288, 140508683993087, -+SNULL, 140508681883647, 140508683993087, -+STORE, 140508681740288, 140508681883647, -+STORE, 140508681883648, 140508683993087, -+ERASE, 140508681883648, 140508683993087, -+STORE, 140508683980800, 140508683988991, -+STORE, 140508683988992, 140508683993087, -+STORE, 140723412070400, 140723412074495, -+STORE, 140723412058112, 140723412070399, -+STORE, 140508683952128, 140508683980799, -+STORE, 140508683943936, 140508683952127, -+STORE, 140508677943296, 140508681740287, -+SNULL, 140508677943296, 140508679602175, -+STORE, 140508679602176, 140508681740287, -+STORE, 140508677943296, 140508679602175, -+SNULL, 140508681699327, 140508681740287, -+STORE, 140508679602176, 140508681699327, -+STORE, 140508681699328, 140508681740287, -+SNULL, 140508681699328, 140508681723903, -+STORE, 140508681723904, 140508681740287, -+STORE, 140508681699328, 140508681723903, -+ERASE, 140508681699328, 140508681723903, -+STORE, 140508681699328, 140508681723903, -+ERASE, 140508681723904, 140508681740287, -+STORE, 140508681723904, 140508681740287, -+SNULL, 140508681715711, 140508681723903, -+STORE, 140508681699328, 140508681715711, -+STORE, 140508681715712, 140508681723903, -+SNULL, 93993770897407, 93993770901503, -+STORE, 93993770889216, 93993770897407, -+STORE, 93993770897408, 93993770901503, -+SNULL, 140508683984895, 140508683988991, -+STORE, 140508683980800, 140508683984895, -+STORE, 140508683984896, 140508683988991, -+ERASE, 140508683952128, 140508683980799, -+STORE, 93993791582208, 93993791717375, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140734685458432, 140737488351231, -+SNULL, 140734685466623, 140737488351231, -+STORE, 140734685458432, 140734685466623, -+STORE, 140734685327360, 140734685466623, -+STORE, 93832321548288, 93832323772415, -+SNULL, 93832321658879, 93832323772415, -+STORE, 93832321548288, 93832321658879, -+STORE, 93832321658880, 93832323772415, -+ERASE, 93832321658880, 93832323772415, -+STORE, 93832323751936, 93832323764223, -+STORE, 93832323764224, 93832323772415, -+STORE, 140650945118208, 140650947371007, -+SNULL, 140650945261567, 140650947371007, -+STORE, 140650945118208, 140650945261567, -+STORE, 140650945261568, 140650947371007, -+ERASE, 140650945261568, 140650947371007, -+STORE, 140650947358720, 140650947366911, -+STORE, 140650947366912, 140650947371007, -+STORE, 140734686081024, 140734686085119, -+STORE, 140734686068736, 140734686081023, -+STORE, 140650947330048, 140650947358719, -+STORE, 140650947321856, 140650947330047, -+STORE, 140650941321216, 140650945118207, -+SNULL, 140650941321216, 140650942980095, -+STORE, 140650942980096, 140650945118207, -+STORE, 140650941321216, 140650942980095, -+SNULL, 140650945077247, 140650945118207, -+STORE, 140650942980096, 140650945077247, -+STORE, 140650945077248, 140650945118207, -+SNULL, 140650945077248, 140650945101823, -+STORE, 140650945101824, 140650945118207, -+STORE, 140650945077248, 140650945101823, -+ERASE, 140650945077248, 140650945101823, -+STORE, 140650945077248, 140650945101823, -+ERASE, 140650945101824, 140650945118207, -+STORE, 140650945101824, 140650945118207, -+SNULL, 140650945093631, 140650945101823, -+STORE, 140650945077248, 140650945093631, -+STORE, 140650945093632, 140650945101823, -+SNULL, 93832323760127, 93832323764223, -+STORE, 93832323751936, 93832323760127, -+STORE, 93832323760128, 93832323764223, -+SNULL, 140650947362815, 140650947366911, -+STORE, 140650947358720, 140650947362815, -+STORE, 140650947362816, 140650947366911, -+ERASE, 140650947330048, 140650947358719, -+STORE, 93832331890688, 93832332025855, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728333520896, 140737488351231, -+SNULL, 140728333529087, 140737488351231, -+STORE, 140728333520896, 140728333529087, -+STORE, 140728333389824, 140728333529087, -+STORE, 94872734732288, 94872736956415, -+SNULL, 94872734842879, 94872736956415, -+STORE, 94872734732288, 94872734842879, -+STORE, 94872734842880, 94872736956415, -+ERASE, 94872734842880, 94872736956415, -+STORE, 94872736935936, 94872736948223, -+STORE, 94872736948224, 94872736956415, -+STORE, 139755193257984, 139755195510783, -+SNULL, 139755193401343, 139755195510783, -+STORE, 139755193257984, 139755193401343, -+STORE, 139755193401344, 139755195510783, -+ERASE, 139755193401344, 139755195510783, -+STORE, 139755195498496, 139755195506687, -+STORE, 139755195506688, 139755195510783, -+STORE, 140728333926400, 140728333930495, -+STORE, 140728333914112, 140728333926399, -+STORE, 139755195469824, 139755195498495, -+STORE, 139755195461632, 139755195469823, -+STORE, 139755189460992, 139755193257983, -+SNULL, 139755189460992, 139755191119871, -+STORE, 139755191119872, 139755193257983, -+STORE, 139755189460992, 139755191119871, -+SNULL, 139755193217023, 139755193257983, -+STORE, 139755191119872, 139755193217023, -+STORE, 139755193217024, 139755193257983, -+SNULL, 139755193217024, 139755193241599, -+STORE, 139755193241600, 139755193257983, -+STORE, 139755193217024, 139755193241599, -+ERASE, 139755193217024, 139755193241599, -+STORE, 139755193217024, 139755193241599, -+ERASE, 139755193241600, 139755193257983, -+STORE, 139755193241600, 139755193257983, -+SNULL, 139755193233407, 139755193241599, -+STORE, 139755193217024, 139755193233407, -+STORE, 139755193233408, 139755193241599, -+SNULL, 94872736944127, 94872736948223, -+STORE, 94872736935936, 94872736944127, -+STORE, 94872736944128, 94872736948223, -+SNULL, 139755195502591, 139755195506687, -+STORE, 139755195498496, 139755195502591, -+STORE, 139755195502592, 139755195506687, -+ERASE, 139755195469824, 139755195498495, -+STORE, 94872749744128, 94872749879295, -+STORE, 94720243642368, 94720243855359, -+STORE, 94720245952512, 94720245956607, -+STORE, 94720245956608, 94720245964799, -+STORE, 94720245964800, 94720245977087, -+STORE, 94720277745664, 94720278151167, -+STORE, 140453174497280, 140453176156159, -+STORE, 140453176156160, 140453178253311, -+STORE, 140453178253312, 140453178269695, -+STORE, 140453178269696, 140453178277887, -+STORE, 140453178277888, 140453178294271, -+STORE, 140453178294272, 140453178306559, -+STORE, 140453178306560, 140453180399615, -+STORE, 140453180399616, 140453180403711, -+STORE, 140453180403712, 140453180407807, -+STORE, 140453180407808, 140453180551167, -+STORE, 140453180919808, 140453182603263, -+STORE, 140453182603264, 140453182619647, -+STORE, 140453182648320, 140453182652415, -+STORE, 140453182652416, 140453182656511, -+STORE, 140453182656512, 140453182660607, -+STORE, 140733223923712, 140733224062975, -+STORE, 140733224808448, 140733224820735, -+STORE, 140733224820736, 140733224824831, -+STORE, 94321091141632, 94321091354623, -+STORE, 94321093451776, 94321093455871, -+STORE, 94321093455872, 94321093464063, -+STORE, 94321093464064, 94321093476351, -+STORE, 94321115873280, 94321117229055, -+STORE, 139695978840064, 139695980498943, -+STORE, 139695980498944, 139695982596095, -+STORE, 139695982596096, 139695982612479, -+STORE, 139695982612480, 139695982620671, -+STORE, 139695982620672, 139695982637055, -+STORE, 139695982637056, 139695982649343, -+STORE, 139695982649344, 139695984742399, -+STORE, 139695984742400, 139695984746495, -+STORE, 139695984746496, 139695984750591, -+STORE, 139695984750592, 139695984893951, -+STORE, 139695985262592, 139695986946047, -+STORE, 139695986946048, 139695986962431, -+STORE, 139695986991104, 139695986995199, -+STORE, 139695986995200, 139695986999295, -+STORE, 139695986999296, 139695987003391, -+STORE, 140734650564608, 140734650703871, -+STORE, 140734650785792, 140734650798079, -+STORE, 140734650798080, 140734650802175, -+STORE, 94523438456832, 94523438669823, -+STORE, 94523440766976, 94523440771071, -+STORE, 94523440771072, 94523440779263, -+STORE, 94523440779264, 94523440791551, -+STORE, 94523464544256, 94523465842687, -+STORE, 140453231493120, 140453233151999, -+STORE, 140453233152000, 140453235249151, -+STORE, 140453235249152, 140453235265535, -+STORE, 140453235265536, 140453235273727, -+STORE, 140453235273728, 140453235290111, -+STORE, 140453235290112, 140453235302399, -+STORE, 140453235302400, 140453237395455, -+STORE, 140453237395456, 140453237399551, -+STORE, 140453237399552, 140453237403647, -+STORE, 140453237403648, 140453237547007, -+STORE, 140453237915648, 140453239599103, -+STORE, 140453239599104, 140453239615487, -+STORE, 140453239644160, 140453239648255, -+STORE, 140453239648256, 140453239652351, -+STORE, 140453239652352, 140453239656447, -+STORE, 140734679445504, 140734679584767, -+STORE, 140734680018944, 140734680031231, -+STORE, 140734680031232, 140734680035327, -+STORE, 94614776987648, 94614777200639, -+STORE, 94614779297792, 94614779301887, -+STORE, 94614779301888, 94614779310079, -+STORE, 94614779310080, 94614779322367, -+STORE, 94614798467072, 94614800699391, -+STORE, 139677037182976, 139677038841855, -+STORE, 139677038841856, 139677040939007, -+STORE, 139677040939008, 139677040955391, -+STORE, 139677040955392, 139677040963583, -+STORE, 139677040963584, 139677040979967, -+STORE, 139677040979968, 139677040992255, -+STORE, 139677040992256, 139677043085311, -+STORE, 139677043085312, 139677043089407, -+STORE, 139677043089408, 139677043093503, -+STORE, 139677043093504, 139677043236863, -+STORE, 139677043605504, 139677045288959, -+STORE, 139677045288960, 139677045305343, -+STORE, 139677045334016, 139677045338111, -+STORE, 139677045338112, 139677045342207, -+STORE, 139677045342208, 139677045346303, -+STORE, 140721604411392, 140721604550655, -+STORE, 140721606135808, 140721606148095, -+STORE, 140721606148096, 140721606152191, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140729280544768, 140737488351231, -+SNULL, 140729280552959, 140737488351231, -+STORE, 140729280544768, 140729280552959, -+STORE, 140729280413696, 140729280552959, -+STORE, 94863939334144, 94863941558271, -+SNULL, 94863939444735, 94863941558271, -+STORE, 94863939334144, 94863939444735, -+STORE, 94863939444736, 94863941558271, -+ERASE, 94863939444736, 94863941558271, -+STORE, 94863941537792, 94863941550079, -+STORE, 94863941550080, 94863941558271, -+STORE, 139691047276544, 139691049529343, -+SNULL, 139691047419903, 139691049529343, -+STORE, 139691047276544, 139691047419903, -+STORE, 139691047419904, 139691049529343, -+ERASE, 139691047419904, 139691049529343, -+STORE, 139691049517056, 139691049525247, -+STORE, 139691049525248, 139691049529343, -+STORE, 140729281679360, 140729281683455, -+STORE, 140729281667072, 140729281679359, -+STORE, 139691049488384, 139691049517055, -+STORE, 139691049480192, 139691049488383, -+STORE, 139691043479552, 139691047276543, -+SNULL, 139691043479552, 139691045138431, -+STORE, 139691045138432, 139691047276543, -+STORE, 139691043479552, 139691045138431, -+SNULL, 139691047235583, 139691047276543, -+STORE, 139691045138432, 139691047235583, -+STORE, 139691047235584, 139691047276543, -+SNULL, 139691047235584, 139691047260159, -+STORE, 139691047260160, 139691047276543, -+STORE, 139691047235584, 139691047260159, -+ERASE, 139691047235584, 139691047260159, -+STORE, 139691047235584, 139691047260159, -+ERASE, 139691047260160, 139691047276543, -+STORE, 139691047260160, 139691047276543, -+SNULL, 139691047251967, 139691047260159, -+STORE, 139691047235584, 139691047251967, -+STORE, 139691047251968, 139691047260159, -+SNULL, 94863941545983, 94863941550079, -+STORE, 94863941537792, 94863941545983, -+STORE, 94863941545984, 94863941550079, -+SNULL, 139691049521151, 139691049525247, -+STORE, 139691049517056, 139691049521151, -+STORE, 139691049521152, 139691049525247, -+ERASE, 139691049488384, 139691049517055, -+STORE, 94863951294464, 94863951429631, -+STORE, 93998209294336, 93998209507327, -+STORE, 93998211604480, 93998211608575, -+STORE, 93998211608576, 93998211616767, -+STORE, 93998211616768, 93998211629055, -+STORE, 93998227210240, 93998227615743, -+STORE, 140243029913600, 140243031572479, -+STORE, 140243031572480, 140243033669631, -+STORE, 140243033669632, 140243033686015, -+STORE, 140243033686016, 140243033694207, -+STORE, 140243033694208, 140243033710591, -+STORE, 140243033710592, 140243033722879, -+STORE, 140243033722880, 140243035815935, -+STORE, 140243035815936, 140243035820031, -+STORE, 140243035820032, 140243035824127, -+STORE, 140243035824128, 140243035967487, -+STORE, 140243036336128, 140243038019583, -+STORE, 140243038019584, 140243038035967, -+STORE, 140243038064640, 140243038068735, -+STORE, 140243038068736, 140243038072831, -+STORE, 140243038072832, 140243038076927, -+STORE, 140734976479232, 140734976618495, -+STORE, 140734977978368, 140734977990655, -+STORE, 140734977990656, 140734977994751, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722742775808, 140737488351231, -+SNULL, 140722742783999, 140737488351231, -+STORE, 140722742775808, 140722742783999, -+STORE, 140722742644736, 140722742783999, -+STORE, 93857673662464, 93857675997183, -+SNULL, 93857673875455, 93857675997183, -+STORE, 93857673662464, 93857673875455, -+STORE, 93857673875456, 93857675997183, -+ERASE, 93857673875456, 93857675997183, -+STORE, 93857675972608, 93857675984895, -+STORE, 93857675984896, 93857675997183, -+STORE, 140629677498368, 140629679751167, -+SNULL, 140629677641727, 140629679751167, -+STORE, 140629677498368, 140629677641727, -+STORE, 140629677641728, 140629679751167, -+ERASE, 140629677641728, 140629679751167, -+STORE, 140629679738880, 140629679747071, -+STORE, 140629679747072, 140629679751167, -+STORE, 140722743222272, 140722743226367, -+STORE, 140722743209984, 140722743222271, -+STORE, 140629679710208, 140629679738879, -+STORE, 140629679702016, 140629679710207, -+STORE, 140629675384832, 140629677498367, -+SNULL, 140629675384832, 140629675397119, -+STORE, 140629675397120, 140629677498367, -+STORE, 140629675384832, 140629675397119, -+SNULL, 140629677490175, 140629677498367, -+STORE, 140629675397120, 140629677490175, -+STORE, 140629677490176, 140629677498367, -+ERASE, 140629677490176, 140629677498367, -+STORE, 140629677490176, 140629677498367, -+STORE, 140629671587840, 140629675384831, -+SNULL, 140629671587840, 140629673246719, -+STORE, 140629673246720, 140629675384831, -+STORE, 140629671587840, 140629673246719, -+SNULL, 140629675343871, 140629675384831, -+STORE, 140629673246720, 140629675343871, -+STORE, 140629675343872, 140629675384831, -+SNULL, 140629675343872, 140629675368447, -+STORE, 140629675368448, 140629675384831, -+STORE, 140629675343872, 140629675368447, -+ERASE, 140629675343872, 140629675368447, -+STORE, 140629675343872, 140629675368447, -+ERASE, 140629675368448, 140629675384831, -+STORE, 140629675368448, 140629675384831, -+STORE, 140629679693824, 140629679710207, -+SNULL, 140629675360255, 140629675368447, -+STORE, 140629675343872, 140629675360255, -+STORE, 140629675360256, 140629675368447, -+SNULL, 140629677494271, 140629677498367, -+STORE, 140629677490176, 140629677494271, -+STORE, 140629677494272, 140629677498367, -+SNULL, 93857675976703, 93857675984895, -+STORE, 93857675972608, 93857675976703, -+STORE, 93857675976704, 93857675984895, -+SNULL, 140629679742975, 140629679747071, -+STORE, 140629679738880, 140629679742975, -+STORE, 140629679742976, 140629679747071, -+ERASE, 140629679710208, 140629679738879, -+STORE, 93857705832448, 93857705967615, -+STORE, 140629678010368, 140629679693823, -+STORE, 93857705832448, 93857706102783, -+STORE, 93857705832448, 93857706237951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735922421760, 140737488351231, -+SNULL, 140735922429951, 140737488351231, -+STORE, 140735922421760, 140735922429951, -+STORE, 140735922290688, 140735922429951, -+STORE, 94651136139264, 94651138363391, -+SNULL, 94651136249855, 94651138363391, -+STORE, 94651136139264, 94651136249855, -+STORE, 94651136249856, 94651138363391, -+ERASE, 94651136249856, 94651138363391, -+STORE, 94651138342912, 94651138355199, -+STORE, 94651138355200, 94651138363391, -+STORE, 140325788266496, 140325790519295, -+SNULL, 140325788409855, 140325790519295, -+STORE, 140325788266496, 140325788409855, -+STORE, 140325788409856, 140325790519295, -+ERASE, 140325788409856, 140325790519295, -+STORE, 140325790507008, 140325790515199, -+STORE, 140325790515200, 140325790519295, -+STORE, 140735923572736, 140735923576831, -+STORE, 140735923560448, 140735923572735, -+STORE, 140325790478336, 140325790507007, -+STORE, 140325790470144, 140325790478335, -+STORE, 140325784469504, 140325788266495, -+SNULL, 140325784469504, 140325786128383, -+STORE, 140325786128384, 140325788266495, -+STORE, 140325784469504, 140325786128383, -+SNULL, 140325788225535, 140325788266495, -+STORE, 140325786128384, 140325788225535, -+STORE, 140325788225536, 140325788266495, -+SNULL, 140325788225536, 140325788250111, -+STORE, 140325788250112, 140325788266495, -+STORE, 140325788225536, 140325788250111, -+ERASE, 140325788225536, 140325788250111, -+STORE, 140325788225536, 140325788250111, -+ERASE, 140325788250112, 140325788266495, -+STORE, 140325788250112, 140325788266495, -+SNULL, 140325788241919, 140325788250111, -+STORE, 140325788225536, 140325788241919, -+STORE, 140325788241920, 140325788250111, -+SNULL, 94651138351103, 94651138355199, -+STORE, 94651138342912, 94651138351103, -+STORE, 94651138351104, 94651138355199, -+SNULL, 140325790511103, 140325790515199, -+STORE, 140325790507008, 140325790511103, -+STORE, 140325790511104, 140325790515199, -+ERASE, 140325790478336, 140325790507007, -+STORE, 94651146297344, 94651146432511, -+STORE, 94212330168320, 94212330381311, -+STORE, 94212332478464, 94212332482559, -+STORE, 94212332482560, 94212332490751, -+STORE, 94212332490752, 94212332503039, -+STORE, 94212348891136, 94212349825023, -+STORE, 140611630604288, 140611632263167, -+STORE, 140611632263168, 140611634360319, -+STORE, 140611634360320, 140611634376703, -+STORE, 140611634376704, 140611634384895, -+STORE, 140611634384896, 140611634401279, -+STORE, 140611634401280, 140611634413567, -+STORE, 140611634413568, 140611636506623, -+STORE, 140611636506624, 140611636510719, -+STORE, 140611636510720, 140611636514815, -+STORE, 140611636514816, 140611636658175, -+STORE, 140611637026816, 140611638710271, -+STORE, 140611638710272, 140611638726655, -+STORE, 140611638755328, 140611638759423, -+STORE, 140611638759424, 140611638763519, -+STORE, 140611638763520, 140611638767615, -+STORE, 140726974533632, 140726974672895, -+STORE, 140726974943232, 140726974955519, -+STORE, 140726974955520, 140726974959615, -+STORE, 94572463521792, 94572463734783, -+STORE, 94572465831936, 94572465836031, -+STORE, 94572465836032, 94572465844223, -+STORE, 94572465844224, 94572465856511, -+STORE, 94572491534336, 94572492865535, -+STORE, 140644351492096, 140644353150975, -+STORE, 140644353150976, 140644355248127, -+STORE, 140644355248128, 140644355264511, -+STORE, 140644355264512, 140644355272703, -+STORE, 140644355272704, 140644355289087, -+STORE, 140644355289088, 140644355301375, -+STORE, 140644355301376, 140644357394431, -+STORE, 140644357394432, 140644357398527, -+STORE, 140644357398528, 140644357402623, -+STORE, 140644357402624, 140644357545983, -+STORE, 140644357914624, 140644359598079, -+STORE, 140644359598080, 140644359614463, -+STORE, 140644359643136, 140644359647231, -+STORE, 140644359647232, 140644359651327, -+STORE, 140644359651328, 140644359655423, -+STORE, 140727841824768, 140727841964031, -+STORE, 140727843188736, 140727843201023, -+STORE, 140727843201024, 140727843205119, -+STORE, 94144315457536, 94144315670527, -+STORE, 94144317767680, 94144317771775, -+STORE, 94144317771776, 94144317779967, -+STORE, 94144317779968, 94144317792255, -+STORE, 94144318369792, 94144320815103, -+STORE, 140316717645824, 140316719304703, -+STORE, 140316719304704, 140316721401855, -+STORE, 140316721401856, 140316721418239, -+STORE, 140316721418240, 140316721426431, -+STORE, 140316721426432, 140316721442815, -+STORE, 140316721442816, 140316721455103, -+STORE, 140316721455104, 140316723548159, -+STORE, 140316723548160, 140316723552255, -+STORE, 140316723552256, 140316723556351, -+STORE, 140316723556352, 140316723699711, -+STORE, 140316724068352, 140316725751807, -+STORE, 140316725751808, 140316725768191, -+STORE, 140316725796864, 140316725800959, -+STORE, 140316725800960, 140316725805055, -+STORE, 140316725805056, 140316725809151, -+STORE, 140725744283648, 140725744422911, -+STORE, 140725745852416, 140725745864703, -+STORE, 140725745864704, 140725745868799, -+STORE, 94646858846208, 94646859059199, -+STORE, 94646861156352, 94646861160447, -+STORE, 94646861160448, 94646861168639, -+STORE, 94646861168640, 94646861180927, -+STORE, 94646879805440, 94646881894399, -+STORE, 140435449745408, 140435451404287, -+STORE, 140435451404288, 140435453501439, -+STORE, 140435453501440, 140435453517823, -+STORE, 140435453517824, 140435453526015, -+STORE, 140435453526016, 140435453542399, -+STORE, 140435453542400, 140435453554687, -+STORE, 140435453554688, 140435455647743, -+STORE, 140435455647744, 140435455651839, -+STORE, 140435455651840, 140435455655935, -+STORE, 140435455655936, 140435455799295, -+STORE, 140435456167936, 140435457851391, -+STORE, 140435457851392, 140435457867775, -+STORE, 140435457896448, 140435457900543, -+STORE, 140435457900544, 140435457904639, -+STORE, 140435457904640, 140435457908735, -+STORE, 140721033818112, 140721033957375, -+STORE, 140721034018816, 140721034031103, -+STORE, 140721034031104, 140721034035199, -+STORE, 94872903438336, 94872903651327, -+STORE, 94872905748480, 94872905752575, -+STORE, 94872905752576, 94872905760767, -+STORE, 94872905760768, 94872905773055, -+STORE, 94872931246080, 94872931651583, -+STORE, 139771607810048, 139771609468927, -+STORE, 139771609468928, 139771611566079, -+STORE, 139771611566080, 139771611582463, -+STORE, 139771611582464, 139771611590655, -+STORE, 139771611590656, 139771611607039, -+STORE, 139771611607040, 139771611619327, -+STORE, 139771611619328, 139771613712383, -+STORE, 139771613712384, 139771613716479, -+STORE, 139771613716480, 139771613720575, -+STORE, 139771613720576, 139771613863935, -+STORE, 139771614232576, 139771615916031, -+STORE, 139771615916032, 139771615932415, -+STORE, 139771615961088, 139771615965183, -+STORE, 139771615965184, 139771615969279, -+STORE, 139771615969280, 139771615973375, -+STORE, 140725402931200, 140725403070463, -+STORE, 140725403852800, 140725403865087, -+STORE, 140725403865088, 140725403869183, -+STORE, 94740737736704, 94740737949695, -+STORE, 94740740046848, 94740740050943, -+STORE, 94740740050944, 94740740059135, -+STORE, 94740740059136, 94740740071423, -+STORE, 94740743249920, 94740744724479, -+STORE, 140640287010816, 140640288669695, -+STORE, 140640288669696, 140640290766847, -+STORE, 140640290766848, 140640290783231, -+STORE, 140640290783232, 140640290791423, -+STORE, 140640290791424, 140640290807807, -+STORE, 140640290807808, 140640290820095, -+STORE, 140640290820096, 140640292913151, -+STORE, 140640292913152, 140640292917247, -+STORE, 140640292917248, 140640292921343, -+STORE, 140640292921344, 140640293064703, -+STORE, 140640293433344, 140640295116799, -+STORE, 140640295116800, 140640295133183, -+STORE, 140640295161856, 140640295165951, -+STORE, 140640295165952, 140640295170047, -+STORE, 140640295170048, 140640295174143, -+STORE, 140725133303808, 140725133443071, -+STORE, 140725133684736, 140725133697023, -+STORE, 140725133697024, 140725133701119, -+STORE, 140737488347136, 140737488351231, -+STORE, 140722826371072, 140737488351231, -+SNULL, 140722826375167, 140737488351231, -+STORE, 140722826371072, 140722826375167, -+STORE, 140722826240000, 140722826375167, -+STORE, 94113818611712, 94113820835839, -+SNULL, 94113818722303, 94113820835839, -+STORE, 94113818611712, 94113818722303, -+STORE, 94113818722304, 94113820835839, -+ERASE, 94113818722304, 94113820835839, -+STORE, 94113820815360, 94113820827647, -+STORE, 94113820827648, 94113820835839, -+STORE, 139628194508800, 139628196761599, -+SNULL, 139628194652159, 139628196761599, -+STORE, 139628194508800, 139628194652159, -+STORE, 139628194652160, 139628196761599, -+ERASE, 139628194652160, 139628196761599, -+STORE, 139628196749312, 139628196757503, -+STORE, 139628196757504, 139628196761599, -+STORE, 140722826727424, 140722826731519, -+STORE, 140722826715136, 140722826727423, -+STORE, 139628196720640, 139628196749311, -+STORE, 139628196712448, 139628196720639, -+STORE, 139628190711808, 139628194508799, -+SNULL, 139628190711808, 139628192370687, -+STORE, 139628192370688, 139628194508799, -+STORE, 139628190711808, 139628192370687, -+SNULL, 139628194467839, 139628194508799, -+STORE, 139628192370688, 139628194467839, -+STORE, 139628194467840, 139628194508799, -+SNULL, 139628194467840, 139628194492415, -+STORE, 139628194492416, 139628194508799, -+STORE, 139628194467840, 139628194492415, -+ERASE, 139628194467840, 139628194492415, -+STORE, 139628194467840, 139628194492415, -+ERASE, 139628194492416, 139628194508799, -+STORE, 139628194492416, 139628194508799, -+SNULL, 139628194484223, 139628194492415, -+STORE, 139628194467840, 139628194484223, -+STORE, 139628194484224, 139628194492415, -+SNULL, 94113820823551, 94113820827647, -+STORE, 94113820815360, 94113820823551, -+STORE, 94113820823552, 94113820827647, -+SNULL, 139628196753407, 139628196757503, -+STORE, 139628196749312, 139628196753407, -+STORE, 139628196753408, 139628196757503, -+ERASE, 139628196720640, 139628196749311, -+STORE, 94113830850560, 94113830985727, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731865833472, 140737488351231, -+SNULL, 140731865841663, 140737488351231, -+STORE, 140731865833472, 140731865841663, -+STORE, 140731865702400, 140731865841663, -+STORE, 94763339386880, 94763341611007, -+SNULL, 94763339497471, 94763341611007, -+STORE, 94763339386880, 94763339497471, -+STORE, 94763339497472, 94763341611007, -+ERASE, 94763339497472, 94763341611007, -+STORE, 94763341590528, 94763341602815, -+STORE, 94763341602816, 94763341611007, -+STORE, 139778398486528, 139778400739327, -+SNULL, 139778398629887, 139778400739327, -+STORE, 139778398486528, 139778398629887, -+STORE, 139778398629888, 139778400739327, -+ERASE, 139778398629888, 139778400739327, -+STORE, 139778400727040, 139778400735231, -+STORE, 139778400735232, 139778400739327, -+STORE, 140731865858048, 140731865862143, -+STORE, 140731865845760, 140731865858047, -+STORE, 139778400698368, 139778400727039, -+STORE, 139778400690176, 139778400698367, -+STORE, 139778394689536, 139778398486527, -+SNULL, 139778394689536, 139778396348415, -+STORE, 139778396348416, 139778398486527, -+STORE, 139778394689536, 139778396348415, -+SNULL, 139778398445567, 139778398486527, -+STORE, 139778396348416, 139778398445567, -+STORE, 139778398445568, 139778398486527, -+SNULL, 139778398445568, 139778398470143, -+STORE, 139778398470144, 139778398486527, -+STORE, 139778398445568, 139778398470143, -+ERASE, 139778398445568, 139778398470143, -+STORE, 139778398445568, 139778398470143, -+ERASE, 139778398470144, 139778398486527, -+STORE, 139778398470144, 139778398486527, -+SNULL, 139778398461951, 139778398470143, -+STORE, 139778398445568, 139778398461951, -+STORE, 139778398461952, 139778398470143, -+SNULL, 94763341598719, 94763341602815, -+STORE, 94763341590528, 94763341598719, -+STORE, 94763341598720, 94763341602815, -+SNULL, 139778400731135, 139778400735231, -+STORE, 139778400727040, 139778400731135, -+STORE, 139778400731136, 139778400735231, -+ERASE, 139778400698368, 139778400727039, -+STORE, 94763362197504, 94763362332671, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140737488338944, 140737488351231, -+STORE, 140732053192704, 140737488351231, -+SNULL, 140732053204991, 140737488351231, -+STORE, 140732053192704, 140732053204991, -+STORE, 140732053061632, 140732053204991, -+STORE, 4194304, 26279935, -+STORE, 28372992, 28454911, -+STORE, 28454912, 29806591, -+STORE, 140176018599936, 140176020852735, -+SNULL, 140176018743295, 140176020852735, -+STORE, 140176018599936, 140176018743295, -+STORE, 140176018743296, 140176020852735, -+ERASE, 140176018743296, 140176020852735, -+STORE, 140176020840448, 140176020848639, -+STORE, 140176020848640, 140176020852735, -+STORE, 140732053381120, 140732053385215, -+STORE, 140732053368832, 140732053381119, -+STORE, 140176020811776, 140176020840447, -+STORE, 140176020803584, 140176020811775, -+STORE, 140176014766080, 140176018599935, -+SNULL, 140176014766080, 140176016474111, -+STORE, 140176016474112, 140176018599935, -+STORE, 140176014766080, 140176016474111, -+SNULL, 140176018567167, 140176018599935, -+STORE, 140176016474112, 140176018567167, -+STORE, 140176018567168, 140176018599935, -+ERASE, 140176018567168, 140176018599935, -+STORE, 140176018567168, 140176018599935, -+STORE, 140176012570624, 140176014766079, -+SNULL, 140176012570624, 140176012664831, -+STORE, 140176012664832, 140176014766079, -+STORE, 140176012570624, 140176012664831, -+SNULL, 140176014757887, 140176014766079, -+STORE, 140176012664832, 140176014757887, -+STORE, 140176014757888, 140176014766079, -+ERASE, 140176014757888, 140176014766079, -+STORE, 140176014757888, 140176014766079, -+STORE, 140176010051584, 140176012570623, -+SNULL, 140176010051584, 140176010465279, -+STORE, 140176010465280, 140176012570623, -+STORE, 140176010051584, 140176010465279, -+SNULL, 140176012558335, 140176012570623, -+STORE, 140176010465280, 140176012558335, -+STORE, 140176012558336, 140176012570623, -+ERASE, 140176012558336, 140176012570623, -+STORE, 140176012558336, 140176012570623, -+STORE, 140176007417856, 140176010051583, -+SNULL, 140176007417856, 140176007946239, -+STORE, 140176007946240, 140176010051583, -+STORE, 140176007417856, 140176007946239, -+SNULL, 140176010043391, 140176010051583, -+STORE, 140176007946240, 140176010043391, -+STORE, 140176010043392, 140176010051583, -+ERASE, 140176010043392, 140176010051583, -+STORE, 140176010043392, 140176010051583, -+STORE, 140176005304320, 140176007417855, -+SNULL, 140176005304320, 140176005316607, -+STORE, 140176005316608, 140176007417855, -+STORE, 140176005304320, 140176005316607, -+SNULL, 140176007409663, 140176007417855, -+STORE, 140176005316608, 140176007409663, -+STORE, 140176007409664, 140176007417855, -+ERASE, 140176007409664, 140176007417855, -+STORE, 140176007409664, 140176007417855, -+STORE, 140176003100672, 140176005304319, -+SNULL, 140176003100672, 140176003203071, -+STORE, 140176003203072, 140176005304319, -+STORE, 140176003100672, 140176003203071, -+SNULL, 140176005296127, 140176005304319, -+STORE, 140176003203072, 140176005296127, -+STORE, 140176005296128, 140176005304319, -+ERASE, 140176005296128, 140176005304319, -+STORE, 140176005296128, 140176005304319, -+STORE, 140176020795392, 140176020811775, -+STORE, 140175999938560, 140176003100671, -+SNULL, 140175999938560, 140176000999423, -+STORE, 140176000999424, 140176003100671, -+STORE, 140175999938560, 140176000999423, -+SNULL, 140176003092479, 140176003100671, -+STORE, 140176000999424, 140176003092479, -+STORE, 140176003092480, 140176003100671, -+ERASE, 140176003092480, 140176003100671, -+STORE, 140176003092480, 140176003100671, -+STORE, 140175996141568, 140175999938559, -+SNULL, 140175996141568, 140175997800447, -+STORE, 140175997800448, 140175999938559, -+STORE, 140175996141568, 140175997800447, -+SNULL, 140175999897599, 140175999938559, -+STORE, 140175997800448, 140175999897599, -+STORE, 140175999897600, 140175999938559, -+SNULL, 140175999897600, 140175999922175, -+STORE, 140175999922176, 140175999938559, -+STORE, 140175999897600, 140175999922175, -+ERASE, 140175999897600, 140175999922175, -+STORE, 140175999897600, 140175999922175, -+ERASE, 140175999922176, 140175999938559, -+STORE, 140175999922176, 140175999938559, -+STORE, 140176020783104, 140176020811775, -+SNULL, 140175999913983, 140175999922175, -+STORE, 140175999897600, 140175999913983, -+STORE, 140175999913984, 140175999922175, -+SNULL, 140176003096575, 140176003100671, -+STORE, 140176003092480, 140176003096575, -+STORE, 140176003096576, 140176003100671, -+SNULL, 140176005300223, 140176005304319, -+STORE, 140176005296128, 140176005300223, -+STORE, 140176005300224, 140176005304319, -+SNULL, 140176007413759, 140176007417855, -+STORE, 140176007409664, 140176007413759, -+STORE, 140176007413760, 140176007417855, -+SNULL, 140176010047487, 140176010051583, -+STORE, 140176010043392, 140176010047487, -+STORE, 140176010047488, 140176010051583, -+SNULL, 140176012566527, 140176012570623, -+STORE, 140176012558336, 140176012566527, -+STORE, 140176012566528, 140176012570623, -+SNULL, 140176014761983, 140176014766079, -+STORE, 140176014757888, 140176014761983, -+STORE, 140176014761984, 140176014766079, -+SNULL, 140176018571263, 140176018599935, -+STORE, 140176018567168, 140176018571263, -+STORE, 140176018571264, 140176018599935, -+SNULL, 28405759, 28454911, -+STORE, 28372992, 28405759, -+STORE, 28405760, 28454911, -+SNULL, 140176020844543, 140176020848639, -+STORE, 140176020840448, 140176020844543, -+STORE, 140176020844544, 140176020848639, -+ERASE, 140176020811776, 140176020840447, -+STORE, 53080064, 53215231, -+STORE, 140176019099648, 140176020783103, -+STORE, 140176020836352, 140176020840447, -+STORE, 140176018964480, 140176019099647, -+STORE, 53080064, 53358591, -+STORE, 140175994044416, 140175996141567, -+STORE, 140176020828160, 140176020840447, -+STORE, 140176020819968, 140176020840447, -+STORE, 140176020783104, 140176020819967, -+STORE, 140176018948096, 140176019099647, -+STORE, 53080064, 53493759, -+STORE, 53080064, 53649407, -+STORE, 140176018939904, 140176019099647, -+STORE, 140176018931712, 140176019099647, -+STORE, 53080064, 53784575, -+STORE, 53080064, 53919743, -+STORE, 140176018915328, 140176019099647, -+STORE, 140176018907136, 140176019099647, -+STORE, 53080064, 54059007, -+STORE, 140175993769984, 140175996141567, -+STORE, 140176018747392, 140176019099647, -+STORE, 53080064, 54198271, -+SNULL, 54190079, 54198271, -+STORE, 53080064, 54190079, -+STORE, 54190080, 54198271, -+ERASE, 54190080, 54198271, -+SNULL, 54181887, 54190079, -+STORE, 53080064, 54181887, -+STORE, 54181888, 54190079, -+ERASE, 54181888, 54190079, -+SNULL, 54173695, 54181887, -+STORE, 53080064, 54173695, -+STORE, 54173696, 54181887, -+ERASE, 54173696, 54181887, -+SNULL, 54165503, 54173695, -+STORE, 53080064, 54165503, -+STORE, 54165504, 54173695, -+ERASE, 54165504, 54173695, -+STORE, 140175993753600, 140175996141567, -+STORE, 140175993688064, 140175996141567, -+STORE, 140175993655296, 140175996141567, -+STORE, 140175991558144, 140175996141567, -+STORE, 140175991492608, 140175996141567, -+STORE, 53080064, 54312959, -+STORE, 140175991361536, 140175996141567, -+STORE, 140175991099392, 140175996141567, -+STORE, 140175991091200, 140175996141567, -+STORE, 140175991074816, 140175996141567, -+STORE, 140175991066624, 140175996141567, -+STORE, 140175991058432, 140175996141567, -+STORE, 53080064, 54448127, -+SNULL, 54439935, 54448127, -+STORE, 53080064, 54439935, -+STORE, 54439936, 54448127, -+ERASE, 54439936, 54448127, -+SNULL, 54431743, 54439935, -+STORE, 53080064, 54431743, -+STORE, 54431744, 54439935, -+ERASE, 54431744, 54439935, -+SNULL, 54419455, 54431743, -+STORE, 53080064, 54419455, -+STORE, 54419456, 54431743, -+ERASE, 54419456, 54431743, -+SNULL, 54403071, 54419455, -+STORE, 53080064, 54403071, -+STORE, 54403072, 54419455, -+ERASE, 54403072, 54419455, -+STORE, 140175991042048, 140175996141567, -+STORE, 53080064, 54538239, -+SNULL, 54534143, 54538239, -+STORE, 53080064, 54534143, -+STORE, 54534144, 54538239, -+ERASE, 54534144, 54538239, -+SNULL, 54530047, 54534143, -+STORE, 53080064, 54530047, -+STORE, 54530048, 54534143, -+ERASE, 54530048, 54534143, -+SNULL, 54525951, 54530047, -+STORE, 53080064, 54525951, -+STORE, 54525952, 54530047, -+ERASE, 54525952, 54530047, -+SNULL, 54521855, 54525951, -+STORE, 53080064, 54521855, -+STORE, 54521856, 54525951, -+ERASE, 54521856, 54525951, -+SNULL, 54517759, 54521855, -+STORE, 53080064, 54517759, -+STORE, 54517760, 54521855, -+ERASE, 54517760, 54521855, -+SNULL, 54513663, 54517759, -+STORE, 53080064, 54513663, -+STORE, 54513664, 54517759, -+ERASE, 54513664, 54517759, -+SNULL, 54509567, 54513663, -+STORE, 53080064, 54509567, -+STORE, 54509568, 54513663, -+ERASE, 54509568, 54513663, -+STORE, 140175991025664, 140175996141567, -+STORE, 140175990992896, 140175996141567, -+STORE, 53080064, 54644735, -+SNULL, 54628351, 54644735, -+STORE, 53080064, 54628351, -+STORE, 54628352, 54644735, -+ERASE, 54628352, 54644735, -+SNULL, 54616063, 54628351, -+STORE, 53080064, 54616063, -+STORE, 54616064, 54628351, -+ERASE, 54616064, 54628351, -+STORE, 140175988895744, 140175996141567, -+STORE, 53080064, 54767615, -+STORE, 140175988879360, 140175996141567, -+STORE, 140175988617216, 140175996141567, -+STORE, 140175988609024, 140175996141567, -+STORE, 140175988600832, 140175996141567, -+STORE, 53080064, 54906879, -+SNULL, 54898687, 54906879, -+STORE, 53080064, 54898687, -+STORE, 54898688, 54906879, -+ERASE, 54898688, 54906879, -+SNULL, 54853631, 54898687, -+STORE, 53080064, 54853631, -+STORE, 54853632, 54898687, -+ERASE, 54853632, 54898687, -+STORE, 140175986503680, 140175996141567, -+STORE, 53080064, 54996991, -+STORE, 140175986495488, 140175996141567, -+STORE, 140175986487296, 140175996141567, -+STORE, 140175985438720, 140175996141567, -+STORE, 53080064, 55136255, -+STORE, 140175985405952, 140175996141567, -+STORE, 140175985139712, 140175996141567, -+SNULL, 140176018964479, 140176019099647, -+STORE, 140176018747392, 140176018964479, -+STORE, 140176018964480, 140176019099647, -+ERASE, 140176018964480, 140176019099647, -+STORE, 140175983042560, 140175996141567, -+STORE, 140175982518272, 140175996141567, -+STORE, 140175980421120, 140175996141567, -+STORE, 53080064, 55287807, -+STORE, 53080064, 55427071, -+STORE, 140176019091456, 140176019099647, -+STORE, 140176019083264, 140176019099647, -+STORE, 140176019075072, 140176019099647, -+STORE, 140176019066880, 140176019099647, -+STORE, 140176019058688, 140176019099647, -+STORE, 140175980158976, 140175996141567, -+STORE, 140176019050496, 140176019099647, -+STORE, 140176019042304, 140176019099647, -+STORE, 140176019034112, 140176019099647, -+STORE, 140176019025920, 140176019099647, -+STORE, 140176019017728, 140176019099647, -+STORE, 140176019009536, 140176019099647, -+STORE, 140176019001344, 140176019099647, -+STORE, 140176018993152, 140176019099647, -+STORE, 140176018984960, 140176019099647, -+STORE, 140176018976768, 140176019099647, -+STORE, 140176018968576, 140176019099647, -+STORE, 140175978061824, 140175996141567, -+STORE, 53080064, 55603199, -+STORE, 140175978029056, 140175996141567, -+STORE, 140175977996288, 140175996141567, -+STORE, 53080064, 55738367, -+STORE, 53080064, 55881727, -+STORE, 140175977963520, 140175996141567, -+STORE, 140175977930752, 140175996141567, -+STORE, 53080064, 56041471, -+STORE, 140175977897984, 140175996141567, -+STORE, 140175977865216, 140175996141567, -+SNULL, 55881727, 56041471, -+STORE, 53080064, 55881727, -+STORE, 55881728, 56041471, -+ERASE, 55881728, 56041471, -+SNULL, 55721983, 55881727, -+STORE, 53080064, 55721983, -+STORE, 55721984, 55881727, -+ERASE, 55721984, 55881727, -+SNULL, 55570431, 55721983, -+STORE, 53080064, 55570431, -+STORE, 55570432, 55721983, -+ERASE, 55570432, 55721983, -+STORE, 140175977857024, 140175996141567, -+STORE, 140175975759872, 140175996141567, -+STORE, 53080064, 55754751, -+STORE, 53080064, 55943167, -+STORE, 140175975751680, 140175996141567, -+STORE, 140175975743488, 140175996141567, -+STORE, 140175975735296, 140175996141567, -+STORE, 140175975727104, 140175996141567, -+STORE, 140175975718912, 140175996141567, -+STORE, 140175975710720, 140175996141567, -+STORE, 140175975702528, 140175996141567, -+STORE, 140175975694336, 140175996141567, -+STORE, 140175975686144, 140175996141567, -+STORE, 140175975677952, 140175996141567, -+STORE, 140175975669760, 140175996141567, -+STORE, 140175974621184, 140175996141567, -+STORE, 140175974612992, 140175996141567, -+STORE, 53080064, 56139775, -+STORE, 140175972515840, 140175996141567, -+STORE, 53080064, 56401919, -+STORE, 140175970418688, 140175996141567, -+STORE, 140175970410496, 140175996141567, -+STORE, 140175970402304, 140175996141567, -+STORE, 140175970394112, 140175996141567, -+STORE, 53080064, 56569855, -+STORE, 140175969865728, 140175996141567, -+SNULL, 140175985139711, 140175996141567, -+STORE, 140175969865728, 140175985139711, -+STORE, 140175985139712, 140175996141567, -+SNULL, 140175985139712, 140175985405951, -+STORE, 140175985405952, 140175996141567, -+STORE, 140175985139712, 140175985405951, -+ERASE, 140175985139712, 140175985405951, -+STORE, 140175965671424, 140175985139711, -+STORE, 140175985397760, 140175996141567, -+STORE, 140175985389568, 140175996141567, -+STORE, 140175985381376, 140175996141567, -+STORE, 140175985373184, 140175996141567, -+STORE, 140175985364992, 140175996141567, -+STORE, 140175985356800, 140175996141567, -+STORE, 140175985348608, 140175996141567, -+STORE, 140175985340416, 140175996141567, -+STORE, 140175985332224, 140175996141567, -+STORE, 140175985324032, 140175996141567, -+STORE, 140175985315840, 140175996141567, -+STORE, 140175985307648, 140175996141567, -+STORE, 140175985299456, 140175996141567, -+STORE, 140175985291264, 140175996141567, -+STORE, 140175985283072, 140175996141567, -+STORE, 140175985274880, 140175996141567, -+STORE, 140175963574272, 140175985139711, -+STORE, 140175985266688, 140175996141567, -+STORE, 140175961477120, 140175985139711, -+STORE, 53080064, 56831999, -+STORE, 140175959379968, 140175985139711, -+STORE, 140175985258496, 140175996141567, -+STORE, 140175957282816, 140175985139711, -+STORE, 140175985250304, 140175996141567, -+STORE, 140175985242112, 140175996141567, -+STORE, 140175985233920, 140175996141567, -+STORE, 140175985225728, 140175996141567, -+STORE, 140175985217536, 140175996141567, -+STORE, 140175957151744, 140175985139711, -+STORE, 140175956627456, 140175985139711, -+SNULL, 140175980158975, 140175985139711, -+STORE, 140175956627456, 140175980158975, -+STORE, 140175980158976, 140175985139711, -+SNULL, 140175980158976, 140175980421119, -+STORE, 140175980421120, 140175985139711, -+STORE, 140175980158976, 140175980421119, -+ERASE, 140175980158976, 140175980421119, -+STORE, 140175954530304, 140175980158975, -+STORE, 140175985209344, 140175996141567, -+STORE, 53080064, 57094143, -+STORE, 140175952433152, 140175980158975, -+STORE, 140175985192960, 140175996141567, -+STORE, 140175985184768, 140175996141567, -+STORE, 140175985176576, 140175996141567, -+STORE, 140175985168384, 140175996141567, -+STORE, 140175985160192, 140175996141567, -+STORE, 140175985152000, 140175996141567, -+STORE, 140175985143808, 140175996141567, -+STORE, 140175980412928, 140175985139711, -+STORE, 140175980404736, 140175985139711, -+STORE, 140175980396544, 140175985139711, -+STORE, 140175980388352, 140175985139711, -+STORE, 140175980380160, 140175985139711, -+STORE, 140175980371968, 140175985139711, -+STORE, 140175980363776, 140175985139711, -+STORE, 140175980355584, 140175985139711, -+STORE, 140175980347392, 140175985139711, -+STORE, 140175980339200, 140175985139711, -+STORE, 53080064, 57356287, -+SNULL, 140176018747392, 140176018907135, -+STORE, 140176018907136, 140176018964479, -+STORE, 140176018747392, 140176018907135, -+ERASE, 140176018747392, 140176018907135, -+STORE, 140175952146432, 140175980158975, -+STORE, 140175950049280, 140175980158975, -+SNULL, 140175952146431, 140175980158975, -+STORE, 140175950049280, 140175952146431, -+STORE, 140175952146432, 140175980158975, -+SNULL, 140175952146432, 140175952433151, -+STORE, 140175952433152, 140175980158975, -+STORE, 140175952146432, 140175952433151, -+ERASE, 140175952146432, 140175952433151, -+STORE, 140176018898944, 140176018964479, -+STORE, 53080064, 57749503, -+STORE, 140175949520896, 140175952146431, -+STORE, 140175947423744, 140175952146431, -+SNULL, 140175993769983, 140175996141567, -+STORE, 140175985143808, 140175993769983, -+STORE, 140175993769984, 140175996141567, -+SNULL, 140175993769984, 140175994044415, -+STORE, 140175994044416, 140175996141567, -+STORE, 140175993769984, 140175994044415, -+ERASE, 140175993769984, 140175994044415, -+STORE, 140176018890752, 140176018964479, -+STORE, 140176018882560, 140176018964479, -+STORE, 140176018874368, 140176018964479, -+STORE, 140176018866176, 140176018964479, -+STORE, 140176018849792, 140176018964479, -+STORE, 140176018841600, 140176018964479, -+STORE, 140176018825216, 140176018964479, -+STORE, 140176018817024, 140176018964479, -+STORE, 140176018800640, 140176018964479, -+STORE, 140176018792448, 140176018964479, -+STORE, 140176018759680, 140176018964479, -+STORE, 140176018751488, 140176018964479, -+STORE, 140175994028032, 140175996141567, -+STORE, 140176018743296, 140176018964479, -+STORE, 140175994011648, 140175996141567, -+STORE, 140175994003456, 140175996141567, -+STORE, 140175993987072, 140175996141567, -+STORE, 140175993978880, 140175996141567, -+STORE, 140175993946112, 140175996141567, -+STORE, 140175993937920, 140175996141567, -+STORE, 140175993921536, 140175996141567, -+STORE, 140175993913344, 140175996141567, -+STORE, 140175993896960, 140175996141567, -+STORE, 140175993888768, 140175996141567, -+STORE, 140175993872384, 140175996141567, -+STORE, 140175993864192, 140175996141567, -+STORE, 140175993831424, 140175996141567, -+STORE, 140175993823232, 140175996141567, -+STORE, 140175993806848, 140175996141567, -+STORE, 140175993798656, 140175996141567, -+STORE, 140175993782272, 140175996141567, -+STORE, 140175993774080, 140175996141567, -+STORE, 140175980322816, 140175985139711, -+STORE, 140175980314624, 140175985139711, -+STORE, 140175980281856, 140175985139711, -+STORE, 140175980273664, 140175985139711, -+STORE, 140175980257280, 140175985139711, -+STORE, 140175945326592, 140175952146431, -+STORE, 140175980249088, 140175985139711, -+STORE, 140175980232704, 140175985139711, -+STORE, 140175980224512, 140175985139711, -+STORE, 140175980208128, 140175985139711, -+STORE, 140175980199936, 140175985139711, -+STORE, 140175980167168, 140175985139711, -+STORE, 140175952433152, 140175985139711, -+STORE, 140175952416768, 140175985139711, -+STORE, 140175952408576, 140175985139711, -+STORE, 140175952392192, 140175985139711, -+STORE, 140175952384000, 140175985139711, -+STORE, 140175952367616, 140175985139711, -+STORE, 140175943229440, 140175952146431, -+STORE, 140175952359424, 140175985139711, -+STORE, 140175952326656, 140175985139711, -+STORE, 140175952318464, 140175985139711, -+STORE, 140175952302080, 140175985139711, -+STORE, 140175952293888, 140175985139711, -+STORE, 140175952277504, 140175985139711, -+STORE, 140175952269312, 140175985139711, -+STORE, 140175952252928, 140175985139711, -+STORE, 140175952244736, 140175985139711, -+STORE, 140175952211968, 140175985139711, -+STORE, 140175952203776, 140175985139711, -+STORE, 140175952187392, 140175985139711, -+STORE, 140175952179200, 140175985139711, -+STORE, 140175952162816, 140175985139711, -+STORE, 140175952154624, 140175985139711, -+STORE, 140175943213056, 140175952146431, -+STORE, 140175943213056, 140175985139711, -+STORE, 140175943180288, 140175985139711, -+STORE, 140175943172096, 140175985139711, -+STORE, 140175943155712, 140175985139711, -+STORE, 140175943147520, 140175985139711, -+STORE, 140175943131136, 140175985139711, -+STORE, 140175943122944, 140175985139711, -+STORE, 140175943106560, 140175985139711, -+STORE, 140175943098368, 140175985139711, -+STORE, 140175943065600, 140175985139711, -+STORE, 140175943057408, 140175985139711, -+STORE, 140175943041024, 140175985139711, -+STORE, 140175943032832, 140175985139711, -+STORE, 140175943016448, 140175985139711, -+STORE, 140175943008256, 140175985139711, -+STORE, 140175942991872, 140175985139711, -+STORE, 140175942983680, 140175985139711, -+STORE, 140175942950912, 140175985139711, -+STORE, 140175942942720, 140175985139711, -+STORE, 140175942926336, 140175985139711, -+STORE, 140175942918144, 140175985139711, -+STORE, 140175942901760, 140175985139711, -+STORE, 140175942893568, 140175985139711, -+STORE, 140175942877184, 140175985139711, -+STORE, 140175942868992, 140175985139711, -+STORE, 140175942836224, 140175985139711, -+STORE, 140175942828032, 140175985139711, -+STORE, 140175942811648, 140175985139711, -+STORE, 140175942803456, 140175985139711, -+STORE, 140175942787072, 140175985139711, -+STORE, 140175942778880, 140175985139711, -+STORE, 140175942762496, 140175985139711, -+STORE, 140175942754304, 140175985139711, -+STORE, 140175942721536, 140175985139711, -+STORE, 140175942713344, 140175985139711, -+STORE, 140175942696960, 140175985139711, -+STORE, 140175942688768, 140175985139711, -+STORE, 140175942672384, 140175985139711, -+STORE, 140175942664192, 140175985139711, -+STORE, 140175942647808, 140175985139711, -+STORE, 140175942639616, 140175985139711, -+STORE, 140175942606848, 140175985139711, -+STORE, 140175942598656, 140175985139711, -+STORE, 140175942582272, 140175985139711, -+STORE, 140175942574080, 140175985139711, -+STORE, 140175942557696, 140175985139711, -+STORE, 140175942549504, 140175985139711, -+STORE, 140175942533120, 140175985139711, -+STORE, 140175942524928, 140175985139711, -+STORE, 140175942492160, 140175985139711, -+STORE, 140175942483968, 140175985139711, -+STORE, 140175942467584, 140175985139711, -+STORE, 140175942459392, 140175985139711, -+STORE, 140175942443008, 140175985139711, -+STORE, 140175942434816, 140175985139711, -+STORE, 140175942418432, 140175985139711, -+STORE, 140175942410240, 140175985139711, -+STORE, 140175942377472, 140175985139711, -+STORE, 140175942369280, 140175985139711, -+STORE, 140175942352896, 140175985139711, -+STORE, 140175942344704, 140175985139711, -+STORE, 140175942328320, 140175985139711, -+STORE, 140175942320128, 140175985139711, -+STORE, 140175942303744, 140175985139711, -+STORE, 140175942295552, 140175985139711, -+STORE, 140175942262784, 140175985139711, -+STORE, 140175942254592, 140175985139711, -+STORE, 140175942238208, 140175985139711, -+STORE, 140175942230016, 140175985139711, -+STORE, 140175942213632, 140175985139711, -+STORE, 140175942205440, 140175985139711, -+STORE, 140175942189056, 140175985139711, -+STORE, 140175942180864, 140175985139711, -+STORE, 140175942148096, 140175985139711, -+STORE, 140175942139904, 140175985139711, -+STORE, 140175942123520, 140175985139711, -+STORE, 140175942115328, 140175985139711, -+STORE, 140175942098944, 140175985139711, -+STORE, 140175942090752, 140175985139711, -+STORE, 140175942074368, 140175985139711, -+STORE, 140175942066176, 140175985139711, -+STORE, 140175942033408, 140175985139711, -+STORE, 140175942025216, 140175985139711, -+STORE, 140175942008832, 140175985139711, -+STORE, 140175942000640, 140175985139711, -+STORE, 140175941984256, 140175985139711, -+STORE, 140175941976064, 140175985139711, -+STORE, 140175941959680, 140175985139711, -+STORE, 140175939862528, 140175985139711, -+STORE, 140175939854336, 140175985139711, -+STORE, 140175939821568, 140175985139711, -+STORE, 140175939813376, 140175985139711, -+STORE, 140175939796992, 140175985139711, -+STORE, 140175939788800, 140175985139711, -+STORE, 140175939772416, 140175985139711, -+STORE, 140175939764224, 140175985139711, -+STORE, 140175939747840, 140175985139711, -+STORE, 140175939739648, 140175985139711, -+STORE, 140175939706880, 140175985139711, -+STORE, 140175939698688, 140175985139711, -+STORE, 140175939682304, 140175985139711, -+STORE, 140175939674112, 140175985139711, -+STORE, 140175939657728, 140175985139711, -+STORE, 140175939649536, 140175985139711, -+STORE, 140175939633152, 140175985139711, -+STORE, 140175939624960, 140175985139711, -+STORE, 140175939592192, 140175985139711, -+STORE, 140175939584000, 140175985139711, -+STORE, 140175939567616, 140175985139711, -+STORE, 140175939559424, 140175985139711, -+STORE, 140175939543040, 140175985139711, -+STORE, 140175939534848, 140175985139711, -+STORE, 140175939518464, 140175985139711, -+STORE, 140175939510272, 140175985139711, -+STORE, 140175939477504, 140175985139711, -+STORE, 140175939469312, 140175985139711, -+STORE, 140175939452928, 140175985139711, -+STORE, 140175939444736, 140175985139711, -+STORE, 140175939428352, 140175985139711, -+STORE, 140175939420160, 140175985139711, -+STORE, 140175939403776, 140175985139711, -+STORE, 140175939395584, 140175985139711, -+STORE, 140175939362816, 140175985139711, -+STORE, 140175939354624, 140175985139711, -+STORE, 140175939338240, 140175985139711, -+STORE, 140175939330048, 140175985139711, -+STORE, 140175939313664, 140175985139711, -+STORE, 140175939305472, 140175985139711, -+STORE, 140175939289088, 140175985139711, -+STORE, 140175939280896, 140175985139711, -+STORE, 140175939248128, 140175985139711, -+STORE, 140175939239936, 140175985139711, -+STORE, 140175939223552, 140175985139711, -+STORE, 140175939215360, 140175985139711, -+STORE, 140175939198976, 140175985139711, -+STORE, 140175939190784, 140175985139711, -+STORE, 140175939174400, 140175985139711, -+STORE, 140175939166208, 140175985139711, -+STORE, 140175939133440, 140175985139711, -+STORE, 140175939125248, 140175985139711, -+STORE, 140175939108864, 140175985139711, -+STORE, 140175939100672, 140175985139711, -+STORE, 140175939084288, 140175985139711, -+STORE, 140175939076096, 140175985139711, -+STORE, 140175939059712, 140175985139711, -+STORE, 140175939051520, 140175985139711, -+STORE, 140175939018752, 140175985139711, -+STORE, 140175939010560, 140175985139711, -+STORE, 140175938994176, 140175985139711, -+STORE, 140175938985984, 140175985139711, -+STORE, 140175938969600, 140175985139711, -+STORE, 140175938961408, 140175985139711, -+STORE, 140175938945024, 140175985139711, -+STORE, 140175938936832, 140175985139711, -+STORE, 140175938904064, 140175985139711, -+STORE, 140175938895872, 140175985139711, -+STORE, 140175938879488, 140175985139711, -+STORE, 140175938871296, 140175985139711, -+STORE, 140175938854912, 140175985139711, -+STORE, 140175938846720, 140175985139711, -+STORE, 140175938830336, 140175985139711, -+STORE, 140175938822144, 140175985139711, -+STORE, 140175938789376, 140175985139711, -+STORE, 140175938781184, 140175985139711, -+STORE, 140175938764800, 140175985139711, -+STORE, 140175938756608, 140175985139711, -+STORE, 140175938740224, 140175985139711, -+STORE, 140175938732032, 140175985139711, -+STORE, 140175938715648, 140175985139711, -+STORE, 140175938707456, 140175985139711, -+STORE, 140175938674688, 140175985139711, -+STORE, 140175938666496, 140175985139711, -+STORE, 140175938650112, 140175985139711, -+STORE, 140175938641920, 140175985139711, -+STORE, 140175938625536, 140175985139711, -+STORE, 140175938617344, 140175985139711, -+STORE, 140175938600960, 140175985139711, -+STORE, 140175938592768, 140175985139711, -+STORE, 140175938560000, 140175985139711, -+STORE, 140175938551808, 140175985139711, -+STORE, 140175938535424, 140175985139711, -+STORE, 140175938527232, 140175985139711, -+STORE, 140175938510848, 140175985139711, -+STORE, 140175938502656, 140175985139711, -+STORE, 140175938486272, 140175985139711, -+STORE, 140175938478080, 140175985139711, -+STORE, 140175938445312, 140175985139711, -+STORE, 140175938437120, 140175985139711, -+STORE, 140175938420736, 140175985139711, -+STORE, 140175938412544, 140175985139711, -+STORE, 140175938396160, 140175985139711, -+STORE, 140175938387968, 140175985139711, -+STORE, 140175938371584, 140175985139711, -+STORE, 140175938363392, 140175985139711, -+STORE, 140175938330624, 140175985139711, -+STORE, 140175938322432, 140175985139711, -+STORE, 140175938306048, 140175985139711, -+STORE, 140175938297856, 140175985139711, -+STORE, 140175938281472, 140175985139711, -+STORE, 140175938273280, 140175985139711, -+STORE, 140175938256896, 140175985139711, -+STORE, 140175938248704, 140175985139711, -+STORE, 140175938215936, 140175985139711, -+STORE, 140175938207744, 140175985139711, -+STORE, 140175938191360, 140175985139711, -+STORE, 140175938183168, 140175985139711, -+STORE, 140175938166784, 140175985139711, -+STORE, 140175938158592, 140175985139711, -+STORE, 140175938142208, 140175985139711, -+STORE, 140175936045056, 140175985139711, -+STORE, 140175936036864, 140175985139711, -+STORE, 140175936004096, 140175985139711, -+STORE, 140175935995904, 140175985139711, -+STORE, 140175935979520, 140175985139711, -+STORE, 140175935971328, 140175985139711, -+STORE, 140175935954944, 140175985139711, -+STORE, 140175935946752, 140175985139711, -+STORE, 140175935930368, 140175985139711, -+STORE, 140175935922176, 140175985139711, -+STORE, 140175935889408, 140175985139711, -+STORE, 140175935881216, 140175985139711, -+STORE, 140175935864832, 140175985139711, -+STORE, 140175935856640, 140175985139711, -+STORE, 140175935840256, 140175985139711, -+STORE, 140175935832064, 140175985139711, -+STORE, 140175935815680, 140175985139711, -+STORE, 140175935807488, 140175985139711, -+STORE, 140175935774720, 140175985139711, -+STORE, 140175935766528, 140175985139711, -+STORE, 140175935750144, 140175985139711, -+STORE, 140175935741952, 140175985139711, -+STORE, 140175935725568, 140175985139711, -+STORE, 140175935717376, 140175985139711, -+STORE, 140175935700992, 140175985139711, -+STORE, 140175935692800, 140175985139711, -+STORE, 140175935660032, 140175985139711, -+STORE, 140175935651840, 140175985139711, -+STORE, 140175935635456, 140175985139711, -+STORE, 140175935627264, 140175985139711, -+STORE, 140175935610880, 140175985139711, -+STORE, 140175935602688, 140175985139711, -+STORE, 140175935586304, 140175985139711, -+STORE, 140175935578112, 140175985139711, -+STORE, 140175935545344, 140175985139711, -+STORE, 140175935537152, 140175985139711, -+STORE, 140175935520768, 140175985139711, -+STORE, 140175935512576, 140175985139711, -+STORE, 140175935496192, 140175985139711, -+STORE, 140175935488000, 140175985139711, -+STORE, 140175935471616, 140175985139711, -+STORE, 140175935463424, 140175985139711, -+STORE, 140175935430656, 140175985139711, -+STORE, 140175935422464, 140175985139711, -+STORE, 140175935406080, 140175985139711, -+STORE, 140175935397888, 140175985139711, -+STORE, 140175935381504, 140175985139711, -+STORE, 140175935373312, 140175985139711, -+STORE, 140175935356928, 140175985139711, -+STORE, 140175935348736, 140175985139711, -+STORE, 140175935315968, 140175985139711, -+STORE, 140175935307776, 140175985139711, -+STORE, 140175935291392, 140175985139711, -+STORE, 140175935283200, 140175985139711, -+STORE, 140175935266816, 140175985139711, -+STORE, 140175935258624, 140175985139711, -+STORE, 140175935242240, 140175985139711, -+STORE, 140175935234048, 140175985139711, -+STORE, 140175935201280, 140175985139711, -+STORE, 140175935193088, 140175985139711, -+STORE, 140175935176704, 140175985139711, -+STORE, 140175935168512, 140175985139711, -+STORE, 140175935152128, 140175985139711, -+STORE, 140175935143936, 140175985139711, -+STORE, 140175935127552, 140175985139711, -+STORE, 140175935119360, 140175985139711, -+STORE, 140175935086592, 140175985139711, -+STORE, 140175935078400, 140175985139711, -+STORE, 140175935062016, 140175985139711, -+STORE, 140175935053824, 140175985139711, -+STORE, 140175935037440, 140175985139711, -+STORE, 140175935029248, 140175985139711, -+STORE, 140175935012864, 140175985139711, -+STORE, 140175935004672, 140175985139711, -+STORE, 140175934971904, 140175985139711, -+STORE, 140175934963712, 140175985139711, -+STORE, 140175934947328, 140175985139711, -+STORE, 140175934939136, 140175985139711, -+STORE, 140175934922752, 140175985139711, -+STORE, 140175934914560, 140175985139711, -+STORE, 140175934898176, 140175985139711, -+STORE, 140175934889984, 140175985139711, -+STORE, 140175934857216, 140175985139711, -+STORE, 140175934849024, 140175985139711, -+STORE, 140175934832640, 140175985139711, -+STORE, 140175934824448, 140175985139711, -+STORE, 140175934808064, 140175985139711, -+STORE, 140175934799872, 140175985139711, -+STORE, 140175934783488, 140175985139711, -+STORE, 140175934775296, 140175985139711, -+STORE, 140175934742528, 140175985139711, -+STORE, 140175934734336, 140175985139711, -+STORE, 140175934717952, 140175985139711, -+STORE, 140175934709760, 140175985139711, -+STORE, 140175934693376, 140175985139711, -+STORE, 140175934685184, 140175985139711, -+STORE, 140175934668800, 140175985139711, -+STORE, 140175934660608, 140175985139711, -+STORE, 140175934627840, 140175985139711, -+STORE, 140175934619648, 140175985139711, -+STORE, 140175934603264, 140175985139711, -+STORE, 140175934595072, 140175985139711, -+STORE, 140175934578688, 140175985139711, -+STORE, 140175934570496, 140175985139711, -+STORE, 140175934554112, 140175985139711, -+STORE, 140175934545920, 140175985139711, -+STORE, 140175934513152, 140175985139711, -+STORE, 140175934504960, 140175985139711, -+STORE, 140175934488576, 140175985139711, -+STORE, 140175934480384, 140175985139711, -+STORE, 140175934464000, 140175985139711, -+STORE, 140175934455808, 140175985139711, -+STORE, 140175934439424, 140175985139711, -+STORE, 140175934431232, 140175985139711, -+STORE, 140175934398464, 140175985139711, -+STORE, 140175934390272, 140175985139711, -+STORE, 140175934373888, 140175985139711, -+STORE, 140175934365696, 140175985139711, -+STORE, 140175934349312, 140175985139711, -+STORE, 140175934341120, 140175985139711, -+STORE, 140175934324736, 140175985139711, -+STORE, 140175932227584, 140175985139711, -+STORE, 140175932219392, 140175985139711, -+STORE, 140175932186624, 140175985139711, -+STORE, 140175932178432, 140175985139711, -+STORE, 140175932162048, 140175985139711, -+STORE, 140175932153856, 140175985139711, -+STORE, 140175932137472, 140175985139711, -+STORE, 53080064, 57884671, -+STORE, 140175932129280, 140175985139711, -+STORE, 140175932112896, 140175985139711, -+STORE, 140175932104704, 140175985139711, -+STORE, 140175932071936, 140175985139711, -+STORE, 140175932063744, 140175985139711, -+STORE, 140175932047360, 140175985139711, -+STORE, 140175932039168, 140175985139711, -+STORE, 140175932022784, 140175985139711, -+STORE, 140175932014592, 140175985139711, -+STORE, 140175931998208, 140175985139711, -+STORE, 140175931990016, 140175985139711, -+STORE, 140175931957248, 140175985139711, -+STORE, 140175931949056, 140175985139711, -+STORE, 140175931932672, 140175985139711, -+STORE, 140175931924480, 140175985139711, -+STORE, 140175931908096, 140175985139711, -+STORE, 140175931899904, 140175985139711, -+STORE, 140175931883520, 140175985139711, -+STORE, 140175931875328, 140175985139711, -+STORE, 140175931842560, 140175985139711, -+STORE, 140175931834368, 140175985139711, -+STORE, 140175931817984, 140175985139711, -+STORE, 140175931809792, 140175985139711, -+STORE, 140175931793408, 140175985139711, -+STORE, 140175931785216, 140175985139711, -+STORE, 140175931768832, 140175985139711, -+STORE, 140175931760640, 140175985139711, -+STORE, 140175931727872, 140175985139711, -+STORE, 140175931719680, 140175985139711, -+STORE, 140175931703296, 140175985139711, -+STORE, 140175931695104, 140175985139711, -+STORE, 140175931678720, 140175985139711, -+STORE, 140175931670528, 140175985139711, -+STORE, 140175931654144, 140175985139711, -+STORE, 140175931645952, 140175985139711, -+STORE, 140175931613184, 140175985139711, -+STORE, 140175931604992, 140175985139711, -+STORE, 140175931588608, 140175985139711, -+STORE, 140175931580416, 140175985139711, -+STORE, 140175931564032, 140175985139711, -+STORE, 140175931555840, 140175985139711, -+STORE, 140175931539456, 140175985139711, -+STORE, 140175931531264, 140175985139711, -+STORE, 140175931498496, 140175985139711, -+STORE, 140175931490304, 140175985139711, -+STORE, 140175931473920, 140175985139711, -+STORE, 140175931465728, 140175985139711, -+STORE, 140175931449344, 140175985139711, -+STORE, 140175931441152, 140175985139711, -+STORE, 140175931424768, 140175985139711, -+STORE, 140175931416576, 140175985139711, -+STORE, 140175931383808, 140175985139711, -+STORE, 140175931375616, 140175985139711, -+STORE, 140175931359232, 140175985139711, -+STORE, 140175931351040, 140175985139711, -+STORE, 140175931334656, 140175985139711, -+STORE, 140175931326464, 140175985139711, -+STORE, 140175931310080, 140175985139711, -+STORE, 140175931301888, 140175985139711, -+STORE, 140175931269120, 140175985139711, -+STORE, 140175931260928, 140175985139711, -+STORE, 140175931244544, 140175985139711, -+STORE, 140175931236352, 140175985139711, -+STORE, 140175931219968, 140175985139711, -+STORE, 140175931211776, 140175985139711, -+STORE, 140175931195392, 140175985139711, -+STORE, 140175931187200, 140175985139711, -+STORE, 140175931154432, 140175985139711, -+STORE, 140175931146240, 140175985139711, -+STORE, 140175931129856, 140175985139711, -+STORE, 140175931121664, 140175985139711, -+STORE, 140175931105280, 140175985139711, -+STORE, 140175931097088, 140175985139711, -+STORE, 140175931080704, 140175985139711, -+STORE, 140175931072512, 140175985139711, -+STORE, 140175931039744, 140175985139711, -+STORE, 140175931031552, 140175985139711, -+STORE, 140175931015168, 140175985139711, -+STORE, 140175931006976, 140175985139711, -+STORE, 140175930990592, 140175985139711, -+STORE, 140175930982400, 140175985139711, -+STORE, 140175930966016, 140175985139711, -+STORE, 140175930957824, 140175985139711, -+STORE, 140175930925056, 140175985139711, -+STORE, 140175930916864, 140175985139711, -+STORE, 140175930900480, 140175985139711, -+STORE, 140175930892288, 140175985139711, -+STORE, 140175930875904, 140175985139711, -+STORE, 140175930867712, 140175985139711, -+STORE, 140175930851328, 140175985139711, -+STORE, 140175930843136, 140175985139711, -+STORE, 140175930810368, 140175985139711, -+STORE, 140175930802176, 140175985139711, -+STORE, 140175930785792, 140175985139711, -+STORE, 140175930777600, 140175985139711, -+STORE, 140175930761216, 140175985139711, -+STORE, 140175930753024, 140175985139711, -+STORE, 140175930736640, 140175985139711, -+STORE, 140175930728448, 140175985139711, -+STORE, 140175930695680, 140175985139711, -+STORE, 140175930687488, 140175985139711, -+STORE, 140175930671104, 140175985139711, -+STORE, 140175930662912, 140175985139711, -+STORE, 140175930646528, 140175985139711, -+STORE, 140175930638336, 140175985139711, -+STORE, 140175930621952, 140175985139711, -+STORE, 140175930613760, 140175985139711, -+STORE, 140175930580992, 140175985139711, -+STORE, 140175930572800, 140175985139711, -+STORE, 140175930556416, 140175985139711, -+STORE, 140175930548224, 140175985139711, -+STORE, 140175930531840, 140175985139711, -+STORE, 140175930523648, 140175985139711, -+STORE, 140175930507264, 140175985139711, -+STORE, 140175928410112, 140175985139711, -+STORE, 140175928401920, 140175985139711, -+STORE, 140175928369152, 140175985139711, -+STORE, 140175928360960, 140175985139711, -+STORE, 140175928344576, 140175985139711, -+STORE, 140175928336384, 140175985139711, -+STORE, 140175928320000, 140175985139711, -+STORE, 140175928311808, 140175985139711, -+STORE, 140175928295424, 140175985139711, -+STORE, 140175927242752, 140175985139711, -+SNULL, 140175956627455, 140175985139711, -+STORE, 140175927242752, 140175956627455, -+STORE, 140175956627456, 140175985139711, -+ }; -+ unsigned long set24[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140735281639424, 140737488351231, -+SNULL, 140735281643519, 140737488351231, -+STORE, 140735281639424, 140735281643519, -+STORE, 140735281508352, 140735281643519, -+STORE, 94717834911744, 94717834928127, -+SNULL, 94717834915839, 94717834928127, -+STORE, 94717834911744, 94717834915839, -+STORE, 94717834915840, 94717834928127, -+ERASE, 94717834915840, 94717834928127, -+STORE, 94717834919936, 94717834928127, -+STORE, 140428246065152, 140428248317951, -+SNULL, 140428246208511, 140428248317951, -+STORE, 140428246065152, 140428246208511, -+STORE, 140428246208512, 140428248317951, -+ERASE, 140428246208512, 140428248317951, -+STORE, 140428248305664, 140428248313855, -+STORE, 140428248313856, 140428248317951, -+STORE, 140735281811456, 140735281815551, -+STORE, 140735281799168, 140735281811455, -+STORE, 140428248297472, 140428248305663, -+STORE, 140428243841024, 140428246065151, -+SNULL, 140428245491711, 140428246065151, -+STORE, 140428243841024, 140428245491711, -+STORE, 140428245491712, 140428246065151, -+SNULL, 140428245491712, 140428246061055, -+STORE, 140428246061056, 140428246065151, -+STORE, 140428245491712, 140428246061055, -+ERASE, 140428245491712, 140428246061055, -+STORE, 140428245491712, 140428246061055, -+ERASE, 140428246061056, 140428246065151, -+STORE, 140428246061056, 140428246065151, -+STORE, 140428248268800, 140428248297471, -+STORE, 140428241625088, 140428243841023, -+SNULL, 140428241625088, 140428241723391, -+STORE, 140428241723392, 140428243841023, -+STORE, 140428241625088, 140428241723391, -+SNULL, 140428243816447, 140428243841023, -+STORE, 140428241723392, 140428243816447, -+STORE, 140428243816448, 140428243841023, -+SNULL, 140428243816448, 140428243824639, -+STORE, 140428243824640, 140428243841023, -+STORE, 140428243816448, 140428243824639, -+ERASE, 140428243816448, 140428243824639, -+STORE, 140428243816448, 140428243824639, -+ERASE, 140428243824640, 140428243841023, -+STORE, 140428243824640, 140428243841023, -+STORE, 140428237828096, 140428241625087, -+SNULL, 140428237828096, 140428239486975, -+STORE, 140428239486976, 140428241625087, -+STORE, 140428237828096, 140428239486975, -+SNULL, 140428241584127, 140428241625087, -+STORE, 140428239486976, 140428241584127, -+STORE, 140428241584128, 140428241625087, -+SNULL, 140428241584128, 140428241608703, -+STORE, 140428241608704, 140428241625087, -+STORE, 140428241584128, 140428241608703, -+ERASE, 140428241584128, 140428241608703, -+STORE, 140428241584128, 140428241608703, -+ERASE, 140428241608704, 140428241625087, -+STORE, 140428241608704, 140428241625087, -+STORE, 140428235567104, 140428237828095, -+SNULL, 140428235567104, 140428235718655, -+STORE, 140428235718656, 140428237828095, -+STORE, 140428235567104, 140428235718655, -+SNULL, 140428237811711, 140428237828095, -+STORE, 140428235718656, 140428237811711, -+STORE, 140428237811712, 140428237828095, -+SNULL, 140428237811712, 140428237819903, -+STORE, 140428237819904, 140428237828095, -+STORE, 140428237811712, 140428237819903, -+ERASE, 140428237811712, 140428237819903, -+STORE, 140428237811712, 140428237819903, -+ERASE, 140428237819904, 140428237828095, -+STORE, 140428237819904, 140428237828095, -+STORE, 140428233445376, 140428235567103, -+SNULL, 140428233445376, 140428233461759, -+STORE, 140428233461760, 140428235567103, -+STORE, 140428233445376, 140428233461759, -+SNULL, 140428235558911, 140428235567103, -+STORE, 140428233461760, 140428235558911, -+STORE, 140428235558912, 140428235567103, -+ERASE, 140428235558912, 140428235567103, -+STORE, 140428235558912, 140428235567103, -+STORE, 140428231315456, 140428233445375, -+SNULL, 140428231315456, 140428231344127, -+STORE, 140428231344128, 140428233445375, -+STORE, 140428231315456, 140428231344127, -+SNULL, 140428233437183, 140428233445375, -+STORE, 140428231344128, 140428233437183, -+STORE, 140428233437184, 140428233445375, -+ERASE, 140428233437184, 140428233445375, -+STORE, 140428233437184, 140428233445375, -+STORE, 140428248260608, 140428248268799, -+STORE, 140428229062656, 140428231315455, -+SNULL, 140428229062656, 140428229214207, -+STORE, 140428229214208, 140428231315455, -+STORE, 140428229062656, 140428229214207, -+SNULL, 140428231307263, 140428231315455, -+STORE, 140428229214208, 140428231307263, -+STORE, 140428231307264, 140428231315455, -+ERASE, 140428231307264, 140428231315455, -+STORE, 140428231307264, 140428231315455, -+STORE, 140428226891776, 140428229062655, -+SNULL, 140428226891776, 140428226961407, -+STORE, 140428226961408, 140428229062655, -+STORE, 140428226891776, 140428226961407, -+SNULL, 140428229054463, 140428229062655, -+STORE, 140428226961408, 140428229054463, -+STORE, 140428229054464, 140428229062655, -+ERASE, 140428229054464, 140428229062655, -+STORE, 140428229054464, 140428229062655, -+STORE, 140428223680512, 140428226891775, -+SNULL, 140428223680512, 140428224757759, -+STORE, 140428224757760, 140428226891775, -+STORE, 140428223680512, 140428224757759, -+SNULL, 140428226854911, 140428226891775, -+STORE, 140428224757760, 140428226854911, -+STORE, 140428226854912, 140428226891775, -+ERASE, 140428226854912, 140428226891775, -+STORE, 140428226854912, 140428226891775, -+STORE, 140428221546496, 140428223680511, -+SNULL, 140428221546496, 140428221575167, -+STORE, 140428221575168, 140428223680511, -+STORE, 140428221546496, 140428221575167, -+SNULL, 140428223672319, 140428223680511, -+STORE, 140428221575168, 140428223672319, -+STORE, 140428223672320, 140428223680511, -+ERASE, 140428223672320, 140428223680511, -+STORE, 140428223672320, 140428223680511, -+STORE, 140428219236352, 140428221546495, -+SNULL, 140428219236352, 140428219441151, -+STORE, 140428219441152, 140428221546495, -+STORE, 140428219236352, 140428219441151, -+SNULL, 140428221538303, 140428221546495, -+STORE, 140428219441152, 140428221538303, -+STORE, 140428221538304, 140428221546495, -+ERASE, 140428221538304, 140428221546495, -+STORE, 140428221538304, 140428221546495, -+STORE, 140428216852480, 140428219236351, -+SNULL, 140428216852480, 140428217044991, -+STORE, 140428217044992, 140428219236351, -+STORE, 140428216852480, 140428217044991, -+SNULL, 140428219138047, 140428219236351, -+STORE, 140428217044992, 140428219138047, -+STORE, 140428219138048, 140428219236351, -+ERASE, 140428219138048, 140428219236351, -+STORE, 140428219138048, 140428219236351, -+STORE, 140428248252416, 140428248268799, -+STORE, 140428214284288, 140428216852479, -+SNULL, 140428214284288, 140428214751231, -+STORE, 140428214751232, 140428216852479, -+STORE, 140428214284288, 140428214751231, -+SNULL, 140428216844287, 140428216852479, -+STORE, 140428214751232, 140428216844287, -+STORE, 140428216844288, 140428216852479, -+ERASE, 140428216844288, 140428216852479, -+STORE, 140428216844288, 140428216852479, -+STORE, 140428212170752, 140428214284287, -+SNULL, 140428212170752, 140428212183039, -+STORE, 140428212183040, 140428214284287, -+STORE, 140428212170752, 140428212183039, -+SNULL, 140428214276095, 140428214284287, -+STORE, 140428212183040, 140428214276095, -+STORE, 140428214276096, 140428214284287, -+ERASE, 140428214276096, 140428214284287, -+STORE, 140428214276096, 140428214284287, -+STORE, 140428209991680, 140428212170751, -+SNULL, 140428209991680, 140428210069503, -+STORE, 140428210069504, 140428212170751, -+STORE, 140428209991680, 140428210069503, -+SNULL, 140428212162559, 140428212170751, -+STORE, 140428210069504, 140428212162559, -+STORE, 140428212162560, 140428212170751, -+ERASE, 140428212162560, 140428212170751, -+STORE, 140428212162560, 140428212170751, -+STORE, 140428207874048, 140428209991679, -+SNULL, 140428207874048, 140428207890431, -+STORE, 140428207890432, 140428209991679, -+STORE, 140428207874048, 140428207890431, -+SNULL, 140428209983487, 140428209991679, -+STORE, 140428207890432, 140428209983487, -+STORE, 140428209983488, 140428209991679, -+ERASE, 140428209983488, 140428209991679, -+STORE, 140428209983488, 140428209991679, -+STORE, 140428248244224, 140428248268799, -+STORE, 140428248231936, 140428248268799, -+SNULL, 140428241600511, 140428241608703, -+STORE, 140428241584128, 140428241600511, -+STORE, 140428241600512, 140428241608703, -+SNULL, 140428209987583, 140428209991679, -+STORE, 140428209983488, 140428209987583, -+STORE, 140428209987584, 140428209991679, -+SNULL, 140428212166655, 140428212170751, -+STORE, 140428212162560, 140428212166655, -+STORE, 140428212166656, 140428212170751, -+SNULL, 140428214280191, 140428214284287, -+STORE, 140428214276096, 140428214280191, -+STORE, 140428214280192, 140428214284287, -+SNULL, 140428243820543, 140428243824639, -+STORE, 140428243816448, 140428243820543, -+STORE, 140428243820544, 140428243824639, -+SNULL, 140428216848383, 140428216852479, -+STORE, 140428216844288, 140428216848383, -+STORE, 140428216848384, 140428216852479, -+SNULL, 140428219232255, 140428219236351, -+STORE, 140428219138048, 140428219232255, -+STORE, 140428219232256, 140428219236351, -+SNULL, 140428221542399, 140428221546495, -+STORE, 140428221538304, 140428221542399, -+STORE, 140428221542400, 140428221546495, -+SNULL, 140428223676415, 140428223680511, -+STORE, 140428223672320, 140428223676415, -+STORE, 140428223676416, 140428223680511, -+SNULL, 140428226863103, 140428226891775, -+STORE, 140428226854912, 140428226863103, -+STORE, 140428226863104, 140428226891775, -+SNULL, 140428229058559, 140428229062655, -+STORE, 140428229054464, 140428229058559, -+STORE, 140428229058560, 140428229062655, -+SNULL, 140428231311359, 140428231315455, -+STORE, 140428231307264, 140428231311359, -+STORE, 140428231311360, 140428231315455, -+SNULL, 140428233441279, 140428233445375, -+STORE, 140428233437184, 140428233441279, -+STORE, 140428233441280, 140428233445375, -+SNULL, 140428235563007, 140428235567103, -+STORE, 140428235558912, 140428235563007, -+STORE, 140428235563008, 140428235567103, -+SNULL, 140428237815807, 140428237819903, -+STORE, 140428237811712, 140428237815807, -+STORE, 140428237815808, 140428237819903, -+SNULL, 140428246056959, 140428246061055, -+STORE, 140428245491712, 140428246056959, -+STORE, 140428246056960, 140428246061055, -+SNULL, 94717834924031, 94717834928127, -+STORE, 94717834919936, 94717834924031, -+STORE, 94717834924032, 94717834928127, -+SNULL, 140428248309759, 140428248313855, -+STORE, 140428248305664, 140428248309759, -+STORE, 140428248309760, 140428248313855, -+ERASE, 140428248268800, 140428248297471, -+STORE, 94717843058688, 94717843193855, -+STORE, 94749677137920, 94749677559807, -+STORE, 94749677563904, 94749677604863, -+STORE, 94749677604864, 94749677608959, -+STORE, 94749710970880, 94749711241215, -+STORE, 140490884894720, 140490884935679, -+STORE, 140490884935680, 140490887032831, -+STORE, 140490887032832, 140490887036927, -+STORE, 140490887036928, 140490887041023, -+STORE, 140490887041024, 140490887065599, -+STORE, 140490887065600, 140490887110655, -+STORE, 140490887110656, 140490889203711, -+STORE, 140490889203712, 140490889207807, -+STORE, 140490889207808, 140490889211903, -+STORE, 140490889211904, 140490889293823, -+STORE, 140490889293824, 140490891390975, -+STORE, 140490891390976, 140490891395071, -+STORE, 140490891395072, 140490891399167, -+STORE, 140490891399168, 140490891407359, -+STORE, 140490891407360, 140490891436031, -+STORE, 140490891436032, 140490893529087, -+STORE, 140490893529088, 140490893533183, -+STORE, 140490893533184, 140490893537279, -+STORE, 140490893537280, 140490901979135, -+STORE, 140490901979136, 140490901991423, -+STORE, 140490901991424, 140490904084479, -+STORE, 140490904084480, 140490904088575, -+STORE, 140490904088576, 140490904092671, -+STORE, 140490904092672, 140490904559615, -+STORE, 140490904559616, 140490906652671, -+STORE, 140490906652672, 140490906656767, -+STORE, 140490906656768, 140490906660863, -+STORE, 140490906660864, 140490906677247, -+STORE, 140490906677248, 140490908770303, -+STORE, 140490908770304, 140490908774399, -+STORE, 140490908774400, 140490908778495, -+STORE, 140490908778496, 140490908794879, -+STORE, 140490908794880, 140490910887935, -+STORE, 140490910887936, 140490910892031, -+STORE, 140490910892032, 140490910896127, -+STORE, 140490910896128, 140490912555007, -+STORE, 140490912555008, 140490914652159, -+STORE, 140490914652160, 140490914668543, -+STORE, 140490914668544, 140490914676735, -+STORE, 140490914676736, 140490914693119, -+STORE, 140490914693120, 140490914791423, -+STORE, 140490914791424, 140490916884479, -+STORE, 140490916884480, 140490916888575, -+STORE, 140490916888576, 140490916892671, -+STORE, 140490916892672, 140490916909055, -+STORE, 140490916909056, 140490916937727, -+STORE, 140490916937728, 140490919030783, -+STORE, 140490919030784, 140490919034879, -+STORE, 140490919034880, 140490919038975, -+STORE, 140490919038976, 140490919190527, -+STORE, 140490919190528, 140490921283583, -+STORE, 140490921283584, 140490921287679, -+STORE, 140490921287680, 140490921291775, -+STORE, 140490921291776, 140490921299967, -+STORE, 140490921299968, 140490921390079, -+STORE, 140490921390080, 140490923483135, -+STORE, 140490923483136, 140490923487231, -+STORE, 140490923487232, 140490923491327, -+STORE, 140490923491328, 140490923757567, -+STORE, 140490923757568, 140490925850623, -+STORE, 140490925850624, 140490925867007, -+STORE, 140490925867008, 140490925871103, -+STORE, 140490925871104, 140490925875199, -+STORE, 140490925875200, 140490925903871, -+STORE, 140490925903872, 140490928001023, -+STORE, 140490928001024, 140490928005119, -+STORE, 140490928005120, 140490928009215, -+STORE, 140490928009216, 140490928152575, -+STORE, 140490930184192, 140490930221055, -+STORE, 140490930221056, 140490930237439, -+STORE, 140490930237440, 140490930241535, -+STORE, 140490930241536, 140490930245631, -+STORE, 140490930245632, 140490930249727, -+STORE, 140490930249728, 140490930253823, -+STORE, 140490930253824, 140490930257919, -+STORE, 140490930257920, 140490930262015, -+STORE, 140724611694592, 140724611829759, -+STORE, 140724612427776, 140724612440063, -+STORE, 140724612440064, 140724612444159, -+STORE, 94103163662336, 94103163772927, -+STORE, 94103165865984, 94103165874175, -+STORE, 94103165874176, 94103165878271, -+STORE, 94103165878272, 94103165886463, -+STORE, 94103182548992, 94103182684159, -+STORE, 140092694708224, 140092696367103, -+STORE, 140092696367104, 140092698464255, -+STORE, 140092698464256, 140092698480639, -+STORE, 140092698480640, 140092698488831, -+STORE, 140092698488832, 140092698505215, -+STORE, 140092698505216, 140092698648575, -+STORE, 140092700708864, 140092700717055, -+STORE, 140092700745728, 140092700749823, -+STORE, 140092700749824, 140092700753919, -+STORE, 140092700753920, 140092700758015, -+STORE, 140736800911360, 140736801046527, -+STORE, 140736802308096, 140736802320383, -+STORE, 140736802320384, 140736802324479, -+STORE, 93948802064384, 93948802174975, -+STORE, 93948804268032, 93948804276223, -+STORE, 93948804276224, 93948804280319, -+STORE, 93948804280320, 93948804288511, -+STORE, 93948806266880, 93948806402047, -+STORE, 140222999113728, 140223000772607, -+STORE, 140223000772608, 140223002869759, -+STORE, 140223002869760, 140223002886143, -+STORE, 140223002886144, 140223002894335, -+STORE, 140223002894336, 140223002910719, -+STORE, 140223002910720, 140223003054079, -+STORE, 140223005114368, 140223005122559, -+STORE, 140223005151232, 140223005155327, -+STORE, 140223005155328, 140223005159423, -+STORE, 140223005159424, 140223005163519, -+STORE, 140720877506560, 140720877641727, -+STORE, 140720878231552, 140720878243839, -+STORE, 140720878243840, 140720878247935, -+STORE, 140737488347136, 140737488351231, -+STORE, 140733232087040, 140737488351231, -+SNULL, 140733232091135, 140737488351231, -+STORE, 140733232087040, 140733232091135, -+STORE, 140733231955968, 140733232091135, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 140161681321984, 140161683574783, -+SNULL, 140161681465343, 140161683574783, -+STORE, 140161681321984, 140161681465343, -+STORE, 140161681465344, 140161683574783, -+ERASE, 140161681465344, 140161683574783, -+STORE, 140161683562496, 140161683570687, -+STORE, 140161683570688, 140161683574783, -+STORE, 140733232214016, 140733232218111, -+STORE, 140733232201728, 140733232214015, -+STORE, 140161683533824, 140161683562495, -+STORE, 140161683525632, 140161683533823, -+STORE, 140161678159872, 140161681321983, -+SNULL, 140161678159872, 140161679220735, -+STORE, 140161679220736, 140161681321983, -+STORE, 140161678159872, 140161679220735, -+SNULL, 140161681313791, 140161681321983, -+STORE, 140161679220736, 140161681313791, -+STORE, 140161681313792, 140161681321983, -+ERASE, 140161681313792, 140161681321983, -+STORE, 140161681313792, 140161681321983, -+STORE, 140161674362880, 140161678159871, -+SNULL, 140161674362880, 140161676021759, -+STORE, 140161676021760, 140161678159871, -+STORE, 140161674362880, 140161676021759, -+SNULL, 140161678118911, 140161678159871, -+STORE, 140161676021760, 140161678118911, -+STORE, 140161678118912, 140161678159871, -+SNULL, 140161678118912, 140161678143487, -+STORE, 140161678143488, 140161678159871, -+STORE, 140161678118912, 140161678143487, -+ERASE, 140161678118912, 140161678143487, -+STORE, 140161678118912, 140161678143487, -+ERASE, 140161678143488, 140161678159871, -+STORE, 140161678143488, 140161678159871, -+STORE, 140161683513344, 140161683533823, -+SNULL, 140161678135295, 140161678143487, -+STORE, 140161678118912, 140161678135295, -+STORE, 140161678135296, 140161678143487, -+SNULL, 140161681317887, 140161681321983, -+STORE, 140161681313792, 140161681317887, -+STORE, 140161681317888, 140161681321983, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 140161683566591, 140161683570687, -+STORE, 140161683562496, 140161683566591, -+STORE, 140161683566592, 140161683570687, -+ERASE, 140161683533824, 140161683562495, -+STORE, 25477120, 25612287, -+STORE, 25477120, 25759743, -+STORE, 140161681829888, 140161683513343, -+STORE, 25477120, 25915391, -+STORE, 25477120, 26054655, -+SNULL, 25800703, 26054655, -+STORE, 25477120, 25800703, -+STORE, 25800704, 26054655, -+ERASE, 25800704, 26054655, -+STORE, 140737488347136, 140737488351231, -+STORE, 140723218452480, 140737488351231, -+SNULL, 140723218456575, 140737488351231, -+STORE, 140723218452480, 140723218456575, -+STORE, 140723218321408, 140723218456575, -+STORE, 4194304, 26279935, -+STORE, 28372992, 28454911, -+STORE, 28454912, 29806591, -+STORE, 140398872264704, 140398874517503, -+SNULL, 140398872408063, 140398874517503, -+STORE, 140398872264704, 140398872408063, -+STORE, 140398872408064, 140398874517503, -+ERASE, 140398872408064, 140398874517503, -+STORE, 140398874505216, 140398874513407, -+STORE, 140398874513408, 140398874517503, -+STORE, 140723219247104, 140723219251199, -+STORE, 140723219234816, 140723219247103, -+STORE, 140398874476544, 140398874505215, -+STORE, 140398874468352, 140398874476543, -+STORE, 140398868430848, 140398872264703, -+SNULL, 140398868430848, 140398870138879, -+STORE, 140398870138880, 140398872264703, -+STORE, 140398868430848, 140398870138879, -+SNULL, 140398872231935, 140398872264703, -+STORE, 140398870138880, 140398872231935, -+STORE, 140398872231936, 140398872264703, -+ERASE, 140398872231936, 140398872264703, -+STORE, 140398872231936, 140398872264703, -+STORE, 140398866235392, 140398868430847, -+SNULL, 140398866235392, 140398866329599, -+STORE, 140398866329600, 140398868430847, -+STORE, 140398866235392, 140398866329599, -+SNULL, 140398868422655, 140398868430847, -+STORE, 140398866329600, 140398868422655, -+STORE, 140398868422656, 140398868430847, -+ERASE, 140398868422656, 140398868430847, -+STORE, 140398868422656, 140398868430847, -+STORE, 140398863716352, 140398866235391, -+SNULL, 140398863716352, 140398864130047, -+STORE, 140398864130048, 140398866235391, -+STORE, 140398863716352, 140398864130047, -+SNULL, 140398866223103, 140398866235391, -+STORE, 140398864130048, 140398866223103, -+STORE, 140398866223104, 140398866235391, -+ERASE, 140398866223104, 140398866235391, -+STORE, 140398866223104, 140398866235391, -+STORE, 140398861082624, 140398863716351, -+SNULL, 140398861082624, 140398861611007, -+STORE, 140398861611008, 140398863716351, -+STORE, 140398861082624, 140398861611007, -+SNULL, 140398863708159, 140398863716351, -+STORE, 140398861611008, 140398863708159, -+STORE, 140398863708160, 140398863716351, -+ERASE, 140398863708160, 140398863716351, -+STORE, 140398863708160, 140398863716351, -+STORE, 140398858969088, 140398861082623, -+SNULL, 140398858969088, 140398858981375, -+STORE, 140398858981376, 140398861082623, -+STORE, 140398858969088, 140398858981375, -+SNULL, 140398861074431, 140398861082623, -+STORE, 140398858981376, 140398861074431, -+STORE, 140398861074432, 140398861082623, -+ERASE, 140398861074432, 140398861082623, -+STORE, 140398861074432, 140398861082623, -+STORE, 140398856765440, 140398858969087, -+SNULL, 140398856765440, 140398856867839, -+STORE, 140398856867840, 140398858969087, -+STORE, 140398856765440, 140398856867839, -+SNULL, 140398858960895, 140398858969087, -+STORE, 140398856867840, 140398858960895, -+STORE, 140398858960896, 140398858969087, -+ERASE, 140398858960896, 140398858969087, -+STORE, 140398858960896, 140398858969087, -+STORE, 140398874460160, 140398874476543, -+STORE, 140398853603328, 140398856765439, -+SNULL, 140398853603328, 140398854664191, -+STORE, 140398854664192, 140398856765439, -+STORE, 140398853603328, 140398854664191, -+SNULL, 140398856757247, 140398856765439, -+STORE, 140398854664192, 140398856757247, -+STORE, 140398856757248, 140398856765439, -+ERASE, 140398856757248, 140398856765439, -+STORE, 140398856757248, 140398856765439, -+STORE, 140398849806336, 140398853603327, -+SNULL, 140398849806336, 140398851465215, -+STORE, 140398851465216, 140398853603327, -+STORE, 140398849806336, 140398851465215, -+SNULL, 140398853562367, 140398853603327, -+STORE, 140398851465216, 140398853562367, -+STORE, 140398853562368, 140398853603327, -+SNULL, 140398853562368, 140398853586943, -+STORE, 140398853586944, 140398853603327, -+STORE, 140398853562368, 140398853586943, -+ERASE, 140398853562368, 140398853586943, -+STORE, 140398853562368, 140398853586943, -+ERASE, 140398853586944, 140398853603327, -+STORE, 140398853586944, 140398853603327, -+STORE, 140398874447872, 140398874476543, -+SNULL, 140398853578751, 140398853586943, -+STORE, 140398853562368, 140398853578751, -+STORE, 140398853578752, 140398853586943, -+SNULL, 140398856761343, 140398856765439, -+STORE, 140398856757248, 140398856761343, -+STORE, 140398856761344, 140398856765439, -+SNULL, 140398858964991, 140398858969087, -+STORE, 140398858960896, 140398858964991, -+STORE, 140398858964992, 140398858969087, -+SNULL, 140398861078527, 140398861082623, -+STORE, 140398861074432, 140398861078527, -+STORE, 140398861078528, 140398861082623, -+SNULL, 140398863712255, 140398863716351, -+STORE, 140398863708160, 140398863712255, -+STORE, 140398863712256, 140398863716351, -+SNULL, 140398866231295, 140398866235391, -+STORE, 140398866223104, 140398866231295, -+STORE, 140398866231296, 140398866235391, -+SNULL, 140398868426751, 140398868430847, -+STORE, 140398868422656, 140398868426751, -+STORE, 140398868426752, 140398868430847, -+SNULL, 140398872236031, 140398872264703, -+STORE, 140398872231936, 140398872236031, -+STORE, 140398872236032, 140398872264703, -+SNULL, 28405759, 28454911, -+STORE, 28372992, 28405759, -+STORE, 28405760, 28454911, -+SNULL, 140398874509311, 140398874513407, -+STORE, 140398874505216, 140398874509311, -+STORE, 140398874509312, 140398874513407, -+ERASE, 140398874476544, 140398874505215, -+STORE, 43278336, 43413503, -+STORE, 140398872764416, 140398874447871, -+STORE, 140398874501120, 140398874505215, -+STORE, 140398872629248, 140398872764415, -+STORE, 43278336, 43556863, -+STORE, 140398847709184, 140398849806335, -+STORE, 140398874492928, 140398874505215, -+STORE, 140398874484736, 140398874505215, -+STORE, 140398874447872, 140398874484735, -+STORE, 140398872612864, 140398872764415, -+STORE, 43278336, 43692031, -+STORE, 43278336, 43880447, -+STORE, 140398872604672, 140398872764415, -+STORE, 140398872596480, 140398872764415, -+STORE, 43278336, 44044287, -+STORE, 140398872580096, 140398872764415, -+STORE, 140737488347136, 140737488351231, -+STORE, 140734403092480, 140737488351231, -+SNULL, 140734403096575, 140737488351231, -+STORE, 140734403092480, 140734403096575, -+STORE, 140734402961408, 140734403096575, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 140240662380544, 140240664633343, -+SNULL, 140240662523903, 140240664633343, -+STORE, 140240662380544, 140240662523903, -+STORE, 140240662523904, 140240664633343, -+ERASE, 140240662523904, 140240664633343, -+STORE, 140240664621056, 140240664629247, -+STORE, 140240664629248, 140240664633343, -+STORE, 140734403145728, 140734403149823, -+STORE, 140734403133440, 140734403145727, -+STORE, 140240664592384, 140240664621055, -+STORE, 140240664584192, 140240664592383, -+STORE, 140240659218432, 140240662380543, -+SNULL, 140240659218432, 140240660279295, -+STORE, 140240660279296, 140240662380543, -+STORE, 140240659218432, 140240660279295, -+SNULL, 140240662372351, 140240662380543, -+STORE, 140240660279296, 140240662372351, -+STORE, 140240662372352, 140240662380543, -+ERASE, 140240662372352, 140240662380543, -+STORE, 140240662372352, 140240662380543, -+STORE, 140240655421440, 140240659218431, -+SNULL, 140240655421440, 140240657080319, -+STORE, 140240657080320, 140240659218431, -+STORE, 140240655421440, 140240657080319, -+SNULL, 140240659177471, 140240659218431, -+STORE, 140240657080320, 140240659177471, -+STORE, 140240659177472, 140240659218431, -+SNULL, 140240659177472, 140240659202047, -+STORE, 140240659202048, 140240659218431, -+STORE, 140240659177472, 140240659202047, -+ERASE, 140240659177472, 140240659202047, -+STORE, 140240659177472, 140240659202047, -+ERASE, 140240659202048, 140240659218431, -+STORE, 140240659202048, 140240659218431, -+STORE, 140240664571904, 140240664592383, -+SNULL, 140240659193855, 140240659202047, -+STORE, 140240659177472, 140240659193855, -+STORE, 140240659193856, 140240659202047, -+SNULL, 140240662376447, 140240662380543, -+STORE, 140240662372352, 140240662376447, -+STORE, 140240662376448, 140240662380543, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 140240664625151, 140240664629247, -+STORE, 140240664621056, 140240664625151, -+STORE, 140240664625152, 140240664629247, -+ERASE, 140240664592384, 140240664621055, -+STORE, 30646272, 30781439, -+STORE, 30646272, 30928895, -+STORE, 140240662888448, 140240664571903, -+STORE, 94256659468288, 94256659578879, -+STORE, 94256661671936, 94256661680127, -+STORE, 94256661680128, 94256661684223, -+STORE, 94256661684224, 94256661692415, -+STORE, 94256687980544, 94256688115711, -+STORE, 139801712504832, 139801714163711, -+STORE, 139801714163712, 139801716260863, -+STORE, 139801716260864, 139801716277247, -+STORE, 139801716277248, 139801716285439, -+STORE, 139801716285440, 139801716301823, -+STORE, 139801716301824, 139801716445183, -+STORE, 139801718505472, 139801718513663, -+STORE, 139801718542336, 139801718546431, -+STORE, 139801718546432, 139801718550527, -+STORE, 139801718550528, 139801718554623, -+STORE, 140721575538688, 140721575673855, -+STORE, 140721577013248, 140721577025535, -+STORE, 140721577025536, 140721577029631, -+STORE, 140737488347136, 140737488351231, -+STORE, 140729259393024, 140737488351231, -+SNULL, 140729259397119, 140737488351231, -+STORE, 140729259393024, 140729259397119, -+STORE, 140729259261952, 140729259397119, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 139682376638464, 139682378891263, -+SNULL, 139682376781823, 139682378891263, -+STORE, 139682376638464, 139682376781823, -+STORE, 139682376781824, 139682378891263, -+ERASE, 139682376781824, 139682378891263, -+STORE, 139682378878976, 139682378887167, -+STORE, 139682378887168, 139682378891263, -+STORE, 140729260462080, 140729260466175, -+STORE, 140729260449792, 140729260462079, -+STORE, 139682378850304, 139682378878975, -+STORE, 139682378842112, 139682378850303, -+STORE, 139682373476352, 139682376638463, -+SNULL, 139682373476352, 139682374537215, -+STORE, 139682374537216, 139682376638463, -+STORE, 139682373476352, 139682374537215, -+SNULL, 139682376630271, 139682376638463, -+STORE, 139682374537216, 139682376630271, -+STORE, 139682376630272, 139682376638463, -+ERASE, 139682376630272, 139682376638463, -+STORE, 139682376630272, 139682376638463, -+STORE, 139682369679360, 139682373476351, -+SNULL, 139682369679360, 139682371338239, -+STORE, 139682371338240, 139682373476351, -+STORE, 139682369679360, 139682371338239, -+SNULL, 139682373435391, 139682373476351, -+STORE, 139682371338240, 139682373435391, -+STORE, 139682373435392, 139682373476351, -+SNULL, 139682373435392, 139682373459967, -+STORE, 139682373459968, 139682373476351, -+STORE, 139682373435392, 139682373459967, -+ERASE, 139682373435392, 139682373459967, -+STORE, 139682373435392, 139682373459967, -+ERASE, 139682373459968, 139682373476351, -+STORE, 139682373459968, 139682373476351, -+STORE, 139682378829824, 139682378850303, -+SNULL, 139682373451775, 139682373459967, -+STORE, 139682373435392, 139682373451775, -+STORE, 139682373451776, 139682373459967, -+SNULL, 139682376634367, 139682376638463, -+STORE, 139682376630272, 139682376634367, -+STORE, 139682376634368, 139682376638463, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 139682378883071, 139682378887167, -+STORE, 139682378878976, 139682378883071, -+STORE, 139682378883072, 139682378887167, -+ERASE, 139682378850304, 139682378878975, -+STORE, 10022912, 10158079, -+STORE, 10022912, 10305535, -+STORE, 139682377146368, 139682378829823, -+STORE, 140737488347136, 140737488351231, -+STORE, 140731831926784, 140737488351231, -+SNULL, 140731831930879, 140737488351231, -+STORE, 140731831926784, 140731831930879, -+STORE, 140731831795712, 140731831930879, -+STORE, 94615305261056, 94615307485183, -+SNULL, 94615305371647, 94615307485183, -+STORE, 94615305261056, 94615305371647, -+STORE, 94615305371648, 94615307485183, -+ERASE, 94615305371648, 94615307485183, -+STORE, 94615307464704, 94615307476991, -+STORE, 94615307476992, 94615307485183, -+STORE, 140163912994816, 140163915247615, -+SNULL, 140163913138175, 140163915247615, -+STORE, 140163912994816, 140163913138175, -+STORE, 140163913138176, 140163915247615, -+ERASE, 140163913138176, 140163915247615, -+STORE, 140163915235328, 140163915243519, -+STORE, 140163915243520, 140163915247615, -+STORE, 140731832217600, 140731832221695, -+STORE, 140731832205312, 140731832217599, -+STORE, 140163915206656, 140163915235327, -+STORE, 140163915198464, 140163915206655, -+STORE, 140163909197824, 140163912994815, -+SNULL, 140163909197824, 140163910856703, -+STORE, 140163910856704, 140163912994815, -+STORE, 140163909197824, 140163910856703, -+SNULL, 140163912953855, 140163912994815, -+STORE, 140163910856704, 140163912953855, -+STORE, 140163912953856, 140163912994815, -+SNULL, 140163912953856, 140163912978431, -+STORE, 140163912978432, 140163912994815, -+STORE, 140163912953856, 140163912978431, -+ERASE, 140163912953856, 140163912978431, -+STORE, 140163912953856, 140163912978431, -+ERASE, 140163912978432, 140163912994815, -+STORE, 140163912978432, 140163912994815, -+SNULL, 140163912970239, 140163912978431, -+STORE, 140163912953856, 140163912970239, -+STORE, 140163912970240, 140163912978431, -+SNULL, 94615307472895, 94615307476991, -+STORE, 94615307464704, 94615307472895, -+STORE, 94615307472896, 94615307476991, -+SNULL, 140163915239423, 140163915243519, -+STORE, 140163915235328, 140163915239423, -+STORE, 140163915239424, 140163915243519, -+ERASE, 140163915206656, 140163915235327, -+STORE, 94615330672640, 94615330807807, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140725254479872, 140737488351231, -+SNULL, 140725254488063, 140737488351231, -+STORE, 140725254479872, 140725254488063, -+STORE, 140725254348800, 140725254488063, -+STORE, 94572781277184, 94572785741823, -+SNULL, 94572783312895, 94572785741823, -+STORE, 94572781277184, 94572783312895, -+STORE, 94572783312896, 94572785741823, -+ERASE, 94572783312896, 94572785741823, -+STORE, 94572785405952, 94572785455103, -+STORE, 94572785455104, 94572785741823, -+STORE, 139636001341440, 139636003594239, -+SNULL, 139636001484799, 139636003594239, -+STORE, 139636001341440, 139636001484799, -+STORE, 139636001484800, 139636003594239, -+ERASE, 139636001484800, 139636003594239, -+STORE, 139636003581952, 139636003590143, -+STORE, 139636003590144, 139636003594239, -+STORE, 140725255557120, 140725255561215, -+STORE, 140725255544832, 140725255557119, -+STORE, 139636003553280, 139636003581951, -+STORE, 139636003545088, 139636003553279, -+STORE, 139635998773248, 139636001341439, -+SNULL, 139635998773248, 139635999240191, -+STORE, 139635999240192, 139636001341439, -+STORE, 139635998773248, 139635999240191, -+SNULL, 139636001333247, 139636001341439, -+STORE, 139635999240192, 139636001333247, -+STORE, 139636001333248, 139636001341439, -+ERASE, 139636001333248, 139636001341439, -+STORE, 139636001333248, 139636001341439, -+STORE, 139635996569600, 139635998773247, -+SNULL, 139635996569600, 139635996671999, -+STORE, 139635996672000, 139635998773247, -+STORE, 139635996569600, 139635996671999, -+SNULL, 139635998765055, 139635998773247, -+STORE, 139635996672000, 139635998765055, -+STORE, 139635998765056, 139635998773247, -+ERASE, 139635998765056, 139635998773247, -+STORE, 139635998765056, 139635998773247, -+STORE, 139635994353664, 139635996569599, -+SNULL, 139635994353664, 139635994451967, -+STORE, 139635994451968, 139635996569599, -+STORE, 139635994353664, 139635994451967, -+SNULL, 139635996545023, 139635996569599, -+STORE, 139635994451968, 139635996545023, -+STORE, 139635996545024, 139635996569599, -+SNULL, 139635996545024, 139635996553215, -+STORE, 139635996553216, 139635996569599, -+STORE, 139635996545024, 139635996553215, -+ERASE, 139635996545024, 139635996553215, -+STORE, 139635996545024, 139635996553215, -+ERASE, 139635996553216, 139635996569599, -+STORE, 139635996553216, 139635996569599, -+STORE, 139635992223744, 139635994353663, -+SNULL, 139635992223744, 139635992252415, -+STORE, 139635992252416, 139635994353663, -+STORE, 139635992223744, 139635992252415, -+SNULL, 139635994345471, 139635994353663, -+STORE, 139635992252416, 139635994345471, -+STORE, 139635994345472, 139635994353663, -+ERASE, 139635994345472, 139635994353663, -+STORE, 139635994345472, 139635994353663, -+STORE, 139635988426752, 139635992223743, -+SNULL, 139635988426752, 139635990085631, -+STORE, 139635990085632, 139635992223743, -+STORE, 139635988426752, 139635990085631, -+SNULL, 139635992182783, 139635992223743, -+STORE, 139635990085632, 139635992182783, -+STORE, 139635992182784, 139635992223743, -+SNULL, 139635992182784, 139635992207359, -+STORE, 139635992207360, 139635992223743, -+STORE, 139635992182784, 139635992207359, -+ERASE, 139635992182784, 139635992207359, -+STORE, 139635992182784, 139635992207359, -+ERASE, 139635992207360, 139635992223743, -+STORE, 139635992207360, 139635992223743, -+STORE, 139636003536896, 139636003553279, -+SNULL, 139635992199167, 139635992207359, -+STORE, 139635992182784, 139635992199167, -+STORE, 139635992199168, 139635992207359, -+SNULL, 139635996549119, 139635996553215, -+STORE, 139635996545024, 139635996549119, -+STORE, 139635996549120, 139635996553215, -+SNULL, 139635994349567, 139635994353663, -+STORE, 139635994345472, 139635994349567, -+STORE, 139635994349568, 139635994353663, -+SNULL, 139635998769151, 139635998773247, -+STORE, 139635998765056, 139635998769151, -+STORE, 139635998769152, 139635998773247, -+SNULL, 139636001337343, 139636001341439, -+STORE, 139636001333248, 139636001337343, -+STORE, 139636001337344, 139636001341439, -+SNULL, 94572785418239, 94572785455103, -+STORE, 94572785405952, 94572785418239, -+STORE, 94572785418240, 94572785455103, -+SNULL, 139636003586047, 139636003590143, -+STORE, 139636003581952, 139636003586047, -+STORE, 139636003586048, 139636003590143, -+ERASE, 139636003553280, 139636003581951, -+STORE, 94572798435328, 94572798570495, -+STORE, 139636001853440, 139636003536895, -+STORE, 139635981426688, 139635988426751, -+STORE, 139635980615680, 139635981426687, -+STORE, 94572798435328, 94572798705663, -+STORE, 94572798435328, 94572798840831, -+STORE, 94572798435328, 94572798975999, -+STORE, 94572798435328, 94572799111167, -+STORE, 94572798435328, 94572799246335, -+STORE, 94572798435328, 94572799381503, -+STORE, 94572798435328, 94572799516671, -+STORE, 94572798435328, 94572799651839, -+STORE, 94572798435328, 94572799787007, -+STORE, 94572798435328, 94572799922175, -+STORE, 94572798435328, 94572800057343, -+STORE, 94572798435328, 94572800192511, -+STORE, 94572798435328, 94572800327679, -+STORE, 94572798435328, 94572800462847, -+STORE, 94572798435328, 94572800598015, -+STORE, 94572798435328, 94572800733183, -+STORE, 94572798435328, 94572800868351, -+STORE, 94572798435328, 94572801003519, -+STORE, 94572798435328, 94572801138687, -+STORE, 94572798435328, 94572801273855, -+STORE, 94572798435328, 94572801409023, -+STORE, 94572798435328, 94572801544191, -+STORE, 94572798435328, 94572801679359, -+STORE, 94572798435328, 94572801814527, -+STORE, 94572798435328, 94572801949695, -+STORE, 94572798435328, 94572802084863, -+STORE, 94572798435328, 94572802220031, -+STORE, 94572798435328, 94572802355199, -+STORE, 94572798435328, 94572802490367, -+STORE, 94572798435328, 94572802625535, -+STORE, 94572798435328, 94572802760703, -+STORE, 94572798435328, 94572802895871, -+STORE, 94572798435328, 94572803031039, -+STORE, 94572798435328, 94572803166207, -+STORE, 94572798435328, 94572803301375, -+STORE, 94572798435328, 94572803436543, -+STORE, 94572798435328, 94572803571711, -+STORE, 94572798435328, 94572803706879, -+STORE, 94572798435328, 94572803842047, -+STORE, 94572798435328, 94572803977215, -+STORE, 94572798435328, 94572804112383, -+STORE, 94572798435328, 94572804247551, -+STORE, 94572798435328, 94572804382719, -+STORE, 94572798435328, 94572804517887, -+STORE, 94572798435328, 94572804653055, -+STORE, 94572798435328, 94572804788223, -+STORE, 94572798435328, 94572804923391, -+STORE, 94572798435328, 94572805058559, -+STORE, 94572798435328, 94572805193727, -+STORE, 94572798435328, 94572805328895, -+STORE, 94572798435328, 94572805464063, -+STORE, 94572798435328, 94572805599231, -+STORE, 94572798435328, 94572805734399, -+STORE, 94572798435328, 94572805869567, -+STORE, 94572798435328, 94572806004735, -+STORE, 94572798435328, 94572806139903, -+STORE, 94572798435328, 94572806275071, -+STORE, 94572798435328, 94572806410239, -+STORE, 94572798435328, 94572806545407, -+STORE, 94572798435328, 94572806680575, -+STORE, 94572798435328, 94572806815743, -+STORE, 94572798435328, 94572806950911, -+STORE, 94572798435328, 94572807086079, -+STORE, 94572798435328, 94572807221247, -+STORE, 94572798435328, 94572807356415, -+STORE, 94572798435328, 94572807491583, -+STORE, 94572798435328, 94572807626751, -+STORE, 94572798435328, 94572807761919, -+STORE, 94572798435328, 94572807897087, -+STORE, 94572798435328, 94572808032255, -+STORE, 94572798435328, 94572808167423, -+STORE, 94572798435328, 94572808302591, -+STORE, 94572798435328, 94572808437759, -+STORE, 94572798435328, 94572808572927, -+ERASE, 139635981426688, 139635988426751, -+STORE, 139635985088512, 139635988426751, -+STORE, 139635778273280, 139635980615679, -+STORE, 139635567632384, 139635778273279, -+STORE, 94572798435328, 94572808716287, -+STORE, 139635984564224, 139635985088511, -+STORE, 139635559239680, 139635567632383, -+SNULL, 139635559243775, 139635567632383, -+STORE, 139635559239680, 139635559243775, -+STORE, 139635559243776, 139635567632383, -+STORE, 139635550846976, 139635559239679, -+SNULL, 139635550851071, 139635559239679, -+STORE, 139635550846976, 139635550851071, -+STORE, 139635550851072, 139635559239679, -+STORE, 139635542454272, 139635550846975, -+STORE, 139635408236544, 139635542454271, -+SNULL, 139635408236544, 139635426590719, -+STORE, 139635426590720, 139635542454271, -+STORE, 139635408236544, 139635426590719, -+ERASE, 139635408236544, 139635426590719, -+STORE, 139635292372992, 139635542454271, -+SNULL, 139635359481855, 139635542454271, -+STORE, 139635292372992, 139635359481855, -+STORE, 139635359481856, 139635542454271, -+SNULL, 139635359481856, 139635426590719, -+STORE, 139635426590720, 139635542454271, -+STORE, 139635359481856, 139635426590719, -+ERASE, 139635359481856, 139635426590719, -+SNULL, 139635542458367, 139635550846975, -+STORE, 139635542454272, 139635542458367, -+STORE, 139635542458368, 139635550846975, -+STORE, 139635418198016, 139635426590719, -+SNULL, 139635493699583, 139635542454271, -+STORE, 139635426590720, 139635493699583, -+STORE, 139635493699584, 139635542454271, -+ERASE, 139635493699584, 139635542454271, -+SNULL, 139635426725887, 139635493699583, -+STORE, 139635426590720, 139635426725887, -+STORE, 139635426725888, 139635493699583, -+SNULL, 139635292508159, 139635359481855, -+STORE, 139635292372992, 139635292508159, -+STORE, 139635292508160, 139635359481855, -+SNULL, 139635418202111, 139635426590719, -+STORE, 139635418198016, 139635418202111, -+STORE, 139635418202112, 139635426590719, -+STORE, 139635225264128, 139635292372991, -+STORE, 139635534061568, 139635542454271, -+SNULL, 139635534065663, 139635542454271, -+STORE, 139635534061568, 139635534065663, -+STORE, 139635534065664, 139635542454271, -+STORE, 139635525668864, 139635534061567, -+SNULL, 139635525672959, 139635534061567, -+STORE, 139635525668864, 139635525672959, -+STORE, 139635525672960, 139635534061567, -+SNULL, 139635225399295, 139635292372991, -+STORE, 139635225264128, 139635225399295, -+STORE, 139635225399296, 139635292372991, -+STORE, 139635091046400, 139635225264127, -+SNULL, 139635158155263, 139635225264127, -+STORE, 139635091046400, 139635158155263, -+STORE, 139635158155264, 139635225264127, -+ERASE, 139635158155264, 139635225264127, -+STORE, 139634956828672, 139635158155263, -+STORE, 139635517276160, 139635525668863, -+SNULL, 139635517280255, 139635525668863, -+STORE, 139635517276160, 139635517280255, -+STORE, 139635517280256, 139635525668863, -+SNULL, 139634956828672, 139635091046399, -+STORE, 139635091046400, 139635158155263, -+STORE, 139634956828672, 139635091046399, -+SNULL, 139635091181567, 139635158155263, -+STORE, 139635091046400, 139635091181567, -+STORE, 139635091181568, 139635158155263, -+SNULL, 139635023937535, 139635091046399, -+STORE, 139634956828672, 139635023937535, -+STORE, 139635023937536, 139635091046399, -+ERASE, 139635023937536, 139635091046399, -+STORE, 139634956828672, 139635091046399, -+SNULL, 139634956828672, 139635023937535, -+STORE, 139635023937536, 139635091046399, -+STORE, 139634956828672, 139635023937535, -+SNULL, 139635024072703, 139635091046399, -+STORE, 139635023937536, 139635024072703, -+STORE, 139635024072704, 139635091046399, -+STORE, 139635508883456, 139635517276159, -+SNULL, 139635508887551, 139635517276159, -+STORE, 139635508883456, 139635508887551, -+STORE, 139635508887552, 139635517276159, -+STORE, 139634822610944, 139635023937535, -+SNULL, 139634822610944, 139634956828671, -+STORE, 139634956828672, 139635023937535, -+STORE, 139634822610944, 139634956828671, -+SNULL, 139634956963839, 139635023937535, -+STORE, 139634956828672, 139634956963839, -+STORE, 139634956963840, 139635023937535, -+STORE, 139635500490752, 139635508883455, -+SNULL, 139634889719807, 139634956828671, -+STORE, 139634822610944, 139634889719807, -+STORE, 139634889719808, 139634956828671, -+ERASE, 139634889719808, 139634956828671, -+SNULL, 139635500494847, 139635508883455, -+STORE, 139635500490752, 139635500494847, -+STORE, 139635500494848, 139635508883455, -+SNULL, 139634822746111, 139634889719807, -+STORE, 139634822610944, 139634822746111, -+STORE, 139634822746112, 139634889719807, -+STORE, 139635409805312, 139635418198015, -+STORE, 139634822746112, 139634956828671, -+SNULL, 139634822746112, 139634889719807, -+STORE, 139634889719808, 139634956828671, -+STORE, 139634822746112, 139634889719807, -+SNULL, 139634889854975, 139634956828671, -+STORE, 139634889719808, 139634889854975, -+STORE, 139634889854976, 139634956828671, -+SNULL, 139635409809407, 139635418198015, -+STORE, 139635409805312, 139635409809407, -+STORE, 139635409809408, 139635418198015, -+STORE, 139635401412608, 139635409805311, -+STORE, 139634688393216, 139634822610943, -+SNULL, 139634755502079, 139634822610943, -+STORE, 139634688393216, 139634755502079, -+STORE, 139634755502080, 139634822610943, -+ERASE, 139634755502080, 139634822610943, -+SNULL, 139635401416703, 139635409805311, -+STORE, 139635401412608, 139635401416703, -+STORE, 139635401416704, 139635409805311, -+STORE, 139634554175488, 139634755502079, -+SNULL, 139634554175488, 139634688393215, -+STORE, 139634688393216, 139634755502079, -+STORE, 139634554175488, 139634688393215, -+SNULL, 139634688528383, 139634755502079, -+STORE, 139634688393216, 139634688528383, -+STORE, 139634688528384, 139634755502079, -+STORE, 139635393019904, 139635401412607, -+SNULL, 139634621284351, 139634688393215, -+STORE, 139634554175488, 139634621284351, -+STORE, 139634621284352, 139634688393215, -+ERASE, 139634621284352, 139634688393215, -+SNULL, 139634554310655, 139634621284351, -+STORE, 139634554175488, 139634554310655, -+STORE, 139634554310656, 139634621284351, -+STORE, 139634554310656, 139634688393215, -+SNULL, 139635393023999, 139635401412607, -+STORE, 139635393019904, 139635393023999, -+STORE, 139635393024000, 139635401412607, -+SNULL, 139634554310656, 139634621284351, -+STORE, 139634621284352, 139634688393215, -+STORE, 139634554310656, 139634621284351, -+SNULL, 139634621419519, 139634688393215, -+STORE, 139634621284352, 139634621419519, -+STORE, 139634621419520, 139634688393215, -+STORE, 139635384627200, 139635393019903, -+SNULL, 139635384631295, 139635393019903, -+STORE, 139635384627200, 139635384631295, -+STORE, 139635384631296, 139635393019903, -+STORE, 139635376234496, 139635384627199, -+SNULL, 139635376238591, 139635384627199, -+STORE, 139635376234496, 139635376238591, -+STORE, 139635376238592, 139635384627199, -+STORE, 139635367841792, 139635376234495, -+SNULL, 139635367845887, 139635376234495, -+STORE, 139635367841792, 139635367845887, -+STORE, 139635367845888, 139635376234495, -+STORE, 139634419957760, 139634554175487, -+SNULL, 139634487066623, 139634554175487, -+STORE, 139634419957760, 139634487066623, -+STORE, 139634487066624, 139634554175487, -+ERASE, 139634487066624, 139634554175487, -+STORE, 139635216871424, 139635225264127, -+SNULL, 139635216875519, 139635225264127, -+STORE, 139635216871424, 139635216875519, -+STORE, 139635216875520, 139635225264127, -+SNULL, 139634420092927, 139634487066623, -+STORE, 139634419957760, 139634420092927, -+STORE, 139634420092928, 139634487066623, -+STORE, 139635208478720, 139635216871423, -+SNULL, 139635208482815, 139635216871423, -+STORE, 139635208478720, 139635208482815, -+STORE, 139635208482816, 139635216871423, -+STORE, 139635200086016, 139635208478719, -+SNULL, 139635200090111, 139635208478719, -+STORE, 139635200086016, 139635200090111, -+STORE, 139635200090112, 139635208478719, -+STORE, 139635191693312, 139635200086015, -+SNULL, 139635191697407, 139635200086015, -+STORE, 139635191693312, 139635191697407, -+STORE, 139635191697408, 139635200086015, -+STORE, 139635183300608, 139635191693311, -+SNULL, 139635183304703, 139635191693311, -+STORE, 139635183300608, 139635183304703, -+STORE, 139635183304704, 139635191693311, -+STORE, 139634420092928, 139634554175487, -+SNULL, 139634420092928, 139634487066623, -+STORE, 139634487066624, 139634554175487, -+STORE, 139634420092928, 139634487066623, -+SNULL, 139634487201791, 139634554175487, -+STORE, 139634487066624, 139634487201791, -+STORE, 139634487201792, 139634554175487, -+ERASE, 139635559239680, 139635559243775, -+ERASE, 139635559243776, 139635567632383, -+ERASE, 139635550846976, 139635550851071, -+ERASE, 139635550851072, 139635559239679, -+ERASE, 139635542454272, 139635542458367, -+ERASE, 139635542458368, 139635550846975, -+ERASE, 139635418198016, 139635418202111, -+ERASE, 139635418202112, 139635426590719, -+ERASE, 139635534061568, 139635534065663, -+ERASE, 139635534065664, 139635542454271, -+ERASE, 139635525668864, 139635525672959, -+ERASE, 139635525672960, 139635534061567, -+ERASE, 139635517276160, 139635517280255, -+ERASE, 139635517280256, 139635525668863, -+ERASE, 139635508883456, 139635508887551, -+ERASE, 139635508887552, 139635517276159, -+ERASE, 139635500490752, 139635500494847, -+ERASE, 139635500494848, 139635508883455, -+ERASE, 139635409805312, 139635409809407, -+ERASE, 139635409809408, 139635418198015, -+ERASE, 139635401412608, 139635401416703, -+ERASE, 139635401416704, 139635409805311, -+ERASE, 139635393019904, 139635393023999, -+ERASE, 139635393024000, 139635401412607, -+ERASE, 139635384627200, 139635384631295, -+ERASE, 139635384631296, 139635393019903, -+ }; -+ unsigned long set25[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722547441664, 140737488351231, -+SNULL, 140722547449855, 140737488351231, -+STORE, 140722547441664, 140722547449855, -+STORE, 140722547310592, 140722547449855, -+STORE, 94827521732608, 94827523956735, -+SNULL, 94827521843199, 94827523956735, -+STORE, 94827521732608, 94827521843199, -+STORE, 94827521843200, 94827523956735, -+ERASE, 94827521843200, 94827523956735, -+STORE, 94827523936256, 94827523948543, -+STORE, 94827523948544, 94827523956735, -+STORE, 139816136847360, 139816139100159, -+SNULL, 139816136990719, 139816139100159, -+STORE, 139816136847360, 139816136990719, -+STORE, 139816136990720, 139816139100159, -+ERASE, 139816136990720, 139816139100159, -+STORE, 139816139087872, 139816139096063, -+STORE, 139816139096064, 139816139100159, -+STORE, 140722548142080, 140722548146175, -+STORE, 140722548129792, 140722548142079, -+STORE, 139816139059200, 139816139087871, -+STORE, 139816139051008, 139816139059199, -+STORE, 139816133050368, 139816136847359, -+SNULL, 139816133050368, 139816134709247, -+STORE, 139816134709248, 139816136847359, -+STORE, 139816133050368, 139816134709247, -+SNULL, 139816136806399, 139816136847359, -+STORE, 139816134709248, 139816136806399, -+STORE, 139816136806400, 139816136847359, -+SNULL, 139816136806400, 139816136830975, -+STORE, 139816136830976, 139816136847359, -+STORE, 139816136806400, 139816136830975, -+ERASE, 139816136806400, 139816136830975, -+STORE, 139816136806400, 139816136830975, -+ERASE, 139816136830976, 139816136847359, -+STORE, 139816136830976, 139816136847359, -+SNULL, 139816136822783, 139816136830975, -+STORE, 139816136806400, 139816136822783, -+STORE, 139816136822784, 139816136830975, -+SNULL, 94827523944447, 94827523948543, -+STORE, 94827523936256, 94827523944447, -+STORE, 94827523944448, 94827523948543, -+SNULL, 139816139091967, 139816139096063, -+STORE, 139816139087872, 139816139091967, -+STORE, 139816139091968, 139816139096063, -+ERASE, 139816139059200, 139816139087871, -+STORE, 94827534970880, 94827535106047, -+STORE, 94114394132480, 94114394345471, -+STORE, 94114396442624, 94114396446719, -+STORE, 94114396446720, 94114396454911, -+STORE, 94114396454912, 94114396467199, -+STORE, 94114421575680, 94114427715583, -+STORE, 139934313955328, 139934315614207, -+STORE, 139934315614208, 139934317711359, -+STORE, 139934317711360, 139934317727743, -+STORE, 139934317727744, 139934317735935, -+STORE, 139934317735936, 139934317752319, -+STORE, 139934317752320, 139934317764607, -+STORE, 139934317764608, 139934319857663, -+STORE, 139934319857664, 139934319861759, -+STORE, 139934319861760, 139934319865855, -+STORE, 139934319865856, 139934320009215, -+STORE, 139934320377856, 139934322061311, -+STORE, 139934322061312, 139934322077695, -+STORE, 139934322106368, 139934322110463, -+STORE, 139934322110464, 139934322114559, -+STORE, 139934322114560, 139934322118655, -+STORE, 140731200376832, 140731200516095, -+STORE, 140731200929792, 140731200942079, -+STORE, 140731200942080, 140731200946175, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140734133174272, 140737488351231, -+SNULL, 140734133182463, 140737488351231, -+STORE, 140734133174272, 140734133182463, -+STORE, 140734133043200, 140734133182463, -+STORE, 94412675600384, 94412677824511, -+SNULL, 94412675710975, 94412677824511, -+STORE, 94412675600384, 94412675710975, -+STORE, 94412675710976, 94412677824511, -+ERASE, 94412675710976, 94412677824511, -+STORE, 94412677804032, 94412677816319, -+STORE, 94412677816320, 94412677824511, -+STORE, 140320087945216, 140320090198015, -+SNULL, 140320088088575, 140320090198015, -+STORE, 140320087945216, 140320088088575, -+STORE, 140320088088576, 140320090198015, -+ERASE, 140320088088576, 140320090198015, -+STORE, 140320090185728, 140320090193919, -+STORE, 140320090193920, 140320090198015, -+STORE, 140734134591488, 140734134595583, -+STORE, 140734134579200, 140734134591487, -+STORE, 140320090157056, 140320090185727, -+STORE, 140320090148864, 140320090157055, -+STORE, 140320084148224, 140320087945215, -+SNULL, 140320084148224, 140320085807103, -+STORE, 140320085807104, 140320087945215, -+STORE, 140320084148224, 140320085807103, -+SNULL, 140320087904255, 140320087945215, -+STORE, 140320085807104, 140320087904255, -+STORE, 140320087904256, 140320087945215, -+SNULL, 140320087904256, 140320087928831, -+STORE, 140320087928832, 140320087945215, -+STORE, 140320087904256, 140320087928831, -+ERASE, 140320087904256, 140320087928831, -+STORE, 140320087904256, 140320087928831, -+ERASE, 140320087928832, 140320087945215, -+STORE, 140320087928832, 140320087945215, -+SNULL, 140320087920639, 140320087928831, -+STORE, 140320087904256, 140320087920639, -+STORE, 140320087920640, 140320087928831, -+SNULL, 94412677812223, 94412677816319, -+STORE, 94412677804032, 94412677812223, -+STORE, 94412677812224, 94412677816319, -+SNULL, 140320090189823, 140320090193919, -+STORE, 140320090185728, 140320090189823, -+STORE, 140320090189824, 140320090193919, -+ERASE, 140320090157056, 140320090185727, -+STORE, 94412684546048, 94412684681215, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723005485056, 140737488351231, -+SNULL, 140723005493247, 140737488351231, -+STORE, 140723005485056, 140723005493247, -+STORE, 140723005353984, 140723005493247, -+STORE, 94387431936000, 94387434160127, -+SNULL, 94387432046591, 94387434160127, -+STORE, 94387431936000, 94387432046591, -+STORE, 94387432046592, 94387434160127, -+ERASE, 94387432046592, 94387434160127, -+STORE, 94387434139648, 94387434151935, -+STORE, 94387434151936, 94387434160127, -+STORE, 140151675392000, 140151677644799, -+SNULL, 140151675535359, 140151677644799, -+STORE, 140151675392000, 140151675535359, -+STORE, 140151675535360, 140151677644799, -+ERASE, 140151675535360, 140151677644799, -+STORE, 140151677632512, 140151677640703, -+STORE, 140151677640704, 140151677644799, -+STORE, 140723005784064, 140723005788159, -+STORE, 140723005771776, 140723005784063, -+STORE, 140151677603840, 140151677632511, -+STORE, 140151677595648, 140151677603839, -+STORE, 140151671595008, 140151675391999, -+SNULL, 140151671595008, 140151673253887, -+STORE, 140151673253888, 140151675391999, -+STORE, 140151671595008, 140151673253887, -+SNULL, 140151675351039, 140151675391999, -+STORE, 140151673253888, 140151675351039, -+STORE, 140151675351040, 140151675391999, -+SNULL, 140151675351040, 140151675375615, -+STORE, 140151675375616, 140151675391999, -+STORE, 140151675351040, 140151675375615, -+ERASE, 140151675351040, 140151675375615, -+STORE, 140151675351040, 140151675375615, -+ERASE, 140151675375616, 140151675391999, -+STORE, 140151675375616, 140151675391999, -+SNULL, 140151675367423, 140151675375615, -+STORE, 140151675351040, 140151675367423, -+STORE, 140151675367424, 140151675375615, -+SNULL, 94387434147839, 94387434151935, -+STORE, 94387434139648, 94387434147839, -+STORE, 94387434147840, 94387434151935, -+SNULL, 140151677636607, 140151677640703, -+STORE, 140151677632512, 140151677636607, -+STORE, 140151677636608, 140151677640703, -+ERASE, 140151677603840, 140151677632511, -+STORE, 94387458818048, 94387458953215, -+STORE, 94909010997248, 94909011210239, -+STORE, 94909013307392, 94909013311487, -+STORE, 94909013311488, 94909013319679, -+STORE, 94909013319680, 94909013331967, -+STORE, 94909014827008, 94909023371263, -+STORE, 140712411975680, 140712413634559, -+STORE, 140712413634560, 140712415731711, -+STORE, 140712415731712, 140712415748095, -+STORE, 140712415748096, 140712415756287, -+STORE, 140712415756288, 140712415772671, -+STORE, 140712415772672, 140712415784959, -+STORE, 140712415784960, 140712417878015, -+STORE, 140712417878016, 140712417882111, -+STORE, 140712417882112, 140712417886207, -+STORE, 140712417886208, 140712418029567, -+STORE, 140712418398208, 140712420081663, -+STORE, 140712420081664, 140712420098047, -+STORE, 140712420126720, 140712420130815, -+STORE, 140712420130816, 140712420134911, -+STORE, 140712420134912, 140712420139007, -+STORE, 140729293111296, 140729293250559, -+STORE, 140729293307904, 140729293320191, -+STORE, 140729293320192, 140729293324287, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140720541691904, 140737488351231, -+SNULL, 140720541700095, 140737488351231, -+STORE, 140720541691904, 140720541700095, -+STORE, 140720541560832, 140720541700095, -+STORE, 94203603419136, 94203605643263, -+SNULL, 94203603529727, 94203605643263, -+STORE, 94203603419136, 94203603529727, -+STORE, 94203603529728, 94203605643263, -+ERASE, 94203603529728, 94203605643263, -+STORE, 94203605622784, 94203605635071, -+STORE, 94203605635072, 94203605643263, -+STORE, 139847623081984, 139847625334783, -+SNULL, 139847623225343, 139847625334783, -+STORE, 139847623081984, 139847623225343, -+STORE, 139847623225344, 139847625334783, -+ERASE, 139847623225344, 139847625334783, -+STORE, 139847625322496, 139847625330687, -+STORE, 139847625330688, 139847625334783, -+STORE, 140720542547968, 140720542552063, -+STORE, 140720542535680, 140720542547967, -+STORE, 139847625293824, 139847625322495, -+STORE, 139847625285632, 139847625293823, -+STORE, 139847619284992, 139847623081983, -+SNULL, 139847619284992, 139847620943871, -+STORE, 139847620943872, 139847623081983, -+STORE, 139847619284992, 139847620943871, -+SNULL, 139847623041023, 139847623081983, -+STORE, 139847620943872, 139847623041023, -+STORE, 139847623041024, 139847623081983, -+SNULL, 139847623041024, 139847623065599, -+STORE, 139847623065600, 139847623081983, -+STORE, 139847623041024, 139847623065599, -+ERASE, 139847623041024, 139847623065599, -+STORE, 139847623041024, 139847623065599, -+ERASE, 139847623065600, 139847623081983, -+STORE, 139847623065600, 139847623081983, -+SNULL, 139847623057407, 139847623065599, -+STORE, 139847623041024, 139847623057407, -+STORE, 139847623057408, 139847623065599, -+SNULL, 94203605630975, 94203605635071, -+STORE, 94203605622784, 94203605630975, -+STORE, 94203605630976, 94203605635071, -+SNULL, 139847625326591, 139847625330687, -+STORE, 139847625322496, 139847625326591, -+STORE, 139847625326592, 139847625330687, -+ERASE, 139847625293824, 139847625322495, -+STORE, 94203634880512, 94203635015679, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140721428738048, 140737488351231, -+SNULL, 140721428746239, 140737488351231, -+STORE, 140721428738048, 140721428746239, -+STORE, 140721428606976, 140721428746239, -+STORE, 93968808378368, 93968810602495, -+SNULL, 93968808488959, 93968810602495, -+STORE, 93968808378368, 93968808488959, -+STORE, 93968808488960, 93968810602495, -+ERASE, 93968808488960, 93968810602495, -+STORE, 93968810582016, 93968810594303, -+STORE, 93968810594304, 93968810602495, -+STORE, 140397757026304, 140397759279103, -+SNULL, 140397757169663, 140397759279103, -+STORE, 140397757026304, 140397757169663, -+STORE, 140397757169664, 140397759279103, -+ERASE, 140397757169664, 140397759279103, -+STORE, 140397759266816, 140397759275007, -+STORE, 140397759275008, 140397759279103, -+STORE, 140721430368256, 140721430372351, -+STORE, 140721430355968, 140721430368255, -+STORE, 140397759238144, 140397759266815, -+STORE, 140397759229952, 140397759238143, -+STORE, 140397753229312, 140397757026303, -+SNULL, 140397753229312, 140397754888191, -+STORE, 140397754888192, 140397757026303, -+STORE, 140397753229312, 140397754888191, -+SNULL, 140397756985343, 140397757026303, -+STORE, 140397754888192, 140397756985343, -+STORE, 140397756985344, 140397757026303, -+SNULL, 140397756985344, 140397757009919, -+STORE, 140397757009920, 140397757026303, -+STORE, 140397756985344, 140397757009919, -+ERASE, 140397756985344, 140397757009919, -+STORE, 140397756985344, 140397757009919, -+ERASE, 140397757009920, 140397757026303, -+STORE, 140397757009920, 140397757026303, -+SNULL, 140397757001727, 140397757009919, -+STORE, 140397756985344, 140397757001727, -+STORE, 140397757001728, 140397757009919, -+SNULL, 93968810590207, 93968810594303, -+STORE, 93968810582016, 93968810590207, -+STORE, 93968810590208, 93968810594303, -+SNULL, 140397759270911, 140397759275007, -+STORE, 140397759266816, 140397759270911, -+STORE, 140397759270912, 140397759275007, -+ERASE, 140397759238144, 140397759266815, -+STORE, 93968837025792, 93968837160959, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140721751044096, 140737488351231, -+SNULL, 140721751052287, 140737488351231, -+STORE, 140721751044096, 140721751052287, -+STORE, 140721750913024, 140721751052287, -+STORE, 94426051657728, 94426053881855, -+SNULL, 94426051768319, 94426053881855, -+STORE, 94426051657728, 94426051768319, -+STORE, 94426051768320, 94426053881855, -+ERASE, 94426051768320, 94426053881855, -+STORE, 94426053861376, 94426053873663, -+STORE, 94426053873664, 94426053881855, -+STORE, 140228456181760, 140228458434559, -+SNULL, 140228456325119, 140228458434559, -+STORE, 140228456181760, 140228456325119, -+STORE, 140228456325120, 140228458434559, -+ERASE, 140228456325120, 140228458434559, -+STORE, 140228458422272, 140228458430463, -+STORE, 140228458430464, 140228458434559, -+STORE, 140721751117824, 140721751121919, -+STORE, 140721751105536, 140721751117823, -+STORE, 140228458393600, 140228458422271, -+STORE, 140228458385408, 140228458393599, -+STORE, 140228452384768, 140228456181759, -+SNULL, 140228452384768, 140228454043647, -+STORE, 140228454043648, 140228456181759, -+STORE, 140228452384768, 140228454043647, -+SNULL, 140228456140799, 140228456181759, -+STORE, 140228454043648, 140228456140799, -+STORE, 140228456140800, 140228456181759, -+SNULL, 140228456140800, 140228456165375, -+STORE, 140228456165376, 140228456181759, -+STORE, 140228456140800, 140228456165375, -+ERASE, 140228456140800, 140228456165375, -+STORE, 140228456140800, 140228456165375, -+ERASE, 140228456165376, 140228456181759, -+STORE, 140228456165376, 140228456181759, -+SNULL, 140228456157183, 140228456165375, -+STORE, 140228456140800, 140228456157183, -+STORE, 140228456157184, 140228456165375, -+SNULL, 94426053869567, 94426053873663, -+STORE, 94426053861376, 94426053869567, -+STORE, 94426053869568, 94426053873663, -+SNULL, 140228458426367, 140228458430463, -+STORE, 140228458422272, 140228458426367, -+STORE, 140228458426368, 140228458430463, -+ERASE, 140228458393600, 140228458422271, -+STORE, 94426073681920, 94426073817087, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140732727623680, 140737488351231, -+SNULL, 140732727631871, 140737488351231, -+STORE, 140732727623680, 140732727631871, -+STORE, 140732727492608, 140732727631871, -+STORE, 94537485996032, 94537488220159, -+SNULL, 94537486106623, 94537488220159, -+STORE, 94537485996032, 94537486106623, -+STORE, 94537486106624, 94537488220159, -+ERASE, 94537486106624, 94537488220159, -+STORE, 94537488199680, 94537488211967, -+STORE, 94537488211968, 94537488220159, -+STORE, 140446578036736, 140446580289535, -+SNULL, 140446578180095, 140446580289535, -+STORE, 140446578036736, 140446578180095, -+STORE, 140446578180096, 140446580289535, -+ERASE, 140446578180096, 140446580289535, -+STORE, 140446580277248, 140446580285439, -+STORE, 140446580285440, 140446580289535, -+STORE, 140732727758848, 140732727762943, -+STORE, 140732727746560, 140732727758847, -+STORE, 140446580248576, 140446580277247, -+STORE, 140446580240384, 140446580248575, -+STORE, 140446574239744, 140446578036735, -+SNULL, 140446574239744, 140446575898623, -+STORE, 140446575898624, 140446578036735, -+STORE, 140446574239744, 140446575898623, -+SNULL, 140446577995775, 140446578036735, -+STORE, 140446575898624, 140446577995775, -+STORE, 140446577995776, 140446578036735, -+SNULL, 140446577995776, 140446578020351, -+STORE, 140446578020352, 140446578036735, -+STORE, 140446577995776, 140446578020351, -+ERASE, 140446577995776, 140446578020351, -+STORE, 140446577995776, 140446578020351, -+ERASE, 140446578020352, 140446578036735, -+STORE, 140446578020352, 140446578036735, -+SNULL, 140446578012159, 140446578020351, -+STORE, 140446577995776, 140446578012159, -+STORE, 140446578012160, 140446578020351, -+SNULL, 94537488207871, 94537488211967, -+STORE, 94537488199680, 94537488207871, -+STORE, 94537488207872, 94537488211967, -+SNULL, 140446580281343, 140446580285439, -+STORE, 140446580277248, 140446580281343, -+STORE, 140446580281344, 140446580285439, -+ERASE, 140446580248576, 140446580277247, -+STORE, 94537489014784, 94537489149951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728766808064, 140737488351231, -+SNULL, 140728766816255, 140737488351231, -+STORE, 140728766808064, 140728766816255, -+STORE, 140728766676992, 140728766816255, -+STORE, 94418513866752, 94418516090879, -+SNULL, 94418513977343, 94418516090879, -+STORE, 94418513866752, 94418513977343, -+STORE, 94418513977344, 94418516090879, -+ERASE, 94418513977344, 94418516090879, -+STORE, 94418516070400, 94418516082687, -+STORE, 94418516082688, 94418516090879, -+STORE, 140556479520768, 140556481773567, -+SNULL, 140556479664127, 140556481773567, -+STORE, 140556479520768, 140556479664127, -+STORE, 140556479664128, 140556481773567, -+ERASE, 140556479664128, 140556481773567, -+STORE, 140556481761280, 140556481769471, -+STORE, 140556481769472, 140556481773567, -+STORE, 140728767148032, 140728767152127, -+STORE, 140728767135744, 140728767148031, -+STORE, 140556481732608, 140556481761279, -+STORE, 140556481724416, 140556481732607, -+STORE, 140556475723776, 140556479520767, -+SNULL, 140556475723776, 140556477382655, -+STORE, 140556477382656, 140556479520767, -+STORE, 140556475723776, 140556477382655, -+SNULL, 140556479479807, 140556479520767, -+STORE, 140556477382656, 140556479479807, -+STORE, 140556479479808, 140556479520767, -+SNULL, 140556479479808, 140556479504383, -+STORE, 140556479504384, 140556479520767, -+STORE, 140556479479808, 140556479504383, -+ERASE, 140556479479808, 140556479504383, -+STORE, 140556479479808, 140556479504383, -+ERASE, 140556479504384, 140556479520767, -+STORE, 140556479504384, 140556479520767, -+SNULL, 140556479496191, 140556479504383, -+STORE, 140556479479808, 140556479496191, -+STORE, 140556479496192, 140556479504383, -+SNULL, 94418516078591, 94418516082687, -+STORE, 94418516070400, 94418516078591, -+STORE, 94418516078592, 94418516082687, -+SNULL, 140556481765375, 140556481769471, -+STORE, 140556481761280, 140556481765375, -+STORE, 140556481765376, 140556481769471, -+ERASE, 140556481732608, 140556481761279, -+STORE, 94418541113344, 94418541248511, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723945873408, 140737488351231, -+SNULL, 140723945881599, 140737488351231, -+STORE, 140723945873408, 140723945881599, -+STORE, 140723945742336, 140723945881599, -+STORE, 94543169773568, 94543171997695, -+SNULL, 94543169884159, 94543171997695, -+STORE, 94543169773568, 94543169884159, -+STORE, 94543169884160, 94543171997695, -+ERASE, 94543169884160, 94543171997695, -+STORE, 94543171977216, 94543171989503, -+STORE, 94543171989504, 94543171997695, -+STORE, 139890420883456, 139890423136255, -+SNULL, 139890421026815, 139890423136255, -+STORE, 139890420883456, 139890421026815, -+STORE, 139890421026816, 139890423136255, -+ERASE, 139890421026816, 139890423136255, -+STORE, 139890423123968, 139890423132159, -+STORE, 139890423132160, 139890423136255, -+STORE, 140723946102784, 140723946106879, -+STORE, 140723946090496, 140723946102783, -+STORE, 139890423095296, 139890423123967, -+STORE, 139890423087104, 139890423095295, -+STORE, 139890417086464, 139890420883455, -+SNULL, 139890417086464, 139890418745343, -+STORE, 139890418745344, 139890420883455, -+STORE, 139890417086464, 139890418745343, -+SNULL, 139890420842495, 139890420883455, -+STORE, 139890418745344, 139890420842495, -+STORE, 139890420842496, 139890420883455, -+SNULL, 139890420842496, 139890420867071, -+STORE, 139890420867072, 139890420883455, -+STORE, 139890420842496, 139890420867071, -+ERASE, 139890420842496, 139890420867071, -+STORE, 139890420842496, 139890420867071, -+ERASE, 139890420867072, 139890420883455, -+STORE, 139890420867072, 139890420883455, -+SNULL, 139890420858879, 139890420867071, -+STORE, 139890420842496, 139890420858879, -+STORE, 139890420858880, 139890420867071, -+SNULL, 94543171985407, 94543171989503, -+STORE, 94543171977216, 94543171985407, -+STORE, 94543171985408, 94543171989503, -+SNULL, 139890423128063, 139890423132159, -+STORE, 139890423123968, 139890423128063, -+STORE, 139890423128064, 139890423132159, -+ERASE, 139890423095296, 139890423123967, -+STORE, 94543197097984, 94543197233151, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140736205979648, 140737488351231, -+SNULL, 140736205987839, 140737488351231, -+STORE, 140736205979648, 140736205987839, -+STORE, 140736205848576, 140736205987839, -+STORE, 94913209913344, 94913212137471, -+SNULL, 94913210023935, 94913212137471, -+STORE, 94913209913344, 94913210023935, -+STORE, 94913210023936, 94913212137471, -+ERASE, 94913210023936, 94913212137471, -+STORE, 94913212116992, 94913212129279, -+STORE, 94913212129280, 94913212137471, -+STORE, 140006323052544, 140006325305343, -+SNULL, 140006323195903, 140006325305343, -+STORE, 140006323052544, 140006323195903, -+STORE, 140006323195904, 140006325305343, -+ERASE, 140006323195904, 140006325305343, -+STORE, 140006325293056, 140006325301247, -+STORE, 140006325301248, 140006325305343, -+STORE, 140736206716928, 140736206721023, -+STORE, 140736206704640, 140736206716927, -+STORE, 140006325264384, 140006325293055, -+STORE, 140006325256192, 140006325264383, -+STORE, 140006319255552, 140006323052543, -+SNULL, 140006319255552, 140006320914431, -+STORE, 140006320914432, 140006323052543, -+STORE, 140006319255552, 140006320914431, -+SNULL, 140006323011583, 140006323052543, -+STORE, 140006320914432, 140006323011583, -+STORE, 140006323011584, 140006323052543, -+SNULL, 140006323011584, 140006323036159, -+STORE, 140006323036160, 140006323052543, -+STORE, 140006323011584, 140006323036159, -+ERASE, 140006323011584, 140006323036159, -+STORE, 140006323011584, 140006323036159, -+ERASE, 140006323036160, 140006323052543, -+STORE, 140006323036160, 140006323052543, -+SNULL, 140006323027967, 140006323036159, -+STORE, 140006323011584, 140006323027967, -+STORE, 140006323027968, 140006323036159, -+SNULL, 94913212125183, 94913212129279, -+STORE, 94913212116992, 94913212125183, -+STORE, 94913212125184, 94913212129279, -+SNULL, 140006325297151, 140006325301247, -+STORE, 140006325293056, 140006325297151, -+STORE, 140006325297152, 140006325301247, -+ERASE, 140006325264384, 140006325293055, -+STORE, 94913239932928, 94913240068095, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140726926897152, 140737488351231, -+SNULL, 140726926905343, 140737488351231, -+STORE, 140726926897152, 140726926905343, -+STORE, 140726926766080, 140726926905343, -+STORE, 94213246820352, 94213249044479, -+SNULL, 94213246930943, 94213249044479, -+STORE, 94213246820352, 94213246930943, -+STORE, 94213246930944, 94213249044479, -+ERASE, 94213246930944, 94213249044479, -+STORE, 94213249024000, 94213249036287, -+STORE, 94213249036288, 94213249044479, -+STORE, 140368830242816, 140368832495615, -+SNULL, 140368830386175, 140368832495615, -+STORE, 140368830242816, 140368830386175, -+STORE, 140368830386176, 140368832495615, -+ERASE, 140368830386176, 140368832495615, -+STORE, 140368832483328, 140368832491519, -+STORE, 140368832491520, 140368832495615, -+STORE, 140726926999552, 140726927003647, -+STORE, 140726926987264, 140726926999551, -+STORE, 140368832454656, 140368832483327, -+STORE, 140368832446464, 140368832454655, -+STORE, 140368826445824, 140368830242815, -+SNULL, 140368826445824, 140368828104703, -+STORE, 140368828104704, 140368830242815, -+STORE, 140368826445824, 140368828104703, -+SNULL, 140368830201855, 140368830242815, -+STORE, 140368828104704, 140368830201855, -+STORE, 140368830201856, 140368830242815, -+SNULL, 140368830201856, 140368830226431, -+STORE, 140368830226432, 140368830242815, -+STORE, 140368830201856, 140368830226431, -+ERASE, 140368830201856, 140368830226431, -+STORE, 140368830201856, 140368830226431, -+ERASE, 140368830226432, 140368830242815, -+STORE, 140368830226432, 140368830242815, -+SNULL, 140368830218239, 140368830226431, -+STORE, 140368830201856, 140368830218239, -+STORE, 140368830218240, 140368830226431, -+SNULL, 94213249032191, 94213249036287, -+STORE, 94213249024000, 94213249032191, -+STORE, 94213249032192, 94213249036287, -+SNULL, 140368832487423, 140368832491519, -+STORE, 140368832483328, 140368832487423, -+STORE, 140368832487424, 140368832491519, -+ERASE, 140368832454656, 140368832483327, -+STORE, 94213267435520, 94213267570687, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728954130432, 140737488351231, -+SNULL, 140728954138623, 140737488351231, -+STORE, 140728954130432, 140728954138623, -+STORE, 140728953999360, 140728954138623, -+STORE, 94672570966016, 94672573190143, -+SNULL, 94672571076607, 94672573190143, -+STORE, 94672570966016, 94672571076607, -+STORE, 94672571076608, 94672573190143, -+ERASE, 94672571076608, 94672573190143, -+STORE, 94672573169664, 94672573181951, -+STORE, 94672573181952, 94672573190143, -+STORE, 140201696735232, 140201698988031, -+SNULL, 140201696878591, 140201698988031, -+STORE, 140201696735232, 140201696878591, -+STORE, 140201696878592, 140201698988031, -+ERASE, 140201696878592, 140201698988031, -+STORE, 140201698975744, 140201698983935, -+STORE, 140201698983936, 140201698988031, -+STORE, 140728954163200, 140728954167295, -+STORE, 140728954150912, 140728954163199, -+STORE, 140201698947072, 140201698975743, -+STORE, 140201698938880, 140201698947071, -+STORE, 140201692938240, 140201696735231, -+SNULL, 140201692938240, 140201694597119, -+STORE, 140201694597120, 140201696735231, -+STORE, 140201692938240, 140201694597119, -+SNULL, 140201696694271, 140201696735231, -+STORE, 140201694597120, 140201696694271, -+STORE, 140201696694272, 140201696735231, -+SNULL, 140201696694272, 140201696718847, -+STORE, 140201696718848, 140201696735231, -+STORE, 140201696694272, 140201696718847, -+ERASE, 140201696694272, 140201696718847, -+STORE, 140201696694272, 140201696718847, -+ERASE, 140201696718848, 140201696735231, -+STORE, 140201696718848, 140201696735231, -+SNULL, 140201696710655, 140201696718847, -+STORE, 140201696694272, 140201696710655, -+STORE, 140201696710656, 140201696718847, -+SNULL, 94672573177855, 94672573181951, -+STORE, 94672573169664, 94672573177855, -+STORE, 94672573177856, 94672573181951, -+SNULL, 140201698979839, 140201698983935, -+STORE, 140201698975744, 140201698979839, -+STORE, 140201698979840, 140201698983935, -+ERASE, 140201698947072, 140201698975743, -+STORE, 94672595689472, 94672595824639, -+STORE, 94114394132480, 94114394345471, -+STORE, 94114396442624, 94114396446719, -+STORE, 94114396446720, 94114396454911, -+STORE, 94114396454912, 94114396467199, -+STORE, 94114421575680, 94114428256255, -+STORE, 139934313955328, 139934315614207, -+STORE, 139934315614208, 139934317711359, -+STORE, 139934317711360, 139934317727743, -+STORE, 139934317727744, 139934317735935, -+STORE, 139934317735936, 139934317752319, -+STORE, 139934317752320, 139934317764607, -+STORE, 139934317764608, 139934319857663, -+STORE, 139934319857664, 139934319861759, -+STORE, 139934319861760, 139934319865855, -+STORE, 139934319865856, 139934320009215, -+STORE, 139934320377856, 139934322061311, -+STORE, 139934322061312, 139934322077695, -+STORE, 139934322106368, 139934322110463, -+STORE, 139934322110464, 139934322114559, -+STORE, 139934322114560, 139934322118655, -+STORE, 140731200376832, 140731200516095, -+STORE, 140731200929792, 140731200942079, -+STORE, 140731200942080, 140731200946175, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140721532362752, 140737488351231, -+SNULL, 140721532370943, 140737488351231, -+STORE, 140721532362752, 140721532370943, -+STORE, 140721532231680, 140721532370943, -+STORE, 94467222597632, 94467224821759, -+SNULL, 94467222708223, 94467224821759, -+STORE, 94467222597632, 94467222708223, -+STORE, 94467222708224, 94467224821759, -+ERASE, 94467222708224, 94467224821759, -+STORE, 94467224801280, 94467224813567, -+STORE, 94467224813568, 94467224821759, -+STORE, 140191433543680, 140191435796479, -+SNULL, 140191433687039, 140191435796479, -+STORE, 140191433543680, 140191433687039, -+STORE, 140191433687040, 140191435796479, -+ERASE, 140191433687040, 140191435796479, -+STORE, 140191435784192, 140191435792383, -+STORE, 140191435792384, 140191435796479, -+STORE, 140721533034496, 140721533038591, -+STORE, 140721533022208, 140721533034495, -+STORE, 140191435755520, 140191435784191, -+STORE, 140191435747328, 140191435755519, -+STORE, 140191429746688, 140191433543679, -+SNULL, 140191429746688, 140191431405567, -+STORE, 140191431405568, 140191433543679, -+STORE, 140191429746688, 140191431405567, -+SNULL, 140191433502719, 140191433543679, -+STORE, 140191431405568, 140191433502719, -+STORE, 140191433502720, 140191433543679, -+SNULL, 140191433502720, 140191433527295, -+STORE, 140191433527296, 140191433543679, -+STORE, 140191433502720, 140191433527295, -+ERASE, 140191433502720, 140191433527295, -+STORE, 140191433502720, 140191433527295, -+ERASE, 140191433527296, 140191433543679, -+STORE, 140191433527296, 140191433543679, -+SNULL, 140191433519103, 140191433527295, -+STORE, 140191433502720, 140191433519103, -+STORE, 140191433519104, 140191433527295, -+SNULL, 94467224809471, 94467224813567, -+STORE, 94467224801280, 94467224809471, -+STORE, 94467224809472, 94467224813567, -+SNULL, 140191435788287, 140191435792383, -+STORE, 140191435784192, 140191435788287, -+STORE, 140191435788288, 140191435792383, -+ERASE, 140191435755520, 140191435784191, -+STORE, 94467251847168, 94467251982335, -+STORE, 94367895400448, 94367895613439, -+STORE, 94367897710592, 94367897714687, -+STORE, 94367897714688, 94367897722879, -+STORE, 94367897722880, 94367897735167, -+STORE, 94367925264384, 94367926861823, -+STORE, 139801317548032, 139801319206911, -+STORE, 139801319206912, 139801321304063, -+STORE, 139801321304064, 139801321320447, -+STORE, 139801321320448, 139801321328639, -+STORE, 139801321328640, 139801321345023, -+STORE, 139801321345024, 139801321357311, -+STORE, 139801321357312, 139801323450367, -+STORE, 139801323450368, 139801323454463, -+STORE, 139801323454464, 139801323458559, -+STORE, 139801323458560, 139801323601919, -+STORE, 139801323970560, 139801325654015, -+STORE, 139801325654016, 139801325670399, -+STORE, 139801325699072, 139801325703167, -+STORE, 139801325703168, 139801325707263, -+STORE, 139801325707264, 139801325711359, -+STORE, 140724442861568, 140724443000831, -+STORE, 140724443611136, 140724443623423, -+STORE, 140724443623424, 140724443627519, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731353149440, 140737488351231, -+SNULL, 140731353157631, 140737488351231, -+STORE, 140731353149440, 140731353157631, -+STORE, 140731353018368, 140731353157631, -+STORE, 94310379503616, 94310381838335, -+SNULL, 94310379716607, 94310381838335, -+STORE, 94310379503616, 94310379716607, -+STORE, 94310379716608, 94310381838335, -+ERASE, 94310379716608, 94310381838335, -+STORE, 94310381813760, 94310381826047, -+STORE, 94310381826048, 94310381838335, -+STORE, 140515434659840, 140515436912639, -+SNULL, 140515434803199, 140515436912639, -+STORE, 140515434659840, 140515434803199, -+STORE, 140515434803200, 140515436912639, -+ERASE, 140515434803200, 140515436912639, -+STORE, 140515436900352, 140515436908543, -+STORE, 140515436908544, 140515436912639, -+STORE, 140731353886720, 140731353890815, -+STORE, 140731353874432, 140731353886719, -+STORE, 140515436871680, 140515436900351, -+STORE, 140515436863488, 140515436871679, -+STORE, 140515432546304, 140515434659839, -+SNULL, 140515432546304, 140515432558591, -+STORE, 140515432558592, 140515434659839, -+STORE, 140515432546304, 140515432558591, -+SNULL, 140515434651647, 140515434659839, -+STORE, 140515432558592, 140515434651647, -+STORE, 140515434651648, 140515434659839, -+ERASE, 140515434651648, 140515434659839, -+STORE, 140515434651648, 140515434659839, -+STORE, 140515428749312, 140515432546303, -+SNULL, 140515428749312, 140515430408191, -+STORE, 140515430408192, 140515432546303, -+STORE, 140515428749312, 140515430408191, -+SNULL, 140515432505343, 140515432546303, -+STORE, 140515430408192, 140515432505343, -+STORE, 140515432505344, 140515432546303, -+SNULL, 140515432505344, 140515432529919, -+STORE, 140515432529920, 140515432546303, -+STORE, 140515432505344, 140515432529919, -+ERASE, 140515432505344, 140515432529919, -+STORE, 140515432505344, 140515432529919, -+ERASE, 140515432529920, 140515432546303, -+STORE, 140515432529920, 140515432546303, -+STORE, 140515436855296, 140515436871679, -+SNULL, 140515432521727, 140515432529919, -+STORE, 140515432505344, 140515432521727, -+STORE, 140515432521728, 140515432529919, -+SNULL, 140515434655743, 140515434659839, -+STORE, 140515434651648, 140515434655743, -+STORE, 140515434655744, 140515434659839, -+SNULL, 94310381817855, 94310381826047, -+STORE, 94310381813760, 94310381817855, -+STORE, 94310381817856, 94310381826047, -+SNULL, 140515436904447, 140515436908543, -+STORE, 140515436900352, 140515436904447, -+STORE, 140515436904448, 140515436908543, -+ERASE, 140515436871680, 140515436900351, -+STORE, 94310395457536, 94310395592703, -+STORE, 140515435171840, 140515436855295, -+STORE, 94310395457536, 94310395727871, -+STORE, 94310395457536, 94310395863039, -+STORE, 94310395457536, 94310396047359, -+SNULL, 94310396022783, 94310396047359, -+STORE, 94310395457536, 94310396022783, -+STORE, 94310396022784, 94310396047359, -+ERASE, 94310396022784, 94310396047359, -+STORE, 94310395457536, 94310396157951, -+STORE, 94310395457536, 94310396293119, -+SNULL, 94310396276735, 94310396293119, -+STORE, 94310395457536, 94310396276735, -+STORE, 94310396276736, 94310396293119, -+ERASE, 94310396276736, 94310396293119, -+STORE, 94310395457536, 94310396411903, -+SNULL, 94310396383231, 94310396411903, -+STORE, 94310395457536, 94310396383231, -+STORE, 94310396383232, 94310396411903, -+ERASE, 94310396383232, 94310396411903, -+STORE, 94310395457536, 94310396522495, -+STORE, 94310395457536, 94310396674047, -+SNULL, 94310396657663, 94310396674047, -+STORE, 94310395457536, 94310396657663, -+STORE, 94310396657664, 94310396674047, -+ERASE, 94310396657664, 94310396674047, -+SNULL, 94310396624895, 94310396657663, -+STORE, 94310395457536, 94310396624895, -+STORE, 94310396624896, 94310396657663, -+ERASE, 94310396624896, 94310396657663, -+STORE, 94310395457536, 94310396776447, -+SNULL, 94310396764159, 94310396776447, -+STORE, 94310395457536, 94310396764159, -+STORE, 94310396764160, 94310396776447, -+ERASE, 94310396764160, 94310396776447, -+SNULL, 94310396739583, 94310396764159, -+STORE, 94310395457536, 94310396739583, -+STORE, 94310396739584, 94310396764159, -+ERASE, 94310396739584, 94310396764159, -+STORE, 94310395457536, 94310396882943, -+STORE, 94310395457536, 94310397018111, -+STORE, 94310395457536, 94310397161471, -+STORE, 94310395457536, 94310397300735, -+SNULL, 94310397292543, 94310397300735, -+STORE, 94310395457536, 94310397292543, -+STORE, 94310397292544, 94310397300735, -+ERASE, 94310397292544, 94310397300735, -+STORE, 94359222210560, 94359222423551, -+STORE, 94359224520704, 94359224524799, -+STORE, 94359224524800, 94359224532991, -+STORE, 94359224532992, 94359224545279, -+STORE, 94359238348800, 94359239385087, -+STORE, 140675699838976, 140675701497855, -+STORE, 140675701497856, 140675703595007, -+STORE, 140675703595008, 140675703611391, -+STORE, 140675703611392, 140675703619583, -+STORE, 140675703619584, 140675703635967, -+STORE, 140675703635968, 140675703648255, -+STORE, 140675703648256, 140675705741311, -+STORE, 140675705741312, 140675705745407, -+STORE, 140675705745408, 140675705749503, -+STORE, 140675705749504, 140675705892863, -+STORE, 140675706261504, 140675707944959, -+STORE, 140675707944960, 140675707961343, -+STORE, 140675707990016, 140675707994111, -+STORE, 140675707994112, 140675707998207, -+STORE, 140675707998208, 140675708002303, -+STORE, 140721324634112, 140721324773375, -+STORE, 140721324810240, 140721324822527, -+STORE, 140721324822528, 140721324826623, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724099678208, 140737488351231, -+SNULL, 140724099686399, 140737488351231, -+STORE, 140724099678208, 140724099686399, -+STORE, 140724099547136, 140724099686399, -+STORE, 94586638516224, 94586640850943, -+SNULL, 94586638729215, 94586640850943, -+STORE, 94586638516224, 94586638729215, -+STORE, 94586638729216, 94586640850943, -+ERASE, 94586638729216, 94586640850943, -+STORE, 94586640826368, 94586640838655, -+STORE, 94586640838656, 94586640850943, -+STORE, 140371033796608, 140371036049407, -+SNULL, 140371033939967, 140371036049407, -+STORE, 140371033796608, 140371033939967, -+STORE, 140371033939968, 140371036049407, -+ERASE, 140371033939968, 140371036049407, -+STORE, 140371036037120, 140371036045311, -+STORE, 140371036045312, 140371036049407, -+STORE, 140724100001792, 140724100005887, -+STORE, 140724099989504, 140724100001791, -+STORE, 140371036008448, 140371036037119, -+STORE, 140371036000256, 140371036008447, -+STORE, 140371031683072, 140371033796607, -+SNULL, 140371031683072, 140371031695359, -+STORE, 140371031695360, 140371033796607, -+STORE, 140371031683072, 140371031695359, -+SNULL, 140371033788415, 140371033796607, -+STORE, 140371031695360, 140371033788415, -+STORE, 140371033788416, 140371033796607, -+ERASE, 140371033788416, 140371033796607, -+STORE, 140371033788416, 140371033796607, -+STORE, 140371027886080, 140371031683071, -+SNULL, 140371027886080, 140371029544959, -+STORE, 140371029544960, 140371031683071, -+STORE, 140371027886080, 140371029544959, -+SNULL, 140371031642111, 140371031683071, -+STORE, 140371029544960, 140371031642111, -+STORE, 140371031642112, 140371031683071, -+SNULL, 140371031642112, 140371031666687, -+STORE, 140371031666688, 140371031683071, -+STORE, 140371031642112, 140371031666687, -+ERASE, 140371031642112, 140371031666687, -+STORE, 140371031642112, 140371031666687, -+ERASE, 140371031666688, 140371031683071, -+STORE, 140371031666688, 140371031683071, -+STORE, 140371035992064, 140371036008447, -+SNULL, 140371031658495, 140371031666687, -+STORE, 140371031642112, 140371031658495, -+STORE, 140371031658496, 140371031666687, -+SNULL, 140371033792511, 140371033796607, -+STORE, 140371033788416, 140371033792511, -+STORE, 140371033792512, 140371033796607, -+SNULL, 94586640830463, 94586640838655, -+STORE, 94586640826368, 94586640830463, -+STORE, 94586640830464, 94586640838655, -+SNULL, 140371036041215, 140371036045311, -+STORE, 140371036037120, 140371036041215, -+STORE, 140371036041216, 140371036045311, -+ERASE, 140371036008448, 140371036037119, -+STORE, 94586663849984, 94586663985151, -+STORE, 140371034308608, 140371035992063, -+STORE, 94586663849984, 94586664120319, -+STORE, 94586663849984, 94586664255487, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140727532937216, 140737488351231, -+SNULL, 140727532945407, 140737488351231, -+STORE, 140727532937216, 140727532945407, -+STORE, 140727532806144, 140727532945407, -+STORE, 94849780191232, 94849782525951, -+SNULL, 94849780404223, 94849782525951, -+STORE, 94849780191232, 94849780404223, -+STORE, 94849780404224, 94849782525951, -+ERASE, 94849780404224, 94849782525951, -+STORE, 94849782501376, 94849782513663, -+STORE, 94849782513664, 94849782525951, -+STORE, 140382070218752, 140382072471551, -+SNULL, 140382070362111, 140382072471551, -+STORE, 140382070218752, 140382070362111, -+STORE, 140382070362112, 140382072471551, -+ERASE, 140382070362112, 140382072471551, -+STORE, 140382072459264, 140382072467455, -+STORE, 140382072467456, 140382072471551, -+STORE, 140727533092864, 140727533096959, -+STORE, 140727533080576, 140727533092863, -+STORE, 140382072430592, 140382072459263, -+STORE, 140382072422400, 140382072430591, -+STORE, 140382068105216, 140382070218751, -+SNULL, 140382068105216, 140382068117503, -+STORE, 140382068117504, 140382070218751, -+STORE, 140382068105216, 140382068117503, -+SNULL, 140382070210559, 140382070218751, -+STORE, 140382068117504, 140382070210559, -+STORE, 140382070210560, 140382070218751, -+ERASE, 140382070210560, 140382070218751, -+STORE, 140382070210560, 140382070218751, -+STORE, 140382064308224, 140382068105215, -+SNULL, 140382064308224, 140382065967103, -+STORE, 140382065967104, 140382068105215, -+STORE, 140382064308224, 140382065967103, -+SNULL, 140382068064255, 140382068105215, -+STORE, 140382065967104, 140382068064255, -+STORE, 140382068064256, 140382068105215, -+SNULL, 140382068064256, 140382068088831, -+STORE, 140382068088832, 140382068105215, -+STORE, 140382068064256, 140382068088831, -+ERASE, 140382068064256, 140382068088831, -+STORE, 140382068064256, 140382068088831, -+ERASE, 140382068088832, 140382068105215, -+STORE, 140382068088832, 140382068105215, -+STORE, 140382072414208, 140382072430591, -+SNULL, 140382068080639, 140382068088831, -+STORE, 140382068064256, 140382068080639, -+STORE, 140382068080640, 140382068088831, -+SNULL, 140382070214655, 140382070218751, -+STORE, 140382070210560, 140382070214655, -+STORE, 140382070214656, 140382070218751, -+SNULL, 94849782505471, 94849782513663, -+STORE, 94849782501376, 94849782505471, -+STORE, 94849782505472, 94849782513663, -+SNULL, 140382072463359, 140382072467455, -+STORE, 140382072459264, 140382072463359, -+STORE, 140382072463360, 140382072467455, -+ERASE, 140382072430592, 140382072459263, -+STORE, 94849782845440, 94849782980607, -+STORE, 140382070730752, 140382072414207, -+STORE, 94849782845440, 94849783115775, -+STORE, 94849782845440, 94849783250943, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722594377728, 140737488351231, -+SNULL, 140722594385919, 140737488351231, -+STORE, 140722594377728, 140722594385919, -+STORE, 140722594246656, 140722594385919, -+STORE, 94421466353664, 94421468577791, -+SNULL, 94421466464255, 94421468577791, -+STORE, 94421466353664, 94421466464255, -+STORE, 94421466464256, 94421468577791, -+ERASE, 94421466464256, 94421468577791, -+STORE, 94421468557312, 94421468569599, -+STORE, 94421468569600, 94421468577791, -+STORE, 140345458057216, 140345460310015, -+SNULL, 140345458200575, 140345460310015, -+STORE, 140345458057216, 140345458200575, -+STORE, 140345458200576, 140345460310015, -+ERASE, 140345458200576, 140345460310015, -+STORE, 140345460297728, 140345460305919, -+STORE, 140345460305920, 140345460310015, -+STORE, 140722595557376, 140722595561471, -+STORE, 140722595545088, 140722595557375, -+STORE, 140345460269056, 140345460297727, -+STORE, 140345460260864, 140345460269055, -+STORE, 140345454260224, 140345458057215, -+SNULL, 140345454260224, 140345455919103, -+STORE, 140345455919104, 140345458057215, -+STORE, 140345454260224, 140345455919103, -+SNULL, 140345458016255, 140345458057215, -+STORE, 140345455919104, 140345458016255, -+STORE, 140345458016256, 140345458057215, -+SNULL, 140345458016256, 140345458040831, -+STORE, 140345458040832, 140345458057215, -+STORE, 140345458016256, 140345458040831, -+ERASE, 140345458016256, 140345458040831, -+STORE, 140345458016256, 140345458040831, -+ERASE, 140345458040832, 140345458057215, -+STORE, 140345458040832, 140345458057215, -+SNULL, 140345458032639, 140345458040831, -+STORE, 140345458016256, 140345458032639, -+STORE, 140345458032640, 140345458040831, -+SNULL, 94421468565503, 94421468569599, -+STORE, 94421468557312, 94421468565503, -+STORE, 94421468565504, 94421468569599, -+SNULL, 140345460301823, 140345460305919, -+STORE, 140345460297728, 140345460301823, -+STORE, 140345460301824, 140345460305919, -+ERASE, 140345460269056, 140345460297727, -+STORE, 94421496004608, 94421496139775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140726096302080, 140737488351231, -+SNULL, 140726096310271, 140737488351231, -+STORE, 140726096302080, 140726096310271, -+STORE, 140726096171008, 140726096310271, -+STORE, 94101992124416, 94101994459135, -+SNULL, 94101992337407, 94101994459135, -+STORE, 94101992124416, 94101992337407, -+STORE, 94101992337408, 94101994459135, -+ERASE, 94101992337408, 94101994459135, -+STORE, 94101994434560, 94101994446847, -+STORE, 94101994446848, 94101994459135, -+STORE, 140192085594112, 140192087846911, -+SNULL, 140192085737471, 140192087846911, -+STORE, 140192085594112, 140192085737471, -+STORE, 140192085737472, 140192087846911, -+ERASE, 140192085737472, 140192087846911, -+STORE, 140192087834624, 140192087842815, -+STORE, 140192087842816, 140192087846911, -+STORE, 140726096375808, 140726096379903, -+STORE, 140726096363520, 140726096375807, -+STORE, 140192087805952, 140192087834623, -+STORE, 140192087797760, 140192087805951, -+STORE, 140192083480576, 140192085594111, -+SNULL, 140192083480576, 140192083492863, -+STORE, 140192083492864, 140192085594111, -+STORE, 140192083480576, 140192083492863, -+SNULL, 140192085585919, 140192085594111, -+STORE, 140192083492864, 140192085585919, -+STORE, 140192085585920, 140192085594111, -+ERASE, 140192085585920, 140192085594111, -+STORE, 140192085585920, 140192085594111, -+STORE, 140192079683584, 140192083480575, -+SNULL, 140192079683584, 140192081342463, -+STORE, 140192081342464, 140192083480575, -+STORE, 140192079683584, 140192081342463, -+SNULL, 140192083439615, 140192083480575, -+STORE, 140192081342464, 140192083439615, -+STORE, 140192083439616, 140192083480575, -+SNULL, 140192083439616, 140192083464191, -+STORE, 140192083464192, 140192083480575, -+STORE, 140192083439616, 140192083464191, -+ERASE, 140192083439616, 140192083464191, -+STORE, 140192083439616, 140192083464191, -+ERASE, 140192083464192, 140192083480575, -+STORE, 140192083464192, 140192083480575, -+STORE, 140192087789568, 140192087805951, -+SNULL, 140192083455999, 140192083464191, -+STORE, 140192083439616, 140192083455999, -+STORE, 140192083456000, 140192083464191, -+SNULL, 140192085590015, 140192085594111, -+STORE, 140192085585920, 140192085590015, -+STORE, 140192085590016, 140192085594111, -+SNULL, 94101994438655, 94101994446847, -+STORE, 94101994434560, 94101994438655, -+STORE, 94101994438656, 94101994446847, -+SNULL, 140192087838719, 140192087842815, -+STORE, 140192087834624, 140192087838719, -+STORE, 140192087838720, 140192087842815, -+ERASE, 140192087805952, 140192087834623, -+STORE, 94102011887616, 94102012022783, -+STORE, 140192086106112, 140192087789567, -+STORE, 94102011887616, 94102012157951, -+STORE, 94102011887616, 94102012293119, -+STORE, 94102011887616, 94102012440575, -+SNULL, 94102012428287, 94102012440575, -+STORE, 94102011887616, 94102012428287, -+STORE, 94102012428288, 94102012440575, -+ERASE, 94102012428288, 94102012440575, -+STORE, 94102011887616, 94102012579839, -+STORE, 94102011887616, 94102012715007, -+SNULL, 94102012694527, 94102012715007, -+STORE, 94102011887616, 94102012694527, -+STORE, 94102012694528, 94102012715007, -+ERASE, 94102012694528, 94102012715007, -+STORE, 94102011887616, 94102012833791, -+STORE, 94102011887616, 94102012968959, -+SNULL, 94102012927999, 94102012968959, -+STORE, 94102011887616, 94102012927999, -+STORE, 94102012928000, 94102012968959, -+ERASE, 94102012928000, 94102012968959, -+STORE, 94102011887616, 94102013091839, -+SNULL, 94102013075455, 94102013091839, -+STORE, 94102011887616, 94102013075455, -+STORE, 94102013075456, 94102013091839, -+ERASE, 94102013075456, 94102013091839, -+STORE, 94102011887616, 94102013210623, -+STORE, 94102011887616, 94102013345791, -+STORE, 93968727965696, 93968728178687, -+STORE, 93968730275840, 93968730279935, -+STORE, 93968730279936, 93968730288127, -+STORE, 93968730288128, 93968730300415, -+STORE, 93968731140096, 93968732704767, -+STORE, 140588443168768, 140588444827647, -+STORE, 140588444827648, 140588446924799, -+STORE, 140588446924800, 140588446941183, -+STORE, 140588446941184, 140588446949375, -+STORE, 140588446949376, 140588446965759, -+STORE, 140588446965760, 140588446978047, -+STORE, 140588446978048, 140588449071103, -+STORE, 140588449071104, 140588449075199, -+STORE, 140588449075200, 140588449079295, -+STORE, 140588449079296, 140588449222655, -+STORE, 140588449591296, 140588451274751, -+STORE, 140588451274752, 140588451291135, -+STORE, 140588451319808, 140588451323903, -+STORE, 140588451323904, 140588451327999, -+STORE, 140588451328000, 140588451332095, -+STORE, 140733877239808, 140733877379071, -+STORE, 140733878702080, 140733878714367, -+STORE, 140733878714368, 140733878718463, -+STORE, 93968727965696, 93968728178687, -+STORE, 93968730275840, 93968730279935, -+STORE, 93968730279936, 93968730288127, -+STORE, 93968730288128, 93968730300415, -+STORE, 93968731140096, 93968732991487, -+STORE, 140588443168768, 140588444827647, -+STORE, 140588444827648, 140588446924799, -+STORE, 140588446924800, 140588446941183, -+STORE, 140588446941184, 140588446949375, -+STORE, 140588446949376, 140588446965759, -+STORE, 140588446965760, 140588446978047, -+STORE, 140588446978048, 140588449071103, -+STORE, 140588449071104, 140588449075199, -+STORE, 140588449075200, 140588449079295, -+STORE, 140588449079296, 140588449222655, -+STORE, 140588449591296, 140588451274751, -+STORE, 140588451274752, 140588451291135, -+STORE, 140588451319808, 140588451323903, -+STORE, 140588451323904, 140588451327999, -+STORE, 140588451328000, 140588451332095, -+STORE, 140733877239808, 140733877379071, -+STORE, 140733878702080, 140733878714367, -+STORE, 140733878714368, 140733878718463, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733054472192, 140737488351231, -+SNULL, 140733054480383, 140737488351231, -+STORE, 140733054472192, 140733054480383, -+STORE, 140733054341120, 140733054480383, -+STORE, 93992873623552, 93992875847679, -+SNULL, 93992873734143, 93992875847679, -+STORE, 93992873623552, 93992873734143, -+STORE, 93992873734144, 93992875847679, -+ERASE, 93992873734144, 93992875847679, -+STORE, 93992875827200, 93992875839487, -+STORE, 93992875839488, 93992875847679, -+STORE, 139790881488896, 139790883741695, -+SNULL, 139790881632255, 139790883741695, -+STORE, 139790881488896, 139790881632255, -+STORE, 139790881632256, 139790883741695, -+ERASE, 139790881632256, 139790883741695, -+STORE, 139790883729408, 139790883737599, -+STORE, 139790883737600, 139790883741695, -+STORE, 140733054754816, 140733054758911, -+STORE, 140733054742528, 140733054754815, -+STORE, 139790883700736, 139790883729407, -+STORE, 139790883692544, 139790883700735, -+STORE, 139790877691904, 139790881488895, -+SNULL, 139790877691904, 139790879350783, -+STORE, 139790879350784, 139790881488895, -+STORE, 139790877691904, 139790879350783, -+SNULL, 139790881447935, 139790881488895, -+STORE, 139790879350784, 139790881447935, -+STORE, 139790881447936, 139790881488895, -+SNULL, 139790881447936, 139790881472511, -+STORE, 139790881472512, 139790881488895, -+STORE, 139790881447936, 139790881472511, -+ERASE, 139790881447936, 139790881472511, -+STORE, 139790881447936, 139790881472511, -+ERASE, 139790881472512, 139790881488895, -+STORE, 139790881472512, 139790881488895, -+SNULL, 139790881464319, 139790881472511, -+STORE, 139790881447936, 139790881464319, -+STORE, 139790881464320, 139790881472511, -+SNULL, 93992875835391, 93992875839487, -+STORE, 93992875827200, 93992875835391, -+STORE, 93992875835392, 93992875839487, -+SNULL, 139790883733503, 139790883737599, -+STORE, 139790883729408, 139790883733503, -+STORE, 139790883733504, 139790883737599, -+ERASE, 139790883700736, 139790883729407, -+STORE, 93992877031424, 93992877166591, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728550887424, 140737488351231, -+SNULL, 140728550895615, 140737488351231, -+STORE, 140728550887424, 140728550895615, -+STORE, 140728550756352, 140728550895615, -+STORE, 94707634077696, 94707636301823, -+SNULL, 94707634188287, 94707636301823, -+STORE, 94707634077696, 94707634188287, -+STORE, 94707634188288, 94707636301823, -+ERASE, 94707634188288, 94707636301823, -+STORE, 94707636281344, 94707636293631, -+STORE, 94707636293632, 94707636301823, -+STORE, 140553545666560, 140553547919359, -+SNULL, 140553545809919, 140553547919359, -+STORE, 140553545666560, 140553545809919, -+STORE, 140553545809920, 140553547919359, -+ERASE, 140553545809920, 140553547919359, -+STORE, 140553547907072, 140553547915263, -+STORE, 140553547915264, 140553547919359, -+STORE, 140728552374272, 140728552378367, -+STORE, 140728552361984, 140728552374271, -+STORE, 140553547878400, 140553547907071, -+STORE, 140553547870208, 140553547878399, -+STORE, 140553541869568, 140553545666559, -+SNULL, 140553541869568, 140553543528447, -+STORE, 140553543528448, 140553545666559, -+STORE, 140553541869568, 140553543528447, -+SNULL, 140553545625599, 140553545666559, -+STORE, 140553543528448, 140553545625599, -+STORE, 140553545625600, 140553545666559, -+SNULL, 140553545625600, 140553545650175, -+STORE, 140553545650176, 140553545666559, -+STORE, 140553545625600, 140553545650175, -+ERASE, 140553545625600, 140553545650175, -+STORE, 140553545625600, 140553545650175, -+ERASE, 140553545650176, 140553545666559, -+STORE, 140553545650176, 140553545666559, -+SNULL, 140553545641983, 140553545650175, -+STORE, 140553545625600, 140553545641983, -+STORE, 140553545641984, 140553545650175, -+SNULL, 94707636289535, 94707636293631, -+STORE, 94707636281344, 94707636289535, -+STORE, 94707636289536, 94707636293631, -+SNULL, 140553547911167, 140553547915263, -+STORE, 140553547907072, 140553547911167, -+STORE, 140553547911168, 140553547915263, -+ERASE, 140553547878400, 140553547907071, -+STORE, 94707651411968, 94707651547135, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140732168695808, 140737488351231, -+SNULL, 140732168703999, 140737488351231, -+STORE, 140732168695808, 140732168703999, -+STORE, 140732168564736, 140732168703999, -+STORE, 94454287859712, 94454290083839, -+SNULL, 94454287970303, 94454290083839, -+STORE, 94454287859712, 94454287970303, -+STORE, 94454287970304, 94454290083839, -+ERASE, 94454287970304, 94454290083839, -+STORE, 94454290063360, 94454290075647, -+STORE, 94454290075648, 94454290083839, -+STORE, 140564947107840, 140564949360639, -+SNULL, 140564947251199, 140564949360639, -+STORE, 140564947107840, 140564947251199, -+STORE, 140564947251200, 140564949360639, -+ERASE, 140564947251200, 140564949360639, -+STORE, 140564949348352, 140564949356543, -+STORE, 140564949356544, 140564949360639, -+STORE, 140732168843264, 140732168847359, -+STORE, 140732168830976, 140732168843263, -+STORE, 140564949319680, 140564949348351, -+STORE, 140564949311488, 140564949319679, -+STORE, 140564943310848, 140564947107839, -+SNULL, 140564943310848, 140564944969727, -+STORE, 140564944969728, 140564947107839, -+STORE, 140564943310848, 140564944969727, -+SNULL, 140564947066879, 140564947107839, -+STORE, 140564944969728, 140564947066879, -+STORE, 140564947066880, 140564947107839, -+SNULL, 140564947066880, 140564947091455, -+STORE, 140564947091456, 140564947107839, -+STORE, 140564947066880, 140564947091455, -+ERASE, 140564947066880, 140564947091455, -+STORE, 140564947066880, 140564947091455, -+ERASE, 140564947091456, 140564947107839, -+STORE, 140564947091456, 140564947107839, -+SNULL, 140564947083263, 140564947091455, -+STORE, 140564947066880, 140564947083263, -+STORE, 140564947083264, 140564947091455, -+SNULL, 94454290071551, 94454290075647, -+STORE, 94454290063360, 94454290071551, -+STORE, 94454290071552, 94454290075647, -+SNULL, 140564949352447, 140564949356543, -+STORE, 140564949348352, 140564949352447, -+STORE, 140564949352448, 140564949356543, -+ERASE, 140564949319680, 140564949348351, -+STORE, 94454316236800, 94454316371967, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735155617792, 140737488351231, -+SNULL, 140735155625983, 140737488351231, -+STORE, 140735155617792, 140735155625983, -+STORE, 140735155486720, 140735155625983, -+STORE, 93915969556480, 93915971780607, -+SNULL, 93915969667071, 93915971780607, -+STORE, 93915969556480, 93915969667071, -+STORE, 93915969667072, 93915971780607, -+ERASE, 93915969667072, 93915971780607, -+STORE, 93915971760128, 93915971772415, -+STORE, 93915971772416, 93915971780607, -+STORE, 140141164605440, 140141166858239, -+SNULL, 140141164748799, 140141166858239, -+STORE, 140141164605440, 140141164748799, -+STORE, 140141164748800, 140141166858239, -+ERASE, 140141164748800, 140141166858239, -+STORE, 140141166845952, 140141166854143, -+STORE, 140141166854144, 140141166858239, -+STORE, 140735155691520, 140735155695615, -+STORE, 140735155679232, 140735155691519, -+STORE, 140141166817280, 140141166845951, -+STORE, 140141166809088, 140141166817279, -+STORE, 140141160808448, 140141164605439, -+SNULL, 140141160808448, 140141162467327, -+STORE, 140141162467328, 140141164605439, -+STORE, 140141160808448, 140141162467327, -+SNULL, 140141164564479, 140141164605439, -+STORE, 140141162467328, 140141164564479, -+STORE, 140141164564480, 140141164605439, -+SNULL, 140141164564480, 140141164589055, -+STORE, 140141164589056, 140141164605439, -+STORE, 140141164564480, 140141164589055, -+ERASE, 140141164564480, 140141164589055, -+STORE, 140141164564480, 140141164589055, -+ERASE, 140141164589056, 140141164605439, -+STORE, 140141164589056, 140141164605439, -+SNULL, 140141164580863, 140141164589055, -+STORE, 140141164564480, 140141164580863, -+STORE, 140141164580864, 140141164589055, -+SNULL, 93915971768319, 93915971772415, -+STORE, 93915971760128, 93915971768319, -+STORE, 93915971768320, 93915971772415, -+SNULL, 140141166850047, 140141166854143, -+STORE, 140141166845952, 140141166850047, -+STORE, 140141166850048, 140141166854143, -+ERASE, 140141166817280, 140141166845951, -+STORE, 93916002775040, 93916002910207, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728988409856, 140737488351231, -+SNULL, 140728988418047, 140737488351231, -+STORE, 140728988409856, 140728988418047, -+STORE, 140728988278784, 140728988418047, -+STORE, 94021634813952, 94021637038079, -+SNULL, 94021634924543, 94021637038079, -+STORE, 94021634813952, 94021634924543, -+STORE, 94021634924544, 94021637038079, -+ERASE, 94021634924544, 94021637038079, -+STORE, 94021637017600, 94021637029887, -+STORE, 94021637029888, 94021637038079, -+STORE, 140638014038016, 140638016290815, -+SNULL, 140638014181375, 140638016290815, -+STORE, 140638014038016, 140638014181375, -+STORE, 140638014181376, 140638016290815, -+ERASE, 140638014181376, 140638016290815, -+STORE, 140638016278528, 140638016286719, -+STORE, 140638016286720, 140638016290815, -+STORE, 140728988536832, 140728988540927, -+STORE, 140728988524544, 140728988536831, -+STORE, 140638016249856, 140638016278527, -+STORE, 140638016241664, 140638016249855, -+STORE, 140638010241024, 140638014038015, -+SNULL, 140638010241024, 140638011899903, -+STORE, 140638011899904, 140638014038015, -+STORE, 140638010241024, 140638011899903, -+SNULL, 140638013997055, 140638014038015, -+STORE, 140638011899904, 140638013997055, -+STORE, 140638013997056, 140638014038015, -+SNULL, 140638013997056, 140638014021631, -+STORE, 140638014021632, 140638014038015, -+STORE, 140638013997056, 140638014021631, -+ERASE, 140638013997056, 140638014021631, -+STORE, 140638013997056, 140638014021631, -+ERASE, 140638014021632, 140638014038015, -+STORE, 140638014021632, 140638014038015, -+SNULL, 140638014013439, 140638014021631, -+STORE, 140638013997056, 140638014013439, -+STORE, 140638014013440, 140638014021631, -+SNULL, 94021637025791, 94021637029887, -+STORE, 94021637017600, 94021637025791, -+STORE, 94021637025792, 94021637029887, -+SNULL, 140638016282623, 140638016286719, -+STORE, 140638016278528, 140638016282623, -+STORE, 140638016282624, 140638016286719, -+ERASE, 140638016249856, 140638016278527, -+STORE, 94021643124736, 94021643259903, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731219275776, 140737488351231, -+SNULL, 140731219283967, 140737488351231, -+STORE, 140731219275776, 140731219283967, -+STORE, 140731219144704, 140731219283967, -+STORE, 93888803647488, 93888805871615, -+SNULL, 93888803758079, 93888805871615, -+STORE, 93888803647488, 93888803758079, -+STORE, 93888803758080, 93888805871615, -+ERASE, 93888803758080, 93888805871615, -+STORE, 93888805851136, 93888805863423, -+STORE, 93888805863424, 93888805871615, -+STORE, 139630576934912, 139630579187711, -+SNULL, 139630577078271, 139630579187711, -+STORE, 139630576934912, 139630577078271, -+STORE, 139630577078272, 139630579187711, -+ERASE, 139630577078272, 139630579187711, -+STORE, 139630579175424, 139630579183615, -+STORE, 139630579183616, 139630579187711, -+STORE, 140731219718144, 140731219722239, -+STORE, 140731219705856, 140731219718143, -+STORE, 139630579146752, 139630579175423, -+STORE, 139630579138560, 139630579146751, -+STORE, 139630573137920, 139630576934911, -+SNULL, 139630573137920, 139630574796799, -+STORE, 139630574796800, 139630576934911, -+STORE, 139630573137920, 139630574796799, -+SNULL, 139630576893951, 139630576934911, -+STORE, 139630574796800, 139630576893951, -+STORE, 139630576893952, 139630576934911, -+SNULL, 139630576893952, 139630576918527, -+STORE, 139630576918528, 139630576934911, -+STORE, 139630576893952, 139630576918527, -+ERASE, 139630576893952, 139630576918527, -+STORE, 139630576893952, 139630576918527, -+ERASE, 139630576918528, 139630576934911, -+STORE, 139630576918528, 139630576934911, -+SNULL, 139630576910335, 139630576918527, -+STORE, 139630576893952, 139630576910335, -+STORE, 139630576910336, 139630576918527, -+SNULL, 93888805859327, 93888805863423, -+STORE, 93888805851136, 93888805859327, -+STORE, 93888805859328, 93888805863423, -+SNULL, 139630579179519, 139630579183615, -+STORE, 139630579175424, 139630579179519, -+STORE, 139630579179520, 139630579183615, -+ERASE, 139630579146752, 139630579175423, -+STORE, 93888822235136, 93888822370303, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733391151104, 140737488351231, -+SNULL, 140733391159295, 140737488351231, -+STORE, 140733391151104, 140733391159295, -+STORE, 140733391020032, 140733391159295, -+STORE, 94393875324928, 94393877549055, -+SNULL, 94393875435519, 94393877549055, -+STORE, 94393875324928, 94393875435519, -+STORE, 94393875435520, 94393877549055, -+ERASE, 94393875435520, 94393877549055, -+STORE, 94393877528576, 94393877540863, -+STORE, 94393877540864, 94393877549055, -+STORE, 140292111740928, 140292113993727, -+SNULL, 140292111884287, 140292113993727, -+STORE, 140292111740928, 140292111884287, -+STORE, 140292111884288, 140292113993727, -+ERASE, 140292111884288, 140292113993727, -+STORE, 140292113981440, 140292113989631, -+STORE, 140292113989632, 140292113993727, -+STORE, 140733391532032, 140733391536127, -+STORE, 140733391519744, 140733391532031, -+STORE, 140292113952768, 140292113981439, -+STORE, 140292113944576, 140292113952767, -+STORE, 140292107943936, 140292111740927, -+SNULL, 140292107943936, 140292109602815, -+STORE, 140292109602816, 140292111740927, -+STORE, 140292107943936, 140292109602815, -+SNULL, 140292111699967, 140292111740927, -+STORE, 140292109602816, 140292111699967, -+STORE, 140292111699968, 140292111740927, -+SNULL, 140292111699968, 140292111724543, -+STORE, 140292111724544, 140292111740927, -+STORE, 140292111699968, 140292111724543, -+ERASE, 140292111699968, 140292111724543, -+STORE, 140292111699968, 140292111724543, -+ERASE, 140292111724544, 140292111740927, -+STORE, 140292111724544, 140292111740927, -+SNULL, 140292111716351, 140292111724543, -+STORE, 140292111699968, 140292111716351, -+STORE, 140292111716352, 140292111724543, -+SNULL, 94393877536767, 94393877540863, -+STORE, 94393877528576, 94393877536767, -+STORE, 94393877536768, 94393877540863, -+SNULL, 140292113985535, 140292113989631, -+STORE, 140292113981440, 140292113985535, -+STORE, 140292113985536, 140292113989631, -+ERASE, 140292113952768, 140292113981439, -+STORE, 94393909342208, 94393909477375, -+STORE, 94458367512576, 94458367725567, -+STORE, 94458369822720, 94458369826815, -+STORE, 94458369826816, 94458369835007, -+STORE, 94458369835008, 94458369847295, -+STORE, 94458393292800, 94458399666175, -+STORE, 140619773841408, 140619775500287, -+STORE, 140619775500288, 140619777597439, -+STORE, 140619777597440, 140619777613823, -+STORE, 140619777613824, 140619777622015, -+STORE, 140619777622016, 140619777638399, -+STORE, 140619777638400, 140619777650687, -+STORE, 140619777650688, 140619779743743, -+STORE, 140619779743744, 140619779747839, -+STORE, 140619779747840, 140619779751935, -+STORE, 140619779751936, 140619779895295, -+STORE, 140619780263936, 140619781947391, -+STORE, 140619781947392, 140619781963775, -+STORE, 140619781992448, 140619781996543, -+STORE, 140619781996544, 140619782000639, -+STORE, 140619782000640, 140619782004735, -+STORE, 140725811675136, 140725811814399, -+STORE, 140725812813824, 140725812826111, -+STORE, 140725812826112, 140725812830207, -+STORE, 94458367512576, 94458367725567, -+STORE, 94458369822720, 94458369826815, -+STORE, 94458369826816, 94458369835007, -+STORE, 94458369835008, 94458369847295, -+STORE, 94458393292800, 94458400366591, -+STORE, 140619773841408, 140619775500287, -+STORE, 140619775500288, 140619777597439, -+STORE, 140619777597440, 140619777613823, -+STORE, 140619777613824, 140619777622015, -+STORE, 140619777622016, 140619777638399, -+STORE, 140619777638400, 140619777650687, -+STORE, 140619777650688, 140619779743743, -+STORE, 140619779743744, 140619779747839, -+STORE, 140619779747840, 140619779751935, -+STORE, 140619779751936, 140619779895295, -+STORE, 140619780263936, 140619781947391, -+STORE, 140619781947392, 140619781963775, -+STORE, 140619781992448, 140619781996543, -+STORE, 140619781996544, 140619782000639, -+STORE, 140619782000640, 140619782004735, -+STORE, 140725811675136, 140725811814399, -+STORE, 140725812813824, 140725812826111, -+STORE, 140725812826112, 140725812830207, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728740679680, 140737488351231, -+SNULL, 140728740687871, 140737488351231, -+STORE, 140728740679680, 140728740687871, -+STORE, 140728740548608, 140728740687871, -+STORE, 94764075249664, 94764077473791, -+SNULL, 94764075360255, 94764077473791, -+STORE, 94764075249664, 94764075360255, -+STORE, 94764075360256, 94764077473791, -+ERASE, 94764075360256, 94764077473791, -+STORE, 94764077453312, 94764077465599, -+STORE, 94764077465600, 94764077473791, -+STORE, 139766406791168, 139766409043967, -+SNULL, 139766406934527, 139766409043967, -+STORE, 139766406791168, 139766406934527, -+STORE, 139766406934528, 139766409043967, -+ERASE, 139766406934528, 139766409043967, -+STORE, 139766409031680, 139766409039871, -+STORE, 139766409039872, 139766409043967, -+STORE, 140728740913152, 140728740917247, -+STORE, 140728740900864, 140728740913151, -+STORE, 139766409003008, 139766409031679, -+STORE, 139766408994816, 139766409003007, -+STORE, 139766402994176, 139766406791167, -+SNULL, 139766402994176, 139766404653055, -+STORE, 139766404653056, 139766406791167, -+STORE, 139766402994176, 139766404653055, -+SNULL, 139766406750207, 139766406791167, -+STORE, 139766404653056, 139766406750207, -+STORE, 139766406750208, 139766406791167, -+SNULL, 139766406750208, 139766406774783, -+STORE, 139766406774784, 139766406791167, -+STORE, 139766406750208, 139766406774783, -+ERASE, 139766406750208, 139766406774783, -+STORE, 139766406750208, 139766406774783, -+ERASE, 139766406774784, 139766406791167, -+STORE, 139766406774784, 139766406791167, -+SNULL, 139766406766591, 139766406774783, -+STORE, 139766406750208, 139766406766591, -+STORE, 139766406766592, 139766406774783, -+SNULL, 94764077461503, 94764077465599, -+STORE, 94764077453312, 94764077461503, -+STORE, 94764077461504, 94764077465599, -+SNULL, 139766409035775, 139766409039871, -+STORE, 139766409031680, 139766409035775, -+STORE, 139766409035776, 139766409039871, -+ERASE, 139766409003008, 139766409031679, -+STORE, 94764090458112, 94764090593279, -+STORE, 94758057480192, 94758057590783, -+STORE, 94758059683840, 94758059692031, -+STORE, 94758059692032, 94758059696127, -+STORE, 94758059696128, 94758059704319, -+STORE, 94758083215360, 94758083350527, -+STORE, 139951456772096, 139951458430975, -+STORE, 139951458430976, 139951460528127, -+STORE, 139951460528128, 139951460544511, -+STORE, 139951460544512, 139951460552703, -+STORE, 139951460552704, 139951460569087, -+STORE, 139951460569088, 139951460712447, -+STORE, 139951462772736, 139951462780927, -+STORE, 139951462809600, 139951462813695, -+STORE, 139951462813696, 139951462817791, -+STORE, 139951462817792, 139951462821887, -+STORE, 140734098313216, 140734098452479, -+STORE, 140734098911232, 140734098923519, -+STORE, 140734098923520, 140734098927615, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724904095744, 140737488351231, -+SNULL, 140724904103935, 140737488351231, -+STORE, 140724904095744, 140724904103935, -+STORE, 140724903964672, 140724904103935, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 140408497864704, 140408500117503, -+SNULL, 140408498008063, 140408500117503, -+STORE, 140408497864704, 140408498008063, -+STORE, 140408498008064, 140408500117503, -+ERASE, 140408498008064, 140408500117503, -+STORE, 140408500105216, 140408500113407, -+STORE, 140408500113408, 140408500117503, -+STORE, 140724905369600, 140724905373695, -+STORE, 140724905357312, 140724905369599, -+STORE, 140408500076544, 140408500105215, -+STORE, 140408500068352, 140408500076543, -+STORE, 140408494702592, 140408497864703, -+SNULL, 140408494702592, 140408495763455, -+STORE, 140408495763456, 140408497864703, -+STORE, 140408494702592, 140408495763455, -+SNULL, 140408497856511, 140408497864703, -+STORE, 140408495763456, 140408497856511, -+STORE, 140408497856512, 140408497864703, -+ERASE, 140408497856512, 140408497864703, -+STORE, 140408497856512, 140408497864703, -+STORE, 140408490905600, 140408494702591, -+SNULL, 140408490905600, 140408492564479, -+STORE, 140408492564480, 140408494702591, -+STORE, 140408490905600, 140408492564479, -+SNULL, 140408494661631, 140408494702591, -+STORE, 140408492564480, 140408494661631, -+STORE, 140408494661632, 140408494702591, -+SNULL, 140408494661632, 140408494686207, -+STORE, 140408494686208, 140408494702591, -+STORE, 140408494661632, 140408494686207, -+ERASE, 140408494661632, 140408494686207, -+STORE, 140408494661632, 140408494686207, -+ERASE, 140408494686208, 140408494702591, -+STORE, 140408494686208, 140408494702591, -+STORE, 140408500056064, 140408500076543, -+SNULL, 140408494678015, 140408494686207, -+STORE, 140408494661632, 140408494678015, -+STORE, 140408494678016, 140408494686207, -+SNULL, 140408497860607, 140408497864703, -+STORE, 140408497856512, 140408497860607, -+STORE, 140408497860608, 140408497864703, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 140408500109311, 140408500113407, -+STORE, 140408500105216, 140408500109311, -+STORE, 140408500109312, 140408500113407, -+ERASE, 140408500076544, 140408500105215, -+STORE, 25235456, 25370623, -+STORE, 25235456, 25518079, -+STORE, 140408498372608, 140408500056063, -+STORE, 94543937388544, 94543937499135, -+STORE, 94543939592192, 94543939600383, -+STORE, 94543939600384, 94543939604479, -+STORE, 94543939604480, 94543939612671, -+STORE, 94543941447680, 94543941582847, -+STORE, 140282621947904, 140282623606783, -+STORE, 140282623606784, 140282625703935, -+STORE, 140282625703936, 140282625720319, -+STORE, 140282625720320, 140282625728511, -+STORE, 140282625728512, 140282625744895, -+STORE, 140282625744896, 140282625888255, -+STORE, 140282627948544, 140282627956735, -+STORE, 140282627985408, 140282627989503, -+STORE, 140282627989504, 140282627993599, -+STORE, 140282627993600, 140282627997695, -+STORE, 140728295723008, 140728295862271, -+STORE, 140728296476672, 140728296488959, -+STORE, 140728296488960, 140728296493055, -+STORE, 94431504838656, 94431505051647, -+STORE, 94431507148800, 94431507152895, -+STORE, 94431507152896, 94431507161087, -+STORE, 94431507161088, 94431507173375, -+STORE, 94431510286336, 94431510691839, -+STORE, 139818797948928, 139818799607807, -+STORE, 139818799607808, 139818801704959, -+STORE, 139818801704960, 139818801721343, -+STORE, 139818801721344, 139818801729535, -+STORE, 139818801729536, 139818801745919, -+STORE, 139818801745920, 139818801758207, -+STORE, 139818801758208, 139818803851263, -+STORE, 139818803851264, 139818803855359, -+STORE, 139818803855360, 139818803859455, -+STORE, 139818803859456, 139818804002815, -+STORE, 139818804371456, 139818806054911, -+STORE, 139818806054912, 139818806071295, -+STORE, 139818806099968, 139818806104063, -+STORE, 139818806104064, 139818806108159, -+STORE, 139818806108160, 139818806112255, -+STORE, 140731430457344, 140731430596607, -+STORE, 140731431227392, 140731431239679, -+STORE, 140731431239680, 140731431243775, -+STORE, 94431504838656, 94431505051647, -+STORE, 94431507148800, 94431507152895, -+STORE, 94431507152896, 94431507161087, -+STORE, 94431507161088, 94431507173375, -+STORE, 94431510286336, 94431510691839, -+STORE, 139818797948928, 139818799607807, -+STORE, 139818799607808, 139818801704959, -+STORE, 139818801704960, 139818801721343, -+STORE, 139818801721344, 139818801729535, -+STORE, 139818801729536, 139818801745919, -+STORE, 139818801745920, 139818801758207, -+STORE, 139818801758208, 139818803851263, -+STORE, 139818803851264, 139818803855359, -+STORE, 139818803855360, 139818803859455, -+STORE, 139818803859456, 139818804002815, -+STORE, 139818804371456, 139818806054911, -+STORE, 139818806054912, 139818806071295, -+STORE, 139818806099968, 139818806104063, -+STORE, 139818806104064, 139818806108159, -+STORE, 139818806108160, 139818806112255, -+STORE, 140731430457344, 140731430596607, -+STORE, 140731431227392, 140731431239679, -+STORE, 140731431239680, 140731431243775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140737488338944, 140737488351231, -+STORE, 140736944451584, 140737488351231, -+SNULL, 140736944463871, 140737488351231, -+STORE, 140736944451584, 140736944463871, -+STORE, 140736944320512, 140736944463871, -+STORE, 4194304, 26279935, -+STORE, 28372992, 28454911, -+STORE, 28454912, 29806591, -+STORE, 139693609893888, 139693612146687, -+SNULL, 139693610037247, 139693612146687, -+STORE, 139693609893888, 139693610037247, -+STORE, 139693610037248, 139693612146687, -+ERASE, 139693610037248, 139693612146687, -+STORE, 139693612134400, 139693612142591, -+STORE, 139693612142592, 139693612146687, -+STORE, 140736945152000, 140736945156095, -+STORE, 140736945139712, 140736945151999, -+STORE, 139693612105728, 139693612134399, -+STORE, 139693612097536, 139693612105727, -+STORE, 139693606060032, 139693609893887, -+SNULL, 139693606060032, 139693607768063, -+STORE, 139693607768064, 139693609893887, -+STORE, 139693606060032, 139693607768063, -+SNULL, 139693609861119, 139693609893887, -+STORE, 139693607768064, 139693609861119, -+STORE, 139693609861120, 139693609893887, -+ERASE, 139693609861120, 139693609893887, -+STORE, 139693609861120, 139693609893887, -+STORE, 139693603864576, 139693606060031, -+SNULL, 139693603864576, 139693603958783, -+STORE, 139693603958784, 139693606060031, -+STORE, 139693603864576, 139693603958783, -+SNULL, 139693606051839, 139693606060031, -+STORE, 139693603958784, 139693606051839, -+STORE, 139693606051840, 139693606060031, -+ERASE, 139693606051840, 139693606060031, -+STORE, 139693606051840, 139693606060031, -+STORE, 139693601345536, 139693603864575, -+SNULL, 139693601345536, 139693601759231, -+STORE, 139693601759232, 139693603864575, -+STORE, 139693601345536, 139693601759231, -+SNULL, 139693603852287, 139693603864575, -+STORE, 139693601759232, 139693603852287, -+STORE, 139693603852288, 139693603864575, -+ERASE, 139693603852288, 139693603864575, -+STORE, 139693603852288, 139693603864575, -+STORE, 139693598711808, 139693601345535, -+SNULL, 139693598711808, 139693599240191, -+STORE, 139693599240192, 139693601345535, -+STORE, 139693598711808, 139693599240191, -+SNULL, 139693601337343, 139693601345535, -+STORE, 139693599240192, 139693601337343, -+STORE, 139693601337344, 139693601345535, -+ERASE, 139693601337344, 139693601345535, -+STORE, 139693601337344, 139693601345535, -+STORE, 139693596598272, 139693598711807, -+SNULL, 139693596598272, 139693596610559, -+STORE, 139693596610560, 139693598711807, -+STORE, 139693596598272, 139693596610559, -+SNULL, 139693598703615, 139693598711807, -+STORE, 139693596610560, 139693598703615, -+STORE, 139693598703616, 139693598711807, -+ERASE, 139693598703616, 139693598711807, -+STORE, 139693598703616, 139693598711807, -+STORE, 139693594394624, 139693596598271, -+SNULL, 139693594394624, 139693594497023, -+STORE, 139693594497024, 139693596598271, -+STORE, 139693594394624, 139693594497023, -+SNULL, 139693596590079, 139693596598271, -+STORE, 139693594497024, 139693596590079, -+STORE, 139693596590080, 139693596598271, -+ERASE, 139693596590080, 139693596598271, -+STORE, 139693596590080, 139693596598271, -+STORE, 139693612089344, 139693612105727, -+STORE, 139693591232512, 139693594394623, -+SNULL, 139693591232512, 139693592293375, -+STORE, 139693592293376, 139693594394623, -+STORE, 139693591232512, 139693592293375, -+SNULL, 139693594386431, 139693594394623, -+STORE, 139693592293376, 139693594386431, -+STORE, 139693594386432, 139693594394623, -+ERASE, 139693594386432, 139693594394623, -+STORE, 139693594386432, 139693594394623, -+STORE, 139693587435520, 139693591232511, -+SNULL, 139693587435520, 139693589094399, -+STORE, 139693589094400, 139693591232511, -+STORE, 139693587435520, 139693589094399, -+SNULL, 139693591191551, 139693591232511, -+STORE, 139693589094400, 139693591191551, -+STORE, 139693591191552, 139693591232511, -+SNULL, 139693591191552, 139693591216127, -+STORE, 139693591216128, 139693591232511, -+STORE, 139693591191552, 139693591216127, -+ERASE, 139693591191552, 139693591216127, -+STORE, 139693591191552, 139693591216127, -+ERASE, 139693591216128, 139693591232511, -+STORE, 139693591216128, 139693591232511, -+STORE, 139693612077056, 139693612105727, -+SNULL, 139693591207935, 139693591216127, -+STORE, 139693591191552, 139693591207935, -+STORE, 139693591207936, 139693591216127, -+SNULL, 139693594390527, 139693594394623, -+STORE, 139693594386432, 139693594390527, -+STORE, 139693594390528, 139693594394623, -+SNULL, 139693596594175, 139693596598271, -+STORE, 139693596590080, 139693596594175, -+STORE, 139693596594176, 139693596598271, -+SNULL, 139693598707711, 139693598711807, -+STORE, 139693598703616, 139693598707711, -+STORE, 139693598707712, 139693598711807, -+SNULL, 139693601341439, 139693601345535, -+STORE, 139693601337344, 139693601341439, -+STORE, 139693601341440, 139693601345535, -+SNULL, 139693603860479, 139693603864575, -+STORE, 139693603852288, 139693603860479, -+STORE, 139693603860480, 139693603864575, -+SNULL, 139693606055935, 139693606060031, -+STORE, 139693606051840, 139693606055935, -+STORE, 139693606055936, 139693606060031, -+SNULL, 139693609865215, 139693609893887, -+STORE, 139693609861120, 139693609865215, -+STORE, 139693609865216, 139693609893887, -+SNULL, 28405759, 28454911, -+STORE, 28372992, 28405759, -+STORE, 28405760, 28454911, -+SNULL, 139693612138495, 139693612142591, -+STORE, 139693612134400, 139693612138495, -+STORE, 139693612138496, 139693612142591, -+ERASE, 139693612105728, 139693612134399, -+STORE, 39976960, 40112127, -+STORE, 139693610393600, 139693612077055, -+STORE, 139693612130304, 139693612134399, -+STORE, 139693610258432, 139693610393599, -+STORE, 39976960, 40255487, -+STORE, 139693585338368, 139693587435519, -+STORE, 139693612122112, 139693612134399, -+STORE, 139693612113920, 139693612134399, -+STORE, 139693612077056, 139693612113919, -+STORE, 139693610242048, 139693610393599, -+STORE, 39976960, 40390655, -+STORE, 39976960, 40546303, -+STORE, 139693610233856, 139693610393599, -+STORE, 139693610225664, 139693610393599, -+STORE, 39976960, 40714239, -+STORE, 139693610209280, 139693610393599, -+STORE, 39976960, 40861695, -+STORE, 94431504838656, 94431505051647, -+STORE, 94431507148800, 94431507152895, -+STORE, 94431507152896, 94431507161087, -+STORE, 94431507161088, 94431507173375, -+STORE, 94431510286336, 94431528759295, -+STORE, 139818797948928, 139818799607807, -+STORE, 139818799607808, 139818801704959, -+STORE, 139818801704960, 139818801721343, -+STORE, 139818801721344, 139818801729535, -+STORE, 139818801729536, 139818801745919, -+STORE, 139818801745920, 139818801758207, -+STORE, 139818801758208, 139818803851263, -+STORE, 139818803851264, 139818803855359, -+STORE, 139818803855360, 139818803859455, -+STORE, 139818803859456, 139818804002815, -+STORE, 139818804371456, 139818806054911, -+STORE, 139818806054912, 139818806071295, -+STORE, 139818806099968, 139818806104063, -+STORE, 139818806104064, 139818806108159, -+STORE, 139818806108160, 139818806112255, -+STORE, 140731430457344, 140731430596607, -+STORE, 140731431227392, 140731431239679, -+STORE, 140731431239680, 140731431243775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140729993904128, 140737488351231, -+SNULL, 140729993912319, 140737488351231, -+STORE, 140729993904128, 140729993912319, -+STORE, 140729993773056, 140729993912319, -+STORE, 93926271991808, 93926274215935, -+SNULL, 93926272102399, 93926274215935, -+STORE, 93926271991808, 93926272102399, -+STORE, 93926272102400, 93926274215935, -+ERASE, 93926272102400, 93926274215935, -+STORE, 93926274195456, 93926274207743, -+STORE, 93926274207744, 93926274215935, -+STORE, 139962167296000, 139962169548799, -+SNULL, 139962167439359, 139962169548799, -+STORE, 139962167296000, 139962167439359, -+STORE, 139962167439360, 139962169548799, -+ERASE, 139962167439360, 139962169548799, -+STORE, 139962169536512, 139962169544703, -+STORE, 139962169544704, 139962169548799, -+STORE, 140729995096064, 140729995100159, -+STORE, 140729995083776, 140729995096063, -+STORE, 139962169507840, 139962169536511, -+STORE, 139962169499648, 139962169507839, -+STORE, 139962163499008, 139962167295999, -+SNULL, 139962163499008, 139962165157887, -+STORE, 139962165157888, 139962167295999, -+STORE, 139962163499008, 139962165157887, -+SNULL, 139962167255039, 139962167295999, -+STORE, 139962165157888, 139962167255039, -+STORE, 139962167255040, 139962167295999, -+SNULL, 139962167255040, 139962167279615, -+STORE, 139962167279616, 139962167295999, -+STORE, 139962167255040, 139962167279615, -+ERASE, 139962167255040, 139962167279615, -+STORE, 139962167255040, 139962167279615, -+ERASE, 139962167279616, 139962167295999, -+STORE, 139962167279616, 139962167295999, -+SNULL, 139962167271423, 139962167279615, -+STORE, 139962167255040, 139962167271423, -+STORE, 139962167271424, 139962167279615, -+SNULL, 93926274203647, 93926274207743, -+STORE, 93926274195456, 93926274203647, -+STORE, 93926274203648, 93926274207743, -+SNULL, 139962169540607, 139962169544703, -+STORE, 139962169536512, 139962169540607, -+STORE, 139962169540608, 139962169544703, -+ERASE, 139962169507840, 139962169536511, -+STORE, 93926291120128, 93926291255295, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724960579584, 140737488351231, -+SNULL, 140724960587775, 140737488351231, -+STORE, 140724960579584, 140724960587775, -+STORE, 140724960448512, 140724960587775, -+STORE, 94246489489408, 94246491713535, -+SNULL, 94246489599999, 94246491713535, -+STORE, 94246489489408, 94246489599999, -+STORE, 94246489600000, 94246491713535, -+ERASE, 94246489600000, 94246491713535, -+STORE, 94246491693056, 94246491705343, -+STORE, 94246491705344, 94246491713535, -+STORE, 140098174926848, 140098177179647, -+SNULL, 140098175070207, 140098177179647, -+STORE, 140098174926848, 140098175070207, -+STORE, 140098175070208, 140098177179647, -+ERASE, 140098175070208, 140098177179647, -+STORE, 140098177167360, 140098177175551, -+STORE, 140098177175552, 140098177179647, -+STORE, 140724961439744, 140724961443839, -+STORE, 140724961427456, 140724961439743, -+STORE, 140098177138688, 140098177167359, -+STORE, 140098177130496, 140098177138687, -+STORE, 140098171129856, 140098174926847, -+SNULL, 140098171129856, 140098172788735, -+STORE, 140098172788736, 140098174926847, -+STORE, 140098171129856, 140098172788735, -+SNULL, 140098174885887, 140098174926847, -+STORE, 140098172788736, 140098174885887, -+STORE, 140098174885888, 140098174926847, -+SNULL, 140098174885888, 140098174910463, -+STORE, 140098174910464, 140098174926847, -+STORE, 140098174885888, 140098174910463, -+ERASE, 140098174885888, 140098174910463, -+STORE, 140098174885888, 140098174910463, -+ERASE, 140098174910464, 140098174926847, -+STORE, 140098174910464, 140098174926847, -+SNULL, 140098174902271, 140098174910463, -+STORE, 140098174885888, 140098174902271, -+STORE, 140098174902272, 140098174910463, -+SNULL, 94246491701247, 94246491705343, -+STORE, 94246491693056, 94246491701247, -+STORE, 94246491701248, 94246491705343, -+SNULL, 140098177171455, 140098177175551, -+STORE, 140098177167360, 140098177171455, -+STORE, 140098177171456, 140098177175551, -+ERASE, 140098177138688, 140098177167359, -+STORE, 94246516998144, 94246517133311, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140730522918912, 140737488351231, -+SNULL, 140730522927103, 140737488351231, -+STORE, 140730522918912, 140730522927103, -+STORE, 140730522787840, 140730522927103, -+STORE, 94196043120640, 94196045344767, -+SNULL, 94196043231231, 94196045344767, -+STORE, 94196043120640, 94196043231231, -+STORE, 94196043231232, 94196045344767, -+ERASE, 94196043231232, 94196045344767, -+STORE, 94196045324288, 94196045336575, -+STORE, 94196045336576, 94196045344767, -+STORE, 139815918940160, 139815921192959, -+SNULL, 139815919083519, 139815921192959, -+STORE, 139815918940160, 139815919083519, -+STORE, 139815919083520, 139815921192959, -+ERASE, 139815919083520, 139815921192959, -+STORE, 139815921180672, 139815921188863, -+STORE, 139815921188864, 139815921192959, -+STORE, 140730523344896, 140730523348991, -+STORE, 140730523332608, 140730523344895, -+STORE, 139815921152000, 139815921180671, -+STORE, 139815921143808, 139815921151999, -+STORE, 139815915143168, 139815918940159, -+SNULL, 139815915143168, 139815916802047, -+STORE, 139815916802048, 139815918940159, -+STORE, 139815915143168, 139815916802047, -+SNULL, 139815918899199, 139815918940159, -+STORE, 139815916802048, 139815918899199, -+STORE, 139815918899200, 139815918940159, -+SNULL, 139815918899200, 139815918923775, -+STORE, 139815918923776, 139815918940159, -+STORE, 139815918899200, 139815918923775, -+ERASE, 139815918899200, 139815918923775, -+STORE, 139815918899200, 139815918923775, -+ERASE, 139815918923776, 139815918940159, -+STORE, 139815918923776, 139815918940159, -+SNULL, 139815918915583, 139815918923775, -+STORE, 139815918899200, 139815918915583, -+STORE, 139815918915584, 139815918923775, -+SNULL, 94196045332479, 94196045336575, -+STORE, 94196045324288, 94196045332479, -+STORE, 94196045332480, 94196045336575, -+SNULL, 139815921184767, 139815921188863, -+STORE, 139815921180672, 139815921184767, -+STORE, 139815921184768, 139815921188863, -+ERASE, 139815921152000, 139815921180671, -+STORE, 94196076183552, 94196076318719, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722460393472, 140737488351231, -+SNULL, 140722460401663, 140737488351231, -+STORE, 140722460393472, 140722460401663, -+STORE, 140722460262400, 140722460401663, -+STORE, 94569810399232, 94569812623359, -+SNULL, 94569810509823, 94569812623359, -+STORE, 94569810399232, 94569810509823, -+STORE, 94569810509824, 94569812623359, -+ERASE, 94569810509824, 94569812623359, -+STORE, 94569812602880, 94569812615167, -+STORE, 94569812615168, 94569812623359, -+STORE, 139681565450240, 139681567703039, -+SNULL, 139681565593599, 139681567703039, -+STORE, 139681565450240, 139681565593599, -+STORE, 139681565593600, 139681567703039, -+ERASE, 139681565593600, 139681567703039, -+STORE, 139681567690752, 139681567698943, -+STORE, 139681567698944, 139681567703039, -+STORE, 140722460569600, 140722460573695, -+STORE, 140722460557312, 140722460569599, -+STORE, 139681567662080, 139681567690751, -+STORE, 139681567653888, 139681567662079, -+STORE, 139681561653248, 139681565450239, -+SNULL, 139681561653248, 139681563312127, -+STORE, 139681563312128, 139681565450239, -+STORE, 139681561653248, 139681563312127, -+SNULL, 139681565409279, 139681565450239, -+STORE, 139681563312128, 139681565409279, -+STORE, 139681565409280, 139681565450239, -+SNULL, 139681565409280, 139681565433855, -+STORE, 139681565433856, 139681565450239, -+STORE, 139681565409280, 139681565433855, -+ERASE, 139681565409280, 139681565433855, -+STORE, 139681565409280, 139681565433855, -+ERASE, 139681565433856, 139681565450239, -+STORE, 139681565433856, 139681565450239, -+SNULL, 139681565425663, 139681565433855, -+STORE, 139681565409280, 139681565425663, -+STORE, 139681565425664, 139681565433855, -+SNULL, 94569812611071, 94569812615167, -+STORE, 94569812602880, 94569812611071, -+STORE, 94569812611072, 94569812615167, -+SNULL, 139681567694847, 139681567698943, -+STORE, 139681567690752, 139681567694847, -+STORE, 139681567694848, 139681567698943, -+ERASE, 139681567662080, 139681567690751, -+STORE, 94569818066944, 94569818202111, -+STORE, 94431504838656, 94431505051647, -+STORE, 94431507148800, 94431507152895, -+STORE, 94431507152896, 94431507161087, -+STORE, 94431507161088, 94431507173375, -+STORE, 94431510286336, 94431534280703, -+STORE, 139818797948928, 139818799607807, -+STORE, 139818799607808, 139818801704959, -+STORE, 139818801704960, 139818801721343, -+STORE, 139818801721344, 139818801729535, -+STORE, 139818801729536, 139818801745919, -+STORE, 139818801745920, 139818801758207, -+STORE, 139818801758208, 139818803851263, -+STORE, 139818803851264, 139818803855359, -+STORE, 139818803855360, 139818803859455, -+STORE, 139818803859456, 139818804002815, -+STORE, 139818804371456, 139818806054911, -+STORE, 139818806054912, 139818806071295, -+STORE, 139818806099968, 139818806104063, -+STORE, 139818806104064, 139818806108159, -+STORE, 139818806108160, 139818806112255, -+STORE, 140731430457344, 140731430596607, -+STORE, 140731431227392, 140731431239679, -+STORE, 140731431239680, 140731431243775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140725452365824, 140737488351231, -+SNULL, 140725452374015, 140737488351231, -+STORE, 140725452365824, 140725452374015, -+STORE, 140725452234752, 140725452374015, -+STORE, 94395067465728, 94395069689855, -+SNULL, 94395067576319, 94395069689855, -+STORE, 94395067465728, 94395067576319, -+STORE, 94395067576320, 94395069689855, -+ERASE, 94395067576320, 94395069689855, -+STORE, 94395069669376, 94395069681663, -+STORE, 94395069681664, 94395069689855, -+STORE, 140269941211136, 140269943463935, -+SNULL, 140269941354495, 140269943463935, -+STORE, 140269941211136, 140269941354495, -+STORE, 140269941354496, 140269943463935, -+ERASE, 140269941354496, 140269943463935, -+STORE, 140269943451648, 140269943459839, -+STORE, 140269943459840, 140269943463935, -+STORE, 140725452558336, 140725452562431, -+STORE, 140725452546048, 140725452558335, -+STORE, 140269943422976, 140269943451647, -+STORE, 140269943414784, 140269943422975, -+STORE, 140269937414144, 140269941211135, -+SNULL, 140269937414144, 140269939073023, -+STORE, 140269939073024, 140269941211135, -+STORE, 140269937414144, 140269939073023, -+SNULL, 140269941170175, 140269941211135, -+STORE, 140269939073024, 140269941170175, -+STORE, 140269941170176, 140269941211135, -+SNULL, 140269941170176, 140269941194751, -+STORE, 140269941194752, 140269941211135, -+STORE, 140269941170176, 140269941194751, -+ERASE, 140269941170176, 140269941194751, -+STORE, 140269941170176, 140269941194751, -+ERASE, 140269941194752, 140269941211135, -+STORE, 140269941194752, 140269941211135, -+SNULL, 140269941186559, 140269941194751, -+STORE, 140269941170176, 140269941186559, -+STORE, 140269941186560, 140269941194751, -+SNULL, 94395069677567, 94395069681663, -+STORE, 94395069669376, 94395069677567, -+STORE, 94395069677568, 94395069681663, -+SNULL, 140269943455743, 140269943459839, -+STORE, 140269943451648, 140269943455743, -+STORE, 140269943455744, 140269943459839, -+ERASE, 140269943422976, 140269943451647, -+STORE, 94395101691904, 94395101827071, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733860118528, 140737488351231, -+SNULL, 140733860126719, 140737488351231, -+STORE, 140733860118528, 140733860126719, -+STORE, 140733859987456, 140733860126719, -+STORE, 94484752990208, 94484755214335, -+SNULL, 94484753100799, 94484755214335, -+STORE, 94484752990208, 94484753100799, -+STORE, 94484753100800, 94484755214335, -+ERASE, 94484753100800, 94484755214335, -+STORE, 94484755193856, 94484755206143, -+STORE, 94484755206144, 94484755214335, -+STORE, 139958922309632, 139958924562431, -+SNULL, 139958922452991, 139958924562431, -+STORE, 139958922309632, 139958922452991, -+STORE, 139958922452992, 139958924562431, -+ERASE, 139958922452992, 139958924562431, -+STORE, 139958924550144, 139958924558335, -+STORE, 139958924558336, 139958924562431, -+STORE, 140733860253696, 140733860257791, -+STORE, 140733860241408, 140733860253695, -+STORE, 139958924521472, 139958924550143, -+STORE, 139958924513280, 139958924521471, -+STORE, 139958918512640, 139958922309631, -+SNULL, 139958918512640, 139958920171519, -+STORE, 139958920171520, 139958922309631, -+STORE, 139958918512640, 139958920171519, -+SNULL, 139958922268671, 139958922309631, -+STORE, 139958920171520, 139958922268671, -+STORE, 139958922268672, 139958922309631, -+SNULL, 139958922268672, 139958922293247, -+STORE, 139958922293248, 139958922309631, -+STORE, 139958922268672, 139958922293247, -+ERASE, 139958922268672, 139958922293247, -+STORE, 139958922268672, 139958922293247, -+ERASE, 139958922293248, 139958922309631, -+STORE, 139958922293248, 139958922309631, -+SNULL, 139958922285055, 139958922293247, -+STORE, 139958922268672, 139958922285055, -+STORE, 139958922285056, 139958922293247, -+SNULL, 94484755202047, 94484755206143, -+STORE, 94484755193856, 94484755202047, -+STORE, 94484755202048, 94484755206143, -+SNULL, 139958924554239, 139958924558335, -+STORE, 139958924550144, 139958924554239, -+STORE, 139958924554240, 139958924558335, -+ERASE, 139958924521472, 139958924550143, -+STORE, 94484777615360, 94484777750527, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731051036672, 140737488351231, -+SNULL, 140731051044863, 140737488351231, -+STORE, 140731051036672, 140731051044863, -+STORE, 140731050905600, 140731051044863, -+STORE, 93945822998528, 93945825222655, -+SNULL, 93945823109119, 93945825222655, -+STORE, 93945822998528, 93945823109119, -+STORE, 93945823109120, 93945825222655, -+ERASE, 93945823109120, 93945825222655, -+STORE, 93945825202176, 93945825214463, -+STORE, 93945825214464, 93945825222655, -+STORE, 140153503997952, 140153506250751, -+SNULL, 140153504141311, 140153506250751, -+STORE, 140153503997952, 140153504141311, -+STORE, 140153504141312, 140153506250751, -+ERASE, 140153504141312, 140153506250751, -+STORE, 140153506238464, 140153506246655, -+STORE, 140153506246656, 140153506250751, -+STORE, 140731051331584, 140731051335679, -+STORE, 140731051319296, 140731051331583, -+STORE, 140153506209792, 140153506238463, -+STORE, 140153506201600, 140153506209791, -+STORE, 140153500200960, 140153503997951, -+SNULL, 140153500200960, 140153501859839, -+STORE, 140153501859840, 140153503997951, -+STORE, 140153500200960, 140153501859839, -+SNULL, 140153503956991, 140153503997951, -+STORE, 140153501859840, 140153503956991, -+STORE, 140153503956992, 140153503997951, -+SNULL, 140153503956992, 140153503981567, -+STORE, 140153503981568, 140153503997951, -+STORE, 140153503956992, 140153503981567, -+ERASE, 140153503956992, 140153503981567, -+STORE, 140153503956992, 140153503981567, -+ERASE, 140153503981568, 140153503997951, -+STORE, 140153503981568, 140153503997951, -+SNULL, 140153503973375, 140153503981567, -+STORE, 140153503956992, 140153503973375, -+STORE, 140153503973376, 140153503981567, -+SNULL, 93945825210367, 93945825214463, -+STORE, 93945825202176, 93945825210367, -+STORE, 93945825210368, 93945825214463, -+SNULL, 140153506242559, 140153506246655, -+STORE, 140153506238464, 140153506242559, -+STORE, 140153506242560, 140153506246655, -+ERASE, 140153506209792, 140153506238463, -+STORE, 93945854537728, 93945854672895, -+STORE, 94431504838656, 94431505051647, -+STORE, 94431507148800, 94431507152895, -+STORE, 94431507152896, 94431507161087, -+STORE, 94431507161088, 94431507173375, -+STORE, 94431510286336, 94431537885183, -+STORE, 139818797948928, 139818799607807, -+STORE, 139818799607808, 139818801704959, -+STORE, 139818801704960, 139818801721343, -+STORE, 139818801721344, 139818801729535, -+STORE, 139818801729536, 139818801745919, -+STORE, 139818801745920, 139818801758207, -+STORE, 139818801758208, 139818803851263, -+STORE, 139818803851264, 139818803855359, -+STORE, 139818803855360, 139818803859455, -+STORE, 139818803859456, 139818804002815, -+STORE, 139818804371456, 139818806054911, -+STORE, 139818806054912, 139818806071295, -+STORE, 139818806099968, 139818806104063, -+STORE, 139818806104064, 139818806108159, -+STORE, 139818806108160, 139818806112255, -+STORE, 140731430457344, 140731430596607, -+STORE, 140731431227392, 140731431239679, -+STORE, 140731431239680, 140731431243775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140736025325568, 140737488351231, -+SNULL, 140736025333759, 140737488351231, -+STORE, 140736025325568, 140736025333759, -+STORE, 140736025194496, 140736025333759, -+STORE, 94809095172096, 94809097396223, -+SNULL, 94809095282687, 94809097396223, -+STORE, 94809095172096, 94809095282687, -+STORE, 94809095282688, 94809097396223, -+ERASE, 94809095282688, 94809097396223, -+STORE, 94809097375744, 94809097388031, -+STORE, 94809097388032, 94809097396223, -+STORE, 140194992517120, 140194994769919, -+SNULL, 140194992660479, 140194994769919, -+STORE, 140194992517120, 140194992660479, -+STORE, 140194992660480, 140194994769919, -+ERASE, 140194992660480, 140194994769919, -+STORE, 140194994757632, 140194994765823, -+STORE, 140194994765824, 140194994769919, -+STORE, 140736026173440, 140736026177535, -+STORE, 140736026161152, 140736026173439, -+STORE, 140194994728960, 140194994757631, -+STORE, 140194994720768, 140194994728959, -+STORE, 140194988720128, 140194992517119, -+SNULL, 140194988720128, 140194990379007, -+STORE, 140194990379008, 140194992517119, -+STORE, 140194988720128, 140194990379007, -+SNULL, 140194992476159, 140194992517119, -+STORE, 140194990379008, 140194992476159, -+STORE, 140194992476160, 140194992517119, -+SNULL, 140194992476160, 140194992500735, -+STORE, 140194992500736, 140194992517119, -+STORE, 140194992476160, 140194992500735, -+ERASE, 140194992476160, 140194992500735, -+STORE, 140194992476160, 140194992500735, -+ERASE, 140194992500736, 140194992517119, -+STORE, 140194992500736, 140194992517119, -+SNULL, 140194992492543, 140194992500735, -+STORE, 140194992476160, 140194992492543, -+STORE, 140194992492544, 140194992500735, -+SNULL, 94809097383935, 94809097388031, -+STORE, 94809097375744, 94809097383935, -+STORE, 94809097383936, 94809097388031, -+SNULL, 140194994761727, 140194994765823, -+STORE, 140194994757632, 140194994761727, -+STORE, 140194994761728, 140194994765823, -+ERASE, 140194994728960, 140194994757631, -+STORE, 94809124286464, 94809124421631, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140726342660096, 140737488351231, -+SNULL, 140726342668287, 140737488351231, -+STORE, 140726342660096, 140726342668287, -+STORE, 140726342529024, 140726342668287, -+STORE, 94140331462656, 94140333686783, -+SNULL, 94140331573247, 94140333686783, -+STORE, 94140331462656, 94140331573247, -+STORE, 94140331573248, 94140333686783, -+ERASE, 94140331573248, 94140333686783, -+STORE, 94140333666304, 94140333678591, -+STORE, 94140333678592, 94140333686783, -+STORE, 140714077208576, 140714079461375, -+SNULL, 140714077351935, 140714079461375, -+STORE, 140714077208576, 140714077351935, -+STORE, 140714077351936, 140714079461375, -+ERASE, 140714077351936, 140714079461375, -+STORE, 140714079449088, 140714079457279, -+STORE, 140714079457280, 140714079461375, -+STORE, 140726343933952, 140726343938047, -+STORE, 140726343921664, 140726343933951, -+STORE, 140714079420416, 140714079449087, -+STORE, 140714079412224, 140714079420415, -+STORE, 140714073411584, 140714077208575, -+SNULL, 140714073411584, 140714075070463, -+STORE, 140714075070464, 140714077208575, -+STORE, 140714073411584, 140714075070463, -+SNULL, 140714077167615, 140714077208575, -+STORE, 140714075070464, 140714077167615, -+STORE, 140714077167616, 140714077208575, -+SNULL, 140714077167616, 140714077192191, -+STORE, 140714077192192, 140714077208575, -+STORE, 140714077167616, 140714077192191, -+ERASE, 140714077167616, 140714077192191, -+STORE, 140714077167616, 140714077192191, -+ERASE, 140714077192192, 140714077208575, -+STORE, 140714077192192, 140714077208575, -+SNULL, 140714077183999, 140714077192191, -+STORE, 140714077167616, 140714077183999, -+STORE, 140714077184000, 140714077192191, -+SNULL, 94140333674495, 94140333678591, -+STORE, 94140333666304, 94140333674495, -+STORE, 94140333674496, 94140333678591, -+SNULL, 140714079453183, 140714079457279, -+STORE, 140714079449088, 140714079453183, -+STORE, 140714079453184, 140714079457279, -+ERASE, 140714079420416, 140714079449087, -+STORE, 94140341432320, 94140341567487, -+STORE, 94431504838656, 94431505051647, -+STORE, 94431507148800, 94431507152895, -+STORE, 94431507152896, 94431507161087, -+STORE, 94431507161088, 94431507173375, -+STORE, 94431510286336, 94431539601407, -+STORE, 139818797948928, 139818799607807, -+STORE, 139818799607808, 139818801704959, -+STORE, 139818801704960, 139818801721343, -+STORE, 139818801721344, 139818801729535, -+STORE, 139818801729536, 139818801745919, -+STORE, 139818801745920, 139818801758207, -+STORE, 139818801758208, 139818803851263, -+STORE, 139818803851264, 139818803855359, -+STORE, 139818803855360, 139818803859455, -+STORE, 139818803859456, 139818804002815, -+STORE, 139818804371456, 139818806054911, -+STORE, 139818806054912, 139818806071295, -+STORE, 139818806099968, 139818806104063, -+STORE, 139818806104064, 139818806108159, -+STORE, 139818806108160, 139818806112255, -+STORE, 140731430457344, 140731430596607, -+STORE, 140731431227392, 140731431239679, -+STORE, 140731431239680, 140731431243775, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140725843607552, 140737488351231, -+SNULL, 140725843615743, 140737488351231, -+STORE, 140725843607552, 140725843615743, -+STORE, 140725843476480, 140725843615743, -+STORE, 94889043505152, 94889045839871, -+SNULL, 94889043718143, 94889045839871, -+STORE, 94889043505152, 94889043718143, -+STORE, 94889043718144, 94889045839871, -+ERASE, 94889043718144, 94889045839871, -+STORE, 94889045815296, 94889045827583, -+STORE, 94889045827584, 94889045839871, -+STORE, 140250965946368, 140250968199167, -+SNULL, 140250966089727, 140250968199167, -+STORE, 140250965946368, 140250966089727, -+STORE, 140250966089728, 140250968199167, -+ERASE, 140250966089728, 140250968199167, -+STORE, 140250968186880, 140250968195071, -+STORE, 140250968195072, 140250968199167, -+STORE, 140725844500480, 140725844504575, -+STORE, 140725844488192, 140725844500479, -+STORE, 140250968158208, 140250968186879, -+STORE, 140250968150016, 140250968158207, -+STORE, 140250963832832, 140250965946367, -+SNULL, 140250963832832, 140250963845119, -+STORE, 140250963845120, 140250965946367, -+STORE, 140250963832832, 140250963845119, -+SNULL, 140250965938175, 140250965946367, -+STORE, 140250963845120, 140250965938175, -+STORE, 140250965938176, 140250965946367, -+ERASE, 140250965938176, 140250965946367, -+STORE, 140250965938176, 140250965946367, -+STORE, 140250960035840, 140250963832831, -+SNULL, 140250960035840, 140250961694719, -+STORE, 140250961694720, 140250963832831, -+STORE, 140250960035840, 140250961694719, -+SNULL, 140250963791871, 140250963832831, -+STORE, 140250961694720, 140250963791871, -+STORE, 140250963791872, 140250963832831, -+SNULL, 140250963791872, 140250963816447, -+STORE, 140250963816448, 140250963832831, -+STORE, 140250963791872, 140250963816447, -+ERASE, 140250963791872, 140250963816447, -+STORE, 140250963791872, 140250963816447, -+ERASE, 140250963816448, 140250963832831, -+STORE, 140250963816448, 140250963832831, -+STORE, 140250968141824, 140250968158207, -+SNULL, 140250963808255, 140250963816447, -+STORE, 140250963791872, 140250963808255, -+STORE, 140250963808256, 140250963816447, -+SNULL, 140250965942271, 140250965946367, -+STORE, 140250965938176, 140250965942271, -+STORE, 140250965942272, 140250965946367, -+SNULL, 94889045819391, 94889045827583, -+STORE, 94889045815296, 94889045819391, -+STORE, 94889045819392, 94889045827583, -+SNULL, 140250968190975, 140250968195071, -+STORE, 140250968186880, 140250968190975, -+STORE, 140250968190976, 140250968195071, -+ERASE, 140250968158208, 140250968186879, -+STORE, 94889052213248, 94889052348415, -+STORE, 140250966458368, 140250968141823, -+STORE, 94889052213248, 94889052483583, -+STORE, 94889052213248, 94889052618751, -+STORE, 94170851819520, 94170852032511, -+STORE, 94170854129664, 94170854133759, -+STORE, 94170854133760, 94170854141951, -+STORE, 94170854141952, 94170854154239, -+STORE, 94170866515968, 94170867740671, -+STORE, 140062030422016, 140062032080895, -+STORE, 140062032080896, 140062034178047, -+STORE, 140062034178048, 140062034194431, -+STORE, 140062034194432, 140062034202623, -+STORE, 140062034202624, 140062034219007, -+STORE, 140062034219008, 140062034231295, -+STORE, 140062034231296, 140062036324351, -+STORE, 140062036324352, 140062036328447, -+STORE, 140062036328448, 140062036332543, -+STORE, 140062036332544, 140062036475903, -+STORE, 140062036844544, 140062038527999, -+STORE, 140062038528000, 140062038544383, -+STORE, 140062038573056, 140062038577151, -+STORE, 140062038577152, 140062038581247, -+STORE, 140062038581248, 140062038585343, -+STORE, 140736210550784, 140736210690047, -+STORE, 140736210759680, 140736210771967, -+STORE, 140736210771968, 140736210776063, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724272365568, 140737488351231, -+SNULL, 140724272373759, 140737488351231, -+STORE, 140724272365568, 140724272373759, -+STORE, 140724272234496, 140724272373759, -+STORE, 94607711965184, 94607714189311, -+SNULL, 94607712075775, 94607714189311, -+STORE, 94607711965184, 94607712075775, -+STORE, 94607712075776, 94607714189311, -+ERASE, 94607712075776, 94607714189311, -+STORE, 94607714168832, 94607714181119, -+STORE, 94607714181120, 94607714189311, -+STORE, 140054949253120, 140054951505919, -+SNULL, 140054949396479, 140054951505919, -+STORE, 140054949253120, 140054949396479, -+STORE, 140054949396480, 140054951505919, -+ERASE, 140054949396480, 140054951505919, -+STORE, 140054951493632, 140054951501823, -+STORE, 140054951501824, 140054951505919, -+STORE, 140724272992256, 140724272996351, -+STORE, 140724272979968, 140724272992255, -+STORE, 140054951464960, 140054951493631, -+STORE, 140054951456768, 140054951464959, -+STORE, 140054945456128, 140054949253119, -+SNULL, 140054945456128, 140054947115007, -+STORE, 140054947115008, 140054949253119, -+STORE, 140054945456128, 140054947115007, -+SNULL, 140054949212159, 140054949253119, -+STORE, 140054947115008, 140054949212159, -+STORE, 140054949212160, 140054949253119, -+SNULL, 140054949212160, 140054949236735, -+STORE, 140054949236736, 140054949253119, -+STORE, 140054949212160, 140054949236735, -+ERASE, 140054949212160, 140054949236735, -+STORE, 140054949212160, 140054949236735, -+ERASE, 140054949236736, 140054949253119, -+STORE, 140054949236736, 140054949253119, -+SNULL, 140054949228543, 140054949236735, -+STORE, 140054949212160, 140054949228543, -+STORE, 140054949228544, 140054949236735, -+SNULL, 94607714177023, 94607714181119, -+STORE, 94607714168832, 94607714177023, -+STORE, 94607714177024, 94607714181119, -+SNULL, 140054951497727, 140054951501823, -+STORE, 140054951493632, 140054951497727, -+STORE, 140054951497728, 140054951501823, -+ERASE, 140054951464960, 140054951493631, -+STORE, 94607733374976, 94607733510143, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733586923520, 140737488351231, -+SNULL, 140733586931711, 140737488351231, -+STORE, 140733586923520, 140733586931711, -+STORE, 140733586792448, 140733586931711, -+STORE, 93901634904064, 93901637128191, -+SNULL, 93901635014655, 93901637128191, -+STORE, 93901634904064, 93901635014655, -+STORE, 93901635014656, 93901637128191, -+ERASE, 93901635014656, 93901637128191, -+STORE, 93901637107712, 93901637119999, -+STORE, 93901637120000, 93901637128191, -+STORE, 140086104784896, 140086107037695, -+SNULL, 140086104928255, 140086107037695, -+STORE, 140086104784896, 140086104928255, -+STORE, 140086104928256, 140086107037695, -+ERASE, 140086104928256, 140086107037695, -+STORE, 140086107025408, 140086107033599, -+STORE, 140086107033600, 140086107037695, -+STORE, 140733587263488, 140733587267583, -+STORE, 140733587251200, 140733587263487, -+STORE, 140086106996736, 140086107025407, -+STORE, 140086106988544, 140086106996735, -+STORE, 140086100987904, 140086104784895, -+SNULL, 140086100987904, 140086102646783, -+STORE, 140086102646784, 140086104784895, -+STORE, 140086100987904, 140086102646783, -+SNULL, 140086104743935, 140086104784895, -+STORE, 140086102646784, 140086104743935, -+STORE, 140086104743936, 140086104784895, -+SNULL, 140086104743936, 140086104768511, -+STORE, 140086104768512, 140086104784895, -+STORE, 140086104743936, 140086104768511, -+ERASE, 140086104743936, 140086104768511, -+STORE, 140086104743936, 140086104768511, -+ERASE, 140086104768512, 140086104784895, -+STORE, 140086104768512, 140086104784895, -+SNULL, 140086104760319, 140086104768511, -+STORE, 140086104743936, 140086104760319, -+STORE, 140086104760320, 140086104768511, -+SNULL, 93901637115903, 93901637119999, -+STORE, 93901637107712, 93901637115903, -+STORE, 93901637115904, 93901637119999, -+SNULL, 140086107029503, 140086107033599, -+STORE, 140086107025408, 140086107029503, -+STORE, 140086107029504, 140086107033599, -+ERASE, 140086106996736, 140086107025407, -+STORE, 93901662715904, 93901662851071, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723365613568, 140737488351231, -+SNULL, 140723365621759, 140737488351231, -+STORE, 140723365613568, 140723365621759, -+STORE, 140723365482496, 140723365621759, -+STORE, 94759193546752, 94759195770879, -+SNULL, 94759193657343, 94759195770879, -+STORE, 94759193546752, 94759193657343, -+STORE, 94759193657344, 94759195770879, -+ERASE, 94759193657344, 94759195770879, -+STORE, 94759195750400, 94759195762687, -+STORE, 94759195762688, 94759195770879, -+STORE, 140607636246528, 140607638499327, -+SNULL, 140607636389887, 140607638499327, -+STORE, 140607636246528, 140607636389887, -+STORE, 140607636389888, 140607638499327, -+ERASE, 140607636389888, 140607638499327, -+STORE, 140607638487040, 140607638495231, -+STORE, 140607638495232, 140607638499327, -+STORE, 140723365900288, 140723365904383, -+STORE, 140723365888000, 140723365900287, -+STORE, 140607638458368, 140607638487039, -+STORE, 140607638450176, 140607638458367, -+STORE, 140607632449536, 140607636246527, -+SNULL, 140607632449536, 140607634108415, -+STORE, 140607634108416, 140607636246527, -+STORE, 140607632449536, 140607634108415, -+SNULL, 140607636205567, 140607636246527, -+STORE, 140607634108416, 140607636205567, -+STORE, 140607636205568, 140607636246527, -+SNULL, 140607636205568, 140607636230143, -+STORE, 140607636230144, 140607636246527, -+STORE, 140607636205568, 140607636230143, -+ERASE, 140607636205568, 140607636230143, -+STORE, 140607636205568, 140607636230143, -+ERASE, 140607636230144, 140607636246527, -+STORE, 140607636230144, 140607636246527, -+SNULL, 140607636221951, 140607636230143, -+STORE, 140607636205568, 140607636221951, -+STORE, 140607636221952, 140607636230143, -+SNULL, 94759195758591, 94759195762687, -+STORE, 94759195750400, 94759195758591, -+STORE, 94759195758592, 94759195762687, -+SNULL, 140607638491135, 140607638495231, -+STORE, 140607638487040, 140607638491135, -+STORE, 140607638491136, 140607638495231, -+ERASE, 140607638458368, 140607638487039, -+STORE, 94759204995072, 94759205130239, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140732503789568, 140737488351231, -+SNULL, 140732503797759, 140737488351231, -+STORE, 140732503789568, 140732503797759, -+STORE, 140732503658496, 140732503797759, -+STORE, 94077792956416, 94077795180543, -+SNULL, 94077793067007, 94077795180543, -+STORE, 94077792956416, 94077793067007, -+STORE, 94077793067008, 94077795180543, -+ERASE, 94077793067008, 94077795180543, -+STORE, 94077795160064, 94077795172351, -+STORE, 94077795172352, 94077795180543, -+STORE, 140359874252800, 140359876505599, -+SNULL, 140359874396159, 140359876505599, -+STORE, 140359874252800, 140359874396159, -+STORE, 140359874396160, 140359876505599, -+ERASE, 140359874396160, 140359876505599, -+STORE, 140359876493312, 140359876501503, -+STORE, 140359876501504, 140359876505599, -+STORE, 140732504465408, 140732504469503, -+STORE, 140732504453120, 140732504465407, -+STORE, 140359876464640, 140359876493311, -+STORE, 140359876456448, 140359876464639, -+STORE, 140359870455808, 140359874252799, -+SNULL, 140359870455808, 140359872114687, -+STORE, 140359872114688, 140359874252799, -+STORE, 140359870455808, 140359872114687, -+SNULL, 140359874211839, 140359874252799, -+STORE, 140359872114688, 140359874211839, -+STORE, 140359874211840, 140359874252799, -+SNULL, 140359874211840, 140359874236415, -+STORE, 140359874236416, 140359874252799, -+STORE, 140359874211840, 140359874236415, -+ERASE, 140359874211840, 140359874236415, -+STORE, 140359874211840, 140359874236415, -+ERASE, 140359874236416, 140359874252799, -+STORE, 140359874236416, 140359874252799, -+SNULL, 140359874228223, 140359874236415, -+STORE, 140359874211840, 140359874228223, -+STORE, 140359874228224, 140359874236415, -+SNULL, 94077795168255, 94077795172351, -+STORE, 94077795160064, 94077795168255, -+STORE, 94077795168256, 94077795172351, -+SNULL, 140359876497407, 140359876501503, -+STORE, 140359876493312, 140359876497407, -+STORE, 140359876497408, 140359876501503, -+ERASE, 140359876464640, 140359876493311, -+STORE, 94077808717824, 94077808852991, -+STORE, 94549486252032, 94549486465023, -+STORE, 94549488562176, 94549488566271, -+STORE, 94549488566272, 94549488574463, -+STORE, 94549488574464, 94549488586751, -+STORE, 94549503492096, 94549506121727, -+STORE, 140085800894464, 140085802553343, -+STORE, 140085802553344, 140085804650495, -+STORE, 140085804650496, 140085804666879, -+STORE, 140085804666880, 140085804675071, -+STORE, 140085804675072, 140085804691455, -+STORE, 140085804691456, 140085804703743, -+STORE, 140085804703744, 140085806796799, -+STORE, 140085806796800, 140085806800895, -+STORE, 140085806800896, 140085806804991, -+STORE, 140085806804992, 140085806948351, -+STORE, 140085807316992, 140085809000447, -+STORE, 140085809000448, 140085809016831, -+STORE, 140085809045504, 140085809049599, -+STORE, 140085809049600, 140085809053695, -+STORE, 140085809053696, 140085809057791, -+STORE, 140731810545664, 140731810684927, -+STORE, 140731810967552, 140731810979839, -+STORE, 140731810979840, 140731810983935, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724752330752, 140737488351231, -+SNULL, 140724752338943, 140737488351231, -+STORE, 140724752330752, 140724752338943, -+STORE, 140724752199680, 140724752338943, -+STORE, 94656357539840, 94656359874559, -+SNULL, 94656357752831, 94656359874559, -+STORE, 94656357539840, 94656357752831, -+STORE, 94656357752832, 94656359874559, -+ERASE, 94656357752832, 94656359874559, -+STORE, 94656359849984, 94656359862271, -+STORE, 94656359862272, 94656359874559, -+STORE, 139632585203712, 139632587456511, -+SNULL, 139632585347071, 139632587456511, -+STORE, 139632585203712, 139632585347071, -+STORE, 139632585347072, 139632587456511, -+ERASE, 139632585347072, 139632587456511, -+STORE, 139632587444224, 139632587452415, -+STORE, 139632587452416, 139632587456511, -+STORE, 139632587440128, 139632587444223, -+STORE, 139632587427840, 139632587440127, -+STORE, 139632587399168, 139632587427839, -+STORE, 139632587390976, 139632587399167, -+STORE, 139632583090176, 139632585203711, -+SNULL, 139632583090176, 139632583102463, -+STORE, 139632583102464, 139632585203711, -+STORE, 139632583090176, 139632583102463, -+SNULL, 139632585195519, 139632585203711, -+STORE, 139632583102464, 139632585195519, -+STORE, 139632585195520, 139632585203711, -+ERASE, 139632585195520, 139632585203711, -+STORE, 139632585195520, 139632585203711, -+STORE, 139632579293184, 139632583090175, -+SNULL, 139632579293184, 139632580952063, -+STORE, 139632580952064, 139632583090175, -+STORE, 139632579293184, 139632580952063, -+SNULL, 139632583049215, 139632583090175, -+STORE, 139632580952064, 139632583049215, -+STORE, 139632583049216, 139632583090175, -+SNULL, 139632583049216, 139632583073791, -+STORE, 139632583073792, 139632583090175, -+STORE, 139632583049216, 139632583073791, -+ERASE, 139632583049216, 139632583073791, -+STORE, 139632583049216, 139632583073791, -+ERASE, 139632583073792, 139632583090175, -+STORE, 139632583073792, 139632583090175, -+STORE, 139632587382784, 139632587399167, -+SNULL, 139632583065599, 139632583073791, -+STORE, 139632583049216, 139632583065599, -+STORE, 139632583065600, 139632583073791, -+SNULL, 139632585199615, 139632585203711, -+STORE, 139632585195520, 139632585199615, -+STORE, 139632585199616, 139632585203711, -+SNULL, 94656359854079, 94656359862271, -+STORE, 94656359849984, 94656359854079, -+STORE, 94656359854080, 94656359862271, -+SNULL, 139632587448319, 139632587452415, -+STORE, 139632587444224, 139632587448319, -+STORE, 139632587448320, 139632587452415, -+ERASE, 139632587399168, 139632587427839, -+STORE, 94656378912768, 94656379047935, -+STORE, 139632585699328, 139632587382783, -+STORE, 94656378912768, 94656379183103, -+STORE, 94656378912768, 94656379318271, -+STORE, 94656378912768, 94656379494399, -+SNULL, 94656379469823, 94656379494399, -+STORE, 94656378912768, 94656379469823, -+STORE, 94656379469824, 94656379494399, -+ERASE, 94656379469824, 94656379494399, -+STORE, 94656378912768, 94656379621375, -+STORE, 94656378912768, 94656379756543, -+STORE, 94656378912768, 94656379912191, -+STORE, 94656378912768, 94656380055551, -+STORE, 94656378912768, 94656380190719, -+STORE, 94656378912768, 94656380338175, -+SNULL, 94656380313599, 94656380338175, -+STORE, 94656378912768, 94656380313599, -+STORE, 94656380313600, 94656380338175, -+ERASE, 94656380313600, 94656380338175, -+STORE, 94656378912768, 94656380448767, -+SNULL, 94656380432383, 94656380448767, -+STORE, 94656378912768, 94656380432383, -+STORE, 94656380432384, 94656380448767, -+ERASE, 94656380432384, 94656380448767, -+STORE, 94656378912768, 94656380567551, -+STORE, 94656378912768, 94656380719103, -+STORE, 94656378912768, 94656380858367, -+STORE, 94656378912768, 94656380997631, -+STORE, 94656378912768, 94656381132799, -+SNULL, 94656381124607, 94656381132799, -+STORE, 94656378912768, 94656381124607, -+STORE, 94656381124608, 94656381132799, -+ERASE, 94656381124608, 94656381132799, -+STORE, 94656378912768, 94656381276159, -+STORE, 94656378912768, 94656381427711, -+STORE, 94604087611392, 94604087824383, -+STORE, 94604089921536, 94604089925631, -+STORE, 94604089925632, 94604089933823, -+STORE, 94604089933824, 94604089946111, -+STORE, 94604105125888, 94604106424319, -+STORE, 140454937694208, 140454939353087, -+STORE, 140454939353088, 140454941450239, -+STORE, 140454941450240, 140454941466623, -+STORE, 140454941466624, 140454941474815, -+STORE, 140454941474816, 140454941491199, -+STORE, 140454941491200, 140454941503487, -+STORE, 140454941503488, 140454943596543, -+STORE, 140454943596544, 140454943600639, -+STORE, 140454943600640, 140454943604735, -+STORE, 140454943604736, 140454943748095, -+STORE, 140454944116736, 140454945800191, -+STORE, 140454945800192, 140454945816575, -+STORE, 140454945845248, 140454945849343, -+STORE, 140454945849344, 140454945853439, -+STORE, 140454945853440, 140454945857535, -+STORE, 140728438214656, 140728438353919, -+STORE, 140728439095296, 140728439107583, -+STORE, 140728439107584, 140728439111679, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140727821099008, 140737488351231, -+SNULL, 140727821107199, 140737488351231, -+STORE, 140727821099008, 140727821107199, -+STORE, 140727820967936, 140727821107199, -+STORE, 94088457240576, 94088459575295, -+SNULL, 94088457453567, 94088459575295, -+STORE, 94088457240576, 94088457453567, -+STORE, 94088457453568, 94088459575295, -+ERASE, 94088457453568, 94088459575295, -+STORE, 94088459550720, 94088459563007, -+STORE, 94088459563008, 94088459575295, -+STORE, 140234378989568, 140234381242367, -+SNULL, 140234379132927, 140234381242367, -+STORE, 140234378989568, 140234379132927, -+STORE, 140234379132928, 140234381242367, -+ERASE, 140234379132928, 140234381242367, -+STORE, 140234381230080, 140234381238271, -+STORE, 140234381238272, 140234381242367, -+STORE, 140727822077952, 140727822082047, -+STORE, 140727822065664, 140727822077951, -+STORE, 140234381201408, 140234381230079, -+STORE, 140234381193216, 140234381201407, -+STORE, 140234376876032, 140234378989567, -+SNULL, 140234376876032, 140234376888319, -+STORE, 140234376888320, 140234378989567, -+STORE, 140234376876032, 140234376888319, -+SNULL, 140234378981375, 140234378989567, -+STORE, 140234376888320, 140234378981375, -+STORE, 140234378981376, 140234378989567, -+ERASE, 140234378981376, 140234378989567, -+STORE, 140234378981376, 140234378989567, -+STORE, 140234373079040, 140234376876031, -+SNULL, 140234373079040, 140234374737919, -+STORE, 140234374737920, 140234376876031, -+STORE, 140234373079040, 140234374737919, -+SNULL, 140234376835071, 140234376876031, -+STORE, 140234374737920, 140234376835071, -+STORE, 140234376835072, 140234376876031, -+SNULL, 140234376835072, 140234376859647, -+STORE, 140234376859648, 140234376876031, -+STORE, 140234376835072, 140234376859647, -+ERASE, 140234376835072, 140234376859647, -+STORE, 140234376835072, 140234376859647, -+ERASE, 140234376859648, 140234376876031, -+STORE, 140234376859648, 140234376876031, -+STORE, 140234381185024, 140234381201407, -+SNULL, 140234376851455, 140234376859647, -+STORE, 140234376835072, 140234376851455, -+STORE, 140234376851456, 140234376859647, -+SNULL, 140234378985471, 140234378989567, -+STORE, 140234378981376, 140234378985471, -+STORE, 140234378985472, 140234378989567, -+SNULL, 94088459554815, 94088459563007, -+STORE, 94088459550720, 94088459554815, -+STORE, 94088459554816, 94088459563007, -+SNULL, 140234381234175, 140234381238271, -+STORE, 140234381230080, 140234381234175, -+STORE, 140234381234176, 140234381238271, -+ERASE, 140234381201408, 140234381230079, -+STORE, 94088468852736, 94088468987903, -+STORE, 140234379501568, 140234381185023, -+STORE, 94088468852736, 94088469123071, -+STORE, 94088468852736, 94088469258239, -+STORE, 94110050402304, 94110050615295, -+STORE, 94110052712448, 94110052716543, -+STORE, 94110052716544, 94110052724735, -+STORE, 94110052724736, 94110052737023, -+STORE, 94110061875200, 94110062415871, -+STORE, 140139439357952, 140139441016831, -+STORE, 140139441016832, 140139443113983, -+STORE, 140139443113984, 140139443130367, -+STORE, 140139443130368, 140139443138559, -+STORE, 140139443138560, 140139443154943, -+STORE, 140139443154944, 140139443167231, -+STORE, 140139443167232, 140139445260287, -+STORE, 140139445260288, 140139445264383, -+STORE, 140139445264384, 140139445268479, -+STORE, 140139445268480, 140139445411839, -+STORE, 140139445780480, 140139447463935, -+STORE, 140139447463936, 140139447480319, -+STORE, 140139447508992, 140139447513087, -+STORE, 140139447513088, 140139447517183, -+STORE, 140139447517184, 140139447521279, -+STORE, 140731901427712, 140731901566975, -+STORE, 140731902259200, 140731902271487, -+STORE, 140731902271488, 140731902275583, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140727282622464, 140737488351231, -+SNULL, 140727282630655, 140737488351231, -+STORE, 140727282622464, 140727282630655, -+STORE, 140727282491392, 140727282630655, -+STORE, 94266649866240, 94266652200959, -+SNULL, 94266650079231, 94266652200959, -+STORE, 94266649866240, 94266650079231, -+STORE, 94266650079232, 94266652200959, -+ERASE, 94266650079232, 94266652200959, -+STORE, 94266652176384, 94266652188671, -+STORE, 94266652188672, 94266652200959, -+STORE, 139888497991680, 139888500244479, -+SNULL, 139888498135039, 139888500244479, -+STORE, 139888497991680, 139888498135039, -+STORE, 139888498135040, 139888500244479, -+ERASE, 139888498135040, 139888500244479, -+STORE, 139888500232192, 139888500240383, -+STORE, 139888500240384, 139888500244479, -+STORE, 140727283113984, 140727283118079, -+STORE, 140727283101696, 140727283113983, -+STORE, 139888500203520, 139888500232191, -+STORE, 139888500195328, 139888500203519, -+STORE, 139888495878144, 139888497991679, -+SNULL, 139888495878144, 139888495890431, -+STORE, 139888495890432, 139888497991679, -+STORE, 139888495878144, 139888495890431, -+SNULL, 139888497983487, 139888497991679, -+STORE, 139888495890432, 139888497983487, -+STORE, 139888497983488, 139888497991679, -+ERASE, 139888497983488, 139888497991679, -+STORE, 139888497983488, 139888497991679, -+STORE, 139888492081152, 139888495878143, -+SNULL, 139888492081152, 139888493740031, -+STORE, 139888493740032, 139888495878143, -+STORE, 139888492081152, 139888493740031, -+SNULL, 139888495837183, 139888495878143, -+STORE, 139888493740032, 139888495837183, -+STORE, 139888495837184, 139888495878143, -+SNULL, 139888495837184, 139888495861759, -+STORE, 139888495861760, 139888495878143, -+STORE, 139888495837184, 139888495861759, -+ERASE, 139888495837184, 139888495861759, -+STORE, 139888495837184, 139888495861759, -+ERASE, 139888495861760, 139888495878143, -+STORE, 139888495861760, 139888495878143, -+STORE, 139888500187136, 139888500203519, -+SNULL, 139888495853567, 139888495861759, -+STORE, 139888495837184, 139888495853567, -+STORE, 139888495853568, 139888495861759, -+SNULL, 139888497987583, 139888497991679, -+STORE, 139888497983488, 139888497987583, -+STORE, 139888497987584, 139888497991679, -+SNULL, 94266652180479, 94266652188671, -+STORE, 94266652176384, 94266652180479, -+STORE, 94266652180480, 94266652188671, -+SNULL, 139888500236287, 139888500240383, -+STORE, 139888500232192, 139888500236287, -+STORE, 139888500236288, 139888500240383, -+ERASE, 139888500203520, 139888500232191, -+STORE, 94266678542336, 94266678677503, -+STORE, 139888498503680, 139888500187135, -+STORE, 94266678542336, 94266678812671, -+STORE, 94266678542336, 94266678947839, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722507702272, 140737488351231, -+SNULL, 140722507710463, 140737488351231, -+STORE, 140722507702272, 140722507710463, -+STORE, 140722507571200, 140722507710463, -+STORE, 94313981394944, 94313983729663, -+SNULL, 94313981607935, 94313983729663, -+STORE, 94313981394944, 94313981607935, -+STORE, 94313981607936, 94313983729663, -+ERASE, 94313981607936, 94313983729663, -+STORE, 94313983705088, 94313983717375, -+STORE, 94313983717376, 94313983729663, -+STORE, 140456286076928, 140456288329727, -+SNULL, 140456286220287, 140456288329727, -+STORE, 140456286076928, 140456286220287, -+STORE, 140456286220288, 140456288329727, -+ERASE, 140456286220288, 140456288329727, -+STORE, 140456288317440, 140456288325631, -+STORE, 140456288325632, 140456288329727, -+STORE, 140722507997184, 140722508001279, -+STORE, 140722507984896, 140722507997183, -+STORE, 140456288288768, 140456288317439, -+STORE, 140456288280576, 140456288288767, -+STORE, 140456283963392, 140456286076927, -+SNULL, 140456283963392, 140456283975679, -+STORE, 140456283975680, 140456286076927, -+STORE, 140456283963392, 140456283975679, -+SNULL, 140456286068735, 140456286076927, -+STORE, 140456283975680, 140456286068735, -+STORE, 140456286068736, 140456286076927, -+ERASE, 140456286068736, 140456286076927, -+STORE, 140456286068736, 140456286076927, -+STORE, 140456280166400, 140456283963391, -+SNULL, 140456280166400, 140456281825279, -+STORE, 140456281825280, 140456283963391, -+STORE, 140456280166400, 140456281825279, -+SNULL, 140456283922431, 140456283963391, -+STORE, 140456281825280, 140456283922431, -+STORE, 140456283922432, 140456283963391, -+SNULL, 140456283922432, 140456283947007, -+STORE, 140456283947008, 140456283963391, -+STORE, 140456283922432, 140456283947007, -+ERASE, 140456283922432, 140456283947007, -+STORE, 140456283922432, 140456283947007, -+ERASE, 140456283947008, 140456283963391, -+STORE, 140456283947008, 140456283963391, -+STORE, 140456288272384, 140456288288767, -+SNULL, 140456283938815, 140456283947007, -+STORE, 140456283922432, 140456283938815, -+STORE, 140456283938816, 140456283947007, -+SNULL, 140456286072831, 140456286076927, -+STORE, 140456286068736, 140456286072831, -+STORE, 140456286072832, 140456286076927, -+SNULL, 94313983709183, 94313983717375, -+STORE, 94313983705088, 94313983709183, -+STORE, 94313983709184, 94313983717375, -+SNULL, 140456288321535, 140456288325631, -+STORE, 140456288317440, 140456288321535, -+STORE, 140456288321536, 140456288325631, -+ERASE, 140456288288768, 140456288317439, -+STORE, 94314006716416, 94314006851583, -+STORE, 140456286588928, 140456288272383, -+STORE, 94314006716416, 94314006986751, -+STORE, 94314006716416, 94314007121919, -+STORE, 93948644454400, 93948644667391, -+STORE, 93948646764544, 93948646768639, -+STORE, 93948646768640, 93948646776831, -+STORE, 93948646776832, 93948646789119, -+STORE, 93948664999936, 93948667142143, -+STORE, 140187350659072, 140187352317951, -+STORE, 140187352317952, 140187354415103, -+STORE, 140187354415104, 140187354431487, -+STORE, 140187354431488, 140187354439679, -+STORE, 140187354439680, 140187354456063, -+STORE, 140187354456064, 140187354468351, -+STORE, 140187354468352, 140187356561407, -+STORE, 140187356561408, 140187356565503, -+STORE, 140187356565504, 140187356569599, -+STORE, 140187356569600, 140187356712959, -+STORE, 140187357081600, 140187358765055, -+STORE, 140187358765056, 140187358781439, -+STORE, 140187358810112, 140187358814207, -+STORE, 140187358814208, 140187358818303, -+STORE, 140187358818304, 140187358822399, -+STORE, 140730484518912, 140730484658175, -+STORE, 140730485690368, 140730485702655, -+STORE, 140730485702656, 140730485706751, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140721211551744, 140737488351231, -+SNULL, 140721211559935, 140737488351231, -+STORE, 140721211551744, 140721211559935, -+STORE, 140721211420672, 140721211559935, -+STORE, 94105221423104, 94105223757823, -+SNULL, 94105221636095, 94105223757823, -+STORE, 94105221423104, 94105221636095, -+STORE, 94105221636096, 94105223757823, -+ERASE, 94105221636096, 94105223757823, -+STORE, 94105223733248, 94105223745535, -+STORE, 94105223745536, 94105223757823, -+STORE, 140474453676032, 140474455928831, -+SNULL, 140474453819391, 140474455928831, -+STORE, 140474453676032, 140474453819391, -+STORE, 140474453819392, 140474455928831, -+ERASE, 140474453819392, 140474455928831, -+STORE, 140474455916544, 140474455924735, -+STORE, 140474455924736, 140474455928831, -+STORE, 140721211703296, 140721211707391, -+STORE, 140721211691008, 140721211703295, -+STORE, 140474455887872, 140474455916543, -+STORE, 140474455879680, 140474455887871, -+STORE, 140474451562496, 140474453676031, -+SNULL, 140474451562496, 140474451574783, -+STORE, 140474451574784, 140474453676031, -+STORE, 140474451562496, 140474451574783, -+SNULL, 140474453667839, 140474453676031, -+STORE, 140474451574784, 140474453667839, -+STORE, 140474453667840, 140474453676031, -+ERASE, 140474453667840, 140474453676031, -+STORE, 140474453667840, 140474453676031, -+STORE, 140474447765504, 140474451562495, -+SNULL, 140474447765504, 140474449424383, -+STORE, 140474449424384, 140474451562495, -+STORE, 140474447765504, 140474449424383, -+SNULL, 140474451521535, 140474451562495, -+STORE, 140474449424384, 140474451521535, -+STORE, 140474451521536, 140474451562495, -+SNULL, 140474451521536, 140474451546111, -+STORE, 140474451546112, 140474451562495, -+STORE, 140474451521536, 140474451546111, -+ERASE, 140474451521536, 140474451546111, -+STORE, 140474451521536, 140474451546111, -+ERASE, 140474451546112, 140474451562495, -+STORE, 140474451546112, 140474451562495, -+STORE, 140474455871488, 140474455887871, -+SNULL, 140474451537919, 140474451546111, -+STORE, 140474451521536, 140474451537919, -+STORE, 140474451537920, 140474451546111, -+SNULL, 140474453671935, 140474453676031, -+STORE, 140474453667840, 140474453671935, -+STORE, 140474453671936, 140474453676031, -+SNULL, 94105223737343, 94105223745535, -+STORE, 94105223733248, 94105223737343, -+STORE, 94105223737344, 94105223745535, -+SNULL, 140474455920639, 140474455924735, -+STORE, 140474455916544, 140474455920639, -+STORE, 140474455920640, 140474455924735, -+ERASE, 140474455887872, 140474455916543, -+STORE, 94105238712320, 94105238847487, -+STORE, 140474454188032, 140474455871487, -+STORE, 94105238712320, 94105238982655, -+STORE, 94105238712320, 94105239117823, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140732356354048, 140737488351231, -+SNULL, 140732356362239, 140737488351231, -+STORE, 140732356354048, 140732356362239, -+STORE, 140732356222976, 140732356362239, -+STORE, 94461165989888, 94461168324607, -+SNULL, 94461166202879, 94461168324607, -+STORE, 94461165989888, 94461166202879, -+STORE, 94461166202880, 94461168324607, -+ERASE, 94461166202880, 94461168324607, -+STORE, 94461168300032, 94461168312319, -+STORE, 94461168312320, 94461168324607, -+STORE, 140317255110656, 140317257363455, -+SNULL, 140317255254015, 140317257363455, -+STORE, 140317255110656, 140317255254015, -+STORE, 140317255254016, 140317257363455, -+ERASE, 140317255254016, 140317257363455, -+STORE, 140317257351168, 140317257359359, -+STORE, 140317257359360, 140317257363455, -+STORE, 140732356583424, 140732356587519, -+STORE, 140732356571136, 140732356583423, -+STORE, 140317257322496, 140317257351167, -+STORE, 140317257314304, 140317257322495, -+STORE, 140317252997120, 140317255110655, -+SNULL, 140317252997120, 140317253009407, -+STORE, 140317253009408, 140317255110655, -+STORE, 140317252997120, 140317253009407, -+SNULL, 140317255102463, 140317255110655, -+STORE, 140317253009408, 140317255102463, -+STORE, 140317255102464, 140317255110655, -+ERASE, 140317255102464, 140317255110655, -+STORE, 140317255102464, 140317255110655, -+STORE, 140317249200128, 140317252997119, -+SNULL, 140317249200128, 140317250859007, -+STORE, 140317250859008, 140317252997119, -+STORE, 140317249200128, 140317250859007, -+SNULL, 140317252956159, 140317252997119, -+STORE, 140317250859008, 140317252956159, -+STORE, 140317252956160, 140317252997119, -+SNULL, 140317252956160, 140317252980735, -+STORE, 140317252980736, 140317252997119, -+STORE, 140317252956160, 140317252980735, -+ERASE, 140317252956160, 140317252980735, -+STORE, 140317252956160, 140317252980735, -+ERASE, 140317252980736, 140317252997119, -+STORE, 140317252980736, 140317252997119, -+STORE, 140317257306112, 140317257322495, -+SNULL, 140317252972543, 140317252980735, -+STORE, 140317252956160, 140317252972543, -+STORE, 140317252972544, 140317252980735, -+SNULL, 140317255106559, 140317255110655, -+STORE, 140317255102464, 140317255106559, -+STORE, 140317255106560, 140317255110655, -+SNULL, 94461168304127, 94461168312319, -+STORE, 94461168300032, 94461168304127, -+STORE, 94461168304128, 94461168312319, -+SNULL, 140317257355263, 140317257359359, -+STORE, 140317257351168, 140317257355263, -+STORE, 140317257355264, 140317257359359, -+ERASE, 140317257322496, 140317257351167, -+STORE, 94461195268096, 94461195403263, -+STORE, 140317255622656, 140317257306111, -+STORE, 94461195268096, 94461195538431, -+STORE, 94461195268096, 94461195673599, -+STORE, 94110050402304, 94110050615295, -+STORE, 94110052712448, 94110052716543, -+STORE, 94110052716544, 94110052724735, -+STORE, 94110052724736, 94110052737023, -+STORE, 94110061875200, 94110062415871, -+STORE, 140139439357952, 140139441016831, -+STORE, 140139441016832, 140139443113983, -+STORE, 140139443113984, 140139443130367, -+STORE, 140139443130368, 140139443138559, -+STORE, 140139443138560, 140139443154943, -+STORE, 140139443154944, 140139443167231, -+STORE, 140139443167232, 140139445260287, -+STORE, 140139445260288, 140139445264383, -+STORE, 140139445264384, 140139445268479, -+STORE, 140139445268480, 140139445411839, -+STORE, 140139445780480, 140139447463935, -+STORE, 140139447463936, 140139447480319, -+STORE, 140139447508992, 140139447513087, -+STORE, 140139447513088, 140139447517183, -+STORE, 140139447517184, 140139447521279, -+STORE, 140731901427712, 140731901566975, -+STORE, 140731902259200, 140731902271487, -+STORE, 140731902271488, 140731902275583, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140720941613056, 140737488351231, -+SNULL, 140720941621247, 140737488351231, -+STORE, 140720941613056, 140720941621247, -+STORE, 140720941481984, 140720941621247, -+STORE, 93902377721856, 93902379945983, -+SNULL, 93902377832447, 93902379945983, -+STORE, 93902377721856, 93902377832447, -+STORE, 93902377832448, 93902379945983, -+ERASE, 93902377832448, 93902379945983, -+STORE, 93902379925504, 93902379937791, -+STORE, 93902379937792, 93902379945983, -+STORE, 139836543635456, 139836545888255, -+SNULL, 139836543778815, 139836545888255, -+STORE, 139836543635456, 139836543778815, -+STORE, 139836543778816, 139836545888255, -+ERASE, 139836543778816, 139836545888255, -+STORE, 139836545875968, 139836545884159, -+STORE, 139836545884160, 139836545888255, -+STORE, 140720941711360, 140720941715455, -+STORE, 140720941699072, 140720941711359, -+STORE, 139836545847296, 139836545875967, -+STORE, 139836545839104, 139836545847295, -+STORE, 139836539838464, 139836543635455, -+SNULL, 139836539838464, 139836541497343, -+STORE, 139836541497344, 139836543635455, -+STORE, 139836539838464, 139836541497343, -+SNULL, 139836543594495, 139836543635455, -+STORE, 139836541497344, 139836543594495, -+STORE, 139836543594496, 139836543635455, -+SNULL, 139836543594496, 139836543619071, -+STORE, 139836543619072, 139836543635455, -+STORE, 139836543594496, 139836543619071, -+ERASE, 139836543594496, 139836543619071, -+STORE, 139836543594496, 139836543619071, -+ERASE, 139836543619072, 139836543635455, -+STORE, 139836543619072, 139836543635455, -+SNULL, 139836543610879, 139836543619071, -+STORE, 139836543594496, 139836543610879, -+STORE, 139836543610880, 139836543619071, -+SNULL, 93902379933695, 93902379937791, -+STORE, 93902379925504, 93902379933695, -+STORE, 93902379933696, 93902379937791, -+SNULL, 139836545880063, 139836545884159, -+STORE, 139836545875968, 139836545880063, -+STORE, 139836545880064, 139836545884159, -+ERASE, 139836545847296, 139836545875967, -+STORE, 93902396891136, 93902397026303, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140736538206208, 140737488351231, -+SNULL, 140736538214399, 140737488351231, -+STORE, 140736538206208, 140736538214399, -+STORE, 140736538075136, 140736538214399, -+STORE, 94173471399936, 94173473734655, -+SNULL, 94173471612927, 94173473734655, -+STORE, 94173471399936, 94173471612927, -+STORE, 94173471612928, 94173473734655, -+ERASE, 94173471612928, 94173473734655, -+STORE, 94173473710080, 94173473722367, -+STORE, 94173473722368, 94173473734655, -+STORE, 140035513556992, 140035515809791, -+SNULL, 140035513700351, 140035515809791, -+STORE, 140035513556992, 140035513700351, -+STORE, 140035513700352, 140035515809791, -+ERASE, 140035513700352, 140035515809791, -+STORE, 140035515797504, 140035515805695, -+STORE, 140035515805696, 140035515809791, -+STORE, 140736538329088, 140736538333183, -+STORE, 140736538316800, 140736538329087, -+STORE, 140035515768832, 140035515797503, -+STORE, 140035515760640, 140035515768831, -+STORE, 140035511443456, 140035513556991, -+SNULL, 140035511443456, 140035511455743, -+STORE, 140035511455744, 140035513556991, -+STORE, 140035511443456, 140035511455743, -+SNULL, 140035513548799, 140035513556991, -+STORE, 140035511455744, 140035513548799, -+STORE, 140035513548800, 140035513556991, -+ERASE, 140035513548800, 140035513556991, -+STORE, 140035513548800, 140035513556991, -+STORE, 140035507646464, 140035511443455, -+SNULL, 140035507646464, 140035509305343, -+STORE, 140035509305344, 140035511443455, -+STORE, 140035507646464, 140035509305343, -+SNULL, 140035511402495, 140035511443455, -+STORE, 140035509305344, 140035511402495, -+STORE, 140035511402496, 140035511443455, -+SNULL, 140035511402496, 140035511427071, -+STORE, 140035511427072, 140035511443455, -+STORE, 140035511402496, 140035511427071, -+ERASE, 140035511402496, 140035511427071, -+STORE, 140035511402496, 140035511427071, -+ERASE, 140035511427072, 140035511443455, -+STORE, 140035511427072, 140035511443455, -+STORE, 140035515752448, 140035515768831, -+SNULL, 140035511418879, 140035511427071, -+STORE, 140035511402496, 140035511418879, -+STORE, 140035511418880, 140035511427071, -+SNULL, 140035513552895, 140035513556991, -+STORE, 140035513548800, 140035513552895, -+STORE, 140035513552896, 140035513556991, -+SNULL, 94173473714175, 94173473722367, -+STORE, 94173473710080, 94173473714175, -+STORE, 94173473714176, 94173473722367, -+SNULL, 140035515801599, 140035515805695, -+STORE, 140035515797504, 140035515801599, -+STORE, 140035515801600, 140035515805695, -+ERASE, 140035515768832, 140035515797503, -+STORE, 94173478645760, 94173478780927, -+STORE, 140035514068992, 140035515752447, -+STORE, 94173478645760, 94173478916095, -+STORE, 94173478645760, 94173479051263, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140724216176640, 140737488351231, -+SNULL, 140724216184831, 140737488351231, -+STORE, 140724216176640, 140724216184831, -+STORE, 140724216045568, 140724216184831, -+STORE, 94870930628608, 94870932963327, -+SNULL, 94870930841599, 94870932963327, -+STORE, 94870930628608, 94870930841599, -+STORE, 94870930841600, 94870932963327, -+ERASE, 94870930841600, 94870932963327, -+STORE, 94870932938752, 94870932951039, -+STORE, 94870932951040, 94870932963327, -+STORE, 140453683736576, 140453685989375, -+SNULL, 140453683879935, 140453685989375, -+STORE, 140453683736576, 140453683879935, -+STORE, 140453683879936, 140453685989375, -+ERASE, 140453683879936, 140453685989375, -+STORE, 140453685977088, 140453685985279, -+STORE, 140453685985280, 140453685989375, -+STORE, 140724216832000, 140724216836095, -+STORE, 140724216819712, 140724216831999, -+STORE, 140453685948416, 140453685977087, -+STORE, 140453685940224, 140453685948415, -+STORE, 140453681623040, 140453683736575, -+SNULL, 140453681623040, 140453681635327, -+STORE, 140453681635328, 140453683736575, -+STORE, 140453681623040, 140453681635327, -+SNULL, 140453683728383, 140453683736575, -+STORE, 140453681635328, 140453683728383, -+STORE, 140453683728384, 140453683736575, -+ERASE, 140453683728384, 140453683736575, -+STORE, 140453683728384, 140453683736575, -+STORE, 140453677826048, 140453681623039, -+SNULL, 140453677826048, 140453679484927, -+STORE, 140453679484928, 140453681623039, -+STORE, 140453677826048, 140453679484927, -+SNULL, 140453681582079, 140453681623039, -+STORE, 140453679484928, 140453681582079, -+STORE, 140453681582080, 140453681623039, -+SNULL, 140453681582080, 140453681606655, -+STORE, 140453681606656, 140453681623039, -+STORE, 140453681582080, 140453681606655, -+ERASE, 140453681582080, 140453681606655, -+STORE, 140453681582080, 140453681606655, -+ERASE, 140453681606656, 140453681623039, -+STORE, 140453681606656, 140453681623039, -+STORE, 140453685932032, 140453685948415, -+SNULL, 140453681598463, 140453681606655, -+STORE, 140453681582080, 140453681598463, -+STORE, 140453681598464, 140453681606655, -+SNULL, 140453683732479, 140453683736575, -+STORE, 140453683728384, 140453683732479, -+STORE, 140453683732480, 140453683736575, -+SNULL, 94870932942847, 94870932951039, -+STORE, 94870932938752, 94870932942847, -+STORE, 94870932942848, 94870932951039, -+SNULL, 140453685981183, 140453685985279, -+STORE, 140453685977088, 140453685981183, -+STORE, 140453685981184, 140453685985279, -+ERASE, 140453685948416, 140453685977087, -+STORE, 94870940565504, 94870940700671, -+STORE, 140453684248576, 140453685932031, -+STORE, 94870940565504, 94870940835839, -+STORE, 94870940565504, 94870940971007, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731275661312, 140737488351231, -+SNULL, 140731275669503, 140737488351231, -+STORE, 140731275661312, 140731275669503, -+STORE, 140731275530240, 140731275669503, -+STORE, 94642788548608, 94642790883327, -+SNULL, 94642788761599, 94642790883327, -+STORE, 94642788548608, 94642788761599, -+STORE, 94642788761600, 94642790883327, -+ERASE, 94642788761600, 94642790883327, -+STORE, 94642790858752, 94642790871039, -+STORE, 94642790871040, 94642790883327, -+STORE, 140228458749952, 140228461002751, -+SNULL, 140228458893311, 140228461002751, -+STORE, 140228458749952, 140228458893311, -+STORE, 140228458893312, 140228461002751, -+ERASE, 140228458893312, 140228461002751, -+STORE, 140228460990464, 140228460998655, -+STORE, 140228460998656, 140228461002751, -+STORE, 140731276349440, 140731276353535, -+STORE, 140731276337152, 140731276349439, -+STORE, 140228460961792, 140228460990463, -+STORE, 140228460953600, 140228460961791, -+STORE, 140228456636416, 140228458749951, -+SNULL, 140228456636416, 140228456648703, -+STORE, 140228456648704, 140228458749951, -+STORE, 140228456636416, 140228456648703, -+SNULL, 140228458741759, 140228458749951, -+STORE, 140228456648704, 140228458741759, -+STORE, 140228458741760, 140228458749951, -+ERASE, 140228458741760, 140228458749951, -+STORE, 140228458741760, 140228458749951, -+STORE, 140228452839424, 140228456636415, -+SNULL, 140228452839424, 140228454498303, -+STORE, 140228454498304, 140228456636415, -+STORE, 140228452839424, 140228454498303, -+SNULL, 140228456595455, 140228456636415, -+STORE, 140228454498304, 140228456595455, -+STORE, 140228456595456, 140228456636415, -+SNULL, 140228456595456, 140228456620031, -+STORE, 140228456620032, 140228456636415, -+STORE, 140228456595456, 140228456620031, -+ERASE, 140228456595456, 140228456620031, -+STORE, 140228456595456, 140228456620031, -+ERASE, 140228456620032, 140228456636415, -+STORE, 140228456620032, 140228456636415, -+STORE, 140228460945408, 140228460961791, -+SNULL, 140228456611839, 140228456620031, -+STORE, 140228456595456, 140228456611839, -+STORE, 140228456611840, 140228456620031, -+SNULL, 140228458745855, 140228458749951, -+STORE, 140228458741760, 140228458745855, -+STORE, 140228458745856, 140228458749951, -+SNULL, 94642790862847, 94642790871039, -+STORE, 94642790858752, 94642790862847, -+STORE, 94642790862848, 94642790871039, -+SNULL, 140228460994559, 140228460998655, -+STORE, 140228460990464, 140228460994559, -+STORE, 140228460994560, 140228460998655, -+ERASE, 140228460961792, 140228460990463, -+STORE, 94642801549312, 94642801684479, -+STORE, 140228459261952, 140228460945407, -+STORE, 94642801549312, 94642801819647, -+STORE, 94642801549312, 94642801954815, -+STORE, 94604087611392, 94604087824383, -+STORE, 94604089921536, 94604089925631, -+STORE, 94604089925632, 94604089933823, -+STORE, 94604089933824, 94604089946111, -+STORE, 94604105125888, 94604106424319, -+STORE, 140454937694208, 140454939353087, -+STORE, 140454939353088, 140454941450239, -+STORE, 140454941450240, 140454941466623, -+STORE, 140454941466624, 140454941474815, -+STORE, 140454941474816, 140454941491199, -+STORE, 140454941491200, 140454941503487, -+STORE, 140454941503488, 140454943596543, -+STORE, 140454943596544, 140454943600639, -+STORE, 140454943600640, 140454943604735, -+STORE, 140454943604736, 140454943748095, -+STORE, 140454944116736, 140454945800191, -+STORE, 140454945800192, 140454945816575, -+STORE, 140454945845248, 140454945849343, -+STORE, 140454945849344, 140454945853439, -+STORE, 140454945853440, 140454945857535, -+STORE, 140728438214656, 140728438353919, -+STORE, 140728439095296, 140728439107583, -+STORE, 140728439107584, 140728439111679, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140721843453952, 140737488351231, -+SNULL, 140721843462143, 140737488351231, -+STORE, 140721843453952, 140721843462143, -+STORE, 140721843322880, 140721843462143, -+STORE, 94465962455040, 94465964789759, -+SNULL, 94465962668031, 94465964789759, -+STORE, 94465962455040, 94465962668031, -+STORE, 94465962668032, 94465964789759, -+ERASE, 94465962668032, 94465964789759, -+STORE, 94465964765184, 94465964777471, -+STORE, 94465964777472, 94465964789759, -+STORE, 139913488314368, 139913490567167, -+SNULL, 139913488457727, 139913490567167, -+STORE, 139913488314368, 139913488457727, -+STORE, 139913488457728, 139913490567167, -+ERASE, 139913488457728, 139913490567167, -+STORE, 139913490554880, 139913490563071, -+STORE, 139913490563072, 139913490567167, -+STORE, 140721843503104, 140721843507199, -+STORE, 140721843490816, 140721843503103, -+STORE, 139913490526208, 139913490554879, -+STORE, 139913490518016, 139913490526207, -+STORE, 139913486200832, 139913488314367, -+SNULL, 139913486200832, 139913486213119, -+STORE, 139913486213120, 139913488314367, -+STORE, 139913486200832, 139913486213119, -+SNULL, 139913488306175, 139913488314367, -+STORE, 139913486213120, 139913488306175, -+STORE, 139913488306176, 139913488314367, -+ERASE, 139913488306176, 139913488314367, -+STORE, 139913488306176, 139913488314367, -+STORE, 139913482403840, 139913486200831, -+SNULL, 139913482403840, 139913484062719, -+STORE, 139913484062720, 139913486200831, -+STORE, 139913482403840, 139913484062719, -+SNULL, 139913486159871, 139913486200831, -+STORE, 139913484062720, 139913486159871, -+STORE, 139913486159872, 139913486200831, -+SNULL, 139913486159872, 139913486184447, -+STORE, 139913486184448, 139913486200831, -+STORE, 139913486159872, 139913486184447, -+ERASE, 139913486159872, 139913486184447, -+STORE, 139913486159872, 139913486184447, -+ERASE, 139913486184448, 139913486200831, -+STORE, 139913486184448, 139913486200831, -+STORE, 139913490509824, 139913490526207, -+SNULL, 139913486176255, 139913486184447, -+STORE, 139913486159872, 139913486176255, -+STORE, 139913486176256, 139913486184447, -+SNULL, 139913488310271, 139913488314367, -+STORE, 139913488306176, 139913488310271, -+STORE, 139913488310272, 139913488314367, -+SNULL, 94465964769279, 94465964777471, -+STORE, 94465964765184, 94465964769279, -+STORE, 94465964769280, 94465964777471, -+SNULL, 139913490558975, 139913490563071, -+STORE, 139913490554880, 139913490558975, -+STORE, 139913490558976, 139913490563071, -+ERASE, 139913490526208, 139913490554879, -+STORE, 94465970024448, 94465970159615, -+STORE, 139913488826368, 139913490509823, -+STORE, 94465970024448, 94465970294783, -+STORE, 94465970024448, 94465970429951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140720583307264, 140737488351231, -+SNULL, 140720583315455, 140737488351231, -+STORE, 140720583307264, 140720583315455, -+STORE, 140720583176192, 140720583315455, -+STORE, 94212322082816, 94212324417535, -+SNULL, 94212322295807, 94212324417535, -+STORE, 94212322082816, 94212322295807, -+STORE, 94212322295808, 94212324417535, -+ERASE, 94212322295808, 94212324417535, -+STORE, 94212324392960, 94212324405247, -+STORE, 94212324405248, 94212324417535, -+STORE, 139659688538112, 139659690790911, -+SNULL, 139659688681471, 139659690790911, -+STORE, 139659688538112, 139659688681471, -+STORE, 139659688681472, 139659690790911, -+ERASE, 139659688681472, 139659690790911, -+STORE, 139659690778624, 139659690786815, -+STORE, 139659690786816, 139659690790911, -+STORE, 140720584781824, 140720584785919, -+STORE, 140720584769536, 140720584781823, -+STORE, 139659690749952, 139659690778623, -+STORE, 139659690741760, 139659690749951, -+STORE, 139659686424576, 139659688538111, -+SNULL, 139659686424576, 139659686436863, -+STORE, 139659686436864, 139659688538111, -+STORE, 139659686424576, 139659686436863, -+SNULL, 139659688529919, 139659688538111, -+STORE, 139659686436864, 139659688529919, -+STORE, 139659688529920, 139659688538111, -+ERASE, 139659688529920, 139659688538111, -+STORE, 139659688529920, 139659688538111, -+STORE, 139659682627584, 139659686424575, -+SNULL, 139659682627584, 139659684286463, -+STORE, 139659684286464, 139659686424575, -+STORE, 139659682627584, 139659684286463, -+SNULL, 139659686383615, 139659686424575, -+STORE, 139659684286464, 139659686383615, -+STORE, 139659686383616, 139659686424575, -+SNULL, 139659686383616, 139659686408191, -+STORE, 139659686408192, 139659686424575, -+STORE, 139659686383616, 139659686408191, -+ERASE, 139659686383616, 139659686408191, -+STORE, 139659686383616, 139659686408191, -+ERASE, 139659686408192, 139659686424575, -+STORE, 139659686408192, 139659686424575, -+STORE, 139659690733568, 139659690749951, -+SNULL, 139659686399999, 139659686408191, -+STORE, 139659686383616, 139659686399999, -+STORE, 139659686400000, 139659686408191, -+SNULL, 139659688534015, 139659688538111, -+STORE, 139659688529920, 139659688534015, -+STORE, 139659688534016, 139659688538111, -+SNULL, 94212324397055, 94212324405247, -+STORE, 94212324392960, 94212324397055, -+STORE, 94212324397056, 94212324405247, -+SNULL, 139659690782719, 139659690786815, -+STORE, 139659690778624, 139659690782719, -+STORE, 139659690782720, 139659690786815, -+ERASE, 139659690749952, 139659690778623, -+STORE, 94212355014656, 94212355149823, -+STORE, 139659689050112, 139659690733567, -+STORE, 94212355014656, 94212355284991, -+STORE, 94212355014656, 94212355420159, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140727689830400, 140737488351231, -+SNULL, 140727689838591, 140737488351231, -+STORE, 140727689830400, 140727689838591, -+STORE, 140727689699328, 140727689838591, -+STORE, 94572390281216, 94572392615935, -+SNULL, 94572390494207, 94572392615935, -+STORE, 94572390281216, 94572390494207, -+STORE, 94572390494208, 94572392615935, -+ERASE, 94572390494208, 94572392615935, -+STORE, 94572392591360, 94572392603647, -+STORE, 94572392603648, 94572392615935, -+STORE, 140575923769344, 140575926022143, -+SNULL, 140575923912703, 140575926022143, -+STORE, 140575923769344, 140575923912703, -+STORE, 140575923912704, 140575926022143, -+ERASE, 140575923912704, 140575926022143, -+STORE, 140575926009856, 140575926018047, -+STORE, 140575926018048, 140575926022143, -+STORE, 140727689871360, 140727689875455, -+STORE, 140727689859072, 140727689871359, -+STORE, 140575925981184, 140575926009855, -+STORE, 140575925972992, 140575925981183, -+STORE, 140575921655808, 140575923769343, -+SNULL, 140575921655808, 140575921668095, -+STORE, 140575921668096, 140575923769343, -+STORE, 140575921655808, 140575921668095, -+SNULL, 140575923761151, 140575923769343, -+STORE, 140575921668096, 140575923761151, -+STORE, 140575923761152, 140575923769343, -+ERASE, 140575923761152, 140575923769343, -+STORE, 140575923761152, 140575923769343, -+STORE, 140575917858816, 140575921655807, -+SNULL, 140575917858816, 140575919517695, -+STORE, 140575919517696, 140575921655807, -+STORE, 140575917858816, 140575919517695, -+SNULL, 140575921614847, 140575921655807, -+STORE, 140575919517696, 140575921614847, -+STORE, 140575921614848, 140575921655807, -+SNULL, 140575921614848, 140575921639423, -+STORE, 140575921639424, 140575921655807, -+STORE, 140575921614848, 140575921639423, -+ERASE, 140575921614848, 140575921639423, -+STORE, 140575921614848, 140575921639423, -+ERASE, 140575921639424, 140575921655807, -+STORE, 140575921639424, 140575921655807, -+STORE, 140575925964800, 140575925981183, -+SNULL, 140575921631231, 140575921639423, -+STORE, 140575921614848, 140575921631231, -+STORE, 140575921631232, 140575921639423, -+SNULL, 140575923765247, 140575923769343, -+STORE, 140575923761152, 140575923765247, -+STORE, 140575923765248, 140575923769343, -+SNULL, 94572392595455, 94572392603647, -+STORE, 94572392591360, 94572392595455, -+STORE, 94572392595456, 94572392603647, -+SNULL, 140575926013951, 140575926018047, -+STORE, 140575926009856, 140575926013951, -+STORE, 140575926013952, 140575926018047, -+ERASE, 140575925981184, 140575926009855, -+STORE, 94572402278400, 94572402413567, -+STORE, 140575924281344, 140575925964799, -+STORE, 94572402278400, 94572402548735, -+STORE, 94572402278400, 94572402683903, -+STORE, 94572402278400, 94572402851839, -+SNULL, 94572402827263, 94572402851839, -+STORE, 94572402278400, 94572402827263, -+STORE, 94572402827264, 94572402851839, -+ERASE, 94572402827264, 94572402851839, -+STORE, 94572402278400, 94572402966527, -+STORE, 94572402278400, 94572403109887, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140725520506880, 140737488351231, -+SNULL, 140725520515071, 140737488351231, -+STORE, 140725520506880, 140725520515071, -+STORE, 140725520375808, 140725520515071, -+STORE, 93829948788736, 93829951012863, -+SNULL, 93829948899327, 93829951012863, -+STORE, 93829948788736, 93829948899327, -+STORE, 93829948899328, 93829951012863, -+ERASE, 93829948899328, 93829951012863, -+STORE, 93829950992384, 93829951004671, -+STORE, 93829951004672, 93829951012863, -+STORE, 140133696794624, 140133699047423, -+SNULL, 140133696937983, 140133699047423, -+STORE, 140133696794624, 140133696937983, -+STORE, 140133696937984, 140133699047423, -+ERASE, 140133696937984, 140133699047423, -+STORE, 140133699035136, 140133699043327, -+STORE, 140133699043328, 140133699047423, -+STORE, 140725520875520, 140725520879615, -+STORE, 140725520863232, 140725520875519, -+STORE, 140133699006464, 140133699035135, -+STORE, 140133698998272, 140133699006463, -+STORE, 140133692997632, 140133696794623, -+SNULL, 140133692997632, 140133694656511, -+STORE, 140133694656512, 140133696794623, -+STORE, 140133692997632, 140133694656511, -+SNULL, 140133696753663, 140133696794623, -+STORE, 140133694656512, 140133696753663, -+STORE, 140133696753664, 140133696794623, -+SNULL, 140133696753664, 140133696778239, -+STORE, 140133696778240, 140133696794623, -+STORE, 140133696753664, 140133696778239, -+ERASE, 140133696753664, 140133696778239, -+STORE, 140133696753664, 140133696778239, -+ERASE, 140133696778240, 140133696794623, -+STORE, 140133696778240, 140133696794623, -+SNULL, 140133696770047, 140133696778239, -+STORE, 140133696753664, 140133696770047, -+STORE, 140133696770048, 140133696778239, -+SNULL, 93829951000575, 93829951004671, -+STORE, 93829950992384, 93829951000575, -+STORE, 93829951000576, 93829951004671, -+SNULL, 140133699039231, 140133699043327, -+STORE, 140133699035136, 140133699039231, -+STORE, 140133699039232, 140133699043327, -+ERASE, 140133699006464, 140133699035135, -+STORE, 93829978693632, 93829978828799, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140736118022144, 140737488351231, -+SNULL, 140736118030335, 140737488351231, -+STORE, 140736118022144, 140736118030335, -+STORE, 140736117891072, 140736118030335, -+STORE, 94467663982592, 94467666206719, -+SNULL, 94467664093183, 94467666206719, -+STORE, 94467663982592, 94467664093183, -+STORE, 94467664093184, 94467666206719, -+ERASE, 94467664093184, 94467666206719, -+STORE, 94467666186240, 94467666198527, -+STORE, 94467666198528, 94467666206719, -+STORE, 140525377327104, 140525379579903, -+SNULL, 140525377470463, 140525379579903, -+STORE, 140525377327104, 140525377470463, -+STORE, 140525377470464, 140525379579903, -+ERASE, 140525377470464, 140525379579903, -+STORE, 140525379567616, 140525379575807, -+STORE, 140525379575808, 140525379579903, -+STORE, 140736118771712, 140736118775807, -+STORE, 140736118759424, 140736118771711, -+STORE, 140525379538944, 140525379567615, -+STORE, 140525379530752, 140525379538943, -+STORE, 140525373530112, 140525377327103, -+SNULL, 140525373530112, 140525375188991, -+STORE, 140525375188992, 140525377327103, -+STORE, 140525373530112, 140525375188991, -+SNULL, 140525377286143, 140525377327103, -+STORE, 140525375188992, 140525377286143, -+STORE, 140525377286144, 140525377327103, -+SNULL, 140525377286144, 140525377310719, -+STORE, 140525377310720, 140525377327103, -+STORE, 140525377286144, 140525377310719, -+ERASE, 140525377286144, 140525377310719, -+STORE, 140525377286144, 140525377310719, -+ERASE, 140525377310720, 140525377327103, -+STORE, 140525377310720, 140525377327103, -+SNULL, 140525377302527, 140525377310719, -+STORE, 140525377286144, 140525377302527, -+STORE, 140525377302528, 140525377310719, -+SNULL, 94467666194431, 94467666198527, -+STORE, 94467666186240, 94467666194431, -+STORE, 94467666194432, 94467666198527, -+SNULL, 140525379571711, 140525379575807, -+STORE, 140525379567616, 140525379571711, -+STORE, 140525379571712, 140525379575807, -+ERASE, 140525379538944, 140525379567615, -+STORE, 94467693379584, 94467693514751, -+STORE, 94200172744704, 94200172957695, -+STORE, 94200175054848, 94200175058943, -+STORE, 94200175058944, 94200175067135, -+STORE, 94200175067136, 94200175079423, -+STORE, 94200196673536, 94200198905855, -+STORE, 140053867720704, 140053869379583, -+STORE, 140053869379584, 140053871476735, -+STORE, 140053871476736, 140053871493119, -+STORE, 140053871493120, 140053871501311, -+STORE, 140053871501312, 140053871517695, -+STORE, 140053871517696, 140053871529983, -+STORE, 140053871529984, 140053873623039, -+STORE, 140053873623040, 140053873627135, -+STORE, 140053873627136, 140053873631231, -+STORE, 140053873631232, 140053873774591, -+STORE, 140053874143232, 140053875826687, -+STORE, 140053875826688, 140053875843071, -+STORE, 140053875871744, 140053875875839, -+STORE, 140053875875840, 140053875879935, -+STORE, 140053875879936, 140053875884031, -+STORE, 140728538484736, 140728538623999, -+STORE, 140728538652672, 140728538664959, -+STORE, 140728538664960, 140728538669055, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140732307775488, 140737488351231, -+SNULL, 140732307783679, 140737488351231, -+STORE, 140732307775488, 140732307783679, -+STORE, 140732307644416, 140732307783679, -+STORE, 93831417630720, 93831419965439, -+SNULL, 93831417843711, 93831419965439, -+STORE, 93831417630720, 93831417843711, -+STORE, 93831417843712, 93831419965439, -+ERASE, 93831417843712, 93831419965439, -+STORE, 93831419940864, 93831419953151, -+STORE, 93831419953152, 93831419965439, -+STORE, 140241062088704, 140241064341503, -+SNULL, 140241062232063, 140241064341503, -+STORE, 140241062088704, 140241062232063, -+STORE, 140241062232064, 140241064341503, -+ERASE, 140241062232064, 140241064341503, -+STORE, 140241064329216, 140241064337407, -+STORE, 140241064337408, 140241064341503, -+STORE, 140732308140032, 140732308144127, -+STORE, 140732308127744, 140732308140031, -+STORE, 140241064300544, 140241064329215, -+STORE, 140241064292352, 140241064300543, -+STORE, 140241059975168, 140241062088703, -+SNULL, 140241059975168, 140241059987455, -+STORE, 140241059987456, 140241062088703, -+STORE, 140241059975168, 140241059987455, -+SNULL, 140241062080511, 140241062088703, -+STORE, 140241059987456, 140241062080511, -+STORE, 140241062080512, 140241062088703, -+ERASE, 140241062080512, 140241062088703, -+STORE, 140241062080512, 140241062088703, -+STORE, 140241056178176, 140241059975167, -+SNULL, 140241056178176, 140241057837055, -+STORE, 140241057837056, 140241059975167, -+STORE, 140241056178176, 140241057837055, -+SNULL, 140241059934207, 140241059975167, -+STORE, 140241057837056, 140241059934207, -+STORE, 140241059934208, 140241059975167, -+SNULL, 140241059934208, 140241059958783, -+STORE, 140241059958784, 140241059975167, -+STORE, 140241059934208, 140241059958783, -+ERASE, 140241059934208, 140241059958783, -+STORE, 140241059934208, 140241059958783, -+ERASE, 140241059958784, 140241059975167, -+STORE, 140241059958784, 140241059975167, -+STORE, 140241064284160, 140241064300543, -+SNULL, 140241059950591, 140241059958783, -+STORE, 140241059934208, 140241059950591, -+STORE, 140241059950592, 140241059958783, -+SNULL, 140241062084607, 140241062088703, -+STORE, 140241062080512, 140241062084607, -+STORE, 140241062084608, 140241062088703, -+SNULL, 93831419944959, 93831419953151, -+STORE, 93831419940864, 93831419944959, -+STORE, 93831419944960, 93831419953151, -+SNULL, 140241064333311, 140241064337407, -+STORE, 140241064329216, 140241064333311, -+STORE, 140241064333312, 140241064337407, -+ERASE, 140241064300544, 140241064329215, -+STORE, 93831435284480, 93831435419647, -+STORE, 140241062600704, 140241064284159, -+STORE, 93831435284480, 93831435554815, -+STORE, 93831435284480, 93831435689983, -+STORE, 93831435284480, 93831435862015, -+SNULL, 93831435837439, 93831435862015, -+STORE, 93831435284480, 93831435837439, -+STORE, 93831435837440, 93831435862015, -+ERASE, 93831435837440, 93831435862015, -+STORE, 93831435284480, 93831435972607, -+STORE, 93831435284480, 93831436107775, -+SNULL, 93831436091391, 93831436107775, -+STORE, 93831435284480, 93831436091391, -+STORE, 93831436091392, 93831436107775, -+ERASE, 93831436091392, 93831436107775, -+STORE, 93831435284480, 93831436226559, -+STORE, 93831435284480, 93831436361727, -+STORE, 93831435284480, 93831436505087, -+STORE, 93831435284480, 93831436652543, -+STORE, 93831435284480, 93831436787711, -+STORE, 93831435284480, 93831436926975, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140728546775040, 140737488351231, -+SNULL, 140728546783231, 140737488351231, -+STORE, 140728546775040, 140728546783231, -+STORE, 140728546643968, 140728546783231, -+STORE, 94456178786304, 94456181010431, -+SNULL, 94456178896895, 94456181010431, -+STORE, 94456178786304, 94456178896895, -+STORE, 94456178896896, 94456181010431, -+ERASE, 94456178896896, 94456181010431, -+STORE, 94456180989952, 94456181002239, -+STORE, 94456181002240, 94456181010431, -+STORE, 140221893091328, 140221895344127, -+SNULL, 140221893234687, 140221895344127, -+STORE, 140221893091328, 140221893234687, -+STORE, 140221893234688, 140221895344127, -+ERASE, 140221893234688, 140221895344127, -+STORE, 140221895331840, 140221895340031, -+STORE, 140221895340032, 140221895344127, -+STORE, 140728547803136, 140728547807231, -+STORE, 140728547790848, 140728547803135, -+STORE, 140221895303168, 140221895331839, -+STORE, 140221895294976, 140221895303167, -+STORE, 140221889294336, 140221893091327, -+SNULL, 140221889294336, 140221890953215, -+STORE, 140221890953216, 140221893091327, -+STORE, 140221889294336, 140221890953215, -+SNULL, 140221893050367, 140221893091327, -+STORE, 140221890953216, 140221893050367, -+STORE, 140221893050368, 140221893091327, -+SNULL, 140221893050368, 140221893074943, -+STORE, 140221893074944, 140221893091327, -+STORE, 140221893050368, 140221893074943, -+ERASE, 140221893050368, 140221893074943, -+STORE, 140221893050368, 140221893074943, -+ERASE, 140221893074944, 140221893091327, -+STORE, 140221893074944, 140221893091327, -+SNULL, 140221893066751, 140221893074943, -+STORE, 140221893050368, 140221893066751, -+STORE, 140221893066752, 140221893074943, -+SNULL, 94456180998143, 94456181002239, -+STORE, 94456180989952, 94456180998143, -+STORE, 94456180998144, 94456181002239, -+SNULL, 140221895335935, 140221895340031, -+STORE, 140221895331840, 140221895335935, -+STORE, 140221895335936, 140221895340031, -+ERASE, 140221895303168, 140221895331839, -+STORE, 94456203730944, 94456203866111, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140734438637568, 140737488351231, -+SNULL, 140734438645759, 140737488351231, -+STORE, 140734438637568, 140734438645759, -+STORE, 140734438506496, 140734438645759, -+STORE, 94652233351168, 94652235575295, -+SNULL, 94652233461759, 94652235575295, -+STORE, 94652233351168, 94652233461759, -+STORE, 94652233461760, 94652235575295, -+ERASE, 94652233461760, 94652235575295, -+STORE, 94652235554816, 94652235567103, -+STORE, 94652235567104, 94652235575295, -+STORE, 140536493195264, 140536495448063, -+SNULL, 140536493338623, 140536495448063, -+STORE, 140536493195264, 140536493338623, -+STORE, 140536493338624, 140536495448063, -+ERASE, 140536493338624, 140536495448063, -+STORE, 140536495435776, 140536495443967, -+STORE, 140536495443968, 140536495448063, -+STORE, 140734439002112, 140734439006207, -+STORE, 140734438989824, 140734439002111, -+STORE, 140536495407104, 140536495435775, -+STORE, 140536495398912, 140536495407103, -+STORE, 140536489398272, 140536493195263, -+SNULL, 140536489398272, 140536491057151, -+STORE, 140536491057152, 140536493195263, -+STORE, 140536489398272, 140536491057151, -+SNULL, 140536493154303, 140536493195263, -+STORE, 140536491057152, 140536493154303, -+STORE, 140536493154304, 140536493195263, -+SNULL, 140536493154304, 140536493178879, -+STORE, 140536493178880, 140536493195263, -+STORE, 140536493154304, 140536493178879, -+ERASE, 140536493154304, 140536493178879, -+STORE, 140536493154304, 140536493178879, -+ERASE, 140536493178880, 140536493195263, -+STORE, 140536493178880, 140536493195263, -+SNULL, 140536493170687, 140536493178879, -+STORE, 140536493154304, 140536493170687, -+STORE, 140536493170688, 140536493178879, -+SNULL, 94652235563007, 94652235567103, -+STORE, 94652235554816, 94652235563007, -+STORE, 94652235563008, 94652235567103, -+SNULL, 140536495439871, 140536495443967, -+STORE, 140536495435776, 140536495439871, -+STORE, 140536495439872, 140536495443967, -+ERASE, 140536495407104, 140536495435775, -+STORE, 94652265619456, 94652265754623, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140721814200320, 140737488351231, -+SNULL, 140721814208511, 140737488351231, -+STORE, 140721814200320, 140721814208511, -+STORE, 140721814069248, 140721814208511, -+STORE, 94062800691200, 94062802915327, -+SNULL, 94062800801791, 94062802915327, -+STORE, 94062800691200, 94062800801791, -+STORE, 94062800801792, 94062802915327, -+ERASE, 94062800801792, 94062802915327, -+STORE, 94062802894848, 94062802907135, -+STORE, 94062802907136, 94062802915327, -+STORE, 139717739700224, 139717741953023, -+SNULL, 139717739843583, 139717741953023, -+STORE, 139717739700224, 139717739843583, -+STORE, 139717739843584, 139717741953023, -+ERASE, 139717739843584, 139717741953023, -+STORE, 139717741940736, 139717741948927, -+STORE, 139717741948928, 139717741953023, -+STORE, 140721814224896, 140721814228991, -+STORE, 140721814212608, 140721814224895, -+STORE, 139717741912064, 139717741940735, -+STORE, 139717741903872, 139717741912063, -+STORE, 139717735903232, 139717739700223, -+SNULL, 139717735903232, 139717737562111, -+STORE, 139717737562112, 139717739700223, -+STORE, 139717735903232, 139717737562111, -+SNULL, 139717739659263, 139717739700223, -+STORE, 139717737562112, 139717739659263, -+STORE, 139717739659264, 139717739700223, -+SNULL, 139717739659264, 139717739683839, -+STORE, 139717739683840, 139717739700223, -+STORE, 139717739659264, 139717739683839, -+ERASE, 139717739659264, 139717739683839, -+STORE, 139717739659264, 139717739683839, -+ERASE, 139717739683840, 139717739700223, -+STORE, 139717739683840, 139717739700223, -+SNULL, 139717739675647, 139717739683839, -+STORE, 139717739659264, 139717739675647, -+STORE, 139717739675648, 139717739683839, -+SNULL, 94062802903039, 94062802907135, -+STORE, 94062802894848, 94062802903039, -+STORE, 94062802903040, 94062802907135, -+SNULL, 139717741944831, 139717741948927, -+STORE, 139717741940736, 139717741944831, -+STORE, 139717741944832, 139717741948927, -+ERASE, 139717741912064, 139717741940735, -+STORE, 94062814060544, 94062814195711, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723945754624, 140737488351231, -+SNULL, 140723945762815, 140737488351231, -+STORE, 140723945754624, 140723945762815, -+STORE, 140723945623552, 140723945762815, -+STORE, 94886119305216, 94886121639935, -+SNULL, 94886119518207, 94886121639935, -+STORE, 94886119305216, 94886119518207, -+STORE, 94886119518208, 94886121639935, -+ERASE, 94886119518208, 94886121639935, -+STORE, 94886121615360, 94886121627647, -+STORE, 94886121627648, 94886121639935, -+STORE, 140152532131840, 140152534384639, -+SNULL, 140152532275199, 140152534384639, -+STORE, 140152532131840, 140152532275199, -+STORE, 140152532275200, 140152534384639, -+ERASE, 140152532275200, 140152534384639, -+STORE, 140152534372352, 140152534380543, -+STORE, 140152534380544, 140152534384639, -+STORE, 140723946213376, 140723946217471, -+STORE, 140723946201088, 140723946213375, -+STORE, 140152534343680, 140152534372351, -+STORE, 140152534335488, 140152534343679, -+STORE, 140152530018304, 140152532131839, -+SNULL, 140152530018304, 140152530030591, -+STORE, 140152530030592, 140152532131839, -+STORE, 140152530018304, 140152530030591, -+SNULL, 140152532123647, 140152532131839, -+STORE, 140152530030592, 140152532123647, -+STORE, 140152532123648, 140152532131839, -+ERASE, 140152532123648, 140152532131839, -+STORE, 140152532123648, 140152532131839, -+STORE, 140152526221312, 140152530018303, -+SNULL, 140152526221312, 140152527880191, -+STORE, 140152527880192, 140152530018303, -+STORE, 140152526221312, 140152527880191, -+SNULL, 140152529977343, 140152530018303, -+STORE, 140152527880192, 140152529977343, -+STORE, 140152529977344, 140152530018303, -+SNULL, 140152529977344, 140152530001919, -+STORE, 140152530001920, 140152530018303, -+STORE, 140152529977344, 140152530001919, -+ERASE, 140152529977344, 140152530001919, -+STORE, 140152529977344, 140152530001919, -+ERASE, 140152530001920, 140152530018303, -+STORE, 140152530001920, 140152530018303, -+STORE, 140152534327296, 140152534343679, -+SNULL, 140152529993727, 140152530001919, -+STORE, 140152529977344, 140152529993727, -+STORE, 140152529993728, 140152530001919, -+SNULL, 140152532127743, 140152532131839, -+STORE, 140152532123648, 140152532127743, -+STORE, 140152532127744, 140152532131839, -+SNULL, 94886121619455, 94886121627647, -+STORE, 94886121615360, 94886121619455, -+STORE, 94886121619456, 94886121627647, -+SNULL, 140152534376447, 140152534380543, -+STORE, 140152534372352, 140152534376447, -+STORE, 140152534376448, 140152534380543, -+ERASE, 140152534343680, 140152534372351, -+STORE, 94886129770496, 94886129905663, -+STORE, 140152532643840, 140152534327295, -+STORE, 94886129770496, 94886130040831, -+STORE, 94886129770496, 94886130175999, -+STORE, 94886129770496, 94886130348031, -+SNULL, 94886130323455, 94886130348031, -+STORE, 94886129770496, 94886130323455, -+STORE, 94886130323456, 94886130348031, -+ERASE, 94886130323456, 94886130348031, -+STORE, 94886129770496, 94886130458623, -+STORE, 94886129770496, 94886130606079, -+SNULL, 94886130573311, 94886130606079, -+STORE, 94886129770496, 94886130573311, -+STORE, 94886130573312, 94886130606079, -+ERASE, 94886130573312, 94886130606079, -+STORE, 94886129770496, 94886130724863, -+STORE, 94886129770496, 94886130876415, -+STORE, 94886129770496, 94886131023871, -+STORE, 94886129770496, 94886131175423, -+STORE, 94886129770496, 94886131318783, -+STORE, 94886129770496, 94886131453951, -+SNULL, 94886131449855, 94886131453951, -+STORE, 94886129770496, 94886131449855, -+STORE, 94886131449856, 94886131453951, -+ERASE, 94886131449856, 94886131453951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735450779648, 140737488351231, -+SNULL, 140735450787839, 140737488351231, -+STORE, 140735450779648, 140735450787839, -+STORE, 140735450648576, 140735450787839, -+STORE, 93947794079744, 93947796414463, -+SNULL, 93947794292735, 93947796414463, -+STORE, 93947794079744, 93947794292735, -+STORE, 93947794292736, 93947796414463, -+ERASE, 93947794292736, 93947796414463, -+STORE, 93947796389888, 93947796402175, -+STORE, 93947796402176, 93947796414463, -+STORE, 139841993433088, 139841995685887, -+SNULL, 139841993576447, 139841995685887, -+STORE, 139841993433088, 139841993576447, -+STORE, 139841993576448, 139841995685887, -+ERASE, 139841993576448, 139841995685887, -+STORE, 139841995673600, 139841995681791, -+STORE, 139841995681792, 139841995685887, -+STORE, 140735451308032, 140735451312127, -+STORE, 140735451295744, 140735451308031, -+STORE, 139841995644928, 139841995673599, -+STORE, 139841995636736, 139841995644927, -+STORE, 139841991319552, 139841993433087, -+SNULL, 139841991319552, 139841991331839, -+STORE, 139841991331840, 139841993433087, -+STORE, 139841991319552, 139841991331839, -+SNULL, 139841993424895, 139841993433087, -+STORE, 139841991331840, 139841993424895, -+STORE, 139841993424896, 139841993433087, -+ERASE, 139841993424896, 139841993433087, -+STORE, 139841993424896, 139841993433087, -+STORE, 139841987522560, 139841991319551, -+SNULL, 139841987522560, 139841989181439, -+STORE, 139841989181440, 139841991319551, -+STORE, 139841987522560, 139841989181439, -+SNULL, 139841991278591, 139841991319551, -+STORE, 139841989181440, 139841991278591, -+STORE, 139841991278592, 139841991319551, -+SNULL, 139841991278592, 139841991303167, -+STORE, 139841991303168, 139841991319551, -+STORE, 139841991278592, 139841991303167, -+ERASE, 139841991278592, 139841991303167, -+STORE, 139841991278592, 139841991303167, -+ERASE, 139841991303168, 139841991319551, -+STORE, 139841991303168, 139841991319551, -+STORE, 139841995628544, 139841995644927, -+SNULL, 139841991294975, 139841991303167, -+STORE, 139841991278592, 139841991294975, -+STORE, 139841991294976, 139841991303167, -+SNULL, 139841993428991, 139841993433087, -+STORE, 139841993424896, 139841993428991, -+STORE, 139841993428992, 139841993433087, -+SNULL, 93947796393983, 93947796402175, -+STORE, 93947796389888, 93947796393983, -+STORE, 93947796393984, 93947796402175, -+SNULL, 139841995677695, 139841995681791, -+STORE, 139841995673600, 139841995677695, -+STORE, 139841995677696, 139841995681791, -+ERASE, 139841995644928, 139841995673599, -+STORE, 93947829739520, 93947829874687, -+STORE, 139841993945088, 139841995628543, -+STORE, 93947829739520, 93947830009855, -+STORE, 93947829739520, 93947830145023, -+STORE, 94659351814144, 94659352027135, -+STORE, 94659354124288, 94659354128383, -+STORE, 94659354128384, 94659354136575, -+STORE, 94659354136576, 94659354148863, -+STORE, 94659383476224, 94659385057279, -+STORE, 139959054557184, 139959056216063, -+STORE, 139959056216064, 139959058313215, -+STORE, 139959058313216, 139959058329599, -+STORE, 139959058329600, 139959058337791, -+STORE, 139959058337792, 139959058354175, -+STORE, 139959058354176, 139959058366463, -+STORE, 139959058366464, 139959060459519, -+STORE, 139959060459520, 139959060463615, -+STORE, 139959060463616, 139959060467711, -+STORE, 139959060467712, 139959060611071, -+STORE, 139959060979712, 139959062663167, -+STORE, 139959062663168, 139959062679551, -+STORE, 139959062708224, 139959062712319, -+STORE, 139959062712320, 139959062716415, -+STORE, 139959062716416, 139959062720511, -+STORE, 140735532539904, 140735532679167, -+STORE, 140735532830720, 140735532843007, -+STORE, 140735532843008, 140735532847103, -+STORE, 93894361829376, 93894362042367, -+STORE, 93894364139520, 93894364143615, -+STORE, 93894364143616, 93894364151807, -+STORE, 93894364151808, 93894364164095, -+STORE, 93894396944384, 93894397624319, -+STORE, 140075612573696, 140075614232575, -+STORE, 140075614232576, 140075616329727, -+STORE, 140075616329728, 140075616346111, -+STORE, 140075616346112, 140075616354303, -+STORE, 140075616354304, 140075616370687, -+STORE, 140075616370688, 140075616382975, -+STORE, 140075616382976, 140075618476031, -+STORE, 140075618476032, 140075618480127, -+STORE, 140075618480128, 140075618484223, -+STORE, 140075618484224, 140075618627583, -+STORE, 140075618996224, 140075620679679, -+STORE, 140075620679680, 140075620696063, -+STORE, 140075620724736, 140075620728831, -+STORE, 140075620728832, 140075620732927, -+STORE, 140075620732928, 140075620737023, -+STORE, 140720830312448, 140720830451711, -+STORE, 140720830631936, 140720830644223, -+STORE, 140720830644224, 140720830648319, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735116226560, 140737488351231, -+SNULL, 140735116234751, 140737488351231, -+STORE, 140735116226560, 140735116234751, -+STORE, 140735116095488, 140735116234751, -+STORE, 94873398054912, 94873400279039, -+SNULL, 94873398165503, 94873400279039, -+STORE, 94873398054912, 94873398165503, -+STORE, 94873398165504, 94873400279039, -+ERASE, 94873398165504, 94873400279039, -+STORE, 94873400258560, 94873400270847, -+STORE, 94873400270848, 94873400279039, -+STORE, 140303828606976, 140303830859775, -+SNULL, 140303828750335, 140303830859775, -+STORE, 140303828606976, 140303828750335, -+STORE, 140303828750336, 140303830859775, -+ERASE, 140303828750336, 140303830859775, -+STORE, 140303830847488, 140303830855679, -+STORE, 140303830855680, 140303830859775, -+STORE, 140735116251136, 140735116255231, -+STORE, 140735116238848, 140735116251135, -+STORE, 140303830818816, 140303830847487, -+STORE, 140303830810624, 140303830818815, -+STORE, 140303824809984, 140303828606975, -+SNULL, 140303824809984, 140303826468863, -+STORE, 140303826468864, 140303828606975, -+STORE, 140303824809984, 140303826468863, -+SNULL, 140303828566015, 140303828606975, -+STORE, 140303826468864, 140303828566015, -+STORE, 140303828566016, 140303828606975, -+SNULL, 140303828566016, 140303828590591, -+STORE, 140303828590592, 140303828606975, -+STORE, 140303828566016, 140303828590591, -+ERASE, 140303828566016, 140303828590591, -+STORE, 140303828566016, 140303828590591, -+ERASE, 140303828590592, 140303828606975, -+STORE, 140303828590592, 140303828606975, -+SNULL, 140303828582399, 140303828590591, -+STORE, 140303828566016, 140303828582399, -+STORE, 140303828582400, 140303828590591, -+SNULL, 94873400266751, 94873400270847, -+STORE, 94873400258560, 94873400266751, -+STORE, 94873400266752, 94873400270847, -+SNULL, 140303830851583, 140303830855679, -+STORE, 140303830847488, 140303830851583, -+STORE, 140303830851584, 140303830855679, -+ERASE, 140303830818816, 140303830847487, -+STORE, 94873413713920, 94873413849087, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140732349956096, 140737488351231, -+SNULL, 140732349964287, 140737488351231, -+STORE, 140732349956096, 140732349964287, -+STORE, 140732349825024, 140732349964287, -+STORE, 94009652736000, 94009655070719, -+SNULL, 94009652948991, 94009655070719, -+STORE, 94009652736000, 94009652948991, -+STORE, 94009652948992, 94009655070719, -+ERASE, 94009652948992, 94009655070719, -+STORE, 94009655046144, 94009655058431, -+STORE, 94009655058432, 94009655070719, -+STORE, 140295688531968, 140295690784767, -+SNULL, 140295688675327, 140295690784767, -+STORE, 140295688531968, 140295688675327, -+STORE, 140295688675328, 140295690784767, -+ERASE, 140295688675328, 140295690784767, -+STORE, 140295690772480, 140295690780671, -+STORE, 140295690780672, 140295690784767, -+STORE, 140732350005248, 140732350009343, -+STORE, 140732349992960, 140732350005247, -+STORE, 140295690743808, 140295690772479, -+STORE, 140295690735616, 140295690743807, -+STORE, 140295686418432, 140295688531967, -+SNULL, 140295686418432, 140295686430719, -+STORE, 140295686430720, 140295688531967, -+STORE, 140295686418432, 140295686430719, -+SNULL, 140295688523775, 140295688531967, -+STORE, 140295686430720, 140295688523775, -+STORE, 140295688523776, 140295688531967, -+ERASE, 140295688523776, 140295688531967, -+STORE, 140295688523776, 140295688531967, -+STORE, 140295682621440, 140295686418431, -+SNULL, 140295682621440, 140295684280319, -+STORE, 140295684280320, 140295686418431, -+STORE, 140295682621440, 140295684280319, -+SNULL, 140295686377471, 140295686418431, -+STORE, 140295684280320, 140295686377471, -+STORE, 140295686377472, 140295686418431, -+SNULL, 140295686377472, 140295686402047, -+STORE, 140295686402048, 140295686418431, -+STORE, 140295686377472, 140295686402047, -+ERASE, 140295686377472, 140295686402047, -+STORE, 140295686377472, 140295686402047, -+ERASE, 140295686402048, 140295686418431, -+STORE, 140295686402048, 140295686418431, -+STORE, 140295690727424, 140295690743807, -+SNULL, 140295686393855, 140295686402047, -+STORE, 140295686377472, 140295686393855, -+STORE, 140295686393856, 140295686402047, -+SNULL, 140295688527871, 140295688531967, -+STORE, 140295688523776, 140295688527871, -+STORE, 140295688527872, 140295688531967, -+SNULL, 94009655050239, 94009655058431, -+STORE, 94009655046144, 94009655050239, -+STORE, 94009655050240, 94009655058431, -+SNULL, 140295690776575, 140295690780671, -+STORE, 140295690772480, 140295690776575, -+STORE, 140295690776576, 140295690780671, -+ERASE, 140295690743808, 140295690772479, -+STORE, 94009672114176, 94009672249343, -+STORE, 140295689043968, 140295690727423, -+STORE, 94009672114176, 94009672384511, -+STORE, 94009672114176, 94009672519679, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722376515584, 140737488351231, -+SNULL, 140722376523775, 140737488351231, -+STORE, 140722376515584, 140722376523775, -+STORE, 140722376384512, 140722376523775, -+STORE, 94089815773184, 94089818107903, -+SNULL, 94089815986175, 94089818107903, -+STORE, 94089815773184, 94089815986175, -+STORE, 94089815986176, 94089818107903, -+ERASE, 94089815986176, 94089818107903, -+STORE, 94089818083328, 94089818095615, -+STORE, 94089818095616, 94089818107903, -+STORE, 140265595711488, 140265597964287, -+SNULL, 140265595854847, 140265597964287, -+STORE, 140265595711488, 140265595854847, -+STORE, 140265595854848, 140265597964287, -+ERASE, 140265595854848, 140265597964287, -+STORE, 140265597952000, 140265597960191, -+STORE, 140265597960192, 140265597964287, -+STORE, 140722378297344, 140722378301439, -+STORE, 140722378285056, 140722378297343, -+STORE, 140265597923328, 140265597951999, -+STORE, 140265597915136, 140265597923327, -+STORE, 140265593597952, 140265595711487, -+SNULL, 140265593597952, 140265593610239, -+STORE, 140265593610240, 140265595711487, -+STORE, 140265593597952, 140265593610239, -+SNULL, 140265595703295, 140265595711487, -+STORE, 140265593610240, 140265595703295, -+STORE, 140265595703296, 140265595711487, -+ERASE, 140265595703296, 140265595711487, -+STORE, 140265595703296, 140265595711487, -+STORE, 140265589800960, 140265593597951, -+SNULL, 140265589800960, 140265591459839, -+STORE, 140265591459840, 140265593597951, -+STORE, 140265589800960, 140265591459839, -+SNULL, 140265593556991, 140265593597951, -+STORE, 140265591459840, 140265593556991, -+STORE, 140265593556992, 140265593597951, -+SNULL, 140265593556992, 140265593581567, -+STORE, 140265593581568, 140265593597951, -+STORE, 140265593556992, 140265593581567, -+ERASE, 140265593556992, 140265593581567, -+STORE, 140265593556992, 140265593581567, -+ERASE, 140265593581568, 140265593597951, -+STORE, 140265593581568, 140265593597951, -+STORE, 140265597906944, 140265597923327, -+SNULL, 140265593573375, 140265593581567, -+STORE, 140265593556992, 140265593573375, -+STORE, 140265593573376, 140265593581567, -+SNULL, 140265595707391, 140265595711487, -+STORE, 140265595703296, 140265595707391, -+STORE, 140265595707392, 140265595711487, -+SNULL, 94089818087423, 94089818095615, -+STORE, 94089818083328, 94089818087423, -+STORE, 94089818087424, 94089818095615, -+SNULL, 140265597956095, 140265597960191, -+STORE, 140265597952000, 140265597956095, -+STORE, 140265597956096, 140265597960191, -+ERASE, 140265597923328, 140265597951999, -+STORE, 94089837146112, 94089837281279, -+STORE, 140265596223488, 140265597906943, -+STORE, 94089837146112, 94089837416447, -+STORE, 94089837146112, 94089837551615, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735265218560, 140737488351231, -+SNULL, 140735265226751, 140737488351231, -+STORE, 140735265218560, 140735265226751, -+STORE, 140735265087488, 140735265226751, -+STORE, 94250422370304, 94250424705023, -+SNULL, 94250422583295, 94250424705023, -+STORE, 94250422370304, 94250422583295, -+STORE, 94250422583296, 94250424705023, -+ERASE, 94250422583296, 94250424705023, -+STORE, 94250424680448, 94250424692735, -+STORE, 94250424692736, 94250424705023, -+STORE, 140344442474496, 140344444727295, -+SNULL, 140344442617855, 140344444727295, -+STORE, 140344442474496, 140344442617855, -+STORE, 140344442617856, 140344444727295, -+ERASE, 140344442617856, 140344444727295, -+STORE, 140344444715008, 140344444723199, -+STORE, 140344444723200, 140344444727295, -+STORE, 140735265341440, 140735265345535, -+STORE, 140735265329152, 140735265341439, -+STORE, 140344444686336, 140344444715007, -+STORE, 140344444678144, 140344444686335, -+STORE, 140344440360960, 140344442474495, -+SNULL, 140344440360960, 140344440373247, -+STORE, 140344440373248, 140344442474495, -+STORE, 140344440360960, 140344440373247, -+SNULL, 140344442466303, 140344442474495, -+STORE, 140344440373248, 140344442466303, -+STORE, 140344442466304, 140344442474495, -+ERASE, 140344442466304, 140344442474495, -+STORE, 140344442466304, 140344442474495, -+STORE, 140344436563968, 140344440360959, -+SNULL, 140344436563968, 140344438222847, -+STORE, 140344438222848, 140344440360959, -+STORE, 140344436563968, 140344438222847, -+SNULL, 140344440319999, 140344440360959, -+STORE, 140344438222848, 140344440319999, -+STORE, 140344440320000, 140344440360959, -+SNULL, 140344440320000, 140344440344575, -+STORE, 140344440344576, 140344440360959, -+STORE, 140344440320000, 140344440344575, -+ERASE, 140344440320000, 140344440344575, -+STORE, 140344440320000, 140344440344575, -+ERASE, 140344440344576, 140344440360959, -+STORE, 140344440344576, 140344440360959, -+STORE, 140344444669952, 140344444686335, -+SNULL, 140344440336383, 140344440344575, -+STORE, 140344440320000, 140344440336383, -+STORE, 140344440336384, 140344440344575, -+SNULL, 140344442470399, 140344442474495, -+STORE, 140344442466304, 140344442470399, -+STORE, 140344442470400, 140344442474495, -+SNULL, 94250424684543, 94250424692735, -+STORE, 94250424680448, 94250424684543, -+STORE, 94250424684544, 94250424692735, -+SNULL, 140344444719103, 140344444723199, -+STORE, 140344444715008, 140344444719103, -+STORE, 140344444719104, 140344444723199, -+ERASE, 140344444686336, 140344444715007, -+STORE, 94250445512704, 94250445647871, -+STORE, 140344442986496, 140344444669951, -+STORE, 94250445512704, 94250445783039, -+STORE, 94250445512704, 94250445918207, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140725762719744, 140737488351231, -+SNULL, 140725762727935, 140737488351231, -+STORE, 140725762719744, 140725762727935, -+STORE, 140725762588672, 140725762727935, -+STORE, 94819009097728, 94819011432447, -+SNULL, 94819009310719, 94819011432447, -+STORE, 94819009097728, 94819009310719, -+STORE, 94819009310720, 94819011432447, -+ERASE, 94819009310720, 94819011432447, -+STORE, 94819011407872, 94819011420159, -+STORE, 94819011420160, 94819011432447, -+STORE, 139987985596416, 139987987849215, -+SNULL, 139987985739775, 139987987849215, -+STORE, 139987985596416, 139987985739775, -+STORE, 139987985739776, 139987987849215, -+ERASE, 139987985739776, 139987987849215, -+STORE, 139987987836928, 139987987845119, -+STORE, 139987987845120, 139987987849215, -+STORE, 140725763072000, 140725763076095, -+STORE, 140725763059712, 140725763071999, -+STORE, 139987987808256, 139987987836927, -+STORE, 139987987800064, 139987987808255, -+STORE, 139987983482880, 139987985596415, -+SNULL, 139987983482880, 139987983495167, -+STORE, 139987983495168, 139987985596415, -+STORE, 139987983482880, 139987983495167, -+SNULL, 139987985588223, 139987985596415, -+STORE, 139987983495168, 139987985588223, -+STORE, 139987985588224, 139987985596415, -+ERASE, 139987985588224, 139987985596415, -+STORE, 139987985588224, 139987985596415, -+STORE, 139987979685888, 139987983482879, -+SNULL, 139987979685888, 139987981344767, -+STORE, 139987981344768, 139987983482879, -+STORE, 139987979685888, 139987981344767, -+SNULL, 139987983441919, 139987983482879, -+STORE, 139987981344768, 139987983441919, -+STORE, 139987983441920, 139987983482879, -+SNULL, 139987983441920, 139987983466495, -+STORE, 139987983466496, 139987983482879, -+STORE, 139987983441920, 139987983466495, -+ERASE, 139987983441920, 139987983466495, -+STORE, 139987983441920, 139987983466495, -+ERASE, 139987983466496, 139987983482879, -+STORE, 139987983466496, 139987983482879, -+STORE, 139987987791872, 139987987808255, -+SNULL, 139987983458303, 139987983466495, -+STORE, 139987983441920, 139987983458303, -+STORE, 139987983458304, 139987983466495, -+SNULL, 139987985592319, 139987985596415, -+STORE, 139987985588224, 139987985592319, -+STORE, 139987985592320, 139987985596415, -+SNULL, 94819011411967, 94819011420159, -+STORE, 94819011407872, 94819011411967, -+STORE, 94819011411968, 94819011420159, -+SNULL, 139987987841023, 139987987845119, -+STORE, 139987987836928, 139987987841023, -+STORE, 139987987841024, 139987987845119, -+ERASE, 139987987808256, 139987987836927, -+STORE, 94819028176896, 94819028312063, -+STORE, 139987986108416, 139987987791871, -+STORE, 94819028176896, 94819028447231, -+STORE, 94819028176896, 94819028582399, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722475413504, 140737488351231, -+SNULL, 140722475421695, 140737488351231, -+STORE, 140722475413504, 140722475421695, -+STORE, 140722475282432, 140722475421695, -+STORE, 94620599119872, 94620601343999, -+SNULL, 94620599230463, 94620601343999, -+STORE, 94620599119872, 94620599230463, -+STORE, 94620599230464, 94620601343999, -+ERASE, 94620599230464, 94620601343999, -+STORE, 94620601323520, 94620601335807, -+STORE, 94620601335808, 94620601343999, -+STORE, 139891763060736, 139891765313535, -+SNULL, 139891763204095, 139891765313535, -+STORE, 139891763060736, 139891763204095, -+STORE, 139891763204096, 139891765313535, -+ERASE, 139891763204096, 139891765313535, -+STORE, 139891765301248, 139891765309439, -+STORE, 139891765309440, 139891765313535, -+STORE, 140722475700224, 140722475704319, -+STORE, 140722475687936, 140722475700223, -+STORE, 139891765272576, 139891765301247, -+STORE, 139891765264384, 139891765272575, -+STORE, 139891759263744, 139891763060735, -+SNULL, 139891759263744, 139891760922623, -+STORE, 139891760922624, 139891763060735, -+STORE, 139891759263744, 139891760922623, -+SNULL, 139891763019775, 139891763060735, -+STORE, 139891760922624, 139891763019775, -+STORE, 139891763019776, 139891763060735, -+SNULL, 139891763019776, 139891763044351, -+STORE, 139891763044352, 139891763060735, -+STORE, 139891763019776, 139891763044351, -+ERASE, 139891763019776, 139891763044351, -+STORE, 139891763019776, 139891763044351, -+ERASE, 139891763044352, 139891763060735, -+STORE, 139891763044352, 139891763060735, -+SNULL, 139891763036159, 139891763044351, -+STORE, 139891763019776, 139891763036159, -+STORE, 139891763036160, 139891763044351, -+SNULL, 94620601331711, 94620601335807, -+STORE, 94620601323520, 94620601331711, -+STORE, 94620601331712, 94620601335807, -+SNULL, 139891765305343, 139891765309439, -+STORE, 139891765301248, 139891765305343, -+STORE, 139891765305344, 139891765309439, -+ERASE, 139891765272576, 139891765301247, -+STORE, 94620610027520, 94620610162687, -+STORE, 94031976210432, 94031976423423, -+STORE, 94031978520576, 94031978524671, -+STORE, 94031978524672, 94031978532863, -+STORE, 94031978532864, 94031978545151, -+STORE, 94031990398976, 94031992565759, -+STORE, 140336240640000, 140336242298879, -+STORE, 140336242298880, 140336244396031, -+STORE, 140336244396032, 140336244412415, -+STORE, 140336244412416, 140336244420607, -+STORE, 140336244420608, 140336244436991, -+STORE, 140336244436992, 140336244449279, -+STORE, 140336244449280, 140336246542335, -+STORE, 140336246542336, 140336246546431, -+STORE, 140336246546432, 140336246550527, -+STORE, 140336246550528, 140336246693887, -+STORE, 140336247062528, 140336248745983, -+STORE, 140336248745984, 140336248762367, -+STORE, 140336248791040, 140336248795135, -+STORE, 140336248795136, 140336248799231, -+STORE, 140336248799232, 140336248803327, -+STORE, 140728500064256, 140728500203519, -+STORE, 140728501501952, 140728501514239, -+STORE, 140728501514240, 140728501518335, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140730503987200, 140737488351231, -+SNULL, 140730503995391, 140737488351231, -+STORE, 140730503987200, 140730503995391, -+STORE, 140730503856128, 140730503995391, -+STORE, 93866544205824, 93866546429951, -+SNULL, 93866544316415, 93866546429951, -+STORE, 93866544205824, 93866544316415, -+STORE, 93866544316416, 93866546429951, -+ERASE, 93866544316416, 93866546429951, -+STORE, 93866546409472, 93866546421759, -+STORE, 93866546421760, 93866546429951, -+STORE, 140216311959552, 140216314212351, -+SNULL, 140216312102911, 140216314212351, -+STORE, 140216311959552, 140216312102911, -+STORE, 140216312102912, 140216314212351, -+ERASE, 140216312102912, 140216314212351, -+STORE, 140216314200064, 140216314208255, -+STORE, 140216314208256, 140216314212351, -+STORE, 140730504626176, 140730504630271, -+STORE, 140730504613888, 140730504626175, -+STORE, 140216314171392, 140216314200063, -+STORE, 140216314163200, 140216314171391, -+STORE, 140216308162560, 140216311959551, -+SNULL, 140216308162560, 140216309821439, -+STORE, 140216309821440, 140216311959551, -+STORE, 140216308162560, 140216309821439, -+SNULL, 140216311918591, 140216311959551, -+STORE, 140216309821440, 140216311918591, -+STORE, 140216311918592, 140216311959551, -+SNULL, 140216311918592, 140216311943167, -+STORE, 140216311943168, 140216311959551, -+STORE, 140216311918592, 140216311943167, -+ERASE, 140216311918592, 140216311943167, -+STORE, 140216311918592, 140216311943167, -+ERASE, 140216311943168, 140216311959551, -+STORE, 140216311943168, 140216311959551, -+SNULL, 140216311934975, 140216311943167, -+STORE, 140216311918592, 140216311934975, -+STORE, 140216311934976, 140216311943167, -+SNULL, 93866546417663, 93866546421759, -+STORE, 93866546409472, 93866546417663, -+STORE, 93866546417664, 93866546421759, -+SNULL, 140216314204159, 140216314208255, -+STORE, 140216314200064, 140216314204159, -+STORE, 140216314204160, 140216314208255, -+ERASE, 140216314171392, 140216314200063, -+STORE, 93866550386688, 93866550521855, -+STORE, 94074292674560, 94074292887551, -+STORE, 94074294984704, 94074294988799, -+STORE, 94074294988800, 94074294996991, -+STORE, 94074294996992, 94074295009279, -+STORE, 94074300219392, 94074301378559, -+STORE, 139781563256832, 139781564915711, -+STORE, 139781564915712, 139781567012863, -+STORE, 139781567012864, 139781567029247, -+STORE, 139781567029248, 139781567037439, -+STORE, 139781567037440, 139781567053823, -+STORE, 139781567053824, 139781567066111, -+STORE, 139781567066112, 139781569159167, -+STORE, 139781569159168, 139781569163263, -+STORE, 139781569163264, 139781569167359, -+STORE, 139781569167360, 139781569310719, -+STORE, 139781569679360, 139781571362815, -+STORE, 139781571362816, 139781571379199, -+STORE, 139781571407872, 139781571411967, -+STORE, 139781571411968, 139781571416063, -+STORE, 139781571416064, 139781571420159, -+STORE, 140723688488960, 140723688628223, -+STORE, 140723689005056, 140723689017343, -+STORE, 140723689017344, 140723689021439, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735189745664, 140737488351231, -+SNULL, 140735189753855, 140737488351231, -+STORE, 140735189745664, 140735189753855, -+STORE, 140735189614592, 140735189753855, -+STORE, 94172072177664, 94172074512383, -+SNULL, 94172072390655, 94172074512383, -+STORE, 94172072177664, 94172072390655, -+STORE, 94172072390656, 94172074512383, -+ERASE, 94172072390656, 94172074512383, -+STORE, 94172074487808, 94172074500095, -+STORE, 94172074500096, 94172074512383, -+STORE, 140687827263488, 140687829516287, -+SNULL, 140687827406847, 140687829516287, -+STORE, 140687827263488, 140687827406847, -+STORE, 140687827406848, 140687829516287, -+ERASE, 140687827406848, 140687829516287, -+STORE, 140687829504000, 140687829512191, -+STORE, 140687829512192, 140687829516287, -+STORE, 140735189766144, 140735189770239, -+STORE, 140735189753856, 140735189766143, -+STORE, 140687829475328, 140687829503999, -+STORE, 140687829467136, 140687829475327, -+STORE, 140687825149952, 140687827263487, -+SNULL, 140687825149952, 140687825162239, -+STORE, 140687825162240, 140687827263487, -+STORE, 140687825149952, 140687825162239, -+SNULL, 140687827255295, 140687827263487, -+STORE, 140687825162240, 140687827255295, -+STORE, 140687827255296, 140687827263487, -+ERASE, 140687827255296, 140687827263487, -+STORE, 140687827255296, 140687827263487, -+STORE, 140687821352960, 140687825149951, -+SNULL, 140687821352960, 140687823011839, -+STORE, 140687823011840, 140687825149951, -+STORE, 140687821352960, 140687823011839, -+SNULL, 140687825108991, 140687825149951, -+STORE, 140687823011840, 140687825108991, -+STORE, 140687825108992, 140687825149951, -+SNULL, 140687825108992, 140687825133567, -+STORE, 140687825133568, 140687825149951, -+STORE, 140687825108992, 140687825133567, -+ERASE, 140687825108992, 140687825133567, -+STORE, 140687825108992, 140687825133567, -+ERASE, 140687825133568, 140687825149951, -+STORE, 140687825133568, 140687825149951, -+STORE, 140687829458944, 140687829475327, -+SNULL, 140687825125375, 140687825133567, -+STORE, 140687825108992, 140687825125375, -+STORE, 140687825125376, 140687825133567, -+SNULL, 140687827259391, 140687827263487, -+STORE, 140687827255296, 140687827259391, -+STORE, 140687827259392, 140687827263487, -+SNULL, 94172074491903, 94172074500095, -+STORE, 94172074487808, 94172074491903, -+STORE, 94172074491904, 94172074500095, -+SNULL, 140687829508095, 140687829512191, -+STORE, 140687829504000, 140687829508095, -+STORE, 140687829508096, 140687829512191, -+ERASE, 140687829475328, 140687829503999, -+STORE, 94172092432384, 94172092567551, -+STORE, 140687827775488, 140687829458943, -+STORE, 94172092432384, 94172092702719, -+STORE, 94172092432384, 94172092837887, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140737229504512, 140737488351231, -+SNULL, 140737229512703, 140737488351231, -+STORE, 140737229504512, 140737229512703, -+STORE, 140737229373440, 140737229512703, -+STORE, 94155246866432, 94155249090559, -+SNULL, 94155246977023, 94155249090559, -+STORE, 94155246866432, 94155246977023, -+STORE, 94155246977024, 94155249090559, -+ERASE, 94155246977024, 94155249090559, -+STORE, 94155249070080, 94155249082367, -+STORE, 94155249082368, 94155249090559, -+STORE, 140640993693696, 140640995946495, -+SNULL, 140640993837055, 140640995946495, -+STORE, 140640993693696, 140640993837055, -+STORE, 140640993837056, 140640995946495, -+ERASE, 140640993837056, 140640995946495, -+STORE, 140640995934208, 140640995942399, -+STORE, 140640995942400, 140640995946495, -+STORE, 140737230004224, 140737230008319, -+STORE, 140737229991936, 140737230004223, -+STORE, 140640995905536, 140640995934207, -+STORE, 140640995897344, 140640995905535, -+STORE, 140640989896704, 140640993693695, -+SNULL, 140640989896704, 140640991555583, -+STORE, 140640991555584, 140640993693695, -+STORE, 140640989896704, 140640991555583, -+SNULL, 140640993652735, 140640993693695, -+STORE, 140640991555584, 140640993652735, -+STORE, 140640993652736, 140640993693695, -+SNULL, 140640993652736, 140640993677311, -+STORE, 140640993677312, 140640993693695, -+STORE, 140640993652736, 140640993677311, -+ERASE, 140640993652736, 140640993677311, -+STORE, 140640993652736, 140640993677311, -+ERASE, 140640993677312, 140640993693695, -+STORE, 140640993677312, 140640993693695, -+SNULL, 140640993669119, 140640993677311, -+STORE, 140640993652736, 140640993669119, -+STORE, 140640993669120, 140640993677311, -+SNULL, 94155249078271, 94155249082367, -+STORE, 94155249070080, 94155249078271, -+STORE, 94155249078272, 94155249082367, -+SNULL, 140640995938303, 140640995942399, -+STORE, 140640995934208, 140640995938303, -+STORE, 140640995938304, 140640995942399, -+ERASE, 140640995905536, 140640995934207, -+STORE, 94155281035264, 94155281170431, -+STORE, 94088066453504, 94088066564095, -+STORE, 94088068657152, 94088068665343, -+STORE, 94088068665344, 94088068669439, -+STORE, 94088068669440, 94088068677631, -+STORE, 94088090214400, 94088090349567, -+STORE, 140503024627712, 140503026286591, -+STORE, 140503026286592, 140503028383743, -+STORE, 140503028383744, 140503028400127, -+STORE, 140503028400128, 140503028408319, -+STORE, 140503028408320, 140503028424703, -+STORE, 140503028424704, 140503028568063, -+STORE, 140503030628352, 140503030636543, -+STORE, 140503030665216, 140503030669311, -+STORE, 140503030669312, 140503030673407, -+STORE, 140503030673408, 140503030677503, -+STORE, 140730894725120, 140730894864383, -+STORE, 140730894880768, 140730894893055, -+STORE, 140730894893056, 140730894897151, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140730434342912, 140737488351231, -+SNULL, 140730434351103, 140737488351231, -+STORE, 140730434342912, 140730434351103, -+STORE, 140730434211840, 140730434351103, -+STORE, 4194304, 5128191, -+STORE, 7221248, 7241727, -+STORE, 7241728, 7249919, -+STORE, 140109041938432, 140109044191231, -+SNULL, 140109042081791, 140109044191231, -+STORE, 140109041938432, 140109042081791, -+STORE, 140109042081792, 140109044191231, -+ERASE, 140109042081792, 140109044191231, -+STORE, 140109044178944, 140109044187135, -+STORE, 140109044187136, 140109044191231, -+STORE, 140730434850816, 140730434854911, -+STORE, 140730434838528, 140730434850815, -+STORE, 140109044150272, 140109044178943, -+STORE, 140109044142080, 140109044150271, -+STORE, 140109038776320, 140109041938431, -+SNULL, 140109038776320, 140109039837183, -+STORE, 140109039837184, 140109041938431, -+STORE, 140109038776320, 140109039837183, -+SNULL, 140109041930239, 140109041938431, -+STORE, 140109039837184, 140109041930239, -+STORE, 140109041930240, 140109041938431, -+ERASE, 140109041930240, 140109041938431, -+STORE, 140109041930240, 140109041938431, -+STORE, 140109034979328, 140109038776319, -+SNULL, 140109034979328, 140109036638207, -+STORE, 140109036638208, 140109038776319, -+STORE, 140109034979328, 140109036638207, -+SNULL, 140109038735359, 140109038776319, -+STORE, 140109036638208, 140109038735359, -+STORE, 140109038735360, 140109038776319, -+SNULL, 140109038735360, 140109038759935, -+STORE, 140109038759936, 140109038776319, -+STORE, 140109038735360, 140109038759935, -+ERASE, 140109038735360, 140109038759935, -+STORE, 140109038735360, 140109038759935, -+ERASE, 140109038759936, 140109038776319, -+STORE, 140109038759936, 140109038776319, -+STORE, 140109044129792, 140109044150271, -+SNULL, 140109038751743, 140109038759935, -+STORE, 140109038735360, 140109038751743, -+STORE, 140109038751744, 140109038759935, -+SNULL, 140109041934335, 140109041938431, -+STORE, 140109041930240, 140109041934335, -+STORE, 140109041934336, 140109041938431, -+SNULL, 7233535, 7241727, -+STORE, 7221248, 7233535, -+STORE, 7233536, 7241727, -+SNULL, 140109044183039, 140109044187135, -+STORE, 140109044178944, 140109044183039, -+STORE, 140109044183040, 140109044187135, -+ERASE, 140109044150272, 140109044178943, -+STORE, 20000768, 20135935, -+STORE, 20000768, 20283391, -+STORE, 140109042446336, 140109044129791, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140730853408768, 140737488351231, -+SNULL, 140730853416959, 140737488351231, -+STORE, 140730853408768, 140730853416959, -+STORE, 140730853277696, 140730853416959, -+STORE, 94865902977024, 94865905311743, -+SNULL, 94865903190015, 94865905311743, -+STORE, 94865902977024, 94865903190015, -+STORE, 94865903190016, 94865905311743, -+ERASE, 94865903190016, 94865905311743, -+STORE, 94865905287168, 94865905299455, -+STORE, 94865905299456, 94865905311743, -+STORE, 139768865738752, 139768867991551, -+SNULL, 139768865882111, 139768867991551, -+STORE, 139768865738752, 139768865882111, -+STORE, 139768865882112, 139768867991551, -+ERASE, 139768865882112, 139768867991551, -+STORE, 139768867979264, 139768867987455, -+STORE, 139768867987456, 139768867991551, -+STORE, 140730853957632, 140730853961727, -+STORE, 140730853945344, 140730853957631, -+STORE, 139768867950592, 139768867979263, -+STORE, 139768867942400, 139768867950591, -+STORE, 139768863625216, 139768865738751, -+SNULL, 139768863625216, 139768863637503, -+STORE, 139768863637504, 139768865738751, -+STORE, 139768863625216, 139768863637503, -+SNULL, 139768865730559, 139768865738751, -+STORE, 139768863637504, 139768865730559, -+STORE, 139768865730560, 139768865738751, -+ERASE, 139768865730560, 139768865738751, -+STORE, 139768865730560, 139768865738751, -+STORE, 139768859828224, 139768863625215, -+SNULL, 139768859828224, 139768861487103, -+STORE, 139768861487104, 139768863625215, -+STORE, 139768859828224, 139768861487103, -+SNULL, 139768863584255, 139768863625215, -+STORE, 139768861487104, 139768863584255, -+STORE, 139768863584256, 139768863625215, -+SNULL, 139768863584256, 139768863608831, -+STORE, 139768863608832, 139768863625215, -+STORE, 139768863584256, 139768863608831, -+ERASE, 139768863584256, 139768863608831, -+STORE, 139768863584256, 139768863608831, -+ERASE, 139768863608832, 139768863625215, -+STORE, 139768863608832, 139768863625215, -+STORE, 139768867934208, 139768867950591, -+SNULL, 139768863600639, 139768863608831, -+STORE, 139768863584256, 139768863600639, -+STORE, 139768863600640, 139768863608831, -+SNULL, 139768865734655, 139768865738751, -+STORE, 139768865730560, 139768865734655, -+STORE, 139768865734656, 139768865738751, -+SNULL, 94865905291263, 94865905299455, -+STORE, 94865905287168, 94865905291263, -+STORE, 94865905291264, 94865905299455, -+SNULL, 139768867983359, 139768867987455, -+STORE, 139768867979264, 139768867983359, -+STORE, 139768867983360, 139768867987455, -+ERASE, 139768867950592, 139768867979263, -+STORE, 94865923670016, 94865923805183, -+STORE, 139768866250752, 139768867934207, -+STORE, 94865923670016, 94865923940351, -+STORE, 94865923670016, 94865924075519, -+STORE, 94865923670016, 94865924222975, -+SNULL, 94865924210687, 94865924222975, -+STORE, 94865923670016, 94865924210687, -+STORE, 94865924210688, 94865924222975, -+ERASE, 94865924210688, 94865924222975, -+STORE, 94865923670016, 94865924349951, -+STORE, 94865923670016, 94865924493311, -+STORE, 94865923670016, 94865924640767, -+SNULL, 94865924603903, 94865924640767, -+STORE, 94865923670016, 94865924603903, -+STORE, 94865924603904, 94865924640767, -+ERASE, 94865924603904, 94865924640767, -+STORE, 94865923670016, 94865924747263, -+STORE, 94865923670016, 94865924898815, -+SNULL, 94865924874239, 94865924898815, -+STORE, 94865923670016, 94865924874239, -+STORE, 94865924874240, 94865924898815, -+ERASE, 94865924874240, 94865924898815, -+STORE, 94865923670016, 94865925025791, -+SNULL, 94865925013503, 94865925025791, -+STORE, 94865923670016, 94865925013503, -+STORE, 94865925013504, 94865925025791, -+ERASE, 94865925013504, 94865925025791, -+SNULL, 94865924988927, 94865925013503, -+STORE, 94865923670016, 94865924988927, -+STORE, 94865924988928, 94865925013503, -+ERASE, 94865924988928, 94865925013503, -+STORE, 94865923670016, 94865925152767, -+SNULL, 94865925136383, 94865925152767, -+STORE, 94865923670016, 94865925136383, -+STORE, 94865925136384, 94865925152767, -+ERASE, 94865925136384, 94865925152767, -+STORE, 94865923670016, 94865925292031, -+SNULL, 94865925279743, 94865925292031, -+STORE, 94865923670016, 94865925279743, -+STORE, 94865925279744, 94865925292031, -+ERASE, 94865925279744, 94865925292031, -+SNULL, 94865925255167, 94865925279743, -+STORE, 94865923670016, 94865925255167, -+STORE, 94865925255168, 94865925279743, -+ERASE, 94865925255168, 94865925279743, -+STORE, 94865923670016, 94865925406719, -+SNULL, 94865925394431, 94865925406719, -+STORE, 94865923670016, 94865925394431, -+STORE, 94865925394432, 94865925406719, -+ERASE, 94865925394432, 94865925406719, -+STORE, 94865923670016, 94865925545983, -+SNULL, 94865925533695, 94865925545983, -+STORE, 94865923670016, 94865925533695, -+STORE, 94865925533696, 94865925545983, -+ERASE, 94865925533696, 94865925545983, -+SNULL, 94865925492735, 94865925533695, -+STORE, 94865923670016, 94865925492735, -+STORE, 94865925492736, 94865925533695, -+ERASE, 94865925492736, 94865925533695, -+STORE, 94865923670016, 94865925627903, -+SNULL, 94865925599231, 94865925627903, -+STORE, 94865923670016, 94865925599231, -+STORE, 94865925599232, 94865925627903, -+ERASE, 94865925599232, 94865925627903, -+STORE, 94865923670016, 94865925738495, -+SNULL, 94865925726207, 94865925738495, -+STORE, 94865923670016, 94865925726207, -+STORE, 94865925726208, 94865925738495, -+ERASE, 94865925726208, 94865925738495, -+STORE, 94865923670016, 94865925877759, -+SNULL, 94865925865471, 94865925877759, -+STORE, 94865923670016, 94865925865471, -+STORE, 94865925865472, 94865925877759, -+ERASE, 94865925865472, 94865925877759, -+STORE, 94865923670016, 94865926021119, -+SNULL, 94865926008831, 94865926021119, -+STORE, 94865923670016, 94865926008831, -+STORE, 94865926008832, 94865926021119, -+ERASE, 94865926008832, 94865926021119, -+SNULL, 94865925971967, 94865926008831, -+STORE, 94865923670016, 94865925971967, -+STORE, 94865925971968, 94865926008831, -+ERASE, 94865925971968, 94865926008831, -+STORE, 94865923670016, 94865926115327, -+STORE, 94865923670016, 94865926254591, -+SNULL, 94865926246399, 94865926254591, -+STORE, 94865923670016, 94865926246399, -+STORE, 94865926246400, 94865926254591, -+ERASE, 94865926246400, 94865926254591, -+STORE, 94865923670016, 94865926385663, -+STORE, 94865923670016, 94865926537215, -+STORE, 94865923670016, 94865926672383, -+STORE, 94865923670016, 94865926815743, -+STORE, 94865923670016, 94865926955007, -+STORE, 94865923670016, 94865927094271, -+STORE, 94865923670016, 94865927233535, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140731148435456, 140737488351231, -+SNULL, 140731148443647, 140737488351231, -+STORE, 140731148435456, 140731148443647, -+STORE, 140731148304384, 140731148443647, -+STORE, 94090775400448, 94090777735167, -+SNULL, 94090775613439, 94090777735167, -+STORE, 94090775400448, 94090775613439, -+STORE, 94090775613440, 94090777735167, -+ERASE, 94090775613440, 94090777735167, -+STORE, 94090777710592, 94090777722879, -+STORE, 94090777722880, 94090777735167, -+STORE, 140301090283520, 140301092536319, -+SNULL, 140301090426879, 140301092536319, -+STORE, 140301090283520, 140301090426879, -+STORE, 140301090426880, 140301092536319, -+ERASE, 140301090426880, 140301092536319, -+STORE, 140301092524032, 140301092532223, -+STORE, 140301092532224, 140301092536319, -+STORE, 140731148570624, 140731148574719, -+STORE, 140731148558336, 140731148570623, -+STORE, 140301092495360, 140301092524031, -+STORE, 140301092487168, 140301092495359, -+STORE, 140301088169984, 140301090283519, -+SNULL, 140301088169984, 140301088182271, -+STORE, 140301088182272, 140301090283519, -+STORE, 140301088169984, 140301088182271, -+SNULL, 140301090275327, 140301090283519, -+STORE, 140301088182272, 140301090275327, -+STORE, 140301090275328, 140301090283519, -+ERASE, 140301090275328, 140301090283519, -+STORE, 140301090275328, 140301090283519, -+STORE, 140301084372992, 140301088169983, -+SNULL, 140301084372992, 140301086031871, -+STORE, 140301086031872, 140301088169983, -+STORE, 140301084372992, 140301086031871, -+SNULL, 140301088129023, 140301088169983, -+STORE, 140301086031872, 140301088129023, -+STORE, 140301088129024, 140301088169983, -+SNULL, 140301088129024, 140301088153599, -+STORE, 140301088153600, 140301088169983, -+STORE, 140301088129024, 140301088153599, -+ERASE, 140301088129024, 140301088153599, -+STORE, 140301088129024, 140301088153599, -+ERASE, 140301088153600, 140301088169983, -+STORE, 140301088153600, 140301088169983, -+STORE, 140301092478976, 140301092495359, -+SNULL, 140301088145407, 140301088153599, -+STORE, 140301088129024, 140301088145407, -+STORE, 140301088145408, 140301088153599, -+SNULL, 140301090279423, 140301090283519, -+STORE, 140301090275328, 140301090279423, -+STORE, 140301090279424, 140301090283519, -+SNULL, 94090777714687, 94090777722879, -+STORE, 94090777710592, 94090777714687, -+STORE, 94090777714688, 94090777722879, -+SNULL, 140301092528127, 140301092532223, -+STORE, 140301092524032, 140301092528127, -+STORE, 140301092528128, 140301092532223, -+ERASE, 140301092495360, 140301092524031, -+STORE, 94090794590208, 94090794725375, -+STORE, 140301090795520, 140301092478975, -+STORE, 94090794590208, 94090794860543, -+STORE, 94090794590208, 94090794995711, -+STORE, 94090794590208, 94090795163647, -+SNULL, 94090795139071, 94090795163647, -+STORE, 94090794590208, 94090795139071, -+STORE, 94090795139072, 94090795163647, -+ERASE, 94090795139072, 94090795163647, -+STORE, 94090794590208, 94090795278335, -+STORE, 94090794590208, 94090795425791, -+SNULL, 94090795388927, 94090795425791, -+STORE, 94090794590208, 94090795388927, -+STORE, 94090795388928, 94090795425791, -+ERASE, 94090795388928, 94090795425791, -+STORE, 94090794590208, 94090795528191, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733084430336, 140737488351231, -+SNULL, 140733084438527, 140737488351231, -+STORE, 140733084430336, 140733084438527, -+STORE, 140733084299264, 140733084438527, -+STORE, 94116169183232, 94116171517951, -+SNULL, 94116169396223, 94116171517951, -+STORE, 94116169183232, 94116169396223, -+STORE, 94116169396224, 94116171517951, -+ERASE, 94116169396224, 94116171517951, -+STORE, 94116171493376, 94116171505663, -+STORE, 94116171505664, 94116171517951, -+STORE, 139772214128640, 139772216381439, -+SNULL, 139772214271999, 139772216381439, -+STORE, 139772214128640, 139772214271999, -+STORE, 139772214272000, 139772216381439, -+ERASE, 139772214272000, 139772216381439, -+STORE, 139772216369152, 139772216377343, -+STORE, 139772216377344, 139772216381439, -+STORE, 140733085270016, 140733085274111, -+STORE, 140733085257728, 140733085270015, -+STORE, 139772216340480, 139772216369151, -+STORE, 139772216332288, 139772216340479, -+STORE, 139772212015104, 139772214128639, -+SNULL, 139772212015104, 139772212027391, -+STORE, 139772212027392, 139772214128639, -+STORE, 139772212015104, 139772212027391, -+SNULL, 139772214120447, 139772214128639, -+STORE, 139772212027392, 139772214120447, -+STORE, 139772214120448, 139772214128639, -+ERASE, 139772214120448, 139772214128639, -+STORE, 139772214120448, 139772214128639, -+STORE, 139772208218112, 139772212015103, -+SNULL, 139772208218112, 139772209876991, -+STORE, 139772209876992, 139772212015103, -+STORE, 139772208218112, 139772209876991, -+SNULL, 139772211974143, 139772212015103, -+STORE, 139772209876992, 139772211974143, -+STORE, 139772211974144, 139772212015103, -+SNULL, 139772211974144, 139772211998719, -+STORE, 139772211998720, 139772212015103, -+STORE, 139772211974144, 139772211998719, -+ERASE, 139772211974144, 139772211998719, -+STORE, 139772211974144, 139772211998719, -+ERASE, 139772211998720, 139772212015103, -+STORE, 139772211998720, 139772212015103, -+STORE, 139772216324096, 139772216340479, -+SNULL, 139772211990527, 139772211998719, -+STORE, 139772211974144, 139772211990527, -+STORE, 139772211990528, 139772211998719, -+SNULL, 139772214124543, 139772214128639, -+STORE, 139772214120448, 139772214124543, -+STORE, 139772214124544, 139772214128639, -+SNULL, 94116171497471, 94116171505663, -+STORE, 94116171493376, 94116171497471, -+STORE, 94116171497472, 94116171505663, -+SNULL, 139772216373247, 139772216377343, -+STORE, 139772216369152, 139772216373247, -+STORE, 139772216373248, 139772216377343, -+ERASE, 139772216340480, 139772216369151, -+STORE, 94116199383040, 94116199518207, -+STORE, 139772214640640, 139772216324095, -+STORE, 94116199383040, 94116199653375, -+STORE, 94116199383040, 94116199788543, -+STORE, 140737488347136, 140737488351231, -+STORE, 140726067826688, 140737488351231, -+SNULL, 140726067830783, 140737488351231, -+STORE, 140726067826688, 140726067830783, -+STORE, 140726067695616, 140726067830783, -+STORE, 94535150673920, 94535152898047, -+SNULL, 94535150784511, 94535152898047, -+STORE, 94535150673920, 94535150784511, -+STORE, 94535150784512, 94535152898047, -+ERASE, 94535150784512, 94535152898047, -+STORE, 94535152877568, 94535152889855, -+STORE, 94535152889856, 94535152898047, -+STORE, 140381257314304, 140381259567103, -+SNULL, 140381257457663, 140381259567103, -+STORE, 140381257314304, 140381257457663, -+STORE, 140381257457664, 140381259567103, -+ERASE, 140381257457664, 140381259567103, -+STORE, 140381259554816, 140381259563007, -+STORE, 140381259563008, 140381259567103, -+STORE, 140726068060160, 140726068064255, -+STORE, 140726068047872, 140726068060159, -+STORE, 140381259526144, 140381259554815, -+STORE, 140381259517952, 140381259526143, -+STORE, 140381253517312, 140381257314303, -+SNULL, 140381253517312, 140381255176191, -+STORE, 140381255176192, 140381257314303, -+STORE, 140381253517312, 140381255176191, -+SNULL, 140381257273343, 140381257314303, -+STORE, 140381255176192, 140381257273343, -+STORE, 140381257273344, 140381257314303, -+SNULL, 140381257273344, 140381257297919, -+STORE, 140381257297920, 140381257314303, -+STORE, 140381257273344, 140381257297919, -+ERASE, 140381257273344, 140381257297919, -+STORE, 140381257273344, 140381257297919, -+ERASE, 140381257297920, 140381257314303, -+STORE, 140381257297920, 140381257314303, -+SNULL, 140381257289727, 140381257297919, -+STORE, 140381257273344, 140381257289727, -+STORE, 140381257289728, 140381257297919, -+SNULL, 94535152885759, 94535152889855, -+STORE, 94535152877568, 94535152885759, -+STORE, 94535152885760, 94535152889855, -+SNULL, 140381259558911, 140381259563007, -+STORE, 140381259554816, 140381259558911, -+STORE, 140381259558912, 140381259563007, -+ERASE, 140381259526144, 140381259554815, -+STORE, 94535186296832, 94535186431999, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140729189425152, 140737488351231, -+SNULL, 140729189433343, 140737488351231, -+STORE, 140729189425152, 140729189433343, -+STORE, 140729189294080, 140729189433343, -+STORE, 94428200128512, 94428202352639, -+SNULL, 94428200239103, 94428202352639, -+STORE, 94428200128512, 94428200239103, -+STORE, 94428200239104, 94428202352639, -+ERASE, 94428200239104, 94428202352639, -+STORE, 94428202332160, 94428202344447, -+STORE, 94428202344448, 94428202352639, -+STORE, 139707216986112, 139707219238911, -+SNULL, 139707217129471, 139707219238911, -+STORE, 139707216986112, 139707217129471, -+STORE, 139707217129472, 139707219238911, -+ERASE, 139707217129472, 139707219238911, -+STORE, 139707219226624, 139707219234815, -+STORE, 139707219234816, 139707219238911, -+STORE, 140729189785600, 140729189789695, -+STORE, 140729189773312, 140729189785599, -+STORE, 139707219197952, 139707219226623, -+STORE, 139707219189760, 139707219197951, -+STORE, 139707213189120, 139707216986111, -+SNULL, 139707213189120, 139707214847999, -+STORE, 139707214848000, 139707216986111, -+STORE, 139707213189120, 139707214847999, -+SNULL, 139707216945151, 139707216986111, -+STORE, 139707214848000, 139707216945151, -+STORE, 139707216945152, 139707216986111, -+SNULL, 139707216945152, 139707216969727, -+STORE, 139707216969728, 139707216986111, -+STORE, 139707216945152, 139707216969727, -+ERASE, 139707216945152, 139707216969727, -+STORE, 139707216945152, 139707216969727, -+ERASE, 139707216969728, 139707216986111, -+STORE, 139707216969728, 139707216986111, -+SNULL, 139707216961535, 139707216969727, -+STORE, 139707216945152, 139707216961535, -+STORE, 139707216961536, 139707216969727, -+SNULL, 94428202340351, 94428202344447, -+STORE, 94428202332160, 94428202340351, -+STORE, 94428202340352, 94428202344447, -+SNULL, 139707219230719, 139707219234815, -+STORE, 139707219226624, 139707219230719, -+STORE, 139707219230720, 139707219234815, -+ERASE, 139707219197952, 139707219226623, -+STORE, 94428208599040, 94428208734207, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722000953344, 140737488351231, -+SNULL, 140722000961535, 140737488351231, -+STORE, 140722000953344, 140722000961535, -+STORE, 140722000822272, 140722000961535, -+STORE, 94636494757888, 94636496982015, -+SNULL, 94636494868479, 94636496982015, -+STORE, 94636494757888, 94636494868479, -+STORE, 94636494868480, 94636496982015, -+ERASE, 94636494868480, 94636496982015, -+STORE, 94636496961536, 94636496973823, -+STORE, 94636496973824, 94636496982015, -+STORE, 140142275100672, 140142277353471, -+SNULL, 140142275244031, 140142277353471, -+STORE, 140142275100672, 140142275244031, -+STORE, 140142275244032, 140142277353471, -+ERASE, 140142275244032, 140142277353471, -+STORE, 140142277341184, 140142277349375, -+STORE, 140142277349376, 140142277353471, -+STORE, 140722002747392, 140722002751487, -+STORE, 140722002735104, 140722002747391, -+STORE, 140142277312512, 140142277341183, -+STORE, 140142277304320, 140142277312511, -+STORE, 140142271303680, 140142275100671, -+SNULL, 140142271303680, 140142272962559, -+STORE, 140142272962560, 140142275100671, -+STORE, 140142271303680, 140142272962559, -+SNULL, 140142275059711, 140142275100671, -+STORE, 140142272962560, 140142275059711, -+STORE, 140142275059712, 140142275100671, -+SNULL, 140142275059712, 140142275084287, -+STORE, 140142275084288, 140142275100671, -+STORE, 140142275059712, 140142275084287, -+ERASE, 140142275059712, 140142275084287, -+STORE, 140142275059712, 140142275084287, -+ERASE, 140142275084288, 140142275100671, -+STORE, 140142275084288, 140142275100671, -+SNULL, 140142275076095, 140142275084287, -+STORE, 140142275059712, 140142275076095, -+STORE, 140142275076096, 140142275084287, -+SNULL, 94636496969727, 94636496973823, -+STORE, 94636496961536, 94636496969727, -+STORE, 94636496969728, 94636496973823, -+SNULL, 140142277345279, 140142277349375, -+STORE, 140142277341184, 140142277345279, -+STORE, 140142277345280, 140142277349375, -+ERASE, 140142277312512, 140142277341183, -+STORE, 94636516286464, 94636516421631, -+STORE, 94071103692800, 94071103905791, -+STORE, 94071106002944, 94071106007039, -+STORE, 94071106007040, 94071106015231, -+STORE, 94071106015232, 94071106027519, -+STORE, 94071138521088, 94071140368383, -+STORE, 140145668190208, 140145669849087, -+STORE, 140145669849088, 140145671946239, -+STORE, 140145671946240, 140145671962623, -+STORE, 140145671962624, 140145671970815, -+STORE, 140145671970816, 140145671987199, -+STORE, 140145671987200, 140145671999487, -+STORE, 140145671999488, 140145674092543, -+STORE, 140145674092544, 140145674096639, -+STORE, 140145674096640, 140145674100735, -+STORE, 140145674100736, 140145674244095, -+STORE, 140145674612736, 140145676296191, -+STORE, 140145676296192, 140145676312575, -+STORE, 140145676341248, 140145676345343, -+STORE, 140145676345344, 140145676349439, -+STORE, 140145676349440, 140145676353535, -+STORE, 140734927740928, 140734927880191, -+STORE, 140734928842752, 140734928855039, -+STORE, 140734928855040, 140734928859135, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722342535168, 140737488351231, -+SNULL, 140722342543359, 140737488351231, -+STORE, 140722342535168, 140722342543359, -+STORE, 140722342404096, 140722342543359, -+STORE, 94399699714048, 94399702048767, -+SNULL, 94399699927039, 94399702048767, -+STORE, 94399699714048, 94399699927039, -+STORE, 94399699927040, 94399702048767, -+ERASE, 94399699927040, 94399702048767, -+STORE, 94399702024192, 94399702036479, -+STORE, 94399702036480, 94399702048767, -+STORE, 139811024748544, 139811027001343, -+SNULL, 139811024891903, 139811027001343, -+STORE, 139811024748544, 139811024891903, -+STORE, 139811024891904, 139811027001343, -+ERASE, 139811024891904, 139811027001343, -+STORE, 139811026989056, 139811026997247, -+STORE, 139811026997248, 139811027001343, -+STORE, 140722342707200, 140722342711295, -+STORE, 140722342694912, 140722342707199, -+STORE, 139811026960384, 139811026989055, -+STORE, 139811026952192, 139811026960383, -+STORE, 139811022635008, 139811024748543, -+SNULL, 139811022635008, 139811022647295, -+STORE, 139811022647296, 139811024748543, -+STORE, 139811022635008, 139811022647295, -+SNULL, 139811024740351, 139811024748543, -+STORE, 139811022647296, 139811024740351, -+STORE, 139811024740352, 139811024748543, -+ERASE, 139811024740352, 139811024748543, -+STORE, 139811024740352, 139811024748543, -+STORE, 139811018838016, 139811022635007, -+SNULL, 139811018838016, 139811020496895, -+STORE, 139811020496896, 139811022635007, -+STORE, 139811018838016, 139811020496895, -+SNULL, 139811022594047, 139811022635007, -+STORE, 139811020496896, 139811022594047, -+STORE, 139811022594048, 139811022635007, -+SNULL, 139811022594048, 139811022618623, -+STORE, 139811022618624, 139811022635007, -+STORE, 139811022594048, 139811022618623, -+ERASE, 139811022594048, 139811022618623, -+STORE, 139811022594048, 139811022618623, -+ERASE, 139811022618624, 139811022635007, -+STORE, 139811022618624, 139811022635007, -+STORE, 139811026944000, 139811026960383, -+SNULL, 139811022610431, 139811022618623, -+STORE, 139811022594048, 139811022610431, -+STORE, 139811022610432, 139811022618623, -+SNULL, 139811024744447, 139811024748543, -+STORE, 139811024740352, 139811024744447, -+STORE, 139811024744448, 139811024748543, -+SNULL, 94399702028287, 94399702036479, -+STORE, 94399702024192, 94399702028287, -+STORE, 94399702028288, 94399702036479, -+SNULL, 139811026993151, 139811026997247, -+STORE, 139811026989056, 139811026993151, -+STORE, 139811026993152, 139811026997247, -+ERASE, 139811026960384, 139811026989055, -+STORE, 94399723880448, 94399724015615, -+STORE, 139811025260544, 139811026943999, -+STORE, 94399723880448, 94399724150783, -+STORE, 94399723880448, 94399724285951, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140735364939776, 140737488351231, -+SNULL, 140735364947967, 140737488351231, -+STORE, 140735364939776, 140735364947967, -+STORE, 140735364808704, 140735364947967, -+STORE, 94421528674304, 94421531009023, -+SNULL, 94421528887295, 94421531009023, -+STORE, 94421528674304, 94421528887295, -+STORE, 94421528887296, 94421531009023, -+ERASE, 94421528887296, 94421531009023, -+STORE, 94421530984448, 94421530996735, -+STORE, 94421530996736, 94421531009023, -+STORE, 140162004742144, 140162006994943, -+SNULL, 140162004885503, 140162006994943, -+STORE, 140162004742144, 140162004885503, -+STORE, 140162004885504, 140162006994943, -+ERASE, 140162004885504, 140162006994943, -+STORE, 140162006982656, 140162006990847, -+STORE, 140162006990848, 140162006994943, -+STORE, 140735365402624, 140735365406719, -+STORE, 140735365390336, 140735365402623, -+STORE, 140162006953984, 140162006982655, -+STORE, 140162006945792, 140162006953983, -+STORE, 140162002628608, 140162004742143, -+SNULL, 140162002628608, 140162002640895, -+STORE, 140162002640896, 140162004742143, -+STORE, 140162002628608, 140162002640895, -+SNULL, 140162004733951, 140162004742143, -+STORE, 140162002640896, 140162004733951, -+STORE, 140162004733952, 140162004742143, -+ERASE, 140162004733952, 140162004742143, -+STORE, 140162004733952, 140162004742143, -+STORE, 140161998831616, 140162002628607, -+SNULL, 140161998831616, 140162000490495, -+STORE, 140162000490496, 140162002628607, -+STORE, 140161998831616, 140162000490495, -+SNULL, 140162002587647, 140162002628607, -+STORE, 140162000490496, 140162002587647, -+STORE, 140162002587648, 140162002628607, -+SNULL, 140162002587648, 140162002612223, -+STORE, 140162002612224, 140162002628607, -+STORE, 140162002587648, 140162002612223, -+ERASE, 140162002587648, 140162002612223, -+STORE, 140162002587648, 140162002612223, -+ERASE, 140162002612224, 140162002628607, -+STORE, 140162002612224, 140162002628607, -+STORE, 140162006937600, 140162006953983, -+SNULL, 140162002604031, 140162002612223, -+STORE, 140162002587648, 140162002604031, -+STORE, 140162002604032, 140162002612223, -+SNULL, 140162004738047, 140162004742143, -+STORE, 140162004733952, 140162004738047, -+STORE, 140162004738048, 140162004742143, -+SNULL, 94421530988543, 94421530996735, -+STORE, 94421530984448, 94421530988543, -+STORE, 94421530988544, 94421530996735, -+SNULL, 140162006986751, 140162006990847, -+STORE, 140162006982656, 140162006986751, -+STORE, 140162006986752, 140162006990847, -+ERASE, 140162006953984, 140162006982655, -+STORE, 94421551697920, 94421551833087, -+STORE, 140162005254144, 140162006937599, -+STORE, 94421551697920, 94421551968255, -+STORE, 94421551697920, 94421552103423, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140733498486784, 140737488351231, -+SNULL, 140733498494975, 140737488351231, -+STORE, 140733498486784, 140733498494975, -+STORE, 140733498355712, 140733498494975, -+STORE, 94567985836032, 94567988170751, -+SNULL, 94567986049023, 94567988170751, -+STORE, 94567985836032, 94567986049023, -+STORE, 94567986049024, 94567988170751, -+ERASE, 94567986049024, 94567988170751, -+STORE, 94567988146176, 94567988158463, -+STORE, 94567988158464, 94567988170751, -+STORE, 139634278572032, 139634280824831, -+SNULL, 139634278715391, 139634280824831, -+STORE, 139634278572032, 139634278715391, -+STORE, 139634278715392, 139634280824831, -+ERASE, 139634278715392, 139634280824831, -+STORE, 139634280812544, 139634280820735, -+STORE, 139634280820736, 139634280824831, -+STORE, 140733498544128, 140733498548223, -+STORE, 140733498531840, 140733498544127, -+STORE, 139634280783872, 139634280812543, -+STORE, 139634280775680, 139634280783871, -+STORE, 139634276458496, 139634278572031, -+SNULL, 139634276458496, 139634276470783, -+STORE, 139634276470784, 139634278572031, -+STORE, 139634276458496, 139634276470783, -+SNULL, 139634278563839, 139634278572031, -+STORE, 139634276470784, 139634278563839, -+STORE, 139634278563840, 139634278572031, -+ERASE, 139634278563840, 139634278572031, -+STORE, 139634278563840, 139634278572031, -+STORE, 139634272661504, 139634276458495, -+SNULL, 139634272661504, 139634274320383, -+STORE, 139634274320384, 139634276458495, -+STORE, 139634272661504, 139634274320383, -+SNULL, 139634276417535, 139634276458495, -+STORE, 139634274320384, 139634276417535, -+STORE, 139634276417536, 139634276458495, -+SNULL, 139634276417536, 139634276442111, -+STORE, 139634276442112, 139634276458495, -+STORE, 139634276417536, 139634276442111, -+ERASE, 139634276417536, 139634276442111, -+STORE, 139634276417536, 139634276442111, -+ERASE, 139634276442112, 139634276458495, -+STORE, 139634276442112, 139634276458495, -+STORE, 139634280767488, 139634280783871, -+SNULL, 139634276433919, 139634276442111, -+STORE, 139634276417536, 139634276433919, -+STORE, 139634276433920, 139634276442111, -+SNULL, 139634278567935, 139634278572031, -+STORE, 139634278563840, 139634278567935, -+STORE, 139634278567936, 139634278572031, -+SNULL, 94567988150271, 94567988158463, -+STORE, 94567988146176, 94567988150271, -+STORE, 94567988150272, 94567988158463, -+SNULL, 139634280816639, 139634280820735, -+STORE, 139634280812544, 139634280816639, -+STORE, 139634280816640, 139634280820735, -+ERASE, 139634280783872, 139634280812543, -+STORE, 94567996379136, 94567996514303, -+STORE, 139634279084032, 139634280767487, -+STORE, 94567996379136, 94567996649471, -+STORE, 94567996379136, 94567996784639, -+STORE, 94567996379136, 94567996960767, -+SNULL, 94567996932095, 94567996960767, -+STORE, 94567996379136, 94567996932095, -+STORE, 94567996932096, 94567996960767, -+ERASE, 94567996932096, 94567996960767, -+STORE, 94567996379136, 94567997071359, -+STORE, 94567996379136, 94567997206527, -+SNULL, 94567997186047, 94567997206527, -+STORE, 94567996379136, 94567997186047, -+STORE, 94567997186048, 94567997206527, -+ERASE, 94567997186048, 94567997206527, -+STORE, 94567996379136, 94567997358079, -+STORE, 94567996379136, 94567997493247, -+SNULL, 94567997476863, 94567997493247, -+STORE, 94567996379136, 94567997476863, -+STORE, 94567997476864, 94567997493247, -+ERASE, 94567997476864, 94567997493247, -+STORE, 94567996379136, 94567997612031, -+STORE, 94567996379136, 94567997767679, -+SNULL, 94567997739007, 94567997767679, -+STORE, 94567996379136, 94567997739007, -+STORE, 94567997739008, 94567997767679, -+ERASE, 94567997739008, 94567997767679, -+SNULL, 94567997698047, 94567997739007, -+STORE, 94567996379136, 94567997698047, -+STORE, 94567997698048, 94567997739007, -+ERASE, 94567997698048, 94567997739007, -+STORE, 94567996379136, 94567997853695, -+STORE, 94567996379136, 94567997988863, -+STORE, 94567996379136, 94567998132223, -+STORE, 94567996379136, 94567998275583, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140723667759104, 140737488351231, -+SNULL, 140723667767295, 140737488351231, -+STORE, 140723667759104, 140723667767295, -+STORE, 140723667628032, 140723667767295, -+STORE, 94231598800896, 94231601135615, -+SNULL, 94231599013887, 94231601135615, -+STORE, 94231598800896, 94231599013887, -+STORE, 94231599013888, 94231601135615, -+ERASE, 94231599013888, 94231601135615, -+STORE, 94231601111040, 94231601123327, -+STORE, 94231601123328, 94231601135615, -+STORE, 140269472649216, 140269474902015, -+SNULL, 140269472792575, 140269474902015, -+STORE, 140269472649216, 140269472792575, -+STORE, 140269472792576, 140269474902015, -+ERASE, 140269472792576, 140269474902015, -+STORE, 140269474889728, 140269474897919, -+STORE, 140269474897920, 140269474902015, -+STORE, 140723667836928, 140723667841023, -+STORE, 140723667824640, 140723667836927, -+STORE, 140269474861056, 140269474889727, -+STORE, 140269474852864, 140269474861055, -+STORE, 140269470535680, 140269472649215, -+SNULL, 140269470535680, 140269470547967, -+STORE, 140269470547968, 140269472649215, -+STORE, 140269470535680, 140269470547967, -+SNULL, 140269472641023, 140269472649215, -+STORE, 140269470547968, 140269472641023, -+STORE, 140269472641024, 140269472649215, -+ERASE, 140269472641024, 140269472649215, -+STORE, 140269472641024, 140269472649215, -+STORE, 140269466738688, 140269470535679, -+SNULL, 140269466738688, 140269468397567, -+STORE, 140269468397568, 140269470535679, -+STORE, 140269466738688, 140269468397567, -+SNULL, 140269470494719, 140269470535679, -+STORE, 140269468397568, 140269470494719, -+STORE, 140269470494720, 140269470535679, -+SNULL, 140269470494720, 140269470519295, -+STORE, 140269470519296, 140269470535679, -+STORE, 140269470494720, 140269470519295, -+ERASE, 140269470494720, 140269470519295, -+STORE, 140269470494720, 140269470519295, -+ERASE, 140269470519296, 140269470535679, -+STORE, 140269470519296, 140269470535679, -+STORE, 140269474844672, 140269474861055, -+SNULL, 140269470511103, 140269470519295, -+STORE, 140269470494720, 140269470511103, -+STORE, 140269470511104, 140269470519295, -+SNULL, 140269472645119, 140269472649215, -+STORE, 140269472641024, 140269472645119, -+STORE, 140269472645120, 140269472649215, -+SNULL, 94231601115135, 94231601123327, -+STORE, 94231601111040, 94231601115135, -+STORE, 94231601115136, 94231601123327, -+SNULL, 140269474893823, 140269474897919, -+STORE, 140269474889728, 140269474893823, -+STORE, 140269474893824, 140269474897919, -+ERASE, 140269474861056, 140269474889727, -+STORE, 94231626592256, 94231626727423, -+STORE, 140269473161216, 140269474844671, -+STORE, 94231626592256, 94231626862591, -+STORE, 94231626592256, 94231626997759, -+STORE, 94327178862592, 94327179075583, -+STORE, 94327181172736, 94327181176831, -+STORE, 94327181176832, 94327181185023, -+STORE, 94327181185024, 94327181197311, -+STORE, 94327185715200, 94327186685951, -+STORE, 140172071755776, 140172073414655, -+STORE, 140172073414656, 140172075511807, -+STORE, 140172075511808, 140172075528191, -+STORE, 140172075528192, 140172075536383, -+STORE, 140172075536384, 140172075552767, -+STORE, 140172075552768, 140172075565055, -+STORE, 140172075565056, 140172077658111, -+STORE, 140172077658112, 140172077662207, -+STORE, 140172077662208, 140172077666303, -+STORE, 140172077666304, 140172077809663, -+STORE, 140172078178304, 140172079861759, -+STORE, 140172079861760, 140172079878143, -+STORE, 140172079878144, 140172079906815, -+STORE, 140172079906816, 140172079910911, -+STORE, 140172079910912, 140172079915007, -+STORE, 140172079915008, 140172079919103, -+STORE, 140720358359040, 140720358494207, -+STORE, 140720358498304, 140720358510591, -+STORE, 140720358510592, 140720358514687, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140722548621312, 140737488351231, -+SNULL, 140722548629503, 140737488351231, -+STORE, 140722548621312, 140722548629503, -+STORE, 140722548490240, 140722548629503, -+STORE, 93949289504768, 93949291728895, -+SNULL, 93949289615359, 93949291728895, -+STORE, 93949289504768, 93949289615359, -+STORE, 93949289615360, 93949291728895, -+ERASE, 93949289615360, 93949291728895, -+STORE, 93949291708416, 93949291720703, -+STORE, 93949291720704, 93949291728895, -+STORE, 140305861902336, 140305864155135, -+SNULL, 140305862045695, 140305864155135, -+STORE, 140305861902336, 140305862045695, -+STORE, 140305862045696, 140305864155135, -+ERASE, 140305862045696, 140305864155135, -+STORE, 140305864142848, 140305864151039, -+STORE, 140305864151040, 140305864155135, -+STORE, 140722549821440, 140722549825535, -+STORE, 140722549809152, 140722549821439, -+STORE, 140305864114176, 140305864142847, -+STORE, 140305864105984, 140305864114175, -+STORE, 140305858105344, 140305861902335, -+SNULL, 140305858105344, 140305859764223, -+STORE, 140305859764224, 140305861902335, -+STORE, 140305858105344, 140305859764223, -+SNULL, 140305861861375, 140305861902335, -+STORE, 140305859764224, 140305861861375, -+STORE, 140305861861376, 140305861902335, -+SNULL, 140305861861376, 140305861885951, -+STORE, 140305861885952, 140305861902335, -+STORE, 140305861861376, 140305861885951, -+ERASE, 140305861861376, 140305861885951, -+STORE, 140305861861376, 140305861885951, -+ERASE, 140305861885952, 140305861902335, -+STORE, 140305861885952, 140305861902335, -+SNULL, 140305861877759, 140305861885951, -+STORE, 140305861861376, 140305861877759, -+STORE, 140305861877760, 140305861885951, -+SNULL, 93949291716607, 93949291720703, -+STORE, 93949291708416, 93949291716607, -+STORE, 93949291716608, 93949291720703, -+SNULL, 140305864146943, 140305864151039, -+STORE, 140305864142848, 140305864146943, -+STORE, 140305864146944, 140305864151039, -+ERASE, 140305864114176, 140305864142847, -+STORE, 93949324136448, 93949324271615, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140725754908672, 140737488351231, -+SNULL, 140725754916863, 140737488351231, -+STORE, 140725754908672, 140725754916863, -+STORE, 140725754777600, 140725754916863, -+STORE, 94831184375808, 94831186599935, -+SNULL, 94831184486399, 94831186599935, -+STORE, 94831184375808, 94831184486399, -+STORE, 94831184486400, 94831186599935, -+ERASE, 94831184486400, 94831186599935, -+STORE, 94831186579456, 94831186591743, -+STORE, 94831186591744, 94831186599935, -+STORE, 140605482479616, 140605484732415, -+SNULL, 140605482622975, 140605484732415, -+STORE, 140605482479616, 140605482622975, -+STORE, 140605482622976, 140605484732415, -+ERASE, 140605482622976, 140605484732415, -+STORE, 140605484720128, 140605484728319, -+STORE, 140605484728320, 140605484732415, -+STORE, 140725755670528, 140725755674623, -+STORE, 140725755658240, 140725755670527, -+STORE, 140605484691456, 140605484720127, -+STORE, 140605484683264, 140605484691455, -+STORE, 140605478682624, 140605482479615, -+SNULL, 140605478682624, 140605480341503, -+STORE, 140605480341504, 140605482479615, -+STORE, 140605478682624, 140605480341503, -+SNULL, 140605482438655, 140605482479615, -+STORE, 140605480341504, 140605482438655, -+STORE, 140605482438656, 140605482479615, -+SNULL, 140605482438656, 140605482463231, -+STORE, 140605482463232, 140605482479615, -+STORE, 140605482438656, 140605482463231, -+ERASE, 140605482438656, 140605482463231, -+STORE, 140605482438656, 140605482463231, -+ERASE, 140605482463232, 140605482479615, -+STORE, 140605482463232, 140605482479615, -+SNULL, 140605482455039, 140605482463231, -+STORE, 140605482438656, 140605482455039, -+STORE, 140605482455040, 140605482463231, -+SNULL, 94831186587647, 94831186591743, -+STORE, 94831186579456, 94831186587647, -+STORE, 94831186587648, 94831186591743, -+SNULL, 140605484724223, 140605484728319, -+STORE, 140605484720128, 140605484724223, -+STORE, 140605484724224, 140605484728319, -+ERASE, 140605484691456, 140605484720127, -+STORE, 94831217156096, 94831217291263, -+STORE, 94327178862592, 94327179075583, -+STORE, 94327181172736, 94327181176831, -+STORE, 94327181176832, 94327181185023, -+STORE, 94327181185024, 94327181197311, -+STORE, 94327185715200, 94327186685951, -+STORE, 140172071755776, 140172073414655, -+STORE, 140172073414656, 140172075511807, -+STORE, 140172075511808, 140172075528191, -+STORE, 140172075528192, 140172075536383, -+STORE, 140172075536384, 140172075552767, -+STORE, 140172075552768, 140172075565055, -+STORE, 140172075565056, 140172077658111, -+STORE, 140172077658112, 140172077662207, -+STORE, 140172077662208, 140172077666303, -+STORE, 140172077666304, 140172077809663, -+STORE, 140172078178304, 140172079861759, -+STORE, 140172079861760, 140172079878143, -+STORE, 140172079878144, 140172079906815, -+STORE, 140172079906816, 140172079910911, -+STORE, 140172079910912, 140172079915007, -+STORE, 140172079915008, 140172079919103, -+STORE, 140720358359040, 140720358494207, -+STORE, 140720358498304, 140720358510591, -+STORE, 140720358510592, 140720358514687, -+STORE, 140737488347136, 140737488351231, -+STORE, 140737488343040, 140737488351231, -+STORE, 140737488338944, 140737488351231, -+STORE, 140734529933312, 140737488351231, -+SNULL, 140734529945599, 140737488351231, -+STORE, 140734529933312, 140734529945599, -+STORE, 140734529802240, 140734529945599, -+STORE, 4194304, 26279935, -+STORE, 28372992, 28454911, -+STORE, 28454912, 29806591, -+STORE, 140249744060416, 140249746313215, -+SNULL, 140249744203775, 140249746313215, -+STORE, 140249744060416, 140249744203775, -+STORE, 140249744203776, 140249746313215, -+ERASE, 140249744203776, 140249746313215, -+STORE, 140249746300928, 140249746309119, -+STORE, 140249746309120, 140249746313215, -+STORE, 140734530174976, 140734530179071, -+STORE, 140734530162688, 140734530174975, -+STORE, 140249746272256, 140249746300927, -+STORE, 140249746264064, 140249746272255, -+STORE, 140249740226560, 140249744060415, -+SNULL, 140249740226560, 140249741934591, -+STORE, 140249741934592, 140249744060415, -+STORE, 140249740226560, 140249741934591, -+SNULL, 140249744027647, 140249744060415, -+STORE, 140249741934592, 140249744027647, -+STORE, 140249744027648, 140249744060415, -+ERASE, 140249744027648, 140249744060415, -+STORE, 140249744027648, 140249744060415, -+STORE, 140249738031104, 140249740226559, -+SNULL, 140249738031104, 140249738125311, -+STORE, 140249738125312, 140249740226559, -+STORE, 140249738031104, 140249738125311, -+SNULL, 140249740218367, 140249740226559, -+STORE, 140249738125312, 140249740218367, -+STORE, 140249740218368, 140249740226559, -+ERASE, 140249740218368, 140249740226559, -+STORE, 140249740218368, 140249740226559, -+STORE, 140249735512064, 140249738031103, -+SNULL, 140249735512064, 140249735925759, -+STORE, 140249735925760, 140249738031103, -+STORE, 140249735512064, 140249735925759, -+SNULL, 140249738018815, 140249738031103, -+STORE, 140249735925760, 140249738018815, -+STORE, 140249738018816, 140249738031103, -+ERASE, 140249738018816, 140249738031103, -+STORE, 140249738018816, 140249738031103, -+STORE, 140249732878336, 140249735512063, -+SNULL, 140249732878336, 140249733406719, -+STORE, 140249733406720, 140249735512063, -+STORE, 140249732878336, 140249733406719, -+SNULL, 140249735503871, 140249735512063, -+STORE, 140249733406720, 140249735503871, -+STORE, 140249735503872, 140249735512063, -+ERASE, 140249735503872, 140249735512063, -+STORE, 140249735503872, 140249735512063, -+STORE, 140249730764800, 140249732878335, -+SNULL, 140249730764800, 140249730777087, -+STORE, 140249730777088, 140249732878335, -+STORE, 140249730764800, 140249730777087, -+SNULL, 140249732870143, 140249732878335, -+STORE, 140249730777088, 140249732870143, -+STORE, 140249732870144, 140249732878335, -+ERASE, 140249732870144, 140249732878335, -+STORE, 140249732870144, 140249732878335, -+STORE, 140249728561152, 140249730764799, -+SNULL, 140249728561152, 140249728663551, -+STORE, 140249728663552, 140249730764799, -+STORE, 140249728561152, 140249728663551, -+SNULL, 140249730756607, 140249730764799, -+STORE, 140249728663552, 140249730756607, -+STORE, 140249730756608, 140249730764799, -+ERASE, 140249730756608, 140249730764799, -+STORE, 140249730756608, 140249730764799, -+STORE, 140249746255872, 140249746272255, -+STORE, 140249725399040, 140249728561151, -+SNULL, 140249725399040, 140249726459903, -+STORE, 140249726459904, 140249728561151, -+STORE, 140249725399040, 140249726459903, -+SNULL, 140249728552959, 140249728561151, -+STORE, 140249726459904, 140249728552959, -+STORE, 140249728552960, 140249728561151, -+ERASE, 140249728552960, 140249728561151, -+STORE, 140249728552960, 140249728561151, -+STORE, 140249721602048, 140249725399039, -+SNULL, 140249721602048, 140249723260927, -+STORE, 140249723260928, 140249725399039, -+STORE, 140249721602048, 140249723260927, -+SNULL, 140249725358079, 140249725399039, -+STORE, 140249723260928, 140249725358079, -+STORE, 140249725358080, 140249725399039, -+SNULL, 140249725358080, 140249725382655, -+STORE, 140249725382656, 140249725399039, -+STORE, 140249725358080, 140249725382655, -+ERASE, 140249725358080, 140249725382655, -+STORE, 140249725358080, 140249725382655, -+ERASE, 140249725382656, 140249725399039, -+STORE, 140249725382656, 140249725399039, -+STORE, 140249746243584, 140249746272255, -+SNULL, 140249725374463, 140249725382655, -+STORE, 140249725358080, 140249725374463, -+STORE, 140249725374464, 140249725382655, -+SNULL, 140249728557055, 140249728561151, -+STORE, 140249728552960, 140249728557055, -+STORE, 140249728557056, 140249728561151, -+SNULL, 140249730760703, 140249730764799, -+STORE, 140249730756608, 140249730760703, -+STORE, 140249730760704, 140249730764799, -+SNULL, 140249732874239, 140249732878335, -+STORE, 140249732870144, 140249732874239, -+STORE, 140249732874240, 140249732878335, -+SNULL, 140249735507967, 140249735512063, -+STORE, 140249735503872, 140249735507967, -+STORE, 140249735507968, 140249735512063, -+SNULL, 140249738027007, 140249738031103, -+STORE, 140249738018816, 140249738027007, -+STORE, 140249738027008, 140249738031103, -+SNULL, 140249740222463, 140249740226559, -+STORE, 140249740218368, 140249740222463, -+STORE, 140249740222464, 140249740226559, -+SNULL, 140249744031743, 140249744060415, -+STORE, 140249744027648, 140249744031743, -+STORE, 140249744031744, 140249744060415, -+SNULL, 28405759, 28454911, -+STORE, 28372992, 28405759, -+STORE, 28405760, 28454911, -+SNULL, 140249746305023, 140249746309119, -+STORE, 140249746300928, 140249746305023, -+STORE, 140249746305024, 140249746309119, -+ERASE, 140249746272256, 140249746300927, -+STORE, 33853440, 33988607, -+STORE, 140249744560128, 140249746243583, -+STORE, 140249746296832, 140249746300927, -+STORE, 140249744424960, 140249744560127, -+STORE, 33853440, 34131967, -+STORE, 140249719504896, 140249721602047, -+STORE, 140249746288640, 140249746300927, -+STORE, 140249746280448, 140249746300927, -+STORE, 140249746243584, 140249746280447, -+STORE, 140249744408576, 140249744560127, -+STORE, 33853440, 34267135, -+STORE, 33853440, 34422783, -+STORE, 140249744400384, 140249744560127, -+STORE, 140249744392192, 140249744560127, -+STORE, 33853440, 34557951, -+STORE, 33853440, 34693119, -+STORE, 140249744375808, 140249744560127, -+STORE, 140249744367616, 140249744560127, -+STORE, 33853440, 34832383, -+STORE, 140249719230464, 140249721602047, -+STORE, 140249744207872, 140249744560127, -+STORE, 33853440, 34971647, -+SNULL, 34963455, 34971647, -+STORE, 33853440, 34963455, -+STORE, 34963456, 34971647, -+ERASE, 34963456, 34971647, -+SNULL, 34955263, 34963455, -+STORE, 33853440, 34955263, -+STORE, 34955264, 34963455, -+ERASE, 34955264, 34963455, -+SNULL, 34947071, 34955263, -+STORE, 33853440, 34947071, -+STORE, 34947072, 34955263, -+ERASE, 34947072, 34955263, -+SNULL, 34938879, 34947071, -+STORE, 33853440, 34938879, -+STORE, 34938880, 34947071, -+ERASE, 34938880, 34947071, -+STORE, 140249719214080, 140249721602047, -+STORE, 140249719148544, 140249721602047, -+STORE, 140249719115776, 140249721602047, -+STORE, 140249717018624, 140249721602047, -+STORE, 140249716953088, 140249721602047, -+STORE, 33853440, 35086335, -+STORE, 140249716822016, 140249721602047, -+STORE, 140249716559872, 140249721602047, -+STORE, 140249716551680, 140249721602047, -+STORE, 140249716535296, 140249721602047, -+STORE, 140249716527104, 140249721602047, -+STORE, 140249716518912, 140249721602047, -+STORE, 33853440, 35221503, -+SNULL, 35213311, 35221503, -+STORE, 33853440, 35213311, -+STORE, 35213312, 35221503, -+ERASE, 35213312, 35221503, -+SNULL, 35205119, 35213311, -+STORE, 33853440, 35205119, -+STORE, 35205120, 35213311, -+ERASE, 35205120, 35213311, -+SNULL, 35192831, 35205119, -+STORE, 33853440, 35192831, -+STORE, 35192832, 35205119, -+ERASE, 35192832, 35205119, -+SNULL, 35176447, 35192831, -+STORE, 33853440, 35176447, -+STORE, 35176448, 35192831, -+ERASE, 35176448, 35192831, -+STORE, 140249716502528, 140249721602047, -+STORE, 33853440, 35311615, -+SNULL, 35307519, 35311615, -+STORE, 33853440, 35307519, -+STORE, 35307520, 35311615, -+ERASE, 35307520, 35311615, -+SNULL, 35303423, 35307519, -+STORE, 33853440, 35303423, -+STORE, 35303424, 35307519, -+ERASE, 35303424, 35307519, -+SNULL, 35299327, 35303423, -+STORE, 33853440, 35299327, -+STORE, 35299328, 35303423, -+ERASE, 35299328, 35303423, -+SNULL, 35295231, 35299327, -+STORE, 33853440, 35295231, -+STORE, 35295232, 35299327, -+ERASE, 35295232, 35299327, -+SNULL, 35291135, 35295231, -+STORE, 33853440, 35291135, -+STORE, 35291136, 35295231, -+ERASE, 35291136, 35295231, -+SNULL, 35287039, 35291135, -+STORE, 33853440, 35287039, -+STORE, 35287040, 35291135, -+ERASE, 35287040, 35291135, -+SNULL, 35282943, 35287039, -+STORE, 33853440, 35282943, -+STORE, 35282944, 35287039, -+ERASE, 35282944, 35287039, -+STORE, 140249716486144, 140249721602047, -+STORE, 140249716453376, 140249721602047, -+STORE, 33853440, 35418111, -+SNULL, 35401727, 35418111, -+STORE, 33853440, 35401727, -+STORE, 35401728, 35418111, -+ERASE, 35401728, 35418111, -+SNULL, 35389439, 35401727, -+STORE, 33853440, 35389439, -+STORE, 35389440, 35401727, -+ERASE, 35389440, 35401727, -+STORE, 140249714356224, 140249721602047, -+STORE, 33853440, 35540991, -+STORE, 140249714339840, 140249721602047, -+STORE, 140249714077696, 140249721602047, -+STORE, 140249714069504, 140249721602047, -+STORE, 140249714061312, 140249721602047, -+STORE, 33853440, 35680255, -+SNULL, 35672063, 35680255, -+STORE, 33853440, 35672063, -+STORE, 35672064, 35680255, -+ERASE, 35672064, 35680255, -+SNULL, 35627007, 35672063, -+STORE, 33853440, 35627007, -+STORE, 35627008, 35672063, -+ERASE, 35627008, 35672063, -+STORE, 140249711964160, 140249721602047, -+STORE, 33853440, 35762175, -+SNULL, 35753983, 35762175, -+STORE, 33853440, 35753983, -+STORE, 35753984, 35762175, -+ERASE, 35753984, 35762175, -+SNULL, 35745791, 35753983, -+STORE, 33853440, 35745791, -+STORE, 35745792, 35753983, -+ERASE, 35745792, 35753983, -+STORE, 140249711955968, 140249721602047, -+STORE, 140249711947776, 140249721602047, -+STORE, 140249710899200, 140249721602047, -+STORE, 140249710866432, 140249721602047, -+STORE, 140249710600192, 140249721602047, -+SNULL, 140249744424959, 140249744560127, -+STORE, 140249744207872, 140249744424959, -+STORE, 140249744424960, 140249744560127, -+ERASE, 140249744424960, 140249744560127, -+STORE, 140249708503040, 140249721602047, -+STORE, 33853440, 35885055, -+STORE, 140249707978752, 140249721602047, -+STORE, 140249705881600, 140249721602047, -+STORE, 33853440, 36036607, -+STORE, 33853440, 36175871, -+STORE, 140249744551936, 140249744560127, -+STORE, 140249744543744, 140249744560127, -+STORE, 140249744535552, 140249744560127, -+STORE, 140249744527360, 140249744560127, -+STORE, 140249744519168, 140249744560127, -+STORE, 140249705619456, 140249721602047, -+STORE, 140249744510976, 140249744560127, -+STORE, 140249744502784, 140249744560127, -+STORE, 140249744494592, 140249744560127, -+STORE, 140249744486400, 140249744560127, -+STORE, 140249744478208, 140249744560127, -+STORE, 140249744470016, 140249744560127, -+STORE, 140249744461824, 140249744560127, -+STORE, 140249744453632, 140249744560127, -+STORE, 140249744445440, 140249744560127, -+STORE, 140249744437248, 140249744560127, -+STORE, 140249744429056, 140249744560127, -+STORE, 140249703522304, 140249721602047, -+STORE, 33853440, 36311039, -+STORE, 140249703489536, 140249721602047, -+STORE, 33853440, 36474879, -+STORE, 140249703456768, 140249721602047, -+STORE, 33853440, 36622335, -+STORE, 140249703424000, 140249721602047, -+STORE, 140249703391232, 140249721602047, -+STORE, 33853440, 36810751, -+STORE, 140249703358464, 140249721602047, -+STORE, 140249703325696, 140249721602047, -+SNULL, 36655103, 36810751, -+STORE, 33853440, 36655103, -+STORE, 36655104, 36810751, -+ERASE, 36655104, 36810751, -+SNULL, 36438015, 36655103, -+STORE, 33853440, 36438015, -+STORE, 36438016, 36655103, -+ERASE, 36438016, 36655103, -+STORE, 140249703317504, 140249721602047, -+STORE, 140249701220352, 140249721602047, -+STORE, 33853440, 36585471, -+STORE, 33853440, 36782079, -+STORE, 140249701212160, 140249721602047, -+STORE, 140249701203968, 140249721602047, -+STORE, 140249701195776, 140249721602047, -+STORE, 140249701187584, 140249721602047, -+STORE, 140249701179392, 140249721602047, -+STORE, 140249701171200, 140249721602047, -+STORE, 140249701163008, 140249721602047, -+STORE, 140249701154816, 140249721602047, -+STORE, 140249701146624, 140249721602047, -+STORE, 140249701138432, 140249721602047, -+STORE, 140249701130240, 140249721602047, -+STORE, 140249700081664, 140249721602047, -+STORE, 140249700073472, 140249721602047, -+STORE, 33853440, 36978687, -+STORE, 140249697976320, 140249721602047, -+STORE, 33853440, 37240831, -+STORE, 140249695879168, 140249721602047, -+STORE, 140249695870976, 140249721602047, -+STORE, 140249695862784, 140249721602047, -+STORE, 140249695854592, 140249721602047, -+STORE, 140249695326208, 140249721602047, -+SNULL, 140249710600191, 140249721602047, -+STORE, 140249695326208, 140249710600191, -+STORE, 140249710600192, 140249721602047, -+SNULL, 140249710600192, 140249710866431, -+STORE, 140249710866432, 140249721602047, -+STORE, 140249710600192, 140249710866431, -+ERASE, 140249710600192, 140249710866431, -+STORE, 140249691131904, 140249710600191, -+STORE, 33853440, 37474303, -+STORE, 140249710858240, 140249721602047, -+STORE, 140249710850048, 140249721602047, -+STORE, 140249710841856, 140249721602047, -+STORE, 140249710833664, 140249721602047, -+STORE, 140249710825472, 140249721602047, -+STORE, 140249710817280, 140249721602047, -+STORE, 140249710809088, 140249721602047, -+STORE, 140249710800896, 140249721602047, -+STORE, 140249710792704, 140249721602047, -+STORE, 140249710784512, 140249721602047, -+STORE, 140249710776320, 140249721602047, -+STORE, 140249710768128, 140249721602047, -+STORE, 140249710759936, 140249721602047, -+STORE, 140249710751744, 140249721602047, -+STORE, 140249710743552, 140249721602047, -+STORE, 140249710735360, 140249721602047, -+STORE, 140249689034752, 140249710600191, -+STORE, 140249710727168, 140249721602047, -+STORE, 140249686937600, 140249710600191, -+STORE, 33853440, 37867519, -+STORE, 140249684840448, 140249710600191, -+STORE, 140249710718976, 140249721602047, -+STORE, 140249682743296, 140249710600191, -+STORE, 140249710710784, 140249721602047, -+STORE, 140249710702592, 140249721602047, -+STORE, 140249710694400, 140249721602047, -+STORE, 140249710686208, 140249721602047, -+STORE, 140249710678016, 140249721602047, -+STORE, 140249682612224, 140249710600191, -+STORE, 140249682087936, 140249710600191, -+SNULL, 140249705619455, 140249710600191, -+STORE, 140249682087936, 140249705619455, -+STORE, 140249705619456, 140249710600191, -+SNULL, 140249705619456, 140249705881599, -+STORE, 140249705881600, 140249710600191, -+STORE, 140249705619456, 140249705881599, -+ERASE, 140249705619456, 140249705881599, -+STORE, 140249679990784, 140249705619455, -+STORE, 140249710669824, 140249721602047, -+STORE, 140249677893632, 140249705619455, -+STORE, 140249710653440, 140249721602047, -+STORE, 140249710645248, 140249721602047, -+STORE, 140249710637056, 140249721602047, -+STORE, 140249710628864, 140249721602047, -+STORE, 140249710620672, 140249721602047, -+STORE, 140249710612480, 140249721602047, -+STORE, 140249710604288, 140249721602047, -+STORE, 140249705873408, 140249710600191, -+STORE, 140249705865216, 140249710600191, -+STORE, 140249705857024, 140249710600191, -+STORE, 140249705848832, 140249710600191, -+STORE, 140249705840640, 140249710600191, -+STORE, 140249705832448, 140249710600191, -+STORE, 140249705824256, 140249710600191, -+STORE, 140249705816064, 140249710600191, -+STORE, 140249705807872, 140249710600191, -+STORE, 140249705799680, 140249710600191, -+STORE, 33853440, 38129663, -+SNULL, 140249744207872, 140249744367615, -+STORE, 140249744367616, 140249744424959, -+STORE, 140249744207872, 140249744367615, -+ERASE, 140249744207872, 140249744367615, -+STORE, 140249677606912, 140249705619455, -+STORE, 140249675509760, 140249705619455, -+SNULL, 140249677606911, 140249705619455, -+STORE, 140249675509760, 140249677606911, -+STORE, 140249677606912, 140249705619455, -+SNULL, 140249677606912, 140249677893631, -+STORE, 140249677893632, 140249705619455, -+STORE, 140249677606912, 140249677893631, -+ERASE, 140249677606912, 140249677893631, -+STORE, 140249744359424, 140249744424959, -+STORE, 33853440, 38391807, -+STORE, 140249674981376, 140249677606911, -+STORE, 140249672884224, 140249677606911, -+SNULL, 140249719230463, 140249721602047, -+STORE, 140249710604288, 140249719230463, -+STORE, 140249719230464, 140249721602047, -+SNULL, 140249719230464, 140249719504895, -+STORE, 140249719504896, 140249721602047, -+STORE, 140249719230464, 140249719504895, -+ERASE, 140249719230464, 140249719504895, -+STORE, 140249744351232, 140249744424959, -+STORE, 140249744343040, 140249744424959, -+STORE, 140249744334848, 140249744424959, -+STORE, 140249744326656, 140249744424959, -+STORE, 140249744310272, 140249744424959, -+STORE, 140249744302080, 140249744424959, -+STORE, 140249744285696, 140249744424959, -+STORE, 140249744277504, 140249744424959, -+STORE, 140249744261120, 140249744424959, -+STORE, 140249744252928, 140249744424959, -+STORE, 140249744220160, 140249744424959, -+STORE, 140249744211968, 140249744424959, -+STORE, 140249719488512, 140249721602047, -+STORE, 140249744203776, 140249744424959, -+STORE, 140249719472128, 140249721602047, -+STORE, 140249719463936, 140249721602047, -+STORE, 140249719447552, 140249721602047, -+STORE, 140249719439360, 140249721602047, -+STORE, 140249719406592, 140249721602047, -+STORE, 140249719398400, 140249721602047, -+STORE, 140249719382016, 140249721602047, -+STORE, 140249719373824, 140249721602047, -+STORE, 140249719357440, 140249721602047, -+STORE, 140249719349248, 140249721602047, -+STORE, 140249719332864, 140249721602047, -+STORE, 140249719324672, 140249721602047, -+STORE, 140249719291904, 140249721602047, -+STORE, 140249719283712, 140249721602047, -+STORE, 140249719267328, 140249721602047, -+STORE, 140249719259136, 140249721602047, -+STORE, 140249719242752, 140249721602047, -+STORE, 140249719234560, 140249721602047, -+STORE, 140249705783296, 140249710600191, -+STORE, 140249705775104, 140249710600191, -+STORE, 140249705742336, 140249710600191, -+STORE, 140249705734144, 140249710600191, -+STORE, 140249705717760, 140249710600191, -+STORE, 140249670787072, 140249677606911, -+STORE, 140249705709568, 140249710600191, -+STORE, 140249705693184, 140249710600191, -+STORE, 140249705684992, 140249710600191, -+STORE, 140249705668608, 140249710600191, -+STORE, 140249705660416, 140249710600191, -+STORE, 140249705627648, 140249710600191, -+STORE, 140249677893632, 140249710600191, -+STORE, 140249677877248, 140249710600191, -+STORE, 140249677869056, 140249710600191, -+STORE, 140249677852672, 140249710600191, -+STORE, 140249677844480, 140249710600191, -+STORE, 140249677828096, 140249710600191, -+STORE, 140249668689920, 140249677606911, -+STORE, 140249677819904, 140249710600191, -+STORE, 140249677787136, 140249710600191, -+STORE, 140249677778944, 140249710600191, -+STORE, 140249677762560, 140249710600191, -+STORE, 140249677754368, 140249710600191, -+STORE, 140249677737984, 140249710600191, -+STORE, 140249677729792, 140249710600191, -+STORE, 140249677713408, 140249710600191, -+STORE, 140249677705216, 140249710600191, -+STORE, 140249677672448, 140249710600191, -+STORE, 140249677664256, 140249710600191, -+STORE, 140249677647872, 140249710600191, -+STORE, 140249677639680, 140249710600191, -+STORE, 140249677623296, 140249710600191, -+STORE, 140249677615104, 140249710600191, -+STORE, 140249668673536, 140249677606911, -+STORE, 140249668673536, 140249710600191, -+STORE, 140249668640768, 140249710600191, -+STORE, 140249668632576, 140249710600191, -+STORE, 140249668616192, 140249710600191, -+STORE, 140249668608000, 140249710600191, -+STORE, 140249668591616, 140249710600191, -+STORE, 140249668583424, 140249710600191, -+STORE, 140249668567040, 140249710600191, -+STORE, 140249668558848, 140249710600191, -+STORE, 140249668526080, 140249710600191, -+STORE, 140249668517888, 140249710600191, -+STORE, 140249668501504, 140249710600191, -+STORE, 140249668493312, 140249710600191, -+STORE, 140249668476928, 140249710600191, -+STORE, 140249668468736, 140249710600191, -+STORE, 140249668452352, 140249710600191, -+STORE, 140249668444160, 140249710600191, -+STORE, 140249668411392, 140249710600191, -+STORE, 140249668403200, 140249710600191, -+STORE, 140249668386816, 140249710600191, -+STORE, 140249668378624, 140249710600191, -+STORE, 140249668362240, 140249710600191, -+STORE, 140249668354048, 140249710600191, -+STORE, 140249668337664, 140249710600191, -+STORE, 140249668329472, 140249710600191, -+STORE, 140249668296704, 140249710600191, -+STORE, 140249668288512, 140249710600191, -+STORE, 140249668272128, 140249710600191, -+STORE, 140249668263936, 140249710600191, -+STORE, 140249668247552, 140249710600191, -+STORE, 140249668239360, 140249710600191, -+STORE, 140249668222976, 140249710600191, -+STORE, 140249668214784, 140249710600191, -+STORE, 140249668182016, 140249710600191, -+STORE, 140249668173824, 140249710600191, -+STORE, 140249668157440, 140249710600191, -+STORE, 140249668149248, 140249710600191, -+STORE, 140249668132864, 140249710600191, -+STORE, 140249668124672, 140249710600191, -+STORE, 140249668108288, 140249710600191, -+STORE, 140249668100096, 140249710600191, -+STORE, 140249668067328, 140249710600191, -+STORE, 140249668059136, 140249710600191, -+STORE, 140249668042752, 140249710600191, -+STORE, 140249668034560, 140249710600191, -+STORE, 140249668018176, 140249710600191, -+STORE, 140249668009984, 140249710600191, -+STORE, 140249667993600, 140249710600191, -+STORE, 140249667985408, 140249710600191, -+STORE, 140249667952640, 140249710600191, -+STORE, 140249667944448, 140249710600191, -+STORE, 140249667928064, 140249710600191, -+STORE, 140249667919872, 140249710600191, -+STORE, 140249667903488, 140249710600191, -+STORE, 140249667895296, 140249710600191, -+STORE, 140249667878912, 140249710600191, -+STORE, 140249667870720, 140249710600191, -+STORE, 140249667837952, 140249710600191, -+STORE, 140249667829760, 140249710600191, -+STORE, 140249667813376, 140249710600191, -+STORE, 140249667805184, 140249710600191, -+STORE, 140249667788800, 140249710600191, -+STORE, 140249667780608, 140249710600191, -+STORE, 140249667764224, 140249710600191, -+STORE, 140249667756032, 140249710600191, -+STORE, 140249667723264, 140249710600191, -+STORE, 140249667715072, 140249710600191, -+STORE, 140249667698688, 140249710600191, -+STORE, 140249667690496, 140249710600191, -+STORE, 140249667674112, 140249710600191, -+STORE, 140249667665920, 140249710600191, -+STORE, 140249667649536, 140249710600191, -+STORE, 140249667641344, 140249710600191, -+STORE, 140249667608576, 140249710600191, -+STORE, 140249667600384, 140249710600191, -+STORE, 140249667584000, 140249710600191, -+STORE, 140249667575808, 140249710600191, -+STORE, 140249667559424, 140249710600191, -+STORE, 140249667551232, 140249710600191, -+STORE, 140249667534848, 140249710600191, -+STORE, 140249667526656, 140249710600191, -+STORE, 140249667493888, 140249710600191, -+STORE, 140249667485696, 140249710600191, -+STORE, 140249667469312, 140249710600191, -+STORE, 140249667461120, 140249710600191, -+STORE, 140249667444736, 140249710600191, -+STORE, 140249667436544, 140249710600191, -+STORE, 140249667420160, 140249710600191, -+STORE, 140249665323008, 140249710600191, -+STORE, 140249665314816, 140249710600191, -+STORE, 140249665282048, 140249710600191, -+STORE, 140249665273856, 140249710600191, -+STORE, 140249665257472, 140249710600191, -+STORE, 140249665249280, 140249710600191, -+STORE, 140249665232896, 140249710600191, -+STORE, 140249665224704, 140249710600191, -+STORE, 140249665208320, 140249710600191, -+STORE, 140249665200128, 140249710600191, -+STORE, 140249665167360, 140249710600191, -+STORE, 140249665159168, 140249710600191, -+STORE, 140249665142784, 140249710600191, -+STORE, 140249665134592, 140249710600191, -+STORE, 140249665118208, 140249710600191, -+STORE, 140249665110016, 140249710600191, -+STORE, 140249665093632, 140249710600191, -+STORE, 140249665085440, 140249710600191, -+STORE, 140249665052672, 140249710600191, -+STORE, 140249665044480, 140249710600191, -+STORE, 140249665028096, 140249710600191, -+STORE, 140249665019904, 140249710600191, -+STORE, 140249665003520, 140249710600191, -+STORE, 140249664995328, 140249710600191, -+STORE, 140249664978944, 140249710600191, -+STORE, 140249664970752, 140249710600191, -+STORE, 140249664937984, 140249710600191, -+STORE, 140249664929792, 140249710600191, -+STORE, 140249664913408, 140249710600191, -+STORE, 140249664905216, 140249710600191, -+STORE, 140249664888832, 140249710600191, -+STORE, 140249664880640, 140249710600191, -+STORE, 140249664864256, 140249710600191, -+STORE, 140249664856064, 140249710600191, -+STORE, 140249664823296, 140249710600191, -+STORE, 140249664815104, 140249710600191, -+STORE, 140249664798720, 140249710600191, -+STORE, 140249664790528, 140249710600191, -+STORE, 140249664774144, 140249710600191, -+STORE, 140249664765952, 140249710600191, -+STORE, 140249664749568, 140249710600191, -+STORE, 140249664741376, 140249710600191, -+STORE, 140249664708608, 140249710600191, -+STORE, 140249664700416, 140249710600191, -+STORE, 140249664684032, 140249710600191, -+STORE, 140249664675840, 140249710600191, -+STORE, 140249664659456, 140249710600191, -+STORE, 140249664651264, 140249710600191, -+STORE, 140249664634880, 140249710600191, -+STORE, 140249664626688, 140249710600191, -+STORE, 140249664593920, 140249710600191, -+STORE, 140249664585728, 140249710600191, -+STORE, 140249664569344, 140249710600191, -+STORE, 140249664561152, 140249710600191, -+STORE, 140249664544768, 140249710600191, -+STORE, 140249664536576, 140249710600191, -+STORE, 140249664520192, 140249710600191, -+STORE, 140249664512000, 140249710600191, -+STORE, 140249664479232, 140249710600191, -+STORE, 140249664471040, 140249710600191, -+STORE, 140249664454656, 140249710600191, -+STORE, 140249664446464, 140249710600191, -+STORE, 140249664430080, 140249710600191, -+STORE, 140249664421888, 140249710600191, -+STORE, 140249664405504, 140249710600191, -+STORE, 140249664397312, 140249710600191, -+STORE, 140249664364544, 140249710600191, -+STORE, 140249664356352, 140249710600191, -+STORE, 140249664339968, 140249710600191, -+STORE, 140249664331776, 140249710600191, -+STORE, 140249664315392, 140249710600191, -+STORE, 140249664307200, 140249710600191, -+STORE, 140249664290816, 140249710600191, -+STORE, 140249664282624, 140249710600191, -+STORE, 140249664249856, 140249710600191, -+STORE, 140249664241664, 140249710600191, -+STORE, 140249664225280, 140249710600191, -+STORE, 140249664217088, 140249710600191, -+STORE, 140249664200704, 140249710600191, -+STORE, 140249664192512, 140249710600191, -+STORE, 140249664176128, 140249710600191, -+STORE, 140249664167936, 140249710600191, -+STORE, 140249664135168, 140249710600191, -+STORE, 140249664126976, 140249710600191, -+STORE, 140249664110592, 140249710600191, -+STORE, 140249664102400, 140249710600191, -+STORE, 140249664086016, 140249710600191, -+STORE, 140249664077824, 140249710600191, -+STORE, 140249664061440, 140249710600191, -+STORE, 140249664053248, 140249710600191, -+STORE, 140249664020480, 140249710600191, -+STORE, 140249664012288, 140249710600191, -+STORE, 140249663995904, 140249710600191, -+STORE, 140249663987712, 140249710600191, -+STORE, 140249663971328, 140249710600191, -+STORE, 140249663963136, 140249710600191, -+STORE, 140249663946752, 140249710600191, -+STORE, 140249663938560, 140249710600191, -+STORE, 140249663905792, 140249710600191, -+STORE, 140249663897600, 140249710600191, -+STORE, 140249663881216, 140249710600191, -+STORE, 140249663873024, 140249710600191, -+STORE, 140249663856640, 140249710600191, -+STORE, 140249663848448, 140249710600191, -+STORE, 140249663832064, 140249710600191, -+STORE, 140249663823872, 140249710600191, -+STORE, 140249663791104, 140249710600191, -+STORE, 140249663782912, 140249710600191, -+STORE, 140249663766528, 140249710600191, -+STORE, 140249663758336, 140249710600191, -+STORE, 140249663741952, 140249710600191, -+STORE, 140249663733760, 140249710600191, -+STORE, 140249663717376, 140249710600191, -+STORE, 140249663709184, 140249710600191, -+STORE, 140249663676416, 140249710600191, -+STORE, 140249663668224, 140249710600191, -+STORE, 140249663651840, 140249710600191, -+STORE, 140249663643648, 140249710600191, -+STORE, 140249663627264, 140249710600191, -+STORE, 33853440, 38526975, -+STORE, 140249663619072, 140249710600191, -+STORE, 140249663602688, 140249710600191, -+STORE, 140249661505536, 140249710600191, -+STORE, 140249661497344, 140249710600191, -+STORE, 140249661464576, 140249710600191, -+STORE, 140249661456384, 140249710600191, -+STORE, 140249661440000, 140249710600191, -+STORE, 140249661431808, 140249710600191, -+STORE, 140249661415424, 140249710600191, -+STORE, 140249661407232, 140249710600191, -+STORE, 140249661390848, 140249710600191, -+STORE, 140249661382656, 140249710600191, -+STORE, 140249661349888, 140249710600191, -+STORE, 140249661341696, 140249710600191, -+STORE, 140249661325312, 140249710600191, -+STORE, 140249661317120, 140249710600191, -+STORE, 140249661300736, 140249710600191, -+STORE, 140249661292544, 140249710600191, -+STORE, 140249661276160, 140249710600191, -+STORE, 140249661267968, 140249710600191, -+STORE, 140249661235200, 140249710600191, -+STORE, 140249661227008, 140249710600191, -+STORE, 140249661210624, 140249710600191, -+STORE, 140249661202432, 140249710600191, -+STORE, 140249661186048, 140249710600191, -+STORE, 140249661177856, 140249710600191, -+STORE, 140249661161472, 140249710600191, -+STORE, 140249661153280, 140249710600191, -+STORE, 140249661120512, 140249710600191, -+STORE, 140249661112320, 140249710600191, -+STORE, 140249661095936, 140249710600191, -+STORE, 140249661087744, 140249710600191, -+STORE, 140249661071360, 140249710600191, -+STORE, 140249661063168, 140249710600191, -+STORE, 140249661046784, 140249710600191, -+STORE, 140249661038592, 140249710600191, -+STORE, 140249661005824, 140249710600191, -+STORE, 140249660997632, 140249710600191, -+STORE, 140249660981248, 140249710600191, -+STORE, 140249660973056, 140249710600191, -+STORE, 140249660956672, 140249710600191, -+STORE, 140249660948480, 140249710600191, -+STORE, 140249660932096, 140249710600191, -+STORE, 140249660923904, 140249710600191, -+STORE, 140249660891136, 140249710600191, -+STORE, 140249660882944, 140249710600191, -+STORE, 140249660866560, 140249710600191, -+STORE, 140249660858368, 140249710600191, -+STORE, 140249660841984, 140249710600191, -+STORE, 140249660833792, 140249710600191, -+STORE, 140249660817408, 140249710600191, -+STORE, 140249660809216, 140249710600191, -+STORE, 140249660776448, 140249710600191, -+STORE, 140249660768256, 140249710600191, -+STORE, 140249660751872, 140249710600191, -+STORE, 140249660743680, 140249710600191, -+STORE, 140249660727296, 140249710600191, -+STORE, 140249660719104, 140249710600191, -+STORE, 140249660702720, 140249710600191, -+STORE, 140249660694528, 140249710600191, -+STORE, 140249660661760, 140249710600191, -+STORE, 140249660653568, 140249710600191, -+STORE, 140249660637184, 140249710600191, -+STORE, 140249660628992, 140249710600191, -+STORE, 140249660612608, 140249710600191, -+STORE, 140249660604416, 140249710600191, -+STORE, 140249660588032, 140249710600191, -+STORE, 140249660579840, 140249710600191, -+STORE, 140249660547072, 140249710600191, -+STORE, 140249660538880, 140249710600191, -+STORE, 140249660522496, 140249710600191, -+STORE, 140249660514304, 140249710600191, -+STORE, 140249660497920, 140249710600191, -+STORE, 140249660489728, 140249710600191, -+STORE, 140249660473344, 140249710600191, -+STORE, 140249660465152, 140249710600191, -+STORE, 140249660432384, 140249710600191, -+STORE, 140249660424192, 140249710600191, -+STORE, 140249660407808, 140249710600191, -+STORE, 140249660399616, 140249710600191, -+STORE, 140249660383232, 140249710600191, -+STORE, 140249660375040, 140249710600191, -+STORE, 140249660358656, 140249710600191, -+STORE, 140249660350464, 140249710600191, -+STORE, 140249660317696, 140249710600191, -+STORE, 140249660309504, 140249710600191, -+STORE, 140249660293120, 140249710600191, -+STORE, 140249660284928, 140249710600191, -+STORE, 140249660268544, 140249710600191, -+STORE, 140249660260352, 140249710600191, -+STORE, 140249660243968, 140249710600191, -+STORE, 140249660235776, 140249710600191, -+STORE, 140249660203008, 140249710600191, -+STORE, 140249660194816, 140249710600191, -+STORE, 140249660178432, 140249710600191, -+STORE, 140249660170240, 140249710600191, -+STORE, 140249660153856, 140249710600191, -+STORE, 140249660145664, 140249710600191, -+STORE, 140249660129280, 140249710600191, -+STORE, 140249660121088, 140249710600191, -+STORE, 140249660088320, 140249710600191, -+STORE, 140249660080128, 140249710600191, -+STORE, 140249660063744, 140249710600191, -+STORE, 140249660055552, 140249710600191, -+STORE, 140249660039168, 140249710600191, -+STORE, 140249660030976, 140249710600191, -+STORE, 140249660014592, 140249710600191, -+STORE, 140249660006400, 140249710600191, -+STORE, 140249659973632, 140249710600191, -+STORE, 140249659965440, 140249710600191, -+STORE, 140249659949056, 140249710600191, -+STORE, 140249659940864, 140249710600191, -+STORE, 140249659924480, 140249710600191, -+STORE, 140249659916288, 140249710600191, -+STORE, 140249659899904, 140249710600191, -+STORE, 140249659891712, 140249710600191, -+STORE, 140249659858944, 140249710600191, -+STORE, 140249659850752, 140249710600191, -+STORE, 140249659834368, 140249710600191, -+STORE, 140249659826176, 140249710600191, -+STORE, 140249659809792, 140249710600191, -+STORE, 140249659801600, 140249710600191, -+STORE, 140249659785216, 140249710600191, -+STORE, 140249657688064, 140249710600191, -+STORE, 140249657679872, 140249710600191, -+STORE, 140249657647104, 140249710600191, -+STORE, 140249657638912, 140249710600191, -+STORE, 140249657622528, 140249710600191, -+STORE, 140249657614336, 140249710600191, -+STORE, 140249657597952, 140249710600191, -+STORE, 140249657589760, 140249710600191, -+STORE, 140249657573376, 140249710600191, -+STORE, 140249657565184, 140249710600191, -+STORE, 140249657532416, 140249710600191, -+STORE, 140249657524224, 140249710600191, -+STORE, 140249657507840, 140249710600191, -+STORE, 140249657499648, 140249710600191, -+STORE, 140249657483264, 140249710600191, -+STORE, 140249657475072, 140249710600191, -+STORE, 140249657458688, 140249710600191, -+STORE, 140249657450496, 140249710600191, -+STORE, 140249657417728, 140249710600191, -+STORE, 140249657409536, 140249710600191, -+STORE, 140249657393152, 140249710600191, -+STORE, 140249657384960, 140249710600191, -+STORE, 140249657368576, 140249710600191, -+STORE, 140249657360384, 140249710600191, -+STORE, 140249657344000, 140249710600191, -+STORE, 140249657335808, 140249710600191, -+STORE, 140249657303040, 140249710600191, -+STORE, 140249657294848, 140249710600191, -+STORE, 140249657278464, 140249710600191, -+STORE, 140249657270272, 140249710600191, -+STORE, 140249657253888, 140249710600191, -+STORE, 140249657245696, 140249710600191, -+STORE, 140249657229312, 140249710600191, -+STORE, 140249657221120, 140249710600191, -+STORE, 140249657188352, 140249710600191, -+STORE, 140249657180160, 140249710600191, -+STORE, 140249657163776, 140249710600191, -+STORE, 140249657155584, 140249710600191, -+STORE, 140249657139200, 140249710600191, -+STORE, 140249657131008, 140249710600191, -+STORE, 140249657114624, 140249710600191, -+STORE, 140249657106432, 140249710600191, -+STORE, 140249657073664, 140249710600191, -+STORE, 140249657065472, 140249710600191, -+STORE, 140249657049088, 140249710600191, -+STORE, 140249657040896, 140249710600191, -+STORE, 140249657024512, 140249710600191, -+STORE, 140249657016320, 140249710600191, -+STORE, 140249656999936, 140249710600191, -+STORE, 140249656991744, 140249710600191, -+STORE, 140249656958976, 140249710600191, -+STORE, 140249656950784, 140249710600191, -+STORE, 140249656934400, 140249710600191, -+STORE, 140249656926208, 140249710600191, -+STORE, 140249656909824, 140249710600191, -+STORE, 140249656901632, 140249710600191, -+STORE, 140249656885248, 140249710600191, -+STORE, 140249656877056, 140249710600191, -+STORE, 140249656844288, 140249710600191, -+STORE, 140249656836096, 140249710600191, -+STORE, 140249656819712, 140249710600191, -+STORE, 140249656811520, 140249710600191, -+STORE, 140249656795136, 140249710600191, -+STORE, 33853440, 38662143, -+STORE, 140249656786944, 140249710600191, -+STORE, 140249656770560, 140249710600191, -+STORE, 140249656762368, 140249710600191, -+STORE, 140249656729600, 140249710600191, -+STORE, 140249656721408, 140249710600191, -+STORE, 140249656705024, 140249710600191, -+STORE, 140249656696832, 140249710600191, -+STORE, 140249656680448, 140249710600191, -+STORE, 140249656672256, 140249710600191, -+STORE, 140249656655872, 140249710600191, -+STORE, 140249656647680, 140249710600191, -+STORE, 140249656614912, 140249710600191, -+STORE, 140249656606720, 140249710600191, -+STORE, 140249656590336, 140249710600191, -+STORE, 140249656582144, 140249710600191, -+STORE, 140249656565760, 140249710600191, -+STORE, 140249656557568, 140249710600191, -+STORE, 140249656541184, 140249710600191, -+STORE, 140249656532992, 140249710600191, -+STORE, 140249656500224, 140249710600191, -+STORE, 140249656492032, 140249710600191, -+STORE, 140249656475648, 140249710600191, -+STORE, 140249656467456, 140249710600191, -+STORE, 140249656451072, 140249710600191, -+STORE, 140249656442880, 140249710600191, -+STORE, 140249656426496, 140249710600191, -+STORE, 140249656418304, 140249710600191, -+STORE, 140249656385536, 140249710600191, -+STORE, 140249656377344, 140249710600191, -+STORE, 140249656360960, 140249710600191, -+STORE, 140249656352768, 140249710600191, -+STORE, 140249656336384, 140249710600191, -+STORE, 140249656328192, 140249710600191, -+STORE, 140249656311808, 140249710600191, -+STORE, 140249656303616, 140249710600191, -+STORE, 140249656270848, 140249710600191, -+STORE, 140249656262656, 140249710600191, -+STORE, 140249656246272, 140249710600191, -+STORE, 140249656238080, 140249710600191, -+STORE, 140249656221696, 140249710600191, -+STORE, 140249656213504, 140249710600191, -+STORE, 140249656197120, 140249710600191, -+STORE, 140249656188928, 140249710600191, -+STORE, 140249656156160, 140249710600191, -+STORE, 140249656147968, 140249710600191, -+STORE, 140249656131584, 140249710600191, -+STORE, 140249656123392, 140249710600191, -+STORE, 140249656107008, 140249710600191, -+STORE, 140249656098816, 140249710600191, -+STORE, 140249656082432, 140249710600191, -+STORE, 140249656074240, 140249710600191, -+STORE, 140249656041472, 140249710600191, -+STORE, 140249656033280, 140249710600191, -+STORE, 140249656016896, 140249710600191, -+STORE, 140249656008704, 140249710600191, -+STORE, 140249655992320, 140249710600191, -+STORE, 140249655984128, 140249710600191, -+STORE, 140249655967744, 140249710600191, -+STORE, 140249653870592, 140249710600191, -+STORE, 140249653862400, 140249710600191, -+STORE, 140249653829632, 140249710600191, -+STORE, 140249653821440, 140249710600191, -+STORE, 140249653805056, 140249710600191, -+STORE, 140249653796864, 140249710600191, -+STORE, 140249653780480, 140249710600191, -+STORE, 140249653772288, 140249710600191, -+STORE, 140249653755904, 140249710600191, -+STORE, 140249652703232, 140249710600191, -+SNULL, 140249682087935, 140249710600191, -+STORE, 140249652703232, 140249682087935, -+STORE, 140249682087936, 140249710600191, -+ }; -+ -+ unsigned long set26[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140729464770560, 140737488351231, -+SNULL, 140729464774655, 140737488351231, -+STORE, 140729464770560, 140729464774655, -+STORE, 140729464639488, 140729464774655, -+STORE, 4194304, 5066751, -+STORE, 7159808, 7172095, -+STORE, 7172096, 7180287, -+STORE, 140729465114624, 140729465118719, -+STORE, 140729465102336, 140729465114623, -+STORE, 30867456, 30875647, -+STORE, 30867456, 31010815, -+STORE, 140109040988160, 140109042671615, -+STORE, 140109040959488, 140109040988159, -+STORE, 140109040943104, 140109040959487, -+ERASE, 140109040943104, 140109040959487, -+STORE, 140109040840704, 140109040959487, -+ERASE, 140109040840704, 140109040959487, -+STORE, 140109040951296, 140109040959487, -+ERASE, 140109040951296, 140109040959487, -+STORE, 140109040955392, 140109040959487, -+ERASE, 140109040955392, 140109040959487, -+ }; -+ unsigned long set27[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140726128070656, 140737488351231, -+SNULL, 140726128074751, 140737488351231, -+STORE, 140726128070656, 140726128074751, -+STORE, 140726127939584, 140726128074751, -+STORE, 94478497189888, 94478499303423, -+SNULL, 94478497202175, 94478499303423, -+STORE, 94478497189888, 94478497202175, -+STORE, 94478497202176, 94478499303423, -+ERASE, 94478497202176, 94478499303423, -+STORE, 94478499295232, 94478499303423, -+STORE, 140415605723136, 140415607975935, -+SNULL, 140415605866495, 140415607975935, -+STORE, 140415605723136, 140415605866495, -+STORE, 140415605866496, 140415607975935, -+ERASE, 140415605866496, 140415607975935, -+STORE, 140415607963648, 140415607971839, -+STORE, 140415607971840, 140415607975935, -+STORE, 140726130024448, 140726130028543, -+STORE, 140726130012160, 140726130024447, -+STORE, 140415607934976, 140415607963647, -+STORE, 140415607926784, 140415607934975, -+STORE, 140415603245056, 140415605723135, -+SNULL, 140415603245056, 140415603613695, -+STORE, 140415603613696, 140415605723135, -+STORE, 140415603245056, 140415603613695, -+SNULL, 140415605710847, 140415605723135, -+STORE, 140415603613696, 140415605710847, -+STORE, 140415605710848, 140415605723135, -+ERASE, 140415605710848, 140415605723135, -+STORE, 140415605710848, 140415605723135, -+STORE, 140415599370240, 140415603245055, -+SNULL, 140415599370240, 140415601111039, -+STORE, 140415601111040, 140415603245055, -+STORE, 140415599370240, 140415601111039, -+SNULL, 140415603208191, 140415603245055, -+STORE, 140415601111040, 140415603208191, -+STORE, 140415603208192, 140415603245055, -+ERASE, 140415603208192, 140415603245055, -+STORE, 140415603208192, 140415603245055, -+STORE, 140415595692032, 140415599370239, -+SNULL, 140415595692032, 140415597207551, -+STORE, 140415597207552, 140415599370239, -+STORE, 140415595692032, 140415597207551, -+SNULL, 140415599304703, 140415599370239, -+STORE, 140415597207552, 140415599304703, -+STORE, 140415599304704, 140415599370239, -+SNULL, 140415599304704, 140415599353855, -+STORE, 140415599353856, 140415599370239, -+STORE, 140415599304704, 140415599353855, -+ERASE, 140415599304704, 140415599353855, -+STORE, 140415599304704, 140415599353855, -+ERASE, 140415599353856, 140415599370239, -+STORE, 140415599353856, 140415599370239, -+STORE, 140415593500672, 140415595692031, -+SNULL, 140415593500672, 140415593590783, -+STORE, 140415593590784, 140415595692031, -+STORE, 140415593500672, 140415593590783, -+SNULL, 140415595683839, 140415595692031, -+STORE, 140415593590784, 140415595683839, -+STORE, 140415595683840, 140415595692031, -+ERASE, 140415595683840, 140415595692031, -+STORE, 140415595683840, 140415595692031, -+STORE, 140415589703680, 140415593500671, -+SNULL, 140415589703680, 140415591362559, -+STORE, 140415591362560, 140415593500671, -+STORE, 140415589703680, 140415591362559, -+SNULL, 140415593459711, 140415593500671, -+STORE, 140415591362560, 140415593459711, -+STORE, 140415593459712, 140415593500671, -+SNULL, 140415593459712, 140415593484287, -+STORE, 140415593484288, 140415593500671, -+STORE, 140415593459712, 140415593484287, -+ERASE, 140415593459712, 140415593484287, -+STORE, 140415593459712, 140415593484287, -+ERASE, 140415593484288, 140415593500671, -+STORE, 140415593484288, 140415593500671, -+STORE, 140415587590144, 140415589703679, -+SNULL, 140415587590144, 140415587602431, -+STORE, 140415587602432, 140415589703679, -+STORE, 140415587590144, 140415587602431, -+SNULL, 140415589695487, 140415589703679, -+STORE, 140415587602432, 140415589695487, -+STORE, 140415589695488, 140415589703679, -+ERASE, 140415589695488, 140415589703679, -+STORE, 140415589695488, 140415589703679, -+STORE, 140415607918592, 140415607934975, -+STORE, 140415585398784, 140415587590143, -+SNULL, 140415585398784, 140415585480703, -+STORE, 140415585480704, 140415587590143, -+STORE, 140415585398784, 140415585480703, -+SNULL, 140415587573759, 140415587590143, -+STORE, 140415585480704, 140415587573759, -+STORE, 140415587573760, 140415587590143, -+SNULL, 140415587573760, 140415587581951, -+STORE, 140415587581952, 140415587590143, -+STORE, 140415587573760, 140415587581951, -+ERASE, 140415587573760, 140415587581951, -+STORE, 140415587573760, 140415587581951, -+ERASE, 140415587581952, 140415587590143, -+STORE, 140415587581952, 140415587590143, -+STORE, 140415583182848, 140415585398783, -+SNULL, 140415583182848, 140415583281151, -+STORE, 140415583281152, 140415585398783, -+STORE, 140415583182848, 140415583281151, -+SNULL, 140415585374207, 140415585398783, -+STORE, 140415583281152, 140415585374207, -+STORE, 140415585374208, 140415585398783, -+SNULL, 140415585374208, 140415585382399, -+STORE, 140415585382400, 140415585398783, -+STORE, 140415585374208, 140415585382399, -+ERASE, 140415585374208, 140415585382399, -+STORE, 140415585374208, 140415585382399, -+ERASE, 140415585382400, 140415585398783, -+STORE, 140415585382400, 140415585398783, -+STORE, 140415580979200, 140415583182847, -+SNULL, 140415580979200, 140415581081599, -+STORE, 140415581081600, 140415583182847, -+STORE, 140415580979200, 140415581081599, -+SNULL, 140415583174655, 140415583182847, -+STORE, 140415581081600, 140415583174655, -+STORE, 140415583174656, 140415583182847, -+ERASE, 140415583174656, 140415583182847, -+STORE, 140415583174656, 140415583182847, -+STORE, 140415578816512, 140415580979199, -+SNULL, 140415578816512, 140415578877951, -+STORE, 140415578877952, 140415580979199, -+STORE, 140415578816512, 140415578877951, -+SNULL, 140415580971007, 140415580979199, -+STORE, 140415578877952, 140415580971007, -+STORE, 140415580971008, 140415580979199, -+ERASE, 140415580971008, 140415580979199, -+STORE, 140415580971008, 140415580979199, -+STORE, 140415576563712, 140415578816511, -+SNULL, 140415576563712, 140415576715263, -+STORE, 140415576715264, 140415578816511, -+STORE, 140415576563712, 140415576715263, -+SNULL, 140415578808319, 140415578816511, -+STORE, 140415576715264, 140415578808319, -+STORE, 140415578808320, 140415578816511, -+ERASE, 140415578808320, 140415578816511, -+STORE, 140415578808320, 140415578816511, -+STORE, 140415574392832, 140415576563711, -+SNULL, 140415574392832, 140415574462463, -+STORE, 140415574462464, 140415576563711, -+STORE, 140415574392832, 140415574462463, -+SNULL, 140415576555519, 140415576563711, -+STORE, 140415574462464, 140415576555519, -+STORE, 140415576555520, 140415576563711, -+ERASE, 140415576555520, 140415576563711, -+STORE, 140415576555520, 140415576563711, -+STORE, 140415607910400, 140415607934975, -+STORE, 140415571230720, 140415574392831, -+SNULL, 140415571230720, 140415572291583, -+STORE, 140415572291584, 140415574392831, -+STORE, 140415571230720, 140415572291583, -+SNULL, 140415574384639, 140415574392831, -+STORE, 140415572291584, 140415574384639, -+STORE, 140415574384640, 140415574392831, -+ERASE, 140415574384640, 140415574392831, -+STORE, 140415574384640, 140415574392831, -+STORE, 140415607902208, 140415607934975, -+SNULL, 140415593476095, 140415593484287, -+STORE, 140415593459712, 140415593476095, -+STORE, 140415593476096, 140415593484287, -+SNULL, 140415574388735, 140415574392831, -+STORE, 140415574384640, 140415574388735, -+STORE, 140415574388736, 140415574392831, -+SNULL, 140415576559615, 140415576563711, -+STORE, 140415576555520, 140415576559615, -+STORE, 140415576559616, 140415576563711, -+SNULL, 140415589699583, 140415589703679, -+STORE, 140415589695488, 140415589699583, -+STORE, 140415589699584, 140415589703679, -+SNULL, 140415585378303, 140415585382399, -+STORE, 140415585374208, 140415585378303, -+STORE, 140415585378304, 140415585382399, -+SNULL, 140415578812415, 140415578816511, -+STORE, 140415578808320, 140415578812415, -+STORE, 140415578812416, 140415578816511, -+SNULL, 140415580975103, 140415580979199, -+STORE, 140415580971008, 140415580975103, -+STORE, 140415580975104, 140415580979199, -+SNULL, 140415583178751, 140415583182847, -+STORE, 140415583174656, 140415583178751, -+STORE, 140415583178752, 140415583182847, -+SNULL, 140415587577855, 140415587581951, -+STORE, 140415587573760, 140415587577855, -+STORE, 140415587577856, 140415587581951, -+SNULL, 140415595687935, 140415595692031, -+STORE, 140415595683840, 140415595687935, -+STORE, 140415595687936, 140415595692031, -+STORE, 140415607894016, 140415607934975, -+SNULL, 140415599345663, 140415599353855, -+STORE, 140415599304704, 140415599345663, -+STORE, 140415599345664, 140415599353855, -+SNULL, 140415603240959, 140415603245055, -+STORE, 140415603208192, 140415603240959, -+STORE, 140415603240960, 140415603245055, -+SNULL, 140415605719039, 140415605723135, -+STORE, 140415605710848, 140415605719039, -+STORE, 140415605719040, 140415605723135, -+SNULL, 94478499299327, 94478499303423, -+STORE, 94478499295232, 94478499299327, -+STORE, 94478499299328, 94478499303423, -+SNULL, 140415607967743, 140415607971839, -+STORE, 140415607963648, 140415607967743, -+STORE, 140415607967744, 140415607971839, -+ERASE, 140415607934976, 140415607963647, -+STORE, 94478511173632, 94478511378431, -+STORE, 140415606210560, 140415607894015, -+STORE, 140415607934976, 140415607963647, -+STORE, 94478511173632, 94478511513599, -+STORE, 94478511173632, 94478511648767, -+SNULL, 94478511615999, 94478511648767, -+STORE, 94478511173632, 94478511615999, -+STORE, 94478511616000, 94478511648767, -+ERASE, 94478511616000, 94478511648767, -+STORE, 94478511173632, 94478511751167, -+SNULL, 94478511747071, 94478511751167, -+STORE, 94478511173632, 94478511747071, -+STORE, 94478511747072, 94478511751167, -+ERASE, 94478511747072, 94478511751167, -+STORE, 94478511173632, 94478511882239, -+SNULL, 94478511878143, 94478511882239, -+STORE, 94478511173632, 94478511878143, -+STORE, 94478511878144, 94478511882239, -+ERASE, 94478511878144, 94478511882239, -+STORE, 94478511173632, 94478512013311, -+SNULL, 94478512009215, 94478512013311, -+STORE, 94478511173632, 94478512009215, -+STORE, 94478512009216, 94478512013311, -+ERASE, 94478512009216, 94478512013311, -+STORE, 94478511173632, 94478512144383, -+STORE, 94478511173632, 94478512279551, -+STORE, 140415606181888, 140415606210559, -+STORE, 140415569100800, 140415571230719, -+SNULL, 140415569100800, 140415569129471, -+STORE, 140415569129472, 140415571230719, -+STORE, 140415569100800, 140415569129471, -+SNULL, 140415571222527, 140415571230719, -+STORE, 140415569129472, 140415571222527, -+STORE, 140415571222528, 140415571230719, -+ERASE, 140415571222528, 140415571230719, -+STORE, 140415571222528, 140415571230719, -+STORE, 140415566905344, 140415569100799, -+SNULL, 140415566905344, 140415566987263, -+STORE, 140415566987264, 140415569100799, -+STORE, 140415566905344, 140415566987263, -+SNULL, 140415569084415, 140415569100799, -+STORE, 140415566987264, 140415569084415, -+STORE, 140415569084416, 140415569100799, -+SNULL, 140415569084416, 140415569092607, -+STORE, 140415569092608, 140415569100799, -+STORE, 140415569084416, 140415569092607, -+ERASE, 140415569084416, 140415569092607, -+STORE, 140415569084416, 140415569092607, -+ERASE, 140415569092608, 140415569100799, -+STORE, 140415569092608, 140415569100799, -+SNULL, 140415569088511, 140415569092607, -+STORE, 140415569084416, 140415569088511, -+STORE, 140415569088512, 140415569092607, -+SNULL, 140415571226623, 140415571230719, -+STORE, 140415571222528, 140415571226623, -+STORE, 140415571226624, 140415571230719, -+ERASE, 140415606181888, 140415606210559, -+STORE, 140415606181888, 140415606210559, -+STORE, 140415564759040, 140415566905343, -+SNULL, 140415564759040, 140415564804095, -+STORE, 140415564804096, 140415566905343, -+STORE, 140415564759040, 140415564804095, -+SNULL, 140415566897151, 140415566905343, -+STORE, 140415564804096, 140415566897151, -+STORE, 140415566897152, 140415566905343, -+ERASE, 140415566897152, 140415566905343, -+STORE, 140415566897152, 140415566905343, -+STORE, 140415562588160, 140415564759039, -+SNULL, 140415562588160, 140415562629119, -+STORE, 140415562629120, 140415564759039, -+STORE, 140415562588160, 140415562629119, -+SNULL, 140415564726271, 140415564759039, -+STORE, 140415562629120, 140415564726271, -+STORE, 140415564726272, 140415564759039, -+SNULL, 140415564726272, 140415564734463, -+STORE, 140415564734464, 140415564759039, -+STORE, 140415564726272, 140415564734463, -+ERASE, 140415564726272, 140415564734463, -+STORE, 140415564726272, 140415564734463, -+ERASE, 140415564734464, 140415564759039, -+STORE, 140415564734464, 140415564759039, -+SNULL, 140415564730367, 140415564734463, -+STORE, 140415564726272, 140415564730367, -+STORE, 140415564730368, 140415564734463, -+SNULL, 140415566901247, 140415566905343, -+STORE, 140415566897152, 140415566901247, -+STORE, 140415566901248, 140415566905343, -+ERASE, 140415606181888, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415605944320, 140415606210559, -+ERASE, 140415605944320, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 94478511173632, 94478512414719, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 140415606206464, 140415606210559, -+ERASE, 140415606206464, 140415606210559, -+STORE, 94478511173632, 94478512652287, -+STORE, 94478511173632, 94478512787455, -+STORE, 94478511173632, 94478512922623, -+STORE, 94478511173632, 94478513057791, -+STORE, 140415537422336, 140415562588159, -+STORE, 94478511173632, 94478513192959, -+STORE, 94478511173632, 94478513356799, -+STORE, 94478511173632, 94478513491967, -+STORE, 94478511173632, 94478513627135, -+STORE, 94478511173632, 94478513790975, -+STORE, 94478511173632, 94478513926143, -+STORE, 94478511173632, 94478514061311, -+STORE, 94478511173632, 94478514196479, -+STORE, 94478511173632, 94478514331647, -+STORE, 94478511173632, 94478514606079, -+STORE, 94478511173632, 94478514741247, -+STORE, 94478511173632, 94478514876415, -+STORE, 94478511173632, 94478515011583, -+STORE, 94478511173632, 94478515146751, -+STORE, 94478511173632, 94478515281919, -+STORE, 94478511173632, 94478515474431, -+STORE, 94478511173632, 94478515609599, -+STORE, 94478511173632, 94478515744767, -+STORE, 140415536922624, 140415562588159, -+STORE, 94478511173632, 94478515879935, -+STORE, 94478511173632, 94478516015103, -+STORE, 94478511173632, 94478516150271, -+STORE, 94478511173632, 94478516285439, -+STORE, 94478511173632, 94478516420607, -+STORE, 94478511173632, 94478516555775, -+STORE, 94478511173632, 94478516690943, -+STORE, 94478511173632, 94478516826111, -+STORE, 94478511173632, 94478516961279, -+STORE, 94478511173632, 94478517231615, -+STORE, 94478511173632, 94478517366783, -+STORE, 94478511173632, 94478517501951, -+STORE, 94478511173632, 94478517637119, -+STORE, 94478511173632, 94478517772287, -+STORE, 94478511173632, 94478517907455, -+STORE, 94478511173632, 94478518042623, -+STORE, 94478511173632, 94478518177791, -+STORE, 94478511173632, 94478518312959, -+STORE, 94478511173632, 94478518448127, -+STORE, 140415535910912, 140415562588159, -+SNULL, 140415536922623, 140415562588159, -+STORE, 140415535910912, 140415536922623, -+STORE, 140415536922624, 140415562588159, -+SNULL, 140415536922624, 140415537422335, -+STORE, 140415537422336, 140415562588159, -+STORE, 140415536922624, 140415537422335, -+ERASE, 140415536922624, 140415537422335, -+STORE, 94478511173632, 94478518583295, -+STORE, 94478511173632, 94478518718463, -+STORE, 94478511173632, 94478518853631, -+STORE, 94478511173632, 94478518988799, -+STORE, 94478511173632, 94478519123967, -+STORE, 94478511173632, 94478519259135, -+STORE, 140415509696512, 140415535910911, -+ERASE, 140415537422336, 140415562588159, -+STORE, 140415482433536, 140415509696511, -+ }; -+ unsigned long set28[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140722475622400, 140737488351231, -+SNULL, 140722475626495, 140737488351231, -+STORE, 140722475622400, 140722475626495, -+STORE, 140722475491328, 140722475626495, -+STORE, 93865834291200, 93865836548095, -+SNULL, 93865834422271, 93865836548095, -+STORE, 93865834291200, 93865834422271, -+STORE, 93865834422272, 93865836548095, -+ERASE, 93865834422272, 93865836548095, -+STORE, 93865836519424, 93865836527615, -+STORE, 93865836527616, 93865836548095, -+STORE, 139918411104256, 139918413357055, -+SNULL, 139918411247615, 139918413357055, -+STORE, 139918411104256, 139918411247615, -+STORE, 139918411247616, 139918413357055, -+ERASE, 139918411247616, 139918413357055, -+STORE, 139918413344768, 139918413352959, -+STORE, 139918413352960, 139918413357055, -+STORE, 140722476642304, 140722476646399, -+STORE, 140722476630016, 140722476642303, -+STORE, 139918413316096, 139918413344767, -+STORE, 139918413307904, 139918413316095, -+STORE, 139918408888320, 139918411104255, -+SNULL, 139918408888320, 139918408986623, -+STORE, 139918408986624, 139918411104255, -+STORE, 139918408888320, 139918408986623, -+SNULL, 139918411079679, 139918411104255, -+STORE, 139918408986624, 139918411079679, -+STORE, 139918411079680, 139918411104255, -+SNULL, 139918411079680, 139918411087871, -+STORE, 139918411087872, 139918411104255, -+STORE, 139918411079680, 139918411087871, -+ERASE, 139918411079680, 139918411087871, -+STORE, 139918411079680, 139918411087871, -+ERASE, 139918411087872, 139918411104255, -+STORE, 139918411087872, 139918411104255, -+STORE, 139918405091328, 139918408888319, -+SNULL, 139918405091328, 139918406750207, -+STORE, 139918406750208, 139918408888319, -+STORE, 139918405091328, 139918406750207, -+SNULL, 139918408847359, 139918408888319, -+STORE, 139918406750208, 139918408847359, -+STORE, 139918408847360, 139918408888319, -+SNULL, 139918408847360, 139918408871935, -+STORE, 139918408871936, 139918408888319, -+STORE, 139918408847360, 139918408871935, -+ERASE, 139918408847360, 139918408871935, -+STORE, 139918408847360, 139918408871935, -+ERASE, 139918408871936, 139918408888319, -+STORE, 139918408871936, 139918408888319, -+STORE, 139918413299712, 139918413316095, -+SNULL, 139918408863743, 139918408871935, -+STORE, 139918408847360, 139918408863743, -+STORE, 139918408863744, 139918408871935, -+SNULL, 139918411083775, 139918411087871, -+STORE, 139918411079680, 139918411083775, -+STORE, 139918411083776, 139918411087871, -+SNULL, 93865836523519, 93865836527615, -+STORE, 93865836519424, 93865836523519, -+STORE, 93865836523520, 93865836527615, -+SNULL, 139918413348863, 139918413352959, -+STORE, 139918413344768, 139918413348863, -+STORE, 139918413348864, 139918413352959, -+ERASE, 139918413316096, 139918413344767, -+STORE, 93865848528896, 93865848664063, -+ }; -+ unsigned long set29[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140734467944448, 140737488351231, -+SNULL, 140734467948543, 140737488351231, -+STORE, 140734467944448, 140734467948543, -+STORE, 140734467813376, 140734467948543, -+STORE, 94880407924736, 94880410177535, -+SNULL, 94880408055807, 94880410177535, -+STORE, 94880407924736, 94880408055807, -+STORE, 94880408055808, 94880410177535, -+ERASE, 94880408055808, 94880410177535, -+STORE, 94880410148864, 94880410157055, -+STORE, 94880410157056, 94880410177535, -+STORE, 140143367815168, 140143370067967, -+SNULL, 140143367958527, 140143370067967, -+STORE, 140143367815168, 140143367958527, -+STORE, 140143367958528, 140143370067967, -+ERASE, 140143367958528, 140143370067967, -+STORE, 140143370055680, 140143370063871, -+STORE, 140143370063872, 140143370067967, -+STORE, 140734468329472, 140734468333567, -+STORE, 140734468317184, 140734468329471, -+STORE, 140143370027008, 140143370055679, -+STORE, 140143370018816, 140143370027007, -+STORE, 140143365599232, 140143367815167, -+SNULL, 140143365599232, 140143365697535, -+STORE, 140143365697536, 140143367815167, -+STORE, 140143365599232, 140143365697535, -+SNULL, 140143367790591, 140143367815167, -+STORE, 140143365697536, 140143367790591, -+STORE, 140143367790592, 140143367815167, -+SNULL, 140143367790592, 140143367798783, -+STORE, 140143367798784, 140143367815167, -+STORE, 140143367790592, 140143367798783, -+ERASE, 140143367790592, 140143367798783, -+STORE, 140143367790592, 140143367798783, -+ERASE, 140143367798784, 140143367815167, -+STORE, 140143367798784, 140143367815167, -+STORE, 140143361802240, 140143365599231, -+SNULL, 140143361802240, 140143363461119, -+STORE, 140143363461120, 140143365599231, -+STORE, 140143361802240, 140143363461119, -+SNULL, 140143365558271, 140143365599231, -+STORE, 140143363461120, 140143365558271, -+STORE, 140143365558272, 140143365599231, -+SNULL, 140143365558272, 140143365582847, -+STORE, 140143365582848, 140143365599231, -+STORE, 140143365558272, 140143365582847, -+ERASE, 140143365558272, 140143365582847, -+STORE, 140143365558272, 140143365582847, -+ERASE, 140143365582848, 140143365599231, -+STORE, 140143365582848, 140143365599231, -+STORE, 140143370010624, 140143370027007, -+SNULL, 140143365574655, 140143365582847, -+STORE, 140143365558272, 140143365574655, -+STORE, 140143365574656, 140143365582847, -+SNULL, 140143367794687, 140143367798783, -+STORE, 140143367790592, 140143367794687, -+STORE, 140143367794688, 140143367798783, -+SNULL, 94880410152959, 94880410157055, -+STORE, 94880410148864, 94880410152959, -+STORE, 94880410152960, 94880410157055, -+SNULL, 140143370059775, 140143370063871, -+STORE, 140143370055680, 140143370059775, -+STORE, 140143370059776, 140143370063871, -+ERASE, 140143370027008, 140143370055679, -+STORE, 94880442400768, 94880442535935, -+STORE, 140143353409536, 140143361802239, -+SNULL, 140143353413631, 140143361802239, -+STORE, 140143353409536, 140143353413631, -+STORE, 140143353413632, 140143361802239, -+STORE, 140143345016832, 140143353409535, -+STORE, 140143210799104, 140143345016831, -+SNULL, 140143210799104, 140143239364607, -+STORE, 140143239364608, 140143345016831, -+STORE, 140143210799104, 140143239364607, -+ERASE, 140143210799104, 140143239364607, -+SNULL, 140143306473471, 140143345016831, -+STORE, 140143239364608, 140143306473471, -+STORE, 140143306473472, 140143345016831, -+ERASE, 140143306473472, 140143345016831, -+SNULL, 140143239499775, 140143306473471, -+STORE, 140143239364608, 140143239499775, -+STORE, 140143239499776, 140143306473471, -+SNULL, 140143345020927, 140143353409535, -+STORE, 140143345016832, 140143345020927, -+STORE, 140143345020928, 140143353409535, -+STORE, 140143336624128, 140143345016831, -+SNULL, 140143336628223, 140143345016831, -+STORE, 140143336624128, 140143336628223, -+STORE, 140143336628224, 140143345016831, -+STORE, 140143328231424, 140143336624127, -+SNULL, 140143328235519, 140143336624127, -+STORE, 140143328231424, 140143328235519, -+STORE, 140143328235520, 140143336624127, -+STORE, 140143319838720, 140143328231423, -+SNULL, 140143319842815, 140143328231423, -+STORE, 140143319838720, 140143319842815, -+STORE, 140143319842816, 140143328231423, -+STORE, 140143311446016, 140143319838719, -+STORE, 140143105146880, 140143239364607, -+STORE, 140143096754176, 140143105146879, -+STORE, 140143029645312, 140143096754175, -+ERASE, 140143029645312, 140143096754175, -+STORE, 140142962536448, 140143096754175, -+SNULL, 140142962536448, 140142970929151, -+STORE, 140142970929152, 140143096754175, -+STORE, 140142962536448, 140142970929151, -+ERASE, 140142962536448, 140142970929151, -+STORE, 140142962536448, 140142970929151, -+STORE, 140142828318720, 140142962536447, -+STORE, 140142819926016, 140142828318719, -+SNULL, 140142828318720, 140142836711423, -+STORE, 140142836711424, 140142962536447, -+STORE, 140142828318720, 140142836711423, -+ERASE, 140142828318720, 140142836711423, -+SNULL, 140143172255743, 140143239364607, -+STORE, 140143105146880, 140143172255743, -+STORE, 140143172255744, 140143239364607, -+ERASE, 140143172255744, 140143239364607, -+SNULL, 140143105282047, 140143172255743, -+STORE, 140143105146880, 140143105282047, -+STORE, 140143105282048, 140143172255743, -+SNULL, 140143038038015, 140143096754175, -+STORE, 140142970929152, 140143038038015, -+STORE, 140143038038016, 140143096754175, -+ERASE, 140143038038016, 140143096754175, -+SNULL, 140142971064319, 140143038038015, -+STORE, 140142970929152, 140142971064319, -+STORE, 140142971064320, 140143038038015, -+SNULL, 140142903820287, 140142962536447, -+STORE, 140142836711424, 140142903820287, -+STORE, 140142903820288, 140142962536447, -+ERASE, 140142903820288, 140142962536447, -+SNULL, 140142836846591, 140142903820287, -+STORE, 140142836711424, 140142836846591, -+STORE, 140142836846592, 140142903820287, -+STORE, 140142685708288, 140142819926015, -+SNULL, 140143311450111, 140143319838719, -+STORE, 140143311446016, 140143311450111, -+STORE, 140143311450112, 140143319838719, -+SNULL, 140142962540543, 140142970929151, -+STORE, 140142962536448, 140142962540543, -+STORE, 140142962540544, 140142970929151, -+SNULL, 140142685708288, 140142702493695, -+STORE, 140142702493696, 140142819926015, -+STORE, 140142685708288, 140142702493695, -+ERASE, 140142685708288, 140142702493695, -+SNULL, 140142769602559, 140142819926015, -+STORE, 140142702493696, 140142769602559, -+STORE, 140142769602560, 140142819926015, -+ERASE, 140142769602560, 140142819926015, -+SNULL, 140142702628863, 140142769602559, -+STORE, 140142702493696, 140142702628863, -+STORE, 140142702628864, 140142769602559, -+STORE, 140143230971904, 140143239364607, -+SNULL, 140143230975999, 140143239364607, -+STORE, 140143230971904, 140143230975999, -+STORE, 140143230976000, 140143239364607, -+SNULL, 140143096758271, 140143105146879, -+STORE, 140143096754176, 140143096758271, -+STORE, 140143096758272, 140143105146879, -+STORE, 140143222579200, 140143230971903, -+SNULL, 140143222583295, 140143230971903, -+STORE, 140143222579200, 140143222583295, -+STORE, 140143222583296, 140143230971903, -+STORE, 140143214186496, 140143222579199, -+SNULL, 140142819930111, 140142828318719, -+STORE, 140142819926016, 140142819930111, -+STORE, 140142819930112, 140142828318719, -+STORE, 140143205793792, 140143222579199, -+SNULL, 140143205793792, 140143214186495, -+STORE, 140143214186496, 140143222579199, -+STORE, 140143205793792, 140143214186495, -+SNULL, 140143214190591, 140143222579199, -+STORE, 140143214186496, 140143214190591, -+STORE, 140143214190592, 140143222579199, -+SNULL, 140143205797887, 140143214186495, -+STORE, 140143205793792, 140143205797887, -+STORE, 140143205797888, 140143214186495, -+STORE, 140143197401088, 140143205793791, -+SNULL, 140143197405183, 140143205793791, -+STORE, 140143197401088, 140143197405183, -+STORE, 140143197405184, 140143205793791, -+STORE, 140143189008384, 140143197401087, -+STORE, 140143180615680, 140143197401087, -+STORE, 140143088361472, 140143096754175, -+SNULL, 140143180619775, 140143197401087, -+STORE, 140143180615680, 140143180619775, -+STORE, 140143180619776, 140143197401087, -+SNULL, 140143180619776, 140143189008383, -+STORE, 140143189008384, 140143197401087, -+STORE, 140143180619776, 140143189008383, -+SNULL, 140143189012479, 140143197401087, -+STORE, 140143189008384, 140143189012479, -+STORE, 140143189012480, 140143197401087, -+SNULL, 140143088365567, 140143096754175, -+STORE, 140143088361472, 140143088365567, -+STORE, 140143088365568, 140143096754175, -+STORE, 140143079968768, 140143088361471, -+SNULL, 140143079972863, 140143088361471, -+STORE, 140143079968768, 140143079972863, -+STORE, 140143079972864, 140143088361471, -+STORE, 140143071576064, 140143079968767, -+SNULL, 140143071580159, 140143079968767, -+STORE, 140143071576064, 140143071580159, -+STORE, 140143071580160, 140143079968767, -+STORE, 140143063183360, 140143071576063, -+STORE, 140143054790656, 140143071576063, -+SNULL, 140143054794751, 140143071576063, -+STORE, 140143054790656, 140143054794751, -+STORE, 140143054794752, 140143071576063, -+SNULL, 140143054794752, 140143063183359, -+STORE, 140143063183360, 140143071576063, -+STORE, 140143054794752, 140143063183359, -+SNULL, 140143063187455, 140143071576063, -+STORE, 140143063183360, 140143063187455, -+STORE, 140143063187456, 140143071576063, -+STORE, 140143046397952, 140143054790655, -+STORE, 140142954143744, 140142962536447, -+STORE, 140142945751040, 140142962536447, -+STORE, 140142937358336, 140142962536447, -+STORE, 140142928965632, 140142962536447, -+STORE, 140142568275968, 140142702493695, -+SNULL, 140142635384831, 140142702493695, -+STORE, 140142568275968, 140142635384831, -+STORE, 140142635384832, 140142702493695, -+ERASE, 140142635384832, 140142702493695, -+STORE, 140142920572928, 140142962536447, -+STORE, 140142912180224, 140142962536447, -+STORE, 140142568275968, 140142702493695, -+SNULL, 140142568275968, 140142635384831, -+STORE, 140142635384832, 140142702493695, -+STORE, 140142568275968, 140142635384831, -+SNULL, 140142635519999, 140142702493695, -+STORE, 140142635384832, 140142635519999, -+STORE, 140142635520000, 140142702493695, -+STORE, 140142819930112, 140142836711423, -+STORE, 140142811533312, 140142819926015, -+STORE, 140142434058240, 140142635384831, -+SNULL, 140142501167103, 140142635384831, -+STORE, 140142434058240, 140142501167103, -+STORE, 140142501167104, 140142635384831, -+SNULL, 140142501167104, 140142568275967, -+STORE, 140142568275968, 140142635384831, -+STORE, 140142501167104, 140142568275967, -+ERASE, 140142501167104, 140142568275967, -+STORE, 140142299840512, 140142501167103, -+STORE, 140142803140608, 140142819926015, -+SNULL, 140142366949375, 140142501167103, -+STORE, 140142299840512, 140142366949375, -+STORE, 140142366949376, 140142501167103, -+SNULL, 140142366949376, 140142434058239, -+STORE, 140142434058240, 140142501167103, -+STORE, 140142366949376, 140142434058239, -+ERASE, 140142366949376, 140142434058239, -+STORE, 140142794747904, 140142819926015, -+STORE, 140142786355200, 140142819926015, -+STORE, 140142299840512, 140142501167103, -+STORE, 140142777962496, 140142819926015, -+STORE, 140142559883264, 140142568275967, -+STORE, 140142232731648, 140142501167103, -+STORE, 140142551490560, 140142568275967, -+SNULL, 140142777962496, 140142803140607, -+STORE, 140142803140608, 140142819926015, -+STORE, 140142777962496, 140142803140607, -+SNULL, 140142803144703, 140142819926015, -+STORE, 140142803140608, 140142803144703, -+STORE, 140142803144704, 140142819926015, -+STORE, 140142543097856, 140142568275967, -+STORE, 140142098513920, 140142501167103, -+SNULL, 140142165622783, 140142501167103, -+STORE, 140142098513920, 140142165622783, -+STORE, 140142165622784, 140142501167103, -+SNULL, 140142165622784, 140142232731647, -+STORE, 140142232731648, 140142501167103, -+STORE, 140142165622784, 140142232731647, -+ERASE, 140142165622784, 140142232731647, -+SNULL, 140142568411135, 140142635384831, -+STORE, 140142568275968, 140142568411135, -+STORE, 140142568411136, 140142635384831, -+STORE, 140141964296192, 140142165622783, -+SNULL, 140142912180224, 140142928965631, -+STORE, 140142928965632, 140142962536447, -+STORE, 140142912180224, 140142928965631, -+SNULL, 140142928969727, 140142962536447, -+STORE, 140142928965632, 140142928969727, -+STORE, 140142928969728, 140142962536447, -+STORE, 140141830078464, 140142165622783, -+SNULL, 140142912184319, 140142928965631, -+STORE, 140142912180224, 140142912184319, -+STORE, 140142912184320, 140142928965631, -+SNULL, 140142232731648, 140142434058239, -+STORE, 140142434058240, 140142501167103, -+STORE, 140142232731648, 140142434058239, -+SNULL, 140142434193407, 140142501167103, -+STORE, 140142434058240, 140142434193407, -+STORE, 140142434193408, 140142501167103, -+SNULL, 140142232731648, 140142299840511, -+STORE, 140142299840512, 140142434058239, -+STORE, 140142232731648, 140142299840511, -+SNULL, 140142299975679, 140142434058239, -+STORE, 140142299840512, 140142299975679, -+STORE, 140142299975680, 140142434058239, -+SNULL, 140142928969728, 140142954143743, -+STORE, 140142954143744, 140142962536447, -+STORE, 140142928969728, 140142954143743, -+SNULL, 140142954147839, 140142962536447, -+STORE, 140142954143744, 140142954147839, -+STORE, 140142954147840, 140142962536447, -+STORE, 140141830078464, 140142299840511, -+SNULL, 140142543097856, 140142559883263, -+STORE, 140142559883264, 140142568275967, -+STORE, 140142543097856, 140142559883263, -+SNULL, 140142559887359, 140142568275967, -+STORE, 140142559883264, 140142559887359, -+STORE, 140142559887360, 140142568275967, -+STORE, 140142534705152, 140142559883263, -+SNULL, 140142928969728, 140142945751039, -+STORE, 140142945751040, 140142954143743, -+STORE, 140142928969728, 140142945751039, -+SNULL, 140142945755135, 140142954143743, -+STORE, 140142945751040, 140142945755135, -+STORE, 140142945755136, 140142954143743, -+SNULL, 140142299975680, 140142366949375, -+STORE, 140142366949376, 140142434058239, -+STORE, 140142299975680, 140142366949375, -+SNULL, 140142367084543, 140142434058239, -+STORE, 140142366949376, 140142367084543, -+STORE, 140142367084544, 140142434058239, -+SNULL, 140142928969728, 140142937358335, -+STORE, 140142937358336, 140142945751039, -+STORE, 140142928969728, 140142937358335, -+SNULL, 140142937362431, 140142945751039, -+STORE, 140142937358336, 140142937362431, -+STORE, 140142937362432, 140142945751039, -+SNULL, 140141830078464, 140142232731647, -+STORE, 140142232731648, 140142299840511, -+STORE, 140141830078464, 140142232731647, -+SNULL, 140142232866815, 140142299840511, -+STORE, 140142232731648, 140142232866815, -+STORE, 140142232866816, 140142299840511, -+SNULL, 140142534705152, 140142543097855, -+STORE, 140142543097856, 140142559883263, -+STORE, 140142534705152, 140142543097855, -+SNULL, 140142543101951, 140142559883263, -+STORE, 140142543097856, 140142543101951, -+STORE, 140142543101952, 140142559883263, -+STORE, 140142526312448, 140142543097855, -+STORE, 140142517919744, 140142543097855, -+SNULL, 140141830078464, 140142098513919, -+STORE, 140142098513920, 140142232731647, -+STORE, 140141830078464, 140142098513919, -+SNULL, 140142098649087, 140142232731647, -+STORE, 140142098513920, 140142098649087, -+STORE, 140142098649088, 140142232731647, -+SNULL, 140142031405055, 140142098513919, -+STORE, 140141830078464, 140142031405055, -+STORE, 140142031405056, 140142098513919, -+ERASE, 140142031405056, 140142098513919, -+SNULL, 140141830078464, 140141964296191, -+STORE, 140141964296192, 140142031405055, -+STORE, 140141830078464, 140141964296191, -+SNULL, 140141964431359, 140142031405055, -+STORE, 140141964296192, 140141964431359, -+STORE, 140141964431360, 140142031405055, -+STORE, 140142509527040, 140142543097855, -+SNULL, 140141897187327, 140141964296191, -+STORE, 140141830078464, 140141897187327, -+STORE, 140141897187328, 140141964296191, -+ERASE, 140141897187328, 140141964296191, -+SNULL, 140141830213631, 140141897187327, -+STORE, 140141830078464, 140141830213631, -+STORE, 140141830213632, 140141897187327, -+SNULL, 140142803144704, 140142811533311, -+STORE, 140142811533312, 140142819926015, -+STORE, 140142803144704, 140142811533311, -+SNULL, 140142811537407, 140142819926015, -+STORE, 140142811533312, 140142811537407, -+STORE, 140142811537408, 140142819926015, -+SNULL, 140142098649088, 140142165622783, -+STORE, 140142165622784, 140142232731647, -+STORE, 140142098649088, 140142165622783, -+SNULL, 140142165757951, 140142232731647, -+STORE, 140142165622784, 140142165757951, -+STORE, 140142165757952, 140142232731647, -+STORE, 140142090121216, 140142098513919, -+SNULL, 140142777962496, 140142786355199, -+STORE, 140142786355200, 140142803140607, -+STORE, 140142777962496, 140142786355199, -+SNULL, 140142786359295, 140142803140607, -+STORE, 140142786355200, 140142786359295, -+STORE, 140142786359296, 140142803140607, -+SNULL, 140142509527040, 140142534705151, -+STORE, 140142534705152, 140142543097855, -+STORE, 140142509527040, 140142534705151, -+SNULL, 140142534709247, 140142543097855, -+STORE, 140142534705152, 140142534709247, -+STORE, 140142534709248, 140142543097855, -+STORE, 140142081728512, 140142098513919, -+SNULL, 140142786359296, 140142794747903, -+STORE, 140142794747904, 140142803140607, -+STORE, 140142786359296, 140142794747903, -+SNULL, 140142794751999, 140142803140607, -+STORE, 140142794747904, 140142794751999, -+STORE, 140142794752000, 140142803140607, -+STORE, 140142073335808, 140142098513919, -+SNULL, 140142073339903, 140142098513919, -+STORE, 140142073335808, 140142073339903, -+STORE, 140142073339904, 140142098513919, -+SNULL, 140142543101952, 140142551490559, -+STORE, 140142551490560, 140142559883263, -+STORE, 140142543101952, 140142551490559, -+SNULL, 140142551494655, 140142559883263, -+STORE, 140142551490560, 140142551494655, -+STORE, 140142551494656, 140142559883263, -+SNULL, 140142509527040, 140142517919743, -+STORE, 140142517919744, 140142534705151, -+STORE, 140142509527040, 140142517919743, -+SNULL, 140142517923839, 140142534705151, -+STORE, 140142517919744, 140142517923839, -+STORE, 140142517923840, 140142534705151, -+STORE, 140142064943104, 140142073335807, -+SNULL, 140142073339904, 140142090121215, -+STORE, 140142090121216, 140142098513919, -+STORE, 140142073339904, 140142090121215, -+SNULL, 140142090125311, 140142098513919, -+STORE, 140142090121216, 140142090125311, -+STORE, 140142090125312, 140142098513919, -+STORE, 140142056550400, 140142073335807, -+SNULL, 140142056554495, 140142073335807, -+STORE, 140142056550400, 140142056554495, -+STORE, 140142056554496, 140142073335807, -+STORE, 140142048157696, 140142056550399, -+SNULL, 140142509531135, 140142517919743, -+STORE, 140142509527040, 140142509531135, -+STORE, 140142509531136, 140142517919743, -+SNULL, 140142777966591, 140142786355199, -+STORE, 140142777962496, 140142777966591, -+STORE, 140142777966592, 140142786355199, -+SNULL, 140143046402047, 140143054790655, -+STORE, 140143046397952, 140143046402047, -+STORE, 140143046402048, 140143054790655, -+SNULL, 140142912184320, 140142920572927, -+STORE, 140142920572928, 140142928965631, -+STORE, 140142912184320, 140142920572927, -+SNULL, 140142920577023, 140142928965631, -+STORE, 140142920572928, 140142920577023, -+STORE, 140142920577024, 140142928965631, -+STORE, 140142039764992, 140142056550399, -+STORE, 140141955903488, 140141964296191, -+SNULL, 140142819930112, 140142828318719, -+STORE, 140142828318720, 140142836711423, -+STORE, 140142819930112, 140142828318719, -+SNULL, 140142828322815, 140142836711423, -+STORE, 140142828318720, 140142828322815, -+STORE, 140142828322816, 140142836711423, -+SNULL, 140142517923840, 140142526312447, -+STORE, 140142526312448, 140142534705151, -+STORE, 140142517923840, 140142526312447, -+SNULL, 140142526316543, 140142534705151, -+STORE, 140142526312448, 140142526316543, -+STORE, 140142526316544, 140142534705151, -+STORE, 140141947510784, 140141964296191, -+SNULL, 140142056554496, 140142064943103, -+STORE, 140142064943104, 140142073335807, -+STORE, 140142056554496, 140142064943103, -+SNULL, 140142064947199, 140142073335807, -+STORE, 140142064943104, 140142064947199, -+STORE, 140142064947200, 140142073335807, -+SNULL, 140142073339904, 140142081728511, -+STORE, 140142081728512, 140142090121215, -+STORE, 140142073339904, 140142081728511, -+SNULL, 140142081732607, 140142090121215, -+STORE, 140142081728512, 140142081732607, -+STORE, 140142081732608, 140142090121215, -+STORE, 140141939118080, 140141964296191, -+STORE, 140141930725376, 140141964296191, -+STORE, 140141922332672, 140141964296191, -+STORE, 140141913939968, 140141964296191, -+SNULL, 140141913939968, 140141922332671, -+STORE, 140141922332672, 140141964296191, -+STORE, 140141913939968, 140141922332671, -+SNULL, 140141922336767, 140141964296191, -+STORE, 140141922332672, 140141922336767, -+STORE, 140141922336768, 140141964296191, -+STORE, 140141905547264, 140141922332671, -+SNULL, 140141905551359, 140141922332671, -+STORE, 140141905547264, 140141905551359, -+STORE, 140141905551360, 140141922332671, -+STORE, 140141821685760, 140141830078463, -+STORE, 140141813293056, 140141830078463, -+STORE, 140141804900352, 140141830078463, -+STORE, 140141796507648, 140141830078463, -+SNULL, 140141796511743, 140141830078463, -+STORE, 140141796507648, 140141796511743, -+STORE, 140141796511744, 140141830078463, -+SNULL, 140141922336768, 140141955903487, -+STORE, 140141955903488, 140141964296191, -+STORE, 140141922336768, 140141955903487, -+SNULL, 140141955907583, 140141964296191, -+STORE, 140141955903488, 140141955907583, -+STORE, 140141955907584, 140141964296191, -+STORE, 140141788114944, 140141796507647, -+STORE, 140141779722240, 140141796507647, -+SNULL, 140141779722240, 140141788114943, -+STORE, 140141788114944, 140141796507647, -+STORE, 140141779722240, 140141788114943, -+SNULL, 140141788119039, 140141796507647, -+STORE, 140141788114944, 140141788119039, -+STORE, 140141788119040, 140141796507647, -+SNULL, 140141922336768, 140141947510783, -+STORE, 140141947510784, 140141955903487, -+STORE, 140141922336768, 140141947510783, -+SNULL, 140141947514879, 140141955903487, -+STORE, 140141947510784, 140141947514879, -+STORE, 140141947514880, 140141955903487, -+SNULL, 140142039764992, 140142048157695, -+STORE, 140142048157696, 140142056550399, -+STORE, 140142039764992, 140142048157695, -+SNULL, 140142048161791, 140142056550399, -+STORE, 140142048157696, 140142048161791, -+STORE, 140142048161792, 140142056550399, -+SNULL, 140142039769087, 140142048157695, -+STORE, 140142039764992, 140142039769087, -+STORE, 140142039769088, 140142048157695, -+SNULL, 140141796511744, 140141804900351, -+STORE, 140141804900352, 140141830078463, -+STORE, 140141796511744, 140141804900351, -+SNULL, 140141804904447, 140141830078463, -+STORE, 140141804900352, 140141804904447, -+STORE, 140141804904448, 140141830078463, -+STORE, 140141771329536, 140141788114943, -+STORE, 140141762936832, 140141788114943, -+STORE, 140141754544128, 140141788114943, -+SNULL, 140141804904448, 140141821685759, -+STORE, 140141821685760, 140141830078463, -+STORE, 140141804904448, 140141821685759, -+SNULL, 140141821689855, 140141830078463, -+STORE, 140141821685760, 140141821689855, -+STORE, 140141821689856, 140141830078463, -+SNULL, 140141922336768, 140141939118079, -+STORE, 140141939118080, 140141947510783, -+STORE, 140141922336768, 140141939118079, -+SNULL, 140141939122175, 140141947510783, -+STORE, 140141939118080, 140141939122175, -+STORE, 140141939122176, 140141947510783, -+SNULL, 140141905551360, 140141913939967, -+STORE, 140141913939968, 140141922332671, -+STORE, 140141905551360, 140141913939967, -+SNULL, 140141913944063, 140141922332671, -+STORE, 140141913939968, 140141913944063, -+STORE, 140141913944064, 140141922332671, -+STORE, 140141746151424, 140141788114943, -+STORE, 140141737758720, 140141788114943, -+SNULL, 140141804904448, 140141813293055, -+STORE, 140141813293056, 140141821685759, -+STORE, 140141804904448, 140141813293055, -+SNULL, 140141813297151, 140141821685759, -+STORE, 140141813293056, 140141813297151, -+STORE, 140141813297152, 140141821685759, -+STORE, 140141729366016, 140141788114943, -+STORE, 140141720973312, 140141788114943, -+STORE, 140141712580608, 140141788114943, -+SNULL, 140141712584703, 140141788114943, -+STORE, 140141712580608, 140141712584703, -+STORE, 140141712584704, 140141788114943, -+SNULL, 140141922336768, 140141930725375, -+STORE, 140141930725376, 140141939118079, -+STORE, 140141922336768, 140141930725375, -+SNULL, 140141930729471, 140141939118079, -+STORE, 140141930725376, 140141930729471, -+STORE, 140141930729472, 140141939118079, -+STORE, 140141704187904, 140141712580607, -+SNULL, 140141704191999, 140141712580607, -+STORE, 140141704187904, 140141704191999, -+STORE, 140141704192000, 140141712580607, -+STORE, 140141695795200, 140141704187903, -+STORE, 140141687402496, 140141704187903, -+SNULL, 140141712584704, 140141771329535, -+STORE, 140141771329536, 140141788114943, -+STORE, 140141712584704, 140141771329535, -+SNULL, 140141771333631, 140141788114943, -+STORE, 140141771329536, 140141771333631, -+STORE, 140141771333632, 140141788114943, -+SNULL, 140141771333632, 140141779722239, -+STORE, 140141779722240, 140141788114943, -+STORE, 140141771333632, 140141779722239, -+SNULL, 140141779726335, 140141788114943, -+STORE, 140141779722240, 140141779726335, -+STORE, 140141779726336, 140141788114943, -+STORE, 140141679009792, 140141704187903, -+SNULL, 140141679013887, 140141704187903, -+STORE, 140141679009792, 140141679013887, -+STORE, 140141679013888, 140141704187903, -+STORE, 140141670617088, 140141679009791, -+SNULL, 140141670621183, 140141679009791, -+STORE, 140141670617088, 140141670621183, -+STORE, 140141670621184, 140141679009791, -+STORE, 140141662224384, 140141670617087, -+SNULL, 140141712584704, 140141737758719, -+STORE, 140141737758720, 140141771329535, -+STORE, 140141712584704, 140141737758719, -+SNULL, 140141737762815, 140141771329535, -+STORE, 140141737758720, 140141737762815, -+STORE, 140141737762816, 140141771329535, -+SNULL, 140141712584704, 140141729366015, -+STORE, 140141729366016, 140141737758719, -+STORE, 140141712584704, 140141729366015, -+SNULL, 140141729370111, 140141737758719, -+STORE, 140141729366016, 140141729370111, -+STORE, 140141729370112, 140141737758719, -+SNULL, 140141737762816, 140141746151423, -+STORE, 140141746151424, 140141771329535, -+STORE, 140141737762816, 140141746151423, -+SNULL, 140141746155519, 140141771329535, -+STORE, 140141746151424, 140141746155519, -+STORE, 140141746155520, 140141771329535, -+STORE, 140141653831680, 140141670617087, -+SNULL, 140141746155520, 140141762936831, -+STORE, 140141762936832, 140141771329535, -+STORE, 140141746155520, 140141762936831, -+SNULL, 140141762940927, 140141771329535, -+STORE, 140141762936832, 140141762940927, -+STORE, 140141762940928, 140141771329535, -+STORE, 140141645438976, 140141670617087, -+SNULL, 140141645443071, 140141670617087, -+STORE, 140141645438976, 140141645443071, -+STORE, 140141645443072, 140141670617087, -+SNULL, 140141712584704, 140141720973311, -+STORE, 140141720973312, 140141729366015, -+STORE, 140141712584704, 140141720973311, -+SNULL, 140141720977407, 140141729366015, -+STORE, 140141720973312, 140141720977407, -+STORE, 140141720977408, 140141729366015, -+STORE, 140141637046272, 140141645438975, -+SNULL, 140141637050367, 140141645438975, -+STORE, 140141637046272, 140141637050367, -+STORE, 140141637050368, 140141645438975, -+STORE, 140141628653568, 140141637046271, -+SNULL, 140141628657663, 140141637046271, -+STORE, 140141628653568, 140141628657663, -+STORE, 140141628657664, 140141637046271, -+STORE, 140141620260864, 140141628653567, -+SNULL, 140141679013888, 140141687402495, -+STORE, 140141687402496, 140141704187903, -+STORE, 140141679013888, 140141687402495, -+SNULL, 140141687406591, 140141704187903, -+STORE, 140141687402496, 140141687406591, -+STORE, 140141687406592, 140141704187903, -+SNULL, 140141746155520, 140141754544127, -+STORE, 140141754544128, 140141762936831, -+STORE, 140141746155520, 140141754544127, -+SNULL, 140141754548223, 140141762936831, -+STORE, 140141754544128, 140141754548223, -+STORE, 140141754548224, 140141762936831, -+SNULL, 140141687406592, 140141695795199, -+STORE, 140141695795200, 140141704187903, -+STORE, 140141687406592, 140141695795199, -+SNULL, 140141695799295, 140141704187903, -+STORE, 140141695795200, 140141695799295, -+STORE, 140141695799296, 140141704187903, -+STORE, 140141611868160, 140141628653567, -+SNULL, 140141611872255, 140141628653567, -+STORE, 140141611868160, 140141611872255, -+STORE, 140141611872256, 140141628653567, -+SNULL, 140141645443072, 140141662224383, -+STORE, 140141662224384, 140141670617087, -+STORE, 140141645443072, 140141662224383, -+SNULL, 140141662228479, 140141670617087, -+STORE, 140141662224384, 140141662228479, -+STORE, 140141662228480, 140141670617087, -+STORE, 140141603475456, 140141611868159, -+SNULL, 140141603479551, 140141611868159, -+STORE, 140141603475456, 140141603479551, -+STORE, 140141603479552, 140141611868159, -+STORE, 140141595082752, 140141603475455, -+SNULL, 140141645443072, 140141653831679, -+STORE, 140141653831680, 140141662224383, -+STORE, 140141645443072, 140141653831679, -+SNULL, 140141653835775, 140141662224383, -+STORE, 140141653831680, 140141653835775, -+STORE, 140141653835776, 140141662224383, -+STORE, 140141586690048, 140141603475455, -+SNULL, 140141611872256, 140141620260863, -+STORE, 140141620260864, 140141628653567, -+STORE, 140141611872256, 140141620260863, -+SNULL, 140141620264959, 140141628653567, -+STORE, 140141620260864, 140141620264959, -+STORE, 140141620264960, 140141628653567, -+SNULL, 140141586690048, 140141595082751, -+STORE, 140141595082752, 140141603475455, -+STORE, 140141586690048, 140141595082751, -+SNULL, 140141595086847, 140141603475455, -+STORE, 140141595082752, 140141595086847, -+STORE, 140141595086848, 140141603475455, -+STORE, 140141578297344, 140141595082751, -+SNULL, 140141578301439, 140141595082751, -+STORE, 140141578297344, 140141578301439, -+STORE, 140141578301440, 140141595082751, -+SNULL, 140141578301440, 140141586690047, -+STORE, 140141586690048, 140141595082751, -+STORE, 140141578301440, 140141586690047, -+SNULL, 140141586694143, 140141595082751, -+STORE, 140141586690048, 140141586694143, -+STORE, 140141586694144, 140141595082751, -+STORE, 140143370027008, 140143370055679, -+STORE, 140143309254656, 140143311446015, -+SNULL, 140143309254656, 140143309344767, -+STORE, 140143309344768, 140143311446015, -+STORE, 140143309254656, 140143309344767, -+SNULL, 140143311437823, 140143311446015, -+STORE, 140143309344768, 140143311437823, -+STORE, 140143311437824, 140143311446015, -+ERASE, 140143311437824, 140143311446015, -+STORE, 140143311437824, 140143311446015, -+SNULL, 140143311441919, 140143311446015, -+STORE, 140143311437824, 140143311441919, -+STORE, 140143311441920, 140143311446015, -+ERASE, 140143370027008, 140143370055679, -+ERASE, 140142912180224, 140142912184319, -+ERASE, 140142912184320, 140142920572927, -+ERASE, 140142945751040, 140142945755135, -+ERASE, 140142945755136, 140142954143743, -+ERASE, 140142090121216, 140142090125311, -+ERASE, 140142090125312, 140142098513919, -+ERASE, 140142794747904, 140142794751999, -+ERASE, 140142794752000, 140142803140607, -+ERASE, 140141913939968, 140141913944063, -+ERASE, 140141913944064, 140141922332671, -+ERASE, 140141746151424, 140141746155519, -+ERASE, 140141746155520, 140141754544127, -+ERASE, 140142954143744, 140142954147839, -+ERASE, 140142954147840, 140142962536447, -+ERASE, 140142081728512, 140142081732607, -+ERASE, 140142081732608, 140142090121215, -+ERASE, 140141905547264, 140141905551359, -+ERASE, 140141905551360, 140141913939967, -+ERASE, 140141729366016, 140141729370111, -+ERASE, 140141729370112, 140141737758719, -+ERASE, 140142920572928, 140142920577023, -+ERASE, 140142920577024, 140142928965631, -+ERASE, 140142039764992, 140142039769087, -+ERASE, 140142039769088, 140142048157695, -+ERASE, 140141679009792, 140141679013887, -+ERASE, 140141679013888, 140141687402495, -+ERASE, 140142551490560, 140142551494655, -+ERASE, 140142551494656, 140142559883263, -+ERASE, 140141947510784, 140141947514879, -+ERASE, 140141947514880, 140141955903487, -+ERASE, 140141771329536, 140141771333631, -+ERASE, 140141771333632, 140141779722239, -+ERASE, 140142928965632, 140142928969727, -+ERASE, 140142928969728, 140142937358335, -+ERASE, 140142073335808, 140142073339903, -+ERASE, 140142073339904, 140142081728511, -+ERASE, 140142543097856, 140142543101951, -+ERASE, 140142543101952, 140142551490559, -+ERASE, 140141955903488, 140141955907583, -+ERASE, 140141955907584, 140141964296191, -+ERASE, 140141704187904, 140141704191999, -+ERASE, 140141704192000, 140141712580607, -+ERASE, 140142786355200, 140142786359295, -+ERASE, 140142786359296, 140142794747903, -+ERASE, 140142056550400, 140142056554495, -+ERASE, 140142056554496, 140142064943103, -+ERASE, 140142828318720, 140142828322815, -+ERASE, 140142828322816, 140142836711423, -+ERASE, 140141788114944, 140141788119039, -+ERASE, 140141788119040, 140141796507647, -+ERASE, 140141695795200, 140141695799295, -+ERASE, 140141695799296, 140141704187903, -+ERASE, 140141578297344, 140141578301439, -+ERASE, 140141578301440, 140141586690047, -+ERASE, 140141611868160, 140141611872255, -+ERASE, 140141611872256, 140141620260863, -+ERASE, 140142811533312, 140142811537407, -+ERASE, 140142811537408, 140142819926015, -+ERASE, 140142064943104, 140142064947199, -+ERASE, 140142064947200, 140142073335807, -+ERASE, 140141628653568, 140141628657663, -+ERASE, 140141628657664, 140141637046271, -+ERASE, 140143046397952, 140143046402047, -+ERASE, 140143046402048, 140143054790655, -+ERASE, 140141796507648, 140141796511743, -+ERASE, 140141796511744, 140141804900351, -+ERASE, 140142803140608, 140142803144703, -+ERASE, 140142803144704, 140142811533311, -+ERASE, 140142509527040, 140142509531135, -+ERASE, 140142509531136, 140142517919743, -+ERASE, 140141821685760, 140141821689855, -+ERASE, 140141821689856, 140141830078463, -+ERASE, 140142777962496, 140142777966591, -+ERASE, 140142777966592, 140142786355199, -+ERASE, 140141804900352, 140141804904447, -+ERASE, 140141804904448, 140141813293055, -+ERASE, 140141930725376, 140141930729471, -+ERASE, 140141930729472, 140141939118079, -+ERASE, 140142937358336, 140142937362431, -+ERASE, 140142937362432, 140142945751039, -+ERASE, 140142559883264, 140142559887359, -+ERASE, 140142559887360, 140142568275967, -+ERASE, 140142534705152, 140142534709247, -+ERASE, 140142534709248, 140142543097855, -+ERASE, 140142048157696, 140142048161791, -+ERASE, 140142048161792, 140142056550399, -+ERASE, 140141754544128, 140141754548223, -+ERASE, 140141754548224, 140141762936831, -+ERASE, 140141939118080, 140141939122175, -+ERASE, 140141939122176, 140141947510783, -+ERASE, 140141653831680, 140141653835775, -+ERASE, 140141653835776, 140141662224383, -+ERASE, 140141712580608, 140141712584703, -+ERASE, 140141712584704, 140141720973311, -+ERASE, 140141645438976, 140141645443071, -+ERASE, 140141645443072, 140141653831679, -+ERASE, 140141687402496, 140141687406591, -+ERASE, 140141687406592, 140141695795199, -+ERASE, 140141662224384, 140141662228479, -+ERASE, 140141662228480, 140141670617087, -+ERASE, 140141922332672, 140141922336767, -+ERASE, 140141922336768, 140141930725375, -+ERASE, 140141737758720, 140141737762815, -+ERASE, 140141737762816, 140141746151423, -+ERASE, 140141637046272, 140141637050367, -+ERASE, 140141637050368, 140141645438975, -+ERASE, 140142517919744, 140142517923839, -+ERASE, 140142517923840, 140142526312447, -+ERASE, 140143096754176, 140143096758271, -+ERASE, 140143096758272, 140143105146879, -+ERASE, 140141595082752, 140141595086847, -+ERASE, 140141595086848, 140141603475455, -+ERASE, 140141762936832, 140141762940927, -+ERASE, 140141762940928, 140141771329535, -+ERASE, 140143311446016, 140143311450111, -+ERASE, 140143311450112, 140143319838719, -+ERASE, 140142526312448, 140142526316543, -+ERASE, 140142526316544, 140142534705151, -+ERASE, 140142819926016, 140142819930111, -+ERASE, 140142819930112, 140142828318719, -+ERASE, 140143180615680, 140143180619775, -+ERASE, 140143180619776, 140143189008383, -+ERASE, 140142962536448, 140142962540543, -+ERASE, 140142962540544, 140142970929151, -+ERASE, 140143214186496, 140143214190591, -+ERASE, 140143214190592, 140143222579199, -+ERASE, 140143088361472, 140143088365567, -+ERASE, 140143088365568, 140143096754175, -+ERASE, 140141586690048, 140141586694143, -+ERASE, 140141586694144, 140141595082751, -+ERASE, 140143230971904, 140143230975999, -+ERASE, 140143230976000, 140143239364607, -+ERASE, 140141779722240, 140141779726335, -+ERASE, 140141779726336, 140141788114943, -+ERASE, 140141670617088, 140141670621183, -+ERASE, 140141670621184, 140141679009791, -+ERASE, 140141813293056, 140141813297151, -+ERASE, 140141813297152, 140141821685759, -+ERASE, 140143222579200, 140143222583295, -+ERASE, 140143222583296, 140143230971903, -+ERASE, 140143189008384, 140143189012479, -+ERASE, 140143189012480, 140143197401087, -+ERASE, 140143071576064, 140143071580159, -+ERASE, 140143071580160, 140143079968767, -+ERASE, 140141620260864, 140141620264959, -+ERASE, 140141620264960, 140141628653567, -+ERASE, 140141603475456, 140141603479551, -+ERASE, 140141603479552, 140141611868159, -+ERASE, 140141720973312, 140141720977407, -+ERASE, 140141720977408, 140141729366015, -+ERASE, 140143079968768, 140143079972863, -+ERASE, 140143079972864, 140143088361471, -+ERASE, 140143205793792, 140143205797887, -+ERASE, 140143205797888, 140143214186495, -+ }; -+ unsigned long set30[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140733436743680, 140737488351231, -+SNULL, 140733436747775, 140737488351231, -+STORE, 140733436743680, 140733436747775, -+STORE, 140733436612608, 140733436747775, -+STORE, 94630728904704, 94630731157503, -+SNULL, 94630729035775, 94630731157503, -+STORE, 94630728904704, 94630729035775, -+STORE, 94630729035776, 94630731157503, -+ERASE, 94630729035776, 94630731157503, -+STORE, 94630731128832, 94630731137023, -+STORE, 94630731137024, 94630731157503, -+STORE, 140165750841344, 140165753094143, -+SNULL, 140165750984703, 140165753094143, -+STORE, 140165750841344, 140165750984703, -+STORE, 140165750984704, 140165753094143, -+ERASE, 140165750984704, 140165753094143, -+STORE, 140165753081856, 140165753090047, -+STORE, 140165753090048, 140165753094143, -+STORE, 140733436887040, 140733436891135, -+STORE, 140733436874752, 140733436887039, -+STORE, 140165753053184, 140165753081855, -+STORE, 140165753044992, 140165753053183, -+STORE, 140165748625408, 140165750841343, -+SNULL, 140165748625408, 140165748723711, -+STORE, 140165748723712, 140165750841343, -+STORE, 140165748625408, 140165748723711, -+SNULL, 140165750816767, 140165750841343, -+STORE, 140165748723712, 140165750816767, -+STORE, 140165750816768, 140165750841343, -+SNULL, 140165750816768, 140165750824959, -+STORE, 140165750824960, 140165750841343, -+STORE, 140165750816768, 140165750824959, -+ERASE, 140165750816768, 140165750824959, -+STORE, 140165750816768, 140165750824959, -+ERASE, 140165750824960, 140165750841343, -+STORE, 140165750824960, 140165750841343, -+STORE, 140165744828416, 140165748625407, -+SNULL, 140165744828416, 140165746487295, -+STORE, 140165746487296, 140165748625407, -+STORE, 140165744828416, 140165746487295, -+SNULL, 140165748584447, 140165748625407, -+STORE, 140165746487296, 140165748584447, -+STORE, 140165748584448, 140165748625407, -+SNULL, 140165748584448, 140165748609023, -+STORE, 140165748609024, 140165748625407, -+STORE, 140165748584448, 140165748609023, -+ERASE, 140165748584448, 140165748609023, -+STORE, 140165748584448, 140165748609023, -+ERASE, 140165748609024, 140165748625407, -+STORE, 140165748609024, 140165748625407, -+STORE, 140165753036800, 140165753053183, -+SNULL, 140165748600831, 140165748609023, -+STORE, 140165748584448, 140165748600831, -+STORE, 140165748600832, 140165748609023, -+SNULL, 140165750820863, 140165750824959, -+STORE, 140165750816768, 140165750820863, -+STORE, 140165750820864, 140165750824959, -+SNULL, 94630731132927, 94630731137023, -+STORE, 94630731128832, 94630731132927, -+STORE, 94630731132928, 94630731137023, -+SNULL, 140165753085951, 140165753090047, -+STORE, 140165753081856, 140165753085951, -+STORE, 140165753085952, 140165753090047, -+ERASE, 140165753053184, 140165753081855, -+STORE, 94630743547904, 94630743683071, -+STORE, 140165736435712, 140165744828415, -+SNULL, 140165736439807, 140165744828415, -+STORE, 140165736435712, 140165736439807, -+STORE, 140165736439808, 140165744828415, -+STORE, 140165728043008, 140165736435711, -+STORE, 140165593825280, 140165728043007, -+SNULL, 140165593825280, 140165653725183, -+STORE, 140165653725184, 140165728043007, -+STORE, 140165593825280, 140165653725183, -+ERASE, 140165593825280, 140165653725183, -+SNULL, 140165720834047, 140165728043007, -+STORE, 140165653725184, 140165720834047, -+STORE, 140165720834048, 140165728043007, -+ERASE, 140165720834048, 140165728043007, -+SNULL, 140165653860351, 140165720834047, -+STORE, 140165653725184, 140165653860351, -+STORE, 140165653860352, 140165720834047, -+SNULL, 140165728047103, 140165736435711, -+STORE, 140165728043008, 140165728047103, -+STORE, 140165728047104, 140165736435711, -+STORE, 140165645332480, 140165653725183, -+SNULL, 140165645336575, 140165653725183, -+STORE, 140165645332480, 140165645336575, -+STORE, 140165645336576, 140165653725183, -+STORE, 140165636939776, 140165645332479, -+SNULL, 140165636943871, 140165645332479, -+STORE, 140165636939776, 140165636943871, -+STORE, 140165636943872, 140165645332479, -+STORE, 140165628547072, 140165636939775, -+SNULL, 140165628551167, 140165636939775, -+STORE, 140165628547072, 140165628551167, -+STORE, 140165628551168, 140165636939775, -+STORE, 140165620154368, 140165628547071, -+STORE, 140165611761664, 140165628547071, -+STORE, 140165603368960, 140165628547071, -+STORE, 140165469151232, 140165603368959, -+SNULL, 140165469151232, 140165519507455, -+STORE, 140165519507456, 140165603368959, -+STORE, 140165469151232, 140165519507455, -+ERASE, 140165469151232, 140165519507455, -+SNULL, 140165586616319, 140165603368959, -+STORE, 140165519507456, 140165586616319, -+STORE, 140165586616320, 140165603368959, -+ERASE, 140165586616320, 140165603368959, -+STORE, 140165594976256, 140165628547071, -+STORE, 140165385289728, 140165586616319, -+SNULL, 140165452398591, 140165586616319, -+STORE, 140165385289728, 140165452398591, -+STORE, 140165452398592, 140165586616319, -+SNULL, 140165452398592, 140165519507455, -+STORE, 140165519507456, 140165586616319, -+STORE, 140165452398592, 140165519507455, -+ERASE, 140165452398592, 140165519507455, -+STORE, 140165251072000, 140165452398591, -+SNULL, 140165318180863, 140165452398591, -+STORE, 140165251072000, 140165318180863, -+STORE, 140165318180864, 140165452398591, -+SNULL, 140165318180864, 140165385289727, -+STORE, 140165385289728, 140165452398591, -+STORE, 140165318180864, 140165385289727, -+ERASE, 140165318180864, 140165385289727, -+SNULL, 140165519642623, 140165586616319, -+STORE, 140165519507456, 140165519642623, -+STORE, 140165519642624, 140165586616319, -+SNULL, 140165594976256, 140165611761663, -+STORE, 140165611761664, 140165628547071, -+STORE, 140165594976256, 140165611761663, -+SNULL, 140165611765759, 140165628547071, -+STORE, 140165611761664, 140165611765759, -+STORE, 140165611765760, 140165628547071, -+STORE, 140165385289728, 140165519507455, -+SNULL, 140165385424895, 140165519507455, -+STORE, 140165385289728, 140165385424895, -+STORE, 140165385424896, 140165519507455, -+SNULL, 140165594976256, 140165603368959, -+STORE, 140165603368960, 140165611761663, -+STORE, 140165594976256, 140165603368959, -+SNULL, 140165603373055, 140165611761663, -+STORE, 140165603368960, 140165603373055, -+STORE, 140165603373056, 140165611761663, -+SNULL, 140165251207167, 140165318180863, -+STORE, 140165251072000, 140165251207167, -+STORE, 140165251207168, 140165318180863, -+STORE, 140165376897024, 140165385289727, -+SNULL, 140165376901119, 140165385289727, -+STORE, 140165376897024, 140165376901119, -+STORE, 140165376901120, 140165385289727, -+SNULL, 140165385424896, 140165452398591, -+STORE, 140165452398592, 140165519507455, -+STORE, 140165385424896, 140165452398591, -+SNULL, 140165452533759, 140165519507455, -+STORE, 140165452398592, 140165452533759, -+STORE, 140165452533760, 140165519507455, -+STORE, 140165368504320, 140165376897023, -+SNULL, 140165594980351, 140165603368959, -+STORE, 140165594976256, 140165594980351, -+STORE, 140165594980352, 140165603368959, -+SNULL, 140165368508415, 140165376897023, -+STORE, 140165368504320, 140165368508415, -+STORE, 140165368508416, 140165376897023, -+SNULL, 140165611765760, 140165620154367, -+STORE, 140165620154368, 140165628547071, -+STORE, 140165611765760, 140165620154367, -+SNULL, 140165620158463, 140165628547071, -+STORE, 140165620154368, 140165620158463, -+STORE, 140165620158464, 140165628547071, -+STORE, 140165360111616, 140165368504319, -+STORE, 140165351718912, 140165368504319, -+STORE, 140165343326208, 140165368504319, -+SNULL, 140165343326208, 140165351718911, -+STORE, 140165351718912, 140165368504319, -+STORE, 140165343326208, 140165351718911, -+SNULL, 140165351723007, 140165368504319, -+STORE, 140165351718912, 140165351723007, -+STORE, 140165351723008, 140165368504319, -+SNULL, 140165343330303, 140165351718911, -+STORE, 140165343326208, 140165343330303, -+STORE, 140165343330304, 140165351718911, -+SNULL, 140165351723008, 140165360111615, -+STORE, 140165360111616, 140165368504319, -+STORE, 140165351723008, 140165360111615, -+SNULL, 140165360115711, 140165368504319, -+STORE, 140165360111616, 140165360115711, -+STORE, 140165360115712, 140165368504319, -+STORE, 140165334933504, 140165343326207, -+SNULL, 140165334937599, 140165343326207, -+STORE, 140165334933504, 140165334937599, -+STORE, 140165334937600, 140165343326207, -+STORE, 140165326540800, 140165334933503, -+STORE, 140165242679296, 140165251071999, -+SNULL, 140165242683391, 140165251071999, -+STORE, 140165242679296, 140165242683391, -+STORE, 140165242683392, 140165251071999, -+STORE, 140165234286592, 140165242679295, -+STORE, 140165225893888, 140165242679295, -+SNULL, 140165225897983, 140165242679295, -+STORE, 140165225893888, 140165225897983, -+STORE, 140165225897984, 140165242679295, -+SNULL, 140165225897984, 140165234286591, -+STORE, 140165234286592, 140165242679295, -+STORE, 140165225897984, 140165234286591, -+SNULL, 140165234290687, 140165242679295, -+STORE, 140165234286592, 140165234290687, -+STORE, 140165234290688, 140165242679295, -+SNULL, 140165326544895, 140165334933503, -+STORE, 140165326540800, 140165326544895, -+STORE, 140165326544896, 140165334933503, -+STORE, 140165217501184, 140165225893887, -+STORE, 140165209108480, 140165225893887, -+SNULL, 140165209108480, 140165217501183, -+STORE, 140165217501184, 140165225893887, -+STORE, 140165209108480, 140165217501183, -+SNULL, 140165217505279, 140165225893887, -+STORE, 140165217501184, 140165217505279, -+STORE, 140165217505280, 140165225893887, -+SNULL, 140165209112575, 140165217501183, -+STORE, 140165209108480, 140165209112575, -+STORE, 140165209112576, 140165217501183, -+STORE, 140165200715776, 140165209108479, -+STORE, 140165066498048, 140165200715775, -+SNULL, 140165066498048, 140165116854271, -+STORE, 140165116854272, 140165200715775, -+STORE, 140165066498048, 140165116854271, -+ERASE, 140165066498048, 140165116854271, -+SNULL, 140165183963135, 140165200715775, -+STORE, 140165116854272, 140165183963135, -+STORE, 140165183963136, 140165200715775, -+ERASE, 140165183963136, 140165200715775, -+SNULL, 140165116989439, 140165183963135, -+STORE, 140165116854272, 140165116989439, -+STORE, 140165116989440, 140165183963135, -+STORE, 140165192323072, 140165209108479, -+STORE, 140165108461568, 140165116854271, -+STORE, 140164974243840, 140165108461567, -+STORE, 140164965851136, 140164974243839, -+SNULL, 140164974243840, 140164982636543, -+STORE, 140164982636544, 140165108461567, -+STORE, 140164974243840, 140164982636543, -+ERASE, 140164974243840, 140164982636543, -+STORE, 140164965851136, 140164982636543, -+STORE, 140164957458432, 140164982636543, -+STORE, 140164949065728, 140164982636543, -+STORE, 140164940673024, 140164982636543, -+STORE, 140164806455296, 140164940673023, -+STORE, 140164798062592, 140164806455295, -+STORE, 140164789669888, 140164806455295, -+STORE, 140164655452160, 140164789669887, -+STORE, 140164647059456, 140164655452159, -+STORE, 140164638666752, 140164655452159, -+SNULL, 140164655452160, 140164714201087, -+STORE, 140164714201088, 140164789669887, -+STORE, 140164655452160, 140164714201087, -+ERASE, 140164655452160, 140164714201087, -+STORE, 140164705808384, 140164714201087, -+STORE, 140164697415680, 140164714201087, -+STORE, 140164504449024, 140164638666751, -+SNULL, 140164504449024, 140164512874495, -+STORE, 140164512874496, 140164638666751, -+STORE, 140164504449024, 140164512874495, -+ERASE, 140164504449024, 140164512874495, -+STORE, 140164689022976, 140164714201087, -+STORE, 140164680630272, 140164714201087, -+SNULL, 140164680634367, 140164714201087, -+STORE, 140164680630272, 140164680634367, -+STORE, 140164680634368, 140164714201087, -+STORE, 140164378656768, 140164638666751, -+SNULL, 140165192323072, 140165200715775, -+STORE, 140165200715776, 140165209108479, -+STORE, 140165192323072, 140165200715775, -+SNULL, 140165200719871, 140165209108479, -+STORE, 140165200715776, 140165200719871, -+STORE, 140165200719872, 140165209108479, -+SNULL, 140165049745407, 140165108461567, -+STORE, 140164982636544, 140165049745407, -+STORE, 140165049745408, 140165108461567, -+ERASE, 140165049745408, 140165108461567, -+SNULL, 140164982771711, 140165049745407, -+STORE, 140164982636544, 140164982771711, -+STORE, 140164982771712, 140165049745407, -+STORE, 140164244439040, 140164638666751, -+SNULL, 140164311547903, 140164638666751, -+STORE, 140164244439040, 140164311547903, -+STORE, 140164311547904, 140164638666751, -+SNULL, 140164311547904, 140164378656767, -+STORE, 140164378656768, 140164638666751, -+STORE, 140164311547904, 140164378656767, -+ERASE, 140164311547904, 140164378656767, -+SNULL, 140164806455296, 140164848418815, -+STORE, 140164848418816, 140164940673023, -+STORE, 140164806455296, 140164848418815, -+ERASE, 140164806455296, 140164848418815, -+SNULL, 140164915527679, 140164940673023, -+STORE, 140164848418816, 140164915527679, -+STORE, 140164915527680, 140164940673023, -+ERASE, 140164915527680, 140164940673023, -+STORE, 140164110221312, 140164311547903, -+SNULL, 140164177330175, 140164311547903, -+STORE, 140164110221312, 140164177330175, -+STORE, 140164177330176, 140164311547903, -+SNULL, 140164177330176, 140164244439039, -+STORE, 140164244439040, 140164311547903, -+STORE, 140164177330176, 140164244439039, -+ERASE, 140164177330176, 140164244439039, -+SNULL, 140164781309951, 140164789669887, -+STORE, 140164714201088, 140164781309951, -+STORE, 140164781309952, 140164789669887, -+ERASE, 140164781309952, 140164789669887, -+STORE, 140163976003584, 140164177330175, -+SNULL, 140164043112447, 140164177330175, -+STORE, 140163976003584, 140164043112447, -+STORE, 140164043112448, 140164177330175, -+SNULL, 140164043112448, 140164110221311, -+STORE, 140164110221312, 140164177330175, -+STORE, 140164043112448, 140164110221311, -+ERASE, 140164043112448, 140164110221311, -+SNULL, 140164579983359, 140164638666751, -+STORE, 140164378656768, 140164579983359, -+STORE, 140164579983360, 140164638666751, -+ERASE, 140164579983360, 140164638666751, -+STORE, 140163841785856, 140164043112447, -+SNULL, 140163908894719, 140164043112447, -+STORE, 140163841785856, 140163908894719, -+STORE, 140163908894720, 140164043112447, -+SNULL, 140163908894720, 140163976003583, -+STORE, 140163976003584, 140164043112447, -+STORE, 140163908894720, 140163976003583, -+ERASE, 140163908894720, 140163976003583, -+SNULL, 140164940673024, 140164965851135, -+STORE, 140164965851136, 140164982636543, -+STORE, 140164940673024, 140164965851135, -+SNULL, 140164965855231, 140164982636543, -+STORE, 140164965851136, 140164965855231, -+STORE, 140164965855232, 140164982636543, -+SNULL, 140164965855232, 140164974243839, -+STORE, 140164974243840, 140164982636543, -+STORE, 140164965855232, 140164974243839, -+SNULL, 140164974247935, 140164982636543, -+STORE, 140164974243840, 140164974247935, -+STORE, 140164974247936, 140164982636543, -+SNULL, 140164445765631, 140164579983359, -+STORE, 140164378656768, 140164445765631, -+STORE, 140164445765632, 140164579983359, -+SNULL, 140164445765632, 140164512874495, -+STORE, 140164512874496, 140164579983359, -+STORE, 140164445765632, 140164512874495, -+ERASE, 140164445765632, 140164512874495, -+SNULL, 140164378791935, 140164445765631, -+STORE, 140164378656768, 140164378791935, -+STORE, 140164378791936, 140164445765631, -+SNULL, 140164789673983, 140164806455295, -+STORE, 140164789669888, 140164789673983, -+STORE, 140164789673984, 140164806455295, -+SNULL, 140164789673984, 140164798062591, -+STORE, 140164798062592, 140164806455295, -+STORE, 140164789673984, 140164798062591, -+SNULL, 140164798066687, 140164806455295, -+STORE, 140164798062592, 140164798066687, -+STORE, 140164798066688, 140164806455295, -+SNULL, 140164638670847, 140164655452159, -+STORE, 140164638666752, 140164638670847, -+STORE, 140164638670848, 140164655452159, -+STORE, 140165100068864, 140165116854271, -+STORE, 140165091676160, 140165116854271, -+STORE, 140165083283456, 140165116854271, -+SNULL, 140164244574207, 140164311547903, -+STORE, 140164244439040, 140164244574207, -+STORE, 140164244574208, 140164311547903, -+SNULL, 140164848553983, 140164915527679, -+STORE, 140164848418816, 140164848553983, -+STORE, 140164848553984, 140164915527679, -+SNULL, 140164110356479, 140164177330175, -+STORE, 140164110221312, 140164110356479, -+STORE, 140164110356480, 140164177330175, -+SNULL, 140164714336255, 140164781309951, -+STORE, 140164714201088, 140164714336255, -+STORE, 140164714336256, 140164781309951, -+SNULL, 140163976138751, 140164043112447, -+STORE, 140163976003584, 140163976138751, -+STORE, 140163976138752, 140164043112447, -+SNULL, 140164513009663, 140164579983359, -+STORE, 140164512874496, 140164513009663, -+STORE, 140164513009664, 140164579983359, -+SNULL, 140163841921023, 140163908894719, -+STORE, 140163841785856, 140163841921023, -+STORE, 140163841921024, 140163908894719, -+SNULL, 140165083283456, 140165100068863, -+STORE, 140165100068864, 140165116854271, -+STORE, 140165083283456, 140165100068863, -+SNULL, 140165100072959, 140165116854271, -+STORE, 140165100068864, 140165100072959, -+STORE, 140165100072960, 140165116854271, -+SNULL, 140165100072960, 140165108461567, -+STORE, 140165108461568, 140165116854271, -+STORE, 140165100072960, 140165108461567, -+SNULL, 140165108465663, 140165116854271, -+STORE, 140165108461568, 140165108465663, -+STORE, 140165108465664, 140165116854271, -+STORE, 140165074890752, 140165100068863, -+SNULL, 140165074894847, 140165100068863, -+STORE, 140165074890752, 140165074894847, -+STORE, 140165074894848, 140165100068863, -+STORE, 140165066498048, 140165074890751, -+STORE, 140165058105344, 140165074890751, -+STORE, 140164932280320, 140164965851135, -+SNULL, 140165192327167, 140165200715775, -+STORE, 140165192323072, 140165192327167, -+STORE, 140165192327168, 140165200715775, -+STORE, 140164923887616, 140164965851135, -+SNULL, 140164923891711, 140164965851135, -+STORE, 140164923887616, 140164923891711, -+STORE, 140164923891712, 140164965851135, -+SNULL, 140164680634368, 140164705808383, -+STORE, 140164705808384, 140164714201087, -+STORE, 140164680634368, 140164705808383, -+SNULL, 140164705812479, 140164714201087, -+STORE, 140164705808384, 140164705812479, -+STORE, 140164705812480, 140164714201087, -+SNULL, 140164680634368, 140164697415679, -+STORE, 140164697415680, 140164705808383, -+STORE, 140164680634368, 140164697415679, -+SNULL, 140164697419775, 140164705808383, -+STORE, 140164697415680, 140164697419775, -+STORE, 140164697419776, 140164705808383, -+STORE, 140164840026112, 140164848418815, -+STORE, 140164831633408, 140164848418815, -+STORE, 140164823240704, 140164848418815, -+SNULL, 140165074894848, 140165083283455, -+STORE, 140165083283456, 140165100068863, -+STORE, 140165074894848, 140165083283455, -+SNULL, 140165083287551, 140165100068863, -+STORE, 140165083283456, 140165083287551, -+STORE, 140165083287552, 140165100068863, -+SNULL, 140165083287552, 140165091676159, -+STORE, 140165091676160, 140165100068863, -+STORE, 140165083287552, 140165091676159, -+SNULL, 140165091680255, 140165100068863, -+STORE, 140165091676160, 140165091680255, -+STORE, 140165091680256, 140165100068863, -+SNULL, 140164638670848, 140164647059455, -+STORE, 140164647059456, 140164655452159, -+STORE, 140164638670848, 140164647059455, -+SNULL, 140164647063551, 140164655452159, -+STORE, 140164647059456, 140164647063551, -+STORE, 140164647063552, 140164655452159, -+SNULL, 140164923891712, 140164940673023, -+STORE, 140164940673024, 140164965851135, -+STORE, 140164923891712, 140164940673023, -+SNULL, 140164940677119, 140164965851135, -+STORE, 140164940673024, 140164940677119, -+STORE, 140164940677120, 140164965851135, -+SNULL, 140164940677120, 140164949065727, -+STORE, 140164949065728, 140164965851135, -+STORE, 140164940677120, 140164949065727, -+SNULL, 140164949069823, 140164965851135, -+STORE, 140164949065728, 140164949069823, -+STORE, 140164949069824, 140164965851135, -+SNULL, 140164949069824, 140164957458431, -+STORE, 140164957458432, 140164965851135, -+STORE, 140164949069824, 140164957458431, -+SNULL, 140164957462527, 140164965851135, -+STORE, 140164957458432, 140164957462527, -+STORE, 140164957462528, 140164965851135, -+SNULL, 140164680634368, 140164689022975, -+STORE, 140164689022976, 140164697415679, -+STORE, 140164680634368, 140164689022975, -+SNULL, 140164689027071, 140164697415679, -+STORE, 140164689022976, 140164689027071, -+STORE, 140164689027072, 140164697415679, -+STORE, 140164814848000, 140164848418815, -+SNULL, 140165058105344, 140165066498047, -+STORE, 140165066498048, 140165074890751, -+STORE, 140165058105344, 140165066498047, -+SNULL, 140165066502143, 140165074890751, -+STORE, 140165066498048, 140165066502143, -+STORE, 140165066502144, 140165074890751, -+SNULL, 140165058109439, 140165066498047, -+STORE, 140165058105344, 140165058109439, -+STORE, 140165058109440, 140165066498047, -+STORE, 140164798066688, 140164814847999, -+SNULL, 140164798066688, 140164806455295, -+STORE, 140164806455296, 140164814847999, -+STORE, 140164798066688, 140164806455295, -+SNULL, 140164806459391, 140164814847999, -+STORE, 140164806455296, 140164806459391, -+STORE, 140164806459392, 140164814847999, -+SNULL, 140164923891712, 140164932280319, -+STORE, 140164932280320, 140164940673023, -+STORE, 140164923891712, 140164932280319, -+SNULL, 140164932284415, 140164940673023, -+STORE, 140164932280320, 140164932284415, -+STORE, 140164932284416, 140164940673023, -+STORE, 140164672237568, 140164680630271, -+STORE, 140164663844864, 140164680630271, -+STORE, 140164647063552, 140164680630271, -+SNULL, 140164647063552, 140164655452159, -+STORE, 140164655452160, 140164680630271, -+STORE, 140164647063552, 140164655452159, -+SNULL, 140164655456255, 140164680630271, -+STORE, 140164655452160, 140164655456255, -+STORE, 140164655456256, 140164680630271, -+STORE, 140164630274048, 140164638666751, -+SNULL, 140164814852095, 140164848418815, -+STORE, 140164814848000, 140164814852095, -+STORE, 140164814852096, 140164848418815, -+SNULL, 140164814852096, 140164831633407, -+STORE, 140164831633408, 140164848418815, -+STORE, 140164814852096, 140164831633407, -+SNULL, 140164831637503, 140164848418815, -+STORE, 140164831633408, 140164831637503, -+STORE, 140164831637504, 140164848418815, -+STORE, 140164621881344, 140164638666751, -+SNULL, 140164831637504, 140164840026111, -+STORE, 140164840026112, 140164848418815, -+STORE, 140164831637504, 140164840026111, -+SNULL, 140164840030207, 140164848418815, -+STORE, 140164840026112, 140164840030207, -+STORE, 140164840030208, 140164848418815, -+STORE, 140164613488640, 140164638666751, -+SNULL, 140164613492735, 140164638666751, -+STORE, 140164613488640, 140164613492735, -+STORE, 140164613492736, 140164638666751, -+STORE, 140164605095936, 140164613488639, -+SNULL, 140164605100031, 140164613488639, -+STORE, 140164605095936, 140164605100031, -+STORE, 140164605100032, 140164613488639, -+STORE, 140164596703232, 140164605095935, -+STORE, 140164588310528, 140164605095935, -+SNULL, 140164588314623, 140164605095935, -+STORE, 140164588310528, 140164588314623, -+STORE, 140164588314624, 140164605095935, -+STORE, 140164504481792, 140164512874495, -+STORE, 140164496089088, 140164512874495, -+SNULL, 140164496089088, 140164504481791, -+STORE, 140164504481792, 140164512874495, -+STORE, 140164496089088, 140164504481791, -+SNULL, 140164504485887, 140164512874495, -+STORE, 140164504481792, 140164504485887, -+STORE, 140164504485888, 140164512874495, -+SNULL, 140164613492736, 140164630274047, -+STORE, 140164630274048, 140164638666751, -+STORE, 140164613492736, 140164630274047, -+SNULL, 140164630278143, 140164638666751, -+STORE, 140164630274048, 140164630278143, -+STORE, 140164630278144, 140164638666751, -+STORE, 140164487696384, 140164504481791, -+STORE, 140164479303680, 140164504481791, -+SNULL, 140164814852096, 140164823240703, -+STORE, 140164823240704, 140164831633407, -+STORE, 140164814852096, 140164823240703, -+SNULL, 140164823244799, 140164831633407, -+STORE, 140164823240704, 140164823244799, -+STORE, 140164823244800, 140164831633407, -+STORE, 140164470910976, 140164504481791, -+SNULL, 140164470910976, 140164496089087, -+STORE, 140164496089088, 140164504481791, -+STORE, 140164470910976, 140164496089087, -+SNULL, 140164496093183, 140164504481791, -+STORE, 140164496089088, 140164496093183, -+STORE, 140164496093184, 140164504481791, -+SNULL, 140164655456256, 140164672237567, -+STORE, 140164672237568, 140164680630271, -+STORE, 140164655456256, 140164672237567, -+SNULL, 140164672241663, 140164680630271, -+STORE, 140164672237568, 140164672241663, -+STORE, 140164672241664, 140164680630271, -+STORE, 140164462518272, 140164496089087, -+STORE, 140164454125568, 140164496089087, -+SNULL, 140164655456256, 140164663844863, -+STORE, 140164663844864, 140164672237567, -+STORE, 140164655456256, 140164663844863, -+SNULL, 140164663848959, 140164672237567, -+STORE, 140164663844864, 140164663848959, -+STORE, 140164663848960, 140164672237567, -+STORE, 140164370264064, 140164378656767, -+STORE, 140164361871360, 140164378656767, -+STORE, 140164353478656, 140164378656767, -+STORE, 140164345085952, 140164378656767, -+SNULL, 140164345085952, 140164353478655, -+STORE, 140164353478656, 140164378656767, -+STORE, 140164345085952, 140164353478655, -+SNULL, 140164353482751, 140164378656767, -+STORE, 140164353478656, 140164353482751, -+STORE, 140164353482752, 140164378656767, -+SNULL, 140164454125568, 140164487696383, -+STORE, 140164487696384, 140164496089087, -+STORE, 140164454125568, 140164487696383, -+SNULL, 140164487700479, 140164496089087, -+STORE, 140164487696384, 140164487700479, -+STORE, 140164487700480, 140164496089087, -+STORE, 140164336693248, 140164353478655, -+SNULL, 140164336697343, 140164353478655, -+STORE, 140164336693248, 140164336697343, -+STORE, 140164336697344, 140164353478655, -+STORE, 140164328300544, 140164336693247, -+SNULL, 140164454125568, 140164479303679, -+STORE, 140164479303680, 140164487696383, -+STORE, 140164454125568, 140164479303679, -+SNULL, 140164479307775, 140164487696383, -+STORE, 140164479303680, 140164479307775, -+STORE, 140164479307776, 140164487696383, -+STORE, 140164319907840, 140164336693247, -+STORE, 140164236046336, 140164244439039, -+SNULL, 140164588314624, 140164596703231, -+STORE, 140164596703232, 140164605095935, -+STORE, 140164588314624, 140164596703231, -+SNULL, 140164596707327, 140164605095935, -+STORE, 140164596703232, 140164596707327, -+STORE, 140164596707328, 140164605095935, -+SNULL, 140164454125568, 140164462518271, -+STORE, 140164462518272, 140164479303679, -+STORE, 140164454125568, 140164462518271, -+SNULL, 140164462522367, 140164479303679, -+STORE, 140164462518272, 140164462522367, -+STORE, 140164462522368, 140164479303679, -+STORE, 140164227653632, 140164244439039, -+SNULL, 140164227657727, 140164244439039, -+STORE, 140164227653632, 140164227657727, -+STORE, 140164227657728, 140164244439039, -+SNULL, 140164462522368, 140164470910975, -+STORE, 140164470910976, 140164479303679, -+STORE, 140164462522368, 140164470910975, -+SNULL, 140164470915071, 140164479303679, -+STORE, 140164470910976, 140164470915071, -+STORE, 140164470915072, 140164479303679, -+SNULL, 140164613492736, 140164621881343, -+STORE, 140164621881344, 140164630274047, -+STORE, 140164613492736, 140164621881343, -+SNULL, 140164621885439, 140164630274047, -+STORE, 140164621881344, 140164621885439, -+STORE, 140164621885440, 140164630274047, -+SNULL, 140164353482752, 140164370264063, -+STORE, 140164370264064, 140164378656767, -+STORE, 140164353482752, 140164370264063, -+SNULL, 140164370268159, 140164378656767, -+STORE, 140164370264064, 140164370268159, -+STORE, 140164370268160, 140164378656767, -+STORE, 140164219260928, 140164227653631, -+SNULL, 140164319911935, 140164336693247, -+STORE, 140164319907840, 140164319911935, -+STORE, 140164319911936, 140164336693247, -+SNULL, 140164336697344, 140164345085951, -+STORE, 140164345085952, 140164353478655, -+STORE, 140164336697344, 140164345085951, -+SNULL, 140164345090047, 140164353478655, -+STORE, 140164345085952, 140164345090047, -+STORE, 140164345090048, 140164353478655, -+SNULL, 140164319911936, 140164328300543, -+STORE, 140164328300544, 140164336693247, -+STORE, 140164319911936, 140164328300543, -+SNULL, 140164328304639, 140164336693247, -+STORE, 140164328300544, 140164328304639, -+STORE, 140164328304640, 140164336693247, -+SNULL, 140164454129663, 140164462518271, -+STORE, 140164454125568, 140164454129663, -+STORE, 140164454129664, 140164462518271, -+STORE, 140164210868224, 140164227653631, -+STORE, 140164202475520, 140164227653631, -+STORE, 140164194082816, 140164227653631, -+SNULL, 140164194086911, 140164227653631, -+STORE, 140164194082816, 140164194086911, -+STORE, 140164194086912, 140164227653631, -+SNULL, 140164353482752, 140164361871359, -+STORE, 140164361871360, 140164370264063, -+STORE, 140164353482752, 140164361871359, -+SNULL, 140164361875455, 140164370264063, -+STORE, 140164361871360, 140164361875455, -+STORE, 140164361875456, 140164370264063, -+SNULL, 140164227657728, 140164236046335, -+STORE, 140164236046336, 140164244439039, -+STORE, 140164227657728, 140164236046335, -+SNULL, 140164236050431, 140164244439039, -+STORE, 140164236046336, 140164236050431, -+STORE, 140164236050432, 140164244439039, -+STORE, 140164185690112, 140164194082815, -+SNULL, 140164194086912, 140164219260927, -+STORE, 140164219260928, 140164227653631, -+STORE, 140164194086912, 140164219260927, -+SNULL, 140164219265023, 140164227653631, -+STORE, 140164219260928, 140164219265023, -+STORE, 140164219265024, 140164227653631, -+STORE, 140164101828608, 140164110221311, -+STORE, 140164093435904, 140164110221311, -+STORE, 140164085043200, 140164110221311, -+SNULL, 140164085047295, 140164110221311, -+STORE, 140164085043200, 140164085047295, -+STORE, 140164085047296, 140164110221311, -+STORE, 140164076650496, 140164085043199, -+SNULL, 140164185694207, 140164194082815, -+STORE, 140164185690112, 140164185694207, -+STORE, 140164185694208, 140164194082815, -+SNULL, 140164085047296, 140164101828607, -+STORE, 140164101828608, 140164110221311, -+STORE, 140164085047296, 140164101828607, -+SNULL, 140164101832703, 140164110221311, -+STORE, 140164101828608, 140164101832703, -+STORE, 140164101832704, 140164110221311, -+SNULL, 140164085047296, 140164093435903, -+STORE, 140164093435904, 140164101828607, -+STORE, 140164085047296, 140164093435903, -+SNULL, 140164093439999, 140164101828607, -+STORE, 140164093435904, 140164093439999, -+STORE, 140164093440000, 140164101828607, -+SNULL, 140164194086912, 140164202475519, -+STORE, 140164202475520, 140164219260927, -+STORE, 140164194086912, 140164202475519, -+SNULL, 140164202479615, 140164219260927, -+STORE, 140164202475520, 140164202479615, -+STORE, 140164202479616, 140164219260927, -+SNULL, 140164202479616, 140164210868223, -+STORE, 140164210868224, 140164219260927, -+STORE, 140164202479616, 140164210868223, -+SNULL, 140164210872319, 140164219260927, -+STORE, 140164210868224, 140164210872319, -+STORE, 140164210872320, 140164219260927, -+SNULL, 140164076654591, 140164085043199, -+STORE, 140164076650496, 140164076654591, -+STORE, 140164076654592, 140164085043199, -+STORE, 140164068257792, 140164076650495, -+SNULL, 140164068261887, 140164076650495, -+STORE, 140164068257792, 140164068261887, -+STORE, 140164068261888, 140164076650495, -+STORE, 140165753053184, 140165753081855, -+STORE, 140165725851648, 140165728043007, -+SNULL, 140165725851648, 140165725941759, -+STORE, 140165725941760, 140165728043007, -+STORE, 140165725851648, 140165725941759, -+SNULL, 140165728034815, 140165728043007, -+STORE, 140165725941760, 140165728034815, -+STORE, 140165728034816, 140165728043007, -+ERASE, 140165728034816, 140165728043007, -+STORE, 140165728034816, 140165728043007, -+SNULL, 140165728038911, 140165728043007, -+STORE, 140165728034816, 140165728038911, -+STORE, 140165728038912, 140165728043007, -+ERASE, 140165753053184, 140165753081855, -+ERASE, 140164638666752, 140164638670847, -+ERASE, 140164638670848, 140164647059455, -+ERASE, 140165091676160, 140165091680255, -+ERASE, 140165091680256, 140165100068863, -+ERASE, 140164613488640, 140164613492735, -+ERASE, 140164613492736, 140164621881343, -+ERASE, 140164319907840, 140164319911935, -+ERASE, 140164319911936, 140164328300543, -+ERASE, 140165620154368, 140165620158463, -+ERASE, 140165620158464, 140165628547071, -+ERASE, 140164798062592, 140164798066687, -+ERASE, 140164798066688, 140164806455295, -+ERASE, 140164789669888, 140164789673983, -+ERASE, 140164789673984, 140164798062591, -+ERASE, 140164965851136, 140164965855231, -+ERASE, 140164965855232, 140164974243839, -+ERASE, 140165074890752, 140165074894847, -+ERASE, 140165074894848, 140165083283455, -+ERASE, 140164672237568, 140164672241663, -+ERASE, 140164672241664, 140164680630271, -+ERASE, 140164454125568, 140164454129663, -+ERASE, 140164454129664, 140164462518271, -+ERASE, 140165200715776, 140165200719871, -+ERASE, 140165200719872, 140165209108479, -+ERASE, 140164932280320, 140164932284415, -+ERASE, 140164932284416, 140164940673023, -+ERASE, 140164663844864, 140164663848959, -+ERASE, 140164663848960, 140164672237567, -+ERASE, 140164697415680, 140164697419775, -+ERASE, 140164697419776, 140164705808383, -+ERASE, 140164831633408, 140164831637503, -+ERASE, 140164831637504, 140164840026111, -+ERASE, 140165192323072, 140165192327167, -+ERASE, 140165192327168, 140165200715775, -+ERASE, 140165108461568, 140165108465663, -+ERASE, 140165108465664, 140165116854271, -+ERASE, 140164840026112, 140164840030207, -+ERASE, 140164840030208, 140164848418815, -+ERASE, 140164647059456, 140164647063551, -+ERASE, 140164647063552, 140164655452159, -+ERASE, 140165083283456, 140165083287551, -+ERASE, 140165083287552, 140165091676159, -+ERASE, 140164923887616, 140164923891711, -+ERASE, 140164923891712, 140164932280319, -+ERASE, 140164823240704, 140164823244799, -+ERASE, 140164823244800, 140164831633407, -+ERASE, 140164227653632, 140164227657727, -+ERASE, 140164227657728, 140164236046335, -+ERASE, 140164957458432, 140164957462527, -+ERASE, 140164957462528, 140164965851135, -+ERASE, 140164680630272, 140164680634367, -+ERASE, 140164680634368, 140164689022975, -+ERASE, 140164974243840, 140164974247935, -+ERASE, 140164974247936, 140164982636543, -+ERASE, 140165066498048, 140165066502143, -+ERASE, 140165066502144, 140165074890751, -+ERASE, 140164621881344, 140164621885439, -+ERASE, 140164621885440, 140164630274047, -+ERASE, 140164949065728, 140164949069823, -+ERASE, 140164949069824, 140164957458431, -+ERASE, 140164588310528, 140164588314623, -+ERASE, 140164588314624, 140164596703231, -+ERASE, 140164806455296, 140164806459391, -+ERASE, 140164806459392, 140164814847999, -+ERASE, 140164940673024, 140164940677119, -+ERASE, 140164940677120, 140164949065727, -+ERASE, 140164596703232, 140164596707327, -+ERASE, 140164596707328, 140164605095935, -+ERASE, 140164605095936, 140164605100031, -+ERASE, 140164605100032, 140164613488639, -+ERASE, 140164655452160, 140164655456255, -+ERASE, 140164655456256, 140164663844863, -+ERASE, 140164705808384, 140164705812479, -+ERASE, 140164705812480, 140164714201087, -+ERASE, 140164689022976, 140164689027071, -+ERASE, 140164689027072, 140164697415679, -+ERASE, 140164630274048, 140164630278143, -+ERASE, 140164630278144, 140164638666751, -+ERASE, 140164479303680, 140164479307775, -+ERASE, 140164479307776, 140164487696383, -+ERASE, 140164236046336, 140164236050431, -+ERASE, 140164236050432, 140164244439039, -+ERASE, 140164085043200, 140164085047295, -+ERASE, 140164085047296, 140164093435903, -+ERASE, 140164345085952, 140164345090047, -+ERASE, 140164345090048, 140164353478655, -+ERASE, 140164101828608, 140164101832703, -+ERASE, 140164101832704, 140164110221311, -+ERASE, 140164370264064, 140164370268159, -+ERASE, 140164370268160, 140164378656767, -+ERASE, 140164336693248, 140164336697343, -+ERASE, 140164336697344, 140164345085951, -+ERASE, 140164194082816, 140164194086911, -+ERASE, 140164194086912, 140164202475519, -+ERASE, 140164353478656, 140164353482751, -+ERASE, 140164353482752, 140164361871359, -+ERASE, 140164210868224, 140164210872319, -+ERASE, 140164210872320, 140164219260927, -+ERASE, 140164814848000, 140164814852095, -+ERASE, 140164814852096, 140164823240703, -+ERASE, 140164504481792, 140164504485887, -+ERASE, 140164504485888, 140164512874495, -+ERASE, 140165100068864, 140165100072959, -+ERASE, 140165100072960, 140165108461567, -+ERASE, 140164361871360, 140164361875455, -+ERASE, 140164361875456, 140164370264063, -+ERASE, 140164470910976, 140164470915071, -+ERASE, 140164470915072, 140164479303679, -+ERASE, 140164076650496, 140164076654591, -+ERASE, 140164076654592, 140164085043199, -+ERASE, 140164202475520, 140164202479615, -+ERASE, 140164202479616, 140164210868223, -+ERASE, 140164462518272, 140164462522367, -+ERASE, 140164462522368, 140164470910975, -+ERASE, 140165351718912, 140165351723007, -+ERASE, 140165351723008, 140165360111615, -+ERASE, 140164328300544, 140164328304639, -+ERASE, 140164328304640, 140164336693247, -+ERASE, 140164093435904, 140164093439999, -+ERASE, 140164093440000, 140164101828607, -+ERASE, 140165603368960, 140165603373055, -+ERASE, 140165603373056, 140165611761663, -+ERASE, 140165368504320, 140165368508415, -+ERASE, 140165368508416, 140165376897023, -+ERASE, 140165334933504, 140165334937599, -+ERASE, 140165334937600, 140165343326207, -+ERASE, 140165594976256, 140165594980351, -+ERASE, 140165594980352, 140165603368959, -+ERASE, 140164487696384, 140164487700479, -+ERASE, 140164487700480, 140164496089087, -+ERASE, 140164219260928, 140164219265023, -+ERASE, 140164219265024, 140164227653631, -+ERASE, 140164185690112, 140164185694207, -+ERASE, 140164185694208, 140164194082815, -+ERASE, 140164068257792, 140164068261887, -+ERASE, 140164068261888, 140164076650495, -+ERASE, 140165225893888, 140165225897983, -+ERASE, 140165225897984, 140165234286591, -+ERASE, 140165058105344, 140165058109439, -+ }; -+ unsigned long set31[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140730890784768, 140737488351231, -+SNULL, 140730890788863, 140737488351231, -+STORE, 140730890784768, 140730890788863, -+STORE, 140730890653696, 140730890788863, -+STORE, 94577123659776, 94577125912575, -+SNULL, 94577123790847, 94577125912575, -+STORE, 94577123659776, 94577123790847, -+STORE, 94577123790848, 94577125912575, -+ERASE, 94577123790848, 94577125912575, -+STORE, 94577125883904, 94577125892095, -+STORE, 94577125892096, 94577125912575, -+STORE, 140624060407808, 140624062660607, -+SNULL, 140624060551167, 140624062660607, -+STORE, 140624060407808, 140624060551167, -+STORE, 140624060551168, 140624062660607, -+ERASE, 140624060551168, 140624062660607, -+STORE, 140624062648320, 140624062656511, -+STORE, 140624062656512, 140624062660607, -+STORE, 140730892140544, 140730892144639, -+STORE, 140730892128256, 140730892140543, -+STORE, 140624062619648, 140624062648319, -+STORE, 140624062611456, 140624062619647, -+STORE, 140624058191872, 140624060407807, -+SNULL, 140624058191872, 140624058290175, -+STORE, 140624058290176, 140624060407807, -+STORE, 140624058191872, 140624058290175, -+SNULL, 140624060383231, 140624060407807, -+STORE, 140624058290176, 140624060383231, -+STORE, 140624060383232, 140624060407807, -+SNULL, 140624060383232, 140624060391423, -+STORE, 140624060391424, 140624060407807, -+STORE, 140624060383232, 140624060391423, -+ERASE, 140624060383232, 140624060391423, -+STORE, 140624060383232, 140624060391423, -+ERASE, 140624060391424, 140624060407807, -+STORE, 140624060391424, 140624060407807, -+STORE, 140624054394880, 140624058191871, -+SNULL, 140624054394880, 140624056053759, -+STORE, 140624056053760, 140624058191871, -+STORE, 140624054394880, 140624056053759, -+SNULL, 140624058150911, 140624058191871, -+STORE, 140624056053760, 140624058150911, -+STORE, 140624058150912, 140624058191871, -+SNULL, 140624058150912, 140624058175487, -+STORE, 140624058175488, 140624058191871, -+STORE, 140624058150912, 140624058175487, -+ERASE, 140624058150912, 140624058175487, -+STORE, 140624058150912, 140624058175487, -+ERASE, 140624058175488, 140624058191871, -+STORE, 140624058175488, 140624058191871, -+STORE, 140624062603264, 140624062619647, -+SNULL, 140624058167295, 140624058175487, -+STORE, 140624058150912, 140624058167295, -+STORE, 140624058167296, 140624058175487, -+SNULL, 140624060387327, 140624060391423, -+STORE, 140624060383232, 140624060387327, -+STORE, 140624060387328, 140624060391423, -+SNULL, 94577125887999, 94577125892095, -+STORE, 94577125883904, 94577125887999, -+STORE, 94577125888000, 94577125892095, -+SNULL, 140624062652415, 140624062656511, -+STORE, 140624062648320, 140624062652415, -+STORE, 140624062652416, 140624062656511, -+ERASE, 140624062619648, 140624062648319, -+STORE, 94577157709824, 94577157844991, -+STORE, 140624046002176, 140624054394879, -+SNULL, 140624046006271, 140624054394879, -+STORE, 140624046002176, 140624046006271, -+STORE, 140624046006272, 140624054394879, -+STORE, 140624037609472, 140624046002175, -+STORE, 140623903391744, 140624037609471, -+SNULL, 140623903391744, 140623940157439, -+STORE, 140623940157440, 140624037609471, -+STORE, 140623903391744, 140623940157439, -+ERASE, 140623903391744, 140623940157439, -+SNULL, 140624007266303, 140624037609471, -+STORE, 140623940157440, 140624007266303, -+STORE, 140624007266304, 140624037609471, -+ERASE, 140624007266304, 140624037609471, -+SNULL, 140623940292607, 140624007266303, -+STORE, 140623940157440, 140623940292607, -+STORE, 140623940292608, 140624007266303, -+SNULL, 140624037613567, 140624046002175, -+STORE, 140624037609472, 140624037613567, -+STORE, 140624037613568, 140624046002175, -+STORE, 140624029216768, 140624037609471, -+SNULL, 140624029220863, 140624037609471, -+STORE, 140624029216768, 140624029220863, -+STORE, 140624029220864, 140624037609471, -+STORE, 140624020824064, 140624029216767, -+SNULL, 140624020828159, 140624029216767, -+STORE, 140624020824064, 140624020828159, -+STORE, 140624020828160, 140624029216767, -+STORE, 140624012431360, 140624020824063, -+SNULL, 140624012435455, 140624020824063, -+STORE, 140624012431360, 140624012435455, -+STORE, 140624012435456, 140624020824063, -+STORE, 140623931764736, 140623940157439, -+STORE, 140623797547008, 140623931764735, -+SNULL, 140623797547008, 140623805939711, -+STORE, 140623805939712, 140623931764735, -+STORE, 140623797547008, 140623805939711, -+ERASE, 140623797547008, 140623805939711, -+SNULL, 140623873048575, 140623931764735, -+STORE, 140623805939712, 140623873048575, -+STORE, 140623873048576, 140623931764735, -+ERASE, 140623873048576, 140623931764735, -+STORE, 140623923372032, 140623940157439, -+STORE, 140623914979328, 140623940157439, -+STORE, 140623906586624, 140623940157439, -+STORE, 140623671721984, 140623873048575, -+SNULL, 140623738830847, 140623873048575, -+STORE, 140623671721984, 140623738830847, -+STORE, 140623738830848, 140623873048575, -+SNULL, 140623738830848, 140623805939711, -+STORE, 140623805939712, 140623873048575, -+STORE, 140623738830848, 140623805939711, -+ERASE, 140623738830848, 140623805939711, -+SNULL, 140623806074879, 140623873048575, -+STORE, 140623805939712, 140623806074879, -+STORE, 140623806074880, 140623873048575, -+SNULL, 140623906586624, 140623931764735, -+STORE, 140623931764736, 140623940157439, -+STORE, 140623906586624, 140623931764735, -+SNULL, 140623931768831, 140623940157439, -+STORE, 140623931764736, 140623931768831, -+STORE, 140623931768832, 140623940157439, -+STORE, 140623537504256, 140623738830847, -+SNULL, 140623537504256, 140623671721983, -+STORE, 140623671721984, 140623738830847, -+STORE, 140623537504256, 140623671721983, -+SNULL, 140623671857151, 140623738830847, -+STORE, 140623671721984, 140623671857151, -+STORE, 140623671857152, 140623738830847, -+SNULL, 140623604613119, 140623671721983, -+STORE, 140623537504256, 140623604613119, -+STORE, 140623604613120, 140623671721983, -+ERASE, 140623604613120, 140623671721983, -+SNULL, 140623537639423, 140623604613119, -+STORE, 140623537504256, 140623537639423, -+STORE, 140623537639424, 140623604613119, -+STORE, 140623537639424, 140623671721983, -+SNULL, 140623537639424, 140623604613119, -+STORE, 140623604613120, 140623671721983, -+STORE, 140623537639424, 140623604613119, -+SNULL, 140623604748287, 140623671721983, -+STORE, 140623604613120, 140623604748287, -+STORE, 140623604748288, 140623671721983, -+STORE, 140623898193920, 140623931764735, -+SNULL, 140623898193920, 140623923372031, -+STORE, 140623923372032, 140623931764735, -+STORE, 140623898193920, 140623923372031, -+SNULL, 140623923376127, 140623931764735, -+STORE, 140623923372032, 140623923376127, -+STORE, 140623923376128, 140623931764735, -+STORE, 140623889801216, 140623923372031, -+SNULL, 140623889801216, 140623898193919, -+STORE, 140623898193920, 140623923372031, -+STORE, 140623889801216, 140623898193919, -+SNULL, 140623898198015, 140623923372031, -+STORE, 140623898193920, 140623898198015, -+STORE, 140623898198016, 140623923372031, -+SNULL, 140623889805311, 140623898193919, -+STORE, 140623889801216, 140623889805311, -+STORE, 140623889805312, 140623898193919, -+SNULL, 140623898198016, 140623906586623, -+STORE, 140623906586624, 140623923372031, -+STORE, 140623898198016, 140623906586623, -+SNULL, 140623906590719, 140623923372031, -+STORE, 140623906586624, 140623906590719, -+STORE, 140623906590720, 140623923372031, -+STORE, 140623881408512, 140623889801215, -+SNULL, 140623906590720, 140623914979327, -+STORE, 140623914979328, 140623923372031, -+STORE, 140623906590720, 140623914979327, -+SNULL, 140623914983423, 140623923372031, -+STORE, 140623914979328, 140623914983423, -+STORE, 140623914983424, 140623923372031, -+SNULL, 140623881412607, 140623889801215, -+STORE, 140623881408512, 140623881412607, -+STORE, 140623881412608, 140623889801215, -+STORE, 140623797547008, 140623805939711, -+STORE, 140623789154304, 140623805939711, -+STORE, 140623780761600, 140623805939711, -+SNULL, 140623780761600, 140623789154303, -+STORE, 140623789154304, 140623805939711, -+STORE, 140623780761600, 140623789154303, -+SNULL, 140623789158399, 140623805939711, -+STORE, 140623789154304, 140623789158399, -+STORE, 140623789158400, 140623805939711, -+STORE, 140623772368896, 140623789154303, -+STORE, 140623763976192, 140623789154303, -+SNULL, 140623763976192, 140623780761599, -+STORE, 140623780761600, 140623789154303, -+STORE, 140623763976192, 140623780761599, -+SNULL, 140623780765695, 140623789154303, -+STORE, 140623780761600, 140623780765695, -+STORE, 140623780765696, 140623789154303, -+SNULL, 140623789158400, 140623797547007, -+STORE, 140623797547008, 140623805939711, -+STORE, 140623789158400, 140623797547007, -+SNULL, 140623797551103, 140623805939711, -+STORE, 140623797547008, 140623797551103, -+STORE, 140623797551104, 140623805939711, -+SNULL, 140623763976192, 140623772368895, -+STORE, 140623772368896, 140623780761599, -+STORE, 140623763976192, 140623772368895, -+SNULL, 140623772372991, 140623780761599, -+STORE, 140623772368896, 140623772372991, -+STORE, 140623772372992, 140623780761599, -+SNULL, 140623763980287, 140623772368895, -+STORE, 140623763976192, 140623763980287, -+STORE, 140623763980288, 140623772368895, -+STORE, 140623755583488, 140623763976191, -+STORE, 140623747190784, 140623763976191, -+SNULL, 140623747190784, 140623755583487, -+STORE, 140623755583488, 140623763976191, -+STORE, 140623747190784, 140623755583487, -+SNULL, 140623755587583, 140623763976191, -+STORE, 140623755583488, 140623755587583, -+STORE, 140623755587584, 140623763976191, -+STORE, 140623529111552, 140623537504255, -+SNULL, 140623747194879, 140623755583487, -+STORE, 140623747190784, 140623747194879, -+STORE, 140623747194880, 140623755583487, -+SNULL, 140623529115647, 140623537504255, -+STORE, 140623529111552, 140623529115647, -+STORE, 140623529115648, 140623537504255, -+STORE, 140623520718848, 140623529111551, -+SNULL, 140623520722943, 140623529111551, -+STORE, 140623520718848, 140623520722943, -+STORE, 140623520722944, 140623529111551, -+STORE, 140623512326144, 140623520718847, -+STORE, 140623503933440, 140623520718847, -+STORE, 140623495540736, 140623520718847, -+STORE, 140623361323008, 140623495540735, -+STORE, 140623227105280, 140623495540735, -+STORE, 140623218712576, 140623227105279, -+STORE, 140623084494848, 140623218712575, -+STORE, 140623076102144, 140623084494847, -+STORE, 140622941884416, 140623076102143, -+SNULL, 140622941884416, 140623000633343, -+STORE, 140623000633344, 140623076102143, -+STORE, 140622941884416, 140623000633343, -+ERASE, 140622941884416, 140623000633343, -+STORE, 140622992240640, 140623000633343, -+STORE, 140622983847936, 140623000633343, -+STORE, 140622849630208, 140622983847935, -+STORE, 140622841237504, 140622849630207, -+SNULL, 140622849630208, 140622866415615, -+STORE, 140622866415616, 140622983847935, -+STORE, 140622849630208, 140622866415615, -+ERASE, 140622849630208, 140622866415615, -+STORE, 140622858022912, 140622866415615, -+SNULL, 140622933524479, 140622983847935, -+STORE, 140622866415616, 140622933524479, -+STORE, 140622933524480, 140622983847935, -+ERASE, 140622933524480, 140622983847935, -+STORE, 140622975455232, 140623000633343, -+STORE, 140622707019776, 140622841237503, -+STORE, 140622967062528, 140623000633343, -+STORE, 140622572802048, 140622841237503, -+STORE, 140622958669824, 140623000633343, -+STORE, 140622438584320, 140622841237503, -+STORE, 140622950277120, 140623000633343, -+SNULL, 140622858027007, 140622866415615, -+STORE, 140622858022912, 140622858027007, -+STORE, 140622858027008, 140622866415615, -+STORE, 140622941884416, 140623000633343, -+STORE, 140622841237504, 140622858022911, -+SNULL, 140622841237504, 140622849630207, -+STORE, 140622849630208, 140622858022911, -+STORE, 140622841237504, 140622849630207, -+SNULL, 140622849634303, 140622858022911, -+STORE, 140622849630208, 140622849634303, -+STORE, 140622849634304, 140622858022911, -+STORE, 140622430191616, 140622438584319, -+SNULL, 140622430195711, 140622438584319, -+STORE, 140622430191616, 140622430195711, -+STORE, 140622430195712, 140622438584319, -+SNULL, 140623361323007, 140623495540735, -+STORE, 140623227105280, 140623361323007, -+STORE, 140623361323008, 140623495540735, -+SNULL, 140623361323008, 140623403286527, -+STORE, 140623403286528, 140623495540735, -+STORE, 140623361323008, 140623403286527, -+ERASE, 140623361323008, 140623403286527, -+SNULL, 140623470395391, 140623495540735, -+STORE, 140623403286528, 140623470395391, -+STORE, 140623470395392, 140623495540735, -+ERASE, 140623470395392, 140623495540735, -+SNULL, 140623227105280, 140623269068799, -+STORE, 140623269068800, 140623361323007, -+STORE, 140623227105280, 140623269068799, -+ERASE, 140623227105280, 140623269068799, -+SNULL, 140623084494848, 140623134851071, -+STORE, 140623134851072, 140623218712575, -+STORE, 140623084494848, 140623134851071, -+ERASE, 140623084494848, 140623134851071, -+SNULL, 140623201959935, 140623218712575, -+STORE, 140623134851072, 140623201959935, -+STORE, 140623201959936, 140623218712575, -+ERASE, 140623201959936, 140623218712575, -+SNULL, 140623067742207, 140623076102143, -+STORE, 140623000633344, 140623067742207, -+STORE, 140623067742208, 140623076102143, -+ERASE, 140623067742208, 140623076102143, -+STORE, 140622295973888, 140622430191615, -+SNULL, 140622295973888, 140622329544703, -+STORE, 140622329544704, 140622430191615, -+STORE, 140622295973888, 140622329544703, -+ERASE, 140622295973888, 140622329544703, -+SNULL, 140622866550783, 140622933524479, -+STORE, 140622866415616, 140622866550783, -+STORE, 140622866550784, 140622933524479, -+SNULL, 140622707019775, 140622841237503, -+STORE, 140622438584320, 140622707019775, -+STORE, 140622707019776, 140622841237503, -+SNULL, 140622707019776, 140622732197887, -+STORE, 140622732197888, 140622841237503, -+STORE, 140622707019776, 140622732197887, -+ERASE, 140622707019776, 140622732197887, -+SNULL, 140622799306751, 140622841237503, -+STORE, 140622732197888, 140622799306751, -+STORE, 140622799306752, 140622841237503, -+ERASE, 140622799306752, 140622841237503, -+SNULL, 140622572802047, 140622707019775, -+STORE, 140622438584320, 140622572802047, -+STORE, 140622572802048, 140622707019775, -+SNULL, 140622572802048, 140622597980159, -+STORE, 140622597980160, 140622707019775, -+STORE, 140622572802048, 140622597980159, -+ERASE, 140622572802048, 140622597980159, -+SNULL, 140622438584320, 140622463762431, -+STORE, 140622463762432, 140622572802047, -+STORE, 140622438584320, 140622463762431, -+ERASE, 140622438584320, 140622463762431, -+SNULL, 140622530871295, 140622572802047, -+STORE, 140622463762432, 140622530871295, -+STORE, 140622530871296, 140622572802047, -+ERASE, 140622530871296, 140622572802047, -+STORE, 140622195326976, 140622430191615, -+SNULL, 140622262435839, 140622430191615, -+STORE, 140622195326976, 140622262435839, -+STORE, 140622262435840, 140622430191615, -+SNULL, 140622262435840, 140622329544703, -+STORE, 140622329544704, 140622430191615, -+STORE, 140622262435840, 140622329544703, -+ERASE, 140622262435840, 140622329544703, -+SNULL, 140622841241599, 140622849630207, -+STORE, 140622841237504, 140622841241599, -+STORE, 140622841241600, 140622849630207, -+STORE, 140623487148032, 140623520718847, -+STORE, 140623478755328, 140623520718847, -+SNULL, 140622941884416, 140622983847935, -+STORE, 140622983847936, 140623000633343, -+STORE, 140622941884416, 140622983847935, -+SNULL, 140622983852031, 140623000633343, -+STORE, 140622983847936, 140622983852031, -+STORE, 140622983852032, 140623000633343, -+STORE, 140623394893824, 140623403286527, -+SNULL, 140623394897919, 140623403286527, -+STORE, 140623394893824, 140623394897919, -+STORE, 140623394897920, 140623403286527, -+SNULL, 140623403421695, 140623470395391, -+STORE, 140623403286528, 140623403421695, -+STORE, 140623403421696, 140623470395391, -+SNULL, 140623478755328, 140623503933439, -+STORE, 140623503933440, 140623520718847, -+STORE, 140623478755328, 140623503933439, -+SNULL, 140623503937535, 140623520718847, -+STORE, 140623503933440, 140623503937535, -+STORE, 140623503937536, 140623520718847, -+SNULL, 140623336177663, 140623361323007, -+STORE, 140623269068800, 140623336177663, -+STORE, 140623336177664, 140623361323007, -+ERASE, 140623336177664, 140623361323007, -+SNULL, 140623269203967, 140623336177663, -+STORE, 140623269068800, 140623269203967, -+STORE, 140623269203968, 140623336177663, -+SNULL, 140623134986239, 140623201959935, -+STORE, 140623134851072, 140623134986239, -+STORE, 140623134986240, 140623201959935, -+SNULL, 140623000768511, 140623067742207, -+STORE, 140623000633344, 140623000768511, -+STORE, 140623000768512, 140623067742207, -+SNULL, 140622396653567, 140622430191615, -+STORE, 140622329544704, 140622396653567, -+STORE, 140622396653568, 140622430191615, -+ERASE, 140622396653568, 140622430191615, -+SNULL, 140622732333055, 140622799306751, -+STORE, 140622732197888, 140622732333055, -+STORE, 140622732333056, 140622799306751, -+SNULL, 140622941884416, 140622975455231, -+STORE, 140622975455232, 140622983847935, -+STORE, 140622941884416, 140622975455231, -+SNULL, 140622975459327, 140622983847935, -+STORE, 140622975455232, 140622975459327, -+STORE, 140622975459328, 140622983847935, -+SNULL, 140622665089023, 140622707019775, -+STORE, 140622597980160, 140622665089023, -+STORE, 140622665089024, 140622707019775, -+ERASE, 140622665089024, 140622707019775, -+SNULL, 140622598115327, 140622665089023, -+STORE, 140622597980160, 140622598115327, -+STORE, 140622598115328, 140622665089023, -+SNULL, 140622463897599, 140622530871295, -+STORE, 140622463762432, 140622463897599, -+STORE, 140622463897600, 140622530871295, -+SNULL, 140622195462143, 140622262435839, -+STORE, 140622195326976, 140622195462143, -+STORE, 140622195462144, 140622262435839, -+STORE, 140623386501120, 140623394893823, -+SNULL, 140622941884416, 140622950277119, -+STORE, 140622950277120, 140622975455231, -+STORE, 140622941884416, 140622950277119, -+SNULL, 140622950281215, 140622975455231, -+STORE, 140622950277120, 140622950281215, -+STORE, 140622950281216, 140622975455231, -+SNULL, 140622941888511, 140622950277119, -+STORE, 140622941884416, 140622941888511, -+STORE, 140622941888512, 140622950277119, -+STORE, 140623378108416, 140623394893823, -+SNULL, 140623478755328, 140623495540735, -+STORE, 140623495540736, 140623503933439, -+STORE, 140623478755328, 140623495540735, -+SNULL, 140623495544831, 140623503933439, -+STORE, 140623495540736, 140623495544831, -+STORE, 140623495544832, 140623503933439, -+SNULL, 140623478755328, 140623487148031, -+STORE, 140623487148032, 140623495540735, -+STORE, 140623478755328, 140623487148031, -+SNULL, 140623487152127, 140623495540735, -+STORE, 140623487148032, 140623487152127, -+STORE, 140623487152128, 140623495540735, -+SNULL, 140623218716671, 140623227105279, -+STORE, 140623218712576, 140623218716671, -+STORE, 140623218716672, 140623227105279, -+SNULL, 140623076106239, 140623084494847, -+STORE, 140623076102144, 140623076106239, -+STORE, 140623076106240, 140623084494847, -+SNULL, 140622329679871, 140622396653567, -+STORE, 140622329544704, 140622329679871, -+STORE, 140622329679872, 140622396653567, -+SNULL, 140622950281216, 140622958669823, -+STORE, 140622958669824, 140622975455231, -+STORE, 140622950281216, 140622958669823, -+SNULL, 140622958673919, 140622975455231, -+STORE, 140622958669824, 140622958673919, -+STORE, 140622958673920, 140622975455231, -+SNULL, 140623503937536, 140623512326143, -+STORE, 140623512326144, 140623520718847, -+STORE, 140623503937536, 140623512326143, -+SNULL, 140623512330239, 140623520718847, -+STORE, 140623512326144, 140623512330239, -+STORE, 140623512330240, 140623520718847, -+SNULL, 140623378108416, 140623386501119, -+STORE, 140623386501120, 140623394893823, -+STORE, 140623378108416, 140623386501119, -+SNULL, 140623386505215, 140623394893823, -+STORE, 140623386501120, 140623386505215, -+STORE, 140623386505216, 140623394893823, -+STORE, 140623369715712, 140623386501119, -+STORE, 140623361323008, 140623386501119, -+STORE, 140623352930304, 140623386501119, -+SNULL, 140623352930304, 140623361323007, -+STORE, 140623361323008, 140623386501119, -+STORE, 140623352930304, 140623361323007, -+SNULL, 140623361327103, 140623386501119, -+STORE, 140623361323008, 140623361327103, -+STORE, 140623361327104, 140623386501119, -+SNULL, 140623478759423, 140623487148031, -+STORE, 140623478755328, 140623478759423, -+STORE, 140623478759424, 140623487148031, -+STORE, 140623344537600, 140623361323007, -+STORE, 140623260676096, 140623269068799, -+SNULL, 140622958673920, 140622967062527, -+STORE, 140622967062528, 140622975455231, -+STORE, 140622958673920, 140622967062527, -+SNULL, 140622967066623, 140622975455231, -+STORE, 140622967062528, 140622967066623, -+STORE, 140622967066624, 140622975455231, -+STORE, 140623252283392, 140623269068799, -+STORE, 140623243890688, 140623269068799, -+SNULL, 140622983852032, 140622992240639, -+STORE, 140622992240640, 140623000633343, -+STORE, 140622983852032, 140622992240639, -+SNULL, 140622992244735, 140623000633343, -+STORE, 140622992240640, 140622992244735, -+STORE, 140622992244736, 140623000633343, -+STORE, 140623235497984, 140623269068799, -+STORE, 140623218716672, 140623235497983, -+STORE, 140623210319872, 140623218712575, -+STORE, 140623126458368, 140623134851071, -+SNULL, 140623210323967, 140623218712575, -+STORE, 140623210319872, 140623210323967, -+STORE, 140623210323968, 140623218712575, -+SNULL, 140623218716672, 140623227105279, -+STORE, 140623227105280, 140623235497983, -+STORE, 140623218716672, 140623227105279, -+SNULL, 140623227109375, 140623235497983, -+STORE, 140623227105280, 140623227109375, -+STORE, 140623227109376, 140623235497983, -+STORE, 140623118065664, 140623134851071, -+STORE, 140623109672960, 140623134851071, -+SNULL, 140623109677055, 140623134851071, -+STORE, 140623109672960, 140623109677055, -+STORE, 140623109677056, 140623134851071, -+STORE, 140623101280256, 140623109672959, -+STORE, 140623092887552, 140623109672959, -+SNULL, 140623092887552, 140623101280255, -+STORE, 140623101280256, 140623109672959, -+STORE, 140623092887552, 140623101280255, -+SNULL, 140623101284351, 140623109672959, -+STORE, 140623101280256, 140623101284351, -+STORE, 140623101284352, 140623109672959, -+SNULL, 140623361327104, 140623378108415, -+STORE, 140623378108416, 140623386501119, -+STORE, 140623361327104, 140623378108415, -+SNULL, 140623378112511, 140623386501119, -+STORE, 140623378108416, 140623378112511, -+STORE, 140623378112512, 140623386501119, -+SNULL, 140623235497984, 140623243890687, -+STORE, 140623243890688, 140623269068799, -+STORE, 140623235497984, 140623243890687, -+SNULL, 140623243894783, 140623269068799, -+STORE, 140623243890688, 140623243894783, -+STORE, 140623243894784, 140623269068799, -+SNULL, 140623361327104, 140623369715711, -+STORE, 140623369715712, 140623378108415, -+STORE, 140623361327104, 140623369715711, -+SNULL, 140623369719807, 140623378108415, -+STORE, 140623369715712, 140623369719807, -+STORE, 140623369719808, 140623378108415, -+SNULL, 140623243894784, 140623252283391, -+STORE, 140623252283392, 140623269068799, -+STORE, 140623243894784, 140623252283391, -+SNULL, 140623252287487, 140623269068799, -+STORE, 140623252283392, 140623252287487, -+STORE, 140623252287488, 140623269068799, -+SNULL, 140623235502079, 140623243890687, -+STORE, 140623235497984, 140623235502079, -+STORE, 140623235502080, 140623243890687, -+SNULL, 140623344541695, 140623361323007, -+STORE, 140623344537600, 140623344541695, -+STORE, 140623344541696, 140623361323007, -+STORE, 140623076106240, 140623092887551, -+SNULL, 140623076106240, 140623084494847, -+STORE, 140623084494848, 140623092887551, -+STORE, 140623076106240, 140623084494847, -+SNULL, 140623084498943, 140623092887551, -+STORE, 140623084494848, 140623084498943, -+STORE, 140623084498944, 140623092887551, -+SNULL, 140623344541696, 140623352930303, -+STORE, 140623352930304, 140623361323007, -+STORE, 140623344541696, 140623352930303, -+SNULL, 140623352934399, 140623361323007, -+STORE, 140623352930304, 140623352934399, -+STORE, 140623352934400, 140623361323007, -+SNULL, 140623109677056, 140623118065663, -+STORE, 140623118065664, 140623134851071, -+STORE, 140623109677056, 140623118065663, -+SNULL, 140623118069759, 140623134851071, -+STORE, 140623118065664, 140623118069759, -+STORE, 140623118069760, 140623134851071, -+STORE, 140622832844800, 140622841237503, -+STORE, 140622824452096, 140622841237503, -+SNULL, 140622824452096, 140622832844799, -+STORE, 140622832844800, 140622841237503, -+STORE, 140622824452096, 140622832844799, -+SNULL, 140622832848895, 140622841237503, -+STORE, 140622832844800, 140622832848895, -+STORE, 140622832848896, 140622841237503, -+STORE, 140622816059392, 140622832844799, -+SNULL, 140623092891647, 140623101280255, -+STORE, 140623092887552, 140623092891647, -+STORE, 140623092891648, 140623101280255, -+SNULL, 140623118069760, 140623126458367, -+STORE, 140623126458368, 140623134851071, -+STORE, 140623118069760, 140623126458367, -+SNULL, 140623126462463, 140623134851071, -+STORE, 140623126458368, 140623126462463, -+STORE, 140623126462464, 140623134851071, -+SNULL, 140623252287488, 140623260676095, -+STORE, 140623260676096, 140623269068799, -+STORE, 140623252287488, 140623260676095, -+SNULL, 140623260680191, 140623269068799, -+STORE, 140623260676096, 140623260680191, -+STORE, 140623260680192, 140623269068799, -+STORE, 140622807666688, 140622832844799, -+STORE, 140622723805184, 140622732197887, -+STORE, 140622715412480, 140622732197887, -+STORE, 140622707019776, 140622732197887, -+SNULL, 140622707023871, 140622732197887, -+STORE, 140622707019776, 140622707023871, -+STORE, 140622707023872, 140622732197887, -+STORE, 140622698627072, 140622707019775, -+STORE, 140622690234368, 140622707019775, -+SNULL, 140622690238463, 140622707019775, -+STORE, 140622690234368, 140622690238463, -+STORE, 140622690238464, 140622707019775, -+SNULL, 140622807666688, 140622816059391, -+STORE, 140622816059392, 140622832844799, -+STORE, 140622807666688, 140622816059391, -+SNULL, 140622816063487, 140622832844799, -+STORE, 140622816059392, 140622816063487, -+STORE, 140622816063488, 140622832844799, -+STORE, 140622681841664, 140622690234367, -+STORE, 140622673448960, 140622690234367, -+SNULL, 140622673453055, 140622690234367, -+STORE, 140622673448960, 140622673453055, -+STORE, 140622673453056, 140622690234367, -+STORE, 140622589587456, 140622597980159, -+SNULL, 140622807670783, 140622816059391, -+STORE, 140622807666688, 140622807670783, -+STORE, 140622807670784, 140622816059391, -+STORE, 140622581194752, 140622597980159, -+SNULL, 140622581198847, 140622597980159, -+STORE, 140622581194752, 140622581198847, -+STORE, 140622581198848, 140622597980159, -+SNULL, 140622816063488, 140622824452095, -+STORE, 140622824452096, 140622832844799, -+STORE, 140622816063488, 140622824452095, -+SNULL, 140622824456191, 140622832844799, -+STORE, 140622824452096, 140622824456191, -+STORE, 140622824456192, 140622832844799, -+STORE, 140622572802048, 140622581194751, -+SNULL, 140622572806143, 140622581194751, -+STORE, 140622572802048, 140622572806143, -+STORE, 140622572806144, 140622581194751, -+STORE, 140622564409344, 140622572802047, -+STORE, 140622556016640, 140622572802047, -+SNULL, 140622556016640, 140622564409343, -+STORE, 140622564409344, 140622572802047, -+STORE, 140622556016640, 140622564409343, -+SNULL, 140622564413439, 140622572802047, -+STORE, 140622564409344, 140622564413439, -+STORE, 140622564413440, 140622572802047, -+SNULL, 140622690238464, 140622698627071, -+STORE, 140622698627072, 140622707019775, -+STORE, 140622690238464, 140622698627071, -+SNULL, 140622698631167, 140622707019775, -+STORE, 140622698627072, 140622698631167, -+STORE, 140622698631168, 140622707019775, -+SNULL, 140622707023872, 140622723805183, -+STORE, 140622723805184, 140622732197887, -+STORE, 140622707023872, 140622723805183, -+SNULL, 140622723809279, 140622732197887, -+STORE, 140622723805184, 140622723809279, -+STORE, 140622723809280, 140622732197887, -+SNULL, 140622707023872, 140622715412479, -+STORE, 140622715412480, 140622723805183, -+STORE, 140622707023872, 140622715412479, -+SNULL, 140622715416575, 140622723805183, -+STORE, 140622715412480, 140622715416575, -+STORE, 140622715416576, 140622723805183, -+STORE, 140622547623936, 140622564409343, -+SNULL, 140622547628031, 140622564409343, -+STORE, 140622547623936, 140622547628031, -+STORE, 140622547628032, 140622564409343, -+STORE, 140622539231232, 140622547623935, -+SNULL, 140622539235327, 140622547623935, -+STORE, 140622539231232, 140622539235327, -+STORE, 140622539235328, 140622547623935, -+SNULL, 140622581198848, 140622589587455, -+STORE, 140622589587456, 140622597980159, -+STORE, 140622581198848, 140622589587455, -+SNULL, 140622589591551, 140622597980159, -+STORE, 140622589587456, 140622589591551, -+STORE, 140622589591552, 140622597980159, -+STORE, 140622455369728, 140622463762431, -+SNULL, 140622455373823, 140622463762431, -+STORE, 140622455369728, 140622455373823, -+STORE, 140622455373824, 140622463762431, -+STORE, 140622446977024, 140622455369727, -+SNULL, 140622446981119, 140622455369727, -+STORE, 140622446977024, 140622446981119, -+STORE, 140622446981120, 140622455369727, -+SNULL, 140622547628032, 140622556016639, -+STORE, 140622556016640, 140622564409343, -+STORE, 140622547628032, 140622556016639, -+SNULL, 140622556020735, 140622564409343, -+STORE, 140622556016640, 140622556020735, -+STORE, 140622556020736, 140622564409343, -+STORE, 140622430195712, 140622446977023, -+STORE, 140622421798912, 140622430191615, -+SNULL, 140622430195712, 140622438584319, -+STORE, 140622438584320, 140622446977023, -+STORE, 140622430195712, 140622438584319, -+SNULL, 140622438588415, 140622446977023, -+STORE, 140622438584320, 140622438588415, -+STORE, 140622438588416, 140622446977023, -+STORE, 140622413406208, 140622430191615, -+STORE, 140622405013504, 140622430191615, -+SNULL, 140622405013504, 140622413406207, -+STORE, 140622413406208, 140622430191615, -+STORE, 140622405013504, 140622413406207, -+SNULL, 140622413410303, 140622430191615, -+STORE, 140622413406208, 140622413410303, -+STORE, 140622413410304, 140622430191615, -+SNULL, 140622673453056, 140622681841663, -+STORE, 140622681841664, 140622690234367, -+STORE, 140622673453056, 140622681841663, -+SNULL, 140622681845759, 140622690234367, -+STORE, 140622681841664, 140622681845759, -+STORE, 140622681845760, 140622690234367, -+STORE, 140622321152000, 140622329544703, -+SNULL, 140622413410304, 140622421798911, -+STORE, 140622421798912, 140622430191615, -+STORE, 140622413410304, 140622421798911, -+SNULL, 140622421803007, 140622430191615, -+STORE, 140622421798912, 140622421803007, -+STORE, 140622421803008, 140622430191615, -+STORE, 140622312759296, 140622329544703, -+SNULL, 140622312763391, 140622329544703, -+STORE, 140622312759296, 140622312763391, -+STORE, 140622312763392, 140622329544703, -+SNULL, 140622405017599, 140622413406207, -+STORE, 140622405013504, 140622405017599, -+STORE, 140622405017600, 140622413406207, -+STORE, 140622304366592, 140622312759295, -+SNULL, 140622304370687, 140622312759295, -+STORE, 140622304366592, 140622304370687, -+STORE, 140622304370688, 140622312759295, -+SNULL, 140622312763392, 140622321151999, -+STORE, 140622321152000, 140622329544703, -+STORE, 140622312763392, 140622321151999, -+SNULL, 140622321156095, 140622329544703, -+STORE, 140622321152000, 140622321156095, -+STORE, 140622321156096, 140622329544703, -+STORE, 140624062619648, 140624062648319, -+STORE, 140624010240000, 140624012431359, -+SNULL, 140624010240000, 140624010330111, -+STORE, 140624010330112, 140624012431359, -+STORE, 140624010240000, 140624010330111, -+SNULL, 140624012423167, 140624012431359, -+STORE, 140624010330112, 140624012423167, -+STORE, 140624012423168, 140624012431359, -+ERASE, 140624012423168, 140624012431359, -+STORE, 140624012423168, 140624012431359, -+SNULL, 140624012427263, 140624012431359, -+STORE, 140624012423168, 140624012427263, -+STORE, 140624012427264, 140624012431359, -+ERASE, 140624062619648, 140624062648319, -+ERASE, 140622849630208, 140622849634303, -+ERASE, 140622849634304, 140622858022911, -+ERASE, 140623394893824, 140623394897919, -+ERASE, 140623394897920, 140623403286527, -+ERASE, 140623361323008, 140623361327103, -+ERASE, 140623361327104, 140623369715711, -+ERASE, 140623084494848, 140623084498943, -+ERASE, 140623084498944, 140623092887551, -+ERASE, 140623931764736, 140623931768831, -+ERASE, 140623931768832, 140623940157439, -+ERASE, 140622841237504, 140622841241599, -+ERASE, 140622841241600, 140622849630207, -+ERASE, 140623487148032, 140623487152127, -+ERASE, 140623487152128, 140623495540735, -+ERASE, 140623109672960, 140623109677055, -+ERASE, 140623109677056, 140623118065663, -+ERASE, 140622983847936, 140622983852031, -+ERASE, 140622983852032, 140622992240639, -+ERASE, 140623352930304, 140623352934399, -+ERASE, 140623352934400, 140623361323007, -+ERASE, 140622564409344, 140622564413439, -+ERASE, 140622564413440, 140622572802047, -+ERASE, 140622430191616, 140622430195711, -+ERASE, 140622430195712, 140622438584319, -+ERASE, 140622958669824, 140622958673919, -+ERASE, 140622958673920, 140622967062527, -+ERASE, 140622992240640, 140622992244735, -+ERASE, 140622992244736, 140623000633343, -+ERASE, 140623227105280, 140623227109375, -+ERASE, 140623227109376, 140623235497983, -+ERASE, 140622321152000, 140622321156095, -+ERASE, 140622321156096, 140622329544703, -+ERASE, 140622858022912, 140622858027007, -+ERASE, 140622858027008, 140622866415615, -+ERASE, 140622975455232, 140622975459327, -+ERASE, 140622975459328, 140622983847935, -+ERASE, 140623378108416, 140623378112511, -+ERASE, 140623378112512, 140623386501119, -+ERASE, 140623495540736, 140623495544831, -+ERASE, 140623495544832, 140623503933439, -+ERASE, 140623118065664, 140623118069759, -+ERASE, 140623118069760, 140623126458367, -+ERASE, 140622572802048, 140622572806143, -+ERASE, 140622572806144, 140622581194751, -+ERASE, 140622421798912, 140622421803007, -+ERASE, 140622421803008, 140622430191615, -+ERASE, 140622967062528, 140622967066623, -+ERASE, 140622967066624, 140622975455231, -+ERASE, 140623252283392, 140623252287487, -+ERASE, 140623252287488, 140623260676095, -+ERASE, 140622673448960, 140622673453055, -+ERASE, 140622673453056, 140622681841663, -+ERASE, 140623076102144, 140623076106239, -+ERASE, 140623076106240, 140623084494847, -+ERASE, 140623101280256, 140623101284351, -+ERASE, 140623101284352, 140623109672959, -+ERASE, 140622715412480, 140622715416575, -+ERASE, 140622715416576, 140622723805183, -+ERASE, 140622405013504, 140622405017599, -+ERASE, 140622405017600, 140622413406207, -+ERASE, 140623478755328, 140623478759423, -+ERASE, 140623478759424, 140623487148031, -+ERASE, 140623906586624, 140623906590719, -+ERASE, 140623906590720, 140623914979327, -+ERASE, 140622950277120, 140622950281215, -+ERASE, 140622950281216, 140622958669823, -+ }; -+ unsigned long set32[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140731244212224, 140737488351231, -+SNULL, 140731244216319, 140737488351231, -+STORE, 140731244212224, 140731244216319, -+STORE, 140731244081152, 140731244216319, -+STORE, 94427773984768, 94427776237567, -+SNULL, 94427774115839, 94427776237567, -+STORE, 94427773984768, 94427774115839, -+STORE, 94427774115840, 94427776237567, -+ERASE, 94427774115840, 94427776237567, -+STORE, 94427776208896, 94427776217087, -+STORE, 94427776217088, 94427776237567, -+STORE, 140401464893440, 140401467146239, -+SNULL, 140401465036799, 140401467146239, -+STORE, 140401464893440, 140401465036799, -+STORE, 140401465036800, 140401467146239, -+ERASE, 140401465036800, 140401467146239, -+STORE, 140401467133952, 140401467142143, -+STORE, 140401467142144, 140401467146239, -+STORE, 140731244507136, 140731244511231, -+STORE, 140731244494848, 140731244507135, -+STORE, 140401467105280, 140401467133951, -+STORE, 140401467097088, 140401467105279, -+STORE, 140401462677504, 140401464893439, -+SNULL, 140401462677504, 140401462775807, -+STORE, 140401462775808, 140401464893439, -+STORE, 140401462677504, 140401462775807, -+SNULL, 140401464868863, 140401464893439, -+STORE, 140401462775808, 140401464868863, -+STORE, 140401464868864, 140401464893439, -+SNULL, 140401464868864, 140401464877055, -+STORE, 140401464877056, 140401464893439, -+STORE, 140401464868864, 140401464877055, -+ERASE, 140401464868864, 140401464877055, -+STORE, 140401464868864, 140401464877055, -+ERASE, 140401464877056, 140401464893439, -+STORE, 140401464877056, 140401464893439, -+STORE, 140401458880512, 140401462677503, -+SNULL, 140401458880512, 140401460539391, -+STORE, 140401460539392, 140401462677503, -+STORE, 140401458880512, 140401460539391, -+SNULL, 140401462636543, 140401462677503, -+STORE, 140401460539392, 140401462636543, -+STORE, 140401462636544, 140401462677503, -+SNULL, 140401462636544, 140401462661119, -+STORE, 140401462661120, 140401462677503, -+STORE, 140401462636544, 140401462661119, -+ERASE, 140401462636544, 140401462661119, -+STORE, 140401462636544, 140401462661119, -+ERASE, 140401462661120, 140401462677503, -+STORE, 140401462661120, 140401462677503, -+STORE, 140401467088896, 140401467105279, -+SNULL, 140401462652927, 140401462661119, -+STORE, 140401462636544, 140401462652927, -+STORE, 140401462652928, 140401462661119, -+SNULL, 140401464872959, 140401464877055, -+STORE, 140401464868864, 140401464872959, -+STORE, 140401464872960, 140401464877055, -+SNULL, 94427776212991, 94427776217087, -+STORE, 94427776208896, 94427776212991, -+STORE, 94427776212992, 94427776217087, -+SNULL, 140401467138047, 140401467142143, -+STORE, 140401467133952, 140401467138047, -+STORE, 140401467138048, 140401467142143, -+ERASE, 140401467105280, 140401467133951, -+STORE, 94427784683520, 94427784818687, -+STORE, 140401450487808, 140401458880511, -+SNULL, 140401450491903, 140401458880511, -+STORE, 140401450487808, 140401450491903, -+STORE, 140401450491904, 140401458880511, -+STORE, 140401442095104, 140401450487807, -+STORE, 140401307877376, 140401442095103, -+SNULL, 140401307877376, 140401340055551, -+STORE, 140401340055552, 140401442095103, -+STORE, 140401307877376, 140401340055551, -+ERASE, 140401307877376, 140401340055551, -+SNULL, 140401407164415, 140401442095103, -+STORE, 140401340055552, 140401407164415, -+STORE, 140401407164416, 140401442095103, -+ERASE, 140401407164416, 140401442095103, -+SNULL, 140401340190719, 140401407164415, -+STORE, 140401340055552, 140401340190719, -+STORE, 140401340190720, 140401407164415, -+SNULL, 140401442099199, 140401450487807, -+STORE, 140401442095104, 140401442099199, -+STORE, 140401442099200, 140401450487807, -+STORE, 140401433702400, 140401442095103, -+SNULL, 140401433706495, 140401442095103, -+STORE, 140401433702400, 140401433706495, -+STORE, 140401433706496, 140401442095103, -+STORE, 140401425309696, 140401433702399, -+SNULL, 140401425313791, 140401433702399, -+STORE, 140401425309696, 140401425313791, -+STORE, 140401425313792, 140401433702399, -+STORE, 140401416916992, 140401425309695, -+SNULL, 140401416921087, 140401425309695, -+STORE, 140401416916992, 140401416921087, -+STORE, 140401416921088, 140401425309695, -+STORE, 140401408524288, 140401416916991, -+STORE, 140401205837824, 140401340055551, -+SNULL, 140401272946687, 140401340055551, -+STORE, 140401205837824, 140401272946687, -+STORE, 140401272946688, 140401340055551, -+ERASE, 140401272946688, 140401340055551, -+SNULL, 140401205972991, 140401272946687, -+STORE, 140401205837824, 140401205972991, -+STORE, 140401205972992, 140401272946687, -+STORE, 140401331662848, 140401340055551, -+STORE, 140401323270144, 140401340055551, -+STORE, 140401138728960, 140401205837823, -+STORE, 140401314877440, 140401340055551, -+SNULL, 140401408528383, 140401416916991, -+STORE, 140401408524288, 140401408528383, -+STORE, 140401408528384, 140401416916991, -+SNULL, 140401138864127, 140401205837823, -+STORE, 140401138728960, 140401138864127, -+STORE, 140401138864128, 140401205837823, -+STORE, 140401004511232, 140401138728959, -+SNULL, 140401071620095, 140401138728959, -+STORE, 140401004511232, 140401071620095, -+STORE, 140401071620096, 140401138728959, -+ERASE, 140401071620096, 140401138728959, -+STORE, 140400870293504, 140401071620095, -+SNULL, 140400937402367, 140401071620095, -+STORE, 140400870293504, 140400937402367, -+STORE, 140400937402368, 140401071620095, -+SNULL, 140400937402368, 140401004511231, -+STORE, 140401004511232, 140401071620095, -+STORE, 140400937402368, 140401004511231, -+ERASE, 140400937402368, 140401004511231, -+STORE, 140401306484736, 140401340055551, -+SNULL, 140401306484736, 140401323270143, -+STORE, 140401323270144, 140401340055551, -+STORE, 140401306484736, 140401323270143, -+SNULL, 140401323274239, 140401340055551, -+STORE, 140401323270144, 140401323274239, -+STORE, 140401323274240, 140401340055551, -+SNULL, 140401004646399, 140401071620095, -+STORE, 140401004511232, 140401004646399, -+STORE, 140401004646400, 140401071620095, -+SNULL, 140400870428671, 140400937402367, -+STORE, 140400870293504, 140400870428671, -+STORE, 140400870428672, 140400937402367, -+SNULL, 140401306488831, 140401323270143, -+STORE, 140401306484736, 140401306488831, -+STORE, 140401306488832, 140401323270143, -+STORE, 140401298092032, 140401306484735, -+SNULL, 140401306488832, 140401314877439, -+STORE, 140401314877440, 140401323270143, -+STORE, 140401306488832, 140401314877439, -+SNULL, 140401314881535, 140401323270143, -+STORE, 140401314877440, 140401314881535, -+STORE, 140401314881536, 140401323270143, -+SNULL, 140401323274240, 140401331662847, -+STORE, 140401331662848, 140401340055551, -+STORE, 140401323274240, 140401331662847, -+SNULL, 140401331666943, 140401340055551, -+STORE, 140401331662848, 140401331666943, -+STORE, 140401331666944, 140401340055551, -+SNULL, 140401298096127, 140401306484735, -+STORE, 140401298092032, 140401298096127, -+STORE, 140401298096128, 140401306484735, -+STORE, 140401289699328, 140401298092031, -+STORE, 140401281306624, 140401298092031, -+STORE, 140401130336256, 140401138728959, -+SNULL, 140401281306624, 140401289699327, -+STORE, 140401289699328, 140401298092031, -+STORE, 140401281306624, 140401289699327, -+SNULL, 140401289703423, 140401298092031, -+STORE, 140401289699328, 140401289703423, -+STORE, 140401289703424, 140401298092031, -+STORE, 140401121943552, 140401138728959, -+STORE, 140401113550848, 140401138728959, -+SNULL, 140401281310719, 140401289699327, -+STORE, 140401281306624, 140401281310719, -+STORE, 140401281310720, 140401289699327, -+SNULL, 140401113550848, 140401121943551, -+STORE, 140401121943552, 140401138728959, -+STORE, 140401113550848, 140401121943551, -+SNULL, 140401121947647, 140401138728959, -+STORE, 140401121943552, 140401121947647, -+STORE, 140401121947648, 140401138728959, -+STORE, 140401105158144, 140401121943551, -+SNULL, 140401121947648, 140401130336255, -+STORE, 140401130336256, 140401138728959, -+STORE, 140401121947648, 140401130336255, -+SNULL, 140401130340351, 140401138728959, -+STORE, 140401130336256, 140401130340351, -+STORE, 140401130340352, 140401138728959, -+STORE, 140401096765440, 140401121943551, -+SNULL, 140401096765440, 140401113550847, -+STORE, 140401113550848, 140401121943551, -+STORE, 140401096765440, 140401113550847, -+SNULL, 140401113554943, 140401121943551, -+STORE, 140401113550848, 140401113554943, -+STORE, 140401113554944, 140401121943551, -+STORE, 140401088372736, 140401113550847, -+SNULL, 140401088372736, 140401096765439, -+STORE, 140401096765440, 140401113550847, -+STORE, 140401088372736, 140401096765439, -+SNULL, 140401096769535, 140401113550847, -+STORE, 140401096765440, 140401096769535, -+STORE, 140401096769536, 140401113550847, -+SNULL, 140401096769536, 140401105158143, -+STORE, 140401105158144, 140401113550847, -+STORE, 140401096769536, 140401105158143, -+SNULL, 140401105162239, 140401113550847, -+STORE, 140401105158144, 140401105162239, -+STORE, 140401105162240, 140401113550847, -+SNULL, 140401088376831, 140401096765439, -+STORE, 140401088372736, 140401088376831, -+STORE, 140401088376832, 140401096765439, -+STORE, 140401079980032, 140401088372735, -+STORE, 140400996118528, 140401004511231, -+SNULL, 140401079984127, 140401088372735, -+STORE, 140401079980032, 140401079984127, -+STORE, 140401079984128, 140401088372735, -+SNULL, 140400996122623, 140401004511231, -+STORE, 140400996118528, 140400996122623, -+STORE, 140400996122624, 140401004511231, -+STORE, 140400987725824, 140400996118527, -+STORE, 140400979333120, 140400996118527, -+STORE, 140400803184640, 140400870293503, -+SNULL, 140400803319807, 140400870293503, -+STORE, 140400803184640, 140400803319807, -+STORE, 140400803319808, 140400870293503, -+SNULL, 140400979333120, 140400987725823, -+STORE, 140400987725824, 140400996118527, -+STORE, 140400979333120, 140400987725823, -+SNULL, 140400987729919, 140400996118527, -+STORE, 140400987725824, 140400987729919, -+STORE, 140400987729920, 140400996118527, -+STORE, 140400970940416, 140400987725823, -+STORE, 140400962547712, 140400987725823, -+STORE, 140400668966912, 140400803184639, -+STORE, 140400954155008, 140400987725823, -+STORE, 140400945762304, 140400987725823, -+STORE, 140400660574208, 140400668966911, -+STORE, 140400593465344, 140400660574207, -+STORE, 140400585072640, 140400593465343, -+STORE, 140400450854912, 140400585072639, -+STORE, 140400442462208, 140400450854911, -+STORE, 140400434069504, 140400450854911, -+STORE, 140400299851776, 140400434069503, -+STORE, 140400291459072, 140400299851775, -+SNULL, 140400299851776, 140400333422591, -+STORE, 140400333422592, 140400434069503, -+STORE, 140400299851776, 140400333422591, -+ERASE, 140400299851776, 140400333422591, -+STORE, 140400325029888, 140400333422591, -+STORE, 140400157241344, 140400291459071, -+STORE, 140400316637184, 140400333422591, -+STORE, 140400308244480, 140400333422591, -+STORE, 140400023023616, 140400291459071, -+STORE, 140400291459072, 140400333422591, -+SNULL, 140400023023616, 140400064987135, -+STORE, 140400064987136, 140400291459071, -+STORE, 140400023023616, 140400064987135, -+ERASE, 140400023023616, 140400064987135, -+STORE, 140400056594432, 140400064987135, -+SNULL, 140400056598527, 140400064987135, -+STORE, 140400056594432, 140400056598527, -+STORE, 140400056598528, 140400064987135, -+STORE, 140399989485568, 140400056594431, -+SNULL, 140400291459072, 140400316637183, -+STORE, 140400316637184, 140400333422591, -+STORE, 140400291459072, 140400316637183, -+SNULL, 140400316641279, 140400333422591, -+STORE, 140400316637184, 140400316641279, -+STORE, 140400316641280, 140400333422591, -+STORE, 140399855267840, 140400056594431, -+SNULL, 140399855267840, 140399863660543, -+STORE, 140399863660544, 140400056594431, -+STORE, 140399855267840, 140399863660543, -+ERASE, 140399855267840, 140399863660543, -+SNULL, 140400736075775, 140400803184639, -+STORE, 140400668966912, 140400736075775, -+STORE, 140400736075776, 140400803184639, -+ERASE, 140400736075776, 140400803184639, -+SNULL, 140400669102079, 140400736075775, -+STORE, 140400668966912, 140400669102079, -+STORE, 140400669102080, 140400736075775, -+STORE, 140400669102080, 140400803184639, -+SNULL, 140400669102080, 140400736075775, -+STORE, 140400736075776, 140400803184639, -+STORE, 140400669102080, 140400736075775, -+SNULL, 140400736210943, 140400803184639, -+STORE, 140400736075776, 140400736210943, -+STORE, 140400736210944, 140400803184639, -+ERASE, 140400593465344, 140400660574207, -+SNULL, 140400450854912, 140400467640319, -+STORE, 140400467640320, 140400585072639, -+STORE, 140400450854912, 140400467640319, -+ERASE, 140400450854912, 140400467640319, -+STORE, 140399729442816, 140400056594431, -+SNULL, 140400400531455, 140400434069503, -+STORE, 140400333422592, 140400400531455, -+STORE, 140400400531456, 140400434069503, -+ERASE, 140400400531456, 140400434069503, -+SNULL, 140400333557759, 140400400531455, -+STORE, 140400333422592, 140400333557759, -+STORE, 140400333557760, 140400400531455, -+SNULL, 140400157241343, 140400291459071, -+STORE, 140400064987136, 140400157241343, -+STORE, 140400157241344, 140400291459071, -+SNULL, 140400157241344, 140400199204863, -+STORE, 140400199204864, 140400291459071, -+STORE, 140400157241344, 140400199204863, -+ERASE, 140400157241344, 140400199204863, -+SNULL, 140400266313727, 140400291459071, -+STORE, 140400199204864, 140400266313727, -+STORE, 140400266313728, 140400291459071, -+ERASE, 140400266313728, 140400291459071, -+SNULL, 140400132095999, 140400157241343, -+STORE, 140400064987136, 140400132095999, -+STORE, 140400132096000, 140400157241343, -+ERASE, 140400132096000, 140400157241343, -+SNULL, 140400065122303, 140400132095999, -+STORE, 140400064987136, 140400065122303, -+STORE, 140400065122304, 140400132095999, -+SNULL, 140400945762304, 140400954155007, -+STORE, 140400954155008, 140400987725823, -+STORE, 140400945762304, 140400954155007, -+SNULL, 140400954159103, 140400987725823, -+STORE, 140400954155008, 140400954159103, -+STORE, 140400954159104, 140400987725823, -+SNULL, 140400434069504, 140400442462207, -+STORE, 140400442462208, 140400450854911, -+STORE, 140400434069504, 140400442462207, -+SNULL, 140400442466303, 140400450854911, -+STORE, 140400442462208, 140400442466303, -+STORE, 140400442466304, 140400450854911, -+SNULL, 140400291463167, 140400316637183, -+STORE, 140400291459072, 140400291463167, -+STORE, 140400291463168, 140400316637183, -+STORE, 140400652181504, 140400668966911, -+STORE, 140400643788800, 140400668966911, -+SNULL, 140400291463168, 140400299851775, -+STORE, 140400299851776, 140400316637183, -+STORE, 140400291463168, 140400299851775, -+SNULL, 140400299855871, 140400316637183, -+STORE, 140400299851776, 140400299855871, -+STORE, 140400299855872, 140400316637183, -+STORE, 140400635396096, 140400668966911, -+SNULL, 140400635396096, 140400643788799, -+STORE, 140400643788800, 140400668966911, -+STORE, 140400635396096, 140400643788799, -+SNULL, 140400643792895, 140400668966911, -+STORE, 140400643788800, 140400643792895, -+STORE, 140400643792896, 140400668966911, -+SNULL, 140399989485567, 140400056594431, -+STORE, 140399729442816, 140399989485567, -+STORE, 140399989485568, 140400056594431, -+ERASE, 140399989485568, 140400056594431, -+SNULL, 140399930769407, 140399989485567, -+STORE, 140399729442816, 140399930769407, -+STORE, 140399930769408, 140399989485567, -+ERASE, 140399930769408, 140399989485567, -+SNULL, 140400945766399, 140400954155007, -+STORE, 140400945762304, 140400945766399, -+STORE, 140400945766400, 140400954155007, -+SNULL, 140400534749183, 140400585072639, -+STORE, 140400467640320, 140400534749183, -+STORE, 140400534749184, 140400585072639, -+ERASE, 140400534749184, 140400585072639, -+SNULL, 140399796551679, 140399930769407, -+STORE, 140399729442816, 140399796551679, -+STORE, 140399796551680, 140399930769407, -+SNULL, 140399796551680, 140399863660543, -+STORE, 140399863660544, 140399930769407, -+STORE, 140399796551680, 140399863660543, -+ERASE, 140399796551680, 140399863660543, -+SNULL, 140400199340031, 140400266313727, -+STORE, 140400199204864, 140400199340031, -+STORE, 140400199340032, 140400266313727, -+STORE, 140400627003392, 140400643788799, -+SNULL, 140400316641280, 140400325029887, -+STORE, 140400325029888, 140400333422591, -+STORE, 140400316641280, 140400325029887, -+SNULL, 140400325033983, 140400333422591, -+STORE, 140400325029888, 140400325033983, -+STORE, 140400325033984, 140400333422591, -+SNULL, 140400627003392, 140400635396095, -+STORE, 140400635396096, 140400643788799, -+STORE, 140400627003392, 140400635396095, -+SNULL, 140400635400191, 140400643788799, -+STORE, 140400635396096, 140400635400191, -+STORE, 140400635400192, 140400643788799, -+SNULL, 140400434073599, 140400442462207, -+STORE, 140400434069504, 140400434073599, -+STORE, 140400434073600, 140400442462207, -+STORE, 140400618610688, 140400635396095, -+STORE, 140400610217984, 140400635396095, -+SNULL, 140400954159104, 140400962547711, -+STORE, 140400962547712, 140400987725823, -+STORE, 140400954159104, 140400962547711, -+SNULL, 140400962551807, 140400987725823, -+STORE, 140400962547712, 140400962551807, -+STORE, 140400962551808, 140400987725823, -+SNULL, 140400299855872, 140400308244479, -+STORE, 140400308244480, 140400316637183, -+STORE, 140400299855872, 140400308244479, -+SNULL, 140400308248575, 140400316637183, -+STORE, 140400308244480, 140400308248575, -+STORE, 140400308248576, 140400316637183, -+STORE, 140400601825280, 140400635396095, -+SNULL, 140400601829375, 140400635396095, -+STORE, 140400601825280, 140400601829375, -+STORE, 140400601829376, 140400635396095, -+STORE, 140400576679936, 140400593465343, -+SNULL, 140400576684031, 140400593465343, -+STORE, 140400576679936, 140400576684031, -+STORE, 140400576684032, 140400593465343, -+SNULL, 140400643792896, 140400652181503, -+STORE, 140400652181504, 140400668966911, -+STORE, 140400643792896, 140400652181503, -+SNULL, 140400652185599, 140400668966911, -+STORE, 140400652181504, 140400652185599, -+STORE, 140400652185600, 140400668966911, -+STORE, 140399595225088, 140399796551679, -+SNULL, 140399662333951, 140399796551679, -+STORE, 140399595225088, 140399662333951, -+STORE, 140399662333952, 140399796551679, -+SNULL, 140399662333952, 140399729442815, -+STORE, 140399729442816, 140399796551679, -+STORE, 140399662333952, 140399729442815, -+ERASE, 140399662333952, 140399729442815, -+SNULL, 140399863795711, 140399930769407, -+STORE, 140399863660544, 140399863795711, -+STORE, 140399863795712, 140399930769407, -+STORE, 140400568287232, 140400576679935, -+SNULL, 140400568291327, 140400576679935, -+STORE, 140400568287232, 140400568291327, -+STORE, 140400568291328, 140400576679935, -+SNULL, 140400467775487, 140400534749183, -+STORE, 140400467640320, 140400467775487, -+STORE, 140400467775488, 140400534749183, -+SNULL, 140399729577983, 140399796551679, -+STORE, 140399729442816, 140399729577983, -+STORE, 140399729577984, 140399796551679, -+SNULL, 140400601829376, 140400627003391, -+STORE, 140400627003392, 140400635396095, -+STORE, 140400601829376, 140400627003391, -+SNULL, 140400627007487, 140400635396095, -+STORE, 140400627003392, 140400627007487, -+STORE, 140400627007488, 140400635396095, -+STORE, 140400559894528, 140400568287231, -+STORE, 140400551501824, 140400568287231, -+STORE, 140400543109120, 140400568287231, -+STORE, 140400459247616, 140400467640319, -+STORE, 140400442466304, 140400467640319, -+SNULL, 140399595360255, 140399662333951, -+STORE, 140399595225088, 140399595360255, -+STORE, 140399595360256, 140399662333951, -+SNULL, 140400962551808, 140400970940415, -+STORE, 140400970940416, 140400987725823, -+STORE, 140400962551808, 140400970940415, -+SNULL, 140400970944511, 140400987725823, -+STORE, 140400970940416, 140400970944511, -+STORE, 140400970944512, 140400987725823, -+SNULL, 140400652185600, 140400660574207, -+STORE, 140400660574208, 140400668966911, -+STORE, 140400652185600, 140400660574207, -+SNULL, 140400660578303, 140400668966911, -+STORE, 140400660574208, 140400660578303, -+STORE, 140400660578304, 140400668966911, -+SNULL, 140400576684032, 140400585072639, -+STORE, 140400585072640, 140400593465343, -+STORE, 140400576684032, 140400585072639, -+SNULL, 140400585076735, 140400593465343, -+STORE, 140400585072640, 140400585076735, -+STORE, 140400585076736, 140400593465343, -+STORE, 140400425676800, 140400434069503, -+STORE, 140400417284096, 140400434069503, -+STORE, 140400408891392, 140400434069503, -+SNULL, 140400408891392, 140400417284095, -+STORE, 140400417284096, 140400434069503, -+STORE, 140400408891392, 140400417284095, -+SNULL, 140400417288191, 140400434069503, -+STORE, 140400417284096, 140400417288191, -+STORE, 140400417288192, 140400434069503, -+STORE, 140400283066368, 140400291459071, -+SNULL, 140400601829376, 140400618610687, -+STORE, 140400618610688, 140400627003391, -+STORE, 140400601829376, 140400618610687, -+SNULL, 140400618614783, 140400627003391, -+STORE, 140400618610688, 140400618614783, -+STORE, 140400618614784, 140400627003391, -+SNULL, 140400601829376, 140400610217983, -+STORE, 140400610217984, 140400618610687, -+STORE, 140400601829376, 140400610217983, -+SNULL, 140400610222079, 140400618610687, -+STORE, 140400610217984, 140400610222079, -+STORE, 140400610222080, 140400618610687, -+STORE, 140400274673664, 140400291459071, -+STORE, 140400190812160, 140400199204863, -+STORE, 140400182419456, 140400199204863, -+SNULL, 140400442466304, 140400450854911, -+STORE, 140400450854912, 140400467640319, -+STORE, 140400442466304, 140400450854911, -+SNULL, 140400450859007, 140400467640319, -+STORE, 140400450854912, 140400450859007, -+STORE, 140400450859008, 140400467640319, -+SNULL, 140400543109120, 140400559894527, -+STORE, 140400559894528, 140400568287231, -+STORE, 140400543109120, 140400559894527, -+SNULL, 140400559898623, 140400568287231, -+STORE, 140400559894528, 140400559898623, -+STORE, 140400559898624, 140400568287231, -+SNULL, 140400450859008, 140400459247615, -+STORE, 140400459247616, 140400467640319, -+STORE, 140400450859008, 140400459247615, -+SNULL, 140400459251711, 140400467640319, -+STORE, 140400459247616, 140400459251711, -+STORE, 140400459251712, 140400467640319, -+SNULL, 140400543113215, 140400559894527, -+STORE, 140400543109120, 140400543113215, -+STORE, 140400543113216, 140400559894527, -+SNULL, 140400970944512, 140400979333119, -+STORE, 140400979333120, 140400987725823, -+STORE, 140400970944512, 140400979333119, -+SNULL, 140400979337215, 140400987725823, -+STORE, 140400979333120, 140400979337215, -+STORE, 140400979337216, 140400987725823, -+STORE, 140400174026752, 140400199204863, -+SNULL, 140400174030847, 140400199204863, -+STORE, 140400174026752, 140400174030847, -+STORE, 140400174030848, 140400199204863, -+SNULL, 140400274673664, 140400283066367, -+STORE, 140400283066368, 140400291459071, -+STORE, 140400274673664, 140400283066367, -+SNULL, 140400283070463, 140400291459071, -+STORE, 140400283066368, 140400283070463, -+STORE, 140400283070464, 140400291459071, -+STORE, 140400165634048, 140400174026751, -+SNULL, 140400165638143, 140400174026751, -+STORE, 140400165634048, 140400165638143, -+STORE, 140400165638144, 140400174026751, -+SNULL, 140400174030848, 140400182419455, -+STORE, 140400182419456, 140400199204863, -+STORE, 140400174030848, 140400182419455, -+SNULL, 140400182423551, 140400199204863, -+STORE, 140400182419456, 140400182423551, -+STORE, 140400182423552, 140400199204863, -+SNULL, 140400182423552, 140400190812159, -+STORE, 140400190812160, 140400199204863, -+STORE, 140400182423552, 140400190812159, -+SNULL, 140400190816255, 140400199204863, -+STORE, 140400190812160, 140400190816255, -+STORE, 140400190816256, 140400199204863, -+STORE, 140400157241344, 140400165634047, -+SNULL, 140400157245439, 140400165634047, -+STORE, 140400157241344, 140400157245439, -+STORE, 140400157245440, 140400165634047, -+SNULL, 140400408895487, 140400417284095, -+STORE, 140400408891392, 140400408895487, -+STORE, 140400408895488, 140400417284095, -+SNULL, 140400417288192, 140400425676799, -+STORE, 140400425676800, 140400434069503, -+STORE, 140400417288192, 140400425676799, -+SNULL, 140400425680895, 140400434069503, -+STORE, 140400425676800, 140400425680895, -+STORE, 140400425680896, 140400434069503, -+STORE, 140400148848640, 140400157241343, -+SNULL, 140400148852735, 140400157241343, -+STORE, 140400148848640, 140400148852735, -+STORE, 140400148852736, 140400157241343, -+SNULL, 140400543113216, 140400551501823, -+STORE, 140400551501824, 140400559894527, -+STORE, 140400543113216, 140400551501823, -+SNULL, 140400551505919, 140400559894527, -+STORE, 140400551501824, 140400551505919, -+STORE, 140400551505920, 140400559894527, -+STORE, 140400140455936, 140400148848639, -+STORE, 140400048201728, 140400056594431, -+SNULL, 140400140460031, 140400148848639, -+STORE, 140400140455936, 140400140460031, -+STORE, 140400140460032, 140400148848639, -+STORE, 140400039809024, 140400056594431, -+SNULL, 140400039813119, 140400056594431, -+STORE, 140400039809024, 140400039813119, -+STORE, 140400039813120, 140400056594431, -+STORE, 140400031416320, 140400039809023, -+STORE, 140400023023616, 140400039809023, -+SNULL, 140400274677759, 140400283066367, -+STORE, 140400274673664, 140400274677759, -+STORE, 140400274677760, 140400283066367, -+STORE, 140400014630912, 140400039809023, -+STORE, 140400006238208, 140400039809023, -+STORE, 140399997845504, 140400039809023, -+SNULL, 140399997849599, 140400039809023, -+STORE, 140399997845504, 140399997849599, -+STORE, 140399997849600, 140400039809023, -+STORE, 140399989452800, 140399997845503, -+SNULL, 140399989456895, 140399997845503, -+STORE, 140399989452800, 140399989456895, -+STORE, 140399989456896, 140399997845503, -+STORE, 140399981060096, 140399989452799, -+SNULL, 140399981064191, 140399989452799, -+STORE, 140399981060096, 140399981064191, -+STORE, 140399981064192, 140399989452799, -+STORE, 140399972667392, 140399981060095, -+STORE, 140399964274688, 140399981060095, -+SNULL, 140399964278783, 140399981060095, -+STORE, 140399964274688, 140399964278783, -+STORE, 140399964278784, 140399981060095, -+SNULL, 140400039813120, 140400048201727, -+STORE, 140400048201728, 140400056594431, -+STORE, 140400039813120, 140400048201727, -+SNULL, 140400048205823, 140400056594431, -+STORE, 140400048201728, 140400048205823, -+STORE, 140400048205824, 140400056594431, -+SNULL, 140399997849600, 140400031416319, -+STORE, 140400031416320, 140400039809023, -+STORE, 140399997849600, 140400031416319, -+SNULL, 140400031420415, 140400039809023, -+STORE, 140400031416320, 140400031420415, -+STORE, 140400031420416, 140400039809023, -+STORE, 140399955881984, 140399964274687, -+SNULL, 140399955886079, 140399964274687, -+STORE, 140399955881984, 140399955886079, -+STORE, 140399955886080, 140399964274687, -+STORE, 140399947489280, 140399955881983, -+STORE, 140399939096576, 140399955881983, -+STORE, 140399855267840, 140399863660543, -+SNULL, 140399939100671, 140399955881983, -+STORE, 140399939096576, 140399939100671, -+STORE, 140399939100672, 140399955881983, -+SNULL, 140399997849600, 140400014630911, -+STORE, 140400014630912, 140400031416319, -+STORE, 140399997849600, 140400014630911, -+SNULL, 140400014635007, 140400031416319, -+STORE, 140400014630912, 140400014635007, -+STORE, 140400014635008, 140400031416319, -+SNULL, 140400014635008, 140400023023615, -+STORE, 140400023023616, 140400031416319, -+STORE, 140400014635008, 140400023023615, -+SNULL, 140400023027711, 140400031416319, -+STORE, 140400023023616, 140400023027711, -+STORE, 140400023027712, 140400031416319, -+SNULL, 140399997849600, 140400006238207, -+STORE, 140400006238208, 140400014630911, -+STORE, 140399997849600, 140400006238207, -+SNULL, 140400006242303, 140400014630911, -+STORE, 140400006238208, 140400006242303, -+STORE, 140400006242304, 140400014630911, -+STORE, 140399846875136, 140399863660543, -+STORE, 140399838482432, 140399863660543, -+SNULL, 140399838486527, 140399863660543, -+STORE, 140399838482432, 140399838486527, -+STORE, 140399838486528, 140399863660543, -+SNULL, 140399939100672, 140399947489279, -+STORE, 140399947489280, 140399955881983, -+STORE, 140399939100672, 140399947489279, -+SNULL, 140399947493375, 140399955881983, -+STORE, 140399947489280, 140399947493375, -+STORE, 140399947493376, 140399955881983, -+SNULL, 140399964278784, 140399972667391, -+STORE, 140399972667392, 140399981060095, -+STORE, 140399964278784, 140399972667391, -+SNULL, 140399972671487, 140399981060095, -+STORE, 140399972667392, 140399972671487, -+STORE, 140399972671488, 140399981060095, -+SNULL, 140399838486528, 140399855267839, -+STORE, 140399855267840, 140399863660543, -+STORE, 140399838486528, 140399855267839, -+SNULL, 140399855271935, 140399863660543, -+STORE, 140399855267840, 140399855271935, -+STORE, 140399855271936, 140399863660543, -+STORE, 140399830089728, 140399838482431, -+SNULL, 140399830093823, 140399838482431, -+STORE, 140399830089728, 140399830093823, -+STORE, 140399830093824, 140399838482431, -+STORE, 140399821697024, 140399830089727, -+SNULL, 140399821701119, 140399830089727, -+STORE, 140399821697024, 140399821701119, -+STORE, 140399821701120, 140399830089727, -+SNULL, 140399838486528, 140399846875135, -+STORE, 140399846875136, 140399855267839, -+STORE, 140399838486528, 140399846875135, -+SNULL, 140399846879231, 140399855267839, -+STORE, 140399846875136, 140399846879231, -+STORE, 140399846879232, 140399855267839, -+STORE, 140399813304320, 140399821697023, -+STORE, 140399804911616, 140399821697023, -+SNULL, 140399804915711, 140399821697023, -+STORE, 140399804911616, 140399804915711, -+STORE, 140399804915712, 140399821697023, -+STORE, 140399721050112, 140399729442815, -+SNULL, 140399804915712, 140399813304319, -+STORE, 140399813304320, 140399821697023, -+STORE, 140399804915712, 140399813304319, -+SNULL, 140399813308415, 140399821697023, -+STORE, 140399813304320, 140399813308415, -+STORE, 140399813308416, 140399821697023, -+SNULL, 140399721054207, 140399729442815, -+STORE, 140399721050112, 140399721054207, -+STORE, 140399721054208, 140399729442815, -+STORE, 140401467105280, 140401467133951, -+STORE, 140401279115264, 140401281306623, -+SNULL, 140401279115264, 140401279205375, -+STORE, 140401279205376, 140401281306623, -+STORE, 140401279115264, 140401279205375, -+SNULL, 140401281298431, 140401281306623, -+STORE, 140401279205376, 140401281298431, -+STORE, 140401281298432, 140401281306623, -+ERASE, 140401281298432, 140401281306623, -+STORE, 140401281298432, 140401281306623, -+SNULL, 140401281302527, 140401281306623, -+STORE, 140401281298432, 140401281302527, -+STORE, 140401281302528, 140401281306623, -+ERASE, 140401467105280, 140401467133951, -+ERASE, 140400056594432, 140400056598527, -+ERASE, 140400056598528, 140400064987135, -+ERASE, 140400635396096, 140400635400191, -+ERASE, 140400635400192, 140400643788799, -+ERASE, 140400408891392, 140400408895487, -+ERASE, 140400408895488, 140400417284095, -+ERASE, 140400299851776, 140400299855871, -+ERASE, 140400299855872, 140400308244479, -+ERASE, 140400627003392, 140400627007487, -+ERASE, 140400627007488, 140400635396095, -+ERASE, 140400954155008, 140400954159103, -+ERASE, 140400954159104, 140400962547711, -+ERASE, 140400291459072, 140400291463167, -+ERASE, 140400291463168, 140400299851775, -+ERASE, 140400643788800, 140400643792895, -+ERASE, 140400643792896, 140400652181503, -+ERASE, 140400325029888, 140400325033983, -+ERASE, 140400325033984, 140400333422591, -+ERASE, 140400610217984, 140400610222079, -+ERASE, 140400610222080, 140400618610687, -+ERASE, 140400190812160, 140400190816255, -+ERASE, 140400190816256, 140400199204863, -+ERASE, 140399964274688, 140399964278783, -+ERASE, 140399964278784, 140399972667391, -+ERASE, 140400945762304, 140400945766399, -+ERASE, 140400945766400, 140400954155007, -+ERASE, 140400568287232, 140400568291327, -+ERASE, 140400568291328, 140400576679935, -+ERASE, 140399972667392, 140399972671487, -+ERASE, 140399972671488, 140399981060095, -+ERASE, 140400962547712, 140400962551807, -+ERASE, 140400962551808, 140400970940415, -+ERASE, 140400987725824, 140400987729919, -+ERASE, 140400987729920, 140400996118527, -+ERASE, 140400652181504, 140400652185599, -+ERASE, 140400652185600, 140400660574207, -+ERASE, 140400450854912, 140400450859007, -+ERASE, 140400450859008, 140400459247615, -+ERASE, 140400031416320, 140400031420415, -+ERASE, 140400031420416, 140400039809023, -+ERASE, 140400308244480, 140400308248575, -+ERASE, 140400308248576, 140400316637183, -+ERASE, 140400434069504, 140400434073599, -+ERASE, 140400434073600, 140400442462207, -+ERASE, 140400543109120, 140400543113215, -+ERASE, 140400543113216, 140400551501823, -+ERASE, 140400023023616, 140400023027711, -+ERASE, 140400023027712, 140400031416319, -+ERASE, 140399813304320, 140399813308415, -+ERASE, 140399813308416, 140399821697023, -+ERASE, 140400316637184, 140400316641279, -+ERASE, 140400316641280, 140400325029887, -+ERASE, 140400585072640, 140400585076735, -+ERASE, 140400585076736, 140400593465343, -+ERASE, 140400148848640, 140400148852735, -+ERASE, 140400148852736, 140400157241343, -+ERASE, 140399955881984, 140399955886079, -+ERASE, 140399955886080, 140399964274687, -+ERASE, 140399821697024, 140399821701119, -+ERASE, 140399821701120, 140399830089727, -+ERASE, 140400601825280, 140400601829375, -+ERASE, 140400601829376, 140400610217983, -+ERASE, 140400979333120, 140400979337215, -+ERASE, 140400979337216, 140400987725823, -+ERASE, 140399997845504, 140399997849599, -+ERASE, 140399997849600, 140400006238207, -+ERASE, 140400459247616, 140400459251711, -+ERASE, 140400459251712, 140400467640319, -+ERASE, 140400551501824, 140400551505919, -+ERASE, 140400551505920, 140400559894527, -+ERASE, 140399939096576, 140399939100671, -+ERASE, 140399939100672, 140399947489279, -+ERASE, 140400442462208, 140400442466303, -+ERASE, 140400442466304, 140400450854911, -+ERASE, 140400576679936, 140400576684031, -+ERASE, 140400576684032, 140400585072639, -+ERASE, 140400559894528, 140400559898623, -+ERASE, 140400559898624, 140400568287231, -+ERASE, 140400417284096, 140400417288191, -+ERASE, 140400417288192, 140400425676799, -+ERASE, 140400283066368, 140400283070463, -+ERASE, 140400283070464, 140400291459071, -+ }; -+ unsigned long set33[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140734562918400, 140737488351231, -+SNULL, 140734562922495, 140737488351231, -+STORE, 140734562918400, 140734562922495, -+STORE, 140734562787328, 140734562922495, -+STORE, 94133878984704, 94133881237503, -+SNULL, 94133879115775, 94133881237503, -+STORE, 94133878984704, 94133879115775, -+STORE, 94133879115776, 94133881237503, -+ERASE, 94133879115776, 94133881237503, -+STORE, 94133881208832, 94133881217023, -+STORE, 94133881217024, 94133881237503, -+STORE, 140583654043648, 140583656296447, -+SNULL, 140583654187007, 140583656296447, -+STORE, 140583654043648, 140583654187007, -+STORE, 140583654187008, 140583656296447, -+ERASE, 140583654187008, 140583656296447, -+STORE, 140583656284160, 140583656292351, -+STORE, 140583656292352, 140583656296447, -+STORE, 140734564319232, 140734564323327, -+STORE, 140734564306944, 140734564319231, -+STORE, 140583656255488, 140583656284159, -+STORE, 140583656247296, 140583656255487, -+STORE, 140583651827712, 140583654043647, -+SNULL, 140583651827712, 140583651926015, -+STORE, 140583651926016, 140583654043647, -+STORE, 140583651827712, 140583651926015, -+SNULL, 140583654019071, 140583654043647, -+STORE, 140583651926016, 140583654019071, -+STORE, 140583654019072, 140583654043647, -+SNULL, 140583654019072, 140583654027263, -+STORE, 140583654027264, 140583654043647, -+STORE, 140583654019072, 140583654027263, -+ERASE, 140583654019072, 140583654027263, -+STORE, 140583654019072, 140583654027263, -+ERASE, 140583654027264, 140583654043647, -+STORE, 140583654027264, 140583654043647, -+STORE, 140583648030720, 140583651827711, -+SNULL, 140583648030720, 140583649689599, -+STORE, 140583649689600, 140583651827711, -+STORE, 140583648030720, 140583649689599, -+SNULL, 140583651786751, 140583651827711, -+STORE, 140583649689600, 140583651786751, -+STORE, 140583651786752, 140583651827711, -+SNULL, 140583651786752, 140583651811327, -+STORE, 140583651811328, 140583651827711, -+STORE, 140583651786752, 140583651811327, -+ERASE, 140583651786752, 140583651811327, -+STORE, 140583651786752, 140583651811327, -+ERASE, 140583651811328, 140583651827711, -+STORE, 140583651811328, 140583651827711, -+STORE, 140583656239104, 140583656255487, -+SNULL, 140583651803135, 140583651811327, -+STORE, 140583651786752, 140583651803135, -+STORE, 140583651803136, 140583651811327, -+SNULL, 140583654023167, 140583654027263, -+STORE, 140583654019072, 140583654023167, -+STORE, 140583654023168, 140583654027263, -+SNULL, 94133881212927, 94133881217023, -+STORE, 94133881208832, 94133881212927, -+STORE, 94133881212928, 94133881217023, -+SNULL, 140583656288255, 140583656292351, -+STORE, 140583656284160, 140583656288255, -+STORE, 140583656288256, 140583656292351, -+ERASE, 140583656255488, 140583656284159, -+STORE, 94133881733120, 94133881868287, -+STORE, 140583639638016, 140583648030719, -+SNULL, 140583639642111, 140583648030719, -+STORE, 140583639638016, 140583639642111, -+STORE, 140583639642112, 140583648030719, -+STORE, 140583631245312, 140583639638015, -+STORE, 140583497027584, 140583631245311, -+SNULL, 140583497027584, 140583540621311, -+STORE, 140583540621312, 140583631245311, -+STORE, 140583497027584, 140583540621311, -+ERASE, 140583497027584, 140583540621311, -+SNULL, 140583607730175, 140583631245311, -+STORE, 140583540621312, 140583607730175, -+STORE, 140583607730176, 140583631245311, -+ERASE, 140583607730176, 140583631245311, -+SNULL, 140583540756479, 140583607730175, -+STORE, 140583540621312, 140583540756479, -+STORE, 140583540756480, 140583607730175, -+SNULL, 140583631249407, 140583639638015, -+STORE, 140583631245312, 140583631249407, -+STORE, 140583631249408, 140583639638015, -+STORE, 140583622852608, 140583631245311, -+SNULL, 140583622856703, 140583631245311, -+STORE, 140583622852608, 140583622856703, -+STORE, 140583622856704, 140583631245311, -+STORE, 140583614459904, 140583622852607, -+SNULL, 140583614463999, 140583622852607, -+STORE, 140583614459904, 140583614463999, -+STORE, 140583614464000, 140583622852607, -+STORE, 140583532228608, 140583540621311, -+SNULL, 140583532232703, 140583540621311, -+STORE, 140583532228608, 140583532232703, -+STORE, 140583532232704, 140583540621311, -+STORE, 140583523835904, 140583532228607, -+STORE, 140583515443200, 140583532228607, -+STORE, 140583507050496, 140583532228607, -+STORE, 140583372832768, 140583507050495, -+STORE, 140583364440064, 140583372832767, -+STORE, 140583230222336, 140583364440063, -+STORE, 140583096004608, 140583364440063, -+SNULL, 140583230222335, 140583364440063, -+STORE, 140583096004608, 140583230222335, -+STORE, 140583230222336, 140583364440063, -+SNULL, 140583230222336, 140583272185855, -+STORE, 140583272185856, 140583364440063, -+STORE, 140583230222336, 140583272185855, -+ERASE, 140583230222336, 140583272185855, -+STORE, 140582961786880, 140583230222335, -+SNULL, 140583372832768, 140583406403583, -+STORE, 140583406403584, 140583507050495, -+STORE, 140583372832768, 140583406403583, -+ERASE, 140583372832768, 140583406403583, -+SNULL, 140583473512447, 140583507050495, -+STORE, 140583406403584, 140583473512447, -+STORE, 140583473512448, 140583507050495, -+ERASE, 140583473512448, 140583507050495, -+SNULL, 140583096004607, 140583230222335, -+STORE, 140582961786880, 140583096004607, -+STORE, 140583096004608, 140583230222335, -+SNULL, 140583096004608, 140583137968127, -+STORE, 140583137968128, 140583230222335, -+STORE, 140583096004608, 140583137968127, -+ERASE, 140583096004608, 140583137968127, -+SNULL, 140583339294719, 140583364440063, -+STORE, 140583272185856, 140583339294719, -+STORE, 140583339294720, 140583364440063, -+ERASE, 140583339294720, 140583364440063, -+SNULL, 140583272321023, 140583339294719, -+STORE, 140583272185856, 140583272321023, -+STORE, 140583272321024, 140583339294719, -+SNULL, 140582961786880, 140583003750399, -+STORE, 140583003750400, 140583096004607, -+STORE, 140582961786880, 140583003750399, -+ERASE, 140582961786880, 140583003750399, -+ }; -+ -+ unsigned long set34[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140731327180800, 140737488351231, -+SNULL, 140731327184895, 140737488351231, -+STORE, 140731327180800, 140731327184895, -+STORE, 140731327049728, 140731327184895, -+STORE, 94632924487680, 94632926740479, -+SNULL, 94632924618751, 94632926740479, -+STORE, 94632924487680, 94632924618751, -+STORE, 94632924618752, 94632926740479, -+ERASE, 94632924618752, 94632926740479, -+STORE, 94632926711808, 94632926719999, -+STORE, 94632926720000, 94632926740479, -+STORE, 140012544888832, 140012547141631, -+SNULL, 140012545032191, 140012547141631, -+STORE, 140012544888832, 140012545032191, -+STORE, 140012545032192, 140012547141631, -+ERASE, 140012545032192, 140012547141631, -+STORE, 140012547129344, 140012547137535, -+STORE, 140012547137536, 140012547141631, -+STORE, 140731327725568, 140731327729663, -+STORE, 140731327713280, 140731327725567, -+STORE, 140012547100672, 140012547129343, -+STORE, 140012547092480, 140012547100671, -+STORE, 140012542672896, 140012544888831, -+SNULL, 140012542672896, 140012542771199, -+STORE, 140012542771200, 140012544888831, -+STORE, 140012542672896, 140012542771199, -+SNULL, 140012544864255, 140012544888831, -+STORE, 140012542771200, 140012544864255, -+STORE, 140012544864256, 140012544888831, -+SNULL, 140012544864256, 140012544872447, -+STORE, 140012544872448, 140012544888831, -+STORE, 140012544864256, 140012544872447, -+ERASE, 140012544864256, 140012544872447, -+STORE, 140012544864256, 140012544872447, -+ERASE, 140012544872448, 140012544888831, -+STORE, 140012544872448, 140012544888831, -+STORE, 140012538875904, 140012542672895, -+SNULL, 140012538875904, 140012540534783, -+STORE, 140012540534784, 140012542672895, -+STORE, 140012538875904, 140012540534783, -+SNULL, 140012542631935, 140012542672895, -+STORE, 140012540534784, 140012542631935, -+STORE, 140012542631936, 140012542672895, -+SNULL, 140012542631936, 140012542656511, -+STORE, 140012542656512, 140012542672895, -+STORE, 140012542631936, 140012542656511, -+ERASE, 140012542631936, 140012542656511, -+STORE, 140012542631936, 140012542656511, -+ERASE, 140012542656512, 140012542672895, -+STORE, 140012542656512, 140012542672895, -+STORE, 140012547084288, 140012547100671, -+SNULL, 140012542648319, 140012542656511, -+STORE, 140012542631936, 140012542648319, -+STORE, 140012542648320, 140012542656511, -+SNULL, 140012544868351, 140012544872447, -+STORE, 140012544864256, 140012544868351, -+STORE, 140012544868352, 140012544872447, -+SNULL, 94632926715903, 94632926719999, -+STORE, 94632926711808, 94632926715903, -+STORE, 94632926715904, 94632926719999, -+SNULL, 140012547133439, 140012547137535, -+STORE, 140012547129344, 140012547133439, -+STORE, 140012547133440, 140012547137535, -+ERASE, 140012547100672, 140012547129343, -+STORE, 94632939606016, 94632939741183, -+STORE, 140012530483200, 140012538875903, -+SNULL, 140012530487295, 140012538875903, -+STORE, 140012530483200, 140012530487295, -+STORE, 140012530487296, 140012538875903, -+STORE, 140012522090496, 140012530483199, -+STORE, 140012387872768, 140012522090495, -+SNULL, 140012387872768, 140012444188671, -+STORE, 140012444188672, 140012522090495, -+STORE, 140012387872768, 140012444188671, -+ERASE, 140012387872768, 140012444188671, -+SNULL, 140012511297535, 140012522090495, -+STORE, 140012444188672, 140012511297535, -+STORE, 140012511297536, 140012522090495, -+ERASE, 140012511297536, 140012522090495, -+SNULL, 140012444323839, 140012511297535, -+STORE, 140012444188672, 140012444323839, -+STORE, 140012444323840, 140012511297535, -+SNULL, 140012522094591, 140012530483199, -+STORE, 140012522090496, 140012522094591, -+STORE, 140012522094592, 140012530483199, -+STORE, 140012513697792, 140012522090495, -+SNULL, 140012513701887, 140012522090495, -+STORE, 140012513697792, 140012513701887, -+STORE, 140012513701888, 140012522090495, -+STORE, 140012435795968, 140012444188671, -+SNULL, 140012435800063, 140012444188671, -+STORE, 140012435795968, 140012435800063, -+STORE, 140012435800064, 140012444188671, -+STORE, 140012427403264, 140012435795967, -+SNULL, 140012427407359, 140012435795967, -+STORE, 140012427403264, 140012427407359, -+STORE, 140012427407360, 140012435795967, -+STORE, 140012419010560, 140012427403263, -+STORE, 140012410617856, 140012427403263, -+STORE, 140012276400128, 140012410617855, -+STORE, 140012268007424, 140012276400127, -+STORE, 140012133789696, 140012268007423, -+SNULL, 140012133789696, 140012175753215, -+STORE, 140012175753216, 140012268007423, -+STORE, 140012133789696, 140012175753215, -+ERASE, 140012133789696, 140012175753215, -+STORE, 140012041535488, 140012268007423, -+SNULL, 140012108644351, 140012268007423, -+STORE, 140012041535488, 140012108644351, -+STORE, 140012108644352, 140012268007423, -+SNULL, 140012108644352, 140012175753215, -+STORE, 140012175753216, 140012268007423, -+STORE, 140012108644352, 140012175753215, -+ERASE, 140012108644352, 140012175753215, -+SNULL, 140012276400128, 140012309970943, -+STORE, 140012309970944, 140012410617855, -+STORE, 140012276400128, 140012309970943, -+ERASE, 140012276400128, 140012309970943, -+STORE, 140012301578240, 140012309970943, -+STORE, 140012041535488, 140012268007423, -+SNULL, 140012242862079, 140012268007423, -+STORE, 140012041535488, 140012242862079, -+STORE, 140012242862080, 140012268007423, -+ERASE, 140012242862080, 140012268007423, -+SNULL, 140012041670655, 140012242862079, -+STORE, 140012041535488, 140012041670655, -+STORE, 140012041670656, 140012242862079, -+SNULL, 140012041670656, 140012108644351, -+STORE, 140012108644352, 140012242862079, -+STORE, 140012041670656, 140012108644351, -+SNULL, 140012108779519, 140012242862079, -+STORE, 140012108644352, 140012108779519, -+STORE, 140012108779520, 140012242862079, -+SNULL, 140012377079807, 140012410617855, -+STORE, 140012309970944, 140012377079807, -+STORE, 140012377079808, 140012410617855, -+ERASE, 140012377079808, 140012410617855, -+SNULL, 140012310106111, 140012377079807, -+STORE, 140012309970944, 140012310106111, -+STORE, 140012310106112, 140012377079807, -+SNULL, 140012410621951, 140012427403263, -+STORE, 140012410617856, 140012410621951, -+STORE, 140012410621952, 140012427403263, -+SNULL, 140012108779520, 140012175753215, -+STORE, 140012175753216, 140012242862079, -+STORE, 140012108779520, 140012175753215, -+SNULL, 140012175888383, 140012242862079, -+STORE, 140012175753216, 140012175888383, -+STORE, 140012175888384, 140012242862079, -+SNULL, 140012301582335, 140012309970943, -+STORE, 140012301578240, 140012301582335, -+STORE, 140012301582336, 140012309970943, -+SNULL, 140012410621952, 140012419010559, -+STORE, 140012419010560, 140012427403263, -+STORE, 140012410621952, 140012419010559, -+SNULL, 140012419014655, 140012427403263, -+STORE, 140012419010560, 140012419014655, -+STORE, 140012419014656, 140012427403263, -+SNULL, 140012268011519, 140012276400127, -+STORE, 140012268007424, 140012268011519, -+STORE, 140012268011520, 140012276400127, -+STORE, 140012402225152, 140012410617855, -+STORE, 140012393832448, 140012410617855, -+SNULL, 140012393832448, 140012402225151, -+STORE, 140012402225152, 140012410617855, -+STORE, 140012393832448, 140012402225151, -+SNULL, 140012402229247, 140012410617855, -+STORE, 140012402225152, 140012402229247, -+STORE, 140012402229248, 140012410617855, -+STORE, 140012385439744, 140012402225151, -+SNULL, 140012385439744, 140012393832447, -+STORE, 140012393832448, 140012402225151, -+STORE, 140012385439744, 140012393832447, -+SNULL, 140012393836543, 140012402225151, -+STORE, 140012393832448, 140012393836543, -+STORE, 140012393836544, 140012402225151, -+STORE, 140012293185536, 140012301578239, -+STORE, 140012284792832, 140012301578239, -+SNULL, 140012284792832, 140012293185535, -+STORE, 140012293185536, 140012301578239, -+STORE, 140012284792832, 140012293185535, -+SNULL, 140012293189631, 140012301578239, -+STORE, 140012293185536, 140012293189631, -+STORE, 140012293189632, 140012301578239, -+STORE, 140012268011520, 140012284792831, -+SNULL, 140012385443839, 140012393832447, -+STORE, 140012385439744, 140012385443839, -+STORE, 140012385443840, 140012393832447, -+STORE, 140012259614720, 140012268007423, -+SNULL, 140012259618815, 140012268007423, -+STORE, 140012259614720, 140012259618815, -+STORE, 140012259618816, 140012268007423, -+STORE, 140012251222016, 140012259614719, -+SNULL, 140012251226111, 140012259614719, -+STORE, 140012251222016, 140012251226111, -+STORE, 140012251226112, 140012259614719, -+SNULL, 140012284796927, 140012293185535, -+STORE, 140012284792832, 140012284796927, -+STORE, 140012284796928, 140012293185535, -+SNULL, 140012268011520, 140012276400127, -+STORE, 140012276400128, 140012284792831, -+STORE, 140012268011520, 140012276400127, -+SNULL, 140012276404223, 140012284792831, -+STORE, 140012276400128, 140012276404223, -+STORE, 140012276404224, 140012284792831, -+STORE, 140012033142784, 140012041535487, -+SNULL, 140012033146879, 140012041535487, -+STORE, 140012033142784, 140012033146879, -+STORE, 140012033146880, 140012041535487, -+STORE, 140012024750080, 140012033142783, -+STORE, 140012016357376, 140012033142783, -+SNULL, 140012016357376, 140012024750079, -+STORE, 140012024750080, 140012033142783, -+STORE, 140012016357376, 140012024750079, -+SNULL, 140012024754175, 140012033142783, -+STORE, 140012024750080, 140012024754175, -+STORE, 140012024754176, 140012033142783, -+SNULL, 140012016361471, 140012024750079, -+STORE, 140012016357376, 140012016361471, -+STORE, 140012016361472, 140012024750079, -+STORE, 140012007964672, 140012016357375, -+SNULL, 140012007968767, 140012016357375, -+STORE, 140012007964672, 140012007968767, -+STORE, 140012007968768, 140012016357375, -+STORE, 140011999571968, 140012007964671, -+STORE, 140011991179264, 140012007964671, -+STORE, 140011856961536, 140011991179263, -+STORE, 140011848568832, 140011856961535, -+STORE, 140011714351104, 140011848568831, -+SNULL, 140011714351104, 140011773100031, -+STORE, 140011773100032, 140011848568831, -+STORE, 140011714351104, 140011773100031, -+ERASE, 140011714351104, 140011773100031, -+STORE, 140011764707328, 140011773100031, -+STORE, 140011756314624, 140011773100031, -+STORE, 140011622096896, 140011756314623, -+STORE, 140011613704192, 140011622096895, -+STORE, 140011479486464, 140011613704191, -+STORE, 140011471093760, 140011479486463, -+SNULL, 140011479486464, 140011504664575, -+STORE, 140011504664576, 140011613704191, -+STORE, 140011479486464, 140011504664575, -+ERASE, 140011479486464, 140011504664575, -+STORE, 140011496271872, 140011504664575, -+STORE, 140011487879168, 140011504664575, -+STORE, 140011336876032, 140011471093759, -+SNULL, 140011336876032, 140011370446847, -+STORE, 140011370446848, 140011471093759, -+STORE, 140011336876032, 140011370446847, -+ERASE, 140011336876032, 140011370446847, -+STORE, 140011471093760, 140011487879167, -+STORE, 140011362054144, 140011370446847, -+SNULL, 140011362058239, 140011370446847, -+STORE, 140011362054144, 140011362058239, -+STORE, 140011362058240, 140011370446847, -+STORE, 140011353661440, 140011362054143, -+STORE, 140011345268736, 140011362054143, -+SNULL, 140011345272831, 140011362054143, -+STORE, 140011345268736, 140011345272831, -+STORE, 140011345272832, 140011362054143, -+STORE, 140011336876032, 140011345268735, -+STORE, 140011328483328, 140011345268735, -+SNULL, 140011328487423, 140011345268735, -+STORE, 140011328483328, 140011328487423, -+STORE, 140011328487424, 140011345268735, -+STORE, 140011320090624, 140011328483327, -+STORE, 140011185872896, 140011320090623, -+SNULL, 140011185872896, 140011236229119, -+STORE, 140011236229120, 140011320090623, -+STORE, 140011185872896, 140011236229119, -+ERASE, 140011185872896, 140011236229119, -+SNULL, 140011856961536, 140011907317759, -+STORE, 140011907317760, 140011991179263, -+STORE, 140011856961536, 140011907317759, -+ERASE, 140011856961536, 140011907317759, -+SNULL, 140011974426623, 140011991179263, -+STORE, 140011907317760, 140011974426623, -+STORE, 140011974426624, 140011991179263, -+ERASE, 140011974426624, 140011991179263, -+SNULL, 140011840208895, 140011848568831, -+STORE, 140011773100032, 140011840208895, -+STORE, 140011840208896, 140011848568831, -+ERASE, 140011840208896, 140011848568831, -+SNULL, 140011773235199, 140011840208895, -+STORE, 140011773100032, 140011773235199, -+STORE, 140011773235200, 140011840208895, -+STORE, 140011102011392, 140011320090623, -+SNULL, 140011169120255, 140011320090623, -+STORE, 140011102011392, 140011169120255, -+STORE, 140011169120256, 140011320090623, -+SNULL, 140011169120256, 140011236229119, -+STORE, 140011236229120, 140011320090623, -+STORE, 140011169120256, 140011236229119, -+ERASE, 140011169120256, 140011236229119, -+SNULL, 140011622096896, 140011638882303, -+STORE, 140011638882304, 140011756314623, -+STORE, 140011622096896, 140011638882303, -+ERASE, 140011622096896, 140011638882303, -+SNULL, 140011705991167, 140011756314623, -+STORE, 140011638882304, 140011705991167, -+STORE, 140011705991168, 140011756314623, -+ERASE, 140011705991168, 140011756314623, -+SNULL, 140011571773439, 140011613704191, -+STORE, 140011504664576, 140011571773439, -+STORE, 140011571773440, 140011613704191, -+ERASE, 140011571773440, 140011613704191, -+STORE, 140010967793664, 140011169120255, -+SNULL, 140011034902527, 140011169120255, -+STORE, 140010967793664, 140011034902527, -+STORE, 140011034902528, 140011169120255, -+SNULL, 140011034902528, 140011102011391, -+STORE, 140011102011392, 140011169120255, -+STORE, 140011034902528, 140011102011391, -+ERASE, 140011034902528, 140011102011391, -+STORE, 140010833575936, 140011034902527, -+SNULL, 140011437555711, 140011471093759, -+STORE, 140011370446848, 140011437555711, -+STORE, 140011437555712, 140011471093759, -+ERASE, 140011437555712, 140011471093759, -+SNULL, 140011370582015, 140011437555711, -+STORE, 140011370446848, 140011370582015, -+STORE, 140011370582016, 140011437555711, -+STORE, 140010699358208, 140011034902527, -+SNULL, 140011487883263, 140011504664575, -+STORE, 140011487879168, 140011487883263, -+STORE, 140011487883264, 140011504664575, -+SNULL, 140011345272832, 140011353661439, -+STORE, 140011353661440, 140011362054143, -+STORE, 140011345272832, 140011353661439, -+SNULL, 140011353665535, 140011362054143, -+STORE, 140011353661440, 140011353665535, -+STORE, 140011353665536, 140011362054143, -+SNULL, 140011328487424, 140011336876031, -+STORE, 140011336876032, 140011345268735, -+STORE, 140011328487424, 140011336876031, -+SNULL, 140011336880127, 140011345268735, -+STORE, 140011336876032, 140011336880127, -+STORE, 140011336880128, 140011345268735, -+SNULL, 140011303337983, 140011320090623, -+STORE, 140011236229120, 140011303337983, -+STORE, 140011303337984, 140011320090623, -+ERASE, 140011303337984, 140011320090623, -+SNULL, 140011907452927, 140011974426623, -+STORE, 140011907317760, 140011907452927, -+STORE, 140011907452928, 140011974426623, -+SNULL, 140011102146559, 140011169120255, -+STORE, 140011102011392, 140011102146559, -+STORE, 140011102146560, 140011169120255, -+SNULL, 140011639017471, 140011705991167, -+STORE, 140011638882304, 140011639017471, -+STORE, 140011639017472, 140011705991167, -+SNULL, 140011504799743, 140011571773439, -+STORE, 140011504664576, 140011504799743, -+STORE, 140011504799744, 140011571773439, -+SNULL, 140011613708287, 140011622096895, -+STORE, 140011613704192, 140011613708287, -+STORE, 140011613708288, 140011622096895, -+SNULL, 140010699358208, 140010967793663, -+STORE, 140010967793664, 140011034902527, -+STORE, 140010699358208, 140010967793663, -+SNULL, 140010967928831, 140011034902527, -+STORE, 140010967793664, 140010967928831, -+STORE, 140010967928832, 140011034902527, -+SNULL, 140010900684799, 140010967793663, -+STORE, 140010699358208, 140010900684799, -+STORE, 140010900684800, 140010967793663, -+ERASE, 140010900684800, 140010967793663, -+SNULL, 140010766467071, 140010900684799, -+STORE, 140010699358208, 140010766467071, -+STORE, 140010766467072, 140010900684799, -+SNULL, 140010766467072, 140010833575935, -+STORE, 140010833575936, 140010900684799, -+STORE, 140010766467072, 140010833575935, -+ERASE, 140010766467072, 140010833575935, -+SNULL, 140010699493375, 140010766467071, -+STORE, 140010699358208, 140010699493375, -+STORE, 140010699493376, 140010766467071, -+SNULL, 140011848572927, 140011856961535, -+STORE, 140011848568832, 140011848572927, -+STORE, 140011848572928, 140011856961535, -+STORE, 140011982786560, 140012007964671, -+STORE, 140011898925056, 140011907317759, -+SNULL, 140011898929151, 140011907317759, -+STORE, 140011898925056, 140011898929151, -+STORE, 140011898929152, 140011907317759, -+SNULL, 140011320094719, 140011328483327, -+STORE, 140011320090624, 140011320094719, -+STORE, 140011320094720, 140011328483327, -+STORE, 140011890532352, 140011898925055, -+STORE, 140011882139648, 140011898925055, -+SNULL, 140011882143743, 140011898925055, -+STORE, 140011882139648, 140011882143743, -+STORE, 140011882143744, 140011898925055, -+STORE, 140011873746944, 140011882139647, -+SNULL, 140011873751039, 140011882139647, -+STORE, 140011873746944, 140011873751039, -+STORE, 140011873751040, 140011882139647, -+SNULL, 140011236364287, 140011303337983, -+STORE, 140011236229120, 140011236364287, -+STORE, 140011236364288, 140011303337983, -+SNULL, 140011756318719, 140011773100031, -+STORE, 140011756314624, 140011756318719, -+STORE, 140011756318720, 140011773100031, -+SNULL, 140011756318720, 140011764707327, -+STORE, 140011764707328, 140011773100031, -+STORE, 140011756318720, 140011764707327, -+SNULL, 140011764711423, 140011773100031, -+STORE, 140011764707328, 140011764711423, -+STORE, 140011764711424, 140011773100031, -+SNULL, 140011471097855, 140011487879167, -+STORE, 140011471093760, 140011471097855, -+STORE, 140011471097856, 140011487879167, -+SNULL, 140010833711103, 140010900684799, -+STORE, 140010833575936, 140010833711103, -+STORE, 140010833711104, 140010900684799, -+SNULL, 140011982790655, 140012007964671, -+STORE, 140011982786560, 140011982790655, -+STORE, 140011982790656, 140012007964671, -+STORE, 140011865354240, 140011873746943, -+STORE, 140011848572928, 140011865354239, -+SNULL, 140011848572928, 140011856961535, -+STORE, 140011856961536, 140011865354239, -+STORE, 140011848572928, 140011856961535, -+SNULL, 140011856965631, 140011865354239, -+STORE, 140011856961536, 140011856965631, -+STORE, 140011856965632, 140011865354239, -+STORE, 140011747921920, 140011756314623, -+STORE, 140011739529216, 140011756314623, -+SNULL, 140011471097856, 140011479486463, -+STORE, 140011479486464, 140011487879167, -+STORE, 140011471097856, 140011479486463, -+SNULL, 140011479490559, 140011487879167, -+STORE, 140011479486464, 140011479490559, -+STORE, 140011479490560, 140011487879167, -+STORE, 140011731136512, 140011756314623, -+STORE, 140011722743808, 140011756314623, -+SNULL, 140011982790656, 140011999571967, -+STORE, 140011999571968, 140012007964671, -+STORE, 140011982790656, 140011999571967, -+SNULL, 140011999576063, 140012007964671, -+STORE, 140011999571968, 140011999576063, -+STORE, 140011999576064, 140012007964671, -+STORE, 140011714351104, 140011756314623, -+SNULL, 140011882143744, 140011890532351, -+STORE, 140011890532352, 140011898925055, -+STORE, 140011882143744, 140011890532351, -+SNULL, 140011890536447, 140011898925055, -+STORE, 140011890532352, 140011890536447, -+STORE, 140011890536448, 140011898925055, -+STORE, 140011630489600, 140011638882303, -+STORE, 140011613708288, 140011638882303, -+STORE, 140011605311488, 140011613704191, -+STORE, 140011596918784, 140011613704191, -+STORE, 140011588526080, 140011613704191, -+SNULL, 140011487883264, 140011496271871, -+STORE, 140011496271872, 140011504664575, -+STORE, 140011487883264, 140011496271871, -+SNULL, 140011496275967, 140011504664575, -+STORE, 140011496271872, 140011496275967, -+STORE, 140011496275968, 140011504664575, -+STORE, 140011580133376, 140011613704191, -+SNULL, 140011580137471, 140011613704191, -+STORE, 140011580133376, 140011580137471, -+STORE, 140011580137472, 140011613704191, -+SNULL, 140011982790656, 140011991179263, -+STORE, 140011991179264, 140011999571967, -+STORE, 140011982790656, 140011991179263, -+SNULL, 140011991183359, 140011999571967, -+STORE, 140011991179264, 140011991183359, -+STORE, 140011991183360, 140011999571967, -+SNULL, 140011865358335, 140011873746943, -+STORE, 140011865354240, 140011865358335, -+STORE, 140011865358336, 140011873746943, -+STORE, 140011462701056, 140011471093759, -+SNULL, 140011714351104, 140011739529215, -+STORE, 140011739529216, 140011756314623, -+STORE, 140011714351104, 140011739529215, -+SNULL, 140011739533311, 140011756314623, -+STORE, 140011739529216, 140011739533311, -+STORE, 140011739533312, 140011756314623, -+SNULL, 140011739533312, 140011747921919, -+STORE, 140011747921920, 140011756314623, -+STORE, 140011739533312, 140011747921919, -+SNULL, 140011747926015, 140011756314623, -+STORE, 140011747921920, 140011747926015, -+STORE, 140011747926016, 140011756314623, -+SNULL, 140011613708288, 140011630489599, -+STORE, 140011630489600, 140011638882303, -+STORE, 140011613708288, 140011630489599, -+SNULL, 140011630493695, 140011638882303, -+STORE, 140011630489600, 140011630493695, -+STORE, 140011630493696, 140011638882303, -+SNULL, 140011714351104, 140011722743807, -+STORE, 140011722743808, 140011739529215, -+STORE, 140011714351104, 140011722743807, -+SNULL, 140011722747903, 140011739529215, -+STORE, 140011722743808, 140011722747903, -+STORE, 140011722747904, 140011739529215, -+SNULL, 140011714355199, 140011722743807, -+STORE, 140011714351104, 140011714355199, -+STORE, 140011714355200, 140011722743807, -+SNULL, 140011722747904, 140011731136511, -+STORE, 140011731136512, 140011739529215, -+STORE, 140011722747904, 140011731136511, -+SNULL, 140011731140607, 140011739529215, -+STORE, 140011731136512, 140011731140607, -+STORE, 140011731140608, 140011739529215, -+STORE, 140011454308352, 140011471093759, -+STORE, 140011445915648, 140011471093759, -+SNULL, 140011580137472, 140011588526079, -+STORE, 140011588526080, 140011613704191, -+STORE, 140011580137472, 140011588526079, -+SNULL, 140011588530175, 140011613704191, -+STORE, 140011588526080, 140011588530175, -+STORE, 140011588530176, 140011613704191, -+SNULL, 140011445915648, 140011462701055, -+STORE, 140011462701056, 140011471093759, -+STORE, 140011445915648, 140011462701055, -+SNULL, 140011462705151, 140011471093759, -+STORE, 140011462701056, 140011462705151, -+STORE, 140011462705152, 140011471093759, -+SNULL, 140011588530176, 140011596918783, -+STORE, 140011596918784, 140011613704191, -+STORE, 140011588530176, 140011596918783, -+SNULL, 140011596922879, 140011613704191, -+STORE, 140011596918784, 140011596922879, -+STORE, 140011596922880, 140011613704191, -+SNULL, 140011596922880, 140011605311487, -+STORE, 140011605311488, 140011613704191, -+STORE, 140011596922880, 140011605311487, -+SNULL, 140011605315583, 140011613704191, -+STORE, 140011605311488, 140011605315583, -+STORE, 140011605315584, 140011613704191, -+SNULL, 140011613708288, 140011622096895, -+STORE, 140011622096896, 140011630489599, -+STORE, 140011613708288, 140011622096895, -+SNULL, 140011622100991, 140011630489599, -+STORE, 140011622096896, 140011622100991, -+STORE, 140011622100992, 140011630489599, -+STORE, 140011311697920, 140011320090623, -+STORE, 140011227836416, 140011236229119, -+STORE, 140011219443712, 140011236229119, -+SNULL, 140011219447807, 140011236229119, -+STORE, 140011219443712, 140011219447807, -+STORE, 140011219447808, 140011236229119, -+STORE, 140011211051008, 140011219443711, -+STORE, 140011202658304, 140011219443711, -+SNULL, 140011202662399, 140011219443711, -+STORE, 140011202658304, 140011202662399, -+STORE, 140011202662400, 140011219443711, -+STORE, 140011194265600, 140011202658303, -+STORE, 140011185872896, 140011202658303, -+STORE, 140011177480192, 140011202658303, -+STORE, 140011093618688, 140011102011391, -+SNULL, 140011445915648, 140011454308351, -+STORE, 140011454308352, 140011462701055, -+STORE, 140011445915648, 140011454308351, -+SNULL, 140011454312447, 140011462701055, -+STORE, 140011454308352, 140011454312447, -+STORE, 140011454312448, 140011462701055, -+STORE, 140011085225984, 140011102011391, -+SNULL, 140011085230079, 140011102011391, -+STORE, 140011085225984, 140011085230079, -+STORE, 140011085230080, 140011102011391, -+SNULL, 140011177484287, 140011202658303, -+STORE, 140011177480192, 140011177484287, -+STORE, 140011177484288, 140011202658303, -+SNULL, 140011445919743, 140011454308351, -+STORE, 140011445915648, 140011445919743, -+STORE, 140011445919744, 140011454308351, -+SNULL, 140011177484288, 140011185872895, -+STORE, 140011185872896, 140011202658303, -+STORE, 140011177484288, 140011185872895, -+SNULL, 140011185876991, 140011202658303, -+STORE, 140011185872896, 140011185876991, -+STORE, 140011185876992, 140011202658303, -+STORE, 140011076833280, 140011085225983, -+SNULL, 140011202662400, 140011211051007, -+STORE, 140011211051008, 140011219443711, -+STORE, 140011202662400, 140011211051007, -+SNULL, 140011211055103, 140011219443711, -+STORE, 140011211051008, 140011211055103, -+STORE, 140011211055104, 140011219443711, -+SNULL, 140011185876992, 140011194265599, -+STORE, 140011194265600, 140011202658303, -+STORE, 140011185876992, 140011194265599, -+SNULL, 140011194269695, 140011202658303, -+STORE, 140011194265600, 140011194269695, -+STORE, 140011194269696, 140011202658303, -+STORE, 140011068440576, 140011085225983, -+SNULL, 140011311702015, 140011320090623, -+STORE, 140011311697920, 140011311702015, -+STORE, 140011311702016, 140011320090623, -+STORE, 140011060047872, 140011085225983, -+SNULL, 140011060051967, 140011085225983, -+STORE, 140011060047872, 140011060051967, -+STORE, 140011060051968, 140011085225983, -+STORE, 140011051655168, 140011060047871, -+STORE, 140011043262464, 140011060047871, -+SNULL, 140011043266559, 140011060047871, -+STORE, 140011043262464, 140011043266559, -+STORE, 140011043266560, 140011060047871, -+SNULL, 140011219447808, 140011227836415, -+STORE, 140011227836416, 140011236229119, -+STORE, 140011219447808, 140011227836415, -+SNULL, 140011227840511, 140011236229119, -+STORE, 140011227836416, 140011227840511, -+STORE, 140011227840512, 140011236229119, -+SNULL, 140011085230080, 140011093618687, -+STORE, 140011093618688, 140011102011391, -+STORE, 140011085230080, 140011093618687, -+SNULL, 140011093622783, 140011102011391, -+STORE, 140011093618688, 140011093622783, -+STORE, 140011093622784, 140011102011391, -+STORE, 140010959400960, 140010967793663, -+STORE, 140010951008256, 140010967793663, -+SNULL, 140010951008256, 140010959400959, -+STORE, 140010959400960, 140010967793663, -+STORE, 140010951008256, 140010959400959, -+SNULL, 140010959405055, 140010967793663, -+STORE, 140010959400960, 140010959405055, -+STORE, 140010959405056, 140010967793663, -+STORE, 140010942615552, 140010959400959, -+STORE, 140010934222848, 140010959400959, -+SNULL, 140011060051968, 140011076833279, -+STORE, 140011076833280, 140011085225983, -+STORE, 140011060051968, 140011076833279, -+SNULL, 140011076837375, 140011085225983, -+STORE, 140011076833280, 140011076837375, -+STORE, 140011076837376, 140011085225983, -+SNULL, 140011043266560, 140011051655167, -+STORE, 140011051655168, 140011060047871, -+STORE, 140011043266560, 140011051655167, -+SNULL, 140011051659263, 140011060047871, -+STORE, 140011051655168, 140011051659263, -+STORE, 140011051659264, 140011060047871, -+STORE, 140010925830144, 140010959400959, -+SNULL, 140011060051968, 140011068440575, -+STORE, 140011068440576, 140011076833279, -+STORE, 140011060051968, 140011068440575, -+SNULL, 140011068444671, 140011076833279, -+STORE, 140011068440576, 140011068444671, -+STORE, 140011068444672, 140011076833279, -+STORE, 140010917437440, 140010959400959, -+STORE, 140010909044736, 140010959400959, -+STORE, 140010825183232, 140010833575935, -+SNULL, 140010909044736, 140010942615551, -+STORE, 140010942615552, 140010959400959, -+STORE, 140010909044736, 140010942615551, -+SNULL, 140010942619647, 140010959400959, -+STORE, 140010942615552, 140010942619647, -+STORE, 140010942619648, 140010959400959, -+SNULL, 140010909044736, 140010934222847, -+STORE, 140010934222848, 140010942615551, -+STORE, 140010909044736, 140010934222847, -+SNULL, 140010934226943, 140010942615551, -+STORE, 140010934222848, 140010934226943, -+STORE, 140010934226944, 140010942615551, -+SNULL, 140010909048831, 140010934222847, -+STORE, 140010909044736, 140010909048831, -+STORE, 140010909048832, 140010934222847, -+STORE, 140010816790528, 140010833575935, -+SNULL, 140010816794623, 140010833575935, -+STORE, 140010816790528, 140010816794623, -+STORE, 140010816794624, 140010833575935, -+STORE, 140010808397824, 140010816790527, -+SNULL, 140010942619648, 140010951008255, -+STORE, 140010951008256, 140010959400959, -+STORE, 140010942619648, 140010951008255, -+SNULL, 140010951012351, 140010959400959, -+STORE, 140010951008256, 140010951012351, -+STORE, 140010951012352, 140010959400959, -+STORE, 140010800005120, 140010816790527, -+SNULL, 140010800009215, 140010816790527, -+STORE, 140010800005120, 140010800009215, -+STORE, 140010800009216, 140010816790527, -+SNULL, 140010909048832, 140010925830143, -+STORE, 140010925830144, 140010934222847, -+STORE, 140010909048832, 140010925830143, -+SNULL, 140010925834239, 140010934222847, -+STORE, 140010925830144, 140010925834239, -+STORE, 140010925834240, 140010934222847, -+SNULL, 140010816794624, 140010825183231, -+STORE, 140010825183232, 140010833575935, -+STORE, 140010816794624, 140010825183231, -+SNULL, 140010825187327, 140010833575935, -+STORE, 140010825183232, 140010825187327, -+STORE, 140010825187328, 140010833575935, -+SNULL, 140010909048832, 140010917437439, -+STORE, 140010917437440, 140010925830143, -+STORE, 140010909048832, 140010917437439, -+SNULL, 140010917441535, 140010925830143, -+STORE, 140010917437440, 140010917441535, -+STORE, 140010917441536, 140010925830143, -+SNULL, 140010800009216, 140010808397823, -+STORE, 140010808397824, 140010816790527, -+STORE, 140010800009216, 140010808397823, -+SNULL, 140010808401919, 140010816790527, -+STORE, 140010808397824, 140010808401919, -+STORE, 140010808401920, 140010816790527, -+STORE, 140010791612416, 140010800005119, -+SNULL, 140010791616511, 140010800005119, -+STORE, 140010791612416, 140010791616511, -+STORE, 140010791616512, 140010800005119, -+STORE, 140012547100672, 140012547129343, -+STORE, 140012511506432, 140012513697791, -+SNULL, 140012511506432, 140012511596543, -+STORE, 140012511596544, 140012513697791, -+STORE, 140012511506432, 140012511596543, -+SNULL, 140012513689599, 140012513697791, -+STORE, 140012511596544, 140012513689599, -+STORE, 140012513689600, 140012513697791, -+ERASE, 140012513689600, 140012513697791, -+STORE, 140012513689600, 140012513697791, -+SNULL, 140012513693695, 140012513697791, -+STORE, 140012513689600, 140012513693695, -+STORE, 140012513693696, 140012513697791, -+ERASE, 140012547100672, 140012547129343, -+ERASE, 140011362054144, 140011362058239, -+ERASE, 140011362058240, 140011370446847, -+ERASE, 140011882139648, 140011882143743, -+ERASE, 140011882143744, 140011890532351, -+ERASE, 140011873746944, 140011873751039, -+ERASE, 140011873751040, 140011882139647, -+ERASE, 140011588526080, 140011588530175, -+ERASE, 140011588530176, 140011596918783, -+ERASE, 140011328483328, 140011328487423, -+ERASE, 140011328487424, 140011336876031, -+ERASE, 140011898925056, 140011898929151, -+ERASE, 140011898929152, 140011907317759, -+ERASE, 140011353661440, 140011353665535, -+ERASE, 140011353665536, 140011362054143, -+ERASE, 140011336876032, 140011336880127, -+ERASE, 140011336880128, 140011345268735, -+ERASE, 140011731136512, 140011731140607, -+ERASE, 140011731140608, 140011739529215, -+ERASE, 140011479486464, 140011479490559, -+ERASE, 140011479490560, 140011487879167, -+ERASE, 140011756314624, 140011756318719, -+ERASE, 140011756318720, 140011764707327, -+ERASE, 140011580133376, 140011580137471, -+ERASE, 140011580137472, 140011588526079, -+ERASE, 140011219443712, 140011219447807, -+ERASE, 140011219447808, 140011227836415, -+ERASE, 140011051655168, 140011051659263, -+ERASE, 140011051659264, 140011060047871, -+ERASE, 140011999571968, 140011999576063, -+ERASE, 140011999576064, 140012007964671, -+ERASE, 140011714351104, 140011714355199, -+ERASE, 140011714355200, 140011722743807, -+ERASE, 140011739529216, 140011739533311, -+ERASE, 140011739533312, 140011747921919, -+ERASE, 140011320090624, 140011320094719, -+ERASE, 140011320094720, 140011328483327, -+ERASE, 140011630489600, 140011630493695, -+ERASE, 140011630493696, 140011638882303, -+ERASE, 140011345268736, 140011345272831, -+ERASE, 140011345272832, 140011353661439, -+ERASE, 140011496271872, 140011496275967, -+ERASE, 140011496275968, 140011504664575, -+ERASE, 140011194265600, 140011194269695, -+ERASE, 140011194269696, 140011202658303, -+ERASE, 140011068440576, 140011068444671, -+ERASE, 140011068444672, 140011076833279, -+ERASE, 140010909044736, 140010909048831, -+ERASE, 140010909048832, 140010917437439, -+ERASE, 140011764707328, 140011764711423, -+ERASE, 140011764711424, 140011773100031, -+ERASE, 140011462701056, 140011462705151, -+ERASE, 140011462705152, 140011471093759, -+ERASE, 140011076833280, 140011076837375, -+ERASE, 140011076837376, 140011085225983, -+ERASE, 140011991179264, 140011991183359, -+ERASE, 140011991183360, 140011999571967, -+ERASE, 140011211051008, 140011211055103, -+ERASE, 140011211055104, 140011219443711, -+ERASE, 140010917437440, 140010917441535, -+ERASE, 140010917441536, 140010925830143, -+ERASE, 140011085225984, 140011085230079, -+ERASE, 140011085230080, 140011093618687, -+ERASE, 140011487879168, 140011487883263, -+ERASE, 140011487883264, 140011496271871, -+ERASE, 140011856961536, 140011856965631, -+ERASE, 140011856965632, 140011865354239, -+ERASE, 140011982786560, 140011982790655, -+ERASE, 140011982790656, 140011991179263, -+ERASE, 140011722743808, 140011722747903, -+ERASE, 140011722747904, 140011731136511, -+ERASE, 140011177480192, 140011177484287, -+ERASE, 140011177484288, 140011185872895, -+ERASE, 140011848568832, 140011848572927, -+ERASE, 140011848572928, 140011856961535, -+ERASE, 140011890532352, 140011890536447, -+ERASE, 140011890536448, 140011898925055, -+ERASE, 140011622096896, 140011622100991, -+ERASE, 140011622100992, 140011630489599, -+ERASE, 140011311697920, 140011311702015, -+ERASE, 140011311702016, 140011320090623, -+ERASE, 140011471093760, 140011471097855, -+ERASE, 140011471097856, 140011479486463, -+ERASE, 140011605311488, 140011605315583, -+ERASE, 140011605315584, 140011613704191, -+ERASE, 140010791612416, 140010791616511, -+ERASE, 140010791616512, 140010800005119, -+ERASE, 140010959400960, 140010959405055, -+ERASE, 140010959405056, 140010967793663, -+ERASE, 140011185872896, 140011185876991, -+ERASE, 140011185876992, 140011194265599, -+ERASE, 140011454308352, 140011454312447, -+ERASE, 140011454312448, 140011462701055, -+ERASE, 140011596918784, 140011596922879, -+ERASE, 140011596922880, 140011605311487, -+ERASE, 140011060047872, 140011060051967, -+ERASE, 140011060051968, 140011068440575, -+ERASE, 140010925830144, 140010925834239, -+ERASE, 140010925834240, 140010934222847, -+ERASE, 140011747921920, 140011747926015, -+ERASE, 140011747926016, 140011756314623, -+ERASE, 140011202658304, 140011202662399, -+ERASE, 140011202662400, 140011211051007, -+ERASE, 140010800005120, 140010800009215, -+ERASE, 140010800009216, 140010808397823, -+ERASE, 140011093618688, 140011093622783, -+ERASE, 140011093622784, 140011102011391, -+ERASE, 140010808397824, 140010808401919, -+ERASE, 140010808401920, 140010816790527, -+ERASE, 140012419010560, 140012419014655, -+ERASE, 140012419014656, 140012427403263, -+ERASE, 140010934222848, 140010934226943, -+ERASE, 140010934226944, 140010942615551, -+ERASE, 140010942615552, 140010942619647, -+ERASE, 140010942619648, 140010951008255, -+ERASE, 140011613704192, 140011613708287, -+ERASE, 140011613708288, 140011622096895, -+ERASE, 140011865354240, 140011865358335, -+ERASE, 140011865358336, 140011873746943, -+ERASE, 140012301578240, 140012301582335, -+ERASE, 140012301582336, 140012309970943, -+ERASE, 140012393832448, 140012393836543, -+ERASE, 140012393836544, 140012402225151, -+ERASE, 140012410617856, 140012410621951, -+ERASE, 140012410621952, 140012419010559, -+ERASE, 140012402225152, 140012402229247, -+ERASE, 140012402229248, 140012410617855, -+ERASE, 140012259614720, 140012259618815, -+ERASE, 140012259618816, 140012268007423, -+ERASE, 140012251222016, 140012251226111, -+ERASE, 140012251226112, 140012259614719, -+ERASE, 140012284792832, 140012284796927, -+ERASE, 140012284796928, 140012293185535, -+ERASE, 140011445915648, 140011445919743, -+ERASE, 140011445919744, 140011454308351, -+ERASE, 140010951008256, 140010951012351, -+ERASE, 140010951012352, 140010959400959, -+ERASE, 140011043262464, 140011043266559, -+ERASE, 140011043266560, 140011051655167, -+ERASE, 140010825183232, 140010825187327, -+ERASE, 140010825187328, 140010833575935, -+ERASE, 140012293185536, 140012293189631, -+ERASE, 140012293189632, 140012301578239, -+ERASE, 140012276400128, 140012276404223, -+ERASE, 140012276404224, 140012284792831, -+ERASE, 140012016357376, 140012016361471, -+ERASE, 140012016361472, 140012024750079, -+ERASE, 140012024750080, 140012024754175, -+ERASE, 140012024754176, 140012033142783, -+ERASE, 140011227836416, 140011227840511, -+ERASE, 140011227840512, 140011236229119, -+ERASE, 140010816790528, 140010816794623, -+ERASE, 140010816794624, 140010825183231, -+ERASE, 140012268007424, 140012268011519, -+ERASE, 140012268011520, 140012276400127, -+ERASE, 140012385439744, 140012385443839, -+ERASE, 140012385443840, 140012393832447, -+ERASE, 140012522090496, 140012522094591, -+ERASE, 140012522094592, 140012530483199, -+ERASE, 140012033142784, 140012033146879, -+ERASE, 140012033146880, 140012041535487, -+ }; -+ unsigned long set35[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140730536939520, 140737488351231, -+SNULL, 140730536943615, 140737488351231, -+STORE, 140730536939520, 140730536943615, -+STORE, 140730536808448, 140730536943615, -+STORE, 94245239877632, 94245242130431, -+SNULL, 94245240008703, 94245242130431, -+STORE, 94245239877632, 94245240008703, -+STORE, 94245240008704, 94245242130431, -+ERASE, 94245240008704, 94245242130431, -+STORE, 94245242101760, 94245242109951, -+STORE, 94245242109952, 94245242130431, -+STORE, 140475575263232, 140475577516031, -+SNULL, 140475575406591, 140475577516031, -+STORE, 140475575263232, 140475575406591, -+STORE, 140475575406592, 140475577516031, -+ERASE, 140475575406592, 140475577516031, -+STORE, 140475577503744, 140475577511935, -+STORE, 140475577511936, 140475577516031, -+STORE, 140730538164224, 140730538168319, -+STORE, 140730538151936, 140730538164223, -+STORE, 140475577475072, 140475577503743, -+STORE, 140475577466880, 140475577475071, -+STORE, 140475573047296, 140475575263231, -+SNULL, 140475573047296, 140475573145599, -+STORE, 140475573145600, 140475575263231, -+STORE, 140475573047296, 140475573145599, -+SNULL, 140475575238655, 140475575263231, -+STORE, 140475573145600, 140475575238655, -+STORE, 140475575238656, 140475575263231, -+SNULL, 140475575238656, 140475575246847, -+STORE, 140475575246848, 140475575263231, -+STORE, 140475575238656, 140475575246847, -+ERASE, 140475575238656, 140475575246847, -+STORE, 140475575238656, 140475575246847, -+ERASE, 140475575246848, 140475575263231, -+STORE, 140475575246848, 140475575263231, -+STORE, 140475569250304, 140475573047295, -+SNULL, 140475569250304, 140475570909183, -+STORE, 140475570909184, 140475573047295, -+STORE, 140475569250304, 140475570909183, -+SNULL, 140475573006335, 140475573047295, -+STORE, 140475570909184, 140475573006335, -+STORE, 140475573006336, 140475573047295, -+SNULL, 140475573006336, 140475573030911, -+STORE, 140475573030912, 140475573047295, -+STORE, 140475573006336, 140475573030911, -+ERASE, 140475573006336, 140475573030911, -+STORE, 140475573006336, 140475573030911, -+ERASE, 140475573030912, 140475573047295, -+STORE, 140475573030912, 140475573047295, -+STORE, 140475577458688, 140475577475071, -+SNULL, 140475573022719, 140475573030911, -+STORE, 140475573006336, 140475573022719, -+STORE, 140475573022720, 140475573030911, -+SNULL, 140475575242751, 140475575246847, -+STORE, 140475575238656, 140475575242751, -+STORE, 140475575242752, 140475575246847, -+SNULL, 94245242105855, 94245242109951, -+STORE, 94245242101760, 94245242105855, -+STORE, 94245242105856, 94245242109951, -+SNULL, 140475577507839, 140475577511935, -+STORE, 140475577503744, 140475577507839, -+STORE, 140475577507840, 140475577511935, -+ERASE, 140475577475072, 140475577503743, -+STORE, 94245271216128, 94245271351295, -+STORE, 140475560857600, 140475569250303, -+SNULL, 140475560861695, 140475569250303, -+STORE, 140475560857600, 140475560861695, -+STORE, 140475560861696, 140475569250303, -+STORE, 140475552464896, 140475560857599, -+STORE, 140475418247168, 140475552464895, -+SNULL, 140475418247168, 140475428241407, -+STORE, 140475428241408, 140475552464895, -+STORE, 140475418247168, 140475428241407, -+ERASE, 140475418247168, 140475428241407, -+SNULL, 140475495350271, 140475552464895, -+STORE, 140475428241408, 140475495350271, -+STORE, 140475495350272, 140475552464895, -+ERASE, 140475495350272, 140475552464895, -+SNULL, 140475428376575, 140475495350271, -+STORE, 140475428241408, 140475428376575, -+STORE, 140475428376576, 140475495350271, -+SNULL, 140475552468991, 140475560857599, -+STORE, 140475552464896, 140475552468991, -+STORE, 140475552468992, 140475560857599, -+STORE, 140475544072192, 140475552464895, -+SNULL, 140475544076287, 140475552464895, -+STORE, 140475544072192, 140475544076287, -+STORE, 140475544076288, 140475552464895, -+STORE, 140475535679488, 140475544072191, -+SNULL, 140475535683583, 140475544072191, -+STORE, 140475535679488, 140475535683583, -+STORE, 140475535683584, 140475544072191, -+STORE, 140475527286784, 140475535679487, -+SNULL, 140475527290879, 140475535679487, -+STORE, 140475527286784, 140475527290879, -+STORE, 140475527290880, 140475535679487, -+STORE, 140475518894080, 140475527286783, -+STORE, 140475510501376, 140475527286783, -+STORE, 140475502108672, 140475527286783, -+STORE, 140475419848704, 140475428241407, -+STORE, 140475285630976, 140475419848703, -+SNULL, 140475285630976, 140475294023679, -+STORE, 140475294023680, 140475419848703, -+STORE, 140475285630976, 140475294023679, -+ERASE, 140475285630976, 140475294023679, -+STORE, 140475159805952, 140475419848703, -+STORE, 140475025588224, 140475419848703, -+SNULL, 140475092697087, 140475419848703, -+STORE, 140475025588224, 140475092697087, -+STORE, 140475092697088, 140475419848703, -+SNULL, 140475092697088, 140475159805951, -+STORE, 140475159805952, 140475419848703, -+STORE, 140475092697088, 140475159805951, -+ERASE, 140475092697088, 140475159805951, -+STORE, 140474891370496, 140475092697087, -+SNULL, 140474958479359, 140475092697087, -+STORE, 140474891370496, 140474958479359, -+STORE, 140474958479360, 140475092697087, -+SNULL, 140474958479360, 140475025588223, -+STORE, 140475025588224, 140475092697087, -+STORE, 140474958479360, 140475025588223, -+ERASE, 140474958479360, 140475025588223, -+SNULL, 140475361132543, 140475419848703, -+STORE, 140475159805952, 140475361132543, -+STORE, 140475361132544, 140475419848703, -+ERASE, 140475361132544, 140475419848703, -+SNULL, 140475159805952, 140475294023679, -+STORE, 140475294023680, 140475361132543, -+STORE, 140475159805952, 140475294023679, -+SNULL, 140475294158847, 140475361132543, -+STORE, 140475294023680, 140475294158847, -+STORE, 140475294158848, 140475361132543, -+SNULL, 140475226914815, 140475294023679, -+STORE, 140475159805952, 140475226914815, -+STORE, 140475226914816, 140475294023679, -+ERASE, 140475226914816, 140475294023679, -+SNULL, 140475025723391, 140475092697087, -+STORE, 140475025588224, 140475025723391, -+STORE, 140475025723392, 140475092697087, -+SNULL, 140475159941119, 140475226914815, -+STORE, 140475159805952, 140475159941119, -+STORE, 140475159941120, 140475226914815, -+SNULL, 140474891505663, 140474958479359, -+STORE, 140474891370496, 140474891505663, -+STORE, 140474891505664, 140474958479359, -+SNULL, 140475502108672, 140475518894079, -+STORE, 140475518894080, 140475527286783, -+STORE, 140475502108672, 140475518894079, -+SNULL, 140475518898175, 140475527286783, -+STORE, 140475518894080, 140475518898175, -+STORE, 140475518898176, 140475527286783, -+STORE, 140475411456000, 140475428241407, -+SNULL, 140475502112767, 140475518894079, -+STORE, 140475502108672, 140475502112767, -+STORE, 140475502112768, 140475518894079, -+SNULL, 140475411460095, 140475428241407, -+STORE, 140475411456000, 140475411460095, -+STORE, 140475411460096, 140475428241407, -+SNULL, 140475411460096, 140475419848703, -+STORE, 140475419848704, 140475428241407, -+STORE, 140475411460096, 140475419848703, -+SNULL, 140475419852799, 140475428241407, -+STORE, 140475419848704, 140475419852799, -+STORE, 140475419852800, 140475428241407, -+STORE, 140475403063296, 140475411455999, -+SNULL, 140475502112768, 140475510501375, -+STORE, 140475510501376, 140475518894079, -+STORE, 140475502112768, 140475510501375, -+SNULL, 140475510505471, 140475518894079, -+STORE, 140475510501376, 140475510505471, -+STORE, 140475510505472, 140475518894079, -+SNULL, 140475403067391, 140475411455999, -+STORE, 140475403063296, 140475403067391, -+STORE, 140475403067392, 140475411455999, -+STORE, 140475394670592, 140475403063295, -+SNULL, 140475394674687, 140475403063295, -+STORE, 140475394670592, 140475394674687, -+STORE, 140475394674688, 140475403063295, -+STORE, 140475386277888, 140475394670591, -+STORE, 140475377885184, 140475394670591, -+STORE, 140475369492480, 140475394670591, -+SNULL, 140475369496575, 140475394670591, -+STORE, 140475369492480, 140475369496575, -+STORE, 140475369496576, 140475394670591, -+SNULL, 140475369496576, 140475377885183, -+STORE, 140475377885184, 140475394670591, -+STORE, 140475369496576, 140475377885183, -+SNULL, 140475377889279, 140475394670591, -+STORE, 140475377885184, 140475377889279, -+STORE, 140475377889280, 140475394670591, -+STORE, 140475285630976, 140475294023679, -+SNULL, 140475377889280, 140475386277887, -+STORE, 140475386277888, 140475394670591, -+STORE, 140475377889280, 140475386277887, -+SNULL, 140475386281983, 140475394670591, -+STORE, 140475386277888, 140475386281983, -+STORE, 140475386281984, 140475394670591, -+SNULL, 140475285635071, 140475294023679, -+STORE, 140475285630976, 140475285635071, -+STORE, 140475285635072, 140475294023679, -+STORE, 140475277238272, 140475285630975, -+STORE, 140475268845568, 140475285630975, -+SNULL, 140475268845568, 140475277238271, -+STORE, 140475277238272, 140475285630975, -+STORE, 140475268845568, 140475277238271, -+SNULL, 140475277242367, 140475285630975, -+STORE, 140475277238272, 140475277242367, -+STORE, 140475277242368, 140475285630975, -+STORE, 140475260452864, 140475277238271, -+SNULL, 140475260452864, 140475268845567, -+STORE, 140475268845568, 140475277238271, -+STORE, 140475260452864, 140475268845567, -+SNULL, 140475268849663, 140475277238271, -+STORE, 140475268845568, 140475268849663, -+STORE, 140475268849664, 140475277238271, -+SNULL, 140475260456959, 140475268845567, -+STORE, 140475260452864, 140475260456959, -+STORE, 140475260456960, 140475268845567, -+STORE, 140475252060160, 140475260452863, -+SNULL, 140475252064255, 140475260452863, -+STORE, 140475252060160, 140475252064255, -+STORE, 140475252064256, 140475260452863, -+STORE, 140475243667456, 140475252060159, -+SNULL, 140475243671551, 140475252060159, -+STORE, 140475243667456, 140475243671551, -+STORE, 140475243671552, 140475252060159, -+STORE, 140475235274752, 140475243667455, -+STORE, 140475151413248, 140475159805951, -+STORE, 140474891505664, 140475025588223, -+STORE, 140475143020544, 140475159805951, -+SNULL, 140474891505664, 140474958479359, -+STORE, 140474958479360, 140475025588223, -+STORE, 140474891505664, 140474958479359, -+SNULL, 140474958614527, 140475025588223, -+STORE, 140474958479360, 140474958614527, -+STORE, 140474958614528, 140475025588223, -+STORE, 140474824261632, 140474891370495, -+SNULL, 140474824396799, 140474891370495, -+STORE, 140474824261632, 140474824396799, -+STORE, 140474824396800, 140474891370495, -+STORE, 140475134627840, 140475159805951, -+STORE, 140474690043904, 140474824261631, -+STORE, 140475126235136, 140475159805951, -+STORE, 140475117842432, 140475159805951, -+STORE, 140474622935040, 140474824261631, -+STORE, 140475109449728, 140475159805951, -+STORE, 140474488717312, 140474824261631, -+STORE, 140475101057024, 140475159805951, -+STORE, 140474480324608, 140474488717311, -+STORE, 140474413215744, 140474480324607, -+STORE, 140474404823040, 140474413215743, -+ERASE, 140474413215744, 140474480324607, -+STORE, 140474471931904, 140474488717311, -+STORE, 140474270605312, 140474404823039, -+SNULL, 140475101057024, 140475126235135, -+STORE, 140475126235136, 140475159805951, -+STORE, 140475101057024, 140475126235135, -+SNULL, 140475126239231, 140475159805951, -+STORE, 140475126235136, 140475126239231, -+STORE, 140475126239232, 140475159805951, -+STORE, 140474463539200, 140474488717311, -+STORE, 140474455146496, 140474488717311, -+SNULL, 140474455150591, 140474488717311, -+STORE, 140474455146496, 140474455150591, -+STORE, 140474455150592, 140474488717311, -+STORE, 140474446753792, 140474455146495, -+SNULL, 140474446757887, 140474455146495, -+STORE, 140474446753792, 140474446757887, -+STORE, 140474446757888, 140474455146495, -+STORE, 140474438361088, 140474446753791, -+STORE, 140474429968384, 140474446753791, -+SNULL, 140474429972479, 140474446753791, -+STORE, 140474429968384, 140474429972479, -+STORE, 140474429972480, 140474446753791, -+SNULL, 140475235278847, 140475243667455, -+STORE, 140475235274752, 140475235278847, -+STORE, 140475235278848, 140475243667455, -+SNULL, 140474757152767, 140474824261631, -+STORE, 140474488717312, 140474757152767, -+STORE, 140474757152768, 140474824261631, -+ERASE, 140474757152768, 140474824261631, -+SNULL, 140474488717312, 140474690043903, -+STORE, 140474690043904, 140474757152767, -+STORE, 140474488717312, 140474690043903, -+SNULL, 140474690179071, 140474757152767, -+STORE, 140474690043904, 140474690179071, -+STORE, 140474690179072, 140474757152767, -+SNULL, 140474488717312, 140474622935039, -+STORE, 140474622935040, 140474690043903, -+STORE, 140474488717312, 140474622935039, -+SNULL, 140474623070207, 140474690043903, -+STORE, 140474622935040, 140474623070207, -+STORE, 140474623070208, 140474690043903, -+SNULL, 140475101057024, 140475117842431, -+STORE, 140475117842432, 140475126235135, -+STORE, 140475101057024, 140475117842431, -+SNULL, 140475117846527, 140475126235135, -+STORE, 140475117842432, 140475117846527, -+STORE, 140475117846528, 140475126235135, -+SNULL, 140474555826175, 140474622935039, -+STORE, 140474488717312, 140474555826175, -+STORE, 140474555826176, 140474622935039, -+ERASE, 140474555826176, 140474622935039, -+STORE, 140474136387584, 140474404823039, -+SNULL, 140474136387584, 140474153172991, -+STORE, 140474153172992, 140474404823039, -+STORE, 140474136387584, 140474153172991, -+ERASE, 140474136387584, 140474153172991, -+STORE, 140474018955264, 140474404823039, -+STORE, 140473884737536, 140474404823039, -+SNULL, 140474086064127, 140474404823039, -+STORE, 140473884737536, 140474086064127, -+STORE, 140474086064128, 140474404823039, -+SNULL, 140474086064128, 140474153172991, -+STORE, 140474153172992, 140474404823039, -+STORE, 140474086064128, 140474153172991, -+ERASE, 140474086064128, 140474153172991, -+STORE, 140473750519808, 140474086064127, -+SNULL, 140473817628671, 140474086064127, -+STORE, 140473750519808, 140473817628671, -+STORE, 140473817628672, 140474086064127, -+SNULL, 140473817628672, 140473884737535, -+STORE, 140473884737536, 140474086064127, -+STORE, 140473817628672, 140473884737535, -+ERASE, 140473817628672, 140473884737535, -+SNULL, 140475126239232, 140475151413247, -+STORE, 140475151413248, 140475159805951, -+STORE, 140475126239232, 140475151413247, -+SNULL, 140475151417343, 140475159805951, -+STORE, 140475151413248, 140475151417343, -+STORE, 140475151417344, 140475159805951, -+SNULL, 140474270605311, 140474404823039, -+STORE, 140474153172992, 140474270605311, -+STORE, 140474270605312, 140474404823039, -+SNULL, 140474270605312, 140474287390719, -+STORE, 140474287390720, 140474404823039, -+STORE, 140474270605312, 140474287390719, -+ERASE, 140474270605312, 140474287390719, -+SNULL, 140474429972480, 140474438361087, -+STORE, 140474438361088, 140474446753791, -+STORE, 140474429972480, 140474438361087, -+SNULL, 140474438365183, 140474446753791, -+STORE, 140474438361088, 140474438365183, -+STORE, 140474438365184, 140474446753791, -+STORE, 140474815868928, 140474824261631, -+SNULL, 140474815873023, 140474824261631, -+STORE, 140474815868928, 140474815873023, -+STORE, 140474815873024, 140474824261631, -+SNULL, 140474220281855, 140474270605311, -+STORE, 140474153172992, 140474220281855, -+STORE, 140474220281856, 140474270605311, -+ERASE, 140474220281856, 140474270605311, -+SNULL, 140474488852479, 140474555826175, -+STORE, 140474488717312, 140474488852479, -+STORE, 140474488852480, 140474555826175, -+SNULL, 140475101057024, 140475109449727, -+STORE, 140475109449728, 140475117842431, -+STORE, 140475101057024, 140475109449727, -+SNULL, 140475109453823, 140475117842431, -+STORE, 140475109449728, 140475109453823, -+STORE, 140475109453824, 140475117842431, -+SNULL, 140473951846399, 140474086064127, -+STORE, 140473884737536, 140473951846399, -+STORE, 140473951846400, 140474086064127, -+SNULL, 140473951846400, 140474018955263, -+STORE, 140474018955264, 140474086064127, -+STORE, 140473951846400, 140474018955263, -+ERASE, 140473951846400, 140474018955263, -+SNULL, 140473884872703, 140473951846399, -+STORE, 140473884737536, 140473884872703, -+STORE, 140473884872704, 140473951846399, -+SNULL, 140474019090431, 140474086064127, -+STORE, 140474018955264, 140474019090431, -+STORE, 140474019090432, 140474086064127, -+SNULL, 140473750654975, 140473817628671, -+STORE, 140473750519808, 140473750654975, -+STORE, 140473750654976, 140473817628671, -+SNULL, 140474455150592, 140474463539199, -+STORE, 140474463539200, 140474488717311, -+STORE, 140474455150592, 140474463539199, -+SNULL, 140474463543295, 140474488717311, -+STORE, 140474463539200, 140474463543295, -+STORE, 140474463543296, 140474488717311, -+STORE, 140474807476224, 140474815868927, -+SNULL, 140474463543296, 140474471931903, -+STORE, 140474471931904, 140474488717311, -+STORE, 140474463543296, 140474471931903, -+SNULL, 140474471935999, 140474488717311, -+STORE, 140474471931904, 140474471935999, -+STORE, 140474471936000, 140474488717311, -+STORE, 140474799083520, 140474815868927, -+STORE, 140474790690816, 140474815868927, -+SNULL, 140474790690816, 140474799083519, -+STORE, 140474799083520, 140474815868927, -+STORE, 140474790690816, 140474799083519, -+SNULL, 140474799087615, 140474815868927, -+STORE, 140474799083520, 140474799087615, -+STORE, 140474799087616, 140474815868927, -+SNULL, 140474354499583, 140474404823039, -+STORE, 140474287390720, 140474354499583, -+STORE, 140474354499584, 140474404823039, -+ERASE, 140474354499584, 140474404823039, -+SNULL, 140474287525887, 140474354499583, -+STORE, 140474287390720, 140474287525887, -+STORE, 140474287525888, 140474354499583, -+STORE, 140474782298112, 140474799083519, -+STORE, 140474773905408, 140474799083519, -+SNULL, 140474773909503, 140474799083519, -+STORE, 140474773905408, 140474773909503, -+STORE, 140474773909504, 140474799083519, -+SNULL, 140475126239232, 140475134627839, -+STORE, 140475134627840, 140475151413247, -+STORE, 140475126239232, 140475134627839, -+SNULL, 140475134631935, 140475151413247, -+STORE, 140475134627840, 140475134631935, -+STORE, 140475134631936, 140475151413247, -+STORE, 140474765512704, 140474773905407, -+STORE, 140474614542336, 140474622935039, -+SNULL, 140474153308159, 140474220281855, -+STORE, 140474153172992, 140474153308159, -+STORE, 140474153308160, 140474220281855, -+SNULL, 140474404827135, 140474413215743, -+STORE, 140474404823040, 140474404827135, -+STORE, 140474404827136, 140474413215743, -+STORE, 140474606149632, 140474622935039, -+SNULL, 140474606153727, 140474622935039, -+STORE, 140474606149632, 140474606153727, -+STORE, 140474606153728, 140474622935039, -+STORE, 140474597756928, 140474606149631, -+SNULL, 140474597761023, 140474606149631, -+STORE, 140474597756928, 140474597761023, -+STORE, 140474597761024, 140474606149631, -+SNULL, 140475134631936, 140475143020543, -+STORE, 140475143020544, 140475151413247, -+STORE, 140475134631936, 140475143020543, -+SNULL, 140475143024639, 140475151413247, -+STORE, 140475143020544, 140475143024639, -+STORE, 140475143024640, 140475151413247, -+STORE, 140474589364224, 140474597756927, -+SNULL, 140474606153728, 140474614542335, -+STORE, 140474614542336, 140474622935039, -+STORE, 140474606153728, 140474614542335, -+SNULL, 140474614546431, 140474622935039, -+STORE, 140474614542336, 140474614546431, -+STORE, 140474614546432, 140474622935039, -+SNULL, 140474765516799, 140474773905407, -+STORE, 140474765512704, 140474765516799, -+STORE, 140474765516800, 140474773905407, -+STORE, 140474580971520, 140474597756927, -+SNULL, 140474773909504, 140474782298111, -+STORE, 140474782298112, 140474799083519, -+STORE, 140474773909504, 140474782298111, -+SNULL, 140474782302207, 140474799083519, -+STORE, 140474782298112, 140474782302207, -+STORE, 140474782302208, 140474799083519, -+SNULL, 140474471936000, 140474480324607, -+STORE, 140474480324608, 140474488717311, -+STORE, 140474471936000, 140474480324607, -+SNULL, 140474480328703, 140474488717311, -+STORE, 140474480324608, 140474480328703, -+STORE, 140474480328704, 140474488717311, -+STORE, 140474572578816, 140474597756927, -+SNULL, 140474572582911, 140474597756927, -+STORE, 140474572578816, 140474572582911, -+STORE, 140474572582912, 140474597756927, -+SNULL, 140474782302208, 140474790690815, -+STORE, 140474790690816, 140474799083519, -+STORE, 140474782302208, 140474790690815, -+SNULL, 140474790694911, 140474799083519, -+STORE, 140474790690816, 140474790694911, -+STORE, 140474790694912, 140474799083519, -+STORE, 140474564186112, 140474572578815, -+STORE, 140474421575680, 140474429968383, -+STORE, 140474396430336, 140474404823039, -+SNULL, 140474396434431, 140474404823039, -+STORE, 140474396430336, 140474396434431, -+STORE, 140474396434432, 140474404823039, -+STORE, 140474388037632, 140474396430335, -+SNULL, 140474799087616, 140474807476223, -+STORE, 140474807476224, 140474815868927, -+STORE, 140474799087616, 140474807476223, -+SNULL, 140474807480319, 140474815868927, -+STORE, 140474807476224, 140474807480319, -+STORE, 140474807480320, 140474815868927, -+SNULL, 140475101061119, 140475109449727, -+STORE, 140475101057024, 140475101061119, -+STORE, 140475101061120, 140475109449727, -+STORE, 140474379644928, 140474396430335, -+SNULL, 140474572582912, 140474589364223, -+STORE, 140474589364224, 140474597756927, -+STORE, 140474572582912, 140474589364223, -+SNULL, 140474589368319, 140474597756927, -+STORE, 140474589364224, 140474589368319, -+STORE, 140474589368320, 140474597756927, -+STORE, 140474371252224, 140474396430335, -+STORE, 140474362859520, 140474396430335, -+STORE, 140474278998016, 140474287390719, -+STORE, 140474270605312, 140474287390719, -+STORE, 140474262212608, 140474287390719, -+SNULL, 140474262216703, 140474287390719, -+STORE, 140474262212608, 140474262216703, -+STORE, 140474262216704, 140474287390719, -+STORE, 140474253819904, 140474262212607, -+SNULL, 140474253823999, 140474262212607, -+STORE, 140474253819904, 140474253823999, -+STORE, 140474253824000, 140474262212607, -+SNULL, 140474362859520, 140474388037631, -+STORE, 140474388037632, 140474396430335, -+STORE, 140474362859520, 140474388037631, -+SNULL, 140474388041727, 140474396430335, -+STORE, 140474388037632, 140474388041727, -+STORE, 140474388041728, 140474396430335, -+SNULL, 140474362859520, 140474379644927, -+STORE, 140474379644928, 140474388037631, -+STORE, 140474362859520, 140474379644927, -+SNULL, 140474379649023, 140474388037631, -+STORE, 140474379644928, 140474379649023, -+STORE, 140474379649024, 140474388037631, -+STORE, 140474245427200, 140474253819903, -+STORE, 140474237034496, 140474253819903, -+STORE, 140474228641792, 140474253819903, -+STORE, 140474144780288, 140474153172991, -+SNULL, 140474228645887, 140474253819903, -+STORE, 140474228641792, 140474228645887, -+STORE, 140474228645888, 140474253819903, -+SNULL, 140474564190207, 140474572578815, -+STORE, 140474564186112, 140474564190207, -+STORE, 140474564190208, 140474572578815, -+STORE, 140474136387584, 140474153172991, -+SNULL, 140474362859520, 140474371252223, -+STORE, 140474371252224, 140474379644927, -+STORE, 140474362859520, 140474371252223, -+SNULL, 140474371256319, 140474379644927, -+STORE, 140474371252224, 140474371256319, -+STORE, 140474371256320, 140474379644927, -+STORE, 140474127994880, 140474153172991, -+STORE, 140474119602176, 140474153172991, -+SNULL, 140474421579775, 140474429968383, -+STORE, 140474421575680, 140474421579775, -+STORE, 140474421579776, 140474429968383, -+STORE, 140474111209472, 140474153172991, -+SNULL, 140474111213567, 140474153172991, -+STORE, 140474111209472, 140474111213567, -+STORE, 140474111213568, 140474153172991, -+SNULL, 140474262216704, 140474270605311, -+STORE, 140474270605312, 140474287390719, -+STORE, 140474262216704, 140474270605311, -+SNULL, 140474270609407, 140474287390719, -+STORE, 140474270605312, 140474270609407, -+STORE, 140474270609408, 140474287390719, -+STORE, 140474102816768, 140474111209471, -+SNULL, 140474102820863, 140474111209471, -+STORE, 140474102816768, 140474102820863, -+STORE, 140474102820864, 140474111209471, -+SNULL, 140474270609408, 140474278998015, -+STORE, 140474278998016, 140474287390719, -+STORE, 140474270609408, 140474278998015, -+SNULL, 140474279002111, 140474287390719, -+STORE, 140474278998016, 140474279002111, -+STORE, 140474279002112, 140474287390719, -+STORE, 140474094424064, 140474102816767, -+SNULL, 140474572582912, 140474580971519, -+STORE, 140474580971520, 140474589364223, -+STORE, 140474572582912, 140474580971519, -+SNULL, 140474580975615, 140474589364223, -+STORE, 140474580971520, 140474580975615, -+STORE, 140474580975616, 140474589364223, -+SNULL, 140474362863615, 140474371252223, -+STORE, 140474362859520, 140474362863615, -+STORE, 140474362863616, 140474371252223, -+STORE, 140474010562560, 140474018955263, -+SNULL, 140474228645888, 140474245427199, -+STORE, 140474245427200, 140474253819903, -+STORE, 140474228645888, 140474245427199, -+SNULL, 140474245431295, 140474253819903, -+STORE, 140474245427200, 140474245431295, -+STORE, 140474245431296, 140474253819903, -+SNULL, 140474111213568, 140474136387583, -+STORE, 140474136387584, 140474153172991, -+STORE, 140474111213568, 140474136387583, -+SNULL, 140474136391679, 140474153172991, -+STORE, 140474136387584, 140474136391679, -+STORE, 140474136391680, 140474153172991, -+STORE, 140474002169856, 140474018955263, -+STORE, 140473993777152, 140474018955263, -+SNULL, 140474111213568, 140474127994879, -+STORE, 140474127994880, 140474136387583, -+STORE, 140474111213568, 140474127994879, -+SNULL, 140474127998975, 140474136387583, -+STORE, 140474127994880, 140474127998975, -+STORE, 140474127998976, 140474136387583, -+SNULL, 140474228645888, 140474237034495, -+STORE, 140474237034496, 140474245427199, -+STORE, 140474228645888, 140474237034495, -+SNULL, 140474237038591, 140474245427199, -+STORE, 140474237034496, 140474237038591, -+STORE, 140474237038592, 140474245427199, -+SNULL, 140474136391680, 140474144780287, -+STORE, 140474144780288, 140474153172991, -+STORE, 140474136391680, 140474144780287, -+SNULL, 140474144784383, 140474153172991, -+STORE, 140474144780288, 140474144784383, -+STORE, 140474144784384, 140474153172991, -+STORE, 140473985384448, 140474018955263, -+STORE, 140473976991744, 140474018955263, -+STORE, 140473968599040, 140474018955263, -+SNULL, 140473968603135, 140474018955263, -+STORE, 140473968599040, 140473968603135, -+STORE, 140473968603136, 140474018955263, -+SNULL, 140474111213568, 140474119602175, -+STORE, 140474119602176, 140474127994879, -+STORE, 140474111213568, 140474119602175, -+SNULL, 140474119606271, 140474127994879, -+STORE, 140474119602176, 140474119606271, -+STORE, 140474119606272, 140474127994879, -+STORE, 140473960206336, 140473968599039, -+SNULL, 140474094428159, 140474102816767, -+STORE, 140474094424064, 140474094428159, -+STORE, 140474094428160, 140474102816767, -+STORE, 140473876344832, 140473884737535, -+STORE, 140473867952128, 140473884737535, -+STORE, 140473859559424, 140473884737535, -+SNULL, 140473859563519, 140473884737535, -+STORE, 140473859559424, 140473859563519, -+STORE, 140473859563520, 140473884737535, -+SNULL, 140473968603136, 140473993777151, -+STORE, 140473993777152, 140474018955263, -+STORE, 140473968603136, 140473993777151, -+SNULL, 140473993781247, 140474018955263, -+STORE, 140473993777152, 140473993781247, -+STORE, 140473993781248, 140474018955263, -+SNULL, 140473960210431, 140473968599039, -+STORE, 140473960206336, 140473960210431, -+STORE, 140473960210432, 140473968599039, -+SNULL, 140473993781248, 140474010562559, -+STORE, 140474010562560, 140474018955263, -+STORE, 140473993781248, 140474010562559, -+SNULL, 140474010566655, 140474018955263, -+STORE, 140474010562560, 140474010566655, -+STORE, 140474010566656, 140474018955263, -+SNULL, 140473968603136, 140473985384447, -+STORE, 140473985384448, 140473993777151, -+STORE, 140473968603136, 140473985384447, -+SNULL, 140473985388543, 140473993777151, -+STORE, 140473985384448, 140473985388543, -+STORE, 140473985388544, 140473993777151, -+SNULL, 140473993781248, 140474002169855, -+STORE, 140474002169856, 140474010562559, -+STORE, 140473993781248, 140474002169855, -+SNULL, 140474002173951, 140474010562559, -+STORE, 140474002169856, 140474002173951, -+STORE, 140474002173952, 140474010562559, -+STORE, 140473851166720, 140473859559423, -+SNULL, 140473851170815, 140473859559423, -+STORE, 140473851166720, 140473851170815, -+STORE, 140473851170816, 140473859559423, -+SNULL, 140473968603136, 140473976991743, -+STORE, 140473976991744, 140473985384447, -+STORE, 140473968603136, 140473976991743, -+SNULL, 140473976995839, 140473985384447, -+STORE, 140473976991744, 140473976995839, -+STORE, 140473976995840, 140473985384447, -+STORE, 140473842774016, 140473851166719, -+SNULL, 140473859563520, 140473867952127, -+STORE, 140473867952128, 140473884737535, -+STORE, 140473859563520, 140473867952127, -+SNULL, 140473867956223, 140473884737535, -+STORE, 140473867952128, 140473867956223, -+STORE, 140473867956224, 140473884737535, -+SNULL, 140473867956224, 140473876344831, -+STORE, 140473876344832, 140473884737535, -+STORE, 140473867956224, 140473876344831, -+SNULL, 140473876348927, 140473884737535, -+STORE, 140473876344832, 140473876348927, -+STORE, 140473876348928, 140473884737535, -+STORE, 140473834381312, 140473851166719, -+SNULL, 140473834385407, 140473851166719, -+STORE, 140473834381312, 140473834385407, -+STORE, 140473834385408, 140473851166719, -+SNULL, 140473834385408, 140473842774015, -+STORE, 140473842774016, 140473851166719, -+STORE, 140473834385408, 140473842774015, -+SNULL, 140473842778111, 140473851166719, -+STORE, 140473842774016, 140473842778111, -+STORE, 140473842778112, 140473851166719, -+STORE, 140473825988608, 140473834381311, -+SNULL, 140473825992703, 140473834381311, -+STORE, 140473825988608, 140473825992703, -+STORE, 140473825992704, 140473834381311, -+STORE, 140475577475072, 140475577503743, -+STORE, 140475499917312, 140475502108671, -+SNULL, 140475499917312, 140475500007423, -+STORE, 140475500007424, 140475502108671, -+STORE, 140475499917312, 140475500007423, -+SNULL, 140475502100479, 140475502108671, -+STORE, 140475500007424, 140475502100479, -+STORE, 140475502100480, 140475502108671, -+ERASE, 140475502100480, 140475502108671, -+STORE, 140475502100480, 140475502108671, -+SNULL, 140475502104575, 140475502108671, -+STORE, 140475502100480, 140475502104575, -+STORE, 140475502104576, 140475502108671, -+ERASE, 140475577475072, 140475577503743, -+ERASE, 140475235274752, 140475235278847, -+ERASE, 140475235278848, 140475243667455, -+ERASE, 140474815868928, 140474815873023, -+ERASE, 140474815873024, 140474824261631, -+ERASE, 140474606149632, 140474606153727, -+ERASE, 140474606153728, 140474614542335, -+ERASE, 140474270605312, 140474270609407, -+ERASE, 140474270609408, 140474278998015, -+ERASE, 140474438361088, 140474438365183, -+ERASE, 140474438365184, 140474446753791, -+ERASE, 140474597756928, 140474597761023, -+ERASE, 140474597761024, 140474606149631, -+ERASE, 140475126235136, 140475126239231, -+ERASE, 140475126239232, 140475134627839, -+ERASE, 140474463539200, 140474463543295, -+ERASE, 140474463543296, 140474471931903, -+ERASE, 140474388037632, 140474388041727, -+ERASE, 140474388041728, 140474396430335, -+ERASE, 140474404823040, 140474404827135, -+ERASE, 140474404827136, 140474413215743, -+ERASE, 140474278998016, 140474279002111, -+ERASE, 140474279002112, 140474287390719, -+ERASE, 140474094424064, 140474094428159, -+ERASE, 140474094428160, 140474102816767, -+ERASE, 140473867952128, 140473867956223, -+ERASE, 140473867956224, 140473876344831, -+ERASE, 140475151413248, 140475151417343, -+ERASE, 140475151417344, 140475159805951, -+ERASE, 140474455146496, 140474455150591, -+ERASE, 140474455150592, 140474463539199, -+ERASE, 140474807476224, 140474807480319, -+ERASE, 140474807480320, 140474815868927, -+ERASE, 140475117842432, 140475117846527, -+ERASE, 140475117846528, 140475126235135, -+ERASE, 140474446753792, 140474446757887, -+ERASE, 140474446757888, 140474455146495, -+ERASE, 140474429968384, 140474429972479, -+ERASE, 140474429972480, 140474438361087, -+ERASE, 140474782298112, 140474782302207, -+ERASE, 140474782302208, 140474790690815, -+ERASE, 140474136387584, 140474136391679, -+ERASE, 140474136391680, 140474144780287, -+ERASE, 140474002169856, 140474002173951, -+ERASE, 140474002173952, 140474010562559, -+ERASE, 140475134627840, 140475134631935, -+ERASE, 140475134631936, 140475143020543, -+ERASE, 140474471931904, 140474471935999, -+ERASE, 140474471936000, 140474480324607, -+ERASE, 140474396430336, 140474396434431, -+ERASE, 140474396434432, 140474404823039, -+ }; -+ unsigned long set36[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140723893125120, 140737488351231, -+SNULL, 140723893129215, 140737488351231, -+STORE, 140723893125120, 140723893129215, -+STORE, 140723892994048, 140723893129215, -+STORE, 94076829786112, 94076832038911, -+SNULL, 94076829917183, 94076832038911, -+STORE, 94076829786112, 94076829917183, -+STORE, 94076829917184, 94076832038911, -+ERASE, 94076829917184, 94076832038911, -+STORE, 94076832010240, 94076832018431, -+STORE, 94076832018432, 94076832038911, -+STORE, 140122444345344, 140122446598143, -+SNULL, 140122444488703, 140122446598143, -+STORE, 140122444345344, 140122444488703, -+STORE, 140122444488704, 140122446598143, -+ERASE, 140122444488704, 140122446598143, -+STORE, 140122446585856, 140122446594047, -+STORE, 140122446594048, 140122446598143, -+STORE, 140723893538816, 140723893542911, -+STORE, 140723893526528, 140723893538815, -+STORE, 140122446557184, 140122446585855, -+STORE, 140122446548992, 140122446557183, -+STORE, 140122442129408, 140122444345343, -+SNULL, 140122442129408, 140122442227711, -+STORE, 140122442227712, 140122444345343, -+STORE, 140122442129408, 140122442227711, -+SNULL, 140122444320767, 140122444345343, -+STORE, 140122442227712, 140122444320767, -+STORE, 140122444320768, 140122444345343, -+SNULL, 140122444320768, 140122444328959, -+STORE, 140122444328960, 140122444345343, -+STORE, 140122444320768, 140122444328959, -+ERASE, 140122444320768, 140122444328959, -+STORE, 140122444320768, 140122444328959, -+ERASE, 140122444328960, 140122444345343, -+STORE, 140122444328960, 140122444345343, -+STORE, 140122438332416, 140122442129407, -+SNULL, 140122438332416, 140122439991295, -+STORE, 140122439991296, 140122442129407, -+STORE, 140122438332416, 140122439991295, -+SNULL, 140122442088447, 140122442129407, -+STORE, 140122439991296, 140122442088447, -+STORE, 140122442088448, 140122442129407, -+SNULL, 140122442088448, 140122442113023, -+STORE, 140122442113024, 140122442129407, -+STORE, 140122442088448, 140122442113023, -+ERASE, 140122442088448, 140122442113023, -+STORE, 140122442088448, 140122442113023, -+ERASE, 140122442113024, 140122442129407, -+STORE, 140122442113024, 140122442129407, -+STORE, 140122446540800, 140122446557183, -+SNULL, 140122442104831, 140122442113023, -+STORE, 140122442088448, 140122442104831, -+STORE, 140122442104832, 140122442113023, -+SNULL, 140122444324863, 140122444328959, -+STORE, 140122444320768, 140122444324863, -+STORE, 140122444324864, 140122444328959, -+SNULL, 94076832014335, 94076832018431, -+STORE, 94076832010240, 94076832014335, -+STORE, 94076832014336, 94076832018431, -+SNULL, 140122446589951, 140122446594047, -+STORE, 140122446585856, 140122446589951, -+STORE, 140122446589952, 140122446594047, -+ERASE, 140122446557184, 140122446585855, -+STORE, 94076845723648, 94076845858815, -+STORE, 140122429939712, 140122438332415, -+SNULL, 140122429943807, 140122438332415, -+STORE, 140122429939712, 140122429943807, -+STORE, 140122429943808, 140122438332415, -+STORE, 140122421547008, 140122429939711, -+STORE, 140122287329280, 140122421547007, -+SNULL, 140122287329280, 140122301399039, -+STORE, 140122301399040, 140122421547007, -+STORE, 140122287329280, 140122301399039, -+ERASE, 140122287329280, 140122301399039, -+SNULL, 140122368507903, 140122421547007, -+STORE, 140122301399040, 140122368507903, -+STORE, 140122368507904, 140122421547007, -+ERASE, 140122368507904, 140122421547007, -+SNULL, 140122301534207, 140122368507903, -+STORE, 140122301399040, 140122301534207, -+STORE, 140122301534208, 140122368507903, -+SNULL, 140122421551103, 140122429939711, -+STORE, 140122421547008, 140122421551103, -+STORE, 140122421551104, 140122429939711, -+STORE, 140122413154304, 140122421547007, -+SNULL, 140122413158399, 140122421547007, -+STORE, 140122413154304, 140122413158399, -+STORE, 140122413158400, 140122421547007, -+STORE, 140122404761600, 140122413154303, -+SNULL, 140122404765695, 140122413154303, -+STORE, 140122404761600, 140122404765695, -+STORE, 140122404765696, 140122413154303, -+STORE, 140122396368896, 140122404761599, -+SNULL, 140122396372991, 140122404761599, -+STORE, 140122396368896, 140122396372991, -+STORE, 140122396372992, 140122404761599, -+STORE, 140122387976192, 140122396368895, -+STORE, 140122167181312, 140122301399039, -+SNULL, 140122234290175, 140122301399039, -+STORE, 140122167181312, 140122234290175, -+STORE, 140122234290176, 140122301399039, -+ERASE, 140122234290176, 140122301399039, -+SNULL, 140122167316479, 140122234290175, -+STORE, 140122167181312, 140122167316479, -+STORE, 140122167316480, 140122234290175, -+STORE, 140122379583488, 140122396368895, -+STORE, 140122371190784, 140122396368895, -+STORE, 140122167316480, 140122301399039, -+STORE, 140122158788608, 140122167181311, -+SNULL, 140122371190784, 140122387976191, -+STORE, 140122387976192, 140122396368895, -+STORE, 140122371190784, 140122387976191, -+SNULL, 140122387980287, 140122396368895, -+STORE, 140122387976192, 140122387980287, -+STORE, 140122387980288, 140122396368895, -+SNULL, 140122167316480, 140122234290175, -+STORE, 140122234290176, 140122301399039, -+STORE, 140122167316480, 140122234290175, -+SNULL, 140122234425343, 140122301399039, -+STORE, 140122234290176, 140122234425343, -+STORE, 140122234425344, 140122301399039, -+STORE, 140122024570880, 140122158788607, -+SNULL, 140122024570880, 140122032963583, -+STORE, 140122032963584, 140122158788607, -+STORE, 140122024570880, 140122032963583, -+ERASE, 140122024570880, 140122032963583, -+STORE, 140121898745856, 140122158788607, -+STORE, 140121890353152, 140121898745855, -+SNULL, 140122100072447, 140122158788607, -+STORE, 140121898745856, 140122100072447, -+STORE, 140122100072448, 140122158788607, -+ERASE, 140122100072448, 140122158788607, -+SNULL, 140121965854719, 140122100072447, -+STORE, 140121898745856, 140121965854719, -+STORE, 140121965854720, 140122100072447, -+SNULL, 140121965854720, 140122032963583, -+STORE, 140122032963584, 140122100072447, -+STORE, 140121965854720, 140122032963583, -+ERASE, 140121965854720, 140122032963583, -+SNULL, 140121898881023, 140121965854719, -+STORE, 140121898745856, 140121898881023, -+STORE, 140121898881024, 140121965854719, -+SNULL, 140121890357247, 140121898745855, -+STORE, 140121890353152, 140121890357247, -+STORE, 140121890357248, 140121898745855, -+SNULL, 140122371190784, 140122379583487, -+STORE, 140122379583488, 140122387976191, -+STORE, 140122371190784, 140122379583487, -+SNULL, 140122379587583, 140122387976191, -+STORE, 140122379583488, 140122379587583, -+STORE, 140122379587584, 140122387976191, -+SNULL, 140122033098751, 140122100072447, -+STORE, 140122032963584, 140122033098751, -+STORE, 140122033098752, 140122100072447, -+SNULL, 140122158792703, 140122167181311, -+STORE, 140122158788608, 140122158792703, -+STORE, 140122158792704, 140122167181311, -+STORE, 140122150395904, 140122158788607, -+STORE, 140122142003200, 140122158788607, -+SNULL, 140122142007295, 140122158788607, -+STORE, 140122142003200, 140122142007295, -+STORE, 140122142007296, 140122158788607, -+SNULL, 140122371194879, 140122379583487, -+STORE, 140122371190784, 140122371194879, -+STORE, 140122371194880, 140122379583487, -+SNULL, 140122142007296, 140122150395903, -+STORE, 140122150395904, 140122158788607, -+STORE, 140122142007296, 140122150395903, -+SNULL, 140122150399999, 140122158788607, -+STORE, 140122150395904, 140122150399999, -+STORE, 140122150400000, 140122158788607, -+STORE, 140122133610496, 140122142003199, -+STORE, 140122125217792, 140122142003199, -+STORE, 140122116825088, 140122142003199, -+SNULL, 140122116829183, 140122142003199, -+STORE, 140122116825088, 140122116829183, -+STORE, 140122116829184, 140122142003199, -+SNULL, 140122116829184, 140122133610495, -+STORE, 140122133610496, 140122142003199, -+STORE, 140122116829184, 140122133610495, -+SNULL, 140122133614591, 140122142003199, -+STORE, 140122133610496, 140122133614591, -+STORE, 140122133614592, 140122142003199, -+SNULL, 140122116829184, 140122125217791, -+STORE, 140122125217792, 140122133610495, -+STORE, 140122116829184, 140122125217791, -+SNULL, 140122125221887, 140122133610495, -+STORE, 140122125217792, 140122125221887, -+STORE, 140122125221888, 140122133610495, -+STORE, 140122108432384, 140122116825087, -+SNULL, 140122108436479, 140122116825087, -+STORE, 140122108432384, 140122108436479, -+STORE, 140122108436480, 140122116825087, -+STORE, 140122024570880, 140122032963583, -+STORE, 140122016178176, 140122032963583, -+SNULL, 140122016182271, 140122032963583, -+STORE, 140122016178176, 140122016182271, -+STORE, 140122016182272, 140122032963583, -+SNULL, 140122016182272, 140122024570879, -+STORE, 140122024570880, 140122032963583, -+STORE, 140122016182272, 140122024570879, -+SNULL, 140122024574975, 140122032963583, -+STORE, 140122024570880, 140122024574975, -+STORE, 140122024574976, 140122032963583, -+STORE, 140122007785472, 140122016178175, -+SNULL, 140122007789567, 140122016178175, -+STORE, 140122007785472, 140122007789567, -+STORE, 140122007789568, 140122016178175, -+STORE, 140121999392768, 140122007785471, -+STORE, 140121991000064, 140122007785471, -+SNULL, 140121991004159, 140122007785471, -+STORE, 140121991000064, 140121991004159, -+STORE, 140121991004160, 140122007785471, -+SNULL, 140121991004160, 140121999392767, -+STORE, 140121999392768, 140122007785471, -+STORE, 140121991004160, 140121999392767, -+SNULL, 140121999396863, 140122007785471, -+STORE, 140121999392768, 140121999396863, -+STORE, 140121999396864, 140122007785471, -+STORE, 140121982607360, 140121991000063, -+STORE, 140121823244288, 140121890353151, -+ERASE, 140121823244288, 140121890353151, -+STORE, 140121756135424, 140121890353151, -+SNULL, 140121756135424, 140121764528127, -+STORE, 140121764528128, 140121890353151, -+STORE, 140121756135424, 140121764528127, -+ERASE, 140121756135424, 140121764528127, -+SNULL, 140121831636991, 140121890353151, -+STORE, 140121764528128, 140121831636991, -+STORE, 140121831636992, 140121890353151, -+ERASE, 140121831636992, 140121890353151, -+STORE, 140121974214656, 140121991000063, -+STORE, 140121630310400, 140121831636991, -+SNULL, 140121697419263, 140121831636991, -+STORE, 140121630310400, 140121697419263, -+STORE, 140121697419264, 140121831636991, -+SNULL, 140121697419264, 140121764528127, -+STORE, 140121764528128, 140121831636991, -+STORE, 140121697419264, 140121764528127, -+ERASE, 140121697419264, 140121764528127, -+STORE, 140121881960448, 140121890353151, -+STORE, 140121630310400, 140121831636991, -+STORE, 140121873567744, 140121890353151, -+SNULL, 140121630310400, 140121697419263, -+STORE, 140121697419264, 140121831636991, -+STORE, 140121630310400, 140121697419263, -+SNULL, 140121697554431, 140121831636991, -+STORE, 140121697419264, 140121697554431, -+STORE, 140121697554432, 140121831636991, -+STORE, 140121865175040, 140121890353151, -+STORE, 140121856782336, 140121890353151, -+STORE, 140121848389632, 140121890353151, -+STORE, 140121839996928, 140121890353151, -+STORE, 140121496092672, 140121697419263, -+STORE, 140121487699968, 140121496092671, -+STORE, 140121420591104, 140121487699967, -+STORE, 140121412198400, 140121420591103, -+ERASE, 140121420591104, 140121487699967, -+STORE, 140121479307264, 140121496092671, -+STORE, 140121277980672, 140121412198399, -+SNULL, 140121277980672, 140121294766079, -+STORE, 140121294766080, 140121412198399, -+STORE, 140121277980672, 140121294766079, -+ERASE, 140121277980672, 140121294766079, -+STORE, 140121470914560, 140121496092671, -+STORE, 140121462521856, 140121496092671, -+STORE, 140121160548352, 140121412198399, -+STORE, 140121454129152, 140121496092671, -+SNULL, 140121227657215, 140121412198399, -+STORE, 140121160548352, 140121227657215, -+STORE, 140121227657216, 140121412198399, -+SNULL, 140121227657216, 140121294766079, -+STORE, 140121294766080, 140121412198399, -+STORE, 140121227657216, 140121294766079, -+ERASE, 140121227657216, 140121294766079, -+STORE, 140121445736448, 140121496092671, -+STORE, 140121437343744, 140121496092671, -+SNULL, 140121437343744, 140121445736447, -+STORE, 140121445736448, 140121496092671, -+STORE, 140121437343744, 140121445736447, -+SNULL, 140121445740543, 140121496092671, -+STORE, 140121445736448, 140121445740543, -+STORE, 140121445740544, 140121496092671, -+SNULL, 140121697554432, 140121764528127, -+STORE, 140121764528128, 140121831636991, -+STORE, 140121697554432, 140121764528127, -+SNULL, 140121764663295, 140121831636991, -+STORE, 140121764528128, 140121764663295, -+STORE, 140121764663296, 140121831636991, -+SNULL, 140121496092672, 140121630310399, -+STORE, 140121630310400, 140121697419263, -+STORE, 140121496092672, 140121630310399, -+SNULL, 140121630445567, 140121697419263, -+STORE, 140121630310400, 140121630445567, -+STORE, 140121630445568, 140121697419263, -+SNULL, 140121445740544, 140121454129151, -+STORE, 140121454129152, 140121496092671, -+STORE, 140121445740544, 140121454129151, -+SNULL, 140121454133247, 140121496092671, -+STORE, 140121454129152, 140121454133247, -+STORE, 140121454133248, 140121496092671, -+STORE, 140121026330624, 140121227657215, -+SNULL, 140121093439487, 140121227657215, -+STORE, 140121026330624, 140121093439487, -+STORE, 140121093439488, 140121227657215, -+SNULL, 140121093439488, 140121160548351, -+STORE, 140121160548352, 140121227657215, -+STORE, 140121093439488, 140121160548351, -+ERASE, 140121093439488, 140121160548351, -+SNULL, 140121563201535, 140121630310399, -+STORE, 140121496092672, 140121563201535, -+STORE, 140121563201536, 140121630310399, -+ERASE, 140121563201536, 140121630310399, -+STORE, 140120892112896, 140121093439487, -+SNULL, 140120959221759, 140121093439487, -+STORE, 140120892112896, 140120959221759, -+STORE, 140120959221760, 140121093439487, -+SNULL, 140120959221760, 140121026330623, -+STORE, 140121026330624, 140121093439487, -+STORE, 140120959221760, 140121026330623, -+ERASE, 140120959221760, 140121026330623, -+STORE, 140120757895168, 140120959221759, -+SNULL, 140121361874943, 140121412198399, -+STORE, 140121294766080, 140121361874943, -+STORE, 140121361874944, 140121412198399, -+ERASE, 140121361874944, 140121412198399, -+SNULL, 140121294901247, 140121361874943, -+STORE, 140121294766080, 140121294901247, -+STORE, 140121294901248, 140121361874943, -+STORE, 140120623677440, 140120959221759, -+SNULL, 140120690786303, 140120959221759, -+STORE, 140120623677440, 140120690786303, -+STORE, 140120690786304, 140120959221759, -+SNULL, 140120690786304, 140120757895167, -+STORE, 140120757895168, 140120959221759, -+STORE, 140120690786304, 140120757895167, -+ERASE, 140120690786304, 140120757895167, -+SNULL, 140121160683519, 140121227657215, -+STORE, 140121160548352, 140121160683519, -+STORE, 140121160683520, 140121227657215, -+SNULL, 140121974214656, 140121982607359, -+STORE, 140121982607360, 140121991000063, -+STORE, 140121974214656, 140121982607359, -+SNULL, 140121982611455, 140121991000063, -+STORE, 140121982607360, 140121982611455, -+STORE, 140121982611456, 140121991000063, -+SNULL, 140121839996928, 140121873567743, -+STORE, 140121873567744, 140121890353151, -+STORE, 140121839996928, 140121873567743, -+SNULL, 140121873571839, 140121890353151, -+STORE, 140121873567744, 140121873571839, -+STORE, 140121873571840, 140121890353151, -+SNULL, 140121873571840, 140121881960447, -+STORE, 140121881960448, 140121890353151, -+STORE, 140121873571840, 140121881960447, -+SNULL, 140121881964543, 140121890353151, -+STORE, 140121881960448, 140121881964543, -+STORE, 140121881964544, 140121890353151, -+SNULL, 140121840001023, 140121873567743, -+STORE, 140121839996928, 140121840001023, -+STORE, 140121840001024, 140121873567743, -+SNULL, 140121840001024, 140121865175039, -+STORE, 140121865175040, 140121873567743, -+STORE, 140121840001024, 140121865175039, -+SNULL, 140121865179135, 140121873567743, -+STORE, 140121865175040, 140121865179135, -+STORE, 140121865179136, 140121873567743, -+SNULL, 140121437347839, 140121445736447, -+STORE, 140121437343744, 140121437347839, -+STORE, 140121437347840, 140121445736447, -+STORE, 140121621917696, 140121630310399, -+STORE, 140121613524992, 140121630310399, -+SNULL, 140121026465791, 140121093439487, -+STORE, 140121026330624, 140121026465791, -+STORE, 140121026465792, 140121093439487, -+SNULL, 140121496227839, 140121563201535, -+STORE, 140121496092672, 140121496227839, -+STORE, 140121496227840, 140121563201535, -+SNULL, 140120757895168, 140120892112895, -+STORE, 140120892112896, 140120959221759, -+STORE, 140120757895168, 140120892112895, -+SNULL, 140120892248063, 140120959221759, -+STORE, 140120892112896, 140120892248063, -+STORE, 140120892248064, 140120959221759, -+SNULL, 140120825004031, 140120892112895, -+STORE, 140120757895168, 140120825004031, -+STORE, 140120825004032, 140120892112895, -+ERASE, 140120825004032, 140120892112895, -+SNULL, 140120623812607, 140120690786303, -+STORE, 140120623677440, 140120623812607, -+STORE, 140120623812608, 140120690786303, -+SNULL, 140120758030335, 140120825004031, -+STORE, 140120757895168, 140120758030335, -+STORE, 140120758030336, 140120825004031, -+SNULL, 140121454133248, 140121462521855, -+STORE, 140121462521856, 140121496092671, -+STORE, 140121454133248, 140121462521855, -+SNULL, 140121462525951, 140121496092671, -+STORE, 140121462521856, 140121462525951, -+STORE, 140121462525952, 140121496092671, -+STORE, 140121605132288, 140121630310399, -+SNULL, 140121605136383, 140121630310399, -+STORE, 140121605132288, 140121605136383, -+STORE, 140121605136384, 140121630310399, -+STORE, 140121596739584, 140121605132287, -+SNULL, 140121605136384, 140121621917695, -+STORE, 140121621917696, 140121630310399, -+STORE, 140121605136384, 140121621917695, -+SNULL, 140121621921791, 140121630310399, -+STORE, 140121621917696, 140121621921791, -+STORE, 140121621921792, 140121630310399, -+STORE, 140121588346880, 140121605132287, -+STORE, 140121579954176, 140121605132287, -+SNULL, 140121412202495, 140121420591103, -+STORE, 140121412198400, 140121412202495, -+STORE, 140121412202496, 140121420591103, -+SNULL, 140121974218751, 140121982607359, -+STORE, 140121974214656, 140121974218751, -+STORE, 140121974218752, 140121982607359, -+SNULL, 140121462525952, 140121479307263, -+STORE, 140121479307264, 140121496092671, -+STORE, 140121462525952, 140121479307263, -+SNULL, 140121479311359, 140121496092671, -+STORE, 140121479307264, 140121479311359, -+STORE, 140121479311360, 140121496092671, -+STORE, 140121571561472, 140121605132287, -+SNULL, 140121571565567, 140121605132287, -+STORE, 140121571561472, 140121571565567, -+STORE, 140121571565568, 140121605132287, -+STORE, 140121428951040, 140121437343743, -+SNULL, 140121428955135, 140121437343743, -+STORE, 140121428951040, 140121428955135, -+STORE, 140121428955136, 140121437343743, -+SNULL, 140121840001024, 140121856782335, -+STORE, 140121856782336, 140121865175039, -+STORE, 140121840001024, 140121856782335, -+SNULL, 140121856786431, 140121865175039, -+STORE, 140121856782336, 140121856786431, -+STORE, 140121856786432, 140121865175039, -+STORE, 140121403805696, 140121412198399, -+SNULL, 140121840001024, 140121848389631, -+STORE, 140121848389632, 140121856782335, -+STORE, 140121840001024, 140121848389631, -+SNULL, 140121848393727, 140121856782335, -+STORE, 140121848389632, 140121848393727, -+STORE, 140121848393728, 140121856782335, -+SNULL, 140121479311360, 140121487699967, -+STORE, 140121487699968, 140121496092671, -+STORE, 140121479311360, 140121487699967, -+SNULL, 140121487704063, 140121496092671, -+STORE, 140121487699968, 140121487704063, -+STORE, 140121487704064, 140121496092671, -+STORE, 140121395412992, 140121412198399, -+STORE, 140121387020288, 140121412198399, -+SNULL, 140121387024383, 140121412198399, -+STORE, 140121387020288, 140121387024383, -+STORE, 140121387024384, 140121412198399, -+SNULL, 140121605136384, 140121613524991, -+STORE, 140121613524992, 140121621917695, -+STORE, 140121605136384, 140121613524991, -+SNULL, 140121613529087, 140121621917695, -+STORE, 140121613524992, 140121613529087, -+STORE, 140121613529088, 140121621917695, -+SNULL, 140121462525952, 140121470914559, -+STORE, 140121470914560, 140121479307263, -+STORE, 140121462525952, 140121470914559, -+SNULL, 140121470918655, 140121479307263, -+STORE, 140121470914560, 140121470918655, -+STORE, 140121470918656, 140121479307263, -+STORE, 140121378627584, 140121387020287, -+SNULL, 140121378631679, 140121387020287, -+STORE, 140121378627584, 140121378631679, -+STORE, 140121378631680, 140121387020287, -+SNULL, 140121571565568, 140121596739583, -+STORE, 140121596739584, 140121605132287, -+STORE, 140121571565568, 140121596739583, -+SNULL, 140121596743679, 140121605132287, -+STORE, 140121596739584, 140121596743679, -+STORE, 140121596743680, 140121605132287, -+SNULL, 140121387024384, 140121403805695, -+STORE, 140121403805696, 140121412198399, -+STORE, 140121387024384, 140121403805695, -+SNULL, 140121403809791, 140121412198399, -+STORE, 140121403805696, 140121403809791, -+STORE, 140121403809792, 140121412198399, -+STORE, 140121370234880, 140121378627583, -+SNULL, 140121387024384, 140121395412991, -+STORE, 140121395412992, 140121403805695, -+STORE, 140121387024384, 140121395412991, -+SNULL, 140121395417087, 140121403805695, -+STORE, 140121395412992, 140121395417087, -+STORE, 140121395417088, 140121403805695, -+SNULL, 140121571565568, 140121588346879, -+STORE, 140121588346880, 140121596739583, -+STORE, 140121571565568, 140121588346879, -+SNULL, 140121588350975, 140121596739583, -+STORE, 140121588346880, 140121588350975, -+STORE, 140121588350976, 140121596739583, -+SNULL, 140121571565568, 140121579954175, -+STORE, 140121579954176, 140121588346879, -+STORE, 140121571565568, 140121579954175, -+SNULL, 140121579958271, 140121588346879, -+STORE, 140121579954176, 140121579958271, -+STORE, 140121579958272, 140121588346879, -+STORE, 140121286373376, 140121294766079, -+STORE, 140121277980672, 140121294766079, -+SNULL, 140121277980672, 140121286373375, -+STORE, 140121286373376, 140121294766079, -+STORE, 140121277980672, 140121286373375, -+SNULL, 140121286377471, 140121294766079, -+STORE, 140121286373376, 140121286377471, -+STORE, 140121286377472, 140121294766079, -+STORE, 140121269587968, 140121286373375, -+STORE, 140121261195264, 140121286373375, -+SNULL, 140121261195264, 140121269587967, -+STORE, 140121269587968, 140121286373375, -+STORE, 140121261195264, 140121269587967, -+SNULL, 140121269592063, 140121286373375, -+STORE, 140121269587968, 140121269592063, -+STORE, 140121269592064, 140121286373375, -+STORE, 140121252802560, 140121269587967, -+SNULL, 140121252806655, 140121269587967, -+STORE, 140121252802560, 140121252806655, -+STORE, 140121252806656, 140121269587967, -+STORE, 140121244409856, 140121252802559, -+STORE, 140121236017152, 140121252802559, -+SNULL, 140121236017152, 140121244409855, -+STORE, 140121244409856, 140121252802559, -+STORE, 140121236017152, 140121244409855, -+SNULL, 140121244413951, 140121252802559, -+STORE, 140121244409856, 140121244413951, -+STORE, 140121244413952, 140121252802559, -+SNULL, 140121370238975, 140121378627583, -+STORE, 140121370234880, 140121370238975, -+STORE, 140121370238976, 140121378627583, -+STORE, 140121152155648, 140121160548351, -+STORE, 140121143762944, 140121160548351, -+STORE, 140121135370240, 140121160548351, -+SNULL, 140121135374335, 140121160548351, -+STORE, 140121135370240, 140121135374335, -+STORE, 140121135374336, 140121160548351, -+STORE, 140121126977536, 140121135370239, -+STORE, 140121118584832, 140121135370239, -+STORE, 140121110192128, 140121135370239, -+SNULL, 140121110192128, 140121118584831, -+STORE, 140121118584832, 140121135370239, -+STORE, 140121110192128, 140121118584831, -+SNULL, 140121118588927, 140121135370239, -+STORE, 140121118584832, 140121118588927, -+STORE, 140121118588928, 140121135370239, -+STORE, 140121101799424, 140121118584831, -+STORE, 140121017937920, 140121026330623, -+STORE, 140121009545216, 140121026330623, -+SNULL, 140121009545216, 140121017937919, -+STORE, 140121017937920, 140121026330623, -+STORE, 140121009545216, 140121017937919, -+SNULL, 140121017942015, 140121026330623, -+STORE, 140121017937920, 140121017942015, -+STORE, 140121017942016, 140121026330623, -+SNULL, 140121269592064, 140121277980671, -+STORE, 140121277980672, 140121286373375, -+STORE, 140121269592064, 140121277980671, -+SNULL, 140121277984767, 140121286373375, -+STORE, 140121277980672, 140121277984767, -+STORE, 140121277984768, 140121286373375, -+STORE, 140121001152512, 140121017937919, -+SNULL, 140121252806656, 140121261195263, -+STORE, 140121261195264, 140121269587967, -+STORE, 140121252806656, 140121261195263, -+SNULL, 140121261199359, 140121269587967, -+STORE, 140121261195264, 140121261199359, -+STORE, 140121261199360, 140121269587967, -+SNULL, 140121135374336, 140121152155647, -+STORE, 140121152155648, 140121160548351, -+STORE, 140121135374336, 140121152155647, -+SNULL, 140121152159743, 140121160548351, -+STORE, 140121152155648, 140121152159743, -+STORE, 140121152159744, 140121160548351, -+STORE, 140120992759808, 140121017937919, -+STORE, 140120984367104, 140121017937919, -+STORE, 140120975974400, 140121017937919, -+SNULL, 140121101799424, 140121110192127, -+STORE, 140121110192128, 140121118584831, -+STORE, 140121101799424, 140121110192127, -+SNULL, 140121110196223, 140121118584831, -+STORE, 140121110192128, 140121110196223, -+STORE, 140121110196224, 140121118584831, -+SNULL, 140121118588928, 140121126977535, -+STORE, 140121126977536, 140121135370239, -+STORE, 140121118588928, 140121126977535, -+SNULL, 140121126981631, 140121135370239, -+STORE, 140121126977536, 140121126981631, -+STORE, 140121126981632, 140121135370239, -+STORE, 140120967581696, 140121017937919, -+STORE, 140120883720192, 140120892112895, -+SNULL, 140120883724287, 140120892112895, -+STORE, 140120883720192, 140120883724287, -+STORE, 140120883724288, 140120892112895, -+STORE, 140120875327488, 140120883720191, -+SNULL, 140121101803519, 140121110192127, -+STORE, 140121101799424, 140121101803519, -+STORE, 140121101803520, 140121110192127, -+SNULL, 140121135374336, 140121143762943, -+STORE, 140121143762944, 140121152155647, -+STORE, 140121135374336, 140121143762943, -+SNULL, 140121143767039, 140121152155647, -+STORE, 140121143762944, 140121143767039, -+STORE, 140121143767040, 140121152155647, -+STORE, 140120866934784, 140120883720191, -+SNULL, 140120967581696, 140120984367103, -+STORE, 140120984367104, 140121017937919, -+STORE, 140120967581696, 140120984367103, -+SNULL, 140120984371199, 140121017937919, -+STORE, 140120984367104, 140120984371199, -+STORE, 140120984371200, 140121017937919, -+STORE, 140120858542080, 140120883720191, -+SNULL, 140121236021247, 140121244409855, -+STORE, 140121236017152, 140121236021247, -+STORE, 140121236021248, 140121244409855, -+SNULL, 140120984371200, 140121009545215, -+STORE, 140121009545216, 140121017937919, -+STORE, 140120984371200, 140121009545215, -+SNULL, 140121009549311, 140121017937919, -+STORE, 140121009545216, 140121009549311, -+STORE, 140121009549312, 140121017937919, -+SNULL, 140120984371200, 140120992759807, -+STORE, 140120992759808, 140121009545215, -+STORE, 140120984371200, 140120992759807, -+SNULL, 140120992763903, 140121009545215, -+STORE, 140120992759808, 140120992763903, -+STORE, 140120992763904, 140121009545215, -+SNULL, 140120992763904, 140121001152511, -+STORE, 140121001152512, 140121009545215, -+STORE, 140120992763904, 140121001152511, -+SNULL, 140121001156607, 140121009545215, -+STORE, 140121001152512, 140121001156607, -+STORE, 140121001156608, 140121009545215, -+STORE, 140120850149376, 140120883720191, -+SNULL, 140120850153471, 140120883720191, -+STORE, 140120850149376, 140120850153471, -+STORE, 140120850153472, 140120883720191, -+SNULL, 140120967585791, 140120984367103, -+STORE, 140120967581696, 140120967585791, -+STORE, 140120967585792, 140120984367103, -+SNULL, 140120850153472, 140120866934783, -+STORE, 140120866934784, 140120883720191, -+STORE, 140120850153472, 140120866934783, -+SNULL, 140120866938879, 140120883720191, -+STORE, 140120866934784, 140120866938879, -+STORE, 140120866938880, 140120883720191, -+STORE, 140120841756672, 140120850149375, -+SNULL, 140120967585792, 140120975974399, -+STORE, 140120975974400, 140120984367103, -+STORE, 140120967585792, 140120975974399, -+SNULL, 140120975978495, 140120984367103, -+STORE, 140120975974400, 140120975978495, -+STORE, 140120975978496, 140120984367103, -+SNULL, 140120866938880, 140120875327487, -+STORE, 140120875327488, 140120883720191, -+STORE, 140120866938880, 140120875327487, -+SNULL, 140120875331583, 140120883720191, -+STORE, 140120875327488, 140120875331583, -+STORE, 140120875331584, 140120883720191, -+STORE, 140120833363968, 140120850149375, -+STORE, 140120749502464, 140120757895167, -+STORE, 140120741109760, 140120757895167, -+STORE, 140120732717056, 140120757895167, -+STORE, 140120724324352, 140120757895167, -+SNULL, 140120724324352, 140120732717055, -+STORE, 140120732717056, 140120757895167, -+STORE, 140120724324352, 140120732717055, -+SNULL, 140120732721151, 140120757895167, -+STORE, 140120732717056, 140120732721151, -+STORE, 140120732721152, 140120757895167, -+STORE, 140120715931648, 140120732717055, -+SNULL, 140120715935743, 140120732717055, -+STORE, 140120715931648, 140120715935743, -+STORE, 140120715935744, 140120732717055, -+SNULL, 140120850153472, 140120858542079, -+STORE, 140120858542080, 140120866934783, -+STORE, 140120850153472, 140120858542079, -+SNULL, 140120858546175, 140120866934783, -+STORE, 140120858542080, 140120858546175, -+STORE, 140120858546176, 140120866934783, -+STORE, 140120707538944, 140120715931647, -+SNULL, 140120707543039, 140120715931647, -+STORE, 140120707538944, 140120707543039, -+STORE, 140120707543040, 140120715931647, -+SNULL, 140120833368063, 140120850149375, -+STORE, 140120833363968, 140120833368063, -+STORE, 140120833368064, 140120850149375, -+SNULL, 140120833368064, 140120841756671, -+STORE, 140120841756672, 140120850149375, -+STORE, 140120833368064, 140120841756671, -+SNULL, 140120841760767, 140120850149375, -+STORE, 140120841756672, 140120841760767, -+STORE, 140120841760768, 140120850149375, -+STORE, 140120699146240, 140120707538943, -+SNULL, 140120715935744, 140120724324351, -+STORE, 140120724324352, 140120732717055, -+STORE, 140120715935744, 140120724324351, -+SNULL, 140120724328447, 140120732717055, -+STORE, 140120724324352, 140120724328447, -+STORE, 140120724328448, 140120732717055, -+SNULL, 140120732721152, 140120741109759, -+STORE, 140120741109760, 140120757895167, -+STORE, 140120732721152, 140120741109759, -+SNULL, 140120741113855, 140120757895167, -+STORE, 140120741109760, 140120741113855, -+STORE, 140120741113856, 140120757895167, -+SNULL, 140120741113856, 140120749502463, -+STORE, 140120749502464, 140120757895167, -+STORE, 140120741113856, 140120749502463, -+SNULL, 140120749506559, 140120757895167, -+STORE, 140120749502464, 140120749506559, -+STORE, 140120749506560, 140120757895167, -+SNULL, 140120699150335, 140120707538943, -+STORE, 140120699146240, 140120699150335, -+STORE, 140120699150336, 140120707538943, -+STORE, 140122446557184, 140122446585855, -+STORE, 140122368999424, 140122371190783, -+SNULL, 140122368999424, 140122369089535, -+STORE, 140122369089536, 140122371190783, -+STORE, 140122368999424, 140122369089535, -+SNULL, 140122371182591, 140122371190783, -+STORE, 140122369089536, 140122371182591, -+STORE, 140122371182592, 140122371190783, -+ERASE, 140122371182592, 140122371190783, -+STORE, 140122371182592, 140122371190783, -+SNULL, 140122371186687, 140122371190783, -+STORE, 140122371182592, 140122371186687, -+STORE, 140122371186688, 140122371190783, -+ERASE, 140122446557184, 140122446585855, -+ERASE, 140121445736448, 140121445740543, -+ERASE, 140121445740544, 140121454129151, -+ERASE, 140121621917696, 140121621921791, -+ERASE, 140121621921792, 140121630310399, -+ERASE, 140121579954176, 140121579958271, -+ERASE, 140121579958272, 140121588346879, -+ERASE, 140121261195264, 140121261199359, -+ERASE, 140121261199360, 140121269587967, -+ERASE, 140121454129152, 140121454133247, -+ERASE, 140121454133248, 140121462521855, -+ERASE, 140121588346880, 140121588350975, -+ERASE, 140121588350976, 140121596739583, -+ERASE, 140121135370240, 140121135374335, -+ERASE, 140121135374336, 140121143762943, -+ERASE, 140121881960448, 140121881964543, -+ERASE, 140121881964544, 140121890353151, -+ERASE, 140121428951040, 140121428955135, -+ERASE, 140121428955136, 140121437343743, -+ERASE, 140121387020288, 140121387024383, -+ERASE, 140121387024384, 140121395412991, -+ERASE, 140121487699968, 140121487704063, -+ERASE, 140121487704064, 140121496092671, -+ERASE, 140121437343744, 140121437347839, -+ERASE, 140121437347840, 140121445736447, -+ERASE, 140121613524992, 140121613529087, -+ERASE, 140121613529088, 140121621917695, -+ERASE, 140121856782336, 140121856786431, -+ERASE, 140121856786432, 140121865175039, -+ERASE, 140121252802560, 140121252806655, -+ERASE, 140121252806656, 140121261195263, -+ERASE, 140121839996928, 140121840001023, -+ERASE, 140121840001024, 140121848389631, -+ERASE, 140121596739584, 140121596743679, -+ERASE, 140121596743680, 140121605132287, -+ERASE, 140121009545216, 140121009549311, -+ERASE, 140121009549312, 140121017937919, -+ERASE, 140120724324352, 140120724328447, -+ERASE, 140120724328448, 140120732717055, -+ERASE, 140120883720192, 140120883724287, -+ERASE, 140120883724288, 140120892112895, -+ERASE, 140121982607360, 140121982611455, -+ERASE, 140121982611456, 140121991000063, -+ERASE, 140121571561472, 140121571565567, -+ERASE, 140121571565568, 140121579954175, -+ERASE, 140121286373376, 140121286377471, -+ERASE, 140121286377472, 140121294766079, -+ERASE, 140120875327488, 140120875331583, -+ERASE, 140120875331584, 140120883720191, -+ERASE, 140121848389632, 140121848393727, -+ERASE, 140121848393728, 140121856782335, -+ERASE, 140121370234880, 140121370238975, -+ERASE, 140121370238976, 140121378627583, -+ERASE, 140121143762944, 140121143767039, -+ERASE, 140121143767040, 140121152155647, -+ERASE, 140121118584832, 140121118588927, -+ERASE, 140121118588928, 140121126977535, -+ERASE, 140120866934784, 140120866938879, -+ERASE, 140120866938880, 140120875327487, -+ERASE, 140120741109760, 140120741113855, -+ERASE, 140120741113856, 140120749502463, -+ERASE, 140121865175040, 140121865179135, -+ERASE, 140121865179136, 140121873567743, -+ERASE, 140121403805696, 140121403809791, -+ERASE, 140121403809792, 140121412198399, -+ERASE, 140121236017152, 140121236021247, -+ERASE, 140121236021248, 140121244409855, -+ERASE, 140120732717056, 140120732721151, -+ERASE, 140120732721152, 140120741109759, -+ERASE, 140121017937920, 140121017942015, -+ERASE, 140121017942016, 140121026330623, -+ERASE, 140121873567744, 140121873571839, -+ERASE, 140121873571840, 140121881960447, -+ERASE, 140121470914560, 140121470918655, -+ERASE, 140121470918656, 140121479307263, -+ERASE, 140121126977536, 140121126981631, -+ERASE, 140121126981632, 140121135370239, -+ERASE, 140120850149376, 140120850153471, -+ERASE, 140120850153472, 140120858542079, -+ERASE, 140120707538944, 140120707543039, -+ERASE, 140120707543040, 140120715931647, -+ERASE, 140121479307264, 140121479311359, -+ERASE, 140121479311360, 140121487699967, -+ERASE, 140120967581696, 140120967585791, -+ERASE, 140120967585792, 140120975974399, -+ERASE, 140120841756672, 140120841760767, -+ERASE, 140120841760768, 140120850149375, -+ERASE, 140121412198400, 140121412202495, -+ERASE, 140121412202496, 140121420591103, -+ERASE, 140122158788608, 140122158792703, -+ERASE, 140122158792704, 140122167181311, -+ERASE, 140122142003200, 140122142007295, -+ERASE, 140122142007296, 140122150395903, -+ERASE, 140121101799424, 140121101803519, -+ERASE, 140121101803520, 140121110192127, -+ERASE, 140120858542080, 140120858546175, -+ERASE, 140120858546176, 140120866934783, -+ERASE, 140120833363968, 140120833368063, -+ERASE, 140120833368064, 140120841756671, -+ERASE, 140121277980672, 140121277984767, -+ERASE, 140121277984768, 140121286373375, -+ERASE, 140121001152512, 140121001156607, -+ERASE, 140121001156608, 140121009545215, -+ERASE, 140120749502464, 140120749506559, -+ERASE, 140120749506560, 140120757895167, -+ERASE, 140121605132288, 140121605136383, -+ERASE, 140121605136384, 140121613524991, -+ERASE, 140121378627584, 140121378631679, -+ERASE, 140121378631680, 140121387020287, -+ERASE, 140121110192128, 140121110196223, -+ERASE, 140121110196224, 140121118584831, -+ERASE, 140121462521856, 140121462525951, -+ERASE, 140121462525952, 140121470914559, -+ERASE, 140121395412992, 140121395417087, -+ERASE, 140121395417088, 140121403805695, -+ERASE, 140121152155648, 140121152159743, -+ERASE, 140121152159744, 140121160548351, -+ERASE, 140120992759808, 140120992763903, -+ERASE, 140120992763904, 140121001152511, -+ERASE, 140122387976192, 140122387980287, -+ERASE, 140122387980288, 140122396368895, -+ERASE, 140121890353152, 140121890357247, -+ERASE, 140121890357248, 140121898745855, -+ERASE, 140121269587968, 140121269592063, -+ERASE, 140121269592064, 140121277980671, -+ }; -+ unsigned long set37[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140722404016128, 140737488351231, -+SNULL, 140722404020223, 140737488351231, -+STORE, 140722404016128, 140722404020223, -+STORE, 140722403885056, 140722404020223, -+STORE, 94637010001920, 94637012254719, -+SNULL, 94637010132991, 94637012254719, -+STORE, 94637010001920, 94637010132991, -+STORE, 94637010132992, 94637012254719, -+ERASE, 94637010132992, 94637012254719, -+STORE, 94637012226048, 94637012234239, -+STORE, 94637012234240, 94637012254719, -+STORE, 139760240594944, 139760242847743, -+SNULL, 139760240738303, 139760242847743, -+STORE, 139760240594944, 139760240738303, -+STORE, 139760240738304, 139760242847743, -+ERASE, 139760240738304, 139760242847743, -+STORE, 139760242835456, 139760242843647, -+STORE, 139760242843648, 139760242847743, -+STORE, 140722405232640, 140722405236735, -+STORE, 140722405220352, 140722405232639, -+STORE, 139760242806784, 139760242835455, -+STORE, 139760242798592, 139760242806783, -+STORE, 139760238379008, 139760240594943, -+SNULL, 139760238379008, 139760238477311, -+STORE, 139760238477312, 139760240594943, -+STORE, 139760238379008, 139760238477311, -+SNULL, 139760240570367, 139760240594943, -+STORE, 139760238477312, 139760240570367, -+STORE, 139760240570368, 139760240594943, -+SNULL, 139760240570368, 139760240578559, -+STORE, 139760240578560, 139760240594943, -+STORE, 139760240570368, 139760240578559, -+ERASE, 139760240570368, 139760240578559, -+STORE, 139760240570368, 139760240578559, -+ERASE, 139760240578560, 139760240594943, -+STORE, 139760240578560, 139760240594943, -+STORE, 139760234582016, 139760238379007, -+SNULL, 139760234582016, 139760236240895, -+STORE, 139760236240896, 139760238379007, -+STORE, 139760234582016, 139760236240895, -+SNULL, 139760238338047, 139760238379007, -+STORE, 139760236240896, 139760238338047, -+STORE, 139760238338048, 139760238379007, -+SNULL, 139760238338048, 139760238362623, -+STORE, 139760238362624, 139760238379007, -+STORE, 139760238338048, 139760238362623, -+ERASE, 139760238338048, 139760238362623, -+STORE, 139760238338048, 139760238362623, -+ERASE, 139760238362624, 139760238379007, -+STORE, 139760238362624, 139760238379007, -+STORE, 139760242790400, 139760242806783, -+SNULL, 139760238354431, 139760238362623, -+STORE, 139760238338048, 139760238354431, -+STORE, 139760238354432, 139760238362623, -+SNULL, 139760240574463, 139760240578559, -+STORE, 139760240570368, 139760240574463, -+STORE, 139760240574464, 139760240578559, -+SNULL, 94637012230143, 94637012234239, -+STORE, 94637012226048, 94637012230143, -+STORE, 94637012230144, 94637012234239, -+SNULL, 139760242839551, 139760242843647, -+STORE, 139760242835456, 139760242839551, -+STORE, 139760242839552, 139760242843647, -+ERASE, 139760242806784, 139760242835455, -+STORE, 94637033324544, 94637033459711, -+STORE, 139760226189312, 139760234582015, -+SNULL, 139760226193407, 139760234582015, -+STORE, 139760226189312, 139760226193407, -+STORE, 139760226193408, 139760234582015, -+STORE, 139760217796608, 139760226189311, -+STORE, 139760083578880, 139760217796607, -+SNULL, 139760083578880, 139760114860031, -+STORE, 139760114860032, 139760217796607, -+STORE, 139760083578880, 139760114860031, -+ERASE, 139760083578880, 139760114860031, -+SNULL, 139760181968895, 139760217796607, -+STORE, 139760114860032, 139760181968895, -+STORE, 139760181968896, 139760217796607, -+ERASE, 139760181968896, 139760217796607, -+SNULL, 139760114995199, 139760181968895, -+STORE, 139760114860032, 139760114995199, -+STORE, 139760114995200, 139760181968895, -+SNULL, 139760217800703, 139760226189311, -+STORE, 139760217796608, 139760217800703, -+STORE, 139760217800704, 139760226189311, -+STORE, 139760209403904, 139760217796607, -+SNULL, 139760209407999, 139760217796607, -+STORE, 139760209403904, 139760209407999, -+STORE, 139760209408000, 139760217796607, -+STORE, 139760201011200, 139760209403903, -+SNULL, 139760201015295, 139760209403903, -+STORE, 139760201011200, 139760201015295, -+STORE, 139760201015296, 139760209403903, -+STORE, 139760192618496, 139760201011199, -+SNULL, 139760192622591, 139760201011199, -+STORE, 139760192618496, 139760192622591, -+STORE, 139760192622592, 139760201011199, -+STORE, 139760184225792, 139760192618495, -+STORE, 139759980642304, 139760114860031, -+STORE, 139759972249600, 139759980642303, -+STORE, 139759963856896, 139759980642303, -+STORE, 139759955464192, 139759980642303, -+STORE, 139759888355328, 139759955464191, -+SNULL, 139760047751167, 139760114860031, -+STORE, 139759980642304, 139760047751167, -+STORE, 139760047751168, 139760114860031, -+ERASE, 139760047751168, 139760114860031, -+SNULL, 139759980777471, 139760047751167, -+STORE, 139759980642304, 139759980777471, -+STORE, 139759980777472, 139760047751167, -+STORE, 139759980777472, 139760114860031, -+SNULL, 139759980777472, 139760047751167, -+STORE, 139760047751168, 139760114860031, -+STORE, 139759980777472, 139760047751167, -+SNULL, 139760047886335, 139760114860031, -+STORE, 139760047751168, 139760047886335, -+STORE, 139760047886336, 139760114860031, -+STORE, 139759821246464, 139759955464191, -+SNULL, 139759821246464, 139759888355327, -+STORE, 139759888355328, 139759955464191, -+STORE, 139759821246464, 139759888355327, -+ERASE, 139759821246464, 139759888355327, -+ERASE, 139759888355328, 139759955464191, -+ }; -+ unsigned long set38[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140730666221568, 140737488351231, -+SNULL, 140730666225663, 140737488351231, -+STORE, 140730666221568, 140730666225663, -+STORE, 140730666090496, 140730666225663, -+STORE, 94177584803840, 94177587056639, -+SNULL, 94177584934911, 94177587056639, -+STORE, 94177584803840, 94177584934911, -+STORE, 94177584934912, 94177587056639, -+ERASE, 94177584934912, 94177587056639, -+STORE, 94177587027968, 94177587036159, -+STORE, 94177587036160, 94177587056639, -+STORE, 140614382714880, 140614384967679, -+SNULL, 140614382858239, 140614384967679, -+STORE, 140614382714880, 140614382858239, -+STORE, 140614382858240, 140614384967679, -+ERASE, 140614382858240, 140614384967679, -+STORE, 140614384955392, 140614384963583, -+STORE, 140614384963584, 140614384967679, -+STORE, 140730666315776, 140730666319871, -+STORE, 140730666303488, 140730666315775, -+STORE, 140614384926720, 140614384955391, -+STORE, 140614384918528, 140614384926719, -+STORE, 140614380498944, 140614382714879, -+SNULL, 140614380498944, 140614380597247, -+STORE, 140614380597248, 140614382714879, -+STORE, 140614380498944, 140614380597247, -+SNULL, 140614382690303, 140614382714879, -+STORE, 140614380597248, 140614382690303, -+STORE, 140614382690304, 140614382714879, -+SNULL, 140614382690304, 140614382698495, -+STORE, 140614382698496, 140614382714879, -+STORE, 140614382690304, 140614382698495, -+ERASE, 140614382690304, 140614382698495, -+STORE, 140614382690304, 140614382698495, -+ERASE, 140614382698496, 140614382714879, -+STORE, 140614382698496, 140614382714879, -+STORE, 140614376701952, 140614380498943, -+SNULL, 140614376701952, 140614378360831, -+STORE, 140614378360832, 140614380498943, -+STORE, 140614376701952, 140614378360831, -+SNULL, 140614380457983, 140614380498943, -+STORE, 140614378360832, 140614380457983, -+STORE, 140614380457984, 140614380498943, -+SNULL, 140614380457984, 140614380482559, -+STORE, 140614380482560, 140614380498943, -+STORE, 140614380457984, 140614380482559, -+ERASE, 140614380457984, 140614380482559, -+STORE, 140614380457984, 140614380482559, -+ERASE, 140614380482560, 140614380498943, -+STORE, 140614380482560, 140614380498943, -+STORE, 140614384910336, 140614384926719, -+SNULL, 140614380474367, 140614380482559, -+STORE, 140614380457984, 140614380474367, -+STORE, 140614380474368, 140614380482559, -+SNULL, 140614382694399, 140614382698495, -+STORE, 140614382690304, 140614382694399, -+STORE, 140614382694400, 140614382698495, -+SNULL, 94177587032063, 94177587036159, -+STORE, 94177587027968, 94177587032063, -+STORE, 94177587032064, 94177587036159, -+SNULL, 140614384959487, 140614384963583, -+STORE, 140614384955392, 140614384959487, -+STORE, 140614384959488, 140614384963583, -+ERASE, 140614384926720, 140614384955391, -+STORE, 94177619791872, 94177619927039, -+STORE, 140614368309248, 140614376701951, -+SNULL, 140614368313343, 140614376701951, -+STORE, 140614368309248, 140614368313343, -+STORE, 140614368313344, 140614376701951, -+STORE, 140614359916544, 140614368309247, -+STORE, 140614225698816, 140614359916543, -+SNULL, 140614225698816, 140614276481023, -+STORE, 140614276481024, 140614359916543, -+STORE, 140614225698816, 140614276481023, -+ERASE, 140614225698816, 140614276481023, -+SNULL, 140614343589887, 140614359916543, -+STORE, 140614276481024, 140614343589887, -+STORE, 140614343589888, 140614359916543, -+ERASE, 140614343589888, 140614359916543, -+SNULL, 140614276616191, 140614343589887, -+STORE, 140614276481024, 140614276616191, -+STORE, 140614276616192, 140614343589887, -+SNULL, 140614359920639, 140614368309247, -+STORE, 140614359916544, 140614359920639, -+STORE, 140614359920640, 140614368309247, -+STORE, 140614351523840, 140614359916543, -+SNULL, 140614351527935, 140614359916543, -+STORE, 140614351523840, 140614351527935, -+STORE, 140614351527936, 140614359916543, -+STORE, 140614268088320, 140614276481023, -+SNULL, 140614268092415, 140614276481023, -+STORE, 140614268088320, 140614268092415, -+STORE, 140614268092416, 140614276481023, -+STORE, 140614259695616, 140614268088319, -+SNULL, 140614259699711, 140614268088319, -+STORE, 140614259695616, 140614259699711, -+STORE, 140614259699712, 140614268088319, -+STORE, 140614251302912, 140614259695615, -+STORE, 140614242910208, 140614259695615, -+STORE, 140614108692480, 140614242910207, -+SNULL, 140614108692480, 140614142263295, -+STORE, 140614142263296, 140614242910207, -+STORE, 140614108692480, 140614142263295, -+ERASE, 140614108692480, 140614142263295, -+STORE, 140614133870592, 140614142263295, -+STORE, 140613999652864, 140614133870591, -+SNULL, 140613999652864, 140614008045567, -+STORE, 140614008045568, 140614133870591, -+STORE, 140613999652864, 140614008045567, -+ERASE, 140613999652864, 140614008045567, -+STORE, 140613999652864, 140614008045567, -+STORE, 140613865435136, 140613999652863, -+SNULL, 140613865435136, 140613873827839, -+STORE, 140613873827840, 140613999652863, -+STORE, 140613865435136, 140613873827839, -+ERASE, 140613865435136, 140613873827839, -+SNULL, 140614209372159, 140614242910207, -+STORE, 140614142263296, 140614209372159, -+STORE, 140614209372160, 140614242910207, -+ERASE, 140614209372160, 140614242910207, -+SNULL, 140614142398463, 140614209372159, -+STORE, 140614142263296, 140614142398463, -+STORE, 140614142398464, 140614209372159, -+SNULL, 140614075154431, 140614133870591, -+STORE, 140614008045568, 140614075154431, -+STORE, 140614075154432, 140614133870591, -+ERASE, 140614075154432, 140614133870591, -+SNULL, 140614008180735, 140614075154431, -+STORE, 140614008045568, 140614008180735, -+STORE, 140614008180736, 140614075154431, -+SNULL, 140613940936703, 140613999652863, -+STORE, 140613873827840, 140613940936703, -+STORE, 140613940936704, 140613999652863, -+ERASE, 140613940936704, 140613999652863, -+SNULL, 140614242914303, 140614259695615, -+STORE, 140614242910208, 140614242914303, -+STORE, 140614242914304, 140614259695615, -+STORE, 140613739610112, 140613940936703, -+STORE, 140614234517504, 140614242910207, -+SNULL, 140614242914304, 140614251302911, -+STORE, 140614251302912, 140614259695615, -+STORE, 140614242914304, 140614251302911, -+SNULL, 140614251307007, 140614259695615, -+STORE, 140614251302912, 140614251307007, -+STORE, 140614251307008, 140614259695615, -+SNULL, 140613739610112, 140613873827839, -+STORE, 140613873827840, 140613940936703, -+STORE, 140613739610112, 140613873827839, -+SNULL, 140613873963007, 140613940936703, -+STORE, 140613873827840, 140613873963007, -+STORE, 140613873963008, 140613940936703, -+SNULL, 140614133874687, 140614142263295, -+STORE, 140614133870592, 140614133874687, -+STORE, 140614133874688, 140614142263295, -+SNULL, 140613806718975, 140613873827839, -+STORE, 140613739610112, 140613806718975, -+STORE, 140613806718976, 140613873827839, -+ERASE, 140613806718976, 140613873827839, -+STORE, 140614226124800, 140614242910207, -+SNULL, 140613739745279, 140613806718975, -+STORE, 140613739610112, 140613739745279, -+STORE, 140613739745280, 140613806718975, -+SNULL, 140613999656959, 140614008045567, -+STORE, 140613999652864, 140613999656959, -+STORE, 140613999656960, 140614008045567, -+SNULL, 140614226124800, 140614234517503, -+STORE, 140614234517504, 140614242910207, -+STORE, 140614226124800, 140614234517503, -+SNULL, 140614234521599, 140614242910207, -+STORE, 140614234517504, 140614234521599, -+STORE, 140614234521600, 140614242910207, -+STORE, 140614217732096, 140614234517503, -+STORE, 140614125477888, 140614133870591, -+SNULL, 140614125481983, 140614133870591, -+STORE, 140614125477888, 140614125481983, -+STORE, 140614125481984, 140614133870591, -+STORE, 140614117085184, 140614125477887, -+SNULL, 140614217736191, 140614234517503, -+STORE, 140614217732096, 140614217736191, -+STORE, 140614217736192, 140614234517503, -+SNULL, 140614117089279, 140614125477887, -+STORE, 140614117085184, 140614117089279, -+STORE, 140614117089280, 140614125477887, -+SNULL, 140614217736192, 140614226124799, -+STORE, 140614226124800, 140614234517503, -+STORE, 140614217736192, 140614226124799, -+SNULL, 140614226128895, 140614234517503, -+STORE, 140614226124800, 140614226128895, -+STORE, 140614226128896, 140614234517503, -+STORE, 140614108692480, 140614117085183, -+STORE, 140614100299776, 140614117085183, -+STORE, 140614091907072, 140614117085183, -+SNULL, 140614091907072, 140614108692479, -+STORE, 140614108692480, 140614117085183, -+STORE, 140614091907072, 140614108692479, -+SNULL, 140614108696575, 140614117085183, -+STORE, 140614108692480, 140614108696575, -+STORE, 140614108696576, 140614117085183, -+SNULL, 140614091907072, 140614100299775, -+STORE, 140614100299776, 140614108692479, -+STORE, 140614091907072, 140614100299775, -+SNULL, 140614100303871, 140614108692479, -+STORE, 140614100299776, 140614100303871, -+STORE, 140614100303872, 140614108692479, -+STORE, 140614083514368, 140614100299775, -+SNULL, 140614083518463, 140614100299775, -+STORE, 140614083514368, 140614083518463, -+STORE, 140614083518464, 140614100299775, -+STORE, 140613991260160, 140613999652863, -+SNULL, 140614083518464, 140614091907071, -+STORE, 140614091907072, 140614100299775, -+STORE, 140614083518464, 140614091907071, -+SNULL, 140614091911167, 140614100299775, -+STORE, 140614091907072, 140614091911167, -+STORE, 140614091911168, 140614100299775, -+SNULL, 140613991264255, 140613999652863, -+STORE, 140613991260160, 140613991264255, -+STORE, 140613991264256, 140613999652863, -+STORE, 140613982867456, 140613991260159, -+SNULL, 140613982871551, 140613991260159, -+STORE, 140613982867456, 140613982871551, -+STORE, 140613982871552, 140613991260159, -+STORE, 140613974474752, 140613982867455, -+SNULL, 140613974478847, 140613982867455, -+STORE, 140613974474752, 140613974478847, -+STORE, 140613974478848, 140613982867455, -+STORE, 140613966082048, 140613974474751, -+STORE, 140613739745280, 140613873827839, -+SNULL, 140613739745280, 140613806718975, -+STORE, 140613806718976, 140613873827839, -+STORE, 140613739745280, 140613806718975, -+SNULL, 140613806854143, 140613873827839, -+STORE, 140613806718976, 140613806854143, -+STORE, 140613806854144, 140613873827839, -+SNULL, 140613966086143, 140613974474751, -+STORE, 140613966082048, 140613966086143, -+STORE, 140613966086144, 140613974474751, -+STORE, 140613957689344, 140613966082047, -+STORE, 140613605392384, 140613739610111, -+STORE, 140613949296640, 140613966082047, -+STORE, 140613596999680, 140613605392383, -+STORE, 140613529890816, 140613596999679, -+STORE, 140613521498112, 140613529890815, -+STORE, 140613513105408, 140613529890815, -+STORE, 140613378887680, 140613513105407, -+SNULL, 140613378887680, 140613404065791, -+STORE, 140613404065792, 140613513105407, -+STORE, 140613378887680, 140613404065791, -+ERASE, 140613378887680, 140613404065791, -+STORE, 140613395673088, 140613404065791, -+STORE, 140613261455360, 140613395673087, -+SNULL, 140613261455360, 140613269848063, -+STORE, 140613269848064, 140613395673087, -+STORE, 140613261455360, 140613269848063, -+ERASE, 140613261455360, 140613269848063, -+STORE, 140613261455360, 140613269848063, -+STORE, 140613253062656, 140613269848063, -+STORE, 140613118844928, 140613253062655, -+STORE, 140613110452224, 140613118844927, -+SNULL, 140613118844928, 140613135630335, -+STORE, 140613135630336, 140613253062655, -+STORE, 140613118844928, 140613135630335, -+ERASE, 140613118844928, 140613135630335, -+STORE, 140613127237632, 140613135630335, -+STORE, 140613110452224, 140613135630335, -+STORE, 140612976234496, 140613110452223, -+STORE, 140612967841792, 140612976234495, -+STORE, 140612833624064, 140612967841791, -+STORE, 140612825231360, 140612833624063, -+STORE, 140612816838656, 140612833624063, -+STORE, 140612682620928, 140612816838655, -+STORE, 140612674228224, 140612682620927, -+SNULL, 140612682620928, 140612732977151, -+STORE, 140612732977152, 140612816838655, -+STORE, 140612682620928, 140612732977151, -+ERASE, 140612682620928, 140612732977151, -+SNULL, 140613672501247, 140613739610111, -+STORE, 140613605392384, 140613672501247, -+STORE, 140613672501248, 140613739610111, -+ERASE, 140613672501248, 140613739610111, -+SNULL, 140613605527551, 140613672501247, -+STORE, 140613605392384, 140613605527551, -+STORE, 140613605527552, 140613672501247, -+ERASE, 140613529890816, 140613596999679, -+STORE, 140612540010496, 140612674228223, -+SNULL, 140612540010496, 140612598759423, -+STORE, 140612598759424, 140612674228223, -+STORE, 140612540010496, 140612598759423, -+ERASE, 140612540010496, 140612598759423, -+SNULL, 140613471174655, 140613513105407, -+STORE, 140613404065792, 140613471174655, -+STORE, 140613471174656, 140613513105407, -+ERASE, 140613471174656, 140613513105407, -+SNULL, 140613404200959, 140613471174655, -+STORE, 140613404065792, 140613404200959, -+STORE, 140613404200960, 140613471174655, -+SNULL, 140613336956927, 140613395673087, -+STORE, 140613269848064, 140613336956927, -+STORE, 140613336956928, 140613395673087, -+ERASE, 140613336956928, 140613395673087, -+SNULL, 140612833624064, 140612867194879, -+STORE, 140612867194880, 140612967841791, -+STORE, 140612833624064, 140612867194879, -+ERASE, 140612833624064, 140612867194879, -+SNULL, 140612976234496, 140613001412607, -+STORE, 140613001412608, 140613110452223, -+STORE, 140612976234496, 140613001412607, -+ERASE, 140612976234496, 140613001412607, -+SNULL, 140613202739199, 140613253062655, -+STORE, 140613135630336, 140613202739199, -+STORE, 140613202739200, 140613253062655, -+ERASE, 140613202739200, 140613253062655, -+SNULL, 140613135765503, 140613202739199, -+STORE, 140613135630336, 140613135765503, -+STORE, 140613135765504, 140613202739199, -+SNULL, 140612816842751, 140612833624063, -+STORE, 140612816838656, 140612816842751, -+STORE, 140612816842752, 140612833624063, -+SNULL, 140613110456319, 140613135630335, -+STORE, 140613110452224, 140613110456319, -+STORE, 140613110456320, 140613135630335, -+SNULL, 140613949300735, 140613966082047, -+STORE, 140613949296640, 140613949300735, -+STORE, 140613949300736, 140613966082047, -+SNULL, 140613110456320, 140613118844927, -+STORE, 140613118844928, 140613135630335, -+STORE, 140613110456320, 140613118844927, -+SNULL, 140613118849023, 140613135630335, -+STORE, 140613118844928, 140613118849023, -+STORE, 140613118849024, 140613135630335, -+SNULL, 140612800086015, 140612816838655, -+STORE, 140612732977152, 140612800086015, -+STORE, 140612800086016, 140612816838655, -+ERASE, 140612800086016, 140612816838655, -+SNULL, 140613253062656, 140613261455359, -+STORE, 140613261455360, 140613269848063, -+STORE, 140613253062656, 140613261455359, -+SNULL, 140613261459455, 140613269848063, -+STORE, 140613261455360, 140613261459455, -+STORE, 140613261459456, 140613269848063, -+SNULL, 140612674232319, 140612682620927, -+STORE, 140612674228224, 140612674232319, -+STORE, 140612674232320, 140612682620927, -+STORE, 140613731217408, 140613739610111, -+STORE, 140613722824704, 140613739610111, -+SNULL, 140613949300736, 140613957689343, -+STORE, 140613957689344, 140613966082047, -+STORE, 140613949300736, 140613957689343, -+SNULL, 140613957693439, 140613966082047, -+STORE, 140613957689344, 140613957693439, -+STORE, 140613957693440, 140613966082047, -+STORE, 140612464541696, 140612674228223, -+SNULL, 140612531650559, 140612674228223, -+STORE, 140612464541696, 140612531650559, -+STORE, 140612531650560, 140612674228223, -+SNULL, 140612531650560, 140612598759423, -+STORE, 140612598759424, 140612674228223, -+STORE, 140612531650560, 140612598759423, -+ERASE, 140612531650560, 140612598759423, -+SNULL, 140612665868287, 140612674228223, -+STORE, 140612598759424, 140612665868287, -+STORE, 140612665868288, 140612674228223, -+ERASE, 140612665868288, 140612674228223, -+SNULL, 140613269983231, 140613336956927, -+STORE, 140613269848064, 140613269983231, -+STORE, 140613269983232, 140613336956927, -+SNULL, 140612934303743, 140612967841791, -+STORE, 140612867194880, 140612934303743, -+STORE, 140612934303744, 140612967841791, -+ERASE, 140612934303744, 140612967841791, -+SNULL, 140613068521471, 140613110452223, -+STORE, 140613001412608, 140613068521471, -+STORE, 140613068521472, 140613110452223, -+ERASE, 140613068521472, 140613110452223, -+STORE, 140613714432000, 140613739610111, -+SNULL, 140613001547775, 140613068521471, -+STORE, 140613001412608, 140613001547775, -+STORE, 140613001547776, 140613068521471, -+SNULL, 140612733112319, 140612800086015, -+STORE, 140612732977152, 140612733112319, -+STORE, 140612733112320, 140612800086015, -+SNULL, 140613513109503, 140613529890815, -+STORE, 140613513105408, 140613513109503, -+STORE, 140613513109504, 140613529890815, -+STORE, 140613706039296, 140613739610111, -+STORE, 140613697646592, 140613739610111, -+STORE, 140613689253888, 140613739610111, -+SNULL, 140613689257983, 140613739610111, -+STORE, 140613689253888, 140613689257983, -+STORE, 140613689257984, 140613739610111, -+SNULL, 140613253066751, 140613261455359, -+STORE, 140613253062656, 140613253066751, -+STORE, 140613253066752, 140613261455359, -+STORE, 140613680861184, 140613689253887, -+STORE, 140613588606976, 140613605392383, -+SNULL, 140613689257984, 140613731217407, -+STORE, 140613731217408, 140613739610111, -+STORE, 140613689257984, 140613731217407, -+SNULL, 140613731221503, 140613739610111, -+STORE, 140613731217408, 140613731221503, -+STORE, 140613731221504, 140613739610111, -+STORE, 140613580214272, 140613605392383, -+SNULL, 140612464676863, 140612531650559, -+STORE, 140612464541696, 140612464676863, -+STORE, 140612464676864, 140612531650559, -+SNULL, 140612598894591, 140612665868287, -+STORE, 140612598759424, 140612598894591, -+STORE, 140612598894592, 140612665868287, -+SNULL, 140612867330047, 140612934303743, -+STORE, 140612867194880, 140612867330047, -+STORE, 140612867330048, 140612934303743, -+STORE, 140613571821568, 140613605392383, -+SNULL, 140613571825663, 140613605392383, -+STORE, 140613571821568, 140613571825663, -+STORE, 140613571825664, 140613605392383, -+SNULL, 140613689257984, 140613722824703, -+STORE, 140613722824704, 140613731217407, -+STORE, 140613689257984, 140613722824703, -+SNULL, 140613722828799, 140613731217407, -+STORE, 140613722824704, 140613722828799, -+STORE, 140613722828800, 140613731217407, -+SNULL, 140613689257984, 140613714431999, -+STORE, 140613714432000, 140613722824703, -+STORE, 140613689257984, 140613714431999, -+SNULL, 140613714436095, 140613722824703, -+STORE, 140613714432000, 140613714436095, -+STORE, 140613714436096, 140613722824703, -+SNULL, 140612816842752, 140612825231359, -+STORE, 140612825231360, 140612833624063, -+STORE, 140612816842752, 140612825231359, -+SNULL, 140612825235455, 140612833624063, -+STORE, 140612825231360, 140612825235455, -+STORE, 140612825235456, 140612833624063, -+SNULL, 140613395677183, 140613404065791, -+STORE, 140613395673088, 140613395677183, -+STORE, 140613395677184, 140613404065791, -+SNULL, 140613689257984, 140613706039295, -+STORE, 140613706039296, 140613714431999, -+STORE, 140613689257984, 140613706039295, -+SNULL, 140613706043391, 140613714431999, -+STORE, 140613706039296, 140613706043391, -+STORE, 140613706043392, 140613714431999, -+SNULL, 140613118849024, 140613127237631, -+STORE, 140613127237632, 140613135630335, -+STORE, 140613118849024, 140613127237631, -+SNULL, 140613127241727, 140613135630335, -+STORE, 140613127237632, 140613127241727, -+STORE, 140613127241728, 140613135630335, -+SNULL, 140613571825664, 140613580214271, -+STORE, 140613580214272, 140613605392383, -+STORE, 140613571825664, 140613580214271, -+SNULL, 140613580218367, 140613605392383, -+STORE, 140613580214272, 140613580218367, -+STORE, 140613580218368, 140613605392383, -+SNULL, 140613689257984, 140613697646591, -+STORE, 140613697646592, 140613706039295, -+STORE, 140613689257984, 140613697646591, -+SNULL, 140613697650687, 140613706039295, -+STORE, 140613697646592, 140613697650687, -+STORE, 140613697650688, 140613706039295, -+SNULL, 140613680865279, 140613689253887, -+STORE, 140613680861184, 140613680865279, -+STORE, 140613680865280, 140613689253887, -+STORE, 140613563428864, 140613571821567, -+SNULL, 140613563432959, 140613571821567, -+STORE, 140613563428864, 140613563432959, -+STORE, 140613563432960, 140613571821567, -+SNULL, 140613580218368, 140613588606975, -+STORE, 140613588606976, 140613605392383, -+STORE, 140613580218368, 140613588606975, -+SNULL, 140613588611071, 140613605392383, -+STORE, 140613588606976, 140613588611071, -+STORE, 140613588611072, 140613605392383, -+SNULL, 140613513109504, 140613521498111, -+STORE, 140613521498112, 140613529890815, -+STORE, 140613513109504, 140613521498111, -+SNULL, 140613521502207, 140613529890815, -+STORE, 140613521498112, 140613521502207, -+STORE, 140613521502208, 140613529890815, -+SNULL, 140613588611072, 140613596999679, -+STORE, 140613596999680, 140613605392383, -+STORE, 140613588611072, 140613596999679, -+SNULL, 140613597003775, 140613605392383, -+STORE, 140613596999680, 140613597003775, -+STORE, 140613597003776, 140613605392383, -+STORE, 140613555036160, 140613563428863, -+SNULL, 140613555040255, 140613563428863, -+STORE, 140613555036160, 140613555040255, -+STORE, 140613555040256, 140613563428863, -+STORE, 140613546643456, 140613555036159, -+STORE, 140613538250752, 140613555036159, -+SNULL, 140613538250752, 140613546643455, -+STORE, 140613546643456, 140613555036159, -+STORE, 140613538250752, 140613546643455, -+SNULL, 140613546647551, 140613555036159, -+STORE, 140613546643456, 140613546647551, -+STORE, 140613546647552, 140613555036159, -+STORE, 140613504712704, 140613513105407, -+STORE, 140613496320000, 140613513105407, -+SNULL, 140613496324095, 140613513105407, -+STORE, 140613496320000, 140613496324095, -+STORE, 140613496324096, 140613513105407, -+STORE, 140613487927296, 140613496319999, -+SNULL, 140613487931391, 140613496319999, -+STORE, 140613487927296, 140613487931391, -+STORE, 140613487931392, 140613496319999, -+STORE, 140613479534592, 140613487927295, -+SNULL, 140612967845887, 140612976234495, -+STORE, 140612967841792, 140612967845887, -+STORE, 140612967845888, 140612976234495, -+STORE, 140613387280384, 140613395673087, -+STORE, 140613378887680, 140613395673087, -+SNULL, 140613378887680, 140613387280383, -+STORE, 140613387280384, 140613395673087, -+STORE, 140613378887680, 140613387280383, -+SNULL, 140613387284479, 140613395673087, -+STORE, 140613387280384, 140613387284479, -+STORE, 140613387284480, 140613395673087, -+STORE, 140613370494976, 140613387280383, -+STORE, 140613362102272, 140613387280383, -+SNULL, 140613479538687, 140613487927295, -+STORE, 140613479534592, 140613479538687, -+STORE, 140613479538688, 140613487927295, -+STORE, 140613353709568, 140613387280383, -+STORE, 140613345316864, 140613387280383, -+STORE, 140613244669952, 140613253062655, -+SNULL, 140613345320959, 140613387280383, -+STORE, 140613345316864, 140613345320959, -+STORE, 140613345320960, 140613387280383, -+SNULL, 140613538254847, 140613546643455, -+STORE, 140613538250752, 140613538254847, -+STORE, 140613538254848, 140613546643455, -+STORE, 140613236277248, 140613253062655, -+STORE, 140613227884544, 140613253062655, -+STORE, 140613219491840, 140613253062655, -+STORE, 140613211099136, 140613253062655, -+SNULL, 140613211103231, 140613253062655, -+STORE, 140613211099136, 140613211103231, -+STORE, 140613211103232, 140613253062655, -+STORE, 140613102059520, 140613110452223, -+STORE, 140613093666816, 140613110452223, -+SNULL, 140613093670911, 140613110452223, -+STORE, 140613093666816, 140613093670911, -+STORE, 140613093670912, 140613110452223, -+STORE, 140613085274112, 140613093666815, -+SNULL, 140613496324096, 140613504712703, -+STORE, 140613504712704, 140613513105407, -+STORE, 140613496324096, 140613504712703, -+SNULL, 140613504716799, 140613513105407, -+STORE, 140613504712704, 140613504716799, -+STORE, 140613504716800, 140613513105407, -+SNULL, 140613345320960, 140613378887679, -+STORE, 140613378887680, 140613387280383, -+STORE, 140613345320960, 140613378887679, -+SNULL, 140613378891775, 140613387280383, -+STORE, 140613378887680, 140613378891775, -+STORE, 140613378891776, 140613387280383, -+SNULL, 140613345320960, 140613362102271, -+STORE, 140613362102272, 140613378887679, -+STORE, 140613345320960, 140613362102271, -+SNULL, 140613362106367, 140613378887679, -+STORE, 140613362102272, 140613362106367, -+STORE, 140613362106368, 140613378887679, -+SNULL, 140613362106368, 140613370494975, -+STORE, 140613370494976, 140613378887679, -+STORE, 140613362106368, 140613370494975, -+SNULL, 140613370499071, 140613378887679, -+STORE, 140613370494976, 140613370499071, -+STORE, 140613370499072, 140613378887679, -+STORE, 140613076881408, 140613093666815, -+STORE, 140612993019904, 140613001412607, -+SNULL, 140613076885503, 140613093666815, -+STORE, 140613076881408, 140613076885503, -+STORE, 140613076885504, 140613093666815, -+SNULL, 140613093670912, 140613102059519, -+STORE, 140613102059520, 140613110452223, -+STORE, 140613093670912, 140613102059519, -+SNULL, 140613102063615, 140613110452223, -+STORE, 140613102059520, 140613102063615, -+STORE, 140613102063616, 140613110452223, -+SNULL, 140613076885504, 140613085274111, -+STORE, 140613085274112, 140613093666815, -+STORE, 140613076885504, 140613085274111, -+SNULL, 140613085278207, 140613093666815, -+STORE, 140613085274112, 140613085278207, -+STORE, 140613085278208, 140613093666815, -+STORE, 140612984627200, 140613001412607, -+STORE, 140612967845888, 140612984627199, -+SNULL, 140613211103232, 140613219491839, -+STORE, 140613219491840, 140613253062655, -+STORE, 140613211103232, 140613219491839, -+SNULL, 140613219495935, 140613253062655, -+STORE, 140613219491840, 140613219495935, -+STORE, 140613219495936, 140613253062655, -+STORE, 140612959449088, 140612967841791, -+STORE, 140612951056384, 140612967841791, -+SNULL, 140612951060479, 140612967841791, -+STORE, 140612951056384, 140612951060479, -+STORE, 140612951060480, 140612967841791, -+SNULL, 140613345320960, 140613353709567, -+STORE, 140613353709568, 140613362102271, -+STORE, 140613345320960, 140613353709567, -+SNULL, 140613353713663, 140613362102271, -+STORE, 140613353709568, 140613353713663, -+STORE, 140613353713664, 140613362102271, -+SNULL, 140613219495936, 140613244669951, -+STORE, 140613244669952, 140613253062655, -+STORE, 140613219495936, 140613244669951, -+SNULL, 140613244674047, 140613253062655, -+STORE, 140613244669952, 140613244674047, -+STORE, 140613244674048, 140613253062655, -+STORE, 140612942663680, 140612951056383, -+SNULL, 140613219495936, 140613236277247, -+STORE, 140613236277248, 140613244669951, -+STORE, 140613219495936, 140613236277247, -+SNULL, 140613236281343, 140613244669951, -+STORE, 140613236277248, 140613236281343, -+STORE, 140613236281344, 140613244669951, -+SNULL, 140613219495936, 140613227884543, -+STORE, 140613227884544, 140613236277247, -+STORE, 140613219495936, 140613227884543, -+SNULL, 140613227888639, 140613236277247, -+STORE, 140613227884544, 140613227888639, -+STORE, 140613227888640, 140613236277247, -+SNULL, 140612984627200, 140612993019903, -+STORE, 140612993019904, 140613001412607, -+STORE, 140612984627200, 140612993019903, -+SNULL, 140612993023999, 140613001412607, -+STORE, 140612993019904, 140612993023999, -+STORE, 140612993024000, 140613001412607, -+STORE, 140612858802176, 140612867194879, -+STORE, 140612850409472, 140612867194879, -+SNULL, 140612951060480, 140612959449087, -+STORE, 140612959449088, 140612967841791, -+STORE, 140612951060480, 140612959449087, -+SNULL, 140612959453183, 140612967841791, -+STORE, 140612959449088, 140612959453183, -+STORE, 140612959453184, 140612967841791, -+SNULL, 140612967845888, 140612976234495, -+STORE, 140612976234496, 140612984627199, -+STORE, 140612967845888, 140612976234495, -+SNULL, 140612976238591, 140612984627199, -+STORE, 140612976234496, 140612976238591, -+STORE, 140612976238592, 140612984627199, -+STORE, 140612842016768, 140612867194879, -+SNULL, 140612842020863, 140612867194879, -+STORE, 140612842016768, 140612842020863, -+STORE, 140612842020864, 140612867194879, -+SNULL, 140612984631295, 140612993019903, -+STORE, 140612984627200, 140612984631295, -+STORE, 140612984631296, 140612993019903, -+STORE, 140612825235456, 140612842016767, -+STORE, 140612808445952, 140612816838655, -+SNULL, 140612942667775, 140612951056383, -+STORE, 140612942663680, 140612942667775, -+STORE, 140612942667776, 140612951056383, -+STORE, 140612724584448, 140612732977151, -+SNULL, 140612724588543, 140612732977151, -+STORE, 140612724584448, 140612724588543, -+STORE, 140612724588544, 140612732977151, -+STORE, 140612716191744, 140612724584447, -+SNULL, 140612842020864, 140612850409471, -+STORE, 140612850409472, 140612867194879, -+STORE, 140612842020864, 140612850409471, -+SNULL, 140612850413567, 140612867194879, -+STORE, 140612850409472, 140612850413567, -+STORE, 140612850413568, 140612867194879, -+SNULL, 140612850413568, 140612858802175, -+STORE, 140612858802176, 140612867194879, -+STORE, 140612850413568, 140612858802175, -+SNULL, 140612858806271, 140612867194879, -+STORE, 140612858802176, 140612858806271, -+STORE, 140612858806272, 140612867194879, -+STORE, 140612707799040, 140612724584447, -+SNULL, 140612707803135, 140612724584447, -+STORE, 140612707799040, 140612707803135, -+STORE, 140612707803136, 140612724584447, -+SNULL, 140612707803136, 140612716191743, -+STORE, 140612716191744, 140612724584447, -+STORE, 140612707803136, 140612716191743, -+SNULL, 140612716195839, 140612724584447, -+STORE, 140612716191744, 140612716195839, -+STORE, 140612716195840, 140612724584447, -+SNULL, 140612808450047, 140612816838655, -+STORE, 140612808445952, 140612808450047, -+STORE, 140612808450048, 140612816838655, -+SNULL, 140612825235456, 140612833624063, -+STORE, 140612833624064, 140612842016767, -+STORE, 140612825235456, 140612833624063, -+SNULL, 140612833628159, 140612842016767, -+STORE, 140612833624064, 140612833628159, -+STORE, 140612833628160, 140612842016767, -+STORE, 140612699406336, 140612707799039, -+SNULL, 140612699410431, 140612707799039, -+STORE, 140612699406336, 140612699410431, -+STORE, 140612699410432, 140612707799039, -+STORE, 140614384926720, 140614384955391, -+STORE, 140614349332480, 140614351523839, -+SNULL, 140614349332480, 140614349422591, -+STORE, 140614349422592, 140614351523839, -+STORE, 140614349332480, 140614349422591, -+SNULL, 140614351515647, 140614351523839, -+STORE, 140614349422592, 140614351515647, -+STORE, 140614351515648, 140614351523839, -+ERASE, 140614351515648, 140614351523839, -+STORE, 140614351515648, 140614351523839, -+SNULL, 140614351519743, 140614351523839, -+STORE, 140614351515648, 140614351519743, -+STORE, 140614351519744, 140614351523839, -+ERASE, 140614384926720, 140614384955391, -+ERASE, 140613949296640, 140613949300735, -+ERASE, 140613949300736, 140613957689343, -+ERASE, 140613689253888, 140613689257983, -+ERASE, 140613689257984, 140613697646591, -+ERASE, 140613563428864, 140613563432959, -+ERASE, 140613563432960, 140613571821567, -+ERASE, 140613211099136, 140613211103231, -+ERASE, 140613211103232, 140613219491839, -+ERASE, 140614133870592, 140614133874687, -+ERASE, 140614133874688, 140614142263295, -+ERASE, 140612967841792, 140612967845887, -+ERASE, 140612967845888, 140612976234495, -+ERASE, 140613076881408, 140613076885503, -+ERASE, 140613076885504, 140613085274111, -+ERASE, 140612850409472, 140612850413567, -+ERASE, 140612850413568, 140612858802175, -+ERASE, 140613110452224, 140613110456319, -+ERASE, 140613110456320, 140613118844927, -+ERASE, 140613706039296, 140613706043391, -+ERASE, 140613706043392, 140613714431999, -+ERASE, 140613521498112, 140613521502207, -+ERASE, 140613521502208, 140613529890815, -+ERASE, 140613362102272, 140613362106367, -+ERASE, 140613362106368, 140613370494975, -+ERASE, 140613253062656, 140613253066751, -+ERASE, 140613253066752, 140613261455359, -+ERASE, 140612816838656, 140612816842751, -+ERASE, 140612816842752, 140612825231359, -+ERASE, 140613261455360, 140613261459455, -+ERASE, 140613261459456, 140613269848063, -+ERASE, 140613118844928, 140613118849023, -+ERASE, 140613118849024, 140613127237631, -+ERASE, 140613714432000, 140613714436095, -+ERASE, 140613714436096, 140613722824703, -+ERASE, 140613496320000, 140613496324095, -+ERASE, 140613496324096, 140613504712703, -+ERASE, 140613513105408, 140613513109503, -+ERASE, 140613513109504, 140613521498111, -+ERASE, 140613697646592, 140613697650687, -+ERASE, 140613697650688, 140613706039295, -+ERASE, 140613093666816, 140613093670911, -+ERASE, 140613093670912, 140613102059519, -+ERASE, 140612993019904, 140612993023999, -+ERASE, 140612993024000, 140613001412607, -+ERASE, 140613127237632, 140613127241727, -+ERASE, 140613127241728, 140613135630335, -+ERASE, 140613957689344, 140613957693439, -+ERASE, 140613957693440, 140613966082047, -+ERASE, 140613571821568, 140613571825663, -+ERASE, 140613571825664, 140613580214271, -+ERASE, 140613479534592, 140613479538687, -+ERASE, 140613479538688, 140613487927295, -+ERASE, 140612984627200, 140612984631295, -+ERASE, 140612984631296, 140612993019903, -+ERASE, 140613588606976, 140613588611071, -+ERASE, 140613588611072, 140613596999679, -+ERASE, 140613680861184, 140613680865279, -+ERASE, 140613680865280, 140613689253887, -+ERASE, 140613345316864, 140613345320959, -+ERASE, 140613345320960, 140613353709567, -+ERASE, 140613596999680, 140613597003775, -+ERASE, 140613597003776, 140613605392383, -+ERASE, 140613966082048, 140613966086143, -+ERASE, 140613966086144, 140613974474751, -+ERASE, 140613731217408, 140613731221503, -+ERASE, 140613731221504, 140613739610111, -+ERASE, 140613395673088, 140613395677183, -+ERASE, 140613395677184, 140613404065791, -+ERASE, 140612825231360, 140612825235455, -+ERASE, 140612825235456, 140612833624063, -+ERASE, 140612674228224, 140612674232319, -+ERASE, 140612674232320, 140612682620927, -+ERASE, 140613722824704, 140613722828799, -+ERASE, 140613722828800, 140613731217407, -+ERASE, 140613487927296, 140613487931391, -+ERASE, 140613487931392, 140613496319999, -+ERASE, 140613102059520, 140613102063615, -+ERASE, 140613102063616, 140613110452223, -+ERASE, 140614242910208, 140614242914303, -+ERASE, 140614242914304, 140614251302911, -+ERASE, 140612808445952, 140612808450047, -+ERASE, 140612808450048, 140612816838655, -+ERASE, 140613236277248, 140613236281343, -+ERASE, 140613236281344, 140613244669951, -+ERASE, 140613580214272, 140613580218367, -+ERASE, 140613580218368, 140613588606975, -+ERASE, 140613370494976, 140613370499071, -+ERASE, 140613370499072, 140613378887679, -+ERASE, 140613244669952, 140613244674047, -+ERASE, 140613244674048, 140613253062655, -+ERASE, 140612724584448, 140612724588543, -+ERASE, 140612724588544, 140612732977151, -+ERASE, 140612707799040, 140612707803135, -+ERASE, 140612707803136, 140612716191743, -+ERASE, 140613504712704, 140613504716799, -+ERASE, 140613504716800, 140613513105407, -+ }; -+ -+ unsigned long set39[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140736271417344, 140737488351231, -+SNULL, 140736271421439, 140737488351231, -+STORE, 140736271417344, 140736271421439, -+STORE, 140736271286272, 140736271421439, -+STORE, 94412930822144, 94412933074943, -+SNULL, 94412930953215, 94412933074943, -+STORE, 94412930822144, 94412930953215, -+STORE, 94412930953216, 94412933074943, -+ERASE, 94412930953216, 94412933074943, -+STORE, 94412933046272, 94412933054463, -+STORE, 94412933054464, 94412933074943, -+STORE, 140326136901632, 140326139154431, -+SNULL, 140326137044991, 140326139154431, -+STORE, 140326136901632, 140326137044991, -+STORE, 140326137044992, 140326139154431, -+ERASE, 140326137044992, 140326139154431, -+STORE, 140326139142144, 140326139150335, -+STORE, 140326139150336, 140326139154431, -+STORE, 140736271585280, 140736271589375, -+STORE, 140736271572992, 140736271585279, -+STORE, 140326139113472, 140326139142143, -+STORE, 140326139105280, 140326139113471, -+STORE, 140326134685696, 140326136901631, -+SNULL, 140326134685696, 140326134783999, -+STORE, 140326134784000, 140326136901631, -+STORE, 140326134685696, 140326134783999, -+SNULL, 140326136877055, 140326136901631, -+STORE, 140326134784000, 140326136877055, -+STORE, 140326136877056, 140326136901631, -+SNULL, 140326136877056, 140326136885247, -+STORE, 140326136885248, 140326136901631, -+STORE, 140326136877056, 140326136885247, -+ERASE, 140326136877056, 140326136885247, -+STORE, 140326136877056, 140326136885247, -+ERASE, 140326136885248, 140326136901631, -+STORE, 140326136885248, 140326136901631, -+STORE, 140326130888704, 140326134685695, -+SNULL, 140326130888704, 140326132547583, -+STORE, 140326132547584, 140326134685695, -+STORE, 140326130888704, 140326132547583, -+SNULL, 140326134644735, 140326134685695, -+STORE, 140326132547584, 140326134644735, -+STORE, 140326134644736, 140326134685695, -+SNULL, 140326134644736, 140326134669311, -+STORE, 140326134669312, 140326134685695, -+STORE, 140326134644736, 140326134669311, -+ERASE, 140326134644736, 140326134669311, -+STORE, 140326134644736, 140326134669311, -+ERASE, 140326134669312, 140326134685695, -+STORE, 140326134669312, 140326134685695, -+STORE, 140326139097088, 140326139113471, -+SNULL, 140326134661119, 140326134669311, -+STORE, 140326134644736, 140326134661119, -+STORE, 140326134661120, 140326134669311, -+SNULL, 140326136881151, 140326136885247, -+STORE, 140326136877056, 140326136881151, -+STORE, 140326136881152, 140326136885247, -+SNULL, 94412933050367, 94412933054463, -+STORE, 94412933046272, 94412933050367, -+STORE, 94412933050368, 94412933054463, -+SNULL, 140326139146239, 140326139150335, -+STORE, 140326139142144, 140326139146239, -+STORE, 140326139146240, 140326139150335, -+ERASE, 140326139113472, 140326139142143, -+STORE, 94412939493376, 94412939628543, -+STORE, 140326122496000, 140326130888703, -+SNULL, 140326122500095, 140326130888703, -+STORE, 140326122496000, 140326122500095, -+STORE, 140326122500096, 140326130888703, -+STORE, 140326114103296, 140326122495999, -+STORE, 140325979885568, 140326114103295, -+SNULL, 140325979885568, 140326043910143, -+STORE, 140326043910144, 140326114103295, -+STORE, 140325979885568, 140326043910143, -+ERASE, 140325979885568, 140326043910143, -+SNULL, 140326111019007, 140326114103295, -+STORE, 140326043910144, 140326111019007, -+STORE, 140326111019008, 140326114103295, -+ERASE, 140326111019008, 140326114103295, -+SNULL, 140326044045311, 140326111019007, -+STORE, 140326043910144, 140326044045311, -+STORE, 140326044045312, 140326111019007, -+SNULL, 140326114107391, 140326122495999, -+STORE, 140326114103296, 140326114107391, -+STORE, 140326114107392, 140326122495999, -+STORE, 140326035517440, 140326043910143, -+SNULL, 140326035521535, 140326043910143, -+STORE, 140326035517440, 140326035521535, -+STORE, 140326035521536, 140326043910143, -+STORE, 140326027124736, 140326035517439, -+SNULL, 140326027128831, 140326035517439, -+STORE, 140326027124736, 140326027128831, -+STORE, 140326027128832, 140326035517439, -+STORE, 140326018732032, 140326027124735, -+SNULL, 140326018736127, 140326027124735, -+STORE, 140326018732032, 140326018736127, -+STORE, 140326018736128, 140326027124735, -+STORE, 140326010339328, 140326018732031, -+STORE, 140326001946624, 140326018732031, -+STORE, 140325993553920, 140326018732031, -+STORE, 140325859336192, 140325993553919, -+SNULL, 140325859336192, 140325909692415, -+STORE, 140325909692416, 140325993553919, -+STORE, 140325859336192, 140325909692415, -+ERASE, 140325859336192, 140325909692415, -+SNULL, 140325976801279, 140325993553919, -+STORE, 140325909692416, 140325976801279, -+STORE, 140325976801280, 140325993553919, -+ERASE, 140325976801280, 140325993553919, -+STORE, 140325985161216, 140326018732031, -+STORE, 140325775474688, 140325976801279, -+STORE, 140325708365824, 140325976801279, -+SNULL, 140325708500991, 140325976801279, -+STORE, 140325708365824, 140325708500991, -+STORE, 140325708500992, 140325976801279, -+SNULL, 140325708500992, 140325909692415, -+STORE, 140325909692416, 140325976801279, -+STORE, 140325708500992, 140325909692415, -+SNULL, 140325909827583, 140325976801279, -+STORE, 140325909692416, 140325909827583, -+STORE, 140325909827584, 140325976801279, -+SNULL, 140325842583551, 140325909692415, -+STORE, 140325708500992, 140325842583551, -+STORE, 140325842583552, 140325909692415, -+ERASE, 140325842583552, 140325909692415, -+SNULL, 140325708500992, 140325775474687, -+STORE, 140325775474688, 140325842583551, -+STORE, 140325708500992, 140325775474687, -+SNULL, 140325775609855, 140325842583551, -+STORE, 140325775474688, 140325775609855, -+STORE, 140325775609856, 140325842583551, -+STORE, 140325775609856, 140325909692415, -+SNULL, 140325775609856, 140325842583551, -+STORE, 140325842583552, 140325909692415, -+STORE, 140325775609856, 140325842583551, -+SNULL, 140325842718719, 140325909692415, -+STORE, 140325842583552, 140325842718719, -+STORE, 140325842718720, 140325909692415, -+SNULL, 140325985161216, 140325993553919, -+STORE, 140325993553920, 140326018732031, -+STORE, 140325985161216, 140325993553919, -+SNULL, 140325993558015, 140326018732031, -+STORE, 140325993553920, 140325993558015, -+STORE, 140325993558016, 140326018732031, -+SNULL, 140325985165311, 140325993553919, -+STORE, 140325985161216, 140325985165311, -+STORE, 140325985165312, 140325993553919, -+SNULL, 140325993558016, 140326001946623, -+STORE, 140326001946624, 140326018732031, -+STORE, 140325993558016, 140326001946623, -+SNULL, 140326001950719, 140326018732031, -+STORE, 140326001946624, 140326001950719, -+STORE, 140326001950720, 140326018732031, -+SNULL, 140326001950720, 140326010339327, -+STORE, 140326010339328, 140326018732031, -+STORE, 140326001950720, 140326010339327, -+SNULL, 140326010343423, 140326018732031, -+STORE, 140326010339328, 140326010343423, -+STORE, 140326010343424, 140326018732031, -+STORE, 140325699973120, 140325708365823, -+STORE, 140325691580416, 140325708365823, -+STORE, 140325683187712, 140325708365823, -+SNULL, 140325683191807, 140325708365823, -+STORE, 140325683187712, 140325683191807, -+STORE, 140325683191808, 140325708365823, -+SNULL, 140325683191808, 140325699973119, -+STORE, 140325699973120, 140325708365823, -+STORE, 140325683191808, 140325699973119, -+SNULL, 140325699977215, 140325708365823, -+STORE, 140325699973120, 140325699977215, -+STORE, 140325699977216, 140325708365823, -+STORE, 140325674795008, 140325683187711, -+STORE, 140325666402304, 140325683187711, -+STORE, 140325658009600, 140325683187711, -+SNULL, 140325658009600, 140325666402303, -+STORE, 140325666402304, 140325683187711, -+STORE, 140325658009600, 140325666402303, -+SNULL, 140325666406399, 140325683187711, -+STORE, 140325666402304, 140325666406399, -+STORE, 140325666406400, 140325683187711, -+SNULL, 140325683191808, 140325691580415, -+STORE, 140325691580416, 140325699973119, -+STORE, 140325683191808, 140325691580415, -+SNULL, 140325691584511, 140325699973119, -+STORE, 140325691580416, 140325691584511, -+STORE, 140325691584512, 140325699973119, -+SNULL, 140325666406400, 140325674795007, -+STORE, 140325674795008, 140325683187711, -+STORE, 140325666406400, 140325674795007, -+SNULL, 140325674799103, 140325683187711, -+STORE, 140325674795008, 140325674799103, -+STORE, 140325674799104, 140325683187711, -+STORE, 140325649616896, 140325666402303, -+SNULL, 140325649616896, 140325658009599, -+STORE, 140325658009600, 140325666402303, -+STORE, 140325649616896, 140325658009599, -+SNULL, 140325658013695, 140325666402303, -+STORE, 140325658009600, 140325658013695, -+STORE, 140325658013696, 140325666402303, -+SNULL, 140325649620991, 140325658009599, -+STORE, 140325649616896, 140325649620991, -+STORE, 140325649620992, 140325658009599, -+STORE, 140325641224192, 140325649616895, -+STORE, 140325632831488, 140325649616895, -+SNULL, 140325632835583, 140325649616895, -+STORE, 140325632831488, 140325632835583, -+STORE, 140325632835584, 140325649616895, -+STORE, 140325624438784, 140325632831487, -+SNULL, 140325624442879, 140325632831487, -+STORE, 140325624438784, 140325624442879, -+STORE, 140325624442880, 140325632831487, -+SNULL, 140325632835584, 140325641224191, -+STORE, 140325641224192, 140325649616895, -+STORE, 140325632835584, 140325641224191, -+SNULL, 140325641228287, 140325649616895, -+STORE, 140325641224192, 140325641228287, -+STORE, 140325641228288, 140325649616895, -+STORE, 140325616046080, 140325624438783, -+SNULL, 140325616050175, 140325624438783, -+STORE, 140325616046080, 140325616050175, -+STORE, 140325616050176, 140325624438783, -+STORE, 140325607653376, 140325616046079, -+SNULL, 140325607657471, 140325616046079, -+STORE, 140325607653376, 140325607657471, -+STORE, 140325607657472, 140325616046079, -+STORE, 140325599260672, 140325607653375, -+STORE, 140325590867968, 140325607653375, -+STORE, 140325456650240, 140325590867967, -+SNULL, 140325456650240, 140325507039231, -+STORE, 140325507039232, 140325590867967, -+STORE, 140325456650240, 140325507039231, -+ERASE, 140325456650240, 140325507039231, -+STORE, 140325498646528, 140325507039231, -+STORE, 140325364428800, 140325498646527, -+SNULL, 140325364428800, 140325372821503, -+STORE, 140325372821504, 140325498646527, -+STORE, 140325364428800, 140325372821503, -+ERASE, 140325364428800, 140325372821503, -+STORE, 140325364428800, 140325372821503, -+STORE, 140325356036096, 140325372821503, -+STORE, 140325221818368, 140325356036095, -+SNULL, 140325221818368, 140325238603775, -+STORE, 140325238603776, 140325356036095, -+STORE, 140325221818368, 140325238603775, -+ERASE, 140325221818368, 140325238603775, -+STORE, 140325230211072, 140325238603775, -+STORE, 140325221818368, 140325238603775, -+STORE, 140325087600640, 140325221818367, -+STORE, 140325079207936, 140325087600639, -+SNULL, 140325087600640, 140325104386047, -+STORE, 140325104386048, 140325221818367, -+STORE, 140325087600640, 140325104386047, -+ERASE, 140325087600640, 140325104386047, -+STORE, 140325095993344, 140325104386047, -+STORE, 140325079207936, 140325104386047, -+STORE, 140324944990208, 140325079207935, -+SNULL, 140324944990208, 140324970168319, -+STORE, 140324970168320, 140325079207935, -+STORE, 140324944990208, 140324970168319, -+ERASE, 140324944990208, 140324970168319, -+STORE, 140324961775616, 140324970168319, -+STORE, 140324953382912, 140324970168319, -+STORE, 140324819165184, 140324953382911, -+STORE, 140324684947456, 140324953382911, -+STORE, 140324676554752, 140324684947455, -+STORE, 140324668162048, 140324684947455, -+STORE, 140324533944320, 140324668162047, -+STORE, 140324525551616, 140324533944319, -+SNULL, 140324533944320, 140324567515135, -+STORE, 140324567515136, 140324668162047, -+STORE, 140324533944320, 140324567515135, -+ERASE, 140324533944320, 140324567515135, -+STORE, 140324559122432, 140324567515135, -+STORE, 140324391333888, 140324525551615, -+SNULL, 140325574148095, 140325590867967, -+STORE, 140325507039232, 140325574148095, -+STORE, 140325574148096, 140325590867967, -+ERASE, 140325574148096, 140325590867967, -+SNULL, 140325439930367, 140325498646527, -+STORE, 140325372821504, 140325439930367, -+STORE, 140325439930368, 140325498646527, -+ERASE, 140325439930368, 140325498646527, -+SNULL, 140325305712639, 140325356036095, -+STORE, 140325238603776, 140325305712639, -+STORE, 140325305712640, 140325356036095, -+ERASE, 140325305712640, 140325356036095, -+SNULL, 140325171494911, 140325221818367, -+STORE, 140325104386048, 140325171494911, -+STORE, 140325171494912, 140325221818367, -+ERASE, 140325171494912, 140325221818367, -+SNULL, 140325104521215, 140325171494911, -+STORE, 140325104386048, 140325104521215, -+STORE, 140325104521216, 140325171494911, -+STORE, 140324257116160, 140324525551615, -+SNULL, 140324257116160, 140324299079679, -+STORE, 140324299079680, 140324525551615, -+STORE, 140324257116160, 140324299079679, -+ERASE, 140324257116160, 140324299079679, -+SNULL, 140325037277183, 140325079207935, -+STORE, 140324970168320, 140325037277183, -+STORE, 140325037277184, 140325079207935, -+ERASE, 140325037277184, 140325079207935, -+SNULL, 140324819165183, 140324953382911, -+STORE, 140324684947456, 140324819165183, -+STORE, 140324819165184, 140324953382911, -+SNULL, 140324819165184, 140324835950591, -+STORE, 140324835950592, 140324953382911, -+STORE, 140324819165184, 140324835950591, -+ERASE, 140324819165184, 140324835950591, -+SNULL, 140324903059455, 140324953382911, -+STORE, 140324835950592, 140324903059455, -+STORE, 140324903059456, 140324953382911, -+ERASE, 140324903059456, 140324953382911, -+SNULL, 140324684947456, 140324701732863, -+STORE, 140324701732864, 140324819165183, -+STORE, 140324684947456, 140324701732863, -+ERASE, 140324684947456, 140324701732863, -+SNULL, 140324768841727, 140324819165183, -+STORE, 140324701732864, 140324768841727, -+STORE, 140324768841728, 140324819165183, -+ERASE, 140324768841728, 140324819165183, -+SNULL, 140324634623999, 140324668162047, -+STORE, 140324567515136, 140324634623999, -+STORE, 140324634624000, 140324668162047, -+ERASE, 140324634624000, 140324668162047, -+SNULL, 140324391333887, 140324525551615, -+STORE, 140324299079680, 140324391333887, -+STORE, 140324391333888, 140324525551615, -+SNULL, 140324391333888, 140324433297407, -+STORE, 140324433297408, 140324525551615, -+STORE, 140324391333888, 140324433297407, -+ERASE, 140324391333888, 140324433297407, -+SNULL, 140325507174399, 140325574148095, -+STORE, 140325507039232, 140325507174399, -+STORE, 140325507174400, 140325574148095, -+SNULL, 140325590867968, 140325599260671, -+STORE, 140325599260672, 140325607653375, -+STORE, 140325590867968, 140325599260671, -+SNULL, 140325599264767, 140325607653375, -+STORE, 140325599260672, 140325599264767, -+STORE, 140325599264768, 140325607653375, -+SNULL, 140325372956671, 140325439930367, -+STORE, 140325372821504, 140325372956671, -+STORE, 140325372956672, 140325439930367, -+SNULL, 140324668166143, 140324684947455, -+STORE, 140324668162048, 140324668166143, -+STORE, 140324668166144, 140324684947455, -+SNULL, 140324525555711, 140324533944319, -+STORE, 140324525551616, 140324525555711, -+STORE, 140324525555712, 140324533944319, -+SNULL, 140324953382912, 140324961775615, -+STORE, 140324961775616, 140324970168319, -+STORE, 140324953382912, 140324961775615, -+SNULL, 140324961779711, 140324970168319, -+STORE, 140324961775616, 140324961779711, -+STORE, 140324961779712, 140324970168319, -+SNULL, 140325079212031, 140325104386047, -+STORE, 140325079207936, 140325079212031, -+STORE, 140325079212032, 140325104386047, -+SNULL, 140325221818368, 140325230211071, -+STORE, 140325230211072, 140325238603775, -+STORE, 140325221818368, 140325230211071, -+SNULL, 140325230215167, 140325238603775, -+STORE, 140325230211072, 140325230215167, -+STORE, 140325230215168, 140325238603775, -+SNULL, 140325356036096, 140325364428799, -+STORE, 140325364428800, 140325372821503, -+STORE, 140325356036096, 140325364428799, -+SNULL, 140325364432895, 140325372821503, -+ }; -+ unsigned long set40[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140734309167104, 140737488351231, -+SNULL, 140734309171199, 140737488351231, -+STORE, 140734309167104, 140734309171199, -+STORE, 140734309036032, 140734309171199, -+STORE, 94270500081664, 94270502334463, -+SNULL, 94270500212735, 94270502334463, -+STORE, 94270500081664, 94270500212735, -+STORE, 94270500212736, 94270502334463, -+ERASE, 94270500212736, 94270502334463, -+STORE, 94270502305792, 94270502313983, -+STORE, 94270502313984, 94270502334463, -+STORE, 140321935110144, 140321937362943, -+SNULL, 140321935253503, 140321937362943, -+STORE, 140321935110144, 140321935253503, -+STORE, 140321935253504, 140321937362943, -+ERASE, 140321935253504, 140321937362943, -+STORE, 140321937350656, 140321937358847, -+STORE, 140321937358848, 140321937362943, -+STORE, 140734309625856, 140734309629951, -+STORE, 140734309613568, 140734309625855, -+STORE, 140321937321984, 140321937350655, -+STORE, 140321937313792, 140321937321983, -+STORE, 140321932894208, 140321935110143, -+SNULL, 140321932894208, 140321932992511, -+STORE, 140321932992512, 140321935110143, -+STORE, 140321932894208, 140321932992511, -+SNULL, 140321935085567, 140321935110143, -+STORE, 140321932992512, 140321935085567, -+STORE, 140321935085568, 140321935110143, -+SNULL, 140321935085568, 140321935093759, -+STORE, 140321935093760, 140321935110143, -+STORE, 140321935085568, 140321935093759, -+ERASE, 140321935085568, 140321935093759, -+STORE, 140321935085568, 140321935093759, -+ERASE, 140321935093760, 140321935110143, -+STORE, 140321935093760, 140321935110143, -+STORE, 140321929097216, 140321932894207, -+SNULL, 140321929097216, 140321930756095, -+STORE, 140321930756096, 140321932894207, -+STORE, 140321929097216, 140321930756095, -+SNULL, 140321932853247, 140321932894207, -+STORE, 140321930756096, 140321932853247, -+STORE, 140321932853248, 140321932894207, -+SNULL, 140321932853248, 140321932877823, -+STORE, 140321932877824, 140321932894207, -+STORE, 140321932853248, 140321932877823, -+ERASE, 140321932853248, 140321932877823, -+STORE, 140321932853248, 140321932877823, -+ERASE, 140321932877824, 140321932894207, -+STORE, 140321932877824, 140321932894207, -+STORE, 140321937305600, 140321937321983, -+SNULL, 140321932869631, 140321932877823, -+STORE, 140321932853248, 140321932869631, -+STORE, 140321932869632, 140321932877823, -+SNULL, 140321935089663, 140321935093759, -+STORE, 140321935085568, 140321935089663, -+STORE, 140321935089664, 140321935093759, -+SNULL, 94270502309887, 94270502313983, -+STORE, 94270502305792, 94270502309887, -+STORE, 94270502309888, 94270502313983, -+SNULL, 140321937354751, 140321937358847, -+STORE, 140321937350656, 140321937354751, -+STORE, 140321937354752, 140321937358847, -+ERASE, 140321937321984, 140321937350655, -+STORE, 94270507364352, 94270507499519, -+STORE, 140321920704512, 140321929097215, -+SNULL, 140321920708607, 140321929097215, -+STORE, 140321920704512, 140321920708607, -+STORE, 140321920708608, 140321929097215, -+STORE, 140321912311808, 140321920704511, -+STORE, 140321778094080, 140321912311807, -+SNULL, 140321778094080, 140321816051711, -+STORE, 140321816051712, 140321912311807, -+STORE, 140321778094080, 140321816051711, -+ERASE, 140321778094080, 140321816051711, -+SNULL, 140321883160575, 140321912311807, -+STORE, 140321816051712, 140321883160575, -+STORE, 140321883160576, 140321912311807, -+ERASE, 140321883160576, 140321912311807, -+SNULL, 140321816186879, 140321883160575, -+STORE, 140321816051712, 140321816186879, -+STORE, 140321816186880, 140321883160575, -+SNULL, 140321912315903, 140321920704511, -+STORE, 140321912311808, 140321912315903, -+STORE, 140321912315904, 140321920704511, -+STORE, 140321903919104, 140321912311807, -+SNULL, 140321903923199, 140321912311807, -+STORE, 140321903919104, 140321903923199, -+STORE, 140321903923200, 140321912311807, -+STORE, 140321895526400, 140321903919103, -+SNULL, 140321895530495, 140321903919103, -+STORE, 140321895526400, 140321895530495, -+STORE, 140321895530496, 140321903919103, -+STORE, 140321887133696, 140321895526399, -+SNULL, 140321887137791, 140321895526399, -+STORE, 140321887133696, 140321887137791, -+STORE, 140321887137792, 140321895526399, -+STORE, 140321807659008, 140321816051711, -+STORE, 140321673441280, 140321807659007, -+SNULL, 140321673441280, 140321681833983, -+STORE, 140321681833984, 140321807659007, -+STORE, 140321673441280, 140321681833983, -+ERASE, 140321673441280, 140321681833983, -+SNULL, 140321748942847, 140321807659007, -+STORE, 140321681833984, 140321748942847, -+STORE, 140321748942848, 140321807659007, -+ERASE, 140321748942848, 140321807659007, -+STORE, 140321799266304, 140321816051711, -+STORE, 140321790873600, 140321816051711, -+STORE, 140321782480896, 140321816051711, -+STORE, 140321547616256, 140321748942847, -+SNULL, 140321614725119, 140321748942847, -+STORE, 140321547616256, 140321614725119, -+STORE, 140321614725120, 140321748942847, -+SNULL, 140321614725120, 140321681833983, -+STORE, 140321681833984, 140321748942847, -+STORE, 140321614725120, 140321681833983, -+ERASE, 140321614725120, 140321681833983, -+SNULL, 140321681969151, 140321748942847, -+STORE, 140321681833984, 140321681969151, -+STORE, 140321681969152, 140321748942847, -+STORE, 140321547616256, 140321681833983, -+SNULL, 140321547616256, 140321614725119, -+STORE, 140321614725120, 140321681833983, -+STORE, 140321547616256, 140321614725119, -+SNULL, 140321614860287, 140321681833983, -+STORE, 140321614725120, 140321614860287, -+STORE, 140321614860288, 140321681833983, -+SNULL, 140321547751423, 140321614725119, -+STORE, 140321547616256, 140321547751423, -+STORE, 140321547751424, 140321614725119, -+STORE, 140321480507392, 140321547616255, -+SNULL, 140321782480896, 140321799266303, -+STORE, 140321799266304, 140321816051711, -+STORE, 140321782480896, 140321799266303, -+SNULL, 140321799270399, 140321816051711, -+STORE, 140321799266304, 140321799270399, -+STORE, 140321799270400, 140321816051711, -+STORE, 140321774088192, 140321799266303, -+SNULL, 140321774088192, 140321790873599, -+STORE, 140321790873600, 140321799266303, -+STORE, 140321774088192, 140321790873599, -+SNULL, 140321790877695, 140321799266303, -+STORE, 140321790873600, 140321790877695, -+STORE, 140321790877696, 140321799266303, -+SNULL, 140321480642559, 140321547616255, -+STORE, 140321480507392, 140321480642559, -+STORE, 140321480642560, 140321547616255, -+SNULL, 140321774088192, 140321782480895, -+STORE, 140321782480896, 140321790873599, -+STORE, 140321774088192, 140321782480895, -+SNULL, 140321782484991, 140321790873599, -+STORE, 140321782480896, 140321782484991, -+STORE, 140321782484992, 140321790873599, -+SNULL, 140321799270400, 140321807659007, -+STORE, 140321807659008, 140321816051711, -+STORE, 140321799270400, 140321807659007, -+SNULL, 140321807663103, 140321816051711, -+STORE, 140321807659008, 140321807663103, -+STORE, 140321807663104, 140321816051711, -+STORE, 140321765695488, 140321782480895, -+STORE, 140321757302784, 140321782480895, -+SNULL, 140321757306879, 140321782480895, -+STORE, 140321757302784, 140321757306879, -+STORE, 140321757306880, 140321782480895, -+STORE, 140321472114688, 140321480507391, -+STORE, 140321463721984, 140321480507391, -+SNULL, 140321463726079, 140321480507391, -+STORE, 140321463721984, 140321463726079, -+STORE, 140321463726080, 140321480507391, -+SNULL, 140321757306880, 140321774088191, -+STORE, 140321774088192, 140321782480895, -+STORE, 140321757306880, 140321774088191, -+SNULL, 140321774092287, 140321782480895, -+STORE, 140321774088192, 140321774092287, -+STORE, 140321774092288, 140321782480895, -+SNULL, 140321463726080, 140321472114687, -+STORE, 140321472114688, 140321480507391, -+STORE, 140321463726080, 140321472114687, -+SNULL, 140321472118783, 140321480507391, -+STORE, 140321472114688, 140321472118783, -+STORE, 140321472118784, 140321480507391, -+SNULL, 140321757306880, 140321765695487, -+STORE, 140321765695488, 140321774088191, -+STORE, 140321757306880, 140321765695487, -+SNULL, 140321765699583, 140321774088191, -+STORE, 140321765695488, 140321765699583, -+STORE, 140321765699584, 140321774088191, -+STORE, 140321455329280, 140321463721983, -+SNULL, 140321455333375, 140321463721983, -+STORE, 140321455329280, 140321455333375, -+STORE, 140321455333376, 140321463721983, -+STORE, 140321446936576, 140321455329279, -+STORE, 140321438543872, 140321455329279, -+STORE, 140321430151168, 140321455329279, -+SNULL, 140321430155263, 140321455329279, -+STORE, 140321430151168, 140321430155263, -+STORE, 140321430155264, 140321455329279, -+SNULL, 140321430155264, 140321446936575, -+STORE, 140321446936576, 140321455329279, -+STORE, 140321430155264, 140321446936575, -+SNULL, 140321446940671, 140321455329279, -+STORE, 140321446936576, 140321446940671, -+STORE, 140321446940672, 140321455329279, -+SNULL, 140321430155264, 140321438543871, -+STORE, 140321438543872, 140321446936575, -+STORE, 140321430155264, 140321438543871, -+SNULL, 140321438547967, 140321446936575, -+STORE, 140321438543872, 140321438547967, -+STORE, 140321438547968, 140321446936575, -+STORE, 140321421758464, 140321430151167, -+SNULL, 140321421762559, 140321430151167, -+STORE, 140321421758464, 140321421762559, -+STORE, 140321421762560, 140321430151167, -+STORE, 140321413365760, 140321421758463, -+SNULL, 140321413369855, 140321421758463, -+STORE, 140321413365760, 140321413369855, -+STORE, 140321413369856, 140321421758463, -+STORE, 140321404973056, 140321413365759, -+SNULL, 140321404977151, 140321413365759, -+STORE, 140321404973056, 140321404977151, -+STORE, 140321404977152, 140321413365759, -+STORE, 140321396580352, 140321404973055, -+STORE, 140321388187648, 140321404973055, -+STORE, 140321253969920, 140321388187647, -+SNULL, 140321253969920, 140321279180799, -+STORE, 140321279180800, 140321388187647, -+STORE, 140321253969920, 140321279180799, -+ERASE, 140321253969920, 140321279180799, -+SNULL, 140321346289663, 140321388187647, -+STORE, 140321279180800, 140321346289663, -+STORE, 140321346289664, 140321388187647, -+ERASE, 140321346289664, 140321388187647, -+STORE, 140321144963072, 140321346289663, -+STORE, 140321379794944, 140321404973055, -+STORE, 140321371402240, 140321404973055, -+STORE, 140321010745344, 140321346289663, -+STORE, 140321363009536, 140321404973055, -+SNULL, 140321077854207, 140321346289663, -+STORE, 140321010745344, 140321077854207, -+STORE, 140321077854208, 140321346289663, -+SNULL, 140321077854208, 140321144963071, -+STORE, 140321144963072, 140321346289663, -+STORE, 140321077854208, 140321144963071, -+ERASE, 140321077854208, 140321144963071, -+STORE, 140321354616832, 140321404973055, -+STORE, 140321136570368, 140321144963071, -+STORE, 140320943636480, 140321077854207, -+STORE, 140320876527616, 140321077854207, -+STORE, 140321128177664, 140321144963071, -+SNULL, 140320876662783, 140321077854207, -+STORE, 140320876527616, 140320876662783, -+STORE, 140320876662784, 140321077854207, -+STORE, 140321119784960, 140321144963071, -+STORE, 140321111392256, 140321144963071, -+STORE, 140320742309888, 140320876527615, -+STORE, 140321102999552, 140321144963071, -+STORE, 140320608092160, 140320876527615, -+SNULL, 140320675201023, 140320876527615, -+STORE, 140320608092160, 140320675201023, -+STORE, 140320675201024, 140320876527615, -+SNULL, 140320675201024, 140320742309887, -+STORE, 140320742309888, 140320876527615, -+STORE, 140320675201024, 140320742309887, -+ERASE, 140320675201024, 140320742309887, -+STORE, 140321094606848, 140321144963071, -+STORE, 140321086214144, 140321144963071, -+STORE, 140320608092160, 140320876527615, -+SNULL, 140320608092160, 140320675201023, -+STORE, 140320675201024, 140320876527615, -+STORE, 140320608092160, 140320675201023, -+SNULL, 140320675336191, 140320876527615, -+STORE, 140320675201024, 140320675336191, -+STORE, 140320675336192, 140320876527615, -+STORE, 140320599699456, 140320608092159, -+STORE, 140320591306752, 140320608092159, -+STORE, 140320457089024, 140320591306751, -+STORE, 140320448696320, 140320457089023, -+STORE, 140320314478592, 140320448696319, -+SNULL, 140321144963072, 140321279180799, -+STORE, 140321279180800, 140321346289663, -+STORE, 140321144963072, 140321279180799, -+SNULL, 140321279315967, 140321346289663, -+STORE, 140321279180800, 140321279315967, -+STORE, 140321279315968, 140321346289663, -+SNULL, 140321086214144, 140321136570367, -+STORE, 140321136570368, 140321144963071, -+STORE, 140321086214144, 140321136570367, -+SNULL, 140321136574463, 140321144963071, -+STORE, 140321136570368, 140321136574463, -+STORE, 140321136574464, 140321144963071, -+SNULL, 140321212071935, 140321279180799, -+STORE, 140321144963072, 140321212071935, -+STORE, 140321212071936, 140321279180799, -+ERASE, 140321212071936, 140321279180799, -+SNULL, 140321145098239, 140321212071935, -+STORE, 140321144963072, 140321145098239, -+STORE, 140321145098240, 140321212071935, -+SNULL, 140320876662784, 140321010745343, -+STORE, 140321010745344, 140321077854207, -+STORE, 140320876662784, 140321010745343, -+SNULL, 140321010880511, 140321077854207, -+STORE, 140321010745344, 140321010880511, -+STORE, 140321010880512, 140321077854207, -+SNULL, 140321354616832, 140321379794943, -+STORE, 140321379794944, 140321404973055, -+STORE, 140321354616832, 140321379794943, -+SNULL, 140321379799039, 140321404973055, -+STORE, 140321379794944, 140321379799039, -+STORE, 140321379799040, 140321404973055, -+SNULL, 140320876662784, 140320943636479, -+STORE, 140320943636480, 140321010745343, -+STORE, 140320876662784, 140320943636479, -+SNULL, 140320943771647, 140321010745343, -+STORE, 140320943636480, 140320943771647, -+STORE, 140320943771648, 140321010745343, -+SNULL, 140320809418751, 140320876527615, -+STORE, 140320675336192, 140320809418751, -+STORE, 140320809418752, 140320876527615, -+ERASE, 140320809418752, 140320876527615, -+SNULL, 140320675336192, 140320742309887, -+STORE, 140320742309888, 140320809418751, -+STORE, 140320675336192, 140320742309887, -+SNULL, 140320742445055, 140320809418751, -+STORE, 140320742309888, 140320742445055, -+STORE, 140320742445056, 140320809418751, -+SNULL, 140320608227327, 140320675201023, -+STORE, 140320608092160, 140320608227327, -+STORE, 140320608227328, 140320675201023, -+SNULL, 140320457089024, 140320473874431, -+STORE, 140320473874432, 140320591306751, -+STORE, 140320457089024, 140320473874431, -+ERASE, 140320457089024, 140320473874431, -+SNULL, 140320540983295, 140320591306751, -+STORE, 140320473874432, 140320540983295, -+STORE, 140320540983296, 140320591306751, -+ERASE, 140320540983296, 140320591306751, -+SNULL, 140320314478592, 140320339656703, -+STORE, 140320339656704, 140320448696319, -+STORE, 140320314478592, 140320339656703, -+ERASE, 140320314478592, 140320339656703, -+SNULL, 140321086214144, 140321128177663, -+STORE, 140321128177664, 140321136570367, -+STORE, 140321086214144, 140321128177663, -+SNULL, 140321128181759, 140321136570367, -+STORE, 140321128177664, 140321128181759, -+STORE, 140321128181760, 140321136570367, -+SNULL, 140321354616832, 140321371402239, -+STORE, 140321371402240, 140321379794943, -+STORE, 140321354616832, 140321371402239, -+SNULL, 140321371406335, 140321379794943, -+STORE, 140321371402240, 140321371406335, -+STORE, 140321371406336, 140321379794943, -+SNULL, 140320591310847, 140320608092159, -+STORE, 140320591306752, 140320591310847, -+STORE, 140320591310848, 140320608092159, -+SNULL, 140321354616832, 140321363009535, -+STORE, 140321363009536, 140321371402239, -+STORE, 140321354616832, 140321363009535, -+SNULL, 140321363013631, 140321371402239, -+STORE, 140321363009536, 140321363013631, -+STORE, 140321363013632, 140321371402239, -+SNULL, 140321086214144, 140321119784959, -+STORE, 140321119784960, 140321128177663, -+STORE, 140321086214144, 140321119784959, -+SNULL, 140321119789055, 140321128177663, -+STORE, 140321119784960, 140321119789055, -+STORE, 140321119789056, 140321128177663, -+SNULL, 140321086218239, 140321119784959, -+STORE, 140321086214144, 140321086218239, -+STORE, 140321086218240, 140321119784959, -+SNULL, 140321086218240, 140321094606847, -+STORE, 140321094606848, 140321119784959, -+STORE, 140321086218240, 140321094606847, -+SNULL, 140321094610943, 140321119784959, -+STORE, 140321094606848, 140321094610943, -+STORE, 140321094610944, 140321119784959, -+SNULL, 140320474009599, 140320540983295, -+STORE, 140320473874432, 140320474009599, -+STORE, 140320474009600, 140320540983295, -+SNULL, 140320406765567, 140320448696319, -+STORE, 140320339656704, 140320406765567, -+STORE, 140320406765568, 140320448696319, -+ERASE, 140320406765568, 140320448696319, -+SNULL, 140320339791871, 140320406765567, -+STORE, 140320339656704, 140320339791871, -+STORE, 140320339791872, 140320406765567, -+STORE, 140321270788096, 140321279180799, -+STORE, 140321262395392, 140321279180799, -+STORE, 140321254002688, 140321279180799, -+SNULL, 140321254002688, 140321262395391, -+STORE, 140321262395392, 140321279180799, -+STORE, 140321254002688, 140321262395391, -+SNULL, 140321262399487, 140321279180799, -+STORE, 140321262395392, 140321262399487, -+STORE, 140321262399488, 140321279180799, -+STORE, 140321245609984, 140321262395391, -+STORE, 140321237217280, 140321262395391, -+SNULL, 140321237217280, 140321245609983, -+STORE, 140321245609984, 140321262395391, -+STORE, 140321237217280, 140321245609983, -+SNULL, 140321245614079, 140321262395391, -+STORE, 140321245609984, 140321245614079, -+STORE, 140321245614080, 140321262395391, -+SNULL, 140321379799040, 140321388187647, -+STORE, 140321388187648, 140321404973055, -+STORE, 140321379799040, 140321388187647, -+SNULL, 140321388191743, 140321404973055, -+STORE, 140321388187648, 140321388191743, -+STORE, 140321388191744, 140321404973055, -+SNULL, 140321354620927, 140321363009535, -+STORE, 140321354616832, 140321354620927, -+STORE, 140321354620928, 140321363009535, -+SNULL, 140321388191744, 140321396580351, -+STORE, 140321396580352, 140321404973055, -+STORE, 140321388191744, 140321396580351, -+SNULL, 140321396584447, 140321404973055, -+STORE, 140321396580352, 140321396584447, -+STORE, 140321396584448, 140321404973055, -+SNULL, 140321094610944, 140321111392255, -+STORE, 140321111392256, 140321119784959, -+STORE, 140321094610944, 140321111392255, -+SNULL, 140321111396351, 140321119784959, -+STORE, 140321111392256, 140321111396351, -+STORE, 140321111396352, 140321119784959, -+STORE, 140321228824576, 140321245609983, -+SNULL, 140321094610944, 140321102999551, -+STORE, 140321102999552, 140321111392255, -+STORE, 140321094610944, 140321102999551, -+SNULL, 140321103003647, 140321111392255, -+STORE, 140321102999552, 140321103003647, -+STORE, 140321103003648, 140321111392255, -+STORE, 140321220431872, 140321245609983, -+SNULL, 140321220435967, 140321245609983, -+STORE, 140321220431872, 140321220435967, -+STORE, 140321220435968, 140321245609983, -+STORE, 140320868134912, 140320876527615, -+SNULL, 140320868139007, 140320876527615, -+STORE, 140320868134912, 140320868139007, -+STORE, 140320868139008, 140320876527615, -+SNULL, 140320591310848, 140320599699455, -+STORE, 140320599699456, 140320608092159, -+STORE, 140320591310848, 140320599699455, -+SNULL, 140320599703551, 140320608092159, -+STORE, 140320599699456, 140320599703551, -+STORE, 140320599703552, 140320608092159, -+STORE, 140320859742208, 140320868134911, -+SNULL, 140321262399488, 140321270788095, -+STORE, 140321270788096, 140321279180799, -+STORE, 140321262399488, 140321270788095, -+SNULL, 140321270792191, 140321279180799, -+STORE, 140321270788096, 140321270792191, -+STORE, 140321270792192, 140321279180799, -+STORE, 140320851349504, 140320868134911, -+STORE, 140320842956800, 140320868134911, -+STORE, 140320834564096, 140320868134911, -+STORE, 140320826171392, 140320868134911, -+SNULL, 140320826171392, 140320834564095, -+STORE, 140320834564096, 140320868134911, -+STORE, 140320826171392, 140320834564095, -+SNULL, 140320834568191, 140320868134911, -+STORE, 140320834564096, 140320834568191, -+STORE, 140320834568192, 140320868134911, -+SNULL, 140321220435968, 140321228824575, -+STORE, 140321228824576, 140321245609983, -+STORE, 140321220435968, 140321228824575, -+SNULL, 140321228828671, 140321245609983, -+STORE, 140321228824576, 140321228828671, -+STORE, 140321228828672, 140321245609983, -+STORE, 140320817778688, 140320834564095, -+SNULL, 140320817782783, 140320834564095, -+STORE, 140320817778688, 140320817782783, -+STORE, 140320817782784, 140320834564095, -+STORE, 140320582914048, 140320591306751, -+SNULL, 140321228828672, 140321237217279, -+STORE, 140321237217280, 140321245609983, -+STORE, 140321228828672, 140321237217279, -+SNULL, 140321237221375, 140321245609983, -+STORE, 140321237217280, 140321237221375, -+STORE, 140321237221376, 140321245609983, -+SNULL, 140320448700415, 140320457089023, -+STORE, 140320448696320, 140320448700415, -+STORE, 140320448700416, 140320457089023, -+SNULL, 140321245614080, 140321254002687, -+STORE, 140321254002688, 140321262395391, -+STORE, 140321245614080, 140321254002687, -+SNULL, 140321254006783, 140321262395391, -+STORE, 140321254002688, 140321254006783, -+STORE, 140321254006784, 140321262395391, -+STORE, 140320574521344, 140320591306751, -+SNULL, 140320574525439, 140320591306751, -+STORE, 140320574521344, 140320574525439, -+STORE, 140320574525440, 140320591306751, -+STORE, 140320566128640, 140320574521343, -+SNULL, 140320566132735, 140320574521343, -+STORE, 140320566128640, 140320566132735, -+STORE, 140320566132736, 140320574521343, -+SNULL, 140320574525440, 140320582914047, -+STORE, 140320582914048, 140320591306751, -+STORE, 140320574525440, 140320582914047, -+SNULL, 140320582918143, 140320591306751, -+STORE, 140320582914048, 140320582918143, -+STORE, 140320582918144, 140320591306751, -+STORE, 140320557735936, 140320566128639, -+SNULL, 140320557740031, 140320566128639, -+STORE, 140320557735936, 140320557740031, -+STORE, 140320557740032, 140320566128639, -+STORE, 140320549343232, 140320557735935, -+STORE, 140320465481728, 140320473874431, -+STORE, 140320448700416, 140320473874431, -+SNULL, 140320834568192, 140320859742207, -+STORE, 140320859742208, 140320868134911, -+STORE, 140320834568192, 140320859742207, -+SNULL, 140320859746303, 140320868134911, -+STORE, 140320859742208, 140320859746303, -+STORE, 140320859746304, 140320868134911, -+STORE, 140320440303616, 140320448696319, -+STORE, 140320431910912, 140320448696319, -+SNULL, 140320834568192, 140320851349503, -+STORE, 140320851349504, 140320859742207, -+STORE, 140320834568192, 140320851349503, -+SNULL, 140320851353599, 140320859742207, -+STORE, 140320851349504, 140320851353599, -+STORE, 140320851353600, 140320859742207, -+SNULL, 140320817782784, 140320826171391, -+STORE, 140320826171392, 140320834564095, -+STORE, 140320817782784, 140320826171391, -+SNULL, 140320826175487, 140320834564095, -+STORE, 140320826171392, 140320826175487, -+STORE, 140320826175488, 140320834564095, -+SNULL, 140320834568192, 140320842956799, -+STORE, 140320842956800, 140320851349503, -+STORE, 140320834568192, 140320842956799, -+SNULL, 140320842960895, 140320851349503, -+STORE, 140320842956800, 140320842960895, -+STORE, 140320842960896, 140320851349503, -+STORE, 140320423518208, 140320448696319, -+SNULL, 140320423522303, 140320448696319, -+STORE, 140320423518208, 140320423522303, -+STORE, 140320423522304, 140320448696319, -+STORE, 140320415125504, 140320423518207, -+STORE, 140320331264000, 140320339656703, -+STORE, 140320322871296, 140320339656703, -+STORE, 140320314478592, 140320339656703, -+SNULL, 140320314482687, 140320339656703, -+STORE, 140320314478592, 140320314482687, -+STORE, 140320314482688, 140320339656703, -+STORE, 140320306085888, 140320314478591, -+SNULL, 140320306089983, 140320314478591, -+STORE, 140320306085888, 140320306089983, -+STORE, 140320306089984, 140320314478591, -+STORE, 140320297693184, 140320306085887, -+SNULL, 140320297697279, 140320306085887, -+STORE, 140320297693184, 140320297697279, -+STORE, 140320297697280, 140320306085887, -+STORE, 140320289300480, 140320297693183, -+STORE, 140320280907776, 140320297693183, -+SNULL, 140320280911871, 140320297693183, -+STORE, 140320280907776, 140320280911871, -+STORE, 140320280911872, 140320297693183, -+SNULL, 140320423522304, 140320431910911, -+STORE, 140320431910912, 140320448696319, -+STORE, 140320423522304, 140320431910911, -+SNULL, 140320431915007, 140320448696319, -+STORE, 140320431910912, 140320431915007, -+STORE, 140320431915008, 140320448696319, -+SNULL, 140320549347327, 140320557735935, -+STORE, 140320549343232, 140320549347327, -+STORE, 140320549347328, 140320557735935, -+STORE, 140320272515072, 140320280907775, -+SNULL, 140320448700416, 140320457089023, -+STORE, 140320457089024, 140320473874431, -+STORE, 140320448700416, 140320457089023, -+SNULL, 140320457093119, 140320473874431, -+STORE, 140320457089024, 140320457093119, -+STORE, 140320457093120, 140320473874431, -+STORE, 140320264122368, 140320280907775, -+SNULL, 140320457093120, 140320465481727, -+STORE, 140320465481728, 140320473874431, -+STORE, 140320457093120, 140320465481727, -+SNULL, 140320465485823, 140320473874431, -+STORE, 140320465481728, 140320465485823, -+STORE, 140320465485824, 140320473874431, -+SNULL, 140320431915008, 140320440303615, -+STORE, 140320440303616, 140320448696319, -+STORE, 140320431915008, 140320440303615, -+SNULL, 140320440307711, 140320448696319, -+STORE, 140320440303616, 140320440307711, -+STORE, 140320440307712, 140320448696319, -+STORE, 140320255729664, 140320280907775, -+STORE, 140320247336960, 140320280907775, -+SNULL, 140320247341055, 140320280907775, -+STORE, 140320247336960, 140320247341055, -+STORE, 140320247341056, 140320280907775, -+STORE, 140320238944256, 140320247336959, -+STORE, 140320230551552, 140320247336959, -+SNULL, 140320230551552, 140320238944255, -+STORE, 140320238944256, 140320247336959, -+STORE, 140320230551552, 140320238944255, -+SNULL, 140320238948351, 140320247336959, -+STORE, 140320238944256, 140320238948351, -+STORE, 140320238948352, 140320247336959, -+SNULL, 140320314482688, 140320331263999, -+STORE, 140320331264000, 140320339656703, -+STORE, 140320314482688, 140320331263999, -+SNULL, 140320331268095, 140320339656703, -+STORE, 140320331264000, 140320331268095, -+STORE, 140320331268096, 140320339656703, -+SNULL, 140320280911872, 140320289300479, -+STORE, 140320289300480, 140320297693183, -+STORE, 140320280911872, 140320289300479, -+SNULL, 140320289304575, 140320297693183, -+STORE, 140320289300480, 140320289304575, -+STORE, 140320289304576, 140320297693183, -+SNULL, 140320415129599, 140320423518207, -+STORE, 140320415125504, 140320415129599, -+STORE, 140320415129600, 140320423518207, -+STORE, 140320222158848, 140320238944255, -+STORE, 140320213766144, 140320238944255, -+STORE, 140320205373440, 140320238944255, -+SNULL, 140320205377535, 140320238944255, -+STORE, 140320205373440, 140320205377535, -+STORE, 140320205377536, 140320238944255, -+SNULL, 140320314482688, 140320322871295, -+STORE, 140320322871296, 140320331263999, -+STORE, 140320314482688, 140320322871295, -+SNULL, 140320322875391, 140320331263999, -+STORE, 140320322871296, 140320322875391, -+STORE, 140320322875392, 140320331263999, -+SNULL, 140320247341056, 140320272515071, -+STORE, 140320272515072, 140320280907775, -+STORE, 140320247341056, 140320272515071, -+SNULL, 140320272519167, 140320280907775, -+STORE, 140320272515072, 140320272519167, -+STORE, 140320272519168, 140320280907775, -+SNULL, 140320247341056, 140320264122367, -+STORE, 140320264122368, 140320272515071, -+STORE, 140320247341056, 140320264122367, -+SNULL, 140320264126463, 140320272515071, -+STORE, 140320264122368, 140320264126463, -+STORE, 140320264126464, 140320272515071, -+SNULL, 140320205377536, 140320230551551, -+STORE, 140320230551552, 140320238944255, -+STORE, 140320205377536, 140320230551551, -+SNULL, 140320230555647, 140320238944255, -+STORE, 140320230551552, 140320230555647, -+STORE, 140320230555648, 140320238944255, -+STORE, 140320196980736, 140320205373439, -+SNULL, 140320196984831, 140320205373439, -+STORE, 140320196980736, 140320196984831, -+STORE, 140320196984832, 140320205373439, -+STORE, 140320188588032, 140320196980735, -+SNULL, 140320247341056, 140320255729663, -+STORE, 140320255729664, 140320264122367, -+STORE, 140320247341056, 140320255729663, -+SNULL, 140320255733759, 140320264122367, -+STORE, 140320255729664, 140320255733759, -+STORE, 140320255733760, 140320264122367, -+STORE, 140320180195328, 140320196980735, -+SNULL, 140320180199423, 140320196980735, -+STORE, 140320180195328, 140320180199423, -+STORE, 140320180199424, 140320196980735, -+STORE, 140320171802624, 140320180195327, -+STORE, 140320163409920, 140320180195327, -+SNULL, 140320163414015, 140320180195327, -+STORE, 140320163409920, 140320163414015, -+STORE, 140320163414016, 140320180195327, -+SNULL, 140320205377536, 140320222158847, -+STORE, 140320222158848, 140320230551551, -+STORE, 140320205377536, 140320222158847, -+SNULL, 140320222162943, 140320230551551, -+STORE, 140320222158848, 140320222162943, -+STORE, 140320222162944, 140320230551551, -+SNULL, 140320205377536, 140320213766143, -+STORE, 140320213766144, 140320222158847, -+STORE, 140320205377536, 140320213766143, -+SNULL, 140320213770239, 140320222158847, -+STORE, 140320213766144, 140320213770239, -+STORE, 140320213770240, 140320222158847, -+STORE, 140320155017216, 140320163409919, -+SNULL, 140320180199424, 140320188588031, -+STORE, 140320188588032, 140320196980735, -+STORE, 140320180199424, 140320188588031, -+SNULL, 140320188592127, 140320196980735, -+STORE, 140320188588032, 140320188592127, -+STORE, 140320188592128, 140320196980735, -+SNULL, 140320155021311, 140320163409919, -+STORE, 140320155017216, 140320155021311, -+STORE, 140320155021312, 140320163409919, -+SNULL, 140320163414016, 140320171802623, -+STORE, 140320171802624, 140320180195327, -+STORE, 140320163414016, 140320171802623, -+SNULL, 140320171806719, 140320180195327, -+STORE, 140320171802624, 140320171806719, -+STORE, 140320171806720, 140320180195327, -+STORE, 140320146624512, 140320155017215, -+SNULL, 140320146628607, 140320155017215, -+STORE, 140320146624512, 140320146628607, -+STORE, 140320146628608, 140320155017215, -+STORE, 140321937321984, 140321937350655, -+STORE, 140321884942336, 140321887133695, -+SNULL, 140321884942336, 140321885032447, -+STORE, 140321885032448, 140321887133695, -+STORE, 140321884942336, 140321885032447, -+SNULL, 140321887125503, 140321887133695, -+STORE, 140321885032448, 140321887125503, -+STORE, 140321887125504, 140321887133695, -+ERASE, 140321887125504, 140321887133695, -+STORE, 140321887125504, 140321887133695, -+SNULL, 140321887129599, 140321887133695, -+STORE, 140321887125504, 140321887129599, -+STORE, 140321887129600, 140321887133695, -+ERASE, 140321937321984, 140321937350655, -+ERASE, 140321086214144, 140321086218239, -+ERASE, 140321086218240, 140321094606847, -+ERASE, 140321119784960, 140321119789055, -+ERASE, 140321119789056, 140321128177663, -+ERASE, 140321245609984, 140321245614079, -+ERASE, 140321245614080, 140321254002687, -+ERASE, 140320574521344, 140320574525439, -+ERASE, 140320574525440, 140320582914047, -+ERASE, 140320297693184, 140320297697279, -+ERASE, 140320297697280, 140320306085887, -+ERASE, 140321354616832, 140321354620927, -+ERASE, 140321354620928, 140321363009535, -+ERASE, 140320834564096, 140320834568191, -+ERASE, 140320834568192, 140320842956799, -+ERASE, 140320591306752, 140320591310847, -+ERASE, 140320591310848, 140320599699455, -+ERASE, 140321136570368, 140321136574463, -+ERASE, 140321136574464, 140321144963071, -+ERASE, 140321237217280, 140321237221375, -+ERASE, 140321237221376, 140321245609983, -+ERASE, 140321363009536, 140321363013631, -+ERASE, 140321363013632, 140321371402239, -+ERASE, 140320599699456, 140320599703551, -+ERASE, 140320599703552, 140320608092159, -+ERASE, 140321396580352, 140321396584447, -+ERASE, 140321396584448, 140321404973055, -+ERASE, 140320566128640, 140320566132735, -+ERASE, 140320566132736, 140320574521343, -+ERASE, 140321094606848, 140321094610943, -+ERASE, 140321094610944, 140321102999551, -+ERASE, 140320582914048, 140320582918143, -+ERASE, 140320582918144, 140320591306751, -+ERASE, 140320289300480, 140320289304575, -+ERASE, 140320289304576, 140320297693183, -+ERASE, 140320163409920, 140320163414015, -+ }; -+ unsigned long set41[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140728157171712, 140737488351231, -+SNULL, 140728157175807, 140737488351231, -+STORE, 140728157171712, 140728157175807, -+STORE, 140728157040640, 140728157175807, -+STORE, 94376106364928, 94376108613631, -+SNULL, 94376106487807, 94376108613631, -+STORE, 94376106364928, 94376106487807, -+STORE, 94376106487808, 94376108613631, -+SNULL, 94376106487808, 94376108613631, -+STORE, 94376108584960, 94376108593151, -+STORE, 94376108593152, 94376108613631, -+STORE, 140113496432640, 140113498685439, -+SNULL, 140113496575999, 140113498685439, -+STORE, 140113496432640, 140113496575999, -+STORE, 140113496576000, 140113498685439, -+SNULL, 140113496576000, 140113498685439, -+STORE, 140113498673152, 140113498681343, -+STORE, 140113498681344, 140113498685439, -+STORE, 140728157609984, 140728157618175, -+STORE, 140728157593600, 140728157609983, -+STORE, 140113498636288, 140113498673151, -+STORE, 140113498628096, 140113498636287, -+STORE, 140113492635648, 140113496432639, -+SNULL, 140113492635648, 140113494294527, -+STORE, 140113494294528, 140113496432639, -+STORE, 140113492635648, 140113494294527, -+SNULL, 140113496391679, 140113496432639, -+STORE, 140113494294528, 140113496391679, -+STORE, 140113496391680, 140113496432639, -+SNULL, 140113496391680, 140113496416255, -+STORE, 140113496416256, 140113496432639, -+STORE, 140113496391680, 140113496416255, -+SNULL, 140113496391680, 140113496416255, -+STORE, 140113496391680, 140113496416255, -+SNULL, 140113496416256, 140113496432639, -+STORE, 140113496416256, 140113496432639, -+SNULL, 140113496408063, 140113496416255, -+STORE, 140113496391680, 140113496408063, -+STORE, 140113496408064, 140113496416255, -+SNULL, 94376108589055, 94376108593151, -+STORE, 94376108584960, 94376108589055, -+STORE, 94376108589056, 94376108593151, -+SNULL, 140113498677247, 140113498681343, -+STORE, 140113498673152, 140113498677247, -+STORE, 140113498677248, 140113498681343, -+SNULL, 140113498636288, 140113498673151, -+STORE, 94376135090176, 94376135094271, -+STORE, 94376135090176, 94376135098367, -+STORE, 94376139288576, 94376139292671, -+STORE, 94376143482880, 94376143486975, -+STORE, 94376147677184, 94376147681279, -+STORE, 94376151871488, 94376151875583, -+STORE, 94376156065792, 94376156069887, -+STORE, 94376160260096, 94376160264191, -+STORE, 94376164454400, 94376164458495, -+STORE, 94376168648704, 94376168652799, -+STORE, 94376172843008, 94376172847103, -+STORE, 94376177037312, 94376177041407, -+STORE, 94376181231616, 94376181235711, -+STORE, 94376185425920, 94376185430015, -+STORE, 94376189620224, 94376189624319, -+STORE, 94376193814528, 94376193818623, -+STORE, 94376198008832, 94376198012927, -+STORE, 94376202203136, 94376202207231, -+STORE, 94376206397440, 94376206401535, -+STORE, 94376210591744, 94376210595839, -+STORE, 94376214786048, 94376214790143, -+STORE, 94376218980352, 94376218984447, -+STORE, 94376223174656, 94376223178751, -+STORE, 94376227368960, 94376227373055, -+STORE, 94376231563264, 94376231567359, -+STORE, 94376235757568, 94376235761663, -+STORE, 94376239951872, 94376239955967, -+STORE, 94376244146176, 94376244150271, -+STORE, 94376248340480, 94376248344575, -+STORE, 94376252534784, 94376252538879, -+STORE, 94376256729088, 94376256733183, -+STORE, 94376260923392, 94376260927487, -+STORE, 94376265117696, 94376265121791, -+STORE, 94376269312000, 94376269316095, -+STORE, 94376273506304, 94376273510399, -+STORE, 94376277700608, 94376277704703, -+STORE, 94376281894912, 94376281899007, -+STORE, 94376286089216, 94376286093311, -+STORE, 94376290283520, 94376290287615, -+STORE, 94376294477824, 94376294481919, -+STORE, 94376298672128, 94376298676223, -+STORE, 94376302866432, 94376302870527, -+STORE, 94376307060736, 94376307064831, -+STORE, 94376311255040, 94376311259135, -+STORE, 94376315449344, 94376315453439, -+STORE, 94376319643648, 94376319647743, -+STORE, 94376323837952, 94376323842047, -+STORE, 94376328032256, 94376328036351, -+STORE, 94376332226560, 94376332230655, -+STORE, 94376336420864, 94376336424959, -+STORE, 94376340615168, 94376340619263, -+STORE, 94376344809472, 94376344813567, -+STORE, 94376349003776, 94376349007871, -+STORE, 94376353198080, 94376353202175, -+STORE, 94376357392384, 94376357396479, -+STORE, 94376361586688, 94376361590783, -+STORE, 94376365780992, 94376365785087, -+STORE, 94376369975296, 94376369979391, -+STORE, 94376374169600, 94376374173695, -+STORE, 94376378363904, 94376378367999, -+STORE, 94376382558208, 94376382562303, -+STORE, 94376386752512, 94376386756607, -+STORE, 94376390946816, 94376390950911, -+STORE, 94376395141120, 94376395145215, -+STORE, 94376399335424, 94376399339519, -+STORE, 94376403529728, 94376403533823, -+STORE, 94376407724032, 94376407728127, -+STORE, 94376411918336, 94376411922431, -+STORE, 94376416112640, 94376416116735, -+STORE, 94376420306944, 94376420311039, -+STORE, 94376424501248, 94376424505343, -+STORE, 94376428695552, 94376428699647, -+STORE, 94376432889856, 94376432893951, -+STORE, 94376437084160, 94376437088255, -+STORE, 94376441278464, 94376441282559, -+STORE, 94376445472768, 94376445476863, -+STORE, 94376449667072, 94376449671167, -+STORE, 94376453861376, 94376453865471, -+STORE, 94376458055680, 94376458059775, -+STORE, 94376462249984, 94376462254079, -+STORE, 94376466444288, 94376466448383, -+STORE, 94376470638592, 94376470642687, -+STORE, 94376474832896, 94376474836991, -+STORE, 94376479027200, 94376479031295, -+STORE, 94376483221504, 94376483225599, -+STORE, 94376487415808, 94376487419903, -+STORE, 94376491610112, 94376491614207, -+STORE, 94376495804416, 94376495808511, -+STORE, 94376499998720, 94376500002815, -+STORE, 94376504193024, 94376504197119, -+STORE, 94376508387328, 94376508391423, -+STORE, 94376512581632, 94376512585727, -+STORE, 94376516775936, 94376516780031, -+STORE, 94376520970240, 94376520974335, -+STORE, 94376525164544, 94376525168639, -+STORE, 94376529358848, 94376529362943, -+STORE, 94376533553152, 94376533557247, -+STORE, 94376537747456, 94376537751551, -+STORE, 94376541941760, 94376541945855, -+STORE, 94376546136064, 94376546140159, -+STORE, 94376550330368, 94376550334463, -+STORE, 94376554524672, 94376554528767, -+STORE, 94376558718976, 94376558723071, -+STORE, 94376562913280, 94376562917375, -+STORE, 94376567107584, 94376567111679, -+STORE, 94376571301888, 94376571305983, -+STORE, 94376575496192, 94376575500287, -+STORE, 94376579690496, 94376579694591, -+STORE, 94376583884800, 94376583888895, -+STORE, 94376588079104, 94376588083199, -+STORE, 94376592273408, 94376592277503, -+STORE, 94376596467712, 94376596471807, -+STORE, 94376600662016, 94376600666111, -+STORE, 94376604856320, 94376604860415, -+STORE, 94376609050624, 94376609054719, -+STORE, 94376613244928, 94376613249023, -+STORE, 94376617439232, 94376617443327, -+STORE, 94376621633536, 94376621637631, -+STORE, 94376625827840, 94376625831935, -+STORE, 94376630022144, 94376630026239, -+STORE, 94376634216448, 94376634220543, -+STORE, 94376638410752, 94376638414847, -+STORE, 94376642605056, 94376642609151, -+STORE, 94376646799360, 94376646803455, -+STORE, 94376650993664, 94376650997759, -+STORE, 94376655187968, 94376655192063, -+STORE, 94376659382272, 94376659386367, -+STORE, 94376663576576, 94376663580671, -+STORE, 94376667770880, 94376667774975, -+STORE, 94376671965184, 94376671969279, -+STORE, 94376676159488, 94376676163583, -+STORE, 94376680353792, 94376680357887, -+STORE, 94376684548096, 94376684552191, -+STORE, 94376688742400, 94376688746495, -+STORE, 94376692936704, 94376692940799, -+STORE, 94376697131008, 94376697135103, -+STORE, 94376701325312, 94376701329407, -+STORE, 94376705519616, 94376705523711, -+STORE, 94376709713920, 94376709718015, -+STORE, 94376713908224, 94376713912319, -+STORE, 94376718102528, 94376718106623, -+STORE, 94376722296832, 94376722300927, -+STORE, 94376726491136, 94376726495231, -+STORE, 94376730685440, 94376730689535, -+STORE, 94376734879744, 94376734883839, -+STORE, 94376739074048, 94376739078143, -+STORE, 94376743268352, 94376743272447, -+STORE, 94376747462656, 94376747466751, -+STORE, 94376751656960, 94376751661055, -+STORE, 94376755851264, 94376755855359, -+STORE, 94376760045568, 94376760049663, -+STORE, 94376764239872, 94376764243967, -+STORE, 94376768434176, 94376768438271, -+STORE, 94376772628480, 94376772632575, -+STORE, 94376776822784, 94376776826879, -+STORE, 94376781017088, 94376781021183, -+STORE, 94376785211392, 94376785215487, -+STORE, 94376789405696, 94376789409791, -+STORE, 94376793600000, 94376793604095, -+STORE, 94376797794304, 94376797798399, -+STORE, 94376801988608, 94376801992703, -+STORE, 94376806182912, 94376806187007, -+STORE, 94376810377216, 94376810381311, -+STORE, 94376814571520, 94376814575615, -+STORE, 94376818765824, 94376818769919, -+STORE, 94376822960128, 94376822964223, -+STORE, 94376827154432, 94376827158527, -+STORE, 94376831348736, 94376831352831, -+STORE, 94376835543040, 94376835547135, -+STORE, 94376839737344, 94376839741439, -+STORE, 94376843931648, 94376843935743, -+STORE, 94376848125952, 94376848130047, -+STORE, 94376852320256, 94376852324351, -+STORE, 94376856514560, 94376856518655, -+STORE, 94376860708864, 94376860712959, -+STORE, 94376864903168, 94376864907263, -+STORE, 94376869097472, 94376869101567, -+STORE, 94376873291776, 94376873295871, -+STORE, 94376877486080, 94376877490175, -+STORE, 94376881680384, 94376881684479, -+STORE, 94376885874688, 94376885878783, -+STORE, 94376890068992, 94376890073087, -+STORE, 94376894263296, 94376894267391, -+STORE, 94376898457600, 94376898461695, -+STORE, 94376902651904, 94376902655999, -+STORE, 94376906846208, 94376906850303, -+STORE, 94376911040512, 94376911044607, -+STORE, 94376915234816, 94376915238911, -+STORE, 94376919429120, 94376919433215, -+STORE, 94376923623424, 94376923627519, -+STORE, 94376927817728, 94376927821823, -+STORE, 94376932012032, 94376932016127, -+STORE, 94376936206336, 94376936210431, -+STORE, 94376940400640, 94376940404735, -+STORE, 94376944594944, 94376944599039, -+STORE, 94376948789248, 94376948793343, -+STORE, 94376952983552, 94376952987647, -+STORE, 94376957177856, 94376957181951, -+STORE, 94376961372160, 94376961376255, -+STORE, 94376965566464, 94376965570559, -+STORE, 94376969760768, 94376969764863, -+STORE, 94376973955072, 94376973959167, -+STORE, 94376978149376, 94376978153471, -+STORE, 94376982343680, 94376982347775, -+STORE, 94376986537984, 94376986542079, -+STORE, 94376990732288, 94376990736383, -+STORE, 94376994926592, 94376994930687, -+STORE, 94376999120896, 94376999124991, -+STORE, 94377003315200, 94377003319295, -+STORE, 94377007509504, 94377007513599, -+STORE, 94377011703808, 94377011707903, -+STORE, 94377015898112, 94377015902207, -+STORE, 94377020092416, 94377020096511, -+STORE, 94377024286720, 94377024290815, -+STORE, 94377028481024, 94377028485119, -+STORE, 94377032675328, 94377032679423, -+STORE, 94377036869632, 94377036873727, -+STORE, 94377041063936, 94377041068031, -+STORE, 94377045258240, 94377045262335, -+STORE, 94377049452544, 94377049456639, -+STORE, 94377053646848, 94377053650943, -+STORE, 94377057841152, 94377057845247, -+STORE, 94377062035456, 94377062039551, -+STORE, 94377066229760, 94377066233855, -+STORE, 94377070424064, 94377070428159, -+STORE, 94377074618368, 94377074622463, -+STORE, 94377078812672, 94377078816767, -+STORE, 94377083006976, 94377083011071, -+STORE, 94377087201280, 94377087205375, -+STORE, 94377091395584, 94377091399679, -+STORE, 94377095589888, 94377095593983, -+STORE, 94377099784192, 94377099788287, -+STORE, 94377103978496, 94377103982591, -+STORE, 94377108172800, 94377108176895, -+STORE, 94377112367104, 94377112371199, -+STORE, 94377116561408, 94377116565503, -+STORE, 94377120755712, 94377120759807, -+STORE, 94377124950016, 94377124954111, -+STORE, 94377129144320, 94377129148415, -+STORE, 94377133338624, 94377133342719, -+STORE, 94377137532928, 94377137537023, -+STORE, 94377141727232, 94377141731327, -+STORE, 94377145921536, 94377145925631, -+STORE, 94377150115840, 94377150119935, -+STORE, 94377154310144, 94377154314239, -+STORE, 94377158504448, 94377158508543, -+STORE, 94377162698752, 94377162702847, -+STORE, 94377166893056, 94377166897151, -+STORE, 94377171087360, 94377171091455, -+STORE, 94377175281664, 94377175285759, -+STORE, 94377179475968, 94377179480063, -+STORE, 94377183670272, 94377183674367, -+STORE, 94377187864576, 94377187868671, -+STORE, 94377192058880, 94377192062975, -+STORE, 94377196253184, 94377196257279, -+STORE, 94377200447488, 94377200451583, -+STORE, 94377204641792, 94377204645887, -+SNULL, 94376135094271, 94376135098367, -+STORE, 94376135090176, 94376135094271, -+STORE, 94376135094272, 94376135098367, -+SNULL, 94376135094272, 94377208836095, -+ }; -+ unsigned long set42[] = { -+STORE, 314572800, 1388314623, -+STORE, 1462157312, 1462169599, -+STORE, 1462169600, 1462185983, -+STORE, 1462185984, 1462190079, -+STORE, 1462190080, 1462194175, -+STORE, 1462194176, 1462198271, -+STORE, 1879986176, 1881800703, -+STORE, 1881800704, 1882034175, -+STORE, 1882034176, 1882193919, -+STORE, 1882193920, 1882406911, -+STORE, 1882406912, 1882451967, -+STORE, 1882451968, 1882996735, -+STORE, 1882996736, 1885892607, -+STORE, 1885892608, 1885896703, -+STORE, 1885896704, 1885904895, -+STORE, 1885904896, 1885908991, -+STORE, 1885908992, 1885913087, -+STORE, 1885913088, 1885966335, -+STORE, 1885966336, 1886232575, -+STORE, 1886232576, 1886236671, -+STORE, 1886236672, 1886240767, -+STORE, 1886240768, 1886244863, -+STORE, 1886244864, 1886248959, -+STORE, 1886248960, 1886294015, -+STORE, 1886294016, 1886494719, -+STORE, 1886494720, 1886498815, -+STORE, 1886498816, 1886502911, -+STORE, 1886502912, 1886507007, -+STORE, 1886507008, 1886511103, -+STORE, 1886511104, 1886556159, -+STORE, 1886556160, 1886629887, -+STORE, 1886629888, 1886633983, -+STORE, 1886633984, 1886638079, -+STORE, 1886638080, 1886642175, -+STORE, 1886642176, 1886646271, -+STORE, 1886646272, 1886666751, -+STORE, 1886666752, 1886670847, -+STORE, 1886670848, 1886674943, -+STORE, 1886674944, 1886679039, -+STORE, 1886679040, 1895419903, -+STORE, 1895419904, 1895550975, -+STORE, 1895550976, 1896148991, -+STORE, 1896148992, 1897189375, -+STORE, 1897189376, 1897701375, -+STORE, 1897701376, 1897803775, -+STORE, 1897803776, 1897816063, -+STORE, 1897816064, 1899913215, -+STORE, 1899913216, 1909379071, -+STORE, 1909379072, 1909387263, -+STORE, 1909387264, 1909391359, -+STORE, 1909391360, 1909432319, -+STORE, 1909432320, 1909436415, -+STORE, 1909436416, 1909440511, -+STORE, 1909440512, 1909460991, -+STORE, 1909460992, 1909547007, -+STORE, 1909547008, 1909551103, -+STORE, 1909551104, 1909555199, -+STORE, 1909555200, 1909559295, -+STORE, 1909559296, 1909563391, -+STORE, 1909563392, 1909739519, -+STORE, 1909739520, 1910566911, -+STORE, 1910566912, 1910571007, -+STORE, 1910571008, 1910575103, -+STORE, 1910575104, 1910579199, -+STORE, 1910579200, 1910583295, -+STORE, 1910583296, 1910587391, -+STORE, 1910587392, 1910620159, -+STORE, 1910620160, 1910624255, -+STORE, 1910624256, 1910628351, -+STORE, 1910628352, 1910632447, -+STORE, 1910632448, 1910652927, -+STORE, 1910652928, 1910657023, -+STORE, 1910657024, 1910661119, -+STORE, 1910661120, 1910665215, -+STORE, 1910665216, 1910669311, -+STORE, 1910669312, 1910677503, -+STORE, 1910677504, 1910681599, -+STORE, 1910681600, 1910685695, -+STORE, 1910685696, 1910689791, -+STORE, 1910689792, 1910697983, -+STORE, 1910697984, 1910702079, -+STORE, 1910702080, 1910706175, -+STORE, 1910706176, 1910710271, -+STORE, 1910710272, 1914093567, -+STORE, 1914093568, 1914097663, -+STORE, 1914097664, 1969434623, -+STORE, 1969434624, 1977819135, -+STORE, 3290435584, 3426750463, -+STORE, 3426750464, 3426754559, -+STORE, 3426754560, 3426762751, -+STORE, 3426762752, 3426766847, -+STORE, 3426766848, 3426770943, -+STORE, 3427037184, 3427061759, -+STORE, 3427061760, 3427135487, -+STORE, 3427135488, 3427143679, -+STORE, 3427143680, 3427147775, -+STORE, 3427147776, 3427209215, -+STORE, 3427319808, 3432116223, -+STORE, 3432116224, 3450130431, -+STORE, 3450130432, 3451027455, -+STORE, 3451027456, 3451031551, -+STORE, 3451031552, 3451461631, -+STORE, 3451736064, 3456688127, -+STORE, 3456688128, 3475222527, -+STORE, 3475222528, 3476119551, -+STORE, 3476119552, 3476127743, -+STORE, 3476127744, 3476553727, -+STORE, 3476631552, 3477315583, -+STORE, 3477315584, 3479949311, -+STORE, 3479949312, 3480002559, -+STORE, 3480002560, 3480006655, -+STORE, 3480006656, 3480432639, -+STORE, 3480539136, 3480543231, -+STORE, 3480543232, 3480547327, -+STORE, 3480547328, 3480555519, -+STORE, 3480854528, 3480903679, -+STORE, 3480903680, 3480969215, -+STORE, 3480969216, 3480977407, -+STORE, 3480977408, 3480981503, -+STORE, 3481030656, 3481092095, -+STORE, 3481092096, 3481235455, -+STORE, 3481235456, 3481243647, -+STORE, 3481243648, 3481247743, -+STORE, 3481436160, 3481444351, -+STORE, 3481444352, 3481456639, -+STORE, 3481456640, 3481460735, -+STORE, 3481460736, 3481464831, -+STORE, 3481587712, 3481645055, -+STORE, 3481645056, 3481772031, -+STORE, 3481772032, 3481776127, -+STORE, 3481776128, 3481780223, -+STORE, 3481874432, 3481935871, -+STORE, 3481935872, 3482030079, -+STORE, 3482030080, 3482038271, -+STORE, 3482038272, 3482042367, -+STORE, 3482198016, 3482230783, -+STORE, 3482230784, 3482271743, -+STORE, 3482271744, 3482279935, -+STORE, 3482279936, 3482284031, -+STORE, 3482562560, 3482566655, -+STORE, 3482566656, 3482570751, -+STORE, 3482570752, 3482574847, -+STORE, 3482636288, 3482689535, -+STORE, 3482689536, 3482746879, -+STORE, 3482746880, 3482755071, -+STORE, 3482755072, 3482759167, -+STORE, 3482972160, 3483062271, -+STORE, 3483062272, 3483242495, -+STORE, 3483242496, 3483246591, -+STORE, 3483246592, 3483250687, -+STORE, 3483398144, 3483688959, -+STORE, 3483688960, 3484114943, -+STORE, 3484114944, 3484131327, -+STORE, 3484131328, 3484135423, -+STORE, 3484135424, 3484143615, -+STORE, 3484184576, 3484475391, -+STORE, 3484475392, 3485028351, -+STORE, 3485028352, 3485057023, -+STORE, 3485057024, 3485061119, -+STORE, 3485360128, 3485364223, -+STORE, 3485364224, 3485368319, -+STORE, 3485368320, 3485372415, -+STORE, 3485589504, 3485593599, -+STORE, 3485593600, 3485597695, -+STORE, 3485597696, 3485601791, -+STORE, 3485913088, 3485937663, -+STORE, 3485937664, 3485974527, -+STORE, 3485974528, 3485982719, -+STORE, 3485982720, 3485986815, -+STORE, 3486052352, 3486056447, -+STORE, 3486056448, 3486064639, -+STORE, 3486064640, 3486068735, -+STORE, 3486068736, 3486072831, -+STORE, 3486294016, 3486302207, -+STORE, 3486302208, 3486306303, -+STORE, 3486306304, 3486310399, -+STORE, 3486310400, 3486314495, -+STORE, 3486670848, 3486679039, -+STORE, 3486679040, 3486683135, -+STORE, 3486683136, 3486687231, -+STORE, 3486687232, 3486691327, -+STORE, 3486863360, 3486871551, -+STORE, 3486871552, 3486875647, -+STORE, 3486875648, 3486879743, -+STORE, 3486879744, 3486883839, -+STORE, 3487584256, 3522543615, -+STORE, 3522543616, 3523321855, -+STORE, 3523321856, 3523342335, -+STORE, 3523342336, 3523387391, -+STORE, 3523387392, 3523391487, -+STORE, 3523391488, 3523395583, -+STORE, 3523477504, 3523686399, -+STORE, 3523686400, 3523981311, -+STORE, 3523981312, 3523997695, -+STORE, 3523997696, 3524001791, -+STORE, 3524177920, 3525013503, -+STORE, 3525013504, 3526582271, -+STORE, 3526582272, 3526606847, -+STORE, 3526606848, 3526610943, -+STORE, 3526610944, 3526615039, -+STORE, 3526672384, 3526746111, -+STORE, 3526746112, 3526860799, -+STORE, 3526860800, 3526868991, -+STORE, 3526868992, 3526873087, -+STORE, 3527000064, 3527475199, -+STORE, 3527475200, 3527479295, -+STORE, 3527479296, 3527573503, -+STORE, 3527573504, 3527581695, -+STORE, 3527581696, 3527585791, -+STORE, 3527585792, 3527606271, -+STORE, 3527909376, 3527913471, -+STORE, 3527913472, 3527917567, -+STORE, 3527917568, 3527921663, -+STORE, 3527950336, 3528011775, -+STORE, 3528011776, 3528093695, -+STORE, 3528093696, 3528101887, -+STORE, 3528101888, 3528105983, -+STORE, 3528228864, 3528241151, -+STORE, 3528241152, 3528261631, -+STORE, 3528261632, 3528265727, -+STORE, 3528273920, 3528593407, -+STORE, 3528593408, 3528609791, -+STORE, 3528609792, 3528638463, -+STORE, 3528638464, 3528642559, -+STORE, 3528642560, 3528646655, -+STORE, 3528880128, 3528912895, -+STORE, 3528912896, 3528962047, -+STORE, 3528962048, 3528966143, -+STORE, 3528966144, 3528970239, -+STORE, 3528982528, 3530293247, -+STORE, 3530366976, 3530825727, -+STORE, 3530825728, 3531317247, -+STORE, 3531317248, 3541041151, -+STORE, 3541041152, 3541303295, -+STORE, 3541430272, 3566206975, -+STORE, 3566206976, 3566993407, -+STORE, 3567239168, 3587571711, -+STORE, 3587571712, 3588284415, -+STORE, 3588284416, 3588661247, -+STORE, 3588661248, 3589066751, -+STORE, 3589066752, 3589574655, -+STORE, 3589574656, 3590078463, -+STORE, 3590078464, 3590373375, -+STORE, 3590373376, 3590668287, -+STORE, 3590668288, 3590963199, -+STORE, 3590963200, 3591294975, -+STORE, 3591294976, 3591602175, -+STORE, 3591602176, 3591933951, -+STORE, 3591933952, 3592241151, -+STORE, 3592241152, 3592572927, -+STORE, 3592572928, 3592876031, -+STORE, 3592876032, 3593211903, -+STORE, 3593211904, 3593547775, -+STORE, 3593547776, 3593650175, -+STORE, 3593650176, 3593928703, -+STORE, 3593928704, 3593936895, -+STORE, 3593936896, 3593940991, -+STORE, 3594006528, 3594301439, -+STORE, 3594301440, 3594739711, -+STORE, 3594739712, 3594756095, -+STORE, 3594756096, 3594760191, -+STORE, 3594760192, 3594768383, -+STORE, 3594952704, 3595051007, -+STORE, 3595051008, 3595223039, -+STORE, 3595223040, 3595227135, -+STORE, 3595227136, 3595235327, -+STORE, 3595431936, 3595775999, -+STORE, 3595776000, 3596701695, -+STORE, 3596701696, 3596742655, -+STORE, 3596742656, 3596746751, -+STORE, 3596746752, 3596750847, -+STORE, 3596767232, 3597070335, -+STORE, 3597070336, 3597402111, -+STORE, 3597402112, 3598188543, -+STORE, 3598262272, 3623428095, -+STORE, 3623428096, 3623432191, -+STORE, 3623432192, 3623436287, -+STORE, 3623436288, 3623440383, -+STORE, 3623616512, 3623878655, -+STORE, 3624169472, 3624300543, -+STORE, 3627524096, 3628523519, -+STORE, 3628523520, 3629522943, -+STORE, 3696631808, 3730186239, -+STORE, 3730186240, 3763740671, -+STORE, 3763740672, 3764027391, -+STORE, 3764027392, 3765133311, -+STORE, 3765133312, 3765145599, -+STORE, 3765145600, 3765149695, -+STORE, 3765178368, 3766022143, -+STORE, 3766022144, 3768791039, -+STORE, 3768791040, 3768840191, -+STORE, 3768840192, 3768844287, -+STORE, 3768897536, 3768913919, -+STORE, 3768913920, 3768934399, -+STORE, 3768934400, 3768938495, -+STORE, 3769016320, 3769147391, -+STORE, 3769147392, 3769233407, -+STORE, 3769233408, 3769356287, -+STORE, 3769356288, 3769360383, -+STORE, 3769360384, 3769368575, -+STORE, 3769376768, 3794542591, -+STORE, 3794542592, 3794599935, -+STORE, 3794599936, 3794731007, -+STORE, 3794731008, 3794735103, -+STORE, 3794735104, 3794743295, -+STORE, 3794849792, 3794980863, -+STORE, 3794980864, 3794984959, -+STORE, 3794984960, 3794989055, -+STORE, 3794989056, 3794993151, -+STORE, 3794993152, 3794997247, -+STORE, 3795103744, 3795128319, -+STORE, 3795128320, 3795165183, -+STORE, 3795165184, 3795169279, -+STORE, 3795169280, 3795173375, -+STORE, 3795210240, 3795357695, -+STORE, 3795357696, 3795365887, -+STORE, 3795365888, 3795374079, -+STORE, 3795374080, 3795378175, -+STORE, 3795378176, 3795382271, -+STORE, 3795406848, 3795738623, -+STORE, 3795738624, 3795742719, -+STORE, 3795742720, 3795755007, -+STORE, 3795755008, 3795759103, -+STORE, 3795763200, 3795894271, -+STORE, 3795894272, 3796041727, -+STORE, 3796041728, 3796054015, -+STORE, 3796054016, 3796066303, -+STORE, 3796066304, 3796070399, -+STORE, 3796176896, 3796205567, -+STORE, 3796205568, 3796250623, -+STORE, 3796250624, 3796254719, -+STORE, 3796254720, 3796258815, -+STORE, 3796262912, 3796393983, -+STORE, 3796393984, 3796516863, -+STORE, 3796516864, 3796873215, -+STORE, 3796873216, 3796885503, -+STORE, 3796885504, 3796889599, -+STORE, 3796963328, 3796967423, -+STORE, 3796967424, 3796975615, -+STORE, 3796975616, 3796979711, -+STORE, 3797000192, 3797307391, -+STORE, 3797307392, 3797311487, -+STORE, 3797311488, 3797315583, -+STORE, 3797315584, 3797323775, -+STORE, 3797327872, 3797450751, -+STORE, 3797450752, 3797458943, -+STORE, 3797458944, 3797471231, -+STORE, 3797471232, 3797475327, -+STORE, 3797577728, 3797700607, -+STORE, 3797700608, 3797721087, -+STORE, 3797721088, 3797733375, -+STORE, 3797733376, 3797741567, -+STORE, 3797741568, 3797864447, -+STORE, 3797864448, 3797995519, -+STORE, 3797995520, 3798048767, -+STORE, 3798048768, 3798179839, -+STORE, 3798179840, 3798188031, -+STORE, 3798188032, 3798192127, -+STORE, 3798290432, 3798302719, -+STORE, 3798302720, 3798323199, -+STORE, 3798323200, 3798327295, -+STORE, 3798327296, 3798331391, -+STORE, 3798429696, 3798433791, -+STORE, 3798433792, 3798552575, -+STORE, 3798552576, 3798556671, -+STORE, 3798556672, 3798568959, -+STORE, 3798568960, 3798573055, -+STORE, 3798573056, 3798581247, -+STORE, 3798618112, 3798749183, -+STORE, 3798749184, 3798855679, -+STORE, 3798855680, 3798966271, -+STORE, 3798966272, 3798982655, -+STORE, 3798982656, 3798986751, -+STORE, 3799101440, 3799171071, -+STORE, 3799171072, 3799240703, -+STORE, 3799240704, 3799248895, -+STORE, 3799248896, 3799252991, -+STORE, 3799326720, 3799650303, -+STORE, 3799650304, 3800629247, -+STORE, 3800629248, 3800641535, -+STORE, 3800641536, 3800645631, -+STORE, 3800645632, 3800649727, -+STORE, 3800649728, 3800903679, -+STORE, 3800903680, 3800936447, -+STORE, 3800936448, 3800969215, -+STORE, 3800969216, 3800981503, -+STORE, 3800981504, 3800985599, -+STORE, 3801001984, 3801133055, -+STORE, 3801133056, 3801202687, -+STORE, 3801202688, 3801591807, -+STORE, 3801591808, 3801599999, -+STORE, 3801600000, 3801604095, -+STORE, 3801604096, 3801608191, -+STORE, 3801608192, 3801739263, -+STORE, 3801739264, 3801755647, -+STORE, 3801755648, 3801796607, -+STORE, 3801796608, 3801804799, -+STORE, 3801804800, 3801808895, -+STORE, 3801878528, 3801944063, -+STORE, 3801944064, 3802116095, -+STORE, 3802116096, 3802124287, -+STORE, 3802124288, 3802128383, -+STORE, 3802136576, 3803447295, -+STORE, 3803492352, 3803553791, -+STORE, 3803553792, 3804233727, -+STORE, 3804233728, 3806068735, -+STORE, 3806121984, 3806253055, -+STORE, 3806253056, 3806674943, -+STORE, 3806674944, 3807117311, -+STORE, 3807117312, 3807379455, -+STORE, 3807379456, 3807432703, -+STORE, 3807432704, 3807563775, -+STORE, 3807563776, 3809202175, -+STORE, 3809202176, 3810250751, -+STORE, 3810250752, 3827027967, -+STORE, 3827027968, 3829125119, -+STORE, 3829125120, 3837513727, -+STORE, 3837513728, 3839610879, -+STORE, 3839610880, 3847999487, -+STORE, 3847999488, 3856392191, -+STORE, 3856392192, 3864784895, -+STORE, 3864784896, 3868983295, -+STORE, 3868983296, 3885760511, -+STORE, 3885760512, 3886809087, -+STORE, 3886809088, 3887857663, -+STORE, 3887857664, 3888119807, -+STORE, 3888144384, 3888148479, -+STORE, 3888148480, 3888218111, -+STORE, 3888218112, 3888222207, -+STORE, 3888222208, 3888353279, -+STORE, 3888353280, 3889172479, -+STORE, 3889172480, 3892314111, -+STORE, 3892314112, 3892576255, -+STORE, 3892588544, 3892637695, -+STORE, 3892637696, 3892686847, -+STORE, 3892686848, 3892744191, -+STORE, 3892748288, 3892785151, -+STORE, 3892785152, 3895459839, -+STORE, 3895459840, 3895721983, -+STORE, 3895738368, 3895885823, -+STORE, 3895885824, 3897081855, -+STORE, 3897081856, 3906482175, -+STORE, 3906482176, 3916144639, -+STORE, 3916144640, 3925766143, -+STORE, 3925766144, 3926974463, -+STORE, 3926974464, 3928367103, -+STORE, 3928367104, 3928911871, -+STORE, 3928911872, 3933995007, -+STORE, 3933995008, 3935830015, -+STORE, 3935830016, 3935846399, -+STORE, 3935879168, 3936010239, -+STORE, 3936010240, 3936026623, -+STORE, 3936026624, 3936034815, -+STORE, 3936034816, 3936051199, -+STORE, 3936051200, 3936055295, -+STORE, 3936071680, 3936137215, -+STORE, 3936137216, 3936202751, -+STORE, 3936202752, 3936219135, -+STORE, 3936235520, 3936251903, -+STORE, 3936268288, 3936276479, -+STORE, 3936276480, 3936284671, -+STORE, 3936284672, 3936288767, -+STORE, 3936288768, 3936292863, -+STORE, 3936296960, 3936354303, -+STORE, 3936354304, 3936616447, -+STORE, 3936628736, 3936669695, -+STORE, 3936669696, 3936747519, -+STORE, 3936747520, 3936870399, -+STORE, 3936870400, 3936874495, -+STORE, 3936874496, 3936878591, -+STORE, 3936882688, 3936903167, -+STORE, 3936911360, 3936948223, -+STORE, 3936948224, 3936964607, -+STORE, 3936964608, 3937103871, -+STORE, 3937103872, 3937107967, -+STORE, 3937132544, 3937161215, -+STORE, 3937189888, 3937255423, -+STORE, 3937255424, 3938512895, -+STORE, 3938512896, 3945435135, -+STORE, 3945435136, 3945476095, -+STORE, 3945476096, 3945484287, -+STORE, 3945484288, 3945496575, -+STORE, 3945500672, 3945541631, -+STORE, 3945558016, 3945566207, -+STORE, 3945566208, 3945594879, -+STORE, 3945594880, 3945598975, -+STORE, 3945598976, 3945603071, -+STORE, 3945611264, 3945742335, -+STORE, 3945742336, 3945844735, -+STORE, 3945844736, 3945848831, -+STORE, 3945848832, 3945861119, -+STORE, 3945861120, 3945865215, -+STORE, 3945869312, 3945897983, -+STORE, 3945897984, 3946303487, -+STORE, 3946303488, 3946397695, -+STORE, 3946397696, 3946569727, -+STORE, 3946569728, 3946573823, -+STORE, 3946573824, 3946594303, -+STORE, 3946594304, 3946663935, -+STORE, 3946663936, 3946708991, -+STORE, 3946708992, 3946823679, -+STORE, 3946823680, 3946827775, -+STORE, 3946827776, 3946831871, -+STORE, 3946831872, 3946860543, -+STORE, 3946893312, 3946897407, -+STORE, 3946897408, 3946905599, -+STORE, 3946905600, 3946909695, -+STORE, 3946909696, 3946913791, -+STORE, 3946913792, 3946930175, -+STORE, 3946930176, 3946967039, -+STORE, 3946967040, 3947102207, -+STORE, 3947102208, 3948412927, -+STORE, 3948441600, 3948556287, -+STORE, 3948556288, 3948576767, -+STORE, 3948576768, 3948597247, -+STORE, 3948597248, 3948605439, -+STORE, 3948605440, 3948609535, -+STORE, 3948609536, 3948654591, -+STORE, 3948654592, 3948781567, -+STORE, 3948781568, 3948822527, -+STORE, 3948822528, 3948904447, -+STORE, 3948904448, 3948908543, -+STORE, 3948908544, 3948912639, -+STORE, 3948945408, 3949043711, -+STORE, 3949043712, 3949174783, -+STORE, 3949174784, 3949191167, -+STORE, 3949191168, 3949195263, -+STORE, 3949207552, 3949252607, -+STORE, 3949252608, 3949256703, -+STORE, 3949256704, 3949363199, -+STORE, 3949363200, 3949367295, -+STORE, 3949367296, 3949379583, -+STORE, 3949379584, 3949383679, -+STORE, 3949383680, 3949400063, -+STORE, 3949400064, 3949404159, -+STORE, 3949416448, 3949481983, -+STORE, 3949481984, 3949486079, -+STORE, 3949486080, 3949592575, -+STORE, 3949592576, 3949596671, -+STORE, 3949596672, 3949621247, -+STORE, 3949621248, 3949662207, -+STORE, 3949662208, 3949666303, -+STORE, 3949694976, 3949727743, -+STORE, 3949727744, 3949731839, -+STORE, 3949731840, 3949838335, -+STORE, 3949838336, 3949842431, -+STORE, 3949842432, 3949846527, -+STORE, 3949846528, 3949854719, -+STORE, 3949854720, 3949858815, -+STORE, 3949858816, 3949862911, -+STORE, 3949867008, 3949891583, -+STORE, 3949891584, 3949928447, -+STORE, 3949928448, 3949993983, -+STORE, 3949993984, 3950043135, -+STORE, 3950043136, 3950059519, -+STORE, 3950059520, 3950096383, -+STORE, 3950096384, 3950100479, -+STORE, 3950100480, 3950104575, -+STORE, 3950104576, 3950157823, -+STORE, 3950157824, 3950292991, -+STORE, 3950292992, 3950346239, -+STORE, 3950346240, 3950477311, -+STORE, 3950477312, 3950485503, -+STORE, 3950485504, 3950489599, -+STORE, 3950493696, 3950510079, -+STORE, 3950510080, 3950661631, -+STORE, 3950661632, 3951005695, -+STORE, 3951005696, 3951026175, -+STORE, 3951026176, 3951030271, -+STORE, 3951030272, 3951054847, -+STORE, 3951054848, 3951116287, -+STORE, 3951116288, 3951144959, -+STORE, 3951144960, 3951149055, -+STORE, 3951149056, 3951194111, -+STORE, 3951194112, 3951202303, -+STORE, 3951202304, 3951206399, -+STORE, 3951210496, 3951226879, -+STORE, 3951226880, 3951329279, -+STORE, 3951329280, 3951366143, -+STORE, 3951366144, 3951411199, -+STORE, 3951411200, 3951415295, -+STORE, 3951415296, 3951419391, -+STORE, 3951419392, 3951452159, -+STORE, 3951452160, 3951566847, -+STORE, 3951566848, 3951812607, -+STORE, 3951812608, 3952173055, -+STORE, 3952173056, 3952214015, -+STORE, 3952214016, 3952218111, -+STORE, 3952222208, 3952250879, -+STORE, 3952250880, 3952369663, -+STORE, 3952369664, 3952488447, -+STORE, 3952488448, 3952627711, -+STORE, 3952627712, 3952635903, -+STORE, 3952635904, 3952639999, -+STORE, 3952652288, 3952668671, -+STORE, 3952668672, 3953000447, -+STORE, 3953000448, 3953004543, -+STORE, 3953004544, 3953008639, -+STORE, 3953008640, 3953012735, -+STORE, 3953012736, 3953037311, -+STORE, 3953037312, 3953151999, -+STORE, 3953152000, 3953291263, -+STORE, 3953291264, 3953324031, -+STORE, 3953324032, 3953364991, -+STORE, 3953364992, 3953373183, -+STORE, 3953373184, 3953377279, -+STORE, 3953381376, 3953410047, -+STORE, 3953410048, 3953491967, -+STORE, 3953491968, 3953643519, -+STORE, 3953643520, 3953651711, -+STORE, 3953651712, 3953655807, -+STORE, 3953659904, 3953766399, -+STORE, 3953766400, 3953774591, -+STORE, 3953774592, 3953786879, -+STORE, 3953786880, 3953790975, -+STORE, 3953790976, 3953823743, -+STORE, 3953823744, 3953963007, -+STORE, 3953963008, 3954024447, -+STORE, 3954024448, 3954118655, -+STORE, 3954118656, 3954122751, -+STORE, 3954122752, 3954126847, -+STORE, 3954130944, 3954184191, -+STORE, 3954184192, 3954294783, -+STORE, 3954294784, 3954323455, -+STORE, 3954323456, 3954393087, -+STORE, 3954393088, 3954397183, -+STORE, 3954397184, 3954401279, -+STORE, 3954401280, 3954405375, -+STORE, 3954409472, 3954528255, -+STORE, 3954528256, 3954737151, -+STORE, 3954737152, 3955052543, -+STORE, 3955052544, 3955060735, -+STORE, 3955060736, 3955064831, -+STORE, 3955068928, 3955105791, -+STORE, 3955105792, 3955167231, -+STORE, 3955167232, 3955277823, -+STORE, 3955277824, 3955310591, -+STORE, 3955310592, 3955351551, -+STORE, 3955351552, 3955359743, -+STORE, 3955359744, 3955363839, -+STORE, 3955363840, 3955392511, -+STORE, 3955392512, 3955453951, -+STORE, 3955453952, 3955601407, -+STORE, 3955601408, 3955777535, -+STORE, 3955777536, 3955982335, -+STORE, 3955982336, 3956011007, -+STORE, 3956011008, 3956015103, -+STORE, 3956023296, 3956039679, -+STORE, 3956039680, 3956125695, -+STORE, 3956125696, 3956129791, -+STORE, 3956129792, 3956133887, -+STORE, 3956133888, 3956137983, -+STORE, 3956142080, 3956449279, -+STORE, 3956449280, 3956543487, -+STORE, 3956543488, 3956719615, -+STORE, 3956719616, 3956731903, -+STORE, 3956731904, 3956735999, -+STORE, 3956744192, 3956793343, -+STORE, 3956793344, 3956887551, -+STORE, 3956887552, 3956953087, -+STORE, 3956953088, 3957035007, -+STORE, 3957035008, 3957039103, -+STORE, 3957039104, 3957047295, -+STORE, 3957047296, 3957071871, -+STORE, 3957071872, 3957231615, -+STORE, 3957231616, 3957563391, -+STORE, 3957563392, 3957579775, -+STORE, 3957579776, 3957583871, -+STORE, 3957592064, 3957608447, -+STORE, 3957608448, 3957878783, -+STORE, 3957878784, 3958591487, -+STORE, 3958591488, 3958599679, -+STORE, 3958599680, 3958607871, -+STORE, 3958607872, 3958620159, -+STORE, 3958620160, 3958624255, -+STORE, 3958624256, 3963199487, -+STORE, 3963199488, 3963285503, -+STORE, 3963285504, 3963371519, -+STORE, 3963371520, 3963428863, -+STORE, 3963428864, 3963555839, -+STORE, 3963555840, 3963559935, -+STORE, 3963559936, 3963564031, -+STORE, 3963568128, 3963596799, -+STORE, 3963596800, 3963682815, -+STORE, 3963682816, 3963695103, -+STORE, 3963695104, 3963711487, -+STORE, 3963711488, 3963715583, -+STORE, 3963719680, 3963752447, -+STORE, 3963752448, 3963846655, -+STORE, 3963846656, 3963932671, -+STORE, 3963932672, 3964444671, -+STORE, 3964444672, 3964448767, -+STORE, 3964448768, 3965808639, -+STORE, 3965808640, 3965845503, -+STORE, 3965845504, 3965849599, -+STORE, 3965853696, 3965935615, -+STORE, 3965935616, 3966017535, -+STORE, 3966017536, 3966103551, -+STORE, 3966103552, 3966685183, -+STORE, 3966685184, 3967705087, -+STORE, 3967705088, 3967758335, -+STORE, 3967758336, 3967762431, -+STORE, 3967762432, 3967770623, -+STORE, 3967770624, 3967799295, -+STORE, 3967799296, 3967848447, -+STORE, 3967848448, 3967868927, -+STORE, 3967868928, 3967901695, -+STORE, 3967901696, 3967905791, -+STORE, 3967905792, 3967909887, -+STORE, 3967909888, 3967995903, -+STORE, 3967995904, 3968077823, -+STORE, 3968077824, 3968159743, -+STORE, 3968159744, 3968167935, -+STORE, 3968167936, 3968172031, -+STORE, 3968172032, 3968192511, -+STORE, 3968192512, 3968196607, -+STORE, 3968196608, 3968200703, -+STORE, 3968208896, 3968516095, -+STORE, 3968516096, 3968528383, -+STORE, 3968528384, 3968552959, -+STORE, 3968552960, 3968557055, -+STORE, 3968561152, 3968593919, -+STORE, 3968593920, 3968626687, -+STORE, 3968626688, 3971153919, -+STORE, 3971153920, 3973754879, -+STORE, 3973754880, 3973804031, -+STORE, 3973804032, 3973820415, -+STORE, 3973820416, 3973832703, -+STORE, 3973840896, 3973873663, -+STORE, 3973873664, 3973967871, -+STORE, 3973967872, 3973976063, -+STORE, 3973976064, 3973984255, -+STORE, 3973984256, 3973988351, -+STORE, 3973988352, 3973992447, -+STORE, 3973996544, 3974008831, -+STORE, 3974008832, 3974045695, -+STORE, 3974045696, 3974139903, -+STORE, 3974139904, 3974254591, -+STORE, 3974254592, 3974275071, -+STORE, 3974275072, 3974291455, -+STORE, 3974291456, 3974295551, -+STORE, 3974295552, 3974373375, -+STORE, 3974373376, 3974524927, -+STORE, 3974524928, 3974529023, -+STORE, 3974529024, 3974537215, -+STORE, 3974537216, 3974541311, -+STORE, 3974541312, 3974545407, -+STORE, 3974545408, 3974627327, -+STORE, 3974627328, 3974680575, -+STORE, 3974680576, 3974811647, -+STORE, 3974811648, 3974819839, -+STORE, 3974819840, 3974823935, -+STORE, 3974832128, 3974918143, -+STORE, 3974918144, 3974963199, -+STORE, 3974963200, 3975077887, -+STORE, 3975077888, 3975090175, -+STORE, 3975090176, 3975094271, -+STORE, 3975094272, 3975102463, -+STORE, 3975102464, 3975114751, -+STORE, 3975114752, 3975266303, -+STORE, 3975266304, 3975274495, -+STORE, 3975274496, 3975286783, -+STORE, 3975286784, 3975290879, -+STORE, 3975290880, 3975299071, -+STORE, 3975299072, 3975315455, -+STORE, 3975315456, 3975430143, -+STORE, 3975430144, 3975536639, -+STORE, 3975536640, 3975651327, -+STORE, 3975651328, 3975655423, -+STORE, 3975655424, 3975659519, -+STORE, 3975659520, 3975770111, -+STORE, 3975770112, 3975778303, -+STORE, 3975778304, 3975790591, -+STORE, 3975790592, 3975794687, -+STORE, 3975794688, 3975798783, -+STORE, 3975798784, 3975831551, -+STORE, 3975831552, 3975872511, -+STORE, 3975872512, 3975987199, -+STORE, 3975987200, 3976134655, -+STORE, 3976134656, 3977175039, -+STORE, 3977175040, 3977183231, -+STORE, 3977183232, 3977191423, -+STORE, 3977191424, 3977195519, -+STORE, 3977199616, 3977248767, -+STORE, 3977248768, 3977539583, -+STORE, 3977539584, 3977965567, -+STORE, 3977965568, 3977981951, -+STORE, 3977981952, 3977986047, -+STORE, 3977986048, 3977994239, -+STORE, 3977994240, 3978002431, -+STORE, 3978002432, 3978084351, -+STORE, 3978084352, 3978125311, -+STORE, 3978125312, 3978174463, -+STORE, 3978174464, 3978178559, -+STORE, 3978178560, 3978182655, -+STORE, 3978182656, 3978207231, -+STORE, 3978207232, 3978297343, -+STORE, 3978297344, 3978301439, -+STORE, 3978301440, 3978305535, -+STORE, 3978305536, 3978309631, -+STORE, 3978309632, 3978317823, -+STORE, 3978317824, 3978625023, -+STORE, 3978625024, 3978657791, -+STORE, 3978657792, 3978727423, -+STORE, 3978727424, 3978735615, -+STORE, 3978735616, 3978739711, -+STORE, 3978739712, 3978760191, -+STORE, 3978760192, 3978842111, -+STORE, 3978842112, 3978850303, -+STORE, 3978850304, 3978858495, -+STORE, 3978858496, 3978862591, -+STORE, 3978862592, 3978895359, -+STORE, 3978895360, 3979014143, -+STORE, 3979014144, 3979132927, -+STORE, 3979132928, 3979288575, -+STORE, 3979288576, 3979481087, -+STORE, 3979481088, 3979489279, -+STORE, 3979489280, 3979493375, -+STORE, 3979497472, 3979583487, -+STORE, 3979583488, 3979673599, -+STORE, 3979673600, 3979718655, -+STORE, 3979718656, 3979829247, -+STORE, 3979829248, 3979841535, -+STORE, 3979841536, 3979882495, -+STORE, 3979882496, 3979964415, -+STORE, 3979964416, 3980013567, -+STORE, 3980013568, 3980148735, -+STORE, 3980148736, 3980152831, -+STORE, 3980152832, 3980320767, -+STORE, 3980320768, 3980337151, -+STORE, 3980337152, 3980341247, -+STORE, 3980345344, 3980365823, -+STORE, 3980365824, 3980423167, -+STORE, 3980423168, 3980460031, -+STORE, 3980460032, 3980500991, -+STORE, 3980500992, 3980509183, -+STORE, 3980509184, 3980513279, -+STORE, 3980513280, 3980546047, -+STORE, 3980546048, 3980660735, -+STORE, 3980660736, 3980951551, -+STORE, 3980951552, 3981500415, -+STORE, 3981500416, 3981529087, -+STORE, 3981529088, 3981533183, -+STORE, 3981537280, 3981549567, -+STORE, 3981549568, 3981598719, -+STORE, 3981598720, 3981717503, -+STORE, 3981717504, 3982127103, -+STORE, 3982127104, 3982675967, -+STORE, 3982675968, 3982733311, -+STORE, 3982733312, 3982737407, -+STORE, 3982741504, 3982860287, -+STORE, 3982860288, 3982905343, -+STORE, 3982905344, 3982966783, -+STORE, 3982966784, 3982974975, -+STORE, 3982974976, 3982979071, -+STORE, 3982979072, 3983032319, -+STORE, 3983032320, 3983085567, -+STORE, 3983085568, 3983208447, -+STORE, 3983208448, 3983212543, -+STORE, 3983212544, 3983220735, -+STORE, 3983220736, 3983224831, -+STORE, 3983224832, 3983237119, -+STORE, 3983237120, 3983351807, -+STORE, 3983351808, 3983376383, -+STORE, 3983376384, 3983392767, -+STORE, 3983392768, 3983396863, -+STORE, 3983396864, 3983400959, -+STORE, 3983400960, 3983417343, -+STORE, 3983417344, 3983753215, -+STORE, 3983753216, 3983757311, -+STORE, 3983757312, 3983761407, -+STORE, 3983761408, 3983765503, -+STORE, 3983765504, 3983769599, -+STORE, 3983769600, 3983880191, -+STORE, 3983880192, 3983892479, -+STORE, 3983892480, 3983900671, -+STORE, 3983900672, 3983904767, -+STORE, 3983904768, 3983908863, -+STORE, 3983908864, 3983941631, -+STORE, 3983941632, 3983990783, -+STORE, 3983990784, 3984097279, -+STORE, 3984097280, 3984105471, -+STORE, 3984105472, 3984117759, -+STORE, 3984117760, 3984121855, -+STORE, 3984121856, 3984125951, -+STORE, 3984125952, 3984134143, -+STORE, 3984134144, 3984150527, -+STORE, 3984150528, 3984416767, -+STORE, 3984416768, 3984470015, -+STORE, 3984470016, 3984564223, -+STORE, 3984564224, 3984568319, -+STORE, 3984572416, 3984629759, -+STORE, 3984629760, 3984805887, -+STORE, 3984805888, 3985096703, -+STORE, 3985096704, 3985104895, -+STORE, 3985104896, 3985108991, -+STORE, 3985113088, 3986862079, -+STORE, 3986862080, 3993640959, -+STORE, 3993640960, 3993739263, -+STORE, 3993739264, 3993743359, -+STORE, 3993743360, 3993759743, -+STORE, 3993759744, 3993780223, -+STORE, 3993780224, 3993784319, -+STORE, 3993784320, 3993792511, -+STORE, 3993792512, 3993796607, -+STORE, 3993796608, 3993800703, -+STORE, 3993804800, 3994214399, -+STORE, 3994214400, 3994218495, -+STORE, 3994218496, 3994222591, -+STORE, 3994222592, 3994226687, -+STORE, 3994230784, 3994243071, -+STORE, 3994243072, 3994255359, -+STORE, 3994255360, 3994304511, -+STORE, 3994304512, 3994386431, -+STORE, 3994386432, 3994509311, -+STORE, 3994509312, 3994521599, -+STORE, 3994521600, 3994525695, -+STORE, 3994529792, 3994542079, -+STORE, 3994542080, 3994660863, -+STORE, 3994660864, 3994705919, -+STORE, 3994705920, 3994796031, -+STORE, 3994796032, 3994800127, -+STORE, 3994800128, 3994804223, -+STORE, 3994804224, 3994812415, -+STORE, 3994812416, 3994845183, -+STORE, 3994845184, 3994898431, -+STORE, 3994898432, 3994902527, -+STORE, 3994902528, 3994906623, -+STORE, 3994910720, 3994931199, -+STORE, 3994931200, 3995181055, -+STORE, 3995181056, 3995222015, -+STORE, 3995222016, 3995275263, -+STORE, 3995275264, 3995279359, -+STORE, 3995279360, 3995283455, -+STORE, 3995283456, 3995291647, -+STORE, 3995291648, 3995324415, -+STORE, 3995324416, 3995451391, -+STORE, 3995451392, 3995697151, -+STORE, 3995697152, 3996078079, -+STORE, 3996078080, 3996086271, -+STORE, 3996086272, 3996090367, -+STORE, 3996094464, 3996119039, -+STORE, 3996119040, 3996200959, -+STORE, 3996200960, 3996229631, -+STORE, 3996229632, 3996233727, -+STORE, 3996233728, 3996282879, -+STORE, 3996282880, 3996291071, -+STORE, 3996291072, 3996295167, -+STORE, 3996299264, 3996311551, -+STORE, 3996311552, 3996430335, -+STORE, 3996430336, 3996467199, -+STORE, 3996467200, 3996504063, -+STORE, 3996504064, 3996512255, -+STORE, 3996512256, 3996516351, -+STORE, 3996516352, 3996540927, -+STORE, 3996540928, 3996671999, -+STORE, 3996672000, 3996676095, -+STORE, 3996676096, 3996684287, -+STORE, 3996684288, 3996688383, -+STORE, 3996688384, 3996692479, -+STORE, 3996692480, 3996717055, -+STORE, 3996717056, 3997048831, -+STORE, 3997048832, 3997057023, -+STORE, 3997057024, 3997073407, -+STORE, 3997073408, 3997077503, -+STORE, 3997077504, 3997081599, -+STORE, 3997081600, 3997097983, -+STORE, 3997097984, 3997179903, -+STORE, 3997179904, 3997356031, -+STORE, 3997356032, 3997650943, -+STORE, 3997650944, 3997675519, -+STORE, 3997675520, 3997679615, -+STORE, 3997683712, 3997700095, -+STORE, 3997700096, 3997745151, -+STORE, 3997745152, 3997802495, -+STORE, 3997802496, 3997810687, -+STORE, 3997810688, 3997814783, -+STORE, 3997814784, 3998064639, -+STORE, 3998064640, 3998081023, -+STORE, 3998081024, 3998085119, -+STORE, 3998085120, 3998130175, -+STORE, 3998130176, 3998134271, -+STORE, 3998134272, 3998142463, -+STORE, 3998142464, 3998179327, -+STORE, 3998179328, 3998212095, -+STORE, 3998212096, 3998326783, -+STORE, 3998326784, 3998351359, -+STORE, 3998351360, 3998392319, -+STORE, 3998392320, 3998396415, -+STORE, 3998396416, 3998400511, -+STORE, 3998400512, 3998433279, -+STORE, 3998433280, 3998466047, -+STORE, 3998466048, 3998613503, -+STORE, 3998613504, 3998666751, -+STORE, 3998666752, 3998724095, -+STORE, 3998724096, 3998732287, -+STORE, 3998732288, 3998736383, -+STORE, 3998736384, 3998760959, -+STORE, 3998760960, 3998777343, -+STORE, 3998777344, 3998822399, -+STORE, 3998822400, 3998826495, -+STORE, 3998826496, 3998830591, -+STORE, 3998830592, 3998863359, -+STORE, 3998863360, 3998900223, -+STORE, 3998900224, 3999043583, -+STORE, 3999043584, 3999121407, -+STORE, 3999121408, 3999215615, -+STORE, 3999215616, 3999223807, -+STORE, 3999223808, 3999227903, -+STORE, 3999227904, 3999236095, -+STORE, 3999236096, 3999268863, -+STORE, 3999268864, 3999301631, -+STORE, 3999301632, 3999354879, -+STORE, 3999354880, 3999428607, -+STORE, 3999428608, 3999436799, -+STORE, 3999436800, 3999440895, -+STORE, 3999444992, 3999461375, -+STORE, 3999461376, 3999584255, -+STORE, 3999584256, 3999760383, -+STORE, 3999760384, 4000219135, -+STORE, 4000219136, 4000235519, -+STORE, 4000235520, 4000251903, -+STORE, 4000251904, 4000501759, -+STORE, 4000501760, 4000505855, -+STORE, 4000505856, 4000509951, -+STORE, 4000509952, 4000518143, -+STORE, 4000518144, 4000522239, -+STORE, 4000522240, 4000587775, -+STORE, 4000587776, 4000645119, -+STORE, 4000645120, 4000813055, -+STORE, 4000813056, 4000817151, -+STORE, 4000821248, 4000837631, -+STORE, 4000837632, 4000870399, -+STORE, 4000870400, 4000874495, -+STORE, 4000874496, 4000878591, -+STORE, 4000878592, 4000882687, -+STORE, 4000882688, 4000886783, -+STORE, 4000886784, 4000890879, -+STORE, 4000890880, 4000907263, -+STORE, 4000907264, 4001214463, -+STORE, 4001214464, 4001558527, -+STORE, 4001558528, 4002484223, -+STORE, 4002484224, 4002525183, -+STORE, 4002525184, 4002529279, -+STORE, 4002529280, 4002533375, -+STORE, 4002533376, 4002537471, -+STORE, 4002537472, 4002660351, -+STORE, 4002660352, 4002779135, -+STORE, 4002779136, 4002791423, -+STORE, 4002791424, 4002799615, -+STORE, 4002799616, 4002807807, -+STORE, 4002807808, 4002811903, -+STORE, 4002811904, 4002828287, -+STORE, 4002828288, 4002910207, -+STORE, 4002910208, 4003028991, -+STORE, 4003028992, 4003037183, -+STORE, 4003037184, 4003045375, -+STORE, 4003045376, 4003049471, -+STORE, 4003049472, 4003053567, -+STORE, 4003053568, 4003057663, -+STORE, 4003057664, 4003065855, -+STORE, 4003065856, 4003135487, -+STORE, 4003135488, 4003446783, -+STORE, 4003446784, 4003450879, -+STORE, 4003450880, 4003454975, -+STORE, 4003454976, 4003459071, -+STORE, 4003459072, 4003463167, -+STORE, 4003463168, 4003495935, -+STORE, 4003495936, 4003569663, -+STORE, 4003569664, 4003573759, -+STORE, 4003573760, 4003704831, -+STORE, 4003704832, 4003708927, -+STORE, 4003708928, 4003713023, -+STORE, 4003713024, 4003737599, -+STORE, 4003737600, 4003770367, -+STORE, 4003770368, 4003876863, -+STORE, 4003876864, 4003880959, -+STORE, 4003880960, 4003885055, -+STORE, 4003885056, 4003889151, -+STORE, 4003889152, 4003893247, -+STORE, 4003893248, 4003897343, -+STORE, 4003897344, 4003962879, -+STORE, 4003962880, 4004069375, -+STORE, 4004069376, 4004093951, -+STORE, 4004093952, 4004118527, -+STORE, 4004118528, 4004122623, -+STORE, 4004122624, 4004126719, -+STORE, 4004126720, 4004155391, -+STORE, 4004155392, 4004286463, -+STORE, 4004286464, 4004384767, -+STORE, 4004384768, 4004388863, -+STORE, 4004388864, 4004646911, -+STORE, 4004646912, 4004655103, -+STORE, 4004655104, 4004659199, -+STORE, 4004659200, 4004667391, -+STORE, 4004667392, 4004683775, -+STORE, 4004683776, 4004814847, -+STORE, 4004814848, 4004818943, -+STORE, 4004818944, 4004823039, -+STORE, 4004823040, 4004827135, -+STORE, 4004827136, 4004835327, -+STORE, 4004835328, 4004954111, -+STORE, 4004954112, 4005085183, -+STORE, 4005085184, 4005306367, -+STORE, 4005306368, 4005765119, -+STORE, 4005765120, 4005789695, -+STORE, 4005789696, 4005793791, -+STORE, 4005793792, 4005801983, -+STORE, 4005801984, 4005920767, -+STORE, 4005920768, 4005945343, -+STORE, 4005945344, 4005949439, -+STORE, 4005949440, 4005986303, -+STORE, 4005986304, 4005990399, -+STORE, 4005990400, 4005994495, -+STORE, 4005994496, 4006002687, -+STORE, 4006002688, 4006109183, -+STORE, 4006109184, 4006117375, -+STORE, 4006117376, 4006121471, -+STORE, 4006121472, 4006133759, -+STORE, 4006133760, 4006137855, -+STORE, 4006137856, 4006141951, -+STORE, 4006141952, 4006150143, -+STORE, 4006150144, 4006391807, -+STORE, 4006391808, 4006445055, -+STORE, 4006445056, 4006563839, -+STORE, 4006563840, 4006572031, -+STORE, 4006572032, 4006576127, -+STORE, 4006576128, 4006584319, -+STORE, 4006584320, 4006694911, -+STORE, 4006694912, 4006739967, -+STORE, 4006739968, 4006776831, -+STORE, 4006776832, 4006785023, -+STORE, 4006785024, 4006789119, -+STORE, 4006789120, 4006797311, -+STORE, 4006797312, 4006813695, -+STORE, 4006813696, 4006846463, -+STORE, 4006846464, 4006977535, -+STORE, 4006977536, 4007006207, -+STORE, 4007006208, 4007010303, -+STORE, 4007010304, 4007067647, -+STORE, 4007067648, 4007075839, -+STORE, 4007075840, 4007084031, -+STORE, 4007084032, 4007100415, -+STORE, 4007100416, 4007116799, -+STORE, 4007116800, 4007133183, -+STORE, 4007133184, 4007153663, -+STORE, 4007153664, 4007178239, -+STORE, 4007178240, 4007202815, -+STORE, 4007202816, 4007206911, -+STORE, 4007206912, 4007272447, -+STORE, 4007272448, 4007276543, -+STORE, 4007276544, 4007280639, -+STORE, 4007280640, 4007284735, -+STORE, 4007284736, 4007292927, -+STORE, 4007292928, 4007423999, -+STORE, 4007424000, 4007448575, -+STORE, 4007448576, 4007452671, -+STORE, 4007452672, 4007505919, -+STORE, 4007505920, 4007510015, -+STORE, 4007510016, 4007514111, -+STORE, 4007514112, 4007645183, -+STORE, 4007645184, 4007776255, -+STORE, 4007776256, 4007780351, -+STORE, 4007780352, 4007784447, -+STORE, 4007784448, 4007788543, -+STORE, 4007788544, 4007809023, -+STORE, 4007809024, 4007829503, -+STORE, 4007829504, 4007960575, -+STORE, 4007960576, 4008091647, -+STORE, 4008091648, 4008296447, -+STORE, 4008296448, 4008890367, -+STORE, 4008890368, 4008898559, -+STORE, 4008898560, 4008902655, -+STORE, 4008902656, 4008996863, -+STORE, 4008996864, 4009041919, -+STORE, 4009041920, 4009082879, -+STORE, 4009082880, 4009091071, -+STORE, 4009091072, 4009107455, -+STORE, 4009107456, 4009349119, -+STORE, 4009349120, 4009373695, -+STORE, 4009373696, 4009414655, -+STORE, 4009414656, 4009422847, -+STORE, 4009422848, 4009426943, -+STORE, 4009426944, 4009447423, -+STORE, 4009447424, 4009471999, -+STORE, 4009472000, 4009512959, -+STORE, 4009512960, 4009594879, -+STORE, 4009594880, 4009598975, -+STORE, 4009598976, 4009697279, -+STORE, 4009697280, 4009713663, -+STORE, 4009713664, 4009717759, -+STORE, 4009717760, 4009721855, -+STORE, 4009721856, 4009730047, -+STORE, 4009730048, 4009861119, -+STORE, 4009861120, 4009951231, -+STORE, 4009951232, 4010131455, -+STORE, 4010131456, 4010135551, -+STORE, 4010135552, 4010139647, -+STORE, 4010139648, 4010143743, -+STORE, 4010143744, 4010164223, -+STORE, 4010164224, 4010295295, -+STORE, 4010295296, 4010299391, -+STORE, 4010299392, 4010491903, -+STORE, 4010491904, 4010495999, -+STORE, 4010496000, 4010668031, -+STORE, 4010668032, 4011028479, -+STORE, 4011028480, 4011053055, -+STORE, 4011053056, 4011057151, -+STORE, 4011057152, 4011118591, -+STORE, 4011118592, 4011126783, -+STORE, 4011126784, 4011130879, -+STORE, 4011130880, 4011143167, -+STORE, 4011143168, 4011147263, -+STORE, 4011147264, 4011167743, -+STORE, 4011167744, 4011171839, -+STORE, 4011171840, 4011360255, -+STORE, 4011360256, 4011364351, -+STORE, 4011364352, 4011626495, -+STORE, 4011626496, 4012216319, -+STORE, 4012216320, 4012228607, -+STORE, 4012228608, 4012232703, -+STORE, 4012232704, 4012236799, -+STORE, 4012236800, 4012240895, -+STORE, 4012240896, 4012261375, -+STORE, 4012261376, 4012392447, -+STORE, 4012392448, 4012466175, -+STORE, 4012466176, 4012597247, -+STORE, 4012597248, 4012601343, -+STORE, 4012601344, 4012605439, -+STORE, 4012605440, 4012609535, -+STORE, 4012609536, 4012679167, -+STORE, 4012679168, 4013563903, -+STORE, 4013563904, 4015366143, -+STORE, 4015366144, 4015411199, -+STORE, 4015411200, 4015415295, -+STORE, 4015415296, 4015419391, -+STORE, 4015419392, 4015542271, -+STORE, 4015542272, 4015550463, -+STORE, 4015550464, 4015558655, -+STORE, 4015558656, 4015562751, -+STORE, 4015562752, 4015583231, -+STORE, 4015583232, 4015587327, -+STORE, 4015587328, 4015603711, -+STORE, 4015665152, 4015669247, -+STORE, 4015669248, 4015812607, -+STORE, 4015812608, 4015816703, -+STORE, 4015816704, 4016111615, -+STORE, 4016111616, 4016467967, -+STORE, 4016467968, 4016508927, -+STORE, 4016508928, 4016517119, -+STORE, 4016517120, 4016525311, -+STORE, 4016525312, 4016586751, -+STORE, 4016586752, 4016664575, -+STORE, 4016664576, 4016697343, -+STORE, 4016697344, 4016742399, -+STORE, 4016742400, 4016746495, -+STORE, 4016746496, 4016750591, -+STORE, 4016750592, 4016758783, -+STORE, 4016799744, 4016844799, -+STORE, 4016844800, 4016902143, -+STORE, 4016902144, 4016992255, -+STORE, 4016992256, 4017000447, -+STORE, 4017000448, 4017004543, -+STORE, 4017004544, 4017008639, -+STORE, 4017008640, 4017016831, -+STORE, 4017016832, 4017020927, -+STORE, 4017020928, 4017127423, -+STORE, 4017127424, 4017131519, -+STORE, 4017131520, 4017229823, -+STORE, 4017229824, 4017422335, -+STORE, 4017422336, 4017438719, -+STORE, 4017438720, 4017442815, -+STORE, 4017442816, 4017446911, -+STORE, 4017446912, 4017455103, -+STORE, 4017455104, 4017766399, -+STORE, 4017766400, 4017909759, -+STORE, 4017909760, 4018081791, -+STORE, 4018081792, 4018089983, -+STORE, 4018089984, 4018094079, -+STORE, 4018094080, 4018098175, -+STORE, 4018098176, 4018327551, -+STORE, 4018327552, 4018331647, -+STORE, 4018331648, 4018339839, -+STORE, 4018339840, 4018348031, -+STORE, 4018348032, 4018610175, -+STORE, 4018610176, 4018626559, -+STORE, 4018626560, 4018647039, -+STORE, 4018647040, 4018651135, -+STORE, 4018651136, 4018749439, -+STORE, 4018749440, 4018761727, -+STORE, 4018761728, 4018802687, -+STORE, 4018802688, 4018806783, -+STORE, 4018806784, 4018810879, -+STORE, 4018810880, 4018814975, -+STORE, 4018814976, 4018823167, -+STORE, 4018823168, 4018954239, -+STORE, 4018954240, 4019007487, -+STORE, 4019007488, 4019068927, -+STORE, 4019068928, 4019077119, -+STORE, 4019077120, 4019081215, -+STORE, 4019081216, 4019093503, -+STORE, 4019093504, 4019208191, -+STORE, 4019208192, 4019232767, -+STORE, 4019232768, 4019265535, -+STORE, 4019265536, 4019269631, -+STORE, 4019269632, 4019277823, -+STORE, 4019277824, 4019458047, -+STORE, 4019458048, 4019519487, -+STORE, 4019519488, 4019613695, -+STORE, 4019613696, 4019621887, -+STORE, 4019621888, 4019625983, -+STORE, 4019625984, 4019630079, -+STORE, 4019630080, 4019744767, -+STORE, 4019744768, 4019822591, -+STORE, 4019822592, 4019929087, -+STORE, 4019929088, 4019941375, -+STORE, 4019941376, 4019945471, -+STORE, 4019945472, 4019961855, -+STORE, 4019961856, 4019994623, -+STORE, 4019994624, 4019998719, -+STORE, 4019998720, 4020002815, -+STORE, 4020002816, 4020006911, -+STORE, 4020006912, 4020011007, -+STORE, 4020011008, 4020256767, -+STORE, 4020256768, 4020326399, -+STORE, 4020326400, 4020457471, -+STORE, 4020457472, 4020469759, -+STORE, 4020469760, 4020473855, -+STORE, 4020473856, 4020482047, -+STORE, 4020482048, 4020711423, -+STORE, 4020711424, 4020715519, -+STORE, 4020715520, 4020719615, -+STORE, 4020719616, 4020723711, -+STORE, 4020723712, 4020805631, -+STORE, 4020805632, 4021051391, -+STORE, 4021051392, 4021460991, -+STORE, 4021460992, 4021469183, -+STORE, 4021469184, 4021473279, -+STORE, 4021473280, 4021571583, -+STORE, 4021571584, 4021633023, -+STORE, 4021633024, 4021727231, -+STORE, 4021727232, 4021735423, -+STORE, 4021735424, 4021739519, -+STORE, 4021739520, 4021747711, -+STORE, 4021747712, 4021829631, -+STORE, 4021829632, 4021866495, -+STORE, 4021866496, 4021919743, -+STORE, 4021919744, 4021927935, -+STORE, 4021927936, 4021932031, -+STORE, 4021932032, 4021944319, -+STORE, 4021944320, 4022157311, -+STORE, 4022157312, 4022161407, -+STORE, 4022161408, 4022173695, -+STORE, 4022173696, 4022177791, -+STORE, 4022177792, 4022472703, -+STORE, 4022472704, 4022509567, -+STORE, 4022509568, 4022583295, -+STORE, 4022583296, 4022587391, -+STORE, 4022587392, 4022591487, -+STORE, 4022591488, 4022607871, -+STORE, 4022607872, 4022657023, -+STORE, 4022657024, 4022722559, -+STORE, 4022722560, 4022730751, -+STORE, 4022730752, 4022734847, -+STORE, 4022734848, 4022865919, -+STORE, 4022865920, 4022943743, -+STORE, 4022943744, 4023062527, -+STORE, 4023062528, 4023074815, -+STORE, 4023074816, 4023078911, -+STORE, 4023078912, 4023128063, -+STORE, 4023128064, 4023218175, -+STORE, 4023218176, 4023361535, -+STORE, 4023361536, 4023373823, -+STORE, 4023373824, 4023377919, -+STORE, 4023377920, 4023558143, -+STORE, 4023558144, 4023631871, -+STORE, 4023631872, 4023816191, -+STORE, 4023816192, 4023820287, -+STORE, 4023820288, 4023824383, -+STORE, 4023824384, 4023832575, -+STORE, 4023832576, 4024078335, -+STORE, 4024078336, 4024197119, -+STORE, 4024197120, 4024389631, -+STORE, 4024389632, 4024406015, -+STORE, 4024406016, 4024410111, -+STORE, 4024410112, 4024422399, -+STORE, 4024422400, 4024619007, -+STORE, 4024619008, 4024639487, -+STORE, 4024639488, 4024655871, -+STORE, 4024655872, 4024664063, -+STORE, 4024664064, 4024668159, -+STORE, 4024668160, 4024676351, -+STORE, 4024676352, 4024905727, -+STORE, 4024905728, 4024909823, -+STORE, 4024909824, 4024918015, -+STORE, 4024918016, 4024922111, -+STORE, 4024922112, 4024930303, -+STORE, 4024930304, 4025110527, -+STORE, 4025110528, 4025176063, -+STORE, 4025176064, 4025208831, -+STORE, 4025208832, 4025212927, -+STORE, 4025212928, 4025217023, -+STORE, 4025217024, 4025348095, -+STORE, 4025348096, 4025372671, -+STORE, 4025372672, 4025458687, -+STORE, 4025458688, 4025466879, -+STORE, 4025466880, 4025565183, -+STORE, 4025565184, 4025757695, -+STORE, 4025757696, 4026249215, -+STORE, 4026249216, 4026261503, -+STORE, 4026261504, 4026265599, -+STORE, 4026265600, 4026269695, -+STORE, 4026269696, 4026302463, -+STORE, 4026302464, 4026306559, -+STORE, 4026306560, 4026314751, -+STORE, 4026314752, 4026318847, -+STORE, 4026318848, 4026322943, -+STORE, 4026322944, 4026327039, -+STORE, 4026327040, 4026654719, -+STORE, 4026654720, 4026671103, -+STORE, 4026671104, 4026720255, -+STORE, 4026720256, 4026724351, -+STORE, 4026724352, 4026728447, -+STORE, 4026728448, 4026732543, -+STORE, 4026732544, 4026863615, -+STORE, 4026863616, 4027027455, -+STORE, 4027027456, 4027031551, -+STORE, 4027031552, 4027514879, -+STORE, 4027514880, 4027531263, -+STORE, 4027531264, 4027535359, -+STORE, 4027535360, 4027539455, -+STORE, 4027539456, 4027785215, -+STORE, 4027785216, 4027789311, -+STORE, 4027789312, 4027793407, -+STORE, 4027793408, 4027797503, -+STORE, 4027797504, 4027863039, -+STORE, 4027863040, 4027899903, -+STORE, 4027899904, 4027949055, -+STORE, 4027949056, 4027957247, -+STORE, 4027957248, 4027961343, -+STORE, 4027961344, 4027965439, -+STORE, 4027965440, 4028194815, -+STORE, 4028194816, 4028252159, -+STORE, 4028252160, 4028338175, -+STORE, 4028338176, 4028350463, -+STORE, 4028350464, 4028354559, -+STORE, 4028354560, 4028452863, -+STORE, 4028452864, 4028489727, -+STORE, 4028489728, 4028530687, -+STORE, 4028530688, 4028538879, -+STORE, 4028538880, 4028542975, -+STORE, 4028542976, 4028551167, -+STORE, 4028551168, 4028665855, -+STORE, 4028665856, 4029349887, -+STORE, 4029349888, 4030468095, -+STORE, 4030468096, 4030513151, -+STORE, 4030513152, 4030517247, -+STORE, 4030517248, 4030525439, -+STORE, 4030525440, 4030529535, -+STORE, 4030529536, 4030758911, -+STORE, 4030758912, 4030828543, -+STORE, 4030828544, 4030943231, -+STORE, 4030943232, 4030951423, -+STORE, 4030951424, 4030955519, -+STORE, 4030955520, 4030967807, -+STORE, 4030967808, 4031131647, -+STORE, 4031131648, 4031135743, -+STORE, 4031135744, 4031139839, -+STORE, 4031139840, 4031148031, -+STORE, 4031148032, 4031152127, -+STORE, 4031152128, 4031160319, -+STORE, 4031160320, 4031504383, -+STORE, 4031504384, 4031598591, -+STORE, 4031598592, 4031754239, -+STORE, 4031754240, 4031766527, -+STORE, 4031766528, 4031770623, -+STORE, 4031770624, 4031774719, -+STORE, 4031774720, 4031782911, -+STORE, 4031782912, 4031799295, -+STORE, 4031799296, 4031856639, -+STORE, 4031856640, 4031983615, -+STORE, 4031983616, 4031987711, -+STORE, 4031987712, 4031991807, -+STORE, 4031991808, 4032270335, -+STORE, 4032270336, 4032274431, -+STORE, 4032274432, 4032282623, -+STORE, 4032282624, 4032286719, -+STORE, 4032286720, 4032290815, -+STORE, 4032290816, 4032389119, -+STORE, 4032389120, 4032397311, -+STORE, 4032397312, 4032405503, -+STORE, 4032405504, 4032413695, -+STORE, 4032413696, 4032417791, -+STORE, 4032417792, 4032565247, -+STORE, 4032565248, 4032593919, -+STORE, 4032593920, 4032737279, -+STORE, 4032737280, 4032741375, -+STORE, 4032741376, 4032745471, -+STORE, 4032745472, 4032770047, -+STORE, 4032770048, 4032933887, -+STORE, 4032933888, 4032999423, -+STORE, 4032999424, 4033032191, -+STORE, 4033032192, 4033036287, -+STORE, 4033036288, 4033040383, -+STORE, 4033040384, 4033105919, -+STORE, 4033105920, 4033396735, -+STORE, 4033396736, 4033822719, -+STORE, 4033822720, 4033839103, -+STORE, 4033839104, 4033843199, -+STORE, 4033843200, 4033851391, -+STORE, 4033851392, 4033863679, -+STORE, 4033863680, 4033880063, -+STORE, 4033880064, 4033933311, -+STORE, 4033933312, 4034023423, -+STORE, 4034023424, 4034031615, -+STORE, 4034031616, 4034035711, -+STORE, 4034035712, 4034043903, -+STORE, 4034043904, 4034142207, -+STORE, 4034142208, 4034191359, -+STORE, 4034191360, 4034260991, -+STORE, 4034260992, 4034269183, -+STORE, 4034269184, 4034273279, -+STORE, 4034273280, 4034281471, -+STORE, 4034281472, 4034412543, -+STORE, 4034412544, 4034445311, -+STORE, 4034445312, 4034490367, -+STORE, 4034490368, 4034494463, -+STORE, 4034494464, 4034498559, -+STORE, 4034498560, 4034662399, -+STORE, 4034662400, 4034666495, -+STORE, 4034666496, 4034670591, -+STORE, 4034670592, 4034674687, -+STORE, 4034674688, 4034678783, -+STORE, 4034678784, 4034682879, -+STORE, 4034682880, 4034781183, -+STORE, 4034781184, 4035043327, -+STORE, 4035043328, 4035047423, -+STORE, 4035047424, 4035055615, -+STORE, 4035055616, 4035059711, -+STORE, 4035059712, 4035063807, -+STORE, 4035063808, 4035067903, -+STORE, 4035067904, 4035100671, -+STORE, 4035100672, 4035375103, -+STORE, 4035375104, 4035383295, -+STORE, 4035383296, 4035395583, -+STORE, 4035395584, 4035399679, -+STORE, 4035399680, 4035403775, -+STORE, 4035403776, 4035407871, -+STORE, 4035407872, 4035411967, -+STORE, 4035411968, 4035477503, -+STORE, 4035477504, 4035608575, -+STORE, 4035608576, 4035641343, -+STORE, 4035641344, 4035682303, -+STORE, 4035682304, 4035686399, -+STORE, 4035686400, 4035690495, -+STORE, 4035690496, 4035694591, -+STORE, 4035694592, 4035743743, -+STORE, 4035743744, 4035784703, -+STORE, 4035784704, 4035829759, -+STORE, 4035829760, 4035837951, -+STORE, 4035837952, 4035842047, -+STORE, 4035842048, 4035846143, -+STORE, 4035846144, 4035850239, -+STORE, 4035850240, 4036001791, -+STORE, 4036001792, 4036005887, -+STORE, 4036005888, 4036214783, -+STORE, 4036214784, 4036218879, -+STORE, 4036218880, 4036603903, -+STORE, 4036603904, 4036648959, -+STORE, 4036648960, 4036653055, -+STORE, 4036653056, 4036657151, -+STORE, 4036657152, 4036665343, -+STORE, 4036665344, 4036780031, -+STORE, 4036780032, 4036829183, -+STORE, 4036829184, 4036984831, -+STORE, 4036984832, 4036993023, -+STORE, 4036993024, 4036997119, -+STORE, 4036997120, 4037001215, -+STORE, 4037001216, 4037009407, -+STORE, 4037009408, 4037025791, -+STORE, 4037025792, 4037095423, -+STORE, 4037095424, 4037181439, -+STORE, 4037181440, 4037193727, -+STORE, 4037193728, 4037197823, -+STORE, 4037197824, 4037206015, -+STORE, 4037206016, 4037320703, -+STORE, 4037320704, 4037337087, -+STORE, 4037337088, 4037349375, -+STORE, 4037349376, 4037357567, -+STORE, 4037357568, 4037361663, -+STORE, 4037369856, 4037386239, -+STORE, 4037386240, 4037672959, -+STORE, 4037672960, 4037689343, -+STORE, 4037689344, 4037730303, -+STORE, 4037730304, 4037734399, -+STORE, 4037734400, 4037738495, -+STORE, 4037738496, 4037742591, -+STORE, 4037742592, 4037758975, -+STORE, 4037758976, 4037890047, -+STORE, 4037890048, 4037931007, -+STORE, 4037931008, 4037976063, -+STORE, 4037976064, 4037984255, -+STORE, 4037984256, 4037988351, -+STORE, 4037988352, 4038053887, -+STORE, 4038053888, 4038184959, -+STORE, 4038184960, 4038189055, -+STORE, 4038189056, 4038197247, -+STORE, 4038197248, 4038201343, -+STORE, 4038201344, 4038205439, -+STORE, 4038205440, 4038209535, -+STORE, 4038217728, 4038250495, -+STORE, 4038250496, 4038512639, -+STORE, 4038512640, 4038516735, -+STORE, 4038516736, 4038520831, -+STORE, 4038520832, 4038524927, -+STORE, 4038524928, 4038529023, -+STORE, 4038529024, 4038533119, -+STORE, 4038541312, 4038623231, -+STORE, 4038623232, 4038754303, -+STORE, 4038754304, 4038885375, -+STORE, 4038885376, 4038889471, -+STORE, 4038897664, 4038963199, -+STORE, 4038963200, 4038967295, -+STORE, 4038967296, 4038983679, -+STORE, 4038983680, 4039114751, -+STORE, 4039114752, 4039245823, -+STORE, 4039245824, 4039376895, -+STORE, 4039376896, 4040687615, -+STORE, 4040687616, 4040691711, -+STORE, 4040691712, 4040806399, -+STORE, 4040806400, 4040937471, -+STORE, 4040937472, 4040941567, -+STORE, 4040945664, 4040949759, -+STORE, 4040949760, 4041080831, -+STORE, 4041080832, 4041211903, -+STORE, 4041211904, 4043046911, -+STORE, 4043046912, 4043051007, -+STORE, 4043051008, 4043055103, -+STORE, 4043055104, 4043137023, -+STORE, 4043137024, 4043141119, -+STORE, 4043141120, 4043145215, -+STORE, 4043145216, 4043153407, -+STORE, 4043153408, 4043186175, -+STORE, 4043186176, 4043317247, -+STORE, 4043317248, 4043448319, -+STORE, 4043448320, 4043579391, -+STORE, 4043579392, 4043583487, -+STORE, 4043583488, 4043599871, -+STORE, 4043599872, 4043661311, -+STORE, 4043661312, 4043792383, -+STORE, 4043792384, 4043796479, -+STORE, 4043796480, 4043800575, -+STORE, 4043800576, 4043816959, -+STORE, 4043816960, 4043821055, -+STORE, 4043821056, 4043825151, -+STORE, 4043825152, 4043829247, -+STORE, 4043829248, 4043833343, -+STORE, 4043833344, 4047241215, -+STORE, 4047241216, 4047249407, -+STORE, 4047249408, 4047253503, -+STORE, 4047253504, 4047323135, -+STORE, 4047323136, 4047327231, -+STORE, 4047327232, 4047458303, -+STORE, 4047458304, 4047589375, -+STORE, 4047589376, 4047720447, -+STORE, 4047720448, 4047773695, -+STORE, 4047773696, 4047790079, -+STORE, 4047790080, 4047921151, -+STORE, 4047921152, 4048052223, -+STORE, 4048052224, 4048183295, -+STORE, 4048183296, 4049002495, -+STORE, 4049002496, 4049133567, -+STORE, 4049133568, 4049154047, -+STORE, 4049154048, 4049158143, -+STORE, 4049158144, 4049162239, -+STORE, 4049162240, 4049166335, -+STORE, 4049166336, 4049174527, -+STORE, 4049174528, 4049182719, -+STORE, 4049182720, 4049186815, -+STORE, 4049186816, 4049190911, -+STORE, 4049190912, 4049195007, -+STORE, 4049195008, 4049203199, -+STORE, 4049203200, 4049207295, -+STORE, 4049207296, 4049211391, -+STORE, 4049211392, 4049215487, -+STORE, 4049215488, 4049219583, -+STORE, 4049219584, 4049227775, -+STORE, 4049227776, 4049231871, -+STORE, 4049231872, 4049235967, -+STORE, 4049235968, 4049244159, -+STORE, 4049244160, 4049248255, -+STORE, 4049248256, 4049252351, -+STORE, 4049252352, 4049256447, -+STORE, 4049256448, 4049268735, -+STORE, 4049268736, 4049272831, -+STORE, 4049272832, 4049313791, -+STORE, 4049313792, 4049723391, -+STORE, 4049723392, 4049727487, -+STORE, 4049727488, 4049858559, -+STORE, 4049858560, 4049989631, -+STORE, 4049989632, 4049993727, -+STORE, 4049993728, 4050026495, -+STORE, 4050026496, 4050030591, -+STORE, 4050030592, 4050161663, -+STORE, 4050161664, 4050169855, -+STORE, 4050169856, 4050223103, -+STORE, 4050223104, 4050632703, -+STORE, 4050632704, 4050636799, -+STORE, 4050636800, 4050640895, -+STORE, 4050640896, 4050644991, -+STORE, 4050644992, 4050661375, -+STORE, 4050661376, 4050665471, -+STORE, 4050665472, 4050673663, -+STORE, 4050673664, 4050677759, -+STORE, 4050677760, 4050694143, -+STORE, 4050694144, 4050702335, -+STORE, 4050702336, 4050956287, -+STORE, 4050956288, 4051963903, -+STORE, 4051963904, 4051980287, -+STORE, 4051980288, 4051988479, -+STORE, 4051988480, 4052000767, -+STORE, 4052000768, 4052004863, -+STORE, 4052004864, 4052029439, -+STORE, 4284014592, 4284018687, -+STORE, 4284018688, 4292403199, -+SNULL, 4041080832, 4041211903, -+SNULL, 3795763200, 3795894271, -+STORE, 3629522944, 3696631807, -+SNULL, 3663077375, 3696631807, -+STORE, 3629522944, 3663077375, -+STORE, 3663077376, 3696631807, -+SNULL, 3663077376, 3696631807, -+STORE, 3663077376, 3696631807, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3626471424, 3627524095, -+SNULL, 3626471424, 3626475519, -+STORE, 3626475520, 3627524095, -+STORE, 3626471424, 3626475519, -+SNULL, 3627519999, 3627524095, -+STORE, 3626475520, 3627519999, -+STORE, 3627520000, 3627524095, -+STORE, 3625418752, 3626475519, -+SNULL, 3625418752, 3625422847, -+STORE, 3625422848, 3626475519, -+STORE, 3625418752, 3625422847, -+SNULL, 3626467327, 3626475519, -+STORE, 3625422848, 3626467327, -+STORE, 3626467328, 3626475519, -+STORE, 3624366080, 3625422847, -+SNULL, 3624366080, 3624370175, -+STORE, 3624370176, 3625422847, -+STORE, 3624366080, 3624370175, -+SNULL, 3625414655, 3625422847, -+STORE, 3624370176, 3625414655, -+STORE, 3625414656, 3625422847, -+STORE, 4041191424, 4041211903, -+SNULL, 4041195519, 4041211903, -+STORE, 4041191424, 4041195519, -+STORE, 4041195520, 4041211903, -+STORE, 4041170944, 4041191423, -+SNULL, 4041175039, 4041191423, -+STORE, 4041170944, 4041175039, -+STORE, 4041175040, 4041191423, -+SNULL, 3625426943, 3626467327, -+STORE, 3625422848, 3625426943, -+STORE, 3625426944, 3626467327, -+STORE, 4041162752, 4041170943, -+SNULL, 3626479615, 3627519999, -+STORE, 3626475520, 3626479615, -+STORE, 3626479616, 3627519999, -+STORE, 4041154560, 4041162751, -+STORE, 4041154560, 4041170943, -+STORE, 4041134080, 4041154559, -+SNULL, 4041138175, 4041154559, -+STORE, 4041134080, 4041138175, -+STORE, 4041138176, 4041154559, -+SNULL, 3624374271, 3625414655, -+STORE, 3624370176, 3624374271, -+STORE, 3624374272, 3625414655, -+STORE, 4041125888, 4041134079, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+STORE, 3487174656, 3487584255, -+STORE, 4041121792, 4041125887, -+SNULL, 4041121792, 4041125887, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 3487174656, 3487584255, -+STORE, 3222274048, 3223326719, -+SNULL, 3222274048, 3222278143, -+STORE, 3222278144, 3223326719, -+STORE, 3222274048, 3222278143, -+SNULL, 3223322623, 3223326719, -+STORE, 3222278144, 3223322623, -+STORE, 3223322624, 3223326719, -+STORE, 3221221376, 3222278143, -+SNULL, 3221221376, 3221225471, -+STORE, 3221225472, 3222278143, -+STORE, 3221221376, 3221225471, -+SNULL, 3222269951, 3222278143, -+STORE, 3221225472, 3222269951, -+STORE, 3222269952, 3222278143, -+STORE, 3220168704, 3221225471, -+SNULL, 3220168704, 3220172799, -+STORE, 3220172800, 3221225471, -+STORE, 3220168704, 3220172799, -+SNULL, 3221217279, 3221225471, -+STORE, 3220172800, 3221217279, -+STORE, 3221217280, 3221225471, -+STORE, 4041117696, 4041125887, -+STORE, 4041117696, 4041134079, -+STORE, 3219083264, 3220172799, -+SNULL, 3219083264, 3219087359, -+STORE, 3219087360, 3220172799, -+STORE, 3219083264, 3219087359, -+SNULL, 3220164607, 3220172799, -+STORE, 3219087360, 3220164607, -+STORE, 3220164608, 3220172799, -+STORE, 4041109504, 4041117695, -+STORE, 4041109504, 4041134079, -+STORE, 3217997824, 3219087359, -+SNULL, 3217997824, 3218001919, -+STORE, 3218001920, 3219087359, -+STORE, 3217997824, 3218001919, -+SNULL, 3219079167, 3219087359, -+STORE, 3218001920, 3219079167, -+STORE, 3219079168, 3219087359, -+STORE, 4041101312, 4041109503, -+STORE, 4041101312, 4041134079, -+STORE, 3216912384, 3218001919, -+SNULL, 3216912384, 3216916479, -+STORE, 3216916480, 3218001919, -+STORE, 3216912384, 3216916479, -+SNULL, 3217993727, 3218001919, -+STORE, 3216916480, 3217993727, -+STORE, 3217993728, 3218001919, -+STORE, 4041093120, 4041101311, -+STORE, 4041093120, 4041134079, -+STORE, 3215826944, 3216916479, -+SNULL, 3215826944, 3215831039, -+STORE, 3215831040, 3216916479, -+STORE, 3215826944, 3215831039, -+SNULL, 3216908287, 3216916479, -+STORE, 3215831040, 3216908287, -+STORE, 3216908288, 3216916479, -+STORE, 4016779264, 4016799743, -+SNULL, 4016783359, 4016799743, -+STORE, 4016779264, 4016783359, -+STORE, 4016783360, 4016799743, -+STORE, 4016758784, 4016779263, -+SNULL, 4016762879, 4016779263, -+STORE, 4016758784, 4016762879, -+STORE, 4016762880, 4016779263, -+SNULL, 3222282239, 3223322623, -+STORE, 3222278144, 3222282239, -+STORE, 3222282240, 3223322623, -+STORE, 4041084928, 4041093119, -+STORE, 4041084928, 4041134079, -+SNULL, 3221229567, 3222269951, -+STORE, 3221225472, 3221229567, -+STORE, 3221229568, 3222269951, -+STORE, 4015644672, 4015665151, -+STORE, 4038889472, 4038897663, -+SNULL, 4015648767, 4015665151, -+STORE, 4015644672, 4015648767, -+STORE, 4015648768, 4015665151, -+STORE, 4015624192, 4015644671, -+SNULL, 4015628287, 4015644671, -+STORE, 4015624192, 4015628287, -+STORE, 4015628288, 4015644671, -+SNULL, 3219091455, 3220164607, -+STORE, 3219087360, 3219091455, -+STORE, 3219091456, 3220164607, -+STORE, 4015603712, 4015624191, -+SNULL, 4015607807, 4015624191, -+STORE, 4015603712, 4015607807, -+STORE, 4015607808, 4015624191, -+SNULL, 3218006015, 3219079167, -+STORE, 3218001920, 3218006015, -+STORE, 3218006016, 3219079167, -+STORE, 3949674496, 3949694975, -+SNULL, 3949678591, 3949694975, -+STORE, 3949674496, 3949678591, -+STORE, 3949678592, 3949694975, -+SNULL, 3216920575, 3217993727, -+STORE, 3216916480, 3216920575, -+STORE, 3216920576, 3217993727, -+STORE, 3948924928, 3948945407, -+SNULL, 3948929023, 3948945407, -+STORE, 3948924928, 3948929023, -+STORE, 3948929024, 3948945407, -+SNULL, 3215835135, 3216908287, -+STORE, 3215831040, 3215835135, -+STORE, 3215835136, 3216908287, -+SNULL, 3220176895, 3221217279, -+STORE, 3220172800, 3220176895, -+STORE, 3220176896, 3221217279, -+STORE, 3214786560, 3215826943, -+STORE, 3213733888, 3214786559, -+SNULL, 3213733888, 3213737983, -+STORE, 3213737984, 3214786559, -+STORE, 3213733888, 3213737983, -+SNULL, 3214782463, 3214786559, -+STORE, 3213737984, 3214782463, -+STORE, 3214782464, 3214786559, -+STORE, 4038533120, 4038541311, -+STORE, 3948421120, 3948441599, -+SNULL, 3948425215, 3948441599, -+STORE, 3948421120, 3948425215, -+STORE, 3948425216, 3948441599, -+SNULL, 3213742079, 3214782463, -+STORE, 3213737984, 3213742079, -+STORE, 3213742080, 3214782463, -+STORE, 4038209536, 4038217727, -+STORE, 3212681216, 3213737983, -+SNULL, 3212681216, 3212685311, -+STORE, 3212685312, 3213737983, -+STORE, 3212681216, 3212685311, -+SNULL, 3213729791, 3213737983, -+STORE, 3212685312, 3213729791, -+STORE, 3213729792, 3213737983, -+STORE, 3795763200, 3795894271, -+STORE, 3946872832, 3946893311, -+SNULL, 3946876927, 3946893311, -+STORE, 3946872832, 3946876927, -+STORE, 3946876928, 3946893311, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+STORE, 3487174656, 3487584255, -+SNULL, 3212689407, 3213729791, -+STORE, 3212685312, 3212689407, -+STORE, 3212689408, 3213729791, -+STORE, 4041080832, 4041084927, -+STORE, 4040941568, 4040945663, -+STORE, 4037361664, 4037369855, -+STORE, 4000817152, 4000821247, -+STORE, 3999440896, 3999444991, -+STORE, 3212161024, 3212681215, -+SNULL, 3212161024, 3212439551, -+STORE, 3212439552, 3212681215, -+STORE, 3212161024, 3212439551, -+SNULL, 3212161024, 3212439551, -+SNULL, 3212464127, 3212681215, -+STORE, 3212439552, 3212464127, -+STORE, 3212464128, 3212681215, -+SNULL, 3212464128, 3212681215, -+SNULL, 3212439552, 3212451839, -+STORE, 3212451840, 3212464127, -+STORE, 3212439552, 3212451839, -+SNULL, 3212439552, 3212451839, -+STORE, 3212439552, 3212451839, -+SNULL, 3212451840, 3212455935, -+STORE, 3212455936, 3212464127, -+STORE, 3212451840, 3212455935, -+SNULL, 3212451840, 3212455935, -+STORE, 3212451840, 3212455935, -+SNULL, 3212455936, 3212460031, -+STORE, 3212460032, 3212464127, -+STORE, 3212455936, 3212460031, -+SNULL, 3212455936, 3212460031, -+STORE, 3212455936, 3212460031, -+SNULL, 3212460032, 3212464127, -+STORE, 3212460032, 3212464127, -+STORE, 3997679616, 3997683711, -+SNULL, 4049235968, 4049240063, -+STORE, 4049240064, 4049244159, -+STORE, 4049235968, 4049240063, -+SNULL, 4049240064, 4049244159, -+STORE, 4049240064, 4049244159, -+SNULL, 3997679616, 3997683711, -+SNULL, 3999440896, 3999444991, -+SNULL, 4000817152, 4000821247, -+SNULL, 4040941568, 4040945663, -+SNULL, 4041080832, 4041084927, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 3487174656, 3487584255, -+SNULL, 3212451840, 3212455935, -+STORE, 3212451840, 3212455935, -+STORE, 4041080832, 4041084927, -+STORE, 3623890944, 3624169471, -+SNULL, 4041080832, 4041084927, -+STORE, 4041080832, 4041084927, -+SNULL, 4041080832, 4041084927, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+STORE, 4041080832, 4041084927, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+STORE, 3211386880, 3212439551, -+SNULL, 3211386880, 3211390975, -+STORE, 3211390976, 3212439551, -+STORE, 3211386880, 3211390975, -+SNULL, 3212435455, 3212439551, -+STORE, 3211390976, 3212435455, -+STORE, 3212435456, 3212439551, -+STORE, 4040941568, 4040945663, -+STORE, 3937169408, 3937189887, -+STORE, 3623485440, 3623616511, -+SNULL, 717225983, 1388314623, -+STORE, 314572800, 717225983, -+STORE, 717225984, 1388314623, -+SNULL, 717225984, 1388314623, -+STORE, 3937112064, 3937132543, -+SNULL, 3937116159, 3937132543, -+STORE, 3937112064, 3937116159, -+STORE, 3937116160, 3937132543, -+SNULL, 3211395071, 3212435455, -+STORE, 3211390976, 3211395071, -+STORE, 3211395072, 3212435455, -+STORE, 4000817152, 4000821247, -+STORE, 3974823936, 3974832127, -+STORE, 3595284480, 3595431935, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+STORE, 3487174656, 3487584255, -+STORE, 3999440896, 3999444991, -+STORE, 3997679616, 3997683711, -+STORE, 3996295168, 3996299263, -+STORE, 3996090368, 3996094463, -+STORE, 3210866688, 3211386879, -+SNULL, 3210866688, 3211001855, -+STORE, 3211001856, 3211386879, -+STORE, 3210866688, 3211001855, -+SNULL, 3210866688, 3211001855, -+SNULL, 3211038719, 3211386879, -+STORE, 3211001856, 3211038719, -+STORE, 3211038720, 3211386879, -+SNULL, 3211038720, 3211386879, -+SNULL, 3211001856, 3211022335, -+STORE, 3211022336, 3211038719, -+STORE, 3211001856, 3211022335, -+SNULL, 3211001856, 3211022335, -+STORE, 3211001856, 3211022335, -+SNULL, 3211022336, 3211030527, -+STORE, 3211030528, 3211038719, -+STORE, 3211022336, 3211030527, -+SNULL, 3211022336, 3211030527, -+STORE, 3211022336, 3211030527, -+SNULL, 3211030528, 3211034623, -+STORE, 3211034624, 3211038719, -+STORE, 3211030528, 3211034623, -+SNULL, 3211030528, 3211034623, -+STORE, 3211030528, 3211034623, -+SNULL, 3211034624, 3211038719, -+STORE, 3211034624, 3211038719, -+STORE, 3994906624, 3994910719, -+SNULL, 4049240064, 4049244159, -+STORE, 4049240064, 4049244159, -+SNULL, 3994906624, 3994910719, -+SNULL, 3996090368, 3996094463, -+SNULL, 3996295168, 3996299263, -+SNULL, 3997679616, 3997683711, -+SNULL, 3999440896, 3999444991, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 3487174656, 3487584255, -+SNULL, 3211022336, 3211030527, -+STORE, 3211022336, 3211030527, -+STORE, 3999440896, 3999444991, -+STORE, 3210199040, 3211001855, -+SNULL, 3999440896, 3999444991, -+STORE, 3999440896, 3999444991, -+SNULL, 3999440896, 3999444991, -+STORE, 3594821632, 3594952703, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 4048183296, 4048592895, -+STORE, 4048592896, 4049002495, -+STORE, 4048183296, 4048592895, -+STORE, 4048183296, 4049002495, -+SNULL, 1914101759, 1969434623, -+STORE, 1914097664, 1914101759, -+STORE, 1914101760, 1969434623, -+STORE, 3567108096, 3567239167, -+STORE, 3973832704, 3973840895, -+STORE, 3209113600, 3210199039, -+SNULL, 3209113600, 3209117695, -+STORE, 3209117696, 3210199039, -+STORE, 3209113600, 3209117695, -+SNULL, 3210194943, 3210199039, -+STORE, 3209117696, 3210194943, -+STORE, 3210194944, 3210199039, -+STORE, 3935858688, 3935879167, -+SNULL, 3935862783, 3935879167, -+STORE, 3935858688, 3935862783, -+STORE, 3935862784, 3935879167, -+SNULL, 3209121791, 3210194943, -+STORE, 3209117696, 3209121791, -+STORE, 3209121792, 3210194943, -+STORE, 3528749056, 3528880127, -+STORE, 3968200704, 3968208895, -+STORE, 3208028160, 3209117695, -+SNULL, 3208028160, 3208032255, -+STORE, 3208032256, 3209117695, -+STORE, 3208028160, 3208032255, -+SNULL, 3209109503, 3209117695, -+STORE, 3208032256, 3209109503, -+STORE, 3209109504, 3209117695, -+STORE, 3888123904, 3888144383, -+SNULL, 3888127999, 3888144383, -+STORE, 3888123904, 3888127999, -+STORE, 3888128000, 3888144383, -+SNULL, 3208036351, 3209109503, -+STORE, 3208032256, 3208036351, -+STORE, 3208036352, 3209109503, -+SNULL, 3968200704, 3968208895, -+SNULL, 3888123904, 3888144383, -+SNULL, 3209109504, 3209113599, -+STORE, 3209113600, 3209117695, -+STORE, 3209109504, 3209113599, -+SNULL, 3208028160, 3209113599, -+STORE, 3208060928, 3209117695, -+SNULL, 3208060928, 3208065023, -+STORE, 3208065024, 3209117695, -+STORE, 3208060928, 3208065023, -+SNULL, 3209109503, 3209117695, -+STORE, 3208065024, 3209109503, -+STORE, 3209109504, 3209117695, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3888123904, 3888144383, -+SNULL, 3888127999, 3888144383, -+STORE, 3888123904, 3888127999, -+STORE, 3888128000, 3888144383, -+SNULL, 3208069119, 3209109503, -+STORE, 3208065024, 3208069119, -+STORE, 3208069120, 3209109503, -+STORE, 3968200704, 3968208895, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3527778304, 3527909375, -+STORE, 3999440896, 3999444991, -+STORE, 3997679616, 3997683711, -+STORE, 1914097664, 1914105855, -+STORE, 1914105856, 1969434623, -+STORE, 3957583872, 3957592063, -+STORE, 3206975488, 3208065023, -+SNULL, 3206975488, 3206979583, -+STORE, 3206979584, 3208065023, -+STORE, 3206975488, 3206979583, -+SNULL, 3208056831, 3208065023, -+STORE, 3206979584, 3208056831, -+STORE, 3208056832, 3208065023, -+STORE, 3956736000, 3956744191, -+STORE, 3205890048, 3206979583, -+SNULL, 3205890048, 3205894143, -+STORE, 3205894144, 3206979583, -+STORE, 3205890048, 3205894143, -+SNULL, 3206971391, 3206979583, -+STORE, 3205894144, 3206971391, -+STORE, 3206971392, 3206979583, -+STORE, 3806101504, 3806121983, -+SNULL, 3806105599, 3806121983, -+STORE, 3806101504, 3806105599, -+STORE, 3806105600, 3806121983, -+SNULL, 3206983679, 3208056831, -+STORE, 3206979584, 3206983679, -+STORE, 3206983680, 3208056831, -+STORE, 3806081024, 3806101503, -+SNULL, 3806085119, 3806101503, -+STORE, 3806081024, 3806085119, -+STORE, 3806085120, 3806101503, -+SNULL, 3205898239, 3206971391, -+STORE, 3205894144, 3205898239, -+STORE, 3205898240, 3206971391, -+STORE, 3956015104, 3956023295, -+STORE, 3204804608, 3205894143, -+SNULL, 3204804608, 3204808703, -+STORE, 3204808704, 3205894143, -+STORE, 3204804608, 3204808703, -+SNULL, 3205885951, 3205894143, -+STORE, 3204808704, 3205885951, -+STORE, 3205885952, 3205894143, -+STORE, 3803471872, 3803492351, -+STORE, 3803451392, 3803471871, -+STORE, 3803451392, 3803492351, -+SNULL, 3957583872, 3957592063, -+SNULL, 3806101504, 3806121983, -+SNULL, 3206975487, 3206979583, -+STORE, 3206971392, 3206975487, -+STORE, 3206975488, 3206979583, -+SNULL, 3208056832, 3208060927, -+STORE, 3208060928, 3208065023, -+STORE, 3208056832, 3208060927, -+SNULL, 3206975488, 3208060927, -+STORE, 3801845760, 3801878527, -+STORE, 3806101504, 3806121983, -+SNULL, 3806105599, 3806121983, -+STORE, 3806101504, 3806105599, -+STORE, 3806105600, 3806121983, -+SNULL, 3204812799, 3205885951, -+STORE, 3204808704, 3204812799, -+STORE, 3204812800, 3205885951, -+STORE, 1914097664, 1914109951, -+STORE, 1914109952, 1969434623, -+STORE, 3957583872, 3957592063, -+STORE, 3206971392, 3208065023, -+SNULL, 3206971392, 3206979583, -+STORE, 3206979584, 3208065023, -+STORE, 3206971392, 3206979583, -+SNULL, 3208056831, 3208065023, -+STORE, 3206979584, 3208056831, -+STORE, 3208056832, 3208065023, -+STORE, 3801825280, 3801845759, -+SNULL, 3801829375, 3801845759, -+STORE, 3801825280, 3801829375, -+STORE, 3801829376, 3801845759, -+SNULL, 3206983679, 3208056831, -+STORE, 3206979584, 3206983679, -+STORE, 3206983680, 3208056831, -+STORE, 3202707456, 3204804607, -+SNULL, 3202707456, 3204804607, -+STORE, 3202707456, 3204804607, -+STORE, 3200610304, 3202707455, -+SNULL, 3202707456, 3204804607, -+SNULL, 3200610304, 3202707455, -+STORE, 3202707456, 3204804607, -+SNULL, 3202707456, 3204804607, -+STORE, 3202707456, 3204804607, -+SNULL, 3202707456, 3204804607, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3527647232, 3527778303, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+STORE, 3487059968, 3487584255, -+SNULL, 3487059968, 3487301631, -+STORE, 3487301632, 3487584255, -+STORE, 3487059968, 3487301631, -+SNULL, 3487059968, 3487301631, -+SNULL, 3487563775, 3487584255, -+STORE, 3487301632, 3487563775, -+STORE, 3487563776, 3487584255, -+SNULL, 3487563776, 3487584255, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3524046848, 3524177919, -+STORE, 3487170560, 3487301631, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3487039488, 3487170559, -+STORE, 3487039488, 3487301631, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3204280320, 3204804607, -+SNULL, 3204280320, 3204448255, -+STORE, 3204448256, 3204804607, -+STORE, 3204280320, 3204448255, -+SNULL, 3204280320, 3204448255, -+SNULL, 3204710399, 3204804607, -+STORE, 3204448256, 3204710399, -+STORE, 3204710400, 3204804607, -+SNULL, 3204710400, 3204804607, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3996295168, 3996299263, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+SNULL, 3996295168, 3996299263, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3486908416, 3487039487, -+STORE, 3486908416, 3487301631, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3223326720, 3290435583, -+SNULL, 3223326720, 3256881151, -+STORE, 3256881152, 3290435583, -+STORE, 3223326720, 3256881151, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+STORE, 3201826816, 3202351103, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+STORE, 3202351104, 3204448255, -+SNULL, 3202351104, 3204448255, -+SNULL, 3803471871, 3803492351, -+STORE, 3803451392, 3803471871, -+STORE, 3803471872, 3803492351, -+SNULL, 3803471872, 3803492351, -+SNULL, 3803451392, 3803471871, -+STORE, 3798999040, 3799101439, -+SNULL, 3798999040, 3799101439, -+STORE, 3952644096, 3952652287, -+STORE, 3203362816, 3204448255, -+SNULL, 3203362816, 3203366911, -+STORE, 3203366912, 3204448255, -+STORE, 3203362816, 3203366911, -+SNULL, 3204444159, 3204448255, -+STORE, 3203366912, 3204444159, -+STORE, 3204444160, 3204448255, -+STORE, 3803471872, 3803492351, -+SNULL, 3803475967, 3803492351, -+STORE, 3803471872, 3803475967, -+STORE, 3803475968, 3803492351, -+SNULL, 3203371007, 3204444159, -+STORE, 3203366912, 3203371007, -+STORE, 3203371008, 3204444159, -+STORE, 3199729664, 3201826815, -+SNULL, 3199729664, 3201826815, -+STORE, 3199729664, 3201826815, -+SNULL, 3199729664, 3201826815, -+STORE, 3199729664, 3201826815, -+SNULL, 3199729664, 3201826815, -+STORE, 3199729664, 3201826815, -+SNULL, 3199729664, 3201826815, -+STORE, 3199729664, 3201826815, -+SNULL, 3199729664, 3201826815, -+STORE, 3200774144, 3201826815, -+SNULL, 3200774144, 3200778239, -+STORE, 3200778240, 3201826815, -+STORE, 3200774144, 3200778239, -+SNULL, 3201822719, 3201826815, -+STORE, 3200778240, 3201822719, -+STORE, 3201822720, 3201826815, -+STORE, 3803451392, 3803471871, -+SNULL, 3803455487, 3803471871, -+STORE, 3803451392, 3803455487, -+STORE, 3803455488, 3803471871, -+SNULL, 3200782335, 3201822719, -+STORE, 3200778240, 3200782335, -+STORE, 3200782336, 3201822719, -+STORE, 3949666304, 3949674495, -+STORE, 3949408256, 3949416447, -+STORE, 3199688704, 3200778239, -+SNULL, 3199688704, 3199692799, -+STORE, 3199692800, 3200778239, -+STORE, 3199688704, 3199692799, -+SNULL, 3200770047, 3200778239, -+STORE, 3199692800, 3200770047, -+STORE, 3200770048, 3200778239, -+STORE, 3799306240, 3799326719, -+SNULL, 3799310335, 3799326719, -+STORE, 3799306240, 3799310335, -+STORE, 3799310336, 3799326719, -+SNULL, 3199696895, 3200770047, -+STORE, 3199692800, 3199696895, -+STORE, 3199696896, 3200770047, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+STORE, 3799277568, 3799306239, -+SNULL, 3799277568, 3799306239, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+SNULL, 4041162751, 4041170943, -+STORE, 4041154560, 4041162751, -+STORE, 4041162752, 4041170943, -+SNULL, 4041162752, 4041170943, -+SNULL, 4041154560, 4041162751, -+SNULL, 4041191424, 4041211903, -+SNULL, 4041170944, 4041191423, -+SNULL, 3626471423, 3626475519, -+STORE, 3626467328, 3626471423, -+STORE, 3626471424, 3626475519, -+SNULL, 3626471424, 3627524095, -+SNULL, 3625418751, 3625422847, -+STORE, 3625414656, 3625418751, -+STORE, 3625418752, 3625422847, -+SNULL, 3625418752, 3626471423, -+STORE, 3627393024, 3627524095, -+STORE, 3627261952, 3627393023, -+STORE, 3627261952, 3627524095, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+STORE, 3195494400, 3197591551, -+SNULL, 3197591552, 3199688703, -+SNULL, 3195494400, 3197591551, -+STORE, 3197591552, 3199688703, -+SNULL, 3197591552, 3199688703, -+STORE, 3197591552, 3199688703, -+STORE, 3195494400, 3197591551, -+SNULL, 3197591552, 3199688703, -+SNULL, 3195494400, 3197591551, -+STORE, 3798999040, 3799101439, -+SNULL, 3798999040, 3799101439, -+/* -+ * mmap: unmapped_area_topdown: ffff9a9f14ddaa80 -+ * Gap was found: mt 4041162752 gap_end 4041183232 -+ * mmap: window was 4052029440 - 4096 size 28672 -+ * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423 -+ * mmap: mas.index 4041162752 align mask 0 offset 0 -+ * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678) -+ */ -+ }; -+ -+ unsigned long set43[] = { -+STORE, 140737488347136, 140737488351231, -+STORE, 140734187720704, 140737488351231, -+SNULL, 140734187724800, 140737488351231, -+STORE, 140734187589632, 140734187724799, -+STORE, 4194304, 6443007, -+STORE, 4337664, 6443007, -+STORE, 4194304, 4337663, -+SNULL, 4337664, 6443007, -+STORE, 6430720, 6443007, -+STORE, 206158430208, 206160674815, -+STORE, 206158569472, 206160674815, -+STORE, 206158430208, 206158569471, -+SNULL, 206158569472, 206160674815, -+STORE, 206160662528, 206160670719, -+STORE, 206160670720, 206160674815, -+STORE, 140734188756992, 140734188765183, -+STORE, 140734188740608, 140734188756991, -+STORE, 140501948112896, 140501948116991, -+ }; -+ -+ int count = 0; -+ void *ptr = NULL; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ mt_set_non_kernel(3); -+ check_erase2_testset(mt, set, ARRAY_SIZE(set)); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set2, ARRAY_SIZE(set2)); -+ start = 140735933894656; -+ MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL)); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(2); -+ mt_init_flags(mt, 0); -+ check_erase2_testset(mt, set3, ARRAY_SIZE(set3)); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, 0); -+ check_erase2_testset(mt, set4, ARRAY_SIZE(set4)); -+ rcu_read_lock(); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ if (xa_is_zero(entry)) -+ continue; -+ } -+ rcu_read_unlock(); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ mt_set_non_kernel(100); -+ check_erase2_testset(mt, set5, ARRAY_SIZE(set5)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set6, ARRAY_SIZE(set6)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set7, ARRAY_SIZE(set7)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set8, ARRAY_SIZE(set8)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set9, ARRAY_SIZE(set9)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set10, ARRAY_SIZE(set10)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set11, ARRAY_SIZE(set11)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); -+ MT_BUG_ON(mt, mas.last != 140014592573439); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mas.tree = mt; -+ count = 0; -+ mas.index = 0; -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set12, ARRAY_SIZE(set12)); -+ rcu_barrier(); -+ mas_for_each(&mas, entry, ULONG_MAX) { -+ if (xa_is_zero(entry)) -+ continue; -+ BUG_ON(count > 12); -+ count++; -+ } -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set13, ARRAY_SIZE(set13)); -+ mtree_erase(mt, 140373516443648); -+ rcu_read_lock(); -+ mas_empty_area_rev(&mas, 0, 140373518663680, 4096); -+ rcu_read_unlock(); -+ mtree_destroy(mt); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set14, ARRAY_SIZE(set14)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set15, ARRAY_SIZE(set15)); -+ rcu_barrier(); -+ mtree_destroy(mt); -+ -+ /* set16 was to find a bug on limit updating at slot 0. */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set16, ARRAY_SIZE(set16)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); -+ MT_BUG_ON(mt, mas.last != 139921865547775); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ /* -+ * set17 found a bug in walking backwards and not counting nulls at -+ * the end. This could cause a gap to be missed if the null had any -+ * size. -+ */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set17, ARRAY_SIZE(set17)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); -+ MT_BUG_ON(mt, mas.last != 139953197322239); -+/* MT_BUG_ON(mt, mas.index != 139953197318144); */ -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ /* -+ * set18 found a bug in walking backwards and not setting the max from -+ * the node, but using the parent node. This was only an issue if the -+ * next slot in the parent had what we needed. -+ */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set18, ARRAY_SIZE(set18)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); -+ MT_BUG_ON(mt, mas.last != 140222968475647); -+ /*MT_BUG_ON(mt, mas.index != 140222966259712); */ -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ /* -+ * set19 found 2 bugs in prev. -+ * 1. If we hit root without finding anything, then there was an -+ * infinite loop. -+ * 2. The first ascending wasn't using the correct slot which may have -+ * caused missed entries. -+ */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set19, ARRAY_SIZE(set19)); -+ rcu_barrier(); -+ mas.index = 140656779083776; -+ entry = mas_find(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, entry != xa_mk_value(140656779083776)); -+ entry = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, entry != xa_mk_value(140656766251008)); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ /* -+ * set20 found a bug in mas_may_move_gap due to the slot being -+ * overwritten during the __mas_add operation and setting it to zero. -+ */ -+ mt_set_non_kernel(99); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set20, ARRAY_SIZE(set20)); -+ rcu_barrier(); -+ check_load(mt, 94849009414144, NULL); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(99); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set21, ARRAY_SIZE(set21)); -+ rcu_barrier(); -+ mt_validate(mt); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(999); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set22, ARRAY_SIZE(set22)); -+ rcu_barrier(); -+ mt_validate(mt); -+ ptr = mtree_load(mt, 140551363362816); -+ MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159)); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(99); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set23, ARRAY_SIZE(set23)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ -+ mt_set_non_kernel(99); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set24, ARRAY_SIZE(set24)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(99); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set25, ARRAY_SIZE(set25)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* Split on NULL followed by delete - causes gap issues. */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set26, ARRAY_SIZE(set26)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); -+ MT_BUG_ON(mt, mas.last != 140109040959487); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* Split on NULL followed by delete - causes gap issues. */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set27, ARRAY_SIZE(set27)); -+ rcu_barrier(); -+ MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336)); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set28, ARRAY_SIZE(set28)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); -+ /* Search for the size of gap then align it (offset 0) */ -+ mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); -+ MT_BUG_ON(mt, mas.index != 139918401601536); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* This test found issues with retry moving rebalanced nodes so the -+ * incorrect parent pivot was updated. -+ */ -+ mt_set_non_kernel(999); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set29, ARRAY_SIZE(set29)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* This test found issues with deleting all entries in a node when -+ * surrounded by entries in the next nodes, then deleting the entries -+ * surrounding the node filled with deleted entries. -+ */ -+ mt_set_non_kernel(999); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set30, ARRAY_SIZE(set30)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* This test found an issue with deleting all entries in a node that was -+ * the end node and mas_gap incorrectly set next = curr, and curr = prev -+ * then moved next to the left, losing data. -+ */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set31, ARRAY_SIZE(set31)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set32, ARRAY_SIZE(set32)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+/* -+ * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found: -+ * mt 140582827569152 gap_end 140582869532672 -+ * mmap: window was 140583656296448 - 4096 size 134217728 -+ * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879 -+ * mmap: mas.index 140582827569152 align mask 0 offset 0 -+ * mmap: rb_find_vma find on -+ * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00) -+ */ -+ -+ /* move gap failed due to an entirely empty node */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set33, ARRAY_SIZE(set33)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); -+ MT_BUG_ON(mt, mas.last != 140583003750399); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* -+ * Incorrect gap in tree caused by mas_prev not setting the limits -+ * correctly while walking down. -+ */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set34, ARRAY_SIZE(set34)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* Empty leaf at the end of a parent caused incorrect gap. */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set35, ARRAY_SIZE(set35)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(99); -+ /* Empty leaf at the end of a parent caused incorrect gap. */ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set36, ARRAY_SIZE(set36)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set37, ARRAY_SIZE(set37)); -+ rcu_barrier(); -+ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set38, ARRAY_SIZE(set38)); -+ rcu_barrier(); -+ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); -+ rcu_barrier(); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); -+ rcu_barrier(); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set41, ARRAY_SIZE(set41)); -+ rcu_barrier(); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* move gap failed due to an entirely empty node. */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set42, ARRAY_SIZE(set42)); -+ rcu_barrier(); -+ mas_empty_area_rev(&mas, 4096, 4052029440, 28672); -+ MT_BUG_ON(mt, mas.last != 4041211903); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* gap calc off by one */ -+ mt_set_non_kernel(99); -+ mas_reset(&mas); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_erase2_testset(mt, set43, ARRAY_SIZE(set43)); -+ rcu_barrier(); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ mtree_destroy(mt); -+} -+ -+static noinline void check_alloc_rev_range(struct maple_tree *mt) -+{ -+ /* -+ * Generated by: -+ * cat /proc/self/maps | awk '{print $1}'| -+ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' -+ */ -+ -+ unsigned long range[] = { -+ /* Inclusive , Exclusive. */ -+ 0x565234af2000, 0x565234af4000, -+ 0x565234af4000, 0x565234af9000, -+ 0x565234af9000, 0x565234afb000, -+ 0x565234afc000, 0x565234afd000, -+ 0x565234afd000, 0x565234afe000, -+ 0x565235def000, 0x565235e10000, -+ 0x7f36d4bfd000, 0x7f36d4ee2000, -+ 0x7f36d4ee2000, 0x7f36d4f04000, -+ 0x7f36d4f04000, 0x7f36d504c000, -+ 0x7f36d504c000, 0x7f36d5098000, -+ 0x7f36d5098000, 0x7f36d5099000, -+ 0x7f36d5099000, 0x7f36d509d000, -+ 0x7f36d509d000, 0x7f36d509f000, -+ 0x7f36d509f000, 0x7f36d50a5000, -+ 0x7f36d50b9000, 0x7f36d50db000, -+ 0x7f36d50db000, 0x7f36d50dc000, -+ 0x7f36d50dc000, 0x7f36d50fa000, -+ 0x7f36d50fa000, 0x7f36d5102000, -+ 0x7f36d5102000, 0x7f36d5103000, -+ 0x7f36d5103000, 0x7f36d5104000, -+ 0x7f36d5104000, 0x7f36d5105000, -+ 0x7fff5876b000, 0x7fff5878d000, -+ 0x7fff5878e000, 0x7fff58791000, -+ 0x7fff58791000, 0x7fff58793000, -+ }; -+ -+ unsigned long holes[] = { -+ /* -+ * Note: start of hole is INCLUSIVE -+ * end of hole is EXCLUSIVE -+ * (opposite of the above table.) -+ * Start of hole, end of hole, size of hole (+1) -+ */ -+ 0x565234afb000, 0x565234afc000, 0x1000, -+ 0x565234afe000, 0x565235def000, 0x12F1000, -+ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, -+ }; -+ -+ /* -+ * req_range consists of 4 values. -+ * 1. min index -+ * 2. max index -+ * 3. size -+ * 4. number that should be returned. -+ * 5. return value -+ */ -+ unsigned long req_range[] = { -+ 0x565234af9000, /* Min */ -+ 0x7fff58791000, /* Max */ -+ 0x1000, /* Size */ -+ 0x7fff5878d << 12, /* First rev hole of size 0x1000 */ -+ 0, /* Return value success. */ -+ -+ 0x0, /* Min */ -+ 0x565234AF1 << 12, /* Max */ -+ 0x3000, /* Size */ -+ 0x565234AEE << 12, /* max - 3. */ -+ 0, /* Return value success. */ -+ -+ 0x0, /* Min */ -+ -1, /* Max */ -+ 0x1000, /* Size */ -+ 562949953421311 << 12,/* First rev hole of size 0x1000 */ -+ 0, /* Return value success. */ -+ -+ 0x0, /* Min */ -+ 0x7F36D510A << 12, /* Max */ -+ 0x4000, /* Size */ -+ 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */ -+ 0, /* Return value success. */ -+ -+ /* Ascend test. */ -+ 0x0, -+ 34148798629 << 12, -+ 19 << 12, -+ 34148797418 << 12, -+ 0x0, -+ -+ /* Too big test. */ -+ 0x0, -+ 18446744073709551615UL, -+ 562915594369134UL << 12, -+ 0x0, -+ -EBUSY, -+ -+ }; -+ -+ int i, range_count = ARRAY_SIZE(range); -+ int req_range_count = ARRAY_SIZE(req_range); -+ unsigned long min = 0; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, -+ GFP_KERNEL); -+#define DEBUG_REV_RANGE 0 -+ for (i = 0; i < range_count; i += 2) { -+ /* Inclusive, Inclusive (with the -1) */ -+ -+#if DEBUG_REV_RANGE -+ pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12, -+ (range[i + 1] >> 12) - 1); -+#endif -+ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, -+ xa_mk_value(range[i] >> 12), 0); -+ mt_validate(mt); -+ } -+ -+ -+ for (i = 0; i < ARRAY_SIZE(holes); i += 3) { -+#if DEBUG_REV_RANGE -+ pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n", -+ min, holes[i+1]>>12, holes[i+2]>>12, -+ holes[i] >> 12); -+#endif -+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, min, -+ holes[i+1] >> 12, -+ holes[i+2] >> 12)); -+#if DEBUG_REV_RANGE -+ pr_debug("Found %lu %lu\n", mas.index, mas.last); -+ pr_debug("gap %lu %lu\n", (holes[i] >> 12), -+ (holes[i+1] >> 12)); -+#endif -+ MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12)); -+ MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12)); -+ min = holes[i+1] >> 12; -+ mas_reset(&mas); -+ } -+ -+ for (i = 0; i < req_range_count; i += 5) { -+#if DEBUG_REV_RANGE -+ pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n", -+ req_range[i] >> 12, -+ (req_range[i + 1] >> 12) - 1, -+ req_range[i+2] >> 12, -+ req_range[i+3] >> 12); -+#endif -+ check_mtree_alloc_rrange(mt, -+ req_range[i] >> 12, /* start */ -+ req_range[i+1] >> 12, /* end */ -+ req_range[i+2] >> 12, /* size */ -+ req_range[i+3] >> 12, /* expected address */ -+ req_range[i+4], /* expected return */ -+ xa_mk_value(req_range[i] >> 12)); /* pointer */ -+ mt_validate(mt); -+ } -+ -+ mt_set_non_kernel(1); -+ mtree_erase(mt, 34148798727); /* create a deleted range. */ -+ check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414, -+ 34148798725, 0, mt); -+ -+ mtree_destroy(mt); -+} -+ -+static noinline void check_alloc_range(struct maple_tree *mt) -+{ -+ /* -+ * Generated by: -+ * cat /proc/self/maps|awk '{print $1}'| -+ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' -+ */ -+ -+ unsigned long range[] = { -+ /* Inclusive , Exclusive. */ -+ 0x565234af2000, 0x565234af4000, -+ 0x565234af4000, 0x565234af9000, -+ 0x565234af9000, 0x565234afb000, -+ 0x565234afc000, 0x565234afd000, -+ 0x565234afd000, 0x565234afe000, -+ 0x565235def000, 0x565235e10000, -+ 0x7f36d4bfd000, 0x7f36d4ee2000, -+ 0x7f36d4ee2000, 0x7f36d4f04000, -+ 0x7f36d4f04000, 0x7f36d504c000, -+ 0x7f36d504c000, 0x7f36d5098000, -+ 0x7f36d5098000, 0x7f36d5099000, -+ 0x7f36d5099000, 0x7f36d509d000, -+ 0x7f36d509d000, 0x7f36d509f000, -+ 0x7f36d509f000, 0x7f36d50a5000, -+ 0x7f36d50b9000, 0x7f36d50db000, -+ 0x7f36d50db000, 0x7f36d50dc000, -+ 0x7f36d50dc000, 0x7f36d50fa000, -+ 0x7f36d50fa000, 0x7f36d5102000, -+ 0x7f36d5102000, 0x7f36d5103000, -+ 0x7f36d5103000, 0x7f36d5104000, -+ 0x7f36d5104000, 0x7f36d5105000, -+ 0x7fff5876b000, 0x7fff5878d000, -+ 0x7fff5878e000, 0x7fff58791000, -+ 0x7fff58791000, 0x7fff58793000, -+ }; -+ unsigned long holes[] = { -+ /* Start of hole, end of hole, size of hole (+1) */ -+ 0x565234afb000, 0x565234afc000, 0x1000, -+ 0x565234afe000, 0x565235def000, 0x12F1000, -+ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, -+ }; -+ -+ /* -+ * req_range consists of 4 values. -+ * 1. min index -+ * 2. max index -+ * 3. size -+ * 4. number that should be returned. -+ * 5. return value -+ */ -+ unsigned long req_range[] = { -+ 0x565234af9000, /* Min */ -+ 0x7fff58791000, /* Max */ -+ 0x1000, /* Size */ -+ 0x565234afb000, /* First hole in our data of size 1000. */ -+ 0, /* Return value success. */ -+ -+ 0x0, /* Min */ -+ 0x7fff58791000, /* Max */ -+ 0x1F00, /* Size */ -+ 0x0, /* First hole in our data of size 2000. */ -+ 0, /* Return value success. */ -+ -+ /* Test ascend. */ -+ 34148797436 << 12, /* Min */ -+ 0x7fff587AF000, /* Max */ -+ 0x3000, /* Size */ -+ 34148798629 << 12, /* Expected location */ -+ 0, /* Return value success. */ -+ -+ /* Test failing. */ -+ 34148798623 << 12, /* Min */ -+ 34148798683 << 12, /* Max */ -+ 0x15000, /* Size */ -+ 0, /* Expected location */ -+ -EBUSY, /* Return value failed. */ -+ -+ /* Test filling entire gap. */ -+ 34148798623 << 12, /* Min */ -+ 0x7fff587AF000, /* Max */ -+ 0x10000, /* Size */ -+ 34148798632 << 12, /* Expected location */ -+ 0, /* Return value success. */ -+ -+ /* Test walking off the end of root. */ -+ 0, /* Min */ -+ -1, /* Max */ -+ -1, /* Size */ -+ 0, /* Expected location */ -+ -EBUSY, /* Return value failure. */ -+ -+ /* Test looking for too large a hole across entire range. */ -+ 0, /* Min */ -+ -1, /* Max */ -+ 4503599618982063UL << 12, /* Size */ -+ 34359052178 << 12, /* Expected location */ -+ -EBUSY, /* Return failure. */ -+ }; -+ int i, range_count = ARRAY_SIZE(range); -+ int req_range_count = ARRAY_SIZE(req_range); -+ unsigned long min = 0x565234af2000; -+ -+ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, -+ GFP_KERNEL); -+ for (i = 0; i < range_count; i += 2) { -+#define DEBUG_ALLOC_RANGE 0 -+#if DEBUG_ALLOC_RANGE -+ pr_debug("\tInsert %lu-%lu\n", range[i] >> 12, -+ (range[i + 1] >> 12) - 1); -+ mt_dump(mt); -+#endif -+ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, -+ xa_mk_value(range[i] >> 12), 0); -+ mt_validate(mt); -+ } -+ -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ for (i = 0; i < ARRAY_SIZE(holes); i += 3) { -+ -+#if DEBUG_ALLOC_RANGE -+ pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12, -+ holes[i+1] >> 12, holes[i+2] >> 12, -+ min, holes[i+1]); -+#endif -+ MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12, -+ holes[i+1] >> 12, -+ holes[i+2] >> 12)); -+ MT_BUG_ON(mt, mas.index != holes[i] >> 12); -+ min = holes[i+1]; -+ mas_reset(&mas); -+ } -+ for (i = 0; i < req_range_count; i += 5) { -+#if DEBUG_ALLOC_RANGE -+ pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n", -+ i/5, req_range[i] >> 12, req_range[i + 1] >> 12, -+ req_range[i + 2] >> 12, req_range[i + 3] >> 12, -+ req_range[i], req_range[i+1]); -+#endif -+ check_mtree_alloc_range(mt, -+ req_range[i] >> 12, /* start */ -+ req_range[i+1] >> 12, /* end */ -+ req_range[i+2] >> 12, /* size */ -+ req_range[i+3] >> 12, /* expected address */ -+ req_range[i+4], /* expected return */ -+ xa_mk_value(req_range[i] >> 12)); /* pointer */ -+ mt_validate(mt); -+#if DEBUG_ALLOC_RANGE -+ mt_dump(mt); -+#endif -+ } -+ -+ mtree_destroy(mt); -+} -+ -+static noinline void check_ranges(struct maple_tree *mt) -+{ -+ int i, val, val2; -+ unsigned long r[] = { -+ 10, 15, -+ 20, 25, -+ 17, 22, /* Overlaps previous range. */ -+ 9, 1000, /* Huge. */ -+ 100, 200, -+ 45, 168, -+ 118, 128, -+ }; -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0); -+ check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0); -+ check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ /* Store */ -+ check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0); -+ check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0); -+ check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ MT_BUG_ON(mt, mt_height(mt)); -+ -+ check_seq(mt, 50, false); -+ mt_set_non_kernel(4); -+ check_store_range(mt, 5, 47, xa_mk_value(47), 0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ /* Create tree of 1-100 */ -+ check_seq(mt, 100, false); -+ /* Store 45-168 */ -+ mt_set_non_kernel(10); -+ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ /* Create tree of 1-200 */ -+ check_seq(mt, 200, false); -+ /* Store 45-168 */ -+ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ check_seq(mt, 30, false); -+ check_store_range(mt, 6, 18, xa_mk_value(6), 0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ /* Overwrite across multiple levels. */ -+ /* Create tree of 1-400 */ -+ check_seq(mt, 400, false); -+ mt_set_non_kernel(50); -+ /* Store 118-128 */ -+ check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0); -+ mt_set_non_kernel(50); -+ mtree_test_erase(mt, 140); -+ mtree_test_erase(mt, 141); -+ mtree_test_erase(mt, 142); -+ mtree_test_erase(mt, 143); -+ mtree_test_erase(mt, 130); -+ mtree_test_erase(mt, 131); -+ mtree_test_erase(mt, 132); -+ mtree_test_erase(mt, 133); -+ mtree_test_erase(mt, 134); -+ mtree_test_erase(mt, 135); -+ check_load(mt, r[12], xa_mk_value(r[12])); -+ check_load(mt, r[13], xa_mk_value(r[12])); -+ check_load(mt, r[13] - 1, xa_mk_value(r[12])); -+ check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); -+ check_load(mt, 135, NULL); -+ check_load(mt, 140, NULL); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ -+ -+ /* Overwrite multiple levels at the end of the tree (slot 7) */ -+ mt_set_non_kernel(50); -+ check_seq(mt, 400, false); -+ check_store_range(mt, 353, 361, xa_mk_value(353), 0); -+ check_store_range(mt, 347, 352, xa_mk_value(347), 0); -+ -+ check_load(mt, 346, xa_mk_value(346)); -+ for (i = 347; i <= 352; i++) -+ check_load(mt, i, xa_mk_value(347)); -+ for (i = 353; i <= 361; i++) -+ check_load(mt, i, xa_mk_value(353)); -+ check_load(mt, 362, xa_mk_value(362)); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(50); -+ check_seq(mt, 400, false); -+ check_store_range(mt, 352, 364, NULL, 0); -+ check_store_range(mt, 351, 363, xa_mk_value(352), 0); -+ check_load(mt, 350, xa_mk_value(350)); -+ check_load(mt, 351, xa_mk_value(352)); -+ for (i = 352; i <= 363; i++) -+ check_load(mt, i, xa_mk_value(352)); -+ check_load(mt, 364, NULL); -+ check_load(mt, 365, xa_mk_value(365)); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ mt_set_non_kernel(5); -+ check_seq(mt, 400, false); -+ check_store_range(mt, 352, 364, NULL, 0); -+ check_store_range(mt, 351, 364, xa_mk_value(352), 0); -+ check_load(mt, 350, xa_mk_value(350)); -+ check_load(mt, 351, xa_mk_value(352)); -+ for (i = 352; i <= 364; i++) -+ check_load(mt, i, xa_mk_value(352)); -+ check_load(mt, 365, xa_mk_value(365)); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ -+ mt_set_non_kernel(50); -+ check_seq(mt, 400, false); -+ check_store_range(mt, 362, 367, xa_mk_value(362), 0); -+ check_store_range(mt, 353, 361, xa_mk_value(353), 0); -+ mt_set_non_kernel(0); -+ mt_validate(mt); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ /* -+ * Interesting cases: -+ * 1. Overwrite the end of a node and end in the first entry of the next -+ * node. -+ * 2. Split a single range -+ * 3. Overwrite the start of a range -+ * 4. Overwrite the end of a range -+ * 5. Overwrite the entire range -+ * 6. Overwrite a range that causes multiple parent nodes to be -+ * combined -+ * 7. Overwrite a range that causes multiple parent nodes and part of -+ * root to be combined -+ * 8. Overwrite the whole tree -+ * 9. Try to overwrite the zero entry of an alloc tree. -+ * 10. Write a range larger than a nodes current pivot -+ */ -+ -+ mt_set_non_kernel(50); -+ for (i = 0; i <= 500; i++) { -+ val = i*5; -+ val2 = (i+1)*5; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ } -+ check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0); -+ check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0); -+ check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0); -+ check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0); -+ check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0); -+ mtree_destroy(mt); -+ mt_set_non_kernel(0); -+ -+ mt_set_non_kernel(50); -+ for (i = 0; i <= 500; i++) { -+ val = i*5; -+ val2 = (i+1)*5; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ } -+ check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0); -+ check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0); -+ check_store_range(mt, 2425, 2425, xa_mk_value(2), 0); -+ check_store_range(mt, 2460, 2470, NULL, 0); -+ check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0); -+ check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ /* Test rebalance gaps */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ mt_set_non_kernel(50); -+ for (i = 0; i <= 50; i++) { -+ val = i*10; -+ val2 = (i+1)*10; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ } -+ check_store_range(mt, 161, 161, xa_mk_value(161), 0); -+ check_store_range(mt, 162, 162, xa_mk_value(162), 0); -+ check_store_range(mt, 163, 163, xa_mk_value(163), 0); -+ check_store_range(mt, 240, 249, NULL, 0); -+ mtree_erase(mt, 200); -+ mtree_erase(mt, 210); -+ mtree_erase(mt, 220); -+ mtree_erase(mt, 230); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= 500; i++) { -+ val = i*10; -+ val2 = (i+1)*10; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ } -+ check_store_range(mt, 4600, 4959, xa_mk_value(1), 0); -+ mt_validate(mt); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= 500; i++) { -+ val = i*10; -+ val2 = (i+1)*10; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ } -+ check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0); -+ check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0); -+ check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0); -+ check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0); -+ check_store_range(mt, 4842, 4849, NULL, 0); -+ mt_validate(mt); -+ MT_BUG_ON(mt, !mt_height(mt)); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= 1300; i++) { -+ val = i*10; -+ val2 = (i+1)*10; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ MT_BUG_ON(mt, mt_height(mt) >= 4); -+ } -+ /* Cause a 3 child split all the way up the tree. */ -+ for (i = 5; i < 215; i += 10) -+ check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); -+ for (i = 5; i < 65; i += 10) -+ check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); -+ -+ MT_BUG_ON(mt, mt_height(mt) >= 4); -+ for (i = 5; i < 45; i += 10) -+ check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); -+ MT_BUG_ON(mt, mt_height(mt) < 4); -+ mtree_destroy(mt); -+ -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= 1200; i++) { -+ val = i*10; -+ val2 = (i+1)*10; -+ check_store_range(mt, val, val2, xa_mk_value(val), 0); -+ MT_BUG_ON(mt, mt_height(mt) >= 4); -+ } -+ /* Fill parents and leaves before split. */ -+ for (i = 5; i < 455; i += 10) -+ check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); -+ -+ for (i = 1; i < 16; i++) -+ check_store_range(mt, 8185 + i, 8185 + i + 1, -+ xa_mk_value(8185+i), 0); -+ MT_BUG_ON(mt, mt_height(mt) >= 4); -+ /* triple split across multiple levels. */ -+ check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); -+ MT_BUG_ON(mt, mt_height(mt) != 4); -+} -+ -+static noinline void check_next_entry(struct maple_tree *mt) -+{ -+ void *entry = NULL; -+ unsigned long limit = 30, i = 0; -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ MA_STATE(mas, mt, i, i); -+ -+ check_seq(mt, limit, false); -+ rcu_read_lock(); -+ -+ /* Check the first one and get ma_state in the correct state. */ -+ MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++)); -+ for ( ; i <= limit + 1; i++) { -+ entry = mas_next(&mas, limit); -+ if (i > limit) -+ MT_BUG_ON(mt, entry != NULL); -+ else -+ MT_BUG_ON(mt, xa_mk_value(i) != entry); -+ } -+ rcu_read_unlock(); -+ mtree_destroy(mt); -+} -+ -+static noinline void check_prev_entry(struct maple_tree *mt) -+{ -+ unsigned long index = 16; -+ void *value; -+ int i; -+ -+ MA_STATE(mas, mt, index, index); -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ check_seq(mt, 30, false); -+ -+ rcu_read_lock(); -+ value = mas_find(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, value != xa_mk_value(index)); -+ value = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, value != xa_mk_value(index - 1)); -+ rcu_read_unlock(); -+ mtree_destroy(mt); -+ -+ /* Check limits on prev */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ mas_lock(&mas); -+ for (i = 0; i <= index; i++) { -+ mas_set_range(&mas, i*10, i*10+5); -+ mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL); -+ } -+ -+ mas_set(&mas, 20); -+ value = mas_walk(&mas); -+ MT_BUG_ON(mt, value != xa_mk_value(2)); -+ -+ value = mas_prev(&mas, 19); -+ MT_BUG_ON(mt, value != NULL); -+ -+ mas_set(&mas, 80); -+ value = mas_walk(&mas); -+ MT_BUG_ON(mt, value != xa_mk_value(8)); -+ -+ value = mas_prev(&mas, 76); -+ MT_BUG_ON(mt, value != NULL); -+ -+ mas_unlock(&mas); -+} -+ -+static noinline void check_root_expand(struct maple_tree *mt) -+{ -+ MA_STATE(mas, mt, 0, 0); -+ void *ptr; -+ -+ -+ mas_lock(&mas); -+ mas_set(&mas, 3); -+ ptr = mas_walk(&mas); -+ MT_BUG_ON(mt, ptr != NULL); -+ MT_BUG_ON(mt, mas.index != 0); -+ MT_BUG_ON(mt, mas.last != ULONG_MAX); -+ -+ ptr = &check_prev_entry; -+ mas_set(&mas, 1); -+ mas_store_gfp(&mas, ptr, GFP_KERNEL); -+ -+ mas_set(&mas, 0); -+ ptr = mas_walk(&mas); -+ MT_BUG_ON(mt, ptr != NULL); -+ -+ mas_set(&mas, 1); -+ ptr = mas_walk(&mas); -+ MT_BUG_ON(mt, ptr != &check_prev_entry); -+ -+ mas_set(&mas, 2); -+ ptr = mas_walk(&mas); -+ MT_BUG_ON(mt, ptr != NULL); -+ mas_unlock(&mas); -+ mtree_destroy(mt); -+ -+ -+ mt_init_flags(mt, 0); -+ mas_lock(&mas); -+ -+ mas_set(&mas, 0); -+ ptr = &check_prev_entry; -+ mas_store_gfp(&mas, ptr, GFP_KERNEL); -+ -+ mas_set(&mas, 5); -+ ptr = mas_walk(&mas); -+ MT_BUG_ON(mt, ptr != NULL); -+ MT_BUG_ON(mt, mas.index != 1); -+ MT_BUG_ON(mt, mas.last != ULONG_MAX); -+ -+ mas_set_range(&mas, 0, 100); -+ ptr = mas_walk(&mas); -+ MT_BUG_ON(mt, ptr != &check_prev_entry); -+ MT_BUG_ON(mt, mas.last != 0); -+ mas_unlock(&mas); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, 0); -+ mas_lock(&mas); -+ -+ mas_set(&mas, 0); -+ ptr = (void *)((unsigned long) check_prev_entry | 1UL); -+ mas_store_gfp(&mas, ptr, GFP_KERNEL); -+ ptr = mas_next(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, ptr != NULL); -+ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); -+ -+ mas_set(&mas, 1); -+ ptr = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); -+ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL)); -+ -+ mas_unlock(&mas); -+ -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, 0); -+ mas_lock(&mas); -+ mas_set(&mas, 0); -+ ptr = (void *)((unsigned long) check_prev_entry | 2UL); -+ mas_store_gfp(&mas, ptr, GFP_KERNEL); -+ ptr = mas_next(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, ptr != NULL); -+ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); -+ -+ mas_set(&mas, 1); -+ ptr = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); -+ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL)); -+ -+ -+ mas_unlock(&mas); -+} -+ -+static noinline void check_prealloc(struct maple_tree *mt) -+{ -+ unsigned long i, max = 100; -+ unsigned long allocated; -+ unsigned char height; -+ struct maple_node *mn; -+ void *ptr = check_prealloc; -+ MA_STATE(mas, mt, 10, 20); -+ -+ mt_set_non_kernel(1000); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mas_destroy(&mas); -+ allocated = mas_allocated(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ mas_destroy(&mas); -+ allocated = mas_allocated(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+ -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); -+ ma_free_rcu(mn); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ mas_destroy(&mas); -+ allocated = mas_allocated(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ mas_destroy(&mas); -+ allocated = mas_allocated(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+ ma_free_rcu(mn); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mn = mas_pop_node(&mas); -+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); -+ mas_push_node(&mas, mn); -+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ mas_destroy(&mas); -+ allocated = mas_allocated(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mas_store_prealloc(&mas, ptr); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mas_store_prealloc(&mas, ptr); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mas_store_prealloc(&mas, ptr); -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mas_store_prealloc(&mas, ptr); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ mt_set_non_kernel(1); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+ mas_destroy(&mas); -+ -+ -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated == 0); -+ MT_BUG_ON(mt, allocated != 1 + height * 3); -+ mas_store_prealloc(&mas, ptr); -+ MT_BUG_ON(mt, mas_allocated(&mas) != 0); -+ mt_set_non_kernel(1); -+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); -+ allocated = mas_allocated(&mas); -+ height = mas_mt_height(&mas); -+ MT_BUG_ON(mt, allocated != 0); -+} -+ -+static noinline void check_spanning_write(struct maple_tree *mt) -+{ -+ unsigned long i, max = 5000; -+ MA_STATE(mas, mt, 1200, 2380); -+ -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 1205); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ for (i = 1; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mtree_lock(mt); -+ mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 1205); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* Test spanning store that requires a right cousin rebalance */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 1205); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* Test non-alloc tree spanning store */ -+ mt_init_flags(mt, 0); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mas_set_range(&mas, 0, 300); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 15); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* Test spanning store that requires a right sibling rebalance */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mas_set_range(&mas, 0, 12865); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 15); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* Test spanning store that requires a left sibling rebalance */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mas_set_range(&mas, 90, 13665); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 95); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* Test spanning store that requires a left cousin rebalance */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mas_set_range(&mas, 46805, 49995); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 46815); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* -+ * Test spanning store that requires a left cousin rebalance all the way -+ * to root -+ */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mas_set_range(&mas, 32395, 49995); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 46815); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* -+ * Test spanning store that requires a right cousin rebalance all the -+ * way to root -+ */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ mas_set_range(&mas, 38875, 43190); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 38900); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* Test spanning store ending at full node (depth 2)*/ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ mtree_lock(mt); -+ mas_set(&mas, 47606); -+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); -+ mas_set(&mas, 47607); -+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); -+ mas_set(&mas, 47608); -+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); -+ mas_set(&mas, 47609); -+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); -+ /* Ensure the parent node is full */ -+ mas_ascend(&mas); -+ MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); -+ mas_set_range(&mas, 11516, 48940); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ /* Test spanning write with many levels of no siblings */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ mas_set_range(&mas, 43200, 49999); -+ mtree_lock(mt); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mas_set(&mas, 43200); -+ MT_BUG_ON(mt, mas_walk(&mas) != NULL); -+ mtree_unlock(mt); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= 100; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ -+ mtree_lock(mt); -+ mas_set_range(&mas, 76, 875); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ mtree_unlock(mt); -+} -+ -+static noinline void check_null_expand(struct maple_tree *mt) -+{ -+ unsigned long i, max = 100; -+ unsigned char data_end; -+ MA_STATE(mas, mt, 959, 959); -+ -+ for (i = 0; i <= max; i++) -+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); -+ /* Test expanding null at start. */ -+ mas_walk(&mas); -+ data_end = mas_data_end(&mas); -+ mas_set_range(&mas, 959, 963); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ MT_BUG_ON(mt, mtree_load(mt, 963) != NULL); -+ MT_BUG_ON(mt, data_end != mas_data_end(&mas)); -+ -+ /* Test expanding null at end. */ -+ mas_set(&mas, 880); -+ mas_walk(&mas); -+ data_end = mas_data_end(&mas); -+ mas_set_range(&mas, 884, 887); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ MT_BUG_ON(mt, mtree_load(mt, 884) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 889) != NULL); -+ MT_BUG_ON(mt, data_end != mas_data_end(&mas)); -+ -+ /* Test expanding null at start and end. */ -+ mas_set(&mas, 890); -+ mas_walk(&mas); -+ data_end = mas_data_end(&mas); -+ mas_set_range(&mas, 900, 905); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ MT_BUG_ON(mt, mtree_load(mt, 899) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 900) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 905) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 906) != NULL); -+ MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); -+ -+ /* Test expanding null across multiple slots. */ -+ mas_set(&mas, 800); -+ mas_walk(&mas); -+ data_end = mas_data_end(&mas); -+ mas_set_range(&mas, 810, 825); -+ mas_store_gfp(&mas, NULL, GFP_KERNEL); -+ MT_BUG_ON(mt, mtree_load(mt, 809) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 810) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 825) != NULL); -+ MT_BUG_ON(mt, mtree_load(mt, 826) != NULL); -+ MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); -+} -+ -+static noinline void check_gap_combining(struct maple_tree *mt) -+{ -+ struct maple_enode *mn1, *mn2; -+ void *entry; -+ -+ unsigned long seq100[] = { -+ /* 0-5 */ -+ 74, 75, 76, -+ 50, 100, 2, -+ -+ /* 6-12 */ -+ 44, 45, 46, 43, -+ 20, 50, 3, -+ -+ /* 13-20*/ -+ 80, 81, 82, -+ 76, 2, 79, 85, 4, -+ }; -+ unsigned long seq2000[] = { -+ 1152, 1151, -+ 1100, 1200, 2, -+ }; -+ unsigned long seq400[] = { -+ 286, 318, -+ 256, 260, 266, 270, 275, 280, 290, 398, -+ 286, 310, -+ }; -+ -+ unsigned long index = seq100[0]; -+ -+ MA_STATE(mas, mt, index, index); -+ -+ MT_BUG_ON(mt, !mtree_empty(mt)); -+ check_seq(mt, 100, false); /* create 100 singletons. */ -+ -+ mt_set_non_kernel(1); -+ mtree_test_erase(mt, seq100[2]); -+ check_load(mt, seq100[2], NULL); -+ mtree_test_erase(mt, seq100[1]); -+ check_load(mt, seq100[1], NULL); -+ -+ rcu_read_lock(); -+ entry = mas_find(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, entry != xa_mk_value(index)); -+ mn1 = mas.node; -+ mas_next(&mas, ULONG_MAX); -+ entry = mas_next(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); -+ mn2 = mas.node; -+ MT_BUG_ON(mt, mn1 == mn2); /* test the test. */ -+ -+ /* -+ * At this point, there is a gap of 2 at index + 1 between seq100[3] and -+ * seq100[4]. Search for the gap. -+ */ -+ mt_set_non_kernel(1); -+ mas_reset(&mas); -+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4], -+ seq100[5])); -+ MT_BUG_ON(mt, mas.index != index + 1); -+ rcu_read_unlock(); -+ -+ mtree_test_erase(mt, seq100[6]); -+ check_load(mt, seq100[6], NULL); -+ mtree_test_erase(mt, seq100[7]); -+ check_load(mt, seq100[7], NULL); -+ mtree_test_erase(mt, seq100[8]); -+ index = seq100[9]; -+ -+ rcu_read_lock(); -+ mas.index = index; -+ mas.last = index; -+ mas_reset(&mas); -+ entry = mas_find(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, entry != xa_mk_value(index)); -+ mn1 = mas.node; -+ entry = mas_next(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); -+ mas_next(&mas, ULONG_MAX); /* go to the next entry. */ -+ mn2 = mas.node; -+ MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */ -+ -+ /* -+ * At this point, there is a gap of 3 at seq100[6]. Find it by -+ * searching 20 - 50 for size 3. -+ */ -+ mas_reset(&mas); -+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11], -+ seq100[12])); -+ MT_BUG_ON(mt, mas.index != seq100[6]); -+ rcu_read_unlock(); -+ -+ mt_set_non_kernel(1); -+ mtree_store(mt, seq100[13], NULL, GFP_KERNEL); -+ check_load(mt, seq100[13], NULL); -+ check_load(mt, seq100[14], xa_mk_value(seq100[14])); -+ mtree_store(mt, seq100[14], NULL, GFP_KERNEL); -+ check_load(mt, seq100[13], NULL); -+ check_load(mt, seq100[14], NULL); -+ -+ mas_reset(&mas); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15], -+ seq100[17])); -+ MT_BUG_ON(mt, mas.index != seq100[13]); -+ mt_validate(mt); -+ rcu_read_unlock(); -+ -+ /* -+ * *DEPRECATED: no retries anymore* Test retry entry in the start of a -+ * gap. -+ */ -+ mt_set_non_kernel(2); -+ mtree_test_store_range(mt, seq100[18], seq100[14], NULL); -+ mtree_test_erase(mt, seq100[15]); -+ mas_reset(&mas); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19], -+ seq100[20])); -+ rcu_read_unlock(); -+ MT_BUG_ON(mt, mas.index != seq100[18]); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* seq 2000 tests are for multi-level tree gaps */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_seq(mt, 2000, false); -+ mt_set_non_kernel(1); -+ mtree_test_erase(mt, seq2000[0]); -+ mtree_test_erase(mt, seq2000[1]); -+ -+ mt_set_non_kernel(2); -+ mas_reset(&mas); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3], -+ seq2000[4])); -+ MT_BUG_ON(mt, mas.index != seq2000[1]); -+ rcu_read_unlock(); -+ mt_validate(mt); -+ mtree_destroy(mt); -+ -+ /* seq 400 tests rebalancing over two levels. */ -+ mt_set_non_kernel(99); -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_seq(mt, 400, false); -+ mtree_test_store_range(mt, seq400[0], seq400[1], NULL); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_seq(mt, 400, false); -+ mt_set_non_kernel(50); -+ mtree_test_store_range(mt, seq400[2], seq400[9], -+ xa_mk_value(seq400[2])); -+ mtree_test_store_range(mt, seq400[3], seq400[9], -+ xa_mk_value(seq400[3])); -+ mtree_test_store_range(mt, seq400[4], seq400[9], -+ xa_mk_value(seq400[4])); -+ mtree_test_store_range(mt, seq400[5], seq400[9], -+ xa_mk_value(seq400[5])); -+ mtree_test_store_range(mt, seq400[0], seq400[9], -+ xa_mk_value(seq400[0])); -+ mtree_test_store_range(mt, seq400[6], seq400[9], -+ xa_mk_value(seq400[6])); -+ mtree_test_store_range(mt, seq400[7], seq400[9], -+ xa_mk_value(seq400[7])); -+ mtree_test_store_range(mt, seq400[8], seq400[9], -+ xa_mk_value(seq400[8])); -+ mtree_test_store_range(mt, seq400[10], seq400[11], -+ xa_mk_value(seq400[10])); -+ mt_validate(mt); -+ mt_set_non_kernel(0); -+ mtree_destroy(mt); -+} -+static noinline void check_node_overwrite(struct maple_tree *mt) -+{ -+ int i, max = 4000; -+ -+ for (i = 0; i < max; i++) -+ mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100)); -+ -+ mtree_test_store_range(mt, 319951, 367950, NULL); -+ /*mt_dump(mt); */ -+ mt_validate(mt); -+} -+ -+static void mas_dfs_preorder(struct ma_state *mas) -+{ -+ -+ struct maple_enode *prev; -+ unsigned char end, slot = 0; -+ -+ if (mas_is_start(mas)) { -+ mas_start(mas); -+ return; -+ } -+ -+ if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) -+ goto done; -+ -+walk_up: -+ end = mas_data_end(mas); -+ if (mte_is_leaf(mas->node) || -+ (slot > end)) { -+ if (mte_is_root(mas->node)) -+ goto done; -+ -+ slot = mte_parent_slot(mas->node) + 1; -+ mas_ascend(mas); -+ goto walk_up; -+ } -+ -+ prev = mas->node; -+ mas->node = mas_get_slot(mas, slot); -+ if (!mas->node || slot > end) { -+ if (mte_is_root(prev)) -+ goto done; -+ -+ mas->node = prev; -+ slot = mte_parent_slot(mas->node) + 1; -+ mas_ascend(mas); -+ goto walk_up; -+ } -+ -+ return; -+done: -+ mas->node = MAS_NONE; -+} -+ -+ -+static void check_dfs_preorder(struct maple_tree *mt) -+{ -+ unsigned long count = 0, max = 1000; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ check_seq(mt, max, false); -+ do { -+ count++; -+ mas_dfs_preorder(&mas); -+ } while (!mas_is_none(&mas)); -+ MT_BUG_ON(mt, count != 74); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ mas_reset(&mas); -+ count = 0; -+ check_seq(mt, max, false); -+ do { -+ count++; -+ mas_dfs_preorder(&mas); -+ } while (!mas_is_none(&mas)); -+ /*printk("count %lu\n", count); */ -+ MT_BUG_ON(mt, count != 77); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ mas_reset(&mas); -+ count = 0; -+ check_rev_seq(mt, max, false); -+ do { -+ count++; -+ mas_dfs_preorder(&mas); -+ } while (!mas_is_none(&mas)); -+ /*printk("count %lu\n", count); */ -+ MT_BUG_ON(mt, count != 77); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ mas_reset(&mas); -+ mt_zero_nr_tallocated(); -+ mt_set_non_kernel(200); -+ mas_expected_entries(&mas, max); -+ for (count = 0; count <= max; count++) { -+ mas.index = mas.last = count; -+ mas_store(&mas, xa_mk_value(count)); -+ MT_BUG_ON(mt, mas_is_err(&mas)); -+ } -+ mas_destroy(&mas); -+ rcu_barrier(); -+ /* -+ * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n", -+ * max, mt_get_alloc_size()/1024, mt_nr_allocated(), -+ * mt_nr_tallocated()); -+ */ -+ -+} -+ -+#if defined(BENCH_SLOT_STORE) -+static noinline void bench_slot_store(struct maple_tree *mt) -+{ -+ int i, brk = 105, max = 1040, brk_start = 100, count = 20000000; -+ -+ for (i = 0; i < max; i += 10) -+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); -+ -+ for (i = 0; i < count; i++) { -+ mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL); -+ mtree_store_range(mt, brk_start, brk, xa_mk_value(brk), -+ GFP_KERNEL); -+ } -+} -+#endif -+ -+#if defined(BENCH_NODE_STORE) -+static noinline void bench_node_store(struct maple_tree *mt) -+{ -+ int i, overwrite = 76, max = 240, count = 20000000; -+ -+ for (i = 0; i < max; i += 10) -+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); -+ -+ for (i = 0; i < count; i++) { -+ mtree_store_range(mt, overwrite, overwrite + 15, -+ xa_mk_value(overwrite), GFP_KERNEL); -+ -+ overwrite += 5; -+ if (overwrite >= 135) -+ overwrite = 76; -+ } -+} -+#endif -+ -+#if defined(BENCH_AWALK) -+static noinline void bench_awalk(struct maple_tree *mt) -+{ -+ int i, max = 2500, count = 50000000; -+ MA_STATE(mas, mt, 1470, 1470); -+ -+ for (i = 0; i < max; i += 10) -+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); -+ -+ mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL); -+ -+ for (i = 0; i < count; i++) { -+ mas_empty_area_rev(&mas, 0, 2000, 10); -+ mas_reset(&mas); -+ } -+} -+#endif -+#if defined(BENCH_WALK) -+static noinline void bench_walk(struct maple_tree *mt) -+{ -+ int i, max = 2500, count = 550000000; -+ MA_STATE(mas, mt, 1470, 1470); -+ -+ for (i = 0; i < max; i += 10) -+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); -+ -+ for (i = 0; i < count; i++) { -+ mas_walk(&mas); -+ mas_reset(&mas); -+ } -+ -+} -+#endif -+ -+#if defined(BENCH_MT_FOR_EACH) -+static noinline void bench_mt_for_each(struct maple_tree *mt) -+{ -+ int i, count = 1000000; -+ unsigned long max = 2500, index = 0; -+ void *entry; -+ -+ for (i = 0; i < max; i += 5) -+ mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL); -+ -+ for (i = 0; i < count; i++) { -+ unsigned long j = 0; -+ -+ mt_for_each(mt, entry, index, max) { -+ MT_BUG_ON(mt, entry != xa_mk_value(j)); -+ j += 5; -+ } -+ -+ index = 0; -+ } -+ -+} -+#endif -+ -+static noinline void check_forking(struct maple_tree *mt) -+{ -+ -+ struct maple_tree newmt; -+ int i, nr_entries = 134; -+ void *val; -+ MA_STATE(mas, mt, 0, 0); -+ MA_STATE(newmas, mt, 0, 0); -+ -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ mt_set_non_kernel(99999); -+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); -+ newmas.tree = &newmt; -+ mas_reset(&newmas); -+ mas_reset(&mas); -+ mas.index = 0; -+ mas.last = 0; -+ if (mas_expected_entries(&newmas, nr_entries)) { -+ pr_err("OOM!"); -+ BUG_ON(1); -+ } -+ mas_for_each(&mas, val, ULONG_MAX) { -+ newmas.index = mas.index; -+ newmas.last = mas.last; -+ mas_store(&newmas, val); -+ } -+ mas_destroy(&newmas); -+ mt_validate(&newmt); -+ mt_set_non_kernel(0); -+ mtree_destroy(&newmt); -+} -+ -+static noinline void check_mas_store_gfp(struct maple_tree *mt) -+{ -+ -+ struct maple_tree newmt; -+ int i, nr_entries = 135; -+ void *val; -+ MA_STATE(mas, mt, 0, 0); -+ MA_STATE(newmas, mt, 0, 0); -+ -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ mt_set_non_kernel(99999); -+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); -+ newmas.tree = &newmt; -+ mas_reset(&newmas); -+ mas_set(&mas, 0); -+ mas_for_each(&mas, val, ULONG_MAX) { -+ newmas.index = mas.index; -+ newmas.last = mas.last; -+ mas_store_gfp(&newmas, val, GFP_KERNEL); -+ } -+ -+ mt_validate(&newmt); -+ mt_set_non_kernel(0); -+ mtree_destroy(&newmt); -+} -+ -+#if defined(BENCH_FORK) -+static noinline void bench_forking(struct maple_tree *mt) -+{ -+ -+ struct maple_tree newmt; -+ int i, nr_entries = 134, nr_fork = 80000; -+ void *val; -+ MA_STATE(mas, mt, 0, 0); -+ MA_STATE(newmas, mt, 0, 0); -+ -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ for (i = 0; i < nr_fork; i++) { -+ mt_set_non_kernel(99999); -+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); -+ newmas.tree = &newmt; -+ mas_reset(&newmas); -+ mas_reset(&mas); -+ mas.index = 0; -+ mas.last = 0; -+ if (mas_expected_entries(&newmas, nr_entries)) { -+ printk("OOM!"); -+ BUG_ON(1); -+ } -+ mas_for_each(&mas, val, ULONG_MAX) { -+ newmas.index = mas.index; -+ newmas.last = mas.last; -+ mas_store(&newmas, val); -+ } -+ mas_destroy(&newmas); -+ mt_validate(&newmt); -+ mt_set_non_kernel(0); -+ mtree_destroy(&newmt); -+ } -+} -+#endif -+ -+static noinline void next_prev_test(struct maple_tree *mt) -+{ -+ int i, nr_entries = 200; -+ void *val; -+ MA_STATE(mas, mt, 0, 0); -+ struct maple_enode *mn; -+ -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ for (i = 0; i <= nr_entries / 2; i++) { -+ mas_next(&mas, 1000); -+ if (mas_is_none(&mas)) -+ break; -+ -+ } -+ mas_reset(&mas); -+ mas_set(&mas, 0); -+ i = 0; -+ mas_for_each(&mas, val, 1000) { -+ i++; -+ } -+ -+ mas_reset(&mas); -+ mas_set(&mas, 0); -+ i = 0; -+ mas_for_each(&mas, val, 1000) { -+ mas_pause(&mas); -+ i++; -+ } -+ -+ /* -+ * 680 - 685 = 0x61a00001930c -+ * 686 - 689 = NULL; -+ * 690 - 695 = 0x61a00001930c -+ * Check simple next/prev -+ */ -+ mas_set(&mas, 686); -+ val = mas_walk(&mas); -+ MT_BUG_ON(mt, val != NULL); -+ -+ val = mas_next(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); -+ MT_BUG_ON(mt, mas.index != 690); -+ MT_BUG_ON(mt, mas.last != 695); -+ -+ val = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(680 / 10)); -+ MT_BUG_ON(mt, mas.index != 680); -+ MT_BUG_ON(mt, mas.last != 685); -+ -+ val = mas_next(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); -+ MT_BUG_ON(mt, mas.index != 690); -+ MT_BUG_ON(mt, mas.last != 695); -+ -+ val = mas_next(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(700 / 10)); -+ MT_BUG_ON(mt, mas.index != 700); -+ MT_BUG_ON(mt, mas.last != 705); -+ -+ /* Check across node boundaries of the tree */ -+ mas_set(&mas, 70); -+ val = mas_walk(&mas); -+ MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); -+ MT_BUG_ON(mt, mas.index != 70); -+ MT_BUG_ON(mt, mas.last != 75); -+ -+ val = mas_next(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(80 / 10)); -+ MT_BUG_ON(mt, mas.index != 80); -+ MT_BUG_ON(mt, mas.last != 85); -+ -+ val = mas_prev(&mas, 70); -+ MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); -+ MT_BUG_ON(mt, mas.index != 70); -+ MT_BUG_ON(mt, mas.last != 75); -+ -+ /* Check across two levels of the tree */ -+ mas_reset(&mas); -+ mas_set(&mas, 707); -+ val = mas_walk(&mas); -+ MT_BUG_ON(mt, val != NULL); -+ val = mas_next(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); -+ MT_BUG_ON(mt, mas.index != 710); -+ MT_BUG_ON(mt, mas.last != 715); -+ mn = mas.node; -+ -+ val = mas_next(&mas, 1000); -+ MT_BUG_ON(mt, val != xa_mk_value(720 / 10)); -+ MT_BUG_ON(mt, mas.index != 720); -+ MT_BUG_ON(mt, mas.last != 725); -+ MT_BUG_ON(mt, mn == mas.node); -+ -+ val = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); -+ MT_BUG_ON(mt, mas.index != 710); -+ MT_BUG_ON(mt, mas.last != 715); -+ -+ /* Check running off the end and back on */ -+ mas_reset(&mas); -+ mas_set(&mas, 2000); -+ val = mas_walk(&mas); -+ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); -+ MT_BUG_ON(mt, mas.index != 2000); -+ MT_BUG_ON(mt, mas.last != 2005); -+ -+ val = mas_next(&mas, ULONG_MAX); -+ MT_BUG_ON(mt, val != NULL); -+ MT_BUG_ON(mt, mas.index != ULONG_MAX); -+ MT_BUG_ON(mt, mas.last != ULONG_MAX); -+ -+ val = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); -+ MT_BUG_ON(mt, mas.index != 2000); -+ MT_BUG_ON(mt, mas.last != 2005); -+ -+ /* Check running off the start and back on */ -+ mas_reset(&mas); -+ mas_set(&mas, 10); -+ val = mas_walk(&mas); -+ MT_BUG_ON(mt, val != xa_mk_value(1)); -+ MT_BUG_ON(mt, mas.index != 10); -+ MT_BUG_ON(mt, mas.last != 15); -+ -+ val = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, val != xa_mk_value(0)); -+ MT_BUG_ON(mt, mas.index != 0); -+ MT_BUG_ON(mt, mas.last != 5); -+ -+ val = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, val != NULL); -+ MT_BUG_ON(mt, mas.index != 0); -+ MT_BUG_ON(mt, mas.last != 0); -+ -+ mas.index = 0; -+ mas.last = 5; -+ mas_store(&mas, NULL); -+ mas_reset(&mas); -+ mas_set(&mas, 10); -+ mas_walk(&mas); -+ -+ val = mas_prev(&mas, 0); -+ MT_BUG_ON(mt, val != NULL); -+ MT_BUG_ON(mt, mas.index != 0); -+ MT_BUG_ON(mt, mas.last != 0); -+ -+ mtree_destroy(mt); -+ -+ mt_init(mt); -+ mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); -+ mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL); -+ mas_set(&mas, 5); -+ val = mas_prev(&mas, 4); -+ MT_BUG_ON(mt, val != NULL); -+} -+ -+#define RCU_RANGE_COUNT 1000 -+#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true;} MT_BUG_ON(test->mt, y);} -+struct rcu_test_struct2 { -+ struct maple_tree *mt; -+ -+ bool start; -+ bool stop; -+ unsigned int thread_count; -+ -+ unsigned int seen_toggle; -+ unsigned int seen_added; -+ unsigned int seen_modified; -+ unsigned int seen_deleted; -+ int pause; -+ -+ unsigned long index[RCU_RANGE_COUNT]; -+ unsigned long last[RCU_RANGE_COUNT]; -+}; -+ -+struct rcu_reader_struct { -+ unsigned int id; -+ int mod; -+ int del; -+ int flip; -+ int add; -+ int next; -+ struct rcu_test_struct2 *test; -+}; -+ -+/* RCU reader helper function */ -+static void rcu_reader_register(struct rcu_test_struct2 *test) -+{ -+ rcu_register_thread(); -+ uatomic_inc(&test->thread_count); -+ -+ while (!test->start) -+ usleep(test->pause * 100); -+} -+ -+static void rcu_reader_setup(struct rcu_reader_struct *reader, -+ unsigned int id, struct rcu_test_struct2 *test) -+{ -+ reader->id = id; -+ reader->test = test; -+ reader->mod = reader->id % 10; -+ reader->del = (reader->mod + 1) % 10; -+ reader->flip = (reader->mod + 2) % 10; -+ reader->add = (reader->mod + 3) % 10; -+ reader->next = (reader->mod + 4) % 10; -+} -+ -+/* RCU reader in increasing index */ -+static void *rcu_reader_fwd(void *ptr) -+{ -+ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; -+ struct rcu_test_struct2 *test = reader->test; -+ unsigned long index = reader->id; -+ bool toggled, modified, deleted, added; -+ int i; -+ void *entry, *prev = NULL; -+ MA_STATE(mas, test->mt, 0, 0); -+ -+ rcu_reader_register(test); -+ toggled = modified = deleted = added = false; -+ -+ while (!test->stop) { -+ i = 0; -+ /* mas_for_each ?*/ -+ rcu_read_lock(); -+ mas_set(&mas, test->index[index]); -+ mas_for_each(&mas, entry, test->last[index + 9]) { -+ unsigned long r_start, r_end, alt_start; -+ void *expected, *alt; -+ -+ r_start = test->index[index + i]; -+ r_end = test->last[index + i]; -+ expected = xa_mk_value(r_start); -+ -+ if (i == reader->del) { -+ if (!deleted) { -+ alt_start = test->index[index + reader->flip]; -+ /* delete occurred. */ -+ if (mas.index == alt_start) { -+ uatomic_inc(&test->seen_deleted); -+ deleted = true; -+ } -+ } -+ if (deleted) { -+ i = reader->flip; -+ r_start = test->index[index + i]; -+ r_end = test->last[index + i]; -+ expected = xa_mk_value(r_start); -+ } -+ } -+ -+ if (!added && (i == reader->add)) { -+ alt_start = test->index[index + reader->next]; -+ if (mas.index == r_start) { -+ uatomic_inc(&test->seen_added); -+ added = true; -+ } else if (mas.index == alt_start) { -+ i = reader->next; -+ r_start = test->index[index + i]; -+ r_end = test->last[index + i]; -+ expected = xa_mk_value(r_start); -+ } -+ } -+ -+ RCU_MT_BUG_ON(test, mas.index != r_start); -+ RCU_MT_BUG_ON(test, mas.last != r_end); -+ -+ if (i == reader->flip) { -+ alt = xa_mk_value(index + i + RCU_RANGE_COUNT); -+ if (prev) { -+ if (toggled && entry == expected) -+ uatomic_inc(&test->seen_toggle); -+ else if (!toggled && entry == alt) -+ uatomic_inc(&test->seen_toggle); -+ } -+ -+ if (entry == expected) -+ toggled = false; -+ else if (entry == alt) -+ toggled = true; -+ else { -+ printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); -+ RCU_MT_BUG_ON(test, 1); -+ } -+ -+ prev = entry; -+ } else if (i == reader->mod) { -+ alt = xa_mk_value(index + i * 2 + 1 + -+ RCU_RANGE_COUNT); -+ if (entry != expected) { -+ if (!modified) -+ uatomic_inc(&test->seen_modified); -+ modified = true; -+ } else { -+ if (modified) -+ uatomic_inc(&test->seen_modified); -+ modified = false; -+ } -+ -+ if (modified) -+ RCU_MT_BUG_ON(test, entry != alt); -+ -+ } else { -+ if (entry != expected) -+ printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); -+ RCU_MT_BUG_ON(test, entry != expected); -+ } -+ -+ i++; -+ } -+ rcu_read_unlock(); -+ usleep(test->pause); -+ } -+ -+ rcu_unregister_thread(); -+ return NULL; -+} -+ -+/* RCU reader in decreasing index */ -+static void *rcu_reader_rev(void *ptr) -+{ -+ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; -+ struct rcu_test_struct2 *test = reader->test; -+ unsigned long index = reader->id; -+ bool toggled, modified, deleted, added; -+ int i; -+ void *prev = NULL; -+ MA_STATE(mas, test->mt, 0, 0); -+ -+ rcu_reader_register(test); -+ toggled = modified = deleted = added = false; -+ -+ -+ while (!test->stop) { -+ void *entry; -+ -+ i = 9; -+ mas_set(&mas, test->index[index + i]); -+ -+ rcu_read_lock(); -+ while (i--) { -+ unsigned long r_start, r_end, alt_start; -+ void *expected, *alt; -+ int line = __LINE__; -+ -+ entry = mas_prev(&mas, test->index[index]); -+ r_start = test->index[index + i]; -+ r_end = test->last[index + i]; -+ expected = xa_mk_value(r_start); -+ -+ if (i == reader->del) { -+ alt_start = test->index[index + reader->mod]; -+ if (mas.index == alt_start) { -+ line = __LINE__; -+ if (!deleted) -+ uatomic_inc(&test->seen_deleted); -+ deleted = true; -+ } -+ if (deleted) { -+ line = __LINE__; -+ i = reader->mod; -+ r_start = test->index[index + i]; -+ r_end = test->last[index + i]; -+ expected = xa_mk_value(r_start); -+ } -+ } -+ if (!added && (i == reader->add)) { -+ alt_start = test->index[index + reader->flip]; -+ if (mas.index == r_start) { -+ line = __LINE__; -+ uatomic_inc(&test->seen_added); -+ added = true; -+ } else if (mas.index == alt_start) { -+ line = __LINE__; -+ i = reader->flip; -+ r_start = test->index[index + i]; -+ r_end = test->last[index + i]; -+ expected = xa_mk_value(r_start); -+ } -+ } -+ -+ if (i == reader->mod) -+ line = __LINE__; -+ else if (i == reader->flip) -+ line = __LINE__; -+ -+ if (mas.index != r_start) { -+ alt = xa_mk_value(index + i * 2 + 1 + -+ RCU_RANGE_COUNT); -+ mt_dump(test->mt); -+ printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n", -+ mas.index, mas.last, entry, -+ r_start, r_end, expected, alt, -+ line, i); -+ } -+ RCU_MT_BUG_ON(test, mas.index != r_start); -+ RCU_MT_BUG_ON(test, mas.last != r_end); -+ -+ if (i == reader->mod) { -+ alt = xa_mk_value(index + i * 2 + 1 + -+ RCU_RANGE_COUNT); -+ -+ if (entry != expected) { -+ if (!modified) -+ uatomic_inc(&test->seen_modified); -+ modified = true; -+ } else { -+ if (modified) -+ uatomic_inc(&test->seen_modified); -+ modified = false; -+ } -+ if (modified) -+ RCU_MT_BUG_ON(test, entry != alt); -+ -+ -+ } else if (i == reader->flip) { -+ alt = xa_mk_value(index + i + -+ RCU_RANGE_COUNT); -+ if (prev) { -+ if (toggled && entry == expected) -+ uatomic_inc(&test->seen_toggle); -+ else if (!toggled && entry == alt) -+ uatomic_inc(&test->seen_toggle); -+ } -+ -+ if (entry == expected) -+ toggled = false; -+ else if (entry == alt) -+ toggled = true; -+ else { -+ printk("%lu-%lu %p != %p or %p\n", -+ mas.index, mas.last, entry, -+ expected, alt); -+ RCU_MT_BUG_ON(test, 1); -+ } -+ -+ prev = entry; -+ } else { -+ if (entry != expected) -+ printk("%lu-%lu %p != %p\n", mas.index, -+ mas.last, entry, expected); -+ RCU_MT_BUG_ON(test, entry != expected); -+ } -+ } -+ rcu_read_unlock(); -+ usleep(test->pause); -+ } -+ -+ rcu_unregister_thread(); -+ return NULL; -+} -+ -+static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test, -+ int count, struct rcu_reader_struct *test_reader) -+{ -+ int i, j = 10000; -+ bool toggle = true; -+ -+ test->start = true; /* Release the hounds! */ -+ usleep(5); -+ -+ while (j--) { -+ toggle = !toggle; -+ i = count; -+ while (i--) { -+ unsigned long start, end; -+ struct rcu_reader_struct *this = &test_reader[i]; -+ -+ /* Mod offset */ -+ if (j == 600) { -+ start = test->index[this->id + this->mod]; -+ end = test->last[this->id + this->mod]; -+ mtree_store_range(mt, start, end, -+ xa_mk_value(this->id + this->mod * 2 + -+ 1 + RCU_RANGE_COUNT), -+ GFP_KERNEL); -+ } -+ -+ /* Toggle */ -+ if (!(j % 5)) { -+ start = test->index[this->id + this->flip]; -+ end = test->last[this->id + this->flip]; -+ mtree_store_range(mt, start, end, -+ xa_mk_value((toggle ? start : -+ this->id + this->flip + -+ RCU_RANGE_COUNT)), -+ GFP_KERNEL); -+ } -+ -+ /* delete */ -+ if (j == 400) { -+ start = test->index[this->id + this->del]; -+ end = test->last[this->id + this->del]; -+ mtree_store_range(mt, start, end, NULL, GFP_KERNEL); -+ } -+ -+ /* add */ -+ if (j == 500) { -+ start = test->index[this->id + this->add]; -+ end = test->last[this->id + this->add]; -+ mtree_store_range(mt, start, end, -+ xa_mk_value(start), GFP_KERNEL); -+ } -+ } -+ usleep(test->pause); -+ /* If a test fails, don't flood the console */ -+ if (test->stop) -+ break; -+ } -+} -+ -+static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test, -+ int count, struct rcu_reader_struct *test_reader) -+{ -+ int j, i; -+ bool toggle = true; -+ -+ test->start = true; /* Release the hounds! */ -+ usleep(5); -+ for (j = 0; j < 10000; j++) { -+ toggle = !toggle; -+ for (i = 0; i < count; i++) { -+ unsigned long start, end; -+ struct rcu_reader_struct *this = &test_reader[i]; -+ -+ /* Mod offset */ -+ if (j == 600) { -+ start = test->index[this->id + this->mod]; -+ end = test->last[this->id + this->mod]; -+ mtree_store_range(mt, start, end, -+ xa_mk_value(this->id + this->mod * 2 + -+ 1 + RCU_RANGE_COUNT), -+ GFP_KERNEL); -+ } -+ -+ /* Toggle */ -+ if (!(j % 5)) { -+ start = test->index[this->id + this->flip]; -+ end = test->last[this->id + this->flip]; -+ mtree_store_range(mt, start, end, -+ xa_mk_value((toggle ? start : -+ this->id + this->flip + -+ RCU_RANGE_COUNT)), -+ GFP_KERNEL); -+ } -+ -+ /* delete */ -+ if (j == 400) { -+ start = test->index[this->id + this->del]; -+ end = test->last[this->id + this->del]; -+ mtree_store_range(mt, start, end, NULL, GFP_KERNEL); -+ } -+ -+ /* add */ -+ if (j == 500) { -+ start = test->index[this->id + this->add]; -+ end = test->last[this->id + this->add]; -+ mtree_store_range(mt, start, end, -+ xa_mk_value(start), GFP_KERNEL); -+ } -+ } -+ usleep(test->pause); -+ /* If a test fails, don't flood the console */ -+ if (test->stop) -+ break; -+ } -+} -+ -+/* -+ * This is to check: -+ * 1. Range that is not ever present -+ * 2. Range that is always present -+ * 3. Things being added but not removed. -+ * 4. Things being removed but not added. -+ * 5. Things are being added and removed, searches my succeed or fail -+ * -+ * This sets up two readers for every 10 entries; one forward and one reverse -+ * reading. -+ */ -+static void rcu_stress(struct maple_tree *mt, bool forward) -+{ -+ unsigned int count, i; -+ unsigned long r, seed; -+ pthread_t readers[RCU_RANGE_COUNT / 5]; -+ struct rcu_test_struct2 test; -+ struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5]; -+ void *(*function)(void *); -+ -+ /* Test setup */ -+ test.mt = mt; -+ test.pause = 5; -+ test.seen_toggle = 0; -+ test.seen_deleted = 0; -+ test.seen_added = 0; -+ test.seen_modified = 0; -+ test.thread_count = 0; -+ test.start = test.stop = false; -+ seed = time(NULL); -+ srand(seed); -+ for (i = 0; i < RCU_RANGE_COUNT; i++) { -+ r = seed + rand(); -+ mtree_store_range(mt, seed, r, -+ xa_mk_value(seed), GFP_KERNEL); -+ -+ /* Record start and end of entry */ -+ test.index[i] = seed; -+ test.last[i] = r; -+ seed = 1 + r + rand() % 10; -+ } -+ -+ i = count = ARRAY_SIZE(readers); -+ while (i--) { -+ unsigned long id; -+ -+ id = i / 2 * 10; -+ if (i % 2) -+ function = rcu_reader_fwd; -+ else -+ function = rcu_reader_rev; -+ -+ rcu_reader_setup(&test_reader[i], id, &test); -+ if (pthread_create(&readers[i], NULL, *function, -+ &test_reader[i])) { -+ perror("creating reader thread"); -+ exit(1); -+ } -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(readers); i++) { -+ struct rcu_reader_struct *this = &test_reader[i]; -+ int add = this->id + this->add; -+ -+ /* Remove add entries from the tree for later addition */ -+ mtree_store_range(mt, test.index[add], test.last[add], -+ NULL, GFP_KERNEL); -+ } -+ -+ mt_set_in_rcu(mt); -+ do { -+ usleep(5); -+ } while (test.thread_count > ARRAY_SIZE(readers)); -+ -+ if (forward) -+ rcu_stress_fwd(mt, &test, count, test_reader); -+ else -+ rcu_stress_rev(mt, &test, count, test_reader); -+ -+ test.stop = true; -+ while (count--) -+ pthread_join(readers[count], NULL); -+ -+ mt_validate(mt); -+} -+ -+ -+struct rcu_test_struct { -+ struct maple_tree *mt; /* the maple tree */ -+ int count; /* Number of times to check value(s) */ -+ unsigned long index; /* The first index to check */ -+ void *entry1; /* The first entry value */ -+ void *entry2; /* The second entry value */ -+ void *entry3; /* The third entry value */ -+ -+ bool update_2; -+ bool update_3; -+ unsigned long range_start; -+ unsigned long range_end; -+ unsigned int loop_sleep; -+ unsigned int val_sleep; -+ -+ unsigned int failed; /* failed detection for other threads */ -+ unsigned int seen_entry2; /* Number of threads that have seen the new value */ -+ unsigned int seen_entry3; /* Number of threads that have seen the new value */ -+ unsigned int seen_both; /* Number of threads that have seen both new values */ -+ unsigned int seen_toggle; -+ unsigned int seen_added; -+ unsigned int seen_removed; -+ unsigned long last; /* The end of the range to write. */ -+ -+ unsigned long removed; /* The index of the removed entry */ -+ unsigned long added; /* The index of the removed entry */ -+ unsigned long toggle; /* The index of the removed entry */ -+}; -+ -+static inline -+int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2, -+ bool *update_3) -+{ -+ if (entry == test->entry1) -+ return 0; -+ -+ if (entry == test->entry2) { -+ if (!(*update_2)) { -+ uatomic_inc(&test->seen_entry2); -+ *update_2 = true; -+ if (update_3) -+ uatomic_inc(&test->seen_both); -+ } -+ return 0; -+ } -+ -+ if (entry == test->entry3) { -+ if (!(*update_3)) { -+ uatomic_inc(&test->seen_entry3); -+ *update_3 = true; -+ if (update_2) -+ uatomic_inc(&test->seen_both); -+ } -+ return 0; -+ } -+ -+ return 1; -+} -+ -+/* -+ * rcu_val() - Read a given value in the tree test->count times using the -+ * regular API -+ * -+ * @ptr: The pointer to the rcu_test_struct -+ */ -+static void *rcu_val(void *ptr) -+{ -+ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; -+ unsigned long count = test->count; -+ bool update_2 = false; -+ bool update_3 = false; -+ void *entry; -+ -+ rcu_register_thread(); -+ while (count--) { -+ usleep(test->val_sleep); -+ /* -+ * No locking required, regular API locking is handled in the -+ * maple tree code -+ */ -+ entry = mtree_load(test->mt, test->index); -+ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2, -+ &update_3)); -+ } -+ rcu_unregister_thread(); -+ return NULL; -+} -+ -+/* -+ * rcu_loop() - Loop over a section of the maple tree, checking for an expected -+ * value using the advanced API -+ * -+ * @ptr - The pointer to the rcu_test_struct -+ */ -+static void *rcu_loop(void *ptr) -+{ -+ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; -+ unsigned long count = test->count; -+ void *entry, *expected; -+ bool update_2 = false; -+ bool update_3 = false; -+ MA_STATE(mas, test->mt, test->range_start, test->range_start); -+ -+ rcu_register_thread(); -+ -+ /* -+ * Loop through the test->range_start - test->range_end test->count -+ * times -+ */ -+ while (count--) { -+ usleep(test->loop_sleep); -+ rcu_read_lock(); -+ mas_for_each(&mas, entry, test->range_end) { -+ /* The expected value is based on the start range. */ -+ expected = xa_mk_value(mas.index ? mas.index / 10 : 0); -+ -+ /* Out of the interesting range */ -+ if (mas.index < test->index || mas.index > test->last) { -+ if (entry != expected) { -+ printk("%lx - %lx = %p not %p\n", -+ mas.index, mas.last, entry, expected); -+ } -+ MT_BUG_ON(test->mt, entry != expected); -+ continue; -+ } -+ -+ if (entry == expected) -+ continue; /* Not seen. */ -+ -+ /* In the interesting range */ -+ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, -+ &update_2, -+ &update_3)); -+ } -+ rcu_read_unlock(); -+ mas_set(&mas, test->range_start); -+ } -+ -+ rcu_unregister_thread(); -+ return NULL; -+} -+ -+static noinline -+void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals) -+{ -+ -+ int i; -+ void *(*function)(void *); -+ pthread_t readers[20]; -+ -+ mt_set_in_rcu(mt); -+ MT_BUG_ON(mt, !mt_in_rcu(mt)); -+ -+ for (i = 0; i < ARRAY_SIZE(readers); i++) { -+ if (i % 2) -+ function = rcu_loop; -+ else -+ function = rcu_val; -+ -+ if (pthread_create(&readers[i], NULL, *function, vals)) { -+ perror("creating reader thread"); -+ exit(1); -+ } -+ } -+ -+ usleep(5); /* small yield to ensure all threads are at least started. */ -+ mtree_store_range(mt, vals->index, vals->last, vals->entry2, -+ GFP_KERNEL); -+ while (i--) -+ pthread_join(readers[i], NULL); -+ -+ /* Make sure the test caught at least one update. */ -+ MT_BUG_ON(mt, !vals->seen_entry2); -+} -+ -+static noinline -+void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals) -+{ -+ -+ int i; -+ void *(*function)(void *); -+ pthread_t readers[20]; -+ unsigned int index = vals->index; -+ -+ mt_set_in_rcu(mt); -+ MT_BUG_ON(mt, !mt_in_rcu(mt)); -+ -+ for (i = 0; i < ARRAY_SIZE(readers); i++) { -+ if (i % 2) -+ function = rcu_loop; -+ else -+ function = rcu_val; -+ -+ if (pthread_create(&readers[i], NULL, *function, vals)) { -+ perror("creating reader thread"); -+ exit(1); -+ } -+ } -+ -+ usleep(5); /* small yield to ensure all threads are at least started. */ -+ -+ while (index <= vals->last) { -+ mtree_store(mt, index, -+ (index % 2 ? vals->entry2 : vals->entry3), -+ GFP_KERNEL); -+ index++; -+ usleep(5); -+ } -+ -+ while (i--) -+ pthread_join(readers[i], NULL); -+ -+ /* Make sure the test caught at least one update. */ -+ MT_BUG_ON(mt, !vals->seen_entry2); -+ MT_BUG_ON(mt, !vals->seen_entry3); -+ MT_BUG_ON(mt, !vals->seen_both); -+} -+static noinline void check_rcu_simulated(struct maple_tree *mt) -+{ -+ unsigned long i, nr_entries = 1000; -+ unsigned long target = 4320; -+ unsigned long val = 0xDEAD; -+ -+ MA_STATE(mas_writer, mt, 0, 0); -+ MA_STATE(mas_reader, mt, target, target); -+ -+ rcu_register_thread(); -+ -+ mt_set_in_rcu(mt); -+ mas_lock(&mas_writer); -+ for (i = 0; i <= nr_entries; i++) { -+ mas_writer.index = i * 10; -+ mas_writer.last = i * 10 + 5; -+ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); -+ } -+ mas_unlock(&mas_writer); -+ -+ /* Overwrite one entry with a new value. */ -+ mas_set_range(&mas_writer, target, target + 5); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ /* Restore value. */ -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ mas_reset(&mas_reader); -+ -+ -+ /* Overwrite 1/2 the entry */ -+ mas_set_range(&mas_writer, target, target + 2); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ -+ /* Restore value. */ -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ mas_reset(&mas_reader); -+ -+ /* Overwrite last 1/2 the entry */ -+ mas_set_range(&mas_writer, target + 2, target + 5); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ rcu_read_unlock(); -+ -+ -+ /* Restore value. */ -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ mas_reset(&mas_reader); -+ -+ /* Overwrite more than the entry */ -+ mas_set_range(&mas_writer, target - 5, target + 15); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ /* Restore value. */ -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ mas_reset(&mas_reader); -+ -+ /* Overwrite more than the node. */ -+ mas_set_range(&mas_writer, target - 400, target + 400); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ /* Restore value. */ -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ mas_reset(&mas_reader); -+ -+ /* Overwrite the tree */ -+ mas_set_range(&mas_writer, 0, ULONG_MAX); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ /* Clear out tree & recreate it */ -+ mas_lock(&mas_writer); -+ mas_set_range(&mas_writer, 0, ULONG_MAX); -+ mas_store_gfp(&mas_writer, NULL, GFP_KERNEL); -+ mas_set_range(&mas_writer, 0, 0); -+ for (i = 0; i <= nr_entries; i++) { -+ mas_writer.index = i * 10; -+ mas_writer.last = i * 10 + 5; -+ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); -+ } -+ mas_unlock(&mas_writer); -+ -+ /* next check */ -+ /* Overwrite one entry with a new value. */ -+ mas_reset(&mas_reader); -+ mas_set_range(&mas_writer, target, target + 5); -+ mas_set_range(&mas_reader, target, target); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_prev(&mas_reader, 0); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ /* Restore value. */ -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ -+ /* prev check */ -+ /* Overwrite one entry with a new value. */ -+ mas_reset(&mas_reader); -+ mas_set_range(&mas_writer, target, target + 5); -+ mas_set_range(&mas_reader, target, target); -+ rcu_read_lock(); -+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); -+ mas_next(&mas_reader, ULONG_MAX); -+ mas_lock(&mas_writer); -+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); -+ mas_unlock(&mas_writer); -+ MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val)); -+ rcu_read_unlock(); -+ -+ rcu_unregister_thread(); -+} -+ -+static noinline void check_rcu_threaded(struct maple_tree *mt) -+{ -+ unsigned long i, nr_entries = 1000; -+ struct rcu_test_struct vals; -+ -+ vals.val_sleep = 200; -+ vals.loop_sleep = 110; -+ -+ rcu_register_thread(); -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ /* Store across several slots. */ -+ vals.count = 1000; -+ vals.mt = mt; -+ vals.index = 8650; -+ vals.last = 8666; -+ vals.entry1 = xa_mk_value(865); -+ vals.entry2 = xa_mk_value(8650); -+ vals.entry3 = xa_mk_value(8650); -+ vals.range_start = 0; -+ vals.range_end = ULONG_MAX; -+ vals.seen_entry2 = 0; -+ vals.seen_entry3 = 0; -+ -+ run_check_rcu(mt, &vals); -+ mtree_destroy(mt); -+ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ /* 4390-4395: value 439 (0x1b7) [0x36f] */ -+ /* Store across several slots. */ -+ /* Spanning store. */ -+ vals.count = 10000; -+ vals.mt = mt; -+ vals.index = 4390; -+ vals.last = 4398; -+ vals.entry1 = xa_mk_value(4390); -+ vals.entry2 = xa_mk_value(439); -+ vals.entry3 = xa_mk_value(439); -+ vals.seen_entry2 = 0; -+ vals.range_start = 4316; -+ vals.range_end = 5035; -+ run_check_rcu(mt, &vals); -+ mtree_destroy(mt); -+ -+ -+ /* Forward writer for rcu stress */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ rcu_stress(mt, true); -+ mtree_destroy(mt); -+ -+ /* Reverse writer for rcu stress */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ rcu_stress(mt, false); -+ mtree_destroy(mt); -+ -+ /* Slow reader test with spanning store. */ -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ /* 4390-4395: value 439 (0x1b7) [0x36f] */ -+ /* Store across several slots. */ -+ /* Spanning store. */ -+ vals.count = 15000; -+ vals.mt = mt; -+ vals.index = 4390; -+ vals.last = 4398; -+ vals.entry1 = xa_mk_value(4390); -+ vals.entry2 = xa_mk_value(439); -+ vals.entry3 = xa_mk_value(4391); -+ vals.seen_toggle = 0; -+ vals.seen_added = 0; -+ vals.seen_removed = 0; -+ vals.range_start = 4316; -+ vals.range_end = 5035; -+ vals.removed = 4360; -+ vals.added = 4396; -+ vals.toggle = 4347; -+ vals.val_sleep = 400; -+ vals.loop_sleep = 200; -+ vals.seen_entry2 = 0; -+ vals.seen_entry3 = 0; -+ vals.seen_both = 0; -+ vals.entry3 = xa_mk_value(438); -+ -+ run_check_rcu_slowread(mt, &vals); -+ rcu_unregister_thread(); -+} -+ -+extern void test_kmem_cache_bulk(void); -+ -+/* Test spanning writes that require balancing right sibling or right cousin */ -+static noinline void check_spanning_relatives(struct maple_tree *mt) -+{ -+ -+ unsigned long i, nr_entries = 1000; -+ -+ for (i = 0; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, i*10 + 5, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ -+ mtree_store_range(mt, 9365, 9955, NULL, GFP_KERNEL); -+} -+ -+static noinline void check_fuzzer(struct maple_tree *mt) -+{ -+ /* -+ * 1. Causes a spanning rebalance of a single root node. -+ * Fixed by setting the correct limit in mast_cp_to_nodes() when the -+ * entire right side is consumed. -+ */ -+ mtree_test_insert(mt, 88, (void *)0xb1); -+ mtree_test_insert(mt, 84, (void *)0xa9); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 4, (void *)0x9); -+ mtree_test_insert(mt, 14, (void *)0x1d); -+ mtree_test_insert(mt, 7, (void *)0xf); -+ mtree_test_insert(mt, 12, (void *)0x19); -+ mtree_test_insert(mt, 18, (void *)0x25); -+ mtree_test_store_range(mt, 8, 18, (void *)0x11); -+ mtree_destroy(mt); -+ -+ -+ /* -+ * 2. Cause a spanning rebalance of two nodes in root. -+ * Fixed by setting mast->r->max correctly. -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_store(mt, 87, (void *)0xaf); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_load(mt, 4); -+ mtree_test_insert(mt, 4, (void *)0x9); -+ mtree_test_store(mt, 8, (void *)0x11); -+ mtree_test_store(mt, 44, (void *)0x59); -+ mtree_test_store(mt, 68, (void *)0x89); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 43, (void *)0x57); -+ mtree_test_insert(mt, 24, (void *)0x31); -+ mtree_test_insert(mt, 844, (void *)0x699); -+ mtree_test_store(mt, 84, (void *)0xa9); -+ mtree_test_store(mt, 4, (void *)0x9); -+ mtree_test_erase(mt, 4); -+ mtree_test_load(mt, 5); -+ mtree_test_erase(mt, 0); -+ mtree_destroy(mt); -+ -+ /* -+ * 3. Cause a node overflow on copy -+ * Fixed by using the correct check for node size in mas_wr_modify() -+ * Also discovered issue with metadata setting. -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_store_range(mt, 0, 18446744073709551615UL, (void *)0x1); -+ mtree_test_store(mt, 4, (void *)0x9); -+ mtree_test_erase(mt, 5); -+ mtree_test_erase(mt, 0); -+ mtree_test_erase(mt, 4); -+ mtree_test_store(mt, 5, (void *)0xb); -+ mtree_test_erase(mt, 5); -+ mtree_test_store(mt, 5, (void *)0xb); -+ mtree_test_erase(mt, 5); -+ mtree_test_erase(mt, 4); -+ mtree_test_store(mt, 4, (void *)0x9); -+ mtree_test_store(mt, 444, (void *)0x379); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_load(mt, 0); -+ mtree_test_store(mt, 5, (void *)0xb); -+ mtree_test_erase(mt, 0); -+ mtree_destroy(mt); -+ -+ /* -+ * 4. spanning store failure due to writing incorrect pivot value at -+ * last slot. -+ * Fixed by setting mast->r->max correctly in mast_cp_to_nodes() -+ * -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_insert(mt, 261, (void *)0x20b); -+ mtree_test_store(mt, 516, (void *)0x409); -+ mtree_test_store(mt, 6, (void *)0xd); -+ mtree_test_insert(mt, 5, (void *)0xb); -+ mtree_test_insert(mt, 1256, (void *)0x9d1); -+ mtree_test_store(mt, 4, (void *)0x9); -+ mtree_test_erase(mt, 1); -+ mtree_test_store(mt, 56, (void *)0x71); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_store(mt, 24, (void *)0x31); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 2263, (void *)0x11af); -+ mtree_test_insert(mt, 446, (void *)0x37d); -+ mtree_test_store_range(mt, 6, 45, (void *)0xd); -+ mtree_test_store_range(mt, 3, 446, (void *)0x7); -+ mtree_destroy(mt); -+ -+ /* -+ * 5. mas_wr_extend_null() may overflow slots. -+ * Fix by checking against wr_mas->node_end. -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_store(mt, 48, (void *)0x61); -+ mtree_test_store(mt, 3, (void *)0x7); -+ mtree_test_load(mt, 0); -+ mtree_test_store(mt, 88, (void *)0xb1); -+ mtree_test_store(mt, 81, (void *)0xa3); -+ mtree_test_insert(mt, 0, (void *)0x1); -+ mtree_test_insert(mt, 8, (void *)0x11); -+ mtree_test_insert(mt, 4, (void *)0x9); -+ mtree_test_insert(mt, 2480, (void *)0x1361); -+ mtree_test_insert(mt, 18446744073709551615UL, -+ (void *)0xffffffffffffffff); -+ mtree_test_erase(mt, 18446744073709551615UL); -+ mtree_destroy(mt); -+ -+ /* -+ * 6. When reusing a node with an implied pivot and the node is -+ * shrinking, old data would be left in the implied slot -+ * Fixed by checking the last pivot for the mas->max and clear -+ * accordingly. This only affected the left-most node as that node is -+ * the only one allowed to end in NULL. -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_erase(mt, 3); -+ mtree_test_insert(mt, 22, (void *)0x2d); -+ mtree_test_insert(mt, 15, (void *)0x1f); -+ mtree_test_load(mt, 2); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 5, (void *)0xb); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 4, (void *)0x9); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 3); -+ mtree_test_insert(mt, 22, (void *)0x2d); -+ mtree_test_insert(mt, 15, (void *)0x1f); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 8, (void *)0x11); -+ mtree_test_load(mt, 2); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 5, (void *)0xb); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 4, (void *)0x9); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 3); -+ mtree_test_insert(mt, 22, (void *)0x2d); -+ mtree_test_insert(mt, 15, (void *)0x1f); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 8, (void *)0x11); -+ mtree_test_insert(mt, 12, (void *)0x19); -+ mtree_test_erase(mt, 1); -+ mtree_test_store_range(mt, 4, 62, (void *)0x9); -+ mtree_test_erase(mt, 62); -+ mtree_test_store_range(mt, 1, 0, (void *)0x3); -+ mtree_test_insert(mt, 11, (void *)0x17); -+ mtree_test_insert(mt, 3, (void *)0x7); -+ mtree_test_insert(mt, 3, (void *)0x7); -+ mtree_test_store(mt, 62, (void *)0x7d); -+ mtree_test_erase(mt, 62); -+ mtree_test_store_range(mt, 1, 15, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 22, (void *)0x2d); -+ mtree_test_insert(mt, 12, (void *)0x19); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 3, (void *)0x7); -+ mtree_test_store(mt, 62, (void *)0x7d); -+ mtree_test_erase(mt, 62); -+ mtree_test_insert(mt, 122, (void *)0xf5); -+ mtree_test_store(mt, 3, (void *)0x7); -+ mtree_test_insert(mt, 0, (void *)0x1); -+ mtree_test_store_range(mt, 0, 1, (void *)0x1); -+ mtree_test_insert(mt, 85, (void *)0xab); -+ mtree_test_insert(mt, 72, (void *)0x91); -+ mtree_test_insert(mt, 81, (void *)0xa3); -+ mtree_test_insert(mt, 726, (void *)0x5ad); -+ mtree_test_insert(mt, 0, (void *)0x1); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_store(mt, 51, (void *)0x67); -+ mtree_test_insert(mt, 611, (void *)0x4c7); -+ mtree_test_insert(mt, 485, (void *)0x3cb); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 0, (void *)0x1); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert_range(mt, 26, 1, (void *)0x35); -+ mtree_test_load(mt, 1); -+ mtree_test_store_range(mt, 1, 22, (void *)0x3); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_load(mt, 53); -+ mtree_test_load(mt, 1); -+ mtree_test_store_range(mt, 1, 1, (void *)0x3); -+ mtree_test_insert(mt, 222, (void *)0x1bd); -+ mtree_test_insert(mt, 485, (void *)0x3cb); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_load(mt, 0); -+ mtree_test_insert(mt, 21, (void *)0x2b); -+ mtree_test_insert(mt, 3, (void *)0x7); -+ mtree_test_store(mt, 621, (void *)0x4db); -+ mtree_test_insert(mt, 0, (void *)0x1); -+ mtree_test_erase(mt, 5); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_store(mt, 62, (void *)0x7d); -+ mtree_test_erase(mt, 62); -+ mtree_test_store_range(mt, 1, 0, (void *)0x3); -+ mtree_test_insert(mt, 22, (void *)0x2d); -+ mtree_test_insert(mt, 12, (void *)0x19); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_store_range(mt, 4, 62, (void *)0x9); -+ mtree_test_erase(mt, 62); -+ mtree_test_erase(mt, 1); -+ mtree_test_load(mt, 1); -+ mtree_test_store_range(mt, 1, 22, (void *)0x3); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_load(mt, 53); -+ mtree_test_load(mt, 1); -+ mtree_test_store_range(mt, 1, 1, (void *)0x3); -+ mtree_test_insert(mt, 222, (void *)0x1bd); -+ mtree_test_insert(mt, 485, (void *)0x3cb); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_load(mt, 0); -+ mtree_test_load(mt, 0); -+ mtree_destroy(mt); -+ -+ /* -+ * 7. Previous fix was incomplete, fix mas_resuse_node() clearing of old -+ * data by overwriting it first - that way metadata is of no concern. -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_load(mt, 1); -+ mtree_test_insert(mt, 102, (void *)0xcd); -+ mtree_test_erase(mt, 2); -+ mtree_test_erase(mt, 0); -+ mtree_test_load(mt, 0); -+ mtree_test_insert(mt, 4, (void *)0x9); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 110, (void *)0xdd); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_insert_range(mt, 5, 0, (void *)0xb); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_store(mt, 112, (void *)0xe1); -+ mtree_test_insert(mt, 21, (void *)0x2b); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_insert_range(mt, 110, 2, (void *)0xdd); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_load(mt, 22); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 210, (void *)0x1a5); -+ mtree_test_store_range(mt, 0, 2, (void *)0x1); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_erase(mt, 2); -+ mtree_test_erase(mt, 22); -+ mtree_test_erase(mt, 1); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_load(mt, 112); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_insert_range(mt, 1, 2, (void *)0x3); -+ mtree_test_erase(mt, 0); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_erase(mt, 0); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_erase(mt, 2); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert_range(mt, 1, 2, (void *)0x3); -+ mtree_test_erase(mt, 0); -+ mtree_test_erase(mt, 2); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_load(mt, 112); -+ mtree_test_store_range(mt, 110, 12, (void *)0xdd); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_load(mt, 110); -+ mtree_test_insert_range(mt, 4, 71, (void *)0x9); -+ mtree_test_load(mt, 2); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_insert_range(mt, 11, 22, (void *)0x17); -+ mtree_test_erase(mt, 12); -+ mtree_test_store(mt, 2, (void *)0x5); -+ mtree_test_load(mt, 22); -+ mtree_destroy(mt); -+ -+ -+ /* -+ * 8. When rebalancing or spanning_rebalance(), the max of the new node -+ * may be set incorrectly to the final pivot and not the right max. -+ * Fix by setting the left max to orig right max if the entire node is -+ * consumed. -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_store(mt, 6, (void *)0xd); -+ mtree_test_store(mt, 67, (void *)0x87); -+ mtree_test_insert(mt, 15, (void *)0x1f); -+ mtree_test_insert(mt, 6716, (void *)0x3479); -+ mtree_test_store(mt, 61, (void *)0x7b); -+ mtree_test_insert(mt, 13, (void *)0x1b); -+ mtree_test_store(mt, 8, (void *)0x11); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_load(mt, 0); -+ mtree_test_erase(mt, 67167); -+ mtree_test_insert_range(mt, 6, 7167, (void *)0xd); -+ mtree_test_insert(mt, 6, (void *)0xd); -+ mtree_test_erase(mt, 67); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 667167); -+ mtree_test_insert(mt, 6, (void *)0xd); -+ mtree_test_store(mt, 67, (void *)0x87); -+ mtree_test_insert(mt, 5, (void *)0xb); -+ mtree_test_erase(mt, 1); -+ mtree_test_insert(mt, 6, (void *)0xd); -+ mtree_test_erase(mt, 67); -+ mtree_test_insert(mt, 15, (void *)0x1f); -+ mtree_test_insert(mt, 67167, (void *)0x20cbf); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_load(mt, 7); -+ mtree_test_insert(mt, 16, (void *)0x21); -+ mtree_test_insert(mt, 36, (void *)0x49); -+ mtree_test_store(mt, 67, (void *)0x87); -+ mtree_test_store(mt, 6, (void *)0xd); -+ mtree_test_insert(mt, 367, (void *)0x2df); -+ mtree_test_insert(mt, 115, (void *)0xe7); -+ mtree_test_store(mt, 0, (void *)0x1); -+ mtree_test_store_range(mt, 1, 3, (void *)0x3); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 67167); -+ mtree_test_insert_range(mt, 6, 47, (void *)0xd); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_insert_range(mt, 1, 67, (void *)0x3); -+ mtree_test_load(mt, 67); -+ mtree_test_insert(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 67167); -+ mtree_destroy(mt); -+ -+ /* -+ * 9. spanning store to the end of data caused an invalid metadata -+ * length which resulted in a crash eventually. -+ * Fix by checking if there is a value in pivot before incrementing the -+ * metadata end in mab_mas_cp(). To ensure this doesn't happen again, -+ * abstract the two locations this happens into a function called -+ * mas_leaf_set_meta(). -+ */ -+ mt_init_flags(mt, 0); -+ mtree_test_insert(mt, 21, (void *)0x2b); -+ mtree_test_insert(mt, 12, (void *)0x19); -+ mtree_test_insert(mt, 6, (void *)0xd); -+ mtree_test_insert(mt, 8, (void *)0x11); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 91, (void *)0xb7); -+ mtree_test_insert(mt, 18, (void *)0x25); -+ mtree_test_insert(mt, 81, (void *)0xa3); -+ mtree_test_store_range(mt, 0, 128, (void *)0x1); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_erase(mt, 8); -+ mtree_test_insert(mt, 11, (void *)0x17); -+ mtree_test_insert(mt, 8, (void *)0x11); -+ mtree_test_insert(mt, 21, (void *)0x2b); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); -+ mtree_test_erase(mt, 18446744073709551605UL); -+ mtree_test_store_range(mt, 0, 281, (void *)0x1); -+ mtree_test_erase(mt, 2); -+ mtree_test_insert(mt, 1211, (void *)0x977); -+ mtree_test_insert(mt, 111, (void *)0xdf); -+ mtree_test_insert(mt, 13, (void *)0x1b); -+ mtree_test_insert(mt, 211, (void *)0x1a7); -+ mtree_test_insert(mt, 11, (void *)0x17); -+ mtree_test_insert(mt, 5, (void *)0xb); -+ mtree_test_insert(mt, 1218, (void *)0x985); -+ mtree_test_insert(mt, 61, (void *)0x7b); -+ mtree_test_store(mt, 1, (void *)0x3); -+ mtree_test_insert(mt, 121, (void *)0xf3); -+ mtree_test_insert(mt, 8, (void *)0x11); -+ mtree_test_insert(mt, 21, (void *)0x2b); -+ mtree_test_insert(mt, 2, (void *)0x5); -+ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); -+ mtree_test_erase(mt, 18446744073709551605UL); -+} -+static noinline void check_dup_gaps(struct maple_tree *mt, -+ unsigned long nr_entries, bool zero_start, -+ unsigned long gap) -+{ -+ unsigned long i = 0; -+ struct maple_tree newmt; -+ int ret; -+ void *tmp; -+ MA_STATE(mas, mt, 0, 0); -+ MA_STATE(newmas, &newmt, 0, 0); -+ -+ -+ if (!zero_start) -+ i = 1; -+ -+ mt_zero_nr_tallocated(); -+ for (; i <= nr_entries; i++) -+ mtree_store_range(mt, i*10, (i+1)*10 - gap, -+ xa_mk_value(i), GFP_KERNEL); -+ -+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); -+ mt_set_non_kernel(99999); -+ ret = mas_expected_entries(&newmas, nr_entries); -+ mt_set_non_kernel(0); -+ MT_BUG_ON(mt, ret != 0); -+ -+ mas_for_each(&mas, tmp, ULONG_MAX) { -+ newmas.index = mas.index; -+ newmas.last = mas.last; -+ mas_store(&newmas, tmp); -+ } -+ -+ mas_destroy(&mas); -+ mas_destroy(&newmas); -+ mtree_destroy(&newmt); -+} -+ -+static noinline void check_dup(struct maple_tree *mt) -+{ -+ int i; -+ -+ /* Check with a value at zero */ -+ for (i = 10; i < 1000; i++) { -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_dup_gaps(mt, i, true, 5); -+ mtree_destroy(mt); -+ } -+ -+ /* Check with a value at zero, no gap */ -+ for (i = 1000; i < 2000; i++) { -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_dup_gaps(mt, i, true, 0); -+ mtree_destroy(mt); -+ } -+ -+ /* Check with a value at zero and unreasonably large */ -+ for (i = 100010; i < 100020; i++) { -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_dup_gaps(mt, i, true, 5); -+ mtree_destroy(mt); -+ } -+ -+ /* Small to medium size not starting at zero*/ -+ for (i = 200; i < 1000; i++) { -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_dup_gaps(mt, i, false, 5); -+ mtree_destroy(mt); -+ } -+ -+ /* Unreasonably large not starting at zero*/ -+ for (i = 100010; i < 100020; i++) { -+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); -+ check_dup_gaps(mt, i, false, 5); -+ mtree_destroy(mt); -+ } -+ -+ /* Check non-allocation tree not starting at zero */ -+ for (i = 1500; i < 3000; i++) { -+ mt_init_flags(mt, 0); -+ check_dup_gaps(mt, i, false, 5); -+ mtree_destroy(mt); -+ } -+ -+ /* Check non-allocation tree starting at zero */ -+ for (i = 200; i < 1000; i++) { -+ mt_init_flags(mt, 0); -+ check_dup_gaps(mt, i, true, 5); -+ mtree_destroy(mt); -+ } -+ -+ /* Unreasonably large */ -+ for (i = 100015; i < 100020; i++) { -+ mt_init_flags(mt, 0); -+ check_dup_gaps(mt, i, true, 5); -+ mtree_destroy(mt); -+ } -+ -+} -+ -+static DEFINE_MTREE(tree); -+static int maple_tree_seed(void) -+{ -+ unsigned long set[] = {5015, 5014, 5017, 25, 1000, -+ 1001, 1002, 1003, 1005, 0, -+ 5003, 5002}; -+ void *ptr = &set; -+ -+ pr_info("\nTEST STARTING\n\n"); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_root_expand(&tree); -+ mtree_destroy(&tree); -+ -+#if defined(BENCH_SLOT_STORE) -+#define BENCH -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ bench_slot_store(&tree); -+ mtree_destroy(&tree); -+ goto skip; -+#endif -+#if defined(BENCH_NODE_STORE) -+#define BENCH -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ bench_node_store(&tree); -+ mtree_destroy(&tree); -+ goto skip; -+#endif -+#if defined(BENCH_AWALK) -+#define BENCH -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ bench_awalk(&tree); -+ mtree_destroy(&tree); -+ goto skip; -+#endif -+#if defined(BENCH_WALK) -+#define BENCH -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ bench_walk(&tree); -+ mtree_destroy(&tree); -+ goto skip; -+#endif -+#if defined(BENCH_FORK) -+#define BENCH -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ bench_forking(&tree); -+ mtree_destroy(&tree); -+ goto skip; -+#endif -+#if defined(BENCH_MT_FOR_EACH) -+#define BENCH -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ bench_mt_for_each(&tree); -+ mtree_destroy(&tree); -+ goto skip; -+#endif -+ -+ test_kmem_cache_bulk(); -+ -+ mt_init_flags(&tree, 0); -+ check_new_node(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_prealloc(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_spanning_write(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_null_expand(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ check_dfs_preorder(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_forking(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_mas_store_gfp(&tree); -+ mtree_destroy(&tree); -+ -+ /* Test ranges (store and insert) */ -+ mt_init_flags(&tree, 0); -+ check_ranges(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_alloc_range(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_alloc_rev_range(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ -+ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ -+ -+ check_insert(&tree, set[9], &tree); /* Insert 0 */ -+ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ -+ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ -+ -+ check_insert(&tree, set[10], ptr); /* Insert 5003 */ -+ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ -+ check_load(&tree, set[11], NULL); /* See if 5002 -> NULL */ -+ check_load(&tree, set[10], ptr); /* See if 5003 -> ptr */ -+ -+ /* Clear out the tree */ -+ mtree_destroy(&tree); -+ -+ /* Try to insert, insert a dup, and load back what was inserted. */ -+ mt_init_flags(&tree, 0); -+ check_insert(&tree, set[0], &tree); /* Insert 5015 */ -+ check_dup_insert(&tree, set[0], &tree); /* Insert 5015 again */ -+ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ -+ -+ /* -+ * Second set of tests try to load a value that doesn't exist, inserts -+ * a second value, then loads the value again -+ */ -+ check_load(&tree, set[1], NULL); /* See if 5014 -> NULL */ -+ check_insert(&tree, set[1], ptr); /* insert 5014 -> ptr */ -+ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ -+ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ -+ /* -+ * Tree currently contains: -+ * p[0]: 14 -> (nil) p[1]: 15 -> ptr p[2]: 16 -> &tree p[3]: 0 -> (nil) -+ */ -+ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ -+ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ -+ -+ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ -+ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ -+ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ -+ check_load(&tree, set[7], &tree); /* 1003 = &tree ? */ -+ -+ /* Clear out tree */ -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ /* Test inserting into a NULL hole. */ -+ check_insert(&tree, set[5], ptr); /* insert 1001 -> ptr */ -+ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ -+ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ -+ check_load(&tree, set[5], ptr); /* See if 1001 -> ptr */ -+ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ -+ check_load(&tree, set[7], &tree); /* See if 1003 -> &tree */ -+ -+ /* Clear out the tree */ -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ check_erase_testset(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ /* -+ * set[] = {5015, 5014, 5017, 25, 1000, -+ * 1001, 1002, 1003, 1005, 0, -+ * 5003, 5002}; -+ */ -+ -+ check_insert(&tree, set[0], ptr); /* 5015 */ -+ check_insert(&tree, set[1], &tree); /* 5014 */ -+ check_insert(&tree, set[2], ptr); /* 5017 */ -+ check_insert(&tree, set[3], &tree); /* 25 */ -+ check_load(&tree, set[0], ptr); -+ check_load(&tree, set[1], &tree); -+ check_load(&tree, set[2], ptr); -+ check_load(&tree, set[3], &tree); -+ check_insert(&tree, set[4], ptr); /* 1000 < Should split. */ -+ check_load(&tree, set[0], ptr); -+ check_load(&tree, set[1], &tree); -+ check_load(&tree, set[2], ptr); -+ check_load(&tree, set[3], &tree); /*25 */ -+ check_load(&tree, set[4], ptr); -+ check_insert(&tree, set[5], &tree); /* 1001 */ -+ check_load(&tree, set[0], ptr); -+ check_load(&tree, set[1], &tree); -+ check_load(&tree, set[2], ptr); -+ check_load(&tree, set[3], &tree); -+ check_load(&tree, set[4], ptr); -+ check_load(&tree, set[5], &tree); -+ check_insert(&tree, set[6], ptr); -+ check_load(&tree, set[0], ptr); -+ check_load(&tree, set[1], &tree); -+ check_load(&tree, set[2], ptr); -+ check_load(&tree, set[3], &tree); -+ check_load(&tree, set[4], ptr); -+ check_load(&tree, set[5], &tree); -+ check_load(&tree, set[6], ptr); -+ check_insert(&tree, set[7], &tree); -+ check_load(&tree, set[0], ptr); -+ check_insert(&tree, set[8], ptr); -+ -+ check_insert(&tree, set[9], &tree); -+ -+ check_load(&tree, set[0], ptr); -+ check_load(&tree, set[1], &tree); -+ check_load(&tree, set[2], ptr); -+ check_load(&tree, set[3], &tree); -+ check_load(&tree, set[4], ptr); -+ check_load(&tree, set[5], &tree); -+ check_load(&tree, set[6], ptr); -+ check_load(&tree, set[9], &tree); -+ mtree_destroy(&tree); -+ -+ check_nomem(&tree); -+ mt_init_flags(&tree, 0); -+ check_seq(&tree, 16, false); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ check_seq(&tree, 1000, true); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_rev_seq(&tree, 1000, true); -+ mtree_destroy(&tree); -+ -+ check_lower_bound_split(&tree); -+ check_upper_bound_split(&tree); -+ check_mid_split(&tree); -+ -+ mt_init_flags(&tree, 0); -+ check_next_entry(&tree); -+ check_find(&tree); -+ check_find_2(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_prev_entry(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ check_erase2_sets(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_gap_combining(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_node_overwrite(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ next_prev_test(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_rcu_simulated(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_rcu_threaded(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_spanning_relatives(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_rev_find(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, 0); -+ check_fuzzer(&tree); -+ mtree_destroy(&tree); -+ -+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); -+ check_dup(&tree); -+ mtree_destroy(&tree); -+ -+#if defined(BENCH) -+skip: -+#endif -+ rcu_barrier(); -+ pr_info("maple_tree: %u of %u tests passed\n", -+ atomic_read(&maple_tree_tests_passed), -+ atomic_read(&maple_tree_tests_run)); -+ if (atomic_read(&maple_tree_tests_run) == -+ atomic_read(&maple_tree_tests_passed)) -+ return 0; -+ -+ return -EINVAL; -+} -+ -+static void maple_tree_harvest(void) -+{ -+ -+} -+ -+module_init(maple_tree_seed); -+module_exit(maple_tree_harvest); -+MODULE_AUTHOR("Liam R. Howlett "); -+MODULE_LICENSE("GPL"); -diff --git a/mm/Kconfig b/mm/Kconfig -index 0331f1461f81..96cd3ae25c6f 100644 ---- a/mm/Kconfig -+++ b/mm/Kconfig -@@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP - purposes. It is required to enable userfaultfd write protection on - file-backed memory types like shmem and hugetlbfs. - -+# multi-gen LRU { -+config LRU_GEN -+ bool "Multi-Gen LRU" -+ depends on MMU -+ # make sure folio->flags has enough spare bits -+ depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP -+ help -+ A high performance LRU implementation to overcommit memory. See -+ Documentation/admin-guide/mm/multigen_lru.rst for details. -+ -+config LRU_GEN_ENABLED -+ bool "Enable by default" -+ depends on LRU_GEN -+ help -+ This option enables the multi-gen LRU by default. -+ -+config LRU_GEN_STATS -+ bool "Full stats for debugging" -+ depends on LRU_GEN -+ help -+ Do not enable this option unless you plan to look at historical stats -+ from evicted generations for debugging purpose. -+ -+ This option has a per-memcg and per-node memory overhead. -+# } -+ - source "mm/damon/Kconfig" - - endmenu -diff --git a/mm/Makefile b/mm/Makefile -index 9a564f836403..8083fa85a348 100644 ---- a/mm/Makefile -+++ b/mm/Makefile -@@ -52,7 +52,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ - readahead.o swap.o truncate.o vmscan.o shmem.o \ - util.o mmzone.o vmstat.o backing-dev.o \ - mm_init.o percpu.o slab_common.o \ -- compaction.o vmacache.o \ -+ compaction.o \ - interval_tree.o list_lru.o workingset.o \ - debug.o gup.o mmap_lock.o $(mmu-y) - -diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h -index d4f55f349100..bce37c487540 100644 ---- a/mm/damon/vaddr-test.h -+++ b/mm/damon/vaddr-test.h -@@ -14,33 +14,19 @@ - - #include - --static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) -+static void __link_vmas(struct maple_tree *mt, struct vm_area_struct *vmas, -+ ssize_t nr_vmas) - { -- int i, j; -- unsigned long largest_gap, gap; -+ int i; -+ MA_STATE(mas, mt, 0, 0); - - if (!nr_vmas) - return; - -- for (i = 0; i < nr_vmas - 1; i++) { -- vmas[i].vm_next = &vmas[i + 1]; -- -- vmas[i].vm_rb.rb_left = NULL; -- vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb; -- -- largest_gap = 0; -- for (j = i; j < nr_vmas; j++) { -- if (j == 0) -- continue; -- gap = vmas[j].vm_start - vmas[j - 1].vm_end; -- if (gap > largest_gap) -- largest_gap = gap; -- } -- vmas[i].rb_subtree_gap = largest_gap; -- } -- vmas[i].vm_next = NULL; -- vmas[i].vm_rb.rb_right = NULL; -- vmas[i].rb_subtree_gap = 0; -+ mas_lock(&mas); -+ for (i = 0; i < nr_vmas; i++) -+ vma_mas_store(&vmas[i], &mas); -+ mas_unlock(&mas); - } - - /* -@@ -72,6 +58,7 @@ static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) - */ - static void damon_test_three_regions_in_vmas(struct kunit *test) - { -+ static struct mm_struct mm; - struct damon_addr_range regions[3] = {0,}; - /* 10-20-25, 200-210-220, 300-305, 307-330 */ - struct vm_area_struct vmas[] = { -@@ -83,9 +70,10 @@ static void damon_test_three_regions_in_vmas(struct kunit *test) - (struct vm_area_struct) {.vm_start = 307, .vm_end = 330}, - }; - -- __link_vmas(vmas, 6); -+ mt_init_flags(&mm.mm_mt, MM_MT_FLAGS); -+ __link_vmas(&mm.mm_mt, vmas, ARRAY_SIZE(vmas)); - -- __damon_va_three_regions(&vmas[0], regions); -+ __damon_va_three_regions(&mm, regions); - - KUNIT_EXPECT_EQ(test, 10ul, regions[0].start); - KUNIT_EXPECT_EQ(test, 25ul, regions[0].end); -diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c -index 3c7b9d6dca95..d24148a8149f 100644 ---- a/mm/damon/vaddr.c -+++ b/mm/damon/vaddr.c -@@ -113,37 +113,38 @@ static unsigned long sz_range(struct damon_addr_range *r) - * - * Returns 0 if success, or negative error code otherwise. - */ --static int __damon_va_three_regions(struct vm_area_struct *vma, -+static int __damon_va_three_regions(struct mm_struct *mm, - struct damon_addr_range regions[3]) - { -- struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0}; -- struct vm_area_struct *last_vma = NULL; -- unsigned long start = 0; -- struct rb_root rbroot; -- -- /* Find two biggest gaps so that first_gap > second_gap > others */ -- for (; vma; vma = vma->vm_next) { -- if (!last_vma) { -- start = vma->vm_start; -- goto next; -- } -+ struct damon_addr_range first_gap = {0}, second_gap = {0}; -+ VMA_ITERATOR(vmi, mm, 0); -+ struct vm_area_struct *vma, *prev = NULL; -+ unsigned long start; - -- if (vma->rb_subtree_gap <= sz_range(&second_gap)) { -- rbroot.rb_node = &vma->vm_rb; -- vma = rb_entry(rb_last(&rbroot), -- struct vm_area_struct, vm_rb); -+ /* -+ * Find the two biggest gaps so that first_gap > second_gap > others. -+ * If this is too slow, it can be optimised to examine the maple -+ * tree gaps. -+ */ -+ for_each_vma(vmi, vma) { -+ unsigned long gap; -+ -+ if (!prev) { -+ start = vma->vm_start; - goto next; - } -- -- gap.start = last_vma->vm_end; -- gap.end = vma->vm_start; -- if (sz_range(&gap) > sz_range(&second_gap)) { -- swap(gap, second_gap); -- if (sz_range(&second_gap) > sz_range(&first_gap)) -- swap(second_gap, first_gap); -+ gap = vma->vm_start - prev->vm_end; -+ -+ if (gap > sz_range(&first_gap)) { -+ second_gap = first_gap; -+ first_gap.start = prev->vm_end; -+ first_gap.end = vma->vm_start; -+ } else if (gap > sz_range(&second_gap)) { -+ second_gap.start = prev->vm_end; -+ second_gap.end = vma->vm_start; - } - next: -- last_vma = vma; -+ prev = vma; - } - - if (!sz_range(&second_gap) || !sz_range(&first_gap)) -@@ -159,7 +160,7 @@ static int __damon_va_three_regions(struct vm_area_struct *vma, - regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); - regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); - regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); -- regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION); -+ regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION); - - return 0; - } -@@ -180,7 +181,7 @@ static int damon_va_three_regions(struct damon_target *t, - return -EINVAL; - - mmap_read_lock(mm); -- rc = __damon_va_three_regions(mm->mmap, regions); -+ rc = __damon_va_three_regions(mm, regions); - mmap_read_unlock(mm); - - mmput(mm); -diff --git a/mm/debug.c b/mm/debug.c -index bef329bf28f0..0fd15ba70d16 100644 ---- a/mm/debug.c -+++ b/mm/debug.c -@@ -139,13 +139,11 @@ EXPORT_SYMBOL(dump_page); - - void dump_vma(const struct vm_area_struct *vma) - { -- pr_emerg("vma %px start %px end %px\n" -- "next %px prev %px mm %px\n" -+ pr_emerg("vma %px start %px end %px mm %px\n" - "prot %lx anon_vma %px vm_ops %px\n" - "pgoff %lx file %px private_data %px\n" - "flags: %#lx(%pGv)\n", -- vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_next, -- vma->vm_prev, vma->vm_mm, -+ vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, - (unsigned long)pgprot_val(vma->vm_page_prot), - vma->anon_vma, vma->vm_ops, vma->vm_pgoff, - vma->vm_file, vma->vm_private_data, -@@ -155,11 +153,11 @@ EXPORT_SYMBOL(dump_vma); - - void dump_mm(const struct mm_struct *mm) - { -- pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n" -+ pr_emerg("mm %px task_size %lu\n" - #ifdef CONFIG_MMU - "get_unmapped_area %px\n" - #endif -- "mmap_base %lu mmap_legacy_base %lu highest_vm_end %lu\n" -+ "mmap_base %lu mmap_legacy_base %lu\n" - "pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n" - "hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n" - "pinned_vm %llx data_vm %lx exec_vm %lx stack_vm %lx\n" -@@ -183,11 +181,11 @@ void dump_mm(const struct mm_struct *mm) - "tlb_flush_pending %d\n" - "def_flags: %#lx(%pGv)\n", - -- mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size, -+ mm, mm->task_size, - #ifdef CONFIG_MMU - mm->get_unmapped_area, - #endif -- mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end, -+ mm->mmap_base, mm->mmap_legacy_base, - mm->pgd, atomic_read(&mm->mm_users), - atomic_read(&mm->mm_count), - mm_pgtables_bytes(mm), -diff --git a/mm/gup.c b/mm/gup.c -index 5abdaf487460..5f3c464dbce1 100644 ---- a/mm/gup.c -+++ b/mm/gup.c -@@ -1667,10 +1667,11 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) - if (!locked) { - locked = 1; - mmap_read_lock(mm); -- vma = find_vma(mm, nstart); -+ vma = find_vma_intersection(mm, nstart, end); - } else if (nstart >= vma->vm_end) -- vma = vma->vm_next; -- if (!vma || vma->vm_start >= end) -+ vma = find_vma_intersection(mm, vma->vm_end, end); -+ -+ if (!vma) - break; - /* - * Set [nstart; nend) to intersection of desired address -diff --git a/mm/huge_memory.c b/mm/huge_memory.c -index e9414ee57c5b..cd0eed90a865 100644 ---- a/mm/huge_memory.c -+++ b/mm/huge_memory.c -@@ -70,9 +70,8 @@ static atomic_t huge_zero_refcount; - struct page *huge_zero_page __read_mostly; - unsigned long huge_zero_pfn __read_mostly = ~0UL; - --bool hugepage_vma_check(struct vm_area_struct *vma, -- unsigned long vm_flags, -- bool smaps, bool in_pf) -+bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, -+ bool smaps, bool in_pf, bool enforce_sysfs) - { - if (!vma->vm_mm) /* vdso */ - return false; -@@ -121,11 +120,10 @@ bool hugepage_vma_check(struct vm_area_struct *vma, - if (!in_pf && shmem_file(vma->vm_file)) - return shmem_huge_enabled(vma); - -- if (!hugepage_flags_enabled()) -- return false; -- -- /* THP settings require madvise. */ -- if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always()) -+ /* Enforce sysfs THP requirements as necessary */ -+ if (enforce_sysfs && -+ (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) && -+ !hugepage_flags_always()))) - return false; - - /* Only regular file is valid */ -@@ -2288,25 +2286,11 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, - void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, - bool freeze, struct folio *folio) - { -- pgd_t *pgd; -- p4d_t *p4d; -- pud_t *pud; -- pmd_t *pmd; -- -- pgd = pgd_offset(vma->vm_mm, address); -- if (!pgd_present(*pgd)) -- return; -+ pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); - -- p4d = p4d_offset(pgd, address); -- if (!p4d_present(*p4d)) -+ if (!pmd) - return; - -- pud = pud_offset(p4d, address); -- if (!pud_present(*pud)) -- return; -- -- pmd = pmd_offset(pud, address); -- - __split_huge_pmd(vma, pmd, address, freeze, folio); - } - -@@ -2334,11 +2318,11 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, - split_huge_pmd_if_needed(vma, end); - - /* -- * If we're also updating the vma->vm_next->vm_start, -+ * If we're also updating the next vma vm_start, - * check if we need to split it. - */ - if (adjust_next > 0) { -- struct vm_area_struct *next = vma->vm_next; -+ struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); - unsigned long nstart = next->vm_start; - nstart += adjust_next; - split_huge_pmd_if_needed(next, nstart); -@@ -2438,7 +2422,8 @@ static void __split_huge_page_tail(struct page *head, int tail, - #ifdef CONFIG_64BIT - (1L << PG_arch_2) | - #endif -- (1L << PG_dirty))); -+ (1L << PG_dirty) | -+ LRU_GEN_MASK | LRU_REFS_MASK)); - - /* ->mapping in first tail page is compound_mapcount */ - VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, -diff --git a/mm/init-mm.c b/mm/init-mm.c -index fbe7844d0912..c9327abb771c 100644 ---- a/mm/init-mm.c -+++ b/mm/init-mm.c -@@ -1,6 +1,6 @@ - // SPDX-License-Identifier: GPL-2.0 - #include --#include -+#include - #include - #include - #include -@@ -28,7 +28,7 @@ - * and size this cpu_bitmask to NR_CPUS. - */ - struct mm_struct init_mm = { -- .mm_rb = RB_ROOT, -+ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, init_mm.mmap_lock), - .pgd = swapper_pg_dir, - .mm_users = ATOMIC_INIT(2), - .mm_count = ATOMIC_INIT(1), -diff --git a/mm/internal.h b/mm/internal.h -index 785409805ed7..0f106a3982e7 100644 ---- a/mm/internal.h -+++ b/mm/internal.h -@@ -83,9 +83,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); - void folio_rotate_reclaimable(struct folio *folio); - bool __folio_end_writeback(struct folio *folio); - void deactivate_file_folio(struct folio *folio); -+void folio_activate(struct folio *folio); - --void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, -- unsigned long floor, unsigned long ceiling); -+void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, -+ struct vm_area_struct *start_vma, unsigned long floor, -+ unsigned long ceiling); - void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); - - struct zap_details; -@@ -187,7 +189,7 @@ extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason - /* - * in mm/rmap.c: - */ --extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); -+pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); - - /* - * in mm/page_alloc.c -@@ -479,9 +481,6 @@ static inline bool is_data_mapping(vm_flags_t flags) - } - - /* mm/util.c */ --void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, -- struct vm_area_struct *prev); --void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma); - struct anon_vma *folio_anon_vma(struct folio *folio); - - #ifdef CONFIG_MMU -diff --git a/mm/khugepaged.c b/mm/khugepaged.c -index 01f71786d530..df890338daed 100644 ---- a/mm/khugepaged.c -+++ b/mm/khugepaged.c -@@ -28,6 +28,7 @@ enum scan_result { - SCAN_FAIL, - SCAN_SUCCEED, - SCAN_PMD_NULL, -+ SCAN_PMD_MAPPED, - SCAN_EXCEED_NONE_PTE, - SCAN_EXCEED_SWAP_PTE, - SCAN_EXCEED_SHARED_PTE, -@@ -73,6 +74,8 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); - * default collapse hugepages if there is at least one pte mapped like - * it would have happened if the vma was large enough during page - * fault. -+ * -+ * Note that these are only respected if collapse was initiated by khugepaged. - */ - static unsigned int khugepaged_max_ptes_none __read_mostly; - static unsigned int khugepaged_max_ptes_swap __read_mostly; -@@ -85,6 +88,16 @@ static struct kmem_cache *mm_slot_cache __read_mostly; - - #define MAX_PTE_MAPPED_THP 8 - -+struct collapse_control { -+ bool is_khugepaged; -+ -+ /* Num pages scanned per node */ -+ u32 node_load[MAX_NUMNODES]; -+ -+ /* Last target selected in hpage_collapse_find_target_node() */ -+ int last_target_node; -+}; -+ - /** - * struct mm_slot - hash lookup from mm to mm_slot - * @hash: hash collision list -@@ -425,7 +438,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm, - hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); - } - --static inline int khugepaged_test_exit(struct mm_struct *mm) -+static inline int hpage_collapse_test_exit(struct mm_struct *mm) - { - return atomic_read(&mm->mm_users) == 0; - } -@@ -440,7 +453,7 @@ void __khugepaged_enter(struct mm_struct *mm) - return; - - /* __khugepaged_exit() must not run from under us */ -- VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); -+ VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); - if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { - free_mm_slot(mm_slot); - return; -@@ -466,7 +479,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, - { - if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && - hugepage_flags_enabled()) { -- if (hugepage_vma_check(vma, vm_flags, false, false)) -+ if (hugepage_vma_check(vma, vm_flags, false, false, true)) - __khugepaged_enter(vma->vm_mm); - } - } -@@ -492,11 +505,10 @@ void __khugepaged_exit(struct mm_struct *mm) - } else if (mm_slot) { - /* - * This is required to serialize against -- * khugepaged_test_exit() (which is guaranteed to run -- * under mmap sem read mode). Stop here (after we -- * return all pagetables will be destroyed) until -- * khugepaged has finished working on the pagetables -- * under the mmap_lock. -+ * hpage_collapse_test_exit() (which is guaranteed to run -+ * under mmap sem read mode). Stop here (after we return all -+ * pagetables will be destroyed) until khugepaged has finished -+ * working on the pagetables under the mmap_lock. - */ - mmap_write_lock(mm); - mmap_write_unlock(mm); -@@ -546,11 +558,12 @@ static bool is_refcount_suitable(struct page *page) - static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - unsigned long address, - pte_t *pte, -+ struct collapse_control *cc, - struct list_head *compound_pagelist) - { - struct page *page = NULL; - pte_t *_pte; -- int none_or_zero = 0, shared = 0, result = 0, referenced = 0; -+ int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; - bool writable = false; - - for (_pte = pte; _pte < pte + HPAGE_PMD_NR; -@@ -558,8 +571,10 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - pte_t pteval = *_pte; - if (pte_none(pteval) || (pte_present(pteval) && - is_zero_pfn(pte_pfn(pteval)))) { -+ ++none_or_zero; - if (!userfaultfd_armed(vma) && -- ++none_or_zero <= khugepaged_max_ptes_none) { -+ (!cc->is_khugepaged || -+ none_or_zero <= khugepaged_max_ptes_none)) { - continue; - } else { - result = SCAN_EXCEED_NONE_PTE; -@@ -579,11 +594,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - - VM_BUG_ON_PAGE(!PageAnon(page), page); - -- if (page_mapcount(page) > 1 && -- ++shared > khugepaged_max_ptes_shared) { -- result = SCAN_EXCEED_SHARED_PTE; -- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -- goto out; -+ if (page_mapcount(page) > 1) { -+ ++shared; -+ if (cc->is_khugepaged && -+ shared > khugepaged_max_ptes_shared) { -+ result = SCAN_EXCEED_SHARED_PTE; -+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -+ goto out; -+ } - } - - if (PageCompound(page)) { -@@ -646,10 +664,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - if (PageCompound(page)) - list_add_tail(&page->lru, compound_pagelist); - next: -- /* There should be enough young pte to collapse the page */ -- if (pte_young(pteval) || -- page_is_young(page) || PageReferenced(page) || -- mmu_notifier_test_young(vma->vm_mm, address)) -+ /* -+ * If collapse was initiated by khugepaged, check that there is -+ * enough young pte to justify collapsing the page -+ */ -+ if (cc->is_khugepaged && -+ (pte_young(pteval) || page_is_young(page) || -+ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, -+ address))) - referenced++; - - if (pte_write(pteval)) -@@ -658,19 +680,19 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, - - if (unlikely(!writable)) { - result = SCAN_PAGE_RO; -- } else if (unlikely(!referenced)) { -+ } else if (unlikely(cc->is_khugepaged && !referenced)) { - result = SCAN_LACK_REFERENCED_PAGE; - } else { - result = SCAN_SUCCEED; - trace_mm_collapse_huge_page_isolate(page, none_or_zero, - referenced, writable, result); -- return 1; -+ return result; - } - out: - release_pte_pages(pte, _pte, compound_pagelist); - trace_mm_collapse_huge_page_isolate(page, none_or_zero, - referenced, writable, result); -- return 0; -+ return result; - } - - static void __collapse_huge_page_copy(pte_t *pte, struct page *page, -@@ -735,9 +757,12 @@ static void khugepaged_alloc_sleep(void) - remove_wait_queue(&khugepaged_wait, &wait); - } - --static int khugepaged_node_load[MAX_NUMNODES]; -+struct collapse_control khugepaged_collapse_control = { -+ .is_khugepaged = true, -+ .last_target_node = NUMA_NO_NODE, -+}; - --static bool khugepaged_scan_abort(int nid) -+static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) - { - int i; - -@@ -749,11 +774,11 @@ static bool khugepaged_scan_abort(int nid) - return false; - - /* If there is a count for this node already, it must be acceptable */ -- if (khugepaged_node_load[nid]) -+ if (cc->node_load[nid]) - return false; - - for (i = 0; i < MAX_NUMNODES; i++) { -- if (!khugepaged_node_load[i]) -+ if (!cc->node_load[i]) - continue; - if (node_distance(nid, i) > node_reclaim_distance) - return true; -@@ -772,146 +797,62 @@ static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) - } - - #ifdef CONFIG_NUMA --static int khugepaged_find_target_node(void) -+static int hpage_collapse_find_target_node(struct collapse_control *cc) - { -- static int last_khugepaged_target_node = NUMA_NO_NODE; - int nid, target_node = 0, max_value = 0; - - /* find first node with max normal pages hit */ - for (nid = 0; nid < MAX_NUMNODES; nid++) -- if (khugepaged_node_load[nid] > max_value) { -- max_value = khugepaged_node_load[nid]; -+ if (cc->node_load[nid] > max_value) { -+ max_value = cc->node_load[nid]; - target_node = nid; - } - - /* do some balance if several nodes have the same hit record */ -- if (target_node <= last_khugepaged_target_node) -- for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; -- nid++) -- if (max_value == khugepaged_node_load[nid]) { -+ if (target_node <= cc->last_target_node) -+ for (nid = cc->last_target_node + 1; nid < MAX_NUMNODES; -+ nid++) -+ if (max_value == cc->node_load[nid]) { - target_node = nid; - break; - } - -- last_khugepaged_target_node = target_node; -+ cc->last_target_node = target_node; - return target_node; - } -- --static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) -+#else -+static int hpage_collapse_find_target_node(struct collapse_control *cc) - { -- if (IS_ERR(*hpage)) { -- if (!*wait) -- return false; -- -- *wait = false; -- *hpage = NULL; -- khugepaged_alloc_sleep(); -- } else if (*hpage) { -- put_page(*hpage); -- *hpage = NULL; -- } -- -- return true; -+ return 0; - } -+#endif - --static struct page * --khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) -+static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node) - { -- VM_BUG_ON_PAGE(*hpage, *hpage); -- - *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); - if (unlikely(!*hpage)) { - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); -- *hpage = ERR_PTR(-ENOMEM); -- return NULL; -+ return false; - } - - prep_transhuge_page(*hpage); - count_vm_event(THP_COLLAPSE_ALLOC); -- return *hpage; --} --#else --static int khugepaged_find_target_node(void) --{ -- return 0; --} -- --static inline struct page *alloc_khugepaged_hugepage(void) --{ -- struct page *page; -- -- page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), -- HPAGE_PMD_ORDER); -- if (page) -- prep_transhuge_page(page); -- return page; --} -- --static struct page *khugepaged_alloc_hugepage(bool *wait) --{ -- struct page *hpage; -- -- do { -- hpage = alloc_khugepaged_hugepage(); -- if (!hpage) { -- count_vm_event(THP_COLLAPSE_ALLOC_FAILED); -- if (!*wait) -- return NULL; -- -- *wait = false; -- khugepaged_alloc_sleep(); -- } else -- count_vm_event(THP_COLLAPSE_ALLOC); -- } while (unlikely(!hpage) && likely(hugepage_flags_enabled())); -- -- return hpage; --} -- --static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) --{ -- /* -- * If the hpage allocated earlier was briefly exposed in page cache -- * before collapse_file() failed, it is possible that racing lookups -- * have not yet completed, and would then be unpleasantly surprised by -- * finding the hpage reused for the same mapping at a different offset. -- * Just release the previous allocation if there is any danger of that. -- */ -- if (*hpage && page_count(*hpage) > 1) { -- put_page(*hpage); -- *hpage = NULL; -- } -- -- if (!*hpage) -- *hpage = khugepaged_alloc_hugepage(wait); -- -- if (unlikely(!*hpage)) -- return false; -- - return true; - } - --static struct page * --khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) --{ -- VM_BUG_ON(!*hpage); -- -- return *hpage; --} --#endif -- - /* - * If mmap_lock temporarily dropped, revalidate vma - * before taking mmap_lock. -- * Return 0 if succeeds, otherwise return none-zero -- * value (scan code). -+ * Returns enum scan_result value. - */ - - static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, -- struct vm_area_struct **vmap) -+ struct vm_area_struct **vmap, -+ struct collapse_control *cc) - { - struct vm_area_struct *vma; - -- if (unlikely(khugepaged_test_exit(mm))) -+ if (unlikely(hpage_collapse_test_exit(mm))) - return SCAN_ANY_PROCESS; - - *vmap = vma = find_vma(mm, address); -@@ -920,7 +861,8 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, - - if (!transhuge_vma_suitable(vma, address)) - return SCAN_ADDRESS_RANGE; -- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, -+ cc->is_khugepaged)) - return SCAN_VMA_CHECK; - /* - * Anon VMA expected, the address may be unmapped then -@@ -931,21 +873,60 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, - */ - if (!vma->anon_vma || !vma_is_anonymous(vma)) - return SCAN_VMA_CHECK; -- return 0; -+ return SCAN_SUCCEED; -+} -+ -+static int find_pmd_or_thp_or_none(struct mm_struct *mm, -+ unsigned long address, -+ pmd_t **pmd) -+{ -+ pmd_t pmde; -+ -+ *pmd = mm_find_pmd(mm, address); -+ if (!*pmd) -+ return SCAN_PMD_NULL; -+ -+ pmde = pmd_read_atomic(*pmd); -+ -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ -+ barrier(); -+#endif -+ if (!pmd_present(pmde)) -+ return SCAN_PMD_NULL; -+ if (pmd_trans_huge(pmde)) -+ return SCAN_PMD_MAPPED; -+ if (pmd_bad(pmde)) -+ return SCAN_PMD_NULL; -+ return SCAN_SUCCEED; -+} -+ -+static int check_pmd_still_valid(struct mm_struct *mm, -+ unsigned long address, -+ pmd_t *pmd) -+{ -+ pmd_t *new_pmd; -+ int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); -+ -+ if (result != SCAN_SUCCEED) -+ return result; -+ if (new_pmd != pmd) -+ return SCAN_FAIL; -+ return SCAN_SUCCEED; - } - - /* - * Bring missing pages in from swap, to complete THP collapse. -- * Only done if khugepaged_scan_pmd believes it is worthwhile. -+ * Only done if hpage_collapse_scan_pmd believes it is worthwhile. - * - * Called and returns without pte mapped or spinlocks held. - * Note that if false is returned, mmap_lock will be released. - */ - --static bool __collapse_huge_page_swapin(struct mm_struct *mm, -- struct vm_area_struct *vma, -- unsigned long haddr, pmd_t *pmd, -- int referenced) -+static int __collapse_huge_page_swapin(struct mm_struct *mm, -+ struct vm_area_struct *vma, -+ unsigned long haddr, pmd_t *pmd, -+ int referenced) - { - int swapped_in = 0; - vm_fault_t ret = 0; -@@ -976,12 +957,13 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, - */ - if (ret & VM_FAULT_RETRY) { - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); -- return false; -+ /* Likely, but not guaranteed, that page lock failed */ -+ return SCAN_PAGE_LOCK; - } - if (ret & VM_FAULT_ERROR) { - mmap_read_unlock(mm); - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); -- return false; -+ return SCAN_FAIL; - } - swapped_in++; - } -@@ -991,30 +973,41 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, - lru_add_drain(); - - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); -- return true; -+ return SCAN_SUCCEED; - } - --static void collapse_huge_page(struct mm_struct *mm, -- unsigned long address, -- struct page **hpage, -- int node, int referenced, int unmapped) -+static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, -+ struct collapse_control *cc) -+{ -+ /* Only allocate from the target node */ -+ gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : -+ GFP_TRANSHUGE) | __GFP_THISNODE; -+ int node = hpage_collapse_find_target_node(cc); -+ -+ if (!hpage_collapse_alloc_page(hpage, gfp, node)) -+ return SCAN_ALLOC_HUGE_PAGE_FAIL; -+ if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) -+ return SCAN_CGROUP_CHARGE_FAIL; -+ count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); -+ return SCAN_SUCCEED; -+} -+ -+static int collapse_huge_page(struct mm_struct *mm, unsigned long address, -+ int referenced, int unmapped, -+ struct collapse_control *cc) - { - LIST_HEAD(compound_pagelist); - pmd_t *pmd, _pmd; - pte_t *pte; - pgtable_t pgtable; -- struct page *new_page; -+ struct page *hpage; - spinlock_t *pmd_ptl, *pte_ptl; -- int isolated = 0, result = 0; -+ int result = SCAN_FAIL; - struct vm_area_struct *vma; - struct mmu_notifier_range range; -- gfp_t gfp; - - VM_BUG_ON(address & ~HPAGE_PMD_MASK); - -- /* Only allocate from the target node */ -- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; -- - /* - * Before allocating the hugepage, release the mmap_lock read lock. - * The allocation can take potentially a long time if it involves -@@ -1022,40 +1015,34 @@ static void collapse_huge_page(struct mm_struct *mm, - * that. We will recheck the vma after taking it again in write mode. - */ - mmap_read_unlock(mm); -- new_page = khugepaged_alloc_page(hpage, gfp, node); -- if (!new_page) { -- result = SCAN_ALLOC_HUGE_PAGE_FAIL; -- goto out_nolock; -- } - -- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { -- result = SCAN_CGROUP_CHARGE_FAIL; -+ result = alloc_charge_hpage(&hpage, mm, cc); -+ if (result != SCAN_SUCCEED) - goto out_nolock; -- } -- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); - - mmap_read_lock(mm); -- result = hugepage_vma_revalidate(mm, address, &vma); -- if (result) { -+ result = hugepage_vma_revalidate(mm, address, &vma, cc); -+ if (result != SCAN_SUCCEED) { - mmap_read_unlock(mm); - goto out_nolock; - } - -- pmd = mm_find_pmd(mm, address); -- if (!pmd) { -- result = SCAN_PMD_NULL; -+ result = find_pmd_or_thp_or_none(mm, address, &pmd); -+ if (result != SCAN_SUCCEED) { - mmap_read_unlock(mm); - goto out_nolock; - } - -- /* -- * __collapse_huge_page_swapin will return with mmap_lock released -- * when it fails. So we jump out_nolock directly in that case. -- * Continuing to collapse causes inconsistency. -- */ -- if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, -- pmd, referenced)) { -- goto out_nolock; -+ if (unmapped) { -+ /* -+ * __collapse_huge_page_swapin will return with mmap_lock -+ * released when it fails. So we jump out_nolock directly in -+ * that case. Continuing to collapse causes inconsistency. -+ */ -+ result = __collapse_huge_page_swapin(mm, vma, address, pmd, -+ referenced); -+ if (result != SCAN_SUCCEED) -+ goto out_nolock; - } - - mmap_read_unlock(mm); -@@ -1065,11 +1052,12 @@ static void collapse_huge_page(struct mm_struct *mm, - * handled by the anon_vma lock + PG_lock. - */ - mmap_write_lock(mm); -- result = hugepage_vma_revalidate(mm, address, &vma); -- if (result) -+ result = hugepage_vma_revalidate(mm, address, &vma, cc); -+ if (result != SCAN_SUCCEED) - goto out_up_write; - /* check if the pmd is still valid */ -- if (mm_find_pmd(mm, address) != pmd) -+ result = check_pmd_still_valid(mm, address, pmd); -+ if (result != SCAN_SUCCEED) - goto out_up_write; - - anon_vma_lock_write(vma->anon_vma); -@@ -1093,11 +1081,11 @@ static void collapse_huge_page(struct mm_struct *mm, - mmu_notifier_invalidate_range_end(&range); - - spin_lock(pte_ptl); -- isolated = __collapse_huge_page_isolate(vma, address, pte, -- &compound_pagelist); -+ result = __collapse_huge_page_isolate(vma, address, pte, cc, -+ &compound_pagelist); - spin_unlock(pte_ptl); - -- if (unlikely(!isolated)) { -+ if (unlikely(result != SCAN_SUCCEED)) { - pte_unmap(pte); - spin_lock(pmd_ptl); - BUG_ON(!pmd_none(*pmd)); -@@ -1109,7 +1097,6 @@ static void collapse_huge_page(struct mm_struct *mm, - pmd_populate(mm, pmd, pmd_pgtable(_pmd)); - spin_unlock(pmd_ptl); - anon_vma_unlock_write(vma->anon_vma); -- result = SCAN_FAIL; - goto out_up_write; - } - -@@ -1119,8 +1106,8 @@ static void collapse_huge_page(struct mm_struct *mm, - */ - anon_vma_unlock_write(vma->anon_vma); - -- __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, -- &compound_pagelist); -+ __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, -+ &compound_pagelist); - pte_unmap(pte); - /* - * spin_lock() below is not the equivalent of smp_wmb(), but -@@ -1128,42 +1115,43 @@ static void collapse_huge_page(struct mm_struct *mm, - * avoid the copy_huge_page writes to become visible after - * the set_pmd_at() write. - */ -- __SetPageUptodate(new_page); -+ __SetPageUptodate(hpage); - pgtable = pmd_pgtable(_pmd); - -- _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); -+ _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); - _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); - - spin_lock(pmd_ptl); - BUG_ON(!pmd_none(*pmd)); -- page_add_new_anon_rmap(new_page, vma, address); -- lru_cache_add_inactive_or_unevictable(new_page, vma); -+ page_add_new_anon_rmap(hpage, vma, address); -+ lru_cache_add_inactive_or_unevictable(hpage, vma); - pgtable_trans_huge_deposit(mm, pmd, pgtable); - set_pmd_at(mm, address, pmd, _pmd); - update_mmu_cache_pmd(vma, address, pmd); - spin_unlock(pmd_ptl); - -- *hpage = NULL; -+ hpage = NULL; - -- khugepaged_pages_collapsed++; - result = SCAN_SUCCEED; - out_up_write: - mmap_write_unlock(mm); - out_nolock: -- if (!IS_ERR_OR_NULL(*hpage)) -- mem_cgroup_uncharge(page_folio(*hpage)); -- trace_mm_collapse_huge_page(mm, isolated, result); -- return; -+ if (hpage) { -+ mem_cgroup_uncharge(page_folio(hpage)); -+ put_page(hpage); -+ } -+ trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); -+ return result; - } - --static int khugepaged_scan_pmd(struct mm_struct *mm, -- struct vm_area_struct *vma, -- unsigned long address, -- struct page **hpage) -+static int hpage_collapse_scan_pmd(struct mm_struct *mm, -+ struct vm_area_struct *vma, -+ unsigned long address, bool *mmap_locked, -+ struct collapse_control *cc) - { - pmd_t *pmd; - pte_t *pte, *_pte; -- int ret = 0, result = 0, referenced = 0; -+ int result = SCAN_FAIL, referenced = 0; - int none_or_zero = 0, shared = 0; - struct page *page = NULL; - unsigned long _address; -@@ -1173,19 +1161,19 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - - VM_BUG_ON(address & ~HPAGE_PMD_MASK); - -- pmd = mm_find_pmd(mm, address); -- if (!pmd) { -- result = SCAN_PMD_NULL; -+ result = find_pmd_or_thp_or_none(mm, address, &pmd); -+ if (result != SCAN_SUCCEED) - goto out; -- } - -- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); -+ memset(cc->node_load, 0, sizeof(cc->node_load)); - pte = pte_offset_map_lock(mm, pmd, address, &ptl); - for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; - _pte++, _address += PAGE_SIZE) { - pte_t pteval = *_pte; - if (is_swap_pte(pteval)) { -- if (++unmapped <= khugepaged_max_ptes_swap) { -+ ++unmapped; -+ if (!cc->is_khugepaged || -+ unmapped <= khugepaged_max_ptes_swap) { - /* - * Always be strict with uffd-wp - * enabled swap entries. Please see -@@ -1203,8 +1191,10 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - } - } - if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { -+ ++none_or_zero; - if (!userfaultfd_armed(vma) && -- ++none_or_zero <= khugepaged_max_ptes_none) { -+ (!cc->is_khugepaged || -+ none_or_zero <= khugepaged_max_ptes_none)) { - continue; - } else { - result = SCAN_EXCEED_NONE_PTE; -@@ -1234,27 +1224,30 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - goto out_unmap; - } - -- if (page_mapcount(page) > 1 && -- ++shared > khugepaged_max_ptes_shared) { -- result = SCAN_EXCEED_SHARED_PTE; -- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -- goto out_unmap; -+ if (page_mapcount(page) > 1) { -+ ++shared; -+ if (cc->is_khugepaged && -+ shared > khugepaged_max_ptes_shared) { -+ result = SCAN_EXCEED_SHARED_PTE; -+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); -+ goto out_unmap; -+ } - } - - page = compound_head(page); - - /* - * Record which node the original page is from and save this -- * information to khugepaged_node_load[]. -+ * information to cc->node_load[]. - * Khugepaged will allocate hugepage from the node has the max - * hit record. - */ - node = page_to_nid(page); -- if (khugepaged_scan_abort(node)) { -+ if (hpage_collapse_scan_abort(node, cc)) { - result = SCAN_SCAN_ABORT; - goto out_unmap; - } -- khugepaged_node_load[node]++; -+ cc->node_load[node]++; - if (!PageLRU(page)) { - result = SCAN_PAGE_LRU; - goto out_unmap; -@@ -1289,31 +1282,38 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, - result = SCAN_PAGE_COUNT; - goto out_unmap; - } -- if (pte_young(pteval) || -- page_is_young(page) || PageReferenced(page) || -- mmu_notifier_test_young(vma->vm_mm, address)) -+ -+ /* -+ * If collapse was initiated by khugepaged, check that there is -+ * enough young pte to justify collapsing the page -+ */ -+ if (cc->is_khugepaged && -+ (pte_young(pteval) || page_is_young(page) || -+ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, -+ address))) - referenced++; - } - if (!writable) { - result = SCAN_PAGE_RO; -- } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { -+ } else if (cc->is_khugepaged && -+ (!referenced || -+ (unmapped && referenced < HPAGE_PMD_NR / 2))) { - result = SCAN_LACK_REFERENCED_PAGE; - } else { - result = SCAN_SUCCEED; -- ret = 1; - } - out_unmap: - pte_unmap_unlock(pte, ptl); -- if (ret) { -- node = khugepaged_find_target_node(); -+ if (result == SCAN_SUCCEED) { -+ result = collapse_huge_page(mm, address, referenced, -+ unmapped, cc); - /* collapse_huge_page will return with the mmap_lock released */ -- collapse_huge_page(mm, address, hpage, node, -- referenced, unmapped); -+ *mmap_locked = false; - } - out: - trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, - none_or_zero, result, unmapped); -- return ret; -+ return result; - } - - static void collect_mm_slot(struct mm_slot *mm_slot) -@@ -1322,7 +1322,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) - - lockdep_assert_held(&khugepaged_mm_lock); - -- if (khugepaged_test_exit(mm)) { -+ if (hpage_collapse_test_exit(mm)) { - /* free mm_slot */ - hash_del(&mm_slot->hash); - list_del(&mm_slot->mm_node); -@@ -1387,7 +1387,7 @@ static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *v - void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) - { - unsigned long haddr = addr & HPAGE_PMD_MASK; -- struct vm_area_struct *vma = find_vma(mm, haddr); -+ struct vm_area_struct *vma = vma_lookup(mm, haddr); - struct page *hpage; - pte_t *start_pte, *pte; - pmd_t *pmd; -@@ -1400,12 +1400,13 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) - return; - - /* -- * This vm_flags may not have VM_HUGEPAGE if the page was not -- * collapsed by this mm. But we can still collapse if the page is -- * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() -- * will not fail the vma for missing VM_HUGEPAGE -+ * If we are here, we've succeeded in replacing all the native pages -+ * in the page cache with a single hugepage. If a mm were to fault-in -+ * this memory (mapped by a suitably aligned VMA), we'd get the hugepage -+ * and map it by a PMD, regardless of sysfs THP settings. As such, let's -+ * analogously elide sysfs THP settings here. - */ -- if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false)) -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) - return; - - /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ -@@ -1420,8 +1421,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) - if (!PageHead(hpage)) - goto drop_hpage; - -- pmd = mm_find_pmd(mm, haddr); -- if (!pmd) -+ if (find_pmd_or_thp_or_none(mm, haddr, &pmd) != SCAN_SUCCEED) - goto drop_hpage; - - start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); -@@ -1495,7 +1495,7 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) - if (!mmap_write_trylock(mm)) - return; - -- if (unlikely(khugepaged_test_exit(mm))) -+ if (unlikely(hpage_collapse_test_exit(mm))) - goto out; - - for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) -@@ -1539,8 +1539,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - if (vma->vm_end < addr + HPAGE_PMD_SIZE) - continue; - mm = vma->vm_mm; -- pmd = mm_find_pmd(mm, addr); -- if (!pmd) -+ if (find_pmd_or_thp_or_none(mm, addr, &pmd) != SCAN_SUCCEED) - continue; - /* - * We need exclusive mmap_lock to retract page table. -@@ -1558,7 +1557,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - * it'll always mapped in small page size for uffd-wp - * registered ranges. - */ -- if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) -+ if (!hpage_collapse_test_exit(mm) && -+ !userfaultfd_wp(vma)) - collapse_and_free_pmd(mm, vma, addr, pmd); - mmap_write_unlock(mm); - } else { -@@ -1575,8 +1575,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - * @mm: process address space where collapse happens - * @file: file that collapse on - * @start: collapse start address -- * @hpage: new allocated huge page for collapse -- * @node: appointed node the new huge page allocate from -+ * @cc: collapse context and scratchpad - * - * Basic scheme is simple, details are more complex: - * - allocate and lock a new huge page; -@@ -1593,13 +1592,11 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) - * + restore gaps in the page cache; - * + unlock and free huge page; - */ --static void collapse_file(struct mm_struct *mm, -- struct file *file, pgoff_t start, -- struct page **hpage, int node) -+static int collapse_file(struct mm_struct *mm, struct file *file, -+ pgoff_t start, struct collapse_control *cc) - { - struct address_space *mapping = file->f_mapping; -- gfp_t gfp; -- struct page *new_page; -+ struct page *hpage; - pgoff_t index, end = start + HPAGE_PMD_NR; - LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); -@@ -1610,20 +1607,9 @@ static void collapse_file(struct mm_struct *mm, - VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); - -- /* Only allocate from the target node */ -- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; -- -- new_page = khugepaged_alloc_page(hpage, gfp, node); -- if (!new_page) { -- result = SCAN_ALLOC_HUGE_PAGE_FAIL; -+ result = alloc_charge_hpage(&hpage, mm, cc); -+ if (result != SCAN_SUCCEED) - goto out; -- } -- -- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { -- result = SCAN_CGROUP_CHARGE_FAIL; -- goto out; -- } -- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); - - /* - * Ensure we have slots for all the pages in the range. This is -@@ -1641,14 +1627,14 @@ static void collapse_file(struct mm_struct *mm, - } - } while (1); - -- __SetPageLocked(new_page); -+ __SetPageLocked(hpage); - if (is_shmem) -- __SetPageSwapBacked(new_page); -- new_page->index = start; -- new_page->mapping = mapping; -+ __SetPageSwapBacked(hpage); -+ hpage->index = start; -+ hpage->mapping = mapping; - - /* -- * At this point the new_page is locked and not up-to-date. -+ * At this point the hpage is locked and not up-to-date. - * It's safe to insert it into the page cache, because nobody would - * be able to map it or use it in another way until we unlock it. - */ -@@ -1676,7 +1662,7 @@ static void collapse_file(struct mm_struct *mm, - result = SCAN_FAIL; - goto xa_locked; - } -- xas_store(&xas, new_page); -+ xas_store(&xas, hpage); - nr_none++; - continue; - } -@@ -1818,19 +1804,19 @@ static void collapse_file(struct mm_struct *mm, - list_add_tail(&page->lru, &pagelist); - - /* Finally, replace with the new page. */ -- xas_store(&xas, new_page); -+ xas_store(&xas, hpage); - continue; - out_unlock: - unlock_page(page); - put_page(page); - goto xa_unlocked; - } -- nr = thp_nr_pages(new_page); -+ nr = thp_nr_pages(hpage); - - if (is_shmem) -- __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); -+ __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); - else { -- __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); -+ __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); - filemap_nr_thps_inc(mapping); - /* - * Paired with smp_mb() in do_dentry_open() to ensure -@@ -1841,21 +1827,21 @@ static void collapse_file(struct mm_struct *mm, - smp_mb(); - if (inode_is_open_for_write(mapping->host)) { - result = SCAN_FAIL; -- __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); -+ __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); - filemap_nr_thps_dec(mapping); - goto xa_locked; - } - } - - if (nr_none) { -- __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); -+ __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); - /* nr_none is always 0 for non-shmem. */ -- __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); -+ __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); - } - - /* Join all the small entries into a single multi-index entry */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); -- xas_store(&xas, new_page); -+ xas_store(&xas, hpage); - xa_locked: - xas_unlock_irq(&xas); - xa_unlocked: -@@ -1877,11 +1863,11 @@ static void collapse_file(struct mm_struct *mm, - index = start; - list_for_each_entry_safe(page, tmp, &pagelist, lru) { - while (index < page->index) { -- clear_highpage(new_page + (index % HPAGE_PMD_NR)); -+ clear_highpage(hpage + (index % HPAGE_PMD_NR)); - index++; - } -- copy_highpage(new_page + (page->index % HPAGE_PMD_NR), -- page); -+ copy_highpage(hpage + (page->index % HPAGE_PMD_NR), -+ page); - list_del(&page->lru); - page->mapping = NULL; - page_ref_unfreeze(page, 1); -@@ -1892,23 +1878,22 @@ static void collapse_file(struct mm_struct *mm, - index++; - } - while (index < end) { -- clear_highpage(new_page + (index % HPAGE_PMD_NR)); -+ clear_highpage(hpage + (index % HPAGE_PMD_NR)); - index++; - } - -- SetPageUptodate(new_page); -- page_ref_add(new_page, HPAGE_PMD_NR - 1); -+ SetPageUptodate(hpage); -+ page_ref_add(hpage, HPAGE_PMD_NR - 1); - if (is_shmem) -- set_page_dirty(new_page); -- lru_cache_add(new_page); -+ set_page_dirty(hpage); -+ lru_cache_add(hpage); - - /* - * Remove pte page tables, so we can re-fault the page as huge. - */ - retract_page_tables(mapping, start); -- *hpage = NULL; -- -- khugepaged_pages_collapsed++; -+ unlock_page(hpage); -+ hpage = NULL; - } else { - struct page *page; - -@@ -1947,19 +1932,23 @@ static void collapse_file(struct mm_struct *mm, - VM_BUG_ON(nr_none); - xas_unlock_irq(&xas); - -- new_page->mapping = NULL; -+ hpage->mapping = NULL; - } - -- unlock_page(new_page); -+ if (hpage) -+ unlock_page(hpage); - out: - VM_BUG_ON(!list_empty(&pagelist)); -- if (!IS_ERR_OR_NULL(*hpage)) -- mem_cgroup_uncharge(page_folio(*hpage)); -+ if (hpage) { -+ mem_cgroup_uncharge(page_folio(hpage)); -+ put_page(hpage); -+ } - /* TODO: tracepoints */ -+ return result; - } - --static void khugepaged_scan_file(struct mm_struct *mm, -- struct file *file, pgoff_t start, struct page **hpage) -+static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, -+ pgoff_t start, struct collapse_control *cc) - { - struct page *page = NULL; - struct address_space *mapping = file->f_mapping; -@@ -1970,14 +1959,16 @@ static void khugepaged_scan_file(struct mm_struct *mm, - - present = 0; - swap = 0; -- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); -+ memset(cc->node_load, 0, sizeof(cc->node_load)); - rcu_read_lock(); - xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { - if (xas_retry(&xas, page)) - continue; - - if (xa_is_value(page)) { -- if (++swap > khugepaged_max_ptes_swap) { -+ ++swap; -+ if (cc->is_khugepaged && -+ swap > khugepaged_max_ptes_swap) { - result = SCAN_EXCEED_SWAP_PTE; - count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); - break; -@@ -1995,11 +1986,11 @@ static void khugepaged_scan_file(struct mm_struct *mm, - } - - node = page_to_nid(page); -- if (khugepaged_scan_abort(node)) { -+ if (hpage_collapse_scan_abort(node, cc)) { - result = SCAN_SCAN_ABORT; - break; - } -- khugepaged_node_load[node]++; -+ cc->node_load[node]++; - - if (!PageLRU(page)) { - result = SCAN_PAGE_LRU; -@@ -2028,20 +2019,21 @@ static void khugepaged_scan_file(struct mm_struct *mm, - rcu_read_unlock(); - - if (result == SCAN_SUCCEED) { -- if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { -+ if (cc->is_khugepaged && -+ present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { - result = SCAN_EXCEED_NONE_PTE; - count_vm_event(THP_SCAN_EXCEED_NONE_PTE); - } else { -- node = khugepaged_find_target_node(); -- collapse_file(mm, file, start, hpage, node); -+ result = collapse_file(mm, file, start, cc); - } - } - - /* TODO: tracepoints */ -+ return result; - } - #else --static void khugepaged_scan_file(struct mm_struct *mm, -- struct file *file, pgoff_t start, struct page **hpage) -+static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, -+ pgoff_t start, struct collapse_control *cc) - { - BUILD_BUG(); - } -@@ -2051,11 +2043,12 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) - } - #endif - --static unsigned int khugepaged_scan_mm_slot(unsigned int pages, -- struct page **hpage) -+static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, -+ struct collapse_control *cc) - __releases(&khugepaged_mm_lock) - __acquires(&khugepaged_mm_lock) - { -+ struct vma_iterator vmi; - struct mm_slot *mm_slot; - struct mm_struct *mm; - struct vm_area_struct *vma; -@@ -2063,6 +2056,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - - VM_BUG_ON(!pages); - lockdep_assert_held(&khugepaged_mm_lock); -+ *result = SCAN_FAIL; - - if (khugepaged_scan.mm_slot) - mm_slot = khugepaged_scan.mm_slot; -@@ -2083,19 +2077,21 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - vma = NULL; - if (unlikely(!mmap_read_trylock(mm))) - goto breakouterloop_mmap_lock; -- if (likely(!khugepaged_test_exit(mm))) -- vma = find_vma(mm, khugepaged_scan.address); - - progress++; -- for (; vma; vma = vma->vm_next) { -+ if (unlikely(hpage_collapse_test_exit(mm))) -+ goto breakouterloop; -+ -+ vma_iter_init(&vmi, mm, khugepaged_scan.address); -+ for_each_vma(vmi, vma) { - unsigned long hstart, hend; - - cond_resched(); -- if (unlikely(khugepaged_test_exit(mm))) { -+ if (unlikely(hpage_collapse_test_exit(mm))) { - progress++; - break; - } -- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) { -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { - skip: - progress++; - continue; -@@ -2109,9 +2105,10 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); - - while (khugepaged_scan.address < hend) { -- int ret; -+ bool mmap_locked = true; -+ - cond_resched(); -- if (unlikely(khugepaged_test_exit(mm))) -+ if (unlikely(hpage_collapse_test_exit(mm))) - goto breakouterloop; - - VM_BUG_ON(khugepaged_scan.address < hstart || -@@ -2123,19 +2120,29 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - khugepaged_scan.address); - - mmap_read_unlock(mm); -- ret = 1; -- khugepaged_scan_file(mm, file, pgoff, hpage); -+ *result = khugepaged_scan_file(mm, file, pgoff, -+ cc); -+ mmap_locked = false; - fput(file); - } else { -- ret = khugepaged_scan_pmd(mm, vma, -- khugepaged_scan.address, -- hpage); -+ *result = hpage_collapse_scan_pmd(mm, vma, -+ khugepaged_scan.address, -+ &mmap_locked, -+ cc); - } -+ if (*result == SCAN_SUCCEED) -+ ++khugepaged_pages_collapsed; - /* move to next address */ - khugepaged_scan.address += HPAGE_PMD_SIZE; - progress += HPAGE_PMD_NR; -- if (ret) -- /* we released mmap_lock so break loop */ -+ if (!mmap_locked) -+ /* -+ * We released mmap_lock so break loop. Note -+ * that we drop mmap_lock before all hugepage -+ * allocations, so if allocation fails, we are -+ * guaranteed to break here and report the -+ * correct result back to caller. -+ */ - goto breakouterloop_mmap_lock; - if (progress >= pages) - goto breakouterloop; -@@ -2151,7 +2158,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - * Release the current mm_slot if this mm is about to die, or - * if we scanned all vmas of this mm. - */ -- if (khugepaged_test_exit(mm) || !vma) { -+ if (hpage_collapse_test_exit(mm) || !vma) { - /* - * Make sure that if mm_users is reaching zero while - * khugepaged runs here, khugepaged_exit will find -@@ -2185,19 +2192,16 @@ static int khugepaged_wait_event(void) - kthread_should_stop(); - } - --static void khugepaged_do_scan(void) -+static void khugepaged_do_scan(struct collapse_control *cc) - { -- struct page *hpage = NULL; - unsigned int progress = 0, pass_through_head = 0; - unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); - bool wait = true; -+ int result = SCAN_SUCCEED; - - lru_add_drain_all(); - -- while (progress < pages) { -- if (!khugepaged_prealloc_page(&hpage, &wait)) -- break; -- -+ while (true) { - cond_resched(); - - if (unlikely(kthread_should_stop() || try_to_freeze())) -@@ -2209,14 +2213,25 @@ static void khugepaged_do_scan(void) - if (khugepaged_has_work() && - pass_through_head < 2) - progress += khugepaged_scan_mm_slot(pages - progress, -- &hpage); -+ &result, cc); - else - progress = pages; - spin_unlock(&khugepaged_mm_lock); -- } - -- if (!IS_ERR_OR_NULL(hpage)) -- put_page(hpage); -+ if (progress >= pages) -+ break; -+ -+ if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { -+ /* -+ * If fail to allocate the first time, try to sleep for -+ * a while. When hit again, cancel the scan. -+ */ -+ if (!wait) -+ break; -+ wait = false; -+ khugepaged_alloc_sleep(); -+ } -+ } - } - - static bool khugepaged_should_wakeup(void) -@@ -2253,7 +2268,7 @@ static int khugepaged(void *none) - set_user_nice(current, MAX_NICE); - - while (!kthread_should_stop()) { -- khugepaged_do_scan(); -+ khugepaged_do_scan(&khugepaged_collapse_control); - khugepaged_wait_work(); - } - -@@ -2352,3 +2367,120 @@ void khugepaged_min_free_kbytes_update(void) - set_recommended_min_free_kbytes(); - mutex_unlock(&khugepaged_mutex); - } -+ -+static int madvise_collapse_errno(enum scan_result r) -+{ -+ /* -+ * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide -+ * actionable feedback to caller, so they may take an appropriate -+ * fallback measure depending on the nature of the failure. -+ */ -+ switch (r) { -+ case SCAN_ALLOC_HUGE_PAGE_FAIL: -+ return -ENOMEM; -+ case SCAN_CGROUP_CHARGE_FAIL: -+ return -EBUSY; -+ /* Resource temporary unavailable - trying again might succeed */ -+ case SCAN_PAGE_LOCK: -+ case SCAN_PAGE_LRU: -+ return -EAGAIN; -+ /* -+ * Other: Trying again likely not to succeed / error intrinsic to -+ * specified memory range. khugepaged likely won't be able to collapse -+ * either. -+ */ -+ default: -+ return -EINVAL; -+ } -+} -+ -+int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, -+ unsigned long start, unsigned long end) -+{ -+ struct collapse_control *cc; -+ struct mm_struct *mm = vma->vm_mm; -+ unsigned long hstart, hend, addr; -+ int thps = 0, last_fail = SCAN_FAIL; -+ bool mmap_locked = true; -+ -+ BUG_ON(vma->vm_start > start); -+ BUG_ON(vma->vm_end < end); -+ -+ *prev = vma; -+ -+ /* TODO: Support file/shmem */ -+ if (!vma->anon_vma || !vma_is_anonymous(vma)) -+ return -EINVAL; -+ -+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) -+ return -EINVAL; -+ -+ cc = kmalloc(sizeof(*cc), GFP_KERNEL); -+ if (!cc) -+ return -ENOMEM; -+ cc->is_khugepaged = false; -+ cc->last_target_node = NUMA_NO_NODE; -+ -+ mmgrab(mm); -+ lru_add_drain_all(); -+ -+ hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; -+ hend = end & HPAGE_PMD_MASK; -+ -+ for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { -+ int result = SCAN_FAIL; -+ -+ if (!mmap_locked) { -+ cond_resched(); -+ mmap_read_lock(mm); -+ mmap_locked = true; -+ result = hugepage_vma_revalidate(mm, addr, &vma, cc); -+ if (result != SCAN_SUCCEED) { -+ last_fail = result; -+ goto out_nolock; -+ } -+ } -+ mmap_assert_locked(mm); -+ memset(cc->node_load, 0, sizeof(cc->node_load)); -+ result = hpage_collapse_scan_pmd(mm, vma, addr, &mmap_locked, -+ cc); -+ if (!mmap_locked) -+ *prev = NULL; /* Tell caller we dropped mmap_lock */ -+ -+ switch (result) { -+ case SCAN_SUCCEED: -+ case SCAN_PMD_MAPPED: -+ ++thps; -+ break; -+ /* Whitelisted set of results where continuing OK */ -+ case SCAN_PMD_NULL: -+ case SCAN_PTE_NON_PRESENT: -+ case SCAN_PTE_UFFD_WP: -+ case SCAN_PAGE_RO: -+ case SCAN_LACK_REFERENCED_PAGE: -+ case SCAN_PAGE_NULL: -+ case SCAN_PAGE_COUNT: -+ case SCAN_PAGE_LOCK: -+ case SCAN_PAGE_COMPOUND: -+ case SCAN_PAGE_LRU: -+ last_fail = result; -+ break; -+ default: -+ last_fail = result; -+ /* Other error, exit */ -+ goto out_maybelock; -+ } -+ } -+ -+out_maybelock: -+ /* Caller expects us to hold mmap_lock on return */ -+ if (!mmap_locked) -+ mmap_read_lock(mm); -+out_nolock: -+ mmap_assert_locked(mm); -+ mmdrop(mm); -+ kfree(cc); -+ -+ return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 -+ : madvise_collapse_errno(last_fail); -+} -diff --git a/mm/ksm.c b/mm/ksm.c -index 4e64adb9adee..b7ee3c87a059 100644 ---- a/mm/ksm.c -+++ b/mm/ksm.c -@@ -981,11 +981,13 @@ static int unmerge_and_remove_all_rmap_items(void) - struct mm_slot, mm_list); - spin_unlock(&ksm_mmlist_lock); - -- for (mm_slot = ksm_scan.mm_slot; -- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { -+ for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; -+ mm_slot = ksm_scan.mm_slot) { -+ VMA_ITERATOR(vmi, mm_slot->mm, 0); -+ - mm = mm_slot->mm; - mmap_read_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (ksm_test_exit(mm)) - break; - if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) -@@ -1134,6 +1136,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, - { - struct mm_struct *mm = vma->vm_mm; - pmd_t *pmd; -+ pmd_t pmde; - pte_t *ptep; - pte_t newpte; - spinlock_t *ptl; -@@ -1148,6 +1151,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, - pmd = mm_find_pmd(mm, addr); - if (!pmd) - goto out; -+ /* -+ * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() -+ * without holding anon_vma lock for write. So when looking for a -+ * genuine pmde (in which to find pte), test present and !THP together. -+ */ -+ pmde = *pmd; -+ barrier(); -+ if (!pmd_present(pmde) || pmd_trans_huge(pmde)) -+ goto out; - - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, - addr + PAGE_SIZE); -@@ -2232,6 +2244,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) - struct mm_slot *slot; - struct vm_area_struct *vma; - struct rmap_item *rmap_item; -+ struct vma_iterator vmi; - int nid; - - if (list_empty(&ksm_mm_head.mm_list)) -@@ -2290,13 +2303,13 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) - } - - mm = slot->mm; -+ vma_iter_init(&vmi, mm, ksm_scan.address); -+ - mmap_read_lock(mm); - if (ksm_test_exit(mm)) -- vma = NULL; -- else -- vma = find_vma(mm, ksm_scan.address); -+ goto no_vmas; - -- for (; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (!(vma->vm_flags & VM_MERGEABLE)) - continue; - if (ksm_scan.address < vma->vm_start) -@@ -2334,6 +2347,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) - } - - if (ksm_test_exit(mm)) { -+no_vmas: - ksm_scan.address = 0; - ksm_scan.rmap_list = &slot->rmap_list; - } -diff --git a/mm/madvise.c b/mm/madvise.c -index 5f0f0948a50e..682e1d161aef 100644 ---- a/mm/madvise.c -+++ b/mm/madvise.c -@@ -59,6 +59,7 @@ static int madvise_need_mmap_write(int behavior) - case MADV_FREE: - case MADV_POPULATE_READ: - case MADV_POPULATE_WRITE: -+ case MADV_COLLAPSE: - return 0; - default: - /* be safe, default to 1. list exceptions explicitly */ -@@ -1057,6 +1058,8 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, - if (error) - goto out; - break; -+ case MADV_COLLAPSE: -+ return madvise_collapse(vma, prev, start, end); - } - - anon_name = anon_vma_name(vma); -@@ -1150,6 +1153,7 @@ madvise_behavior_valid(int behavior) - #ifdef CONFIG_TRANSPARENT_HUGEPAGE - case MADV_HUGEPAGE: - case MADV_NOHUGEPAGE: -+ case MADV_COLLAPSE: - #endif - case MADV_DONTDUMP: - case MADV_DODUMP: -@@ -1166,13 +1170,13 @@ madvise_behavior_valid(int behavior) - } - } - --static bool --process_madvise_behavior_valid(int behavior) -+static bool process_madvise_behavior_valid(int behavior) - { - switch (behavior) { - case MADV_COLD: - case MADV_PAGEOUT: - case MADV_WILLNEED: -+ case MADV_COLLAPSE: - return true; - default: - return false; -@@ -1238,7 +1242,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, - if (start >= end) - break; - if (prev) -- vma = prev->vm_next; -+ vma = find_vma(mm, prev->vm_end); - else /* madvise_remove dropped mmap_lock */ - vma = find_vma(mm, start); - } -@@ -1339,6 +1343,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, - * MADV_NOHUGEPAGE - mark the given range as not worth being backed by - * transparent huge pages so the existing pages will not be - * coalesced into THP and new pages will not be allocated as THP. -+ * MADV_COLLAPSE - synchronously coalesce pages into new THP. - * MADV_DONTDUMP - the application wants to prevent pages in the given range - * from being included in its core dump. - * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. -diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index 7eadbafc006b..454a283b9e85 100644 ---- a/mm/memcontrol.c -+++ b/mm/memcontrol.c -@@ -2789,6 +2789,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - */ - folio->memcg_data = (unsigned long)memcg; - } -@@ -5170,6 +5171,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) - - static void mem_cgroup_free(struct mem_cgroup *memcg) - { -+ lru_gen_exit_memcg(memcg); - memcg_wb_domain_exit(memcg); - __mem_cgroup_free(memcg); - } -@@ -5228,6 +5230,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) - memcg->deferred_split_queue.split_queue_len = 0; - #endif - idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); -+ lru_gen_init_memcg(memcg); - return memcg; - fail: - mem_cgroup_id_remove(memcg); -@@ -5871,7 +5874,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) - unsigned long precharge; - - mmap_read_lock(mm); -- walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL); -+ walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL); - mmap_read_unlock(mm); - - precharge = mc.precharge; -@@ -6169,9 +6172,7 @@ static void mem_cgroup_move_charge(void) - * When we have consumed all precharges and failed in doing - * additional charge, the page walk just aborts. - */ -- walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops, -- NULL); -- -+ walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL); - mmap_read_unlock(mc.mm); - atomic_dec(&mc.from->moving_account); - } -@@ -6196,6 +6197,30 @@ static void mem_cgroup_move_task(void) - } - #endif - -+#ifdef CONFIG_LRU_GEN -+static void mem_cgroup_attach(struct cgroup_taskset *tset) -+{ -+ struct task_struct *task; -+ struct cgroup_subsys_state *css; -+ -+ /* find the first leader if there is any */ -+ cgroup_taskset_for_each_leader(task, css, tset) -+ break; -+ -+ if (!task) -+ return; -+ -+ task_lock(task); -+ if (task->mm && READ_ONCE(task->mm->owner) == task) -+ lru_gen_migrate_mm(task->mm); -+ task_unlock(task); -+} -+#else -+static void mem_cgroup_attach(struct cgroup_taskset *tset) -+{ -+} -+#endif /* CONFIG_LRU_GEN */ -+ - static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) - { - if (value == PAGE_COUNTER_MAX) -@@ -6601,6 +6626,7 @@ struct cgroup_subsys memory_cgrp_subsys = { - .css_reset = mem_cgroup_css_reset, - .css_rstat_flush = mem_cgroup_css_rstat_flush, - .can_attach = mem_cgroup_can_attach, -+ .attach = mem_cgroup_attach, - .cancel_attach = mem_cgroup_cancel_attach, - .post_attach = mem_cgroup_move_task, - .dfl_cftypes = memory_files, -diff --git a/mm/memory.c b/mm/memory.c -index 4ba73f5aa8bb..4fd25d3aba64 100644 ---- a/mm/memory.c -+++ b/mm/memory.c -@@ -125,18 +125,6 @@ int randomize_va_space __read_mostly = - 2; - #endif - --#ifndef arch_faults_on_old_pte --static inline bool arch_faults_on_old_pte(void) --{ -- /* -- * Those arches which don't have hw access flag feature need to -- * implement their own helper. By default, "true" means pagefault -- * will be hit on old pte. -- */ -- return true; --} --#endif -- - #ifndef arch_wants_old_prefaulted_pte - static inline bool arch_wants_old_prefaulted_pte(void) - { -@@ -402,12 +390,21 @@ void free_pgd_range(struct mmu_gather *tlb, - } while (pgd++, addr = next, addr != end); - } - --void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, -- unsigned long floor, unsigned long ceiling) -+void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, -+ struct vm_area_struct *vma, unsigned long floor, -+ unsigned long ceiling) - { -- while (vma) { -- struct vm_area_struct *next = vma->vm_next; -+ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); -+ -+ do { - unsigned long addr = vma->vm_start; -+ struct vm_area_struct *next; -+ -+ /* -+ * Note: USER_PGTABLES_CEILING may be passed as ceiling and may -+ * be 0. This will underflow and is okay. -+ */ -+ next = mas_find(&mas, ceiling - 1); - - /* - * Hide vma from rmap and truncate_pagecache before freeing -@@ -426,7 +423,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - while (next && next->vm_start <= vma->vm_end + PMD_SIZE - && !is_vm_hugetlb_page(next)) { - vma = next; -- next = vma->vm_next; -+ next = mas_find(&mas, ceiling - 1); - unlink_anon_vmas(vma); - unlink_file_vma(vma); - } -@@ -434,7 +431,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - floor, next ? next->vm_start : ceiling); - } - vma = next; -- } -+ } while (vma); - } - - void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) -@@ -1698,6 +1695,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, - /** - * unmap_vmas - unmap a range of memory covered by a list of vma's - * @tlb: address of the caller's struct mmu_gather -+ * @mt: the maple tree - * @vma: the starting vma - * @start_addr: virtual address at which to start unmapping - * @end_addr: virtual address at which to end unmapping -@@ -1713,7 +1711,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, - * ensure that any thus-far unmapped pages are flushed before unmap_vmas() - * drops the lock and schedules. - */ --void unmap_vmas(struct mmu_gather *tlb, -+void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, - struct vm_area_struct *vma, unsigned long start_addr, - unsigned long end_addr) - { -@@ -1723,12 +1721,14 @@ void unmap_vmas(struct mmu_gather *tlb, - /* Careful - we need to zap private pages too! */ - .even_cows = true, - }; -+ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); - - mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, - start_addr, end_addr); - mmu_notifier_invalidate_range_start(&range); -- for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) -+ do { - unmap_single_vma(tlb, vma, start_addr, end_addr, &details); -+ } while ((vma = mas_find(&mas, end_addr - 1)) != NULL); - mmu_notifier_invalidate_range_end(&range); - } - -@@ -1743,8 +1743,11 @@ void unmap_vmas(struct mmu_gather *tlb, - void zap_page_range(struct vm_area_struct *vma, unsigned long start, - unsigned long size) - { -+ struct maple_tree *mt = &vma->vm_mm->mm_mt; -+ unsigned long end = start + size; - struct mmu_notifier_range range; - struct mmu_gather tlb; -+ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); - - lru_add_drain(); - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, -@@ -1752,8 +1755,9 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, - tlb_gather_mmu(&tlb, vma->vm_mm); - update_hiwater_rss(vma->vm_mm); - mmu_notifier_invalidate_range_start(&range); -- for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) -+ do { - unmap_single_vma(&tlb, vma, start, range.end, NULL); -+ } while ((vma = mas_find(&mas, end - 1)) != NULL); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -@@ -2870,7 +2874,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, - * On architectures with software "accessed" bits, we would - * take a double page fault, so mark it accessed here. - */ -- if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { -+ if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { - pte_t entry; - - vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); -@@ -4985,7 +4989,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, - return VM_FAULT_OOM; - retry_pud: - if (pud_none(*vmf.pud) && -- hugepage_vma_check(vma, vm_flags, false, true)) { -+ hugepage_vma_check(vma, vm_flags, false, true, true)) { - ret = create_huge_pud(&vmf); - if (!(ret & VM_FAULT_FALLBACK)) - return ret; -@@ -5019,7 +5023,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, - goto retry_pud; - - if (pmd_none(*vmf.pmd) && -- hugepage_vma_check(vma, vm_flags, false, true)) { -+ hugepage_vma_check(vma, vm_flags, false, true, true)) { - ret = create_huge_pmd(&vmf); - if (!(ret & VM_FAULT_FALLBACK)) - return ret; -@@ -5114,6 +5118,27 @@ static inline void mm_account_fault(struct pt_regs *regs, - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); - } - -+#ifdef CONFIG_LRU_GEN -+static void lru_gen_enter_fault(struct vm_area_struct *vma) -+{ -+ /* the LRU algorithm doesn't apply to sequential or random reads */ -+ current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); -+} -+ -+static void lru_gen_exit_fault(void) -+{ -+ current->in_lru_fault = false; -+} -+#else -+static void lru_gen_enter_fault(struct vm_area_struct *vma) -+{ -+} -+ -+static void lru_gen_exit_fault(void) -+{ -+} -+#endif /* CONFIG_LRU_GEN */ -+ - /* - * By the time we get here, we already hold the mm semaphore - * -@@ -5145,11 +5170,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, - if (flags & FAULT_FLAG_USER) - mem_cgroup_enter_user_fault(); - -+ lru_gen_enter_fault(vma); -+ - if (unlikely(is_vm_hugetlb_page(vma))) - ret = hugetlb_fault(vma->vm_mm, vma, address, flags); - else - ret = __handle_mm_fault(vma, address, flags); - -+ lru_gen_exit_fault(); -+ - if (flags & FAULT_FLAG_USER) { - mem_cgroup_exit_user_fault(); - /* -diff --git a/mm/mempolicy.c b/mm/mempolicy.c -index b73d3248d976..6c27acb6cd63 100644 ---- a/mm/mempolicy.c -+++ b/mm/mempolicy.c -@@ -381,9 +381,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) - void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) - { - struct vm_area_struct *vma; -+ VMA_ITERATOR(vmi, mm, 0); - - mmap_write_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) -+ for_each_vma(vmi, vma) - mpol_rebind_policy(vma->vm_policy, new); - mmap_write_unlock(mm); - } -@@ -654,7 +655,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma, - static int queue_pages_test_walk(unsigned long start, unsigned long end, - struct mm_walk *walk) - { -- struct vm_area_struct *vma = walk->vma; -+ struct vm_area_struct *next, *vma = walk->vma; - struct queue_pages *qp = walk->private; - unsigned long endvma = vma->vm_end; - unsigned long flags = qp->flags; -@@ -669,9 +670,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, - /* hole at head side of range */ - return -EFAULT; - } -+ next = find_vma(vma->vm_mm, vma->vm_end); - if (!(flags & MPOL_MF_DISCONTIG_OK) && - ((vma->vm_end < qp->end) && -- (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) -+ (!next || vma->vm_end < next->vm_start))) - /* hole at middle or tail of range */ - return -EFAULT; - -@@ -785,26 +787,24 @@ static int vma_replace_policy(struct vm_area_struct *vma, - static int mbind_range(struct mm_struct *mm, unsigned long start, - unsigned long end, struct mempolicy *new_pol) - { -+ MA_STATE(mas, &mm->mm_mt, start - 1, start - 1); - struct vm_area_struct *prev; - struct vm_area_struct *vma; - int err = 0; - pgoff_t pgoff; -- unsigned long vmstart; -- unsigned long vmend; -- -- vma = find_vma(mm, start); -- VM_BUG_ON(!vma); - -- prev = vma->vm_prev; -- if (start > vma->vm_start) -- prev = vma; -+ prev = mas_find_rev(&mas, 0); -+ if (prev && (start < prev->vm_end)) -+ vma = prev; -+ else -+ vma = mas_next(&mas, end - 1); - -- for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) { -- vmstart = max(start, vma->vm_start); -- vmend = min(end, vma->vm_end); -+ for (; vma; vma = mas_next(&mas, end - 1)) { -+ unsigned long vmstart = max(start, vma->vm_start); -+ unsigned long vmend = min(end, vma->vm_end); - - if (mpol_equal(vma_policy(vma), new_pol)) -- continue; -+ goto next; - - pgoff = vma->vm_pgoff + - ((vmstart - vma->vm_start) >> PAGE_SHIFT); -@@ -813,6 +813,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, - new_pol, vma->vm_userfaultfd_ctx, - anon_vma_name(vma)); - if (prev) { -+ /* vma_merge() invalidated the mas */ -+ mas_pause(&mas); - vma = prev; - goto replace; - } -@@ -820,19 +822,25 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, - err = split_vma(vma->vm_mm, vma, vmstart, 1); - if (err) - goto out; -+ /* split_vma() invalidated the mas */ -+ mas_pause(&mas); - } - if (vma->vm_end != vmend) { - err = split_vma(vma->vm_mm, vma, vmend, 0); - if (err) - goto out; -+ /* split_vma() invalidated the mas */ -+ mas_pause(&mas); - } -- replace: -+replace: - err = vma_replace_policy(vma, new_pol); - if (err) - goto out; -+next: -+ prev = vma; - } - -- out: -+out: - return err; - } - -@@ -1047,6 +1055,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, - int flags) - { - nodemask_t nmask; -+ struct vm_area_struct *vma; - LIST_HEAD(pagelist); - int err = 0; - struct migration_target_control mtc = { -@@ -1062,8 +1071,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, - * need migration. Between passing in the full user address - * space range and MPOL_MF_DISCONTIG_OK, this call can not fail. - */ -+ vma = find_vma(mm, 0); - VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))); -- queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, -+ queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask, - flags | MPOL_MF_DISCONTIG_OK, &pagelist); - - if (!list_empty(&pagelist)) { -@@ -1193,14 +1203,13 @@ static struct page *new_page(struct page *page, unsigned long start) - struct folio *dst, *src = page_folio(page); - struct vm_area_struct *vma; - unsigned long address; -+ VMA_ITERATOR(vmi, current->mm, start); - gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL; - -- vma = find_vma(current->mm, start); -- while (vma) { -+ for_each_vma(vmi, vma) { - address = page_address_in_vma(page, vma); - if (address != -EFAULT) - break; -- vma = vma->vm_next; - } - - if (folio_test_hugetlb(src)) -@@ -1478,6 +1487,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le - unsigned long vmend; - unsigned long end; - int err = -ENOENT; -+ VMA_ITERATOR(vmi, mm, start); - - start = untagged_addr(start); - if (start & ~PAGE_MASK) -@@ -1503,9 +1513,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le - if (end == start) - return 0; - mmap_write_lock(mm); -- vma = find_vma(mm, start); -- for (; vma && vma->vm_start < end; vma = vma->vm_next) { -- -+ for_each_vma_range(vmi, vma, end) { - vmstart = max(start, vma->vm_start); - vmend = min(end, vma->vm_end); - new = mpol_dup(vma_policy(vma)); -diff --git a/mm/mlock.c b/mm/mlock.c -index b14e929084cc..7032f6dd0ce1 100644 ---- a/mm/mlock.c -+++ b/mm/mlock.c -@@ -471,6 +471,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, - unsigned long nstart, end, tmp; - struct vm_area_struct *vma, *prev; - int error; -+ MA_STATE(mas, ¤t->mm->mm_mt, start, start); - - VM_BUG_ON(offset_in_page(start)); - VM_BUG_ON(len != PAGE_ALIGN(len)); -@@ -479,13 +480,14 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, - return -EINVAL; - if (end == start) - return 0; -- vma = find_vma(current->mm, start); -- if (!vma || vma->vm_start > start) -+ vma = mas_walk(&mas); -+ if (!vma) - return -ENOMEM; - -- prev = vma->vm_prev; - if (start > vma->vm_start) - prev = vma; -+ else -+ prev = mas_prev(&mas, 0); - - for (nstart = start ; ; ) { - vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; -@@ -505,7 +507,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, - if (nstart >= end) - break; - -- vma = prev->vm_next; -+ vma = find_vma(prev->vm_mm, prev->vm_end); - if (!vma || vma->vm_start != nstart) { - error = -ENOMEM; - break; -@@ -526,24 +528,21 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, - { - struct vm_area_struct *vma; - unsigned long count = 0; -+ unsigned long end; -+ VMA_ITERATOR(vmi, mm, start); - -- if (mm == NULL) -- mm = current->mm; -+ /* Don't overflow past ULONG_MAX */ -+ if (unlikely(ULONG_MAX - len < start)) -+ end = ULONG_MAX; -+ else -+ end = start + len; - -- vma = find_vma(mm, start); -- if (vma == NULL) -- return 0; -- -- for (; vma ; vma = vma->vm_next) { -- if (start >= vma->vm_end) -- continue; -- if (start + len <= vma->vm_start) -- break; -+ for_each_vma_range(vmi, vma, end) { - if (vma->vm_flags & VM_LOCKED) { - if (start > vma->vm_start) - count -= (start - vma->vm_start); -- if (start + len < vma->vm_end) { -- count += start + len - vma->vm_start; -+ if (end < vma->vm_end) { -+ count += end - vma->vm_start; - break; - } - count += vma->vm_end - vma->vm_start; -@@ -659,6 +658,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) - */ - static int apply_mlockall_flags(int flags) - { -+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); - struct vm_area_struct *vma, *prev = NULL; - vm_flags_t to_add = 0; - -@@ -679,7 +679,7 @@ static int apply_mlockall_flags(int flags) - to_add |= VM_LOCKONFAULT; - } - -- for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { -+ mas_for_each(&mas, vma, ULONG_MAX) { - vm_flags_t newflags; - - newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; -@@ -687,6 +687,7 @@ static int apply_mlockall_flags(int flags) - - /* Ignore errors */ - mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); -+ mas_pause(&mas); - cond_resched(); - } - out: -diff --git a/mm/mm_init.c b/mm/mm_init.c -index 9ddaf0e1b0ab..0d7b2bd2454a 100644 ---- a/mm/mm_init.c -+++ b/mm/mm_init.c -@@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) - - shift = 8 * sizeof(unsigned long); - width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH -- - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; -+ - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; - mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", -- "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", -+ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", - SECTIONS_WIDTH, - NODES_WIDTH, - ZONES_WIDTH, - LAST_CPUPID_WIDTH, - KASAN_TAG_WIDTH, -+ LRU_GEN_WIDTH, -+ LRU_REFS_WIDTH, - NR_PAGEFLAGS); - mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", - "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", -diff --git a/mm/mmap.c b/mm/mmap.c -index 9d780f415be3..6fc054c55cd8 100644 ---- a/mm/mmap.c -+++ b/mm/mmap.c -@@ -14,7 +14,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -39,7 +38,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -77,9 +75,10 @@ int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; - static bool ignore_rlimit_data; - core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); - --static void unmap_region(struct mm_struct *mm, -+static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, - struct vm_area_struct *vma, struct vm_area_struct *prev, -- unsigned long start, unsigned long end); -+ struct vm_area_struct *next, unsigned long start, -+ unsigned long end); - - static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) - { -@@ -132,12 +131,10 @@ void unlink_file_vma(struct vm_area_struct *vma) - } - - /* -- * Close a vm structure and free it, returning the next. -+ * Close a vm structure and free it. - */ --static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) -+static void remove_vma(struct vm_area_struct *vma) - { -- struct vm_area_struct *next = vma->vm_next; -- - might_sleep(); - if (vma->vm_ops && vma->vm_ops->close) - vma->vm_ops->close(vma); -@@ -145,20 +142,41 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) - fput(vma->vm_file); - mpol_put(vma_policy(vma)); - vm_area_free(vma); -- return next; - } - --static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, -- struct list_head *uf); -+/* -+ * check_brk_limits() - Use platform specific check of range & verify mlock -+ * limits. -+ * @addr: The address to check -+ * @len: The size of increase. -+ * -+ * Return: 0 on success. -+ */ -+static int check_brk_limits(unsigned long addr, unsigned long len) -+{ -+ unsigned long mapped_addr; -+ -+ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); -+ if (IS_ERR_VALUE(mapped_addr)) -+ return mapped_addr; -+ -+ return mlock_future_check(current->mm, current->mm->def_flags, len); -+} -+static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, -+ unsigned long newbrk, unsigned long oldbrk, -+ struct list_head *uf); -+static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma, -+ unsigned long addr, unsigned long request, unsigned long flags); - SYSCALL_DEFINE1(brk, unsigned long, brk) - { - unsigned long newbrk, oldbrk, origbrk; - struct mm_struct *mm = current->mm; -- struct vm_area_struct *next; -+ struct vm_area_struct *brkvma, *next = NULL; - unsigned long min_brk; - bool populate; - bool downgraded = false; - LIST_HEAD(uf); -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - if (mmap_write_lock_killable(mm)) - return -EINTR; -@@ -200,35 +218,51 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) - - /* - * Always allow shrinking brk. -- * __do_munmap() may downgrade mmap_lock to read. -+ * do_brk_munmap() may downgrade mmap_lock to read. - */ - if (brk <= mm->brk) { - int ret; - -+ /* Search one past newbrk */ -+ mas_set(&mas, newbrk); -+ brkvma = mas_find(&mas, oldbrk); -+ BUG_ON(brkvma == NULL); -+ if (brkvma->vm_start >= oldbrk) -+ goto out; /* mapping intersects with an existing non-brk vma. */ - /* -- * mm->brk must to be protected by write mmap_lock so update it -- * before downgrading mmap_lock. When __do_munmap() fails, -- * mm->brk will be restored from origbrk. -+ * mm->brk must be protected by write mmap_lock. -+ * do_brk_munmap() may downgrade the lock, so update it -+ * before calling do_brk_munmap(). - */ - mm->brk = brk; -- ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true); -- if (ret < 0) { -- mm->brk = origbrk; -- goto out; -- } else if (ret == 1) { -+ ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf); -+ if (ret == 1) { - downgraded = true; -- } -- goto success; -+ goto success; -+ } else if (!ret) -+ goto success; -+ -+ mm->brk = origbrk; -+ goto out; - } - -- /* Check against existing mmap mappings. */ -- next = find_vma(mm, oldbrk); -+ if (check_brk_limits(oldbrk, newbrk - oldbrk)) -+ goto out; -+ -+ /* -+ * Only check if the next VMA is within the stack_guard_gap of the -+ * expansion area -+ */ -+ mas_set(&mas, oldbrk); -+ next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap); - if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) - goto out; - -+ brkvma = mas_prev(&mas, mm->start_brk); - /* Ok, looks good - let it rip. */ -- if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) -+ if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) - goto out; -+ - mm->brk = brk; - - success: -@@ -247,104 +281,45 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) - return origbrk; - } - --static inline unsigned long vma_compute_gap(struct vm_area_struct *vma) --{ -- unsigned long gap, prev_end; -- -- /* -- * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we -- * allow two stack_guard_gaps between them here, and when choosing -- * an unmapped area; whereas when expanding we only require one. -- * That's a little inconsistent, but keeps the code here simpler. -- */ -- gap = vm_start_gap(vma); -- if (vma->vm_prev) { -- prev_end = vm_end_gap(vma->vm_prev); -- if (gap > prev_end) -- gap -= prev_end; -- else -- gap = 0; -- } -- return gap; --} -- --#ifdef CONFIG_DEBUG_VM_RB --static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma) --{ -- unsigned long max = vma_compute_gap(vma), subtree_gap; -- if (vma->vm_rb.rb_left) { -- subtree_gap = rb_entry(vma->vm_rb.rb_left, -- struct vm_area_struct, vm_rb)->rb_subtree_gap; -- if (subtree_gap > max) -- max = subtree_gap; -- } -- if (vma->vm_rb.rb_right) { -- subtree_gap = rb_entry(vma->vm_rb.rb_right, -- struct vm_area_struct, vm_rb)->rb_subtree_gap; -- if (subtree_gap > max) -- max = subtree_gap; -- } -- return max; --} -- --static int browse_rb(struct mm_struct *mm) --{ -- struct rb_root *root = &mm->mm_rb; -- int i = 0, j, bug = 0; -- struct rb_node *nd, *pn = NULL; -- unsigned long prev = 0, pend = 0; -- -- for (nd = rb_first(root); nd; nd = rb_next(nd)) { -- struct vm_area_struct *vma; -- vma = rb_entry(nd, struct vm_area_struct, vm_rb); -- if (vma->vm_start < prev) { -- pr_emerg("vm_start %lx < prev %lx\n", -- vma->vm_start, prev); -- bug = 1; -- } -- if (vma->vm_start < pend) { -- pr_emerg("vm_start %lx < pend %lx\n", -- vma->vm_start, pend); -- bug = 1; -- } -- if (vma->vm_start > vma->vm_end) { -- pr_emerg("vm_start %lx > vm_end %lx\n", -- vma->vm_start, vma->vm_end); -- bug = 1; -- } -- spin_lock(&mm->page_table_lock); -- if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) { -- pr_emerg("free gap %lx, correct %lx\n", -- vma->rb_subtree_gap, -- vma_compute_subtree_gap(vma)); -- bug = 1; -+#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) -+extern void mt_validate(struct maple_tree *mt); -+extern void mt_dump(const struct maple_tree *mt); -+ -+/* Validate the maple tree */ -+static void validate_mm_mt(struct mm_struct *mm) -+{ -+ struct maple_tree *mt = &mm->mm_mt; -+ struct vm_area_struct *vma_mt; -+ -+ MA_STATE(mas, mt, 0, 0); -+ -+ mt_validate(&mm->mm_mt); -+ mas_for_each(&mas, vma_mt, ULONG_MAX) { -+ if ((vma_mt->vm_start != mas.index) || -+ (vma_mt->vm_end - 1 != mas.last)) { -+ pr_emerg("issue in %s\n", current->comm); -+ dump_stack(); -+ dump_vma(vma_mt); -+ pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, -+ mas.index, mas.last); -+ pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, -+ vma_mt->vm_start, vma_mt->vm_end); -+ -+ mt_dump(mas.tree); -+ if (vma_mt->vm_end != mas.last + 1) { -+ pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", -+ mm, vma_mt->vm_start, vma_mt->vm_end, -+ mas.index, mas.last); -+ mt_dump(mas.tree); -+ } -+ VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); -+ if (vma_mt->vm_start != mas.index) { -+ pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", -+ mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); -+ mt_dump(mas.tree); -+ } -+ VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); - } -- spin_unlock(&mm->page_table_lock); -- i++; -- pn = nd; -- prev = vma->vm_start; -- pend = vma->vm_end; -- } -- j = 0; -- for (nd = pn; nd; nd = rb_prev(nd)) -- j++; -- if (i != j) { -- pr_emerg("backwards %d, forwards %d\n", j, i); -- bug = 1; -- } -- return bug ? -1 : i; --} -- --static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore) --{ -- struct rb_node *nd; -- -- for (nd = rb_first(root); nd; nd = rb_next(nd)) { -- struct vm_area_struct *vma; -- vma = rb_entry(nd, struct vm_area_struct, vm_rb); -- VM_BUG_ON_VMA(vma != ignore && -- vma->rb_subtree_gap != vma_compute_subtree_gap(vma), -- vma); - } - } - -@@ -352,10 +327,13 @@ static void validate_mm(struct mm_struct *mm) - { - int bug = 0; - int i = 0; -- unsigned long highest_address = 0; -- struct vm_area_struct *vma = mm->mmap; -+ struct vm_area_struct *vma; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - -- while (vma) { -+ validate_mm_mt(mm); -+ -+ mas_for_each(&mas, vma, ULONG_MAX) { -+#ifdef CONFIG_DEBUG_VM_RB - struct anon_vma *anon_vma = vma->anon_vma; - struct anon_vma_chain *avc; - -@@ -365,93 +343,20 @@ static void validate_mm(struct mm_struct *mm) - anon_vma_interval_tree_verify(avc); - anon_vma_unlock_read(anon_vma); - } -- -- highest_address = vm_end_gap(vma); -- vma = vma->vm_next; -+#endif - i++; - } - if (i != mm->map_count) { -- pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); -- bug = 1; -- } -- if (highest_address != mm->highest_vm_end) { -- pr_emerg("mm->highest_vm_end %lx, found %lx\n", -- mm->highest_vm_end, highest_address); -- bug = 1; -- } -- i = browse_rb(mm); -- if (i != mm->map_count) { -- if (i != -1) -- pr_emerg("map_count %d rb %d\n", mm->map_count, i); -+ pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i); - bug = 1; - } - VM_BUG_ON_MM(bug, mm); - } --#else --#define validate_mm_rb(root, ignore) do { } while (0) --#define validate_mm(mm) do { } while (0) --#endif -- --RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks, -- struct vm_area_struct, vm_rb, -- unsigned long, rb_subtree_gap, vma_compute_gap) -- --/* -- * Update augmented rbtree rb_subtree_gap values after vma->vm_start or -- * vma->vm_prev->vm_end values changed, without modifying the vma's position -- * in the rbtree. -- */ --static void vma_gap_update(struct vm_area_struct *vma) --{ -- /* -- * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created -- * a callback function that does exactly what we want. -- */ -- vma_gap_callbacks_propagate(&vma->vm_rb, NULL); --} -- --static inline void vma_rb_insert(struct vm_area_struct *vma, -- struct rb_root *root) --{ -- /* All rb_subtree_gap values must be consistent prior to insertion */ -- validate_mm_rb(root, NULL); -- -- rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks); --} -- --static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) --{ -- /* -- * Note rb_erase_augmented is a fairly large inline function, -- * so make sure we instantiate it only once with our desired -- * augmented rbtree callbacks. -- */ -- rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); --} -- --static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, -- struct rb_root *root, -- struct vm_area_struct *ignore) --{ -- /* -- * All rb_subtree_gap values must be consistent prior to erase, -- * with the possible exception of -- * -- * a. the "next" vma being erased if next->vm_start was reduced in -- * __vma_adjust() -> __vma_unlink() -- * b. the vma being erased in detach_vmas_to_be_unmapped() -> -- * vma_rb_erase() -- */ -- validate_mm_rb(root, ignore); -- -- __vma_rb_erase(vma, root); --} - --static __always_inline void vma_rb_erase(struct vm_area_struct *vma, -- struct rb_root *root) --{ -- vma_rb_erase_ignore(vma, root, vma); --} -+#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ -+#define validate_mm_mt(root) do { } while (0) -+#define validate_mm(mm) do { } while (0) -+#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ - - /* - * vma has some anon_vma assigned, and is already inserted on that -@@ -485,208 +390,220 @@ anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) - anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); - } - --static int find_vma_links(struct mm_struct *mm, unsigned long addr, -- unsigned long end, struct vm_area_struct **pprev, -- struct rb_node ***rb_link, struct rb_node **rb_parent) -+static unsigned long count_vma_pages_range(struct mm_struct *mm, -+ unsigned long addr, unsigned long end) - { -- struct rb_node **__rb_link, *__rb_parent, *rb_prev; -+ VMA_ITERATOR(vmi, mm, addr); -+ struct vm_area_struct *vma; -+ unsigned long nr_pages = 0; - -- mmap_assert_locked(mm); -- __rb_link = &mm->mm_rb.rb_node; -- rb_prev = __rb_parent = NULL; -+ for_each_vma_range(vmi, vma, end) { -+ unsigned long vm_start = max(addr, vma->vm_start); -+ unsigned long vm_end = min(end, vma->vm_end); - -- while (*__rb_link) { -- struct vm_area_struct *vma_tmp; -+ nr_pages += PHYS_PFN(vm_end - vm_start); -+ } - -- __rb_parent = *__rb_link; -- vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb); -+ return nr_pages; -+} - -- if (vma_tmp->vm_end > addr) { -- /* Fail if an existing vma overlaps the area */ -- if (vma_tmp->vm_start < end) -- return -ENOMEM; -- __rb_link = &__rb_parent->rb_left; -- } else { -- rb_prev = __rb_parent; -- __rb_link = &__rb_parent->rb_right; -- } -- } -+static void __vma_link_file(struct vm_area_struct *vma, -+ struct address_space *mapping) -+{ -+ if (vma->vm_flags & VM_SHARED) -+ mapping_allow_writable(mapping); - -- *pprev = NULL; -- if (rb_prev) -- *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); -- *rb_link = __rb_link; -- *rb_parent = __rb_parent; -- return 0; -+ flush_dcache_mmap_lock(mapping); -+ vma_interval_tree_insert(vma, &mapping->i_mmap); -+ flush_dcache_mmap_unlock(mapping); - } - - /* -- * vma_next() - Get the next VMA. -- * @mm: The mm_struct. -- * @vma: The current vma. -+ * vma_mas_store() - Store a VMA in the maple tree. -+ * @vma: The vm_area_struct -+ * @mas: The maple state - * -- * If @vma is NULL, return the first vma in the mm. -+ * Efficient way to store a VMA in the maple tree when the @mas has already -+ * walked to the correct location. - * -- * Returns: The next VMA after @vma. -+ * Note: the end address is inclusive in the maple tree. - */ --static inline struct vm_area_struct *vma_next(struct mm_struct *mm, -- struct vm_area_struct *vma) -+void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) - { -- if (!vma) -- return mm->mmap; -- -- return vma->vm_next; -+ trace_vma_store(mas->tree, vma); -+ mas_set_range(mas, vma->vm_start, vma->vm_end - 1); -+ mas_store_prealloc(mas, vma); - } - - /* -- * munmap_vma_range() - munmap VMAs that overlap a range. -- * @mm: The mm struct -- * @start: The start of the range. -- * @len: The length of the range. -- * @pprev: pointer to the pointer that will be set to previous vm_area_struct -- * @rb_link: the rb_node -- * @rb_parent: the parent rb_node -- * -- * Find all the vm_area_struct that overlap from @start to -- * @end and munmap them. Set @pprev to the previous vm_area_struct. -+ * vma_mas_remove() - Remove a VMA from the maple tree. -+ * @vma: The vm_area_struct -+ * @mas: The maple state - * -- * Returns: -ENOMEM on munmap failure or 0 on success. -+ * Efficient way to remove a VMA from the maple tree when the @mas has already -+ * been established and points to the correct location. -+ * Note: the end address is inclusive in the maple tree. - */ --static inline int --munmap_vma_range(struct mm_struct *mm, unsigned long start, unsigned long len, -- struct vm_area_struct **pprev, struct rb_node ***link, -- struct rb_node **parent, struct list_head *uf) -+void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) - { -- -- while (find_vma_links(mm, start, start + len, pprev, link, parent)) -- if (do_munmap(mm, start, len, uf)) -- return -ENOMEM; -- -- return 0; -+ trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); -+ mas->index = vma->vm_start; -+ mas->last = vma->vm_end - 1; -+ mas_store_prealloc(mas, NULL); - } --static unsigned long count_vma_pages_range(struct mm_struct *mm, -- unsigned long addr, unsigned long end) -+ -+/* -+ * vma_mas_szero() - Set a given range to zero. Used when modifying a -+ * vm_area_struct start or end. -+ * -+ * @mm: The struct_mm -+ * @start: The start address to zero -+ * @end: The end address to zero. -+ */ -+static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, -+ unsigned long end) - { -- unsigned long nr_pages = 0; -- struct vm_area_struct *vma; -+ trace_vma_mas_szero(mas->tree, start, end - 1); -+ mas_set_range(mas, start, end - 1); -+ mas_store_prealloc(mas, NULL); -+} - -- /* Find first overlapping mapping */ -- vma = find_vma_intersection(mm, addr, end); -- if (!vma) -- return 0; -+static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) -+{ -+ MA_STATE(mas, &mm->mm_mt, 0, 0); -+ struct address_space *mapping = NULL; - -- nr_pages = (min(end, vma->vm_end) - -- max(addr, vma->vm_start)) >> PAGE_SHIFT; -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) -+ return -ENOMEM; - -- /* Iterate over the rest of the overlaps */ -- for (vma = vma->vm_next; vma; vma = vma->vm_next) { -- unsigned long overlap_len; -+ if (vma->vm_file) { -+ mapping = vma->vm_file->f_mapping; -+ i_mmap_lock_write(mapping); -+ } - -- if (vma->vm_start > end) -- break; -+ vma_mas_store(vma, &mas); - -- overlap_len = min(end, vma->vm_end) - vma->vm_start; -- nr_pages += overlap_len >> PAGE_SHIFT; -+ if (mapping) { -+ __vma_link_file(vma, mapping); -+ i_mmap_unlock_write(mapping); - } - -- return nr_pages; -+ mm->map_count++; -+ validate_mm(mm); -+ return 0; - } - --void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, -- struct rb_node **rb_link, struct rb_node *rb_parent) -+/* -+ * vma_expand - Expand an existing VMA -+ * -+ * @mas: The maple state -+ * @vma: The vma to expand -+ * @start: The start of the vma -+ * @end: The exclusive end of the vma -+ * @pgoff: The page offset of vma -+ * @next: The current of next vma. -+ * -+ * Expand @vma to @start and @end. Can expand off the start and end. Will -+ * expand over @next if it's different from @vma and @end == @next->vm_end. -+ * Checking if the @vma can expand and merge with @next needs to be handled by -+ * the caller. -+ * -+ * Returns: 0 on success -+ */ -+inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, -+ unsigned long start, unsigned long end, pgoff_t pgoff, -+ struct vm_area_struct *next) - { -- /* Update tracking information for the gap following the new vma. */ -- if (vma->vm_next) -- vma_gap_update(vma->vm_next); -- else -- mm->highest_vm_end = vm_end_gap(vma); -+ struct mm_struct *mm = vma->vm_mm; -+ struct address_space *mapping = NULL; -+ struct rb_root_cached *root = NULL; -+ struct anon_vma *anon_vma = vma->anon_vma; -+ struct file *file = vma->vm_file; -+ bool remove_next = false; - -- /* -- * vma->vm_prev wasn't known when we followed the rbtree to find the -- * correct insertion point for that vma. As a result, we could not -- * update the vma vm_rb parents rb_subtree_gap values on the way down. -- * So, we first insert the vma with a zero rb_subtree_gap value -- * (to be consistent with what we did on the way down), and then -- * immediately update the gap to the correct value. Finally we -- * rebalance the rbtree after all augmented values have been set. -- */ -- rb_link_node(&vma->vm_rb, rb_parent, rb_link); -- vma->rb_subtree_gap = 0; -- vma_gap_update(vma); -- vma_rb_insert(vma, &mm->mm_rb); --} -+ if (next && (vma != next) && (end == next->vm_end)) { -+ remove_next = true; -+ if (next->anon_vma && !vma->anon_vma) { -+ int error; - --static void __vma_link_file(struct vm_area_struct *vma) --{ -- struct file *file; -+ anon_vma = next->anon_vma; -+ vma->anon_vma = anon_vma; -+ error = anon_vma_clone(vma, next); -+ if (error) -+ return error; -+ } -+ } -+ -+ /* Not merging but overwriting any part of next is not handled. */ -+ VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start); -+ /* Only handles expanding */ -+ VM_BUG_ON(vma->vm_start < start || vma->vm_end > end); -+ -+ if (mas_preallocate(mas, vma, GFP_KERNEL)) -+ goto nomem; -+ -+ vma_adjust_trans_huge(vma, start, end, 0); - -- file = vma->vm_file; - if (file) { -- struct address_space *mapping = file->f_mapping; -+ mapping = file->f_mapping; -+ root = &mapping->i_mmap; -+ uprobe_munmap(vma, vma->vm_start, vma->vm_end); -+ i_mmap_lock_write(mapping); -+ } - -- if (vma->vm_flags & VM_SHARED) -- mapping_allow_writable(mapping); -+ if (anon_vma) { -+ anon_vma_lock_write(anon_vma); -+ anon_vma_interval_tree_pre_update_vma(vma); -+ } - -+ if (file) { - flush_dcache_mmap_lock(mapping); -- vma_interval_tree_insert(vma, &mapping->i_mmap); -- flush_dcache_mmap_unlock(mapping); -+ vma_interval_tree_remove(vma, root); - } --} - --static void --__vma_link(struct mm_struct *mm, struct vm_area_struct *vma, -- struct vm_area_struct *prev, struct rb_node **rb_link, -- struct rb_node *rb_parent) --{ -- __vma_link_list(mm, vma, prev); -- __vma_link_rb(mm, vma, rb_link, rb_parent); --} -+ vma->vm_start = start; -+ vma->vm_end = end; -+ vma->vm_pgoff = pgoff; -+ /* Note: mas must be pointing to the expanding VMA */ -+ vma_mas_store(vma, mas); - --static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, -- struct vm_area_struct *prev, struct rb_node **rb_link, -- struct rb_node *rb_parent) --{ -- struct address_space *mapping = NULL; -+ if (file) { -+ vma_interval_tree_insert(vma, root); -+ flush_dcache_mmap_unlock(mapping); -+ } - -- if (vma->vm_file) { -- mapping = vma->vm_file->f_mapping; -- i_mmap_lock_write(mapping); -+ /* Expanding over the next vma */ -+ if (remove_next && file) { -+ __remove_shared_vm_struct(next, file, mapping); - } - -- __vma_link(mm, vma, prev, rb_link, rb_parent); -- __vma_link_file(vma); -+ if (anon_vma) { -+ anon_vma_interval_tree_post_update_vma(vma); -+ anon_vma_unlock_write(anon_vma); -+ } - -- if (mapping) -+ if (file) { - i_mmap_unlock_write(mapping); -+ uprobe_mmap(vma); -+ } - -- mm->map_count++; -- validate_mm(mm); --} -- --/* -- * Helper for vma_adjust() in the split_vma insert case: insert a vma into the -- * mm's list and rbtree. It has already been inserted into the interval tree. -- */ --static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) --{ -- struct vm_area_struct *prev; -- struct rb_node **rb_link, *rb_parent; -+ if (remove_next) { -+ if (file) { -+ uprobe_munmap(next, next->vm_start, next->vm_end); -+ fput(file); -+ } -+ if (next->anon_vma) -+ anon_vma_merge(vma, next); -+ mm->map_count--; -+ mpol_put(vma_policy(next)); -+ vm_area_free(next); -+ } - -- if (find_vma_links(mm, vma->vm_start, vma->vm_end, -- &prev, &rb_link, &rb_parent)) -- BUG(); -- __vma_link(mm, vma, prev, rb_link, rb_parent); -- mm->map_count++; --} -+ validate_mm(mm); -+ return 0; - --static __always_inline void __vma_unlink(struct mm_struct *mm, -- struct vm_area_struct *vma, -- struct vm_area_struct *ignore) --{ -- vma_rb_erase_ignore(vma, &mm->mm_rb, ignore); -- __vma_unlink_list(mm, vma); -- /* Kill the cache */ -- vmacache_invalidate(mm); -+nomem: -+ return -ENOMEM; - } - - /* -@@ -701,18 +618,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - struct vm_area_struct *expand) - { - struct mm_struct *mm = vma->vm_mm; -- struct vm_area_struct *next = vma->vm_next, *orig_vma = vma; -+ struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); -+ struct vm_area_struct *orig_vma = vma; - struct address_space *mapping = NULL; - struct rb_root_cached *root = NULL; - struct anon_vma *anon_vma = NULL; - struct file *file = vma->vm_file; -- bool start_changed = false, end_changed = false; -+ bool vma_changed = false; - long adjust_next = 0; - int remove_next = 0; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); -+ struct vm_area_struct *exporter = NULL, *importer = NULL; - - if (next && !insert) { -- struct vm_area_struct *exporter = NULL, *importer = NULL; -- - if (end >= next->vm_end) { - /* - * vma expands, overlapping all the next, and -@@ -741,10 +659,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - * remove_next == 1 is case 1 or 7. - */ - remove_next = 1 + (end > next->vm_end); -+ if (remove_next == 2) -+ next_next = find_vma(mm, next->vm_end); -+ - VM_WARN_ON(remove_next == 2 && -- end != next->vm_next->vm_end); -- /* trim end to next, for case 6 first pass */ -- end = next->vm_end; -+ end != next_next->vm_end); - } - - exporter = next; -@@ -755,7 +674,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - * next, if the vma overlaps with it. - */ - if (remove_next == 2 && !next->anon_vma) -- exporter = next->vm_next; -+ exporter = next_next; - - } else if (end > next->vm_start) { - /* -@@ -792,9 +711,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - return error; - } - } --again: -- vma_adjust_trans_huge(orig_vma, start, end, adjust_next); - -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) -+ return -ENOMEM; -+ -+ vma_adjust_trans_huge(orig_vma, start, end, adjust_next); - if (file) { - mapping = file->f_mapping; - root = &mapping->i_mmap; -@@ -804,14 +725,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - uprobe_munmap(next, next->vm_start, next->vm_end); - - i_mmap_lock_write(mapping); -- if (insert) { -+ if (insert && insert->vm_file) { - /* - * Put into interval tree now, so instantiated pages - * are visible to arm/parisc __flush_dcache_page - * throughout; but we cannot insert into address - * space until vma start or end is updated. - */ -- __vma_link_file(insert); -+ __vma_link_file(insert, insert->vm_file->f_mapping); - } - } - -@@ -835,17 +756,37 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - } - - if (start != vma->vm_start) { -+ if ((vma->vm_start < start) && -+ (!insert || (insert->vm_end != start))) { -+ vma_mas_szero(&mas, vma->vm_start, start); -+ VM_WARN_ON(insert && insert->vm_start > vma->vm_start); -+ } else { -+ vma_changed = true; -+ } - vma->vm_start = start; -- start_changed = true; - } - if (end != vma->vm_end) { -+ if (vma->vm_end > end) { -+ if (!insert || (insert->vm_start != end)) { -+ vma_mas_szero(&mas, end, vma->vm_end); -+ mas_reset(&mas); -+ VM_WARN_ON(insert && -+ insert->vm_end < vma->vm_end); -+ } -+ } else { -+ vma_changed = true; -+ } - vma->vm_end = end; -- end_changed = true; - } -+ -+ if (vma_changed) -+ vma_mas_store(vma, &mas); -+ - vma->vm_pgoff = pgoff; - if (adjust_next) { - next->vm_start += adjust_next; - next->vm_pgoff += adjust_next >> PAGE_SHIFT; -+ vma_mas_store(next, &mas); - } - - if (file) { -@@ -855,42 +796,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - flush_dcache_mmap_unlock(mapping); - } - -- if (remove_next) { -- /* -- * vma_merge has merged next into vma, and needs -- * us to remove next before dropping the locks. -- */ -- if (remove_next != 3) -- __vma_unlink(mm, next, next); -- else -- /* -- * vma is not before next if they've been -- * swapped. -- * -- * pre-swap() next->vm_start was reduced so -- * tell validate_mm_rb to ignore pre-swap() -- * "next" (which is stored in post-swap() -- * "vma"). -- */ -- __vma_unlink(mm, next, vma); -- if (file) -- __remove_shared_vm_struct(next, file, mapping); -+ if (remove_next && file) { -+ __remove_shared_vm_struct(next, file, mapping); -+ if (remove_next == 2) -+ __remove_shared_vm_struct(next_next, file, mapping); - } else if (insert) { - /* - * split_vma has split insert from vma, and needs - * us to insert it before dropping the locks - * (it may either follow vma or precede it). - */ -- __insert_vm_struct(mm, insert); -- } else { -- if (start_changed) -- vma_gap_update(vma); -- if (end_changed) { -- if (!next) -- mm->highest_vm_end = vm_end_gap(vma); -- else if (!adjust_next) -- vma_gap_update(next); -- } -+ mas_reset(&mas); -+ vma_mas_store(insert, &mas); -+ mm->map_count++; - } - - if (anon_vma) { -@@ -909,6 +827,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - } - - if (remove_next) { -+again: - if (file) { - uprobe_munmap(next, next->vm_start, next->vm_end); - fput(file); -@@ -917,66 +836,24 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - anon_vma_merge(vma, next); - mm->map_count--; - mpol_put(vma_policy(next)); -+ if (remove_next != 2) -+ BUG_ON(vma->vm_end < next->vm_end); - vm_area_free(next); -+ - /* - * In mprotect's case 6 (see comments on vma_merge), -- * we must remove another next too. It would clutter -- * up the code too much to do both in one go. -+ * we must remove next_next too. - */ -- if (remove_next != 3) { -- /* -- * If "next" was removed and vma->vm_end was -- * expanded (up) over it, in turn -- * "next->vm_prev->vm_end" changed and the -- * "vma->vm_next" gap must be updated. -- */ -- next = vma->vm_next; -- } else { -- /* -- * For the scope of the comment "next" and -- * "vma" considered pre-swap(): if "vma" was -- * removed, next->vm_start was expanded (down) -- * over it and the "next" gap must be updated. -- * Because of the swap() the post-swap() "vma" -- * actually points to pre-swap() "next" -- * (post-swap() "next" as opposed is now a -- * dangling pointer). -- */ -- next = vma; -- } - if (remove_next == 2) { - remove_next = 1; -- end = next->vm_end; -+ next = next_next; - goto again; - } -- else if (next) -- vma_gap_update(next); -- else { -- /* -- * If remove_next == 2 we obviously can't -- * reach this path. -- * -- * If remove_next == 3 we can't reach this -- * path because pre-swap() next is always not -- * NULL. pre-swap() "next" is not being -- * removed and its next->vm_end is not altered -- * (and furthermore "end" already matches -- * next->vm_end in remove_next == 3). -- * -- * We reach this only in the remove_next == 1 -- * case if the "next" vma that was removed was -- * the highest vma of the mm. However in such -- * case next->vm_end == "end" and the extended -- * "vma" has vma->vm_end == next->vm_end so -- * mm->highest_vm_end doesn't need any update -- * in remove_next == 1 case. -- */ -- VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); -- } - } - if (insert && file) - uprobe_mmap(insert); - -+ mas_destroy(&mas); - validate_mm(mm); - - return 0; -@@ -1138,10 +1015,10 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, - if (vm_flags & VM_SPECIAL) - return NULL; - -- next = vma_next(mm, prev); -+ next = find_vma(mm, prev ? prev->vm_end : 0); - area = next; - if (area && area->vm_end == end) /* cases 6, 7, 8 */ -- next = next->vm_next; -+ next = find_vma(mm, next->vm_end); - - /* verify some invariant that must be enforced by the caller */ - VM_WARN_ON(prev && addr <= prev->vm_start); -@@ -1275,18 +1152,24 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_ - */ - struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) - { -+ MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); - struct anon_vma *anon_vma = NULL; -+ struct vm_area_struct *prev, *next; - - /* Try next first. */ -- if (vma->vm_next) { -- anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next); -+ next = mas_walk(&mas); -+ if (next) { -+ anon_vma = reusable_anon_vma(next, vma, next); - if (anon_vma) - return anon_vma; - } - -+ prev = mas_prev(&mas, 0); -+ VM_BUG_ON_VMA(prev != vma, vma); -+ prev = mas_prev(&mas, 0); - /* Try prev next. */ -- if (vma->vm_prev) -- anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma); -+ if (prev) -+ anon_vma = reusable_anon_vma(prev, prev, vma); - - /* - * We might reach here with anon_vma == NULL if we can't find -@@ -1375,6 +1258,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, - vm_flags_t vm_flags; - int pkey = 0; - -+ validate_mm(mm); - *populate = 0; - - if (!len) -@@ -1678,388 +1562,63 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) - return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; - } - --unsigned long mmap_region(struct file *file, unsigned long addr, -- unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, -- struct list_head *uf) -+/** -+ * unmapped_area() - Find an area between the low_limit and the high_limit with -+ * the correct alignment and offset, all from @info. Note: current->mm is used -+ * for the search. -+ * -+ * @info: The unmapped area information including the range (low_limit - -+ * hight_limit), the alignment offset and mask. -+ * -+ * Return: A memory address or -ENOMEM. -+ */ -+static unsigned long unmapped_area(struct vm_unmapped_area_info *info) - { -- struct mm_struct *mm = current->mm; -- struct vm_area_struct *vma, *prev, *merge; -- int error; -- struct rb_node **rb_link, *rb_parent; -- unsigned long charged = 0; -- -- /* Check against address space limit. */ -- if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { -- unsigned long nr_pages; -- -- /* -- * MAP_FIXED may remove pages of mappings that intersects with -- * requested mapping. Account for the pages it would unmap. -- */ -- nr_pages = count_vma_pages_range(mm, addr, addr + len); -- -- if (!may_expand_vm(mm, vm_flags, -- (len >> PAGE_SHIFT) - nr_pages)) -- return -ENOMEM; -- } -- -- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ -- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) -- return -ENOMEM; -- /* -- * Private writable mapping: check memory availability -- */ -- if (accountable_mapping(file, vm_flags)) { -- charged = len >> PAGE_SHIFT; -- if (security_vm_enough_memory_mm(mm, charged)) -- return -ENOMEM; -- vm_flags |= VM_ACCOUNT; -- } -- -- /* -- * Can we just expand an old mapping? -- */ -- vma = vma_merge(mm, prev, addr, addr + len, vm_flags, -- NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); -- if (vma) -- goto out; -- -- /* -- * Determine the object being mapped and call the appropriate -- * specific mapper. the address has already been validated, but -- * not unmapped, but the maps are removed from the list. -- */ -- vma = vm_area_alloc(mm); -- if (!vma) { -- error = -ENOMEM; -- goto unacct_error; -- } -- -- vma->vm_start = addr; -- vma->vm_end = addr + len; -- vma->vm_flags = vm_flags; -- vma->vm_page_prot = vm_get_page_prot(vm_flags); -- vma->vm_pgoff = pgoff; -- -- if (file) { -- if (vm_flags & VM_SHARED) { -- error = mapping_map_writable(file->f_mapping); -- if (error) -- goto free_vma; -- } -- -- vma->vm_file = get_file(file); -- error = call_mmap(file, vma); -- if (error) -- goto unmap_and_free_vma; -- -- /* Can addr have changed?? -- * -- * Answer: Yes, several device drivers can do it in their -- * f_op->mmap method. -DaveM -- * Bug: If addr is changed, prev, rb_link, rb_parent should -- * be updated for vma_link() -- */ -- WARN_ON_ONCE(addr != vma->vm_start); -- -- addr = vma->vm_start; -- -- /* If vm_flags changed after call_mmap(), we should try merge vma again -- * as we may succeed this time. -- */ -- if (unlikely(vm_flags != vma->vm_flags && prev)) { -- merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, -- NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); -- if (merge) { -- /* ->mmap() can change vma->vm_file and fput the original file. So -- * fput the vma->vm_file here or we would add an extra fput for file -- * and cause general protection fault ultimately. -- */ -- fput(vma->vm_file); -- vm_area_free(vma); -- vma = merge; -- /* Update vm_flags to pick up the change. */ -- vm_flags = vma->vm_flags; -- goto unmap_writable; -- } -- } -- -- vm_flags = vma->vm_flags; -- } else if (vm_flags & VM_SHARED) { -- error = shmem_zero_setup(vma); -- if (error) -- goto free_vma; -- } else { -- vma_set_anonymous(vma); -- } -- -- /* Allow architectures to sanity-check the vm_flags */ -- if (!arch_validate_flags(vma->vm_flags)) { -- error = -EINVAL; -- if (file) -- goto unmap_and_free_vma; -- else -- goto free_vma; -- } -- -- vma_link(mm, vma, prev, rb_link, rb_parent); -- -- /* -- * vma_merge() calls khugepaged_enter_vma() either, the below -- * call covers the non-merge case. -- */ -- khugepaged_enter_vma(vma, vma->vm_flags); -- -- /* Once vma denies write, undo our temporary denial count */ --unmap_writable: -- if (file && vm_flags & VM_SHARED) -- mapping_unmap_writable(file->f_mapping); -- file = vma->vm_file; --out: -- perf_event_mmap(vma); -- -- vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); -- if (vm_flags & VM_LOCKED) { -- if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || -- is_vm_hugetlb_page(vma) || -- vma == get_gate_vma(current->mm)) -- vma->vm_flags &= VM_LOCKED_CLEAR_MASK; -- else -- mm->locked_vm += (len >> PAGE_SHIFT); -- } -+ unsigned long length, gap; - -- if (file) -- uprobe_mmap(vma); -- -- /* -- * New (or expanded) vma always get soft dirty status. -- * Otherwise user-space soft-dirty page tracker won't -- * be able to distinguish situation when vma area unmapped, -- * then new mapped in-place (which must be aimed as -- * a completely new data area). -- */ -- vma->vm_flags |= VM_SOFTDIRTY; -- -- vma_set_page_prot(vma); -- -- return addr; -- --unmap_and_free_vma: -- fput(vma->vm_file); -- vma->vm_file = NULL; -- -- /* Undo any partial mapping done by a device driver. */ -- unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); -- if (vm_flags & VM_SHARED) -- mapping_unmap_writable(file->f_mapping); --free_vma: -- vm_area_free(vma); --unacct_error: -- if (charged) -- vm_unacct_memory(charged); -- return error; --} -- --static unsigned long unmapped_area(struct vm_unmapped_area_info *info) --{ -- /* -- * We implement the search by looking for an rbtree node that -- * immediately follows a suitable gap. That is, -- * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length; -- * - gap_end = vma->vm_start >= info->low_limit + length; -- * - gap_end - gap_start >= length -- */ -- -- struct mm_struct *mm = current->mm; -- struct vm_area_struct *vma; -- unsigned long length, low_limit, high_limit, gap_start, gap_end; -+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); - - /* Adjust search length to account for worst case alignment overhead */ - length = info->length + info->align_mask; - if (length < info->length) - return -ENOMEM; - -- /* Adjust search limits by the desired length */ -- if (info->high_limit < length) -- return -ENOMEM; -- high_limit = info->high_limit - length; -- -- if (info->low_limit > high_limit) -- return -ENOMEM; -- low_limit = info->low_limit + length; -- -- /* Check if rbtree root looks promising */ -- if (RB_EMPTY_ROOT(&mm->mm_rb)) -- goto check_highest; -- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); -- if (vma->rb_subtree_gap < length) -- goto check_highest; -- -- while (true) { -- /* Visit left subtree if it looks promising */ -- gap_end = vm_start_gap(vma); -- if (gap_end >= low_limit && vma->vm_rb.rb_left) { -- struct vm_area_struct *left = -- rb_entry(vma->vm_rb.rb_left, -- struct vm_area_struct, vm_rb); -- if (left->rb_subtree_gap >= length) { -- vma = left; -- continue; -- } -- } -- -- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; --check_current: -- /* Check if current node has a suitable gap */ -- if (gap_start > high_limit) -- return -ENOMEM; -- if (gap_end >= low_limit && -- gap_end > gap_start && gap_end - gap_start >= length) -- goto found; -- -- /* Visit right subtree if it looks promising */ -- if (vma->vm_rb.rb_right) { -- struct vm_area_struct *right = -- rb_entry(vma->vm_rb.rb_right, -- struct vm_area_struct, vm_rb); -- if (right->rb_subtree_gap >= length) { -- vma = right; -- continue; -- } -- } -- -- /* Go back up the rbtree to find next candidate node */ -- while (true) { -- struct rb_node *prev = &vma->vm_rb; -- if (!rb_parent(prev)) -- goto check_highest; -- vma = rb_entry(rb_parent(prev), -- struct vm_area_struct, vm_rb); -- if (prev == vma->vm_rb.rb_left) { -- gap_start = vm_end_gap(vma->vm_prev); -- gap_end = vm_start_gap(vma); -- goto check_current; -- } -- } -- } -- --check_highest: -- /* Check highest gap, which does not precede any rbtree node */ -- gap_start = mm->highest_vm_end; -- gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */ -- if (gap_start > high_limit) -+ if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, -+ length)) - return -ENOMEM; - --found: -- /* We found a suitable gap. Clip it with the original low_limit. */ -- if (gap_start < info->low_limit) -- gap_start = info->low_limit; -- -- /* Adjust gap address to the desired alignment */ -- gap_start += (info->align_offset - gap_start) & info->align_mask; -- -- VM_BUG_ON(gap_start + info->length > info->high_limit); -- VM_BUG_ON(gap_start + info->length > gap_end); -- return gap_start; -+ gap = mas.index; -+ gap += (info->align_offset - gap) & info->align_mask; -+ return gap; - } - -+/** -+ * unmapped_area_topdown() - Find an area between the low_limit and the -+ * high_limit with * the correct alignment and offset at the highest available -+ * address, all from @info. Note: current->mm is used for the search. -+ * -+ * @info: The unmapped area information including the range (low_limit - -+ * hight_limit), the alignment offset and mask. -+ * -+ * Return: A memory address or -ENOMEM. -+ */ - static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) - { -- struct mm_struct *mm = current->mm; -- struct vm_area_struct *vma; -- unsigned long length, low_limit, high_limit, gap_start, gap_end; -+ unsigned long length, gap; - -+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); - /* Adjust search length to account for worst case alignment overhead */ - length = info->length + info->align_mask; - if (length < info->length) - return -ENOMEM; - -- /* -- * Adjust search limits by the desired length. -- * See implementation comment at top of unmapped_area(). -- */ -- gap_end = info->high_limit; -- if (gap_end < length) -- return -ENOMEM; -- high_limit = gap_end - length; -- -- if (info->low_limit > high_limit) -+ if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, -+ length)) - return -ENOMEM; -- low_limit = info->low_limit + length; - -- /* Check highest gap, which does not precede any rbtree node */ -- gap_start = mm->highest_vm_end; -- if (gap_start <= high_limit) -- goto found_highest; -- -- /* Check if rbtree root looks promising */ -- if (RB_EMPTY_ROOT(&mm->mm_rb)) -- return -ENOMEM; -- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); -- if (vma->rb_subtree_gap < length) -- return -ENOMEM; -- -- while (true) { -- /* Visit right subtree if it looks promising */ -- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; -- if (gap_start <= high_limit && vma->vm_rb.rb_right) { -- struct vm_area_struct *right = -- rb_entry(vma->vm_rb.rb_right, -- struct vm_area_struct, vm_rb); -- if (right->rb_subtree_gap >= length) { -- vma = right; -- continue; -- } -- } -- --check_current: -- /* Check if current node has a suitable gap */ -- gap_end = vm_start_gap(vma); -- if (gap_end < low_limit) -- return -ENOMEM; -- if (gap_start <= high_limit && -- gap_end > gap_start && gap_end - gap_start >= length) -- goto found; -- -- /* Visit left subtree if it looks promising */ -- if (vma->vm_rb.rb_left) { -- struct vm_area_struct *left = -- rb_entry(vma->vm_rb.rb_left, -- struct vm_area_struct, vm_rb); -- if (left->rb_subtree_gap >= length) { -- vma = left; -- continue; -- } -- } -- -- /* Go back up the rbtree to find next candidate node */ -- while (true) { -- struct rb_node *prev = &vma->vm_rb; -- if (!rb_parent(prev)) -- return -ENOMEM; -- vma = rb_entry(rb_parent(prev), -- struct vm_area_struct, vm_rb); -- if (prev == vma->vm_rb.rb_right) { -- gap_start = vma->vm_prev ? -- vm_end_gap(vma->vm_prev) : 0; -- goto check_current; -- } -- } -- } -- --found: -- /* We found a suitable gap. Clip it with the original high_limit. */ -- if (gap_end > info->high_limit) -- gap_end = info->high_limit; -- --found_highest: -- /* Compute highest gap address at the desired alignment */ -- gap_end -= info->length; -- gap_end -= (gap_end - info->align_offset) & info->align_mask; -- -- VM_BUG_ON(gap_end < info->low_limit); -- VM_BUG_ON(gap_end < gap_start); -- return gap_end; -+ gap = mas.last + 1 - info->length; -+ gap -= (gap - info->align_offset) & info->align_mask; -+ return gap; - } - - /* -@@ -2249,58 +1808,67 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - - EXPORT_SYMBOL(get_unmapped_area); - --/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ --struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) -+/** -+ * find_vma_intersection() - Look up the first VMA which intersects the interval -+ * @mm: The process address space. -+ * @start_addr: The inclusive start user address. -+ * @end_addr: The exclusive end user address. -+ * -+ * Returns: The first VMA within the provided range, %NULL otherwise. Assumes -+ * start_addr < end_addr. -+ */ -+struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, -+ unsigned long start_addr, -+ unsigned long end_addr) - { -- struct rb_node *rb_node; -- struct vm_area_struct *vma; -+ unsigned long index = start_addr; - - mmap_assert_locked(mm); -- /* Check the cache first. */ -- vma = vmacache_find(mm, addr); -- if (likely(vma)) -- return vma; -- -- rb_node = mm->mm_rb.rb_node; -- -- while (rb_node) { -- struct vm_area_struct *tmp; -- -- tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb); -+ return mt_find(&mm->mm_mt, &index, end_addr - 1); -+} -+EXPORT_SYMBOL(find_vma_intersection); - -- if (tmp->vm_end > addr) { -- vma = tmp; -- if (tmp->vm_start <= addr) -- break; -- rb_node = rb_node->rb_left; -- } else -- rb_node = rb_node->rb_right; -- } -+/** -+ * find_vma() - Find the VMA for a given address, or the next VMA. -+ * @mm: The mm_struct to check -+ * @addr: The address -+ * -+ * Returns: The VMA associated with addr, or the next VMA. -+ * May return %NULL in the case of no VMA at addr or above. -+ */ -+struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) -+{ -+ unsigned long index = addr; - -- if (vma) -- vmacache_update(addr, vma); -- return vma; -+ mmap_assert_locked(mm); -+ return mt_find(&mm->mm_mt, &index, ULONG_MAX); - } -- - EXPORT_SYMBOL(find_vma); - --/* -- * Same as find_vma, but also return a pointer to the previous VMA in *pprev. -+/** -+ * find_vma_prev() - Find the VMA for a given address, or the next vma and -+ * set %pprev to the previous VMA, if any. -+ * @mm: The mm_struct to check -+ * @addr: The address -+ * @pprev: The pointer to set to the previous VMA -+ * -+ * Note that RCU lock is missing here since the external mmap_lock() is used -+ * instead. -+ * -+ * Returns: The VMA associated with @addr, or the next vma. -+ * May return %NULL in the case of no vma at addr or above. - */ - struct vm_area_struct * - find_vma_prev(struct mm_struct *mm, unsigned long addr, - struct vm_area_struct **pprev) - { - struct vm_area_struct *vma; -+ MA_STATE(mas, &mm->mm_mt, addr, addr); - -- vma = find_vma(mm, addr); -- if (vma) { -- *pprev = vma->vm_prev; -- } else { -- struct rb_node *rb_node = rb_last(&mm->mm_rb); -- -- *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL; -- } -+ vma = mas_walk(&mas); -+ *pprev = mas_prev(&mas, 0); -+ if (!vma) -+ vma = mas_next(&mas, ULONG_MAX); - return vma; - } - -@@ -2354,6 +1922,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) - struct vm_area_struct *next; - unsigned long gap_addr; - int error = 0; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - if (!(vma->vm_flags & VM_GROWSUP)) - return -EFAULT; -@@ -2371,16 +1940,21 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) - if (gap_addr < address || gap_addr > TASK_SIZE) - gap_addr = TASK_SIZE; - -- next = vma->vm_next; -- if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { -+ next = find_vma_intersection(mm, vma->vm_end, gap_addr); -+ if (next && vma_is_accessible(next)) { - if (!(next->vm_flags & VM_GROWSUP)) - return -ENOMEM; - /* Check that both stack segments have the same anon_vma? */ - } - -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) -+ return -ENOMEM; -+ - /* We must make sure the anon_vma is allocated. */ -- if (unlikely(anon_vma_prepare(vma))) -+ if (unlikely(anon_vma_prepare(vma))) { -+ mas_destroy(&mas); - return -ENOMEM; -+ } - - /* - * vma->vm_start/vm_end cannot change under us because the caller -@@ -2401,15 +1975,13 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) - error = acct_stack_growth(vma, size, grow); - if (!error) { - /* -- * vma_gap_update() doesn't support concurrent -- * updates, but we only hold a shared mmap_lock -- * lock here, so we need to protect against -- * concurrent vma expansions. -- * anon_vma_lock_write() doesn't help here, as -- * we don't guarantee that all growable vmas -- * in a mm share the same root anon vma. -- * So, we reuse mm->page_table_lock to guard -- * against concurrent vma expansions. -+ * We only hold a shared mmap_lock lock here, so -+ * we need to protect against concurrent vma -+ * expansions. anon_vma_lock_write() doesn't -+ * help here, as we don't guarantee that all -+ * growable vmas in a mm share the same root -+ * anon vma. So, we reuse mm->page_table_lock -+ * to guard against concurrent vma expansions. - */ - spin_lock(&mm->page_table_lock); - if (vma->vm_flags & VM_LOCKED) -@@ -2417,11 +1989,9 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) - vm_stat_account(mm, vma->vm_flags, grow); - anon_vma_interval_tree_pre_update_vma(vma); - vma->vm_end = address; -+ /* Overwrite old entry in mtree. */ -+ vma_mas_store(vma, &mas); - anon_vma_interval_tree_post_update_vma(vma); -- if (vma->vm_next) -- vma_gap_update(vma->vm_next); -- else -- mm->highest_vm_end = vm_end_gap(vma); - spin_unlock(&mm->page_table_lock); - - perf_event_mmap(vma); -@@ -2430,7 +2000,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) - } - anon_vma_unlock_write(vma->anon_vma); - khugepaged_enter_vma(vma, vma->vm_flags); -- validate_mm(mm); -+ mas_destroy(&mas); - return error; - } - #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ -@@ -2438,10 +2008,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) - /* - * vma is the first one with address < vma->vm_start. Have to extend vma. - */ --int expand_downwards(struct vm_area_struct *vma, -- unsigned long address) -+int expand_downwards(struct vm_area_struct *vma, unsigned long address) - { - struct mm_struct *mm = vma->vm_mm; -+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); - struct vm_area_struct *prev; - int error = 0; - -@@ -2450,7 +2020,7 @@ int expand_downwards(struct vm_area_struct *vma, - return -EPERM; - - /* Enforce stack_guard_gap */ -- prev = vma->vm_prev; -+ prev = mas_prev(&mas, 0); - /* Check that both stack segments have the same anon_vma? */ - if (prev && !(prev->vm_flags & VM_GROWSDOWN) && - vma_is_accessible(prev)) { -@@ -2458,9 +2028,14 @@ int expand_downwards(struct vm_area_struct *vma, - return -ENOMEM; - } - -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) -+ return -ENOMEM; -+ - /* We must make sure the anon_vma is allocated. */ -- if (unlikely(anon_vma_prepare(vma))) -+ if (unlikely(anon_vma_prepare(vma))) { -+ mas_destroy(&mas); - return -ENOMEM; -+ } - - /* - * vma->vm_start/vm_end cannot change under us because the caller -@@ -2481,15 +2056,13 @@ int expand_downwards(struct vm_area_struct *vma, - error = acct_stack_growth(vma, size, grow); - if (!error) { - /* -- * vma_gap_update() doesn't support concurrent -- * updates, but we only hold a shared mmap_lock -- * lock here, so we need to protect against -- * concurrent vma expansions. -- * anon_vma_lock_write() doesn't help here, as -- * we don't guarantee that all growable vmas -- * in a mm share the same root anon vma. -- * So, we reuse mm->page_table_lock to guard -- * against concurrent vma expansions. -+ * We only hold a shared mmap_lock lock here, so -+ * we need to protect against concurrent vma -+ * expansions. anon_vma_lock_write() doesn't -+ * help here, as we don't guarantee that all -+ * growable vmas in a mm share the same root -+ * anon vma. So, we reuse mm->page_table_lock -+ * to guard against concurrent vma expansions. - */ - spin_lock(&mm->page_table_lock); - if (vma->vm_flags & VM_LOCKED) -@@ -2498,8 +2071,9 @@ int expand_downwards(struct vm_area_struct *vma, - anon_vma_interval_tree_pre_update_vma(vma); - vma->vm_start = address; - vma->vm_pgoff -= grow; -+ /* Overwrite old entry in mtree. */ -+ vma_mas_store(vma, &mas); - anon_vma_interval_tree_post_update_vma(vma); -- vma_gap_update(vma); - spin_unlock(&mm->page_table_lock); - - perf_event_mmap(vma); -@@ -2508,7 +2082,7 @@ int expand_downwards(struct vm_area_struct *vma, - } - anon_vma_unlock_write(vma->anon_vma); - khugepaged_enter_vma(vma, vma->vm_flags); -- validate_mm(mm); -+ mas_destroy(&mas); - return error; - } - -@@ -2581,25 +2155,26 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) - EXPORT_SYMBOL_GPL(find_extend_vma); - - /* -- * Ok - we have the memory areas we should free on the vma list, -- * so release them, and do the vma updates. -+ * Ok - we have the memory areas we should free on a maple tree so release them, -+ * and do the vma updates. - * - * Called with the mm semaphore held. - */ --static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) -+static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) - { - unsigned long nr_accounted = 0; -+ struct vm_area_struct *vma; - - /* Update high watermark before we lower total_vm */ - update_hiwater_vm(mm); -- do { -+ mas_for_each(mas, vma, ULONG_MAX) { - long nrpages = vma_pages(vma); - - if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += nrpages; - vm_stat_account(mm, vma->vm_flags, -nrpages); -- vma = remove_vma(vma); -- } while (vma); -+ remove_vma(vma); -+ } - vm_unacct_memory(nr_accounted); - validate_mm(mm); - } -@@ -2609,75 +2184,32 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) - * - * Called with the mm semaphore held. - */ --static void unmap_region(struct mm_struct *mm, -+static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, - struct vm_area_struct *vma, struct vm_area_struct *prev, -+ struct vm_area_struct *next, - unsigned long start, unsigned long end) - { -- struct vm_area_struct *next = vma_next(mm, prev); - struct mmu_gather tlb; - - lru_add_drain(); - tlb_gather_mmu(&tlb, mm); - update_hiwater_rss(mm); -- unmap_vmas(&tlb, vma, start, end); -- free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, -+ unmap_vmas(&tlb, mt, vma, start, end); -+ free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, - next ? next->vm_start : USER_PGTABLES_CEILING); - tlb_finish_mmu(&tlb); - } - - /* -- * Create a list of vma's touched by the unmap, removing them from the mm's -- * vma list as we go.. -+ * __split_vma() bypasses sysctl_max_map_count checking. We use this where it -+ * has already been checked or doesn't make sense to fail. - */ --static bool --detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, -- struct vm_area_struct *prev, unsigned long end) -+int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, -+ unsigned long addr, int new_below) - { -- struct vm_area_struct **insertion_point; -- struct vm_area_struct *tail_vma = NULL; -- -- insertion_point = (prev ? &prev->vm_next : &mm->mmap); -- vma->vm_prev = NULL; -- do { -- vma_rb_erase(vma, &mm->mm_rb); -- if (vma->vm_flags & VM_LOCKED) -- mm->locked_vm -= vma_pages(vma); -- mm->map_count--; -- tail_vma = vma; -- vma = vma->vm_next; -- } while (vma && vma->vm_start < end); -- *insertion_point = vma; -- if (vma) { -- vma->vm_prev = prev; -- vma_gap_update(vma); -- } else -- mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; -- tail_vma->vm_next = NULL; -- -- /* Kill the cache */ -- vmacache_invalidate(mm); -- -- /* -- * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or -- * VM_GROWSUP VMA. Such VMAs can change their size under -- * down_read(mmap_lock) and collide with the VMA we are about to unmap. -- */ -- if (vma && (vma->vm_flags & VM_GROWSDOWN)) -- return false; -- if (prev && (prev->vm_flags & VM_GROWSUP)) -- return false; -- return true; --} -- --/* -- * __split_vma() bypasses sysctl_max_map_count checking. We use this where it -- * has already been checked or doesn't make sense to fail. -- */ --int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, -- unsigned long addr, int new_below) --{ -- struct vm_area_struct *new; -- int err; -+ struct vm_area_struct *new; -+ int err; -+ validate_mm_mt(mm); - - if (vma->vm_ops && vma->vm_ops->may_split) { - err = vma->vm_ops->may_split(vma, addr); -@@ -2720,6 +2252,9 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - if (!err) - return 0; - -+ /* Avoid vm accounting in close() operation */ -+ new->vm_start = new->vm_end; -+ new->vm_pgoff = 0; - /* Clean everything up if vma_adjust failed. */ - if (new->vm_ops && new->vm_ops->close) - new->vm_ops->close(new); -@@ -2730,6 +2265,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - mpol_put(vma_policy(new)); - out_free_vma: - vm_area_free(new); -+ validate_mm_mt(mm); - return err; - } - -@@ -2746,38 +2282,48 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - return __split_vma(mm, vma, addr, new_below); - } - --/* Munmap is split into 2 main parts -- this part which finds -- * what needs doing, and the areas themselves, which do the -- * work. This now handles partial unmappings. -- * Jeremy Fitzhardinge -- */ --int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, -- struct list_head *uf, bool downgrade) -+static inline int munmap_sidetree(struct vm_area_struct *vma, -+ struct ma_state *mas_detach) - { -- unsigned long end; -- struct vm_area_struct *vma, *prev, *last; -- -- if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) -- return -EINVAL; -+ mas_set_range(mas_detach, vma->vm_start, vma->vm_end - 1); -+ if (mas_store_gfp(mas_detach, vma, GFP_KERNEL)) -+ return -ENOMEM; - -- len = PAGE_ALIGN(len); -- end = start + len; -- if (len == 0) -- return -EINVAL; -+ if (vma->vm_flags & VM_LOCKED) -+ vma->vm_mm->locked_vm -= vma_pages(vma); - -- /* -- * arch_unmap() might do unmaps itself. It must be called -- * and finish any rbtree manipulation before this code -- * runs and also starts to manipulate the rbtree. -- */ -- arch_unmap(mm, start, end); -+ return 0; -+} - -- /* Find the first overlapping VMA where start < vma->vm_end */ -- vma = find_vma_intersection(mm, start, end); -- if (!vma) -- return 0; -- prev = vma->vm_prev; -+/* -+ * do_mas_align_munmap() - munmap the aligned region from @start to @end. -+ * @mas: The maple_state, ideally set up to alter the correct tree location. -+ * @vma: The starting vm_area_struct -+ * @mm: The mm_struct -+ * @start: The aligned start address to munmap. -+ * @end: The aligned end address to munmap. -+ * @uf: The userfaultfd list_head -+ * @downgrade: Set to true to attempt a write downgrade of the mmap_sem -+ * -+ * If @downgrade is true, check return code for potential release of the lock. -+ */ -+static int -+do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, -+ struct mm_struct *mm, unsigned long start, -+ unsigned long end, struct list_head *uf, bool downgrade) -+{ -+ struct vm_area_struct *prev, *next = NULL; -+ struct maple_tree mt_detach; -+ int count = 0; -+ int error = -ENOMEM; -+ MA_STATE(mas_detach, &mt_detach, 0, 0); -+ mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN); -+ mt_set_external_lock(&mt_detach, &mm->mmap_lock); -+ -+ if (mas_preallocate(mas, vma, GFP_KERNEL)) -+ return -ENOMEM; - -+ mas->last = end - 1; - /* - * If we need to split any vma, do it now to save pain later. - * -@@ -2785,8 +2331,9 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - * unmapped vm_area_struct will remain in use: so lower split_vma - * places tmp vma above, and higher split_vma places tmp vma below. - */ -+ -+ /* Does it split the first one? */ - if (start > vma->vm_start) { -- int error; - - /* - * Make sure that map_count on return from munmap() will -@@ -2794,22 +2341,61 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - * its limit temporarily, to help free resources as expected. - */ - if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) -- return -ENOMEM; -+ goto map_count_exceeded; - -+ /* -+ * mas_pause() is not needed since mas->index needs to be set -+ * differently than vma->vm_end anyways. -+ */ - error = __split_vma(mm, vma, start, 0); - if (error) -- return error; -- prev = vma; -+ goto start_split_failed; -+ -+ mas_set(mas, start); -+ vma = mas_walk(mas); - } - -- /* Does it split the last one? */ -- last = find_vma(mm, end); -- if (last && end > last->vm_start) { -- int error = __split_vma(mm, last, end, 1); -+ prev = mas_prev(mas, 0); -+ if (unlikely((!prev))) -+ mas_set(mas, start); -+ -+ /* -+ * Detach a range of VMAs from the mm. Using next as a temp variable as -+ * it is always overwritten. -+ */ -+ mas_for_each(mas, next, end - 1) { -+ /* Does it split the end? */ -+ if (next->vm_end > end) { -+ struct vm_area_struct *split; -+ -+ error = __split_vma(mm, next, end, 1); -+ if (error) -+ goto end_split_failed; -+ -+ mas_set(mas, end); -+ split = mas_prev(mas, 0); -+ error = munmap_sidetree(split, &mas_detach); -+ if (error) -+ goto munmap_sidetree_failed; -+ -+ count++; -+ if (vma == next) -+ vma = split; -+ break; -+ } -+ error = munmap_sidetree(next, &mas_detach); - if (error) -- return error; -+ goto munmap_sidetree_failed; -+ -+ count++; -+#ifdef CONFIG_DEBUG_VM_MAPLE_TREE -+ BUG_ON(next->vm_start < start); -+ BUG_ON(next->vm_start > end); -+#endif - } -- vma = vma_next(mm, prev); -+ -+ if (!next) -+ next = mas_next(mas, ULONG_MAX); - - if (unlikely(uf)) { - /* -@@ -2821,30 +2407,366 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - * split, despite we could. This is unlikely enough - * failure that it's not worth optimizing it for. - */ -- int error = userfaultfd_unmap_prep(vma, start, end, uf); -+ error = userfaultfd_unmap_prep(mm, start, end, uf); -+ - if (error) -- return error; -+ goto userfaultfd_error; -+ } -+ -+ /* Point of no return */ -+ mas_set_range(mas, start, end - 1); -+#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) -+ /* Make sure no VMAs are about to be lost. */ -+ { -+ MA_STATE(test, &mt_detach, start, end - 1); -+ struct vm_area_struct *vma_mas, *vma_test; -+ int test_count = 0; -+ -+ rcu_read_lock(); -+ vma_test = mas_find(&test, end - 1); -+ mas_for_each(mas, vma_mas, end - 1) { -+ BUG_ON(vma_mas != vma_test); -+ test_count++; -+ vma_test = mas_next(&test, end - 1); -+ } -+ rcu_read_unlock(); -+ BUG_ON(count != test_count); -+ mas_set_range(mas, start, end - 1); -+ } -+#endif -+ mas_store_prealloc(mas, NULL); -+ mm->map_count -= count; -+ /* -+ * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or -+ * VM_GROWSUP VMA. Such VMAs can change their size under -+ * down_read(mmap_lock) and collide with the VMA we are about to unmap. -+ */ -+ if (downgrade) { -+ if (next && (next->vm_flags & VM_GROWSDOWN)) -+ downgrade = false; -+ else if (prev && (prev->vm_flags & VM_GROWSUP)) -+ downgrade = false; -+ else -+ mmap_write_downgrade(mm); - } - -- /* Detach vmas from rbtree */ -- if (!detach_vmas_to_be_unmapped(mm, vma, prev, end)) -- downgrade = false; -+ unmap_region(mm, &mt_detach, vma, prev, next, start, end); -+ /* Statistics and freeing VMAs */ -+ mas_set(&mas_detach, start); -+ remove_mt(mm, &mas_detach); -+ __mt_destroy(&mt_detach); - -- if (downgrade) -- mmap_write_downgrade(mm); - -- unmap_region(mm, vma, prev, start, end); -+ validate_mm(mm); -+ return downgrade ? 1 : 0; - -- /* Fix up all other VM information */ -- remove_vma_list(mm, vma); -+userfaultfd_error: -+munmap_sidetree_failed: -+end_split_failed: -+ __mt_destroy(&mt_detach); -+start_split_failed: -+map_count_exceeded: -+ mas_destroy(mas); -+ return error; -+} - -- return downgrade ? 1 : 0; -+/* -+ * do_mas_munmap() - munmap a given range. -+ * @mas: The maple state -+ * @mm: The mm_struct -+ * @start: The start address to munmap -+ * @len: The length of the range to munmap -+ * @uf: The userfaultfd list_head -+ * @downgrade: set to true if the user wants to attempt to write_downgrade the -+ * mmap_sem -+ * -+ * This function takes a @mas that is either pointing to the previous VMA or set -+ * to MA_START and sets it up to remove the mapping(s). The @len will be -+ * aligned and any arch_unmap work will be preformed. -+ * -+ * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise. -+ */ -+int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, -+ unsigned long start, size_t len, struct list_head *uf, -+ bool downgrade) -+{ -+ unsigned long end; -+ struct vm_area_struct *vma; -+ -+ if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) -+ return -EINVAL; -+ -+ end = start + PAGE_ALIGN(len); -+ if (end == start) -+ return -EINVAL; -+ -+ /* arch_unmap() might do unmaps itself. */ -+ arch_unmap(mm, start, end); -+ -+ /* Find the first overlapping VMA */ -+ vma = mas_find(mas, end - 1); -+ if (!vma) -+ return 0; -+ -+ return do_mas_align_munmap(mas, vma, mm, start, end, uf, downgrade); - } - -+/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. -+ * @mm: The mm_struct -+ * @start: The start address to munmap -+ * @len: The length to be munmapped. -+ * @uf: The userfaultfd list_head -+ */ - int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - struct list_head *uf) - { -- return __do_munmap(mm, start, len, uf, false); -+ MA_STATE(mas, &mm->mm_mt, start, start); -+ -+ return do_mas_munmap(&mas, mm, start, len, uf, false); -+} -+ -+unsigned long mmap_region(struct file *file, unsigned long addr, -+ unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, -+ struct list_head *uf) -+{ -+ struct mm_struct *mm = current->mm; -+ struct vm_area_struct *vma = NULL; -+ struct vm_area_struct *next, *prev, *merge; -+ pgoff_t pglen = len >> PAGE_SHIFT; -+ unsigned long charged = 0; -+ unsigned long end = addr + len; -+ unsigned long merge_start = addr, merge_end = end; -+ pgoff_t vm_pgoff; -+ int error; -+ MA_STATE(mas, &mm->mm_mt, addr, end - 1); -+ -+ /* Check against address space limit. */ -+ if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { -+ unsigned long nr_pages; -+ -+ /* -+ * MAP_FIXED may remove pages of mappings that intersects with -+ * requested mapping. Account for the pages it would unmap. -+ */ -+ nr_pages = count_vma_pages_range(mm, addr, end); -+ -+ if (!may_expand_vm(mm, vm_flags, -+ (len >> PAGE_SHIFT) - nr_pages)) -+ return -ENOMEM; -+ } -+ -+ /* Unmap any existing mapping in the area */ -+ if (do_mas_munmap(&mas, mm, addr, len, uf, false)) -+ return -ENOMEM; -+ -+ /* -+ * Private writable mapping: check memory availability -+ */ -+ if (accountable_mapping(file, vm_flags)) { -+ charged = len >> PAGE_SHIFT; -+ if (security_vm_enough_memory_mm(mm, charged)) -+ return -ENOMEM; -+ vm_flags |= VM_ACCOUNT; -+ } -+ -+ next = mas_next(&mas, ULONG_MAX); -+ prev = mas_prev(&mas, 0); -+ if (vm_flags & VM_SPECIAL) -+ goto cannot_expand; -+ -+ /* Attempt to expand an old mapping */ -+ /* Check next */ -+ if (next && next->vm_start == end && !vma_policy(next) && -+ can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, -+ NULL_VM_UFFD_CTX, NULL)) { -+ merge_end = next->vm_end; -+ vma = next; -+ vm_pgoff = next->vm_pgoff - pglen; -+ } -+ -+ /* Check prev */ -+ if (prev && prev->vm_end == addr && !vma_policy(prev) && -+ (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, -+ pgoff, vma->vm_userfaultfd_ctx, NULL) : -+ can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, -+ NULL_VM_UFFD_CTX, NULL))) { -+ merge_start = prev->vm_start; -+ vma = prev; -+ vm_pgoff = prev->vm_pgoff; -+ } -+ -+ -+ /* Actually expand, if possible */ -+ if (vma && -+ !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) { -+ khugepaged_enter_vma(vma, vm_flags); -+ goto expanded; -+ } -+ -+ mas.index = addr; -+ mas.last = end - 1; -+cannot_expand: -+ /* -+ * Determine the object being mapped and call the appropriate -+ * specific mapper. the address has already been validated, but -+ * not unmapped, but the maps are removed from the list. -+ */ -+ vma = vm_area_alloc(mm); -+ if (!vma) { -+ error = -ENOMEM; -+ goto unacct_error; -+ } -+ -+ vma->vm_start = addr; -+ vma->vm_end = end; -+ vma->vm_flags = vm_flags; -+ vma->vm_page_prot = vm_get_page_prot(vm_flags); -+ vma->vm_pgoff = pgoff; -+ -+ if (file) { -+ if (vm_flags & VM_SHARED) { -+ error = mapping_map_writable(file->f_mapping); -+ if (error) -+ goto free_vma; -+ } -+ -+ vma->vm_file = get_file(file); -+ error = call_mmap(file, vma); -+ if (error) -+ goto unmap_and_free_vma; -+ -+ /* Can addr have changed?? -+ * -+ * Answer: Yes, several device drivers can do it in their -+ * f_op->mmap method. -DaveM -+ */ -+ WARN_ON_ONCE(addr != vma->vm_start); -+ -+ addr = vma->vm_start; -+ mas_reset(&mas); -+ -+ /* -+ * If vm_flags changed after call_mmap(), we should try merge -+ * vma again as we may succeed this time. -+ */ -+ if (unlikely(vm_flags != vma->vm_flags && prev)) { -+ merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, -+ NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); -+ if (merge) { -+ /* -+ * ->mmap() can change vma->vm_file and fput -+ * the original file. So fput the vma->vm_file -+ * here or we would add an extra fput for file -+ * and cause general protection fault -+ * ultimately. -+ */ -+ fput(vma->vm_file); -+ vm_area_free(vma); -+ vma = merge; -+ /* Update vm_flags to pick up the change. */ -+ addr = vma->vm_start; -+ vm_flags = vma->vm_flags; -+ goto unmap_writable; -+ } -+ } -+ -+ vm_flags = vma->vm_flags; -+ } else if (vm_flags & VM_SHARED) { -+ error = shmem_zero_setup(vma); -+ if (error) -+ goto free_vma; -+ } else { -+ vma_set_anonymous(vma); -+ } -+ -+ /* Allow architectures to sanity-check the vm_flags */ -+ if (!arch_validate_flags(vma->vm_flags)) { -+ error = -EINVAL; -+ if (file) -+ goto unmap_and_free_vma; -+ else -+ goto free_vma; -+ } -+ -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { -+ error = -ENOMEM; -+ if (file) -+ goto unmap_and_free_vma; -+ else -+ goto free_vma; -+ } -+ -+ if (vma->vm_file) -+ i_mmap_lock_write(vma->vm_file->f_mapping); -+ -+ vma_mas_store(vma, &mas); -+ mm->map_count++; -+ if (vma->vm_file) { -+ if (vma->vm_flags & VM_SHARED) -+ mapping_allow_writable(vma->vm_file->f_mapping); -+ -+ flush_dcache_mmap_lock(vma->vm_file->f_mapping); -+ vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap); -+ flush_dcache_mmap_unlock(vma->vm_file->f_mapping); -+ i_mmap_unlock_write(vma->vm_file->f_mapping); -+ } -+ -+ /* -+ * vma_merge() calls khugepaged_enter_vma() either, the below -+ * call covers the non-merge case. -+ */ -+ khugepaged_enter_vma(vma, vma->vm_flags); -+ -+ /* Once vma denies write, undo our temporary denial count */ -+unmap_writable: -+ if (file && vm_flags & VM_SHARED) -+ mapping_unmap_writable(file->f_mapping); -+ file = vma->vm_file; -+expanded: -+ perf_event_mmap(vma); -+ -+ vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); -+ if (vm_flags & VM_LOCKED) { -+ if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || -+ is_vm_hugetlb_page(vma) || -+ vma == get_gate_vma(current->mm)) -+ vma->vm_flags &= VM_LOCKED_CLEAR_MASK; -+ else -+ mm->locked_vm += (len >> PAGE_SHIFT); -+ } -+ -+ if (file) -+ uprobe_mmap(vma); -+ -+ /* -+ * New (or expanded) vma always get soft dirty status. -+ * Otherwise user-space soft-dirty page tracker won't -+ * be able to distinguish situation when vma area unmapped, -+ * then new mapped in-place (which must be aimed as -+ * a completely new data area). -+ */ -+ vma->vm_flags |= VM_SOFTDIRTY; -+ -+ vma_set_page_prot(vma); -+ -+ validate_mm(mm); -+ return addr; -+ -+unmap_and_free_vma: -+ fput(vma->vm_file); -+ vma->vm_file = NULL; -+ -+ /* Undo any partial mapping done by a device driver. */ -+ unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end); -+ if (vm_flags & VM_SHARED) -+ mapping_unmap_writable(file->f_mapping); -+free_vma: -+ vm_area_free(vma); -+unacct_error: -+ if (charged) -+ vm_unacct_memory(charged); -+ validate_mm(mm); -+ return error; - } - - static int __vm_munmap(unsigned long start, size_t len, bool downgrade) -@@ -2852,11 +2774,12 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) - int ret; - struct mm_struct *mm = current->mm; - LIST_HEAD(uf); -+ MA_STATE(mas, &mm->mm_mt, start, start); - - if (mmap_write_lock_killable(mm)) - return -EINTR; - -- ret = __do_munmap(mm, start, len, &uf, downgrade); -+ ret = do_mas_munmap(&mas, mm, start, len, &uf, downgrade); - /* - * Returning 1 indicates mmap_lock is downgraded. - * But 1 is not legal return value of vm_munmap() and munmap(), reset -@@ -2922,11 +2845,12 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, - goto out; - - if (start + size > vma->vm_end) { -- struct vm_area_struct *next; -+ VMA_ITERATOR(vmi, mm, vma->vm_end); -+ struct vm_area_struct *next, *prev = vma; - -- for (next = vma->vm_next; next; next = next->vm_next) { -+ for_each_vma_range(vmi, next, start + size) { - /* hole between vmas ? */ -- if (next->vm_start != next->vm_prev->vm_end) -+ if (next->vm_start != prev->vm_end) - goto out; - - if (next->vm_file != vma->vm_file) -@@ -2935,8 +2859,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, - if (next->vm_flags != vma->vm_flags) - goto out; - -- if (start + size <= next->vm_end) -- break; -+ prev = next; - } - - if (!next) -@@ -2966,37 +2889,53 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, - } - - /* -- * this is really a simplified "do_mmap". it only handles -- * anonymous maps. eventually we may be able to do some -- * brk-specific accounting here. -+ * brk_munmap() - Unmap a parital vma. -+ * @mas: The maple tree state. -+ * @vma: The vma to be modified -+ * @newbrk: the start of the address to unmap -+ * @oldbrk: The end of the address to unmap -+ * @uf: The userfaultfd list_head -+ * -+ * Returns: 1 on success. -+ * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if -+ * possible. - */ --static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf) -+static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, -+ unsigned long newbrk, unsigned long oldbrk, -+ struct list_head *uf) - { -- struct mm_struct *mm = current->mm; -- struct vm_area_struct *vma, *prev; -- struct rb_node **rb_link, *rb_parent; -- pgoff_t pgoff = addr >> PAGE_SHIFT; -- int error; -- unsigned long mapped_addr; -- -- /* Until we need other flags, refuse anything except VM_EXEC. */ -- if ((flags & (~VM_EXEC)) != 0) -- return -EINVAL; -- flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; -- -- mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); -- if (IS_ERR_VALUE(mapped_addr)) -- return mapped_addr; -+ struct mm_struct *mm = vma->vm_mm; -+ int ret; - -- error = mlock_future_check(mm, mm->def_flags, len); -- if (error) -- return error; -+ arch_unmap(mm, newbrk, oldbrk); -+ ret = do_mas_align_munmap(mas, vma, mm, newbrk, oldbrk, uf, true); -+ validate_mm_mt(mm); -+ return ret; -+} - -- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ -- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) -- return -ENOMEM; -+/* -+ * do_brk_flags() - Increase the brk vma if the flags match. -+ * @mas: The maple tree state. -+ * @addr: The start address -+ * @len: The length of the increase -+ * @vma: The vma, -+ * @flags: The VMA Flags -+ * -+ * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags -+ * do not match then create a new anonymous VMA. Eventually we may be able to -+ * do some brk-specific accounting here. -+ */ -+static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma, -+ unsigned long addr, unsigned long len, unsigned long flags) -+{ -+ struct mm_struct *mm = current->mm; - -- /* Check against address space limits *after* clearing old maps... */ -+ validate_mm_mt(mm); -+ /* -+ * Check against address space limits by the changed size -+ * Note: This happens *after* clearing old mappings in some code paths. -+ */ -+ flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; - if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) - return -ENOMEM; - -@@ -3006,28 +2945,49 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla - if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) - return -ENOMEM; - -- /* Can we just expand an old private anonymous mapping? */ -- vma = vma_merge(mm, prev, addr, addr + len, flags, -- NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); -- if (vma) -- goto out; -- - /* -- * create a vma struct for an anonymous mapping -+ * Expand the existing vma if possible; Note that singular lists do not -+ * occur after forking, so the expand will only happen on new VMAs. - */ -- vma = vm_area_alloc(mm); -- if (!vma) { -- vm_unacct_memory(len >> PAGE_SHIFT); -- return -ENOMEM; -+ if (vma && -+ (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) && -+ ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) { -+ mas->index = vma->vm_start; -+ mas->last = addr + len - 1; -+ vma_adjust_trans_huge(vma, addr, addr + len, 0); -+ if (vma->anon_vma) { -+ anon_vma_lock_write(vma->anon_vma); -+ anon_vma_interval_tree_pre_update_vma(vma); -+ } -+ vma->vm_end = addr + len; -+ vma->vm_flags |= VM_SOFTDIRTY; -+ if (mas_store_gfp(mas, vma, GFP_KERNEL)) -+ goto mas_expand_failed; -+ -+ if (vma->anon_vma) { -+ anon_vma_interval_tree_post_update_vma(vma); -+ anon_vma_unlock_write(vma->anon_vma); -+ } -+ khugepaged_enter_vma(vma, flags); -+ goto out; - } - -+ /* create a vma struct for an anonymous mapping */ -+ vma = vm_area_alloc(mm); -+ if (!vma) -+ goto vma_alloc_fail; -+ - vma_set_anonymous(vma); - vma->vm_start = addr; - vma->vm_end = addr + len; -- vma->vm_pgoff = pgoff; -+ vma->vm_pgoff = addr >> PAGE_SHIFT; - vma->vm_flags = flags; - vma->vm_page_prot = vm_get_page_prot(flags); -- vma_link(mm, vma, prev, rb_link, rb_parent); -+ mas_set_range(mas, vma->vm_start, addr + len - 1); -+ if (mas_store_gfp(mas, vma, GFP_KERNEL)) -+ goto mas_store_fail; -+ -+ mm->map_count++; - out: - perf_event_mmap(vma); - mm->total_vm += len >> PAGE_SHIFT; -@@ -3035,16 +2995,32 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla - if (flags & VM_LOCKED) - mm->locked_vm += (len >> PAGE_SHIFT); - vma->vm_flags |= VM_SOFTDIRTY; -+ validate_mm(mm); - return 0; -+ -+mas_store_fail: -+ vm_area_free(vma); -+vma_alloc_fail: -+ vm_unacct_memory(len >> PAGE_SHIFT); -+ return -ENOMEM; -+ -+mas_expand_failed: -+ if (vma->anon_vma) { -+ anon_vma_interval_tree_post_update_vma(vma); -+ anon_vma_unlock_write(vma->anon_vma); -+ } -+ return -ENOMEM; - } - - int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) - { - struct mm_struct *mm = current->mm; -+ struct vm_area_struct *vma = NULL; - unsigned long len; - int ret; - bool populate; - LIST_HEAD(uf); -+ MA_STATE(mas, &mm->mm_mt, addr, addr); - - len = PAGE_ALIGN(request); - if (len < request) -@@ -3055,13 +3031,36 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) - if (mmap_write_lock_killable(mm)) - return -EINTR; - -- ret = do_brk_flags(addr, len, flags, &uf); -+ /* Until we need other flags, refuse anything except VM_EXEC. */ -+ if ((flags & (~VM_EXEC)) != 0) -+ return -EINVAL; -+ -+ ret = check_brk_limits(addr, len); -+ if (ret) -+ goto limits_failed; -+ -+ ret = do_mas_munmap(&mas, mm, addr, len, &uf, 0); -+ if (ret) -+ goto munmap_failed; -+ -+ vma = mas_prev(&mas, 0); -+ if (!vma || vma->vm_end != addr || vma_policy(vma) || -+ !can_vma_merge_after(vma, flags, NULL, NULL, -+ addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) -+ vma = NULL; -+ -+ ret = do_brk_flags(&mas, vma, addr, len, flags); - populate = ((mm->def_flags & VM_LOCKED) != 0); - mmap_write_unlock(mm); - userfaultfd_unmap_complete(mm, &uf); - if (populate && !ret) - mm_populate(addr, len); - return ret; -+ -+munmap_failed: -+limits_failed: -+ mmap_write_unlock(mm); -+ return ret; - } - EXPORT_SYMBOL(vm_brk_flags); - -@@ -3077,6 +3076,8 @@ void exit_mmap(struct mm_struct *mm) - struct mmu_gather tlb; - struct vm_area_struct *vma; - unsigned long nr_accounted = 0; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); -+ int count = 0; - - /* mm's last user has gone, and its about to be pulled down */ - mmu_notifier_release(mm); -@@ -3101,7 +3102,7 @@ void exit_mmap(struct mm_struct *mm) - mmap_write_lock(mm); - arch_exit_mmap(mm); - -- vma = mm->mmap; -+ vma = mas_find(&mas, ULONG_MAX); - if (!vma) { - /* Can happen if dup_mmap() received an OOM */ - mmap_write_unlock(mm); -@@ -3112,19 +3113,29 @@ void exit_mmap(struct mm_struct *mm) - flush_cache_mm(mm); - tlb_gather_mmu_fullmm(&tlb, mm); - /* update_hiwater_rss(mm) here? but nobody should be looking */ -- /* Use -1 here to ensure all VMAs in the mm are unmapped */ -- unmap_vmas(&tlb, vma, 0, -1); -- free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); -+ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ -+ unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX); -+ free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS, -+ USER_PGTABLES_CEILING); - tlb_finish_mmu(&tlb); - -- /* Walk the list again, actually closing and freeing it. */ -- while (vma) { -+ /* -+ * Walk the list again, actually closing and freeing it, with preemption -+ * enabled, without holding any MM locks besides the unreachable -+ * mmap_write_lock. -+ */ -+ do { - if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += vma_pages(vma); -- vma = remove_vma(vma); -+ remove_vma(vma); -+ count++; - cond_resched(); -- } -- mm->mmap = NULL; -+ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); -+ -+ BUG_ON(count != mm->map_count); -+ -+ trace_exit_mmap(mm); -+ __mt_destroy(&mm->mm_mt); - mmap_write_unlock(mm); - vm_unacct_memory(nr_accounted); - } -@@ -3135,14 +3146,14 @@ void exit_mmap(struct mm_struct *mm) - */ - int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) - { -- struct vm_area_struct *prev; -- struct rb_node **rb_link, *rb_parent; -+ unsigned long charged = vma_pages(vma); -+ - -- if (find_vma_links(mm, vma->vm_start, vma->vm_end, -- &prev, &rb_link, &rb_parent)) -+ if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) - return -ENOMEM; -+ - if ((vma->vm_flags & VM_ACCOUNT) && -- security_vm_enough_memory_mm(mm, vma_pages(vma))) -+ security_vm_enough_memory_mm(mm, charged)) - return -ENOMEM; - - /* -@@ -3162,7 +3173,11 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; - } - -- vma_link(mm, vma, prev, rb_link, rb_parent); -+ if (vma_link(mm, vma)) { -+ vm_unacct_memory(charged); -+ return -ENOMEM; -+ } -+ - return 0; - } - -@@ -3178,9 +3193,9 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long vma_start = vma->vm_start; - struct mm_struct *mm = vma->vm_mm; - struct vm_area_struct *new_vma, *prev; -- struct rb_node **rb_link, *rb_parent; - bool faulted_in_anon_vma = true; - -+ validate_mm_mt(mm); - /* - * If anonymous vma has not yet been faulted, update new pgoff - * to match new location, to increase its chance of merging. -@@ -3190,8 +3205,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - faulted_in_anon_vma = false; - } - -- if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) -+ new_vma = find_vma_prev(mm, addr, &prev); -+ if (new_vma && new_vma->vm_start < addr + len) - return NULL; /* should never get here */ -+ - new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, - vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx, anon_vma_name(vma)); -@@ -3232,16 +3249,22 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - get_file(new_vma->vm_file); - if (new_vma->vm_ops && new_vma->vm_ops->open) - new_vma->vm_ops->open(new_vma); -- vma_link(mm, new_vma, prev, rb_link, rb_parent); -+ if (vma_link(mm, new_vma)) -+ goto out_vma_link; - *need_rmap_locks = false; - } -+ validate_mm_mt(mm); - return new_vma; - -+out_vma_link: -+ if (new_vma->vm_ops && new_vma->vm_ops->close) -+ new_vma->vm_ops->close(new_vma); - out_free_mempol: - mpol_put(vma_policy(new_vma)); - out_free_vma: - vm_area_free(new_vma); - out: -+ validate_mm_mt(mm); - return NULL; - } - -@@ -3378,6 +3401,7 @@ static struct vm_area_struct *__install_special_mapping( - int ret; - struct vm_area_struct *vma; - -+ validate_mm_mt(mm); - vma = vm_area_alloc(mm); - if (unlikely(vma == NULL)) - return ERR_PTR(-ENOMEM); -@@ -3400,10 +3424,12 @@ static struct vm_area_struct *__install_special_mapping( - - perf_event_mmap(vma); - -+ validate_mm_mt(mm); - return vma; - - out: - vm_area_free(vma); -+ validate_mm_mt(mm); - return ERR_PTR(ret); - } - -@@ -3528,12 +3554,13 @@ int mm_take_all_locks(struct mm_struct *mm) - { - struct vm_area_struct *vma; - struct anon_vma_chain *avc; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - mmap_assert_write_locked(mm); - - mutex_lock(&mm_all_locks_mutex); - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ mas_for_each(&mas, vma, ULONG_MAX) { - if (signal_pending(current)) - goto out_unlock; - if (vma->vm_file && vma->vm_file->f_mapping && -@@ -3541,7 +3568,8 @@ int mm_take_all_locks(struct mm_struct *mm) - vm_lock_mapping(mm, vma->vm_file->f_mapping); - } - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ mas_set(&mas, 0); -+ mas_for_each(&mas, vma, ULONG_MAX) { - if (signal_pending(current)) - goto out_unlock; - if (vma->vm_file && vma->vm_file->f_mapping && -@@ -3549,7 +3577,8 @@ int mm_take_all_locks(struct mm_struct *mm) - vm_lock_mapping(mm, vma->vm_file->f_mapping); - } - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ mas_set(&mas, 0); -+ mas_for_each(&mas, vma, ULONG_MAX) { - if (signal_pending(current)) - goto out_unlock; - if (vma->anon_vma) -@@ -3608,11 +3637,12 @@ void mm_drop_all_locks(struct mm_struct *mm) - { - struct vm_area_struct *vma; - struct anon_vma_chain *avc; -+ MA_STATE(mas, &mm->mm_mt, 0, 0); - - mmap_assert_write_locked(mm); - BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); - -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ mas_for_each(&mas, vma, ULONG_MAX) { - if (vma->anon_vma) - list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - vm_unlock_anon_vma(avc->anon_vma); -diff --git a/mm/mmzone.c b/mm/mmzone.c -index 0ae7571e35ab..68e1511be12d 100644 ---- a/mm/mmzone.c -+++ b/mm/mmzone.c -@@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) - * Poison its list head, so that any operations on it would crash. - */ - list_del(&lruvec->lists[LRU_UNEVICTABLE]); -+ -+ lru_gen_init_lruvec(lruvec); - } - - #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) -diff --git a/mm/mprotect.c b/mm/mprotect.c -index bc6bddd156ca..bf030bbe7e97 100644 ---- a/mm/mprotect.c -+++ b/mm/mprotect.c -@@ -669,6 +669,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, - const bool rier = (current->personality & READ_IMPLIES_EXEC) && - (prot & PROT_READ); - struct mmu_gather tlb; -+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); - - start = untagged_addr(start); - -@@ -700,7 +701,8 @@ static int do_mprotect_pkey(unsigned long start, size_t len, - if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) - goto out; - -- vma = find_vma(current->mm, start); -+ mas_set(&mas, start); -+ vma = mas_find(&mas, ULONG_MAX); - error = -ENOMEM; - if (!vma) - goto out; -@@ -726,7 +728,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, - if (start > vma->vm_start) - prev = vma; - else -- prev = vma->vm_prev; -+ prev = mas_prev(&mas, 0); - - tlb_gather_mmu(&tlb, current->mm); - for (nstart = start ; ; ) { -@@ -789,7 +791,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, - if (nstart >= end) - break; - -- vma = prev->vm_next; -+ vma = find_vma(current->mm, prev->vm_end); - if (!vma || vma->vm_start != nstart) { - error = -ENOMEM; - break; -diff --git a/mm/mremap.c b/mm/mremap.c -index b522cd0259a0..8644ff278f02 100644 ---- a/mm/mremap.c -+++ b/mm/mremap.c -@@ -716,7 +716,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, - if (excess) { - vma->vm_flags |= VM_ACCOUNT; - if (split) -- vma->vm_next->vm_flags |= VM_ACCOUNT; -+ find_vma(mm, vma->vm_end)->vm_flags |= VM_ACCOUNT; - } - - return new_addr; -@@ -866,9 +866,10 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, - static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) - { - unsigned long end = vma->vm_end + delta; -+ - if (end < vma->vm_end) /* overflow */ - return 0; -- if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ -+ if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) - return 0; - if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, - 0, MAP_FIXED) & ~PAGE_MASK) -@@ -975,20 +976,23 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, - /* - * Always allow a shrinking remap: that just unmaps - * the unnecessary pages.. -- * __do_munmap does all the needed commit accounting, and -+ * do_mas_munmap does all the needed commit accounting, and - * downgrades mmap_lock to read if so directed. - */ - if (old_len >= new_len) { - int retval; -+ MA_STATE(mas, &mm->mm_mt, addr + new_len, addr + new_len); - -- retval = __do_munmap(mm, addr+new_len, old_len - new_len, -- &uf_unmap, true); -- if (retval < 0 && old_len != new_len) { -- ret = retval; -- goto out; -+ retval = do_mas_munmap(&mas, mm, addr + new_len, -+ old_len - new_len, &uf_unmap, true); - /* Returning 1 indicates mmap_lock is downgraded to read. */ -- } else if (retval == 1) -+ if (retval == 1) { - downgraded = true; -+ } else if (retval < 0 && old_len != new_len) { -+ ret = retval; -+ goto out; -+ } -+ - ret = addr; - goto out; - } -diff --git a/mm/msync.c b/mm/msync.c -index 137d1c104f3e..ac4c9bfea2e7 100644 ---- a/mm/msync.c -+++ b/mm/msync.c -@@ -104,7 +104,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) - error = 0; - goto out_unlock; - } -- vma = vma->vm_next; -+ vma = find_vma(mm, vma->vm_end); - } - } - out_unlock: -diff --git a/mm/nommu.c b/mm/nommu.c -index e819cbc21b39..214c70e1d059 100644 ---- a/mm/nommu.c -+++ b/mm/nommu.c -@@ -19,7 +19,6 @@ - #include - #include - #include --#include - #include - #include - #include -@@ -545,26 +544,27 @@ static void put_nommu_region(struct vm_region *region) - __put_nommu_region(region); - } - --/* -- * add a VMA into a process's mm_struct in the appropriate place in the list -- * and tree and add to the address space's page tree also if not an anonymous -- * page -- * - should be called with mm->mmap_lock held writelocked -- */ --static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) -+void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) - { -- struct vm_area_struct *pvma, *prev; -- struct address_space *mapping; -- struct rb_node **p, *parent, *rb_prev; -+ mas_set_range(mas, vma->vm_start, vma->vm_end - 1); -+ mas_store_prealloc(mas, vma); -+} - -- BUG_ON(!vma->vm_region); -+void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) -+{ -+ mas->index = vma->vm_start; -+ mas->last = vma->vm_end - 1; -+ mas_store_prealloc(mas, NULL); -+} - -+static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) -+{ - mm->map_count++; - vma->vm_mm = mm; - - /* add the VMA to the mapping */ - if (vma->vm_file) { -- mapping = vma->vm_file->f_mapping; -+ struct address_space *mapping = vma->vm_file->f_mapping; - - i_mmap_lock_write(mapping); - flush_dcache_mmap_lock(mapping); -@@ -572,67 +572,51 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) - flush_dcache_mmap_unlock(mapping); - i_mmap_unlock_write(mapping); - } -+} - -- /* add the VMA to the tree */ -- parent = rb_prev = NULL; -- p = &mm->mm_rb.rb_node; -- while (*p) { -- parent = *p; -- pvma = rb_entry(parent, struct vm_area_struct, vm_rb); -- -- /* sort by: start addr, end addr, VMA struct addr in that order -- * (the latter is necessary as we may get identical VMAs) */ -- if (vma->vm_start < pvma->vm_start) -- p = &(*p)->rb_left; -- else if (vma->vm_start > pvma->vm_start) { -- rb_prev = parent; -- p = &(*p)->rb_right; -- } else if (vma->vm_end < pvma->vm_end) -- p = &(*p)->rb_left; -- else if (vma->vm_end > pvma->vm_end) { -- rb_prev = parent; -- p = &(*p)->rb_right; -- } else if (vma < pvma) -- p = &(*p)->rb_left; -- else if (vma > pvma) { -- rb_prev = parent; -- p = &(*p)->rb_right; -- } else -- BUG(); -- } -- -- rb_link_node(&vma->vm_rb, parent, p); -- rb_insert_color(&vma->vm_rb, &mm->mm_rb); -+/* -+ * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm(). -+ * @mas: The maple state with preallocations. -+ * @mm: The mm_struct -+ * @vma: The vma to add -+ * -+ */ -+static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm, -+ struct vm_area_struct *vma) -+{ -+ BUG_ON(!vma->vm_region); - -- /* add VMA to the VMA list also */ -- prev = NULL; -- if (rb_prev) -- prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); -+ setup_vma_to_mm(vma, mm); - -- __vma_link_list(mm, vma, prev); -+ /* add the VMA to the tree */ -+ vma_mas_store(vma, mas); - } - - /* -- * delete a VMA from its owning mm_struct and address space -+ * add a VMA into a process's mm_struct in the appropriate place in the list -+ * and tree and add to the address space's page tree also if not an anonymous -+ * page -+ * - should be called with mm->mmap_lock held writelocked - */ --static void delete_vma_from_mm(struct vm_area_struct *vma) --{ -- int i; -- struct address_space *mapping; -- struct mm_struct *mm = vma->vm_mm; -- struct task_struct *curr = current; -- -- mm->map_count--; -- for (i = 0; i < VMACACHE_SIZE; i++) { -- /* if the vma is cached, invalidate the entire cache */ -- if (curr->vmacache.vmas[i] == vma) { -- vmacache_invalidate(mm); -- break; -- } -+static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) -+{ -+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); -+ -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { -+ pr_warn("Allocation of vma tree for process %d failed\n", -+ current->pid); -+ return -ENOMEM; - } -+ mas_add_vma_to_mm(&mas, mm, vma); -+ return 0; -+} - -+static void cleanup_vma_from_mm(struct vm_area_struct *vma) -+{ -+ vma->vm_mm->map_count--; - /* remove the VMA from the mapping */ - if (vma->vm_file) { -+ struct address_space *mapping; - mapping = vma->vm_file->f_mapping; - - i_mmap_lock_write(mapping); -@@ -641,11 +625,24 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) - flush_dcache_mmap_unlock(mapping); - i_mmap_unlock_write(mapping); - } -+} -+/* -+ * delete a VMA from its owning mm_struct and address space -+ */ -+static int delete_vma_from_mm(struct vm_area_struct *vma) -+{ -+ MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0); - -- /* remove from the MM's tree and list */ -- rb_erase(&vma->vm_rb, &mm->mm_rb); -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { -+ pr_warn("Allocation of vma tree for process %d failed\n", -+ current->pid); -+ return -ENOMEM; -+ } -+ cleanup_vma_from_mm(vma); - -- __vma_unlink_list(mm, vma); -+ /* remove from the MM's tree and list */ -+ vma_mas_remove(vma, &mas); -+ return 0; - } - - /* -@@ -661,31 +658,26 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) - vm_area_free(vma); - } - -+struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, -+ unsigned long start_addr, -+ unsigned long end_addr) -+{ -+ unsigned long index = start_addr; -+ -+ mmap_assert_locked(mm); -+ return mt_find(&mm->mm_mt, &index, end_addr - 1); -+} -+EXPORT_SYMBOL(find_vma_intersection); -+ - /* - * look up the first VMA in which addr resides, NULL if none - * - should be called with mm->mmap_lock at least held readlocked - */ - struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) - { -- struct vm_area_struct *vma; -+ MA_STATE(mas, &mm->mm_mt, addr, addr); - -- /* check the cache first */ -- vma = vmacache_find(mm, addr); -- if (likely(vma)) -- return vma; -- -- /* trawl the list (there may be multiple mappings in which addr -- * resides) */ -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -- if (vma->vm_start > addr) -- return NULL; -- if (vma->vm_end > addr) { -- vmacache_update(addr, vma); -- return vma; -- } -- } -- -- return NULL; -+ return mas_walk(&mas); - } - EXPORT_SYMBOL(find_vma); - -@@ -717,26 +709,17 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, - { - struct vm_area_struct *vma; - unsigned long end = addr + len; -+ MA_STATE(mas, &mm->mm_mt, addr, addr); - -- /* check the cache first */ -- vma = vmacache_find_exact(mm, addr, end); -- if (vma) -- return vma; -- -- /* trawl the list (there may be multiple mappings in which addr -- * resides) */ -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -- if (vma->vm_start < addr) -- continue; -- if (vma->vm_start > addr) -- return NULL; -- if (vma->vm_end == end) { -- vmacache_update(addr, vma); -- return vma; -- } -- } -+ vma = mas_walk(&mas); -+ if (!vma) -+ return NULL; -+ if (vma->vm_start != addr) -+ return NULL; -+ if (vma->vm_end != end) -+ return NULL; - -- return NULL; -+ return vma; - } - - /* -@@ -1069,6 +1052,7 @@ unsigned long do_mmap(struct file *file, - vm_flags_t vm_flags; - unsigned long capabilities, result; - int ret; -+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); - - *populate = 0; - -@@ -1087,6 +1071,7 @@ unsigned long do_mmap(struct file *file, - * now know into VMA flags */ - vm_flags = determine_vm_flags(file, prot, flags, capabilities); - -+ - /* we're going to need to record the mapping */ - region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); - if (!region) -@@ -1096,6 +1081,9 @@ unsigned long do_mmap(struct file *file, - if (!vma) - goto error_getting_vma; - -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) -+ goto error_maple_preallocate; -+ - region->vm_usage = 1; - region->vm_flags = vm_flags; - region->vm_pgoff = pgoff; -@@ -1236,7 +1224,7 @@ unsigned long do_mmap(struct file *file, - current->mm->total_vm += len >> PAGE_SHIFT; - - share: -- add_vma_to_mm(current->mm, vma); -+ mas_add_vma_to_mm(&mas, current->mm, vma); - - /* we flush the region from the icache only when the first executable - * mapping of it is made */ -@@ -1262,6 +1250,7 @@ unsigned long do_mmap(struct file *file, - - sharing_violation: - up_write(&nommu_region_sem); -+ mas_destroy(&mas); - pr_warn("Attempt to share mismatched mappings\n"); - ret = -EINVAL; - goto error; -@@ -1278,6 +1267,14 @@ unsigned long do_mmap(struct file *file, - len, current->pid); - show_free_areas(0, NULL); - return -ENOMEM; -+ -+error_maple_preallocate: -+ kmem_cache_free(vm_region_jar, region); -+ vm_area_free(vma); -+ pr_warn("Allocation of vma tree for process %d failed\n", current->pid); -+ show_free_areas(0, NULL); -+ return -ENOMEM; -+ - } - - unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, -@@ -1343,6 +1340,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *new; - struct vm_region *region; - unsigned long npages; -+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); - - /* we're only permitted to split anonymous regions (these should have - * only a single usage on the region) */ -@@ -1357,9 +1355,13 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - return -ENOMEM; - - new = vm_area_dup(vma); -- if (!new) { -- kmem_cache_free(vm_region_jar, region); -- return -ENOMEM; -+ if (!new) -+ goto err_vma_dup; -+ -+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { -+ pr_warn("Allocation of vma tree for process %d failed\n", -+ current->pid); -+ goto err_mas_preallocate; - } - - /* most fields are the same, copy all, and then fixup */ -@@ -1378,7 +1380,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - if (new->vm_ops && new->vm_ops->open) - new->vm_ops->open(new); - -- delete_vma_from_mm(vma); - down_write(&nommu_region_sem); - delete_nommu_region(vma->vm_region); - if (new_below) { -@@ -1391,9 +1392,19 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - add_nommu_region(vma->vm_region); - add_nommu_region(new->vm_region); - up_write(&nommu_region_sem); -- add_vma_to_mm(mm, vma); -- add_vma_to_mm(mm, new); -+ -+ setup_vma_to_mm(vma, mm); -+ setup_vma_to_mm(new, mm); -+ mas_set_range(&mas, vma->vm_start, vma->vm_end - 1); -+ mas_store(&mas, vma); -+ vma_mas_store(new, &mas); - return 0; -+ -+err_mas_preallocate: -+ vm_area_free(new); -+err_vma_dup: -+ kmem_cache_free(vm_region_jar, region); -+ return -ENOMEM; - } - - /* -@@ -1408,12 +1419,14 @@ static int shrink_vma(struct mm_struct *mm, - - /* adjust the VMA's pointers, which may reposition it in the MM's tree - * and list */ -- delete_vma_from_mm(vma); -+ if (delete_vma_from_mm(vma)) -+ return -ENOMEM; - if (from > vma->vm_start) - vma->vm_end = from; - else - vma->vm_start = to; -- add_vma_to_mm(mm, vma); -+ if (add_vma_to_mm(mm, vma)) -+ return -ENOMEM; - - /* cut the backing region down to size */ - region = vma->vm_region; -@@ -1441,9 +1454,10 @@ static int shrink_vma(struct mm_struct *mm, - */ - int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) - { -+ MA_STATE(mas, &mm->mm_mt, start, start); - struct vm_area_struct *vma; - unsigned long end; -- int ret; -+ int ret = 0; - - len = PAGE_ALIGN(len); - if (len == 0) -@@ -1452,7 +1466,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list - end = start + len; - - /* find the first potentially overlapping VMA */ -- vma = find_vma(mm, start); -+ vma = mas_find(&mas, end - 1); - if (!vma) { - static int limit; - if (limit < 5) { -@@ -1471,7 +1485,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list - return -EINVAL; - if (end == vma->vm_end) - goto erase_whole_vma; -- vma = vma->vm_next; -+ vma = mas_next(&mas, end - 1); - } while (vma); - return -EINVAL; - } else { -@@ -1493,9 +1507,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list - } - - erase_whole_vma: -- delete_vma_from_mm(vma); -+ if (delete_vma_from_mm(vma)) -+ ret = -ENOMEM; - delete_vma(mm, vma); -- return 0; -+ return ret; - } - - int vm_munmap(unsigned long addr, size_t len) -@@ -1520,6 +1535,7 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) - */ - void exit_mmap(struct mm_struct *mm) - { -+ VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - - if (!mm) -@@ -1527,12 +1543,18 @@ void exit_mmap(struct mm_struct *mm) - - mm->total_vm = 0; - -- while ((vma = mm->mmap)) { -- mm->mmap = vma->vm_next; -- delete_vma_from_mm(vma); -+ /* -+ * Lock the mm to avoid assert complaining even though this is the only -+ * user of the mm -+ */ -+ mmap_write_lock(mm); -+ for_each_vma(vmi, vma) { -+ cleanup_vma_from_mm(vma); - delete_vma(mm, vma); - cond_resched(); - } -+ __mt_destroy(&mm->mm_mt); -+ mmap_write_unlock(mm); - } - - int vm_brk(unsigned long addr, unsigned long len) -diff --git a/mm/oom_kill.c b/mm/oom_kill.c -index 3c6cf9e3cd66..3996301450e8 100644 ---- a/mm/oom_kill.c -+++ b/mm/oom_kill.c -@@ -513,6 +513,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) - { - struct vm_area_struct *vma; - bool ret = true; -+ VMA_ITERATOR(vmi, mm, 0); - - /* - * Tell all users of get_user/copy_from_user etc... that the content -@@ -522,7 +523,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) - */ - set_bit(MMF_UNSTABLE, &mm->flags); - -- for (vma = mm->mmap ; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) - continue; - -diff --git a/mm/pagewalk.c b/mm/pagewalk.c -index fa7a3d21a751..c09bb9c01f7e 100644 ---- a/mm/pagewalk.c -+++ b/mm/pagewalk.c -@@ -460,7 +460,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, - } else { /* inside vma */ - walk.vma = vma; - next = min(end, vma->vm_end); -- vma = vma->vm_next; -+ vma = find_vma(mm, vma->vm_end); - - err = walk_page_test(start, next, &walk); - if (err > 0) { -diff --git a/mm/rmap.c b/mm/rmap.c -index 93d5a6f793d2..3080edeaa3ad 100644 ---- a/mm/rmap.c -+++ b/mm/rmap.c -@@ -770,13 +770,17 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) - return vma_address(page, vma); - } - -+/* -+ * Returns the actual pmd_t* where we expect 'address' to be mapped from, or -+ * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* -+ * represents. -+ */ - pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) - { - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd = NULL; -- pmd_t pmde; - - pgd = pgd_offset(mm, address); - if (!pgd_present(*pgd)) -@@ -791,15 +795,6 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) - goto out; - - pmd = pmd_offset(pud, address); -- /* -- * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() -- * without holding anon_vma lock for write. So when looking for a -- * genuine pmde (in which to find pte), test present and !THP together. -- */ -- pmde = *pmd; -- barrier(); -- if (!pmd_present(pmde) || pmd_trans_huge(pmde)) -- pmd = NULL; - out: - return pmd; - } -@@ -833,6 +828,12 @@ static bool folio_referenced_one(struct folio *folio, - } - - if (pvmw.pte) { -+ if (lru_gen_enabled() && pte_young(*pvmw.pte) && -+ !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { -+ lru_gen_look_around(&pvmw); -+ referenced++; -+ } -+ - if (ptep_clear_flush_young_notify(vma, address, - pvmw.pte)) { - /* -diff --git a/mm/swap.c b/mm/swap.c -index 186b4e5dcecf..027943b341a0 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu) - folio_batch_move_lru(fbatch, folio_activate_fn); - } - --static void folio_activate(struct folio *folio) -+void folio_activate(struct folio *folio) - { - if (folio_test_lru(folio) && !folio_test_active(folio) && - !folio_test_unevictable(folio)) { -@@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu) - { - } - --static void folio_activate(struct folio *folio) -+void folio_activate(struct folio *folio) - { - struct lruvec *lruvec; - -@@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio) - local_unlock(&cpu_fbatches.lock); - } - -+#ifdef CONFIG_LRU_GEN -+static void folio_inc_refs(struct folio *folio) -+{ -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); -+ -+ if (folio_test_unevictable(folio)) -+ return; -+ -+ if (!folio_test_referenced(folio)) { -+ folio_set_referenced(folio); -+ return; -+ } -+ -+ if (!folio_test_workingset(folio)) { -+ folio_set_workingset(folio); -+ return; -+ } -+ -+ /* see the comment on MAX_NR_TIERS */ -+ do { -+ new_flags = old_flags & LRU_REFS_MASK; -+ if (new_flags == LRU_REFS_MASK) -+ break; -+ -+ new_flags += BIT(LRU_REFS_PGOFF); -+ new_flags |= old_flags & ~LRU_REFS_MASK; -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); -+} -+#else -+static void folio_inc_refs(struct folio *folio) -+{ -+} -+#endif /* CONFIG_LRU_GEN */ -+ - /* - * Mark a page as having seen activity. - * -@@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio) - */ - void folio_mark_accessed(struct folio *folio) - { -+ if (lru_gen_enabled()) { -+ folio_inc_refs(folio); -+ return; -+ } -+ - if (!folio_test_referenced(folio)) { - folio_set_referenced(folio); - } else if (folio_test_unevictable(folio)) { -@@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio) - folio_test_unevictable(folio), folio); - VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); - -+ /* see the comment in lru_gen_add_folio() */ -+ if (lru_gen_enabled() && !folio_test_unevictable(folio) && -+ lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) -+ folio_set_active(folio); -+ - folio_get(folio); - local_lock(&cpu_fbatches.lock); - fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); -@@ -575,7 +619,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) - - static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) - { -- if (folio_test_active(folio) && !folio_test_unevictable(folio)) { -+ if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { - long nr_pages = folio_nr_pages(folio); - - lruvec_del_folio(lruvec, folio); -@@ -688,8 +732,8 @@ void deactivate_page(struct page *page) - { - struct folio *folio = page_folio(page); - -- if (folio_test_lru(folio) && folio_test_active(folio) && -- !folio_test_unevictable(folio)) { -+ if (folio_test_lru(folio) && !folio_test_unevictable(folio) && -+ (folio_test_active(folio) || lru_gen_enabled())) { - struct folio_batch *fbatch; - - folio_get(folio); -diff --git a/mm/swapfile.c b/mm/swapfile.c -index 1fdccd2f1422..5c8681a3f1d9 100644 ---- a/mm/swapfile.c -+++ b/mm/swapfile.c -@@ -1990,14 +1990,16 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type) - { - struct vm_area_struct *vma; - int ret = 0; -+ VMA_ITERATOR(vmi, mm, 0); - - mmap_read_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ for_each_vma(vmi, vma) { - if (vma->anon_vma) { - ret = unuse_vma(vma, type); - if (ret) - break; - } -+ - cond_resched(); - } - mmap_read_unlock(mm); -diff --git a/mm/util.c b/mm/util.c -index c9439c66d8cf..1266a33a49ea 100644 ---- a/mm/util.c -+++ b/mm/util.c -@@ -272,38 +272,6 @@ void *memdup_user_nul(const void __user *src, size_t len) - } - EXPORT_SYMBOL(memdup_user_nul); - --void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, -- struct vm_area_struct *prev) --{ -- struct vm_area_struct *next; -- -- vma->vm_prev = prev; -- if (prev) { -- next = prev->vm_next; -- prev->vm_next = vma; -- } else { -- next = mm->mmap; -- mm->mmap = vma; -- } -- vma->vm_next = next; -- if (next) -- next->vm_prev = vma; --} -- --void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma) --{ -- struct vm_area_struct *prev, *next; -- -- next = vma->vm_next; -- prev = vma->vm_prev; -- if (prev) -- prev->vm_next = next; -- else -- mm->mmap = next; -- if (next) -- next->vm_prev = prev; --} -- - /* Check if the vma is being used as a stack by this task */ - int vma_is_stack_for_current(struct vm_area_struct *vma) - { -diff --git a/mm/vmacache.c b/mm/vmacache.c -deleted file mode 100644 -index 01a6e6688ec1..000000000000 ---- a/mm/vmacache.c -+++ /dev/null -@@ -1,117 +0,0 @@ --// SPDX-License-Identifier: GPL-2.0 --/* -- * Copyright (C) 2014 Davidlohr Bueso. -- */ --#include --#include --#include --#include -- --/* -- * Hash based on the pmd of addr if configured with MMU, which provides a good -- * hit rate for workloads with spatial locality. Otherwise, use pages. -- */ --#ifdef CONFIG_MMU --#define VMACACHE_SHIFT PMD_SHIFT --#else --#define VMACACHE_SHIFT PAGE_SHIFT --#endif --#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK) -- --/* -- * This task may be accessing a foreign mm via (for example) -- * get_user_pages()->find_vma(). The vmacache is task-local and this -- * task's vmacache pertains to a different mm (ie, its own). There is -- * nothing we can do here. -- * -- * Also handle the case where a kernel thread has adopted this mm via -- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm. -- */ --static inline bool vmacache_valid_mm(struct mm_struct *mm) --{ -- return current->mm == mm && !(current->flags & PF_KTHREAD); --} -- --void vmacache_update(unsigned long addr, struct vm_area_struct *newvma) --{ -- if (vmacache_valid_mm(newvma->vm_mm)) -- current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma; --} -- --static bool vmacache_valid(struct mm_struct *mm) --{ -- struct task_struct *curr; -- -- if (!vmacache_valid_mm(mm)) -- return false; -- -- curr = current; -- if (mm->vmacache_seqnum != curr->vmacache.seqnum) { -- /* -- * First attempt will always be invalid, initialize -- * the new cache for this task here. -- */ -- curr->vmacache.seqnum = mm->vmacache_seqnum; -- vmacache_flush(curr); -- return false; -- } -- return true; --} -- --struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr) --{ -- int idx = VMACACHE_HASH(addr); -- int i; -- -- count_vm_vmacache_event(VMACACHE_FIND_CALLS); -- -- if (!vmacache_valid(mm)) -- return NULL; -- -- for (i = 0; i < VMACACHE_SIZE; i++) { -- struct vm_area_struct *vma = current->vmacache.vmas[idx]; -- -- if (vma) { --#ifdef CONFIG_DEBUG_VM_VMACACHE -- if (WARN_ON_ONCE(vma->vm_mm != mm)) -- break; --#endif -- if (vma->vm_start <= addr && vma->vm_end > addr) { -- count_vm_vmacache_event(VMACACHE_FIND_HITS); -- return vma; -- } -- } -- if (++idx == VMACACHE_SIZE) -- idx = 0; -- } -- -- return NULL; --} -- --#ifndef CONFIG_MMU --struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, -- unsigned long start, -- unsigned long end) --{ -- int idx = VMACACHE_HASH(start); -- int i; -- -- count_vm_vmacache_event(VMACACHE_FIND_CALLS); -- -- if (!vmacache_valid(mm)) -- return NULL; -- -- for (i = 0; i < VMACACHE_SIZE; i++) { -- struct vm_area_struct *vma = current->vmacache.vmas[idx]; -- -- if (vma && vma->vm_start == start && vma->vm_end == end) { -- count_vm_vmacache_event(VMACACHE_FIND_HITS); -- return vma; -- } -- if (++idx == VMACACHE_SIZE) -- idx = 0; -- } -- -- return NULL; --} --#endif -diff --git a/mm/vmscan.c b/mm/vmscan.c -index 0fc65ace3a4e..f38fd15935df 100644 ---- a/mm/vmscan.c -+++ b/mm/vmscan.c -@@ -49,6 +49,10 @@ - #include - #include - #include -+#include -+#include -+#include -+#include - - #include - #include -@@ -129,6 +133,12 @@ struct scan_control { - /* Always discard instead of demoting to lower tier memory */ - unsigned int no_demotion:1; - -+#ifdef CONFIG_LRU_GEN -+ /* help kswapd make better choices among multiple memcgs */ -+ unsigned int memcgs_need_aging:1; -+ unsigned long last_reclaimed; -+#endif -+ - /* Allocation order */ - s8 order; - -@@ -1338,9 +1348,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, - - if (folio_test_swapcache(folio)) { - swp_entry_t swap = folio_swap_entry(folio); -- mem_cgroup_swapout(folio, swap); -+ -+ /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ - if (reclaimed && !mapping_exiting(mapping)) - shadow = workingset_eviction(folio, target_memcg); -+ mem_cgroup_swapout(folio, swap); - __delete_from_swap_cache(folio, swap, shadow); - xa_unlock_irq(&mapping->i_pages); - put_swap_page(&folio->page, swap); -@@ -1637,6 +1649,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, - if (!sc->may_unmap && folio_mapped(folio)) - goto keep_locked; - -+ /* folio_update_gen() tried to promote this page? */ -+ if (lru_gen_enabled() && !ignore_references && -+ folio_mapped(folio) && folio_test_referenced(folio)) -+ goto keep_locked; -+ - /* - * The number of dirty pages determines if a node is marked - * reclaim_congested. kswapd will stall and start writing -@@ -2732,6 +2749,112 @@ enum scan_balance { - SCAN_FILE, - }; - -+static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) -+{ -+ unsigned long file; -+ struct lruvec *target_lruvec; -+ -+ if (lru_gen_enabled()) -+ return; -+ -+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); -+ -+ /* -+ * Flush the memory cgroup stats, so that we read accurate per-memcg -+ * lruvec stats for heuristics. -+ */ -+ mem_cgroup_flush_stats(); -+ -+ /* -+ * Determine the scan balance between anon and file LRUs. -+ */ -+ spin_lock_irq(&target_lruvec->lru_lock); -+ sc->anon_cost = target_lruvec->anon_cost; -+ sc->file_cost = target_lruvec->file_cost; -+ spin_unlock_irq(&target_lruvec->lru_lock); -+ -+ /* -+ * Target desirable inactive:active list ratios for the anon -+ * and file LRU lists. -+ */ -+ if (!sc->force_deactivate) { -+ unsigned long refaults; -+ -+ /* -+ * When refaults are being observed, it means a new -+ * workingset is being established. Deactivate to get -+ * rid of any stale active pages quickly. -+ */ -+ refaults = lruvec_page_state(target_lruvec, -+ WORKINGSET_ACTIVATE_ANON); -+ if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || -+ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) -+ sc->may_deactivate |= DEACTIVATE_ANON; -+ else -+ sc->may_deactivate &= ~DEACTIVATE_ANON; -+ -+ refaults = lruvec_page_state(target_lruvec, -+ WORKINGSET_ACTIVATE_FILE); -+ if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || -+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) -+ sc->may_deactivate |= DEACTIVATE_FILE; -+ else -+ sc->may_deactivate &= ~DEACTIVATE_FILE; -+ } else -+ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; -+ -+ /* -+ * If we have plenty of inactive file pages that aren't -+ * thrashing, try to reclaim those first before touching -+ * anonymous pages. -+ */ -+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); -+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) -+ sc->cache_trim_mode = 1; -+ else -+ sc->cache_trim_mode = 0; -+ -+ /* -+ * Prevent the reclaimer from falling into the cache trap: as -+ * cache pages start out inactive, every cache fault will tip -+ * the scan balance towards the file LRU. And as the file LRU -+ * shrinks, so does the window for rotation from references. -+ * This means we have a runaway feedback loop where a tiny -+ * thrashing file LRU becomes infinitely more attractive than -+ * anon pages. Try to detect this based on file LRU size. -+ */ -+ if (!cgroup_reclaim(sc)) { -+ unsigned long total_high_wmark = 0; -+ unsigned long free, anon; -+ int z; -+ -+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); -+ file = node_page_state(pgdat, NR_ACTIVE_FILE) + -+ node_page_state(pgdat, NR_INACTIVE_FILE); -+ -+ for (z = 0; z < MAX_NR_ZONES; z++) { -+ struct zone *zone = &pgdat->node_zones[z]; -+ -+ if (!managed_zone(zone)) -+ continue; -+ -+ total_high_wmark += high_wmark_pages(zone); -+ } -+ -+ /* -+ * Consider anon: if that's low too, this isn't a -+ * runaway file reclaim problem, but rather just -+ * extreme pressure. Reclaim as per usual then. -+ */ -+ anon = node_page_state(pgdat, NR_INACTIVE_ANON); -+ -+ sc->file_is_tiny = -+ file + free <= total_high_wmark && -+ !(sc->may_deactivate & DEACTIVATE_ANON) && -+ anon >> sc->priority; -+ } -+} -+ - /* - * Determine how aggressively the anon and file LRU lists should be - * scanned. -@@ -2951,159 +3074,2906 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, - return can_demote(pgdat->node_id, sc); - } - --static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) --{ -- unsigned long nr[NR_LRU_LISTS]; -- unsigned long targets[NR_LRU_LISTS]; -- unsigned long nr_to_scan; -- enum lru_list lru; -- unsigned long nr_reclaimed = 0; -- unsigned long nr_to_reclaim = sc->nr_to_reclaim; -- struct blk_plug plug; -- bool scan_adjusted; -+#ifdef CONFIG_LRU_GEN - -- get_scan_count(lruvec, sc, nr); -+#ifdef CONFIG_LRU_GEN_ENABLED -+DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); -+#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) -+#else -+DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); -+#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) -+#endif - -- /* Record the original scan target for proportional adjustments later */ -- memcpy(targets, nr, sizeof(nr)); -+/****************************************************************************** -+ * shorthand helpers -+ ******************************************************************************/ - -- /* -- * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal -- * event that can occur when there is little memory pressure e.g. -- * multiple streaming readers/writers. Hence, we do not abort scanning -- * when the requested number of pages are reclaimed when scanning at -- * DEF_PRIORITY on the assumption that the fact we are direct -- * reclaiming implies that kswapd is not keeping up and it is best to -- * do a batch of work at once. For memcg reclaim one check is made to -- * abort proportional reclaim if either the file or anon lru has already -- * dropped to zero at the first pass. -- */ -- scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && -- sc->priority == DEF_PRIORITY); -+#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) - -- blk_start_plug(&plug); -- while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || -- nr[LRU_INACTIVE_FILE]) { -- unsigned long nr_anon, nr_file, percentage; -- unsigned long nr_scanned; -+#define DEFINE_MAX_SEQ(lruvec) \ -+ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) - -- for_each_evictable_lru(lru) { -- if (nr[lru]) { -- nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); -- nr[lru] -= nr_to_scan; -+#define DEFINE_MIN_SEQ(lruvec) \ -+ unsigned long min_seq[ANON_AND_FILE] = { \ -+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ -+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ -+ } - -- nr_reclaimed += shrink_list(lru, nr_to_scan, -- lruvec, sc); -- } -- } -+#define for_each_gen_type_zone(gen, type, zone) \ -+ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ -+ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ -+ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) - -- cond_resched(); -+static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) -+{ -+ struct pglist_data *pgdat = NODE_DATA(nid); - -- if (nr_reclaimed < nr_to_reclaim || scan_adjusted) -- continue; -+#ifdef CONFIG_MEMCG -+ if (memcg) { -+ struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; - -- /* -- * For kswapd and memcg, reclaim at least the number of pages -- * requested. Ensure that the anon and file LRUs are scanned -- * proportionally what was requested by get_scan_count(). We -- * stop reclaiming one LRU and reduce the amount scanning -- * proportional to the original scan target. -- */ -- nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; -- nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; -+ /* for hotadd_new_pgdat() */ -+ if (!lruvec->pgdat) -+ lruvec->pgdat = pgdat; - -- /* -- * It's just vindictive to attack the larger once the smaller -- * has gone to zero. And given the way we stop scanning the -- * smaller below, this makes sure that we only make one nudge -- * towards proportionality once we've got nr_to_reclaim. -- */ -- if (!nr_file || !nr_anon) -- break; -+ return lruvec; -+ } -+#endif -+ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); - -- if (nr_file > nr_anon) { -- unsigned long scan_target = targets[LRU_INACTIVE_ANON] + -- targets[LRU_ACTIVE_ANON] + 1; -- lru = LRU_BASE; -- percentage = nr_anon * 100 / scan_target; -- } else { -- unsigned long scan_target = targets[LRU_INACTIVE_FILE] + -- targets[LRU_ACTIVE_FILE] + 1; -- lru = LRU_FILE; -- percentage = nr_file * 100 / scan_target; -- } -+ return pgdat ? &pgdat->__lruvec : NULL; -+} - -- /* Stop scanning the smaller of the LRU */ -- nr[lru] = 0; -- nr[lru + LRU_ACTIVE] = 0; -+static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(lruvec); - -- /* -- * Recalculate the other LRU scan count based on its original -- * scan target and the percentage scanning already complete -- */ -- lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; -- nr_scanned = targets[lru] - nr[lru]; -- nr[lru] = targets[lru] * (100 - percentage) / 100; -- nr[lru] -= min(nr[lru], nr_scanned); -+ if (!can_demote(pgdat->node_id, sc) && -+ mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) -+ return 0; - -- lru += LRU_ACTIVE; -- nr_scanned = targets[lru] - nr[lru]; -- nr[lru] = targets[lru] * (100 - percentage) / 100; -- nr[lru] -= min(nr[lru], nr_scanned); -+ return mem_cgroup_swappiness(memcg); -+} - -- scan_adjusted = true; -- } -- blk_finish_plug(&plug); -- sc->nr_reclaimed += nr_reclaimed; -+static int get_nr_gens(struct lruvec *lruvec, int type) -+{ -+ return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; -+} - -- /* -- * Even if we did not try to evict anon pages at all, we want to -- * rebalance the anon lru active/inactive ratio. -- */ -- if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && -- inactive_is_low(lruvec, LRU_INACTIVE_ANON)) -- shrink_active_list(SWAP_CLUSTER_MAX, lruvec, -- sc, LRU_ACTIVE_ANON); -+static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) -+{ -+ /* see the comment on lru_gen_struct */ -+ return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && -+ get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && -+ get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; - } - --/* Use reclaim/compaction for costly allocs or under memory pressure */ --static bool in_reclaim_compaction(struct scan_control *sc) -+/****************************************************************************** -+ * mm_struct list -+ ******************************************************************************/ -+ -+static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) - { -- if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && -- (sc->order > PAGE_ALLOC_COSTLY_ORDER || -- sc->priority < DEF_PRIORITY - 2)) -- return true; -+ static struct lru_gen_mm_list mm_list = { -+ .fifo = LIST_HEAD_INIT(mm_list.fifo), -+ .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), -+ }; - -- return false; -+#ifdef CONFIG_MEMCG -+ if (memcg) -+ return &memcg->mm_list; -+#endif -+ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); -+ -+ return &mm_list; - } - --/* -- * Reclaim/compaction is used for high-order allocation requests. It reclaims -- * order-0 pages before compacting the zone. should_continue_reclaim() returns -- * true if more pages should be reclaimed such that when the page allocator -- * calls try_to_compact_pages() that it will have enough free pages to succeed. -- * It will give up earlier than that if there is difficulty reclaiming pages. -- */ --static inline bool should_continue_reclaim(struct pglist_data *pgdat, -- unsigned long nr_reclaimed, -- struct scan_control *sc) -+void lru_gen_add_mm(struct mm_struct *mm) - { -- unsigned long pages_for_compaction; -- unsigned long inactive_lru_pages; -- int z; -+ int nid; -+ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); - -- /* If not in reclaim/compaction mode, stop */ -- if (!in_reclaim_compaction(sc)) -- return false; -+ VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); -+#ifdef CONFIG_MEMCG -+ VM_WARN_ON_ONCE(mm->lru_gen.memcg); -+ mm->lru_gen.memcg = memcg; -+#endif -+ spin_lock(&mm_list->lock); - -- /* -- * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX -- * number of pages that were scanned. This will return to the caller -- * with the risk reclaim/compaction and the resulting allocation attempt -- * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL -- * allocations through requiring that the full LRU list has been scanned -- * first, by assuming that zero delta of sc->nr_scanned means full LRU -- * scan, but that approximation was wrong, and there were corner cases -+ for_each_node_state(nid, N_MEMORY) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ if (!lruvec) -+ continue; -+ -+ /* the first addition since the last iteration */ -+ if (lruvec->mm_state.tail == &mm_list->fifo) -+ lruvec->mm_state.tail = &mm->lru_gen.list; -+ } -+ -+ list_add_tail(&mm->lru_gen.list, &mm_list->fifo); -+ -+ spin_unlock(&mm_list->lock); -+} -+ -+void lru_gen_del_mm(struct mm_struct *mm) -+{ -+ int nid; -+ struct lru_gen_mm_list *mm_list; -+ struct mem_cgroup *memcg = NULL; -+ -+ if (list_empty(&mm->lru_gen.list)) -+ return; -+ -+#ifdef CONFIG_MEMCG -+ memcg = mm->lru_gen.memcg; -+#endif -+ mm_list = get_mm_list(memcg); -+ -+ spin_lock(&mm_list->lock); -+ -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ if (!lruvec) -+ continue; -+ -+ /* where the last iteration ended (exclusive) */ -+ if (lruvec->mm_state.tail == &mm->lru_gen.list) -+ lruvec->mm_state.tail = lruvec->mm_state.tail->next; -+ -+ /* where the current iteration continues (inclusive) */ -+ if (lruvec->mm_state.head != &mm->lru_gen.list) -+ continue; -+ -+ lruvec->mm_state.head = lruvec->mm_state.head->next; -+ /* the deletion ends the current iteration */ -+ if (lruvec->mm_state.head == &mm_list->fifo) -+ WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); -+ } -+ -+ list_del_init(&mm->lru_gen.list); -+ -+ spin_unlock(&mm_list->lock); -+ -+#ifdef CONFIG_MEMCG -+ mem_cgroup_put(mm->lru_gen.memcg); -+ mm->lru_gen.memcg = NULL; -+#endif -+} -+ -+#ifdef CONFIG_MEMCG -+void lru_gen_migrate_mm(struct mm_struct *mm) -+{ -+ struct mem_cgroup *memcg; -+ struct task_struct *task = rcu_dereference_protected(mm->owner, true); -+ -+ VM_WARN_ON_ONCE(task->mm != mm); -+ lockdep_assert_held(&task->alloc_lock); -+ -+ /* for mm_update_next_owner() */ -+ if (mem_cgroup_disabled()) -+ return; -+ -+ rcu_read_lock(); -+ memcg = mem_cgroup_from_task(task); -+ rcu_read_unlock(); -+ if (memcg == mm->lru_gen.memcg) -+ return; -+ -+ VM_WARN_ON_ONCE(!mm->lru_gen.memcg); -+ VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); -+ -+ lru_gen_del_mm(mm); -+ lru_gen_add_mm(mm); -+} -+#endif -+ -+/* -+ * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when -+ * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of -+ * bits in a bitmap, k is the number of hash functions and n is the number of -+ * inserted items. -+ * -+ * Page table walkers use one of the two filters to reduce their search space. -+ * To get rid of non-leaf entries that no longer have enough leaf entries, the -+ * aging uses the double-buffering technique to flip to the other filter each -+ * time it produces a new generation. For non-leaf entries that have enough -+ * leaf entries, the aging carries them over to the next generation in -+ * walk_pmd_range(); the eviction also report them when walking the rmap -+ * in lru_gen_look_around(). -+ * -+ * For future optimizations: -+ * 1. It's not necessary to keep both filters all the time. The spare one can be -+ * freed after the RCU grace period and reallocated if needed again. -+ * 2. And when reallocating, it's worth scaling its size according to the number -+ * of inserted entries in the other filter, to reduce the memory overhead on -+ * small systems and false positives on large systems. -+ * 3. Jenkins' hash function is an alternative to Knuth's. -+ */ -+#define BLOOM_FILTER_SHIFT 15 -+ -+static inline int filter_gen_from_seq(unsigned long seq) -+{ -+ return seq % NR_BLOOM_FILTERS; -+} -+ -+static void get_item_key(void *item, int *key) -+{ -+ u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); -+ -+ BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); -+ -+ key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); -+ key[1] = hash >> BLOOM_FILTER_SHIFT; -+} -+ -+static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) -+{ -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); -+ -+ filter = lruvec->mm_state.filters[gen]; -+ if (filter) { -+ bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); -+ return; -+ } -+ -+ filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), -+ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); -+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter); -+} -+ -+static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) -+{ -+ int key[2]; -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); -+ -+ filter = READ_ONCE(lruvec->mm_state.filters[gen]); -+ if (!filter) -+ return; -+ -+ get_item_key(item, key); -+ -+ if (!test_bit(key[0], filter)) -+ set_bit(key[0], filter); -+ if (!test_bit(key[1], filter)) -+ set_bit(key[1], filter); -+} -+ -+static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) -+{ -+ int key[2]; -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); -+ -+ filter = READ_ONCE(lruvec->mm_state.filters[gen]); -+ if (!filter) -+ return true; -+ -+ get_item_key(item, key); -+ -+ return test_bit(key[0], filter) && test_bit(key[1], filter); -+} -+ -+static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) -+{ -+ int i; -+ int hist; -+ -+ lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); -+ -+ if (walk) { -+ hist = lru_hist_from_seq(walk->max_seq); -+ -+ for (i = 0; i < NR_MM_STATS; i++) { -+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], -+ lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); -+ walk->mm_stats[i] = 0; -+ } -+ } -+ -+ if (NR_HIST_GENS > 1 && last) { -+ hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); -+ -+ for (i = 0; i < NR_MM_STATS; i++) -+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); -+ } -+} -+ -+static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) -+{ -+ int type; -+ unsigned long size = 0; -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); -+ -+ if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) -+ return true; -+ -+ clear_bit(key, &mm->lru_gen.bitmap); -+ -+ for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { -+ size += type ? get_mm_counter(mm, MM_FILEPAGES) : -+ get_mm_counter(mm, MM_ANONPAGES) + -+ get_mm_counter(mm, MM_SHMEMPAGES); -+ } -+ -+ if (size < MIN_LRU_BATCH) -+ return true; -+ -+ return !mmget_not_zero(mm); -+} -+ -+static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, -+ struct mm_struct **iter) -+{ -+ bool first = false; -+ bool last = true; -+ struct mm_struct *mm = NULL; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); -+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; -+ -+ /* -+ * There are four interesting cases for this page table walker: -+ * 1. It tries to start a new iteration of mm_list with a stale max_seq; -+ * there is nothing left to do. -+ * 2. It's the first of the current generation, and it needs to reset -+ * the Bloom filter for the next generation. -+ * 3. It reaches the end of mm_list, and it needs to increment -+ * mm_state->seq; the iteration is done. -+ * 4. It's the last of the current generation, and it needs to reset the -+ * mm stats counters for the next generation. -+ */ -+ spin_lock(&mm_list->lock); -+ -+ VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); -+ VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); -+ VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); -+ -+ if (walk->max_seq <= mm_state->seq) { -+ if (!*iter) -+ last = false; -+ goto done; -+ } -+ -+ if (!mm_state->nr_walkers) { -+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); -+ -+ mm_state->head = mm_list->fifo.next; -+ first = true; -+ } -+ -+ while (!mm && mm_state->head != &mm_list->fifo) { -+ mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); -+ -+ mm_state->head = mm_state->head->next; -+ -+ /* force scan for those added after the last iteration */ -+ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { -+ mm_state->tail = mm_state->head; -+ walk->force_scan = true; -+ } -+ -+ if (should_skip_mm(mm, walk)) -+ mm = NULL; -+ } -+ -+ if (mm_state->head == &mm_list->fifo) -+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); -+done: -+ if (*iter && !mm) -+ mm_state->nr_walkers--; -+ if (!*iter && mm) -+ mm_state->nr_walkers++; -+ -+ if (mm_state->nr_walkers) -+ last = false; -+ -+ if (*iter || last) -+ reset_mm_stats(lruvec, walk, last); -+ -+ spin_unlock(&mm_list->lock); -+ -+ if (mm && first) -+ reset_bloom_filter(lruvec, walk->max_seq + 1); -+ -+ if (*iter) -+ mmput_async(*iter); -+ -+ *iter = mm; -+ -+ return last; -+} -+ -+static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) -+{ -+ bool success = false; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); -+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; -+ -+ spin_lock(&mm_list->lock); -+ -+ VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); -+ -+ if (max_seq > mm_state->seq && !mm_state->nr_walkers) { -+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); -+ -+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); -+ reset_mm_stats(lruvec, NULL, true); -+ success = true; -+ } -+ -+ spin_unlock(&mm_list->lock); -+ -+ return success; -+} -+ -+/****************************************************************************** -+ * refault feedback loop -+ ******************************************************************************/ -+ -+/* -+ * A feedback loop based on Proportional-Integral-Derivative (PID) controller. -+ * -+ * The P term is refaulted/(evicted+protected) from a tier in the generation -+ * currently being evicted; the I term is the exponential moving average of the -+ * P term over the generations previously evicted, using the smoothing factor -+ * 1/2; the D term isn't supported. -+ * -+ * The setpoint (SP) is always the first tier of one type; the process variable -+ * (PV) is either any tier of the other type or any other tier of the same -+ * type. -+ * -+ * The error is the difference between the SP and the PV; the correction is to -+ * turn off protection when SP>PV or turn on protection when SPlrugen; -+ int hist = lru_hist_from_seq(lrugen->min_seq[type]); -+ -+ pos->refaulted = lrugen->avg_refaulted[type][tier] + -+ atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ pos->total = lrugen->avg_total[type][tier] + -+ atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ pos->total += lrugen->protected[hist][type][tier - 1]; -+ pos->gain = gain; -+} -+ -+static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) -+{ -+ int hist, tier; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; -+ unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; -+ -+ lockdep_assert_held(&lruvec->lru_lock); -+ -+ if (!carryover && !clear) -+ return; -+ -+ hist = lru_hist_from_seq(seq); -+ -+ for (tier = 0; tier < MAX_NR_TIERS; tier++) { -+ if (carryover) { -+ unsigned long sum; -+ -+ sum = lrugen->avg_refaulted[type][tier] + -+ atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); -+ -+ sum = lrugen->avg_total[type][tier] + -+ atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ sum += lrugen->protected[hist][type][tier - 1]; -+ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); -+ } -+ -+ if (clear) { -+ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); -+ atomic_long_set(&lrugen->evicted[hist][type][tier], 0); -+ if (tier) -+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); -+ } -+ } -+} -+ -+static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) -+{ -+ /* -+ * Return true if the PV has a limited number of refaults or a lower -+ * refaulted/total than the SP. -+ */ -+ return pv->refaulted < MIN_LRU_BATCH || -+ pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= -+ (sp->refaulted + 1) * pv->total * pv->gain; -+} -+ -+/****************************************************************************** -+ * the aging -+ ******************************************************************************/ -+ -+/* promote pages accessed through page tables */ -+static int folio_update_gen(struct folio *folio, int gen) -+{ -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); -+ -+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(!rcu_read_lock_held()); -+ -+ do { -+ /* lru_gen_del_folio() has isolated this page? */ -+ if (!(old_flags & LRU_GEN_MASK)) { -+ /* for shrink_page_list() */ -+ new_flags = old_flags | BIT(PG_referenced); -+ continue; -+ } -+ -+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); -+ new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); -+ -+ return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+} -+ -+/* protect pages accessed multiple times through file descriptors */ -+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ int type = folio_is_file_lru(folio); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); -+ -+ VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); -+ -+ do { -+ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+ /* folio_update_gen() has promoted this page? */ -+ if (new_gen >= 0 && new_gen != old_gen) -+ return new_gen; -+ -+ new_gen = (old_gen + 1) % MAX_NR_GENS; -+ -+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); -+ new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; -+ /* for folio_end_writeback() */ -+ if (reclaiming) -+ new_flags |= BIT(PG_reclaim); -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); -+ -+ lru_gen_update_size(lruvec, folio, old_gen, new_gen); -+ -+ return new_gen; -+} -+ -+static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, -+ int old_gen, int new_gen) -+{ -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ -+ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); -+ -+ walk->batched++; -+ -+ walk->nr_pages[old_gen][type][zone] -= delta; -+ walk->nr_pages[new_gen][type][zone] += delta; -+} -+ -+static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) -+{ -+ int gen, type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ walk->batched = 0; -+ -+ for_each_gen_type_zone(gen, type, zone) { -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ int delta = walk->nr_pages[gen][type][zone]; -+ -+ if (!delta) -+ continue; -+ -+ walk->nr_pages[gen][type][zone] = 0; -+ WRITE_ONCE(lrugen->nr_pages[gen][type][zone], -+ lrugen->nr_pages[gen][type][zone] + delta); -+ -+ if (lru_gen_is_active(lruvec, gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, delta); -+ } -+} -+ -+static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) -+{ -+ struct address_space *mapping; -+ struct vm_area_struct *vma = args->vma; -+ struct lru_gen_mm_walk *walk = args->private; -+ -+ if (!vma_is_accessible(vma)) -+ return true; -+ -+ if (is_vm_hugetlb_page(vma)) -+ return true; -+ -+ if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) -+ return true; -+ -+ if (vma == get_gate_vma(vma->vm_mm)) -+ return true; -+ -+ if (vma_is_anonymous(vma)) -+ return !walk->can_swap; -+ -+ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) -+ return true; -+ -+ mapping = vma->vm_file->f_mapping; -+ if (mapping_unevictable(mapping)) -+ return true; -+ -+ if (shmem_mapping(mapping)) -+ return !walk->can_swap; -+ -+ /* to exclude special mappings like dax, etc. */ -+ return !mapping->a_ops->read_folio; -+} -+ -+/* -+ * Some userspace memory allocators map many single-page VMAs. Instead of -+ * returning back to the PGD table for each of such VMAs, finish an entire PMD -+ * table to reduce zigzags and improve cache performance. -+ */ -+static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, -+ unsigned long *vm_start, unsigned long *vm_end) -+{ -+ unsigned long start = round_up(*vm_end, size); -+ unsigned long end = (start | ~mask) + 1; -+ VMA_ITERATOR(vmi, args->mm, start); -+ -+ VM_WARN_ON_ONCE(mask & size); -+ VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); -+ -+ for_each_vma(vmi, args->vma) { -+ if (end && end <= args->vma->vm_start) -+ return false; -+ -+ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) -+ continue; -+ -+ *vm_start = max(start, args->vma->vm_start); -+ *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; -+ -+ return true; -+ } -+ -+ return false; -+} -+ -+static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) -+{ -+ unsigned long pfn = pte_pfn(pte); -+ -+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); -+ -+ if (!pte_present(pte) || is_zero_pfn(pfn)) -+ return -1; -+ -+ if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) -+ return -1; -+ -+ if (WARN_ON_ONCE(!pfn_valid(pfn))) -+ return -1; -+ -+ return pfn; -+} -+ -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -+static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) -+{ -+ unsigned long pfn = pmd_pfn(pmd); -+ -+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); -+ -+ if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) -+ return -1; -+ -+ if (WARN_ON_ONCE(pmd_devmap(pmd))) -+ return -1; -+ -+ if (WARN_ON_ONCE(!pfn_valid(pfn))) -+ return -1; -+ -+ return pfn; -+} -+#endif -+ -+static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, -+ struct pglist_data *pgdat, bool can_swap) -+{ -+ struct folio *folio; -+ -+ /* try to avoid unnecessary memory loads */ -+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) -+ return NULL; -+ -+ folio = pfn_folio(pfn); -+ if (folio_nid(folio) != pgdat->node_id) -+ return NULL; -+ -+ if (folio_memcg_rcu(folio) != memcg) -+ return NULL; -+ -+ /* file VMAs can contain anon pages from COW */ -+ if (!folio_is_file_lru(folio) && !can_swap) -+ return NULL; -+ -+ return folio; -+} -+ -+static bool suitable_to_scan(int total, int young) -+{ -+ int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); -+ -+ /* suitable if the average number of young PTEs per cacheline is >=1 */ -+ return young * n >= total; -+} -+ -+static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, -+ struct mm_walk *args) -+{ -+ int i; -+ pte_t *pte; -+ spinlock_t *ptl; -+ unsigned long addr; -+ int total = 0; -+ int young = 0; -+ struct lru_gen_mm_walk *walk = args->private; -+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); -+ -+ VM_WARN_ON_ONCE(pmd_leaf(*pmd)); -+ -+ ptl = pte_lockptr(args->mm, pmd); -+ if (!spin_trylock(ptl)) -+ return false; -+ -+ arch_enter_lazy_mmu_mode(); -+ -+ pte = pte_offset_map(pmd, start & PMD_MASK); -+restart: -+ for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { -+ unsigned long pfn; -+ struct folio *folio; -+ -+ total++; -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ -+ pfn = get_pte_pfn(pte[i], args->vma, addr); -+ if (pfn == -1) -+ continue; -+ -+ if (!pte_young(pte[i])) { -+ walk->mm_stats[MM_LEAF_OLD]++; -+ continue; -+ } -+ -+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); -+ if (!folio) -+ continue; -+ -+ if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) -+ VM_WARN_ON_ONCE(true); -+ -+ young++; -+ walk->mm_stats[MM_LEAF_YOUNG]++; -+ -+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); -+ -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen >= 0 && old_gen != new_gen) -+ update_batch_size(walk, folio, old_gen, new_gen); -+ } -+ -+ if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) -+ goto restart; -+ -+ pte_unmap(pte); -+ -+ arch_leave_lazy_mmu_mode(); -+ spin_unlock(ptl); -+ -+ return suitable_to_scan(total, young); -+} -+ -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, -+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) -+{ -+ int i; -+ pmd_t *pmd; -+ spinlock_t *ptl; -+ struct lru_gen_mm_walk *walk = args->private; -+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); -+ -+ VM_WARN_ON_ONCE(pud_leaf(*pud)); -+ -+ /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ -+ if (*start == -1) { -+ *start = next; -+ return; -+ } -+ -+ i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); -+ if (i && i <= MIN_LRU_BATCH) { -+ __set_bit(i - 1, bitmap); -+ return; -+ } -+ -+ pmd = pmd_offset(pud, *start); -+ -+ ptl = pmd_lockptr(args->mm, pmd); -+ if (!spin_trylock(ptl)) -+ goto done; -+ -+ arch_enter_lazy_mmu_mode(); -+ -+ do { -+ unsigned long pfn; -+ struct folio *folio; -+ unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; -+ -+ pfn = get_pmd_pfn(pmd[i], vma, addr); -+ if (pfn == -1) -+ goto next; -+ -+ if (!pmd_trans_huge(pmd[i])) { -+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && -+ get_cap(LRU_GEN_NONLEAF_YOUNG)) -+ pmdp_test_and_clear_young(vma, addr, pmd + i); -+ goto next; -+ } -+ -+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); -+ if (!folio) -+ goto next; -+ -+ if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) -+ goto next; -+ -+ walk->mm_stats[MM_LEAF_YOUNG]++; -+ -+ if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); -+ -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen >= 0 && old_gen != new_gen) -+ update_batch_size(walk, folio, old_gen, new_gen); -+next: -+ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; -+ } while (i <= MIN_LRU_BATCH); -+ -+ arch_leave_lazy_mmu_mode(); -+ spin_unlock(ptl); -+done: -+ *start = -1; -+ bitmap_zero(bitmap, MIN_LRU_BATCH); -+} -+#else -+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, -+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) -+{ -+} -+#endif -+ -+static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, -+ struct mm_walk *args) -+{ -+ int i; -+ pmd_t *pmd; -+ unsigned long next; -+ unsigned long addr; -+ struct vm_area_struct *vma; -+ unsigned long pos = -1; -+ struct lru_gen_mm_walk *walk = args->private; -+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; -+ -+ VM_WARN_ON_ONCE(pud_leaf(*pud)); -+ -+ /* -+ * Finish an entire PMD in two passes: the first only reaches to PTE -+ * tables to avoid taking the PMD lock; the second, if necessary, takes -+ * the PMD lock to clear the accessed bit in PMD entries. -+ */ -+ pmd = pmd_offset(pud, start & PUD_MASK); -+restart: -+ /* walk_pte_range() may call get_next_vma() */ -+ vma = args->vma; -+ for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { -+ pmd_t val = pmd_read_atomic(pmd + i); -+ -+ /* for pmd_read_atomic() */ -+ barrier(); -+ -+ next = pmd_addr_end(addr, end); -+ -+ if (!pmd_present(val) || is_huge_zero_pmd(val)) { -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ continue; -+ } -+ -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ if (pmd_trans_huge(val)) { -+ unsigned long pfn = pmd_pfn(val); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ -+ if (!pmd_young(val)) { -+ walk->mm_stats[MM_LEAF_OLD]++; -+ continue; -+ } -+ -+ /* try to avoid unnecessary memory loads */ -+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) -+ continue; -+ -+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); -+ continue; -+ } -+#endif -+ walk->mm_stats[MM_NONLEAF_TOTAL]++; -+ -+#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG -+ if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { -+ if (!pmd_young(val)) -+ continue; -+ -+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); -+ } -+#endif -+ if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) -+ continue; -+ -+ walk->mm_stats[MM_NONLEAF_FOUND]++; -+ -+ if (!walk_pte_range(&val, addr, next, args)) -+ continue; -+ -+ walk->mm_stats[MM_NONLEAF_ADDED]++; -+ -+ /* carry over to the next generation */ -+ update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); -+ } -+ -+ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); -+ -+ if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) -+ goto restart; -+} -+ -+static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, -+ struct mm_walk *args) -+{ -+ int i; -+ pud_t *pud; -+ unsigned long addr; -+ unsigned long next; -+ struct lru_gen_mm_walk *walk = args->private; -+ -+ VM_WARN_ON_ONCE(p4d_leaf(*p4d)); -+ -+ pud = pud_offset(p4d, start & P4D_MASK); -+restart: -+ for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { -+ pud_t val = READ_ONCE(pud[i]); -+ -+ next = pud_addr_end(addr, end); -+ -+ if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) -+ continue; -+ -+ walk_pmd_range(&val, addr, next, args); -+ -+ /* a racy check to curtail the waiting time */ -+ if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) -+ return 1; -+ -+ if (need_resched() || walk->batched >= MAX_LRU_BATCH) { -+ end = (addr | ~PUD_MASK) + 1; -+ goto done; -+ } -+ } -+ -+ if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) -+ goto restart; -+ -+ end = round_up(end, P4D_SIZE); -+done: -+ if (!end || !args->vma) -+ return 1; -+ -+ walk->next_addr = max(end, args->vma->vm_start); -+ -+ return -EAGAIN; -+} -+ -+static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) -+{ -+ static const struct mm_walk_ops mm_walk_ops = { -+ .test_walk = should_skip_vma, -+ .p4d_entry = walk_pud_range, -+ }; -+ -+ int err; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ -+ walk->next_addr = FIRST_USER_ADDRESS; -+ -+ do { -+ err = -EBUSY; -+ -+ /* folio_update_gen() requires stable folio_memcg() */ -+ if (!mem_cgroup_trylock_pages(memcg)) -+ break; -+ -+ /* the caller might be holding the lock for write */ -+ if (mmap_read_trylock(mm)) { -+ err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); -+ -+ mmap_read_unlock(mm); -+ } -+ -+ mem_cgroup_unlock_pages(); -+ -+ if (walk->batched) { -+ spin_lock_irq(&lruvec->lru_lock); -+ reset_batch_size(lruvec, walk); -+ spin_unlock_irq(&lruvec->lru_lock); -+ } -+ -+ cond_resched(); -+ } while (err == -EAGAIN); -+} -+ -+static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) -+{ -+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; -+ -+ if (pgdat && current_is_kswapd()) { -+ VM_WARN_ON_ONCE(walk); -+ -+ walk = &pgdat->mm_walk; -+ } else if (!pgdat && !walk) { -+ VM_WARN_ON_ONCE(current_is_kswapd()); -+ -+ walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); -+ } -+ -+ current->reclaim_state->mm_walk = walk; -+ -+ return walk; -+} -+ -+static void clear_mm_walk(void) -+{ -+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; -+ -+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); -+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); -+ -+ current->reclaim_state->mm_walk = NULL; -+ -+ if (!current_is_kswapd()) -+ kfree(walk); -+} -+ -+static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) -+{ -+ int zone; -+ int remaining = MAX_LRU_BATCH; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ -+ if (type == LRU_GEN_ANON && !can_swap) -+ goto done; -+ -+ /* prevent cold/hot inversion if force_scan is true */ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ struct list_head *head = &lrugen->lists[old_gen][type][zone]; -+ -+ while (!list_empty(head)) { -+ struct folio *folio = lru_to_folio(head); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); -+ -+ new_gen = folio_inc_gen(lruvec, folio, false); -+ list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); -+ -+ if (!--remaining) -+ return false; -+ } -+ } -+done: -+ reset_ctrl_pos(lruvec, type, true); -+ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); -+ -+ return true; -+} -+ -+static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) -+{ -+ int gen, type, zone; -+ bool success = false; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ DEFINE_MIN_SEQ(lruvec); -+ -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ -+ /* find the oldest populated generation */ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { -+ gen = lru_gen_from_seq(min_seq[type]); -+ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ if (!list_empty(&lrugen->lists[gen][type][zone])) -+ goto next; -+ } -+ -+ min_seq[type]++; -+ } -+next: -+ ; -+ } -+ -+ /* see the comment on lru_gen_struct */ -+ if (can_swap) { -+ min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); -+ min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); -+ } -+ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ if (min_seq[type] == lrugen->min_seq[type]) -+ continue; -+ -+ reset_ctrl_pos(lruvec, type, true); -+ WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); -+ success = true; -+ } -+ -+ return success; -+} -+ -+static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) -+{ -+ int prev, next; -+ int type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ -+ for (type = ANON_AND_FILE - 1; type >= 0; type--) { -+ if (get_nr_gens(lruvec, type) != MAX_NR_GENS) -+ continue; -+ -+ VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); -+ -+ while (!inc_min_seq(lruvec, type, can_swap)) { -+ spin_unlock_irq(&lruvec->lru_lock); -+ cond_resched(); -+ spin_lock_irq(&lruvec->lru_lock); -+ } -+ } -+ -+ /* -+ * Update the active/inactive LRU sizes for compatibility. Both sides of -+ * the current max_seq need to be covered, since max_seq+1 can overlap -+ * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do -+ * overlap, cold/hot inversion happens. -+ */ -+ prev = lru_gen_from_seq(lrugen->max_seq - 1); -+ next = lru_gen_from_seq(lrugen->max_seq + 1); -+ -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ long delta = lrugen->nr_pages[prev][type][zone] - -+ lrugen->nr_pages[next][type][zone]; -+ -+ if (!delta) -+ continue; -+ -+ __update_lru_size(lruvec, lru, zone, delta); -+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); -+ } -+ } -+ -+ for (type = 0; type < ANON_AND_FILE; type++) -+ reset_ctrl_pos(lruvec, type, false); -+ -+ WRITE_ONCE(lrugen->timestamps[next], jiffies); -+ /* make sure preceding modifications appear */ -+ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+} -+ -+static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, -+ struct scan_control *sc, bool can_swap, bool force_scan) -+{ -+ bool success; -+ struct lru_gen_mm_walk *walk; -+ struct mm_struct *mm = NULL; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); -+ -+ /* see the comment in iterate_mm_list() */ -+ if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { -+ success = false; -+ goto done; -+ } -+ -+ /* -+ * If the hardware doesn't automatically set the accessed bit, fallback -+ * to lru_gen_look_around(), which only clears the accessed bit in a -+ * handful of PTEs. Spreading the work out over a period of time usually -+ * is less efficient, but it avoids bursty page faults. -+ */ -+ if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { -+ success = iterate_mm_list_nowalk(lruvec, max_seq); -+ goto done; -+ } -+ -+ walk = set_mm_walk(NULL); -+ if (!walk) { -+ success = iterate_mm_list_nowalk(lruvec, max_seq); -+ goto done; -+ } -+ -+ walk->lruvec = lruvec; -+ walk->max_seq = max_seq; -+ walk->can_swap = can_swap; -+ walk->force_scan = force_scan; -+ -+ do { -+ success = iterate_mm_list(lruvec, walk, &mm); -+ if (mm) -+ walk_mm(lruvec, mm, walk); -+ -+ cond_resched(); -+ } while (mm); -+done: -+ if (!success) { -+ if (sc->priority <= DEF_PRIORITY - 2) -+ wait_event_killable(lruvec->mm_state.wait, -+ max_seq < READ_ONCE(lrugen->max_seq)); -+ -+ return max_seq < READ_ONCE(lrugen->max_seq); -+ } -+ -+ VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); -+ -+ inc_max_seq(lruvec, can_swap, force_scan); -+ /* either this sees any waiters or they will see updated max_seq */ -+ if (wq_has_sleeper(&lruvec->mm_state.wait)) -+ wake_up_all(&lruvec->mm_state.wait); -+ -+ wakeup_flusher_threads(WB_REASON_VMSCAN); -+ -+ return true; -+} -+ -+static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, -+ struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) -+{ -+ int gen, type, zone; -+ unsigned long old = 0; -+ unsigned long young = 0; -+ unsigned long total = 0; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ unsigned long seq; -+ -+ for (seq = min_seq[type]; seq <= max_seq; seq++) { -+ unsigned long size = 0; -+ -+ gen = lru_gen_from_seq(seq); -+ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) -+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); -+ -+ total += size; -+ if (seq == max_seq) -+ young += size; -+ else if (seq + MIN_NR_GENS == max_seq) -+ old += size; -+ } -+ } -+ -+ /* try to scrape all its memory if this memcg was deleted */ -+ *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; -+ -+ /* -+ * The aging tries to be lazy to reduce the overhead, while the eviction -+ * stalls when the number of generations reaches MIN_NR_GENS. Hence, the -+ * ideal number of generations is MIN_NR_GENS+1. -+ */ -+ if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) -+ return true; -+ if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) -+ return false; -+ -+ /* -+ * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) -+ * of the total number of pages for each generation. A reasonable range -+ * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The -+ * aging cares about the upper bound of hot pages, while the eviction -+ * cares about the lower bound of cold pages. -+ */ -+ if (young * MIN_NR_GENS > total) -+ return true; -+ if (old * (MIN_NR_GENS + 2) < total) -+ return true; -+ -+ return false; -+} -+ -+static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) -+{ -+ bool need_aging; -+ unsigned long nr_to_scan; -+ int swappiness = get_swappiness(lruvec, sc); -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ VM_WARN_ON_ONCE(sc->memcg_low_reclaim); -+ -+ mem_cgroup_calculate_protection(NULL, memcg); -+ -+ if (mem_cgroup_below_min(memcg)) -+ return false; -+ -+ need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); -+ -+ if (min_ttl) { -+ int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); -+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); -+ -+ if (time_is_after_jiffies(birth + min_ttl)) -+ return false; -+ -+ /* the size is likely too small to be helpful */ -+ if (!nr_to_scan && sc->priority != DEF_PRIORITY) -+ return false; -+ } -+ -+ if (need_aging) -+ try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); -+ -+ return true; -+} -+ -+/* to protect the working set of the last N jiffies */ -+static unsigned long lru_gen_min_ttl __read_mostly; -+ -+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -+{ -+ struct mem_cgroup *memcg; -+ bool success = false; -+ unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); -+ -+ VM_WARN_ON_ONCE(!current_is_kswapd()); -+ -+ sc->last_reclaimed = sc->nr_reclaimed; -+ -+ /* -+ * To reduce the chance of going into the aging path, which can be -+ * costly, optimistically skip it if the flag below was cleared in the -+ * eviction path. This improves the overall performance when multiple -+ * memcgs are available. -+ */ -+ if (!sc->memcgs_need_aging) { -+ sc->memcgs_need_aging = true; -+ return; -+ } -+ -+ set_mm_walk(pgdat); -+ -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ -+ if (age_lruvec(lruvec, sc, min_ttl)) -+ success = true; -+ -+ cond_resched(); -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+ -+ clear_mm_walk(); -+ -+ /* check the order to exclude compaction-induced reclaim */ -+ if (success || !min_ttl || sc->order) -+ return; -+ -+ /* -+ * The main goal is to OOM kill if every generation from all memcgs is -+ * younger than min_ttl. However, another possibility is all memcgs are -+ * either below min or empty. -+ */ -+ if (mutex_trylock(&oom_lock)) { -+ struct oom_control oc = { -+ .gfp_mask = sc->gfp_mask, -+ }; -+ -+ out_of_memory(&oc); -+ -+ mutex_unlock(&oom_lock); -+ } -+} -+ -+/* -+ * This function exploits spatial locality when shrink_page_list() walks the -+ * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If -+ * the scan was done cacheline efficiently, it adds the PMD entry pointing to -+ * the PTE table to the Bloom filter. This forms a feedback loop between the -+ * eviction and the aging. -+ */ -+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) -+{ -+ int i; -+ pte_t *pte; -+ unsigned long start; -+ unsigned long end; -+ unsigned long addr; -+ struct lru_gen_mm_walk *walk; -+ int young = 0; -+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; -+ struct folio *folio = pfn_folio(pvmw->pfn); -+ struct mem_cgroup *memcg = folio_memcg(folio); -+ struct pglist_data *pgdat = folio_pgdat(folio); -+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ DEFINE_MAX_SEQ(lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(max_seq); -+ -+ lockdep_assert_held(pvmw->ptl); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); -+ -+ if (spin_is_contended(pvmw->ptl)) -+ return; -+ -+ /* avoid taking the LRU lock under the PTL when possible */ -+ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; -+ -+ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); -+ end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; -+ -+ if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { -+ if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) -+ end = start + MIN_LRU_BATCH * PAGE_SIZE; -+ else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) -+ start = end - MIN_LRU_BATCH * PAGE_SIZE; -+ else { -+ start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; -+ end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; -+ } -+ } -+ -+ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; -+ -+ rcu_read_lock(); -+ arch_enter_lazy_mmu_mode(); -+ -+ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { -+ unsigned long pfn; -+ -+ pfn = get_pte_pfn(pte[i], pvmw->vma, addr); -+ if (pfn == -1) -+ continue; -+ -+ if (!pte_young(pte[i])) -+ continue; -+ -+ folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); -+ if (!folio) -+ continue; -+ -+ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) -+ VM_WARN_ON_ONCE(true); -+ -+ young++; -+ -+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); -+ -+ old_gen = folio_lru_gen(folio); -+ if (old_gen < 0) -+ folio_set_referenced(folio); -+ else if (old_gen != new_gen) -+ __set_bit(i, bitmap); -+ } -+ -+ arch_leave_lazy_mmu_mode(); -+ rcu_read_unlock(); -+ -+ /* feedback from rmap walkers to page table walkers */ -+ if (suitable_to_scan(i, young)) -+ update_bloom_filter(lruvec, max_seq, pvmw->pmd); -+ -+ if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { -+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { -+ folio = pfn_folio(pte_pfn(pte[i])); -+ folio_activate(folio); -+ } -+ return; -+ } -+ -+ /* folio_update_gen() requires stable folio_memcg() */ -+ if (!mem_cgroup_trylock_pages(memcg)) -+ return; -+ -+ if (!walk) { -+ spin_lock_irq(&lruvec->lru_lock); -+ new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); -+ } -+ -+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { -+ folio = pfn_folio(pte_pfn(pte[i])); -+ if (folio_memcg_rcu(folio) != memcg) -+ continue; -+ -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen < 0 || old_gen == new_gen) -+ continue; -+ -+ if (walk) -+ update_batch_size(walk, folio, old_gen, new_gen); -+ else -+ lru_gen_update_size(lruvec, folio, old_gen, new_gen); -+ } -+ -+ if (!walk) -+ spin_unlock_irq(&lruvec->lru_lock); -+ -+ mem_cgroup_unlock_pages(); -+} -+ -+/****************************************************************************** -+ * the eviction -+ ******************************************************************************/ -+ -+static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) -+{ -+ bool success; -+ int gen = folio_lru_gen(folio); -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ int refs = folio_lru_refs(folio); -+ int tier = lru_tier_from_refs(refs); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); -+ -+ /* unevictable */ -+ if (!folio_evictable(folio)) { -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ folio_set_unevictable(folio); -+ lruvec_add_folio(lruvec, folio); -+ __count_vm_events(UNEVICTABLE_PGCULLED, delta); -+ return true; -+ } -+ -+ /* dirty lazyfree */ -+ if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ folio_set_swapbacked(folio); -+ lruvec_add_folio_tail(lruvec, folio); -+ return true; -+ } -+ -+ /* promoted */ -+ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { -+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); -+ return true; -+ } -+ -+ /* protected */ -+ if (tier > tier_idx) { -+ int hist = lru_hist_from_seq(lrugen->min_seq[type]); -+ -+ gen = folio_inc_gen(lruvec, folio, false); -+ list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); -+ -+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], -+ lrugen->protected[hist][type][tier - 1] + delta); -+ __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); -+ return true; -+ } -+ -+ /* waiting for writeback */ -+ if (folio_test_locked(folio) || folio_test_writeback(folio) || -+ (type == LRU_GEN_FILE && folio_test_dirty(folio))) { -+ gen = folio_inc_gen(lruvec, folio, true); -+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); -+ return true; -+ } -+ -+ return false; -+} -+ -+static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) -+{ -+ bool success; -+ -+ /* unmapping inhibited */ -+ if (!sc->may_unmap && folio_mapped(folio)) -+ return false; -+ -+ /* swapping inhibited */ -+ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && -+ (folio_test_dirty(folio) || -+ (folio_test_anon(folio) && !folio_test_swapcache(folio)))) -+ return false; -+ -+ /* raced with release_pages() */ -+ if (!folio_try_get(folio)) -+ return false; -+ -+ /* raced with another isolation */ -+ if (!folio_test_clear_lru(folio)) { -+ folio_put(folio); -+ return false; -+ } -+ -+ /* see the comment on MAX_NR_TIERS */ -+ if (!folio_test_referenced(folio)) -+ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); -+ -+ /* for shrink_page_list() */ -+ folio_clear_reclaim(folio); -+ folio_clear_referenced(folio); -+ -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ -+ return true; -+} -+ -+static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, -+ int type, int tier, struct list_head *list) -+{ -+ int gen, zone; -+ enum vm_event_item item; -+ int sorted = 0; -+ int scanned = 0; -+ int isolated = 0; -+ int remaining = MAX_LRU_BATCH; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ -+ VM_WARN_ON_ONCE(!list_empty(list)); -+ -+ if (get_nr_gens(lruvec, type) == MIN_NR_GENS) -+ return 0; -+ -+ gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ -+ for (zone = sc->reclaim_idx; zone >= 0; zone--) { -+ LIST_HEAD(moved); -+ int skipped = 0; -+ struct list_head *head = &lrugen->lists[gen][type][zone]; -+ -+ while (!list_empty(head)) { -+ struct folio *folio = lru_to_folio(head); -+ int delta = folio_nr_pages(folio); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); -+ -+ scanned += delta; -+ -+ if (sort_folio(lruvec, folio, tier)) -+ sorted += delta; -+ else if (isolate_folio(lruvec, folio, sc)) { -+ list_add(&folio->lru, list); -+ isolated += delta; -+ } else { -+ list_move(&folio->lru, &moved); -+ skipped += delta; -+ } -+ -+ if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) -+ break; -+ } -+ -+ if (skipped) { -+ list_splice(&moved, head); -+ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); -+ } -+ -+ if (!remaining || isolated >= MIN_LRU_BATCH) -+ break; -+ } -+ -+ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; -+ if (!cgroup_reclaim(sc)) { -+ __count_vm_events(item, isolated); -+ __count_vm_events(PGREFILL, sorted); -+ } -+ __count_memcg_events(memcg, item, isolated); -+ __count_memcg_events(memcg, PGREFILL, sorted); -+ __count_vm_events(PGSCAN_ANON + type, isolated); -+ -+ /* -+ * There might not be eligible pages due to reclaim_idx, may_unmap and -+ * may_writepage. Check the remaining to prevent livelock if it's not -+ * making progress. -+ */ -+ return isolated || !remaining ? scanned : 0; -+} -+ -+static int get_tier_idx(struct lruvec *lruvec, int type) -+{ -+ int tier; -+ struct ctrl_pos sp, pv; -+ -+ /* -+ * To leave a margin for fluctuations, use a larger gain factor (1:2). -+ * This value is chosen because any other tier would have at least twice -+ * as many refaults as the first tier. -+ */ -+ read_ctrl_pos(lruvec, type, 0, 1, &sp); -+ for (tier = 1; tier < MAX_NR_TIERS; tier++) { -+ read_ctrl_pos(lruvec, type, tier, 2, &pv); -+ if (!positive_ctrl_err(&sp, &pv)) -+ break; -+ } -+ -+ return tier - 1; -+} -+ -+static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) -+{ -+ int type, tier; -+ struct ctrl_pos sp, pv; -+ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; -+ -+ /* -+ * Compare the first tier of anon with that of file to determine which -+ * type to scan. Also need to compare other tiers of the selected type -+ * with the first tier of the other type to determine the last tier (of -+ * the selected type) to evict. -+ */ -+ read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); -+ read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); -+ type = positive_ctrl_err(&sp, &pv); -+ -+ read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); -+ for (tier = 1; tier < MAX_NR_TIERS; tier++) { -+ read_ctrl_pos(lruvec, type, tier, gain[type], &pv); -+ if (!positive_ctrl_err(&sp, &pv)) -+ break; -+ } -+ -+ *tier_idx = tier - 1; -+ -+ return type; -+} -+ -+static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, -+ int *type_scanned, struct list_head *list) -+{ -+ int i; -+ int type; -+ int scanned; -+ int tier = -1; -+ DEFINE_MIN_SEQ(lruvec); -+ -+ /* -+ * Try to make the obvious choice first. When anon and file are both -+ * available from the same generation, interpret swappiness 1 as file -+ * first and 200 as anon first. -+ */ -+ if (!swappiness) -+ type = LRU_GEN_FILE; -+ else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) -+ type = LRU_GEN_ANON; -+ else if (swappiness == 1) -+ type = LRU_GEN_FILE; -+ else if (swappiness == 200) -+ type = LRU_GEN_ANON; -+ else -+ type = get_type_to_scan(lruvec, swappiness, &tier); -+ -+ for (i = !swappiness; i < ANON_AND_FILE; i++) { -+ if (tier < 0) -+ tier = get_tier_idx(lruvec, type); -+ -+ scanned = scan_folios(lruvec, sc, type, tier, list); -+ if (scanned) -+ break; -+ -+ type = !type; -+ tier = -1; -+ } -+ -+ *type_scanned = type; -+ -+ return scanned; -+} -+ -+static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, -+ bool *need_swapping) -+{ -+ int type; -+ int scanned; -+ int reclaimed; -+ LIST_HEAD(list); -+ struct folio *folio; -+ enum vm_event_item item; -+ struct reclaim_stat stat; -+ struct lru_gen_mm_walk *walk; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(lruvec); -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); -+ -+ scanned += try_to_inc_min_seq(lruvec, swappiness); -+ -+ if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) -+ scanned = 0; -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+ -+ if (list_empty(&list)) -+ return scanned; -+ -+ reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); -+ -+ list_for_each_entry(folio, &list, lru) { -+ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ -+ if (folio_test_workingset(folio)) -+ folio_set_referenced(folio); -+ -+ /* don't add rejected pages to the oldest generation */ -+ if (folio_test_reclaim(folio) && -+ (folio_test_dirty(folio) || folio_test_writeback(folio))) -+ folio_clear_active(folio); -+ else -+ folio_set_active(folio); -+ } -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ move_pages_to_lru(lruvec, &list); -+ -+ walk = current->reclaim_state->mm_walk; -+ if (walk && walk->batched) -+ reset_batch_size(lruvec, walk); -+ -+ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; -+ if (!cgroup_reclaim(sc)) -+ __count_vm_events(item, reclaimed); -+ __count_memcg_events(memcg, item, reclaimed); -+ __count_vm_events(PGSTEAL_ANON + type, reclaimed); -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+ -+ mem_cgroup_uncharge_list(&list); -+ free_unref_page_list(&list); -+ -+ sc->nr_reclaimed += reclaimed; -+ -+ if (need_swapping && type == LRU_GEN_ANON) -+ *need_swapping = true; -+ -+ return scanned; -+} -+ -+/* -+ * For future optimizations: -+ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg -+ * reclaim. -+ */ -+static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, -+ bool can_swap, bool *need_aging) -+{ -+ unsigned long nr_to_scan; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (mem_cgroup_below_min(memcg) || -+ (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) -+ return 0; -+ -+ *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); -+ if (!*need_aging) -+ return nr_to_scan; -+ -+ /* skip the aging path at the default priority */ -+ if (sc->priority == DEF_PRIORITY) -+ goto done; -+ -+ /* leave the work to lru_gen_age_node() */ -+ if (current_is_kswapd()) -+ return 0; -+ -+ if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) -+ return nr_to_scan; -+done: -+ return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; -+} -+ -+static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, -+ struct scan_control *sc, bool need_swapping) -+{ -+ int i; -+ DEFINE_MAX_SEQ(lruvec); -+ -+ if (!current_is_kswapd()) { -+ /* age each memcg once to ensure fairness */ -+ if (max_seq - seq > 1) -+ return true; -+ -+ /* over-swapping can increase allocation latency */ -+ if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) -+ return true; -+ -+ /* give this thread a chance to exit and free its memory */ -+ if (fatal_signal_pending(current)) { -+ sc->nr_reclaimed += MIN_LRU_BATCH; -+ return true; -+ } -+ -+ if (cgroup_reclaim(sc)) -+ return false; -+ } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) -+ return false; -+ -+ /* keep scanning at low priorities to ensure fairness */ -+ if (sc->priority > DEF_PRIORITY - 2) -+ return false; -+ -+ /* -+ * A minimum amount of work was done under global memory pressure. For -+ * kswapd, it may be overshooting. For direct reclaim, the target isn't -+ * met, and yet the allocation may still succeed, since kswapd may have -+ * caught up. In either case, it's better to stop now, and restart if -+ * necessary. -+ */ -+ for (i = 0; i <= sc->reclaim_idx; i++) { -+ unsigned long wmark; -+ struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; -+ -+ if (!managed_zone(zone)) -+ continue; -+ -+ wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); -+ if (wmark > zone_page_state(zone, NR_FREE_PAGES)) -+ return false; -+ } -+ -+ sc->nr_reclaimed += MIN_LRU_BATCH; -+ -+ return true; -+} -+ -+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ struct blk_plug plug; -+ bool need_aging = false; -+ bool need_swapping = false; -+ unsigned long scanned = 0; -+ unsigned long reclaimed = sc->nr_reclaimed; -+ DEFINE_MAX_SEQ(lruvec); -+ -+ lru_add_drain(); -+ -+ blk_start_plug(&plug); -+ -+ set_mm_walk(lruvec_pgdat(lruvec)); -+ -+ while (true) { -+ int delta; -+ int swappiness; -+ unsigned long nr_to_scan; -+ -+ if (sc->may_swap) -+ swappiness = get_swappiness(lruvec, sc); -+ else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) -+ swappiness = 1; -+ else -+ swappiness = 0; -+ -+ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); -+ if (!nr_to_scan) -+ goto done; -+ -+ delta = evict_folios(lruvec, sc, swappiness, &need_swapping); -+ if (!delta) -+ goto done; -+ -+ scanned += delta; -+ if (scanned >= nr_to_scan) -+ break; -+ -+ if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) -+ break; -+ -+ cond_resched(); -+ } -+ -+ /* see the comment in lru_gen_age_node() */ -+ if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) -+ sc->memcgs_need_aging = false; -+done: -+ clear_mm_walk(); -+ -+ blk_finish_plug(&plug); -+} -+ -+/****************************************************************************** -+ * state change -+ ******************************************************************************/ -+ -+static bool __maybe_unused state_is_valid(struct lruvec *lruvec) -+{ -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ if (lrugen->enabled) { -+ enum lru_list lru; -+ -+ for_each_evictable_lru(lru) { -+ if (!list_empty(&lruvec->lists[lru])) -+ return false; -+ } -+ } else { -+ int gen, type, zone; -+ -+ for_each_gen_type_zone(gen, type, zone) { -+ if (!list_empty(&lrugen->lists[gen][type][zone])) -+ return false; -+ } -+ } -+ -+ return true; -+} -+ -+static bool fill_evictable(struct lruvec *lruvec) -+{ -+ enum lru_list lru; -+ int remaining = MAX_LRU_BATCH; -+ -+ for_each_evictable_lru(lru) { -+ int type = is_file_lru(lru); -+ bool active = is_active_lru(lru); -+ struct list_head *head = &lruvec->lists[lru]; -+ -+ while (!list_empty(head)) { -+ bool success; -+ struct folio *folio = lru_to_folio(head); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); -+ -+ lruvec_del_folio(lruvec, folio); -+ success = lru_gen_add_folio(lruvec, folio, false); -+ VM_WARN_ON_ONCE(!success); -+ -+ if (!--remaining) -+ return false; -+ } -+ } -+ -+ return true; -+} -+ -+static bool drain_evictable(struct lruvec *lruvec) -+{ -+ int gen, type, zone; -+ int remaining = MAX_LRU_BATCH; -+ -+ for_each_gen_type_zone(gen, type, zone) { -+ struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; -+ -+ while (!list_empty(head)) { -+ bool success; -+ struct folio *folio = lru_to_folio(head); -+ -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); -+ -+ success = lru_gen_del_folio(lruvec, folio, false); -+ VM_WARN_ON_ONCE(!success); -+ lruvec_add_folio(lruvec, folio); -+ -+ if (!--remaining) -+ return false; -+ } -+ } -+ -+ return true; -+} -+ -+static void lru_gen_change_state(bool enabled) -+{ -+ static DEFINE_MUTEX(state_mutex); -+ -+ struct mem_cgroup *memcg; -+ -+ cgroup_lock(); -+ cpus_read_lock(); -+ get_online_mems(); -+ mutex_lock(&state_mutex); -+ -+ if (enabled == lru_gen_enabled()) -+ goto unlock; -+ -+ if (enabled) -+ static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); -+ else -+ static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); -+ -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ int nid; -+ -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ if (!lruvec) -+ continue; -+ -+ spin_lock_irq(&lruvec->lru_lock); -+ -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ VM_WARN_ON_ONCE(!state_is_valid(lruvec)); -+ -+ lruvec->lrugen.enabled = enabled; -+ -+ while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { -+ spin_unlock_irq(&lruvec->lru_lock); -+ cond_resched(); -+ spin_lock_irq(&lruvec->lru_lock); -+ } -+ -+ spin_unlock_irq(&lruvec->lru_lock); -+ } -+ -+ cond_resched(); -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+unlock: -+ mutex_unlock(&state_mutex); -+ put_online_mems(); -+ cpus_read_unlock(); -+ cgroup_unlock(); -+} -+ -+/****************************************************************************** -+ * sysfs interface -+ ******************************************************************************/ -+ -+static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, -+ const char *buf, size_t len) -+{ -+ unsigned int msecs; -+ -+ if (kstrtouint(buf, 0, &msecs)) -+ return -EINVAL; -+ -+ WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); -+ -+ return len; -+} -+ -+static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( -+ min_ttl_ms, 0644, show_min_ttl, store_min_ttl -+); -+ -+static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -+{ -+ unsigned int caps = 0; -+ -+ if (get_cap(LRU_GEN_CORE)) -+ caps |= BIT(LRU_GEN_CORE); -+ -+ if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) -+ caps |= BIT(LRU_GEN_MM_WALK); -+ -+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) -+ caps |= BIT(LRU_GEN_NONLEAF_YOUNG); -+ -+ return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, -+ const char *buf, size_t len) -+{ -+ int i; -+ unsigned int caps; -+ -+ if (tolower(*buf) == 'n') -+ caps = 0; -+ else if (tolower(*buf) == 'y') -+ caps = -1; -+ else if (kstrtouint(buf, 0, &caps)) -+ return -EINVAL; -+ -+ for (i = 0; i < NR_LRU_GEN_CAPS; i++) { -+ bool enabled = caps & BIT(i); -+ -+ if (i == LRU_GEN_CORE) -+ lru_gen_change_state(enabled); -+ else if (enabled) -+ static_branch_enable(&lru_gen_caps[i]); -+ else -+ static_branch_disable(&lru_gen_caps[i]); -+ } -+ -+ return len; -+} -+ -+static struct kobj_attribute lru_gen_enabled_attr = __ATTR( -+ enabled, 0644, show_enabled, store_enabled -+); -+ -+static struct attribute *lru_gen_attrs[] = { -+ &lru_gen_min_ttl_attr.attr, -+ &lru_gen_enabled_attr.attr, -+ NULL -+}; -+ -+static struct attribute_group lru_gen_attr_group = { -+ .name = "lru_gen", -+ .attrs = lru_gen_attrs, -+}; -+ -+/****************************************************************************** -+ * debugfs interface -+ ******************************************************************************/ -+ -+static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) -+{ -+ struct mem_cgroup *memcg; -+ loff_t nr_to_skip = *pos; -+ -+ m->private = kvmalloc(PATH_MAX, GFP_KERNEL); -+ if (!m->private) -+ return ERR_PTR(-ENOMEM); -+ -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ int nid; -+ -+ for_each_node_state(nid, N_MEMORY) { -+ if (!nr_to_skip--) -+ return get_lruvec(memcg, nid); -+ } -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+ -+ return NULL; -+} -+ -+static void lru_gen_seq_stop(struct seq_file *m, void *v) -+{ -+ if (!IS_ERR_OR_NULL(v)) -+ mem_cgroup_iter_break(NULL, lruvec_memcg(v)); -+ -+ kvfree(m->private); -+ m->private = NULL; -+} -+ -+static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) -+{ -+ int nid = lruvec_pgdat(v)->node_id; -+ struct mem_cgroup *memcg = lruvec_memcg(v); -+ -+ ++*pos; -+ -+ nid = next_memory_node(nid); -+ if (nid == MAX_NUMNODES) { -+ memcg = mem_cgroup_iter(NULL, memcg, NULL); -+ if (!memcg) -+ return NULL; -+ -+ nid = first_memory_node; -+ } -+ -+ return get_lruvec(memcg, nid); -+} -+ -+static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, -+ unsigned long max_seq, unsigned long *min_seq, -+ unsigned long seq) -+{ -+ int i; -+ int type, tier; -+ int hist = lru_hist_from_seq(seq); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ for (tier = 0; tier < MAX_NR_TIERS; tier++) { -+ seq_printf(m, " %10d", tier); -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ const char *s = " "; -+ unsigned long n[3] = {}; -+ -+ if (seq == max_seq) { -+ s = "RT "; -+ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); -+ n[1] = READ_ONCE(lrugen->avg_total[type][tier]); -+ } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { -+ s = "rep"; -+ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); -+ } -+ -+ for (i = 0; i < 3; i++) -+ seq_printf(m, " %10lu%c", n[i], s[i]); -+ } -+ seq_putc(m, '\n'); -+ } -+ -+ seq_puts(m, " "); -+ for (i = 0; i < NR_MM_STATS; i++) { -+ const char *s = " "; -+ unsigned long n = 0; -+ -+ if (seq == max_seq && NR_HIST_GENS == 1) { -+ s = "LOYNFA"; -+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); -+ } else if (seq != max_seq && NR_HIST_GENS > 1) { -+ s = "loynfa"; -+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); -+ } -+ -+ seq_printf(m, " %10lu%c", n, s[i]); -+ } -+ seq_putc(m, '\n'); -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static int lru_gen_seq_show(struct seq_file *m, void *v) -+{ -+ unsigned long seq; -+ bool full = !debugfs_real_fops(m->file)->write; -+ struct lruvec *lruvec = v; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int nid = lruvec_pgdat(lruvec)->node_id; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (nid == first_memory_node) { -+ const char *path = memcg ? m->private : ""; -+ -+#ifdef CONFIG_MEMCG -+ if (memcg) -+ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); -+#endif -+ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); -+ } -+ -+ seq_printf(m, " node %5d\n", nid); -+ -+ if (!full) -+ seq = min_seq[LRU_GEN_ANON]; -+ else if (max_seq >= MAX_NR_GENS) -+ seq = max_seq - MAX_NR_GENS + 1; -+ else -+ seq = 0; -+ -+ for (; seq <= max_seq; seq++) { -+ int type, zone; -+ int gen = lru_gen_from_seq(seq); -+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); -+ -+ seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); -+ -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ unsigned long size = 0; -+ char mark = full && seq < min_seq[type] ? 'x' : ' '; -+ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) -+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); -+ -+ seq_printf(m, " %10lu%c", size, mark); -+ } -+ -+ seq_putc(m, '\n'); -+ -+ if (full) -+ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); -+ } -+ -+ return 0; -+} -+ -+static const struct seq_operations lru_gen_seq_ops = { -+ .start = lru_gen_seq_start, -+ .stop = lru_gen_seq_stop, -+ .next = lru_gen_seq_next, -+ .show = lru_gen_seq_show, -+}; -+ -+static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, -+ bool can_swap, bool force_scan) -+{ -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (seq < max_seq) -+ return 0; -+ -+ if (seq > max_seq) -+ return -EINVAL; -+ -+ if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) -+ return -ERANGE; -+ -+ try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); -+ -+ return 0; -+} -+ -+static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, -+ int swappiness, unsigned long nr_to_reclaim) -+{ -+ DEFINE_MAX_SEQ(lruvec); -+ -+ if (seq + MIN_NR_GENS > max_seq) -+ return -EINVAL; -+ -+ sc->nr_reclaimed = 0; -+ -+ while (!signal_pending(current)) { -+ DEFINE_MIN_SEQ(lruvec); -+ -+ if (seq < min_seq[!swappiness]) -+ return 0; -+ -+ if (sc->nr_reclaimed >= nr_to_reclaim) -+ return 0; -+ -+ if (!evict_folios(lruvec, sc, swappiness, NULL)) -+ return 0; -+ -+ cond_resched(); -+ } -+ -+ return -EINTR; -+} -+ -+static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, -+ struct scan_control *sc, int swappiness, unsigned long opt) -+{ -+ struct lruvec *lruvec; -+ int err = -EINVAL; -+ struct mem_cgroup *memcg = NULL; -+ -+ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) -+ return -EINVAL; -+ -+ if (!mem_cgroup_disabled()) { -+ rcu_read_lock(); -+ memcg = mem_cgroup_from_id(memcg_id); -+#ifdef CONFIG_MEMCG -+ if (memcg && !css_tryget(&memcg->css)) -+ memcg = NULL; -+#endif -+ rcu_read_unlock(); -+ -+ if (!memcg) -+ return -EINVAL; -+ } -+ -+ if (memcg_id != mem_cgroup_id(memcg)) -+ goto done; -+ -+ lruvec = get_lruvec(memcg, nid); -+ -+ if (swappiness < 0) -+ swappiness = get_swappiness(lruvec, sc); -+ else if (swappiness > 200) -+ goto done; -+ -+ switch (cmd) { -+ case '+': -+ err = run_aging(lruvec, seq, sc, swappiness, opt); -+ break; -+ case '-': -+ err = run_eviction(lruvec, seq, sc, swappiness, opt); -+ break; -+ } -+done: -+ mem_cgroup_put(memcg); -+ -+ return err; -+} -+ -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, -+ size_t len, loff_t *pos) -+{ -+ void *buf; -+ char *cur, *next; -+ unsigned int flags; -+ struct blk_plug plug; -+ int err = -EINVAL; -+ struct scan_control sc = { -+ .may_writepage = true, -+ .may_unmap = true, -+ .may_swap = true, -+ .reclaim_idx = MAX_NR_ZONES - 1, -+ .gfp_mask = GFP_KERNEL, -+ }; -+ -+ buf = kvmalloc(len + 1, GFP_KERNEL); -+ if (!buf) -+ return -ENOMEM; -+ -+ if (copy_from_user(buf, src, len)) { -+ kvfree(buf); -+ return -EFAULT; -+ } -+ -+ set_task_reclaim_state(current, &sc.reclaim_state); -+ flags = memalloc_noreclaim_save(); -+ blk_start_plug(&plug); -+ if (!set_mm_walk(NULL)) { -+ err = -ENOMEM; -+ goto done; -+ } -+ -+ next = buf; -+ next[len] = '\0'; -+ -+ while ((cur = strsep(&next, ",;\n"))) { -+ int n; -+ int end; -+ char cmd; -+ unsigned int memcg_id; -+ unsigned int nid; -+ unsigned long seq; -+ unsigned int swappiness = -1; -+ unsigned long opt = -1; -+ -+ cur = skip_spaces(cur); -+ if (!*cur) -+ continue; -+ -+ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, -+ &seq, &end, &swappiness, &end, &opt, &end); -+ if (n < 4 || cur[end]) { -+ err = -EINVAL; -+ break; -+ } -+ -+ err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); -+ if (err) -+ break; -+ } -+done: -+ clear_mm_walk(); -+ blk_finish_plug(&plug); -+ memalloc_noreclaim_restore(flags); -+ set_task_reclaim_state(current, NULL); -+ -+ kvfree(buf); -+ -+ return err ? : len; -+} -+ -+static int lru_gen_seq_open(struct inode *inode, struct file *file) -+{ -+ return seq_open(file, &lru_gen_seq_ops); -+} -+ -+static const struct file_operations lru_gen_rw_fops = { -+ .open = lru_gen_seq_open, -+ .read = seq_read, -+ .write = lru_gen_seq_write, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+static const struct file_operations lru_gen_ro_fops = { -+ .open = lru_gen_seq_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+/****************************************************************************** -+ * initialization -+ ******************************************************************************/ -+ -+void lru_gen_init_lruvec(struct lruvec *lruvec) -+{ -+ int i; -+ int gen, type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ lrugen->max_seq = MIN_NR_GENS + 1; -+ lrugen->enabled = lru_gen_enabled(); -+ -+ for (i = 0; i <= MIN_NR_GENS + 1; i++) -+ lrugen->timestamps[i] = jiffies; -+ -+ for_each_gen_type_zone(gen, type, zone) -+ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); -+ -+ lruvec->mm_state.seq = MIN_NR_GENS; -+ init_waitqueue_head(&lruvec->mm_state.wait); -+} -+ -+#ifdef CONFIG_MEMCG -+void lru_gen_init_memcg(struct mem_cgroup *memcg) -+{ -+ INIT_LIST_HEAD(&memcg->mm_list.fifo); -+ spin_lock_init(&memcg->mm_list.lock); -+} -+ -+void lru_gen_exit_memcg(struct mem_cgroup *memcg) -+{ -+ int i; -+ int nid; -+ -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); -+ -+ VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, -+ sizeof(lruvec->lrugen.nr_pages))); -+ -+ for (i = 0; i < NR_BLOOM_FILTERS; i++) { -+ bitmap_free(lruvec->mm_state.filters[i]); -+ lruvec->mm_state.filters[i] = NULL; -+ } -+ } -+} -+#endif -+ -+static int __init init_lru_gen(void) -+{ -+ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); -+ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); -+ -+ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) -+ pr_err("lru_gen: failed to create sysfs group\n"); -+ -+ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); -+ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); -+ -+ return 0; -+}; -+late_initcall(init_lru_gen); -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -+{ -+} -+ -+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ -+static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ unsigned long nr[NR_LRU_LISTS]; -+ unsigned long targets[NR_LRU_LISTS]; -+ unsigned long nr_to_scan; -+ enum lru_list lru; -+ unsigned long nr_reclaimed = 0; -+ unsigned long nr_to_reclaim = sc->nr_to_reclaim; -+ bool proportional_reclaim; -+ struct blk_plug plug; -+ -+ if (lru_gen_enabled()) { -+ lru_gen_shrink_lruvec(lruvec, sc); -+ return; -+ } -+ -+ get_scan_count(lruvec, sc, nr); -+ -+ /* Record the original scan target for proportional adjustments later */ -+ memcpy(targets, nr, sizeof(nr)); -+ -+ /* -+ * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal -+ * event that can occur when there is little memory pressure e.g. -+ * multiple streaming readers/writers. Hence, we do not abort scanning -+ * when the requested number of pages are reclaimed when scanning at -+ * DEF_PRIORITY on the assumption that the fact we are direct -+ * reclaiming implies that kswapd is not keeping up and it is best to -+ * do a batch of work at once. For memcg reclaim one check is made to -+ * abort proportional reclaim if either the file or anon lru has already -+ * dropped to zero at the first pass. -+ */ -+ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && -+ sc->priority == DEF_PRIORITY); -+ -+ blk_start_plug(&plug); -+ while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || -+ nr[LRU_INACTIVE_FILE]) { -+ unsigned long nr_anon, nr_file, percentage; -+ unsigned long nr_scanned; -+ -+ for_each_evictable_lru(lru) { -+ if (nr[lru]) { -+ nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); -+ nr[lru] -= nr_to_scan; -+ -+ nr_reclaimed += shrink_list(lru, nr_to_scan, -+ lruvec, sc); -+ } -+ } -+ -+ cond_resched(); -+ -+ if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) -+ continue; -+ -+ /* -+ * For kswapd and memcg, reclaim at least the number of pages -+ * requested. Ensure that the anon and file LRUs are scanned -+ * proportionally what was requested by get_scan_count(). We -+ * stop reclaiming one LRU and reduce the amount scanning -+ * proportional to the original scan target. -+ */ -+ nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; -+ nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; -+ -+ /* -+ * It's just vindictive to attack the larger once the smaller -+ * has gone to zero. And given the way we stop scanning the -+ * smaller below, this makes sure that we only make one nudge -+ * towards proportionality once we've got nr_to_reclaim. -+ */ -+ if (!nr_file || !nr_anon) -+ break; -+ -+ if (nr_file > nr_anon) { -+ unsigned long scan_target = targets[LRU_INACTIVE_ANON] + -+ targets[LRU_ACTIVE_ANON] + 1; -+ lru = LRU_BASE; -+ percentage = nr_anon * 100 / scan_target; -+ } else { -+ unsigned long scan_target = targets[LRU_INACTIVE_FILE] + -+ targets[LRU_ACTIVE_FILE] + 1; -+ lru = LRU_FILE; -+ percentage = nr_file * 100 / scan_target; -+ } -+ -+ /* Stop scanning the smaller of the LRU */ -+ nr[lru] = 0; -+ nr[lru + LRU_ACTIVE] = 0; -+ -+ /* -+ * Recalculate the other LRU scan count based on its original -+ * scan target and the percentage scanning already complete -+ */ -+ lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; -+ nr_scanned = targets[lru] - nr[lru]; -+ nr[lru] = targets[lru] * (100 - percentage) / 100; -+ nr[lru] -= min(nr[lru], nr_scanned); -+ -+ lru += LRU_ACTIVE; -+ nr_scanned = targets[lru] - nr[lru]; -+ nr[lru] = targets[lru] * (100 - percentage) / 100; -+ nr[lru] -= min(nr[lru], nr_scanned); -+ } -+ blk_finish_plug(&plug); -+ sc->nr_reclaimed += nr_reclaimed; -+ -+ /* -+ * Even if we did not try to evict anon pages at all, we want to -+ * rebalance the anon lru active/inactive ratio. -+ */ -+ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && -+ inactive_is_low(lruvec, LRU_INACTIVE_ANON)) -+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec, -+ sc, LRU_ACTIVE_ANON); -+} -+ -+/* Use reclaim/compaction for costly allocs or under memory pressure */ -+static bool in_reclaim_compaction(struct scan_control *sc) -+{ -+ if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && -+ (sc->order > PAGE_ALLOC_COSTLY_ORDER || -+ sc->priority < DEF_PRIORITY - 2)) -+ return true; -+ -+ return false; -+} -+ -+/* -+ * Reclaim/compaction is used for high-order allocation requests. It reclaims -+ * order-0 pages before compacting the zone. should_continue_reclaim() returns -+ * true if more pages should be reclaimed such that when the page allocator -+ * calls try_to_compact_pages() that it will have enough free pages to succeed. -+ * It will give up earlier than that if there is difficulty reclaiming pages. -+ */ -+static inline bool should_continue_reclaim(struct pglist_data *pgdat, -+ unsigned long nr_reclaimed, -+ struct scan_control *sc) -+{ -+ unsigned long pages_for_compaction; -+ unsigned long inactive_lru_pages; -+ int z; -+ -+ /* If not in reclaim/compaction mode, stop */ -+ if (!in_reclaim_compaction(sc)) -+ return false; -+ -+ /* -+ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX -+ * number of pages that were scanned. This will return to the caller -+ * with the risk reclaim/compaction and the resulting allocation attempt -+ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL -+ * allocations through requiring that the full LRU list has been scanned -+ * first, by assuming that zero delta of sc->nr_scanned means full LRU -+ * scan, but that approximation was wrong, and there were corner cases - * where always a non-zero amount of pages were scanned. - */ - if (!nr_reclaimed) -@@ -3201,109 +6071,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) - unsigned long nr_reclaimed, nr_scanned; - struct lruvec *target_lruvec; - bool reclaimable = false; -- unsigned long file; - - target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); - - again: -- /* -- * Flush the memory cgroup stats, so that we read accurate per-memcg -- * lruvec stats for heuristics. -- */ -- mem_cgroup_flush_stats(); -- - memset(&sc->nr, 0, sizeof(sc->nr)); - - nr_reclaimed = sc->nr_reclaimed; - nr_scanned = sc->nr_scanned; - -- /* -- * Determine the scan balance between anon and file LRUs. -- */ -- spin_lock_irq(&target_lruvec->lru_lock); -- sc->anon_cost = target_lruvec->anon_cost; -- sc->file_cost = target_lruvec->file_cost; -- spin_unlock_irq(&target_lruvec->lru_lock); -- -- /* -- * Target desirable inactive:active list ratios for the anon -- * and file LRU lists. -- */ -- if (!sc->force_deactivate) { -- unsigned long refaults; -- -- refaults = lruvec_page_state(target_lruvec, -- WORKINGSET_ACTIVATE_ANON); -- if (refaults != target_lruvec->refaults[0] || -- inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) -- sc->may_deactivate |= DEACTIVATE_ANON; -- else -- sc->may_deactivate &= ~DEACTIVATE_ANON; -- -- /* -- * When refaults are being observed, it means a new -- * workingset is being established. Deactivate to get -- * rid of any stale active pages quickly. -- */ -- refaults = lruvec_page_state(target_lruvec, -- WORKINGSET_ACTIVATE_FILE); -- if (refaults != target_lruvec->refaults[1] || -- inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) -- sc->may_deactivate |= DEACTIVATE_FILE; -- else -- sc->may_deactivate &= ~DEACTIVATE_FILE; -- } else -- sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; -- -- /* -- * If we have plenty of inactive file pages that aren't -- * thrashing, try to reclaim those first before touching -- * anonymous pages. -- */ -- file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); -- if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) -- sc->cache_trim_mode = 1; -- else -- sc->cache_trim_mode = 0; -- -- /* -- * Prevent the reclaimer from falling into the cache trap: as -- * cache pages start out inactive, every cache fault will tip -- * the scan balance towards the file LRU. And as the file LRU -- * shrinks, so does the window for rotation from references. -- * This means we have a runaway feedback loop where a tiny -- * thrashing file LRU becomes infinitely more attractive than -- * anon pages. Try to detect this based on file LRU size. -- */ -- if (!cgroup_reclaim(sc)) { -- unsigned long total_high_wmark = 0; -- unsigned long free, anon; -- int z; -- -- free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); -- file = node_page_state(pgdat, NR_ACTIVE_FILE) + -- node_page_state(pgdat, NR_INACTIVE_FILE); -- -- for (z = 0; z < MAX_NR_ZONES; z++) { -- struct zone *zone = &pgdat->node_zones[z]; -- if (!managed_zone(zone)) -- continue; -- -- total_high_wmark += high_wmark_pages(zone); -- } -- -- /* -- * Consider anon: if that's low too, this isn't a -- * runaway file reclaim problem, but rather just -- * extreme pressure. Reclaim as per usual then. -- */ -- anon = node_page_state(pgdat, NR_INACTIVE_ANON); -- -- sc->file_is_tiny = -- file + free <= total_high_wmark && -- !(sc->may_deactivate & DEACTIVATE_ANON) && -- anon >> sc->priority; -- } -+ prepare_scan_count(pgdat, sc); - - shrink_node_memcgs(pgdat, sc); - -@@ -3561,11 +6338,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) - struct lruvec *target_lruvec; - unsigned long refaults; - -+ if (lru_gen_enabled()) -+ return; -+ - target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); -- target_lruvec->refaults[0] = refaults; -+ target_lruvec->refaults[WORKINGSET_ANON] = refaults; - refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); -- target_lruvec->refaults[1] = refaults; -+ target_lruvec->refaults[WORKINGSET_FILE] = refaults; - } - - /* -@@ -3927,12 +6707,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - } - #endif - --static void age_active_anon(struct pglist_data *pgdat, -- struct scan_control *sc) -+static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) - { - struct mem_cgroup *memcg; - struct lruvec *lruvec; - -+ if (lru_gen_enabled()) { -+ lru_gen_age_node(pgdat, sc); -+ return; -+ } -+ - if (!can_age_anon_pages(pgdat, sc)) - return; - -@@ -4252,12 +7036,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) - sc.may_swap = !nr_boost_reclaim; - - /* -- * Do some background aging of the anon list, to give -- * pages a chance to be referenced before reclaiming. All -- * pages are rotated regardless of classzone as this is -- * about consistent aging. -+ * Do some background aging, to give pages a chance to be -+ * referenced before reclaiming. All pages are rotated -+ * regardless of classzone as this is about consistent aging. - */ -- age_active_anon(pgdat, &sc); -+ kswapd_age_node(pgdat, &sc); - - /* - * If we're getting trouble reclaiming, start doing writepage -diff --git a/mm/vmstat.c b/mm/vmstat.c -index 90af9a8572f5..4b85d3d5f7d7 100644 ---- a/mm/vmstat.c -+++ b/mm/vmstat.c -@@ -1389,10 +1389,6 @@ const char * const vmstat_text[] = { - "nr_tlb_local_flush_one", - #endif /* CONFIG_DEBUG_TLBFLUSH */ - --#ifdef CONFIG_DEBUG_VM_VMACACHE -- "vmacache_find_calls", -- "vmacache_find_hits", --#endif - #ifdef CONFIG_SWAP - "swap_ra", - "swap_ra_hit", -diff --git a/mm/workingset.c b/mm/workingset.c -index a5e84862fc86..ae7e984b23c6 100644 ---- a/mm/workingset.c -+++ b/mm/workingset.c -@@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; - static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, - bool workingset) - { -- eviction >>= bucket_order; - eviction &= EVICTION_MASK; - eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; - eviction = (eviction << NODES_SHIFT) | pgdat->node_id; -@@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, - - *memcgidp = memcgid; - *pgdat = NODE_DATA(nid); -- *evictionp = entry << bucket_order; -+ *evictionp = entry; - *workingsetp = workingset; - } - -+#ifdef CONFIG_LRU_GEN -+ -+static void *lru_gen_eviction(struct folio *folio) -+{ -+ int hist; -+ unsigned long token; -+ unsigned long min_seq; -+ struct lruvec *lruvec; -+ struct lru_gen_struct *lrugen; -+ int type = folio_is_file_lru(folio); -+ int delta = folio_nr_pages(folio); -+ int refs = folio_lru_refs(folio); -+ int tier = lru_tier_from_refs(refs); -+ struct mem_cgroup *memcg = folio_memcg(folio); -+ struct pglist_data *pgdat = folio_pgdat(folio); -+ -+ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); -+ -+ lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ lrugen = &lruvec->lrugen; -+ min_seq = READ_ONCE(lrugen->min_seq[type]); -+ token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); -+ -+ hist = lru_hist_from_seq(min_seq); -+ atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); -+ -+ return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); -+} -+ -+static void lru_gen_refault(struct folio *folio, void *shadow) -+{ -+ int hist, tier, refs; -+ int memcg_id; -+ bool workingset; -+ unsigned long token; -+ unsigned long min_seq; -+ struct lruvec *lruvec; -+ struct lru_gen_struct *lrugen; -+ struct mem_cgroup *memcg; -+ struct pglist_data *pgdat; -+ int type = folio_is_file_lru(folio); -+ int delta = folio_nr_pages(folio); -+ -+ unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); -+ -+ if (pgdat != folio_pgdat(folio)) -+ return; -+ -+ rcu_read_lock(); -+ -+ memcg = folio_memcg_rcu(folio); -+ if (memcg_id != mem_cgroup_id(memcg)) -+ goto unlock; -+ -+ lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ lrugen = &lruvec->lrugen; -+ -+ min_seq = READ_ONCE(lrugen->min_seq[type]); -+ if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) -+ goto unlock; -+ -+ hist = lru_hist_from_seq(min_seq); -+ /* see the comment in folio_lru_refs() */ -+ refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; -+ tier = lru_tier_from_refs(refs); -+ -+ atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); -+ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); -+ -+ /* -+ * Count the following two cases as stalls: -+ * 1. For pages accessed through page tables, hotter pages pushed out -+ * hot pages which refaulted immediately. -+ * 2. For pages accessed multiple times through file descriptors, -+ * numbers of accesses might have been out of the range. -+ */ -+ if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { -+ folio_set_workingset(folio); -+ mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); -+ } -+unlock: -+ rcu_read_unlock(); -+} -+ -+#else /* !CONFIG_LRU_GEN */ -+ -+static void *lru_gen_eviction(struct folio *folio) -+{ -+ return NULL; -+} -+ -+static void lru_gen_refault(struct folio *folio, void *shadow) -+{ -+} -+ -+#endif /* CONFIG_LRU_GEN */ -+ - /** - * workingset_age_nonresident - age non-resident entries as LRU ages - * @lruvec: the lruvec that was aged -@@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) - VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); - VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); - -+ if (lru_gen_enabled()) -+ return lru_gen_eviction(folio); -+ - lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - /* XXX: target_memcg can be NULL, go through lruvec */ - memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); - eviction = atomic_long_read(&lruvec->nonresident_age); -+ eviction >>= bucket_order; - workingset_age_nonresident(lruvec, folio_nr_pages(folio)); - return pack_shadow(memcgid, pgdat, eviction, - folio_test_workingset(folio)); -@@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) - int memcgid; - long nr; - -+ if (lru_gen_enabled()) { -+ lru_gen_refault(folio, shadow); -+ return; -+ } -+ - unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); -+ eviction <<= bucket_order; - - rcu_read_lock(); - /* -diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h -index 0616409513eb..311759ea25e9 100644 ---- a/tools/include/linux/slab.h -+++ b/tools/include/linux/slab.h -@@ -41,4 +41,8 @@ struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, - unsigned int align, unsigned int flags, - void (*ctor)(void *)); - -+void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); -+int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, -+ void **list); -+ - #endif /* _TOOLS_SLAB_H */ -diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h -index 6c1aa92a92e4..6ce1f1ceb432 100644 ---- a/tools/include/uapi/asm-generic/mman-common.h -+++ b/tools/include/uapi/asm-generic/mman-common.h -@@ -77,6 +77,8 @@ - - #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ - -+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ -+ - /* compatibility flags */ - #define MAP_FILE 0 - -diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore -index d971516401e6..c901d96dd013 100644 ---- a/tools/testing/radix-tree/.gitignore -+++ b/tools/testing/radix-tree/.gitignore -@@ -6,3 +6,5 @@ main - multiorder - radix-tree.c - xarray -+maple -+ma_xa_benchmark -diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile -index c4ea4fbb0bfc..89d613e0505b 100644 ---- a/tools/testing/radix-tree/Makefile -+++ b/tools/testing/radix-tree/Makefile -@@ -4,9 +4,9 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ - -fsanitize=undefined - LDFLAGS += -fsanitize=address -fsanitize=undefined - LDLIBS+= -lpthread -lurcu --TARGETS = main idr-test multiorder xarray -+TARGETS = main idr-test multiorder xarray maple - CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \ -- slab.o -+ slab.o maple.o - OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ - regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ - iteration_check_2.o benchmark.o -@@ -29,6 +29,8 @@ idr-test: idr-test.o $(CORE_OFILES) - - xarray: $(CORE_OFILES) - -+maple: $(CORE_OFILES) -+ - multiorder: multiorder.o $(CORE_OFILES) - - clean: -@@ -40,6 +42,7 @@ $(OFILES): Makefile *.h */*.h generated/map-shift.h \ - ../../include/linux/*.h \ - ../../include/asm/*.h \ - ../../../include/linux/xarray.h \ -+ ../../../include/linux/maple_tree.h \ - ../../../include/linux/radix-tree.h \ - ../../../include/linux/idr.h - -@@ -51,6 +54,8 @@ idr.c: ../../../lib/idr.c - - xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c - -+maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c -+ - generated/map-shift.h: - @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ - echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ -diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h -index 2218b3cc184e..e7da80350236 100644 ---- a/tools/testing/radix-tree/generated/autoconf.h -+++ b/tools/testing/radix-tree/generated/autoconf.h -@@ -1 +1,2 @@ - #define CONFIG_XARRAY_MULTI 1 -+#define CONFIG_64BIT 1 -diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c -index d5c1bcba86fe..2048d12c31df 100644 ---- a/tools/testing/radix-tree/linux.c -+++ b/tools/testing/radix-tree/linux.c -@@ -23,15 +23,47 @@ struct kmem_cache { - int nr_objs; - void *objs; - void (*ctor)(void *); -+ unsigned int non_kernel; -+ unsigned long nr_allocated; -+ unsigned long nr_tallocated; - }; - -+void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) -+{ -+ cachep->non_kernel = val; -+} -+ -+unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) -+{ -+ return cachep->size * cachep->nr_allocated; -+} -+ -+unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) -+{ -+ return cachep->nr_allocated; -+} -+ -+unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) -+{ -+ return cachep->nr_tallocated; -+} -+ -+void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) -+{ -+ cachep->nr_tallocated = 0; -+} -+ - void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, - int gfp) - { - void *p; - -- if (!(gfp & __GFP_DIRECT_RECLAIM)) -- return NULL; -+ if (!(gfp & __GFP_DIRECT_RECLAIM)) { -+ if (!cachep->non_kernel) -+ return NULL; -+ -+ cachep->non_kernel--; -+ } - - pthread_mutex_lock(&cachep->lock); - if (cachep->nr_objs) { -@@ -53,19 +85,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, - memset(p, 0, cachep->size); - } - -+ uatomic_inc(&cachep->nr_allocated); - uatomic_inc(&nr_allocated); -+ uatomic_inc(&cachep->nr_tallocated); - if (kmalloc_verbose) - printf("Allocating %p from slab\n", p); - return p; - } - --void kmem_cache_free(struct kmem_cache *cachep, void *objp) -+void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) - { - assert(objp); - uatomic_dec(&nr_allocated); -+ uatomic_dec(&cachep->nr_allocated); - if (kmalloc_verbose) - printf("Freeing %p to slab\n", objp); -- pthread_mutex_lock(&cachep->lock); - if (cachep->nr_objs > 10 || cachep->align) { - memset(objp, POISON_FREE, cachep->size); - free(objp); -@@ -75,9 +109,80 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) - node->parent = cachep->objs; - cachep->objs = node; - } -+} -+ -+void kmem_cache_free(struct kmem_cache *cachep, void *objp) -+{ -+ pthread_mutex_lock(&cachep->lock); -+ kmem_cache_free_locked(cachep, objp); - pthread_mutex_unlock(&cachep->lock); - } - -+void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) -+{ -+ if (kmalloc_verbose) -+ pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); -+ -+ pthread_mutex_lock(&cachep->lock); -+ for (int i = 0; i < size; i++) -+ kmem_cache_free_locked(cachep, list[i]); -+ pthread_mutex_unlock(&cachep->lock); -+} -+ -+int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, -+ void **p) -+{ -+ size_t i; -+ -+ if (kmalloc_verbose) -+ pr_debug("Bulk alloc %lu\n", size); -+ -+ if (!(gfp & __GFP_DIRECT_RECLAIM)) { -+ if (cachep->non_kernel < size) -+ return 0; -+ -+ cachep->non_kernel -= size; -+ } -+ -+ pthread_mutex_lock(&cachep->lock); -+ if (cachep->nr_objs >= size) { -+ struct radix_tree_node *node; -+ -+ for (i = 0; i < size; i++) { -+ node = cachep->objs; -+ cachep->nr_objs--; -+ cachep->objs = node->parent; -+ p[i] = node; -+ node->parent = NULL; -+ } -+ pthread_mutex_unlock(&cachep->lock); -+ } else { -+ pthread_mutex_unlock(&cachep->lock); -+ for (i = 0; i < size; i++) { -+ if (cachep->align) { -+ posix_memalign(&p[i], cachep->align, -+ cachep->size * size); -+ } else { -+ p[i] = malloc(cachep->size * size); -+ } -+ if (cachep->ctor) -+ cachep->ctor(p[i]); -+ else if (gfp & __GFP_ZERO) -+ memset(p[i], 0, cachep->size); -+ } -+ } -+ -+ for (i = 0; i < size; i++) { -+ uatomic_inc(&nr_allocated); -+ uatomic_inc(&cachep->nr_allocated); -+ uatomic_inc(&cachep->nr_tallocated); -+ if (kmalloc_verbose) -+ printf("Allocating %p from slab\n", p[i]); -+ } -+ -+ return size; -+} -+ - struct kmem_cache * - kmem_cache_create(const char *name, unsigned int size, unsigned int align, - unsigned int flags, void (*ctor)(void *)) -@@ -88,7 +193,54 @@ kmem_cache_create(const char *name, unsigned int size, unsigned int align, - ret->size = size; - ret->align = align; - ret->nr_objs = 0; -+ ret->nr_allocated = 0; -+ ret->nr_tallocated = 0; - ret->objs = NULL; - ret->ctor = ctor; -+ ret->non_kernel = 0; - return ret; - } -+ -+/* -+ * Test the test infrastructure for kem_cache_alloc/free and bulk counterparts. -+ */ -+void test_kmem_cache_bulk(void) -+{ -+ int i; -+ void *list[12]; -+ static struct kmem_cache *test_cache, *test_cache2; -+ -+ /* -+ * Testing the bulk allocators without aligned kmem_cache to force the -+ * bulk alloc/free to reuse -+ */ -+ test_cache = kmem_cache_create("test_cache", 256, 0, SLAB_PANIC, NULL); -+ -+ for (i = 0; i < 5; i++) -+ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); -+ -+ for (i = 0; i < 5; i++) -+ kmem_cache_free(test_cache, list[i]); -+ assert(test_cache->nr_objs == 5); -+ -+ kmem_cache_alloc_bulk(test_cache, __GFP_DIRECT_RECLAIM, 5, list); -+ kmem_cache_free_bulk(test_cache, 5, list); -+ -+ for (i = 0; i < 12 ; i++) -+ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); -+ -+ for (i = 0; i < 12; i++) -+ kmem_cache_free(test_cache, list[i]); -+ -+ /* The last free will not be kept around */ -+ assert(test_cache->nr_objs == 11); -+ -+ /* Aligned caches will immediately free */ -+ test_cache2 = kmem_cache_create("test_cache2", 128, 128, SLAB_PANIC, NULL); -+ -+ kmem_cache_alloc_bulk(test_cache2, __GFP_DIRECT_RECLAIM, 10, list); -+ kmem_cache_free_bulk(test_cache2, 10, list); -+ assert(!test_cache2->nr_objs); -+ -+ -+} -diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h -index 39867fd80c8f..c5c9d05f29da 100644 ---- a/tools/testing/radix-tree/linux/kernel.h -+++ b/tools/testing/radix-tree/linux/kernel.h -@@ -14,6 +14,7 @@ - #include "../../../include/linux/kconfig.h" - - #define printk printf -+#define pr_err printk - #define pr_info printk - #define pr_debug printk - #define pr_cont printk -diff --git a/tools/testing/radix-tree/linux/lockdep.h b/tools/testing/radix-tree/linux/lockdep.h -index 016cff473cfc..62473ab57f99 100644 ---- a/tools/testing/radix-tree/linux/lockdep.h -+++ b/tools/testing/radix-tree/linux/lockdep.h -@@ -11,4 +11,6 @@ static inline void lockdep_set_class(spinlock_t *lock, - struct lock_class_key *key) - { - } -+ -+extern int lockdep_is_held(const void *); - #endif /* _LINUX_LOCKDEP_H */ -diff --git a/tools/testing/radix-tree/linux/maple_tree.h b/tools/testing/radix-tree/linux/maple_tree.h -new file mode 100644 -index 000000000000..7d8d1f445b89 ---- /dev/null -+++ b/tools/testing/radix-tree/linux/maple_tree.h -@@ -0,0 +1,7 @@ -+/* SPDX-License-Identifier: GPL-2.0+ */ -+#define atomic_t int32_t -+#include "../../../../include/linux/maple_tree.h" -+#define atomic_inc(x) uatomic_inc(x) -+#define atomic_read(x) uatomic_read(x) -+#define atomic_set(x, y) do {} while (0) -+#define U8_MAX UCHAR_MAX -diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c -new file mode 100644 -index 000000000000..35082671928a ---- /dev/null -+++ b/tools/testing/radix-tree/maple.c -@@ -0,0 +1,59 @@ -+// SPDX-License-Identifier: GPL-2.0+ -+/* -+ * maple_tree.c: Userspace shim for maple tree test-suite -+ * Copyright (c) 2018 Liam R. Howlett -+ */ -+ -+#define CONFIG_DEBUG_MAPLE_TREE -+#define CONFIG_MAPLE_SEARCH -+#include "test.h" -+ -+#define module_init(x) -+#define module_exit(x) -+#define MODULE_AUTHOR(x) -+#define MODULE_LICENSE(x) -+#define dump_stack() assert(0) -+ -+#include "../../../lib/maple_tree.c" -+#undef CONFIG_DEBUG_MAPLE_TREE -+#include "../../../lib/test_maple_tree.c" -+ -+void farmer_tests(void) -+{ -+ struct maple_node *node; -+ DEFINE_MTREE(tree); -+ -+ mt_dump(&tree); -+ -+ tree.ma_root = xa_mk_value(0); -+ mt_dump(&tree); -+ -+ node = mt_alloc_one(GFP_KERNEL); -+ node->parent = (void *)((unsigned long)(&tree) | 1); -+ node->slot[0] = xa_mk_value(0); -+ node->slot[1] = xa_mk_value(1); -+ node->mr64.pivot[0] = 0; -+ node->mr64.pivot[1] = 1; -+ node->mr64.pivot[2] = 0; -+ tree.ma_root = mt_mk_node(node, maple_leaf_64); -+ mt_dump(&tree); -+ -+ ma_free_rcu(node); -+} -+ -+void maple_tree_tests(void) -+{ -+ farmer_tests(); -+ maple_tree_seed(); -+ maple_tree_harvest(); -+} -+ -+int __weak main(void) -+{ -+ maple_tree_init(); -+ maple_tree_tests(); -+ rcu_barrier(); -+ if (nr_allocated) -+ printf("nr_allocated = %d\n", nr_allocated); -+ return 0; -+} -diff --git a/tools/testing/radix-tree/trace/events/maple_tree.h b/tools/testing/radix-tree/trace/events/maple_tree.h -new file mode 100644 -index 000000000000..97d0e1ddcf08 ---- /dev/null -+++ b/tools/testing/radix-tree/trace/events/maple_tree.h -@@ -0,0 +1,5 @@ -+/* SPDX-License-Identifier: GPL-2.0+ */ -+ -+#define trace_ma_op(a, b) do {} while (0) -+#define trace_ma_read(a, b) do {} while (0) -+#define trace_ma_write(a, b, c, d) do {} while (0) -diff --git a/tools/testing/selftests/vm/khugepaged.c b/tools/testing/selftests/vm/khugepaged.c -index 155120b67a16..b77b1e28cdb3 100644 ---- a/tools/testing/selftests/vm/khugepaged.c -+++ b/tools/testing/selftests/vm/khugepaged.c -@@ -14,6 +14,9 @@ - #ifndef MADV_PAGEOUT - #define MADV_PAGEOUT 21 - #endif -+#ifndef MADV_COLLAPSE -+#define MADV_COLLAPSE 25 -+#endif - - #define BASE_ADDR ((void *)(1UL << 30)) - static unsigned long hpage_pmd_size; -@@ -23,6 +26,11 @@ static int hpage_pmd_nr; - #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" - #define PID_SMAPS "/proc/self/smaps" - -+struct collapse_context { -+ void (*collapse)(const char *msg, char *p, int nr_hpages, bool expect); -+ bool enforce_pte_scan_limits; -+}; -+ - enum thp_enabled { - THP_ALWAYS, - THP_MADVISE, -@@ -90,18 +98,6 @@ struct settings { - struct khugepaged_settings khugepaged; - }; - --static struct settings default_settings = { -- .thp_enabled = THP_MADVISE, -- .thp_defrag = THP_DEFRAG_ALWAYS, -- .shmem_enabled = SHMEM_NEVER, -- .use_zero_page = 0, -- .khugepaged = { -- .defrag = 1, -- .alloc_sleep_millisecs = 10, -- .scan_sleep_millisecs = 10, -- }, --}; -- - static struct settings saved_settings; - static bool skip_settings_restore; - -@@ -279,6 +275,39 @@ static void write_settings(struct settings *settings) - write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); - } - -+#define MAX_SETTINGS_DEPTH 4 -+static struct settings settings_stack[MAX_SETTINGS_DEPTH]; -+static int settings_index; -+ -+static struct settings *current_settings(void) -+{ -+ if (!settings_index) { -+ printf("Fail: No settings set"); -+ exit(EXIT_FAILURE); -+ } -+ return settings_stack + settings_index - 1; -+} -+ -+static void push_settings(struct settings *settings) -+{ -+ if (settings_index >= MAX_SETTINGS_DEPTH) { -+ printf("Fail: Settings stack exceeded"); -+ exit(EXIT_FAILURE); -+ } -+ settings_stack[settings_index++] = *settings; -+ write_settings(current_settings()); -+} -+ -+static void pop_settings(void) -+{ -+ if (settings_index <= 0) { -+ printf("Fail: Settings stack empty"); -+ exit(EXIT_FAILURE); -+ } -+ --settings_index; -+ write_settings(current_settings()); -+} -+ - static void restore_settings(int sig) - { - if (skip_settings_restore) -@@ -322,14 +351,6 @@ static void save_settings(void) - signal(SIGQUIT, restore_settings); - } - --static void adjust_settings(void) --{ -- -- printf("Adjust settings..."); -- write_settings(&default_settings); -- success("OK"); --} -- - #define MAX_LINE_LENGTH 500 - - static bool check_for_pattern(FILE *fp, char *pattern, char *buf) -@@ -341,7 +362,7 @@ static bool check_for_pattern(FILE *fp, char *pattern, char *buf) - return false; - } - --static bool check_huge(void *addr) -+static bool check_huge(void *addr, int nr_hpages) - { - bool thp = false; - int ret; -@@ -366,7 +387,7 @@ static bool check_huge(void *addr) - goto err_out; - - ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB", -- hpage_pmd_size >> 10); -+ nr_hpages * (hpage_pmd_size >> 10)); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); - exit(EXIT_FAILURE); -@@ -434,12 +455,12 @@ static bool check_swap(void *addr, unsigned long size) - return swap; - } - --static void *alloc_mapping(void) -+static void *alloc_mapping(int nr) - { - void *p; - -- p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, -- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); -+ p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, -+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (p != BASE_ADDR) { - printf("Failed to allocate VMA at %p\n", BASE_ADDR); - exit(EXIT_FAILURE); -@@ -456,6 +477,25 @@ static void fill_memory(int *p, unsigned long start, unsigned long end) - p[i * page_size / sizeof(*p)] = i + 0xdead0000; - } - -+/* -+ * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with -+ * validate_memory()'able contents. -+ */ -+static void *alloc_hpage(void) -+{ -+ void *p; -+ -+ p = alloc_mapping(1); -+ printf("Allocate huge page..."); -+ madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -+ fill_memory(p, 0, hpage_pmd_size); -+ if (check_huge(p, 1)) -+ success("OK"); -+ else -+ fail("Fail"); -+ return p; -+} -+ - static void validate_memory(int *p, unsigned long start, unsigned long end) - { - int i; -@@ -469,26 +509,59 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) - } - } - -+static void madvise_collapse(const char *msg, char *p, int nr_hpages, -+ bool expect) -+{ -+ int ret; -+ struct settings settings = *current_settings(); -+ -+ printf("%s...", msg); -+ /* Sanity check */ -+ if (!check_huge(p, 0)) { -+ printf("Unexpected huge page\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ /* -+ * Prevent khugepaged interference and tests that MADV_COLLAPSE -+ * ignores /sys/kernel/mm/transparent_hugepage/enabled -+ */ -+ settings.thp_enabled = THP_NEVER; -+ push_settings(&settings); -+ -+ /* Clear VM_NOHUGEPAGE */ -+ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); -+ ret = madvise(p, nr_hpages * hpage_pmd_size, MADV_COLLAPSE); -+ if (((bool)ret) == expect) -+ fail("Fail: Bad return value"); -+ else if (check_huge(p, nr_hpages) != expect) -+ fail("Fail: check_huge()"); -+ else -+ success("OK"); -+ -+ pop_settings(); -+} -+ - #define TICK 500000 --static bool wait_for_scan(const char *msg, char *p) -+static bool wait_for_scan(const char *msg, char *p, int nr_hpages) - { - int full_scans; - int timeout = 6; /* 3 seconds */ - - /* Sanity check */ -- if (check_huge(p)) { -+ if (!check_huge(p, 0)) { - printf("Unexpected huge page\n"); - exit(EXIT_FAILURE); - } - -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -+ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); - - /* Wait until the second full_scan completed */ - full_scans = read_num("khugepaged/full_scans") + 2; - - printf("%s...", msg); - while (timeout--) { -- if (check_huge(p)) -+ if (check_huge(p, nr_hpages)) - break; - if (read_num("khugepaged/full_scans") >= full_scans) - break; -@@ -496,121 +569,121 @@ static bool wait_for_scan(const char *msg, char *p) - usleep(TICK); - } - -- madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -+ madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); - - return timeout == -1; - } - -+static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, -+ bool expect) -+{ -+ if (wait_for_scan(msg, p, nr_hpages)) { -+ if (expect) -+ fail("Timeout"); -+ else -+ success("OK"); -+ return; -+ } else if (check_huge(p, nr_hpages) == expect) { -+ success("OK"); -+ } else { -+ fail("Fail"); -+ } -+} -+ - static void alloc_at_fault(void) - { -- struct settings settings = default_settings; -+ struct settings settings = *current_settings(); - char *p; - - settings.thp_enabled = THP_ALWAYS; -- write_settings(&settings); -+ push_settings(&settings); - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - *p = 1; - printf("Allocate huge page on fault..."); -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); - -- write_settings(&default_settings); -+ pop_settings(); - - madvise(p, page_size, MADV_DONTNEED); - printf("Split huge PMD on MADV_DONTNEED..."); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - munmap(p, hpage_pmd_size); - } - --static void collapse_full(void) -+static void collapse_full(struct collapse_context *c) - { - void *p; -- -- p = alloc_mapping(); -- fill_memory(p, 0, hpage_pmd_size); -- if (wait_for_scan("Collapse fully populated PTE table", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- validate_memory(p, 0, hpage_pmd_size); -- munmap(p, hpage_pmd_size); -+ int nr_hpages = 4; -+ unsigned long size = nr_hpages * hpage_pmd_size; -+ -+ p = alloc_mapping(nr_hpages); -+ fill_memory(p, 0, size); -+ c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, -+ true); -+ validate_memory(p, 0, size); -+ munmap(p, size); - } - --static void collapse_empty(void) -+static void collapse_empty(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -- if (wait_for_scan("Do not collapse empty PTE table", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- fail("Fail"); -- else -- success("OK"); -+ p = alloc_mapping(1); -+ c->collapse("Do not collapse empty PTE table", p, 1, false); - munmap(p, hpage_pmd_size); - } - --static void collapse_single_pte_entry(void) -+static void collapse_single_pte_entry(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - fill_memory(p, 0, page_size); -- if (wait_for_scan("Collapse PTE table with single PTE entry present", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table with single PTE entry present", p, -+ 1, true); - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_max_ptes_none(void) -+static void collapse_max_ptes_none(struct collapse_context *c) - { - int max_ptes_none = hpage_pmd_nr / 2; -- struct settings settings = default_settings; -+ struct settings settings = *current_settings(); - void *p; - - settings.khugepaged.max_ptes_none = max_ptes_none; -- write_settings(&settings); -+ push_settings(&settings); - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); -- if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- fail("Fail"); -- else -- success("OK"); -+ c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, -+ !c->enforce_pte_scan_limits); - validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); - -- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); -- if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); -+ if (c->enforce_pte_scan_limits) { -+ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); -+ c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, -+ true); -+ validate_memory(p, 0, -+ (hpage_pmd_nr - max_ptes_none) * page_size); -+ } - - munmap(p, hpage_pmd_size); -- write_settings(&default_settings); -+ pop_settings(); - } - --static void collapse_swapin_single_pte(void) -+static void collapse_swapin_single_pte(struct collapse_context *c) - { - void *p; -- p = alloc_mapping(); -+ p = alloc_mapping(1); - fill_memory(p, 0, hpage_pmd_size); - - printf("Swapout one page..."); -@@ -625,23 +698,18 @@ static void collapse_swapin_single_pte(void) - goto out; - } - -- if (wait_for_scan("Collapse with swapping in single PTE entry", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse with swapping in single PTE entry", p, 1, true); - validate_memory(p, 0, hpage_pmd_size); - out: - munmap(p, hpage_pmd_size); - } - --static void collapse_max_ptes_swap(void) -+static void collapse_max_ptes_swap(struct collapse_context *c) - { - int max_ptes_swap = read_num("khugepaged/max_ptes_swap"); - void *p; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - - fill_memory(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); -@@ -656,115 +724,83 @@ static void collapse_max_ptes_swap(void) - goto out; - } - -- if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- fail("Fail"); -- else -- success("OK"); -+ c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, -+ !c->enforce_pte_scan_limits); - validate_memory(p, 0, hpage_pmd_size); - -- fill_memory(p, 0, hpage_pmd_size); -- printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); -- if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { -- perror("madvise(MADV_PAGEOUT)"); -- exit(EXIT_FAILURE); -- } -- if (check_swap(p, max_ptes_swap * page_size)) { -- success("OK"); -- } else { -- fail("Fail"); -- goto out; -- } -+ if (c->enforce_pte_scan_limits) { -+ fill_memory(p, 0, hpage_pmd_size); -+ printf("Swapout %d of %d pages...", max_ptes_swap, -+ hpage_pmd_nr); -+ if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { -+ perror("madvise(MADV_PAGEOUT)"); -+ exit(EXIT_FAILURE); -+ } -+ if (check_swap(p, max_ptes_swap * page_size)) { -+ success("OK"); -+ } else { -+ fail("Fail"); -+ goto out; -+ } - -- if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- validate_memory(p, 0, hpage_pmd_size); -+ c->collapse("Collapse with max_ptes_swap pages swapped out", p, -+ 1, true); -+ validate_memory(p, 0, hpage_pmd_size); -+ } - out: - munmap(p, hpage_pmd_size); - } - --static void collapse_single_pte_entry_compound(void) -+static void collapse_single_pte_entry_compound(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ p = alloc_hpage(); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -- - printf("Split huge page leaving single PTE mapping compound page..."); - madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table with single PTE mapping compound page", -+ p, 1, true); - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_full_of_compound(void) -+static void collapse_full_of_compound(struct collapse_context *c) - { - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -+ p = alloc_hpage(); - printf("Split huge page leaving single PTE page table full of compound pages..."); - madvise(p, page_size, MADV_NOHUGEPAGE); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Collapse PTE table full of compound pages", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table full of compound pages", p, 1, true); - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_compound_extreme(void) -+static void collapse_compound_extreme(struct collapse_context *c) - { - void *p; - int i; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - for (i = 0; i < hpage_pmd_nr; i++) { - printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", - i + 1, hpage_pmd_nr); - - madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(BASE_ADDR, 0, hpage_pmd_size); -- if (!check_huge(BASE_ADDR)) { -+ if (!check_huge(BASE_ADDR, 1)) { - printf("Failed to allocate huge page\n"); - exit(EXIT_FAILURE); - } -@@ -793,32 +829,28 @@ static void collapse_compound_extreme(void) - - munmap(BASE_ADDR, hpage_pmd_size); - fill_memory(p, 0, hpage_pmd_size); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Collapse PTE table full of different compound pages", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table full of different compound pages", p, 1, -+ true); - - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); - } - --static void collapse_fork(void) -+static void collapse_fork(struct collapse_context *c) - { - int wstatus; - void *p; - -- p = alloc_mapping(); -+ p = alloc_mapping(1); - - printf("Allocate small page..."); - fill_memory(p, 0, page_size); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); -@@ -829,19 +861,14 @@ static void collapse_fork(void) - skip_settings_restore = true; - exit_status = 0; - -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - - fill_memory(p, page_size, 2 * page_size); -- -- if (wait_for_scan("Collapse PTE table with single page shared with parent process", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table with single page shared with parent process", -+ p, 1, true); - - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); -@@ -852,7 +879,7 @@ static void collapse_fork(void) - exit_status += WEXITSTATUS(wstatus); - - printf("Check if parent still has small page..."); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); -@@ -860,28 +887,19 @@ static void collapse_fork(void) - munmap(p, hpage_pmd_size); - } - --static void collapse_fork_compound(void) -+static void collapse_fork_compound(struct collapse_context *c) - { - int wstatus; - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -+ p = alloc_hpage(); - printf("Share huge page over fork()..."); - if (!fork()) { - /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -889,21 +907,17 @@ static void collapse_fork_compound(void) - printf("Split huge page PMD in child process..."); - madvise(p, page_size, MADV_NOHUGEPAGE); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - fill_memory(p, 0, page_size); - - write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); -- if (wait_for_scan("Collapse PTE table full of compound pages in child", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Collapse PTE table full of compound pages in child", -+ p, 1, true); - write_num("khugepaged/max_ptes_shared", -- default_settings.khugepaged.max_ptes_shared); -+ current_settings()->khugepaged.max_ptes_shared); - - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); -@@ -914,7 +928,7 @@ static void collapse_fork_compound(void) - exit_status += WEXITSTATUS(wstatus); - - printf("Check if parent still has huge page..."); -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -922,29 +936,20 @@ static void collapse_fork_compound(void) - munmap(p, hpage_pmd_size); - } - --static void collapse_max_ptes_shared() -+static void collapse_max_ptes_shared(struct collapse_context *c) - { - int max_ptes_shared = read_num("khugepaged/max_ptes_shared"); - int wstatus; - void *p; - -- p = alloc_mapping(); -- -- printf("Allocate huge page..."); -- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); -- fill_memory(p, 0, hpage_pmd_size); -- if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -+ p = alloc_hpage(); - printf("Share huge page over fork()..."); - if (!fork()) { - /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -952,33 +957,27 @@ static void collapse_max_ptes_shared() - printf("Trigger CoW on page %d of %d...", - hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); -- if (!check_huge(p)) -+ if (check_huge(p, 0)) - success("OK"); - else - fail("Fail"); - -- if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p)) -- fail("Timeout"); -- else if (!check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -- printf("Trigger CoW on page %d of %d...", -- hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); -- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); -- if (!check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -- -- -- if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p)) -- fail("Timeout"); -- else if (check_huge(p)) -- success("OK"); -- else -- fail("Fail"); -+ c->collapse("Maybe collapse with max_ptes_shared exceeded", p, -+ 1, !c->enforce_pte_scan_limits); -+ -+ if (c->enforce_pte_scan_limits) { -+ printf("Trigger CoW on page %d of %d...", -+ hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); -+ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * -+ page_size); -+ if (check_huge(p, 0)) -+ success("OK"); -+ else -+ fail("Fail"); -+ -+ c->collapse("Collapse with max_ptes_shared PTEs shared", -+ p, 1, true); -+ } - - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); -@@ -989,7 +988,7 @@ static void collapse_max_ptes_shared() - exit_status += WEXITSTATUS(wstatus); - - printf("Check if parent still has huge page..."); -- if (check_huge(p)) -+ if (check_huge(p, 1)) - success("OK"); - else - fail("Fail"); -@@ -997,8 +996,52 @@ static void collapse_max_ptes_shared() - munmap(p, hpage_pmd_size); - } - --int main(void) -+static void madvise_collapse_existing_thps(void) - { -+ void *p; -+ int err; -+ -+ p = alloc_mapping(1); -+ fill_memory(p, 0, hpage_pmd_size); -+ -+ printf("Collapse fully populated PTE table..."); -+ /* -+ * Note that we don't set MADV_HUGEPAGE here, which -+ * also tests that VM_HUGEPAGE isn't required for -+ * MADV_COLLAPSE in "madvise" mode. -+ */ -+ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); -+ if (err == 0 && check_huge(p, 1)) { -+ success("OK"); -+ printf("Re-collapse PMD-mapped hugepage"); -+ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); -+ if (err == 0 && check_huge(p, 1)) -+ success("OK"); -+ else -+ fail("Fail"); -+ } else { -+ fail("Fail"); -+ } -+ validate_memory(p, 0, hpage_pmd_size); -+ munmap(p, hpage_pmd_size); -+} -+ -+int main(int argc, const char **argv) -+{ -+ struct collapse_context c; -+ struct settings default_settings = { -+ .thp_enabled = THP_MADVISE, -+ .thp_defrag = THP_DEFRAG_ALWAYS, -+ .shmem_enabled = SHMEM_NEVER, -+ .use_zero_page = 0, -+ .khugepaged = { -+ .defrag = 1, -+ .alloc_sleep_millisecs = 10, -+ .scan_sleep_millisecs = 10, -+ }, -+ }; -+ const char *tests = argc == 1 ? "all" : argv[1]; -+ - setbuf(stdout, NULL); - - page_size = getpagesize(); -@@ -1011,21 +1054,47 @@ int main(void) - default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8; - - save_settings(); -- adjust_settings(); -+ push_settings(&default_settings); - - alloc_at_fault(); -- collapse_full(); -- collapse_empty(); -- collapse_single_pte_entry(); -- collapse_max_ptes_none(); -- collapse_swapin_single_pte(); -- collapse_max_ptes_swap(); -- collapse_single_pte_entry_compound(); -- collapse_full_of_compound(); -- collapse_compound_extreme(); -- collapse_fork(); -- collapse_fork_compound(); -- collapse_max_ptes_shared(); -+ -+ if (!strcmp(tests, "khugepaged") || !strcmp(tests, "all")) { -+ printf("\n*** Testing context: khugepaged ***\n"); -+ c.collapse = &khugepaged_collapse; -+ c.enforce_pte_scan_limits = true; -+ -+ collapse_full(&c); -+ collapse_empty(&c); -+ collapse_single_pte_entry(&c); -+ collapse_max_ptes_none(&c); -+ collapse_swapin_single_pte(&c); -+ collapse_max_ptes_swap(&c); -+ collapse_single_pte_entry_compound(&c); -+ collapse_full_of_compound(&c); -+ collapse_compound_extreme(&c); -+ collapse_fork(&c); -+ collapse_fork_compound(&c); -+ collapse_max_ptes_shared(&c); -+ } -+ if (!strcmp(tests, "madvise") || !strcmp(tests, "all")) { -+ printf("\n*** Testing context: madvise ***\n"); -+ c.collapse = &madvise_collapse; -+ c.enforce_pte_scan_limits = false; -+ -+ collapse_full(&c); -+ collapse_empty(&c); -+ collapse_single_pte_entry(&c); -+ collapse_max_ptes_none(&c); -+ collapse_swapin_single_pte(&c); -+ collapse_max_ptes_swap(&c); -+ collapse_single_pte_entry_compound(&c); -+ collapse_full_of_compound(&c); -+ collapse_compound_extreme(&c); -+ collapse_fork(&c); -+ collapse_fork_compound(&c); -+ collapse_max_ptes_shared(&c); -+ madvise_collapse_existing_thps(); -+ } - - restore_settings(0); - } --- -2.38.0.rc1.8.g2a7d63a245 - -From ee83b3c6bc5cedb0d989a4f52368fd31c0736f28 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:18:41 +0200 -Subject: [PATCH 06/13] mm-cleanup - -Signed-off-by: Peter Jung ---- - include/linux/gfp.h | 23 ------------- - include/linux/memory_hotplug.h | 8 ----- - include/linux/mm.h | 2 +- - include/linux/mmzone.h | 12 ------- - mm/internal.h | 3 -- - mm/page_alloc.c | 63 ++++++++++++++++------------------ - 6 files changed, 31 insertions(+), 80 deletions(-) - -diff --git a/include/linux/gfp.h b/include/linux/gfp.h -index f314be58fa77..5c7d752b18d9 100644 ---- a/include/linux/gfp.h -+++ b/include/linux/gfp.h -@@ -33,29 +33,6 @@ static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) - return !!(gfp_flags & __GFP_DIRECT_RECLAIM); - } - --/** -- * gfpflags_normal_context - is gfp_flags a normal sleepable context? -- * @gfp_flags: gfp_flags to test -- * -- * Test whether @gfp_flags indicates that the allocation is from the -- * %current context and allowed to sleep. -- * -- * An allocation being allowed to block doesn't mean it owns the %current -- * context. When direct reclaim path tries to allocate memory, the -- * allocation context is nested inside whatever %current was doing at the -- * time of the original allocation. The nested allocation may be allowed -- * to block but modifying anything %current owns can corrupt the outer -- * context's expectations. -- * -- * %true result from this function indicates that the allocation context -- * can sleep and use anything that's associated with %current. -- */ --static inline bool gfpflags_normal_context(const gfp_t gfp_flags) --{ -- return (gfp_flags & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC)) == -- __GFP_DIRECT_RECLAIM; --} -- - #ifdef CONFIG_HIGHMEM - #define OPT_ZONE_HIGHMEM ZONE_HIGHMEM - #else -diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h -index e0b2209ab71c..7ac0e5c78648 100644 ---- a/include/linux/memory_hotplug.h -+++ b/include/linux/memory_hotplug.h -@@ -44,11 +44,6 @@ extern void arch_refresh_nodedata(int nid, pg_data_t *pgdat); - ({ \ - memblock_alloc(sizeof(*pgdat), SMP_CACHE_BYTES); \ - }) --/* -- * This definition is just for error path in node hotadd. -- * For node hotremove, we have to replace this. -- */ --#define generic_free_nodedata(pgdat) kfree(pgdat) - - extern pg_data_t *node_data[]; - static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) -@@ -64,9 +59,6 @@ static inline pg_data_t *generic_alloc_nodedata(int nid) - BUG(); - return NULL; - } --static inline void generic_free_nodedata(pg_data_t *pgdat) --{ --} - static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) - { - } -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 9ac0e02e2238..9c3d3474033f 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -3032,7 +3032,7 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, - unsigned long address, unsigned long size, - pte_fn_t fn, void *data); - --extern void init_mem_debugging_and_hardening(void); -+extern void __init init_mem_debugging_and_hardening(void); - #ifdef CONFIG_PAGE_POISONING - extern void __kernel_poison_pages(struct page *page, int numpages); - extern void __kernel_unpoison_pages(struct page *page, int numpages); -diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index 1543001feba9..21d39f152d0c 100644 ---- a/include/linux/mmzone.h -+++ b/include/linux/mmzone.h -@@ -577,13 +577,6 @@ enum zone_watermarks { - #define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1)) - #define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP) - --/* -- * Shift to encode migratetype and order in the same integer, with order -- * in the least significant bits. -- */ --#define NR_PCP_ORDER_WIDTH 8 --#define NR_PCP_ORDER_MASK ((1<_watermark[WMARK_MIN] + z->watermark_boost) - #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost) - #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost) -@@ -1241,11 +1234,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) - return pgdat->node_start_pfn + pgdat->node_spanned_pages; - } - --static inline bool pgdat_is_empty(pg_data_t *pgdat) --{ -- return !pgdat->node_start_pfn && !pgdat->node_spanned_pages; --} -- - #include - - void build_all_zonelists(pg_data_t *pgdat); -diff --git a/mm/internal.h b/mm/internal.h -index 0f106a3982e7..4ea69814259a 100644 ---- a/mm/internal.h -+++ b/mm/internal.h -@@ -367,7 +367,6 @@ extern int user_min_free_kbytes; - extern void free_unref_page(struct page *page, unsigned int order); - extern void free_unref_page_list(struct list_head *list); - --extern void zone_pcp_update(struct zone *zone, int cpu_online); - extern void zone_pcp_reset(struct zone *zone); - extern void zone_pcp_disable(struct zone *zone); - extern void zone_pcp_enable(struct zone *zone); -@@ -859,8 +858,6 @@ int migrate_device_coherent_page(struct page *page); - */ - struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags); - --DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); -- - extern bool mirrored_kernelcore; - - static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) -diff --git a/mm/page_alloc.c b/mm/page_alloc.c -index cf131d6e08fb..292ed1bb6a5a 100644 ---- a/mm/page_alloc.c -+++ b/mm/page_alloc.c -@@ -870,7 +870,8 @@ static inline bool set_page_guard(struct zone *zone, struct page *page, - INIT_LIST_HEAD(&page->buddy_list); - set_page_private(page, order); - /* Guard pages are not available for any usage */ -- __mod_zone_freepage_state(zone, -(1 << order), migratetype); -+ if (!is_migrate_isolate(migratetype)) -+ __mod_zone_freepage_state(zone, -(1 << order), migratetype); - - return true; - } -@@ -900,7 +901,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, - * order of appearance. So we need to first gather the full picture of what was - * enabled, and then make decisions. - */ --void init_mem_debugging_and_hardening(void) -+void __init init_mem_debugging_and_hardening(void) - { - bool page_poisoning_requested = false; - -@@ -1105,7 +1106,7 @@ static inline void __free_one_page(struct page *page, - int migratetype, fpi_t fpi_flags) - { - struct capture_control *capc = task_capc(zone); -- unsigned long buddy_pfn; -+ unsigned long buddy_pfn = 0; - unsigned long combined_pfn; - struct page *buddy; - bool to_tail; -@@ -1575,7 +1576,6 @@ static void free_pcppages_bulk(struct zone *zone, int count, - - order = pindex_to_order(pindex); - nr_pages = 1 << order; -- BUILD_BUG_ON(MAX_ORDER >= (1< PAGE_ALLOC_COSTLY_ORDER, gfp_mask); -+ WARN_ON_ONCE_GFP(costly_order, gfp_mask); - - /* - * Help non-failing allocations by giving them access to memory -@@ -6507,7 +6502,7 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta - #define BOOT_PAGESET_BATCH 1 - static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); - static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); --DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); -+static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); - - static void __build_all_zonelists(void *data) - { -@@ -6810,7 +6805,7 @@ void __ref memmap_init_zone_device(struct zone *zone, - unsigned long start = jiffies; - int nid = pgdat->node_id; - -- if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) -+ if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) - return; - - /* -@@ -6986,7 +6981,7 @@ static int zone_batchsize(struct zone *zone) - * size is striking a balance between allocation latency - * and zone lock contention. - */ -- batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); -+ batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); - batch /= 4; /* We effectively *= 4 below */ - if (batch < 1) - batch = 1; -@@ -7171,6 +7166,17 @@ void __meminit setup_zone_pageset(struct zone *zone) - zone_set_pageset_high_and_batch(zone, 0); - } - -+/* -+ * The zone indicated has a new number of managed_pages; batch sizes and percpu -+ * page high values need to be recalculated. -+ */ -+static void zone_pcp_update(struct zone *zone, int cpu_online) -+{ -+ mutex_lock(&pcp_batch_high_lock); -+ zone_set_pageset_high_and_batch(zone, cpu_online); -+ mutex_unlock(&pcp_batch_high_lock); -+} -+ - /* - * Allocate per cpu pagesets and initialize them. - * Before this call only boot pagesets were available. -@@ -8461,8 +8467,8 @@ void __init mem_init_print_info(void) - #endif - ")\n", - K(nr_free_pages()), K(physpages), -- codesize >> 10, datasize >> 10, rosize >> 10, -- (init_data_size + init_code_size) >> 10, bss_size >> 10, -+ codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, -+ (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, - K(physpages - totalram_pages() - totalcma_pages), - K(totalcma_pages) - #ifdef CONFIG_HIGHMEM -@@ -8987,8 +8993,8 @@ void *__init alloc_large_system_hash(const char *tablename, - numentries -= arch_reserved_kernel_pages(); - - /* It isn't necessary when PAGE_SIZE >= 1MB */ -- if (PAGE_SHIFT < 20) -- numentries = round_up(numentries, (1<<20)/PAGE_SIZE); -+ if (PAGE_SIZE < SZ_1M) -+ numentries = round_up(numentries, SZ_1M / PAGE_SIZE); - - #if __BITS_PER_LONG > 32 - if (!high_limit) { -@@ -9412,17 +9418,6 @@ void free_contig_range(unsigned long pfn, unsigned long nr_pages) - } - EXPORT_SYMBOL(free_contig_range); - --/* -- * The zone indicated has a new number of managed_pages; batch sizes and percpu -- * page high values need to be recalculated. -- */ --void zone_pcp_update(struct zone *zone, int cpu_online) --{ -- mutex_lock(&pcp_batch_high_lock); -- zone_set_pageset_high_and_batch(zone, cpu_online); -- mutex_unlock(&pcp_batch_high_lock); --} -- - /* - * Effectively disable pcplists for the zone by setting the high limit to 0 - * and draining all cpus. A concurrent page freeing on another CPU that's about -@@ -9455,9 +9450,11 @@ void zone_pcp_reset(struct zone *zone) - drain_zonestat(zone, pzstats); - } - free_percpu(zone->per_cpu_pageset); -- free_percpu(zone->per_cpu_zonestats); - zone->per_cpu_pageset = &boot_pageset; -- zone->per_cpu_zonestats = &boot_zonestats; -+ if (zone->per_cpu_zonestats != &boot_zonestats) { -+ free_percpu(zone->per_cpu_zonestats); -+ zone->per_cpu_zonestats = &boot_zonestats; -+ } - } - } - --- -2.38.0.rc1.8.g2a7d63a245 - -From 4400a1f1fd24bc600b4557fda2baee2850e5e693 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Fri, 19 Aug 2022 17:06:47 +0200 -Subject: [PATCH 07/13] rtw88 - -Signed-off-by: Peter Jung ---- - drivers/net/wireless/realtek/rtw88/Kconfig | 47 + - drivers/net/wireless/realtek/rtw88/Makefile | 14 + - drivers/net/wireless/realtek/rtw88/coex.c | 3 +- - drivers/net/wireless/realtek/rtw88/debug.c | 15 + - drivers/net/wireless/realtek/rtw88/fw.c | 13 +- - drivers/net/wireless/realtek/rtw88/hci.h | 9 +- - drivers/net/wireless/realtek/rtw88/mac.c | 3 + - drivers/net/wireless/realtek/rtw88/mac80211.c | 2 +- - drivers/net/wireless/realtek/rtw88/main.c | 9 +- - drivers/net/wireless/realtek/rtw88/main.h | 11 +- - drivers/net/wireless/realtek/rtw88/phy.c | 6 +- - drivers/net/wireless/realtek/rtw88/ps.c | 2 +- - drivers/net/wireless/realtek/rtw88/reg.h | 1 + - drivers/net/wireless/realtek/rtw88/rtw8723d.c | 19 + - drivers/net/wireless/realtek/rtw88/rtw8723d.h | 1 + - .../net/wireless/realtek/rtw88/rtw8723du.c | 40 + - .../net/wireless/realtek/rtw88/rtw8723du.h | 13 + - drivers/net/wireless/realtek/rtw88/rtw8821c.c | 23 + - drivers/net/wireless/realtek/rtw88/rtw8821c.h | 21 + - .../net/wireless/realtek/rtw88/rtw8821cu.c | 69 ++ - .../net/wireless/realtek/rtw88/rtw8821cu.h | 15 + - drivers/net/wireless/realtek/rtw88/rtw8822b.c | 19 + - .../net/wireless/realtek/rtw88/rtw8822bu.c | 62 + - .../net/wireless/realtek/rtw88/rtw8822bu.h | 15 + - drivers/net/wireless/realtek/rtw88/rtw8822c.c | 24 + - .../net/wireless/realtek/rtw88/rtw8822cu.c | 40 + - .../net/wireless/realtek/rtw88/rtw8822cu.h | 15 + - drivers/net/wireless/realtek/rtw88/tx.h | 31 + - drivers/net/wireless/realtek/rtw88/usb.c | 1051 +++++++++++++++++ - drivers/net/wireless/realtek/rtw88/usb.h | 109 ++ - drivers/net/wireless/realtek/rtw88/util.c | 92 ++ - drivers/net/wireless/realtek/rtw88/util.h | 12 +- - 32 files changed, 1770 insertions(+), 36 deletions(-) - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8723du.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8723du.h - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8821cu.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8821cu.h - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822bu.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822bu.h - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822cu.c - create mode 100644 drivers/net/wireless/realtek/rtw88/rtw8822cu.h - create mode 100644 drivers/net/wireless/realtek/rtw88/usb.c - create mode 100644 drivers/net/wireless/realtek/rtw88/usb.h - -diff --git a/drivers/net/wireless/realtek/rtw88/Kconfig b/drivers/net/wireless/realtek/rtw88/Kconfig -index e3d7cb6c1290..651ab56d9c6b 100644 ---- a/drivers/net/wireless/realtek/rtw88/Kconfig -+++ b/drivers/net/wireless/realtek/rtw88/Kconfig -@@ -16,6 +16,9 @@ config RTW88_CORE - config RTW88_PCI - tristate - -+config RTW88_USB -+ tristate -+ - config RTW88_8822B - tristate - -@@ -39,6 +42,17 @@ config RTW88_8822BE - - 802.11ac PCIe wireless network adapter - -+config RTW88_8822BU -+ tristate "Realtek 8822BU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8822B -+ help -+ Select this option will enable support for 8822BU chipset -+ -+ 802.11ac USB wireless network adapter -+ - config RTW88_8822CE - tristate "Realtek 8822CE PCI wireless network adapter" - depends on PCI -@@ -50,6 +64,17 @@ config RTW88_8822CE - - 802.11ac PCIe wireless network adapter - -+config RTW88_8822CU -+ tristate "Realtek 8822CU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8822C -+ help -+ Select this option will enable support for 8822CU chipset -+ -+ 802.11ac USB wireless network adapter -+ - config RTW88_8723DE - tristate "Realtek 8723DE PCI wireless network adapter" - depends on PCI -@@ -61,6 +86,17 @@ config RTW88_8723DE - - 802.11n PCIe wireless network adapter - -+config RTW88_8723DU -+ tristate "Realtek 8723DU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8723D -+ help -+ Select this option will enable support for 8723DU chipset -+ -+ 802.11n USB wireless network adapter -+ - config RTW88_8821CE - tristate "Realtek 8821CE PCI wireless network adapter" - depends on PCI -@@ -72,6 +108,17 @@ config RTW88_8821CE - - 802.11ac PCIe wireless network adapter - -+config RTW88_8821CU -+ tristate "Realtek 8821CU USB wireless network adapter" -+ depends on USB -+ select RTW88_CORE -+ select RTW88_USB -+ select RTW88_8821C -+ help -+ Select this option will enable support for 8821CU chipset -+ -+ 802.11ac USB wireless network adapter -+ - config RTW88_DEBUG - bool "Realtek rtw88 debug support" - depends on RTW88_CORE -diff --git a/drivers/net/wireless/realtek/rtw88/Makefile b/drivers/net/wireless/realtek/rtw88/Makefile -index 834c66ec0af9..e0950dbc2565 100644 ---- a/drivers/net/wireless/realtek/rtw88/Makefile -+++ b/drivers/net/wireless/realtek/rtw88/Makefile -@@ -26,23 +26,37 @@ rtw88_8822b-objs := rtw8822b.o rtw8822b_table.o - obj-$(CONFIG_RTW88_8822BE) += rtw88_8822be.o - rtw88_8822be-objs := rtw8822be.o - -+obj-$(CONFIG_RTW88_8822BU) += rtw88_8822bu.o -+rtw88_8822bu-objs := rtw8822bu.o -+ - obj-$(CONFIG_RTW88_8822C) += rtw88_8822c.o - rtw88_8822c-objs := rtw8822c.o rtw8822c_table.o - - obj-$(CONFIG_RTW88_8822CE) += rtw88_8822ce.o - rtw88_8822ce-objs := rtw8822ce.o - -+obj-$(CONFIG_RTW88_8822CU) += rtw88_8822cu.o -+rtw88_8822cu-objs := rtw8822cu.o -+ - obj-$(CONFIG_RTW88_8723D) += rtw88_8723d.o - rtw88_8723d-objs := rtw8723d.o rtw8723d_table.o - - obj-$(CONFIG_RTW88_8723DE) += rtw88_8723de.o - rtw88_8723de-objs := rtw8723de.o - -+obj-$(CONFIG_RTW88_8723DU) += rtw88_8723du.o -+rtw88_8723du-objs := rtw8723du.o -+ - obj-$(CONFIG_RTW88_8821C) += rtw88_8821c.o - rtw88_8821c-objs := rtw8821c.o rtw8821c_table.o - - obj-$(CONFIG_RTW88_8821CE) += rtw88_8821ce.o - rtw88_8821ce-objs := rtw8821ce.o - -+obj-$(CONFIG_RTW88_8821CU) += rtw88_8821cu.o -+rtw88_8821cu-objs := rtw8821cu.o -+ - obj-$(CONFIG_RTW88_PCI) += rtw88_pci.o -+obj-$(CONFIG_RTW88_USB) += rtw88_usb.o - rtw88_pci-objs := pci.o -+rtw88_usb-objs := usb.o -diff --git a/drivers/net/wireless/realtek/rtw88/coex.c b/drivers/net/wireless/realtek/rtw88/coex.c -index cac053f485c3..b156f8c48ffb 100644 ---- a/drivers/net/wireless/realtek/rtw88/coex.c -+++ b/drivers/net/wireless/realtek/rtw88/coex.c -@@ -633,7 +633,7 @@ static struct sk_buff *rtw_coex_info_request(struct rtw_dev *rtwdev, - struct rtw_coex *coex = &rtwdev->coex; - struct sk_buff *skb_resp = NULL; - -- mutex_lock(&coex->mutex); -+ lockdep_assert_held(&rtwdev->mutex); - - rtw_fw_query_bt_mp_info(rtwdev, req); - -@@ -650,7 +650,6 @@ static struct sk_buff *rtw_coex_info_request(struct rtw_dev *rtwdev, - } - - out: -- mutex_unlock(&coex->mutex); - return skb_resp; - } - -diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c -index 7cde6bcf253b..28db952be4ff 100644 ---- a/drivers/net/wireless/realtek/rtw88/debug.c -+++ b/drivers/net/wireless/realtek/rtw88/debug.c -@@ -144,7 +144,9 @@ static int rtw_debugfs_get_rf_read(struct seq_file *m, void *v) - addr = debugfs_priv->rf_addr; - mask = debugfs_priv->rf_mask; - -+ mutex_lock(&rtwdev->mutex); - val = rtw_read_rf(rtwdev, path, addr, mask); -+ mutex_unlock(&rtwdev->mutex); - - seq_printf(m, "rf_read path:%d addr:0x%08x mask:0x%08x val=0x%08x\n", - path, addr, mask, val); -@@ -390,7 +392,9 @@ static ssize_t rtw_debugfs_set_h2c(struct file *filp, - return -EINVAL; - } - -+ mutex_lock(&rtwdev->mutex); - rtw_fw_h2c_cmd_dbg(rtwdev, param); -+ mutex_unlock(&rtwdev->mutex); - - return count; - } -@@ -414,7 +418,9 @@ static ssize_t rtw_debugfs_set_rf_write(struct file *filp, - return count; - } - -+ mutex_lock(&rtwdev->mutex); - rtw_write_rf(rtwdev, path, addr, mask, val); -+ mutex_unlock(&rtwdev->mutex); - rtw_dbg(rtwdev, RTW_DBG_DEBUGFS, - "write_rf path:%d addr:0x%08x mask:0x%08x, val:0x%08x\n", - path, addr, mask, val); -@@ -519,6 +525,8 @@ static int rtw_debug_get_rf_dump(struct seq_file *m, void *v) - u32 addr, offset, data; - u8 path; - -+ mutex_lock(&rtwdev->mutex); -+ - for (path = 0; path < rtwdev->hal.rf_path_num; path++) { - seq_printf(m, "RF path:%d\n", path); - for (addr = 0; addr < 0x100; addr += 4) { -@@ -533,6 +541,8 @@ static int rtw_debug_get_rf_dump(struct seq_file *m, void *v) - seq_puts(m, "\n"); - } - -+ mutex_unlock(&rtwdev->mutex); -+ - return 0; - } - -@@ -828,7 +838,9 @@ static int rtw_debugfs_get_coex_info(struct seq_file *m, void *v) - struct rtw_debugfs_priv *debugfs_priv = m->private; - struct rtw_dev *rtwdev = debugfs_priv->rtwdev; - -+ mutex_lock(&rtwdev->mutex); - rtw_coex_display_coex_info(rtwdev, m); -+ mutex_unlock(&rtwdev->mutex); - - return 0; - } -@@ -1023,6 +1035,8 @@ static void dump_gapk_status(struct rtw_dev *rtwdev, struct seq_file *m) - dm_info->dm_flags & BIT(RTW_DM_CAP_TXGAPK) ? '-' : '+', - rtw_dm_cap_strs[RTW_DM_CAP_TXGAPK]); - -+ mutex_lock(&rtwdev->mutex); -+ - for (path = 0; path < rtwdev->hal.rf_path_num; path++) { - val = rtw_read_rf(rtwdev, path, RF_GAINTX, RFREG_MASK); - seq_printf(m, "path %d:\n0x%x = 0x%x\n", path, RF_GAINTX, val); -@@ -1032,6 +1046,7 @@ static void dump_gapk_status(struct rtw_dev *rtwdev, struct seq_file *m) - txgapk->rf3f_fs[path][i], i); - seq_puts(m, "\n"); - } -+ mutex_unlock(&rtwdev->mutex); - } - - static int rtw_debugfs_get_dm_cap(struct seq_file *m, void *v) -diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c -index 4fdab0329695..5e912421408e 100644 ---- a/drivers/net/wireless/realtek/rtw88/fw.c -+++ b/drivers/net/wireless/realtek/rtw88/fw.c -@@ -320,7 +320,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - h2c[3], h2c[2], h2c[1], h2c[0], - h2c[7], h2c[6], h2c[5], h2c[4]); - -- spin_lock(&rtwdev->h2c.lock); -+ lockdep_assert_held(&rtwdev->mutex); - - box = rtwdev->h2c.last_box_num; - switch (box) { -@@ -342,7 +342,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - break; - default: - WARN(1, "invalid h2c mail box number\n"); -- goto out; -+ return; - } - - ret = read_poll_timeout_atomic(rtw_read8, box_state, -@@ -351,7 +351,7 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - - if (ret) { - rtw_err(rtwdev, "failed to send h2c command\n"); -- goto out; -+ return; - } - - for (idx = 0; idx < 4; idx++) -@@ -361,9 +361,6 @@ static void rtw_fw_send_h2c_command(struct rtw_dev *rtwdev, - - if (++rtwdev->h2c.last_box_num >= 4) - rtwdev->h2c.last_box_num = 0; -- --out: -- spin_unlock(&rtwdev->h2c.lock); - } - - void rtw_fw_h2c_cmd_dbg(struct rtw_dev *rtwdev, u8 *h2c) -@@ -375,15 +372,13 @@ static void rtw_fw_send_h2c_packet(struct rtw_dev *rtwdev, u8 *h2c_pkt) - { - int ret; - -- spin_lock(&rtwdev->h2c.lock); -+ lockdep_assert_held(&rtwdev->mutex); - - FW_OFFLOAD_H2C_SET_SEQ_NUM(h2c_pkt, rtwdev->h2c.seq); - ret = rtw_hci_write_data_h2c(rtwdev, h2c_pkt, H2C_PKT_SIZE); - if (ret) - rtw_err(rtwdev, "failed to send h2c packet\n"); - rtwdev->h2c.seq++; -- -- spin_unlock(&rtwdev->h2c.lock); - } - - void -diff --git a/drivers/net/wireless/realtek/rtw88/hci.h b/drivers/net/wireless/realtek/rtw88/hci.h -index 4c6fc6fb3f83..830d7532f2a3 100644 ---- a/drivers/net/wireless/realtek/rtw88/hci.h -+++ b/drivers/net/wireless/realtek/rtw88/hci.h -@@ -166,12 +166,11 @@ static inline u32 - rtw_read_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, - u32 addr, u32 mask) - { -- unsigned long flags; - u32 val; - -- spin_lock_irqsave(&rtwdev->rf_lock, flags); -+ lockdep_assert_held(&rtwdev->mutex); -+ - val = rtwdev->chip->ops->read_rf(rtwdev, rf_path, addr, mask); -- spin_unlock_irqrestore(&rtwdev->rf_lock, flags); - - return val; - } -@@ -180,11 +179,9 @@ static inline void - rtw_write_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, - u32 addr, u32 mask, u32 data) - { -- unsigned long flags; -+ lockdep_assert_held(&rtwdev->mutex); - -- spin_lock_irqsave(&rtwdev->rf_lock, flags); - rtwdev->chip->ops->write_rf(rtwdev, rf_path, addr, mask, data); -- spin_unlock_irqrestore(&rtwdev->rf_lock, flags); - } - - static inline u32 -diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c -index caf2603da2d6..c3c24850a427 100644 ---- a/drivers/net/wireless/realtek/rtw88/mac.c -+++ b/drivers/net/wireless/realtek/rtw88/mac.c -@@ -1032,6 +1032,9 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev) - if (rtw_chip_wcpu_11ac(rtwdev)) - rtw_write32(rtwdev, REG_H2CQ_CSR, BIT_H2CQ_FULL); - -+ if (rtw_hci_type(rtwdev) == RTW_HCI_TYPE_USB) -+ rtw_write8_set(rtwdev, REG_TXDMA_PQ_MAP, BIT_RXDMA_ARBBW_EN); -+ - return 0; - } - -diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c -index c7b98a0599d5..bc30400e67a1 100644 ---- a/drivers/net/wireless/realtek/rtw88/mac80211.c -+++ b/drivers/net/wireless/realtek/rtw88/mac80211.c -@@ -483,8 +483,8 @@ static int rtw_ops_sta_remove(struct ieee80211_hw *hw, - { - struct rtw_dev *rtwdev = hw->priv; - -- rtw_fw_beacon_filter_config(rtwdev, false, vif); - mutex_lock(&rtwdev->mutex); -+ rtw_fw_beacon_filter_config(rtwdev, false, vif); - rtw_sta_remove(rtwdev, sta, true); - mutex_unlock(&rtwdev->mutex); - -diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c -index 76dc9da88f6c..0edf22d9c1ef 100644 ---- a/drivers/net/wireless/realtek/rtw88/main.c -+++ b/drivers/net/wireless/realtek/rtw88/main.c -@@ -1733,6 +1733,10 @@ static int rtw_chip_parameter_setup(struct rtw_dev *rtwdev) - rtwdev->hci.rpwm_addr = 0x03d9; - rtwdev->hci.cpwm_addr = 0x03da; - break; -+ case RTW_HCI_TYPE_USB: -+ rtwdev->hci.rpwm_addr = 0xfe58; -+ rtwdev->hci.cpwm_addr = 0xfe57; -+ break; - default: - rtw_err(rtwdev, "unsupported hci type\n"); - return -EINVAL; -@@ -2017,13 +2021,10 @@ int rtw_core_init(struct rtw_dev *rtwdev) - skb_queue_head_init(&rtwdev->coex.queue); - skb_queue_head_init(&rtwdev->tx_report.queue); - -- spin_lock_init(&rtwdev->rf_lock); -- spin_lock_init(&rtwdev->h2c.lock); - spin_lock_init(&rtwdev->txq_lock); - spin_lock_init(&rtwdev->tx_report.q_lock); - - mutex_init(&rtwdev->mutex); -- mutex_init(&rtwdev->coex.mutex); - mutex_init(&rtwdev->hal.tx_power_mutex); - - init_waitqueue_head(&rtwdev->coex.wait); -@@ -2091,7 +2092,6 @@ void rtw_core_deinit(struct rtw_dev *rtwdev) - } - - mutex_destroy(&rtwdev->mutex); -- mutex_destroy(&rtwdev->coex.mutex); - mutex_destroy(&rtwdev->hal.tx_power_mutex); - } - EXPORT_SYMBOL(rtw_core_deinit); -@@ -2132,6 +2132,7 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) - hw->wiphy->available_antennas_rx = hal->antenna_rx; - - hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS | -+ WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL | - WIPHY_FLAG_TDLS_EXTERNAL_SETUP; - - hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR; -diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h -index 7db627fc26be..663b8dc453da 100644 ---- a/drivers/net/wireless/realtek/rtw88/main.h -+++ b/drivers/net/wireless/realtek/rtw88/main.h -@@ -875,6 +875,10 @@ struct rtw_chip_ops { - bool is_tx2_path); - void (*config_txrx_mode)(struct rtw_dev *rtwdev, u8 tx_path, - u8 rx_path, bool is_tx2_path); -+ /* for USB/SDIO only */ -+ void (*fill_txdesc_checksum)(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc); - - /* for coex */ - void (*coex_set_init)(struct rtw_dev *rtwdev); -@@ -1504,8 +1508,6 @@ struct rtw_coex_stat { - }; - - struct rtw_coex { -- /* protects coex info request section */ -- struct mutex mutex; - struct sk_buff_head queue; - wait_queue_head_t wait; - -@@ -1994,9 +1996,6 @@ struct rtw_dev { - /* ensures exclusive access from mac80211 callbacks */ - struct mutex mutex; - -- /* read/write rf register */ -- spinlock_t rf_lock; -- - /* watch dog every 2 sec */ - struct delayed_work watch_dog_work; - u32 watch_dog_cnt; -@@ -2022,8 +2021,6 @@ struct rtw_dev { - struct { - /* incicate the mail box to use with fw */ - u8 last_box_num; -- /* protect to send h2c to fw */ -- spinlock_t lock; - u32 seq; - } h2c; - -diff --git a/drivers/net/wireless/realtek/rtw88/phy.c b/drivers/net/wireless/realtek/rtw88/phy.c -index 8982e0c98dac..f6de6568f962 100644 ---- a/drivers/net/wireless/realtek/rtw88/phy.c -+++ b/drivers/net/wireless/realtek/rtw88/phy.c -@@ -300,7 +300,7 @@ static void rtw_phy_stat_rssi(struct rtw_dev *rtwdev) - - data.rtwdev = rtwdev; - data.min_rssi = U8_MAX; -- rtw_iterate_stas_atomic(rtwdev, rtw_phy_stat_rssi_iter, &data); -+ rtw_iterate_stas(rtwdev, rtw_phy_stat_rssi_iter, &data); - - dm_info->pre_min_rssi = dm_info->min_rssi; - dm_info->min_rssi = data.min_rssi; -@@ -544,7 +544,7 @@ static void rtw_phy_ra_info_update(struct rtw_dev *rtwdev) - if (rtwdev->watch_dog_cnt & 0x3) - return; - -- rtw_iterate_stas_atomic(rtwdev, rtw_phy_ra_info_update_iter, rtwdev); -+ rtw_iterate_stas(rtwdev, rtw_phy_ra_info_update_iter, rtwdev); - } - - static u32 rtw_phy_get_rrsr_mask(struct rtw_dev *rtwdev, u8 rate_idx) -@@ -597,7 +597,7 @@ static void rtw_phy_rrsr_update(struct rtw_dev *rtwdev) - struct rtw_dm_info *dm_info = &rtwdev->dm_info; - - dm_info->rrsr_mask_min = RRSR_RATE_ORDER_MAX; -- rtw_iterate_stas_atomic(rtwdev, rtw_phy_rrsr_mask_min_iter, rtwdev); -+ rtw_iterate_stas(rtwdev, rtw_phy_rrsr_mask_min_iter, rtwdev); - rtw_write32(rtwdev, REG_RRSR, dm_info->rrsr_val_init & dm_info->rrsr_mask_min); - } - -diff --git a/drivers/net/wireless/realtek/rtw88/ps.c b/drivers/net/wireless/realtek/rtw88/ps.c -index bfa64c038f5f..a7213ff2c224 100644 ---- a/drivers/net/wireless/realtek/rtw88/ps.c -+++ b/drivers/net/wireless/realtek/rtw88/ps.c -@@ -58,7 +58,7 @@ int rtw_leave_ips(struct rtw_dev *rtwdev) - return ret; - } - -- rtw_iterate_vifs_atomic(rtwdev, rtw_restore_port_cfg_iter, rtwdev); -+ rtw_iterate_vifs(rtwdev, rtw_restore_port_cfg_iter, rtwdev); - - rtw_coex_ips_notify(rtwdev, COEX_IPS_LEAVE); - -diff --git a/drivers/net/wireless/realtek/rtw88/reg.h b/drivers/net/wireless/realtek/rtw88/reg.h -index 03bd8dc53f72..8852b24d6c2a 100644 ---- a/drivers/net/wireless/realtek/rtw88/reg.h -+++ b/drivers/net/wireless/realtek/rtw88/reg.h -@@ -184,6 +184,7 @@ - #define BIT_TXDMA_VIQ_MAP(x) \ - (((x) & BIT_MASK_TXDMA_VIQ_MAP) << BIT_SHIFT_TXDMA_VIQ_MAP) - #define REG_TXDMA_PQ_MAP 0x010C -+#define BIT_RXDMA_ARBBW_EN BIT(0) - #define BIT_SHIFT_TXDMA_BEQ_MAP 8 - #define BIT_MASK_TXDMA_BEQ_MAP 0x3 - #define BIT_TXDMA_BEQ_MAP(x) \ -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.c b/drivers/net/wireless/realtek/rtw88/rtw8723d.c -index 993bd6b1d723..d99c10f34f92 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8723d.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.c -@@ -210,6 +210,12 @@ static void rtw8723de_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8723du_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8723d_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - static int rtw8723d_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - { - struct rtw_efuse *efuse = &rtwdev->efuse; -@@ -239,6 +245,9 @@ static int rtw8723d_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8723de_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8723du_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -1945,6 +1954,15 @@ static void rtw8723d_pwr_track(struct rtw_dev *rtwdev) - dm_info->pwr_trk_triggered = false; - } - -+static void rtw8723d_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */ -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static struct rtw_chip_ops rtw8723d_ops = { - .phy_set_param = rtw8723d_phy_set_param, - .read_efuse = rtw8723d_read_efuse, -@@ -1965,6 +1983,7 @@ static struct rtw_chip_ops rtw8723d_ops = { - .config_bfee = NULL, - .set_gid_table = NULL, - .cfg_csi_rate = NULL, -+ .fill_txdesc_checksum = rtw8723d_fill_txdesc_checksum, - - .coex_set_init = rtw8723d_coex_cfg_init, - .coex_set_ant_switch = NULL, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.h b/drivers/net/wireless/realtek/rtw88/rtw8723d.h -index 4641f6e047b4..7d9fc3dde97d 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8723d.h -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.h -@@ -70,6 +70,7 @@ struct rtw8723d_efuse { - u8 country_code[2]; - u8 res[3]; - struct rtw8723de_efuse e; -+ struct rtw8723de_efuse u; - }; - - extern const struct rtw_chip_info rtw8723d_hw_spec; -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723du.c b/drivers/net/wireless/realtek/rtw88/rtw8723du.c -new file mode 100644 -index 000000000000..910f64c16813 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723du.c -@@ -0,0 +1,40 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8723du.h" -+#include "usb.h" -+ -+static const struct usb_device_id rtw_8723du_id_table[] = { -+ /* -+ * ULLI : -+ * ID found in rtw8822bu sources -+ */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xD723, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8723d_hw_spec) }, /* 8723DU 1*1 */ -+ { }, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8723du_id_table); -+ -+static int rtw8723du_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8723du_driver = { -+ .name = "rtw_8723du", -+ .id_table = rtw_8723du_id_table, -+ .probe = rtw8723du_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8723du_driver); -+ -+MODULE_AUTHOR("Hans Ulli Kroll "); -+MODULE_DESCRIPTION("Realtek 802.11n wireless 8723du driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723du.h b/drivers/net/wireless/realtek/rtw88/rtw8723du.h -new file mode 100644 -index 000000000000..2e069f65c055 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8723du.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8723DU_H_ -+#define __RTW_8723DU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+ -+extern struct rtw_chip_info rtw8723d_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.c b/drivers/net/wireless/realtek/rtw88/rtw8821c.c -index 025262a8970e..5b4ea9990b78 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8821c.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.c -@@ -26,6 +26,12 @@ static void rtw8821ce_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8821cu_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8821c_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - enum rtw8821ce_rf_set { - SWITCH_TO_BTG, - SWITCH_TO_WLG, -@@ -68,6 +74,9 @@ static int rtw8821c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8821ce_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8821cu_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -1148,6 +1157,18 @@ static void rtw8821c_phy_cck_pd_set(struct rtw_dev *rtwdev, u8 new_lvl) - dm_info->cck_pd_default + new_lvl * 2); - } - -+static void rtw8821c_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ size_t words; -+ -+ words = (pkt_info->pkt_offset * 8 + chip->tx_pkt_desc_sz) / 2; -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8821c[] = { - {0x0086, - RTW_PWR_CUT_ALL_MSK, -@@ -1521,6 +1542,7 @@ static const struct rtw_rfe_def rtw8821c_rfe_defs[] = { - [2] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2), - [4] = RTW_DEF_RFE_EXT(8821c, 0, 0, 2), - [6] = RTW_DEF_RFE(8821c, 0, 0), -+ [34] = RTW_DEF_RFE(8821c, 0, 0), - }; - - static struct rtw_hw_reg rtw8821c_dig[] = { -@@ -1595,6 +1617,7 @@ static struct rtw_chip_ops rtw8821c_ops = { - .config_bfee = rtw8821c_bf_config_bfee, - .set_gid_table = rtw_bf_set_gid_table, - .cfg_csi_rate = rtw_bf_cfg_csi_rate, -+ .fill_txdesc_checksum = rtw8821c_fill_txdesc_checksum, - - .coex_set_init = rtw8821c_coex_cfg_init, - .coex_set_ant_switch = rtw8821c_coex_cfg_ant_switch, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.h b/drivers/net/wireless/realtek/rtw88/rtw8821c.h -index 2698801fc35d..71ce01623d49 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8821c.h -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.h -@@ -9,6 +9,26 @@ - - #define RCR_VHT_ACK BIT(26) - -+struct rtw8821cu_efuse { -+ u8 res4[4]; /* 0xd0 */ -+ u8 usb_optional_function; -+ u8 res5[0x1e]; -+ u8 res6[2]; -+ u8 serial[0x0b]; /* 0xf5 */ -+ u8 vid; /* 0x100 */ -+ u8 res7; -+ u8 pid; -+ u8 res8[4]; -+ u8 mac_addr[ETH_ALEN]; /* 0x107 */ -+ u8 res9[2]; -+ u8 vendor_name[0x07]; -+ u8 res10[2]; -+ u8 device_name[0x14]; -+ u8 res11[0xcf]; -+ u8 package_type; /* 0x1fb */ -+ u8 res12[0x4]; -+}; -+ - struct rtw8821ce_efuse { - u8 mac_addr[ETH_ALEN]; /* 0xd0 */ - u8 vender_id[2]; -@@ -73,6 +93,7 @@ struct rtw8821c_efuse { - u8 res[3]; - union { - struct rtw8821ce_efuse e; -+ struct rtw8821cu_efuse u; - }; - }; - -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c -new file mode 100644 -index 000000000000..e6710c5ebdfc ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c -@@ -0,0 +1,69 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8821cu.h" -+#include "usb.h" -+ -+static const struct usb_device_id rtw_8821cu_id_table[] = { -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xb82b, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xb820, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC821, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC820, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC82A, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC82B, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8821CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0xC811, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8811CU */ -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ 0x8811, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* 8811CU */ -+ /*=== Customer ID ===*/ -+ { USB_DEVICE(0x0bda, 0x2006), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Totolink */ -+ { USB_DEVICE(0x0bda, 0xc811), -+ .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Simplecom NW602 */ -+ {}, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8821cu_id_table); -+ -+static int rtw_8821cu_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8821cu_driver = { -+ .name = "rtw_8821cu", -+ .id_table = rtw_8821cu_id_table, -+ .probe = rtw_8821cu_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8821cu_driver); -+ -+MODULE_AUTHOR("Hans Ulli Kroll "); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8821cu driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.h b/drivers/net/wireless/realtek/rtw88/rtw8821cu.h -new file mode 100644 -index 000000000000..bddbd96aa45f ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8821CU_H_ -+#define __RTW_8821CU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+#define RTW_USB_PRODUCT_ID_REALTEK_8811C 0xC811 -+#define RTW_USB_PRODUCT_ID_REALTEK_8821C 0xC81C -+ -+extern struct rtw_chip_info rtw8821c_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c -index 321848870561..fd1e9a47ccef 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c -@@ -26,6 +26,12 @@ static void rtw8822be_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8822bu_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8822b_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - { - struct rtw_efuse *efuse = &rtwdev->efuse; -@@ -56,6 +62,9 @@ static int rtw8822b_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8822be_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8822bu_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -1588,6 +1597,15 @@ static void rtw8822b_adaptivity(struct rtw_dev *rtwdev) - rtw_phy_set_edcca_th(rtwdev, l2h, h2l); - } - -+static void rtw8822b_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ size_t words = 32 / 2; /* calculate the first 32 bytes (16 words) */ -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8822b[] = { - {0x0086, - RTW_PWR_CUT_ALL_MSK, -@@ -2163,6 +2181,7 @@ static struct rtw_chip_ops rtw8822b_ops = { - .cfg_csi_rate = rtw_bf_cfg_csi_rate, - .adaptivity_init = rtw8822b_adaptivity_init, - .adaptivity = rtw8822b_adaptivity, -+ .fill_txdesc_checksum = rtw8822b_fill_txdesc_checksum, - - .coex_set_init = rtw8822b_coex_cfg_init, - .coex_set_ant_switch = rtw8822b_coex_cfg_ant_switch, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bu.c b/drivers/net/wireless/realtek/rtw88/rtw8822bu.c -new file mode 100644 -index 000000000000..5becebdc3247 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822bu.c -@@ -0,0 +1,62 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8822bu.h" -+#include "usb.h" -+ -+#define RTW_USB_VENDER_ID_EDIMAX 0x7392 -+ -+static const struct usb_device_id rtw_8822bu_id_table[] = { -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8812B, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8822B, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDER_ID_EDIMAX, -+ 0xB822, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDER_ID_EDIMAX, -+ 0xC822, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x0b05, 0x184c), /* ASUS AC53 Nano */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x0b05, 0x1841), /* ASUS AC55 B1 */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x2001, 0x331c), /* D-Link DWA-182 rev D1 */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x13b1, 0x0043), /* Linksys WUSB6400M */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x2357, 0x012D), /* TP-Link AC1300 T3U */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ { USB_DEVICE(0x2357, 0x0138), /* TP-Link AC1300 T3U */ -+ .driver_info = (kernel_ulong_t)&(rtw8822b_hw_spec) }, -+ {}, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8822bu_id_table); -+ -+static int rtw8822bu_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8822bu_driver = { -+ .name = "rtw_8822bu", -+ .id_table = rtw_8822bu_id_table, -+ .probe = rtw8822bu_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8822bu_driver); -+ -+MODULE_AUTHOR("Realtek Corporation"); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822bu driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822bu.h b/drivers/net/wireless/realtek/rtw88/rtw8822bu.h -new file mode 100644 -index 000000000000..20f01ecd7441 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822bu.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8822BU_H_ -+#define __RTW_8822BU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+#define RTW_USB_PRODUCT_ID_REALTEK_8812B 0xB812 -+#define RTW_USB_PRODUCT_ID_REALTEK_8822B 0xB82C -+ -+extern struct rtw_chip_info rtw8822b_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c -index 09f9e4adcf34..c0283c8278da 100644 ---- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c -@@ -29,6 +29,12 @@ static void rtw8822ce_efuse_parsing(struct rtw_efuse *efuse, - ether_addr_copy(efuse->addr, map->e.mac_addr); - } - -+static void rtw8822cu_efuse_parsing(struct rtw_efuse *efuse, -+ struct rtw8822c_efuse *map) -+{ -+ ether_addr_copy(efuse->addr, map->u.mac_addr); -+} -+ - static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - { - struct rtw_efuse *efuse = &rtwdev->efuse; -@@ -58,6 +64,9 @@ static int rtw8822c_read_efuse(struct rtw_dev *rtwdev, u8 *log_map) - case RTW_HCI_TYPE_PCIE: - rtw8822ce_efuse_parsing(efuse, map); - break; -+ case RTW_HCI_TYPE_USB: -+ rtw8822cu_efuse_parsing(efuse, map); -+ break; - default: - /* unsupported now */ - return -ENOTSUPP; -@@ -4557,6 +4566,18 @@ static void rtw8822c_adaptivity(struct rtw_dev *rtwdev) - rtw_phy_set_edcca_th(rtwdev, l2h, h2l); - } - -+static void rtw8822c_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ size_t words; -+ -+ words = (pkt_info->pkt_offset * 8 + chip->tx_pkt_desc_sz) / 2; -+ -+ fill_txdesc_checksum_common(txdesc, words); -+} -+ - static const struct rtw_pwr_seq_cmd trans_carddis_to_cardemu_8822c[] = { - {0x0086, - RTW_PWR_CUT_ALL_MSK, -@@ -4895,6 +4916,8 @@ static const struct rtw_rfe_def rtw8822c_rfe_defs[] = { - [0] = RTW_DEF_RFE(8822c, 0, 0), - [1] = RTW_DEF_RFE(8822c, 0, 0), - [2] = RTW_DEF_RFE(8822c, 0, 0), -+ [3] = RTW_DEF_RFE(8822c, 0, 0), -+ [4] = RTW_DEF_RFE(8822c, 0, 0), - [5] = RTW_DEF_RFE(8822c, 0, 5), - [6] = RTW_DEF_RFE(8822c, 0, 0), - }; -@@ -4978,6 +5001,7 @@ static struct rtw_chip_ops rtw8822c_ops = { - .cfo_track = rtw8822c_cfo_track, - .config_tx_path = rtw8822c_config_tx_path, - .config_txrx_mode = rtw8822c_config_trx_mode, -+ .fill_txdesc_checksum = rtw8822c_fill_txdesc_checksum, - - .coex_set_init = rtw8822c_coex_cfg_init, - .coex_set_ant_switch = NULL, -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.c b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c -new file mode 100644 -index 000000000000..36dc734f76eb ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.c -@@ -0,0 +1,40 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include "main.h" -+#include "rtw8822cu.h" -+#include "usb.h" -+ -+static const struct usb_device_id rtw_8822cu_id_table[] = { -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8822C, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, -+ { USB_DEVICE_AND_INTERFACE_INFO(RTW_USB_VENDOR_ID_REALTEK, -+ RTW_USB_PRODUCT_ID_REALTEK_8812C, -+ 0xff, 0xff, 0xff), -+ .driver_info = (kernel_ulong_t)&(rtw8822c_hw_spec) }, -+ {}, -+}; -+MODULE_DEVICE_TABLE(usb, rtw_8822cu_id_table); -+ -+static int rtw8822bu_probe(struct usb_interface *intf, -+ const struct usb_device_id *id) -+{ -+ return rtw_usb_probe(intf, id); -+} -+ -+static struct usb_driver rtw_8822cu_driver = { -+ .name = "rtw_8822cu", -+ .id_table = rtw_8822cu_id_table, -+ .probe = rtw8822bu_probe, -+ .disconnect = rtw_usb_disconnect, -+}; -+module_usb_driver(rtw_8822cu_driver); -+ -+MODULE_AUTHOR("Realtek Corporation"); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless 8822cu driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822cu.h b/drivers/net/wireless/realtek/rtw88/rtw8822cu.h -new file mode 100644 -index 000000000000..16afe22a8216 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/rtw8822cu.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_8822CU_H_ -+#define __RTW_8822CU_H_ -+ -+/* USB Vendor/Product IDs */ -+#define RTW_USB_VENDOR_ID_REALTEK 0x0BDA -+#define RTW_USB_PRODUCT_ID_REALTEK_8812C 0xC812 -+#define RTW_USB_PRODUCT_ID_REALTEK_8822C 0xC82C -+ -+extern struct rtw_chip_info rtw8822c_hw_spec; -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/tx.h b/drivers/net/wireless/realtek/rtw88/tx.h -index 8419603adce4..d283bb05d1dd 100644 ---- a/drivers/net/wireless/realtek/rtw88/tx.h -+++ b/drivers/net/wireless/realtek/rtw88/tx.h -@@ -71,6 +71,14 @@ - le32p_replace_bits((__le32 *)(txdesc) + 0x03, value, BIT(15)) - #define SET_TX_DESC_BT_NULL(txdesc, value) \ - le32p_replace_bits((__le32 *)(txdesc) + 0x02, value, BIT(23)) -+#define SET_TX_DESC_TXDESC_CHECKSUM(txdesc, value) \ -+ le32p_replace_bits((__le32 *)(txdesc) + 0x07, value, GENMASK(15, 0)) -+#define SET_TX_DESC_DMA_TXAGG_NUM(txdesc, value) \ -+ le32p_replace_bits((__le32 *)(txdesc) + 0x07, value, GENMASK(31, 24)) -+#define GET_TX_DESC_PKT_OFFSET(txdesc) \ -+ le32_get_bits(*((__le32 *)(txdesc) + 0x01), GENMASK(28, 24)) -+#define GET_TX_DESC_QSEL(txdesc) \ -+ le32_get_bits(*((__le32 *)(txdesc) + 0x01), GENMASK(12, 8)) - - enum rtw_tx_desc_queue_select { - TX_DESC_QSEL_TID0 = 0, -@@ -123,4 +131,27 @@ rtw_tx_write_data_h2c_get(struct rtw_dev *rtwdev, - struct rtw_tx_pkt_info *pkt_info, - u8 *buf, u32 size); - -+static inline -+void fill_txdesc_checksum_common(u8 *txdesc, size_t words) -+{ -+ __le16 chksum = 0; -+ __le16 *data = (__le16 *)(txdesc); -+ -+ SET_TX_DESC_TXDESC_CHECKSUM(txdesc, 0x0000); -+ -+ while (words--) -+ chksum ^= *data++; -+ -+ SET_TX_DESC_TXDESC_CHECKSUM(txdesc, __le16_to_cpu(chksum)); -+} -+ -+static inline void rtw_tx_fill_txdesc_checksum(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *txdesc) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ -+ chip->ops->fill_txdesc_checksum(rtwdev, pkt_info, txdesc); -+} -+ - #endif -diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c -new file mode 100644 -index 000000000000..7641ea6f6ad1 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/usb.c -@@ -0,0 +1,1051 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#include -+#include -+#include -+#include "main.h" -+#include "debug.h" -+#include "reg.h" -+#include "tx.h" -+#include "rx.h" -+#include "fw.h" -+#include "ps.h" -+#include "usb.h" -+ -+#define RTW_USB_MAX_RXQ_LEN 128 -+ -+struct rtw_usb_txcb { -+ struct rtw_dev *rtwdev; -+ struct sk_buff_head tx_ack_queue; -+}; -+ -+static void rtw_usb_fill_tx_checksum(struct rtw_usb *rtwusb, -+ struct sk_buff *skb, int agg_num) -+{ -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct rtw_tx_pkt_info pkt_info; -+ -+ SET_TX_DESC_DMA_TXAGG_NUM(skb->data, agg_num); -+ pkt_info.pkt_offset = GET_TX_DESC_PKT_OFFSET(skb->data); -+ rtw_tx_fill_txdesc_checksum(rtwdev, &pkt_info, skb->data); -+} -+ -+static void usbctrl_async_callback(struct urb *urb) -+{ -+ /* free dr */ -+ kfree(urb->setup_packet); -+ /* free databuf */ -+ kfree(urb->transfer_buffer); -+} -+ -+static int usbctrl_vendorreq_async_write(struct usb_device *udev, u8 request, -+ u16 value, u16 index, void *pdata, -+ u16 len) -+{ -+ int rc; -+ unsigned int pipe; -+ u8 reqtype; -+ struct usb_ctrlrequest *dr; -+ struct urb *urb; -+ const u16 databuf_maxlen = RTW_USB_VENQT_MAX_BUF_SIZE; -+ u8 *databuf; -+ -+ if (WARN_ON_ONCE(len > databuf_maxlen)) -+ len = databuf_maxlen; -+ -+ pipe = usb_sndctrlpipe(udev, 0); /* write_out */ -+ reqtype = RTW_USB_CMD_WRITE; -+ -+ dr = kzalloc(sizeof(*dr), GFP_ATOMIC); -+ if (!dr) -+ return -ENOMEM; -+ -+ databuf = kmemdup(pdata, len, GFP_ATOMIC); -+ if (!databuf) { -+ kfree(dr); -+ return -ENOMEM; -+ } -+ -+ urb = usb_alloc_urb(0, GFP_ATOMIC); -+ if (!urb) { -+ kfree(databuf); -+ kfree(dr); -+ return -ENOMEM; -+ } -+ -+ dr->bRequestType = reqtype; -+ dr->bRequest = request; -+ dr->wValue = cpu_to_le16(value); -+ dr->wIndex = cpu_to_le16(index); -+ dr->wLength = cpu_to_le16(len); -+ -+ usb_fill_control_urb(urb, udev, pipe, -+ (unsigned char *)dr, databuf, len, -+ usbctrl_async_callback, NULL); -+ rc = usb_submit_urb(urb, GFP_ATOMIC); -+ if (rc < 0) { -+ kfree(databuf); -+ kfree(dr); -+ } -+ -+ usb_free_urb(urb); -+ -+ return rc; -+} -+ -+static u32 rtw_usb_read_sync(struct rtw_dev *rtwdev, u32 addr, u16 len) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *udev = rtwusb->udev; -+ __le32 *data; -+ unsigned long flags; -+ int ret; -+ static int count; -+ -+ spin_lock_irqsave(&rtwusb->usb_lock, flags); -+ -+ if (++rtwusb->usb_data_index >= RTW_USB_MAX_RX_COUNT) -+ rtwusb->usb_data_index = 0; -+ data = &rtwusb->usb_data[rtwusb->usb_data_index]; -+ -+ spin_unlock_irqrestore(&rtwusb->usb_lock, flags); -+ -+ ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), -+ RTW_USB_CMD_REQ, RTW_USB_CMD_READ, addr, -+ RTW_USB_VENQT_CMD_IDX, data, len, 1000); -+ if (ret < 0 && ret != -ENODEV && count++ < 4) -+ rtw_err(rtwdev, "reg 0x%x, usbctrl_vendorreq failed with %d\n", -+ addr, ret); -+ -+ return le32_to_cpu(*data); -+} -+ -+static u8 rtw_usb_read8_sync(struct rtw_dev *rtwdev, u32 addr) -+{ -+ return (u8)rtw_usb_read_sync(rtwdev, addr, 1); -+} -+ -+static u16 rtw_usb_read16_sync(struct rtw_dev *rtwdev, u32 addr) -+{ -+ return (u16)rtw_usb_read_sync(rtwdev, addr, 2); -+} -+ -+static u32 rtw_usb_read32_sync(struct rtw_dev *rtwdev, u32 addr) -+{ -+ return (u32)rtw_usb_read_sync(rtwdev, addr, 4); -+} -+ -+static void rtw_usb_write_async(struct rtw_dev *rtwdev, u32 addr, u32 val, -+ u16 len) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *udev = rtwusb->udev; -+ u8 request; -+ u16 wvalue; -+ u16 index; -+ __le32 data; -+ -+ request = RTW_USB_CMD_REQ; -+ index = RTW_USB_VENQT_CMD_IDX; /* n/a */ -+ wvalue = (u16)(addr & 0x0000ffff); -+ data = cpu_to_le32(val); -+ usbctrl_vendorreq_async_write(udev, request, wvalue, index, &data, len); -+} -+ -+static void rtw_usb_write8_async(struct rtw_dev *rtwdev, u32 addr, u8 val) -+{ -+ rtw_usb_write_async(rtwdev, addr, val, 1); -+} -+ -+static void rtw_usb_write16_async(struct rtw_dev *rtwdev, u32 addr, u16 val) -+{ -+ rtw_usb_write_async(rtwdev, addr, val, 2); -+} -+ -+static void rtw_usb_write32_async(struct rtw_dev *rtwdev, u32 addr, u32 val) -+{ -+ rtw_usb_write_async(rtwdev, addr, val, 4); -+} -+ -+static int rtw_usb_parse(struct rtw_dev *rtwdev, -+ struct usb_interface *interface) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_host_interface *host_interface = &interface->altsetting[0]; -+ struct usb_interface_descriptor *interface_desc = &host_interface->desc; -+ struct usb_endpoint_descriptor *endpoint; -+ struct usb_device *usbd = interface_to_usbdev(interface); -+ int num_out_pipes = 0; -+ int i; -+ u8 num; -+ -+ for (i = 0; i < interface_desc->bNumEndpoints; i++) { -+ endpoint = &host_interface->endpoint[i].desc; -+ num = usb_endpoint_num(endpoint); -+ -+ if (usb_endpoint_dir_in(endpoint) && -+ usb_endpoint_xfer_bulk(endpoint)) { -+ if (rtwusb->pipe_in) { -+ rtw_err(rtwdev, "IN pipes overflow\n"); -+ return -EINVAL; -+ } -+ -+ rtwusb->pipe_in = num; -+ } -+ -+ if (usb_endpoint_dir_in(endpoint) && -+ usb_endpoint_xfer_int(endpoint)) { -+ if (rtwusb->pipe_interrupt) { -+ rtw_err(rtwdev, "INT pipes overflow\n"); -+ return -EINVAL; -+ } -+ -+ rtwusb->pipe_interrupt = num; -+ } -+ -+ if (usb_endpoint_dir_out(endpoint) && -+ usb_endpoint_xfer_bulk(endpoint)) { -+ if (num_out_pipes >= ARRAY_SIZE(rtwusb->out_ep)) { -+ rtw_err(rtwdev, "OUT pipes overflow\n"); -+ return -EINVAL; -+ } -+ -+ rtwusb->out_ep[num_out_pipes++] = num; -+ } -+ } -+ -+ switch (usbd->speed) { -+ case USB_SPEED_LOW: -+ case USB_SPEED_FULL: -+ rtwusb->bulkout_size = RTW_USB_FULL_SPEED_BULK_SIZE; -+ break; -+ case USB_SPEED_HIGH: -+ rtwusb->bulkout_size = RTW_USB_HIGH_SPEED_BULK_SIZE; -+ break; -+ case USB_SPEED_SUPER: -+ rtwusb->bulkout_size = RTW_USB_SUPER_SPEED_BULK_SIZE; -+ break; -+ default: -+ rtw_err(rtwdev, "failed to detect usb speed\n"); -+ return -EINVAL; -+ } -+ -+ rtwdev->hci.bulkout_num = num_out_pipes; -+ -+ switch (num_out_pipes) { -+ case 4: -+ case 3: -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 2; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID4] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID5] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID6] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID7] = 1; -+ break; -+ case 2: -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID0] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID1] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID2] = 1; -+ rtwusb->qsel_to_ep[TX_DESC_QSEL_TID3] = 1; -+ break; -+ case 1: -+ break; -+ default: -+ rtw_err(rtwdev, "failed to get out_pipes(%d)\n", num_out_pipes); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+static void rtw_usb_txcb_enqueue(struct rtw_usb_txcb *txcb, struct sk_buff *skb) -+{ -+ skb_queue_tail(&txcb->tx_ack_queue, skb); -+} -+ -+static void rtw_usb_tx_agg_skb(struct rtw_usb *rtwusb, struct sk_buff_head *list, -+ struct sk_buff *skb_head, struct sk_buff *skb, -+ struct rtw_usb_txcb *txcb) -+{ -+ struct sk_buff *skb_iter; -+ unsigned long flags; -+ u8 *data_ptr; -+ int agg_num = 0, len, max_len; -+ -+ data_ptr = skb_head->data; -+ skb_iter = skb; -+ -+ while (skb_iter) { -+ memcpy(data_ptr, skb_iter->data, skb_iter->len); -+ len = ALIGN(skb_iter->len, 8); -+ skb_put(skb_head, len); -+ data_ptr += len; -+ agg_num++; -+ -+ rtw_usb_txcb_enqueue(txcb, skb_iter); -+ -+ spin_lock_irqsave(&list->lock, flags); -+ -+ skb_iter = skb_peek(list); -+ max_len = RTW_USB_MAX_XMITBUF_SZ - skb_head->len; -+ -+ if (skb_iter && skb_iter->len < max_len) -+ __skb_unlink(skb_iter, list); -+ else -+ skb_iter = NULL; -+ spin_unlock_irqrestore(&list->lock, flags); -+ } -+ -+ if (agg_num > 1) -+ rtw_usb_fill_tx_checksum(rtwusb, skb_head, agg_num); -+} -+ -+static void rtw_usb_indicate_tx_status(struct rtw_dev *rtwdev, -+ struct sk_buff *skb) -+{ -+ struct ieee80211_hw *hw = rtwdev->hw; -+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); -+ struct rtw_usb_tx_data *tx_data = rtw_usb_get_tx_data(skb); -+ -+ /* enqueue to wait for tx report */ -+ if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { -+ rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); -+ return; -+ } -+ -+ /* always ACK for others, then they won't be marked as drop */ -+ ieee80211_tx_info_clear_status(info); -+ if (info->flags & IEEE80211_TX_CTL_NO_ACK) -+ info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; -+ else -+ info->flags |= IEEE80211_TX_STAT_ACK; -+ -+ ieee80211_tx_status_irqsafe(hw, skb); -+} -+ -+static void rtw_usb_write_port_tx_complete(struct urb *urb) -+{ -+ struct rtw_usb_txcb *txcb = urb->context; -+ struct rtw_dev *rtwdev = txcb->rtwdev; -+ -+ while (true) { -+ struct sk_buff *skb = skb_dequeue(&txcb->tx_ack_queue); -+ if (!skb) -+ break; -+ -+ if (GET_TX_DESC_QSEL(skb->data) <= TX_DESC_QSEL_TID7) -+ rtw_usb_indicate_tx_status(rtwdev, skb); -+ else -+ dev_kfree_skb_any(skb); -+ } -+ -+ kfree(txcb); -+} -+ -+static int rtw_usb_write_port(struct rtw_dev *rtwdev, u8 qsel, struct sk_buff *skb, -+ usb_complete_t cb, void *context) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *usbd = rtwusb->udev; -+ struct urb *urb; -+ unsigned int pipe; -+ int ret; -+ int ep = rtwusb->qsel_to_ep[qsel]; -+ -+ pipe = usb_sndbulkpipe(usbd, rtwusb->out_ep[ep]); -+ urb = usb_alloc_urb(0, GFP_ATOMIC); -+ if (!urb) -+ return -ENOMEM; -+ -+ usb_fill_bulk_urb(urb, usbd, pipe, skb->data, skb->len, cb, context); -+ ret = usb_submit_urb(urb, GFP_ATOMIC); -+ -+ usb_free_urb(urb); -+ -+ return ret; -+} -+ -+static struct sk_buff *rtw_usb_tx_agg_check(struct rtw_usb *rtwusb, -+ struct sk_buff *skb, -+ int index, -+ struct rtw_usb_txcb *txcb) -+{ -+ struct sk_buff_head *list; -+ struct sk_buff *skb_head; -+ -+ list = &rtwusb->tx_queue[index]; -+ if (skb_queue_empty(list)) -+ return NULL; -+ -+ skb_head = dev_alloc_skb(RTW_USB_MAX_XMITBUF_SZ); -+ if (!skb_head) -+ return NULL; -+ -+ rtw_usb_tx_agg_skb(rtwusb, list, skb_head, skb, txcb); -+ -+ return skb_head; -+} -+ -+static void rtw_usb_tx_agg(struct rtw_usb *rtwusb, struct sk_buff *skb, int index) -+{ -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct sk_buff *skb_head; -+ struct rtw_usb_txcb *txcb; -+ u8 qsel; -+ -+ txcb = kmalloc(sizeof(*txcb), GFP_ATOMIC); -+ if (!txcb) -+ return; -+ -+ txcb->rtwdev = rtwdev; -+ skb_queue_head_init(&txcb->tx_ack_queue); -+ -+ skb_head = rtw_usb_tx_agg_check(rtwusb, skb, index, txcb); -+ if (!skb_head) { -+ skb_head = skb; -+ rtw_usb_txcb_enqueue(txcb, skb); -+ } -+ -+ qsel = GET_TX_DESC_QSEL(skb->data); -+ -+ rtw_usb_write_port(rtwdev, qsel, skb_head, -+ rtw_usb_write_port_tx_complete, txcb); -+ -+ if (skb_head != skb) -+ dev_kfree_skb(skb_head); -+} -+ -+static void rtw_usb_tx_handler(struct work_struct *work) -+{ -+ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, tx_work); -+ struct sk_buff *skb; -+ int index, limit; -+ -+ for (index = ARRAY_SIZE(rtwusb->tx_queue) - 1; index >= 0; index--) { -+ for (limit = 0; limit < 200; limit++) { -+ skb = skb_dequeue(&rtwusb->tx_queue[index]); -+ if (skb) -+ rtw_usb_tx_agg(rtwusb, skb, index); -+ else -+ break; -+ } -+ } -+} -+ -+static void rtw_usb_tx_queue_purge(struct rtw_usb *rtwusb) -+{ -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) -+ skb_queue_purge(&rtwusb->tx_queue[i]); -+} -+ -+static void rtw_usb_write_port_complete(struct urb *urb) -+{ -+ struct sk_buff *skb = urb->context; -+ -+ dev_kfree_skb_any(skb); -+} -+ -+static int rtw_usb_write_data(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ u8 *buf) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct sk_buff *skb; -+ unsigned int desclen, headsize, size; -+ u8 qsel; -+ int ret = 0; -+ -+ size = pkt_info->tx_pkt_size; -+ qsel = pkt_info->qsel; -+ desclen = chip->tx_pkt_desc_sz; -+ headsize = pkt_info->offset ? pkt_info->offset : desclen; -+ -+ skb = dev_alloc_skb(headsize + size); -+ if (unlikely(!skb)) -+ return -ENOMEM; -+ -+ skb_reserve(skb, headsize); -+ skb_put_data(skb, buf, size); -+ skb_push(skb, headsize); -+ memset(skb->data, 0, headsize); -+ rtw_tx_fill_tx_desc(pkt_info, skb); -+ rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, skb->data); -+ -+ ret = rtw_usb_write_port(rtwdev, qsel, skb, -+ rtw_usb_write_port_complete, skb); -+ if (unlikely(ret)) -+ rtw_err(rtwdev, "failed to do USB write, ret=%d\n", ret); -+ -+ return ret; -+} -+ -+static int rtw_usb_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, -+ u32 size) -+{ -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct rtw_usb *rtwusb; -+ struct rtw_tx_pkt_info pkt_info = {0}; -+ u32 len, desclen; -+ -+ if (unlikely(!rtwdev)) -+ return -EINVAL; -+ -+ rtwusb = rtw_get_usb_priv(rtwdev); -+ if (unlikely(!rtwusb)) -+ return -EINVAL; -+ -+ pkt_info.tx_pkt_size = size; -+ pkt_info.qsel = TX_DESC_QSEL_BEACON; -+ -+ desclen = chip->tx_pkt_desc_sz; -+ len = desclen + size; -+ if (len % rtwusb->bulkout_size == 0) { -+ len += RTW_USB_PACKET_OFFSET_SZ; -+ pkt_info.offset = desclen + RTW_USB_PACKET_OFFSET_SZ; -+ pkt_info.pkt_offset = 1; -+ } else { -+ pkt_info.offset = desclen; -+ } -+ -+ return rtw_usb_write_data(rtwdev, &pkt_info, buf); -+} -+ -+static int rtw_usb_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) -+{ -+ struct rtw_tx_pkt_info pkt_info = {0}; -+ -+ pkt_info.tx_pkt_size = size; -+ pkt_info.qsel = TX_DESC_QSEL_H2C; -+ -+ return rtw_usb_write_data(rtwdev, &pkt_info, buf); -+} -+ -+static u8 rtw_usb_tx_queue_mapping_to_qsel(struct sk_buff *skb) -+{ -+ struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; -+ __le16 fc = hdr->frame_control; -+ u8 qsel; -+ -+ if (unlikely(ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc))) -+ qsel = TX_DESC_QSEL_MGMT; -+ else if (skb_get_queue_mapping(skb) <= IEEE80211_AC_BK) -+ qsel = skb->priority; -+ else -+ qsel = TX_DESC_QSEL_BEACON; -+ -+ return qsel; -+} -+ -+static int rtw_usb_tx_write(struct rtw_dev *rtwdev, -+ struct rtw_tx_pkt_info *pkt_info, -+ struct sk_buff *skb) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct rtw_usb_tx_data *tx_data; -+ u8 *pkt_desc; -+ int ep; -+ -+ pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); -+ memset(pkt_desc, 0, chip->tx_pkt_desc_sz); -+ pkt_info->qsel = rtw_usb_tx_queue_mapping_to_qsel(skb); -+ ep = rtwusb->qsel_to_ep[pkt_info->qsel]; -+ rtw_tx_fill_tx_desc(pkt_info, skb); -+ rtw_tx_fill_txdesc_checksum(rtwdev, pkt_info, skb->data); -+ tx_data = rtw_usb_get_tx_data(skb); -+ tx_data->sn = pkt_info->sn; -+ -+ skb_queue_tail(&rtwusb->tx_queue[ep], skb); -+ -+ return 0; -+} -+ -+static void rtw_usb_tx_kick_off(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ queue_work(rtwusb->txwq, &rtwusb->tx_work); -+} -+ -+static void rtw_usb_rx_handler(struct work_struct *work) -+{ -+ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, rx_work); -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct rtw_chip_info *chip = rtwdev->chip; -+ struct rtw_rx_pkt_stat pkt_stat; -+ struct ieee80211_rx_status rx_status; -+ struct sk_buff *skb; -+ u32 pkt_desc_sz = chip->rx_pkt_desc_sz; -+ u32 pkt_offset; -+ u8 *rx_desc; -+ int limit; -+ -+ for (limit = 0; limit < 200; limit++) { -+ skb = skb_dequeue(&rtwusb->rx_queue); -+ if (!skb) -+ break; -+ -+ rx_desc = skb->data; -+ chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, -+ &rx_status); -+ pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + -+ pkt_stat.shift; -+ -+ if (pkt_stat.is_c2h) { -+ skb_put(skb, pkt_stat.pkt_len + pkt_offset); -+ rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, -+ skb); -+ continue; -+ } -+ -+ if (skb_queue_len(&rtwusb->rx_queue) >= RTW_USB_MAX_RXQ_LEN) { -+ rtw_err(rtwdev, "failed to get rx_queue, overflow\n"); -+ dev_kfree_skb_any(skb); -+ continue; -+ } -+ -+ skb_put(skb, pkt_stat.pkt_len); -+ skb_reserve(skb, pkt_offset); -+ memcpy(skb->cb, &rx_status, sizeof(rx_status)); -+ ieee80211_rx_irqsafe(rtwdev->hw, skb); -+ } -+} -+ -+static void rtw_usb_rx_data_put(struct rtw_usb *rtwusb, -+ struct rx_usb_ctrl_block *rxcb) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_move(&rxcb->list, &rtwusb->rx_data_free); -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+} -+ -+static void rtw_usb_read_port_complete(struct urb *urb) -+{ -+ struct rx_usb_ctrl_block *rxcb = urb->context; -+ struct rtw_dev *rtwdev = rxcb->rtwdev; -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct sk_buff *skb = rxcb->rx_skb; -+ -+ if (urb->status == 0) { -+ if (urb->actual_length >= RTW_USB_MAX_RECVBUF_SZ || -+ urb->actual_length < 24) { -+ rtw_err(rtwdev, "failed to get urb length:%d\n", -+ urb->actual_length); -+ if (skb) -+ dev_kfree_skb_any(skb); -+ } else { -+ skb_queue_tail(&rtwusb->rx_queue, skb); -+ queue_work(rtwusb->rxwq, &rtwusb->rx_work); -+ } -+ -+ rtw_usb_rx_data_put(rtwusb, rxcb); -+ queue_work(rtwusb->rxwq, &rtwusb->rx_refill_work); -+ } else { -+ switch (urb->status) { -+ case -EINVAL: -+ case -EPIPE: -+ case -ENODEV: -+ case -ESHUTDOWN: -+ case -ENOENT: -+ case -EPROTO: -+ case -EILSEQ: -+ case -ETIME: -+ case -ECOMM: -+ case -EOVERFLOW: -+ case -EINPROGRESS: -+ break; -+ default: -+ rtw_err(rtwdev, "status unknown=%d\n", urb->status); -+ break; -+ } -+ if (skb) -+ dev_kfree_skb_any(skb); -+ } -+} -+ -+static void rtw_usb_rx_refill_work(struct work_struct *work) -+{ -+ struct rtw_usb *rtwusb = container_of(work, struct rtw_usb, rx_refill_work); -+ struct rtw_dev *rtwdev = rtwusb->rtwdev; -+ struct rx_usb_ctrl_block *rxcb; -+ unsigned long flags; -+ int error; -+ -+ do { -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ -+ rxcb = list_first_entry_or_null(&rtwusb->rx_data_free, -+ struct rx_usb_ctrl_block, list); -+ -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ if (!rxcb) -+ return; -+ -+ rxcb->rx_skb = alloc_skb(RTW_USB_MAX_RECVBUF_SZ, GFP_KERNEL); -+ if (!rxcb->rx_skb) { -+ rtw_err(rtwdev, "could not allocate rx skbuff\n"); -+ return; -+ } -+ -+ usb_fill_bulk_urb(rxcb->rx_urb, rtwusb->udev, -+ usb_rcvbulkpipe(rtwusb->udev, rtwusb->pipe_in), -+ rxcb->rx_skb->data, RTW_USB_MAX_RECVBUF_SZ, -+ rtw_usb_read_port_complete, rxcb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_move(&rxcb->list, &rtwusb->rx_data_used); -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ -+ error = usb_submit_urb(rxcb->rx_urb, GFP_KERNEL); -+ if (error) { -+ kfree_skb(rxcb->rx_skb); -+ if (error != -ENODEV) -+ rtw_err(rtwdev, "Err sending rx data urb %d\n", -+ error); -+ rtw_usb_rx_data_put(rtwusb, rxcb); -+ -+ return; -+ } -+ } while (true); -+} -+ -+static void rtw_usb_cancel_rx_bufs(struct rtw_usb *rtwusb) -+{ -+ struct rx_usb_ctrl_block *rxcb; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ -+ while (true) { -+ rxcb = list_first_entry_or_null(&rtwusb->rx_data_used, -+ struct rx_usb_ctrl_block, list); -+ -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ -+ if (!rxcb) -+ break; -+ -+ usb_kill_urb(rxcb->rx_urb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_move(&rxcb->list, &rtwusb->rx_data_free); -+ } -+} -+ -+static void rtw_usb_free_rx_bufs(struct rtw_usb *rtwusb) -+{ -+ struct rx_usb_ctrl_block *rxcb; -+ unsigned long flags; -+ -+ rtw_usb_cancel_rx_bufs(rtwusb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ -+ while (true) { -+ rxcb = list_first_entry_or_null(&rtwusb->rx_data_free, struct rx_usb_ctrl_block, list); -+ -+ spin_unlock_irqrestore(&rtwusb->rx_data_list_lock, flags); -+ -+ if (!rxcb) -+ break; -+ -+ usb_free_urb(rxcb->rx_urb); -+ -+ spin_lock_irqsave(&rtwusb->rx_data_list_lock, flags); -+ list_del(&rxcb->list); -+ } -+} -+ -+static int rtw_usb_alloc_rx_bufs(struct rtw_usb *rtwusb) -+{ -+ int i; -+ -+ for (i = 0; i < RTW_USB_RXCB_NUM; i++) { -+ struct rx_usb_ctrl_block *rxcb = &rtwusb->rx_cb[i]; -+ -+ rxcb->rtwdev = rtwusb->rtwdev; -+ rxcb->rx_urb = usb_alloc_urb(0, GFP_KERNEL); -+ if (!rxcb->rx_urb) -+ goto err; -+ list_add_tail(&rxcb->list, &rtwusb->rx_data_free); -+ } -+ -+ return 0; -+err: -+ rtw_usb_free_rx_bufs(rtwusb); -+ return -ENOMEM; -+} -+ -+static int rtw_usb_setup(struct rtw_dev *rtwdev) -+{ -+ /* empty function for rtw_hci_ops */ -+ return 0; -+} -+ -+static int rtw_usb_start(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ queue_work(rtwusb->rxwq, &rtwusb->rx_refill_work); -+ -+ return 0; -+} -+ -+static void rtw_usb_stop(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ cancel_work_sync(&rtwusb->rx_refill_work); -+ rtw_usb_cancel_rx_bufs(rtwusb); -+} -+ -+static void rtw_usb_deep_ps(struct rtw_dev *rtwdev, bool enter) -+{ -+ /* empty function for rtw_hci_ops */ -+} -+ -+static void rtw_usb_link_ps(struct rtw_dev *rtwdev, bool enter) -+{ -+ /* empty function for rtw_hci_ops */ -+} -+ -+static void rtw_usb_interface_cfg(struct rtw_dev *rtwdev) -+{ -+ /* empty function for rtw_hci_ops */ -+} -+ -+static struct rtw_hci_ops rtw_usb_ops = { -+ .tx_write = rtw_usb_tx_write, -+ .tx_kick_off = rtw_usb_tx_kick_off, -+ .setup = rtw_usb_setup, -+ .start = rtw_usb_start, -+ .stop = rtw_usb_stop, -+ .deep_ps = rtw_usb_deep_ps, -+ .link_ps = rtw_usb_link_ps, -+ .interface_cfg = rtw_usb_interface_cfg, -+ -+ .write8 = rtw_usb_write8_async, -+ .write16 = rtw_usb_write16_async, -+ .write32 = rtw_usb_write32_async, -+ .read8 = rtw_usb_read8_sync, -+ .read16 = rtw_usb_read16_sync, -+ .read32 = rtw_usb_read32_sync, -+ -+ .write_data_rsvd_page = rtw_usb_write_data_rsvd_page, -+ .write_data_h2c = rtw_usb_write_data_h2c, -+}; -+ -+static int rtw_usb_init_rx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ rtwusb->rxwq = create_singlethread_workqueue("rtw88_usb: rx wq"); -+ if (!rtwusb->rxwq) { -+ rtw_err(rtwdev, "failed to create RX work queue\n"); -+ return -ENOMEM; -+ } -+ -+ skb_queue_head_init(&rtwusb->rx_queue); -+ -+ INIT_WORK(&rtwusb->rx_work, rtw_usb_rx_handler); -+ -+ return 0; -+} -+ -+static void rtw_usb_deinit_rx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ skb_queue_purge(&rtwusb->rx_queue); -+ -+ flush_workqueue(rtwusb->rxwq); -+ destroy_workqueue(rtwusb->rxwq); -+} -+ -+static int rtw_usb_init_tx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ int i; -+ -+ rtwusb->txwq = create_singlethread_workqueue("rtw88_usb: tx wq"); -+ if (!rtwusb->txwq) { -+ rtw_err(rtwdev, "failed to create TX work queue\n"); -+ return -ENOMEM; -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(rtwusb->tx_queue); i++) -+ skb_queue_head_init(&rtwusb->tx_queue[i]); -+ -+ INIT_WORK(&rtwusb->tx_work, rtw_usb_tx_handler); -+ -+ return 0; -+} -+ -+static void rtw_usb_deinit_tx(struct rtw_dev *rtwdev) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ rtw_usb_tx_queue_purge(rtwusb); -+ flush_workqueue(rtwusb->txwq); -+ destroy_workqueue(rtwusb->txwq); -+} -+ -+static int rtw_usb_intf_init(struct rtw_dev *rtwdev, -+ struct usb_interface *intf) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ struct usb_device *udev = usb_get_dev(interface_to_usbdev(intf)); -+ int ret; -+ -+ rtwusb->udev = udev; -+ ret = rtw_usb_parse(rtwdev, intf); -+ if (ret) -+ return ret; -+ -+ rtwusb->usb_data = kcalloc(RTW_USB_MAX_RX_COUNT, sizeof(u32), -+ GFP_KERNEL); -+ if (!rtwusb->usb_data) -+ return -ENOMEM; -+ -+ usb_set_intfdata(intf, rtwdev->hw); -+ -+ SET_IEEE80211_DEV(rtwdev->hw, &intf->dev); -+ spin_lock_init(&rtwusb->usb_lock); -+ -+ return 0; -+} -+ -+static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev, -+ struct usb_interface *intf) -+{ -+ struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ usb_put_dev(rtwusb->udev); -+ usb_set_intfdata(intf, NULL); -+} -+ -+int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) -+{ -+ struct rtw_dev *rtwdev; -+ struct ieee80211_hw *hw; -+ struct rtw_usb *rtwusb; -+ int drv_data_size; -+ int ret; -+ -+ drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_usb); -+ hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); -+ if (!hw) -+ return -ENOMEM; -+ -+ rtwdev = hw->priv; -+ rtwdev->hw = hw; -+ rtwdev->dev = &intf->dev; -+ rtwdev->chip = (struct rtw_chip_info *)id->driver_info; -+ rtwdev->hci.ops = &rtw_usb_ops; -+ rtwdev->hci.type = RTW_HCI_TYPE_USB; -+ -+ rtwusb = rtw_get_usb_priv(rtwdev); -+ rtwusb->rtwdev = rtwdev; -+ -+ INIT_WORK(&rtwusb->rx_refill_work, rtw_usb_rx_refill_work); -+ INIT_LIST_HEAD(&rtwusb->rx_data_free); -+ INIT_LIST_HEAD(&rtwusb->rx_data_used); -+ spin_lock_init(&rtwusb->rx_data_list_lock); -+ -+ ret = rtw_usb_alloc_rx_bufs(rtwusb); -+ if (ret) -+ return ret; -+ -+ ret = rtw_core_init(rtwdev); -+ if (ret) -+ goto err_release_hw; -+ -+ ret = rtw_usb_intf_init(rtwdev, intf); -+ if (ret) { -+ rtw_err(rtwdev, "failed to init USB interface\n"); -+ goto err_deinit_core; -+ } -+ -+ ret = rtw_usb_init_tx(rtwdev); -+ if (ret) { -+ rtw_err(rtwdev, "failed to init USB TX\n"); -+ goto err_destroy_usb; -+ } -+ -+ ret = rtw_usb_init_rx(rtwdev); -+ if (ret) { -+ rtw_err(rtwdev, "failed to init USB RX\n"); -+ goto err_destroy_txwq; -+ } -+ -+ ret = rtw_chip_info_setup(rtwdev); -+ if (ret) { -+ rtw_err(rtwdev, "failed to setup chip information\n"); -+ goto err_destroy_rxwq; -+ } -+ -+ ret = rtw_register_hw(rtwdev, rtwdev->hw); -+ if (ret) { -+ rtw_err(rtwdev, "failed to register hw\n"); -+ goto err_destroy_rxwq; -+ } -+ -+ return 0; -+ -+err_destroy_rxwq: -+ rtw_usb_deinit_rx(rtwdev); -+ -+err_destroy_txwq: -+ rtw_usb_deinit_tx(rtwdev); -+ -+err_destroy_usb: -+ rtw_usb_intf_deinit(rtwdev, intf); -+ -+err_deinit_core: -+ rtw_core_deinit(rtwdev); -+ -+err_release_hw: -+ ieee80211_free_hw(hw); -+ -+ return ret; -+} -+EXPORT_SYMBOL(rtw_usb_probe); -+ -+void rtw_usb_disconnect(struct usb_interface *intf) -+{ -+ struct ieee80211_hw *hw = usb_get_intfdata(intf); -+ struct rtw_dev *rtwdev; -+ struct rtw_usb *rtwusb; -+ -+ if (!hw) -+ return; -+ -+ rtwdev = hw->priv; -+ rtwusb = rtw_get_usb_priv(rtwdev); -+ -+ rtw_unregister_hw(rtwdev, hw); -+ rtw_usb_deinit_tx(rtwdev); -+ rtw_usb_deinit_rx(rtwdev); -+ -+ if (rtwusb->udev->state != USB_STATE_NOTATTACHED) -+ usb_reset_device(rtwusb->udev); -+ -+ rtw_usb_free_rx_bufs(rtwusb); -+ -+ rtw_usb_intf_deinit(rtwdev, intf); -+ rtw_core_deinit(rtwdev); -+ ieee80211_free_hw(hw); -+} -+EXPORT_SYMBOL(rtw_usb_disconnect); -+ -+MODULE_AUTHOR("Realtek Corporation"); -+MODULE_DESCRIPTION("Realtek 802.11ac wireless USB driver"); -+MODULE_LICENSE("Dual BSD/GPL"); -diff --git a/drivers/net/wireless/realtek/rtw88/usb.h b/drivers/net/wireless/realtek/rtw88/usb.h -new file mode 100644 -index 000000000000..4d714372f265 ---- /dev/null -+++ b/drivers/net/wireless/realtek/rtw88/usb.h -@@ -0,0 +1,109 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -+/* Copyright(c) 2018-2019 Realtek Corporation -+ */ -+ -+#ifndef __RTW_USB_H_ -+#define __RTW_USB_H_ -+ -+#define FW_8192C_START_ADDRESS 0x1000 -+#define FW_8192C_END_ADDRESS 0x5FFF -+ -+#define RTW_USB_MAX_RX_COUNT 100 -+#define RTW_USB_VENQT_MAX_BUF_SIZE 254 -+#define MAX_USBCTRL_VENDORREQ_TIMES 10 -+ -+#define RTW_USB_CMD_READ 0xc0 -+#define RTW_USB_CMD_WRITE 0x40 -+#define RTW_USB_CMD_REQ 0x05 -+ -+#define RTW_USB_VENQT_CMD_IDX 0x00 -+ -+#define RTW_USB_SUPER_SPEED_BULK_SIZE 1024 -+#define RTW_USB_HIGH_SPEED_BULK_SIZE 512 -+#define RTW_USB_FULL_SPEED_BULK_SIZE 64 -+ -+#define RTW_USB_TX_SEL_HQ BIT(0) -+#define RTW_USB_TX_SEL_LQ BIT(1) -+#define RTW_USB_TX_SEL_NQ BIT(2) -+#define RTW_USB_TX_SEL_EQ BIT(3) -+ -+#define RTW_USB_BULK_IN_ADDR 0x80 -+#define RTW_USB_INT_IN_ADDR 0x81 -+ -+#define RTW_USB_HW_QUEUE_ENTRY 8 -+ -+#define RTW_USB_PACKET_OFFSET_SZ 8 -+#define RTW_USB_MAX_XMITBUF_SZ (1592 * 3) -+#define RTW_USB_MAX_RECVBUF_SZ 32768 -+ -+#define RTW_USB_RECVBUFF_ALIGN_SZ 8 -+ -+#define RTW_USB_RXAGG_SIZE 6 -+#define RTW_USB_RXAGG_TIMEOUT 10 -+ -+#define RTW_USB_RXCB_NUM 4 -+ -+#define RTW_USB_EP_MAX 4 -+ -+#define TX_DESC_QSEL_MAX 20 -+ -+static inline struct rtw_usb *rtw_get_usb_priv(struct rtw_dev *rtwdev) -+{ -+ return (struct rtw_usb *)rtwdev->priv; -+} -+ -+struct rx_usb_ctrl_block { -+ struct rtw_dev *rtwdev; -+ struct urb *rx_urb; -+ struct sk_buff *rx_skb; -+ struct list_head list; -+}; -+ -+struct rtw_usb_tx_data { -+ u8 sn; -+}; -+ -+struct rtw_usb { -+ struct rtw_dev *rtwdev; -+ struct usb_device *udev; -+ -+ spinlock_t rx_data_list_lock; -+ struct work_struct rx_refill_work; -+ struct list_head rx_data_free; -+ struct list_head rx_data_used; -+ -+ spinlock_t usb_lock; -+ __le32 *usb_data; -+ int usb_data_index; -+ -+ u32 bulkout_size; -+ u8 pipe_interrupt; -+ u8 pipe_in; -+ u8 out_ep[RTW_USB_EP_MAX]; -+ u8 qsel_to_ep[TX_DESC_QSEL_MAX]; -+ u8 usb_txagg_num; -+ -+ struct workqueue_struct *txwq, *rxwq; -+ -+ struct sk_buff_head tx_queue[RTW_USB_EP_MAX]; -+ struct work_struct tx_work; -+ -+ struct rx_usb_ctrl_block rx_cb[RTW_USB_RXCB_NUM]; -+ struct sk_buff_head rx_queue; -+ struct work_struct rx_work; -+}; -+ -+static inline struct rtw_usb_tx_data *rtw_usb_get_tx_data(struct sk_buff *skb) -+{ -+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); -+ -+ BUILD_BUG_ON(sizeof(struct rtw_usb_tx_data) > -+ sizeof(info->status.status_driver_data)); -+ -+ return (struct rtw_usb_tx_data *)info->status.status_driver_data; -+} -+ -+int rtw_usb_probe(struct usb_interface *intf, const struct usb_device_id *id); -+void rtw_usb_disconnect(struct usb_interface *intf); -+ -+#endif -diff --git a/drivers/net/wireless/realtek/rtw88/util.c b/drivers/net/wireless/realtek/rtw88/util.c -index 2c515af214e7..db55dbd5c533 100644 ---- a/drivers/net/wireless/realtek/rtw88/util.c -+++ b/drivers/net/wireless/realtek/rtw88/util.c -@@ -105,3 +105,95 @@ void rtw_desc_to_mcsrate(u16 rate, u8 *mcs, u8 *nss) - *mcs = rate - DESC_RATEMCS0; - } - } -+ -+struct rtw_stas_entry { -+ struct list_head list; -+ struct ieee80211_sta *sta; -+}; -+ -+struct rtw_iter_stas_data { -+ struct rtw_dev *rtwdev; -+ struct list_head list; -+}; -+ -+void rtw_collect_sta_iter(void *data, struct ieee80211_sta *sta) -+{ -+ struct rtw_iter_stas_data *iter_stas = data; -+ struct rtw_stas_entry *stas_entry; -+ -+ stas_entry = kmalloc(sizeof(*stas_entry), GFP_ATOMIC); -+ if (!stas_entry) -+ return; -+ -+ stas_entry->sta = sta; -+ list_add_tail(&stas_entry->list, &iter_stas->list); -+} -+ -+void rtw_iterate_stas(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, -+ struct ieee80211_sta *sta), -+ void *data) -+{ -+ struct rtw_iter_stas_data iter_data; -+ struct rtw_stas_entry *sta_entry, *tmp; -+ -+ iter_data.rtwdev = rtwdev; -+ INIT_LIST_HEAD(&iter_data.list); -+ -+ ieee80211_iterate_stations_atomic(rtwdev->hw, rtw_collect_sta_iter, -+ &iter_data); -+ -+ list_for_each_entry_safe(sta_entry, tmp, &iter_data.list, -+ list) { -+ list_del_init(&sta_entry->list); -+ iterator(data, sta_entry->sta); -+ kfree(sta_entry); -+ } -+} -+ -+struct rtw_vifs_entry { -+ struct list_head list; -+ struct ieee80211_vif *vif; -+ u8 mac[ETH_ALEN]; -+}; -+ -+struct rtw_iter_vifs_data { -+ struct rtw_dev *rtwdev; -+ struct list_head list; -+}; -+ -+void rtw_collect_vif_iter(void *data, u8 *mac, struct ieee80211_vif *vif) -+{ -+ struct rtw_iter_vifs_data *iter_stas = data; -+ struct rtw_vifs_entry *vifs_entry; -+ -+ vifs_entry = kmalloc(sizeof(*vifs_entry), GFP_ATOMIC); -+ if (!vifs_entry) -+ return; -+ -+ vifs_entry->vif = vif; -+ ether_addr_copy(vifs_entry->mac, mac); -+ list_add_tail(&vifs_entry->list, &iter_stas->list); -+} -+ -+void rtw_iterate_vifs(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, u8 *mac, -+ struct ieee80211_vif *vif), -+ void *data) -+{ -+ struct rtw_iter_vifs_data iter_data; -+ struct rtw_vifs_entry *vif_entry, *tmp; -+ -+ iter_data.rtwdev = rtwdev; -+ INIT_LIST_HEAD(&iter_data.list); -+ -+ ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, -+ IEEE80211_IFACE_ITER_NORMAL, rtw_collect_vif_iter, &iter_data); -+ -+ list_for_each_entry_safe(vif_entry, tmp, &iter_data.list, -+ list) { -+ list_del_init(&vif_entry->list); -+ iterator(data, vif_entry->mac, vif_entry->vif); -+ kfree(vif_entry); -+ } -+} -diff --git a/drivers/net/wireless/realtek/rtw88/util.h b/drivers/net/wireless/realtek/rtw88/util.h -index 0c23b5069be0..dc8965525400 100644 ---- a/drivers/net/wireless/realtek/rtw88/util.h -+++ b/drivers/net/wireless/realtek/rtw88/util.h -@@ -7,9 +7,6 @@ - - struct rtw_dev; - --#define rtw_iterate_vifs(rtwdev, iterator, data) \ -- ieee80211_iterate_active_interfaces(rtwdev->hw, \ -- IEEE80211_IFACE_ITER_NORMAL, iterator, data) - #define rtw_iterate_vifs_atomic(rtwdev, iterator, data) \ - ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, \ - IEEE80211_IFACE_ITER_NORMAL, iterator, data) -@@ -20,6 +17,15 @@ struct rtw_dev; - #define rtw_iterate_keys_rcu(rtwdev, vif, iterator, data) \ - ieee80211_iter_keys_rcu((rtwdev)->hw, vif, iterator, data) - -+void rtw_iterate_vifs(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, u8 *mac, -+ struct ieee80211_vif *vif), -+ void *data); -+void rtw_iterate_stas(struct rtw_dev *rtwdev, -+ void (*iterator)(void *data, -+ struct ieee80211_sta *sta), -+ void *data); -+ - static inline u8 *get_hdr_bssid(struct ieee80211_hdr *hdr) - { - __le16 fc = hdr->frame_control; --- -2.38.0.rc1.8.g2a7d63a245 - -From bc469031493ed241d08816cacca6881482c8a46e Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Wed, 14 Sep 2022 14:40:34 +0200 -Subject: [PATCH 08/13] rcu - -Signed-off-by: Peter Jung ---- - kernel/rcu/tree_nocb.h | 34 +++++++++++----------------------- - 1 file changed, 11 insertions(+), 23 deletions(-) - -diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h -index a8f574d8850d..4017ebecec91 100644 ---- a/kernel/rcu/tree_nocb.h -+++ b/kernel/rcu/tree_nocb.h -@@ -1210,45 +1210,33 @@ EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload); - void __init rcu_init_nohz(void) - { - int cpu; -- bool need_rcu_nocb_mask = false; -- bool offload_all = false; - struct rcu_data *rdp; -- --#if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) -- if (!rcu_state.nocb_is_setup) { -- need_rcu_nocb_mask = true; -- offload_all = true; -- } --#endif /* #if defined(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) */ -+ const struct cpumask *cpumask = NULL; - - #if defined(CONFIG_NO_HZ_FULL) -- if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) { -- need_rcu_nocb_mask = true; -- offload_all = false; /* NO_HZ_FULL has its own mask. */ -- } --#endif /* #if defined(CONFIG_NO_HZ_FULL) */ -+ if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask)) -+ cpumask = tick_nohz_full_mask; -+#endif -+ -+ if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) && -+ !rcu_state.nocb_is_setup && !cpumask) -+ cpumask = cpu_possible_mask; - -- if (need_rcu_nocb_mask) { -+ if (cpumask) { - if (!cpumask_available(rcu_nocb_mask)) { - if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) { - pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n"); - return; - } - } -+ -+ cpumask_or(rcu_nocb_mask, rcu_nocb_mask, cpumask); - rcu_state.nocb_is_setup = true; - } - - if (!rcu_state.nocb_is_setup) - return; - --#if defined(CONFIG_NO_HZ_FULL) -- if (tick_nohz_full_running) -- cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask); --#endif /* #if defined(CONFIG_NO_HZ_FULL) */ -- -- if (offload_all) -- cpumask_setall(rcu_nocb_mask); -- - if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) { - pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n"); - cpumask_and(rcu_nocb_mask, cpu_possible_mask, --- -2.38.0.rc1.8.g2a7d63a245 - -From 3c963a73af0422079aabc28646162c1ae73cfad2 Mon Sep 17 00:00:00 2001 -From: Piotr Gorski -Date: Tue, 6 Sep 2022 20:04:11 +0200 -Subject: [PATCH 09/13] lrng - -Signed-off-by: Piotr Gorski ---- - MAINTAINERS | 7 + - crypto/drbg.c | 16 +- - crypto/jitterentropy-kcapi.c | 3 +- - crypto/jitterentropy.c | 2 +- - drivers/char/Kconfig | 2 + - drivers/char/Makefile | 5 +- - drivers/char/lrng/Kconfig | 907 ++++++++++++++++++ - drivers/char/lrng/Makefile | 39 + - drivers/char/lrng/lrng_definitions.h | 151 +++ - drivers/char/lrng/lrng_drng_atomic.c | 171 ++++ - drivers/char/lrng/lrng_drng_atomic.h | 23 + - drivers/char/lrng/lrng_drng_chacha20.c | 195 ++++ - drivers/char/lrng/lrng_drng_chacha20.h | 42 + - drivers/char/lrng/lrng_drng_drbg.c | 179 ++++ - drivers/char/lrng/lrng_drng_drbg.h | 13 + - drivers/char/lrng/lrng_drng_kcapi.c | 208 ++++ - drivers/char/lrng/lrng_drng_kcapi.h | 13 + - drivers/char/lrng/lrng_drng_mgr.c | 657 +++++++++++++ - drivers/char/lrng/lrng_drng_mgr.h | 86 ++ - drivers/char/lrng/lrng_es_aux.c | 335 +++++++ - drivers/char/lrng/lrng_es_aux.h | 44 + - drivers/char/lrng/lrng_es_cpu.c | 280 ++++++ - drivers/char/lrng/lrng_es_cpu.h | 17 + - drivers/char/lrng/lrng_es_irq.c | 727 ++++++++++++++ - drivers/char/lrng/lrng_es_irq.h | 24 + - drivers/char/lrng/lrng_es_jent.c | 136 +++ - drivers/char/lrng/lrng_es_jent.h | 17 + - drivers/char/lrng/lrng_es_krng.c | 100 ++ - drivers/char/lrng/lrng_es_krng.h | 17 + - drivers/char/lrng/lrng_es_mgr.c | 460 +++++++++ - drivers/char/lrng/lrng_es_mgr.h | 51 + - drivers/char/lrng/lrng_es_mgr_cb.h | 87 ++ - drivers/char/lrng/lrng_es_sched.c | 562 +++++++++++ - drivers/char/lrng/lrng_es_sched.h | 20 + - drivers/char/lrng/lrng_es_timer_common.c | 144 +++ - drivers/char/lrng/lrng_es_timer_common.h | 83 ++ - drivers/char/lrng/lrng_hash_kcapi.c | 140 +++ - drivers/char/lrng/lrng_health.c | 420 ++++++++ - drivers/char/lrng/lrng_health.h | 42 + - drivers/char/lrng/lrng_interface_aux.c | 126 +++ - drivers/char/lrng/lrng_interface_dev.c | 35 + - drivers/char/lrng/lrng_interface_dev_common.c | 314 ++++++ - drivers/char/lrng/lrng_interface_dev_common.h | 51 + - drivers/char/lrng/lrng_interface_hwrand.c | 68 ++ - drivers/char/lrng/lrng_interface_kcapi.c | 129 +++ - .../char/lrng/lrng_interface_random_kernel.c | 214 +++++ - .../char/lrng/lrng_interface_random_kernel.h | 15 + - .../char/lrng/lrng_interface_random_user.c | 104 ++ - drivers/char/lrng/lrng_numa.c | 124 +++ - drivers/char/lrng/lrng_numa.h | 15 + - drivers/char/lrng/lrng_proc.c | 74 ++ - drivers/char/lrng/lrng_proc.h | 15 + - drivers/char/lrng/lrng_selftest.c | 397 ++++++++ - drivers/char/lrng/lrng_sha.h | 14 + - drivers/char/lrng/lrng_sha1.c | 88 ++ - drivers/char/lrng/lrng_sha256.c | 72 ++ - drivers/char/lrng/lrng_switch.c | 286 ++++++ - drivers/char/lrng/lrng_sysctl.c | 140 +++ - drivers/char/lrng/lrng_sysctl.h | 15 + - drivers/char/lrng/lrng_testing.c | 901 +++++++++++++++++ - drivers/char/lrng/lrng_testing.h | 85 ++ - include/crypto/drbg.h | 7 + - .../crypto/internal}/jitterentropy.h | 0 - include/linux/lrng.h | 251 +++++ - kernel/sched/core.c | 3 + - 65 files changed, 9958 insertions(+), 10 deletions(-) - create mode 100644 drivers/char/lrng/Kconfig - create mode 100644 drivers/char/lrng/Makefile - create mode 100644 drivers/char/lrng/lrng_definitions.h - create mode 100644 drivers/char/lrng/lrng_drng_atomic.c - create mode 100644 drivers/char/lrng/lrng_drng_atomic.h - create mode 100644 drivers/char/lrng/lrng_drng_chacha20.c - create mode 100644 drivers/char/lrng/lrng_drng_chacha20.h - create mode 100644 drivers/char/lrng/lrng_drng_drbg.c - create mode 100644 drivers/char/lrng/lrng_drng_drbg.h - create mode 100644 drivers/char/lrng/lrng_drng_kcapi.c - create mode 100644 drivers/char/lrng/lrng_drng_kcapi.h - create mode 100644 drivers/char/lrng/lrng_drng_mgr.c - create mode 100644 drivers/char/lrng/lrng_drng_mgr.h - create mode 100644 drivers/char/lrng/lrng_es_aux.c - create mode 100644 drivers/char/lrng/lrng_es_aux.h - create mode 100644 drivers/char/lrng/lrng_es_cpu.c - create mode 100644 drivers/char/lrng/lrng_es_cpu.h - create mode 100644 drivers/char/lrng/lrng_es_irq.c - create mode 100644 drivers/char/lrng/lrng_es_irq.h - create mode 100644 drivers/char/lrng/lrng_es_jent.c - create mode 100644 drivers/char/lrng/lrng_es_jent.h - create mode 100644 drivers/char/lrng/lrng_es_krng.c - create mode 100644 drivers/char/lrng/lrng_es_krng.h - create mode 100644 drivers/char/lrng/lrng_es_mgr.c - create mode 100644 drivers/char/lrng/lrng_es_mgr.h - create mode 100644 drivers/char/lrng/lrng_es_mgr_cb.h - create mode 100644 drivers/char/lrng/lrng_es_sched.c - create mode 100644 drivers/char/lrng/lrng_es_sched.h - create mode 100644 drivers/char/lrng/lrng_es_timer_common.c - create mode 100644 drivers/char/lrng/lrng_es_timer_common.h - create mode 100644 drivers/char/lrng/lrng_hash_kcapi.c - create mode 100644 drivers/char/lrng/lrng_health.c - create mode 100644 drivers/char/lrng/lrng_health.h - create mode 100644 drivers/char/lrng/lrng_interface_aux.c - create mode 100644 drivers/char/lrng/lrng_interface_dev.c - create mode 100644 drivers/char/lrng/lrng_interface_dev_common.c - create mode 100644 drivers/char/lrng/lrng_interface_dev_common.h - create mode 100644 drivers/char/lrng/lrng_interface_hwrand.c - create mode 100644 drivers/char/lrng/lrng_interface_kcapi.c - create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.c - create mode 100644 drivers/char/lrng/lrng_interface_random_kernel.h - create mode 100644 drivers/char/lrng/lrng_interface_random_user.c - create mode 100644 drivers/char/lrng/lrng_numa.c - create mode 100644 drivers/char/lrng/lrng_numa.h - create mode 100644 drivers/char/lrng/lrng_proc.c - create mode 100644 drivers/char/lrng/lrng_proc.h - create mode 100644 drivers/char/lrng/lrng_selftest.c - create mode 100644 drivers/char/lrng/lrng_sha.h - create mode 100644 drivers/char/lrng/lrng_sha1.c - create mode 100644 drivers/char/lrng/lrng_sha256.c - create mode 100644 drivers/char/lrng/lrng_switch.c - create mode 100644 drivers/char/lrng/lrng_sysctl.c - create mode 100644 drivers/char/lrng/lrng_sysctl.h - create mode 100644 drivers/char/lrng/lrng_testing.c - create mode 100644 drivers/char/lrng/lrng_testing.h - rename {crypto => include/crypto/internal}/jitterentropy.h (100%) - create mode 100644 include/linux/lrng.h - -diff --git a/MAINTAINERS b/MAINTAINERS -index 96a09757feb3..e3c1b29c60a0 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -11741,6 +11741,13 @@ F: Documentation/litmus-tests/ - F: Documentation/memory-barriers.txt - F: tools/memory-model/ - -+LINUX RANDOM NUMBER GENERATOR (LRNG) DRIVER -+M: Stephan Mueller -+S: Maintained -+W: https://www.chronox.de/lrng.html -+F: drivers/char/lrng/ -+F: include/linux/lrng.h -+ - LIS3LV02D ACCELEROMETER DRIVER - M: Eric Piel - S: Maintained -diff --git a/crypto/drbg.c b/crypto/drbg.c -index 177983b6ae38..ea79a31014a4 100644 ---- a/crypto/drbg.c -+++ b/crypto/drbg.c -@@ -115,7 +115,7 @@ - * the SHA256 / AES 256 over other ciphers. Thus, the favored - * DRBGs are the latest entries in this array. - */ --static const struct drbg_core drbg_cores[] = { -+const struct drbg_core drbg_cores[] = { - #ifdef CONFIG_CRYPTO_DRBG_CTR - { - .flags = DRBG_CTR | DRBG_STRENGTH128, -@@ -192,6 +192,7 @@ static const struct drbg_core drbg_cores[] = { - }, - #endif /* CONFIG_CRYPTO_DRBG_HMAC */ - }; -+EXPORT_SYMBOL(drbg_cores); - - static int drbg_uninstantiate(struct drbg_state *drbg); - -@@ -207,7 +208,7 @@ static int drbg_uninstantiate(struct drbg_state *drbg); - * Return: normalized strength in *bytes* value or 32 as default - * to counter programming errors - */ --static inline unsigned short drbg_sec_strength(drbg_flag_t flags) -+unsigned short drbg_sec_strength(drbg_flag_t flags) - { - switch (flags & DRBG_STRENGTH_MASK) { - case DRBG_STRENGTH128: -@@ -220,6 +221,7 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) - return 32; - } - } -+EXPORT_SYMBOL(drbg_sec_strength); - - /* - * FIPS 140-2 continuous self test for the noise source -@@ -1252,7 +1254,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, - } - - /* Free all substructures in a DRBG state without the DRBG state structure */ --static inline void drbg_dealloc_state(struct drbg_state *drbg) -+void drbg_dealloc_state(struct drbg_state *drbg) - { - if (!drbg) - return; -@@ -1273,12 +1275,13 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) - drbg->fips_primed = false; - } - } -+EXPORT_SYMBOL(drbg_dealloc_state); - - /* - * Allocate all sub-structures for a DRBG state. - * The DRBG state structure must already be allocated. - */ --static inline int drbg_alloc_state(struct drbg_state *drbg) -+int drbg_alloc_state(struct drbg_state *drbg) - { - int ret = -ENOMEM; - unsigned int sb_size = 0; -@@ -1359,6 +1362,7 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) - drbg_dealloc_state(drbg); - return ret; - } -+EXPORT_SYMBOL(drbg_alloc_state); - - /************************************************************************* - * DRBG interface functions -@@ -1895,8 +1899,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg, - * - * return: flags - */ --static inline void drbg_convert_tfm_core(const char *cra_driver_name, -- int *coreref, bool *pr) -+void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, bool *pr) - { - int i = 0; - size_t start = 0; -@@ -1923,6 +1926,7 @@ static inline void drbg_convert_tfm_core(const char *cra_driver_name, - } - } - } -+EXPORT_SYMBOL(drbg_convert_tfm_core); - - static int drbg_kcapi_init(struct crypto_tfm *tfm) - { -diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c -index 2d115bec15ae..e7dac734d237 100644 ---- a/crypto/jitterentropy-kcapi.c -+++ b/crypto/jitterentropy-kcapi.c -@@ -42,8 +42,7 @@ - #include - #include - #include -- --#include "jitterentropy.h" -+#include - - /*************************************************************************** - * Helper function -diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c -index 93bff3213823..81b80a4d3d3a 100644 ---- a/crypto/jitterentropy.c -+++ b/crypto/jitterentropy.c -@@ -133,7 +133,7 @@ struct rand_data { - #define JENT_ENTROPY_SAFETY_FACTOR 64 - - #include --#include "jitterentropy.h" -+#include - - /*************************************************************************** - * Adaptive Proportion Test -diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig -index 0f378d29dab0..39c91f09cbe0 100644 ---- a/drivers/char/Kconfig -+++ b/drivers/char/Kconfig -@@ -459,4 +459,6 @@ config RANDOM_TRUST_BOOTLOADER - believe its RNG facilities may be faulty. This may also be configured - at boot time with "random.trust_bootloader=on/off". - -+source "drivers/char/lrng/Kconfig" -+ - endmenu -diff --git a/drivers/char/Makefile b/drivers/char/Makefile -index 1b35d1724565..0d56f0a24fcd 100644 ---- a/drivers/char/Makefile -+++ b/drivers/char/Makefile -@@ -3,7 +3,8 @@ - # Makefile for the kernel character device drivers. - # - --obj-y += mem.o random.o -+obj-y += mem.o -+obj-$(CONFIG_RANDOM_DEFAULT_IMPL) += random.o - obj-$(CONFIG_TTY_PRINTK) += ttyprintk.o - obj-y += misc.o - obj-$(CONFIG_ATARI_DSP56K) += dsp56k.o -@@ -45,3 +46,5 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o - obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/ - obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o - obj-$(CONFIG_ADI) += adi.o -+ -+obj-$(CONFIG_LRNG) += lrng/ -diff --git a/drivers/char/lrng/Kconfig b/drivers/char/lrng/Kconfig -new file mode 100644 -index 000000000000..bfd6b3ea4f1b ---- /dev/null -+++ b/drivers/char/lrng/Kconfig -@@ -0,0 +1,907 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# Linux Random Number Generator configuration -+# -+ -+config RANDOM_DEFAULT_IMPL -+ bool "Kernel RNG Default Implementation" -+ default y -+ help -+ The default random number generator as provided with -+ drivers/char/random.c is selected with this option. -+ -+config LRNG -+ bool "Linux Random Number Generator" -+ default n -+ select CRYPTO_LIB_SHA256 if CRYPTO -+ help -+ The Linux Random Number Generator (LRNG) generates entropy -+ from different entropy sources. Each entropy source can -+ be enabled and configured independently. The interrupt -+ entropy source can be configured to be SP800-90B compliant. -+ The entire LRNG can be configured to be SP800-90C compliant. -+ Runtime-switchable cryptographic support is available. -+ The LRNG delivers significant entropy during boot. -+ -+ The LRNG also provides compliance to SP800-90A/B/C. -+ -+menu "Linux Random Number Generator Configuration" -+ depends on LRNG -+ -+if LRNG -+ -+config LRNG_SHA256 -+ bool -+ default y if CRYPTO_LIB_SHA256 -+ -+config LRNG_SHA1 -+ bool -+ default y if !CRYPTO_LIB_SHA256 -+ -+config LRNG_COMMON_DEV_IF -+ bool -+ -+config LRNG_DRNG_ATOMIC -+ bool -+ select LRNG_DRNG_CHACHA20 -+ -+config LRNG_SYSCTL -+ bool -+ depends on SYSCTL -+ -+config LRNG_RANDOM_IF -+ bool -+ default n if RANDOM_DEFAULT_IMPL -+ default y if !RANDOM_DEFAULT_IMPL -+ select LRNG_COMMON_DEV_IF -+ select LRNG_DRNG_ATOMIC -+ select LRNG_SYSCTL -+ -+menu "LRNG Interfaces" -+ -+config LRNG_KCAPI_IF -+ tristate "Interface with Kernel Crypto API" -+ depends on CRYPTO_RNG -+ help -+ The LRNG can be registered with the kernel crypto API's -+ random number generator framework. This offers a random -+ number generator with the name "lrng" and a priority that -+ is intended to be higher than the existing RNG -+ implementations. -+ -+config LRNG_HWRAND_IF -+ tristate "Interface with Hardware Random Number Generator Framework" -+ depends on HW_RANDOM -+ select LRNG_DRNG_ATOMIC -+ help -+ The LRNG can be registered with the hardware random number -+ generator framework. This offers a random number generator -+ with the name "lrng" that is accessible via the framework. -+ For example it allows pulling data from the LRNG via the -+ /dev/hwrng file. -+ -+config LRNG_DEV_IF -+ bool "Character device file interface" -+ select LRNG_COMMON_DEV_IF -+ help -+ The LRNG can create a character device file that operates -+ identically to /dev/random including IOCTL, read and write -+ operations. -+ -+endmenu # "LRNG Interfaces" -+ -+menu "Entropy Source Configuration" -+ -+config LRNG_RUNTIME_ES_CONFIG -+ bool "Enable runtime configuration of entropy sources" -+ help -+ When enabling this option, the LRNG provides the mechanism -+ allowing to alter the entropy rate of each entropy source -+ during boot time and runtime. -+ -+ Each entropy source allows its entropy rate changed with -+ a kernel command line option. When not providing any -+ option, the default specified during kernel compilation -+ is applied. -+ -+comment "Common Timer-based Entropy Source Configuration" -+ -+config LRNG_IRQ_DFLT_TIMER_ES -+ bool -+ -+config LRNG_SCHED_DFLT_TIMER_ES -+ bool -+ -+config LRNG_TIMER_COMMON -+ bool -+ -+choice -+ prompt "Default Timer-based Entropy Source" -+ default LRNG_IRQ_DFLT_TIMER_ES -+ depends on LRNG_TIMER_COMMON -+ help -+ Select the timer-based entropy source that is credited -+ with entropy. The other timer-based entropy sources may -+ be operational and provide data, but are credited with no -+ entropy. -+ -+ config LRNG_IRQ_DFLT_TIMER_ES -+ bool "Interrupt Entropy Source" -+ depends on LRNG_IRQ -+ help -+ The interrupt entropy source is selected as a timer-based -+ entropy source to provide entropy. -+ -+ config LRNG_SCHED_DFLT_TIMER_ES -+ bool "Scheduler Entropy Source" -+ depends on LRNG_SCHED -+ help -+ The scheduler entropy source is selected as timer-based -+ entropy source to provide entropy. -+endchoice -+ -+choice -+ prompt "LRNG Entropy Collection Pool Size" -+ default LRNG_COLLECTION_SIZE_1024 -+ depends on LRNG_TIMER_COMMON -+ help -+ Select the size of the LRNG entropy collection pool -+ storing data for the interrupt as well as the scheduler -+ entropy sources without performing a compression -+ operation. The larger the collection size is, the faster -+ the average interrupt handling will be. The collection -+ size represents the number of bytes of the per-CPU memory -+ used to batch up entropy event data. -+ -+ The default value is good for regular operations. Choose -+ larger sizes for servers that have no memory limitations. -+ If runtime memory is precious, choose a smaller size. -+ -+ The collection size is unrelated to the entropy rate -+ or the amount of entropy the LRNG can process. -+ -+ config LRNG_COLLECTION_SIZE_32 -+ depends on LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ depends on !LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION -+ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES -+ bool "32 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_256 -+ depends on !LRNG_OVERSAMPLE_ENTROPY_SOURCES -+ bool "256 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_512 -+ bool "512 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_1024 -+ bool "1024 interrupt events (default)" -+ -+ config LRNG_COLLECTION_SIZE_2048 -+ bool "2048 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_4096 -+ bool "4096 interrupt events" -+ -+ config LRNG_COLLECTION_SIZE_8192 -+ bool "8192 interrupt events" -+ -+endchoice -+ -+config LRNG_COLLECTION_SIZE -+ int -+ default 32 if LRNG_COLLECTION_SIZE_32 -+ default 256 if LRNG_COLLECTION_SIZE_256 -+ default 512 if LRNG_COLLECTION_SIZE_512 -+ default 1024 if LRNG_COLLECTION_SIZE_1024 -+ default 2048 if LRNG_COLLECTION_SIZE_2048 -+ default 4096 if LRNG_COLLECTION_SIZE_4096 -+ default 8192 if LRNG_COLLECTION_SIZE_8192 -+ -+config LRNG_HEALTH_TESTS -+ bool "Enable internal entropy source online health tests" -+ depends on LRNG_TIMER_COMMON -+ help -+ The online health tests applied to the interrupt entropy -+ source and to the scheduler entropy source to validate -+ the noise source at runtime for fatal errors. These tests -+ include SP800-90B compliant tests which are invoked if -+ the system is booted with fips=1. In case of fatal errors -+ during active SP800-90B tests, the issue is logged and -+ the noise data is discarded. These tests are required for -+ full compliance of the interrupt entropy source with -+ SP800-90B. -+ -+ If both, the scheduler and the interrupt entropy sources, -+ are enabled, the health tests for both are applied -+ independent of each other. -+ -+ If unsure, say Y. -+ -+config LRNG_RCT_BROKEN -+ bool "SP800-90B RCT with dangerous low cutoff value" -+ depends on LRNG_HEALTH_TESTS -+ depends on BROKEN -+ default n -+ help -+ This option enables a dangerously low SP800-90B repetitive -+ count test (RCT) cutoff value which makes it very likely -+ that the RCT is triggered to raise a self test failure. -+ -+ This option is ONLY intended for developers wanting to -+ test the effectiveness of the SP800-90B RCT health test. -+ -+ If unsure, say N. -+ -+config LRNG_APT_BROKEN -+ bool "SP800-90B APT with dangerous low cutoff value" -+ depends on LRNG_HEALTH_TESTS -+ depends on BROKEN -+ default n -+ help -+ This option enables a dangerously low SP800-90B adaptive -+ proportion test (APT) cutoff value which makes it very -+ likely that the APT is triggered to raise a self test -+ failure. -+ -+ This option is ONLY intended for developers wanting to -+ test the effectiveness of the SP800-90B APT health test. -+ -+ If unsure, say N. -+ -+# Default taken from SP800-90B sec 4.4.1 - significance level 2^-30 -+config LRNG_RCT_CUTOFF -+ int -+ default 31 if !LRNG_RCT_BROKEN -+ default 1 if LRNG_RCT_BROKEN -+ -+# Default taken from SP800-90B sec 4.4.2 - significance level 2^-30 -+config LRNG_APT_CUTOFF -+ int -+ default 325 if !LRNG_APT_BROKEN -+ default 32 if LRNG_APT_BROKEN -+ -+comment "Interrupt Entropy Source" -+ -+config LRNG_IRQ -+ bool "Enable Interrupt Entropy Source as LRNG Seed Source" -+ default y -+ depends on !RANDOM_DEFAULT_IMPL -+ select LRNG_TIMER_COMMON -+ help -+ The LRNG models an entropy source based on the timing of the -+ occurrence of interrupts. Enable this option to enable this -+ IRQ entropy source. -+ -+ The IRQ entropy source is triggered every time an interrupt -+ arrives and thus causes the interrupt handler to execute -+ slightly longer. Disabling the IRQ entropy source implies -+ that the performance penalty on the interrupt handler added -+ by the LRNG is eliminated. Yet, this entropy source is -+ considered to be an internal entropy source of the LRNG. -+ Thus, only disable it if you ensured that other entropy -+ sources are available that supply the LRNG with entropy. -+ -+ If you disable the IRQ entropy source, you MUST ensure -+ one or more entropy sources collectively have the -+ capability to deliver sufficient entropy with one invocation -+ at a rate compliant to the security strength of the DRNG -+ (usually 256 bits of entropy). In addition, if those -+ entropy sources do not deliver sufficient entropy during -+ first request, the reseed must be triggered from user -+ space or kernel space when sufficient entropy is considered -+ to be present. -+ -+ If unsure, say Y. -+ -+choice -+ prompt "Continuous entropy compression boot time setting" -+ default LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ depends on LRNG_IRQ -+ help -+ Select the default behavior of the interrupt entropy source -+ continuous compression operation. -+ -+ The LRNG IRQ ES collects entropy data during each interrupt. -+ For performance reasons, a amount of entropy data defined by -+ the LRNG entropy collection pool size is concatenated into -+ an array. When that array is filled up, a hash is calculated -+ to compress the entropy. That hash is calculated in -+ interrupt context. -+ -+ In case such hash calculation in interrupt context is deemed -+ too time-consuming, the continuous compression operation -+ can be disabled. If disabled, the collection of entropy will -+ not trigger a hash compression operation in interrupt context. -+ The compression happens only when the DRNG is reseeded which is -+ in process context. This implies that old entropy data -+ collected after the last DRNG-reseed is overwritten with newer -+ entropy data once the collection pool is full instead of -+ retaining its entropy with the compression operation. -+ -+ config LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ bool "Enable continuous compression (default)" -+ -+ config LRNG_CONTINUOUS_COMPRESSION_DISABLED -+ bool "Disable continuous compression" -+ -+endchoice -+ -+config LRNG_ENABLE_CONTINUOUS_COMPRESSION -+ bool -+ default y if LRNG_CONTINUOUS_COMPRESSION_ENABLED -+ default n if LRNG_CONTINUOUS_COMPRESSION_DISABLED -+ -+config LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION -+ bool "Runtime-switchable continuous entropy compression" -+ depends on LRNG_IRQ -+ help -+ Per default, the interrupt entropy source continuous -+ compression operation behavior is hard-wired into the kernel. -+ Enable this option to allow it to be configurable at boot time. -+ -+ To modify the default behavior of the continuous -+ compression operation, use the kernel command line option -+ of lrng_sw_noise.lrng_pcpu_continuous_compression. -+ -+ If unsure, say N. -+ -+config LRNG_IRQ_ENTROPY_RATE -+ int "Interrupt Entropy Source Entropy Rate" -+ depends on LRNG_IRQ -+ range 256 4294967295 if LRNG_IRQ_DFLT_TIMER_ES -+ range 4294967295 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES -+ default 256 if LRNG_IRQ_DFLT_TIMER_ES -+ default 4294967295 if !LRNG_IRQ_DFLT_TIMER_ES -+ help -+ The LRNG will collect the configured number of interrupts to -+ obtain 256 bits of entropy. This value can be set to any between -+ 256 and 4294967295. The LRNG guarantees that this value is not -+ lower than 256. This lower limit implies that one interrupt event -+ is credited with one bit of entropy. This value is subject to the -+ increase by the oversampling factor, if no high-resolution timer -+ is found. -+ -+ In order to effectively disable the interrupt entropy source, -+ the option has to be set to 4294967295. In this case, the -+ interrupt entropy source will still deliver data but without -+ being credited with entropy. -+ -+comment "Jitter RNG Entropy Source" -+ -+config LRNG_JENT -+ bool "Enable Jitter RNG as LRNG Seed Source" -+ depends on CRYPTO -+ select CRYPTO_JITTERENTROPY -+ help -+ The LRNG may use the Jitter RNG as entropy source. Enabling -+ this option enables the use of the Jitter RNG. Its default -+ entropy level is 16 bits of entropy per 256 data bits delivered -+ by the Jitter RNG. This entropy level can be changed at boot -+ time or at runtime with the lrng_base.jitterrng configuration -+ variable. -+ -+config LRNG_JENT_ENTROPY_RATE -+ int "Jitter RNG Entropy Source Entropy Rate" -+ depends on LRNG_JENT -+ range 0 256 -+ default 16 -+ help -+ The option defines the amount of entropy the LRNG applies to 256 -+ bits of data obtained from the Jitter RNG entropy source. The -+ LRNG enforces the limit that this value must be in the range -+ between 0 and 256. -+ -+ When configuring this value to 0, the Jitter RNG entropy source -+ will provide 256 bits of data without being credited to contain -+ entropy. -+ -+comment "CPU Entropy Source" -+ -+config LRNG_CPU -+ bool "Enable CPU Entropy Source as LRNG Seed Source" -+ default y -+ help -+ Current CPUs commonly contain entropy sources which can be -+ used to seed the LRNG. For example, the Intel RDSEED -+ instruction, or the POWER DARN instruction will be sourced -+ to seed the LRNG if this option is enabled. -+ -+ Note, if this option is enabled and the underlying CPU -+ does not offer such entropy source, the LRNG will automatically -+ detect this and ignore the hardware. -+ -+config LRNG_CPU_FULL_ENT_MULTIPLIER -+ int -+ default 1 if !LRNG_TEST_CPU_ES_COMPRESSION -+ default 123 if LRNG_TEST_CPU_ES_COMPRESSION -+ -+config LRNG_CPU_ENTROPY_RATE -+ int "CPU Entropy Source Entropy Rate" -+ depends on LRNG_CPU -+ range 0 256 -+ default 8 -+ help -+ The option defines the amount of entropy the LRNG applies to 256 -+ bits of data obtained from the CPU entropy source. The LRNG -+ enforces the limit that this value must be in the range between -+ 0 and 256. -+ -+ When configuring this value to 0, the CPU entropy source will -+ provide 256 bits of data without being credited to contain -+ entropy. -+ -+ Note, this option is overwritten when the option -+ CONFIG_RANDOM_TRUST_CPU is set. -+ -+comment "Scheduler Entropy Source" -+ -+config LRNG_SCHED -+ bool "Enable Scheduer Entropy Source as LRNG Seed Source" -+ select LRNG_TIMER_COMMON -+ help -+ The LRNG models an entropy source based on the timing of the -+ occurrence of scheduler-triggered context switches. Enable -+ this option to enable this scheduler entropy source. -+ -+ The scheduler entropy source is triggered every time a -+ context switch is triggered thus causes the scheduler to -+ execute slightly longer. Disabling the scheduler entropy -+ source implies that the performance penalty on the scheduler -+ added by the LRNG is eliminated. Yet, this entropy source is -+ considered to be an internal entropy source of the LRNG. -+ Thus, only disable it if you ensured that other entropy -+ sources are available that supply the LRNG with entropy. -+ -+ If you disable the scheduler entropy source, you MUST -+ ensure one or more entropy sources collectively have the -+ capability to deliver sufficient entropy with one invocation -+ at a rate compliant to the security strength of the DRNG -+ (usually 256 bits of entropy). In addition, if those -+ entropy sources do not deliver sufficient entropy during -+ first request, the reseed must be triggered from user -+ space or kernel space when sufficient entropy is considered -+ to be present. -+ -+ If unsure, say Y. -+ -+config LRNG_SCHED_ENTROPY_RATE -+ int "Scheduler Entropy Source Entropy Rate" -+ depends on LRNG_SCHED -+ range 256 4294967295 if LRNG_SCHED_DFLT_TIMER_ES -+ range 4294967295 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES -+ default 256 if LRNG_SCHED_DFLT_TIMER_ES -+ default 4294967295 if !LRNG_SCHED_DFLT_TIMER_ES -+ help -+ The LRNG will collect the configured number of context switches -+ triggered by the scheduler to obtain 256 bits of entropy. This -+ value can be set to any between 256 and 4294967295. The LRNG -+ guarantees that this value is not lower than 256. This lower -+ limit implies that one interrupt event is credited with one bit -+ of entropy. This value is subject to the increase by the -+ oversampling factor, if no high-resolution timer is found. -+ -+ In order to effectively disable the scheduler entropy source, -+ the option has to be set to 4294967295. In this case, the -+ scheduler entropy source will still deliver data but without -+ being credited with entropy. -+ -+comment "Kernel RNG Entropy Source" -+ -+config LRNG_KERNEL_RNG -+ bool "Enable Kernel RNG as LRNG Seed Source" -+ depends on RANDOM_DEFAULT_IMPL -+ help -+ The LRNG may use the kernel RNG (random.c) as entropy -+ source. -+ -+config LRNG_KERNEL_RNG_ENTROPY_RATE -+ int "Kernel RNG Entropy Source Entropy Rate" -+ depends on LRNG_KERNEL_RNG -+ range 0 256 -+ default 256 -+ help -+ The option defines the amount of entropy the LRNG applies to 256 -+ bits of data obtained from the kernel RNG entropy source. The -+ LRNG enforces the limit that this value must be in the range -+ between 0 and 256. -+ -+ When configuring this value to 0, the kernel RNG entropy source -+ will provide 256 bits of data without being credited to contain -+ entropy. -+ -+ Note: This value is set to 0 automatically when booting the -+ kernel in FIPS mode (with fips=1 kernel command line option). -+ This is due to the fact that random.c is not SP800-90B -+ compliant. -+ -+endmenu # "Entropy Source Configuration" -+ -+config LRNG_DRNG_CHACHA20 -+ tristate -+ -+config LRNG_DRBG -+ tristate -+ depends on CRYPTO -+ select CRYPTO_DRBG_MENU -+ -+config LRNG_DRNG_KCAPI -+ tristate -+ depends on CRYPTO -+ select CRYPTO_RNG -+ -+config LRNG_SWITCH -+ bool -+ -+menuconfig LRNG_SWITCH_HASH -+ bool "Support conditioning hash runtime switching" -+ select LRNG_SWITCH -+ help -+ The LRNG uses a default message digest. With this -+ configuration option other message digests can be selected -+ and loaded at runtime. -+ -+if LRNG_SWITCH_HASH -+ -+config LRNG_HASH_KCAPI -+ tristate "Kernel crypto API hashing support for LRNG" -+ select CRYPTO_HASH -+ select CRYPTO_SHA512 -+ help -+ Enable the kernel crypto API support for entropy compression -+ and conditioning functions. -+ -+endif # LRNG_SWITCH_HASH -+ -+menuconfig LRNG_SWITCH_DRNG -+ bool "Support DRNG runtime switching" -+ select LRNG_SWITCH -+ help -+ The LRNG uses a default DRNG With this configuration -+ option other DRNGs or message digests can be selected and -+ loaded at runtime. -+ -+if LRNG_SWITCH_DRNG -+ -+config LRNG_SWITCH_DRNG_CHACHA20 -+ tristate "ChaCha20-based DRNG support for LRNG" -+ depends on !LRNG_DFLT_DRNG_CHACHA20 -+ select LRNG_DRNG_CHACHA20 -+ help -+ Enable the ChaCha20-based DRNG. This DRNG implementation -+ does not depend on the kernel crypto API presence. -+ -+config LRNG_SWITCH_DRBG -+ tristate "SP800-90A support for the LRNG" -+ depends on !LRNG_DFLT_DRNG_DRBG -+ select LRNG_DRBG -+ help -+ Enable the SP800-90A DRBG support for the LRNG. Once the -+ module is loaded, output from /dev/random, /dev/urandom, -+ getrandom(2), or get_random_bytes_full is provided by a DRBG. -+ -+config LRNG_SWITCH_DRNG_KCAPI -+ tristate "Kernel Crypto API support for the LRNG" -+ depends on !LRNG_DFLT_DRNG_KCAPI -+ depends on !LRNG_SWITCH_DRBG -+ select LRNG_DRNG_KCAPI -+ help -+ Enable the support for generic pseudo-random number -+ generators offered by the kernel crypto API with the -+ LRNG. Once the module is loaded, output from /dev/random, -+ /dev/urandom, getrandom(2), or get_random_bytes is -+ provided by the selected kernel crypto API RNG. -+ -+endif # LRNG_SWITCH_DRNG -+ -+choice -+ prompt "LRNG Default DRNG" -+ default LRNG_DFLT_DRNG_CHACHA20 -+ help -+ Select the default deterministic random number generator -+ that is used by the LRNG. When enabling the switchable -+ cryptographic mechanism support, this DRNG can be -+ replaced at runtime. -+ -+ config LRNG_DFLT_DRNG_CHACHA20 -+ bool "ChaCha20-based DRNG" -+ select LRNG_DRNG_CHACHA20 -+ -+ config LRNG_DFLT_DRNG_DRBG -+ bool "SP800-90A DRBG" -+ select LRNG_DRBG -+ -+ config LRNG_DFLT_DRNG_KCAPI -+ bool "Kernel Crypto API DRNG" -+ select LRNG_DRNG_KCAPI -+endchoice -+ -+menuconfig LRNG_TESTING_MENU -+ bool "LRNG testing interfaces" -+ depends on DEBUG_FS -+ help -+ Enable one or more of the following test interfaces. -+ -+ If unsure, say N. -+ -+if LRNG_TESTING_MENU -+ -+config LRNG_TESTING -+ bool -+ -+config LRNG_TESTING_RECORDING -+ bool -+ -+comment "Interrupt Entropy Source Test Interfaces" -+ -+config LRNG_RAW_HIRES_ENTROPY -+ bool "Interface to obtain raw unprocessed IRQ noise source data" -+ default y -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned high resolution time stamp noise that -+ is collected by the LRNG for statistical analysis. Extracted -+ noise data is not used to seed the LRNG. -+ -+ The raw noise data can be obtained using the lrng_raw_hires -+ debugfs file. Using the option lrng_testing.boot_raw_hires_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_JIFFIES_ENTROPY -+ bool "Entropy test interface to Jiffies of IRQ noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned Jiffies that is collected by -+ the LRNG for statistical analysis. This data is used for -+ seeding the LRNG if a high-resolution time stamp is not -+ available. If a high-resolution time stamp is detected, -+ the Jiffies value is not collected by the LRNG and no -+ data is provided via the test interface. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the lrng_raw_jiffies -+ debugfs file. Using the option lrng_testing.boot_raw_jiffies_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_IRQ_ENTROPY -+ bool "Entropy test interface to IRQ number noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned interrupt number that is collected by -+ the LRNG for statistical analysis. Extracted noise data is -+ not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the lrng_raw_irq -+ debugfs file. Using the option lrng_testing.boot_raw_irq_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_RETIP_ENTROPY -+ bool "Entropy test interface to RETIP value of IRQ noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned return instruction pointer value -+ that is collected by the LRNG for statistical analysis. -+ Extracted noise data is not used to seed the random number -+ generator. -+ -+ The raw noise data can be obtained using the lrng_raw_retip -+ debugfs file. Using the option lrng_testing.boot_raw_retip_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_REGS_ENTROPY -+ bool "Entropy test interface to IRQ register value noise source" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned interrupt register value that is -+ collected by the LRNG for statistical analysis. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the lrng_raw_regs -+ debugfs file. Using the option lrng_testing.boot_raw_regs_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_ARRAY -+ bool "Test interface to LRNG raw entropy IRQ storage array" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw noise data that is collected by the LRNG -+ in the per-CPU array for statistical analysis. The purpose -+ of this interface is to verify that the array handling code -+ truly only concatenates data and provides the same entropy -+ rate as the raw unconditioned noise source when assessing -+ the collected data byte-wise. -+ -+ The data can be obtained using the lrng_raw_array debugfs -+ file. Using the option lrng_testing.boot_raw_array=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_IRQ_PERF -+ bool "LRNG interrupt entropy source performance monitor" -+ depends on LRNG_IRQ -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ With this option, the performance monitor of the LRNG -+ interrupt handling code is enabled. The file provides -+ the execution time of the interrupt handler in -+ cycles. -+ -+ The interrupt performance data can be obtained using -+ the lrng_irq_perf debugfs file. Using the option -+ lrng_testing.boot_irq_perf=1 the performance data of -+ the first 1000 entropy events since boot can be sampled. -+ -+comment "Scheduler Entropy Source Test Interfaces" -+ -+config LRNG_RAW_SCHED_HIRES_ENTROPY -+ bool "Interface to obtain raw unprocessed scheduler noise source data" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned high resolution time stamp noise that -+ is collected by the LRNG for the Scheduler-based noise source -+ for statistical analysis. Extracted noise data is not used to -+ seed the LRNG. -+ -+ The raw noise data can be obtained using the lrng_raw_sched_hires -+ debugfs file. Using the option -+ lrng_testing.boot_raw_sched_hires_test=1 the raw noise of the -+ first 1000 entropy events since boot can be sampled. -+ -+config LRNG_RAW_SCHED_PID_ENTROPY -+ bool "Entropy test interface to PID value" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned PID value that is collected by the -+ LRNG for statistical analysis. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the -+ lrng_raw_sched_pid debugfs file. Using the option -+ lrng_testing.boot_raw_sched_pid_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_RAW_SCHED_START_TIME_ENTROPY -+ bool "Entropy test interface to task start time value" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned task start time value that is collected -+ by the LRNG for statistical analysis. Extracted noise -+ data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the -+ lrng_raw_sched_starttime debugfs file. Using the option -+ lrng_testing.boot_raw_sched_starttime_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+ -+config LRNG_RAW_SCHED_NVCSW_ENTROPY -+ bool "Entropy test interface to task context switch numbers" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ The test interface allows a privileged process to capture -+ the raw unconditioned task numbers of context switches that -+ are collected by the LRNG for statistical analysis. Extracted -+ noise data is not used to seed the random number generator. -+ -+ The raw noise data can be obtained using the -+ lrng_raw_sched_nvcsw debugfs file. Using the option -+ lrng_testing.boot_raw_sched_nvcsw_test=1 -+ the raw noise of the first 1000 entropy events since boot -+ can be sampled. -+ -+config LRNG_SCHED_PERF -+ bool "LRNG scheduler entropy source performance monitor" -+ depends on LRNG_SCHED -+ select LRNG_TESTING -+ select LRNG_TESTING_RECORDING -+ help -+ With this option, the performance monitor of the LRNG -+ scheduler event handling code is enabled. The file provides -+ the execution time of the interrupt handler in cycles. -+ -+ The scheduler performance data can be obtained using -+ the lrng_sched_perf debugfs file. Using the option -+ lrng_testing.boot_sched_perf=1 the performance data of -+ the first 1000 entropy events since boot can be sampled. -+ -+comment "Auxiliary Test Interfaces" -+ -+config LRNG_ACVT_HASH -+ bool "Enable LRNG ACVT Hash interface" -+ select LRNG_TESTING -+ help -+ With this option, the LRNG built-in hash function used for -+ auxiliary pool management and prior to switching the -+ cryptographic backends is made available for ACVT. The -+ interface allows writing of the data to be hashed -+ into the interface. The read operation triggers the hash -+ operation to generate message digest. -+ -+ The ACVT interface is available with the lrng_acvt_hash -+ debugfs file. -+ -+config LRNG_RUNTIME_MAX_WO_RESEED_CONFIG -+ bool "Enable runtime configuration of max reseed threshold" -+ help -+ When enabling this option, the LRNG provides an interface -+ allowing the setting of the maximum number of DRNG generate -+ operations without a reseed that has full entropy. The -+ interface is lrng_drng.max_wo_reseed. -+ -+config LRNG_TEST_CPU_ES_COMPRESSION -+ bool "Force CPU ES compression operation" -+ help -+ When enabling this option, the CPU ES compression operation -+ is forced by setting an arbitrary value > 1 for the data -+ multiplier even when the CPU ES would deliver full entropy. -+ This allows testing of the compression operation. It -+ therefore forces to pull more data from the CPU ES -+ than what may be required. -+ -+endif #LRNG_TESTING_MENU -+ -+config LRNG_SELFTEST -+ bool "Enable power-on and on-demand self-tests" -+ help -+ The power-on self-tests are executed during boot time -+ covering the ChaCha20 DRNG, the hash operation used for -+ processing the entropy pools and the auxiliary pool, and -+ the time stamp management of the LRNG. -+ -+ The on-demand self-tests are triggered by writing any -+ value into the SysFS file selftest_status. At the same -+ time, when reading this file, the test status is -+ returned. A zero indicates that all tests were executed -+ successfully. -+ -+ If unsure, say Y. -+ -+if LRNG_SELFTEST -+ -+config LRNG_SELFTEST_PANIC -+ bool "Panic the kernel upon self-test failure" -+ help -+ If the option is enabled, the kernel is terminated if an -+ LRNG power-on self-test failure is detected. -+ -+endif # LRNG_SELFTEST -+ -+endif # LRNG -+ -+endmenu # LRNG -diff --git a/drivers/char/lrng/Makefile b/drivers/char/lrng/Makefile -new file mode 100644 -index 000000000000..f61fc40f4620 ---- /dev/null -+++ b/drivers/char/lrng/Makefile -@@ -0,0 +1,39 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# Makefile for the Entropy Source and DRNG Manager. -+# -+ -+obj-y += lrng_es_mgr.o lrng_drng_mgr.o \ -+ lrng_es_aux.o -+obj-$(CONFIG_LRNG_SHA256) += lrng_sha256.o -+obj-$(CONFIG_LRNG_SHA1) += lrng_sha1.o -+ -+obj-$(CONFIG_SYSCTL) += lrng_proc.o -+obj-$(CONFIG_LRNG_SYSCTL) += lrng_sysctl.o -+obj-$(CONFIG_NUMA) += lrng_numa.o -+ -+obj-$(CONFIG_LRNG_SWITCH) += lrng_switch.o -+obj-$(CONFIG_LRNG_HASH_KCAPI) += lrng_hash_kcapi.o -+obj-$(CONFIG_LRNG_DRNG_CHACHA20) += lrng_drng_chacha20.o -+obj-$(CONFIG_LRNG_DRBG) += lrng_drng_drbg.o -+obj-$(CONFIG_LRNG_DRNG_KCAPI) += lrng_drng_kcapi.o -+obj-$(CONFIG_LRNG_DRNG_ATOMIC) += lrng_drng_atomic.o -+ -+obj-$(CONFIG_LRNG_TIMER_COMMON) += lrng_es_timer_common.o -+obj-$(CONFIG_LRNG_IRQ) += lrng_es_irq.o -+obj-$(CONFIG_LRNG_KERNEL_RNG) += lrng_es_krng.o -+obj-$(CONFIG_LRNG_SCHED) += lrng_es_sched.o -+obj-$(CONFIG_LRNG_CPU) += lrng_es_cpu.o -+obj-$(CONFIG_LRNG_JENT) += lrng_es_jent.o -+ -+obj-$(CONFIG_LRNG_HEALTH_TESTS) += lrng_health.o -+obj-$(CONFIG_LRNG_TESTING) += lrng_testing.o -+obj-$(CONFIG_LRNG_SELFTEST) += lrng_selftest.o -+ -+obj-$(CONFIG_LRNG_COMMON_DEV_IF) += lrng_interface_dev_common.o -+obj-$(CONFIG_LRNG_RANDOM_IF) += lrng_interface_random_user.o \ -+ lrng_interface_random_kernel.o \ -+ lrng_interface_aux.o -+obj-$(CONFIG_LRNG_KCAPI_IF) += lrng_interface_kcapi.o -+obj-$(CONFIG_LRNG_DEV_IF) += lrng_interface_dev.o -+obj-$(CONFIG_LRNG_HWRAND_IF) += lrng_interface_hwrand.o -diff --git a/drivers/char/lrng/lrng_definitions.h b/drivers/char/lrng/lrng_definitions.h -new file mode 100644 -index 000000000000..50f87c40f701 ---- /dev/null -+++ b/drivers/char/lrng/lrng_definitions.h -@@ -0,0 +1,151 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DEFINITIONS_H -+#define _LRNG_DEFINITIONS_H -+ -+#include -+#include -+#include -+ -+/*************************** General LRNG parameter ***************************/ -+ -+/* -+ * Specific settings for different use cases -+ */ -+#ifdef CONFIG_CRYPTO_FIPS -+# define LRNG_OVERSAMPLE_ES_BITS 64 -+# define LRNG_SEED_BUFFER_INIT_ADD_BITS 128 -+#else /* CONFIG_CRYPTO_FIPS */ -+# define LRNG_OVERSAMPLE_ES_BITS 0 -+# define LRNG_SEED_BUFFER_INIT_ADD_BITS 0 -+#endif /* CONFIG_CRYPTO_FIPS */ -+ -+/* Security strength of LRNG -- this must match DRNG security strength */ -+#define LRNG_DRNG_SECURITY_STRENGTH_BYTES 32 -+#define LRNG_DRNG_SECURITY_STRENGTH_BITS (LRNG_DRNG_SECURITY_STRENGTH_BYTES * 8) -+#define LRNG_DRNG_INIT_SEED_SIZE_BITS \ -+ (LRNG_DRNG_SECURITY_STRENGTH_BITS + LRNG_SEED_BUFFER_INIT_ADD_BITS) -+#define LRNG_DRNG_INIT_SEED_SIZE_BYTES (LRNG_DRNG_INIT_SEED_SIZE_BITS >> 3) -+ -+/* -+ * SP800-90A defines a maximum request size of 1<<16 bytes. The given value is -+ * considered a safer margin. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_DRNG_MAX_REQSIZE (1<<12) -+ -+/* -+ * SP800-90A defines a maximum number of requests between reseeds of 2^48. -+ * The given value is considered a much safer margin, balancing requests for -+ * frequent reseeds with the need to conserve entropy. This value MUST NOT be -+ * larger than INT_MAX because it is used in an atomic_t. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_DRNG_RESEED_THRESH (1<<20) -+ -+/* -+ * Maximum DRNG generation operations without reseed having full entropy -+ * This value defines the absolute maximum value of DRNG generation operations -+ * without a reseed holding full entropy. LRNG_DRNG_RESEED_THRESH is the -+ * threshold when a new reseed is attempted. But it is possible that this fails -+ * to deliver full entropy. In this case the DRNG will continue to provide data -+ * even though it was not reseeded with full entropy. To avoid in the extreme -+ * case that no reseed is performed for too long, this threshold is enforced. -+ * If that absolute low value is reached, the LRNG is marked as not operational. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_DRNG_MAX_WITHOUT_RESEED (1<<30) -+ -+/* -+ * Min required seed entropy is 128 bits covering the minimum entropy -+ * requirement of SP800-131A and the German BSI's TR02102. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_FULL_SEED_ENTROPY_BITS LRNG_DRNG_SECURITY_STRENGTH_BITS -+#define LRNG_MIN_SEED_ENTROPY_BITS 128 -+#define LRNG_INIT_ENTROPY_BITS 32 -+ -+/* -+ * Wakeup value -+ * -+ * This value is allowed to be changed but must not be larger than the -+ * digest size of the hash operation used update the aux_pool. -+ */ -+#ifdef CONFIG_LRNG_SHA256 -+# define LRNG_ATOMIC_DIGEST_SIZE SHA256_DIGEST_SIZE -+#else -+# define LRNG_ATOMIC_DIGEST_SIZE SHA1_DIGEST_SIZE -+#endif -+#define LRNG_WRITE_WAKEUP_ENTROPY LRNG_ATOMIC_DIGEST_SIZE -+ -+/* -+ * If the switching support is configured, we must provide support up to -+ * the largest digest size. Without switching support, we know it is only -+ * the built-in digest size. -+ */ -+#ifdef CONFIG_LRNG_SWITCH -+# define LRNG_MAX_DIGESTSIZE 64 -+#else -+# define LRNG_MAX_DIGESTSIZE LRNG_ATOMIC_DIGEST_SIZE -+#endif -+ -+/* -+ * Oversampling factor of timer-based events to obtain -+ * LRNG_DRNG_SECURITY_STRENGTH_BYTES. This factor is used when a -+ * high-resolution time stamp is not available. In this case, jiffies and -+ * register contents are used to fill the entropy pool. These noise sources -+ * are much less entropic than the high-resolution timer. The entropy content -+ * is the entropy content assumed with LRNG_[IRQ|SCHED]_ENTROPY_BITS divided by -+ * LRNG_ES_OVERSAMPLING_FACTOR. -+ * -+ * This value is allowed to be changed. -+ */ -+#define LRNG_ES_OVERSAMPLING_FACTOR 10 -+ -+/* Alignmask that is intended to be identical to CRYPTO_MINALIGN */ -+#define LRNG_KCAPI_ALIGN ARCH_KMALLOC_MINALIGN -+ -+/* -+ * This definition must provide a buffer that is equal to SHASH_DESC_ON_STACK -+ * as it will be casted into a struct shash_desc. -+ */ -+#define LRNG_POOL_SIZE (sizeof(struct shash_desc) + HASH_MAX_DESCSIZE) -+ -+/****************************** Helper code ***********************************/ -+ -+static inline u32 lrng_fast_noise_entropylevel(u32 ent_bits, u32 requested_bits) -+{ -+ /* Obtain entropy statement */ -+ ent_bits = ent_bits * requested_bits / LRNG_DRNG_SECURITY_STRENGTH_BITS; -+ /* Cap entropy to buffer size in bits */ -+ ent_bits = min_t(u32, ent_bits, requested_bits); -+ return ent_bits; -+} -+ -+/* Convert entropy in bits into nr. of events with the same entropy content. */ -+static inline u32 lrng_entropy_to_data(u32 entropy_bits, u32 entropy_rate) -+{ -+ return ((entropy_bits * entropy_rate) / -+ LRNG_DRNG_SECURITY_STRENGTH_BITS); -+} -+ -+/* Convert number of events into entropy value. */ -+static inline u32 lrng_data_to_entropy(u32 num, u32 entropy_rate) -+{ -+ return ((num * LRNG_DRNG_SECURITY_STRENGTH_BITS) / -+ entropy_rate); -+} -+ -+static inline u32 atomic_read_u32(atomic_t *v) -+{ -+ return (u32)atomic_read(v); -+} -+ -+#endif /* _LRNG_DEFINITIONS_H */ -diff --git a/drivers/char/lrng/lrng_drng_atomic.c b/drivers/char/lrng/lrng_drng_atomic.c -new file mode 100644 -index 000000000000..c112366677b6 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_atomic.c -@@ -0,0 +1,171 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG DRNG for atomic contexts -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include "lrng_drng_atomic.h" -+#include "lrng_drng_chacha20.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_sha.h" -+ -+static struct chacha20_state chacha20_atomic = { -+ LRNG_CC20_INIT_RFC7539(.block) -+}; -+ -+/* -+ * DRNG usable in atomic context. This DRNG will always use the ChaCha20 -+ * DRNG. It will never benefit from a DRNG switch like the "regular" DRNG. If -+ * there was no DRNG switch, the atomic DRNG is identical to the "regular" DRNG. -+ * -+ * The reason for having this is due to the fact that DRNGs other than -+ * the ChaCha20 DRNG may sleep. -+ */ -+static struct lrng_drng lrng_drng_atomic = { -+ LRNG_DRNG_STATE_INIT(lrng_drng_atomic, -+ &chacha20_atomic, NULL, -+ &lrng_cc20_drng_cb, &lrng_sha_hash_cb), -+ .spin_lock = __SPIN_LOCK_UNLOCKED(lrng_drng_atomic.spin_lock) -+}; -+ -+void lrng_drng_atomic_reset(void) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); -+ lrng_drng_reset(&lrng_drng_atomic); -+ spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); -+} -+ -+void lrng_drng_atomic_force_reseed(void) -+{ -+ lrng_drng_atomic.force_reseed = lrng_drng_atomic.fully_seeded; -+} -+ -+static bool lrng_drng_atomic_must_reseed(struct lrng_drng *drng) -+{ -+ return (!drng->fully_seeded || -+ atomic_read(&lrng_drng_atomic.requests) == 0 || -+ drng->force_reseed || -+ time_after(jiffies, -+ drng->last_seeded + lrng_drng_reseed_max_time * HZ)); -+} -+ -+void lrng_drng_atomic_seed_drng(struct lrng_drng *regular_drng) -+{ -+ u8 seedbuf[LRNG_DRNG_SECURITY_STRENGTH_BYTES] -+ __aligned(LRNG_KCAPI_ALIGN); -+ int ret; -+ -+ if (!lrng_drng_atomic_must_reseed(&lrng_drng_atomic)) -+ return; -+ -+ /* -+ * Reseed atomic DRNG another DRNG "regular" while this regular DRNG -+ * is reseeded. Therefore, this code operates in non-atomic context and -+ * thus can use the lrng_drng_get function to get random numbers from -+ * the just freshly seeded DRNG. -+ */ -+ ret = lrng_drng_get(regular_drng, seedbuf, sizeof(seedbuf)); -+ -+ if (ret < 0) { -+ pr_warn("Error generating random numbers for atomic DRNG: %d\n", -+ ret); -+ } else { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&lrng_drng_atomic.spin_lock, flags); -+ lrng_drng_inject(&lrng_drng_atomic, seedbuf, ret, -+ regular_drng->fully_seeded, "atomic"); -+ spin_unlock_irqrestore(&lrng_drng_atomic.spin_lock, flags); -+ } -+ memzero_explicit(&seedbuf, sizeof(seedbuf)); -+} -+ -+void lrng_drng_atomic_seed_es(void) -+{ -+ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); -+ struct lrng_drng *drng = &lrng_drng_atomic; -+ unsigned long flags; -+ u32 requested_bits = LRNG_MIN_SEED_ENTROPY_BITS; -+ -+ if (lrng_drng_atomic.fully_seeded) -+ return; -+ -+ /* -+ * When the LRNG reached the minimally seeded level, leave 256 bits of -+ * entropy for the regular DRNG. In addition ensure that additional -+ * 256 bits are available for the atomic DRNG to get to the fully -+ * seeded stage of the LRNG. -+ */ -+ if (lrng_state_min_seeded()) { -+ u32 avail_bits = lrng_avail_entropy(); -+ -+ requested_bits = -+ (avail_bits >= 2 * LRNG_FULL_SEED_ENTROPY_BITS) ? -+ LRNG_FULL_SEED_ENTROPY_BITS : 0; -+ -+ if (!requested_bits) -+ return; -+ } -+ -+ pr_debug("atomic DRNG seeding attempt to pull %u bits of entropy directly from entropy sources\n", -+ requested_bits); -+ -+ lrng_fill_seed_buffer(&seedbuf, requested_bits); -+ spin_lock_irqsave(&drng->spin_lock, flags); -+ lrng_drng_inject(&lrng_drng_atomic, (u8 *)&seedbuf, sizeof(seedbuf), -+ lrng_fully_seeded(drng->fully_seeded, -+ lrng_entropy_rate_eb(&seedbuf)), -+ "atomic"); -+ spin_unlock_irqrestore(&drng->spin_lock, flags); -+ lrng_init_ops(&seedbuf); -+ -+ /* -+ * Do not call lrng_init_ops(seedbuf) here as the atomic DRNG does not -+ * serve common users. -+ */ -+ memzero_explicit(&seedbuf, sizeof(seedbuf)); -+} -+ -+static void lrng_drng_atomic_get(u8 *outbuf, u32 outbuflen) -+{ -+ struct lrng_drng *drng = &lrng_drng_atomic; -+ unsigned long flags; -+ -+ if (!outbuf || !outbuflen) -+ return; -+ -+ outbuflen = min_t(size_t, outbuflen, INT_MAX); -+ -+ while (outbuflen) { -+ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); -+ int ret; -+ -+ atomic_dec(&drng->requests); -+ -+ spin_lock_irqsave(&drng->spin_lock, flags); -+ ret = drng->drng_cb->drng_generate(drng->drng, outbuf, todo); -+ spin_unlock_irqrestore(&drng->spin_lock, flags); -+ if (ret <= 0) { -+ pr_warn("getting random data from DRNG failed (%d)\n", -+ ret); -+ return; -+ } -+ outbuflen -= ret; -+ outbuf += ret; -+ } -+} -+ -+void lrng_get_random_bytes(void *buf, int nbytes) -+{ -+ lrng_drng_atomic_get((u8 *)buf, (u32)nbytes); -+ lrng_debug_report_seedlevel("lrng_get_random_bytes"); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes); -diff --git a/drivers/char/lrng/lrng_drng_atomic.h b/drivers/char/lrng/lrng_drng_atomic.h -new file mode 100644 -index 000000000000..d5d24fc94f2e ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_atomic.h -@@ -0,0 +1,23 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DRNG_ATOMIC_H -+#define _LRNG_DRNG_ATOMIC_H -+ -+#include "lrng_drng_mgr.h" -+ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+void lrng_drng_atomic_reset(void); -+void lrng_drng_atomic_seed_drng(struct lrng_drng *drng); -+void lrng_drng_atomic_seed_es(void); -+void lrng_drng_atomic_force_reseed(void); -+#else /* CONFIG_LRNG_DRNG_ATOMIC */ -+static inline void lrng_drng_atomic_reset(void) { } -+static inline void lrng_drng_atomic_seed_drng(struct lrng_drng *drng) { } -+static inline void lrng_drng_atomic_seed_es(void) { } -+static inline void lrng_drng_atomic_force_reseed(void) { } -+#endif /* CONFIG_LRNG_DRNG_ATOMIC */ -+ -+#endif /* _LRNG_DRNG_ATOMIC_H */ -diff --git a/drivers/char/lrng/lrng_drng_chacha20.c b/drivers/char/lrng/lrng_drng_chacha20.c -new file mode 100644 -index 000000000000..31be102e3007 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_chacha20.c -@@ -0,0 +1,195 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the cryptographic primitives using -+ * ChaCha20 cipher implementations. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_chacha20.h" -+ -+/******************************* ChaCha20 DRNG *******************************/ -+ -+#define CHACHA_BLOCK_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) -+ -+/* -+ * Update of the ChaCha20 state by either using an unused buffer part or by -+ * generating one ChaCha20 block which is half of the state of the ChaCha20. -+ * The block is XORed into the key part of the state. This shall ensure -+ * backtracking resistance as well as a proper mix of the ChaCha20 state once -+ * the key is injected. -+ */ -+static void lrng_chacha20_update(struct chacha20_state *chacha20_state, -+ __le32 *buf, u32 used_words) -+{ -+ struct chacha20_block *chacha20 = &chacha20_state->block; -+ u32 i; -+ __le32 tmp[CHACHA_BLOCK_WORDS]; -+ -+ BUILD_BUG_ON(sizeof(struct chacha20_block) != CHACHA_BLOCK_SIZE); -+ BUILD_BUG_ON(CHACHA_BLOCK_SIZE != 2 * CHACHA_KEY_SIZE); -+ -+ if (used_words > CHACHA_KEY_SIZE_WORDS) { -+ chacha20_block(&chacha20->constants[0], (u8 *)tmp); -+ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) -+ chacha20->key.u[i] ^= le32_to_cpu(tmp[i]); -+ memzero_explicit(tmp, sizeof(tmp)); -+ } else { -+ for (i = 0; i < CHACHA_KEY_SIZE_WORDS; i++) -+ chacha20->key.u[i] ^= le32_to_cpu(buf[i + used_words]); -+ } -+ -+ /* Deterministic increment of nonce as required in RFC 7539 chapter 4 */ -+ chacha20->nonce[0]++; -+ if (chacha20->nonce[0] == 0) { -+ chacha20->nonce[1]++; -+ if (chacha20->nonce[1] == 0) -+ chacha20->nonce[2]++; -+ } -+ -+ /* Leave counter untouched as it is start value is undefined in RFC */ -+} -+ -+/* -+ * Seed the ChaCha20 DRNG by injecting the input data into the key part of -+ * the ChaCha20 state. If the input data is longer than the ChaCha20 key size, -+ * perform a ChaCha20 operation after processing of key size input data. -+ * This operation shall spread out the entropy into the ChaCha20 state before -+ * new entropy is injected into the key part. -+ */ -+static int lrng_cc20_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) -+{ -+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; -+ struct chacha20_block *chacha20 = &chacha20_state->block; -+ -+ while (inbuflen) { -+ u32 i, todo = min_t(u32, inbuflen, CHACHA_KEY_SIZE); -+ -+ for (i = 0; i < todo; i++) -+ chacha20->key.b[i] ^= inbuf[i]; -+ -+ /* Break potential dependencies between the inbuf key blocks */ -+ lrng_chacha20_update(chacha20_state, NULL, -+ CHACHA_BLOCK_WORDS); -+ inbuf += todo; -+ inbuflen -= todo; -+ } -+ -+ return 0; -+} -+ -+/* -+ * Chacha20 DRNG generation of random numbers: the stream output of ChaCha20 -+ * is the random number. After the completion of the generation of the -+ * stream, the entire ChaCha20 state is updated. -+ * -+ * Note, as the ChaCha20 implements a 32 bit counter, we must ensure -+ * that this function is only invoked for at most 2^32 - 1 ChaCha20 blocks -+ * before a reseed or an update happens. This is ensured by the variable -+ * outbuflen which is a 32 bit integer defining the number of bytes to be -+ * generated by the ChaCha20 DRNG. At the end of this function, an update -+ * operation is invoked which implies that the 32 bit counter will never be -+ * overflown in this implementation. -+ */ -+static int lrng_cc20_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) -+{ -+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; -+ struct chacha20_block *chacha20 = &chacha20_state->block; -+ __le32 aligned_buf[CHACHA_BLOCK_WORDS]; -+ u32 ret = outbuflen, used = CHACHA_BLOCK_WORDS; -+ int zeroize_buf = 0; -+ -+ while (outbuflen >= CHACHA_BLOCK_SIZE) { -+ chacha20_block(&chacha20->constants[0], outbuf); -+ outbuf += CHACHA_BLOCK_SIZE; -+ outbuflen -= CHACHA_BLOCK_SIZE; -+ } -+ -+ if (outbuflen) { -+ chacha20_block(&chacha20->constants[0], (u8 *)aligned_buf); -+ memcpy(outbuf, aligned_buf, outbuflen); -+ used = ((outbuflen + sizeof(aligned_buf[0]) - 1) / -+ sizeof(aligned_buf[0])); -+ zeroize_buf = 1; -+ } -+ -+ lrng_chacha20_update(chacha20_state, aligned_buf, used); -+ -+ if (zeroize_buf) -+ memzero_explicit(aligned_buf, sizeof(aligned_buf)); -+ -+ return ret; -+} -+ -+/* -+ * Allocation of the DRNG state -+ */ -+static void *lrng_cc20_drng_alloc(u32 sec_strength) -+{ -+ struct chacha20_state *state = NULL; -+ -+ if (sec_strength > CHACHA_KEY_SIZE) { -+ pr_err("Security strength of ChaCha20 DRNG (%u bits) lower than requested by LRNG (%u bits)\n", -+ CHACHA_KEY_SIZE * 8, sec_strength * 8); -+ return ERR_PTR(-EINVAL); -+ } -+ if (sec_strength < CHACHA_KEY_SIZE) -+ pr_warn("Security strength of ChaCha20 DRNG (%u bits) higher than requested by LRNG (%u bits)\n", -+ CHACHA_KEY_SIZE * 8, sec_strength * 8); -+ -+ state = kmalloc(sizeof(struct chacha20_state), GFP_KERNEL); -+ if (!state) -+ return ERR_PTR(-ENOMEM); -+ pr_debug("memory for ChaCha20 core allocated\n"); -+ -+ lrng_cc20_init_rfc7539(&state->block); -+ -+ return state; -+} -+ -+static void lrng_cc20_drng_dealloc(void *drng) -+{ -+ struct chacha20_state *chacha20_state = (struct chacha20_state *)drng; -+ -+ pr_debug("ChaCha20 core zeroized and freed\n"); -+ kfree_sensitive(chacha20_state); -+} -+ -+static const char *lrng_cc20_drng_name(void) -+{ -+ return "ChaCha20 DRNG"; -+} -+ -+const struct lrng_drng_cb lrng_cc20_drng_cb = { -+ .drng_name = lrng_cc20_drng_name, -+ .drng_alloc = lrng_cc20_drng_alloc, -+ .drng_dealloc = lrng_cc20_drng_dealloc, -+ .drng_seed = lrng_cc20_drng_seed_helper, -+ .drng_generate = lrng_cc20_drng_generate_helper, -+}; -+ -+#if !defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) && \ -+ !defined(CONFIG_LRNG_DRNG_ATOMIC) -+static int __init lrng_cc20_drng_init(void) -+{ -+ return lrng_set_drng_cb(&lrng_cc20_drng_cb); -+} -+ -+static void __exit lrng_cc20_drng_exit(void) -+{ -+ lrng_set_drng_cb(NULL); -+} -+ -+late_initcall(lrng_cc20_drng_init); -+module_exit(lrng_cc20_drng_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - ChaCha20-based DRNG backend"); -+#endif /* CONFIG_LRNG_DFLT_DRNG_CHACHA20 */ -diff --git a/drivers/char/lrng/lrng_drng_chacha20.h b/drivers/char/lrng/lrng_drng_chacha20.h -new file mode 100644 -index 000000000000..fee6571281b6 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_chacha20.h -@@ -0,0 +1,42 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG ChaCha20 definitions -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_CHACHA20_H -+#define _LRNG_CHACHA20_H -+ -+#include -+ -+/* State according to RFC 7539 section 2.3 */ -+struct chacha20_block { -+ u32 constants[4]; -+ union { -+#define CHACHA_KEY_SIZE_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) -+ u32 u[CHACHA_KEY_SIZE_WORDS]; -+ u8 b[CHACHA_KEY_SIZE]; -+ } key; -+ u32 counter; -+ u32 nonce[3]; -+}; -+ -+struct chacha20_state { -+ struct chacha20_block block; -+}; -+ -+static inline void lrng_cc20_init_rfc7539(struct chacha20_block *chacha20) -+{ -+ chacha_init_consts(chacha20->constants); -+} -+ -+#define LRNG_CC20_INIT_RFC7539(x) \ -+ x.constants[0] = 0x61707865, \ -+ x.constants[1] = 0x3320646e, \ -+ x.constants[2] = 0x79622d32, \ -+ x.constants[3] = 0x6b206574, -+ -+extern const struct lrng_drng_cb lrng_cc20_drng_cb; -+ -+#endif /* _LRNG_CHACHA20_H */ -diff --git a/drivers/char/lrng/lrng_drng_drbg.c b/drivers/char/lrng/lrng_drng_drbg.c -new file mode 100644 -index 000000000000..1feec2c095a5 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_drbg.c -@@ -0,0 +1,179 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the cryptographic primitives using the -+ * kernel crypto API and its DRBG. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_drbg.h" -+ -+/* -+ * Define a DRBG plus a hash / MAC used to extract data from the entropy pool. -+ * For LRNG_HASH_NAME you can use a hash or a MAC (HMAC or CMAC) of your choice -+ * (Note, you should use the suggested selections below -- using SHA-1 or MD5 -+ * is not wise). The idea is that the used cipher primitive can be selected to -+ * be the same as used for the DRBG. I.e. the LRNG only uses one cipher -+ * primitive using the same cipher implementation with the options offered in -+ * the following. This means, if the CTR DRBG is selected and AES-NI is present, -+ * both the CTR DRBG and the selected cmac(aes) use AES-NI. -+ * -+ * The security strengths of the DRBGs are all 256 bits according to -+ * SP800-57 section 5.6.1. -+ * -+ * This definition is allowed to be changed. -+ */ -+#ifdef CONFIG_CRYPTO_DRBG_CTR -+static unsigned int lrng_drbg_type = 0; -+#elif defined CONFIG_CRYPTO_DRBG_HMAC -+static unsigned int lrng_drbg_type = 1; -+#elif defined CONFIG_CRYPTO_DRBG_HASH -+static unsigned int lrng_drbg_type = 2; -+#else -+#error "Unknown DRBG in use" -+#endif -+ -+/* The parameter must be r/o in sysfs as otherwise races appear. */ -+module_param(lrng_drbg_type, uint, 0444); -+MODULE_PARM_DESC(lrng_drbg_type, "DRBG type used for LRNG (0->CTR_DRBG, 1->HMAC_DRBG, 2->Hash_DRBG)"); -+ -+struct lrng_drbg { -+ const char *hash_name; -+ const char *drbg_core; -+}; -+ -+static const struct lrng_drbg lrng_drbg_types[] = { -+ { /* CTR_DRBG with AES-256 using derivation function */ -+ .drbg_core = "drbg_nopr_ctr_aes256", -+ }, { /* HMAC_DRBG with SHA-512 */ -+ .drbg_core = "drbg_nopr_hmac_sha512", -+ }, { /* Hash_DRBG with SHA-512 using derivation function */ -+ .drbg_core = "drbg_nopr_sha512" -+ } -+}; -+ -+static int lrng_drbg_drng_seed_helper(void *drng, const u8 *inbuf, u32 inbuflen) -+{ -+ struct drbg_state *drbg = (struct drbg_state *)drng; -+ LIST_HEAD(seedlist); -+ struct drbg_string data; -+ int ret; -+ -+ drbg_string_fill(&data, inbuf, inbuflen); -+ list_add_tail(&data.list, &seedlist); -+ ret = drbg->d_ops->update(drbg, &seedlist, drbg->seeded); -+ -+ if (ret >= 0) -+ drbg->seeded = true; -+ -+ return ret; -+} -+ -+static int lrng_drbg_drng_generate_helper(void *drng, u8 *outbuf, u32 outbuflen) -+{ -+ struct drbg_state *drbg = (struct drbg_state *)drng; -+ -+ return drbg->d_ops->generate(drbg, outbuf, outbuflen, NULL); -+} -+ -+static void *lrng_drbg_drng_alloc(u32 sec_strength) -+{ -+ struct drbg_state *drbg; -+ int coreref = -1; -+ bool pr = false; -+ int ret; -+ -+ drbg_convert_tfm_core(lrng_drbg_types[lrng_drbg_type].drbg_core, -+ &coreref, &pr); -+ if (coreref < 0) -+ return ERR_PTR(-EFAULT); -+ -+ drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); -+ if (!drbg) -+ return ERR_PTR(-ENOMEM); -+ -+ drbg->core = &drbg_cores[coreref]; -+ drbg->seeded = false; -+ ret = drbg_alloc_state(drbg); -+ if (ret) -+ goto err; -+ -+ if (sec_strength > drbg_sec_strength(drbg->core->flags)) { -+ pr_err("Security strength of DRBG (%u bits) lower than requested by LRNG (%u bits)\n", -+ drbg_sec_strength(drbg->core->flags) * 8, -+ sec_strength * 8); -+ goto dealloc; -+ } -+ -+ if (sec_strength < drbg_sec_strength(drbg->core->flags)) -+ pr_warn("Security strength of DRBG (%u bits) higher than requested by LRNG (%u bits)\n", -+ drbg_sec_strength(drbg->core->flags) * 8, -+ sec_strength * 8); -+ -+ pr_info("DRBG with %s core allocated\n", drbg->core->backend_cra_name); -+ -+ return drbg; -+ -+dealloc: -+ if (drbg->d_ops) -+ drbg->d_ops->crypto_fini(drbg); -+ drbg_dealloc_state(drbg); -+err: -+ kfree(drbg); -+ return ERR_PTR(-EINVAL); -+} -+ -+static void lrng_drbg_drng_dealloc(void *drng) -+{ -+ struct drbg_state *drbg = (struct drbg_state *)drng; -+ -+ if (drbg && drbg->d_ops) -+ drbg->d_ops->crypto_fini(drbg); -+ drbg_dealloc_state(drbg); -+ kfree_sensitive(drbg); -+ pr_info("DRBG deallocated\n"); -+} -+ -+static const char *lrng_drbg_name(void) -+{ -+ return lrng_drbg_types[lrng_drbg_type].drbg_core; -+} -+ -+const struct lrng_drng_cb lrng_drbg_cb = { -+ .drng_name = lrng_drbg_name, -+ .drng_alloc = lrng_drbg_drng_alloc, -+ .drng_dealloc = lrng_drbg_drng_dealloc, -+ .drng_seed = lrng_drbg_drng_seed_helper, -+ .drng_generate = lrng_drbg_drng_generate_helper, -+}; -+ -+#ifndef CONFIG_LRNG_DFLT_DRNG_DRBG -+static int __init lrng_drbg_init(void) -+{ -+ if (lrng_drbg_type >= ARRAY_SIZE(lrng_drbg_types)) { -+ pr_err("lrng_drbg_type parameter too large (given %u - max: %lu)", -+ lrng_drbg_type, -+ (unsigned long)ARRAY_SIZE(lrng_drbg_types) - 1); -+ return -EAGAIN; -+ } -+ return lrng_set_drng_cb(&lrng_drbg_cb); -+} -+ -+static void __exit lrng_drbg_exit(void) -+{ -+ lrng_set_drng_cb(NULL); -+} -+ -+late_initcall(lrng_drbg_init); -+module_exit(lrng_drbg_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - SP800-90A DRBG backend"); -+#endif /* CONFIG_LRNG_DFLT_DRNG_DRBG */ -diff --git a/drivers/char/lrng/lrng_drng_drbg.h b/drivers/char/lrng/lrng_drng_drbg.h -new file mode 100644 -index 000000000000..b3d556caf294 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_drbg.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG SP800-90A definitions -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DRBG_H -+#define _LRNG_DRBG_H -+ -+extern const struct lrng_drng_cb lrng_drbg_cb; -+ -+#endif /* _LRNG_DRBG_H */ -diff --git a/drivers/char/lrng/lrng_drng_kcapi.c b/drivers/char/lrng/lrng_drng_kcapi.c -new file mode 100644 -index 000000000000..a204bcf52a9a ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_kcapi.c -@@ -0,0 +1,208 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the cryptographic primitives using the -+ * kernel crypto API. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_kcapi.h" -+ -+static char *drng_name = NULL; -+module_param(drng_name, charp, 0444); -+MODULE_PARM_DESC(drng_name, "Kernel crypto API name of DRNG"); -+ -+static char *seed_hash = NULL; -+module_param(seed_hash, charp, 0444); -+MODULE_PARM_DESC(seed_hash, -+ "Kernel crypto API name of hash with output size equal to seedsize of DRNG to bring seed string to the size required by the DRNG"); -+ -+struct lrng_drng_info { -+ struct crypto_rng *kcapi_rng; -+ struct crypto_shash *hash_tfm; -+}; -+ -+static int lrng_kcapi_drng_seed_helper(void *drng, const u8 *inbuf, -+ u32 inbuflen) -+{ -+ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; -+ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; -+ struct crypto_shash *hash_tfm = lrng_drng_info->hash_tfm; -+ SHASH_DESC_ON_STACK(shash, hash_tfm); -+ u32 digestsize; -+ u8 digest[HASH_MAX_DIGESTSIZE] __aligned(8); -+ int ret; -+ -+ if (!hash_tfm) -+ return crypto_rng_reset(kcapi_rng, inbuf, inbuflen); -+ -+ shash->tfm = hash_tfm; -+ digestsize = crypto_shash_digestsize(hash_tfm); -+ -+ ret = crypto_shash_digest(shash, inbuf, inbuflen, digest); -+ shash_desc_zero(shash); -+ if (ret) -+ return ret; -+ -+ ret = crypto_rng_reset(kcapi_rng, digest, digestsize); -+ if (ret) -+ return ret; -+ -+ memzero_explicit(digest, digestsize); -+ return 0; -+} -+ -+static int lrng_kcapi_drng_generate_helper(void *drng, u8 *outbuf, -+ u32 outbuflen) -+{ -+ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; -+ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; -+ int ret = crypto_rng_get_bytes(kcapi_rng, outbuf, outbuflen); -+ -+ if (ret < 0) -+ return ret; -+ -+ return outbuflen; -+} -+ -+static void *lrng_kcapi_drng_alloc(u32 sec_strength) -+{ -+ struct lrng_drng_info *lrng_drng_info; -+ struct crypto_rng *kcapi_rng; -+ u32 time = random_get_entropy(); -+ int seedsize, rv; -+ void *ret = ERR_PTR(-ENOMEM); -+ -+ if (!drng_name) { -+ pr_err("DRNG name missing\n"); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ if (!memcmp(drng_name, "stdrng", 6) || -+ !memcmp(drng_name, "lrng", 4) || -+ !memcmp(drng_name, "drbg", 4) || -+ !memcmp(drng_name, "jitterentropy_rng", 17)) { -+ pr_err("Refusing to load the requested random number generator\n"); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ lrng_drng_info = kzalloc(sizeof(*lrng_drng_info), GFP_KERNEL); -+ if (!lrng_drng_info) -+ return ERR_PTR(-ENOMEM); -+ -+ kcapi_rng = crypto_alloc_rng(drng_name, 0, 0); -+ if (IS_ERR(kcapi_rng)) { -+ pr_err("DRNG %s cannot be allocated\n", drng_name); -+ ret = ERR_CAST(kcapi_rng); -+ goto free; -+ } -+ -+ lrng_drng_info->kcapi_rng = kcapi_rng; -+ -+ seedsize = crypto_rng_seedsize(kcapi_rng); -+ if (seedsize) { -+ struct crypto_shash *hash_tfm; -+ -+ if (!seed_hash) { -+ switch (seedsize) { -+ case 32: -+ seed_hash = "sha256"; -+ break; -+ case 48: -+ seed_hash = "sha384"; -+ break; -+ case 64: -+ seed_hash = "sha512"; -+ break; -+ default: -+ pr_err("Seed size %d cannot be processed\n", -+ seedsize); -+ goto dealloc; -+ } -+ } -+ -+ hash_tfm = crypto_alloc_shash(seed_hash, 0, 0); -+ if (IS_ERR(hash_tfm)) { -+ ret = ERR_CAST(hash_tfm); -+ goto dealloc; -+ } -+ -+ if (seedsize != crypto_shash_digestsize(hash_tfm)) { -+ pr_err("Seed hash output size not equal to DRNG seed size\n"); -+ crypto_free_shash(hash_tfm); -+ ret = ERR_PTR(-EINVAL); -+ goto dealloc; -+ } -+ -+ lrng_drng_info->hash_tfm = hash_tfm; -+ -+ pr_info("Seed hash %s allocated\n", seed_hash); -+ } -+ -+ rv = lrng_kcapi_drng_seed_helper(lrng_drng_info, (u8 *)(&time), -+ sizeof(time)); -+ if (rv) { -+ ret = ERR_PTR(rv); -+ goto dealloc; -+ } -+ -+ pr_info("Kernel crypto API DRNG %s allocated\n", drng_name); -+ -+ return lrng_drng_info; -+ -+dealloc: -+ crypto_free_rng(kcapi_rng); -+free: -+ kfree(lrng_drng_info); -+ return ret; -+} -+ -+static void lrng_kcapi_drng_dealloc(void *drng) -+{ -+ struct lrng_drng_info *lrng_drng_info = (struct lrng_drng_info *)drng; -+ struct crypto_rng *kcapi_rng = lrng_drng_info->kcapi_rng; -+ -+ crypto_free_rng(kcapi_rng); -+ if (lrng_drng_info->hash_tfm) -+ crypto_free_shash(lrng_drng_info->hash_tfm); -+ kfree(lrng_drng_info); -+ pr_info("DRNG %s deallocated\n", drng_name); -+} -+ -+static const char *lrng_kcapi_drng_name(void) -+{ -+ return drng_name; -+} -+ -+const struct lrng_drng_cb lrng_kcapi_drng_cb = { -+ .drng_name = lrng_kcapi_drng_name, -+ .drng_alloc = lrng_kcapi_drng_alloc, -+ .drng_dealloc = lrng_kcapi_drng_dealloc, -+ .drng_seed = lrng_kcapi_drng_seed_helper, -+ .drng_generate = lrng_kcapi_drng_generate_helper, -+}; -+ -+#ifndef CONFIG_LRNG_DFLT_DRNG_KCAPI -+static int __init lrng_kcapi_init(void) -+{ -+ return lrng_set_drng_cb(&lrng_kcapi_drng_cb); -+} -+static void __exit lrng_kcapi_exit(void) -+{ -+ lrng_set_drng_cb(NULL); -+} -+ -+late_initcall(lrng_kcapi_init); -+module_exit(lrng_kcapi_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - kernel crypto API DRNG backend"); -+#endif /* CONFIG_LRNG_DFLT_DRNG_KCAPI */ -diff --git a/drivers/char/lrng/lrng_drng_kcapi.h b/drivers/char/lrng/lrng_drng_kcapi.h -new file mode 100644 -index 000000000000..5db25aaf830c ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_kcapi.h -@@ -0,0 +1,13 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG kernel crypto API DRNG definition -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_KCAPI_DRNG_H -+#define _LRNG_KCAPI_DRNG_H -+ -+extern const struct lrng_drng_cb lrng_kcapi_drng_cb; -+ -+#endif /* _LRNG_KCAPI_DRNG_H */ -diff --git a/drivers/char/lrng/lrng_drng_mgr.c b/drivers/char/lrng/lrng_drng_mgr.c -new file mode 100644 -index 000000000000..e9e08c186530 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_mgr.c -@@ -0,0 +1,657 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG DRNG management -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_atomic.h" -+#include "lrng_drng_chacha20.h" -+#include "lrng_drng_drbg.h" -+#include "lrng_drng_kcapi.h" -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_random_kernel.h" -+#include "lrng_numa.h" -+#include "lrng_sha.h" -+ -+/* -+ * Maximum number of seconds between DRNG reseed intervals of the DRNG. Note, -+ * this is enforced with the next request of random numbers from the -+ * DRNG. Setting this value to zero implies a reseeding attempt before every -+ * generated random number. -+ */ -+int lrng_drng_reseed_max_time = 600; -+ -+/* -+ * Is LRNG for general-purpose use (i.e. is at least the lrng_drng_init -+ * fully allocated)? -+ */ -+static atomic_t lrng_avail = ATOMIC_INIT(0); -+ -+/* Guard protecting all crypto callback update operation of all DRNGs. */ -+DEFINE_MUTEX(lrng_crypto_cb_update); -+ -+/* -+ * Default hash callback that provides the crypto primitive right from the -+ * kernel start. It must not perform any memory allocation operation, but -+ * simply perform the hash calculation. -+ */ -+const struct lrng_hash_cb *lrng_default_hash_cb = &lrng_sha_hash_cb; -+ -+/* -+ * Default DRNG callback that provides the crypto primitive which is -+ * allocated either during late kernel boot stage. So, it is permissible for -+ * the callback to perform memory allocation operations. -+ */ -+const struct lrng_drng_cb *lrng_default_drng_cb = -+#if defined(CONFIG_LRNG_DFLT_DRNG_CHACHA20) -+ &lrng_cc20_drng_cb; -+#elif defined(CONFIG_LRNG_DFLT_DRNG_DRBG) -+ &lrng_drbg_cb; -+#elif defined(CONFIG_LRNG_DFLT_DRNG_KCAPI) -+ &lrng_kcapi_drng_cb; -+#else -+#error "Unknown default DRNG selected" -+#endif -+ -+/* DRNG for non-atomic use cases */ -+static struct lrng_drng lrng_drng_init = { -+ LRNG_DRNG_STATE_INIT(lrng_drng_init, NULL, NULL, NULL, -+ &lrng_sha_hash_cb), -+ .lock = __MUTEX_INITIALIZER(lrng_drng_init.lock), -+}; -+ -+/* Prediction-resistance DRNG: only deliver as much data as received entropy */ -+static struct lrng_drng lrng_drng_pr = { -+ LRNG_DRNG_STATE_INIT(lrng_drng_pr, NULL, NULL, NULL, -+ &lrng_sha_hash_cb), -+ .lock = __MUTEX_INITIALIZER(lrng_drng_pr.lock), -+}; -+ -+static u32 max_wo_reseed = LRNG_DRNG_MAX_WITHOUT_RESEED; -+#ifdef CONFIG_LRNG_RUNTIME_MAX_WO_RESEED_CONFIG -+module_param(max_wo_reseed, uint, 0444); -+MODULE_PARM_DESC(max_wo_reseed, -+ "Maximum number of DRNG generate operation without full reseed\n"); -+#endif -+ -+/* Wait queue to wait until the LRNG is initialized - can freely be used */ -+DECLARE_WAIT_QUEUE_HEAD(lrng_init_wait); -+ -+/********************************** Helper ************************************/ -+ -+bool lrng_get_available(void) -+{ -+ return likely(atomic_read(&lrng_avail)); -+} -+ -+struct lrng_drng *lrng_drng_init_instance(void) -+{ -+ return &lrng_drng_init; -+} -+ -+struct lrng_drng *lrng_drng_pr_instance(void) -+{ -+ return &lrng_drng_pr; -+} -+ -+struct lrng_drng *lrng_drng_node_instance(void) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ int node = numa_node_id(); -+ -+ if (lrng_drng && lrng_drng[node]) -+ return lrng_drng[node]; -+ -+ return lrng_drng_init_instance(); -+} -+ -+void lrng_drng_reset(struct lrng_drng *drng) -+{ -+ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); -+ atomic_set(&drng->requests_since_fully_seeded, 0); -+ drng->last_seeded = jiffies; -+ drng->fully_seeded = false; -+ drng->force_reseed = true; -+ pr_debug("reset DRNG\n"); -+} -+ -+/* Initialize the DRNG, except the mutex lock */ -+int lrng_drng_alloc_common(struct lrng_drng *drng, -+ const struct lrng_drng_cb *drng_cb) -+{ -+ if (!drng || !drng_cb) -+ return -EINVAL; -+ if (!IS_ERR_OR_NULL(drng->drng)) -+ return 0; -+ -+ drng->drng_cb = drng_cb; -+ drng->drng = drng_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); -+ if (IS_ERR(drng->drng)) -+ return PTR_ERR(drng->drng); -+ -+ lrng_drng_reset(drng); -+ return 0; -+} -+ -+/* Initialize the default DRNG during boot and perform its seeding */ -+int lrng_drng_initalize(void) -+{ -+ int ret; -+ -+ if (lrng_get_available()) -+ return 0; -+ -+ /* Catch programming error */ -+ WARN_ON(lrng_drng_init.hash_cb != lrng_default_hash_cb); -+ -+ mutex_lock(&lrng_drng_init.lock); -+ if (lrng_get_available()) { -+ mutex_unlock(&lrng_drng_init.lock); -+ return 0; -+ } -+ -+ /* Initialize the PR DRNG inside init lock as it guards lrng_avail. */ -+ mutex_lock(&lrng_drng_pr.lock); -+ ret = lrng_drng_alloc_common(&lrng_drng_pr, lrng_default_drng_cb); -+ mutex_unlock(&lrng_drng_pr.lock); -+ -+ if (!ret) { -+ ret = lrng_drng_alloc_common(&lrng_drng_init, -+ lrng_default_drng_cb); -+ if (!ret) -+ atomic_set(&lrng_avail, 1); -+ } -+ mutex_unlock(&lrng_drng_init.lock); -+ if (ret) -+ return ret; -+ -+ pr_debug("LRNG for general use is available\n"); -+ -+ /* Seed the DRNG with any entropy available */ -+ if (lrng_pool_trylock()) { -+ pr_info("Initial DRNG initialized triggering first seeding\n"); -+ lrng_drng_seed_work(NULL); -+ } else { -+ pr_info("Initial DRNG initialized without seeding\n"); -+ } -+ -+ return 0; -+} -+ -+static int __init lrng_drng_make_available(void) -+{ -+ return lrng_drng_initalize(); -+} -+late_initcall(lrng_drng_make_available); -+ -+bool lrng_sp80090c_compliant(void) -+{ -+ /* SP800-90C compliant oversampling is only requested in FIPS mode */ -+ return fips_enabled; -+} -+ -+bool lrng_ntg1_compliant(void) -+{ -+ /* Implies using of /dev/random with O_SYNC */ -+ return true; -+} -+ -+/************************* Random Number Generation ***************************/ -+ -+/* Inject a data buffer into the DRNG - caller must hold its lock */ -+void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, -+ bool fully_seeded, const char *drng_type) -+{ -+ BUILD_BUG_ON(LRNG_DRNG_RESEED_THRESH > INT_MAX); -+ pr_debug("seeding %s DRNG with %u bytes\n", drng_type, inbuflen); -+ if (drng->drng_cb->drng_seed(drng->drng, inbuf, inbuflen) < 0) { -+ pr_warn("seeding of %s DRNG failed\n", drng_type); -+ drng->force_reseed = true; -+ } else { -+ int gc = LRNG_DRNG_RESEED_THRESH - atomic_read(&drng->requests); -+ -+ pr_debug("%s DRNG stats since last seeding: %lu secs; generate calls: %d\n", -+ drng_type, -+ (time_after(jiffies, drng->last_seeded) ? -+ (jiffies - drng->last_seeded) : 0) / HZ, gc); -+ -+ /* Count the numbers of generate ops since last fully seeded */ -+ if (fully_seeded) -+ atomic_set(&drng->requests_since_fully_seeded, 0); -+ else -+ atomic_add(gc, &drng->requests_since_fully_seeded); -+ -+ drng->last_seeded = jiffies; -+ atomic_set(&drng->requests, LRNG_DRNG_RESEED_THRESH); -+ drng->force_reseed = false; -+ -+ if (!drng->fully_seeded) { -+ drng->fully_seeded = fully_seeded; -+ if (drng->fully_seeded) -+ pr_debug("%s DRNG fully seeded\n", drng_type); -+ } -+ } -+} -+ -+/* -+ * Perform the seeding of the DRNG with data from entropy source. -+ * The function returns the entropy injected into the DRNG in bits. -+ */ -+static u32 lrng_drng_seed_es_nolock(struct lrng_drng *drng) -+{ -+ struct entropy_buf seedbuf __aligned(LRNG_KCAPI_ALIGN); -+ u32 collected_entropy; -+ -+ lrng_fill_seed_buffer(&seedbuf, -+ lrng_get_seed_entropy_osr(drng->fully_seeded)); -+ -+ collected_entropy = lrng_entropy_rate_eb(&seedbuf); -+ lrng_drng_inject(drng, (u8 *)&seedbuf, sizeof(seedbuf), -+ lrng_fully_seeded(drng->fully_seeded, -+ collected_entropy), -+ "regular"); -+ -+ /* Set the seeding state of the LRNG */ -+ lrng_init_ops(&seedbuf); -+ -+ memzero_explicit(&seedbuf, sizeof(seedbuf)); -+ -+ return collected_entropy; -+} -+ -+static void lrng_drng_seed_es(struct lrng_drng *drng) -+{ -+ mutex_lock(&drng->lock); -+ lrng_drng_seed_es_nolock(drng); -+ mutex_unlock(&drng->lock); -+} -+ -+static void lrng_drng_seed(struct lrng_drng *drng) -+{ -+ BUILD_BUG_ON(LRNG_MIN_SEED_ENTROPY_BITS > -+ LRNG_DRNG_SECURITY_STRENGTH_BITS); -+ -+ if (lrng_get_available()) { -+ /* (Re-)Seed DRNG */ -+ lrng_drng_seed_es(drng); -+ /* (Re-)Seed atomic DRNG from regular DRNG */ -+ lrng_drng_atomic_seed_drng(drng); -+ } else { -+ /* -+ * If no-one is waiting for the DRNG, seed the atomic DRNG -+ * directly from the entropy sources. -+ */ -+ if (!wq_has_sleeper(&lrng_init_wait)) -+ lrng_drng_atomic_seed_es(); -+ else -+ lrng_init_ops(NULL); -+ } -+} -+ -+static void _lrng_drng_seed_work(struct lrng_drng *drng, u32 node) -+{ -+ pr_debug("reseed triggered by system events for DRNG on NUMA node %d\n", -+ node); -+ lrng_drng_seed(drng); -+ if (drng->fully_seeded) { -+ /* Prevent reseed storm */ -+ drng->last_seeded += node * 100 * HZ; -+ } -+} -+ -+/* -+ * DRNG reseed trigger: Kernel thread handler triggered by the schedule_work() -+ */ -+void lrng_drng_seed_work(struct work_struct *dummy) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ u32 node; -+ -+ if (lrng_drng) { -+ for_each_online_node(node) { -+ struct lrng_drng *drng = lrng_drng[node]; -+ -+ if (drng && !drng->fully_seeded) { -+ _lrng_drng_seed_work(drng, node); -+ goto out; -+ } -+ } -+ } else { -+ if (!lrng_drng_init.fully_seeded) { -+ _lrng_drng_seed_work(&lrng_drng_init, 0); -+ goto out; -+ } -+ } -+ -+ if (!lrng_drng_pr.fully_seeded) { -+ _lrng_drng_seed_work(&lrng_drng_pr, 0); -+ goto out; -+ } -+ -+ lrng_pool_all_numa_nodes_seeded(true); -+ -+out: -+ /* Allow the seeding operation to be called again */ -+ lrng_pool_unlock(); -+} -+ -+/* Force all DRNGs to reseed before next generation */ -+void lrng_drng_force_reseed(void) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ u32 node; -+ -+ /* -+ * If the initial DRNG is over the reseed threshold, allow a forced -+ * reseed only for the initial DRNG as this is the fallback for all. It -+ * must be kept seeded before all others to keep the LRNG operational. -+ */ -+ if (!lrng_drng || -+ (atomic_read_u32(&lrng_drng_init.requests_since_fully_seeded) > -+ LRNG_DRNG_RESEED_THRESH)) { -+ lrng_drng_init.force_reseed = lrng_drng_init.fully_seeded; -+ pr_debug("force reseed of initial DRNG\n"); -+ return; -+ } -+ for_each_online_node(node) { -+ struct lrng_drng *drng = lrng_drng[node]; -+ -+ if (!drng) -+ continue; -+ -+ drng->force_reseed = drng->fully_seeded; -+ pr_debug("force reseed of DRNG on node %u\n", node); -+ } -+ lrng_drng_atomic_force_reseed(); -+} -+EXPORT_SYMBOL(lrng_drng_force_reseed); -+ -+static bool lrng_drng_must_reseed(struct lrng_drng *drng) -+{ -+ return (atomic_dec_and_test(&drng->requests) || -+ drng->force_reseed || -+ time_after(jiffies, -+ drng->last_seeded + lrng_drng_reseed_max_time * HZ)); -+} -+ -+/* -+ * lrng_drng_get() - Get random data out of the DRNG which is reseeded -+ * frequently. -+ * -+ * @drng: DRNG instance -+ * @outbuf: buffer for storing random data -+ * @outbuflen: length of outbuf -+ * -+ * Return: -+ * * < 0 in error case (DRNG generation or update failed) -+ * * >=0 returning the returned number of bytes -+ */ -+int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen) -+{ -+ u32 processed = 0; -+ bool pr = (drng == &lrng_drng_pr) ? true : false; -+ -+ if (!outbuf || !outbuflen) -+ return 0; -+ -+ if (!lrng_get_available()) -+ return -EOPNOTSUPP; -+ -+ outbuflen = min_t(size_t, outbuflen, INT_MAX); -+ -+ /* If DRNG operated without proper reseed for too long, block LRNG */ -+ BUILD_BUG_ON(LRNG_DRNG_MAX_WITHOUT_RESEED < LRNG_DRNG_RESEED_THRESH); -+ if (atomic_read_u32(&drng->requests_since_fully_seeded) > max_wo_reseed) -+ lrng_unset_fully_seeded(drng); -+ -+ while (outbuflen) { -+ u32 todo = min_t(u32, outbuflen, LRNG_DRNG_MAX_REQSIZE); -+ int ret; -+ -+ /* In normal operation, check whether to reseed */ -+ if (!pr && lrng_drng_must_reseed(drng)) { -+ if (!lrng_pool_trylock()) { -+ drng->force_reseed = true; -+ } else { -+ lrng_drng_seed(drng); -+ lrng_pool_unlock(); -+ } -+ } -+ -+ mutex_lock(&drng->lock); -+ -+ if (pr) { -+ /* If async reseed did not deliver entropy, try now */ -+ if (!drng->fully_seeded) { -+ u32 coll_ent_bits; -+ -+ /* If we cannot get the pool lock, try again. */ -+ if (!lrng_pool_trylock()) { -+ mutex_unlock(&drng->lock); -+ continue; -+ } -+ -+ coll_ent_bits = lrng_drng_seed_es_nolock(drng); -+ -+ lrng_pool_unlock(); -+ -+ /* If no new entropy was received, stop now. */ -+ if (!coll_ent_bits) { -+ mutex_unlock(&drng->lock); -+ goto out; -+ } -+ -+ /* Produce no more data than received entropy */ -+ todo = min_t(u32, todo, coll_ent_bits >> 3); -+ } -+ -+ /* Do not produce more than DRNG security strength */ -+ todo = min_t(u32, todo, lrng_security_strength() >> 3); -+ } -+ ret = drng->drng_cb->drng_generate(drng->drng, -+ outbuf + processed, todo); -+ -+ mutex_unlock(&drng->lock); -+ if (ret <= 0) { -+ pr_warn("getting random data from DRNG failed (%d)\n", -+ ret); -+ return -EFAULT; -+ } -+ processed += ret; -+ outbuflen -= ret; -+ -+ if (pr) { -+ /* Force the async reseed for PR DRNG */ -+ lrng_unset_fully_seeded(drng); -+ if (outbuflen) -+ cond_resched(); -+ } -+ } -+ -+out: -+ return processed; -+} -+ -+int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *drng = &lrng_drng_init; -+ int ret, node = numa_node_id(); -+ -+ might_sleep(); -+ -+ if (pr) -+ drng = &lrng_drng_pr; -+ else if (lrng_drng && lrng_drng[node] && lrng_drng[node]->fully_seeded) -+ drng = lrng_drng[node]; -+ -+ ret = lrng_drng_initalize(); -+ if (ret) -+ return ret; -+ -+ return lrng_drng_get(drng, outbuf, outbuflen); -+} -+ -+/* Reset LRNG such that all existing entropy is gone */ -+static void _lrng_reset(struct work_struct *work) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ -+ if (!lrng_drng) { -+ mutex_lock(&lrng_drng_init.lock); -+ lrng_drng_reset(&lrng_drng_init); -+ mutex_unlock(&lrng_drng_init.lock); -+ } else { -+ u32 node; -+ -+ for_each_online_node(node) { -+ struct lrng_drng *drng = lrng_drng[node]; -+ -+ if (!drng) -+ continue; -+ mutex_lock(&drng->lock); -+ lrng_drng_reset(drng); -+ mutex_unlock(&drng->lock); -+ } -+ } -+ -+ mutex_lock(&lrng_drng_pr.lock); -+ lrng_drng_reset(&lrng_drng_pr); -+ mutex_unlock(&lrng_drng_pr.lock); -+ -+ lrng_drng_atomic_reset(); -+ lrng_set_entropy_thresh(LRNG_INIT_ENTROPY_BITS); -+ -+ lrng_reset_state(); -+} -+ -+static DECLARE_WORK(lrng_reset_work, _lrng_reset); -+ -+void lrng_reset(void) -+{ -+ schedule_work(&lrng_reset_work); -+} -+ -+/******************* Generic LRNG kernel output interfaces ********************/ -+ -+static int lrng_drng_sleep_while_not_all_nodes_seeded(unsigned int nonblock) -+{ -+ if (lrng_pool_all_numa_nodes_seeded_get()) -+ return 0; -+ if (nonblock) -+ return -EAGAIN; -+ wait_event_interruptible(lrng_init_wait, -+ lrng_pool_all_numa_nodes_seeded_get()); -+ return 0; -+} -+ -+int lrng_drng_sleep_while_nonoperational(int nonblock) -+{ -+ if (likely(lrng_state_operational())) -+ return 0; -+ if (nonblock) -+ return -EAGAIN; -+ return wait_event_interruptible(lrng_init_wait, -+ lrng_state_operational()); -+} -+ -+int lrng_drng_sleep_while_non_min_seeded(void) -+{ -+ if (likely(lrng_state_min_seeded())) -+ return 0; -+ return wait_event_interruptible(lrng_init_wait, -+ lrng_state_min_seeded()); -+} -+ -+ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags) -+{ -+ struct entropy_buf *eb = (struct entropy_buf *)(buf + 2); -+ u64 buflen = sizeof(struct entropy_buf) + 2 * sizeof(u64); -+ u64 collected_bits = 0; -+ int ret; -+ -+ /* Ensure buffer is aligned as required */ -+ BUILD_BUG_ON(sizeof(buflen) < LRNG_KCAPI_ALIGN); -+ if (nbytes < sizeof(buflen)) -+ return -EINVAL; -+ -+ /* Write buffer size into first word */ -+ buf[0] = buflen; -+ if (nbytes < buflen) -+ return -EMSGSIZE; -+ -+ ret = lrng_drng_sleep_while_not_all_nodes_seeded( -+ flags & LRNG_GET_SEED_NONBLOCK); -+ if (ret) -+ return ret; -+ -+ /* Try to get the pool lock and sleep on it to get it. */ -+ lrng_pool_lock(); -+ -+ /* If an LRNG DRNG becomes unseeded, give this DRNG precedence. */ -+ if (!lrng_pool_all_numa_nodes_seeded_get()) { -+ lrng_pool_unlock(); -+ return 0; -+ } -+ -+ /* -+ * Try to get seed data - a rarely used busyloop is cheaper than a wait -+ * queue that is constantly woken up by the hot code path of -+ * lrng_init_ops. -+ */ -+ for (;;) { -+ lrng_fill_seed_buffer(eb, -+ lrng_get_seed_entropy_osr(flags & -+ LRNG_GET_SEED_FULLY_SEEDED)); -+ collected_bits = lrng_entropy_rate_eb(eb); -+ -+ /* Break the collection loop if we got entropy, ... */ -+ if (collected_bits || -+ /* ... a DRNG becomes unseeded, give DRNG precedence, ... */ -+ !lrng_pool_all_numa_nodes_seeded_get() || -+ /* ... if the caller does not want a blocking behavior. */ -+ (flags & LRNG_GET_SEED_NONBLOCK)) -+ break; -+ -+ schedule(); -+ } -+ -+ lrng_pool_unlock(); -+ -+ /* Write collected entropy size into second word */ -+ buf[1] = collected_bits; -+ -+ return (ssize_t)buflen; -+} -+ -+void lrng_get_random_bytes_full(void *buf, int nbytes) -+{ -+ lrng_drng_sleep_while_nonoperational(0); -+ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes_full); -+ -+void lrng_get_random_bytes_min(void *buf, int nbytes) -+{ -+ lrng_drng_sleep_while_non_min_seeded(); -+ lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, false); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes_min); -+ -+int lrng_get_random_bytes_pr(void *buf, int nbytes) -+{ -+ lrng_drng_sleep_while_nonoperational(0); -+ return lrng_drng_get_sleep((u8 *)buf, (u32)nbytes, true); -+} -+EXPORT_SYMBOL(lrng_get_random_bytes_pr); -diff --git a/drivers/char/lrng/lrng_drng_mgr.h b/drivers/char/lrng/lrng_drng_mgr.h -new file mode 100644 -index 000000000000..ec120e4980b5 ---- /dev/null -+++ b/drivers/char/lrng/lrng_drng_mgr.h -@@ -0,0 +1,86 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_DRNG_H -+#define _LRNG_DRNG_H -+ -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+ -+extern struct wait_queue_head lrng_init_wait; -+extern int lrng_drng_reseed_max_time; -+extern struct mutex lrng_crypto_cb_update; -+extern const struct lrng_drng_cb *lrng_default_drng_cb; -+extern const struct lrng_hash_cb *lrng_default_hash_cb; -+ -+/* DRNG state handle */ -+struct lrng_drng { -+ void *drng; /* DRNG handle */ -+ void *hash; /* Hash handle */ -+ const struct lrng_drng_cb *drng_cb; /* DRNG callbacks */ -+ const struct lrng_hash_cb *hash_cb; /* Hash callbacks */ -+ atomic_t requests; /* Number of DRNG requests */ -+ atomic_t requests_since_fully_seeded; /* Number DRNG requests since -+ * last fully seeded -+ */ -+ unsigned long last_seeded; /* Last time it was seeded */ -+ bool fully_seeded; /* Is DRNG fully seeded? */ -+ bool force_reseed; /* Force a reseed */ -+ -+ rwlock_t hash_lock; /* Lock hash_cb replacement */ -+ /* Lock write operations on DRNG state, DRNG replacement of drng_cb */ -+ struct mutex lock; /* Non-atomic DRNG operation */ -+ spinlock_t spin_lock; /* Atomic DRNG operation */ -+}; -+ -+#define LRNG_DRNG_STATE_INIT(x, d, h, d_cb, h_cb) \ -+ .drng = d, \ -+ .hash = h, \ -+ .drng_cb = d_cb, \ -+ .hash_cb = h_cb, \ -+ .requests = ATOMIC_INIT(LRNG_DRNG_RESEED_THRESH),\ -+ .requests_since_fully_seeded = ATOMIC_INIT(0), \ -+ .last_seeded = 0, \ -+ .fully_seeded = false, \ -+ .force_reseed = true, \ -+ .hash_lock = __RW_LOCK_UNLOCKED(x.hash_lock) -+ -+struct lrng_drng *lrng_drng_init_instance(void); -+struct lrng_drng *lrng_drng_pr_instance(void); -+struct lrng_drng *lrng_drng_node_instance(void); -+ -+void lrng_reset(void); -+int lrng_drng_alloc_common(struct lrng_drng *drng, -+ const struct lrng_drng_cb *crypto_cb); -+int lrng_drng_initalize(void); -+bool lrng_sp80090c_compliant(void); -+bool lrng_ntg1_compliant(void); -+bool lrng_get_available(void); -+void lrng_drng_reset(struct lrng_drng *drng); -+void lrng_drng_inject(struct lrng_drng *drng, const u8 *inbuf, u32 inbuflen, -+ bool fully_seeded, const char *drng_type); -+int lrng_drng_get(struct lrng_drng *drng, u8 *outbuf, u32 outbuflen); -+int lrng_drng_sleep_while_nonoperational(int nonblock); -+int lrng_drng_sleep_while_non_min_seeded(void); -+int lrng_drng_get_sleep(u8 *outbuf, u32 outbuflen, bool pr); -+void lrng_drng_seed_work(struct work_struct *dummy); -+void lrng_drng_force_reseed(void); -+ -+static inline u32 lrng_compress_osr(void) -+{ -+ return lrng_sp80090c_compliant() ? LRNG_OVERSAMPLE_ES_BITS : 0; -+} -+ -+static inline u32 lrng_reduce_by_osr(u32 entropy_bits) -+{ -+ u32 osr_bits = lrng_compress_osr(); -+ -+ return (entropy_bits >= osr_bits) ? (entropy_bits - osr_bits) : 0; -+} -+ -+#endif /* _LRNG_DRNG_H */ -diff --git a/drivers/char/lrng/lrng_es_aux.c b/drivers/char/lrng/lrng_es_aux.c -new file mode 100644 -index 000000000000..245bc829998b ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_aux.c -@@ -0,0 +1,335 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Auxiliary entropy pool -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_sysctl.h" -+ -+/* -+ * This is the auxiliary pool -+ * -+ * The aux pool array is aligned to 8 bytes to comfort the kernel crypto API -+ * cipher implementations of the hash functions used to read the pool: for some -+ * accelerated implementations, we need an alignment to avoid a realignment -+ * which involves memcpy(). The alignment to 8 bytes should satisfy all crypto -+ * implementations. -+ */ -+struct lrng_pool { -+ u8 aux_pool[LRNG_POOL_SIZE]; /* Aux pool: digest state */ -+ atomic_t aux_entropy_bits; -+ atomic_t digestsize; /* Digest size of used hash */ -+ bool initialized; /* Aux pool initialized? */ -+ -+ /* Serialize read of entropy pool and update of aux pool */ -+ spinlock_t lock; -+}; -+ -+static struct lrng_pool lrng_pool __aligned(LRNG_KCAPI_ALIGN) = { -+ .aux_entropy_bits = ATOMIC_INIT(0), -+ .digestsize = ATOMIC_INIT(LRNG_ATOMIC_DIGEST_SIZE), -+ .initialized = false, -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_pool.lock) -+}; -+ -+/********************************** Helper ***********************************/ -+ -+/* Entropy in bits present in aux pool */ -+static u32 lrng_aux_avail_entropy(u32 __unused) -+{ -+ /* Cap available entropy with max entropy */ -+ u32 avail_bits = min_t(u32, lrng_get_digestsize(), -+ atomic_read_u32(&lrng_pool.aux_entropy_bits)); -+ -+ /* Consider oversampling rate due to aux pool conditioning */ -+ return lrng_reduce_by_osr(avail_bits); -+} -+ -+/* Set the digest size of the used hash in bytes */ -+static void lrng_set_digestsize(u32 digestsize) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ u32 ent_bits = atomic_xchg_relaxed(&pool->aux_entropy_bits, 0), -+ old_digestsize = lrng_get_digestsize(); -+ -+ atomic_set(&lrng_pool.digestsize, digestsize); -+ -+ /* -+ * Update the write wakeup threshold which must not be larger -+ * than the digest size of the current conditioning hash. -+ */ -+ digestsize = lrng_reduce_by_osr(digestsize << 3); -+ lrng_sysctl_update_max_write_thresh(digestsize); -+ lrng_write_wakeup_bits = digestsize; -+ -+ /* -+ * In case the new digest is larger than the old one, cap the available -+ * entropy to the old message digest used to process the existing data. -+ */ -+ ent_bits = min_t(u32, ent_bits, old_digestsize); -+ atomic_add(ent_bits, &pool->aux_entropy_bits); -+} -+ -+static int __init lrng_init_wakeup_bits(void) -+{ -+ u32 digestsize = lrng_reduce_by_osr(lrng_get_digestsize()); -+ -+ lrng_sysctl_update_max_write_thresh(digestsize); -+ lrng_write_wakeup_bits = digestsize; -+ return 0; -+} -+core_initcall(lrng_init_wakeup_bits); -+ -+/* Obtain the digest size provided by the used hash in bits */ -+u32 lrng_get_digestsize(void) -+{ -+ return atomic_read_u32(&lrng_pool.digestsize) << 3; -+} -+ -+/* Set entropy content in user-space controllable aux pool */ -+void lrng_pool_set_entropy(u32 entropy_bits) -+{ -+ atomic_set(&lrng_pool.aux_entropy_bits, entropy_bits); -+} -+ -+static void lrng_aux_reset(void) -+{ -+ lrng_pool_set_entropy(0); -+} -+ -+/* -+ * Replace old with new hash for auxiliary pool handling -+ * -+ * Assumption: the caller must guarantee that the new_cb is available during the -+ * entire operation (e.g. it must hold the write lock against pointer updating). -+ */ -+static int -+lrng_aux_switch_hash(struct lrng_drng *drng, int __unused, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ struct lrng_drng *init_drng = lrng_drng_init_instance(); -+ struct lrng_pool *pool = &lrng_pool; -+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ int ret; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ if (unlikely(!pool->initialized)) -+ return 0; -+ -+ /* We only switch if the processed DRNG is the initial DRNG. */ -+ if (init_drng != drng) -+ return 0; -+ -+ /* Get the aux pool hash with old digest ... */ -+ ret = old_cb->hash_final(shash, digest) ?: -+ /* ... re-initialize the hash with the new digest ... */ -+ new_cb->hash_init(shash, new_hash) ?: -+ /* -+ * ... feed the old hash into the new state. We may feed -+ * uninitialized memory into the new state, but this is -+ * considered no issue and even good as we have some more -+ * uncertainty here. -+ */ -+ new_cb->hash_update(shash, digest, sizeof(digest)); -+ if (!ret) { -+ lrng_set_digestsize(new_cb->hash_digestsize(new_hash)); -+ pr_debug("Re-initialize aux entropy pool with hash %s\n", -+ new_cb->hash_name()); -+ } -+ -+ memzero_explicit(digest, sizeof(digest)); -+ return ret; -+} -+ -+/* Insert data into auxiliary pool by using the hash update function. */ -+static int -+lrng_aux_pool_insert_locked(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ const struct lrng_hash_cb *hash_cb; -+ unsigned long flags; -+ void *hash; -+ int ret; -+ -+ entropy_bits = min_t(u32, entropy_bits, inbuflen << 3); -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ if (unlikely(!pool->initialized)) { -+ ret = hash_cb->hash_init(shash, hash); -+ if (ret) -+ goto out; -+ pool->initialized = true; -+ } -+ -+ ret = hash_cb->hash_update(shash, inbuf, inbuflen); -+ if (ret) -+ goto out; -+ -+ /* -+ * Cap the available entropy to the hash output size compliant to -+ * SP800-90B section 3.1.5.1 table 1. -+ */ -+ entropy_bits += atomic_read_u32(&pool->aux_entropy_bits); -+ atomic_set(&pool->aux_entropy_bits, -+ min_t(u32, entropy_bits, -+ hash_cb->hash_digestsize(hash) << 3)); -+ -+out: -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ return ret; -+} -+ -+int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ unsigned long flags; -+ int ret; -+ -+ spin_lock_irqsave(&pool->lock, flags); -+ ret = lrng_aux_pool_insert_locked(inbuf, inbuflen, entropy_bits); -+ spin_unlock_irqrestore(&pool->lock, flags); -+ -+ lrng_es_add_entropy(); -+ -+ return ret; -+} -+EXPORT_SYMBOL(lrng_pool_insert_aux); -+ -+/************************* Get data from entropy pool *************************/ -+ -+/* -+ * Get auxiliary entropy pool and its entropy content for seed buffer. -+ * Caller must hold lrng_pool.pool->lock. -+ * @outbuf: buffer to store data in with size requested_bits -+ * @requested_bits: Requested amount of entropy -+ * @return: amount of entropy in outbuf in bits. -+ */ -+static u32 lrng_aux_get_pool(u8 *outbuf, u32 requested_bits) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ struct shash_desc *shash = (struct shash_desc *)pool->aux_pool; -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ const struct lrng_hash_cb *hash_cb; -+ unsigned long flags; -+ void *hash; -+ u32 collected_ent_bits, returned_ent_bits, unused_bits = 0, -+ digestsize, digestsize_bits, requested_bits_osr; -+ u8 aux_output[LRNG_MAX_DIGESTSIZE]; -+ -+ if (unlikely(!pool->initialized)) -+ return 0; -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ digestsize = hash_cb->hash_digestsize(hash); -+ digestsize_bits = digestsize << 3; -+ -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(digestsize_bits, requested_bits); -+ -+ /* Ensure that no more than the size of aux_pool can be requested */ -+ requested_bits = min_t(u32, requested_bits, (LRNG_MAX_DIGESTSIZE << 3)); -+ requested_bits_osr = requested_bits + lrng_compress_osr(); -+ -+ /* Cap entropy with entropy counter from aux pool and the used digest */ -+ collected_ent_bits = min_t(u32, digestsize_bits, -+ atomic_xchg_relaxed(&pool->aux_entropy_bits, 0)); -+ -+ /* We collected too much entropy and put the overflow back */ -+ if (collected_ent_bits > requested_bits_osr) { -+ /* Amount of bits we collected too much */ -+ unused_bits = collected_ent_bits - requested_bits_osr; -+ /* Put entropy back */ -+ atomic_add(unused_bits, &pool->aux_entropy_bits); -+ /* Fix collected entropy */ -+ collected_ent_bits = requested_bits_osr; -+ } -+ -+ /* Apply oversampling: discount requested oversampling rate */ -+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); -+ -+ pr_debug("obtained %u bits by collecting %u bits of entropy from aux pool, %u bits of entropy remaining\n", -+ returned_ent_bits, collected_ent_bits, unused_bits); -+ -+ /* Get the digest for the aux pool to be returned to the caller ... */ -+ if (hash_cb->hash_final(shash, aux_output) || -+ /* -+ * ... and re-initialize the aux state. Do not add the aux pool -+ * digest for backward secrecy as it will be added with the -+ * insertion of the complete seed buffer after it has been filled. -+ */ -+ hash_cb->hash_init(shash, hash)) { -+ returned_ent_bits = 0; -+ } else { -+ /* -+ * Do not truncate the output size exactly to collected_ent_bits -+ * as the aux pool may contain data that is not credited with -+ * entropy, but we want to use them to stir the DRNG state. -+ */ -+ memcpy(outbuf, aux_output, requested_bits >> 3); -+ } -+ -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ memzero_explicit(aux_output, digestsize); -+ return returned_ent_bits; -+} -+ -+static void lrng_aux_get_backtrack(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ struct lrng_pool *pool = &lrng_pool; -+ unsigned long flags; -+ -+ /* Ensure aux pool extraction and backtracking op are atomic */ -+ spin_lock_irqsave(&pool->lock, flags); -+ -+ eb->e_bits[lrng_ext_es_aux] = lrng_aux_get_pool(eb->e[lrng_ext_es_aux], -+ requested_bits); -+ -+ /* Mix the extracted data back into pool for backtracking resistance */ -+ if (lrng_aux_pool_insert_locked((u8 *)eb, -+ sizeof(struct entropy_buf), 0)) -+ pr_warn("Backtracking resistance operation failed\n"); -+ -+ spin_unlock_irqrestore(&pool->lock, flags); -+} -+ -+static void lrng_aux_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for operating entropy pool: %s\n" -+ " Available entropy: %u\n", -+ lrng_drng_init->hash_cb->hash_name(), -+ lrng_aux_avail_entropy(0)); -+} -+ -+struct lrng_es_cb lrng_es_aux = { -+ .name = "Auxiliary", -+ .get_ent = lrng_aux_get_backtrack, -+ .curr_entropy = lrng_aux_avail_entropy, -+ .max_entropy = lrng_get_digestsize, -+ .state = lrng_aux_es_state, -+ .reset = lrng_aux_reset, -+ .switch_hash = lrng_aux_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_aux.h b/drivers/char/lrng/lrng_es_aux.h -new file mode 100644 -index 000000000000..bc41e6474aad ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_aux.h -@@ -0,0 +1,44 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_AUX_H -+#define _LRNG_ES_AUX_H -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_mgr_cb.h" -+ -+u32 lrng_get_digestsize(void); -+void lrng_pool_set_entropy(u32 entropy_bits); -+int lrng_pool_insert_aux(const u8 *inbuf, u32 inbuflen, u32 entropy_bits); -+ -+extern struct lrng_es_cb lrng_es_aux; -+ -+/****************************** Helper code ***********************************/ -+ -+/* Obtain the security strength of the LRNG in bits */ -+static inline u32 lrng_security_strength(void) -+{ -+ /* -+ * We use a hash to read the entropy in the entropy pool. According to -+ * SP800-90B table 1, the entropy can be at most the digest size. -+ * Considering this together with the last sentence in section 3.1.5.1.2 -+ * the security strength of a (approved) hash is equal to its output -+ * size. On the other hand the entropy cannot be larger than the -+ * security strength of the used DRBG. -+ */ -+ return min_t(u32, LRNG_FULL_SEED_ENTROPY_BITS, lrng_get_digestsize()); -+} -+ -+static inline u32 lrng_get_seed_entropy_osr(bool fully_seeded) -+{ -+ u32 requested_bits = lrng_security_strength(); -+ -+ /* Apply oversampling during initialization according to SP800-90C */ -+ if (lrng_sp80090c_compliant() && !fully_seeded) -+ requested_bits += LRNG_SEED_BUFFER_INIT_ADD_BITS; -+ return requested_bits; -+} -+ -+#endif /* _LRNG_ES_AUX_H */ -diff --git a/drivers/char/lrng/lrng_es_cpu.c b/drivers/char/lrng/lrng_es_cpu.c -new file mode 100644 -index 000000000000..f982cc31df4e ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_cpu.c -@@ -0,0 +1,280 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Fast Entropy Source: CPU-based entropy source -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_cpu.h" -+ -+/* -+ * Estimated entropy of data is a 32th of LRNG_DRNG_SECURITY_STRENGTH_BITS. -+ * As we have no ability to review the implementation of those noise sources, -+ * it is prudent to have a conservative estimate here. -+ */ -+#define LRNG_ARCHRANDOM_DEFAULT_STRENGTH CONFIG_LRNG_CPU_ENTROPY_RATE -+#define LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH LRNG_DRNG_SECURITY_STRENGTH_BITS -+#ifdef CONFIG_RANDOM_TRUST_CPU -+static u32 cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; -+#else -+static u32 cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; -+#endif -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(cpu_entropy, uint, 0644); -+MODULE_PARM_DESC(cpu_entropy, "Entropy in bits of 256 data bits from CPU noise source (e.g. RDSEED)"); -+#endif -+ -+static int __init lrng_parse_trust_cpu(char *arg) -+{ -+ int ret; -+ bool trust_cpu = false; -+ -+ ret = kstrtobool(arg, &trust_cpu); -+ if (ret) -+ return ret; -+ -+ if (trust_cpu) { -+ cpu_entropy = LRNG_ARCHRANDOM_TRUST_CPU_STRENGTH; -+ lrng_es_add_entropy(); -+ } else { -+ cpu_entropy = LRNG_ARCHRANDOM_DEFAULT_STRENGTH; -+ } -+ -+ return 0; -+} -+early_param("random.trust_cpu", lrng_parse_trust_cpu); -+ -+static u32 lrng_cpu_entropylevel(u32 requested_bits) -+{ -+ return lrng_fast_noise_entropylevel(cpu_entropy, requested_bits); -+} -+ -+static u32 lrng_cpu_poolsize(void) -+{ -+ return lrng_cpu_entropylevel(lrng_security_strength()); -+} -+ -+static u32 lrng_get_cpu_data(u8 *outbuf, u32 requested_bits) -+{ -+ size_t longs = 0; -+ u32 i, req = requested_bits >> 3; -+ -+ /* operate on full blocks */ -+ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BYTES % sizeof(unsigned long)); -+ BUILD_BUG_ON(LRNG_SEED_BUFFER_INIT_ADD_BITS % sizeof(unsigned long)); -+ /* ensure we have aligned buffers */ -+ BUILD_BUG_ON(LRNG_KCAPI_ALIGN % sizeof(unsigned long)); -+ -+ for (i = 0; i < req; i += longs) { -+ longs = arch_get_random_seed_longs( -+ (unsigned long *)(outbuf + i), req - i); -+ if (longs) -+ continue; -+ longs = arch_get_random_longs((unsigned long *)(outbuf + i), -+ req - i); -+ if (!longs) { -+ cpu_entropy = 0; -+ return 0; -+ } -+ } -+ -+ return requested_bits; -+} -+ -+static u32 lrng_get_cpu_data_compress(u8 *outbuf, u32 requested_bits, -+ u32 data_multiplier) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb; -+ struct lrng_drng *drng = lrng_drng_node_instance(); -+ unsigned long flags; -+ u32 ent_bits = 0, i, partial_bits = 0, digestsize, digestsize_bits, -+ full_bits; -+ void *hash; -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ digestsize = hash_cb->hash_digestsize(hash); -+ digestsize_bits = digestsize << 3; -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(digestsize_bits, requested_bits); -+ full_bits = requested_bits * data_multiplier; -+ -+ /* Calculate oversampling for SP800-90C */ -+ if (lrng_sp80090c_compliant()) { -+ /* Complete amount of bits to be pulled */ -+ full_bits += LRNG_OVERSAMPLE_ES_BITS * data_multiplier; -+ /* Full blocks that will be pulled */ -+ data_multiplier = full_bits / requested_bits; -+ /* Partial block in bits to be pulled */ -+ partial_bits = full_bits - (data_multiplier * requested_bits); -+ } -+ -+ if (hash_cb->hash_init(shash, hash)) -+ goto out; -+ -+ /* Hash all data from the CPU entropy source */ -+ for (i = 0; i < data_multiplier; i++) { -+ ent_bits = lrng_get_cpu_data(outbuf, requested_bits); -+ if (!ent_bits) -+ goto out; -+ -+ if (hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) -+ goto err; -+ } -+ -+ /* Hash partial block, if applicable */ -+ ent_bits = lrng_get_cpu_data(outbuf, partial_bits); -+ if (ent_bits && -+ hash_cb->hash_update(shash, outbuf, ent_bits >> 3)) -+ goto err; -+ -+ pr_debug("pulled %u bits from CPU RNG entropy source\n", full_bits); -+ ent_bits = requested_bits; -+ -+ /* Generate the compressed data to be returned to the caller */ -+ if (requested_bits < digestsize_bits) { -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ -+ if (hash_cb->hash_final(shash, digest)) -+ goto err; -+ -+ /* Truncate output data to requested size */ -+ memcpy(outbuf, digest, requested_bits >> 3); -+ memzero_explicit(digest, digestsize); -+ } else { -+ if (hash_cb->hash_final(shash, outbuf)) -+ goto err; -+ } -+ -+out: -+ hash_cb->hash_desc_zero(shash); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ return ent_bits; -+ -+err: -+ ent_bits = 0; -+ goto out; -+} -+ -+/* -+ * If CPU entropy source requires does not return full entropy, return the -+ * multiplier of how much data shall be sampled from it. -+ */ -+static u32 lrng_cpu_multiplier(void) -+{ -+ static u32 data_multiplier = 0; -+ unsigned long v; -+ -+ if (data_multiplier > 0) -+ return data_multiplier; -+ -+ if (IS_ENABLED(CONFIG_X86) && !arch_get_random_seed_longs(&v, 1)) { -+ /* -+ * Intel SPEC: pulling 512 blocks from RDRAND ensures -+ * one reseed making it logically equivalent to RDSEED. -+ */ -+ data_multiplier = 512; -+ } else if (IS_ENABLED(CONFIG_PPC)) { -+ /* -+ * PowerISA defines DARN to deliver at least 0.5 bits of -+ * entropy per data bit. -+ */ -+ data_multiplier = 2; -+ } else if (IS_ENABLED(CONFIG_RISCV)) { -+ /* -+ * riscv-crypto-spec-scalar-1.0.0-rc6.pdf section 4.2 defines -+ * this requirement. -+ */ -+ data_multiplier = 2; -+ } else { -+ /* CPU provides full entropy */ -+ data_multiplier = CONFIG_LRNG_CPU_FULL_ENT_MULTIPLIER; -+ } -+ return data_multiplier; -+} -+ -+static int -+lrng_cpu_switch_hash(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ u32 digestsize, multiplier; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ digestsize = lrng_get_digestsize(); -+ multiplier = lrng_cpu_multiplier(); -+ -+ /* -+ * It would be security violation if the new digestsize is smaller than -+ * the set CPU entropy rate. -+ */ -+ WARN_ON(multiplier > 1 && digestsize < cpu_entropy); -+ cpu_entropy = min_t(u32, digestsize, cpu_entropy); -+ return 0; -+} -+ -+/* -+ * lrng_get_arch() - Get CPU entropy source entropy -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: requested entropy in bits -+ */ -+static void lrng_cpu_get(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ u32 ent_bits, data_multiplier = lrng_cpu_multiplier(); -+ -+ if (data_multiplier <= 1) { -+ ent_bits = lrng_get_cpu_data(eb->e[lrng_ext_es_cpu], -+ requested_bits); -+ } else { -+ ent_bits = lrng_get_cpu_data_compress(eb->e[lrng_ext_es_cpu], -+ requested_bits, -+ data_multiplier); -+ } -+ -+ ent_bits = lrng_cpu_entropylevel(ent_bits); -+ pr_debug("obtained %u bits of entropy from CPU RNG entropy source\n", -+ ent_bits); -+ eb->e_bits[lrng_ext_es_cpu] = ent_bits; -+} -+ -+static void lrng_cpu_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ u32 data_multiplier = lrng_cpu_multiplier(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for compressing data: %s\n" -+ " Available entropy: %u\n" -+ " Data multiplier: %u\n", -+ (data_multiplier <= 1) ? -+ "N/A" : lrng_drng_init->hash_cb->hash_name(), -+ lrng_cpu_poolsize(), -+ data_multiplier); -+} -+ -+struct lrng_es_cb lrng_es_cpu = { -+ .name = "CPU", -+ .get_ent = lrng_cpu_get, -+ .curr_entropy = lrng_cpu_entropylevel, -+ .max_entropy = lrng_cpu_poolsize, -+ .state = lrng_cpu_es_state, -+ .reset = NULL, -+ .switch_hash = lrng_cpu_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_cpu.h b/drivers/char/lrng/lrng_es_cpu.h -new file mode 100644 -index 000000000000..8dbb4d9a2926 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_cpu.h -@@ -0,0 +1,17 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_CPU_H -+#define _LRNG_ES_CPU_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_CPU -+ -+extern struct lrng_es_cb lrng_es_cpu; -+ -+#endif /* CONFIG_LRNG_CPU */ -+ -+#endif /* _LRNG_ES_CPU_H */ -diff --git a/drivers/char/lrng/lrng_es_irq.c b/drivers/char/lrng/lrng_es_irq.c -new file mode 100644 -index 000000000000..97c5c1d26447 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_irq.c -@@ -0,0 +1,727 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Interrupt data collection -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_timer_common.h" -+#include "lrng_health.h" -+#include "lrng_numa.h" -+#include "lrng_testing.h" -+ -+/* -+ * Number of interrupts to be recorded to assume that DRNG security strength -+ * bits of entropy are received. -+ * Note: a value below the DRNG security strength should not be defined as this -+ * may imply the DRNG can never be fully seeded in case other noise -+ * sources are unavailable. -+ */ -+#define LRNG_IRQ_ENTROPY_BITS LRNG_UINT32_C(CONFIG_LRNG_IRQ_ENTROPY_RATE) -+ -+ -+/* Number of interrupts required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ -+static u32 lrng_irq_entropy_bits = LRNG_IRQ_ENTROPY_BITS; -+ -+static u32 irq_entropy __read_mostly = LRNG_IRQ_ENTROPY_BITS; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(irq_entropy, uint, 0444); -+MODULE_PARM_DESC(irq_entropy, -+ "How many interrupts must be collected for obtaining 256 bits of entropy\n"); -+#endif -+ -+/* Per-CPU array holding concatenated IRQ entropy events */ -+static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_irq_array) -+ __aligned(LRNG_KCAPI_ALIGN); -+static DEFINE_PER_CPU(u32, lrng_irq_array_ptr) = 0; -+static DEFINE_PER_CPU(atomic_t, lrng_irq_array_irqs) = ATOMIC_INIT(0); -+ -+/* -+ * The entropy collection is performed by executing the following steps: -+ * 1. fill up the per-CPU array holding the time stamps -+ * 2. once the per-CPU array is full, a compression of the data into -+ * the entropy pool is performed - this happens in interrupt context -+ * -+ * If step 2 is not desired in interrupt context, the following boolean -+ * needs to be set to false. This implies that old entropy data in the -+ * per-CPU array collected since the last DRNG reseed is overwritten with -+ * new entropy data instead of retaining the entropy with the compression -+ * operation. -+ * -+ * Impact on entropy: -+ * -+ * If continuous compression is enabled, the maximum entropy that is collected -+ * per CPU between DRNG reseeds is equal to the digest size of the used hash. -+ * -+ * If continuous compression is disabled, the maximum number of entropy events -+ * that can be collected per CPU is equal to LRNG_DATA_ARRAY_SIZE. This amount -+ * of events is converted into an entropy statement which then represents the -+ * maximum amount of entropy collectible per CPU between DRNG reseeds. -+ */ -+static bool lrng_irq_continuous_compression __read_mostly = -+ IS_ENABLED(CONFIG_LRNG_ENABLE_CONTINUOUS_COMPRESSION); -+ -+#ifdef CONFIG_LRNG_SWITCHABLE_CONTINUOUS_COMPRESSION -+module_param(lrng_irq_continuous_compression, bool, 0444); -+MODULE_PARM_DESC(lrng_irq_continuous_compression, -+ "Perform entropy compression if per-CPU entropy data array is full\n"); -+#endif -+ -+/* -+ * Per-CPU entropy pool with compressed entropy event -+ * -+ * The per-CPU entropy pool is defined as the hash state. New data is simply -+ * inserted into the entropy pool by performing a hash update operation. -+ * To read the entropy pool, a hash final must be invoked. However, before -+ * the entropy pool is released again after a hash final, the hash init must -+ * be performed. -+ */ -+static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_irq_pool) -+ __aligned(LRNG_KCAPI_ALIGN); -+/* -+ * Lock to allow other CPUs to read the pool - as this is only done during -+ * reseed which is infrequent, this lock is hardly contended. -+ */ -+static DEFINE_PER_CPU(spinlock_t, lrng_irq_lock); -+static DEFINE_PER_CPU(bool, lrng_irq_lock_init) = false; -+ -+static bool lrng_irq_pool_online(int cpu) -+{ -+ return per_cpu(lrng_irq_lock_init, cpu); -+} -+ -+static void __init lrng_irq_check_compression_state(void) -+{ -+ /* One pool must hold sufficient entropy for disabled compression */ -+ if (!lrng_irq_continuous_compression) { -+ u32 max_ent = min_t(u32, lrng_get_digestsize(), -+ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, -+ lrng_irq_entropy_bits)); -+ if (max_ent < lrng_security_strength()) { -+ pr_warn("Force continuous compression operation to ensure LRNG can hold enough entropy\n"); -+ lrng_irq_continuous_compression = true; -+ } -+ } -+} -+ -+void __init lrng_irq_es_init(bool highres_timer) -+{ -+ /* Set a minimum number of interrupts that must be collected */ -+ irq_entropy = max_t(u32, LRNG_IRQ_ENTROPY_BITS, irq_entropy); -+ -+ if (highres_timer) { -+ lrng_irq_entropy_bits = irq_entropy; -+ } else { -+ u32 new_entropy = irq_entropy * LRNG_ES_OVERSAMPLING_FACTOR; -+ -+ lrng_irq_entropy_bits = (irq_entropy < new_entropy) ? -+ new_entropy : irq_entropy; -+ pr_warn("operating without high-resolution timer and applying IRQ oversampling factor %u\n", -+ LRNG_ES_OVERSAMPLING_FACTOR); -+ } -+ -+ lrng_irq_check_compression_state(); -+} -+ -+/* -+ * Reset all per-CPU pools - reset entropy estimator but leave the pool data -+ * that may or may not have entropy unchanged. -+ */ -+static void lrng_irq_reset(void) -+{ -+ int cpu; -+ -+ /* Trigger GCD calculation anew. */ -+ lrng_gcd_set(0); -+ -+ for_each_online_cpu(cpu) -+ atomic_set(per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); -+} -+ -+static u32 lrng_irq_avail_pool_size(void) -+{ -+ u32 max_size = 0, max_pool = lrng_get_digestsize(); -+ int cpu; -+ -+ if (!lrng_irq_continuous_compression) -+ max_pool = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); -+ -+ for_each_online_cpu(cpu) { -+ if (lrng_irq_pool_online(cpu)) -+ max_size += max_pool; -+ } -+ -+ return max_size; -+} -+ -+/* Return entropy of unused IRQs present in all per-CPU pools. */ -+static u32 lrng_irq_avail_entropy(u32 __unused) -+{ -+ u32 digestsize_irqs, irq = 0; -+ int cpu; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) -+ return 0; -+ -+ /* Obtain the cap of maximum numbers of IRQs we count */ -+ digestsize_irqs = lrng_entropy_to_data(lrng_get_digestsize(), -+ lrng_irq_entropy_bits); -+ if (!lrng_irq_continuous_compression) { -+ /* Cap to max. number of IRQs the array can hold */ -+ digestsize_irqs = min_t(u32, digestsize_irqs, -+ LRNG_DATA_NUM_VALUES); -+ } -+ -+ for_each_online_cpu(cpu) { -+ if (!lrng_irq_pool_online(cpu)) -+ continue; -+ irq += min_t(u32, digestsize_irqs, -+ atomic_read_u32(per_cpu_ptr(&lrng_irq_array_irqs, -+ cpu))); -+ } -+ -+ /* Consider oversampling rate */ -+ return lrng_reduce_by_osr(lrng_data_to_entropy(irq, -+ lrng_irq_entropy_bits)); -+} -+ -+/* -+ * Trigger a switch of the hash implementation for the per-CPU pool. -+ * -+ * For each per-CPU pool, obtain the message digest with the old hash -+ * implementation, initialize the per-CPU pool again with the new hash -+ * implementation and inject the message digest into the new state. -+ * -+ * Assumption: the caller must guarantee that the new_cb is available during the -+ * entire operation (e.g. it must hold the lock against pointer updating). -+ */ -+static int -+lrng_irq_switch_hash(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ u32 digestsize_irqs, found_irqs; -+ int ret = 0, cpu; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ for_each_online_cpu(cpu) { -+ struct shash_desc *pcpu_shash; -+ -+ /* -+ * Only switch the per-CPU pools for the current node because -+ * the hash_cb only applies NUMA-node-wide. -+ */ -+ if (cpu_to_node(cpu) != node || !lrng_irq_pool_online(cpu)) -+ continue; -+ -+ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, -+ cpu); -+ -+ digestsize_irqs = old_cb->hash_digestsize(pcpu_shash); -+ digestsize_irqs = lrng_entropy_to_data(digestsize_irqs << 3, -+ lrng_irq_entropy_bits); -+ -+ if (pcpu_shash->tfm == new_hash) -+ continue; -+ -+ /* Get the per-CPU pool hash with old digest ... */ -+ ret = old_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash with the new digest ... */ -+ new_cb->hash_init(pcpu_shash, new_hash) ?: -+ /* -+ * ... feed the old hash into the new state. We may feed -+ * uninitialized memory into the new state, but this is -+ * considered no issue and even good as we have some more -+ * uncertainty here. -+ */ -+ new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); -+ if (ret) -+ goto out; -+ -+ /* -+ * In case the new digest is larger than the old one, cap -+ * the available entropy to the old message digest used to -+ * process the existing data. -+ */ -+ found_irqs = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); -+ found_irqs = min_t(u32, found_irqs, digestsize_irqs); -+ atomic_add_return_relaxed(found_irqs, -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu)); -+ -+ pr_debug("Re-initialize per-CPU interrupt entropy pool for CPU %d on NUMA node %d with hash %s\n", -+ cpu, node, new_cb->hash_name()); -+ } -+ -+out: -+ memzero_explicit(digest, sizeof(digest)); -+ return ret; -+} -+ -+/* -+ * When reading the per-CPU message digest, make sure we use the crypto -+ * callbacks defined for the NUMA node the per-CPU pool is defined for because -+ * the LRNG crypto switch support is only atomic per NUMA node. -+ */ -+static u32 -+lrng_irq_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, -+ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) -+{ -+ struct shash_desc *pcpu_shash = -+ (struct shash_desc *)per_cpu_ptr(lrng_irq_pool, cpu); -+ spinlock_t *lock = per_cpu_ptr(&lrng_irq_lock, cpu); -+ unsigned long flags; -+ u32 digestsize_irqs, found_irqs; -+ -+ /* Lock guarding against reading / writing to per-CPU pool */ -+ spin_lock_irqsave(lock, flags); -+ -+ *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); -+ digestsize_irqs = lrng_entropy_to_data(*digestsize << 3, -+ lrng_irq_entropy_bits); -+ -+ /* Obtain entropy statement like for the entropy pool */ -+ found_irqs = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu), 0); -+ /* Cap to maximum amount of data we can hold in hash */ -+ found_irqs = min_t(u32, found_irqs, digestsize_irqs); -+ -+ /* Cap to maximum amount of data we can hold in array */ -+ if (!lrng_irq_continuous_compression) -+ found_irqs = min_t(u32, found_irqs, LRNG_DATA_NUM_VALUES); -+ -+ /* Store all not-yet compressed data in data array into hash, ... */ -+ if (pcpu_hash_cb->hash_update(pcpu_shash, -+ (u8 *)per_cpu_ptr(lrng_irq_array, cpu), -+ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: -+ /* ... get the per-CPU pool digest, ... */ -+ pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash, ... */ -+ pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: -+ /* ... feed the old hash into the new state. */ -+ pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) -+ found_irqs = 0; -+ -+ spin_unlock_irqrestore(lock, flags); -+ return found_irqs; -+} -+ -+/* -+ * Hash all per-CPU pools and return the digest to be used as seed data for -+ * seeding a DRNG. The caller must guarantee backtracking resistance. -+ * The function will only copy as much data as entropy is available into the -+ * caller-provided output buffer. -+ * -+ * This function handles the translation from the number of received interrupts -+ * into an entropy statement. The conversion depends on LRNG_IRQ_ENTROPY_BITS -+ * which defines how many interrupts must be received to obtain 256 bits of -+ * entropy. With this value, the function lrng_data_to_entropy converts a given -+ * data size (received interrupts, requested amount of data, etc.) into an -+ * entropy statement. lrng_entropy_to_data does the reverse. -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: Requested amount of entropy -+ * @fully_seeded: indicator whether LRNG is fully seeded -+ */ -+static void lrng_irq_pool_hash(struct entropy_buf *eb, u32 requested_bits, -+ bool fully_seeded) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb; -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ unsigned long flags, flags2; -+ u32 found_irqs, collected_irqs = 0, collected_ent_bits, requested_irqs, -+ returned_ent_bits; -+ int ret, cpu; -+ void *hash; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) { -+ eb->e_bits[lrng_int_es_irq] = 0; -+ return; -+ } -+ -+ /* Lock guarding replacement of per-NUMA hash */ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ /* The hash state of filled with all per-CPU pool hashes. */ -+ ret = hash_cb->hash_init(shash, hash); -+ if (ret) -+ goto err; -+ -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); -+ requested_irqs = lrng_entropy_to_data(requested_bits + -+ lrng_compress_osr(), -+ lrng_irq_entropy_bits); -+ -+ /* -+ * Harvest entropy from each per-CPU hash state - even though we may -+ * have collected sufficient entropy, we will hash all per-CPU pools. -+ */ -+ for_each_online_cpu(cpu) { -+ struct lrng_drng *pcpu_drng = drng; -+ u32 digestsize, pcpu_unused_irqs = 0; -+ int node = cpu_to_node(cpu); -+ -+ /* If pool is not online, then no entropy is present. */ -+ if (!lrng_irq_pool_online(cpu)) -+ continue; -+ -+ if (lrng_drng && lrng_drng[node]) -+ pcpu_drng = lrng_drng[node]; -+ -+ if (pcpu_drng == drng) { -+ found_irqs = lrng_irq_pool_hash_one(hash_cb, hash, -+ cpu, digest, -+ &digestsize); -+ } else { -+ read_lock_irqsave(&pcpu_drng->hash_lock, flags2); -+ found_irqs = -+ lrng_irq_pool_hash_one(pcpu_drng->hash_cb, -+ pcpu_drng->hash, cpu, -+ digest, &digestsize); -+ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); -+ } -+ -+ /* Inject the digest into the state of all per-CPU pools */ -+ ret = hash_cb->hash_update(shash, digest, digestsize); -+ if (ret) -+ goto err; -+ -+ collected_irqs += found_irqs; -+ if (collected_irqs > requested_irqs) { -+ pcpu_unused_irqs = collected_irqs - requested_irqs; -+ atomic_add_return_relaxed(pcpu_unused_irqs, -+ per_cpu_ptr(&lrng_irq_array_irqs, cpu)); -+ collected_irqs = requested_irqs; -+ } -+ pr_debug("%u interrupts used from entropy pool of CPU %d, %u interrupts remain unused\n", -+ found_irqs - pcpu_unused_irqs, cpu, pcpu_unused_irqs); -+ } -+ -+ ret = hash_cb->hash_final(shash, digest); -+ if (ret) -+ goto err; -+ -+ collected_ent_bits = lrng_data_to_entropy(collected_irqs, -+ lrng_irq_entropy_bits); -+ /* Apply oversampling: discount requested oversampling rate */ -+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); -+ -+ pr_debug("obtained %u bits by collecting %u bits of entropy from entropy pool noise source\n", -+ returned_ent_bits, collected_ent_bits); -+ -+ /* -+ * Truncate to available entropy as implicitly allowed by SP800-90B -+ * section 3.1.5.1.1 table 1 which awards truncated hashes full -+ * entropy. -+ * -+ * During boot time, we read requested_bits data with -+ * returned_ent_bits entropy. In case our conservative entropy -+ * estimate underestimates the available entropy we can transport as -+ * much available entropy as possible. -+ */ -+ memcpy(eb->e[lrng_int_es_irq], digest, -+ fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); -+ eb->e_bits[lrng_int_es_irq] = returned_ent_bits; -+ -+out: -+ hash_cb->hash_desc_zero(shash); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ memzero_explicit(digest, sizeof(digest)); -+ return; -+ -+err: -+ eb->e_bits[lrng_int_es_irq] = 0; -+ goto out; -+} -+ -+/* Compress the lrng_irq_array array into lrng_irq_pool */ -+static void lrng_irq_array_compress(void) -+{ -+ struct shash_desc *shash = -+ (struct shash_desc *)this_cpu_ptr(lrng_irq_pool); -+ struct lrng_drng *drng = lrng_drng_node_instance(); -+ const struct lrng_hash_cb *hash_cb; -+ spinlock_t *lock = this_cpu_ptr(&lrng_irq_lock); -+ unsigned long flags, flags2; -+ void *hash; -+ bool init = false; -+ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ if (unlikely(!this_cpu_read(lrng_irq_lock_init))) { -+ init = true; -+ spin_lock_init(lock); -+ this_cpu_write(lrng_irq_lock_init, true); -+ pr_debug("Initializing per-CPU entropy pool for CPU %d on NUMA node %d with hash %s\n", -+ raw_smp_processor_id(), numa_node_id(), -+ hash_cb->hash_name()); -+ } -+ -+ spin_lock_irqsave(lock, flags2); -+ -+ if (unlikely(init) && hash_cb->hash_init(shash, hash)) { -+ this_cpu_write(lrng_irq_lock_init, false); -+ pr_warn("Initialization of hash failed\n"); -+ } else if (lrng_irq_continuous_compression) { -+ /* Add entire per-CPU data array content into entropy pool. */ -+ if (hash_cb->hash_update(shash, -+ (u8 *)this_cpu_ptr(lrng_irq_array), -+ LRNG_DATA_ARRAY_SIZE * sizeof(u32))) -+ pr_warn_ratelimited("Hashing of entropy data failed\n"); -+ } -+ -+ spin_unlock_irqrestore(lock, flags2); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+} -+ -+/* Compress data array into hash */ -+static void lrng_irq_array_to_hash(u32 ptr) -+{ -+ u32 *array = this_cpu_ptr(lrng_irq_array); -+ -+ /* -+ * During boot time the hash operation is triggered more often than -+ * during regular operation. -+ */ -+ if (unlikely(!lrng_state_fully_seeded())) { -+ if ((ptr & 31) && (ptr < LRNG_DATA_WORD_MASK)) -+ return; -+ } else if (ptr < LRNG_DATA_WORD_MASK) { -+ return; -+ } -+ -+ if (lrng_raw_array_entropy_store(*array)) { -+ u32 i; -+ -+ /* -+ * If we fed even a part of the array to external analysis, we -+ * mark that the entire array and the per-CPU pool to have no -+ * entropy. This is due to the non-IID property of the data as -+ * we do not fully know whether the existing dependencies -+ * diminish the entropy beyond to what we expect it has. -+ */ -+ atomic_set(this_cpu_ptr(&lrng_irq_array_irqs), 0); -+ -+ for (i = 1; i < LRNG_DATA_ARRAY_SIZE; i++) -+ lrng_raw_array_entropy_store(*(array + i)); -+ } else { -+ lrng_irq_array_compress(); -+ /* Ping pool handler about received entropy */ -+ if (lrng_sp80090b_startup_complete_es(lrng_int_es_irq)) -+ lrng_es_add_entropy(); -+ } -+} -+ -+/* -+ * Concatenate full 32 bit word at the end of time array even when current -+ * ptr is not aligned to sizeof(data). -+ */ -+static void _lrng_irq_array_add_u32(u32 data) -+{ -+ /* Increment pointer by number of slots taken for input value */ -+ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_irq_array_ptr, -+ LRNG_DATA_SLOTS_PER_UINT); -+ unsigned int pre_array; -+ -+ /* -+ * This function injects a unit into the array - guarantee that -+ * array unit size is equal to data type of input data. -+ */ -+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != (sizeof(data) << 3)); -+ -+ /* -+ * The following logic requires at least two units holding -+ * the data as otherwise the pointer would immediately wrap when -+ * injection an u32 word. -+ */ -+ BUILD_BUG_ON(LRNG_DATA_NUM_VALUES <= LRNG_DATA_SLOTS_PER_UINT); -+ -+ lrng_data_split_u32(&ptr, &pre_ptr, &mask); -+ -+ /* MSB of data go into previous unit */ -+ pre_array = lrng_data_idx2array(pre_ptr); -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_irq_array[pre_array], ~(0xffffffff & ~mask)); -+ this_cpu_or(lrng_irq_array[pre_array], data & ~mask); -+ -+ /* Invoke compression as we just filled data array completely */ -+ if (unlikely(pre_ptr > ptr)) -+ lrng_irq_array_to_hash(LRNG_DATA_WORD_MASK); -+ -+ /* LSB of data go into current unit */ -+ this_cpu_write(lrng_irq_array[lrng_data_idx2array(ptr)], -+ data & mask); -+ -+ if (likely(pre_ptr <= ptr)) -+ lrng_irq_array_to_hash(ptr); -+} -+ -+/* Concatenate a 32-bit word at the end of the per-CPU array */ -+void lrng_irq_array_add_u32(u32 data) -+{ -+ /* -+ * Disregard entropy-less data without continuous compression to -+ * avoid it overwriting data with entropy when array ptr wraps. -+ */ -+ if (lrng_irq_continuous_compression) -+ _lrng_irq_array_add_u32(data); -+} -+ -+/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ -+static void lrng_irq_array_add_slot(u32 data) -+{ -+ /* Get slot */ -+ u32 ptr = this_cpu_inc_return(lrng_irq_array_ptr) & -+ LRNG_DATA_WORD_MASK; -+ unsigned int array = lrng_data_idx2array(ptr); -+ unsigned int slot = lrng_data_idx2slot(ptr); -+ -+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS % LRNG_DATA_SLOTSIZE_BITS); -+ /* Ensure consistency of values */ -+ BUILD_BUG_ON(LRNG_DATA_ARRAY_MEMBER_BITS != -+ sizeof(lrng_irq_array[0]) << 3); -+ -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_irq_array[array], -+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, -+ slot))); -+ /* Store data into slot */ -+ this_cpu_or(lrng_irq_array[array], lrng_data_slot_val(data, slot)); -+ -+ lrng_irq_array_to_hash(ptr); -+} -+ -+static void -+lrng_time_process_common(u32 time, void(*add_time)(u32 data)) -+{ -+ enum lrng_health_res health_test; -+ -+ if (lrng_raw_hires_entropy_store(time)) -+ return; -+ -+ health_test = lrng_health_test(time, lrng_int_es_irq); -+ if (health_test > lrng_health_fail_use) -+ return; -+ -+ if (health_test == lrng_health_pass) -+ atomic_inc_return(this_cpu_ptr(&lrng_irq_array_irqs)); -+ -+ add_time(time); -+} -+ -+/* -+ * Batching up of entropy in per-CPU array before injecting into entropy pool. -+ */ -+static void lrng_time_process(void) -+{ -+ u32 now_time = random_get_entropy(); -+ -+ if (unlikely(!lrng_gcd_tested())) { -+ /* When GCD is unknown, we process the full time stamp */ -+ lrng_time_process_common(now_time, _lrng_irq_array_add_u32); -+ lrng_gcd_add_value(now_time); -+ } else { -+ /* GCD is known and applied */ -+ lrng_time_process_common((now_time / lrng_gcd_get()) & -+ LRNG_DATA_SLOTSIZE_MASK, -+ lrng_irq_array_add_slot); -+ } -+ -+ lrng_perf_time(now_time); -+} -+ -+/* Hot code path - Callback for interrupt handler */ -+void add_interrupt_randomness(int irq) -+{ -+ if (lrng_highres_timer()) { -+ lrng_time_process(); -+ } else { -+ struct pt_regs *regs = get_irq_regs(); -+ static atomic_t reg_idx = ATOMIC_INIT(0); -+ u64 ip; -+ u32 tmp; -+ -+ if (regs) { -+ u32 *ptr = (u32 *)regs; -+ int reg_ptr = atomic_add_return_relaxed(1, ®_idx); -+ size_t n = (sizeof(struct pt_regs) / sizeof(u32)); -+ -+ ip = instruction_pointer(regs); -+ tmp = *(ptr + (reg_ptr % n)); -+ tmp = lrng_raw_regs_entropy_store(tmp) ? 0 : tmp; -+ _lrng_irq_array_add_u32(tmp); -+ } else { -+ ip = _RET_IP_; -+ } -+ -+ lrng_time_process(); -+ -+ /* -+ * The XOR operation combining the different values is not -+ * considered to destroy entropy since the entirety of all -+ * processed values delivers the entropy (and not each -+ * value separately of the other values). -+ */ -+ tmp = lrng_raw_jiffies_entropy_store(jiffies) ? 0 : jiffies; -+ tmp ^= lrng_raw_irq_entropy_store(irq) ? 0 : irq; -+ tmp ^= lrng_raw_retip_entropy_store(ip) ? 0 : ip; -+ tmp ^= ip >> 32; -+ _lrng_irq_array_add_u32(tmp); -+ } -+} -+EXPORT_SYMBOL(add_interrupt_randomness); -+ -+static void lrng_irq_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for operating entropy pool: %s\n" -+ " Available entropy: %u\n" -+ " per-CPU interrupt collection size: %u\n" -+ " Standards compliance: %s\n" -+ " High-resolution timer: %s\n" -+ " Continuous compression: %s\n", -+ lrng_drng_init->hash_cb->hash_name(), -+ lrng_irq_avail_entropy(0), -+ LRNG_DATA_NUM_VALUES, -+ lrng_sp80090b_compliant(lrng_int_es_irq) ? "SP800-90B " : "", -+ lrng_highres_timer() ? "true" : "false", -+ lrng_irq_continuous_compression ? "true" : "false"); -+} -+ -+struct lrng_es_cb lrng_es_irq = { -+ .name = "IRQ", -+ .get_ent = lrng_irq_pool_hash, -+ .curr_entropy = lrng_irq_avail_entropy, -+ .max_entropy = lrng_irq_avail_pool_size, -+ .state = lrng_irq_es_state, -+ .reset = lrng_irq_reset, -+ .switch_hash = lrng_irq_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_irq.h b/drivers/char/lrng/lrng_es_irq.h -new file mode 100644 -index 000000000000..2cd746611cf0 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_irq.h -@@ -0,0 +1,24 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_IRQ_H -+#define _LRNG_ES_IRQ_H -+ -+#include -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_IRQ -+void lrng_irq_es_init(bool highres_timer); -+void lrng_irq_array_add_u32(u32 data); -+ -+extern struct lrng_es_cb lrng_es_irq; -+ -+#else /* CONFIG_LRNG_IRQ */ -+static inline void lrng_irq_es_init(bool highres_timer) { } -+static inline void lrng_irq_array_add_u32(u32 data) { } -+#endif /* CONFIG_LRNG_IRQ */ -+ -+#endif /* _LRNG_ES_IRQ_H */ -diff --git a/drivers/char/lrng/lrng_es_jent.c b/drivers/char/lrng/lrng_es_jent.c -new file mode 100644 -index 000000000000..dbb2fce591f0 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_jent.c -@@ -0,0 +1,136 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Fast Entropy Source: Jitter RNG -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_jent.h" -+ -+/* -+ * Estimated entropy of data is a 16th of LRNG_DRNG_SECURITY_STRENGTH_BITS. -+ * Albeit a full entropy assessment is provided for the noise source indicating -+ * that it provides high entropy rates and considering that it deactivates -+ * when it detects insufficient hardware, the chosen under estimation of -+ * entropy is considered to be acceptable to all reviewers. -+ */ -+static u32 jent_entropy = CONFIG_LRNG_JENT_ENTROPY_RATE; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(jent_entropy, uint, 0644); -+MODULE_PARM_DESC(jent_entropy, "Entropy in bits of 256 data bits from Jitter RNG noise source"); -+#endif -+ -+static bool lrng_jent_initialized = false; -+static struct rand_data *lrng_jent_state; -+ -+static int __init lrng_jent_initialize(void) -+{ -+ /* Initialize the Jitter RNG after the clocksources are initialized. */ -+ if (jent_entropy_init() || -+ (lrng_jent_state = jent_entropy_collector_alloc(1, 0)) == NULL) { -+ jent_entropy = 0; -+ pr_info("Jitter RNG unusable on current system\n"); -+ return 0; -+ } -+ lrng_jent_initialized = true; -+ pr_debug("Jitter RNG working on current system\n"); -+ -+ /* -+ * In FIPS mode, the Jitter RNG is defined to have full of entropy -+ * unless a different value has been specified at the command line -+ * (i.e. the user overrides the default), and the default value is -+ * larger than zero (if it is zero, it is assumed that an RBG2(P) or -+ * RBG2(NP) construction is attempted that intends to exclude the -+ * Jitter RNG). -+ */ -+ if (fips_enabled && -+ CONFIG_LRNG_JENT_ENTROPY_RATE > 0 && -+ jent_entropy == CONFIG_LRNG_JENT_ENTROPY_RATE) -+ jent_entropy = LRNG_DRNG_SECURITY_STRENGTH_BITS; -+ -+ lrng_drng_force_reseed(); -+ if (jent_entropy) -+ lrng_es_add_entropy(); -+ -+ return 0; -+} -+device_initcall(lrng_jent_initialize); -+ -+static u32 lrng_jent_entropylevel(u32 requested_bits) -+{ -+ return lrng_fast_noise_entropylevel(lrng_jent_initialized ? -+ jent_entropy : 0, requested_bits); -+} -+ -+static u32 lrng_jent_poolsize(void) -+{ -+ return lrng_jent_entropylevel(lrng_security_strength()); -+} -+ -+/* -+ * lrng_get_jent() - Get Jitter RNG entropy -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: requested entropy in bits -+ */ -+static void lrng_jent_get(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ int ret; -+ u32 ent_bits = lrng_jent_entropylevel(requested_bits); -+ unsigned long flags; -+ static DEFINE_SPINLOCK(lrng_jent_lock); -+ -+ spin_lock_irqsave(&lrng_jent_lock, flags); -+ -+ if (!lrng_jent_initialized) { -+ spin_unlock_irqrestore(&lrng_jent_lock, flags); -+ goto err; -+ } -+ -+ ret = jent_read_entropy(lrng_jent_state, eb->e[lrng_ext_es_jitter], -+ requested_bits >> 3); -+ spin_unlock_irqrestore(&lrng_jent_lock, flags); -+ -+ if (ret) { -+ pr_debug("Jitter RNG failed with %d\n", ret); -+ goto err; -+ } -+ -+ pr_debug("obtained %u bits of entropy from Jitter RNG noise source\n", -+ ent_bits); -+ -+ eb->e_bits[lrng_ext_es_jitter] = ent_bits; -+ return; -+ -+err: -+ eb->e_bits[lrng_ext_es_jitter] = 0; -+} -+ -+static void lrng_jent_es_state(unsigned char *buf, size_t buflen) -+{ -+ snprintf(buf, buflen, -+ " Available entropy: %u\n" -+ " Enabled: %s\n", -+ lrng_jent_poolsize(), -+ lrng_jent_initialized ? "true" : "false"); -+} -+ -+struct lrng_es_cb lrng_es_jent = { -+ .name = "JitterRNG", -+ .get_ent = lrng_jent_get, -+ .curr_entropy = lrng_jent_entropylevel, -+ .max_entropy = lrng_jent_poolsize, -+ .state = lrng_jent_es_state, -+ .reset = NULL, -+ .switch_hash = NULL, -+}; -diff --git a/drivers/char/lrng/lrng_es_jent.h b/drivers/char/lrng/lrng_es_jent.h -new file mode 100644 -index 000000000000..32882d4bdf99 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_jent.h -@@ -0,0 +1,17 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_JENT_H -+#define _LRNG_ES_JENT_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_JENT -+ -+extern struct lrng_es_cb lrng_es_jent; -+ -+#endif /* CONFIG_LRNG_JENT */ -+ -+#endif /* _LRNG_ES_JENT_H */ -diff --git a/drivers/char/lrng/lrng_es_krng.c b/drivers/char/lrng/lrng_es_krng.c -new file mode 100644 -index 000000000000..519ba640cc75 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_krng.c -@@ -0,0 +1,100 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Fast Entropy Source: Linux kernel RNG (random.c) -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_krng.h" -+ -+static u32 krng_entropy = CONFIG_LRNG_KERNEL_RNG_ENTROPY_RATE; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(krng_entropy, uint, 0644); -+MODULE_PARM_DESC(krng_entropy, "Entropy in bits of 256 data bits from the kernel RNG noise source"); -+#endif -+ -+static atomic_t lrng_krng_initial_rate = ATOMIC_INIT(0); -+ -+static u32 lrng_krng_fips_entropylevel(u32 entropylevel) -+{ -+ return fips_enabled ? 0 : entropylevel; -+} -+ -+static int lrng_krng_adjust_entropy(void) -+{ -+ u32 entropylevel; -+ -+ krng_entropy = atomic_read_u32(&lrng_krng_initial_rate); -+ -+ entropylevel = lrng_krng_fips_entropylevel(krng_entropy); -+ pr_debug("Kernel RNG is fully seeded, setting entropy rate to %u bits of entropy\n", -+ entropylevel); -+ lrng_drng_force_reseed(); -+ if (entropylevel) -+ lrng_es_add_entropy(); -+ return 0; -+} -+ -+static u32 lrng_krng_entropylevel(u32 requested_bits) -+{ -+ static bool init = false; -+ -+ if (unlikely(!init) && rng_is_initialized()) { -+ init = true; -+ lrng_krng_adjust_entropy(); -+ } -+ -+ return lrng_fast_noise_entropylevel( -+ lrng_krng_fips_entropylevel(krng_entropy), requested_bits); -+} -+ -+static u32 lrng_krng_poolsize(void) -+{ -+ return lrng_krng_entropylevel(lrng_security_strength()); -+} -+ -+/* -+ * lrng_krng_get() - Get kernel RNG entropy -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: requested entropy in bits -+ */ -+static void lrng_krng_get(struct entropy_buf *eb, u32 requested_bits, -+ bool __unused) -+{ -+ u32 ent_bits = lrng_krng_entropylevel(requested_bits); -+ -+ get_random_bytes(eb->e[lrng_ext_es_krng], requested_bits >> 3); -+ -+ pr_debug("obtained %u bits of entropy from kernel RNG noise source\n", -+ ent_bits); -+ -+ eb->e_bits[lrng_ext_es_krng] = ent_bits; -+} -+ -+static void lrng_krng_es_state(unsigned char *buf, size_t buflen) -+{ -+ snprintf(buf, buflen, -+ " Available entropy: %u\n" -+ " Entropy Rate per 256 data bits: %u\n", -+ lrng_krng_poolsize(), -+ lrng_krng_entropylevel(256)); -+} -+ -+struct lrng_es_cb lrng_es_krng = { -+ .name = "KernelRNG", -+ .get_ent = lrng_krng_get, -+ .curr_entropy = lrng_krng_entropylevel, -+ .max_entropy = lrng_krng_poolsize, -+ .state = lrng_krng_es_state, -+ .reset = NULL, -+ .switch_hash = NULL, -+}; -diff --git a/drivers/char/lrng/lrng_es_krng.h b/drivers/char/lrng/lrng_es_krng.h -new file mode 100644 -index 000000000000..cf982b9eea05 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_krng.h -@@ -0,0 +1,17 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_RANDOM_H -+#define _LRNG_ES_RANDOM_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_KERNEL_RNG -+ -+extern struct lrng_es_cb lrng_es_krng; -+ -+#endif /* CONFIG_LRNG_KERNEL_RNG */ -+ -+#endif /* _LRNG_ES_RANDOM_H */ -diff --git a/drivers/char/lrng/lrng_es_mgr.c b/drivers/char/lrng/lrng_es_mgr.c -new file mode 100644 -index 000000000000..23eba26b9f45 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_mgr.c -@@ -0,0 +1,460 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Entropy sources management -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_cpu.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_jent.h" -+#include "lrng_es_krng.h" -+#include "lrng_es_mgr.h" -+#include "lrng_es_sched.h" -+#include "lrng_interface_dev_common.h" -+#include "lrng_interface_random_kernel.h" -+ -+struct lrng_state { -+ bool can_invalidate; /* Can invalidate batched entropy? */ -+ bool perform_seedwork; /* Can seed work be performed? */ -+ bool lrng_operational; /* Is DRNG operational? */ -+ bool lrng_fully_seeded; /* Is DRNG fully seeded? */ -+ bool lrng_min_seeded; /* Is DRNG minimally seeded? */ -+ bool all_online_numa_node_seeded;/* All NUMA DRNGs seeded? */ -+ -+ /* -+ * To ensure that external entropy providers cannot dominate the -+ * internal noise sources but yet cannot be dominated by internal -+ * noise sources, the following booleans are intended to allow -+ * external to provide seed once when a DRNG reseed occurs. This -+ * triggering of external noise source is performed even when the -+ * entropy pool has sufficient entropy. -+ */ -+ -+ atomic_t boot_entropy_thresh; /* Reseed threshold */ -+ struct mutex reseed_in_progress; /* Flag for on executing reseed */ -+ struct work_struct lrng_seed_work; /* (re)seed work queue */ -+}; -+ -+static struct lrng_state lrng_state = { -+ false, false, false, false, false, false, -+ .boot_entropy_thresh = ATOMIC_INIT(LRNG_INIT_ENTROPY_BITS), -+ .reseed_in_progress = -+ __MUTEX_INITIALIZER(lrng_state.reseed_in_progress), -+}; -+ -+/* -+ * If the entropy count falls under this number of bits, then we -+ * should wake up processes which are selecting or polling on write -+ * access to /dev/random. -+ */ -+u32 lrng_write_wakeup_bits = (LRNG_WRITE_WAKEUP_ENTROPY << 3); -+ -+/* -+ * The entries must be in the same order as defined by enum lrng_internal_es and -+ * enum lrng_external_es -+ */ -+struct lrng_es_cb *lrng_es[] = { -+#ifdef CONFIG_LRNG_IRQ -+ &lrng_es_irq, -+#endif -+#ifdef CONFIG_LRNG_SCHED -+ &lrng_es_sched, -+#endif -+#ifdef CONFIG_LRNG_JENT -+ &lrng_es_jent, -+#endif -+#ifdef CONFIG_LRNG_CPU -+ &lrng_es_cpu, -+#endif -+#ifdef CONFIG_LRNG_KERNEL_RNG -+ &lrng_es_krng, -+#endif -+ &lrng_es_aux -+}; -+ -+/********************************** Helper ***********************************/ -+ -+void lrng_debug_report_seedlevel(const char *name) -+{ -+#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM -+ static void *previous = NULL; -+ void *caller = (void *) _RET_IP_; -+ -+ if (READ_ONCE(previous) == caller) -+ return; -+ -+ if (!lrng_state_min_seeded()) -+ pr_notice("%pS %s called without reaching minimally seeded level (available entropy %u)\n", -+ caller, name, lrng_avail_entropy()); -+ -+ WRITE_ONCE(previous, caller); -+#endif -+} -+ -+/* -+ * Reading of the LRNG pool is only allowed by one caller. The reading is -+ * only performed to (re)seed DRNGs. Thus, if this "lock" is already taken, -+ * the reseeding operation is in progress. The caller is not intended to wait -+ * but continue with its other operation. -+ */ -+int lrng_pool_trylock(void) -+{ -+ return mutex_trylock(&lrng_state.reseed_in_progress); -+} -+ -+void lrng_pool_lock(void) -+{ -+ mutex_lock(&lrng_state.reseed_in_progress); -+} -+ -+void lrng_pool_unlock(void) -+{ -+ mutex_unlock(&lrng_state.reseed_in_progress); -+} -+ -+/* Set new entropy threshold for reseeding during boot */ -+void lrng_set_entropy_thresh(u32 new_entropy_bits) -+{ -+ atomic_set(&lrng_state.boot_entropy_thresh, new_entropy_bits); -+} -+ -+/* -+ * Reset LRNG state - the entropy counters are reset, but the data that may -+ * or may not have entropy remains in the pools as this data will not hurt. -+ */ -+void lrng_reset_state(void) -+{ -+ u32 i; -+ -+ for_each_lrng_es(i) { -+ if (lrng_es[i]->reset) -+ lrng_es[i]->reset(); -+ } -+ lrng_state.lrng_operational = false; -+ lrng_state.lrng_fully_seeded = false; -+ lrng_state.lrng_min_seeded = false; -+ lrng_state.all_online_numa_node_seeded = false; -+ pr_debug("reset LRNG\n"); -+} -+ -+/* Set flag that all DRNGs are fully seeded */ -+void lrng_pool_all_numa_nodes_seeded(bool set) -+{ -+ lrng_state.all_online_numa_node_seeded = set; -+ if (set) -+ wake_up_all(&lrng_init_wait); -+} -+ -+bool lrng_pool_all_numa_nodes_seeded_get(void) -+{ -+ return lrng_state.all_online_numa_node_seeded; -+} -+ -+/* Return boolean whether LRNG reached minimally seed level */ -+bool lrng_state_min_seeded(void) -+{ -+ return lrng_state.lrng_min_seeded; -+} -+ -+/* Return boolean whether LRNG reached fully seed level */ -+bool lrng_state_fully_seeded(void) -+{ -+ return lrng_state.lrng_fully_seeded; -+} -+ -+/* Return boolean whether LRNG is considered fully operational */ -+bool lrng_state_operational(void) -+{ -+ return lrng_state.lrng_operational; -+} -+ -+static void lrng_init_wakeup(void) -+{ -+ wake_up_all(&lrng_init_wait); -+ lrng_init_wakeup_dev(); -+} -+ -+bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy) -+{ -+ return (collected_entropy >= lrng_get_seed_entropy_osr(fully_seeded)); -+} -+ -+u32 lrng_entropy_rate_eb(struct entropy_buf *eb) -+{ -+ u32 i, collected_entropy = 0; -+ -+ for_each_lrng_es(i) -+ collected_entropy += eb->e_bits[i]; -+ -+ return collected_entropy; -+} -+ -+/* Mark one DRNG as not fully seeded */ -+void lrng_unset_fully_seeded(struct lrng_drng *drng) -+{ -+ drng->fully_seeded = false; -+ lrng_pool_all_numa_nodes_seeded(false); -+ -+ /* -+ * The init DRNG instance must always be fully seeded as this instance -+ * is the fall-back if any of the per-NUMA node DRNG instances is -+ * insufficiently seeded. Thus, we mark the entire LRNG as -+ * non-operational if the initial DRNG becomes not fully seeded. -+ */ -+ if (drng == lrng_drng_init_instance() && lrng_state_operational()) { -+ pr_debug("LRNG set to non-operational\n"); -+ lrng_state.lrng_operational = false; -+ lrng_state.lrng_fully_seeded = false; -+ -+ /* If sufficient entropy is available, reseed now. */ -+ lrng_es_add_entropy(); -+ } -+} -+ -+/* Policy to enable LRNG operational mode */ -+static void lrng_set_operational(void) -+{ -+ /* -+ * LRNG is operational if the initial DRNG is fully seeded. This state -+ * can only occur if either the external entropy sources provided -+ * sufficient entropy, or the SP800-90B startup test completed for -+ * the internal ES to supply also entropy data. -+ */ -+ if (lrng_state.lrng_fully_seeded) { -+ lrng_state.lrng_operational = true; -+ lrng_init_wakeup(); -+ pr_info("LRNG fully operational\n"); -+ } -+} -+ -+static u32 lrng_avail_entropy_thresh(void) -+{ -+ u32 ent_thresh = lrng_security_strength(); -+ -+ /* -+ * Apply oversampling during initialization according to SP800-90C as -+ * we request a larger buffer from the ES. -+ */ -+ if (lrng_sp80090c_compliant() && -+ !lrng_state.all_online_numa_node_seeded) -+ ent_thresh += LRNG_SEED_BUFFER_INIT_ADD_BITS; -+ -+ return ent_thresh; -+} -+ -+/* Available entropy in the entire LRNG considering all entropy sources */ -+u32 lrng_avail_entropy(void) -+{ -+ u32 i, ent = 0, ent_thresh = lrng_avail_entropy_thresh(); -+ -+ BUILD_BUG_ON(ARRAY_SIZE(lrng_es) != lrng_ext_es_last); -+ for_each_lrng_es(i) -+ ent += lrng_es[i]->curr_entropy(ent_thresh); -+ return ent; -+} -+ -+u32 lrng_avail_entropy_aux(void) -+{ -+ u32 ent_thresh = lrng_avail_entropy_thresh(); -+ -+ return lrng_es[lrng_ext_es_aux]->curr_entropy(ent_thresh); -+} -+ -+/* -+ * lrng_init_ops() - Set seed stages of LRNG -+ * -+ * Set the slow noise source reseed trigger threshold. The initial threshold -+ * is set to the minimum data size that can be read from the pool: a word. Upon -+ * reaching this value, the next seed threshold of 128 bits is set followed -+ * by 256 bits. -+ * -+ * @eb: buffer containing the size of entropy currently injected into DRNG - if -+ * NULL, the function obtains the available entropy from the ES. -+ */ -+void lrng_init_ops(struct entropy_buf *eb) -+{ -+ struct lrng_state *state = &lrng_state; -+ u32 i, requested_bits, seed_bits = 0; -+ -+ if (state->lrng_operational) -+ return; -+ -+ requested_bits = lrng_get_seed_entropy_osr( -+ state->all_online_numa_node_seeded); -+ -+ if (eb) { -+ for_each_lrng_es(i) -+ seed_bits += eb->e_bits[i]; -+ } else { -+ u32 ent_thresh = lrng_avail_entropy_thresh(); -+ -+ for_each_lrng_es(i) -+ seed_bits += lrng_es[i]->curr_entropy(ent_thresh); -+ } -+ -+ /* DRNG is seeded with full security strength */ -+ if (state->lrng_fully_seeded) { -+ lrng_set_operational(); -+ lrng_set_entropy_thresh(requested_bits); -+ } else if (lrng_fully_seeded(state->all_online_numa_node_seeded, -+ seed_bits)) { -+ if (state->can_invalidate) -+ invalidate_batched_entropy(); -+ -+ state->lrng_fully_seeded = true; -+ lrng_set_operational(); -+ state->lrng_min_seeded = true; -+ pr_info("LRNG fully seeded with %u bits of entropy\n", -+ seed_bits); -+ lrng_set_entropy_thresh(requested_bits); -+ } else if (!state->lrng_min_seeded) { -+ -+ /* DRNG is seeded with at least 128 bits of entropy */ -+ if (seed_bits >= LRNG_MIN_SEED_ENTROPY_BITS) { -+ if (state->can_invalidate) -+ invalidate_batched_entropy(); -+ -+ state->lrng_min_seeded = true; -+ pr_info("LRNG minimally seeded with %u bits of entropy\n", -+ seed_bits); -+ lrng_set_entropy_thresh(requested_bits); -+ lrng_init_wakeup(); -+ -+ /* DRNG is seeded with at least LRNG_INIT_ENTROPY_BITS bits */ -+ } else if (seed_bits >= LRNG_INIT_ENTROPY_BITS) { -+ pr_info("LRNG initial entropy level %u bits of entropy\n", -+ seed_bits); -+ lrng_set_entropy_thresh(LRNG_MIN_SEED_ENTROPY_BITS); -+ } -+ } -+} -+ -+int __init lrng_rand_initialize(void) -+{ -+ struct seed { -+ ktime_t time; -+ unsigned long data[((LRNG_MAX_DIGESTSIZE + -+ sizeof(unsigned long) - 1) / -+ sizeof(unsigned long))]; -+ struct new_utsname utsname; -+ } seed __aligned(LRNG_KCAPI_ALIGN); -+ size_t longs = 0; -+ unsigned int i; -+ -+ seed.time = ktime_get_real(); -+ -+ for (i = 0; i < ARRAY_SIZE(seed.data); i += longs) { -+#ifdef CONFIG_LRNG_RANDOM_IF -+ longs = arch_get_random_seed_longs_early( -+ seed.data + i, ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+ longs = arch_get_random_longs_early(seed.data + i, -+ ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+#else -+ longs = arch_get_random_seed_longs(seed.data + i, -+ ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+ longs = arch_get_random_longs(seed.data + i, -+ ARRAY_SIZE(seed.data) - i); -+ if (longs) -+ continue; -+#endif -+ seed.data[i] = random_get_entropy(); -+ longs = 1; -+ } -+ memcpy(&seed.utsname, utsname(), sizeof(*(utsname()))); -+ -+ lrng_pool_insert_aux((u8 *)&seed, sizeof(seed), 0); -+ memzero_explicit(&seed, sizeof(seed)); -+ -+ /* Initialize the seed work queue */ -+ INIT_WORK(&lrng_state.lrng_seed_work, lrng_drng_seed_work); -+ lrng_state.perform_seedwork = true; -+ -+ invalidate_batched_entropy(); -+ -+ lrng_state.can_invalidate = true; -+ -+ return 0; -+} -+ -+#ifndef CONFIG_LRNG_RANDOM_IF -+early_initcall(lrng_rand_initialize); -+#endif -+ -+/* Interface requesting a reseed of the DRNG */ -+void lrng_es_add_entropy(void) -+{ -+ /* -+ * Once all DRNGs are fully seeded, the system-triggered arrival of -+ * entropy will not cause any reseeding any more. -+ */ -+ if (likely(lrng_state.all_online_numa_node_seeded)) -+ return; -+ -+ /* Only trigger the DRNG reseed if we have collected entropy. */ -+ if (lrng_avail_entropy() < -+ atomic_read_u32(&lrng_state.boot_entropy_thresh)) -+ return; -+ -+ /* Ensure that the seeding only occurs once at any given time. */ -+ if (!lrng_pool_trylock()) -+ return; -+ -+ /* Seed the DRNG with any available noise. */ -+ if (lrng_state.perform_seedwork) -+ schedule_work(&lrng_state.lrng_seed_work); -+ else -+ lrng_drng_seed_work(NULL); -+} -+ -+/* Fill the seed buffer with data from the noise sources */ -+void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits) -+{ -+ struct lrng_state *state = &lrng_state; -+ u32 i, req_ent = lrng_sp80090c_compliant() ? -+ lrng_security_strength() : LRNG_MIN_SEED_ENTROPY_BITS; -+ -+ /* Guarantee that requested bits is a multiple of bytes */ -+ BUILD_BUG_ON(LRNG_DRNG_SECURITY_STRENGTH_BITS % 8); -+ -+ /* always reseed the DRNG with the current time stamp */ -+ eb->now = random_get_entropy(); -+ -+ /* -+ * Require at least 128 bits of entropy for any reseed. If the LRNG is -+ * operated SP800-90C compliant we want to comply with SP800-90A section -+ * 9.2 mandating that DRNG is reseeded with the security strength. -+ */ -+ if (state->lrng_fully_seeded && (lrng_avail_entropy() < req_ent)) { -+ for_each_lrng_es(i) -+ eb->e_bits[i] = 0; -+ -+ goto wakeup; -+ } -+ -+ /* Concatenate the output of the entropy sources. */ -+ for_each_lrng_es(i) { -+ lrng_es[i]->get_ent(eb, requested_bits, -+ state->lrng_fully_seeded); -+ } -+ -+ /* allow external entropy provider to provide seed */ -+ lrng_state_exseed_allow_all(); -+ -+wakeup: -+ lrng_writer_wakeup(); -+} -diff --git a/drivers/char/lrng/lrng_es_mgr.h b/drivers/char/lrng/lrng_es_mgr.h -new file mode 100644 -index 000000000000..f9bbd4863120 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_mgr.h -@@ -0,0 +1,51 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_MGR_H -+#define _LRNG_ES_MGR_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+/*************************** General LRNG parameter ***************************/ -+ -+#define LRNG_DRNG_BLOCKSIZE 64 /* Maximum of DRNG block sizes */ -+ -+/* Helper to concatenate a macro with an integer type */ -+#define LRNG_PASTER(x, y) x ## y -+#define LRNG_UINT32_C(x) LRNG_PASTER(x, U) -+ -+/************************* Entropy sources management *************************/ -+ -+extern struct lrng_es_cb *lrng_es[]; -+ -+#define for_each_lrng_es(ctr) \ -+ for ((ctr) = 0; (ctr) < lrng_ext_es_last; (ctr)++) -+ -+bool lrng_pool_all_numa_nodes_seeded_get(void); -+bool lrng_state_min_seeded(void); -+void lrng_debug_report_seedlevel(const char *name); -+int lrng_rand_initialize(void); -+bool lrng_state_operational(void); -+ -+extern u32 lrng_write_wakeup_bits; -+void lrng_set_entropy_thresh(u32 new); -+u32 lrng_avail_entropy(void); -+u32 lrng_avail_entropy_aux(void); -+void lrng_reset_state(void); -+ -+bool lrng_state_fully_seeded(void); -+ -+int lrng_pool_trylock(void); -+void lrng_pool_lock(void); -+void lrng_pool_unlock(void); -+void lrng_pool_all_numa_nodes_seeded(bool set); -+ -+bool lrng_fully_seeded(bool fully_seeded, u32 collected_entropy); -+u32 lrng_entropy_rate_eb(struct entropy_buf *eb); -+void lrng_unset_fully_seeded(struct lrng_drng *drng); -+void lrng_fill_seed_buffer(struct entropy_buf *eb, u32 requested_bits); -+void lrng_init_ops(struct entropy_buf *eb); -+ -+#endif /* _LRNG_ES_MGR_H */ -diff --git a/drivers/char/lrng/lrng_es_mgr_cb.h b/drivers/char/lrng/lrng_es_mgr_cb.h -new file mode 100644 -index 000000000000..08b24e1b7766 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_mgr_cb.h -@@ -0,0 +1,87 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ * -+ * Definition of an entropy source. -+ */ -+ -+#ifndef _LRNG_ES_MGR_CB_H -+#define _LRNG_ES_MGR_CB_H -+ -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_drng_mgr.h" -+ -+enum lrng_internal_es { -+#ifdef CONFIG_LRNG_IRQ -+ lrng_int_es_irq, /* IRQ-based entropy source */ -+#endif -+#ifdef CONFIG_LRNG_SCHED -+ lrng_int_es_sched, /* Scheduler entropy source */ -+#endif -+ lrng_int_es_last, /* MUST be the last entry */ -+}; -+ -+enum lrng_external_es { -+ lrng_ext_link = lrng_int_es_last - 1, /* Link entry */ -+#ifdef CONFIG_LRNG_JENT -+ lrng_ext_es_jitter, /* Jitter RNG */ -+#endif -+#ifdef CONFIG_LRNG_CPU -+ lrng_ext_es_cpu, /* CPU-based, e.g. RDSEED */ -+#endif -+#ifdef CONFIG_LRNG_KERNEL_RNG -+ lrng_ext_es_krng, /* random.c */ -+#endif -+ lrng_ext_es_aux, /* MUST BE LAST ES! */ -+ lrng_ext_es_last /* MUST be the last entry */ -+}; -+ -+struct entropy_buf { -+ u8 e[lrng_ext_es_last][LRNG_DRNG_INIT_SEED_SIZE_BYTES]; -+ u32 now, e_bits[lrng_ext_es_last]; -+}; -+ -+/* -+ * struct lrng_es_cb - callback defining an entropy source -+ * @name: Name of the entropy source. -+ * @get_ent: Fetch entropy into the entropy_buf. The ES shall only deliver -+ * data if its internal initialization is complete, including any -+ * SP800-90B startup testing or similar. -+ * @curr_entropy: Return amount of currently available entropy. -+ * @max_entropy: Maximum amount of entropy the entropy source is able to -+ * maintain. -+ * @state: Buffer with human-readable ES state. -+ * @reset: Reset entropy source (drop all entropy and reinitialize). -+ * This callback may be NULL. -+ * @switch_hash: callback to switch from an old hash callback definition to -+ * a new one. This callback may be NULL. -+ */ -+struct lrng_es_cb { -+ const char *name; -+ void (*get_ent)(struct entropy_buf *eb, u32 requested_bits, -+ bool fully_seeded); -+ u32 (*curr_entropy)(u32 requested_bits); -+ u32 (*max_entropy)(void); -+ void (*state)(unsigned char *buf, size_t buflen); -+ void (*reset)(void); -+ int (*switch_hash)(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb); -+}; -+ -+/* Allow entropy sources to tell the ES manager that new entropy is there */ -+void lrng_es_add_entropy(void); -+ -+/* Cap to maximum entropy that can ever be generated with given hash */ -+#define lrng_cap_requested(__digestsize_bits, __requested_bits) \ -+ do { \ -+ if (__digestsize_bits < __requested_bits) { \ -+ pr_debug("Cannot satisfy requested entropy %u due to insufficient hash size %u\n",\ -+ __requested_bits, __digestsize_bits); \ -+ __requested_bits = __digestsize_bits; \ -+ } \ -+ } while (0) -+ -+#endif /* _LRNG_ES_MGR_CB_H */ -diff --git a/drivers/char/lrng/lrng_es_sched.c b/drivers/char/lrng/lrng_es_sched.c -new file mode 100644 -index 000000000000..c3abaa7ff310 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_sched.c -@@ -0,0 +1,562 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Scheduler-based data collection -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_sched.h" -+#include "lrng_es_timer_common.h" -+#include "lrng_health.h" -+#include "lrng_numa.h" -+#include "lrng_testing.h" -+ -+/* -+ * Number of scheduler-based context switches to be recorded to assume that -+ * DRNG security strength bits of entropy are received. -+ * Note: a value below the DRNG security strength should not be defined as this -+ * may imply the DRNG can never be fully seeded in case other noise -+ * sources are unavailable. -+ */ -+#define LRNG_SCHED_ENTROPY_BITS \ -+ LRNG_UINT32_C(CONFIG_LRNG_SCHED_ENTROPY_RATE) -+ -+/* Number of events required for LRNG_DRNG_SECURITY_STRENGTH_BITS entropy */ -+static u32 lrng_sched_entropy_bits = LRNG_SCHED_ENTROPY_BITS; -+ -+static u32 sched_entropy __read_mostly = LRNG_SCHED_ENTROPY_BITS; -+#ifdef CONFIG_LRNG_RUNTIME_ES_CONFIG -+module_param(sched_entropy, uint, 0444); -+MODULE_PARM_DESC(sched_entropy, -+ "How many scheduler-based context switches must be collected for obtaining 256 bits of entropy\n"); -+#endif -+ -+/* Per-CPU array holding concatenated entropy events */ -+static DEFINE_PER_CPU(u32 [LRNG_DATA_ARRAY_SIZE], lrng_sched_array) -+ __aligned(LRNG_KCAPI_ALIGN); -+static DEFINE_PER_CPU(u32, lrng_sched_array_ptr) = 0; -+static DEFINE_PER_CPU(atomic_t, lrng_sched_array_events) = ATOMIC_INIT(0); -+ -+/* -+ * Per-CPU entropy pool with compressed entropy event -+ * -+ * The per-CPU entropy pool is defined as the hash state. New data is simply -+ * inserted into the entropy pool by performing a hash update operation. -+ * To read the entropy pool, a hash final must be invoked. However, before -+ * the entropy pool is released again after a hash final, the hash init must -+ * be performed. -+ */ -+static DEFINE_PER_CPU(u8 [LRNG_POOL_SIZE], lrng_sched_pool) -+ __aligned(LRNG_KCAPI_ALIGN); -+/* -+ * Lock to allow other CPUs to read the pool - as this is only done during -+ * reseed which is infrequent, this lock is hardly contended. -+ */ -+static DEFINE_PER_CPU(spinlock_t, lrng_sched_lock); -+static DEFINE_PER_CPU(bool, lrng_sched_lock_init) = false; -+ -+static bool lrng_sched_pool_online(int cpu) -+{ -+ return per_cpu(lrng_sched_lock_init, cpu); -+} -+ -+static void __init lrng_sched_check_compression_state(void) -+{ -+ /* One pool should hold sufficient entropy for disabled compression */ -+ u32 max_ent = min_t(u32, lrng_get_digestsize(), -+ lrng_data_to_entropy(LRNG_DATA_NUM_VALUES, -+ lrng_sched_entropy_bits)); -+ if (max_ent < lrng_security_strength()) { -+ pr_devel("Scheduler entropy source will never provide %u bits of entropy required for fully seeding the DRNG all by itself\n", -+ lrng_security_strength()); -+ } -+} -+ -+void __init lrng_sched_es_init(bool highres_timer) -+{ -+ /* Set a minimum number of scheduler events that must be collected */ -+ sched_entropy = max_t(u32, LRNG_SCHED_ENTROPY_BITS, sched_entropy); -+ -+ if (highres_timer) { -+ lrng_sched_entropy_bits = sched_entropy; -+ } else { -+ u32 new_entropy = sched_entropy * LRNG_ES_OVERSAMPLING_FACTOR; -+ -+ lrng_sched_entropy_bits = (sched_entropy < new_entropy) ? -+ new_entropy : sched_entropy; -+ pr_warn("operating without high-resolution timer and applying oversampling factor %u\n", -+ LRNG_ES_OVERSAMPLING_FACTOR); -+ } -+ -+ lrng_sched_check_compression_state(); -+} -+ -+static u32 lrng_sched_avail_pool_size(void) -+{ -+ u32 max_pool = lrng_get_digestsize(), -+ max_size = min_t(u32, max_pool, LRNG_DATA_NUM_VALUES); -+ int cpu; -+ -+ for_each_online_cpu(cpu) -+ max_size += max_pool; -+ -+ return max_size; -+} -+ -+/* Return entropy of unused scheduler events present in all per-CPU pools. */ -+static u32 lrng_sched_avail_entropy(u32 __unused) -+{ -+ u32 digestsize_events, events = 0; -+ int cpu; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) -+ return 0; -+ -+ /* Obtain the cap of maximum numbers of scheduler events we count */ -+ digestsize_events = lrng_entropy_to_data(lrng_get_digestsize(), -+ lrng_sched_entropy_bits); -+ /* Cap to max. number of scheduler events the array can hold */ -+ digestsize_events = min_t(u32, digestsize_events, LRNG_DATA_NUM_VALUES); -+ -+ for_each_online_cpu(cpu) { -+ events += min_t(u32, digestsize_events, -+ atomic_read_u32(per_cpu_ptr(&lrng_sched_array_events, -+ cpu))); -+ } -+ -+ /* Consider oversampling rate */ -+ return lrng_reduce_by_osr( -+ lrng_data_to_entropy(events, lrng_sched_entropy_bits)); -+} -+ -+/* -+ * Reset all per-CPU pools - reset entropy estimator but leave the pool data -+ * that may or may not have entropy unchanged. -+ */ -+static void lrng_sched_reset(void) -+{ -+ int cpu; -+ -+ /* Trigger GCD calculation anew. */ -+ lrng_gcd_set(0); -+ -+ for_each_online_cpu(cpu) -+ atomic_set(per_cpu_ptr(&lrng_sched_array_events, cpu), 0); -+} -+ -+/* -+ * Trigger a switch of the hash implementation for the per-CPU pool. -+ * -+ * For each per-CPU pool, obtain the message digest with the old hash -+ * implementation, initialize the per-CPU pool again with the new hash -+ * implementation and inject the message digest into the new state. -+ * -+ * Assumption: the caller must guarantee that the new_cb is available during the -+ * entire operation (e.g. it must hold the lock against pointer updating). -+ */ -+static int -+lrng_sched_switch_hash(struct lrng_drng *drng, int node, -+ const struct lrng_hash_cb *new_cb, void *new_hash, -+ const struct lrng_hash_cb *old_cb) -+{ -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ u32 digestsize_events, found_events; -+ int ret = 0, cpu; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH)) -+ return -EOPNOTSUPP; -+ -+ for_each_online_cpu(cpu) { -+ struct shash_desc *pcpu_shash; -+ -+ /* -+ * Only switch the per-CPU pools for the current node because -+ * the hash_cb only applies NUMA-node-wide. -+ */ -+ if (cpu_to_node(cpu) != node || !lrng_sched_pool_online(cpu)) -+ continue; -+ -+ pcpu_shash = (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, -+ cpu); -+ -+ digestsize_events = old_cb->hash_digestsize(pcpu_shash); -+ digestsize_events = lrng_entropy_to_data(digestsize_events << 3, -+ lrng_sched_entropy_bits); -+ -+ if (pcpu_shash->tfm == new_hash) -+ continue; -+ -+ /* Get the per-CPU pool hash with old digest ... */ -+ ret = old_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash with the new digest ... */ -+ new_cb->hash_init(pcpu_shash, new_hash) ?: -+ /* -+ * ... feed the old hash into the new state. We may feed -+ * uninitialized memory into the new state, but this is -+ * considered no issue and even good as we have some more -+ * uncertainty here. -+ */ -+ new_cb->hash_update(pcpu_shash, digest, sizeof(digest)); -+ if (ret) -+ goto out; -+ -+ /* -+ * In case the new digest is larger than the old one, cap -+ * the available entropy to the old message digest used to -+ * process the existing data. -+ */ -+ found_events = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_sched_array_events, cpu), 0); -+ found_events = min_t(u32, found_events, digestsize_events); -+ atomic_add_return_relaxed(found_events, -+ per_cpu_ptr(&lrng_sched_array_events, cpu)); -+ -+ pr_debug("Re-initialize per-CPU scheduler entropy pool for CPU %d on NUMA node %d with hash %s\n", -+ cpu, node, new_cb->hash_name()); -+ } -+ -+out: -+ memzero_explicit(digest, sizeof(digest)); -+ return ret; -+} -+ -+static u32 -+lrng_sched_pool_hash_one(const struct lrng_hash_cb *pcpu_hash_cb, -+ void *pcpu_hash, int cpu, u8 *digest, u32 *digestsize) -+{ -+ struct shash_desc *pcpu_shash = -+ (struct shash_desc *)per_cpu_ptr(lrng_sched_pool, cpu); -+ spinlock_t *lock = per_cpu_ptr(&lrng_sched_lock, cpu); -+ unsigned long flags; -+ u32 digestsize_events, found_events; -+ -+ if (unlikely(!per_cpu(lrng_sched_lock_init, cpu))) { -+ if (pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash)) { -+ pr_warn("Initialization of hash failed\n"); -+ return 0; -+ } -+ spin_lock_init(lock); -+ per_cpu(lrng_sched_lock_init, cpu) = true; -+ pr_debug("Initializing per-CPU scheduler entropy pool for CPU %d with hash %s\n", -+ raw_smp_processor_id(), pcpu_hash_cb->hash_name()); -+ } -+ -+ /* Lock guarding against reading / writing to per-CPU pool */ -+ spin_lock_irqsave(lock, flags); -+ -+ *digestsize = pcpu_hash_cb->hash_digestsize(pcpu_hash); -+ digestsize_events = lrng_entropy_to_data(*digestsize << 3, -+ lrng_sched_entropy_bits); -+ -+ /* Obtain entropy statement like for the entropy pool */ -+ found_events = atomic_xchg_relaxed( -+ per_cpu_ptr(&lrng_sched_array_events, cpu), 0); -+ /* Cap to maximum amount of data we can hold in hash */ -+ found_events = min_t(u32, found_events, digestsize_events); -+ -+ /* Cap to maximum amount of data we can hold in array */ -+ found_events = min_t(u32, found_events, LRNG_DATA_NUM_VALUES); -+ -+ /* Store all not-yet compressed data in data array into hash, ... */ -+ if (pcpu_hash_cb->hash_update(pcpu_shash, -+ (u8 *)per_cpu_ptr(lrng_sched_array, cpu), -+ LRNG_DATA_ARRAY_SIZE * sizeof(u32)) ?: -+ /* ... get the per-CPU pool digest, ... */ -+ pcpu_hash_cb->hash_final(pcpu_shash, digest) ?: -+ /* ... re-initialize the hash, ... */ -+ pcpu_hash_cb->hash_init(pcpu_shash, pcpu_hash) ?: -+ /* ... feed the old hash into the new state. */ -+ pcpu_hash_cb->hash_update(pcpu_shash, digest, *digestsize)) -+ found_events = 0; -+ -+ spin_unlock_irqrestore(lock, flags); -+ return found_events; -+} -+ -+/* -+ * Hash all per-CPU arrays and return the digest to be used as seed data for -+ * seeding a DRNG. The caller must guarantee backtracking resistance. -+ * The function will only copy as much data as entropy is available into the -+ * caller-provided output buffer. -+ * -+ * This function handles the translation from the number of received scheduler -+ * events into an entropy statement. The conversion depends on -+ * LRNG_SCHED_ENTROPY_BITS which defines how many scheduler events must be -+ * received to obtain 256 bits of entropy. With this value, the function -+ * lrng_data_to_entropy converts a given data size (received scheduler events, -+ * requested amount of data, etc.) into an entropy statement. -+ * lrng_entropy_to_data does the reverse. -+ * -+ * @eb: entropy buffer to store entropy -+ * @requested_bits: Requested amount of entropy -+ * @fully_seeded: indicator whether LRNG is fully seeded -+ */ -+static void lrng_sched_pool_hash(struct entropy_buf *eb, u32 requested_bits, -+ bool fully_seeded) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb; -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *drng = lrng_drng_init_instance(); -+ u8 digest[LRNG_MAX_DIGESTSIZE]; -+ unsigned long flags, flags2; -+ u32 found_events, collected_events = 0, collected_ent_bits, -+ requested_events, returned_ent_bits; -+ int ret, cpu; -+ void *hash; -+ -+ /* Only deliver entropy when SP800-90B self test is completed */ -+ if (!lrng_sp80090b_startup_complete_es(lrng_int_es_sched)) { -+ eb->e_bits[lrng_int_es_sched] = 0; -+ return; -+ } -+ -+ /* Lock guarding replacement of per-NUMA hash */ -+ read_lock_irqsave(&drng->hash_lock, flags); -+ -+ hash_cb = drng->hash_cb; -+ hash = drng->hash; -+ -+ /* The hash state of filled with all per-CPU pool hashes. */ -+ ret = hash_cb->hash_init(shash, hash); -+ if (ret) -+ goto err; -+ -+ /* Cap to maximum entropy that can ever be generated with given hash */ -+ lrng_cap_requested(hash_cb->hash_digestsize(hash) << 3, requested_bits); -+ requested_events = lrng_entropy_to_data(requested_bits + -+ lrng_compress_osr(), -+ lrng_sched_entropy_bits); -+ -+ /* -+ * Harvest entropy from each per-CPU hash state - even though we may -+ * have collected sufficient entropy, we will hash all per-CPU pools. -+ */ -+ for_each_online_cpu(cpu) { -+ struct lrng_drng *pcpu_drng = drng; -+ u32 digestsize, unused_events = 0; -+ int node = cpu_to_node(cpu); -+ -+ if (lrng_drng && lrng_drng[node]) -+ pcpu_drng = lrng_drng[node]; -+ -+ if (pcpu_drng == drng) { -+ found_events = lrng_sched_pool_hash_one(hash_cb, hash, -+ cpu, digest, -+ &digestsize); -+ } else { -+ read_lock_irqsave(&pcpu_drng->hash_lock, flags2); -+ found_events = -+ lrng_sched_pool_hash_one(pcpu_drng->hash_cb, -+ pcpu_drng->hash, cpu, -+ digest, &digestsize); -+ read_unlock_irqrestore(&pcpu_drng->hash_lock, flags2); -+ } -+ -+ /* Store all not-yet compressed data in data array into hash */ -+ ret = hash_cb->hash_update(shash, digest, digestsize); -+ if (ret) -+ goto err; -+ -+ collected_events += found_events; -+ if (collected_events > requested_events) { -+ unused_events = collected_events - requested_events; -+ atomic_add_return_relaxed(unused_events, -+ per_cpu_ptr(&lrng_sched_array_events, cpu)); -+ collected_events = requested_events; -+ } -+ pr_debug("%u scheduler-based events used from entropy array of CPU %d, %u scheduler-based events remain unused\n", -+ found_events - unused_events, cpu, unused_events); -+ } -+ -+ ret = hash_cb->hash_final(shash, digest); -+ if (ret) -+ goto err; -+ -+ collected_ent_bits = lrng_data_to_entropy(collected_events, -+ lrng_sched_entropy_bits); -+ /* Apply oversampling: discount requested oversampling rate */ -+ returned_ent_bits = lrng_reduce_by_osr(collected_ent_bits); -+ -+ pr_debug("obtained %u bits by collecting %u bits of entropy from scheduler-based noise source\n", -+ returned_ent_bits, collected_ent_bits); -+ -+ /* -+ * Truncate to available entropy as implicitly allowed by SP800-90B -+ * section 3.1.5.1.1 table 1 which awards truncated hashes full -+ * entropy. -+ * -+ * During boot time, we read requested_bits data with -+ * returned_ent_bits entropy. In case our conservative entropy -+ * estimate underestimates the available entropy we can transport as -+ * much available entropy as possible. -+ */ -+ memcpy(eb->e[lrng_int_es_sched], digest, -+ fully_seeded ? returned_ent_bits >> 3 : requested_bits >> 3); -+ eb->e_bits[lrng_int_es_sched] = returned_ent_bits; -+ -+out: -+ hash_cb->hash_desc_zero(shash); -+ read_unlock_irqrestore(&drng->hash_lock, flags); -+ memzero_explicit(digest, sizeof(digest)); -+ return; -+ -+err: -+ eb->e_bits[lrng_int_es_sched] = 0; -+ goto out; -+} -+ -+/* -+ * Concatenate full 32 bit word at the end of time array even when current -+ * ptr is not aligned to sizeof(data). -+ */ -+static void lrng_sched_array_add_u32(u32 data) -+{ -+ /* Increment pointer by number of slots taken for input value */ -+ u32 pre_ptr, mask, ptr = this_cpu_add_return(lrng_sched_array_ptr, -+ LRNG_DATA_SLOTS_PER_UINT); -+ unsigned int pre_array; -+ -+ lrng_data_split_u32(&ptr, &pre_ptr, &mask); -+ -+ /* MSB of data go into previous unit */ -+ pre_array = lrng_data_idx2array(pre_ptr); -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_sched_array[pre_array], ~(0xffffffff & ~mask)); -+ this_cpu_or(lrng_sched_array[pre_array], data & ~mask); -+ -+ /* -+ * Continuous compression is not allowed for scheduler noise source, -+ * so do not call lrng_sched_array_to_hash here. -+ */ -+ -+ /* LSB of data go into current unit */ -+ this_cpu_write(lrng_sched_array[lrng_data_idx2array(ptr)], -+ data & mask); -+} -+ -+/* Concatenate data of max LRNG_DATA_SLOTSIZE_MASK at the end of time array */ -+static void lrng_sched_array_add_slot(u32 data) -+{ -+ /* Get slot */ -+ u32 ptr = this_cpu_inc_return(lrng_sched_array_ptr) & -+ LRNG_DATA_WORD_MASK; -+ unsigned int array = lrng_data_idx2array(ptr); -+ unsigned int slot = lrng_data_idx2slot(ptr); -+ -+ /* zeroization of slot to ensure the following OR adds the data */ -+ this_cpu_and(lrng_sched_array[array], -+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, -+ slot))); -+ /* Store data into slot */ -+ this_cpu_or(lrng_sched_array[array], lrng_data_slot_val(data, slot)); -+ -+ /* -+ * Continuous compression is not allowed for scheduler noise source, -+ * so do not call lrng_sched_array_to_hash here. -+ */ -+} -+ -+static void -+lrng_time_process_common(u32 time, void(*add_time)(u32 data)) -+{ -+ enum lrng_health_res health_test; -+ -+ if (lrng_raw_sched_hires_entropy_store(time)) -+ return; -+ -+ health_test = lrng_health_test(time, lrng_int_es_sched); -+ if (health_test > lrng_health_fail_use) -+ return; -+ -+ if (health_test == lrng_health_pass) -+ atomic_inc_return(this_cpu_ptr(&lrng_sched_array_events)); -+ -+ add_time(time); -+ -+ /* -+ * We cannot call lrng_es_add_entropy() as this would call a schedule -+ * operation that is not permissible in scheduler context. -+ * As the scheduler ES provides a high bandwidth of entropy, we assume -+ * that other reseed triggers happen to pick up the scheduler ES -+ * entropy in due time. -+ */ -+} -+ -+/* Batching up of entropy in per-CPU array */ -+static void lrng_sched_time_process(void) -+{ -+ u32 now_time = random_get_entropy(); -+ -+ if (unlikely(!lrng_gcd_tested())) { -+ /* When GCD is unknown, we process the full time stamp */ -+ lrng_time_process_common(now_time, lrng_sched_array_add_u32); -+ lrng_gcd_add_value(now_time); -+ } else { -+ /* GCD is known and applied */ -+ lrng_time_process_common((now_time / lrng_gcd_get()) & -+ LRNG_DATA_SLOTSIZE_MASK, -+ lrng_sched_array_add_slot); -+ } -+ -+ lrng_sched_perf_time(now_time); -+} -+ -+void add_sched_randomness(const struct task_struct *p, int cpu) -+{ -+ if (lrng_highres_timer()) { -+ lrng_sched_time_process(); -+ } else { -+ u32 tmp = cpu; -+ -+ tmp ^= lrng_raw_sched_pid_entropy_store(p->pid) ? -+ 0 : (u32)p->pid; -+ tmp ^= lrng_raw_sched_starttime_entropy_store(p->start_time) ? -+ 0 : (u32)p->start_time; -+ tmp ^= lrng_raw_sched_nvcsw_entropy_store(p->nvcsw) ? -+ 0 : (u32)p->nvcsw; -+ -+ lrng_sched_time_process(); -+ lrng_sched_array_add_u32(tmp); -+ } -+} -+EXPORT_SYMBOL(add_sched_randomness); -+ -+static void lrng_sched_es_state(unsigned char *buf, size_t buflen) -+{ -+ const struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ -+ /* Assume the lrng_drng_init lock is taken by caller */ -+ snprintf(buf, buflen, -+ " Hash for operating entropy pool: %s\n" -+ " Available entropy: %u\n" -+ " per-CPU scheduler event collection size: %u\n" -+ " Standards compliance: %s\n" -+ " High-resolution timer: %s\n", -+ lrng_drng_init->hash_cb->hash_name(), -+ lrng_sched_avail_entropy(0), -+ LRNG_DATA_NUM_VALUES, -+ lrng_sp80090b_compliant(lrng_int_es_sched) ? "SP800-90B " : "", -+ lrng_highres_timer() ? "true" : "false"); -+} -+ -+struct lrng_es_cb lrng_es_sched = { -+ .name = "Scheduler", -+ .get_ent = lrng_sched_pool_hash, -+ .curr_entropy = lrng_sched_avail_entropy, -+ .max_entropy = lrng_sched_avail_pool_size, -+ .state = lrng_sched_es_state, -+ .reset = lrng_sched_reset, -+ .switch_hash = lrng_sched_switch_hash, -+}; -diff --git a/drivers/char/lrng/lrng_es_sched.h b/drivers/char/lrng/lrng_es_sched.h -new file mode 100644 -index 000000000000..f1e596dd89d9 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_sched.h -@@ -0,0 +1,20 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_SCHED_H -+#define _LRNG_ES_SCHED_H -+ -+#include "lrng_es_mgr_cb.h" -+ -+#ifdef CONFIG_LRNG_SCHED -+void lrng_sched_es_init(bool highres_timer); -+ -+extern struct lrng_es_cb lrng_es_sched; -+ -+#else /* CONFIG_LRNG_SCHED */ -+static inline void lrng_sched_es_init(bool highres_timer) { } -+#endif /* CONFIG_LRNG_SCHED */ -+ -+#endif /* _LRNG_ES_SCHED_H */ -diff --git a/drivers/char/lrng/lrng_es_timer_common.c b/drivers/char/lrng/lrng_es_timer_common.c -new file mode 100644 -index 000000000000..70f3ff074fe6 ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_timer_common.c -@@ -0,0 +1,144 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Slow Entropy Source: Interrupt data collection -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_es_irq.h" -+#include "lrng_es_sched.h" -+#include "lrng_es_timer_common.h" -+#include "lrng_health.h" -+ -+/* Is high-resolution timer present? */ -+static bool lrng_highres_timer_val = false; -+ -+/* Number of time stamps analyzed to calculate a GCD */ -+#define LRNG_GCD_WINDOW_SIZE 100 -+static u32 lrng_gcd_history[LRNG_GCD_WINDOW_SIZE]; -+static atomic_t lrng_gcd_history_ptr = ATOMIC_INIT(-1); -+ -+/* The common divisor for all timestamps */ -+static u32 lrng_gcd_timer = 0; -+ -+bool lrng_gcd_tested(void) -+{ -+ return (lrng_gcd_timer != 0); -+} -+ -+u32 lrng_gcd_get(void) -+{ -+ return lrng_gcd_timer; -+} -+ -+/* Set the GCD for use in IRQ ES - if 0, the GCD calculation is restarted. */ -+void lrng_gcd_set(u32 running_gcd) -+{ -+ lrng_gcd_timer = running_gcd; -+ /* Ensure that update to global variable lrng_gcd_timer is visible */ -+ mb(); -+} -+ -+static void lrng_gcd_set_check(u32 running_gcd) -+{ -+ if (!lrng_gcd_tested()) { -+ lrng_gcd_set(running_gcd); -+ pr_debug("Setting GCD to %u\n", running_gcd); -+ } -+} -+ -+u32 lrng_gcd_analyze(u32 *history, size_t nelem) -+{ -+ u32 running_gcd = 0; -+ size_t i; -+ -+ /* Now perform the analysis on the accumulated time data. */ -+ for (i = 0; i < nelem; i++) { -+ /* -+ * NOTE: this would be the place to add more analysis on the -+ * appropriateness of the timer like checking the presence -+ * of sufficient variations in the timer. -+ */ -+ -+ /* -+ * This calculates the gcd of all the time values. that is -+ * gcd(time_1, time_2, ..., time_nelem) -+ * -+ * Some timers increment by a fixed (non-1) amount each step. -+ * This code checks for such increments, and allows the library -+ * to output the number of such changes have occurred. -+ */ -+ running_gcd = (u32)gcd(history[i], running_gcd); -+ -+ /* Zeroize data */ -+ history[i] = 0; -+ } -+ -+ return running_gcd; -+} -+ -+void lrng_gcd_add_value(u32 time) -+{ -+ u32 ptr = (u32)atomic_inc_return_relaxed(&lrng_gcd_history_ptr); -+ -+ if (ptr < LRNG_GCD_WINDOW_SIZE) { -+ lrng_gcd_history[ptr] = time; -+ } else if (ptr == LRNG_GCD_WINDOW_SIZE) { -+ u32 gcd = lrng_gcd_analyze(lrng_gcd_history, -+ LRNG_GCD_WINDOW_SIZE); -+ -+ if (!gcd) -+ gcd = 1; -+ -+ /* -+ * Ensure that we have variations in the time stamp below the -+ * given value. This is just a safety measure to prevent the GCD -+ * becoming too large. -+ */ -+ if (gcd >= 1000) { -+ pr_warn("calculated GCD is larger than expected: %u\n", -+ gcd); -+ gcd = 1000; -+ } -+ -+ /* Adjust all deltas by the observed (small) common factor. */ -+ lrng_gcd_set_check(gcd); -+ atomic_set(&lrng_gcd_history_ptr, 0); -+ } -+} -+ -+/* Return boolean whether LRNG identified presence of high-resolution timer */ -+bool lrng_highres_timer(void) -+{ -+ return lrng_highres_timer_val; -+} -+ -+static int __init lrng_init_time_source(void) -+{ -+ if ((random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK) || -+ (random_get_entropy() & LRNG_DATA_SLOTSIZE_MASK)) { -+ /* -+ * As the highres timer is identified here, previous interrupts -+ * obtained during boot time are treated like a lowres-timer -+ * would have been present. -+ */ -+ lrng_highres_timer_val = true; -+ } else { -+ lrng_health_disable(); -+ lrng_highres_timer_val = false; -+ } -+ -+ lrng_irq_es_init(lrng_highres_timer_val); -+ lrng_sched_es_init(lrng_highres_timer_val); -+ -+ /* Ensure that changes to global variables are visible */ -+ mb(); -+ -+ return 0; -+} -+core_initcall(lrng_init_time_source); -diff --git a/drivers/char/lrng/lrng_es_timer_common.h b/drivers/char/lrng/lrng_es_timer_common.h -new file mode 100644 -index 000000000000..b45b9f683dea ---- /dev/null -+++ b/drivers/char/lrng/lrng_es_timer_common.h -@@ -0,0 +1,83 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG Slow Noise Source: Time stamp array handling -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_ES_TIMER_COMMON_H -+#define _LRNG_ES_TIMER_COMMON_H -+ -+bool lrng_gcd_tested(void); -+void lrng_gcd_set(u32 running_gcd); -+u32 lrng_gcd_get(void); -+u32 lrng_gcd_analyze(u32 *history, size_t nelem); -+void lrng_gcd_add_value(u32 time); -+bool lrng_highres_timer(void); -+ -+/* -+ * To limit the impact on the interrupt handling, the LRNG concatenates -+ * entropic LSB parts of the time stamps in a per-CPU array and only -+ * injects them into the entropy pool when the array is full. -+ */ -+ -+/* Store multiple integers in one u32 */ -+#define LRNG_DATA_SLOTSIZE_BITS (8) -+#define LRNG_DATA_SLOTSIZE_MASK ((1 << LRNG_DATA_SLOTSIZE_BITS) - 1) -+#define LRNG_DATA_ARRAY_MEMBER_BITS (4 << 3) /* ((sizeof(u32)) << 3) */ -+#define LRNG_DATA_SLOTS_PER_UINT (LRNG_DATA_ARRAY_MEMBER_BITS / \ -+ LRNG_DATA_SLOTSIZE_BITS) -+ -+/* -+ * Number of time values to store in the array - in small environments -+ * only one atomic_t variable per CPU is used. -+ */ -+#define LRNG_DATA_NUM_VALUES (CONFIG_LRNG_COLLECTION_SIZE) -+/* Mask of LSB of time stamp to store */ -+#define LRNG_DATA_WORD_MASK (LRNG_DATA_NUM_VALUES - 1) -+ -+#define LRNG_DATA_SLOTS_MASK (LRNG_DATA_SLOTS_PER_UINT - 1) -+#define LRNG_DATA_ARRAY_SIZE (LRNG_DATA_NUM_VALUES / \ -+ LRNG_DATA_SLOTS_PER_UINT) -+ -+/* Starting bit index of slot */ -+static inline unsigned int lrng_data_slot2bitindex(unsigned int slot) -+{ -+ return (LRNG_DATA_SLOTSIZE_BITS * slot); -+} -+ -+/* Convert index into the array index */ -+static inline unsigned int lrng_data_idx2array(unsigned int idx) -+{ -+ return idx / LRNG_DATA_SLOTS_PER_UINT; -+} -+ -+/* Convert index into the slot of a given array index */ -+static inline unsigned int lrng_data_idx2slot(unsigned int idx) -+{ -+ return idx & LRNG_DATA_SLOTS_MASK; -+} -+ -+/* Convert value into slot value */ -+static inline unsigned int lrng_data_slot_val(unsigned int val, -+ unsigned int slot) -+{ -+ return val << lrng_data_slot2bitindex(slot); -+} -+ -+/* -+ * Return the pointers for the previous and current units to inject a u32 into. -+ * Also return the mask which the u32 word is to be processed. -+ */ -+static inline void lrng_data_split_u32(u32 *ptr, u32 *pre_ptr, u32 *mask) -+{ -+ /* ptr to previous unit */ -+ *pre_ptr = (*ptr - LRNG_DATA_SLOTS_PER_UINT) & LRNG_DATA_WORD_MASK; -+ *ptr &= LRNG_DATA_WORD_MASK; -+ -+ /* mask to split data into the two parts for the two units */ -+ *mask = ((1 << (*pre_ptr & (LRNG_DATA_SLOTS_PER_UINT - 1)) * -+ LRNG_DATA_SLOTSIZE_BITS)) - 1; -+} -+ -+#endif /* _LRNG_ES_TIMER_COMMON_H */ -\ No newline at end of file -diff --git a/drivers/char/lrng/lrng_hash_kcapi.c b/drivers/char/lrng/lrng_hash_kcapi.c -new file mode 100644 -index 000000000000..13e62db9b6c8 ---- /dev/null -+++ b/drivers/char/lrng/lrng_hash_kcapi.c -@@ -0,0 +1,140 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for providing the hash primitive using the kernel crypto API. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+ -+static char *lrng_hash_name = "sha512"; -+ -+/* The parameter must be r/o in sysfs as otherwise races appear. */ -+module_param(lrng_hash_name, charp, 0444); -+MODULE_PARM_DESC(lrng_hash_name, "Kernel crypto API hash name"); -+ -+struct lrng_hash_info { -+ struct crypto_shash *tfm; -+}; -+ -+static const char *lrng_kcapi_hash_name(void) -+{ -+ return lrng_hash_name; -+} -+ -+static void _lrng_kcapi_hash_free(struct lrng_hash_info *lrng_hash) -+{ -+ struct crypto_shash *tfm = lrng_hash->tfm; -+ -+ crypto_free_shash(tfm); -+ kfree(lrng_hash); -+} -+ -+static void *lrng_kcapi_hash_alloc(const char *name) -+{ -+ struct lrng_hash_info *lrng_hash; -+ struct crypto_shash *tfm; -+ int ret; -+ -+ if (!name) { -+ pr_err("Hash name missing\n"); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ tfm = crypto_alloc_shash(name, 0, 0); -+ if (IS_ERR(tfm)) { -+ pr_err("could not allocate hash %s\n", name); -+ return ERR_CAST(tfm); -+ } -+ -+ ret = sizeof(struct lrng_hash_info); -+ lrng_hash = kmalloc(ret, GFP_KERNEL); -+ if (!lrng_hash) { -+ crypto_free_shash(tfm); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ lrng_hash->tfm = tfm; -+ -+ pr_info("Hash %s allocated\n", name); -+ -+ return lrng_hash; -+} -+ -+static void *lrng_kcapi_hash_name_alloc(void) -+{ -+ return lrng_kcapi_hash_alloc(lrng_kcapi_hash_name()); -+} -+ -+static u32 lrng_kcapi_hash_digestsize(void *hash) -+{ -+ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; -+ struct crypto_shash *tfm = lrng_hash->tfm; -+ -+ return crypto_shash_digestsize(tfm); -+} -+ -+static void lrng_kcapi_hash_dealloc(void *hash) -+{ -+ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; -+ -+ _lrng_kcapi_hash_free(lrng_hash); -+ pr_info("Hash deallocated\n"); -+} -+ -+static int lrng_kcapi_hash_init(struct shash_desc *shash, void *hash) -+{ -+ struct lrng_hash_info *lrng_hash = (struct lrng_hash_info *)hash; -+ struct crypto_shash *tfm = lrng_hash->tfm; -+ -+ shash->tfm = tfm; -+ return crypto_shash_init(shash); -+} -+ -+static int lrng_kcapi_hash_update(struct shash_desc *shash, const u8 *inbuf, -+ u32 inbuflen) -+{ -+ return crypto_shash_update(shash, inbuf, inbuflen); -+} -+ -+static int lrng_kcapi_hash_final(struct shash_desc *shash, u8 *digest) -+{ -+ return crypto_shash_final(shash, digest); -+} -+ -+static void lrng_kcapi_hash_zero(struct shash_desc *shash) -+{ -+ shash_desc_zero(shash); -+} -+ -+static const struct lrng_hash_cb lrng_kcapi_hash_cb = { -+ .hash_name = lrng_kcapi_hash_name, -+ .hash_alloc = lrng_kcapi_hash_name_alloc, -+ .hash_dealloc = lrng_kcapi_hash_dealloc, -+ .hash_digestsize = lrng_kcapi_hash_digestsize, -+ .hash_init = lrng_kcapi_hash_init, -+ .hash_update = lrng_kcapi_hash_update, -+ .hash_final = lrng_kcapi_hash_final, -+ .hash_desc_zero = lrng_kcapi_hash_zero, -+}; -+ -+static int __init lrng_kcapi_init(void) -+{ -+ return lrng_set_hash_cb(&lrng_kcapi_hash_cb); -+} -+ -+static void __exit lrng_kcapi_exit(void) -+{ -+ lrng_set_hash_cb(NULL); -+} -+ -+late_initcall(lrng_kcapi_init); -+module_exit(lrng_kcapi_exit); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager - Kernel crypto API hash backend"); -diff --git a/drivers/char/lrng/lrng_health.c b/drivers/char/lrng/lrng_health.c -new file mode 100644 -index 000000000000..a60c8378eea5 ---- /dev/null -+++ b/drivers/char/lrng/lrng_health.c -@@ -0,0 +1,420 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Entropy Source and DRNG Manager (LRNG) Health Testing -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_es_mgr.h" -+#include "lrng_health.h" -+ -+/* Stuck Test */ -+struct lrng_stuck_test { -+ u32 last_time; /* Stuck test: time of previous IRQ */ -+ u32 last_delta; /* Stuck test: delta of previous IRQ */ -+ u32 last_delta2; /* Stuck test: 2. time derivation of prev IRQ */ -+}; -+ -+/* Repetition Count Test */ -+struct lrng_rct { -+ atomic_t rct_count; /* Number of stuck values */ -+}; -+ -+/* Adaptive Proportion Test */ -+struct lrng_apt { -+ /* Data window size */ -+#define LRNG_APT_WINDOW_SIZE 512 -+ /* LSB of time stamp to process */ -+#define LRNG_APT_LSB 16 -+#define LRNG_APT_WORD_MASK (LRNG_APT_LSB - 1) -+ atomic_t apt_count; /* APT counter */ -+ atomic_t apt_base; /* APT base reference */ -+ -+ atomic_t apt_trigger; -+ bool apt_base_set; /* Is APT base set? */ -+}; -+ -+/* Health data collected for one entropy source */ -+struct lrng_health_es_state { -+ struct lrng_rct rct; -+ struct lrng_apt apt; -+ -+ /* SP800-90B startup health tests */ -+#define LRNG_SP80090B_STARTUP_SAMPLES 1024 -+#define LRNG_SP80090B_STARTUP_BLOCKS ((LRNG_SP80090B_STARTUP_SAMPLES + \ -+ LRNG_APT_WINDOW_SIZE - 1) / \ -+ LRNG_APT_WINDOW_SIZE) -+ bool sp80090b_startup_done; -+ atomic_t sp80090b_startup_blocks; -+}; -+ -+#define LRNG_HEALTH_ES_INIT(x) \ -+ x.rct.rct_count = ATOMIC_INIT(0), \ -+ x.apt.apt_count = ATOMIC_INIT(0), \ -+ x.apt.apt_base = ATOMIC_INIT(-1), \ -+ x.apt.apt_trigger = ATOMIC_INIT(LRNG_APT_WINDOW_SIZE), \ -+ x.apt.apt_base_set = false, \ -+ x.sp80090b_startup_blocks = ATOMIC_INIT(LRNG_SP80090B_STARTUP_BLOCKS), \ -+ x.sp80090b_startup_done = false, -+ -+/* The health test code must operate lock-less */ -+struct lrng_health { -+ bool health_test_enabled; -+ struct lrng_health_es_state es_state[lrng_int_es_last]; -+}; -+ -+static struct lrng_health lrng_health = { -+ .health_test_enabled = true, -+ -+#ifdef CONFIG_LRNG_IRQ -+ LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_irq]) -+#endif -+#ifdef CONFIG_LRNG_SCHED -+ LRNG_HEALTH_ES_INIT(.es_state[lrng_int_es_sched]) -+#endif -+}; -+ -+static DEFINE_PER_CPU(struct lrng_stuck_test[lrng_int_es_last], -+ lrng_stuck_test_array); -+ -+static bool lrng_sp80090b_health_requested(void) -+{ -+ /* Health tests are only requested in FIPS mode */ -+ return fips_enabled; -+} -+ -+static bool lrng_sp80090b_health_enabled(void) -+{ -+ struct lrng_health *health = &lrng_health; -+ -+ return lrng_sp80090b_health_requested() && health->health_test_enabled; -+} -+ -+/*************************************************************************** -+ * SP800-90B Compliance -+ * -+ * If the Linux-RNG is booted into FIPS mode, the following interfaces -+ * provide an SP800-90B compliant noise source: -+ * -+ * * /dev/random -+ * * getrandom(2) -+ * * get_random_bytes_full -+ * -+ * All other interfaces, including /dev/urandom or get_random_bytes without -+ * the add_random_ready_callback cannot claim to use an SP800-90B compliant -+ * noise source. -+ ***************************************************************************/ -+ -+/* -+ * Perform SP800-90B startup testing -+ */ -+static void lrng_sp80090b_startup(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ if (!es_state->sp80090b_startup_done && -+ atomic_dec_and_test(&es_state->sp80090b_startup_blocks)) { -+ es_state->sp80090b_startup_done = true; -+ pr_info("SP800-90B startup health tests for internal entropy source %u completed\n", -+ es); -+ lrng_drng_force_reseed(); -+ -+ /* -+ * We cannot call lrng_es_add_entropy() as this may cause a -+ * schedule operation while in scheduler context for the -+ * scheduler ES. -+ */ -+ } -+} -+ -+/* -+ * Handle failure of SP800-90B startup testing -+ */ -+static void lrng_sp80090b_startup_failure(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ /* Reset of LRNG and its entropy - NOTE: we are in atomic context */ -+ lrng_reset(); -+ -+ /* -+ * Reset the SP800-90B startup test. -+ * -+ * NOTE SP800-90B section 4.3 bullet 4 does not specify what -+ * exactly is to be done in case of failure! Thus, we do what -+ * makes sense, i.e. restarting the health test and thus gating -+ * the output function of /dev/random and getrandom(2). -+ */ -+ atomic_set(&es_state->sp80090b_startup_blocks, -+ LRNG_SP80090B_STARTUP_BLOCKS); -+} -+ -+/* -+ * Handle failure of SP800-90B runtime testing -+ */ -+static void lrng_sp80090b_runtime_failure(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ lrng_sp80090b_startup_failure(health, es); -+ es_state->sp80090b_startup_done = false; -+} -+ -+static void lrng_sp80090b_failure(struct lrng_health *health, -+ enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ if (es_state->sp80090b_startup_done) { -+ pr_err("SP800-90B runtime health test failure for internal entropy source %u - invalidating all existing entropy and initiate SP800-90B startup\n", es); -+ lrng_sp80090b_runtime_failure(health, es); -+ } else { -+ pr_err("SP800-90B startup test failure for internal entropy source %u - resetting\n", es); -+ lrng_sp80090b_startup_failure(health, es); -+ } -+} -+ -+bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) -+{ -+ struct lrng_health *health = &lrng_health; -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ if (!lrng_sp80090b_health_enabled()) -+ return true; -+ -+ return es_state->sp80090b_startup_done; -+} -+ -+bool lrng_sp80090b_compliant(enum lrng_internal_es es) -+{ -+ struct lrng_health *health = &lrng_health; -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ -+ return lrng_sp80090b_health_enabled() && -+ es_state->sp80090b_startup_done; -+} -+ -+/*************************************************************************** -+ * Adaptive Proportion Test -+ * -+ * This test complies with SP800-90B section 4.4.2. -+ ***************************************************************************/ -+ -+/* -+ * Reset the APT counter -+ * -+ * @health [in] Reference to health state -+ */ -+static void lrng_apt_reset(struct lrng_apt *apt, unsigned int time_masked) -+{ -+ /* Reset APT */ -+ atomic_set(&apt->apt_count, 0); -+ atomic_set(&apt->apt_base, time_masked); -+} -+ -+static void lrng_apt_restart(struct lrng_apt *apt) -+{ -+ atomic_set(&apt->apt_trigger, LRNG_APT_WINDOW_SIZE); -+} -+ -+/* -+ * Insert a new entropy event into APT -+ * -+ * This function does is void as it does not decide about the fate of a time -+ * stamp. An APT failure can only happen at the same time of a stuck test -+ * failure. Thus, the stuck failure will already decide how the time stamp -+ * is handled. -+ * -+ * @health [in] Reference to health state -+ * @now_time [in] Time stamp to process -+ */ -+static void lrng_apt_insert(struct lrng_health *health, -+ unsigned int now_time, enum lrng_internal_es es) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ struct lrng_apt *apt = &es_state->apt; -+ -+ if (!lrng_sp80090b_health_requested()) -+ return; -+ -+ now_time &= LRNG_APT_WORD_MASK; -+ -+ /* Initialization of APT */ -+ if (!apt->apt_base_set) { -+ atomic_set(&apt->apt_base, now_time); -+ apt->apt_base_set = true; -+ return; -+ } -+ -+ if (now_time == (unsigned int)atomic_read(&apt->apt_base)) { -+ u32 apt_val = (u32)atomic_inc_return_relaxed(&apt->apt_count); -+ -+ if (apt_val >= CONFIG_LRNG_APT_CUTOFF) -+ lrng_sp80090b_failure(health, es); -+ } -+ -+ if (atomic_dec_and_test(&apt->apt_trigger)) { -+ lrng_apt_restart(apt); -+ lrng_apt_reset(apt, now_time); -+ lrng_sp80090b_startup(health, es); -+ } -+} -+ -+/*************************************************************************** -+ * Repetition Count Test -+ * -+ * The LRNG uses an enhanced version of the Repetition Count Test -+ * (RCT) specified in SP800-90B section 4.4.1. Instead of counting identical -+ * back-to-back values, the input to the RCT is the counting of the stuck -+ * values while filling the entropy pool. -+ * -+ * The RCT is applied with an alpha of 2^-30 compliant to FIPS 140-2 IG 9.8. -+ * -+ * During the counting operation, the LRNG always calculates the RCT -+ * cut-off value of C. If that value exceeds the allowed cut-off value, -+ * the LRNG will invalidate all entropy for the entropy pool which implies -+ * that no data can be extracted from the entropy pool unless new entropy -+ * is received. -+ ***************************************************************************/ -+ -+/* -+ * Hot code path - Insert data for Repetition Count Test -+ * -+ * @health: Reference to health information -+ * @stuck: Decision of stuck test -+ */ -+static void lrng_rct(struct lrng_health *health, enum lrng_internal_es es, -+ int stuck) -+{ -+ struct lrng_health_es_state *es_state = &health->es_state[es]; -+ struct lrng_rct *rct = &es_state->rct; -+ -+ if (!lrng_sp80090b_health_requested()) -+ return; -+ -+ if (stuck) { -+ u32 rct_count = atomic_add_return_relaxed(1, &rct->rct_count); -+ -+ /* -+ * The cutoff value is based on the following consideration: -+ * alpha = 2^-30 as recommended in FIPS 140-2 IG 9.8. -+ * In addition, we imply an entropy value H of 1 bit as this -+ * is the minimum entropy required to provide full entropy. -+ * -+ * Note, rct_count (which equals to value B in the -+ * pseudo code of SP800-90B section 4.4.1) starts with zero. -+ * Hence we need to subtract one from the cutoff value as -+ * calculated following SP800-90B. -+ */ -+ if (rct_count >= CONFIG_LRNG_RCT_CUTOFF) { -+ atomic_set(&rct->rct_count, 0); -+ -+ /* -+ * APT must start anew as we consider all previously -+ * recorded data to contain no entropy. -+ */ -+ lrng_apt_restart(&es_state->apt); -+ -+ lrng_sp80090b_failure(health, es); -+ } -+ } else { -+ atomic_set(&rct->rct_count, 0); -+ } -+} -+ -+/*************************************************************************** -+ * Stuck Test -+ * -+ * Checking the: -+ * 1st derivative of the event occurrence (time delta) -+ * 2nd derivative of the event occurrence (delta of time deltas) -+ * 3rd derivative of the event occurrence (delta of delta of time deltas) -+ * -+ * All values must always be non-zero. The stuck test is only valid disabled if -+ * high-resolution time stamps are identified after initialization. -+ ***************************************************************************/ -+ -+static u32 lrng_delta(u32 prev, u32 next) -+{ -+ /* -+ * Note that this (unsigned) subtraction does yield the correct value -+ * in the wraparound-case, i.e. when next < prev. -+ */ -+ return (next - prev); -+} -+ -+/* -+ * Hot code path -+ * -+ * @health: Reference to health information -+ * @now: Event time -+ * @return: 0 event occurrence not stuck (good time stamp) -+ * != 0 event occurrence stuck (reject time stamp) -+ */ -+static int lrng_irq_stuck(enum lrng_internal_es es, u32 now_time) -+{ -+ struct lrng_stuck_test *stuck = this_cpu_ptr(lrng_stuck_test_array); -+ u32 delta = lrng_delta(stuck[es].last_time, now_time); -+ u32 delta2 = lrng_delta(stuck[es].last_delta, delta); -+ u32 delta3 = lrng_delta(stuck[es].last_delta2, delta2); -+ -+ stuck[es].last_time = now_time; -+ stuck[es].last_delta = delta; -+ stuck[es].last_delta2 = delta2; -+ -+ if (!delta || !delta2 || !delta3) -+ return 1; -+ -+ return 0; -+} -+ -+/*************************************************************************** -+ * Health test interfaces -+ ***************************************************************************/ -+ -+/* -+ * Disable all health tests -+ */ -+void lrng_health_disable(void) -+{ -+ struct lrng_health *health = &lrng_health; -+ -+ health->health_test_enabled = false; -+ -+ if (lrng_sp80090b_health_requested()) -+ pr_warn("SP800-90B compliance requested but the Linux RNG is NOT SP800-90B compliant\n"); -+} -+ -+/* -+ * Hot code path - Perform health test on time stamp received from an event -+ * -+ * @now_time Time stamp -+ */ -+enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es) -+{ -+ struct lrng_health *health = &lrng_health; -+ int stuck; -+ -+ if (!health->health_test_enabled) -+ return lrng_health_pass; -+ -+ lrng_apt_insert(health, now_time, es); -+ -+ stuck = lrng_irq_stuck(es, now_time); -+ lrng_rct(health, es, stuck); -+ if (stuck) { -+ /* SP800-90B disallows using a failing health test time stamp */ -+ return lrng_sp80090b_health_requested() ? -+ lrng_health_fail_drop : lrng_health_fail_use; -+ } -+ -+ return lrng_health_pass; -+} -diff --git a/drivers/char/lrng/lrng_health.h b/drivers/char/lrng/lrng_health.h -new file mode 100644 -index 000000000000..4f9f5033fc30 ---- /dev/null -+++ b/drivers/char/lrng/lrng_health.h -@@ -0,0 +1,42 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_HEALTH_H -+#define _LRNG_HEALTH_H -+ -+#include "lrng_es_mgr.h" -+ -+enum lrng_health_res { -+ lrng_health_pass, /* Health test passes on time stamp */ -+ lrng_health_fail_use, /* Time stamp unhealthy, but mix in */ -+ lrng_health_fail_drop /* Time stamp unhealthy, drop it */ -+}; -+ -+#ifdef CONFIG_LRNG_HEALTH_TESTS -+bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es); -+bool lrng_sp80090b_compliant(enum lrng_internal_es es); -+ -+enum lrng_health_res lrng_health_test(u32 now_time, enum lrng_internal_es es); -+void lrng_health_disable(void); -+#else /* CONFIG_LRNG_HEALTH_TESTS */ -+static inline bool lrng_sp80090b_startup_complete_es(enum lrng_internal_es es) -+{ -+ return true; -+} -+ -+static inline bool lrng_sp80090b_compliant(enum lrng_internal_es es) -+{ -+ return false; -+} -+ -+static inline enum lrng_health_res -+lrng_health_test(u32 now_time, enum lrng_internal_es es) -+{ -+ return lrng_health_pass; -+} -+static inline void lrng_health_disable(void) { } -+#endif /* CONFIG_LRNG_HEALTH_TESTS */ -+ -+#endif /* _LRNG_HEALTH_H */ -diff --git a/drivers/char/lrng/lrng_interface_aux.c b/drivers/char/lrng/lrng_interface_aux.c -new file mode 100644 -index 000000000000..0eb49bd6b48f ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_aux.c -@@ -0,0 +1,126 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG auxiliary interfaces -+ * -+ * Copyright (C) 2022 Stephan Mueller -+ * Copyright (C) 2017 Jason A. Donenfeld . All -+ * Rights Reserved. -+ * Copyright (C) 2016 Jason Cooper -+ */ -+ -+#include -+#include -+#include -+ -+#include "lrng_es_mgr.h" -+#include "lrng_interface_random_kernel.h" -+ -+struct batched_entropy { -+ union { -+ u64 entropy_u64[LRNG_DRNG_BLOCKSIZE / sizeof(u64)]; -+ u32 entropy_u32[LRNG_DRNG_BLOCKSIZE / sizeof(u32)]; -+ }; -+ unsigned int position; -+ spinlock_t batch_lock; -+}; -+ -+/* -+ * Get a random word for internal kernel use only. The quality of the random -+ * number is as good as /dev/urandom, but there is no backtrack protection, -+ * with the goal of being quite fast and not depleting entropy. -+ */ -+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { -+ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), -+}; -+ -+u64 get_random_u64(void) -+{ -+ u64 ret; -+ unsigned long flags; -+ struct batched_entropy *batch; -+ -+ lrng_debug_report_seedlevel("get_random_u64"); -+ -+ batch = raw_cpu_ptr(&batched_entropy_u64); -+ spin_lock_irqsave(&batch->batch_lock, flags); -+ if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { -+ lrng_get_random_bytes(batch->entropy_u64, LRNG_DRNG_BLOCKSIZE); -+ batch->position = 0; -+ } -+ ret = batch->entropy_u64[batch->position++]; -+ spin_unlock_irqrestore(&batch->batch_lock, flags); -+ return ret; -+} -+EXPORT_SYMBOL(get_random_u64); -+ -+static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { -+ .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), -+}; -+ -+u32 get_random_u32(void) -+{ -+ u32 ret; -+ unsigned long flags; -+ struct batched_entropy *batch; -+ -+ lrng_debug_report_seedlevel("get_random_u32"); -+ -+ batch = raw_cpu_ptr(&batched_entropy_u32); -+ spin_lock_irqsave(&batch->batch_lock, flags); -+ if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { -+ lrng_get_random_bytes(batch->entropy_u32, LRNG_DRNG_BLOCKSIZE); -+ batch->position = 0; -+ } -+ ret = batch->entropy_u32[batch->position++]; -+ spin_unlock_irqrestore(&batch->batch_lock, flags); -+ return ret; -+} -+EXPORT_SYMBOL(get_random_u32); -+ -+#ifdef CONFIG_SMP -+/* -+ * This function is called when the CPU is coming up, with entry -+ * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP. -+ */ -+int random_prepare_cpu(unsigned int cpu) -+{ -+ /* -+ * When the cpu comes back online, immediately invalidate all batches, -+ * so that we serve fresh randomness. -+ */ -+ per_cpu_ptr(&batched_entropy_u32, cpu)->position = 0; -+ per_cpu_ptr(&batched_entropy_u64, cpu)->position = 0; -+ return 0; -+} -+ -+int random_online_cpu(unsigned int cpu) -+{ -+ return 0; -+} -+#endif -+ -+/* -+ * It's important to invalidate all potential batched entropy that might -+ * be stored before the crng is initialized, which we can do lazily by -+ * simply resetting the counter to zero so that it's re-extracted on the -+ * next usage. -+ */ -+void invalidate_batched_entropy(void) -+{ -+ int cpu; -+ unsigned long flags; -+ -+ for_each_possible_cpu(cpu) { -+ struct batched_entropy *batched_entropy; -+ -+ batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); -+ spin_lock_irqsave(&batched_entropy->batch_lock, flags); -+ batched_entropy->position = 0; -+ spin_unlock(&batched_entropy->batch_lock); -+ -+ batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); -+ spin_lock(&batched_entropy->batch_lock); -+ batched_entropy->position = 0; -+ spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); -+ } -+} -diff --git a/drivers/char/lrng/lrng_interface_dev.c b/drivers/char/lrng/lrng_interface_dev.c -new file mode 100644 -index 000000000000..e60060d402b3 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_dev.c -@@ -0,0 +1,35 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG user space device file interface -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+ -+#include "lrng_interface_dev_common.h" -+ -+static const struct file_operations lrng_fops = { -+ .read = lrng_drng_read_block, -+ .write = lrng_drng_write, -+ .poll = lrng_random_poll, -+ .unlocked_ioctl = lrng_ioctl, -+ .compat_ioctl = compat_ptr_ioctl, -+ .fasync = lrng_fasync, -+ .llseek = noop_llseek, -+}; -+ -+static struct miscdevice lrng_miscdev = { -+ .minor = MISC_DYNAMIC_MINOR, -+ .name = "lrng", -+ .nodename = "lrng", -+ .fops = &lrng_fops, -+ .mode = 0666 -+}; -+ -+static int __init lrng_dev_if_mod_init(void) -+{ -+ return misc_register(&lrng_miscdev); -+} -+device_initcall(lrng_dev_if_mod_init); -diff --git a/drivers/char/lrng/lrng_interface_dev_common.c b/drivers/char/lrng/lrng_interface_dev_common.c -new file mode 100644 -index 000000000000..09d0563074a4 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_dev_common.c -@@ -0,0 +1,314 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG User and kernel space interfaces -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+ -+DECLARE_WAIT_QUEUE_HEAD(lrng_write_wait); -+static struct fasync_struct *fasync; -+ -+static bool lrng_seed_hw = true; /* Allow HW to provide seed */ -+static bool lrng_seed_user = true; /* Allow user space to provide seed */ -+ -+/********************************** Helper ***********************************/ -+ -+static u32 lrng_get_aux_ent(void) -+{ -+ return lrng_es[lrng_ext_es_aux]->curr_entropy(0); -+} -+ -+/* Is the DRNG seed level too low? */ -+bool lrng_need_entropy(void) -+{ -+ return (lrng_get_aux_ent() < lrng_write_wakeup_bits); -+} -+ -+void lrng_writer_wakeup(void) -+{ -+ if (lrng_need_entropy() && wq_has_sleeper(&lrng_write_wait)) { -+ wake_up_interruptible(&lrng_write_wait); -+ kill_fasync(&fasync, SIGIO, POLL_OUT); -+ } -+} -+ -+void lrng_init_wakeup_dev(void) -+{ -+ kill_fasync(&fasync, SIGIO, POLL_IN); -+} -+ -+/* External entropy provider is allowed to provide seed data */ -+bool lrng_state_exseed_allow(enum lrng_external_noise_source source) -+{ -+ if (source == lrng_noise_source_hw) -+ return lrng_seed_hw; -+ return lrng_seed_user; -+} -+ -+/* Enable / disable external entropy provider to furnish seed */ -+void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) -+{ -+ /* -+ * If the LRNG is not yet operational, allow all entropy sources -+ * to deliver data unconditionally to get fully seeded asap. -+ */ -+ if (!lrng_state_operational()) -+ return; -+ -+ if (source == lrng_noise_source_hw) -+ lrng_seed_hw = type; -+ else -+ lrng_seed_user = type; -+} -+ -+void lrng_state_exseed_allow_all(void) -+{ -+ lrng_state_exseed_set(lrng_noise_source_hw, true); -+ lrng_state_exseed_set(lrng_noise_source_user, true); -+} -+ -+/************************ LRNG user output interfaces *************************/ -+ -+ssize_t lrng_read_seed(char __user *buf, size_t nbytes, unsigned int flags) -+{ -+ ssize_t ret = 0; -+ u64 t[(sizeof(struct entropy_buf) + 3 * sizeof(u64) - 1) / sizeof(u64)]; -+ -+ memset(t, 0, sizeof(t)); -+ ret = lrng_get_seed(t, min_t(size_t, nbytes, sizeof(t)), flags); -+ if (ret == -EMSGSIZE && copy_to_user(buf, t, sizeof(u64))) { -+ ret = -EFAULT; -+ } else if (ret > 0 && copy_to_user(buf, t, ret)) { -+ ret = -EFAULT; -+ } -+ memzero_explicit(t, sizeof(t)); -+ -+ return ret; -+} -+ -+ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr) -+{ -+ ssize_t ret = 0; -+ u8 tmpbuf[LRNG_DRNG_BLOCKSIZE] __aligned(LRNG_KCAPI_ALIGN); -+ u8 *tmp_large = NULL, *tmp = tmpbuf; -+ u32 tmplen = sizeof(tmpbuf); -+ -+ if (nbytes == 0) -+ return 0; -+ -+ /* -+ * Satisfy large read requests -- as the common case are smaller -+ * request sizes, such as 16 or 32 bytes, avoid a kmalloc overhead for -+ * those by using the stack variable of tmpbuf. -+ */ -+ if (!CONFIG_BASE_SMALL && (nbytes > sizeof(tmpbuf))) { -+ tmplen = min_t(u32, nbytes, LRNG_DRNG_MAX_REQSIZE); -+ tmp_large = kmalloc(tmplen + LRNG_KCAPI_ALIGN, GFP_KERNEL); -+ if (!tmp_large) -+ tmplen = sizeof(tmpbuf); -+ else -+ tmp = PTR_ALIGN(tmp_large, LRNG_KCAPI_ALIGN); -+ } -+ -+ while (nbytes) { -+ u32 todo = min_t(u32, nbytes, tmplen); -+ int rc = 0; -+ -+ /* Reschedule if we received a large request. */ -+ if ((tmp_large) && need_resched()) { -+ if (signal_pending(current)) { -+ if (ret == 0) -+ ret = -ERESTARTSYS; -+ break; -+ } -+ schedule(); -+ } -+ -+ rc = lrng_drng_get_sleep(tmp, todo, pr); -+ if (rc <= 0) { -+ if (rc < 0) -+ ret = rc; -+ break; -+ } -+ if (copy_to_user(buf, tmp, rc)) { -+ ret = -EFAULT; -+ break; -+ } -+ -+ nbytes -= rc; -+ buf += rc; -+ ret += rc; -+ } -+ -+ /* Wipe data just returned from memory */ -+ if (tmp_large) -+ kfree_sensitive(tmp_large); -+ else -+ memzero_explicit(tmpbuf, sizeof(tmpbuf)); -+ -+ return ret; -+} -+ -+ssize_t lrng_read_common_block(int nonblock, int pr, -+ char __user *buf, size_t nbytes) -+{ -+ int ret; -+ -+ if (nbytes == 0) -+ return 0; -+ -+ ret = lrng_drng_sleep_while_nonoperational(nonblock); -+ if (ret) -+ return ret; -+ -+ return lrng_read_common(buf, nbytes, !!pr); -+} -+ -+ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, -+ loff_t *ppos) -+{ -+ return lrng_read_common_block(file->f_flags & O_NONBLOCK, -+ file->f_flags & O_SYNC, buf, nbytes); -+} -+ -+__poll_t lrng_random_poll(struct file *file, poll_table *wait) -+{ -+ __poll_t mask; -+ -+ poll_wait(file, &lrng_init_wait, wait); -+ poll_wait(file, &lrng_write_wait, wait); -+ mask = 0; -+ if (lrng_state_operational()) -+ mask |= EPOLLIN | EPOLLRDNORM; -+ if (lrng_need_entropy() || -+ lrng_state_exseed_allow(lrng_noise_source_user)) { -+ lrng_state_exseed_set(lrng_noise_source_user, false); -+ mask |= EPOLLOUT | EPOLLWRNORM; -+ } -+ return mask; -+} -+ -+ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, -+ u32 entropy_bits) -+{ -+ ssize_t ret = 0; -+ u8 buf[64] __aligned(LRNG_KCAPI_ALIGN); -+ const char __user *p = buffer; -+ u32 orig_entropy_bits = entropy_bits; -+ -+ if (!lrng_get_available()) { -+ ret = lrng_drng_initalize(); -+ if (!ret) -+ return ret; -+ } -+ -+ count = min_t(size_t, count, INT_MAX); -+ while (count > 0) { -+ size_t bytes = min_t(size_t, count, sizeof(buf)); -+ u32 ent = min_t(u32, bytes<<3, entropy_bits); -+ -+ if (copy_from_user(&buf, p, bytes)) -+ return -EFAULT; -+ /* Inject data into entropy pool */ -+ lrng_pool_insert_aux(buf, bytes, ent); -+ -+ count -= bytes; -+ p += bytes; -+ ret += bytes; -+ entropy_bits -= ent; -+ -+ cond_resched(); -+ } -+ -+ /* Force reseed of DRNG during next data request. */ -+ if (!orig_entropy_bits) -+ lrng_drng_force_reseed(); -+ -+ return ret; -+} -+ -+ssize_t lrng_drng_write(struct file *file, const char __user *buffer, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_drng_write_common(buffer, count, 0); -+} -+ -+long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg) -+{ -+ u32 digestsize_bits; -+ int size, ent_count_bits; -+ int __user *p = (int __user *)arg; -+ -+ switch (cmd) { -+ case RNDGETENTCNT: -+ ent_count_bits = lrng_avail_entropy_aux(); -+ if (put_user(ent_count_bits, p)) -+ return -EFAULT; -+ return 0; -+ case RNDADDTOENTCNT: -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ if (get_user(ent_count_bits, p)) -+ return -EFAULT; -+ ent_count_bits = (int)lrng_get_aux_ent() + ent_count_bits; -+ if (ent_count_bits < 0) -+ ent_count_bits = 0; -+ digestsize_bits = lrng_get_digestsize(); -+ if (ent_count_bits > digestsize_bits) -+ ent_count_bits = digestsize_bits; -+ lrng_pool_set_entropy(ent_count_bits); -+ return 0; -+ case RNDADDENTROPY: -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ if (get_user(ent_count_bits, p++)) -+ return -EFAULT; -+ if (ent_count_bits < 0) -+ return -EINVAL; -+ if (get_user(size, p++)) -+ return -EFAULT; -+ if (size < 0) -+ return -EINVAL; -+ /* there cannot be more entropy than data */ -+ ent_count_bits = min(ent_count_bits, size<<3); -+ return lrng_drng_write_common((const char __user *)p, size, -+ ent_count_bits); -+ case RNDZAPENTCNT: -+ case RNDCLEARPOOL: -+ /* Clear the entropy pool counter. */ -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ lrng_pool_set_entropy(0); -+ return 0; -+ case RNDRESEEDCRNG: -+ /* -+ * We leave the capability check here since it is present -+ * in the upstream's RNG implementation. Yet, user space -+ * can trigger a reseed as easy as writing into /dev/random -+ * or /dev/urandom where no privilege is needed. -+ */ -+ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ /* Force a reseed of all DRNGs */ -+ lrng_drng_force_reseed(); -+ return 0; -+ default: -+ return -EINVAL; -+ } -+} -+EXPORT_SYMBOL(lrng_ioctl); -+ -+int lrng_fasync(int fd, struct file *filp, int on) -+{ -+ return fasync_helper(fd, filp, on, &fasync); -+} -diff --git a/drivers/char/lrng/lrng_interface_dev_common.h b/drivers/char/lrng/lrng_interface_dev_common.h -new file mode 100644 -index 000000000000..9e6603ad8af4 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_dev_common.h -@@ -0,0 +1,51 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_INTERFACE_DEV_COMMON_H -+#define _LRNG_INTERFACE_DEV_COMMON_H -+ -+#include -+#include -+ -+/******************* Upstream functions hooked into the LRNG ******************/ -+enum lrng_external_noise_source { -+ lrng_noise_source_hw, -+ lrng_noise_source_user -+}; -+ -+#ifdef CONFIG_LRNG_COMMON_DEV_IF -+void lrng_writer_wakeup(void); -+void lrng_init_wakeup_dev(void); -+void lrng_state_exseed_set(enum lrng_external_noise_source source, bool type); -+void lrng_state_exseed_allow_all(void); -+#else /* CONFIG_LRNG_COMMON_DEV_IF */ -+static inline void lrng_writer_wakeup(void) { } -+static inline void lrng_init_wakeup_dev(void) { } -+static inline void -+lrng_state_exseed_set(enum lrng_external_noise_source source, bool type) { } -+static inline void lrng_state_exseed_allow_all(void) { } -+#endif /* CONFIG_LRNG_COMMON_DEV_IF */ -+ -+/****** Downstream service functions to actual interface implementations ******/ -+ -+bool lrng_state_exseed_allow(enum lrng_external_noise_source source); -+int lrng_fasync(int fd, struct file *filp, int on); -+long lrng_ioctl(struct file *f, unsigned int cmd, unsigned long arg); -+ssize_t lrng_drng_write(struct file *file, const char __user *buffer, -+ size_t count, loff_t *ppos); -+ssize_t lrng_drng_write_common(const char __user *buffer, size_t count, -+ u32 entropy_bits); -+__poll_t lrng_random_poll(struct file *file, poll_table *wait); -+ssize_t lrng_read_common_block(int nonblock, int pr, -+ char __user *buf, size_t nbytes); -+ssize_t lrng_drng_read_block(struct file *file, char __user *buf, size_t nbytes, -+ loff_t *ppos); -+ssize_t lrng_read_seed(char __user *buf, size_t nbytes, unsigned int flags); -+ssize_t lrng_read_common(char __user *buf, size_t nbytes, bool pr); -+bool lrng_need_entropy(void); -+ -+extern struct wait_queue_head lrng_write_wait; -+ -+#endif /* _LRNG_INTERFACE_DEV_COMMON_H */ -diff --git a/drivers/char/lrng/lrng_interface_hwrand.c b/drivers/char/lrng/lrng_interface_hwrand.c -new file mode 100644 -index 000000000000..e841eea13348 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_hwrand.c -@@ -0,0 +1,68 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG interface with the HW-Random framework -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+ -+static int lrng_hwrand_if_random(struct hwrng *rng, void *buf, size_t max, -+ bool wait) -+{ -+ /* -+ * lrng_get_random_bytes_full not called as we cannot block. -+ * -+ * Note: We should either adjust .quality below depending on -+ * rng_is_initialized() or block here, but neither is not supported by -+ * the hw_rand framework. -+ */ -+ lrng_get_random_bytes(buf, max); -+ return (int)max; -+} -+ -+static struct hwrng lrng_hwrand = { -+ .name = "lrng", -+ .init = NULL, -+ .cleanup = NULL, -+ .read = lrng_hwrand_if_random, -+ -+ /* -+ * We set .quality only in case the LRNG does not provide the common -+ * interfaces or does not use the legacy RNG as entropy source. This -+ * shall avoid that the LRNG automatically spawns the hw_rand -+ * framework's hwrng kernel thread to feed data into -+ * add_hwgenerator_randomness. When the LRNG implements the common -+ * interfaces, this function feeds the data directly into the LRNG. -+ * If the LRNG uses the legacy RNG as entropy source, -+ * add_hwgenerator_randomness is implemented by the legacy RNG, but -+ * still eventually feeds the data into the LRNG. We should avoid such -+ * circular loops. -+ * -+ * We can specify full entropy here, because the LRNG is designed -+ * to provide full entropy. -+ */ -+#if !defined(CONFIG_LRNG_RANDOM_IF) && \ -+ !defined(CONFIG_LRNG_KERNEL_RNG) -+ .quality = 1024, -+#endif -+}; -+ -+static int __init lrng_hwrand_if_mod_init(void) -+{ -+ return hwrng_register(&lrng_hwrand); -+} -+ -+static void __exit lrng_hwrand_if_mod_exit(void) -+{ -+ hwrng_unregister(&lrng_hwrand); -+} -+ -+module_init(lrng_hwrand_if_mod_init); -+module_exit(lrng_hwrand_if_mod_exit); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager HW-Random Interface"); -diff --git a/drivers/char/lrng/lrng_interface_kcapi.c b/drivers/char/lrng/lrng_interface_kcapi.c -new file mode 100644 -index 000000000000..4cb511f8088e ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_kcapi.c -@@ -0,0 +1,129 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG interface with the RNG framework of the kernel crypto API -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+ -+static int lrng_kcapi_if_init(struct crypto_tfm *tfm) -+{ -+ return 0; -+} -+ -+static void lrng_kcapi_if_cleanup(struct crypto_tfm *tfm) { } -+ -+static int lrng_kcapi_if_reseed(const u8 *src, unsigned int slen) -+{ -+ int ret; -+ -+ if (!slen) -+ return 0; -+ -+ /* Insert caller-provided data without crediting entropy */ -+ ret = lrng_pool_insert_aux((u8 *)src, slen, 0); -+ if (ret) -+ return ret; -+ -+ /* Make sure the new data is immediately available to DRNG */ -+ lrng_drng_force_reseed(); -+ -+ return 0; -+} -+ -+static int lrng_kcapi_if_random(struct crypto_rng *tfm, -+ const u8 *src, unsigned int slen, -+ u8 *rdata, unsigned int dlen) -+{ -+ int ret = lrng_kcapi_if_reseed(src, slen); -+ -+ if (!ret) -+ lrng_get_random_bytes_full(rdata, dlen); -+ -+ return ret; -+} -+ -+static int lrng_kcapi_if_reset(struct crypto_rng *tfm, -+ const u8 *seed, unsigned int slen) -+{ -+ return lrng_kcapi_if_reseed(seed, slen); -+} -+ -+static struct rng_alg lrng_alg = { -+ .generate = lrng_kcapi_if_random, -+ .seed = lrng_kcapi_if_reset, -+ .seedsize = 0, -+ .base = { -+ .cra_name = "stdrng", -+ .cra_driver_name = "lrng", -+ .cra_priority = 500, -+ .cra_ctxsize = 0, -+ .cra_module = THIS_MODULE, -+ .cra_init = lrng_kcapi_if_init, -+ .cra_exit = lrng_kcapi_if_cleanup, -+ -+ } -+}; -+ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+static int lrng_kcapi_if_random_atomic(struct crypto_rng *tfm, -+ const u8 *src, unsigned int slen, -+ u8 *rdata, unsigned int dlen) -+{ -+ int ret = lrng_kcapi_if_reseed(src, slen); -+ -+ if (!ret) -+ lrng_get_random_bytes(rdata, dlen); -+ -+ return ret; -+} -+ -+static struct rng_alg lrng_alg_atomic = { -+ .generate = lrng_kcapi_if_random_atomic, -+ .seed = lrng_kcapi_if_reset, -+ .seedsize = 0, -+ .base = { -+ .cra_name = "lrng_atomic", -+ .cra_driver_name = "lrng_atomic", -+ .cra_priority = 100, -+ .cra_ctxsize = 0, -+ .cra_module = THIS_MODULE, -+ .cra_init = lrng_kcapi_if_init, -+ .cra_exit = lrng_kcapi_if_cleanup, -+ -+ } -+}; -+#endif /* CONFIG_LRNG_DRNG_ATOMIC */ -+ -+static int __init lrng_kcapi_if_mod_init(void) -+{ -+ return -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+ crypto_register_rng(&lrng_alg_atomic) ?: -+#endif -+ crypto_register_rng(&lrng_alg); -+} -+ -+static void __exit lrng_kcapi_if_mod_exit(void) -+{ -+ crypto_unregister_rng(&lrng_alg); -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+ crypto_unregister_rng(&lrng_alg_atomic); -+#endif -+} -+ -+module_init(lrng_kcapi_if_mod_init); -+module_exit(lrng_kcapi_if_mod_exit); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Stephan Mueller "); -+MODULE_DESCRIPTION("Entropy Source and DRNG Manager kernel crypto API RNG framework interface"); -+MODULE_ALIAS_CRYPTO("lrng"); -+MODULE_ALIAS_CRYPTO("lrng_atomic"); -+MODULE_ALIAS_CRYPTO("stdrng"); -diff --git a/drivers/char/lrng/lrng_interface_random_kernel.c b/drivers/char/lrng/lrng_interface_random_kernel.c -new file mode 100644 -index 000000000000..471390bfd70c ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_random_kernel.c -@@ -0,0 +1,214 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Kernel space interfaces API/ABI compliant to linux/random.h -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+#include "lrng_interface_random_kernel.h" -+ -+/********************************** Helper ***********************************/ -+ -+static bool lrng_trust_bootloader __initdata = -+ IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER); -+ -+static int __init lrng_parse_trust_bootloader(char *arg) -+{ -+ return kstrtobool(arg, &lrng_trust_bootloader); -+} -+early_param("random.trust_bootloader", lrng_parse_trust_bootloader); -+ -+int __init random_init(const char *command_line) -+{ -+ int ret = lrng_rand_initialize(); -+ -+ lrng_pool_insert_aux(command_line, strlen(command_line), 0); -+ return ret; -+} -+ -+/************************ LRNG kernel input interfaces ************************/ -+ -+/* -+ * add_hwgenerator_randomness() - Interface for in-kernel drivers of true -+ * hardware RNGs. -+ * -+ * Those devices may produce endless random bits and will be throttled -+ * when our pool is full. -+ * -+ * @buffer: buffer holding the entropic data from HW noise sources to be used to -+ * insert into entropy pool. -+ * @count: length of buffer -+ * @entropy_bits: amount of entropy in buffer (value is in bits) -+ */ -+void add_hwgenerator_randomness(const void *buffer, size_t count, -+ size_t entropy_bits) -+{ -+ /* -+ * Suspend writing if we are fully loaded with entropy. -+ * We'll be woken up again once below lrng_write_wakeup_thresh, -+ * or when the calling thread is about to terminate. -+ */ -+ wait_event_interruptible(lrng_write_wait, -+ lrng_need_entropy() || -+ lrng_state_exseed_allow(lrng_noise_source_hw) || -+ kthread_should_stop()); -+ lrng_state_exseed_set(lrng_noise_source_hw, false); -+ lrng_pool_insert_aux(buffer, count, entropy_bits); -+} -+EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); -+ -+/* -+ * add_bootloader_randomness() - Handle random seed passed by bootloader. -+ * -+ * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise -+ * it would be regarded as device data. -+ * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. -+ * -+ * @buf: buffer holding the entropic data from HW noise sources to be used to -+ * insert into entropy pool. -+ * @size: length of buffer -+ */ -+void __init add_bootloader_randomness(const void *buf, size_t size) -+{ -+ lrng_pool_insert_aux(buf, size, lrng_trust_bootloader ? size * 8 : 0); -+} -+ -+/* -+ * Callback for HID layer -- use the HID event values to stir the entropy pool -+ */ -+void add_input_randomness(unsigned int type, unsigned int code, -+ unsigned int value) -+{ -+ static unsigned char last_value; -+ -+ /* ignore autorepeat and the like */ -+ if (value == last_value) -+ return; -+ -+ last_value = value; -+ -+ lrng_irq_array_add_u32((type << 4) ^ code ^ (code >> 4) ^ value); -+} -+EXPORT_SYMBOL_GPL(add_input_randomness); -+ -+/* -+ * add_device_randomness() - Add device- or boot-specific data to the entropy -+ * pool to help initialize it. -+ * -+ * None of this adds any entropy; it is meant to avoid the problem of -+ * the entropy pool having similar initial state across largely -+ * identical devices. -+ * -+ * @buf: buffer holding the entropic data from HW noise sources to be used to -+ * insert into entropy pool. -+ * @size: length of buffer -+ */ -+void add_device_randomness(const void *buf, size_t size) -+{ -+ lrng_pool_insert_aux((u8 *)buf, size, 0); -+} -+EXPORT_SYMBOL(add_device_randomness); -+ -+#ifdef CONFIG_BLOCK -+void rand_initialize_disk(struct gendisk *disk) { } -+void add_disk_randomness(struct gendisk *disk) { } -+EXPORT_SYMBOL(add_disk_randomness); -+#endif -+ -+#ifndef CONFIG_LRNG_IRQ -+void add_interrupt_randomness(int irq) { } -+EXPORT_SYMBOL(add_interrupt_randomness); -+#endif -+ -+#if IS_ENABLED(CONFIG_VMGENID) -+static BLOCKING_NOTIFIER_HEAD(lrng_vmfork_chain); -+ -+/* -+ * Handle a new unique VM ID, which is unique, not secret, so we -+ * don't credit it, but we do immediately force a reseed after so -+ * that it's used by the crng posthaste. -+ */ -+void add_vmfork_randomness(const void *unique_vm_id, size_t size) -+{ -+ add_device_randomness(unique_vm_id, size); -+ if (lrng_state_operational()) -+ lrng_drng_force_reseed(); -+ blocking_notifier_call_chain(&lrng_vmfork_chain, 0, NULL); -+} -+#if IS_MODULE(CONFIG_VMGENID) -+EXPORT_SYMBOL_GPL(add_vmfork_randomness); -+#endif -+ -+int register_random_vmfork_notifier(struct notifier_block *nb) -+{ -+ return blocking_notifier_chain_register(&lrng_vmfork_chain, nb); -+} -+EXPORT_SYMBOL_GPL(register_random_vmfork_notifier); -+ -+int unregister_random_vmfork_notifier(struct notifier_block *nb) -+{ -+ return blocking_notifier_chain_unregister(&lrng_vmfork_chain, nb); -+} -+EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier); -+#endif -+ -+/*********************** LRNG kernel output interfaces ************************/ -+ -+/* -+ * get_random_bytes() - Provider of cryptographic strong random numbers for -+ * kernel-internal usage. -+ * -+ * This function is appropriate for all in-kernel use cases. However, -+ * it will always use the ChaCha20 DRNG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+void get_random_bytes(void *buf, size_t nbytes) -+{ -+ lrng_get_random_bytes(buf, nbytes); -+} -+EXPORT_SYMBOL(get_random_bytes); -+ -+/* -+ * wait_for_random_bytes() - Wait for the LRNG to be seeded and thus -+ * guaranteed to supply cryptographically secure random numbers. -+ * -+ * This applies to: the /dev/urandom device, the get_random_bytes function, -+ * and the get_random_{u32,u64,int,long} family of functions. Using any of -+ * these functions without first calling this function forfeits the guarantee -+ * of security. -+ * -+ * Return: -+ * * 0 if the LRNG has been seeded. -+ * * -ERESTARTSYS if the function was interrupted by a signal. -+ */ -+int wait_for_random_bytes(void) -+{ -+ return lrng_drng_sleep_while_non_min_seeded(); -+} -+EXPORT_SYMBOL(wait_for_random_bytes); -+ -+/* -+ * Returns whether or not the LRNG has been seeded. -+ * -+ * Returns: true if the urandom pool has been seeded. -+ * false if the urandom pool has not been seeded. -+ */ -+bool rng_is_initialized(void) -+{ -+ return lrng_state_operational(); -+} -+EXPORT_SYMBOL(rng_is_initialized); -diff --git a/drivers/char/lrng/lrng_interface_random_kernel.h b/drivers/char/lrng/lrng_interface_random_kernel.h -new file mode 100644 -index 000000000000..f4d09d1fbb14 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_random_kernel.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_INTERFACE_RANDOM_H -+#define _LRNG_INTERFACE_RANDOM_H -+ -+#ifdef CONFIG_LRNG_RANDOM_IF -+void invalidate_batched_entropy(void); -+#else /* CONFIG_LRNG_RANDOM_IF */ -+static inline void invalidate_batched_entropy(void) { } -+#endif /* CONFIG_LRNG_RANDOM_IF */ -+ -+#endif /* _LRNG_INTERFACE_RANDOM_H */ -diff --git a/drivers/char/lrng/lrng_interface_random_user.c b/drivers/char/lrng/lrng_interface_random_user.c -new file mode 100644 -index 000000000000..d12e883804d9 ---- /dev/null -+++ b/drivers/char/lrng/lrng_interface_random_user.c -@@ -0,0 +1,104 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG Common user space interfaces compliant to random(4), random(7) and -+ * getrandom(2) man pages. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+ -+static ssize_t lrng_drng_read(struct file *file, char __user *buf, -+ size_t nbytes, loff_t *ppos) -+{ -+ if (!lrng_state_min_seeded()) -+ pr_notice_ratelimited("%s - use of insufficiently seeded DRNG (%zu bytes read)\n", -+ current->comm, nbytes); -+ else if (!lrng_state_operational()) -+ pr_debug_ratelimited("%s - use of not fully seeded DRNG (%zu bytes read)\n", -+ current->comm, nbytes); -+ -+ return lrng_read_common(buf, nbytes, false); -+} -+ -+const struct file_operations random_fops = { -+ .read = lrng_drng_read_block, -+ .write = lrng_drng_write, -+ .poll = lrng_random_poll, -+ .unlocked_ioctl = lrng_ioctl, -+ .compat_ioctl = compat_ptr_ioctl, -+ .fasync = lrng_fasync, -+ .llseek = noop_llseek, -+}; -+ -+const struct file_operations urandom_fops = { -+ .read = lrng_drng_read, -+ .write = lrng_drng_write, -+ .unlocked_ioctl = lrng_ioctl, -+ .compat_ioctl = compat_ptr_ioctl, -+ .fasync = lrng_fasync, -+ .llseek = noop_llseek, -+}; -+ -+/* -+ * GRND_SEED -+ * -+ * This flag requests to provide the data directly from the entropy sources. -+ * -+ * The behavior of the call is exactly as outlined for the function -+ * lrng_get_seed in lrng.h. -+ */ -+#define GRND_SEED 0x0010 -+ -+/* -+ * GRND_FULLY_SEEDED -+ * -+ * This flag indicates whether the caller wants to reseed a DRNG that is already -+ * fully seeded. See esdm_get_seed in lrng.h for details. -+ */ -+#define GRND_FULLY_SEEDED 0x0020 -+ -+SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, -+ unsigned int, flags) -+{ -+ if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE| -+ GRND_SEED|GRND_FULLY_SEEDED)) -+ return -EINVAL; -+ -+ /* -+ * Requesting insecure and blocking randomness at the same time makes -+ * no sense. -+ */ -+ if ((flags & -+ (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) -+ return -EINVAL; -+ if ((flags & -+ (GRND_INSECURE|GRND_SEED)) == (GRND_INSECURE|GRND_SEED)) -+ return -EINVAL; -+ if ((flags & -+ (GRND_RANDOM|GRND_SEED)) == (GRND_RANDOM|GRND_SEED)) -+ return -EINVAL; -+ -+ if (count > INT_MAX) -+ count = INT_MAX; -+ -+ if (flags & GRND_INSECURE) { -+ return lrng_drng_read(NULL, buf, count, NULL); -+ } else if (flags & GRND_SEED) { -+ unsigned int seed_flags = (flags & GRND_NONBLOCK) ? -+ LRNG_GET_SEED_NONBLOCK : 0; -+ -+ seed_flags |= (flags & GRND_FULLY_SEEDED) ? -+ LRNG_GET_SEED_FULLY_SEEDED : 0; -+ return lrng_read_seed(buf, count, seed_flags); -+ } -+ -+ return lrng_read_common_block(flags & GRND_NONBLOCK, -+ flags & GRND_RANDOM, buf, count); -+} -diff --git a/drivers/char/lrng/lrng_numa.c b/drivers/char/lrng/lrng_numa.c -new file mode 100644 -index 000000000000..d74dd8df2843 ---- /dev/null -+++ b/drivers/char/lrng/lrng_numa.c -@@ -0,0 +1,124 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG NUMA support -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_irq.h" -+#include "lrng_es_mgr.h" -+#include "lrng_numa.h" -+#include "lrng_proc.h" -+ -+static struct lrng_drng **lrng_drng __read_mostly = NULL; -+ -+struct lrng_drng **lrng_drng_instances(void) -+{ -+ /* counterpart to cmpxchg_release in _lrng_drngs_numa_alloc */ -+ return READ_ONCE(lrng_drng); -+} -+ -+/* Allocate the data structures for the per-NUMA node DRNGs */ -+static void _lrng_drngs_numa_alloc(struct work_struct *work) -+{ -+ struct lrng_drng **drngs; -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ u32 node; -+ bool init_drng_used = false; -+ -+ mutex_lock(&lrng_crypto_cb_update); -+ -+ /* per-NUMA-node DRNGs are already present */ -+ if (lrng_drng) -+ goto unlock; -+ -+ /* Make sure the initial DRNG is initialized and its drng_cb is set */ -+ if (lrng_drng_initalize()) -+ goto err; -+ -+ drngs = kcalloc(nr_node_ids, sizeof(void *), GFP_KERNEL|__GFP_NOFAIL); -+ for_each_online_node(node) { -+ struct lrng_drng *drng; -+ -+ if (!init_drng_used) { -+ drngs[node] = lrng_drng_init; -+ init_drng_used = true; -+ continue; -+ } -+ -+ drng = kmalloc_node(sizeof(struct lrng_drng), -+ GFP_KERNEL|__GFP_NOFAIL, node); -+ memset(drng, 0, sizeof(lrng_drng)); -+ -+ if (lrng_drng_alloc_common(drng, lrng_drng_init->drng_cb)) { -+ kfree(drng); -+ goto err; -+ } -+ -+ drng->hash_cb = lrng_drng_init->hash_cb; -+ drng->hash = lrng_drng_init->hash_cb->hash_alloc(); -+ if (IS_ERR(drng->hash)) { -+ lrng_drng_init->drng_cb->drng_dealloc(drng->drng); -+ kfree(drng); -+ goto err; -+ } -+ -+ mutex_init(&drng->lock); -+ rwlock_init(&drng->hash_lock); -+ -+ /* -+ * No reseeding of NUMA DRNGs from previous DRNGs as this -+ * would complicate the code. Let it simply reseed. -+ */ -+ drngs[node] = drng; -+ -+ lrng_pool_inc_numa_node(); -+ pr_info("DRNG and entropy pool read hash for NUMA node %d allocated\n", -+ node); -+ } -+ -+ /* counterpart to READ_ONCE in lrng_drng_instances */ -+ if (!cmpxchg_release(&lrng_drng, NULL, drngs)) { -+ lrng_pool_all_numa_nodes_seeded(false); -+ goto unlock; -+ } -+ -+err: -+ for_each_online_node(node) { -+ struct lrng_drng *drng = drngs[node]; -+ -+ if (drng == lrng_drng_init) -+ continue; -+ -+ if (drng) { -+ drng->hash_cb->hash_dealloc(drng->hash); -+ drng->drng_cb->drng_dealloc(drng->drng); -+ kfree(drng); -+ } -+ } -+ kfree(drngs); -+ -+unlock: -+ mutex_unlock(&lrng_crypto_cb_update); -+} -+ -+static DECLARE_WORK(lrng_drngs_numa_alloc_work, _lrng_drngs_numa_alloc); -+ -+static void lrng_drngs_numa_alloc(void) -+{ -+ schedule_work(&lrng_drngs_numa_alloc_work); -+} -+ -+static int __init lrng_numa_init(void) -+{ -+ lrng_drngs_numa_alloc(); -+ return 0; -+} -+ -+late_initcall(lrng_numa_init); -diff --git a/drivers/char/lrng/lrng_numa.h b/drivers/char/lrng/lrng_numa.h -new file mode 100644 -index 000000000000..dc8dff9816ee ---- /dev/null -+++ b/drivers/char/lrng/lrng_numa.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_NUMA_H -+#define _LRNG_NUMA_H -+ -+#ifdef CONFIG_NUMA -+struct lrng_drng **lrng_drng_instances(void); -+#else /* CONFIG_NUMA */ -+static inline struct lrng_drng **lrng_drng_instances(void) { return NULL; } -+#endif /* CONFIG_NUMA */ -+ -+#endif /* _LRNG_NUMA_H */ -diff --git a/drivers/char/lrng/lrng_proc.c b/drivers/char/lrng/lrng_proc.c -new file mode 100644 -index 000000000000..92329cc6ef28 ---- /dev/null -+++ b/drivers/char/lrng/lrng_proc.c -@@ -0,0 +1,74 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG proc interfaces -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_proc.h" -+ -+/* Number of online DRNGs */ -+static u32 numa_drngs = 1; -+ -+void lrng_pool_inc_numa_node(void) -+{ -+ numa_drngs++; -+} -+ -+static int lrng_proc_type_show(struct seq_file *m, void *v) -+{ -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ unsigned char buf[270]; -+ u32 i; -+ -+ mutex_lock(&lrng_drng_init->lock); -+ snprintf(buf, sizeof(buf), -+ "DRNG name: %s\n" -+ "LRNG security strength in bits: %d\n" -+ "Number of DRNG instances: %u\n" -+ "Standards compliance: %s%s\n" -+ "LRNG minimally seeded: %s\n" -+ "LRNG fully seeded: %s\n" -+ "LRNG entropy level: %u\n", -+ lrng_drng_init->drng_cb->drng_name(), -+ lrng_security_strength(), -+ numa_drngs, -+ lrng_sp80090c_compliant() ? "SP800-90C " : "", -+ lrng_ntg1_compliant() ? "NTG.1 " : "", -+ lrng_state_min_seeded() ? "true" : "false", -+ lrng_state_fully_seeded() ? "true" : "false", -+ lrng_avail_entropy()); -+ seq_write(m, buf, strlen(buf)); -+ -+ for_each_lrng_es(i) { -+ snprintf(buf, sizeof(buf), -+ "Entropy Source %u properties:\n" -+ " Name: %s\n", -+ i, lrng_es[i]->name); -+ seq_write(m, buf, strlen(buf)); -+ -+ buf[0] = '\0'; -+ lrng_es[i]->state(buf, sizeof(buf)); -+ seq_write(m, buf, strlen(buf)); -+ } -+ -+ mutex_unlock(&lrng_drng_init->lock); -+ -+ return 0; -+} -+ -+static int __init lrng_proc_type_init(void) -+{ -+ proc_create_single("lrng_type", 0444, NULL, &lrng_proc_type_show); -+ return 0; -+} -+ -+module_init(lrng_proc_type_init); -diff --git a/drivers/char/lrng/lrng_proc.h b/drivers/char/lrng/lrng_proc.h -new file mode 100644 -index 000000000000..c653274f1954 ---- /dev/null -+++ b/drivers/char/lrng/lrng_proc.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_PROC_H -+#define _LRNG_PROC_H -+ -+#ifdef CONFIG_SYSCTL -+void lrng_pool_inc_numa_node(void); -+#else -+static inline void lrng_pool_inc_numa_node(void) { } -+#endif -+ -+#endif /* _LRNG_PROC_H */ -diff --git a/drivers/char/lrng/lrng_selftest.c b/drivers/char/lrng/lrng_selftest.c -new file mode 100644 -index 000000000000..15f1e4a2a719 ---- /dev/null -+++ b/drivers/char/lrng/lrng_selftest.c -@@ -0,0 +1,397 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG power-on and on-demand self-test -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+/* -+ * In addition to the self-tests below, the following LRNG components -+ * are covered with self-tests during regular operation: -+ * -+ * * power-on self-test: SP800-90A DRBG provided by the Linux kernel crypto API -+ * * power-on self-test: PRNG provided by the Linux kernel crypto API -+ * * runtime test: Raw noise source data testing including SP800-90B compliant -+ * tests when enabling CONFIG_LRNG_HEALTH_TESTS -+ * -+ * Additional developer tests present with LRNG code: -+ * * SP800-90B APT and RCT test enforcement validation when enabling -+ * CONFIG_LRNG_APT_BROKEN or CONFIG_LRNG_RCT_BROKEN. -+ * * Collection of raw entropy from the interrupt noise source when enabling -+ * CONFIG_LRNG_TESTING and pulling the data from the kernel with the provided -+ * interface. -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "lrng_drng_chacha20.h" -+#include "lrng_sha.h" -+ -+#define LRNG_SELFTEST_PASSED 0 -+#define LRNG_SEFLTEST_ERROR_TIME (1 << 0) -+#define LRNG_SEFLTEST_ERROR_CHACHA20 (1 << 1) -+#define LRNG_SEFLTEST_ERROR_HASH (1 << 2) -+#define LRNG_SEFLTEST_ERROR_GCD (1 << 3) -+#define LRNG_SELFTEST_NOT_EXECUTED 0xffffffff -+ -+#ifdef CONFIG_LRNG_TIMER_COMMON -+ -+#include "lrng_es_timer_common.h" -+ -+static u32 lrng_data_selftest_ptr = 0; -+static u32 lrng_data_selftest[LRNG_DATA_ARRAY_SIZE]; -+ -+static void lrng_data_process_selftest_insert(u32 time) -+{ -+ u32 ptr = lrng_data_selftest_ptr++ & LRNG_DATA_WORD_MASK; -+ unsigned int array = lrng_data_idx2array(ptr); -+ unsigned int slot = lrng_data_idx2slot(ptr); -+ -+ /* zeroization of slot to ensure the following OR adds the data */ -+ lrng_data_selftest[array] &= -+ ~(lrng_data_slot_val(0xffffffff & LRNG_DATA_SLOTSIZE_MASK, -+ slot)); -+ lrng_data_selftest[array] |= -+ lrng_data_slot_val(time & LRNG_DATA_SLOTSIZE_MASK, slot); -+} -+ -+static void lrng_data_process_selftest_u32(u32 data) -+{ -+ u32 pre_ptr, ptr, mask; -+ unsigned int pre_array; -+ -+ /* Increment pointer by number of slots taken for input value */ -+ lrng_data_selftest_ptr += LRNG_DATA_SLOTS_PER_UINT; -+ -+ /* ptr to current unit */ -+ ptr = lrng_data_selftest_ptr; -+ -+ lrng_data_split_u32(&ptr, &pre_ptr, &mask); -+ -+ /* MSB of data go into previous unit */ -+ pre_array = lrng_data_idx2array(pre_ptr); -+ /* zeroization of slot to ensure the following OR adds the data */ -+ lrng_data_selftest[pre_array] &= ~(0xffffffff & ~mask); -+ lrng_data_selftest[pre_array] |= data & ~mask; -+ -+ /* LSB of data go into current unit */ -+ lrng_data_selftest[lrng_data_idx2array(ptr)] = data & mask; -+} -+ -+static unsigned int lrng_data_process_selftest(void) -+{ -+ u32 time; -+ u32 idx_zero_compare = (0 << 0) | (1 << 8) | (2 << 16) | (3 << 24); -+ u32 idx_one_compare = (4 << 0) | (5 << 8) | (6 << 16) | (7 << 24); -+ u32 idx_last_compare = -+ (((LRNG_DATA_NUM_VALUES - 4) & LRNG_DATA_SLOTSIZE_MASK) << 0) | -+ (((LRNG_DATA_NUM_VALUES - 3) & LRNG_DATA_SLOTSIZE_MASK) << 8) | -+ (((LRNG_DATA_NUM_VALUES - 2) & LRNG_DATA_SLOTSIZE_MASK) << 16) | -+ (((LRNG_DATA_NUM_VALUES - 1) & LRNG_DATA_SLOTSIZE_MASK) << 24); -+ -+ (void)idx_one_compare; -+ -+ /* "poison" the array to verify the operation of the zeroization */ -+ lrng_data_selftest[0] = 0xffffffff; -+ lrng_data_selftest[1] = 0xffffffff; -+ -+ lrng_data_process_selftest_insert(0); -+ /* -+ * Note, when using lrng_data_process_u32() on unaligned ptr, -+ * the first slots will go into next word, and the last slots go -+ * into the previous word. -+ */ -+ lrng_data_process_selftest_u32((4 << 0) | (1 << 8) | (2 << 16) | -+ (3 << 24)); -+ lrng_data_process_selftest_insert(5); -+ lrng_data_process_selftest_insert(6); -+ lrng_data_process_selftest_insert(7); -+ -+ if ((lrng_data_selftest[0] != idx_zero_compare) || -+ (lrng_data_selftest[1] != idx_one_compare)) -+ goto err; -+ -+ /* Reset for next test */ -+ lrng_data_selftest[0] = 0; -+ lrng_data_selftest[1] = 0; -+ lrng_data_selftest_ptr = 0; -+ -+ for (time = 0; time < LRNG_DATA_NUM_VALUES; time++) -+ lrng_data_process_selftest_insert(time); -+ -+ if ((lrng_data_selftest[0] != idx_zero_compare) || -+ (lrng_data_selftest[1] != idx_one_compare) || -+ (lrng_data_selftest[LRNG_DATA_ARRAY_SIZE - 1] != idx_last_compare)) -+ goto err; -+ -+ return LRNG_SELFTEST_PASSED; -+ -+err: -+ pr_err("LRNG data array self-test FAILED\n"); -+ return LRNG_SEFLTEST_ERROR_TIME; -+} -+ -+static unsigned int lrng_gcd_selftest(void) -+{ -+ u32 history[10]; -+ unsigned int i; -+ -+#define LRNG_GCD_SELFTEST 3 -+ for (i = 0; i < ARRAY_SIZE(history); i++) -+ history[i] = i * LRNG_GCD_SELFTEST; -+ -+ if (lrng_gcd_analyze(history, ARRAY_SIZE(history)) == LRNG_GCD_SELFTEST) -+ return LRNG_SELFTEST_PASSED; -+ -+ pr_err("LRNG GCD self-test FAILED\n"); -+ return LRNG_SEFLTEST_ERROR_GCD; -+} -+ -+#else /* CONFIG_LRNG_TIMER_COMMON */ -+ -+static unsigned int lrng_data_process_selftest(void) -+{ -+ return LRNG_SELFTEST_PASSED; -+} -+ -+static unsigned int lrng_gcd_selftest(void) -+{ -+ return LRNG_SELFTEST_PASSED; -+} -+ -+#endif /* CONFIG_LRNG_TIMER_COMMON */ -+ -+/* The test vectors are taken from crypto/testmgr.h */ -+static unsigned int lrng_hash_selftest(void) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; -+ static const u8 lrng_hash_selftest_result[] = -+#ifdef CONFIG_CRYPTO_LIB_SHA256 -+ { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, -+ 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, -+ 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, -+ 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad }; -+#else /* CONFIG_CRYPTO_LIB_SHA256 */ -+ { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e, -+ 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d }; -+#endif /* CONFIG_CRYPTO_LIB_SHA256 */ -+ static const u8 hash_input[] = { 0x61, 0x62, 0x63 }; /* "abc" */ -+ u8 digest[sizeof(lrng_hash_selftest_result)] __aligned(sizeof(u32)); -+ -+ if (sizeof(digest) != hash_cb->hash_digestsize(NULL)) -+ return LRNG_SEFLTEST_ERROR_HASH; -+ -+ if (!hash_cb->hash_init(shash, NULL) && -+ !hash_cb->hash_update(shash, hash_input, -+ sizeof(hash_input)) && -+ !hash_cb->hash_final(shash, digest) && -+ !memcmp(digest, lrng_hash_selftest_result, sizeof(digest))) -+ return 0; -+ -+ pr_err("LRNG %s Hash self-test FAILED\n", hash_cb->hash_name()); -+ return LRNG_SEFLTEST_ERROR_HASH; -+} -+ -+#ifdef CONFIG_LRNG_DRNG_CHACHA20 -+ -+static void lrng_selftest_bswap32(u32 *ptr, u32 words) -+{ -+ u32 i; -+ -+ /* Byte-swap data which is an LE representation */ -+ for (i = 0; i < words; i++) { -+ __le32 *p = (__le32 *)ptr; -+ -+ *p = cpu_to_le32(*ptr); -+ ptr++; -+ } -+} -+ -+/* -+ * The test vectors were generated using the ChaCha20 DRNG from -+ * https://www.chronox.de/chacha20.html -+ */ -+static unsigned int lrng_chacha20_drng_selftest(void) -+{ -+ const struct lrng_drng_cb *drng_cb = &lrng_cc20_drng_cb; -+ u8 seed[CHACHA_KEY_SIZE * 2] = { -+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, -+ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, -+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, -+ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, -+ }; -+ struct chacha20_block chacha20; -+ int ret; -+ u8 outbuf[CHACHA_KEY_SIZE * 2] __aligned(sizeof(u32)); -+ -+ /* -+ * Expected result when ChaCha20 DRNG state is zero: -+ * * constants are set to "expand 32-byte k" -+ * * remaining state is 0 -+ * and pulling one half ChaCha20 DRNG block. -+ */ -+ static const u8 expected_halfblock[CHACHA_KEY_SIZE] = { -+ 0x76, 0xb8, 0xe0, 0xad, 0xa0, 0xf1, 0x3d, 0x90, -+ 0x40, 0x5d, 0x6a, 0xe5, 0x53, 0x86, 0xbd, 0x28, -+ 0xbd, 0xd2, 0x19, 0xb8, 0xa0, 0x8d, 0xed, 0x1a, -+ 0xa8, 0x36, 0xef, 0xcc, 0x8b, 0x77, 0x0d, 0xc7 }; -+ -+ /* -+ * Expected result when ChaCha20 DRNG state is zero: -+ * * constants are set to "expand 32-byte k" -+ * * remaining state is 0 -+ * followed by a reseed with two keyblocks -+ * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ * 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, -+ * 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, -+ * 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, -+ * 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f -+ * and pulling one ChaCha20 DRNG block. -+ */ -+ static const u8 expected_oneblock[CHACHA_KEY_SIZE * 2] = { -+ 0xe3, 0xb0, 0x8a, 0xcc, 0x34, 0xc3, 0x17, 0x0e, -+ 0xc3, 0xd8, 0xc3, 0x40, 0xe7, 0x73, 0xe9, 0x0d, -+ 0xd1, 0x62, 0xa3, 0x5d, 0x7d, 0xf2, 0xf1, 0x4a, -+ 0x24, 0x42, 0xb7, 0x1e, 0xb0, 0x05, 0x17, 0x07, -+ 0xb9, 0x35, 0x10, 0x69, 0x8b, 0x46, 0xfb, 0x51, -+ 0xe9, 0x91, 0x3f, 0x46, 0xf2, 0x4d, 0xea, 0xd0, -+ 0x81, 0xc1, 0x1b, 0xa9, 0x5d, 0x52, 0x91, 0x5f, -+ 0xcd, 0xdc, 0xc6, 0xd6, 0xc3, 0x7c, 0x50, 0x23 }; -+ -+ /* -+ * Expected result when ChaCha20 DRNG state is zero: -+ * * constants are set to "expand 32-byte k" -+ * * remaining state is 0 -+ * followed by a reseed with one key block plus one byte -+ * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, -+ * 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, -+ * 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, -+ * 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, -+ * 0x20 -+ * and pulling less than one ChaCha20 DRNG block. -+ */ -+ static const u8 expected_block_nonalinged[CHACHA_KEY_SIZE + 4] = { -+ 0x9c, 0xfc, 0x5e, 0x31, 0x21, 0x62, 0x11, 0x85, -+ 0xd3, 0x77, 0xd3, 0x69, 0x0f, 0xa8, 0x16, 0x55, -+ 0xb4, 0x4c, 0xf6, 0x52, 0xf3, 0xa8, 0x37, 0x99, -+ 0x38, 0x76, 0xa0, 0x66, 0xec, 0xbb, 0xce, 0xa9, -+ 0x9c, 0x95, 0xa1, 0xfd }; -+ -+ BUILD_BUG_ON(sizeof(seed) % sizeof(u32)); -+ -+ memset(&chacha20, 0, sizeof(chacha20)); -+ lrng_cc20_init_rfc7539(&chacha20); -+ lrng_selftest_bswap32((u32 *)seed, sizeof(seed) / sizeof(u32)); -+ -+ /* Generate with zero state */ -+ ret = drng_cb->drng_generate(&chacha20, outbuf, -+ sizeof(expected_halfblock)); -+ if (ret != sizeof(expected_halfblock)) -+ goto err; -+ if (memcmp(outbuf, expected_halfblock, sizeof(expected_halfblock))) -+ goto err; -+ -+ /* Clear state of DRNG */ -+ memset(&chacha20.key.u[0], 0, 48); -+ -+ /* Reseed with 2 key blocks */ -+ ret = drng_cb->drng_seed(&chacha20, seed, sizeof(expected_oneblock)); -+ if (ret < 0) -+ goto err; -+ ret = drng_cb->drng_generate(&chacha20, outbuf, -+ sizeof(expected_oneblock)); -+ if (ret != sizeof(expected_oneblock)) -+ goto err; -+ if (memcmp(outbuf, expected_oneblock, sizeof(expected_oneblock))) -+ goto err; -+ -+ /* Clear state of DRNG */ -+ memset(&chacha20.key.u[0], 0, 48); -+ -+ /* Reseed with 1 key block and one byte */ -+ ret = drng_cb->drng_seed(&chacha20, seed, -+ sizeof(expected_block_nonalinged)); -+ if (ret < 0) -+ goto err; -+ ret = drng_cb->drng_generate(&chacha20, outbuf, -+ sizeof(expected_block_nonalinged)); -+ if (ret != sizeof(expected_block_nonalinged)) -+ goto err; -+ if (memcmp(outbuf, expected_block_nonalinged, -+ sizeof(expected_block_nonalinged))) -+ goto err; -+ -+ return LRNG_SELFTEST_PASSED; -+ -+err: -+ pr_err("LRNG ChaCha20 DRNG self-test FAILED\n"); -+ return LRNG_SEFLTEST_ERROR_CHACHA20; -+} -+ -+#else /* CONFIG_LRNG_DRNG_CHACHA20 */ -+ -+static unsigned int lrng_chacha20_drng_selftest(void) -+{ -+ return LRNG_SELFTEST_PASSED; -+} -+ -+#endif /* CONFIG_LRNG_DRNG_CHACHA20 */ -+ -+static unsigned int lrng_selftest_status = LRNG_SELFTEST_NOT_EXECUTED; -+ -+static int lrng_selftest(void) -+{ -+ unsigned int ret = lrng_data_process_selftest(); -+ -+ ret |= lrng_chacha20_drng_selftest(); -+ ret |= lrng_hash_selftest(); -+ ret |= lrng_gcd_selftest(); -+ -+ if (ret) { -+ if (IS_ENABLED(CONFIG_LRNG_SELFTEST_PANIC)) -+ panic("LRNG self-tests failed: %u\n", ret); -+ } else { -+ pr_info("LRNG self-tests passed\n"); -+ } -+ -+ lrng_selftest_status = ret; -+ -+ if (lrng_selftest_status) -+ return -EFAULT; -+ return 0; -+} -+ -+#ifdef CONFIG_SYSFS -+/* Re-perform self-test when any value is written to the sysfs file. */ -+static int lrng_selftest_sysfs_set(const char *val, -+ const struct kernel_param *kp) -+{ -+ return lrng_selftest(); -+} -+ -+static const struct kernel_param_ops lrng_selftest_sysfs = { -+ .set = lrng_selftest_sysfs_set, -+ .get = param_get_uint, -+}; -+module_param_cb(selftest_status, &lrng_selftest_sysfs, &lrng_selftest_status, -+ 0644); -+#endif /* CONFIG_SYSFS */ -+ -+static int __init lrng_selftest_init(void) -+{ -+ return lrng_selftest(); -+} -+ -+module_init(lrng_selftest_init); -diff --git a/drivers/char/lrng/lrng_sha.h b/drivers/char/lrng/lrng_sha.h -new file mode 100644 -index 000000000000..d2f134f54773 ---- /dev/null -+++ b/drivers/char/lrng/lrng_sha.h -@@ -0,0 +1,14 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * LRNG SHA definition usable in atomic contexts right from the start of the -+ * kernel. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_SHA_H -+#define _LRNG_SHA_H -+ -+extern const struct lrng_hash_cb lrng_sha_hash_cb; -+ -+#endif /* _LRNG_SHA_H */ -diff --git a/drivers/char/lrng/lrng_sha1.c b/drivers/char/lrng/lrng_sha1.c -new file mode 100644 -index 000000000000..9cbc7a6fee49 ---- /dev/null -+++ b/drivers/char/lrng/lrng_sha1.c -@@ -0,0 +1,88 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the SHA-1 implementation that can be used -+ * without the kernel crypto API available including during early boot and in -+ * atomic contexts. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+ -+#include "lrng_sha.h" -+ -+/* -+ * If the SHA-256 support is not compiled, we fall back to SHA-1 that is always -+ * compiled and present in the kernel. -+ */ -+static u32 lrng_sha1_hash_digestsize(void *hash) -+{ -+ return SHA1_DIGEST_SIZE; -+} -+ -+static void lrng_sha1_block_fn(struct sha1_state *sctx, const u8 *src, -+ int blocks) -+{ -+ u32 temp[SHA1_WORKSPACE_WORDS]; -+ -+ while (blocks--) { -+ sha1_transform(sctx->state, src, temp); -+ src += SHA1_BLOCK_SIZE; -+ } -+ memzero_explicit(temp, sizeof(temp)); -+} -+ -+static int lrng_sha1_hash_init(struct shash_desc *shash, void *hash) -+{ -+ /* -+ * We do not need a TFM - we only need sufficient space for -+ * struct sha1_state on the stack. -+ */ -+ sha1_base_init(shash); -+ return 0; -+} -+ -+static int lrng_sha1_hash_update(struct shash_desc *shash, -+ const u8 *inbuf, u32 inbuflen) -+{ -+ return sha1_base_do_update(shash, inbuf, inbuflen, lrng_sha1_block_fn); -+} -+ -+static int lrng_sha1_hash_final(struct shash_desc *shash, u8 *digest) -+{ -+ return sha1_base_do_finalize(shash, lrng_sha1_block_fn) ?: -+ sha1_base_finish(shash, digest); -+} -+ -+static const char *lrng_sha1_hash_name(void) -+{ -+ return "SHA-1"; -+} -+ -+static void lrng_sha1_hash_desc_zero(struct shash_desc *shash) -+{ -+ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha1_state)); -+} -+ -+static void *lrng_sha1_hash_alloc(void) -+{ -+ pr_info("Hash %s allocated\n", lrng_sha1_hash_name()); -+ return NULL; -+} -+ -+static void lrng_sha1_hash_dealloc(void *hash) { } -+ -+const struct lrng_hash_cb lrng_sha_hash_cb = { -+ .hash_name = lrng_sha1_hash_name, -+ .hash_alloc = lrng_sha1_hash_alloc, -+ .hash_dealloc = lrng_sha1_hash_dealloc, -+ .hash_digestsize = lrng_sha1_hash_digestsize, -+ .hash_init = lrng_sha1_hash_init, -+ .hash_update = lrng_sha1_hash_update, -+ .hash_final = lrng_sha1_hash_final, -+ .hash_desc_zero = lrng_sha1_hash_desc_zero, -+}; -diff --git a/drivers/char/lrng/lrng_sha256.c b/drivers/char/lrng/lrng_sha256.c -new file mode 100644 -index 000000000000..50705351a71c ---- /dev/null -+++ b/drivers/char/lrng/lrng_sha256.c -@@ -0,0 +1,72 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * Backend for the LRNG providing the SHA-256 implementation that can be used -+ * without the kernel crypto API available including during early boot and in -+ * atomic contexts. -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+ -+#include "lrng_sha.h" -+ -+static u32 lrng_sha256_hash_digestsize(void *hash) -+{ -+ return SHA256_DIGEST_SIZE; -+} -+ -+static int lrng_sha256_hash_init(struct shash_desc *shash, void *hash) -+{ -+ /* -+ * We do not need a TFM - we only need sufficient space for -+ * struct sha256_state on the stack. -+ */ -+ sha256_init(shash_desc_ctx(shash)); -+ return 0; -+} -+ -+static int lrng_sha256_hash_update(struct shash_desc *shash, -+ const u8 *inbuf, u32 inbuflen) -+{ -+ sha256_update(shash_desc_ctx(shash), inbuf, inbuflen); -+ return 0; -+} -+ -+static int lrng_sha256_hash_final(struct shash_desc *shash, u8 *digest) -+{ -+ sha256_final(shash_desc_ctx(shash), digest); -+ return 0; -+} -+ -+static const char *lrng_sha256_hash_name(void) -+{ -+ return "SHA-256"; -+} -+ -+static void lrng_sha256_hash_desc_zero(struct shash_desc *shash) -+{ -+ memzero_explicit(shash_desc_ctx(shash), sizeof(struct sha256_state)); -+} -+ -+static void *lrng_sha256_hash_alloc(void) -+{ -+ pr_info("Hash %s allocated\n", lrng_sha256_hash_name()); -+ return NULL; -+} -+ -+static void lrng_sha256_hash_dealloc(void *hash) { } -+ -+const struct lrng_hash_cb lrng_sha_hash_cb = { -+ .hash_name = lrng_sha256_hash_name, -+ .hash_alloc = lrng_sha256_hash_alloc, -+ .hash_dealloc = lrng_sha256_hash_dealloc, -+ .hash_digestsize = lrng_sha256_hash_digestsize, -+ .hash_init = lrng_sha256_hash_init, -+ .hash_update = lrng_sha256_hash_update, -+ .hash_final = lrng_sha256_hash_final, -+ .hash_desc_zero = lrng_sha256_hash_desc_zero, -+}; -diff --git a/drivers/char/lrng/lrng_switch.c b/drivers/char/lrng/lrng_switch.c -new file mode 100644 -index 000000000000..aae75594926f ---- /dev/null -+++ b/drivers/char/lrng/lrng_switch.c -@@ -0,0 +1,286 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG DRNG switching support -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+ -+#include "lrng_es_aux.h" -+#include "lrng_es_mgr.h" -+#include "lrng_interface_dev_common.h" -+#include "lrng_numa.h" -+ -+static int __maybe_unused -+lrng_hash_switch(struct lrng_drng *drng_store, const void *cb, int node) -+{ -+ const struct lrng_hash_cb *new_cb = (const struct lrng_hash_cb *)cb; -+ const struct lrng_hash_cb *old_cb = drng_store->hash_cb; -+ unsigned long flags; -+ u32 i; -+ void *new_hash, *old_hash; -+ int ret; -+ -+ if (node == -1) -+ return 0; -+ -+ new_hash = new_cb->hash_alloc(); -+ old_hash = drng_store->hash; -+ -+ if (IS_ERR(new_hash)) { -+ pr_warn("could not allocate new LRNG pool hash (%ld)\n", -+ PTR_ERR(new_hash)); -+ return PTR_ERR(new_hash); -+ } -+ -+ if (new_cb->hash_digestsize(new_hash) > LRNG_MAX_DIGESTSIZE) { -+ pr_warn("digest size of newly requested hash too large\n"); -+ new_cb->hash_dealloc(new_hash); -+ return -EINVAL; -+ } -+ -+ write_lock_irqsave(&drng_store->hash_lock, flags); -+ -+ /* Trigger the switch for each entropy source */ -+ for_each_lrng_es(i) { -+ if (!lrng_es[i]->switch_hash) -+ continue; -+ ret = lrng_es[i]->switch_hash(drng_store, node, new_cb, -+ new_hash, old_cb); -+ if (ret) { -+ u32 j; -+ -+ /* Revert all already executed operations */ -+ for (j = 0; j < i; j++) { -+ if (!lrng_es[j]->switch_hash) -+ continue; -+ WARN_ON(lrng_es[j]->switch_hash(drng_store, -+ node, old_cb, -+ old_hash, -+ new_cb)); -+ } -+ goto err; -+ } -+ } -+ -+ drng_store->hash = new_hash; -+ drng_store->hash_cb = new_cb; -+ old_cb->hash_dealloc(old_hash); -+ pr_info("Conditioning function allocated for DRNG for NUMA node %d\n", -+ node); -+ -+err: -+ write_unlock_irqrestore(&drng_store->hash_lock, flags); -+ return ret; -+} -+ -+static int __maybe_unused -+lrng_drng_switch(struct lrng_drng *drng_store, const void *cb, int node) -+{ -+ const struct lrng_drng_cb *new_cb = (const struct lrng_drng_cb *)cb; -+ const struct lrng_drng_cb *old_cb = drng_store->drng_cb; -+ int ret; -+ u8 seed[LRNG_DRNG_SECURITY_STRENGTH_BYTES]; -+ void *new_drng = new_cb->drng_alloc(LRNG_DRNG_SECURITY_STRENGTH_BYTES); -+ void *old_drng = drng_store->drng; -+ u32 current_security_strength; -+ bool reset_drng = !lrng_get_available(); -+ -+ if (IS_ERR(new_drng)) { -+ pr_warn("could not allocate new DRNG for NUMA node %d (%ld)\n", -+ node, PTR_ERR(new_drng)); -+ return PTR_ERR(new_drng); -+ } -+ -+ current_security_strength = lrng_security_strength(); -+ mutex_lock(&drng_store->lock); -+ -+ /* -+ * Pull from existing DRNG to seed new DRNG regardless of seed status -+ * of old DRNG -- the entropy state for the DRNG is left unchanged which -+ * implies that als the new DRNG is reseeded when deemed necessary. This -+ * seeding of the new DRNG shall only ensure that the new DRNG has the -+ * same entropy as the old DRNG. -+ */ -+ ret = old_cb->drng_generate(old_drng, seed, sizeof(seed)); -+ mutex_unlock(&drng_store->lock); -+ -+ if (ret < 0) { -+ reset_drng = true; -+ pr_warn("getting random data from DRNG failed for NUMA node %d (%d)\n", -+ node, ret); -+ } else { -+ /* seed new DRNG with data */ -+ ret = new_cb->drng_seed(new_drng, seed, ret); -+ memzero_explicit(seed, sizeof(seed)); -+ if (ret < 0) { -+ reset_drng = true; -+ pr_warn("seeding of new DRNG failed for NUMA node %d (%d)\n", -+ node, ret); -+ } else { -+ pr_debug("seeded new DRNG of NUMA node %d instance from old DRNG instance\n", -+ node); -+ } -+ } -+ -+ mutex_lock(&drng_store->lock); -+ -+ if (reset_drng) -+ lrng_drng_reset(drng_store); -+ -+ drng_store->drng = new_drng; -+ drng_store->drng_cb = new_cb; -+ -+ /* Reseed if previous LRNG security strength was insufficient */ -+ if (current_security_strength < lrng_security_strength()) -+ drng_store->force_reseed = true; -+ -+ /* Force oversampling seeding as we initialize DRNG */ -+ if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) -+ lrng_unset_fully_seeded(drng_store); -+ -+ if (lrng_state_min_seeded()) -+ lrng_set_entropy_thresh(lrng_get_seed_entropy_osr( -+ drng_store->fully_seeded)); -+ -+ old_cb->drng_dealloc(old_drng); -+ -+ pr_info("DRNG of NUMA node %d switched\n", node); -+ -+ mutex_unlock(&drng_store->lock); -+ return ret; -+} -+ -+/* -+ * Switch the existing DRNG and hash instances with new using the new crypto -+ * callbacks. The caller must hold the lrng_crypto_cb_update lock. -+ */ -+static int lrng_switch(const void *cb, -+ int(*switcher)(struct lrng_drng *drng_store, -+ const void *cb, int node)) -+{ -+ struct lrng_drng **lrng_drng = lrng_drng_instances(); -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ struct lrng_drng *lrng_drng_pr = lrng_drng_pr_instance(); -+ int ret = 0; -+ -+ if (lrng_drng) { -+ u32 node; -+ -+ for_each_online_node(node) { -+ if (lrng_drng[node]) -+ ret |= switcher(lrng_drng[node], cb, node); -+ } -+ } else { -+ ret |= switcher(lrng_drng_init, cb, 0); -+ } -+ -+ ret |= switcher(lrng_drng_pr, cb, -1); -+ -+ return ret; -+} -+ -+/* -+ * lrng_set_drng_cb - Register new cryptographic callback functions for DRNG -+ * The registering implies that all old DRNG states are replaced with new -+ * DRNG states. -+ * -+ * drng_cb: Callback functions to be registered -- if NULL, use the default -+ * callbacks defined at compile time. -+ * -+ * Return: -+ * * 0 on success -+ * * < 0 on error -+ */ -+int lrng_set_drng_cb(const struct lrng_drng_cb *drng_cb) -+{ -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ int ret; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH_DRNG)) -+ return -EOPNOTSUPP; -+ -+ if (!drng_cb) -+ drng_cb = lrng_default_drng_cb; -+ -+ mutex_lock(&lrng_crypto_cb_update); -+ -+ /* -+ * If a callback other than the default is set, allow it only to be -+ * set back to the default callback. This ensures that multiple -+ * different callbacks can be registered at the same time. If a -+ * callback different from the current callback and the default -+ * callback shall be set, the current callback must be deregistered -+ * (e.g. the kernel module providing it must be unloaded) and the new -+ * implementation can be registered. -+ */ -+ if ((drng_cb != lrng_default_drng_cb) && -+ (lrng_drng_init->drng_cb != lrng_default_drng_cb)) { -+ pr_warn("disallow setting new DRNG callbacks, unload the old callbacks first!\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ ret = lrng_switch(drng_cb, lrng_drng_switch); -+ /* The switch may imply new entropy due to larger DRNG sec strength. */ -+ if (!ret) -+ lrng_es_add_entropy(); -+ -+out: -+ mutex_unlock(&lrng_crypto_cb_update); -+ return ret; -+} -+EXPORT_SYMBOL(lrng_set_drng_cb); -+ -+/* -+ * lrng_set_hash_cb - Register new cryptographic callback functions for hash -+ * The registering implies that all old hash states are replaced with new -+ * hash states. -+ * -+ * @hash_cb: Callback functions to be registered -- if NULL, use the default -+ * callbacks defined at compile time. -+ * -+ * Return: -+ * * 0 on success -+ * * < 0 on error -+ */ -+int lrng_set_hash_cb(const struct lrng_hash_cb *hash_cb) -+{ -+ struct lrng_drng *lrng_drng_init = lrng_drng_init_instance(); -+ int ret; -+ -+ if (!IS_ENABLED(CONFIG_LRNG_SWITCH_HASH)) -+ return -EOPNOTSUPP; -+ -+ if (!hash_cb) -+ hash_cb = lrng_default_hash_cb; -+ -+ mutex_lock(&lrng_crypto_cb_update); -+ -+ /* Comment from lrng_set_drng_cb applies. */ -+ if ((hash_cb != lrng_default_hash_cb) && -+ (lrng_drng_init->hash_cb != lrng_default_hash_cb)) { -+ pr_warn("disallow setting new hash callbacks, unload the old callbacks first!\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ ret = lrng_switch(hash_cb, lrng_hash_switch); -+ /* -+ * The switch may imply new entropy due to larger digest size. But -+ * it may also offer more room in the aux pool which means we ping -+ * any waiting entropy providers. -+ */ -+ if (!ret) { -+ lrng_es_add_entropy(); -+ lrng_writer_wakeup(); -+ } -+ -+out: -+ mutex_unlock(&lrng_crypto_cb_update); -+ return ret; -+} -+EXPORT_SYMBOL(lrng_set_hash_cb); -diff --git a/drivers/char/lrng/lrng_sysctl.c b/drivers/char/lrng/lrng_sysctl.c -new file mode 100644 -index 000000000000..ecdd96a842b4 ---- /dev/null -+++ b/drivers/char/lrng/lrng_sysctl.c -@@ -0,0 +1,140 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG sysctl interfaces -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#include -+#include -+#include -+#include -+ -+#include "lrng_drng_mgr.h" -+#include "lrng_es_mgr.h" -+#include "lrng_sysctl.h" -+ -+/* -+ * This function is used to return both the bootid UUID, and random -+ * UUID. The difference is in whether table->data is NULL; if it is, -+ * then a new UUID is generated and returned to the user. -+ * -+ * If the user accesses this via the proc interface, the UUID will be -+ * returned as an ASCII string in the standard UUID format; if via the -+ * sysctl system call, as 16 bytes of binary data. -+ */ -+static int lrng_sysctl_do_uuid(struct ctl_table *table, int write, -+ void *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table fake_table; -+ unsigned char buf[64], tmp_uuid[16], *uuid; -+ -+ uuid = table->data; -+ if (!uuid) { -+ uuid = tmp_uuid; -+ generate_random_uuid(uuid); -+ } else { -+ static DEFINE_SPINLOCK(bootid_spinlock); -+ -+ spin_lock(&bootid_spinlock); -+ if (!uuid[8]) -+ generate_random_uuid(uuid); -+ spin_unlock(&bootid_spinlock); -+ } -+ -+ sprintf(buf, "%pU", uuid); -+ -+ fake_table.data = buf; -+ fake_table.maxlen = sizeof(buf); -+ -+ return proc_dostring(&fake_table, write, buffer, lenp, ppos); -+} -+ -+static int lrng_sysctl_do_entropy(struct ctl_table *table, int write, -+ void *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table fake_table; -+ int entropy_count = lrng_avail_entropy_aux(); -+ -+ fake_table.data = &entropy_count; -+ fake_table.maxlen = sizeof(entropy_count); -+ -+ return proc_dointvec(&fake_table, write, buffer, lenp, ppos); -+} -+ -+static int lrng_sysctl_do_poolsize(struct ctl_table *table, int write, -+ void *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table fake_table; -+ u32 entropy_count = lrng_es[lrng_ext_es_aux]->max_entropy(); -+ -+ fake_table.data = &entropy_count; -+ fake_table.maxlen = sizeof(entropy_count); -+ -+ return proc_dointvec(&fake_table, write, buffer, lenp, ppos); -+} -+ -+static int lrng_min_write_thresh; -+static int lrng_max_write_thresh = (LRNG_WRITE_WAKEUP_ENTROPY << 3); -+static char lrng_sysctl_bootid[16]; -+static int lrng_drng_reseed_max_min; -+ -+void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) -+{ -+ lrng_max_write_thresh = (int)new_digestsize; -+ /* Ensure that changes to the global variable are visible */ -+ mb(); -+} -+ -+static struct ctl_table random_table[] = { -+ { -+ .procname = "poolsize", -+ .maxlen = sizeof(int), -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_poolsize, -+ }, -+ { -+ .procname = "entropy_avail", -+ .maxlen = sizeof(int), -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_entropy, -+ }, -+ { -+ .procname = "write_wakeup_threshold", -+ .data = &lrng_write_wakeup_bits, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec_minmax, -+ .extra1 = &lrng_min_write_thresh, -+ .extra2 = &lrng_max_write_thresh, -+ }, -+ { -+ .procname = "boot_id", -+ .data = &lrng_sysctl_bootid, -+ .maxlen = 16, -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_uuid, -+ }, -+ { -+ .procname = "uuid", -+ .maxlen = 16, -+ .mode = 0444, -+ .proc_handler = lrng_sysctl_do_uuid, -+ }, -+ { -+ .procname = "urandom_min_reseed_secs", -+ .data = &lrng_drng_reseed_max_time, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ .extra1 = &lrng_drng_reseed_max_min, -+ }, -+ { } -+}; -+ -+static int __init random_sysctls_init(void) -+{ -+ register_sysctl_init("kernel/random", random_table); -+ return 0; -+} -+device_initcall(random_sysctls_init); -diff --git a/drivers/char/lrng/lrng_sysctl.h b/drivers/char/lrng/lrng_sysctl.h -new file mode 100644 -index 000000000000..4b487e5077ed ---- /dev/null -+++ b/drivers/char/lrng/lrng_sysctl.h -@@ -0,0 +1,15 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_SYSCTL_H -+#define _LRNG_SYSCTL_H -+ -+#ifdef CONFIG_LRNG_SYSCTL -+void lrng_sysctl_update_max_write_thresh(u32 new_digestsize); -+#else -+static inline void lrng_sysctl_update_max_write_thresh(u32 new_digestsize) { } -+#endif -+ -+#endif /* _LRNG_SYSCTL_H */ -diff --git a/drivers/char/lrng/lrng_testing.c b/drivers/char/lrng/lrng_testing.c -new file mode 100644 -index 000000000000..101140085d81 ---- /dev/null -+++ b/drivers/char/lrng/lrng_testing.c -@@ -0,0 +1,901 @@ -+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -+/* -+ * LRNG testing interfaces to obtain raw entropy -+ * -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "lrng_definitions.h" -+#include "lrng_drng_chacha20.h" -+#include "lrng_sha.h" -+#include "lrng_testing.h" -+ -+#if defined(CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY) || \ -+ defined(CONFIG_LRNG_RAW_SCHED_PID_ENTROPY) || \ -+ defined(CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY) || \ -+ defined(CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY) || \ -+ defined(CONFIG_LRNG_SCHED_PERF) -+#define LRNG_TESTING_USE_BUSYLOOP -+#endif -+ -+#ifdef CONFIG_LRNG_TESTING_RECORDING -+ -+#define LRNG_TESTING_RINGBUFFER_SIZE 1024 -+#define LRNG_TESTING_RINGBUFFER_MASK (LRNG_TESTING_RINGBUFFER_SIZE - 1) -+ -+struct lrng_testing { -+ u32 lrng_testing_rb[LRNG_TESTING_RINGBUFFER_SIZE]; -+ u32 rb_reader; -+ atomic_t rb_writer; -+ atomic_t lrng_testing_enabled; -+ spinlock_t lock; -+ wait_queue_head_t read_wait; -+}; -+ -+/*************************** Generic Data Handling ****************************/ -+ -+/* -+ * boot variable: -+ * 0 ==> No boot test, gathering of runtime data allowed -+ * 1 ==> Boot test enabled and ready for collecting data, gathering runtime -+ * data is disabled -+ * 2 ==> Boot test completed and disabled, gathering of runtime data is -+ * disabled -+ */ -+ -+static void lrng_testing_reset(struct lrng_testing *data) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&data->lock, flags); -+ data->rb_reader = 0; -+ atomic_set(&data->rb_writer, 0); -+ spin_unlock_irqrestore(&data->lock, flags); -+} -+ -+static void lrng_testing_init(struct lrng_testing *data, u32 boot) -+{ -+ /* -+ * The boot time testing implies we have a running test. If the -+ * caller wants to clear it, he has to unset the boot_test flag -+ * at runtime via sysfs to enable regular runtime testing -+ */ -+ if (boot) -+ return; -+ -+ lrng_testing_reset(data); -+ atomic_set(&data->lrng_testing_enabled, 1); -+ pr_warn("Enabling data collection\n"); -+} -+ -+static void lrng_testing_fini(struct lrng_testing *data, u32 boot) -+{ -+ /* If we have boot data, we do not reset yet to allow data to be read */ -+ if (boot) -+ return; -+ -+ atomic_set(&data->lrng_testing_enabled, 0); -+ lrng_testing_reset(data); -+ pr_warn("Disabling data collection\n"); -+} -+ -+static bool lrng_testing_store(struct lrng_testing *data, u32 value, -+ u32 *boot) -+{ -+ unsigned long flags; -+ -+ if (!atomic_read(&data->lrng_testing_enabled) && (*boot != 1)) -+ return false; -+ -+ spin_lock_irqsave(&data->lock, flags); -+ -+ /* -+ * Disable entropy testing for boot time testing after ring buffer -+ * is filled. -+ */ -+ if (*boot) { -+ if (((u32)atomic_read(&data->rb_writer)) > -+ LRNG_TESTING_RINGBUFFER_SIZE) { -+ *boot = 2; -+ pr_warn_once("One time data collection test disabled\n"); -+ spin_unlock_irqrestore(&data->lock, flags); -+ return false; -+ } -+ -+ if (atomic_read(&data->rb_writer) == 1) -+ pr_warn("One time data collection test enabled\n"); -+ } -+ -+ data->lrng_testing_rb[((u32)atomic_read(&data->rb_writer)) & -+ LRNG_TESTING_RINGBUFFER_MASK] = value; -+ atomic_inc(&data->rb_writer); -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+#ifndef LRNG_TESTING_USE_BUSYLOOP -+ if (wq_has_sleeper(&data->read_wait)) -+ wake_up_interruptible(&data->read_wait); -+#endif -+ -+ return true; -+} -+ -+static bool lrng_testing_have_data(struct lrng_testing *data) -+{ -+ return ((((u32)atomic_read(&data->rb_writer)) & -+ LRNG_TESTING_RINGBUFFER_MASK) != -+ (data->rb_reader & LRNG_TESTING_RINGBUFFER_MASK)); -+} -+ -+static int lrng_testing_reader(struct lrng_testing *data, u32 *boot, -+ u8 *outbuf, u32 outbuflen) -+{ -+ unsigned long flags; -+ int collected_data = 0; -+ -+ lrng_testing_init(data, *boot); -+ -+ while (outbuflen) { -+ u32 writer = (u32)atomic_read(&data->rb_writer); -+ -+ spin_lock_irqsave(&data->lock, flags); -+ -+ /* We have no data or reached the writer. */ -+ if (!writer || (writer == data->rb_reader)) { -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+ /* -+ * Now we gathered all boot data, enable regular data -+ * collection. -+ */ -+ if (*boot) { -+ *boot = 0; -+ goto out; -+ } -+ -+#ifdef LRNG_TESTING_USE_BUSYLOOP -+ while (!lrng_testing_have_data(data)) -+ ; -+#else -+ wait_event_interruptible(data->read_wait, -+ lrng_testing_have_data(data)); -+#endif -+ if (signal_pending(current)) { -+ collected_data = -ERESTARTSYS; -+ goto out; -+ } -+ -+ continue; -+ } -+ -+ /* We copy out word-wise */ -+ if (outbuflen < sizeof(u32)) { -+ spin_unlock_irqrestore(&data->lock, flags); -+ goto out; -+ } -+ -+ memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], -+ sizeof(u32)); -+ data->rb_reader++; -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+ outbuf += sizeof(u32); -+ outbuflen -= sizeof(u32); -+ collected_data += sizeof(u32); -+ } -+ -+out: -+ lrng_testing_fini(data, *boot); -+ return collected_data; -+} -+ -+static int lrng_testing_extract_user(struct file *file, char __user *buf, -+ size_t nbytes, loff_t *ppos, -+ int (*reader)(u8 *outbuf, u32 outbuflen)) -+{ -+ u8 *tmp, *tmp_aligned; -+ int ret = 0, large_request = (nbytes > 256); -+ -+ if (!nbytes) -+ return 0; -+ -+ /* -+ * The intention of this interface is for collecting at least -+ * 1000 samples due to the SP800-90B requirements. So, we make no -+ * effort in avoiding allocating more memory that actually needed -+ * by the user. Hence, we allocate sufficient memory to always hold -+ * that amount of data. -+ */ -+ tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); -+ if (!tmp) -+ return -ENOMEM; -+ -+ tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); -+ -+ while (nbytes) { -+ int i; -+ -+ if (large_request && need_resched()) { -+ if (signal_pending(current)) { -+ if (ret == 0) -+ ret = -ERESTARTSYS; -+ break; -+ } -+ schedule(); -+ } -+ -+ i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); -+ i = reader(tmp_aligned, i); -+ if (i <= 0) { -+ if (i < 0) -+ ret = i; -+ break; -+ } -+ if (copy_to_user(buf, tmp_aligned, i)) { -+ ret = -EFAULT; -+ break; -+ } -+ -+ nbytes -= i; -+ buf += i; -+ ret += i; -+ } -+ -+ kfree_sensitive(tmp); -+ -+ if (ret > 0) -+ *ppos += ret; -+ -+ return ret; -+} -+ -+#endif /* CONFIG_LRNG_TESTING_RECORDING */ -+ -+/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+ -+static u32 boot_raw_hires_test = 0; -+module_param(boot_raw_hires_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); -+ -+static struct lrng_testing lrng_raw_hires = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) -+}; -+ -+bool lrng_raw_hires_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); -+} -+ -+static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_hires_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_hires_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_hires_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+ -+/********************* Raw Jiffies Entropy Data Handling **********************/ -+ -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+ -+static u32 boot_raw_jiffies_test = 0; -+module_param(boot_raw_jiffies_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); -+ -+static struct lrng_testing lrng_raw_jiffies = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) -+}; -+ -+bool lrng_raw_jiffies_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_jiffies, value, -+ &boot_raw_jiffies_test); -+} -+ -+static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_jiffies_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_jiffies_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_jiffies_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+ -+/************************** Raw IRQ Data Handling ****************************/ -+ -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+ -+static u32 boot_raw_irq_test = 0; -+module_param(boot_raw_irq_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); -+ -+static struct lrng_testing lrng_raw_irq = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) -+}; -+ -+bool lrng_raw_irq_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); -+} -+ -+static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_irq_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_irq_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_irq_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+ -+/************************ Raw _RET_IP_ Data Handling **************************/ -+ -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+ -+static u32 boot_raw_retip_test = 0; -+module_param(boot_raw_retip_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); -+ -+static struct lrng_testing lrng_raw_retip = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) -+}; -+ -+bool lrng_raw_retip_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); -+} -+ -+static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_retip_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_retip_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_retip_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+ -+/********************** Raw IRQ register Data Handling ************************/ -+ -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+ -+static u32 boot_raw_regs_test = 0; -+module_param(boot_raw_regs_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); -+ -+static struct lrng_testing lrng_raw_regs = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) -+}; -+ -+bool lrng_raw_regs_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); -+} -+ -+static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_regs_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_regs_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_regs_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+ -+/********************** Raw Entropy Array Data Handling ***********************/ -+ -+#ifdef CONFIG_LRNG_RAW_ARRAY -+ -+static u32 boot_raw_array = 0; -+module_param(boot_raw_array, uint, 0644); -+MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); -+ -+static struct lrng_testing lrng_raw_array = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) -+}; -+ -+bool lrng_raw_array_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); -+} -+ -+static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_raw_array_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_array_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_array_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_array_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_ARRAY */ -+ -+/******************** Interrupt Performance Data Handling *********************/ -+ -+#ifdef CONFIG_LRNG_IRQ_PERF -+ -+static u32 boot_irq_perf = 0; -+module_param(boot_irq_perf, uint, 0644); -+MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); -+ -+static struct lrng_testing lrng_irq_perf = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) -+}; -+ -+bool lrng_perf_time(u32 start) -+{ -+ return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, -+ &boot_irq_perf); -+} -+ -+static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_irq_perf_reader); -+} -+ -+static const struct file_operations lrng_irq_perf_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_irq_perf_read, -+}; -+ -+#endif /* CONFIG_LRNG_IRQ_PERF */ -+ -+/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+ -+static u32 boot_raw_sched_hires_test = 0; -+module_param(boot_raw_sched_hires_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); -+ -+static struct lrng_testing lrng_raw_sched_hires = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), -+ .read_wait = -+ __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) -+}; -+ -+bool lrng_raw_sched_hires_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_hires, value, -+ &boot_raw_sched_hires_test); -+} -+ -+static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_hires, -+ &boot_raw_sched_hires_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_hires_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_hires_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_hires_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+ -+/******************** Interrupt Performance Data Handling *********************/ -+ -+#ifdef CONFIG_LRNG_SCHED_PERF -+ -+static u32 boot_sched_perf = 0; -+module_param(boot_sched_perf, uint, 0644); -+MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); -+ -+static struct lrng_testing lrng_sched_perf = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) -+}; -+ -+bool lrng_sched_perf_time(u32 start) -+{ -+ return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, -+ &boot_sched_perf); -+} -+ -+static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_sched_perf_reader); -+} -+ -+static const struct file_operations lrng_sched_perf_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_sched_perf_read, -+}; -+ -+#endif /* CONFIG_LRNG_SCHED_PERF */ -+ -+/*************** Raw Scheduler task_struct->pid Data Handling *****************/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+ -+static u32 boot_raw_sched_pid_test = 0; -+module_param(boot_raw_sched_pid_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); -+ -+static struct lrng_testing lrng_raw_sched_pid = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) -+}; -+ -+bool lrng_raw_sched_pid_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_pid, value, -+ &boot_raw_sched_pid_test); -+} -+ -+static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_pid, -+ &boot_raw_sched_pid_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_pid_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_pid_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_pid_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+ -+ -+/*********** Raw Scheduler task_struct->start_time Data Handling **************/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+ -+static u32 boot_raw_sched_starttime_test = 0; -+module_param(boot_raw_sched_starttime_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); -+ -+static struct lrng_testing lrng_raw_sched_starttime = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) -+}; -+ -+bool lrng_raw_sched_starttime_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_starttime, value, -+ &boot_raw_sched_starttime_test); -+} -+ -+static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_starttime, -+ &boot_raw_sched_starttime_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_starttime_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_starttime_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_starttime_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+ -+/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+ -+static u32 boot_raw_sched_nvcsw_test = 0; -+module_param(boot_raw_sched_nvcsw_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); -+ -+static struct lrng_testing lrng_raw_sched_nvcsw = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) -+}; -+ -+bool lrng_raw_sched_nvcsw_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_nvcsw, value, -+ &boot_raw_sched_nvcsw_test); -+} -+ -+static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_nvcsw, -+ &boot_raw_sched_nvcsw_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_nvcsw_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_sched_nvcsw_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_nvcsw_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+ -+/*********************************** ACVT ************************************/ -+ -+#ifdef CONFIG_LRNG_ACVT_HASH -+ -+/* maximum amount of data to be hashed as defined by ACVP */ -+#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) -+ -+/* -+ * As we use static variables to store the data, it is clear that the -+ * test interface is only able to handle single threaded testing. This is -+ * considered to be sufficient for testing. If multi-threaded use of the -+ * ACVT test interface would be performed, the caller would get garbage -+ * but the kernel operation is unaffected by this. -+ */ -+static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] -+ __aligned(LRNG_KCAPI_ALIGN); -+static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); -+static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; -+ -+static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, -+ size_t nbytes, loff_t *ppos) -+{ -+ if (nbytes > LRNG_ACVT_MAX_SHA_MSG) -+ return -EINVAL; -+ -+ atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); -+ -+ return simple_write_to_buffer(lrng_acvt_hash_data, -+ LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); -+} -+ -+static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; -+ ssize_t ret; -+ -+ if (count > LRNG_ATOMIC_DIGEST_SIZE) -+ return -EINVAL; -+ -+ ret = hash_cb->hash_init(shash, NULL) ?: -+ hash_cb->hash_update(shash, lrng_acvt_hash_data, -+ atomic_read_u32(&lrng_acvt_hash_data_size)) ?: -+ hash_cb->hash_final(shash, lrng_acvt_hash_digest); -+ if (ret) -+ return ret; -+ -+ return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, -+ sizeof(lrng_acvt_hash_digest)); -+} -+ -+static const struct file_operations lrng_acvt_hash_fops = { -+ .owner = THIS_MODULE, -+ .open = simple_open, -+ .llseek = default_llseek, -+ .read = lrng_acvt_hash_read, -+ .write = lrng_acvt_hash_write, -+}; -+ -+#endif /* CONFIG_LRNG_ACVT_DRNG */ -+ -+/************************************************************************** -+ * Debugfs interface -+ **************************************************************************/ -+ -+static int __init lrng_raw_init(void) -+{ -+ struct dentry *lrng_raw_debugfs_root; -+ -+ lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_hires", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_hires_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_jiffies_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, -+ NULL, &lrng_raw_irq_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_retip", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_retip_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_regs", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_regs_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_ARRAY -+ debugfs_create_file_unsafe("lrng_raw_array", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_array_fops); -+#endif -+#ifdef CONFIG_LRNG_IRQ_PERF -+ debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, -+ NULL, &lrng_irq_perf_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, -+ lrng_raw_debugfs_root, -+ NULL, &lrng_raw_sched_hires_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_pid_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_starttime_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_nvcsw_fops); -+#endif -+#ifdef CONFIG_LRNG_SCHED_PERF -+ debugfs_create_file_unsafe("lrng_sched_perf", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_sched_perf_fops); -+#endif -+#ifdef CONFIG_LRNG_ACVT_HASH -+ debugfs_create_file_unsafe("lrng_acvt_hash", 0600, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_acvt_hash_fops); -+#endif -+ -+ return 0; -+} -+ -+module_init(lrng_raw_init); -diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h -new file mode 100644 -index 000000000000..672a7fca4595 ---- /dev/null -+++ b/drivers/char/lrng/lrng_testing.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_TESTING_H -+#define _LRNG_TESTING_H -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+bool lrng_raw_hires_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+bool lrng_raw_jiffies_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+bool lrng_raw_irq_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+bool lrng_raw_retip_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+bool lrng_raw_regs_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_ARRAY -+bool lrng_raw_array_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_ARRAY */ -+static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_ARRAY */ -+ -+#ifdef CONFIG_LRNG_IRQ_PERF -+bool lrng_perf_time(u32 start); -+#else /* CONFIG_LRNG_IRQ_PERF */ -+static inline bool lrng_perf_time(u32 start) { return false; } -+#endif /*CONFIG_LRNG_IRQ_PERF */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+bool lrng_raw_sched_hires_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+static inline bool -+lrng_raw_sched_hires_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+bool lrng_raw_sched_pid_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+static inline bool -+lrng_raw_sched_pid_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+bool lrng_raw_sched_starttime_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+static inline bool -+lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+bool lrng_raw_sched_nvcsw_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+static inline bool -+lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_SCHED_PERF -+bool lrng_sched_perf_time(u32 start); -+#else /* CONFIG_LRNG_SCHED_PERF */ -+static inline bool lrng_sched_perf_time(u32 start) { return false; } -+#endif /*CONFIG_LRNG_SCHED_PERF */ -+ -+#endif /* _LRNG_TESTING_H */ -diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h -index af5ad51d3eef..b12ae9bdebf4 100644 ---- a/include/crypto/drbg.h -+++ b/include/crypto/drbg.h -@@ -283,4 +283,11 @@ enum drbg_prefixes { - DRBG_PREFIX3 - }; - -+extern int drbg_alloc_state(struct drbg_state *drbg); -+extern void drbg_dealloc_state(struct drbg_state *drbg); -+extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, -+ bool *pr); -+extern const struct drbg_core drbg_cores[]; -+extern unsigned short drbg_sec_strength(drbg_flag_t flags); -+ - #endif /* _DRBG_H */ -diff --git a/crypto/jitterentropy.h b/include/crypto/internal/jitterentropy.h -similarity index 100% -rename from crypto/jitterentropy.h -rename to include/crypto/internal/jitterentropy.h -diff --git a/include/linux/lrng.h b/include/linux/lrng.h -new file mode 100644 -index 000000000000..c0d31a03d51f ---- /dev/null -+++ b/include/linux/lrng.h -@@ -0,0 +1,251 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_H -+#define _LRNG_H -+ -+#include -+#include -+#include -+#include -+ -+/* -+ * struct lrng_drng_cb - cryptographic callback functions defining a DRNG -+ * @drng_name Name of DRNG -+ * @drng_alloc: Allocate DRNG -- the provided integer should be used for -+ * sanity checks. -+ * return: allocated data structure or PTR_ERR on error -+ * @drng_dealloc: Deallocate DRNG -+ * @drng_seed: Seed the DRNG with data of arbitrary length drng: is -+ * pointer to data structure allocated with drng_alloc -+ * return: >= 0 on success, < 0 on error -+ * @drng_generate: Generate random numbers from the DRNG with arbitrary -+ * length -+ */ -+struct lrng_drng_cb { -+ const char *(*drng_name)(void); -+ void *(*drng_alloc)(u32 sec_strength); -+ void (*drng_dealloc)(void *drng); -+ int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); -+ int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); -+}; -+ -+/* -+ * struct lrng_hash_cb - cryptographic callback functions defining a hash -+ * @hash_name Name of Hash used for reading entropy pool arbitrary -+ * length -+ * @hash_alloc: Allocate the hash for reading the entropy pool -+ * return: allocated data structure (NULL is success too) -+ * or ERR_PTR on error -+ * @hash_dealloc: Deallocate Hash -+ * @hash_digestsize: Return the digestsize for the used hash to read out -+ * entropy pool -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: size of digest of hash in bytes -+ * @hash_init: Initialize hash -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_update: Update hash operation -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_final Final hash operation -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_desc_zero Zeroization of hash state buffer -+ * -+ * Assumptions: -+ * -+ * 1. Hash operation will not sleep -+ * 2. The hash' volatile state information is provided with *shash by caller. -+ */ -+struct lrng_hash_cb { -+ const char *(*hash_name)(void); -+ void *(*hash_alloc)(void); -+ void (*hash_dealloc)(void *hash); -+ u32 (*hash_digestsize)(void *hash); -+ int (*hash_init)(struct shash_desc *shash, void *hash); -+ int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, -+ u32 inbuflen); -+ int (*hash_final)(struct shash_desc *shash, u8 *digest); -+ void (*hash_desc_zero)(struct shash_desc *shash); -+}; -+ -+/* Register cryptographic backend */ -+#ifdef CONFIG_LRNG_SWITCH -+int lrng_set_drng_cb(const struct lrng_drng_cb *cb); -+int lrng_set_hash_cb(const struct lrng_hash_cb *cb); -+#else /* CONFIG_LRNG_SWITCH */ -+static inline int -+lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } -+static inline int -+lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } -+#endif /* CONFIG_LRNG_SWITCH */ -+ -+/* Callback to feed events to the scheduler entropy source */ -+#ifdef CONFIG_LRNG_SCHED -+extern void add_sched_randomness(const struct task_struct *p, int cpu); -+#else -+static inline void -+add_sched_randomness(const struct task_struct *p, int cpu) { } -+#endif -+ -+/* -+ * lrng_get_random_bytes() - Provider of cryptographic strong random numbers -+ * for kernel-internal usage. -+ * -+ * This function is appropriate for in-kernel use cases operating in atomic -+ * contexts. It will always use the ChaCha20 DRNG and it may be the case that -+ * it is not fully seeded when being used. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+void lrng_get_random_bytes(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_random_bytes_full() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from a fully initialized LRNG. -+ * -+ * This function will always return random numbers from a fully seeded and -+ * fully initialized LRNG. -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG -+void lrng_get_random_bytes_full(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_random_bytes_min() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from at least a minimally seeded -+ * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C -+ * oversampling applied in FIPS mode is not applied yet). -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG -+void lrng_get_random_bytes_min(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_random_bytes_pr() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from a fully initialized LRNG and -+ * requiring a reseed from the entropy sources before. -+ * -+ * This function will always return random numbers from a fully seeded and -+ * fully initialized LRNG. -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * This call only returns no more data than entropy was pulled from the -+ * entropy sources. Thus, it is likely that this call returns less data -+ * than requested by the caller. Also, the caller MUST be prepared that this -+ * call returns 0 bytes, i.e. it did not generate data. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ * -+ * @return: positive number indicates amount of generated bytes, < 0 on error -+ */ -+#ifdef CONFIG_LRNG -+int lrng_get_random_bytes_pr(void *buf, int nbytes); -+#endif -+ -+/* -+ * lrng_get_seed() - Fill buffer with data from entropy sources -+ * -+ * This call allows accessing the entropy sources directly and fill the buffer -+ * with data from all available entropy sources. This filled buffer is -+ * identical to the temporary seed buffer used by the LRNG to seed its DRNGs. -+ * -+ * The call is to allows users to seed their DRNG directly from the entropy -+ * sources in case the caller does not want to use the LRNG's DRNGs. This -+ * buffer can be directly used to seed the caller's DRNG from. -+ * -+ * The call blocks as long as one LRNG DRNG is not yet fully seeded. If -+ * LRNG_GET_SEED_NONBLOCK is specified, it does not block in this case, but -+ * returns with an error. -+ * -+ * Considering SP800-90C, there is a differentiation between the seeding -+ * requirements during instantiating a DRNG and at runtime of the DRNG. When -+ * specifying LRNG_GET_SEED_FULLY_SEEDED the caller indicates the DRNG was -+ * already fully seeded and the regular amount of entropy is requested. -+ * Otherwise, the LRNG will obtain the entropy rate required for initial -+ * seeding. The following minimum entropy rates will be obtained: -+ * -+ * * FIPS mode: -+ * * Initial seeding: 384 bits of entropy -+ * * Runtime seeding: 256 bits of entropy -+ * * Non-FIPS mode: -+ * * 128 bits of entropy in any case -+ * -+ * Albeit these are minimum entropy rates, the LRNG tries to request the -+ * given amount of entropy from each entropy source individually. If the -+ * minimum amount of entropy cannot be obtained collectively by all entropy -+ * sources, the LRNG will not fill the buffer. -+ * -+ * The return data in buf is structurally equivalent to the following -+ * definition: -+ * -+ * struct { -+ * u64 seedlen; -+ * u64 entropy_rate; -+ * struct entropy_buf seed; -+ * } __attribute((__packed__)); -+ * -+ * As struct entropy_buf is not known outsize of the LRNG, the LRNG fills -+ * seedlen first with the size of struct entropy_buf. If the caller-provided -+ * buffer buf is smaller than u64, then -EINVAL is returned -+ * and buf is not touched. If it is u64 or larger but smaller -+ * than the size of the structure above, -EMSGSIZE is returned and seedlen -+ * is filled with the size of the buffer. Finally, if buf is large -+ * enough to hold all data, it is filled with the seed data and the seedlen -+ * is set to sizeof(struct entropy_buf). The entropy rate is returned with -+ * the variable entropy_rate and provides the value in bits. -+ * -+ * The seed buffer is the data that should be handed to the caller's DRNG as -+ * seed data. -+ * -+ * @buf [out] Buffer to be filled with data from the entropy sources - note, the -+ * buffer is marked as u64 to ensure it is aligned to 64 bits. -+ * @nbytes [in] Size of the buffer allocated by the caller - this value -+ * provides size of @param buf in bytes. -+ * @flags [in] Flags field to adjust the behavior -+ * -+ * @return -EINVAL or -EMSGSIZE indicating the buffer is too small, -EAGAIN when -+ * the call would block, but NONBLOCK is specified, > 0 the size of -+ * the filled buffer. -+ */ -+#ifdef CONFIG_LRNG -+enum lrng_get_seed_flags { -+ LRNG_GET_SEED_NONBLOCK = 0x0001, /**< Do not block the call */ -+ LRNG_GET_SEED_FULLY_SEEDED = 0x0002, /**< DRNG is fully seeded */ -+}; -+ -+ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags); -+#endif -+ -+#endif /* _LRNG_H */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 265944f14948..e9f0ec6c9aeb 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -6,6 +6,7 @@ - * - * Copyright (C) 1991-2002 Linus Torvalds - */ -+#include - #include - #include - #include -@@ -3586,6 +3587,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) - { - struct rq *rq; - -+ add_sched_randomness(p, cpu); -+ - if (!schedstat_enabled()) - return; - --- -2.38.0.rc1.8.g2a7d63a245 - -From 0d38511c9f1327acc3ab92d5ff32d615ef59fb70 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 19 Sep 2022 14:40:14 +0200 -Subject: [PATCH 10/13] folios - -Signed-off-by: Peter Jung ---- - fs/afs/write.c | 114 +++++++++++++++++---------------- - fs/btrfs/extent_io.c | 57 +++++++++-------- - fs/ceph/addr.c | 138 ++++++++++++++++++++-------------------- - fs/cifs/file.c | 33 +++++++++- - fs/ext4/inode.c | 55 ++++++++-------- - fs/f2fs/checkpoint.c | 49 +++++++------- - fs/f2fs/compress.c | 13 ++-- - fs/f2fs/data.c | 69 ++++++++++---------- - fs/f2fs/f2fs.h | 5 +- - fs/f2fs/node.c | 72 +++++++++++---------- - fs/gfs2/aops.c | 64 ++++++++++--------- - fs/nilfs2/btree.c | 14 ++-- - fs/nilfs2/page.c | 59 ++++++++--------- - fs/nilfs2/segment.c | 44 +++++++------ - include/linux/pagemap.h | 32 +++++++--- - include/linux/pagevec.h | 8 --- - mm/filemap.c | 87 ++++++++++++------------- - mm/page-writeback.c | 44 +++++++------ - mm/swap.c | 10 --- - 19 files changed, 507 insertions(+), 460 deletions(-) - -diff --git a/fs/afs/write.c b/fs/afs/write.c -index 9ebdd36eaf2f..c17dbd82a38c 100644 ---- a/fs/afs/write.c -+++ b/fs/afs/write.c -@@ -699,82 +699,86 @@ static int afs_writepages_region(struct address_space *mapping, - loff_t start, loff_t end, loff_t *_next) - { - struct folio *folio; -- struct page *head_page; -+ struct folio_batch fbatch; - ssize_t ret; -+ unsigned int i; - int n, skips = 0; - - _enter("%llx,%llx,", start, end); -+ folio_batch_init(&fbatch); - - do { - pgoff_t index = start / PAGE_SIZE; - -- n = find_get_pages_range_tag(mapping, &index, end / PAGE_SIZE, -- PAGECACHE_TAG_DIRTY, 1, &head_page); -+ n = filemap_get_folios_tag(mapping, &index, end / PAGE_SIZE, -+ PAGECACHE_TAG_DIRTY, &fbatch); -+ - if (!n) - break; -+ for (i = 0; i < n; i++) { -+ folio = fbatch.folios[i]; -+ start = folio_pos(folio); /* May regress with THPs */ - -- folio = page_folio(head_page); -- start = folio_pos(folio); /* May regress with THPs */ -- -- _debug("wback %lx", folio_index(folio)); -+ _debug("wback %lx", folio_index(folio)); - -- /* At this point we hold neither the i_pages lock nor the -- * page lock: the page may be truncated or invalidated -- * (changing page->mapping to NULL), or even swizzled -- * back from swapper_space to tmpfs file mapping -- */ -- if (wbc->sync_mode != WB_SYNC_NONE) { -- ret = folio_lock_killable(folio); -- if (ret < 0) { -- folio_put(folio); -- return ret; -- } -- } else { -- if (!folio_trylock(folio)) { -- folio_put(folio); -- return 0; -+ /* At this point we hold neither the i_pages lock nor the -+ * page lock: the page may be truncated or invalidated -+ * (changing page->mapping to NULL), or even swizzled -+ * back from swapper_space to tmpfs file mapping -+ */ -+ if (wbc->sync_mode != WB_SYNC_NONE) { -+ ret = folio_lock_killable(folio); -+ if (ret < 0) { -+ folio_batch_release(&fbatch); -+ return ret; -+ } -+ } else { -+ if (!folio_trylock(folio)) -+ continue; - } -- } - -- if (folio_mapping(folio) != mapping || -- !folio_test_dirty(folio)) { -- start += folio_size(folio); -- folio_unlock(folio); -- folio_put(folio); -- continue; -- } -+ if (folio->mapping != mapping || -+ !folio_test_dirty(folio)) { -+ start += folio_size(folio); -+ folio_unlock(folio); -+ continue; -+ } - -- if (folio_test_writeback(folio) || -- folio_test_fscache(folio)) { -- folio_unlock(folio); -- if (wbc->sync_mode != WB_SYNC_NONE) { -- folio_wait_writeback(folio); -+ if (folio_test_writeback(folio) || -+ folio_test_fscache(folio)) { -+ folio_unlock(folio); -+ if (wbc->sync_mode != WB_SYNC_NONE) { -+ folio_wait_writeback(folio); - #ifdef CONFIG_AFS_FSCACHE -- folio_wait_fscache(folio); -+ folio_wait_fscache(folio); - #endif -- } else { -- start += folio_size(folio); -+ } else { -+ start += folio_size(folio); -+ } -+ if (wbc->sync_mode == WB_SYNC_NONE) { -+ if (skips >= 5 || need_resched()) { -+ *_next = start; -+ _leave(" = 0 [%llx]", *_next); -+ return 0; -+ } -+ skips++; -+ } -+ continue; - } -- folio_put(folio); -- if (wbc->sync_mode == WB_SYNC_NONE) { -- if (skips >= 5 || need_resched()) -- break; -- skips++; -+ -+ if (!folio_clear_dirty_for_io(folio)) -+ BUG(); -+ ret = afs_write_back_from_locked_folio(mapping, wbc, -+ folio, start, end); -+ if (ret < 0) { -+ _leave(" = %zd", ret); -+ folio_batch_release(&fbatch); -+ return ret; - } -- continue; -- } - -- if (!folio_clear_dirty_for_io(folio)) -- BUG(); -- ret = afs_write_back_from_locked_folio(mapping, wbc, folio, start, end); -- folio_put(folio); -- if (ret < 0) { -- _leave(" = %zd", ret); -- return ret; -+ start += ret; - } -- -- start += ret; -- -+ folio_batch_release(&fbatch); - cond_resched(); - } while (wbc->nr_to_write > 0); - -diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c -index cf4f19e80e2f..80fe313f8461 100644 ---- a/fs/btrfs/extent_io.c -+++ b/fs/btrfs/extent_io.c -@@ -4844,14 +4844,14 @@ int btree_write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int nr_to_write_done = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - int scanned = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* Start from prev offset */ - end = -1; -@@ -4874,14 +4874,15 @@ int btree_write_cache_pages(struct address_space *mapping, - if (wbc->sync_mode == WB_SYNC_ALL) - tag_pages_for_writeback(mapping, index, end); - while (!done && !nr_to_write_done && (index <= end) && -- (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag))) { -+ (nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch))) { - unsigned i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- ret = submit_eb_page(page, wbc, &epd, &eb_context); -+ ret = submit_eb_page(&folio->page, wbc, &epd, -+ &eb_context); - if (ret == 0) - continue; - if (ret < 0) { -@@ -4896,7 +4897,7 @@ int btree_write_cache_pages(struct address_space *mapping, - */ - nr_to_write_done = wbc->nr_to_write <= 0; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - if (!scanned && !done) { -@@ -4971,8 +4972,8 @@ static int extent_write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int nr_to_write_done = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; -@@ -4992,7 +4993,7 @@ static int extent_write_cache_pages(struct address_space *mapping, - if (!igrab(inode)) - return 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* Start from prev offset */ - end = -1; -@@ -5030,14 +5031,14 @@ static int extent_write_cache_pages(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; - while (!done && !nr_to_write_done && (index <= end) && -- (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, -- &index, end, tag))) { -+ (nr_folios = filemap_get_folios_tag(mapping, &index, -+ end, tag, &fbatch))) { - unsigned i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- done_index = page->index + 1; -+ done_index = folio->index + folio_nr_pages(folio); - /* - * At this point we hold neither the i_pages lock nor - * the page lock: the page may be truncated or -@@ -5045,29 +5046,29 @@ static int extent_write_cache_pages(struct address_space *mapping, - * or even swizzled back from swapper_space to - * tmpfs file mapping - */ -- if (!trylock_page(page)) { -+ if (!folio_trylock(folio)) { - submit_write_bio(epd, 0); -- lock_page(page); -+ folio_lock(folio); - } - -- if (unlikely(page->mapping != mapping)) { -- unlock_page(page); -+ if (unlikely(folio->mapping != mapping)) { -+ folio_unlock(folio); - continue; - } - - if (wbc->sync_mode != WB_SYNC_NONE) { -- if (PageWriteback(page)) -+ if (folio_test_writeback(folio)) - submit_write_bio(epd, 0); -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - } - -- if (PageWriteback(page) || -- !clear_page_dirty_for_io(page)) { -- unlock_page(page); -+ if (folio_test_writeback(folio) || -+ !folio_clear_dirty_for_io(folio)) { -+ folio_unlock(folio); - continue; - } - -- ret = __extent_writepage(page, wbc, epd); -+ ret = __extent_writepage(&folio->page, wbc, epd); - if (ret < 0) { - done = 1; - break; -@@ -5080,7 +5081,7 @@ static int extent_write_cache_pages(struct address_space *mapping, - */ - nr_to_write_done = wbc->nr_to_write <= 0; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - if (!scanned && !done) { -diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c -index dcf701b05cc1..33dbe55b08be 100644 ---- a/fs/ceph/addr.c -+++ b/fs/ceph/addr.c -@@ -792,7 +792,7 @@ static int ceph_writepages_start(struct address_space *mapping, - struct ceph_vino vino = ceph_vino(inode); - pgoff_t index, start_index, end = -1; - struct ceph_snap_context *snapc = NULL, *last_snapc = NULL, *pgsnapc; -- struct pagevec pvec; -+ struct folio_batch fbatch; - int rc = 0; - unsigned int wsize = i_blocksize(inode); - struct ceph_osd_request *req = NULL; -@@ -821,7 +821,7 @@ static int ceph_writepages_start(struct address_space *mapping, - if (fsc->mount_options->wsize < wsize) - wsize = fsc->mount_options->wsize; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - - start_index = wbc->range_cyclic ? mapping->writeback_index : 0; - index = start_index; -@@ -869,9 +869,9 @@ static int ceph_writepages_start(struct address_space *mapping, - - while (!done && index <= end) { - int num_ops = 0, op_idx; -- unsigned i, pvec_pages, max_pages, locked_pages = 0; -+ unsigned i, nr_folios, max_pages, locked_pages = 0; - struct page **pages = NULL, **data_pages; -- struct page *page; -+ struct folio *folio; - pgoff_t strip_unit_end = 0; - u64 offset = 0, len = 0; - bool from_pool = false; -@@ -879,28 +879,28 @@ static int ceph_writepages_start(struct address_space *mapping, - max_pages = wsize >> PAGE_SHIFT; - - get_more_pages: -- pvec_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, -- end, PAGECACHE_TAG_DIRTY); -- dout("pagevec_lookup_range_tag got %d\n", pvec_pages); -- if (!pvec_pages && !locked_pages) -+ nr_folios = filemap_get_folios_tag(mapping, &index, -+ end, PAGECACHE_TAG_DIRTY, &fbatch); -+ dout("filemap_get_folios_tag got %d\n", nr_folios); -+ if (!nr_folios && !locked_pages) - break; -- for (i = 0; i < pvec_pages && locked_pages < max_pages; i++) { -- page = pvec.pages[i]; -- dout("? %p idx %lu\n", page, page->index); -+ for (i = 0; i < nr_folios && locked_pages < max_pages; i++) { -+ folio = fbatch.folios[i]; -+ dout("? %p idx %lu\n", folio, folio->index); - if (locked_pages == 0) -- lock_page(page); /* first page */ -- else if (!trylock_page(page)) -+ folio_lock(folio); /* first folio */ -+ else if (!folio_trylock(folio)) - break; - - /* only dirty pages, or our accounting breaks */ -- if (unlikely(!PageDirty(page)) || -- unlikely(page->mapping != mapping)) { -- dout("!dirty or !mapping %p\n", page); -- unlock_page(page); -+ if (unlikely(!folio_test_dirty(folio)) || -+ unlikely(folio->mapping != mapping)) { -+ dout("!dirty or !mapping %p\n", folio); -+ folio_unlock(folio); - continue; - } - /* only if matching snap context */ -- pgsnapc = page_snap_context(page); -+ pgsnapc = page_snap_context(&folio->page); - if (pgsnapc != snapc) { - dout("page snapc %p %lld != oldest %p %lld\n", - pgsnapc, pgsnapc->seq, snapc, snapc->seq); -@@ -908,11 +908,10 @@ static int ceph_writepages_start(struct address_space *mapping, - !ceph_wbc.head_snapc && - wbc->sync_mode != WB_SYNC_NONE) - should_loop = true; -- unlock_page(page); -+ folio_unlock(folio); - continue; - } -- if (page_offset(page) >= ceph_wbc.i_size) { -- struct folio *folio = page_folio(page); -+ if (folio_pos(folio) >= ceph_wbc.i_size) { - - dout("folio at %lu beyond eof %llu\n", - folio->index, ceph_wbc.i_size); -@@ -924,25 +923,26 @@ static int ceph_writepages_start(struct address_space *mapping, - folio_unlock(folio); - continue; - } -- if (strip_unit_end && (page->index > strip_unit_end)) { -- dout("end of strip unit %p\n", page); -- unlock_page(page); -+ if (strip_unit_end && (folio->index > strip_unit_end)) { -+ dout("end of strip unit %p\n", folio); -+ folio_unlock(folio); - break; - } -- if (PageWriteback(page) || PageFsCache(page)) { -+ if (folio_test_writeback(folio) || -+ folio_test_fscache(folio)) { - if (wbc->sync_mode == WB_SYNC_NONE) { -- dout("%p under writeback\n", page); -- unlock_page(page); -+ dout("%p under writeback\n", folio); -+ folio_unlock(folio); - continue; - } -- dout("waiting on writeback %p\n", page); -- wait_on_page_writeback(page); -- wait_on_page_fscache(page); -+ dout("waiting on writeback %p\n", folio); -+ folio_wait_writeback(folio); -+ folio_wait_fscache(folio); - } - -- if (!clear_page_dirty_for_io(page)) { -- dout("%p !clear_page_dirty_for_io\n", page); -- unlock_page(page); -+ if (!folio_clear_dirty_for_io(folio)) { -+ dout("%p !clear_page_dirty_for_io\n", folio); -+ folio_unlock(folio); - continue; - } - -@@ -958,7 +958,7 @@ static int ceph_writepages_start(struct address_space *mapping, - u32 xlen; - - /* prepare async write request */ -- offset = (u64)page_offset(page); -+ offset = (u64)folio_pos(folio); - ceph_calc_file_object_mapping(&ci->i_layout, - offset, wsize, - &objnum, &objoff, -@@ -966,7 +966,7 @@ static int ceph_writepages_start(struct address_space *mapping, - len = xlen; - - num_ops = 1; -- strip_unit_end = page->index + -+ strip_unit_end = folio->index + - ((len - 1) >> PAGE_SHIFT); - - BUG_ON(pages); -@@ -981,54 +981,53 @@ static int ceph_writepages_start(struct address_space *mapping, - } - - len = 0; -- } else if (page->index != -+ } else if (folio->index != - (offset + len) >> PAGE_SHIFT) { - if (num_ops >= (from_pool ? CEPH_OSD_SLAB_OPS : - CEPH_OSD_MAX_OPS)) { -- redirty_page_for_writepage(wbc, page); -- unlock_page(page); -+ folio_redirty_for_writepage(wbc, folio); -+ folio_unlock(folio); - break; - } - - num_ops++; -- offset = (u64)page_offset(page); -+ offset = (u64)folio_pos(folio); - len = 0; - } - -- /* note position of first page in pvec */ -+ /* note position of first page in fbatch */ - dout("%p will write page %p idx %lu\n", -- inode, page, page->index); -+ inode, folio, folio->index); - - if (atomic_long_inc_return(&fsc->writeback_count) > - CONGESTION_ON_THRESH( - fsc->mount_options->congestion_kb)) - fsc->write_congested = true; - -- pages[locked_pages++] = page; -- pvec.pages[i] = NULL; -+ pages[locked_pages++] = &folio->page; -+ fbatch.folios[i] = NULL; - -- len += thp_size(page); -+ len += folio_size(folio); - } - - /* did we get anything? */ - if (!locked_pages) -- goto release_pvec_pages; -+ goto release_folio_batches; - if (i) { - unsigned j, n = 0; -- /* shift unused page to beginning of pvec */ -- for (j = 0; j < pvec_pages; j++) { -- if (!pvec.pages[j]) -+ /* shift unused folio to the beginning of fbatch */ -+ for (j = 0; j < nr_folios; j++) { -+ if (!fbatch.folios[j]) - continue; - if (n < j) -- pvec.pages[n] = pvec.pages[j]; -+ fbatch.folios[n] = fbatch.folios[j]; - n++; - } -- pvec.nr = n; -- -- if (pvec_pages && i == pvec_pages && -+ fbatch.nr = n; -+ if (nr_folios && i == nr_folios && - locked_pages < max_pages) { -- dout("reached end pvec, trying for more\n"); -- pagevec_release(&pvec); -+ dout("reached end of fbatch, trying for more\n"); -+ folio_batch_release(&fbatch); - goto get_more_pages; - } - } -@@ -1056,7 +1055,7 @@ static int ceph_writepages_start(struct address_space *mapping, - BUG_ON(IS_ERR(req)); - } - BUG_ON(len < page_offset(pages[locked_pages - 1]) + -- thp_size(page) - offset); -+ folio_size(folio) - offset); - - req->r_callback = writepages_finish; - req->r_inode = inode; -@@ -1098,7 +1097,7 @@ static int ceph_writepages_start(struct address_space *mapping, - set_page_writeback(pages[i]); - if (caching) - ceph_set_page_fscache(pages[i]); -- len += thp_size(page); -+ len += folio_size(folio); - } - ceph_fscache_write_to_cache(inode, offset, len, caching); - -@@ -1108,7 +1107,7 @@ static int ceph_writepages_start(struct address_space *mapping, - /* writepages_finish() clears writeback pages - * according to the data length, so make sure - * data length covers all locked pages */ -- u64 min_len = len + 1 - thp_size(page); -+ u64 min_len = len + 1 - folio_size(folio); - len = get_writepages_data_length(inode, pages[i - 1], - offset); - len = max(len, min_len); -@@ -1164,10 +1163,10 @@ static int ceph_writepages_start(struct address_space *mapping, - if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) - done = true; - --release_pvec_pages: -- dout("pagevec_release on %d pages (%p)\n", (int)pvec.nr, -- pvec.nr ? pvec.pages[0] : NULL); -- pagevec_release(&pvec); -+release_folio_batches: -+ dout("folio_batch_release on %d batches (%p)", (int) fbatch.nr, -+ fbatch.nr ? fbatch.folios[0] : NULL); -+ folio_batch_release(&fbatch); - } - - if (should_loop && !done) { -@@ -1180,19 +1179,22 @@ static int ceph_writepages_start(struct address_space *mapping, - if (wbc->sync_mode != WB_SYNC_NONE && - start_index == 0 && /* all dirty pages were checked */ - !ceph_wbc.head_snapc) { -- struct page *page; -+ struct folio *folio; - unsigned i, nr; - index = 0; - while ((index <= end) && -- (nr = pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_WRITEBACK))) { -+ (nr = filemap_get_folios_tag(mapping, &index, -+ (pgoff_t)-1, -+ PAGECACHE_TAG_WRITEBACK, -+ &fbatch))) { - for (i = 0; i < nr; i++) { -- page = pvec.pages[i]; -- if (page_snap_context(page) != snapc) -+ folio = fbatch.folios[i]; -+ if (page_snap_context(&folio->page) != -+ snapc) - continue; -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/fs/cifs/file.c b/fs/cifs/file.c -index 6f38b134a346..249a94896f78 100644 ---- a/fs/cifs/file.c -+++ b/fs/cifs/file.c -@@ -2517,14 +2517,41 @@ wdata_alloc_and_fillpages(pgoff_t tofind, struct address_space *mapping, - unsigned int *found_pages) - { - struct cifs_writedata *wdata; -- -+ struct folio_batch fbatch; -+ unsigned int i, idx, p, nr; - wdata = cifs_writedata_alloc((unsigned int)tofind, - cifs_writev_complete); - if (!wdata) - return NULL; - -- *found_pages = find_get_pages_range_tag(mapping, index, end, -- PAGECACHE_TAG_DIRTY, tofind, wdata->pages); -+ folio_batch_init(&fbatch); -+ *found_pages = 0; -+ -+again: -+ nr = filemap_get_folios_tag(mapping, index, end, -+ PAGECACHE_TAG_DIRTY, &fbatch); -+ if (!nr) -+ goto out; /* No dirty pages left in the range */ -+ -+ for (i = 0; i < nr; i++) { -+ struct folio *folio = fbatch.folios[i]; -+ -+ idx = 0; -+ p = folio_nr_pages(folio); -+add_more: -+ wdata->pages[*found_pages] = folio_page(folio, idx); -+ if (++*found_pages == tofind) { -+ folio_batch_release(&fbatch); -+ goto out; -+ } -+ if (++idx < p) { -+ folio_ref_inc(folio); -+ goto add_more; -+ } -+ } -+ folio_batch_release(&fbatch); -+ goto again; -+out: - return wdata; - } - -diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c -index 601214453c3a..fbd876e10a85 100644 ---- a/fs/ext4/inode.c -+++ b/fs/ext4/inode.c -@@ -2565,8 +2565,8 @@ static int ext4_da_writepages_trans_blocks(struct inode *inode) - static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - { - struct address_space *mapping = mpd->inode->i_mapping; -- struct pagevec pvec; -- unsigned int nr_pages; -+ struct folio_batch fbatch; -+ unsigned int nr_folios; - long left = mpd->wbc->nr_to_write; - pgoff_t index = mpd->first_page; - pgoff_t end = mpd->last_page; -@@ -2580,18 +2580,17 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - tag = PAGECACHE_TAG_TOWRITE; - else - tag = PAGECACHE_TAG_DIRTY; -- -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - mpd->map.m_len = 0; - mpd->next_page = index; - while (index <= end) { -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - - /* - * Accumulated enough dirty pages? This doesn't apply -@@ -2605,10 +2604,10 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - goto out; - - /* If we can't merge this page, we are done. */ -- if (mpd->map.m_len > 0 && mpd->next_page != page->index) -+ if (mpd->map.m_len > 0 && mpd->next_page != folio->index) - goto out; - -- lock_page(page); -+ folio_lock(folio); - /* - * If the page is no longer dirty, or its mapping no - * longer corresponds to inode we are writing (which -@@ -2616,16 +2615,16 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - * page is already under writeback and we are not doing - * a data integrity writeback, skip the page - */ -- if (!PageDirty(page) || -- (PageWriteback(page) && -+ if (!folio_test_dirty(folio) || -+ (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || -- unlikely(page->mapping != mapping)) { -- unlock_page(page); -+ unlikely(folio->mapping != mapping)) { -+ folio_unlock(folio); - continue; - } - -- wait_on_page_writeback(page); -- BUG_ON(PageWriteback(page)); -+ folio_wait_writeback(folio); -+ BUG_ON(folio_test_writeback(folio)); - - /* - * Should never happen but for buggy code in -@@ -2636,33 +2635,33 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) - * - * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz - */ -- if (!page_has_buffers(page)) { -- ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index); -- ClearPageDirty(page); -- unlock_page(page); -+ if (!folio_buffers(folio)) { -+ ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index); -+ folio_clear_dirty(folio); -+ folio_unlock(folio); - continue; - } - - if (mpd->map.m_len == 0) -- mpd->first_page = page->index; -- mpd->next_page = page->index + 1; -+ mpd->first_page = folio->index; -+ mpd->next_page = folio->index + folio_nr_pages(folio); - /* Add all dirty buffers to mpd */ -- lblk = ((ext4_lblk_t)page->index) << -+ lblk = ((ext4_lblk_t)folio->index) << - (PAGE_SHIFT - blkbits); -- head = page_buffers(page); -+ head = folio_buffers(folio); - err = mpage_process_page_bufs(mpd, head, head, lblk); - if (err <= 0) - goto out; - err = 0; -- left--; -+ left -= folio_nr_pages(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - mpd->scanned_until_end = 1; - return 0; - out: -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - return err; - } - -diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c -index 8259e0fa97e1..9f6694f7d723 100644 ---- a/fs/f2fs/checkpoint.c -+++ b/fs/f2fs/checkpoint.c -@@ -377,59 +377,62 @@ long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type, - { - struct address_space *mapping = META_MAPPING(sbi); - pgoff_t index = 0, prev = ULONG_MAX; -- struct pagevec pvec; -+ struct folio_batch fbatch; - long nwritten = 0; -- int nr_pages; -+ int nr_folios; - struct writeback_control wbc = { - .for_reclaim = 0, - }; - struct blk_plug plug; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - - blk_start_plug(&plug); - -- while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(mapping, &index, -+ (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- if (prev == ULONG_MAX) -- prev = page->index - 1; -- if (nr_to_write != LONG_MAX && page->index != prev + 1) { -- pagevec_release(&pvec); -+ if (nr_to_write != LONG_MAX && i != 0 && -+ folio->index != prev + -+ folio_nr_pages(fbatch.folios[i-1])) { -+ folio_batch_release(&fbatch); - goto stop; - } - -- lock_page(page); -+ folio_lock(folio); - -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- f2fs_wait_on_page_writeback(page, META, true, true); -+ f2fs_wait_on_page_writeback(&folio->page, META, -+ true, true); - -- if (!clear_page_dirty_for_io(page)) -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - -- if (__f2fs_write_meta_page(page, &wbc, io_type)) { -- unlock_page(page); -+ if (__f2fs_write_meta_page(&folio->page, &wbc, -+ io_type)) { -+ folio_unlock(folio); - break; - } -- nwritten++; -- prev = page->index; -+ nwritten += folio_nr_pages(folio); -+ prev = folio->index; - if (unlikely(nwritten >= nr_to_write)) - break; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - stop: -@@ -1381,7 +1384,7 @@ static void commit_checkpoint(struct f2fs_sb_info *sbi, - }; - - /* -- * pagevec_lookup_tag and lock_page again will take -+ * filemap_get_folios_tag and lock_page again will take - * some extra time. Therefore, f2fs_update_meta_pages and - * f2fs_sync_meta_pages are combined in this function. - */ -diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c -index 70e97075e535..e1bd2e859f64 100644 ---- a/fs/f2fs/compress.c -+++ b/fs/f2fs/compress.c -@@ -841,10 +841,11 @@ bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) - return is_page_in_cluster(cc, index); - } - --bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, -- int index, int nr_pages, bool uptodate) -+bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, -+ struct folio_batch *fbatch, -+ int index, int nr_folios, bool uptodate) - { -- unsigned long pgidx = pages[index]->index; -+ unsigned long pgidx = fbatch->folios[index]->index; - int i = uptodate ? 0 : 1; - - /* -@@ -854,13 +855,13 @@ bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, - if (uptodate && (pgidx % cc->cluster_size)) - return false; - -- if (nr_pages - index < cc->cluster_size) -+ if (nr_folios - index < cc->cluster_size) - return false; - - for (; i < cc->cluster_size; i++) { -- if (pages[index + i]->index != pgidx + i) -+ if (fbatch->folios[index + i]->index != pgidx + i) - return false; -- if (uptodate && !PageUptodate(pages[index + i])) -+ if (uptodate && !folio_test_uptodate(fbatch->folios[index + i])) - return false; - } - -diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c -index aa3ccddfa037..b14747598b39 100644 ---- a/fs/f2fs/data.c -+++ b/fs/f2fs/data.c -@@ -2917,7 +2917,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - { - int ret = 0; - int done = 0, retry = 0; -- struct page *pages[F2FS_ONSTACK_PAGES]; -+ struct folio_batch fbatch; - struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); - struct bio *bio = NULL; - sector_t last_block; -@@ -2938,7 +2938,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - .private = NULL, - }; - #endif -- int nr_pages; -+ int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; -@@ -2948,6 +2948,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - int submitted = 0; - int i; - -+ folio_batch_init(&fbatch); -+ - if (get_dirty_pages(mapping->host) <= - SM_I(F2FS_M_SB(mapping))->min_hot_blocks) - set_inode_flag(mapping->host, FI_HOT_DATA); -@@ -2973,13 +2975,13 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; - while (!done && !retry && (index <= end)) { -- nr_pages = find_get_pages_range_tag(mapping, &index, end, -- tag, F2FS_ONSTACK_PAGES, pages); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - bool need_readd; - readd: - need_readd = false; -@@ -2996,7 +2998,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - } - - if (!f2fs_cluster_can_merge_page(&cc, -- page->index)) { -+ folio->index)) { - ret = f2fs_write_multi_pages(&cc, - &submitted, wbc, io_type); - if (!ret) -@@ -3005,27 +3007,28 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - } - - if (unlikely(f2fs_cp_error(sbi))) -- goto lock_page; -+ goto lock_folio; - - if (!f2fs_cluster_is_empty(&cc)) -- goto lock_page; -+ goto lock_folio; - - if (f2fs_all_cluster_page_ready(&cc, -- pages, i, nr_pages, true)) -- goto lock_page; -+ &fbatch, i, nr_folios, true)) -+ goto lock_folio; - - ret2 = f2fs_prepare_compress_overwrite( - inode, &pagep, -- page->index, &fsdata); -+ folio->index, &fsdata); - if (ret2 < 0) { - ret = ret2; - done = 1; - break; - } else if (ret2 && - (!f2fs_compress_write_end(inode, -- fsdata, page->index, 1) || -+ fsdata, folio->index, 1) || - !f2fs_all_cluster_page_ready(&cc, -- pages, i, nr_pages, false))) { -+ &fbatch, i, nr_folios, -+ false))) { - retry = 1; - break; - } -@@ -3038,46 +3041,47 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - break; - } - #ifdef CONFIG_F2FS_FS_COMPRESSION --lock_page: -+lock_folio: - #endif -- done_index = page->index; -+ done_index = folio->index; - retry_write: -- lock_page(page); -+ folio_lock(folio); - -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- f2fs_wait_on_page_writeback(page, -+ f2fs_wait_on_page_writeback( -+ &folio->page, - DATA, true, true); - else - goto continue_unlock; - } - -- if (!clear_page_dirty_for_io(page)) -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - #ifdef CONFIG_F2FS_FS_COMPRESSION - if (f2fs_compressed_file(inode)) { -- get_page(page); -- f2fs_compress_ctx_add_page(&cc, page); -+ folio_get(folio); -+ f2fs_compress_ctx_add_page(&cc, &folio->page); - continue; - } - #endif -- ret = f2fs_write_single_data_page(page, &submitted, -- &bio, &last_block, wbc, io_type, -- 0, true); -+ ret = f2fs_write_single_data_page(&folio->page, -+ &submitted, &bio, &last_block, -+ wbc, io_type, 0, true); - if (ret == AOP_WRITEPAGE_ACTIVATE) -- unlock_page(page); -+ folio_unlock(folio); - #ifdef CONFIG_F2FS_FS_COMPRESSION - result: - #endif -@@ -3101,7 +3105,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - } - goto next; - } -- done_index = page->index + 1; -+ done_index = folio->index + -+ folio_nr_pages(folio); - done = 1; - break; - } -@@ -3115,7 +3120,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, - if (need_readd) - goto readd; - } -- release_pages(pages, nr_pages); -+ folio_batch_release(&fbatch); - cond_resched(); - } - #ifdef CONFIG_F2FS_FS_COMPRESSION -diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h -index 3c7cdb70fe2e..dcb28240f724 100644 ---- a/fs/f2fs/f2fs.h -+++ b/fs/f2fs/f2fs.h -@@ -4196,8 +4196,9 @@ void f2fs_end_read_compressed_page(struct page *page, bool failed, - block_t blkaddr, bool in_task); - bool f2fs_cluster_is_empty(struct compress_ctx *cc); - bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index); --bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, -- int index, int nr_pages, bool uptodate); -+bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, -+ struct folio_batch *fbatch, int index, int nr_folios, -+ bool uptodate); - bool f2fs_sanity_check_cluster(struct dnode_of_data *dn); - void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page); - int f2fs_write_multi_pages(struct compress_ctx *cc, -diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c -index e06a0c478b39..b993be76013e 100644 ---- a/fs/f2fs/node.c -+++ b/fs/f2fs/node.c -@@ -1513,23 +1513,24 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) - static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) - { - pgoff_t index; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct page *last_page = NULL; -- int nr_pages; -+ int nr_folios; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - index = 0; - -- while ((nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, -- PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - - if (unlikely(f2fs_cp_error(sbi))) { - f2fs_put_page(last_page, 0); -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - return ERR_PTR(-EIO); - } - -@@ -1560,7 +1561,7 @@ static struct page *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) - last_page = page; - unlock_page(page); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - return last_page; -@@ -1726,12 +1727,12 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - unsigned int *seq_id) - { - pgoff_t index; -- struct pagevec pvec; -+ struct folio_batch fbatch; - int ret = 0; - struct page *last_page = NULL; - bool marked = false; - nid_t ino = inode->i_ino; -- int nr_pages; -+ int nr_folios; - int nwritten = 0; - - if (atomic) { -@@ -1740,20 +1741,21 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - return PTR_ERR_OR_ZERO(last_page); - } - retry: -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - index = 0; - -- while ((nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, -- PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - bool submitted = false; - - if (unlikely(f2fs_cp_error(sbi))) { - f2fs_put_page(last_page, 0); -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - ret = -EIO; - goto out; - } -@@ -1819,7 +1821,7 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, - break; - } - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (ret || marked) -@@ -1884,17 +1886,18 @@ static bool flush_dirty_inode(struct page *page) - void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) - { - pgoff_t index = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while ((nr_pages = pagevec_lookup_tag(&pvec, -- NODE_MAPPING(sbi), &index, PAGECACHE_TAG_DIRTY))) { -+ while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - - if (!IS_DNODE(page)) - continue; -@@ -1921,7 +1924,7 @@ void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) - } - unlock_page(page); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -@@ -1931,23 +1934,24 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, - bool do_balance, enum iostat_type io_type) - { - pgoff_t index; -- struct pagevec pvec; -+ struct folio_batch fbatch; - int step = 0; - int nwritten = 0; - int ret = 0; -- int nr_pages, done = 0; -+ int nr_folios, done = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - - next_step: - index = 0; - -- while (!done && (nr_pages = pagevec_lookup_tag(&pvec, -- NODE_MAPPING(sbi), &index, PAGECACHE_TAG_DIRTY))) { -+ while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), -+ &index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY, -+ &fbatch))) { - int i; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct page *page = &fbatch.folios[i]->page; - bool submitted = false; - - /* give a priority to WB_SYNC threads */ -@@ -2022,7 +2026,7 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, - if (--wbc->nr_to_write == 0) - break; - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (wbc->nr_to_write == 0) { -diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c -index 05bee80ac7de..8f87c2551a3d 100644 ---- a/fs/gfs2/aops.c -+++ b/fs/gfs2/aops.c -@@ -195,67 +195,71 @@ static int gfs2_writepages(struct address_space *mapping, - } - - /** -- * gfs2_write_jdata_pagevec - Write back a pagevec's worth of pages -+ * gfs2_write_jdata_batch - Write back a folio batch's worth of folios - * @mapping: The mapping - * @wbc: The writeback control -- * @pvec: The vector of pages -- * @nr_pages: The number of pages to write -+ * @fbatch: The batch of folios - * @done_index: Page index - * - * Returns: non-zero if loop should terminate, zero otherwise - */ - --static int gfs2_write_jdata_pagevec(struct address_space *mapping, -+static int gfs2_write_jdata_batch(struct address_space *mapping, - struct writeback_control *wbc, -- struct pagevec *pvec, -- int nr_pages, -+ struct folio_batch *fbatch, - pgoff_t *done_index) - { - struct inode *inode = mapping->host; - struct gfs2_sbd *sdp = GFS2_SB(inode); -- unsigned nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); -+ unsigned nrblocks; - int i; - int ret; -+ int nr_pages = 0; -+ int nr_folios = folio_batch_count(fbatch); -+ -+ for (i = 0; i < nr_folios; i++) -+ nr_pages += folio_nr_pages(fbatch->folios[i]); -+ nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); - - ret = gfs2_trans_begin(sdp, nrblocks, nrblocks); - if (ret < 0) - return ret; - -- for(i = 0; i < nr_pages; i++) { -- struct page *page = pvec->pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch->folios[i]; - -- *done_index = page->index; -+ *done_index = folio->index; - -- lock_page(page); -+ folio_lock(folio); - -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - else - goto continue_unlock; - } - -- BUG_ON(PageWriteback(page)); -- if (!clear_page_dirty_for_io(page)) -+ BUG_ON(folio_test_writeback(folio)); -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - trace_wbc_writepage(wbc, inode_to_bdi(inode)); - -- ret = __gfs2_jdata_writepage(page, wbc); -+ ret = __gfs2_jdata_writepage(&folio->page, wbc); - if (unlikely(ret)) { - if (ret == AOP_WRITEPAGE_ACTIVATE) { -- unlock_page(page); -+ folio_unlock(folio); - ret = 0; - } else { - -@@ -268,7 +272,8 @@ static int gfs2_write_jdata_pagevec(struct address_space *mapping, - * not be suitable for data integrity - * writeout). - */ -- *done_index = page->index + 1; -+ *done_index = folio->index + -+ folio_nr_pages(folio); - ret = 1; - break; - } -@@ -305,8 +310,8 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - { - int ret = 0; - int done = 0; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - pgoff_t writeback_index; - pgoff_t index; - pgoff_t end; -@@ -315,7 +320,7 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - int range_whole = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - writeback_index = mapping->writeback_index; /* prev offset */ - index = writeback_index; -@@ -341,17 +346,18 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; - while (!done && (index <= end)) { -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ if (nr_folios == 0) - break; - -- ret = gfs2_write_jdata_pagevec(mapping, wbc, &pvec, nr_pages, &done_index); -+ ret = gfs2_write_jdata_batch(mapping, wbc, &fbatch, -+ &done_index); - if (ret) - done = 1; - if (ret > 0) - ret = 0; -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c -index 9f4d9432d38a..1e26f32a4e36 100644 ---- a/fs/nilfs2/btree.c -+++ b/fs/nilfs2/btree.c -@@ -2143,7 +2143,7 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, - struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; - struct address_space *btcache = btnc_inode->i_mapping; - struct list_head lists[NILFS_BTREE_LEVEL_MAX]; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct buffer_head *bh, *head; - pgoff_t index = 0; - int level, i; -@@ -2153,19 +2153,19 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, - level++) - INIT_LIST_HEAD(&lists[level]); - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while (pagevec_lookup_tag(&pvec, btcache, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- bh = head = page_buffers(pvec.pages[i]); -+ while (filemap_get_folios_tag(btcache, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ bh = head = folio_buffers(fbatch.folios[i]); - do { - if (buffer_dirty(bh)) - nilfs_btree_add_dirty_buffer(btree, - lists, bh); - } while ((bh = bh->b_this_page) != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c -index 3267e96c256c..b66f4e988016 100644 ---- a/fs/nilfs2/page.c -+++ b/fs/nilfs2/page.c -@@ -240,42 +240,43 @@ static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty) - int nilfs_copy_dirty_pages(struct address_space *dmap, - struct address_space *smap) - { -- struct pagevec pvec; -+ struct folio_batch fbatch; - unsigned int i; - pgoff_t index = 0; - int err = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - repeat: -- if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY)) -+ if (!filemap_get_folios_tag(smap, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) - return 0; - -- for (i = 0; i < pagevec_count(&pvec); i++) { -- struct page *page = pvec.pages[i], *dpage; -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ struct folio *folio = fbatch.folios[i], *dfolio; - -- lock_page(page); -- if (unlikely(!PageDirty(page))) -- NILFS_PAGE_BUG(page, "inconsistent dirty state"); -+ folio_lock(folio); -+ if (unlikely(!folio_test_dirty(folio))) -+ NILFS_PAGE_BUG(&folio->page, "inconsistent dirty state"); - -- dpage = grab_cache_page(dmap, page->index); -- if (unlikely(!dpage)) { -+ dfolio = filemap_grab_folio(dmap, folio->index); -+ if (unlikely(!dfolio)) { - /* No empty page is added to the page cache */ - err = -ENOMEM; -- unlock_page(page); -+ folio_unlock(folio); - break; - } -- if (unlikely(!page_has_buffers(page))) -- NILFS_PAGE_BUG(page, -+ if (unlikely(!folio_buffers(folio))) -+ NILFS_PAGE_BUG(&folio->page, - "found empty page in dat page cache"); - -- nilfs_copy_page(dpage, page, 1); -- __set_page_dirty_nobuffers(dpage); -+ nilfs_copy_page(&dfolio->page, &folio->page, 1); -+ filemap_dirty_folio(folio_mapping(dfolio), dfolio); - -- unlock_page(dpage); -- put_page(dpage); -- unlock_page(page); -+ folio_unlock(dfolio); -+ folio_put(dfolio); -+ folio_unlock(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (likely(!err)) -@@ -357,22 +358,22 @@ void nilfs_copy_back_pages(struct address_space *dmap, - */ - void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent) - { -- struct pagevec pvec; -+ struct folio_batch fbatch; - unsigned int i; - pgoff_t index = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while (pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- struct page *page = pvec.pages[i]; -+ while (filemap_get_folios_tag(mapping, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ struct folio *folio = fbatch.folios[i]; - -- lock_page(page); -- nilfs_clear_dirty_page(page, silent); -- unlock_page(page); -+ folio_lock(folio); -+ nilfs_clear_dirty_page(&folio->page, silent); -+ folio_unlock(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c -index 0afe0832c754..6f2ca279d230 100644 ---- a/fs/nilfs2/segment.c -+++ b/fs/nilfs2/segment.c -@@ -680,7 +680,7 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - loff_t start, loff_t end) - { - struct address_space *mapping = inode->i_mapping; -- struct pagevec pvec; -+ struct folio_batch fbatch; - pgoff_t index = 0, last = ULONG_MAX; - size_t ndirties = 0; - int i; -@@ -694,23 +694,26 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - index = start >> PAGE_SHIFT; - last = end >> PAGE_SHIFT; - } -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - repeat: - if (unlikely(index > last) || -- !pagevec_lookup_range_tag(&pvec, mapping, &index, last, -- PAGECACHE_TAG_DIRTY)) -+ !filemap_get_folios_tag(mapping, &index, last, -+ PAGECACHE_TAG_DIRTY, &fbatch)) - return ndirties; - -- for (i = 0; i < pagevec_count(&pvec); i++) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { - struct buffer_head *bh, *head; -- struct page *page = pvec.pages[i]; -+ struct folio *folio = fbatch.folios[i]; - -- lock_page(page); -- if (!page_has_buffers(page)) -- create_empty_buffers(page, i_blocksize(inode), 0); -- unlock_page(page); -+ folio_lock(folio); -+ head = folio_buffers(folio); -+ if (!head) { -+ create_empty_buffers(&folio->page, i_blocksize(inode), 0); -+ head = folio_buffers(folio); -+ } -+ folio_unlock(folio); - -- bh = head = page_buffers(page); -+ bh = head; - do { - if (!buffer_dirty(bh) || buffer_async_write(bh)) - continue; -@@ -718,13 +721,13 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - list_add_tail(&bh->b_assoc_buffers, listp); - ndirties++; - if (unlikely(ndirties >= nlimit)) { -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - return ndirties; - } - } while (bh = bh->b_this_page, bh != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - goto repeat; - } -@@ -734,20 +737,19 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, - { - struct nilfs_inode_info *ii = NILFS_I(inode); - struct inode *btnc_inode = ii->i_assoc_inode; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct buffer_head *bh, *head; - unsigned int i; - pgoff_t index = 0; - - if (!btnc_inode) - return; -+ folio_batch_init(&fbatch); - -- pagevec_init(&pvec); -- -- while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- bh = head = page_buffers(pvec.pages[i]); -+ while (filemap_get_folios_tag(btnc_inode->i_mapping, &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ bh = head = folio_buffers(fbatch.folios[i]); - do { - if (buffer_dirty(bh) && - !buffer_async_write(bh)) { -@@ -758,7 +760,7 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, - bh = bh->b_this_page; - } while (bh != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index 39f05e00dac4..9fa4c4e0809b 100644 ---- a/include/linux/pagemap.h -+++ b/include/linux/pagemap.h -@@ -547,6 +547,26 @@ static inline struct folio *filemap_lock_folio(struct address_space *mapping, - return __filemap_get_folio(mapping, index, FGP_LOCK, 0); - } - -+/** -+ * filemap_grab_folio - grab a folio from the page cache -+ * @mapping: The address space to search -+ * @index: The page index -+ * -+ * Looks up the page cache entry at @mapping & @index. If no folio is found, -+ * a new folio is created. The folio is locked, marked as accessed, and -+ * returned. -+ * -+ * Return: A found or created folio. NULL if no folio is found and failed to -+ * create a folio. -+ */ -+static inline struct folio *filemap_grab_folio(struct address_space *mapping, -+ pgoff_t index) -+{ -+ return __filemap_get_folio(mapping, index, -+ FGP_LOCK | FGP_ACCESSED | FGP_CREAT, -+ mapping_gfp_mask(mapping)); -+} -+ - /** - * find_get_page - find and get a page reference - * @mapping: the address_space to search -@@ -720,16 +740,8 @@ unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, - pgoff_t end, struct folio_batch *fbatch); - unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, - unsigned int nr_pages, struct page **pages); --unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, -- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages); --static inline unsigned find_get_pages_tag(struct address_space *mapping, -- pgoff_t *index, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages) --{ -- return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag, -- nr_pages, pages); --} -+unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, -+ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch); - - struct page *grab_cache_page_write_begin(struct address_space *mapping, - pgoff_t index); -diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h -index 215eb6c3bdc9..a520632297ac 100644 ---- a/include/linux/pagevec.h -+++ b/include/linux/pagevec.h -@@ -26,14 +26,6 @@ struct pagevec { - }; - - void __pagevec_release(struct pagevec *pvec); --unsigned pagevec_lookup_range_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, pgoff_t end, -- xa_mark_t tag); --static inline unsigned pagevec_lookup_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, xa_mark_t tag) --{ -- return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag); --} - - static inline void pagevec_init(struct pagevec *pvec) - { -diff --git a/mm/filemap.c b/mm/filemap.c -index 15800334147b..b986f246a6ae 100644 ---- a/mm/filemap.c -+++ b/mm/filemap.c -@@ -503,28 +503,30 @@ static void __filemap_fdatawait_range(struct address_space *mapping, - { - pgoff_t index = start_byte >> PAGE_SHIFT; - pgoff_t end = end_byte >> PAGE_SHIFT; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned nr_folios; - - if (end_byte < start_byte) - return; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); -+ - while (index <= end) { - unsigned i; - -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, -- end, PAGECACHE_TAG_WRITEBACK); -- if (!nr_pages) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ PAGECACHE_TAG_WRITEBACK, &fbatch); -+ -+ if (!nr_folios) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- wait_on_page_writeback(page); -- ClearPageError(page); -+ folio_wait_writeback(folio); -+ folio_clear_error(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -@@ -2255,64 +2257,57 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, - EXPORT_SYMBOL(find_get_pages_contig); - - /** -- * find_get_pages_range_tag - Find and return head pages matching @tag. -- * @mapping: the address_space to search -- * @index: the starting page index -- * @end: The final page index (inclusive) -- * @tag: the tag index -- * @nr_pages: the maximum number of pages -- * @pages: where the resulting pages are placed -+ * filemap_get_folios_tag - Get a batch of folios matching @tag. -+ * @mapping: The address_space to search -+ * @start: The starting page index -+ * @end: The final page index (inclusive) -+ * @tag: The tag index -+ * @fbatch: The batch to fill - * -- * Like find_get_pages_range(), except we only return head pages which are -- * tagged with @tag. @index is updated to the index immediately after the -- * last page we return, ready for the next iteration. -+ * Same as filemap_get_folios, but only returning folios tagged with @tag - * -- * Return: the number of pages which were found. -+ * Return: The number of folios found -+ * Also update @start to index the next folio for traversal - */ --unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, -- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages) -+unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, -+ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) - { -- XA_STATE(xas, &mapping->i_pages, *index); -+ XA_STATE(xas, &mapping->i_pages, *start); - struct folio *folio; -- unsigned ret = 0; -- -- if (unlikely(!nr_pages)) -- return 0; - - rcu_read_lock(); -- while ((folio = find_get_entry(&xas, end, tag))) { -- /* -- * Shadow entries should never be tagged, but this iteration -+ while ((folio = find_get_entry(&xas, end, tag)) != NULL) { -+ /* Shadow entries should never be tagged, but this iteration - * is lockless so there is a window for page reclaim to evict -- * a page we saw tagged. Skip over it. -+ * a page we saw tagged. Skip over it. - */ - if (xa_is_value(folio)) - continue; -+ if (!folio_batch_add(fbatch, folio)) { -+ unsigned long nr = folio_nr_pages(folio); - -- pages[ret] = &folio->page; -- if (++ret == nr_pages) { -- *index = folio->index + folio_nr_pages(folio); -+ if (folio_test_hugetlb(folio)) -+ nr = 1; -+ *start = folio->index + nr; - goto out; - } - } -- - /* -- * We come here when we got to @end. We take care to not overflow the -- * index @index as it confuses some of the callers. This breaks the -- * iteration when there is a page at index -1 but that is already -- * broken anyway. -+ * We come here when there is no page beyond @end. We take care to not -+ * overflow the index @start as it confuses some of the callers. This -+ * breaks the iteration when there is a page at index -1 but that is -+ * already broke anyway. - */ - if (end == (pgoff_t)-1) -- *index = (pgoff_t)-1; -+ *start = (pgoff_t)-1; - else -- *index = end + 1; -+ *start = end + 1; - out: - rcu_read_unlock(); - -- return ret; -+ return folio_batch_count(fbatch); - } --EXPORT_SYMBOL(find_get_pages_range_tag); -+EXPORT_SYMBOL(filemap_get_folios_tag); - - /* - * CD/DVDs are error prone. When a medium error occurs, the driver may fail -diff --git a/mm/page-writeback.c b/mm/page-writeback.c -index cbcecd565c16..8f20c037a87a 100644 ---- a/mm/page-writeback.c -+++ b/mm/page-writeback.c -@@ -2293,15 +2293,15 @@ int write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int error; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; - int range_whole = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* prev offset */ - end = -1; -@@ -2321,17 +2321,18 @@ int write_cache_pages(struct address_space *mapping, - while (!done && (index <= end)) { - int i; - -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); -+ -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- done_index = page->index; -+ done_index = folio->index; - -- lock_page(page); -+ folio_lock(folio); - - /* - * Page truncated or invalidated. We can freely skip it -@@ -2341,30 +2342,30 @@ int write_cache_pages(struct address_space *mapping, - * even if there is now a new, dirty page at the same - * pagecache address. - */ -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - else - goto continue_unlock; - } - -- BUG_ON(PageWriteback(page)); -- if (!clear_page_dirty_for_io(page)) -+ BUG_ON(folio_test_writeback(folio)); -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); -- error = (*writepage)(page, wbc, data); -+ error = writepage(&folio->page, wbc, data); - if (unlikely(error)) { - /* - * Handle errors according to the type of -@@ -2379,11 +2380,12 @@ int write_cache_pages(struct address_space *mapping, - * the first error. - */ - if (error == AOP_WRITEPAGE_ACTIVATE) { -- unlock_page(page); -+ folio_unlock(folio); - error = 0; - } else if (wbc->sync_mode != WB_SYNC_ALL) { - ret = error; -- done_index = page->index + 1; -+ done_index = folio->index + -+ folio_nr_pages(folio); - done = 1; - break; - } -@@ -2403,7 +2405,7 @@ int write_cache_pages(struct address_space *mapping, - break; - } - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/mm/swap.c b/mm/swap.c -index 027943b341a0..29bf97746da0 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -1099,16 +1099,6 @@ void folio_batch_remove_exceptionals(struct folio_batch *fbatch) - fbatch->nr = j; - } - --unsigned pagevec_lookup_range_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, pgoff_t end, -- xa_mark_t tag) --{ -- pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, -- PAGEVEC_SIZE, pvec->pages); -- return pagevec_count(pvec); --} --EXPORT_SYMBOL(pagevec_lookup_range_tag); -- - /* - * Perform any setup for the swap system - */ --- -2.38.0.rc1.8.g2a7d63a245 - -From d30c8771521031923117e12623f501a6cf17ec66 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 19 Sep 2022 14:42:00 +0200 -Subject: [PATCH 11/13] fixes - -Signed-off-by: Peter Jung ---- - Documentation/ABI/stable/sysfs-block | 10 + - .../testing/sysfs-class-led-trigger-blkdev | 68 + - Documentation/leds/index.rst | 1 + - Documentation/leds/ledtrig-blkdev.rst | 155 +++ - arch/x86/events/amd/uncore.c | 1 + - arch/x86/kernel/alternative.c | 9 + - arch/x86/net/bpf_jit_comp.c | 4 +- - drivers/leds/trigger/Kconfig | 9 + - drivers/leds/trigger/Makefile | 1 + - drivers/leds/trigger/ledtrig-blkdev.c | 1177 +++++++++++++++++ - include/linux/pageblock-flags.h | 2 +- - include/net/bluetooth/rfcomm.h | 3 + - net/bluetooth/rfcomm/core.c | 2 + - net/bluetooth/rfcomm/sock.c | 34 +- - tools/objtool/check.c | 3 - - 15 files changed, 1462 insertions(+), 17 deletions(-) - create mode 100644 Documentation/ABI/testing/sysfs-class-led-trigger-blkdev - create mode 100644 Documentation/leds/ledtrig-blkdev.rst - create mode 100644 drivers/leds/trigger/ledtrig-blkdev.c - -diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block -index cd14ecb3c9a5..d11d04d804fe 100644 ---- a/Documentation/ABI/stable/sysfs-block -+++ b/Documentation/ABI/stable/sysfs-block -@@ -101,6 +101,16 @@ Description: - devices that support receiving integrity metadata. - - -+What: /sys/block//linked_leds -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Directory that contains symbolic links to all LEDs that -+ are associated with (linked to) this block device by the -+ blkdev LED trigger. Only present when at least one LED -+ is linked. (See Documentation/leds/ledtrig-blkdev.rst.) -+ -+ - What: /sys/block///alignment_offset - Date: April 2009 - Contact: Martin K. Petersen -diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev -new file mode 100644 -index 000000000000..d8d403569b21 ---- /dev/null -+++ b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev -@@ -0,0 +1,68 @@ -+What: /sys/class/leds//blink_time -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Time (in milliseconds) that the LED will be on during a single -+ "blink". -+ -+What: /sys/class/leds//check_interval -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Frequency (in milliseconds) with which block devices linked to -+ this LED will be checked for activity and the LED will -+ (potentially) be blinked. -+ -+What: /sys/class/leds//blink_on_read -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to read activity on any of its linked block devices. -+ -+What: /sys/class/leds//blink_on_write -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to write activity on any of its linked block devices. -+ -+What: /sys/class/leds//blink_on_discard -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to discard activity on any of its linked block devices. -+ -+What: /sys/class/leds//blink_on_flush -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to cache flush activity on any of its linked block devices. -+ -+What: /sys/class/leds//link_dev_by_path -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Associate a block device with this LED by writing the path to -+ the device special file (e.g. /dev/sda) to this attribute. -+ Symbolic links are followed. -+ -+What: /sys/class/leds//unlink_dev_by_path -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Remove the association between this LED and a block device by -+ writing the path to the device special file (e.g. /dev/sda) to -+ this attribute. Symbolic links are followed. -+ -+What: /sys/class/leds//linked_devices -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Directory containing links to all block devices that are -+ associated with this LED. (Note that the names of the -+ symbolic links in this directory are *kernel* names, which -+ may not match the device special file paths written to -+ link_device and unlink_device.) -diff --git a/Documentation/leds/index.rst b/Documentation/leds/index.rst -index e5d63b940045..e3c24e468cbc 100644 ---- a/Documentation/leds/index.rst -+++ b/Documentation/leds/index.rst -@@ -10,6 +10,7 @@ LEDs - leds-class - leds-class-flash - leds-class-multicolor -+ ledtrig-blkdev - ledtrig-oneshot - ledtrig-transient - ledtrig-usbport -diff --git a/Documentation/leds/ledtrig-blkdev.rst b/Documentation/leds/ledtrig-blkdev.rst -new file mode 100644 -index 000000000000..ae92aa559257 ---- /dev/null -+++ b/Documentation/leds/ledtrig-blkdev.rst -@@ -0,0 +1,155 @@ -+.. SPDX-License-Identifier: GPL-2.0 -+ -+================================= -+Block Device (blkdev) LED Trigger -+================================= -+ -+Available when ``CONFIG_LEDS_TRIGGER_BLKDEV=y`` or -+``CONFIG_LEDS_TRIGGER_BLKDEV=m``. -+ -+See also: -+ -+* ``Documentation/ABI/testing/sysfs-class-led-trigger-blkdev`` -+* ``Documentation/ABI/stable/sysfs-block`` (``/sys/block//linked_leds``) -+ -+Overview -+======== -+ -+.. note:: -+ The examples below use ```` to refer to the name of a -+ system-specific LED. If no suitable LED is available on a test -+ system (in a virtual machine, for example), it is possible to -+ use a userspace LED. (See ``Documentation/leds/uleds.rst``.) -+ -+Verify that the ``blkdev`` LED trigger is available:: -+ -+ # grep blkdev /sys/class/leds//trigger -+ ... rfkill-none blkdev -+ -+(If the previous command produces no output, you may need to load the trigger -+module - ``modprobe ledtrig_blkdev``. If the module is not available, check -+the value of ``CONFIG_LEDS_TRIGGER_BLKDEV`` in your kernel configuration.) -+ -+Associate the LED with the ``blkdev`` LED trigger:: -+ -+ # echo blkdev > /sys/class/leds//trigger -+ -+ # cat /sys/class/leds//trigger -+ ... rfkill-none [blkdev] -+ -+Note that several new device attributes are available in the -+``/sys/class/leds/`` directory. -+ -+* ``link_dev_by_path`` and ``unlink_dev_by_path`` are used to manage the set of -+ block devices associated with this LED. The LED will blink in response to -+ read or write activity on its linked devices. -+ -+* ``blink_on_read``, ``blink_on_write``, ``blink_on_discard``, and -+ ``blink_on_flush`` are boolean values that determine whether the LED will -+ blink when a particular type of activity is detected on one of its linked -+ block devices. -+ -+* ``blink_time`` is the duration (in milliseconds) of each blink of this LED. -+ (The minimum value is 10 milliseconds.) -+ -+* ``check_interval`` is the frequency (in milliseconds) with which block devices -+ linked to this LED will be checked for activity and the LED blinked (if the -+ correct type of activity has occurred). -+ -+* The ``linked_devices`` directory will contain a symbolic link to every device -+ that is associated with this LED. -+ -+Link a block device to the LED:: -+ -+ # echo /dev/sda > /sys/class/leds//link_dev_by_path -+ -+ # ls /sys/class/leds//linked_devices -+ sda -+ -+(The value written to ``link_dev_by_path`` must be the path of the device -+special file, such as ``/dev/sda``, that represents the block device - or the -+path of a symbolic link to such a device special file.) -+ -+Activity on the device will now cause the LED to blink. The duration of each -+blink (in milliseconds) can be adjusted by setting -+``/sys/class/leds//blink_time``. (But see **check_interval and -+blink_time** below.) -+ -+Associate a second device with the LED:: -+ -+ # echo /dev/sdb > /sys/class/leds//link_dev_by_path -+ -+ # ls /sys/class/leds//linked_devices -+ sda sdb -+ -+When a block device is linked to one or more LEDs, the LEDs are linked from -+the device's ``linked_leds`` directory:: -+ -+ # ls /sys/class/block/sd{a,b}/linked_leds -+ /sys/class/block/sda/linked_leds: -+ -+ -+ /sys/class/block/sdb/linked_leds: -+ -+ -+(The ``linked_leds`` directory only exists when the block device is linked to -+at least one LED.) -+ -+``check_interval`` and ``blink_time`` -+===================================== -+ -+* By default, linked block devices are checked for activity every 100 -+ milliseconds. This frequency can be changed for an LED via the -+ ``/sys/class/leds//check_interval`` attribute. (The minimum value is 25 -+ milliseconds.) -+ -+* All block devices associated with an LED are checked for activity every -+ ``check_interval`` milliseconds, and a blink is triggered if the correct type -+ of activity (as determined by the LED's ``blink_on_*`` attributes) is -+ detected. The duration of an LED's blink is determined by its ``blink_time`` -+ attribute. Thus (when the correct type of activity is detected), the LED will -+ be on for ``blink_time`` milliseconds and off for -+ ``check_interval - blink_time`` milliseconds. -+ -+* The LED subsystem ignores new blink requests for an LED that is already in -+ in the process of blinking, so setting a ``blink_time`` greater than or equal -+ to ``check_interval`` will cause some blinks to be missed. -+ -+* Because of processing times, scheduling latencies, etc., avoiding missed -+ blinks actually requires a difference of at least a few milliseconds between -+ the ``blink_time`` and ``check_interval``. The required difference is likely -+ to vary from system to system. As a reference, a Thecus N5550 NAS requires a -+ difference of 7 milliseconds (e.g. ``check_interval == 100``, -+ ``blink_time == 93``). -+ -+* The default values (``check_interval == 100``, ``blink_time == 75``) cause the -+ LED associated with a continuously active device to blink rapidly. For a more -+ "always on" effect, increase the ``blink_time`` (but not too much; see the -+ previous bullet). -+ -+Other Notes -+=========== -+ -+* Many (possibly all) types of block devices work with this trigger, including: -+ -+ * SCSI (including SATA and USB) hard disk drives and SSDs -+ * SCSI (including SATA and USB) optical drives -+ * NVMe SSDs -+ * SD cards -+ * loopback block devices (``/dev/loop*``) -+ * device mapper devices, such as LVM logical volumes -+ * MD RAID devices -+ * zRAM compressed RAM-disks -+ * partitions on block devics that support them -+ -+* The names of the symbolic links in ``/sys/class/leds//linked_devices`` -+ are **kernel** names, which may not match the paths used for -+ ``link_dev_by_path`` and ``unlink_dev_by_path``. This is most likely when a -+ symbolic link is used to refer to the device (as is common with logical -+ volumes), but it can be true for any device, because nothing prevents the -+ creation of device special files with arbitrary names (e.g. -+ ``sudo mknod /foo b 8 0``). -+ -+* The ``blkdev`` LED trigger supports many-to-many device/LED associations. -+ A device can be associated with multiple LEDs, and an LED can be associated -+ with multiple devices. -diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c -index d568afc705d2..83f15fe411b3 100644 ---- a/arch/x86/events/amd/uncore.c -+++ b/arch/x86/events/amd/uncore.c -@@ -553,6 +553,7 @@ static void uncore_clean_online(void) - - hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { - hlist_del(&uncore->node); -+ kfree(uncore->events); - kfree(uncore); - } - } -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index f9c9b5850847..3582e61ddce6 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) - return ret; - i += ret; - -+ /* -+ * The compiler is supposed to EMIT an INT3 after every unconditional -+ * JMP instruction due to AMD BTC. However, if the compiler is too old -+ * or SLS isn't enabled, we still need an INT3 after indirect JMPs -+ * even on Intel. -+ */ -+ if (op == JMP32_INSN_OPCODE && i < insn->length) -+ bytes[i++] = INT3_INSN_OPCODE; -+ - for (; i < insn->length;) - bytes[i++] = BYTES_NOP1; - -diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c -index c1f6c1c51d99..4922517ddb0d 100644 ---- a/arch/x86/net/bpf_jit_comp.c -+++ b/arch/x86/net/bpf_jit_comp.c -@@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) - OPTIMIZER_HIDE_VAR(reg); - emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); - } else { -- EMIT2(0xFF, 0xE0 + reg); -+ EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ -+ if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS)) -+ EMIT1(0xCC); /* int3 */ - } - - *pprog = prog; -diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig -index dc6816d36d06..bda249068182 100644 ---- a/drivers/leds/trigger/Kconfig -+++ b/drivers/leds/trigger/Kconfig -@@ -154,4 +154,13 @@ config LEDS_TRIGGER_TTY - - When build as a module this driver will be called ledtrig-tty. - -+config LEDS_TRIGGER_BLKDEV -+ tristate "LED Trigger for block devices" -+ depends on BLOCK -+ help -+ The blkdev LED trigger allows LEDs to be controlled by block device -+ activity (reads and writes). -+ -+ See Documentation/leds/ledtrig-blkdev.rst. -+ - endif # LEDS_TRIGGERS -diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile -index 25c4db97cdd4..d53bab5d93f1 100644 ---- a/drivers/leds/trigger/Makefile -+++ b/drivers/leds/trigger/Makefile -@@ -16,3 +16,4 @@ obj-$(CONFIG_LEDS_TRIGGER_NETDEV) += ledtrig-netdev.o - obj-$(CONFIG_LEDS_TRIGGER_PATTERN) += ledtrig-pattern.o - obj-$(CONFIG_LEDS_TRIGGER_AUDIO) += ledtrig-audio.o - obj-$(CONFIG_LEDS_TRIGGER_TTY) += ledtrig-tty.o -+obj-$(CONFIG_LEDS_TRIGGER_BLKDEV) += ledtrig-blkdev.o -diff --git a/drivers/leds/trigger/ledtrig-blkdev.c b/drivers/leds/trigger/ledtrig-blkdev.c -new file mode 100644 -index 000000000000..0d2eb36fbe38 ---- /dev/null -+++ b/drivers/leds/trigger/ledtrig-blkdev.c -@@ -0,0 +1,1177 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+ -+/* -+ * Block device LED trigger -+ * -+ * Copyright 2021-2022 Ian Pilcher -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+/** -+ * DOC: Overview -+ * -+ * The ``blkdev`` LED trigger works by periodically checking the activity -+ * counters of block devices that have been linked to one or more LEDs and -+ * blinking those LED(s) if the correct type of activity has occurred. The -+ * periodic check is scheduled with the Linux kernel's deferred work facility. -+ * -+ * Trigger-specific data about block devices and LEDs is stored in two data -+ * structures --- &struct blkdev_trig_bdev (a "BTB") and &struct blkdev_trig_led -+ * (a "BTL"). Each structure contains a &struct xarray that holds links to any -+ * linked devices of the other type. I.e. &blkdev_trig_bdev.linked_btls -+ * contains links to all BTLs whose LEDs have been linked to the BTB's block -+ * device, and &blkdev_trig_led.linked_btbs contains links to all BTBs whose -+ * block devices have been linked to the BTL's LED. Thus, a block device can -+ * be linked to more than one LED, and an LED can be linked to more than one -+ * block device. -+ */ -+ -+/* Default, minimum & maximum blink duration (milliseconds) */ -+#define BLKDEV_TRIG_BLINK_DEF 75 -+#define BLKDEV_TRIG_BLINK_MIN 10 -+#define BLKDEV_TRIG_BLINK_MAX 86400000 /* 24 hours */ -+ -+/* Default, minimum & maximum activity check interval (milliseconds) */ -+#define BLKDEV_TRIG_CHECK_DEF 100 -+#define BLKDEV_TRIG_CHECK_MIN 25 -+#define BLKDEV_TRIG_CHECK_MAX 86400000 /* 24 hours */ -+ -+/* -+ * If blkdev_trig_check() can't lock the mutex, how long to wait before trying -+ * again (milliseconds) -+ */ -+#define BLKDEV_TRIG_CHECK_RETRY 5 -+ -+/* Mode argument for calls to blkdev_get_by_path() and blkdev_put() */ -+#define BLKDEV_TRIG_FMODE 0 -+ -+/** -+ * struct blkdev_trig_bdev - Trigger-specific data about a block device. -+ * @last_checked: Time (in jiffies) at which the trigger last checked this -+ * block device for activity. -+ * @last_activity: Time (in jiffies) at which the trigger last detected -+ * activity of each type. -+ * @ios: Activity counter values for each type, corresponding to -+ * the timestamps in &last_activity. -+ * @index: &xarray index, so the BTB can be included in one or more -+ * &blkdev_trig_led.linked_btbs. -+ * @bdev: The block device. -+ * @linked_btls: The BTLs that represent the LEDs linked to the BTB's -+ * block device. -+ * -+ * Every block device linked to at least one LED gets a "BTB." A BTB is created -+ * when a block device that is not currently linked to any LEDs is linked to an -+ * LED. -+ * -+ * A BTB is freed when one of the following occurs: -+ * -+ * * The number of LEDs linked to the block device becomes zero, because it has -+ * been unlinked from its last LED using the trigger's &sysfs interface. -+ * -+ * * The number of LEDs linked to the block device becomes zero, because the -+ * last LED to which it was linked has been disassociated from the trigger -+ * (which happens automatically if the LED device is removed from the system). -+ * -+ * * The BTB's block device is removed from the system. To accomodate this -+ * scenario, BTB's are created as device resources, so that the release -+ * function will be called by the driver core when the device is removed. -+ */ -+struct blkdev_trig_bdev { -+ unsigned long last_checked; -+ unsigned long last_activity[NR_STAT_GROUPS]; -+ unsigned long ios[NR_STAT_GROUPS]; -+ unsigned long index; -+ struct block_device *bdev; -+ struct xarray linked_btls; -+}; -+ -+/** -+ * struct blkdev_trig_led - Trigger-specific data about an LED. -+ * @last_checked: Time (in jiffies) at which the trigger last checked the -+ * the block devices linked to this LED for activity. -+ * @index: &xarray index, so the BTL can be included in one or more -+ * &blkdev_trig_bdev.linked_btls. -+ * @mode: Bitmask for types of block device activity that will -+ * cause this LED to blink --- reads, writes, discards, -+ * etc. -+ * @led: The LED device. -+ * @blink_msec: Duration of a blink (milliseconds). -+ * @check_jiffies: Frequency with which block devices linked to this LED -+ * should be checked for activity (jiffies). -+ * @linked_btbs: The BTBs that represent the block devices linked to the -+ * BTL's LED. -+ * @all_btls_node: The BTL's node in the module's list of all BTLs. -+ * -+ * Every LED associated with the block device trigger gets a "BTL." A BTL is -+ * created when the trigger is "activated" on an LED (usually by writing -+ * ``blkdev`` to the LED's &sysfs &trigger attribute). A BTL is freed wnen its -+ * LED is disassociated from the trigger, either through the trigger's &sysfs -+ * interface or because the LED device is removed from the system. -+ */ -+struct blkdev_trig_led { -+ unsigned long last_checked; -+ unsigned long index; -+ unsigned long mode; /* must be ulong for atomic bit ops */ -+ struct led_classdev *led; -+ unsigned int blink_msec; -+ unsigned int check_jiffies; -+ struct xarray linked_btbs; -+ struct hlist_node all_btls_node; -+}; -+ -+/* Protects everything except atomic LED attributes */ -+static DEFINE_MUTEX(blkdev_trig_mutex); -+ -+/* BTB device resource release function */ -+static void blkdev_trig_btb_release(struct device *dev, void *res); -+ -+/* Index for next BTB or BTL */ -+static unsigned long blkdev_trig_next_index; -+ -+/* All LEDs associated with the trigger */ -+static HLIST_HEAD(blkdev_trig_all_btls); -+ -+/* Delayed work to periodically check for activity & blink LEDs */ -+static void blkdev_trig_check(struct work_struct *work); -+static DECLARE_DELAYED_WORK(blkdev_trig_work, blkdev_trig_check); -+ -+/* When is the delayed work scheduled to run next (jiffies) */ -+static unsigned long blkdev_trig_next_check; -+ -+/* Total number of BTB-to-BTL links */ -+static unsigned int blkdev_trig_link_count; -+ -+/* Empty sysfs attribute list for next 2 declarations */ -+static struct attribute *blkdev_trig_attrs_empty[] = { NULL }; -+ -+/* linked_leds sysfs directory for block devs linked to 1 or more LEDs */ -+static const struct attribute_group blkdev_trig_linked_leds = { -+ .name = "linked_leds", -+ .attrs = blkdev_trig_attrs_empty, -+}; -+ -+/* linked_devices sysfs directory for each LED associated with the trigger */ -+static const struct attribute_group blkdev_trig_linked_devs = { -+ .name = "linked_devices", -+ .attrs = blkdev_trig_attrs_empty, -+}; -+ -+ -+/* -+ * -+ * Delayed work to check for activity & blink LEDs -+ * -+ */ -+ -+/** -+ * blkdev_trig_blink() - Blink an LED, if the correct type of activity has -+ * occurred on the block device. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: &true if the LED is blinked, &false if not. -+ */ -+static bool blkdev_trig_blink(const struct blkdev_trig_led *btl, -+ const struct blkdev_trig_bdev *btb) -+{ -+ unsigned long mode, mask, delay_on, delay_off; -+ enum stat_group i; -+ -+ mode = READ_ONCE(btl->mode); -+ -+ for (i = STAT_READ, mask = 1; i <= STAT_FLUSH; ++i, mask <<= 1) { -+ -+ if (!(mode & mask)) -+ continue; -+ -+ if (time_before_eq(btb->last_activity[i], btl->last_checked)) -+ continue; -+ -+ delay_on = READ_ONCE(btl->blink_msec); -+ delay_off = 1; /* 0 leaves LED turned on */ -+ -+ led_blink_set_oneshot(btl->led, &delay_on, &delay_off, 0); -+ return true; -+ } -+ -+ return false; -+} -+ -+/** -+ * blkdev_trig_update_btb() - Update a BTB's activity counters and timestamps. -+ * @btb: The BTB -+ * @now: Timestamp (in jiffies) -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_update_btb(struct blkdev_trig_bdev *btb, -+ unsigned long now) -+{ -+ unsigned long new_ios; -+ enum stat_group i; -+ -+ for (i = STAT_READ; i <= STAT_FLUSH; ++i) { -+ -+ new_ios = part_stat_read(btb->bdev, ios[i]); -+ -+ if (new_ios != btb->ios[i]) { -+ btb->ios[i] = new_ios; -+ btb->last_activity[i] = now; -+ } -+ } -+ -+ btb->last_checked = now; -+} -+ -+/** -+ * blkdev_trig_check() - Check linked devices for activity and blink LEDs. -+ * @work: Delayed work (&blkdev_trig_work) -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_check(struct work_struct *work) -+{ -+ struct blkdev_trig_led *btl; -+ struct blkdev_trig_bdev *btb; -+ unsigned long index, delay, now, led_check, led_delay; -+ bool blinked; -+ -+ if (!mutex_trylock(&blkdev_trig_mutex)) { -+ delay = msecs_to_jiffies(BLKDEV_TRIG_CHECK_RETRY); -+ goto exit_reschedule; -+ } -+ -+ now = jiffies; -+ delay = ULONG_MAX; -+ -+ hlist_for_each_entry (btl, &blkdev_trig_all_btls, all_btls_node) { -+ -+ led_check = btl->last_checked + btl->check_jiffies; -+ -+ if (time_before_eq(led_check, now)) { -+ -+ blinked = false; -+ -+ xa_for_each (&btl->linked_btbs, index, btb) { -+ -+ if (btb->last_checked != now) -+ blkdev_trig_update_btb(btb, now); -+ if (!blinked) -+ blinked = blkdev_trig_blink(btl, btb); -+ } -+ -+ btl->last_checked = now; -+ led_delay = btl->check_jiffies; -+ -+ } else { -+ led_delay = led_check - now; -+ } -+ -+ if (led_delay < delay) -+ delay = led_delay; -+ } -+ -+ mutex_unlock(&blkdev_trig_mutex); -+ -+exit_reschedule: -+ WARN_ON_ONCE(delay == ULONG_MAX); -+ WARN_ON_ONCE(!schedule_delayed_work(&blkdev_trig_work, delay)); -+} -+ -+/** -+ * blkdev_trig_sched_led() - Set the schedule of the delayed work when a new -+ * LED is added to the schedule. -+ * @btl: The BTL that represents the LED -+ * -+ * Called when the number of block devices to which an LED is linked becomes -+ * non-zero. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_sched_led(const struct blkdev_trig_led *btl) -+{ -+ unsigned long delay = READ_ONCE(btl->check_jiffies); -+ unsigned long check_by = jiffies + delay; -+ -+ /* -+ * If no other LED-to-block device links exist, simply schedule the -+ * delayed work according to this LED's check_interval attribute -+ * (check_jiffies). -+ */ -+ if (blkdev_trig_link_count == 0) { -+ WARN_ON(!schedule_delayed_work(&blkdev_trig_work, delay)); -+ blkdev_trig_next_check = check_by; -+ return; -+ } -+ -+ /* -+ * If the next check is already scheduled to occur soon enough to -+ * accomodate this LED's check_interval, the schedule doesn't need -+ * to be changed. -+ */ -+ if (time_after_eq(check_by, blkdev_trig_next_check)) -+ return; -+ -+ /* -+ * Modify the schedule, so that the delayed work runs soon enough for -+ * this LED. -+ */ -+ WARN_ON(!mod_delayed_work(system_wq, &blkdev_trig_work, delay)); -+ blkdev_trig_next_check = check_by; -+} -+ -+ -+/* -+ * -+ * Linking and unlinking LEDs and block devices -+ * -+ */ -+ -+/** -+ * blkdev_trig_link() - Link a block device to an LED. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: &0 on success, negative &errno on error. -+ */ -+static int blkdev_trig_link(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ bool led_first_link; -+ int err; -+ -+ led_first_link = xa_empty(&btl->linked_btbs); -+ -+ err = xa_insert(&btb->linked_btls, btl->index, btl, GFP_KERNEL); -+ if (err) -+ return err; -+ -+ err = xa_insert(&btl->linked_btbs, btb->index, btb, GFP_KERNEL); -+ if (err) -+ goto error_erase_btl; -+ -+ /* Create /sys/class/block//linked_leds/ symlink */ -+ err = sysfs_add_link_to_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ &btl->led->dev->kobj, btl->led->name); -+ if (err) -+ goto error_erase_btb; -+ -+ /* Create /sys/class/leds//linked_devices/ symlink */ -+ err = sysfs_add_link_to_group(&btl->led->dev->kobj, -+ blkdev_trig_linked_devs.name, -+ bdev_kobj(btb->bdev), -+ dev_name(&btb->bdev->bd_device)); -+ if (err) -+ goto error_remove_symlink; -+ -+ /* -+ * If this is the first block device linked to this LED, the delayed -+ * work schedule may need to be changed. -+ */ -+ if (led_first_link) -+ blkdev_trig_sched_led(btl); -+ -+ ++blkdev_trig_link_count; -+ -+ return 0; -+ -+error_remove_symlink: -+ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ btl->led->name); -+error_erase_btb: -+ xa_erase(&btl->linked_btbs, btb->index); -+error_erase_btl: -+ xa_erase(&btb->linked_btls, btl->index); -+ return err; -+} -+ -+/** -+ * blkdev_trig_put_btb() - Remove and free a BTB, if it is no longer needed. -+ * @btb: The BTB -+ * -+ * Does nothing if the BTB (block device) is still linked to at least one LED. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_put_btb(struct blkdev_trig_bdev *btb) -+{ -+ struct block_device *bdev = btb->bdev; -+ int err; -+ -+ if (xa_empty(&btb->linked_btls)) { -+ -+ sysfs_remove_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); -+ err = devres_destroy(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ WARN_ON(err); -+ } -+} -+ -+/** -+ * _blkdev_trig_unlink_always() - Perform the unconditionally required steps of -+ * unlinking a block device from an LED. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * When a block device is unlinked from an LED, certain steps must be performed -+ * only if the block device is **not** being released. This function performs -+ * those steps that are **always** required, whether or not the block device is -+ * being released. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void _blkdev_trig_unlink_always(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ --blkdev_trig_link_count; -+ -+ if (blkdev_trig_link_count == 0) -+ WARN_ON(!cancel_delayed_work_sync(&blkdev_trig_work)); -+ -+ xa_erase(&btb->linked_btls, btl->index); -+ xa_erase(&btl->linked_btbs, btb->index); -+ -+ /* Remove /sys/class/leds//linked_devices/ symlink */ -+ sysfs_remove_link_from_group(&btl->led->dev->kobj, -+ blkdev_trig_linked_devs.name, -+ dev_name(&btb->bdev->bd_device)); -+} -+ -+/** -+ * blkdev_trig_unlink_norelease() - Unlink an LED from a block device that is -+ * **not** being released. -+ * @btl: The BTL that represents the LED. -+ * @btb: The BTB that represents the block device. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_unlink_norelease(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ _blkdev_trig_unlink_always(btl, btb); -+ -+ /* Remove /sys/class/block//linked_leds/ symlink */ -+ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ btl->led->name); -+ -+ blkdev_trig_put_btb(btb); -+} -+ -+/** -+ * blkdev_trig_unlink_release() - Unlink an LED from a block device that is -+ * being released. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_unlink_release(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ _blkdev_trig_unlink_always(btl, btb); -+ -+ /* -+ * If the BTB is being released, the driver core has already removed the -+ * device's attribute groups, and the BTB will be freed automatically, -+ * so there's nothing else to do. -+ */ -+} -+ -+ -+/* -+ * -+ * BTB creation -+ * -+ */ -+ -+/** -+ * blkdev_trig_btb_release() - BTB device resource release function. -+ * @dev: The block device -+ * @res: The BTB -+ * -+ * Called by the driver core when a block device with a BTB is removed. -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_btb_release(struct device *dev, void *res) -+{ -+ struct blkdev_trig_bdev *btb = res; -+ struct blkdev_trig_led *btl; -+ unsigned long index; -+ -+ mutex_lock(&blkdev_trig_mutex); -+ -+ xa_for_each (&btb->linked_btls, index, btl) -+ blkdev_trig_unlink_release(btl, btb); -+ -+ mutex_unlock(&blkdev_trig_mutex); -+} -+ -+/** -+ * blkdev_trig_get_bdev() - Get a block device by path. -+ * @path: The value written to an LED's &link_dev_by_path or -+ * &unlink_dev_by_path attribute, which should be the path to a -+ * special file that represents a block device -+ * @len: The number of characters in &path (not including its -+ * terminating null) -+ * -+ * The caller must call blkdev_put() when finished with the device. -+ * -+ * Context: Process context. -+ * Return: The block device, or an error pointer. -+ */ -+static struct block_device *blkdev_trig_get_bdev(const char *path, size_t len) -+{ -+ struct block_device *bdev; -+ char *buf; -+ -+ buf = kmemdup(path, len + 1, GFP_KERNEL); /* +1 to include null */ -+ if (buf == NULL) -+ return ERR_PTR(-ENOMEM); -+ -+ bdev = blkdev_get_by_path(strim(buf), BLKDEV_TRIG_FMODE, THIS_MODULE); -+ kfree(buf); -+ return bdev; -+} -+ -+/** -+ * blkdev_trig_get_btb() - Find or create the BTB for a block device. -+ * @path: The value written to an LED's &link_dev_by_path attribute, -+ * which should be the path to a special file that represents a -+ * block device -+ * @len: The number of characters in &path -+ * -+ * If a new BTB is created, because the block device was not previously linked -+ * to any LEDs, the block device's &linked_leds &sysfs directory is created. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: Pointer to the BTB, error pointer on error. -+ */ -+static struct blkdev_trig_bdev *blkdev_trig_get_btb(const char *path, -+ size_t len) -+{ -+ struct block_device *bdev; -+ struct blkdev_trig_bdev *btb; -+ int err; -+ -+ bdev = blkdev_trig_get_bdev(path, len); -+ if (IS_ERR(bdev)) -+ return ERR_CAST(bdev); -+ -+ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ if (btb != NULL) { -+ err = 0; -+ goto exit_put_bdev; -+ } -+ -+ if (blkdev_trig_next_index == ULONG_MAX) { -+ err = -EOVERFLOW; -+ goto exit_put_bdev; -+ } -+ -+ btb = devres_alloc(blkdev_trig_btb_release, sizeof(*btb), GFP_KERNEL); -+ if (btb == NULL) { -+ err = -ENOMEM; -+ goto exit_put_bdev; -+ } -+ -+ err = sysfs_create_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); -+ if (err) -+ goto exit_free_btb; -+ -+ btb->index = blkdev_trig_next_index++; -+ btb->bdev = bdev; -+ xa_init(&btb->linked_btls); -+ -+ /* Populate BTB activity counters */ -+ blkdev_trig_update_btb(btb, jiffies); -+ -+ devres_add(&bdev->bd_device, btb); -+ -+exit_free_btb: -+ if (err) -+ devres_free(btb); -+exit_put_bdev: -+ blkdev_put(bdev, BLKDEV_TRIG_FMODE); -+ return err ? ERR_PTR(err) : btb; -+} -+ -+ -+/* -+ * -+ * Activating and deactivating the trigger on an LED -+ * -+ */ -+ -+/** -+ * blkdev_trig_activate() - Called by the LEDs subsystem when an LED is -+ * associated with the trigger. -+ * @led: The LED -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &0 on success, negative &errno on error. -+ */ -+static int blkdev_trig_activate(struct led_classdev *led) -+{ -+ struct blkdev_trig_led *btl; -+ int err; -+ -+ btl = kzalloc(sizeof(*btl), GFP_KERNEL); -+ if (btl == NULL) -+ return -ENOMEM; -+ -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ goto exit_free; -+ -+ if (blkdev_trig_next_index == ULONG_MAX) { -+ err = -EOVERFLOW; -+ goto exit_unlock; -+ } -+ -+ btl->index = blkdev_trig_next_index++; -+ btl->last_checked = jiffies; -+ btl->mode = -1; /* set all bits */ -+ btl->led = led; -+ btl->blink_msec = BLKDEV_TRIG_BLINK_DEF; -+ btl->check_jiffies = msecs_to_jiffies(BLKDEV_TRIG_CHECK_DEF); -+ xa_init(&btl->linked_btbs); -+ -+ hlist_add_head(&btl->all_btls_node, &blkdev_trig_all_btls); -+ led_set_trigger_data(led, btl); -+ -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+exit_free: -+ if (err) -+ kfree(btl); -+ return err; -+} -+ -+/** -+ * blkdev_trig_deactivate() - Called by the the LEDs subsystem when an LED is -+ * disassociated from the trigger. -+ * @led: The LED -+ * -+ * The LEDs subsystem also calls this function when an LED associated with the -+ * trigger is removed or when the trigger is unregistered (if the module is -+ * unloaded). -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_deactivate(struct led_classdev *led) -+{ -+ struct blkdev_trig_led *btl = led_get_trigger_data(led); -+ struct blkdev_trig_bdev *btb; -+ unsigned long index; -+ -+ mutex_lock(&blkdev_trig_mutex); -+ -+ xa_for_each (&btl->linked_btbs, index, btb) -+ blkdev_trig_unlink_norelease(btl, btb); -+ -+ hlist_del(&btl->all_btls_node); -+ kfree(btl); -+ -+ mutex_unlock(&blkdev_trig_mutex); -+} -+ -+ -+/* -+ * -+ * Link-related attribute store functions -+ * -+ */ -+ -+/** -+ * link_dev_by_path_store() - &link_dev_by_path device attribute store function. -+ * @dev: The LED device -+ * @attr: The &link_dev_by_path attribute (&dev_attr_link_dev_by_path) -+ * @buf: The value written to the attribute, which should be the path to -+ * a special file that represents a block device to be linked to -+ * the LED (e.g. ``/dev/sda``) -+ * @count: The number of characters in &buf -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t link_dev_by_path_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ struct blkdev_trig_bdev *btb; -+ int err; -+ -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ return err; -+ -+ btb = blkdev_trig_get_btb(buf, count); -+ if (IS_ERR(btb)) { -+ err = PTR_ERR(btb); -+ goto exit_unlock; -+ } -+ -+ if (xa_load(&btb->linked_btls, btl->index) != NULL) { -+ err = -EEXIST; -+ goto exit_put_btb; -+ } -+ -+ err = blkdev_trig_link(btl, btb); -+ -+exit_put_btb: -+ if (err) -+ blkdev_trig_put_btb(btb); -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+ return err ? : count; -+} -+ -+/** -+ * unlink_dev_by_path_store() - &unlink_dev_by_path device attribute store -+ * function. -+ * @dev: The LED device -+ * @attr: The &unlink_dev_by_path attribute (&dev_attr_unlink_dev_by_path) -+ * @buf: The value written to the attribute, which should be the path to -+ * a special file that represents a block device to be unlinked -+ * from the LED (e.g. ``/dev/sda``) -+ * @count: The number of characters in &buf -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t unlink_dev_by_path_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ struct block_device *bdev; -+ struct blkdev_trig_bdev *btb; -+ int err; -+ -+ bdev = blkdev_trig_get_bdev(buf, count); -+ if (IS_ERR(bdev)) -+ return PTR_ERR(bdev); -+ -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ goto exit_put_bdev; -+ -+ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ if (btb == NULL) { -+ err = -EUNATCH; /* bdev isn't linked to any LED */ -+ goto exit_unlock; -+ } -+ -+ if (xa_load(&btb->linked_btls, btl->index) == NULL) { -+ err = -EUNATCH; /* bdev isn't linked to this LED */ -+ goto exit_unlock; -+ } -+ -+ blkdev_trig_unlink_norelease(btl, btb); -+ -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+exit_put_bdev: -+ blkdev_put(bdev, BLKDEV_TRIG_FMODE); -+ return err ? : count; -+} -+ -+ -+/* -+ * -+ * Atomic attribute show & store functions -+ * -+ */ -+ -+/** -+ * blink_time_show() - &blink_time device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_time attribute (&dev_attr_blink_time) -+ * @buf: Output buffer -+ * -+ * Writes the value of &blkdev_trig_led.blink_msec to &buf. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_time_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ const struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ -+ return sprintf(buf, "%u\n", READ_ONCE(btl->blink_msec)); -+} -+ -+/** -+ * blink_time_store() - &blink_time device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_time attribute (&dev_attr_blink_time) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets &blkdev_trig_led.blink_msec to the value in &buf. -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_time_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ unsigned int value; -+ int err; -+ -+ err = kstrtouint(buf, 0, &value); -+ if (err) -+ return err; -+ -+ if (value < BLKDEV_TRIG_BLINK_MIN || value > BLKDEV_TRIG_BLINK_MAX) -+ return -ERANGE; -+ -+ WRITE_ONCE(btl->blink_msec, value); -+ return count; -+} -+ -+/** -+ * check_interval_show() - &check_interval device attribute show function. -+ * @dev: The LED device -+ * @attr: The &check_interval attribute (&dev_attr_check_interval) -+ * @buf: Output buffer -+ * -+ * Writes the value of &blkdev_trig_led.check_jiffies (converted to -+ * milliseconds) to &buf. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t check_interval_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ -+ return sprintf(buf, "%u\n", -+ jiffies_to_msecs(READ_ONCE(btl->check_jiffies))); -+} -+ -+/** -+ * check_interval_store() - &check_interval device attribute store function -+ * @dev: The LED device -+ * @attr: The &check_interval attribute (&dev_attr_check_interval) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets &blkdev_trig_led.check_jiffies to the value in &buf (after converting -+ * from milliseconds). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t check_interval_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *led = led_trigger_get_drvdata(dev); -+ unsigned int value; -+ int err; -+ -+ err = kstrtouint(buf, 0, &value); -+ if (err) -+ return err; -+ -+ if (value < BLKDEV_TRIG_CHECK_MIN || value > BLKDEV_TRIG_CHECK_MAX) -+ return -ERANGE; -+ -+ WRITE_ONCE(led->check_jiffies, msecs_to_jiffies(value)); -+ -+ return count; -+} -+ -+/** -+ * blkdev_trig_mode_show() - Helper for boolean attribute show functions. -+ * @led: The LED -+ * @buf: Output buffer -+ * @bit: Which bit to show -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static int blkdev_trig_mode_show(const struct blkdev_trig_led *led, char *buf, -+ enum stat_group bit) -+{ -+ return sprintf(buf, READ_ONCE(led->mode) & (1 << bit) ? "Y\n" : "N\n"); -+} -+ -+/** -+ * blkdev_trig_mode_store() - Helper for boolean attribute store functions. -+ * @led: The LED -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * @bit: Which bit to set -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static int blkdev_trig_mode_store(struct blkdev_trig_led *led, -+ const char *buf, size_t count, -+ enum stat_group bit) -+{ -+ bool set; -+ int err; -+ -+ err = kstrtobool(buf, &set); -+ if (err) -+ return err; -+ -+ if (set) -+ set_bit(bit, &led->mode); -+ else -+ clear_bit(bit, &led->mode); -+ -+ return count; -+} -+ -+/** -+ * blink_on_read_show() - &blink_on_read device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_READ bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_read_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_READ); -+} -+ -+/** -+ * blink_on_read_store() - &blink_on_read device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_READ bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_read_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_READ); -+} -+ -+/** -+ * blink_on_write_show() - &blink_on_write device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_WRITE bit in -+ * in &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_write_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_WRITE); -+} -+ -+/** -+ * blink_on_write_store() - &blink_on_write device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_WRITE bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_write_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_WRITE); -+} -+ -+/** -+ * blink_on_flush_show() - &blink_on_flush device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending whether the &STAT_FLUSH bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_flush_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_FLUSH); -+} -+ -+/** -+ * blink_on_flush_store() - &blink_on_flush device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_FLUSH bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_flush_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_FLUSH); -+} -+ -+/** -+ * blink_on_discard_show() - &blink_on_discard device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_DISCARD bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_discard_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_DISCARD); -+} -+ -+/** -+ * blink_on_discard_store() - &blink_on_discard device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_DISCARD bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_discard_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_DISCARD); -+} -+ -+/* Device attributes */ -+static DEVICE_ATTR_WO(link_dev_by_path); -+static DEVICE_ATTR_WO(unlink_dev_by_path); -+static DEVICE_ATTR_RW(blink_time); -+static DEVICE_ATTR_RW(check_interval); -+static DEVICE_ATTR_RW(blink_on_read); -+static DEVICE_ATTR_RW(blink_on_write); -+static DEVICE_ATTR_RW(blink_on_flush); -+static DEVICE_ATTR_RW(blink_on_discard); -+ -+/* Device attributes in LED directory (/sys/class/leds//...) */ -+static struct attribute *blkdev_trig_attrs[] = { -+ &dev_attr_link_dev_by_path.attr, -+ &dev_attr_unlink_dev_by_path.attr, -+ &dev_attr_blink_time.attr, -+ &dev_attr_check_interval.attr, -+ &dev_attr_blink_on_read.attr, -+ &dev_attr_blink_on_write.attr, -+ &dev_attr_blink_on_flush.attr, -+ &dev_attr_blink_on_discard.attr, -+ NULL -+}; -+ -+/* Unnamed attribute group == no subdirectory */ -+static const struct attribute_group blkdev_trig_attr_group = { -+ .attrs = blkdev_trig_attrs, -+}; -+ -+/* Attribute groups for the trigger */ -+static const struct attribute_group *blkdev_trig_attr_groups[] = { -+ &blkdev_trig_attr_group, /* /sys/class/leds//... */ -+ &blkdev_trig_linked_devs, /* /sys/class/leds//linked_devices/ */ -+ NULL -+}; -+ -+/* Trigger registration data */ -+static struct led_trigger blkdev_trig_trigger = { -+ .name = "blkdev", -+ .activate = blkdev_trig_activate, -+ .deactivate = blkdev_trig_deactivate, -+ .groups = blkdev_trig_attr_groups, -+}; -+ -+/** -+ * blkdev_trig_init() - Block device LED trigger initialization. -+ * -+ * Registers the ``blkdev`` LED trigger. -+ * -+ * Return: &0 on success, negative &errno on failure. -+ */ -+static int __init blkdev_trig_init(void) -+{ -+ return led_trigger_register(&blkdev_trig_trigger); -+} -+module_init(blkdev_trig_init); -+ -+/** -+ * blkdev_trig_exit() - Block device LED trigger module exit. -+ * -+ * Unregisters the ``blkdev`` LED trigger. -+ */ -+static void __exit blkdev_trig_exit(void) -+{ -+ led_trigger_unregister(&blkdev_trig_trigger); -+} -+module_exit(blkdev_trig_exit); -+ -+MODULE_DESCRIPTION("Block device LED trigger"); -+MODULE_AUTHOR("Ian Pilcher "); -+MODULE_LICENSE("GPL v2"); -diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h -index 83c7248053a1..d2b8741eabf7 100644 ---- a/include/linux/pageblock-flags.h -+++ b/include/linux/pageblock-flags.h -@@ -48,7 +48,7 @@ extern unsigned int pageblock_order; - #else /* CONFIG_HUGETLB_PAGE */ - - /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ --#define pageblock_order (MAX_ORDER-1) -+#define pageblock_order PAGE_ALLOC_COSTLY_ORDER - - #endif /* CONFIG_HUGETLB_PAGE */ - -diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h -index 99d26879b02a..a92799fc5e74 100644 ---- a/include/net/bluetooth/rfcomm.h -+++ b/include/net/bluetooth/rfcomm.h -@@ -171,6 +171,7 @@ struct rfcomm_dlc { - struct rfcomm_session *session; - struct sk_buff_head tx_queue; - struct timer_list timer; -+ struct work_struct state_change_work; - - struct mutex lock; - unsigned long state; -@@ -186,6 +187,7 @@ struct rfcomm_dlc { - u8 sec_level; - u8 role_switch; - u32 defer_setup; -+ int err; - - uint mtu; - uint cfc; -@@ -310,6 +312,7 @@ struct rfcomm_pinfo { - u8 role_switch; - }; - -+void __rfcomm_sk_state_change(struct work_struct *work); - int rfcomm_init_sockets(void); - void rfcomm_cleanup_sockets(void); - -diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c -index 7324764384b6..c6494e85cd68 100644 ---- a/net/bluetooth/rfcomm/core.c -+++ b/net/bluetooth/rfcomm/core.c -@@ -289,6 +289,7 @@ static void rfcomm_dlc_clear_state(struct rfcomm_dlc *d) - d->flags = 0; - d->mscex = 0; - d->sec_level = BT_SECURITY_LOW; -+ d->err = 0; - d->mtu = RFCOMM_DEFAULT_MTU; - d->v24_sig = RFCOMM_V24_RTC | RFCOMM_V24_RTR | RFCOMM_V24_DV; - -@@ -306,6 +307,7 @@ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio) - timer_setup(&d->timer, rfcomm_dlc_timeout, 0); - - skb_queue_head_init(&d->tx_queue); -+ INIT_WORK(&d->state_change_work, __rfcomm_sk_state_change); - mutex_init(&d->lock); - refcount_set(&d->refcnt, 1); - -diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c -index 4bf4ea6cbb5e..4850dafbaa05 100644 ---- a/net/bluetooth/rfcomm/sock.c -+++ b/net/bluetooth/rfcomm/sock.c -@@ -61,19 +61,22 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) - rfcomm_dlc_throttle(d); - } - --static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) -+void __rfcomm_sk_state_change(struct work_struct *work) - { -+ struct rfcomm_dlc *d = container_of(work, struct rfcomm_dlc, -+ state_change_work); - struct sock *sk = d->owner, *parent; - - if (!sk) - return; - -- BT_DBG("dlc %p state %ld err %d", d, d->state, err); -- - lock_sock(sk); -+ rfcomm_dlc_lock(d); - -- if (err) -- sk->sk_err = err; -+ BT_DBG("dlc %p state %ld err %d", d, d->state, d->err); -+ -+ if (d->err) -+ sk->sk_err = d->err; - - sk->sk_state = d->state; - -@@ -91,15 +94,22 @@ static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) - sk->sk_state_change(sk); - } - -+ rfcomm_dlc_unlock(d); - release_sock(sk); -+ sock_put(sk); -+} - -- if (parent && sock_flag(sk, SOCK_ZAPPED)) { -- /* We have to drop DLC lock here, otherwise -- * rfcomm_sock_destruct() will dead lock. */ -- rfcomm_dlc_unlock(d); -- rfcomm_sock_kill(sk); -- rfcomm_dlc_lock(d); -- } -+static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) -+{ -+ struct sock *sk = d->owner; -+ -+ if (!sk) -+ return; -+ -+ d->err = err; -+ sock_hold(sk); -+ if (!schedule_work(&d->state_change_work)) -+ sock_put(sk); - } - - /* ---- Socket functions ---- */ -diff --git a/tools/objtool/check.c b/tools/objtool/check.c -index e55fdf952a3a..445741f36d6d 100644 ---- a/tools/objtool/check.c -+++ b/tools/objtool/check.c -@@ -2104,9 +2104,6 @@ static int read_noendbr_hints(struct objtool_file *file) - return -1; - } - -- if (insn->type == INSN_ENDBR) -- WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); -- - insn->noendbr = 1; - } - --- -2.38.0.rc1.8.g2a7d63a245 - -From 0d94dd6f072f2cf4c73fba0862b5c877828269d6 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:19:51 +0200 -Subject: [PATCH 12/13] kallsyms - -Signed-off-by: Peter Jung ---- - include/linux/kallsyms.h | 12 +- - include/linux/module.h | 4 +- - init/Kconfig | 13 ++ - kernel/Makefile | 1 + - kernel/kallsyms.c | 195 +++++++++++++++-- - kernel/kallsyms_internal.h | 1 + - kernel/kallsyms_selftest.c | 421 +++++++++++++++++++++++++++++++++++++ - kernel/livepatch/core.c | 31 ++- - kernel/module/kallsyms.c | 15 +- - kernel/trace/ftrace.c | 3 +- - scripts/kallsyms.c | 88 +++++--- - 11 files changed, 717 insertions(+), 67 deletions(-) - create mode 100644 kernel/kallsyms_selftest.c - -diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h -index ad39636e0c3f..c7219d74e290 100644 ---- a/include/linux/kallsyms.h -+++ b/include/linux/kallsyms.h -@@ -66,9 +66,11 @@ static inline void *dereference_symbol_descriptor(void *ptr) - } - - #ifdef CONFIG_KALLSYMS --int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, -- unsigned long), -+unsigned long kallsyms_sym_address(int idx); -+int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), - void *data); -+int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data); - - /* Lookup the address for a symbol. Returns 0 if not found. */ - unsigned long kallsyms_lookup_name(const char *name); -@@ -168,6 +170,12 @@ static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct - { - return -EOPNOTSUPP; - } -+ -+static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data) -+{ -+ return -EOPNOTSUPP; -+} - #endif /*CONFIG_KALLSYMS*/ - - static inline void print_ip_sym(const char *loglvl, unsigned long ip) -diff --git a/include/linux/module.h b/include/linux/module.h -index 518296ea7f73..6e1a531d78e7 100644 ---- a/include/linux/module.h -+++ b/include/linux/module.h -@@ -879,8 +879,8 @@ static inline bool module_sig_ok(struct module *module) - } - #endif /* CONFIG_MODULE_SIG */ - --int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, -- struct module *, unsigned long), -+int module_kallsyms_on_each_symbol(const char *modname, -+ int (*fn)(void *, const char *, unsigned long), - void *data); - - #endif /* _LINUX_MODULE_H */ -diff --git a/init/Kconfig b/init/Kconfig -index 442a945ca6ae..b3a9ec8aa753 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1742,6 +1742,19 @@ config KALLSYMS - symbolic stack backtraces. This increases the size of the kernel - somewhat, as all symbols have to be loaded into the kernel image. - -+config KALLSYMS_SELFTEST -+ bool "Test the basic functions and performance of kallsyms" -+ depends on KALLSYMS -+ default n -+ help -+ Test the basic functions and performance of some interfaces, such as -+ kallsyms_lookup_name. It also calculates the compression rate of the -+ kallsyms compression algorithm for the current symbol set. -+ -+ Start self-test automatically after system startup. Suggest executing -+ "dmesg | grep kallsyms_selftest" to collect test results. "finish" is -+ displayed in the last line, indicating that the test is complete. -+ - config KALLSYMS_ALL - bool "Include all symbols in kallsyms" - depends on DEBUG_KERNEL && KALLSYMS -diff --git a/kernel/Makefile b/kernel/Makefile -index 318789c728d3..122a5fed457b 100644 ---- a/kernel/Makefile -+++ b/kernel/Makefile -@@ -68,6 +68,7 @@ endif - obj-$(CONFIG_UID16) += uid16.o - obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o - obj-$(CONFIG_KALLSYMS) += kallsyms.o -+obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o - obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o - obj-$(CONFIG_CRASH_CORE) += crash_core.o - obj-$(CONFIG_KEXEC_CORE) += kexec_core.o -diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c -index 3e7e2c2ad2f7..1512db69aa0a 100644 ---- a/kernel/kallsyms.c -+++ b/kernel/kallsyms.c -@@ -87,6 +87,86 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, - return off; - } - -+static unsigned char *find_token(unsigned char *str, int len, -+ const unsigned char *token) -+{ -+ int i; -+ -+ for (i = 0; i < len - 1; i++) { -+ if (str[i] == token[0] && str[i+1] == token[1]) -+ return &str[i]; -+ } -+ return NULL; -+} -+ -+static int kallsyms_compress_symbol_name(const char *name, char *buf, size_t size) -+{ -+ int i, j, n, len; -+ unsigned char *p1, *p2; -+ const unsigned char *token; -+ -+ len = strscpy(buf, name, size); -+ if (WARN_ON_ONCE(len <= 0)) -+ return 0; -+ -+ /* -+ * For each entry in kallsyms_best_token_table[], the storage -+ * format is: -+ * 1. For tokens that cannot be used to compress characters, the value -+ * at [j] is 0, and the value at [j+1] is the number of consecutive -+ * tokens with this feature. -+ * 2. For each token: the larger the token value, the higher the -+ * frequency, and the lower the index. -+ * -+ * ------------------------------- -+ * | j | [j] [j+1] | token | -+ * -----|---------------|---------| -+ * | 0 | ?? ?? | 255 | -+ * | 2 | ?? ?? | 254 | -+ * | ... | ?? ?? | ... | -+ * | n-2 | ?? ?? | x | -+ * | n | 00 len | x-1 | -+ * | n+2 | ?? ?? | x-1-len | -+ * above '??' is non-zero -+ */ -+ for (i = 255, j = 0; i >= 0; i--, j += 2) { -+ if (!kallsyms_best_token_table[j]) { -+ i -= kallsyms_best_token_table[j + 1]; -+ if (i < 0) -+ break; -+ j += 2; -+ } -+ token = &kallsyms_best_token_table[j]; -+ -+ p1 = buf; -+ -+ /* find the token on the symbol */ -+ p2 = find_token(p1, len, token); -+ if (!p2) -+ continue; -+ -+ n = len; -+ -+ do { -+ *p2 = i; -+ p2++; -+ n -= (p2 - p1); -+ memmove(p2, p2 + 1, n); -+ p1 = p2; -+ len--; -+ -+ if (n < 2) -+ break; -+ -+ /* find the token on the symbol */ -+ p2 = find_token(p1, n, token); -+ -+ } while (p2); -+ } -+ -+ return len; -+} -+ - /* - * Get symbol type information. This is encoded as a single char at the - * beginning of the symbol name. -@@ -128,7 +208,7 @@ static unsigned int get_symbol_offset(unsigned long pos) - return name - kallsyms_names; - } - --static unsigned long kallsyms_sym_address(int idx) -+unsigned long kallsyms_sym_address(int idx) - { - if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE)) - return kallsyms_addresses[idx]; -@@ -186,26 +266,95 @@ static bool cleanup_symbol_name(char *s) - return false; - } - -+static int kallsyms_lookup_clang_name(unsigned char *namebuf, int len, -+ const char *name, -+ unsigned long *addr) -+{ -+ unsigned long i; -+ unsigned int off; -+ -+ if (!IS_ENABLED(CONFIG_LTO_CLANG)) -+ return -ENOENT; -+ -+ for (i = 0, off = 0; i < kallsyms_num_syms; i++) { -+ off = kallsyms_expand_symbol(off, namebuf, len); -+ -+ if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) { -+ *addr = kallsyms_sym_address(i); -+ return 0; -+ } -+ } -+ -+ return -ENOENT; -+} -+ -+static int __kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, -+ unsigned int *index, -+ unsigned int *offset, -+ unsigned long *addr) -+{ -+ unsigned int i = *index; -+ unsigned int off = *offset; -+ unsigned int name_len; -+ const unsigned char *name; -+ -+ for (; len && i < kallsyms_num_syms; i++) { -+ /* -+ * For each entry in kallsyms_names[], the storage format is: -+ * ---------------------------- -+ * | len(1) | type(1) | name(x) | -+ * ---------------------------- -+ * -+ * Number of bytes in parentheses, and: len = 1 + x -+ */ -+ name_len = kallsyms_names[off] - 1; -+ name = &kallsyms_names[off + 2]; -+ off += name_len + 2; -+ -+ if (name_len != len) -+ continue; -+ -+ if (!memcmp(name, namebuf, len)) { -+ /* Prepare for the next iteration */ -+ *index = i + 1; -+ *offset = off; -+ -+ *addr = kallsyms_sym_address(i); -+ return 0; -+ } -+ } -+ -+ return -ENOENT; -+} -+ -+static int kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, -+ unsigned long *addr) -+{ -+ unsigned int i = 0, off = 0; -+ -+ return __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, addr); -+} -+ - /* Lookup the address for this symbol. Returns 0 if not found. */ - unsigned long kallsyms_lookup_name(const char *name) - { - char namebuf[KSYM_NAME_LEN]; -- unsigned long i; -- unsigned int off; -+ unsigned long addr; -+ int ret, len; - - /* Skip the search for empty string. */ - if (!*name) - return 0; - -- for (i = 0, off = 0; i < kallsyms_num_syms; i++) { -- off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); -+ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); -+ ret = kallsyms_lookup_compressed_name(namebuf, len, &addr); -+ if (!ret) -+ return addr; - -- if (strcmp(namebuf, name) == 0) -- return kallsyms_sym_address(i); -+ ret = kallsyms_lookup_clang_name(namebuf, len, name, &addr); -+ if (!ret) -+ return addr; - -- if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) -- return kallsyms_sym_address(i); -- } - return module_kallsyms_lookup_name(name); - } - -@@ -213,8 +362,7 @@ unsigned long kallsyms_lookup_name(const char *name) - * Iterate over all symbols in vmlinux. For symbols from modules use - * module_kallsyms_on_each_symbol instead. - */ --int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, -- unsigned long), -+int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), - void *data) - { - char namebuf[KSYM_NAME_LEN]; -@@ -224,7 +372,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - - for (i = 0, off = 0; i < kallsyms_num_syms; i++) { - off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); -- ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); -+ ret = fn(data, namebuf, kallsyms_sym_address(i)); - if (ret != 0) - return ret; - cond_resched(); -@@ -232,6 +380,27 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - return 0; - } - -+int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data) -+{ -+ int ret, len; -+ unsigned long addr; -+ unsigned int i = 0, off = 0; -+ char namebuf[KSYM_NAME_LEN]; -+ -+ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); -+ do { -+ ret = __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, &addr); -+ if (ret) -+ return 0; /* end of lookup */ -+ -+ ret = fn(data, addr); -+ cond_resched(); -+ } while (!ret); -+ -+ return ret; -+} -+ - static unsigned long get_symbol_pos(unsigned long addr, - unsigned long *symbolsize, - unsigned long *offset) -diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h -index 2d0c6f2f0243..d9672ede8cfc 100644 ---- a/kernel/kallsyms_internal.h -+++ b/kernel/kallsyms_internal.h -@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; - extern const u16 kallsyms_token_index[] __weak; - - extern const unsigned int kallsyms_markers[] __weak; -+extern const unsigned char kallsyms_best_token_table[] __weak; - - #endif // LINUX_KALLSYMS_INTERNAL_H_ -diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c -new file mode 100644 -index 000000000000..f7538a70d36c ---- /dev/null -+++ b/kernel/kallsyms_selftest.c -@@ -0,0 +1,421 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Test the function and performance of kallsyms -+ * -+ * Copyright (C) Huawei Technologies Co., Ltd., 2022 -+ * -+ * Authors: Zhen Lei Huawei -+ */ -+ -+#define pr_fmt(fmt) "kallsyms_selftest: " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "kallsyms_internal.h" -+ -+ -+#define MAX_NUM_OF_RECORDS 64 -+ -+struct test_stat { -+ int min; -+ int max; -+ int save_cnt; -+ int real_cnt; -+ u64 sum; -+ char *name; -+ unsigned long addr; -+ unsigned long addrs[MAX_NUM_OF_RECORDS]; -+}; -+ -+struct test_item { -+ char *name; -+ unsigned long addr; -+}; -+ -+#define ITEM_FUNC(s) \ -+ { \ -+ .name = #s, \ -+ .addr = (unsigned long)s, \ -+ } -+ -+#define ITEM_DATA(s) \ -+ { \ -+ .name = #s, \ -+ .addr = (unsigned long)&s, \ -+ } -+ -+static int test_var_bss_static; -+static int test_var_data_static = 1; -+int test_var_bss; -+int test_var_data = 1; -+ -+static int test_func_static(void) -+{ -+ test_var_bss_static++; -+ test_var_data_static++; -+ -+ return 0; -+} -+ -+int test_func(void) -+{ -+ return test_func_static(); -+} -+ -+__weak int test_func_weak(void) -+{ -+ test_var_bss++; -+ test_var_data++; -+ return 0; -+} -+ -+static struct test_item test_items[] = { -+ ITEM_FUNC(test_func_static), -+ ITEM_FUNC(test_func), -+ ITEM_FUNC(test_func_weak), -+ ITEM_FUNC(vmalloc), -+ ITEM_FUNC(vfree), -+#ifdef CONFIG_KALLSYMS_ALL -+ ITEM_DATA(test_var_bss_static), -+ ITEM_DATA(test_var_data_static), -+ ITEM_DATA(test_var_bss), -+ ITEM_DATA(test_var_data), -+ ITEM_DATA(vmap_area_list), -+#endif -+}; -+ -+static char stub_name[KSYM_NAME_LEN]; -+ -+static int stat_symbol_len(void *data, const char *name, unsigned long addr) -+{ -+ *(u32 *)data += strlen(name); -+ -+ return 0; -+} -+ -+static void test_kallsyms_compression_ratio(void) -+{ -+ int i; -+ const u8 *name; -+ u32 pos; -+ u32 ratio, total_size, total_len = 0; -+ -+ kallsyms_on_each_symbol(stat_symbol_len, &total_len); -+ -+ /* -+ * A symbol name cannot start with a number. This stub name helps us -+ * traverse the entire symbol table without finding a match. It's used -+ * for subsequent performance tests, and its length is the average -+ * length of all symbol names. -+ */ -+ memset(stub_name, '4', sizeof(stub_name)); -+ pos = total_len / kallsyms_num_syms; -+ stub_name[pos] = 0; -+ -+ pos = kallsyms_num_syms - 1; -+ name = &kallsyms_names[kallsyms_markers[pos >> 8]]; -+ for (i = 0; i <= (pos & 0xff); i++) -+ name = name + (*name) + 1; -+ -+ /* -+ * 1. The length fields is not counted -+ * 2. The memory occupied by array kallsyms_token_table[] and -+ * kallsyms_token_index[] needs to be counted. -+ */ -+ total_size = (name - kallsyms_names) - kallsyms_num_syms; -+ pos = kallsyms_token_index[0xff]; -+ total_size += pos + strlen(&kallsyms_token_table[pos]) + 1; -+ total_size += 0x100 * sizeof(u16); -+ -+ pr_info(" ---------------------------------------------------------\n"); -+ pr_info("| nr_symbols | compressed size | original size | ratio(%%) |\n"); -+ pr_info("|---------------------------------------------------------|\n"); -+ ratio = 10000ULL * total_size / total_len; -+ pr_info("| %10d | %10d | %10d | %2d.%-2d |\n", -+ kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100); -+ pr_info(" ---------------------------------------------------------\n"); -+} -+ -+static int lookup_name(void *data, const char *name, unsigned long addr) -+{ -+ u64 t0, t1, t; -+ unsigned long flags; -+ struct test_stat *stat = (struct test_stat *)data; -+ -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ (void)kallsyms_lookup_name(name); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ -+ t = t1 - t0; -+ if (t < stat->min) -+ stat->min = t; -+ -+ if (t > stat->max) -+ stat->max = t; -+ -+ stat->real_cnt++; -+ stat->sum += t; -+ -+ return 0; -+} -+ -+static void test_perf_kallsyms_lookup_name(void) -+{ -+ struct test_stat stat; -+ -+ memset(&stat, 0, sizeof(stat)); -+ stat.min = INT_MAX; -+ kallsyms_on_each_symbol(lookup_name, &stat); -+ pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt); -+ pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n", -+ stat.min, stat.max, stat.sum / stat.real_cnt); -+} -+ -+static int find_symbol(void *data, const char *name, unsigned long addr) -+{ -+ struct test_stat *stat = (struct test_stat *)data; -+ -+ if (strcmp(name, stat->name) == 0) { -+ stat->real_cnt++; -+ stat->addr = addr; -+ -+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { -+ stat->addrs[stat->save_cnt] = addr; -+ stat->save_cnt++; -+ } -+ -+ if (stat->real_cnt == stat->max) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void test_perf_kallsyms_on_each_symbol(void) -+{ -+ u64 t0, t1; -+ unsigned long flags; -+ struct test_stat stat; -+ -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = stub_name; -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ kallsyms_on_each_symbol(find_symbol, &stat); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0); -+} -+ -+static int match_symbol(void *data, unsigned long addr) -+{ -+ struct test_stat *stat = (struct test_stat *)data; -+ -+ stat->real_cnt++; -+ stat->addr = addr; -+ -+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { -+ stat->addrs[stat->save_cnt] = addr; -+ stat->save_cnt++; -+ } -+ -+ if (stat->real_cnt == stat->max) -+ return 1; -+ -+ return 0; -+} -+ -+static void test_perf_kallsyms_on_each_match_symbol(void) -+{ -+ u64 t0, t1; -+ unsigned long flags; -+ struct test_stat stat; -+ -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = stub_name; -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0); -+} -+ -+static int test_kallsyms_basic_function(void) -+{ -+ int i, j, ret; -+ int next = 0, nr_failed = 0; -+ char *prefix; -+ unsigned short rand; -+ unsigned long addr; -+ char namebuf[KSYM_NAME_LEN]; -+ struct test_stat stat, stat1, stat2; -+ -+ prefix = "kallsyms_lookup_name() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ addr = kallsyms_lookup_name(test_items[i].name); -+ if (addr != test_items[i].addr) { -+ nr_failed++; -+ pr_info("%s %s failed: addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, addr, test_items[i].addr); -+ } -+ } -+ -+ prefix = "kallsyms_on_each_symbol() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = test_items[i].name; -+ kallsyms_on_each_symbol(find_symbol, &stat); -+ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { -+ nr_failed++; -+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, -+ stat.real_cnt, stat.addr, test_items[i].addr); -+ } -+ } -+ -+ prefix = "kallsyms_on_each_match_symbol() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = test_items[i].name; -+ kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, &stat); -+ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { -+ nr_failed++; -+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, -+ stat.real_cnt, stat.addr, test_items[i].addr); -+ } -+ } -+ -+ if (nr_failed) -+ return -ESRCH; -+ -+ for (i = 0; i < kallsyms_num_syms; i++) { -+ addr = kallsyms_sym_address(i); -+ if (!is_ksym_addr(addr)) -+ continue; -+ -+ ret = lookup_symbol_name(addr, namebuf); -+ if (unlikely(ret)) { -+ namebuf[0] = 0; -+ goto failed; -+ } -+ -+ stat.addr = kallsyms_lookup_name(namebuf); -+ -+ memset(&stat1, 0, sizeof(stat1)); -+ stat1.max = INT_MAX; -+ kallsyms_on_each_match_symbol(match_symbol, namebuf, &stat1); -+ -+ /* -+ * kallsyms_on_each_symbol() is too slow, randomly select some -+ * symbols for test. -+ */ -+ if (i >= next) { -+ memset(&stat2, 0, sizeof(stat2)); -+ stat2.max = INT_MAX; -+ stat2.name = namebuf; -+ kallsyms_on_each_symbol(find_symbol, &stat2); -+ -+ /* -+ * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol() -+ * need to get the same traversal result. -+ */ -+ if (stat1.addr != stat2.addr || -+ stat1.real_cnt != stat2.real_cnt || -+ memcmp(stat1.addrs, stat2.addrs, -+ stat1.save_cnt * sizeof(stat1.addrs[0]))) -+ goto failed; -+ -+ /* -+ * The average of random increments is 128, that is, one of -+ * them is tested every 128 symbols. -+ */ -+ get_random_bytes(&rand, sizeof(rand)); -+ next = i + (rand & 0xff) + 1; -+ } -+ -+ /* Need to be found at least once */ -+ if (!stat1.real_cnt) -+ goto failed; -+ -+ /* -+ * kallsyms_lookup_name() returns the address of the first -+ * symbol found and cannot be NULL. -+ */ -+ if (!stat.addr || stat.addr != stat1.addrs[0]) -+ goto failed; -+ -+ /* -+ * If the addresses of all matching symbols are recorded, the -+ * target address needs to be exist. -+ */ -+ if (stat1.real_cnt <= MAX_NUM_OF_RECORDS) { -+ for (j = 0; j < stat1.save_cnt; j++) { -+ if (stat1.addrs[j] == addr) -+ break; -+ } -+ -+ if (j == stat1.save_cnt) -+ goto failed; -+ } -+ } -+ -+ return 0; -+ -+failed: -+ pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr); -+ return -ESRCH; -+} -+ -+static int test_entry(void *p) -+{ -+ int ret; -+ -+ do { -+ schedule_timeout(5 * HZ); -+ } while (system_state != SYSTEM_RUNNING); -+ -+ pr_info("start\n"); -+ ret = test_kallsyms_basic_function(); -+ if (ret) { -+ pr_info("abort\n"); -+ return 0; -+ } -+ -+ test_kallsyms_compression_ratio(); -+ test_perf_kallsyms_lookup_name(); -+ test_perf_kallsyms_on_each_symbol(); -+ test_perf_kallsyms_on_each_match_symbol(); -+ pr_info("finish\n"); -+ -+ return 0; -+} -+ -+static int __init kallsyms_test_init(void) -+{ -+ struct task_struct *t; -+ -+ t = kthread_create(test_entry, NULL, "kallsyms_test"); -+ if (IS_ERR(t)) { -+ pr_info("Create kallsyms selftest task failed\n"); -+ return PTR_ERR(t); -+ } -+ kthread_bind(t, 0); -+ wake_up_process(t); -+ -+ return 0; -+} -+late_initcall(kallsyms_test_init); -diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c -index bc475e62279d..074a91e2cfd5 100644 ---- a/kernel/livepatch/core.c -+++ b/kernel/livepatch/core.c -@@ -118,26 +118,36 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, - } - - struct klp_find_arg { -- const char *objname; - const char *name; - unsigned long addr; - unsigned long count; - unsigned long pos; - }; - --static int klp_find_callback(void *data, const char *name, -- struct module *mod, unsigned long addr) -+static int klp_find_callback(void *data, const char *name, unsigned long addr) - { - struct klp_find_arg *args = data; - -- if ((mod && !args->objname) || (!mod && args->objname)) -- return 0; -- - if (strcmp(args->name, name)) - return 0; - -- if (args->objname && strcmp(args->objname, mod->name)) -- return 0; -+ args->addr = addr; -+ args->count++; -+ -+ /* -+ * Finish the search when the symbol is found for the desired position -+ * or the position is not defined for a non-unique symbol. -+ */ -+ if ((args->pos && (args->count == args->pos)) || -+ (!args->pos && (args->count > 1))) -+ return 1; -+ -+ return 0; -+} -+ -+static int klp_match_callback(void *data, unsigned long addr) -+{ -+ struct klp_find_arg *args = data; - - args->addr = addr; - args->count++; -@@ -157,7 +167,6 @@ static int klp_find_object_symbol(const char *objname, const char *name, - unsigned long sympos, unsigned long *addr) - { - struct klp_find_arg args = { -- .objname = objname, - .name = name, - .addr = 0, - .count = 0, -@@ -165,9 +174,9 @@ static int klp_find_object_symbol(const char *objname, const char *name, - }; - - if (objname) -- module_kallsyms_on_each_symbol(klp_find_callback, &args); -+ module_kallsyms_on_each_symbol(objname, klp_find_callback, &args); - else -- kallsyms_on_each_symbol(klp_find_callback, &args); -+ kallsyms_on_each_match_symbol(klp_match_callback, name, &args); - - /* - * Ensure an address was found. If sympos is 0, ensure symbol is unique; -diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c -index f5c5c9175333..329cef573675 100644 ---- a/kernel/module/kallsyms.c -+++ b/kernel/module/kallsyms.c -@@ -495,8 +495,8 @@ unsigned long module_kallsyms_lookup_name(const char *name) - } - - #ifdef CONFIG_LIVEPATCH --int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, -- struct module *, unsigned long), -+int module_kallsyms_on_each_symbol(const char *modname, -+ int (*fn)(void *, const char *, unsigned long), - void *data) - { - struct module *mod; -@@ -510,6 +510,9 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, - if (mod->state == MODULE_STATE_UNFORMED) - continue; - -+ if (strcmp(modname, mod->name)) -+ continue; -+ - /* Use rcu_dereference_sched() to remain compliant with the sparse tool */ - preempt_disable(); - kallsyms = rcu_dereference_sched(mod->kallsyms); -@@ -522,10 +525,16 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, - continue; - - ret = fn(data, kallsyms_symbol_name(kallsyms, i), -- mod, kallsyms_symbol_value(sym)); -+ kallsyms_symbol_value(sym)); - if (ret != 0) - goto out; - } -+ -+ /* -+ * The given module is found, the subsequent modules do not -+ * need to be compared. -+ */ -+ break; - } - out: - mutex_unlock(&module_mutex); -diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c -index 439e2ab6905e..f135a0a334a3 100644 ---- a/kernel/trace/ftrace.c -+++ b/kernel/trace/ftrace.c -@@ -8250,8 +8250,7 @@ struct kallsyms_data { - size_t found; - }; - --static int kallsyms_callback(void *data, const char *name, -- struct module *mod, unsigned long addr) -+static int kallsyms_callback(void *data, const char *name, unsigned long addr) - { - struct kallsyms_data *args = data; - const char **sym; -diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c -index f18e6dfc68c5..40a6fe6d14ef 100644 ---- a/scripts/kallsyms.c -+++ b/scripts/kallsyms.c -@@ -34,7 +34,8 @@ struct sym_entry { - unsigned int len; - unsigned int start_pos; - unsigned int percpu_absolute; -- unsigned char sym[]; -+ unsigned char type; -+ unsigned char name[]; - }; - - struct addr_range { -@@ -75,11 +76,6 @@ static void usage(void) - exit(1); - } - --static char *sym_name(const struct sym_entry *s) --{ -- return (char *)s->sym + 1; --} -- - static bool is_ignored_symbol(const char *name, char type) - { - /* Symbol names that exactly match to the following are ignored.*/ -@@ -227,11 +223,7 @@ static struct sym_entry *read_symbol(FILE *in) - check_symbol_range(name, addr, text_ranges, ARRAY_SIZE(text_ranges)); - check_symbol_range(name, addr, &percpu_range, 1); - -- /* include the type field in the symbol name, so that it gets -- * compressed together */ -- -- len = strlen(name) + 1; -- -+ len = strlen(name); - sym = malloc(sizeof(*sym) + len + 1); - if (!sym) { - fprintf(stderr, "kallsyms failure: " -@@ -240,8 +232,8 @@ static struct sym_entry *read_symbol(FILE *in) - } - sym->addr = addr; - sym->len = len; -- sym->sym[0] = type; -- strcpy(sym_name(sym), name); -+ sym->type = type; -+ strcpy((char *)sym->name, name); - sym->percpu_absolute = 0; - - return sym; -@@ -265,7 +257,7 @@ static int symbol_in_range(const struct sym_entry *s, - - static int symbol_valid(const struct sym_entry *s) - { -- const char *name = sym_name(s); -+ const char *name = (char *)s->name; - - /* if --all-symbols is not specified, then symbols outside the text - * and inittext sections are discarded */ -@@ -471,12 +463,18 @@ static void write_src(void) - if ((i & 0xFF) == 0) - markers[i >> 8] = off; - -- printf("\t.byte 0x%02x", table[i]->len); -+ /* -+ * Store the symbol type togerher with symbol name. -+ * It helps to reduce the size. -+ */ -+ printf("\t.byte 0x%02x", table[i]->len + 1); -+ printf(", 0x%02x", table[i]->type); - for (k = 0; k < table[i]->len; k++) -- printf(", 0x%02x", table[i]->sym[k]); -+ printf(", 0x%02x", table[i]->name[k]); - printf("\n"); - -- off += table[i]->len + 1; -+ /* fields 'len' and 'type' occupy one byte each */ -+ off += table[i]->len + 1 + 1; - } - printf("\n"); - -@@ -501,6 +499,24 @@ static void write_src(void) - for (i = 0; i < 256; i++) - printf("\t.short\t%d\n", best_idx[i]); - printf("\n"); -+ -+ output_label("kallsyms_best_token_table"); -+ for (i = 255, k = 0; (int)i >= 0; i--) { -+ if (best_table_len[i] <= 1) { -+ k++; -+ continue; -+ } -+ -+ if (k) { -+ printf("\t.byte 0x00, 0x%02x\n", k); -+ k = 0; -+ } -+ -+ printf("\t.byte 0x%02x, 0x%02x\n", best_table[i][0], best_table[i][1]); -+ } -+ if (k) -+ printf("\t.byte 0x00, 0x%02x\n", k); -+ printf("\n"); - } - - -@@ -525,12 +541,12 @@ static void forget_symbol(const unsigned char *symbol, int len) - } - - /* do the initial token count */ --static void build_initial_tok_table(void) -+static void build_initial_token_table(void) - { - unsigned int i; - - for (i = 0; i < table_cnt; i++) -- learn_symbol(table[i]->sym, table[i]->len); -+ learn_symbol(table[i]->name, table[i]->len); - } - - static unsigned char *find_token(unsigned char *str, int len, -@@ -555,14 +571,14 @@ static void compress_symbols(const unsigned char *str, int idx) - for (i = 0; i < table_cnt; i++) { - - len = table[i]->len; -- p1 = table[i]->sym; -+ p1 = table[i]->name; - - /* find the token on the symbol */ - p2 = find_token(p1, len, str); - if (!p2) continue; - - /* decrease the counts for this symbol's tokens */ -- forget_symbol(table[i]->sym, len); -+ forget_symbol(table[i]->name, len); - - size = len; - -@@ -584,7 +600,7 @@ static void compress_symbols(const unsigned char *str, int idx) - table[i]->len = len; - - /* increase the counts for this symbol's new tokens */ -- learn_symbol(table[i]->sym, len); -+ learn_symbol(table[i]->name, len); - } - } - -@@ -637,20 +653,24 @@ static void optimize_result(void) - /* start by placing the symbols that are actually used on the table */ - static void insert_real_symbols_in_table(void) - { -- unsigned int i, j, c; -+ unsigned int i, j; -+ unsigned char c; - - for (i = 0; i < table_cnt; i++) { - for (j = 0; j < table[i]->len; j++) { -- c = table[i]->sym[j]; -- best_table[c][0]=c; -- best_table_len[c]=1; -+ c = table[i]->name[j]; -+ best_table[c][0] = c; -+ best_table_len[c] = 1; - } -+ c = table[i]->type; -+ best_table[c][0] = c; -+ best_table_len[c] = 1; - } - } - - static void optimize_token_table(void) - { -- build_initial_tok_table(); -+ build_initial_token_table(); - - insert_real_symbols_in_table(); - -@@ -660,8 +680,8 @@ static void optimize_token_table(void) - /* guess for "linker script provide" symbol */ - static int may_be_linker_script_provide_symbol(const struct sym_entry *se) - { -- const char *symbol = sym_name(se); -- int len = se->len - 1; -+ const char *symbol = (char *)se->name; -+ int len = se->len; - - if (len < 8) - return 0; -@@ -705,8 +725,8 @@ static int compare_symbols(const void *a, const void *b) - return -1; - - /* sort by "weakness" type */ -- wa = (sa->sym[0] == 'w') || (sa->sym[0] == 'W'); -- wb = (sb->sym[0] == 'w') || (sb->sym[0] == 'W'); -+ wa = (sa->type == 'w') || (sa->type == 'W'); -+ wb = (sb->type == 'w') || (sb->type == 'W'); - if (wa != wb) - return wa - wb; - -@@ -717,8 +737,8 @@ static int compare_symbols(const void *a, const void *b) - return wa - wb; - - /* sort by the number of prefix underscores */ -- wa = strspn(sym_name(sa), "_"); -- wb = strspn(sym_name(sb), "_"); -+ wa = strspn((char *)sa->name, "_"); -+ wb = strspn((char *)sb->name, "_"); - if (wa != wb) - return wa - wb; - -@@ -742,7 +762,7 @@ static void make_percpus_absolute(void) - * ensure consistent behavior compared to older - * versions of this tool. - */ -- table[i]->sym[0] = 'A'; -+ table[i]->type = 'A'; - table[i]->percpu_absolute = 1; - } - } --- -2.38.0.rc1.8.g2a7d63a245 - -From 1c53b3346447afadb2f1fbc5c94c11ef14500a15 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Tue, 27 Sep 2022 18:01:48 +0200 -Subject: [PATCH 13/13] bitmap - -Signed-off-by: Peter Jung ---- - arch/loongarch/kernel/setup.c | 2 +- - arch/mips/kernel/setup.c | 2 +- - arch/powerpc/kernel/head_64.S | 4 + - arch/x86/kernel/smpboot.c | 4 +- - arch/x86/xen/smp_pv.c | 2 +- - fs/ntfs3/bitmap.c | 4 +- - include/linux/bitmap.h | 13 +- - include/linux/bitops.h | 19 +++ - include/linux/cpumask.h | 114 +++++++++---- - include/linux/find.h | 272 ++++++++++++++++++++++++++----- - include/linux/netdevice.h | 10 +- - include/linux/nodemask.h | 3 +- - kernel/smp.c | 6 +- - lib/Kconfig | 9 ++ - lib/bitmap.c | 68 +++----- - lib/cpumask.c | 40 ++--- - lib/find_bit.c | 224 +++++++++++++++++--------- - lib/find_bit_benchmark.c | 18 +++ - lib/test_bitmap.c | 291 +++++++++++++++++++++++++++++++++- - tools/include/linux/find.h | 61 ++----- - tools/lib/find_bit.c | 149 ++++++++--------- - 21 files changed, 950 insertions(+), 365 deletions(-) - -diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c -index 8f5c2f9a1a83..18a81edd3ac5 100644 ---- a/arch/loongarch/kernel/setup.c -+++ b/arch/loongarch/kernel/setup.c -@@ -346,7 +346,7 @@ static void __init prefill_possible_map(void) - for (; i < NR_CPUS; i++) - set_cpu_possible(i, false); - -- nr_cpu_ids = possible; -+ set_nr_cpu_ids(possible); - } - #endif - -diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c -index 2ca156a5b231..e8a0759cb4d0 100644 ---- a/arch/mips/kernel/setup.c -+++ b/arch/mips/kernel/setup.c -@@ -750,7 +750,7 @@ static void __init prefill_possible_map(void) - for (; i < NR_CPUS; i++) - set_cpu_possible(i, false); - -- nr_cpu_ids = possible; -+ set_nr_cpu_ids(possible); - } - #else - static inline void prefill_possible_map(void) {} -diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S -index cf2c08902c05..d36939029701 100644 ---- a/arch/powerpc/kernel/head_64.S -+++ b/arch/powerpc/kernel/head_64.S -@@ -400,8 +400,12 @@ generic_secondary_common_init: - #else - LOAD_REG_ADDR(r8, paca_ptrs) /* Load paca_ptrs pointe */ - ld r8,0(r8) /* Get base vaddr of array */ -+#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) -+ LOAD_REG_IMMEDIATE(r7, NR_CPUS) -+#else - LOAD_REG_ADDR(r7, nr_cpu_ids) /* Load nr_cpu_ids address */ - lwz r7,0(r7) /* also the max paca allocated */ -+#endif - li r5,0 /* logical cpu id */ - 1: - sldi r9,r5,3 /* get paca_ptrs[] index from cpu id */ -diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c -index f24227bc3220..3f3ea0287f69 100644 ---- a/arch/x86/kernel/smpboot.c -+++ b/arch/x86/kernel/smpboot.c -@@ -1316,7 +1316,7 @@ static void __init smp_sanity_check(void) - nr++; - } - -- nr_cpu_ids = 8; -+ set_nr_cpu_ids(8); - } - #endif - -@@ -1569,7 +1569,7 @@ __init void prefill_possible_map(void) - possible = i; - } - -- nr_cpu_ids = possible; -+ set_nr_cpu_ids(possible); - - pr_info("Allowing %d CPUs, %d hotplug CPUs\n", - possible, max_t(int, possible - num_processors, 0)); -diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c -index ba7af2eca755..480be82e9b7b 100644 ---- a/arch/x86/xen/smp_pv.c -+++ b/arch/x86/xen/smp_pv.c -@@ -179,7 +179,7 @@ static void __init _get_smp_config(unsigned int early) - * hypercall to expand the max number of VCPUs an already - * running guest has. So cap it up to X. */ - if (subtract) -- nr_cpu_ids = nr_cpu_ids - subtract; -+ set_nr_cpu_ids(nr_cpu_ids - subtract); - #endif - - } -diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c -index 5d44ceac855b..e92bbd754365 100644 ---- a/fs/ntfs3/bitmap.c -+++ b/fs/ntfs3/bitmap.c -@@ -560,7 +560,7 @@ static int wnd_rescan(struct wnd_bitmap *wnd) - - buf = (ulong *)bh->b_data; - -- used = __bitmap_weight(buf, wbits); -+ used = bitmap_weight(buf, wbits); - if (used < wbits) { - frb = wbits - used; - wnd->free_bits[iw] = frb; -@@ -1364,7 +1364,7 @@ int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits) - buf = (ulong *)bh->b_data; - - __bitmap_clear(buf, b0, blocksize * 8 - b0); -- frb = wbits - __bitmap_weight(buf, wbits); -+ frb = wbits - bitmap_weight(buf, wbits); - wnd->total_zeroes += frb - wnd->free_bits[iw]; - wnd->free_bits[iw] = frb; - -diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h -index f65410a49fda..7d6d73b78147 100644 ---- a/include/linux/bitmap.h -+++ b/include/linux/bitmap.h -@@ -51,6 +51,7 @@ struct device; - * bitmap_empty(src, nbits) Are all bits zero in *src? - * bitmap_full(src, nbits) Are all bits set in *src? - * bitmap_weight(src, nbits) Hamming Weight: number set bits -+ * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap - * bitmap_set(dst, pos, nbits) Set specified bit area - * bitmap_clear(dst, pos, nbits) Clear specified bit area - * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area -@@ -164,6 +165,8 @@ bool __bitmap_intersects(const unsigned long *bitmap1, - bool __bitmap_subset(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int nbits); - unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); -+unsigned int __bitmap_weight_and(const unsigned long *bitmap1, -+ const unsigned long *bitmap2, unsigned int nbits); - void __bitmap_set(unsigned long *map, unsigned int start, int len); - void __bitmap_clear(unsigned long *map, unsigned int start, int len); - -@@ -222,7 +225,6 @@ void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int n - #else - #define bitmap_copy_le bitmap_copy - #endif --unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits); - int bitmap_print_to_pagebuf(bool list, char *buf, - const unsigned long *maskp, int nmaskbits); - -@@ -439,6 +441,15 @@ unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits) - return __bitmap_weight(src, nbits); - } - -+static __always_inline -+unsigned long bitmap_weight_and(const unsigned long *src1, -+ const unsigned long *src2, unsigned int nbits) -+{ -+ if (small_const_nbits(nbits)) -+ return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)); -+ return __bitmap_weight_and(src1, src2, nbits); -+} -+ - static __always_inline void bitmap_set(unsigned long *map, unsigned int start, - unsigned int nbits) - { -diff --git a/include/linux/bitops.h b/include/linux/bitops.h -index 3b89c64bcfd8..d7dd83fafeba 100644 ---- a/include/linux/bitops.h -+++ b/include/linux/bitops.h -@@ -247,6 +247,25 @@ static inline unsigned long __ffs64(u64 word) - return __ffs((unsigned long)word); - } - -+/** -+ * fns - find N'th set bit in a word -+ * @word: The word to search -+ * @n: Bit to find -+ */ -+static inline unsigned long fns(unsigned long word, unsigned int n) -+{ -+ unsigned int bit; -+ -+ while (word) { -+ bit = __ffs(word); -+ if (n-- == 0) -+ return bit; -+ __clear_bit(bit, &word); -+ } -+ -+ return BITS_PER_LONG; -+} -+ - /** - * assign_bit - Assign value to a bit in memory - * @nr: the bit to set -diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h -index e8ad12b5b9d2..286804bfe3b7 100644 ---- a/include/linux/cpumask.h -+++ b/include/linux/cpumask.h -@@ -35,19 +35,23 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; - */ - #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) - --#if NR_CPUS == 1 --#define nr_cpu_ids 1U -+#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) -+#define nr_cpu_ids ((unsigned int)NR_CPUS) - #else - extern unsigned int nr_cpu_ids; - #endif - --#ifdef CONFIG_CPUMASK_OFFSTACK --/* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also, -- * not all bits may be allocated. */ --#define nr_cpumask_bits nr_cpu_ids -+static inline void set_nr_cpu_ids(unsigned int nr) -+{ -+#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) -+ WARN_ON(nr != nr_cpu_ids); - #else --#define nr_cpumask_bits ((unsigned int)NR_CPUS) -+ nr_cpu_ids = nr; - #endif -+} -+ -+/* Deprecated. Always use nr_cpu_ids. */ -+#define nr_cpumask_bits nr_cpu_ids - - /* - * The following particular system cpumasks and operations manage -@@ -67,10 +71,6 @@ extern unsigned int nr_cpu_ids; - * cpu_online_mask is the dynamic subset of cpu_present_mask, - * indicating those CPUs available for scheduling. - * -- * If HOTPLUG is enabled, then cpu_possible_mask is forced to have -- * all NR_CPUS bits set, otherwise it is just the set of CPUs that -- * ACPI reports present at boot. -- * - * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, - * depending on what ACPI reports as currently plugged in, otherwise - * cpu_present_mask is just a copy of cpu_possible_mask. -@@ -174,9 +174,8 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) - static inline - unsigned int cpumask_next(int n, const struct cpumask *srcp) - { -- /* -1 is a legal arg here. */ -- if (n != -1) -- cpumask_check(n); -+ /* n is a prior cpu */ -+ cpumask_check(n + 1); - return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); - } - -@@ -189,9 +188,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) - */ - static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) - { -- /* -1 is a legal arg here. */ -- if (n != -1) -- cpumask_check(n); -+ /* n is a prior cpu */ -+ cpumask_check(n + 1); - return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); - } - -@@ -231,9 +229,8 @@ static inline - unsigned int cpumask_next_and(int n, const struct cpumask *src1p, - const struct cpumask *src2p) - { -- /* -1 is a legal arg here. */ -- if (n != -1) -- cpumask_check(n); -+ /* n is a prior cpu */ -+ cpumask_check(n + 1); - return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), - nr_cpumask_bits, n + 1); - } -@@ -246,9 +243,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, - * After the loop, cpu is >= nr_cpu_ids. - */ - #define for_each_cpu(cpu, mask) \ -- for ((cpu) = -1; \ -- (cpu) = cpumask_next((cpu), (mask)), \ -- (cpu) < nr_cpu_ids;) -+ for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) - - /** - * for_each_cpu_not - iterate over every cpu in a complemented mask -@@ -258,17 +253,15 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, - * After the loop, cpu is >= nr_cpu_ids. - */ - #define for_each_cpu_not(cpu, mask) \ -- for ((cpu) = -1; \ -- (cpu) = cpumask_next_zero((cpu), (mask)), \ -- (cpu) < nr_cpu_ids;) -+ for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) - - #if NR_CPUS == 1 - static inline - unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) - { - cpumask_check(start); -- if (n != -1) -- cpumask_check(n); -+ /* n is a prior cpu */ -+ cpumask_check(n + 1); - - /* - * Return the first available CPU when wrapping, or when starting before cpu0, -@@ -293,10 +286,8 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta - * - * After the loop, cpu is >= nr_cpu_ids. - */ --#define for_each_cpu_wrap(cpu, mask, start) \ -- for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \ -- (cpu) < nr_cpumask_bits; \ -- (cpu) = cpumask_next_wrap((cpu), (mask), (start), true)) -+#define for_each_cpu_wrap(cpu, mask, start) \ -+ for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start) - - /** - * for_each_cpu_and - iterate over every cpu in both masks -@@ -313,9 +304,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta - * After the loop, cpu is >= nr_cpu_ids. - */ - #define for_each_cpu_and(cpu, mask1, mask2) \ -- for ((cpu) = -1; \ -- (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ -- (cpu) < nr_cpu_ids;) -+ for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) - - /** - * cpumask_any_but - return a "random" in a cpumask, but not this one. -@@ -337,6 +326,50 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) - return i; - } - -+/** -+ * cpumask_nth - get the first cpu in a cpumask -+ * @srcp: the cpumask pointer -+ * @cpu: the N'th cpu to find, starting from 0 -+ * -+ * Returns >= nr_cpu_ids if such cpu doesn't exist. -+ */ -+static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) -+{ -+ return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu)); -+} -+ -+/** -+ * cpumask_nth_and - get the first cpu in 2 cpumasks -+ * @srcp1: the cpumask pointer -+ * @srcp2: the cpumask pointer -+ * @cpu: the N'th cpu to find, starting from 0 -+ * -+ * Returns >= nr_cpu_ids if such cpu doesn't exist. -+ */ -+static inline -+unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, -+ const struct cpumask *srcp2) -+{ -+ return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), -+ nr_cpumask_bits, cpumask_check(cpu)); -+} -+ -+/** -+ * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. -+ * @srcp1: the cpumask pointer -+ * @srcp2: the cpumask pointer -+ * @cpu: the N'th cpu to find, starting from 0 -+ * -+ * Returns >= nr_cpu_ids if such cpu doesn't exist. -+ */ -+static inline -+unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, -+ const struct cpumask *srcp2) -+{ -+ return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), -+ nr_cpumask_bits, cpumask_check(cpu)); -+} -+ - #define CPU_BITS_NONE \ - { \ - [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ -@@ -586,6 +619,17 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) - return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits); - } - -+/** -+ * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) -+ * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. -+ * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. -+ */ -+static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, -+ const struct cpumask *srcp2) -+{ -+ return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); -+} -+ - /** - * cpumask_shift_right - *dstp = *srcp >> n - * @dstp: the cpumask result -diff --git a/include/linux/find.h b/include/linux/find.h -index 424ef67d4a42..0cdfab9734a6 100644 ---- a/include/linux/find.h -+++ b/include/linux/find.h -@@ -8,15 +8,31 @@ - - #include - --extern unsigned long _find_next_bit(const unsigned long *addr1, -- const unsigned long *addr2, unsigned long nbits, -- unsigned long start, unsigned long invert, unsigned long le); -+unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, -+ unsigned long start); -+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long nbits, unsigned long start); -+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, -+ unsigned long start); - extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); -+unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n); -+unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long size, unsigned long n); -+unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long size, unsigned long n); - extern unsigned long _find_first_and_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long size); - extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); - extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); - -+#ifdef __BIG_ENDIAN -+unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size); -+unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned -+ long size, unsigned long offset); -+unsigned long _find_next_bit_le(const unsigned long *addr, unsigned -+ long size, unsigned long offset); -+#endif -+ - #ifndef find_next_bit - /** - * find_next_bit - find the next set bit in a memory region -@@ -41,7 +57,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, - return val ? __ffs(val) : size; - } - -- return _find_next_bit(addr, NULL, size, offset, 0UL, 0); -+ return _find_next_bit(addr, size, offset); - } - #endif - -@@ -71,7 +87,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, - return val ? __ffs(val) : size; - } - -- return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); -+ return _find_next_and_bit(addr1, addr2, size, offset); - } - #endif - -@@ -99,7 +115,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, - return val == ~0UL ? size : ffz(val); - } - -- return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); -+ return _find_next_zero_bit(addr, size, offset); - } - #endif - -@@ -125,6 +141,87 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) - } - #endif - -+/** -+ * find_nth_bit - find N'th set bit in a memory region -+ * @addr: The address to start the search at -+ * @size: The maximum number of bits to search -+ * @n: The number of set bit, which position is needed, counting from 0 -+ * -+ * The following is semantically equivalent: -+ * idx = find_nth_bit(addr, size, 0); -+ * idx = find_first_bit(addr, size); -+ * -+ * Returns the bit number of the N'th set bit. -+ * If no such, returns @size. -+ */ -+static inline -+unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) -+{ -+ if (n >= size) -+ return size; -+ -+ if (small_const_nbits(size)) { -+ unsigned long val = *addr & GENMASK(size - 1, 0); -+ -+ return val ? fns(val, n) : size; -+ } -+ -+ return __find_nth_bit(addr, size, n); -+} -+ -+/** -+ * find_nth_and_bit - find N'th set bit in 2 memory regions -+ * @addr1: The 1st address to start the search at -+ * @addr2: The 2nd address to start the search at -+ * @size: The maximum number of bits to search -+ * @n: The number of set bit, which position is needed, counting from 0 -+ * -+ * Returns the bit number of the N'th set bit. -+ * If no such, returns @size. -+ */ -+static inline -+unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long size, unsigned long n) -+{ -+ if (n >= size) -+ return size; -+ -+ if (small_const_nbits(size)) { -+ unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0); -+ -+ return val ? fns(val, n) : size; -+ } -+ -+ return __find_nth_and_bit(addr1, addr2, size, n); -+} -+ -+/** -+ * find_nth_andnot_bit - find N'th set bit in 2 memory regions, -+ * flipping bits in 2nd region -+ * @addr1: The 1st address to start the search at -+ * @addr2: The 2nd address to start the search at -+ * @size: The maximum number of bits to search -+ * @n: The number of set bit, which position is needed, counting from 0 -+ * -+ * Returns the bit number of the N'th set bit. -+ * If no such, returns @size. -+ */ -+static inline -+unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long size, unsigned long n) -+{ -+ if (n >= size) -+ return size; -+ -+ if (small_const_nbits(size)) { -+ unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0); -+ -+ return val ? fns(val, n) : size; -+ } -+ -+ return __find_nth_andnot_bit(addr1, addr2, size, n); -+} -+ - #ifndef find_first_and_bit - /** - * find_first_and_bit - find the first set bit in both memory regions -@@ -193,6 +290,78 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) - } - #endif - -+/** -+ * find_next_and_bit_wrap - find the next set bit in both memory regions -+ * @addr1: The first address to base the search on -+ * @addr2: The second address to base the search on -+ * @size: The bitmap size in bits -+ * @offset: The bitnumber to start searching at -+ * -+ * Returns the bit number for the next set bit, or first set bit up to @offset -+ * If no bits are set, returns @size. -+ */ -+static inline -+unsigned long find_next_and_bit_wrap(const unsigned long *addr1, -+ const unsigned long *addr2, -+ unsigned long size, unsigned long offset) -+{ -+ unsigned long bit = find_next_and_bit(addr1, addr2, size, offset); -+ -+ if (bit < size) -+ return bit; -+ -+ bit = find_first_and_bit(addr1, addr2, offset); -+ return bit < offset ? bit : size; -+} -+ -+/** -+ * find_next_bit_wrap - find the next set bit in both memory regions -+ * @addr: The first address to base the search on -+ * @size: The bitmap size in bits -+ * @offset: The bitnumber to start searching at -+ * -+ * Returns the bit number for the next set bit, or first set bit up to @offset -+ * If no bits are set, returns @size. -+ */ -+static inline -+unsigned long find_next_bit_wrap(const unsigned long *addr, -+ unsigned long size, unsigned long offset) -+{ -+ unsigned long bit = find_next_bit(addr, size, offset); -+ -+ if (bit < size) -+ return bit; -+ -+ bit = find_first_bit(addr, offset); -+ return bit < offset ? bit : size; -+} -+ -+/* -+ * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing -+ * before using it alone. -+ */ -+static inline -+unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size, -+ unsigned long start, unsigned long n) -+{ -+ unsigned long bit; -+ -+ /* If not wrapped around */ -+ if (n > start) { -+ /* and have a bit, just return it. */ -+ bit = find_next_bit(bitmap, size, n); -+ if (bit < size) -+ return bit; -+ -+ /* Otherwise, wrap around and ... */ -+ n = 0; -+ } -+ -+ /* Search the other part. */ -+ bit = find_next_bit(bitmap, start, n); -+ return bit < start ? bit : size; -+} -+ - /** - * find_next_clump8 - find next 8-bit clump with set bits in a memory region - * @clump: location to store copy of found clump -@@ -247,7 +416,21 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned - return val == ~0UL ? size : ffz(val); - } - -- return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); -+ return _find_next_zero_bit_le(addr, size, offset); -+} -+#endif -+ -+#ifndef find_first_zero_bit_le -+static inline -+unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) -+{ -+ if (small_const_nbits(size)) { -+ unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0); -+ -+ return val == ~0UL ? size : ffz(val); -+ } -+ -+ return _find_first_zero_bit_le(addr, size); - } - #endif - -@@ -266,40 +449,34 @@ unsigned long find_next_bit_le(const void *addr, unsigned - return val ? __ffs(val) : size; - } - -- return _find_next_bit(addr, NULL, size, offset, 0UL, 1); -+ return _find_next_bit_le(addr, size, offset); - } - #endif - --#ifndef find_first_zero_bit_le --#define find_first_zero_bit_le(addr, size) \ -- find_next_zero_bit_le((addr), (size), 0) --#endif -- - #else - #error "Please fix " - #endif - - #define for_each_set_bit(bit, addr, size) \ -- for ((bit) = find_next_bit((addr), (size), 0); \ -- (bit) < (size); \ -- (bit) = find_next_bit((addr), (size), (bit) + 1)) -+ for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) -+ -+#define for_each_and_bit(bit, addr1, addr2, size) \ -+ for ((bit) = 0; \ -+ (bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ -+ (bit)++) - - /* same as for_each_set_bit() but use bit as value to start with */ - #define for_each_set_bit_from(bit, addr, size) \ -- for ((bit) = find_next_bit((addr), (size), (bit)); \ -- (bit) < (size); \ -- (bit) = find_next_bit((addr), (size), (bit) + 1)) -+ for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) - - #define for_each_clear_bit(bit, addr, size) \ -- for ((bit) = find_next_zero_bit((addr), (size), 0); \ -- (bit) < (size); \ -- (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) -+ for ((bit) = 0; \ -+ (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); \ -+ (bit)++) - - /* same as for_each_clear_bit() but use bit as value to start with */ - #define for_each_clear_bit_from(bit, addr, size) \ -- for ((bit) = find_next_zero_bit((addr), (size), (bit)); \ -- (bit) < (size); \ -- (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) -+ for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++) - - /** - * for_each_set_bitrange - iterate over all set bit ranges [b; e) -@@ -309,11 +486,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned - * @size: bitmap size in number of bits - */ - #define for_each_set_bitrange(b, e, addr, size) \ -- for ((b) = find_next_bit((addr), (size), 0), \ -- (e) = find_next_zero_bit((addr), (size), (b) + 1); \ -+ for ((b) = 0; \ -+ (b) = find_next_bit((addr), (size), b), \ -+ (e) = find_next_zero_bit((addr), (size), (b) + 1), \ - (b) < (size); \ -- (b) = find_next_bit((addr), (size), (e) + 1), \ -- (e) = find_next_zero_bit((addr), (size), (b) + 1)) -+ (b) = (e) + 1) - - /** - * for_each_set_bitrange_from - iterate over all set bit ranges [b; e) -@@ -323,11 +500,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned - * @size: bitmap size in number of bits - */ - #define for_each_set_bitrange_from(b, e, addr, size) \ -- for ((b) = find_next_bit((addr), (size), (b)), \ -- (e) = find_next_zero_bit((addr), (size), (b) + 1); \ -+ for (; \ -+ (b) = find_next_bit((addr), (size), (b)), \ -+ (e) = find_next_zero_bit((addr), (size), (b) + 1), \ - (b) < (size); \ -- (b) = find_next_bit((addr), (size), (e) + 1), \ -- (e) = find_next_zero_bit((addr), (size), (b) + 1)) -+ (b) = (e) + 1) - - /** - * for_each_clear_bitrange - iterate over all unset bit ranges [b; e) -@@ -337,11 +514,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned - * @size: bitmap size in number of bits - */ - #define for_each_clear_bitrange(b, e, addr, size) \ -- for ((b) = find_next_zero_bit((addr), (size), 0), \ -- (e) = find_next_bit((addr), (size), (b) + 1); \ -+ for ((b) = 0; \ -+ (b) = find_next_zero_bit((addr), (size), (b)), \ -+ (e) = find_next_bit((addr), (size), (b) + 1), \ - (b) < (size); \ -- (b) = find_next_zero_bit((addr), (size), (e) + 1), \ -- (e) = find_next_bit((addr), (size), (b) + 1)) -+ (b) = (e) + 1) - - /** - * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e) -@@ -351,11 +528,24 @@ unsigned long find_next_bit_le(const void *addr, unsigned - * @size: bitmap size in number of bits - */ - #define for_each_clear_bitrange_from(b, e, addr, size) \ -- for ((b) = find_next_zero_bit((addr), (size), (b)), \ -- (e) = find_next_bit((addr), (size), (b) + 1); \ -+ for (; \ -+ (b) = find_next_zero_bit((addr), (size), (b)), \ -+ (e) = find_next_bit((addr), (size), (b) + 1), \ - (b) < (size); \ -- (b) = find_next_zero_bit((addr), (size), (e) + 1), \ -- (e) = find_next_bit((addr), (size), (b) + 1)) -+ (b) = (e) + 1) -+ -+/** -+ * for_each_set_bit_wrap - iterate over all set bits starting from @start, and -+ * wrapping around the end of bitmap. -+ * @bit: offset for current iteration -+ * @addr: bitmap address to base the search on -+ * @size: bitmap size in number of bits -+ * @start: Starting bit for bitmap traversing, wrapping around the bitmap end -+ */ -+#define for_each_set_bit_wrap(bit, addr, size, start) \ -+ for ((bit) = find_next_bit_wrap((addr), (size), (start)); \ -+ (bit) < (size); \ -+ (bit) = __for_each_wrap((addr), (size), (start), (bit) + 1)) - - /** - * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits -diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h -index 05d6f3facd5a..4d6d5a2dd82e 100644 ---- a/include/linux/netdevice.h -+++ b/include/linux/netdevice.h -@@ -3643,9 +3643,8 @@ static inline bool netif_attr_test_online(unsigned long j, - static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp, - unsigned int nr_bits) - { -- /* -1 is a legal arg here. */ -- if (n != -1) -- cpu_max_bits_warn(n, nr_bits); -+ /* n is a prior cpu */ -+ cpu_max_bits_warn(n + 1, nr_bits); - - if (srcp) - return find_next_bit(srcp, nr_bits, n + 1); -@@ -3666,9 +3665,8 @@ static inline int netif_attrmask_next_and(int n, const unsigned long *src1p, - const unsigned long *src2p, - unsigned int nr_bits) - { -- /* -1 is a legal arg here. */ -- if (n != -1) -- cpu_max_bits_warn(n, nr_bits); -+ /* n is a prior cpu */ -+ cpu_max_bits_warn(n + 1, nr_bits); - - if (src1p && src2p) - return find_next_and_bit(src1p, src2p, nr_bits, n + 1); -diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h -index 3a0eec9f2faa..ef7c6fc71e10 100644 ---- a/include/linux/nodemask.h -+++ b/include/linux/nodemask.h -@@ -509,8 +509,7 @@ static inline int node_random(const nodemask_t *maskp) - - w = nodes_weight(*maskp); - if (w) -- bit = bitmap_ord_to_pos(maskp->bits, -- get_random_int() % w, MAX_NUMNODES); -+ bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_int() % w); - return bit; - #else - return 0; -diff --git a/kernel/smp.c b/kernel/smp.c -index 650810a6f29b..661d09ae5d6a 100644 ---- a/kernel/smp.c -+++ b/kernel/smp.c -@@ -1070,7 +1070,7 @@ static int __init nrcpus(char *str) - int nr_cpus; - - if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids) -- nr_cpu_ids = nr_cpus; -+ set_nr_cpu_ids(nr_cpus); - - return 0; - } -@@ -1088,14 +1088,16 @@ static int __init maxcpus(char *str) - - early_param("maxcpus", maxcpus); - -+#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS) - /* Setup number of possible processor ids */ - unsigned int nr_cpu_ids __read_mostly = NR_CPUS; - EXPORT_SYMBOL(nr_cpu_ids); -+#endif - - /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */ - void __init setup_nr_cpu_ids(void) - { -- nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; -+ set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); - } - - /* Called by boot processor to activate the rest. */ -diff --git a/lib/Kconfig b/lib/Kconfig -index 3ea8941ab18d..d628235f7934 100644 ---- a/lib/Kconfig -+++ b/lib/Kconfig -@@ -531,6 +531,15 @@ config CPUMASK_OFFSTACK - them on the stack. This is a bit more expensive, but avoids - stack overflow. - -+config FORCE_NR_CPUS -+ bool "NR_CPUS is set to an actual number of CPUs" -+ depends on SMP -+ help -+ Say Yes if you have NR_CPUS set to an actual number of possible -+ CPUs in your system, not to a default value. This forces the core -+ code to rely on compile-time value and optimize kernel routines -+ better. -+ - config CPU_RMAP - bool - depends on SMP -diff --git a/lib/bitmap.c b/lib/bitmap.c -index 488e6c3e5acc..1c81413c51f8 100644 ---- a/lib/bitmap.c -+++ b/lib/bitmap.c -@@ -333,20 +333,32 @@ bool __bitmap_subset(const unsigned long *bitmap1, - } - EXPORT_SYMBOL(__bitmap_subset); - -+#define BITMAP_WEIGHT(FETCH, bits) \ -+({ \ -+ unsigned int __bits = (bits), idx, w = 0; \ -+ \ -+ for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \ -+ w += hweight_long(FETCH); \ -+ \ -+ if (__bits % BITS_PER_LONG) \ -+ w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \ -+ \ -+ w; \ -+}) -+ - unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) - { -- unsigned int k, lim = bits/BITS_PER_LONG, w = 0; -- -- for (k = 0; k < lim; k++) -- w += hweight_long(bitmap[k]); -- -- if (bits % BITS_PER_LONG) -- w += hweight_long(bitmap[k] & BITMAP_LAST_WORD_MASK(bits)); -- -- return w; -+ return BITMAP_WEIGHT(bitmap[idx], bits); - } - EXPORT_SYMBOL(__bitmap_weight); - -+unsigned int __bitmap_weight_and(const unsigned long *bitmap1, -+ const unsigned long *bitmap2, unsigned int bits) -+{ -+ return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits); -+} -+EXPORT_SYMBOL(__bitmap_weight_and); -+ - void __bitmap_set(unsigned long *map, unsigned int start, int len) - { - unsigned long *p = map + BIT_WORD(start); -@@ -953,37 +965,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne - if (pos >= nbits || !test_bit(pos, buf)) - return -1; - -- return __bitmap_weight(buf, pos); --} -- --/** -- * bitmap_ord_to_pos - find position of n-th set bit in bitmap -- * @buf: pointer to bitmap -- * @ord: ordinal bit position (n-th set bit, n >= 0) -- * @nbits: number of valid bit positions in @buf -- * -- * Map the ordinal offset of bit @ord in @buf to its position in @buf. -- * Value of @ord should be in range 0 <= @ord < weight(buf). If @ord -- * >= weight(buf), returns @nbits. -- * -- * If for example, just bits 4 through 7 are set in @buf, then @ord -- * values 0 through 3 will get mapped to 4 through 7, respectively, -- * and all other @ord values returns @nbits. When @ord value 3 -- * gets mapped to (returns) @pos value 7 in this example, that means -- * that the 3rd set bit (starting with 0th) is at position 7 in @buf. -- * -- * The bit positions 0 through @nbits-1 are valid positions in @buf. -- */ --unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsigned int nbits) --{ -- unsigned int pos; -- -- for (pos = find_first_bit(buf, nbits); -- pos < nbits && ord; -- pos = find_next_bit(buf, nbits, pos + 1)) -- ord--; -- -- return pos; -+ return bitmap_weight(buf, pos); - } - - /** -@@ -1035,7 +1017,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, - if (n < 0 || w == 0) - set_bit(oldbit, dst); /* identity map */ - else -- set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); -+ set_bit(find_nth_bit(new, nbits, n % w), dst); - } - } - EXPORT_SYMBOL(bitmap_remap); -@@ -1074,7 +1056,7 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, - if (n < 0 || w == 0) - return oldbit; - else -- return bitmap_ord_to_pos(new, n % w, bits); -+ return find_nth_bit(new, bits, n % w); - } - EXPORT_SYMBOL(bitmap_bitremap); - -@@ -1198,7 +1180,7 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig, - * The following code is a more efficient, but less - * obvious, equivalent to the loop: - * for (m = 0; m < bitmap_weight(relmap, bits); m++) { -- * n = bitmap_ord_to_pos(orig, m, bits); -+ * n = find_nth_bit(orig, bits, m); - * if (test_bit(m, orig)) - * set_bit(n, dst); - * } -diff --git a/lib/cpumask.c b/lib/cpumask.c -index f0ae119be8c4..c7c392514fd3 100644 ---- a/lib/cpumask.c -+++ b/lib/cpumask.c -@@ -128,23 +128,21 @@ unsigned int cpumask_local_spread(unsigned int i, int node) - i %= num_online_cpus(); - - if (node == NUMA_NO_NODE) { -- for_each_cpu(cpu, cpu_online_mask) -- if (i-- == 0) -- return cpu; -+ cpu = cpumask_nth(i, cpu_online_mask); -+ if (cpu < nr_cpu_ids) -+ return cpu; - } else { - /* NUMA first. */ -- for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask) -- if (i-- == 0) -- return cpu; -- -- for_each_cpu(cpu, cpu_online_mask) { -- /* Skip NUMA nodes, done above. */ -- if (cpumask_test_cpu(cpu, cpumask_of_node(node))) -- continue; -- -- if (i-- == 0) -- return cpu; -- } -+ cpu = cpumask_nth_and(i, cpu_online_mask, cpumask_of_node(node)); -+ if (cpu < nr_cpu_ids) -+ return cpu; -+ -+ i -= cpumask_weight_and(cpu_online_mask, cpumask_of_node(node)); -+ -+ /* Skip NUMA nodes, done above. */ -+ cpu = cpumask_nth_andnot(i, cpu_online_mask, cpumask_of_node(node)); -+ if (cpu < nr_cpu_ids) -+ return cpu; - } - BUG(); - } -@@ -168,10 +166,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, - /* NOTE: our first selection will skip 0. */ - prev = __this_cpu_read(distribute_cpu_mask_prev); - -- next = cpumask_next_and(prev, src1p, src2p); -- if (next >= nr_cpu_ids) -- next = cpumask_first_and(src1p, src2p); -- -+ next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p), -+ nr_cpumask_bits, prev + 1); - if (next < nr_cpu_ids) - __this_cpu_write(distribute_cpu_mask_prev, next); - -@@ -185,11 +181,7 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp) - - /* NOTE: our first selection will skip 0. */ - prev = __this_cpu_read(distribute_cpu_mask_prev); -- -- next = cpumask_next(prev, srcp); -- if (next >= nr_cpu_ids) -- next = cpumask_first(srcp); -- -+ next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1); - if (next < nr_cpu_ids) - __this_cpu_write(distribute_cpu_mask_prev, next); - -diff --git a/lib/find_bit.c b/lib/find_bit.c -index 1b8e4b2a9cba..25609974cbe4 100644 ---- a/lib/find_bit.c -+++ b/lib/find_bit.c -@@ -19,57 +19,78 @@ - #include - #include - --#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ -- !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ -- !defined(find_next_and_bit) - /* -- * This is a common helper function for find_next_bit, find_next_zero_bit, and -- * find_next_and_bit. The differences are: -- * - The "invert" argument, which is XORed with each fetched word before -- * searching it for one bits. -- * - The optional "addr2", which is anded with "addr1" if present. -+ * Common helper for find_bit() function family -+ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) -+ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) -+ * @size: The bitmap size in bits - */ --unsigned long _find_next_bit(const unsigned long *addr1, -- const unsigned long *addr2, unsigned long nbits, -- unsigned long start, unsigned long invert, unsigned long le) --{ -- unsigned long tmp, mask; -- -- if (unlikely(start >= nbits)) -- return nbits; -- -- tmp = addr1[start / BITS_PER_LONG]; -- if (addr2) -- tmp &= addr2[start / BITS_PER_LONG]; -- tmp ^= invert; -- -- /* Handle 1st word. */ -- mask = BITMAP_FIRST_WORD_MASK(start); -- if (le) -- mask = swab(mask); -- -- tmp &= mask; -- -- start = round_down(start, BITS_PER_LONG); -- -- while (!tmp) { -- start += BITS_PER_LONG; -- if (start >= nbits) -- return nbits; -- -- tmp = addr1[start / BITS_PER_LONG]; -- if (addr2) -- tmp &= addr2[start / BITS_PER_LONG]; -- tmp ^= invert; -- } -+#define FIND_FIRST_BIT(FETCH, MUNGE, size) \ -+({ \ -+ unsigned long idx, val, sz = (size); \ -+ \ -+ for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ -+ val = (FETCH); \ -+ if (val) { \ -+ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ -+ break; \ -+ } \ -+ } \ -+ \ -+ sz; \ -+}) - -- if (le) -- tmp = swab(tmp); -- -- return min(start + __ffs(tmp), nbits); --} --EXPORT_SYMBOL(_find_next_bit); --#endif -+/* -+ * Common helper for find_next_bit() function family -+ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) -+ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) -+ * @size: The bitmap size in bits -+ * @start: The bitnumber to start searching at -+ */ -+#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ -+({ \ -+ unsigned long mask, idx, tmp, sz = (size), __start = (start); \ -+ \ -+ if (unlikely(__start >= sz)) \ -+ goto out; \ -+ \ -+ mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ -+ idx = __start / BITS_PER_LONG; \ -+ \ -+ for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ -+ if ((idx + 1) * BITS_PER_LONG >= sz) \ -+ goto out; \ -+ idx++; \ -+ } \ -+ \ -+ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ -+out: \ -+ sz; \ -+}) -+ -+#define FIND_NTH_BIT(FETCH, size, num) \ -+({ \ -+ unsigned long sz = (size), nr = (num), idx, w, tmp; \ -+ \ -+ for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) { \ -+ if (idx * BITS_PER_LONG + nr >= sz) \ -+ goto out; \ -+ \ -+ tmp = (FETCH); \ -+ w = hweight_long(tmp); \ -+ if (w > nr) \ -+ goto found; \ -+ \ -+ nr -= w; \ -+ } \ -+ \ -+ if (sz % BITS_PER_LONG) \ -+ tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz); \ -+found: \ -+ sz = min(idx * BITS_PER_LONG + fns(tmp, nr), sz); \ -+out: \ -+ sz; \ -+}) - - #ifndef find_first_bit - /* -@@ -77,14 +98,7 @@ EXPORT_SYMBOL(_find_next_bit); - */ - unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) - { -- unsigned long idx; -- -- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { -- if (addr[idx]) -- return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); -- } -- -- return size; -+ return FIND_FIRST_BIT(addr[idx], /* nop */, size); - } - EXPORT_SYMBOL(_find_first_bit); - #endif -@@ -97,15 +111,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, - const unsigned long *addr2, - unsigned long size) - { -- unsigned long idx, val; -- -- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { -- val = addr1[idx] & addr2[idx]; -- if (val) -- return min(idx * BITS_PER_LONG + __ffs(val), size); -- } -- -- return size; -+ return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); - } - EXPORT_SYMBOL(_find_first_and_bit); - #endif -@@ -116,16 +122,55 @@ EXPORT_SYMBOL(_find_first_and_bit); - */ - unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) - { -- unsigned long idx; -+ return FIND_FIRST_BIT(~addr[idx], /* nop */, size); -+} -+EXPORT_SYMBOL(_find_first_zero_bit); -+#endif - -- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { -- if (addr[idx] != ~0UL) -- return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); -- } -+#ifndef find_next_bit -+unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) -+{ -+ return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); -+} -+EXPORT_SYMBOL(_find_next_bit); -+#endif - -- return size; -+unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) -+{ -+ return FIND_NTH_BIT(addr[idx], size, n); - } --EXPORT_SYMBOL(_find_first_zero_bit); -+EXPORT_SYMBOL(__find_nth_bit); -+ -+unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long size, unsigned long n) -+{ -+ return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n); -+} -+EXPORT_SYMBOL(__find_nth_and_bit); -+ -+unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long size, unsigned long n) -+{ -+ return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n); -+} -+EXPORT_SYMBOL(__find_nth_andnot_bit); -+ -+#ifndef find_next_and_bit -+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long nbits, unsigned long start) -+{ -+ return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); -+} -+EXPORT_SYMBOL(_find_next_and_bit); -+#endif -+ -+#ifndef find_next_zero_bit -+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, -+ unsigned long start) -+{ -+ return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); -+} -+EXPORT_SYMBOL(_find_next_zero_bit); - #endif - - #ifndef find_last_bit -@@ -161,3 +206,38 @@ unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr, - return offset; - } - EXPORT_SYMBOL(find_next_clump8); -+ -+#ifdef __BIG_ENDIAN -+ -+#ifndef find_first_zero_bit_le -+/* -+ * Find the first cleared bit in an LE memory region. -+ */ -+unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size) -+{ -+ return FIND_FIRST_BIT(~addr[idx], swab, size); -+} -+EXPORT_SYMBOL(_find_first_zero_bit_le); -+ -+#endif -+ -+#ifndef find_next_zero_bit_le -+unsigned long _find_next_zero_bit_le(const unsigned long *addr, -+ unsigned long size, unsigned long offset) -+{ -+ return FIND_NEXT_BIT(~addr[idx], swab, size, offset); -+} -+EXPORT_SYMBOL(_find_next_zero_bit_le); -+#endif -+ -+#ifndef find_next_bit_le -+unsigned long _find_next_bit_le(const unsigned long *addr, -+ unsigned long size, unsigned long offset) -+{ -+ return FIND_NEXT_BIT(addr[idx], swab, size, offset); -+} -+EXPORT_SYMBOL(_find_next_bit_le); -+ -+#endif -+ -+#endif /* __BIG_ENDIAN */ -diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c -index db904b57d4b8..10754586403b 100644 ---- a/lib/find_bit_benchmark.c -+++ b/lib/find_bit_benchmark.c -@@ -115,6 +115,22 @@ static int __init test_find_last_bit(const void *bitmap, unsigned long len) - return 0; - } - -+static int __init test_find_nth_bit(const unsigned long *bitmap, unsigned long len) -+{ -+ unsigned long l, n, w = bitmap_weight(bitmap, len); -+ ktime_t time; -+ -+ time = ktime_get(); -+ for (n = 0; n < w; n++) { -+ l = find_nth_bit(bitmap, len, n); -+ WARN_ON(l >= len); -+ } -+ time = ktime_get() - time; -+ pr_err("find_nth_bit: %18llu ns, %6ld iterations\n", time, w); -+ -+ return 0; -+} -+ - static int __init test_find_next_and_bit(const void *bitmap, - const void *bitmap2, unsigned long len) - { -@@ -142,6 +158,7 @@ static int __init find_bit_test(void) - test_find_next_bit(bitmap, BITMAP_LEN); - test_find_next_zero_bit(bitmap, BITMAP_LEN); - test_find_last_bit(bitmap, BITMAP_LEN); -+ test_find_nth_bit(bitmap, BITMAP_LEN / 10); - - /* - * test_find_first_bit() may take some time, so -@@ -164,6 +181,7 @@ static int __init find_bit_test(void) - test_find_next_bit(bitmap, BITMAP_LEN); - test_find_next_zero_bit(bitmap, BITMAP_LEN); - test_find_last_bit(bitmap, BITMAP_LEN); -+ test_find_nth_bit(bitmap, BITMAP_LEN); - test_find_first_bit(bitmap, BITMAP_LEN); - test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN); - test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN); -diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c -index 98754ff9fe68..a8005ad3bd58 100644 ---- a/lib/test_bitmap.c -+++ b/lib/test_bitmap.c -@@ -16,6 +16,8 @@ - - #include "../tools/testing/selftests/kselftest_module.h" - -+#define EXP1_IN_BITS (sizeof(exp1) * 8) -+ - KSTM_MODULE_GLOBALS(); - - static char pbl_buffer[PAGE_SIZE] __initdata; -@@ -219,6 +221,47 @@ static void __init test_zero_clear(void) - expect_eq_pbl("", bmap, 1024); - } - -+static void __init test_find_nth_bit(void) -+{ -+ unsigned long b, bit, cnt = 0; -+ DECLARE_BITMAP(bmap, 64 * 3); -+ -+ bitmap_zero(bmap, 64 * 3); -+ __set_bit(10, bmap); -+ __set_bit(20, bmap); -+ __set_bit(30, bmap); -+ __set_bit(40, bmap); -+ __set_bit(50, bmap); -+ __set_bit(60, bmap); -+ __set_bit(80, bmap); -+ __set_bit(123, bmap); -+ -+ expect_eq_uint(10, find_nth_bit(bmap, 64 * 3, 0)); -+ expect_eq_uint(20, find_nth_bit(bmap, 64 * 3, 1)); -+ expect_eq_uint(30, find_nth_bit(bmap, 64 * 3, 2)); -+ expect_eq_uint(40, find_nth_bit(bmap, 64 * 3, 3)); -+ expect_eq_uint(50, find_nth_bit(bmap, 64 * 3, 4)); -+ expect_eq_uint(60, find_nth_bit(bmap, 64 * 3, 5)); -+ expect_eq_uint(80, find_nth_bit(bmap, 64 * 3, 6)); -+ expect_eq_uint(123, find_nth_bit(bmap, 64 * 3, 7)); -+ expect_eq_uint(64 * 3, find_nth_bit(bmap, 64 * 3, 8)); -+ -+ expect_eq_uint(10, find_nth_bit(bmap, 64 * 3 - 1, 0)); -+ expect_eq_uint(20, find_nth_bit(bmap, 64 * 3 - 1, 1)); -+ expect_eq_uint(30, find_nth_bit(bmap, 64 * 3 - 1, 2)); -+ expect_eq_uint(40, find_nth_bit(bmap, 64 * 3 - 1, 3)); -+ expect_eq_uint(50, find_nth_bit(bmap, 64 * 3 - 1, 4)); -+ expect_eq_uint(60, find_nth_bit(bmap, 64 * 3 - 1, 5)); -+ expect_eq_uint(80, find_nth_bit(bmap, 64 * 3 - 1, 6)); -+ expect_eq_uint(123, find_nth_bit(bmap, 64 * 3 - 1, 7)); -+ expect_eq_uint(64 * 3 - 1, find_nth_bit(bmap, 64 * 3 - 1, 8)); -+ -+ for_each_set_bit(bit, exp1, EXP1_IN_BITS) { -+ b = find_nth_bit(exp1, EXP1_IN_BITS, cnt++); -+ expect_eq_uint(b, bit); -+ } -+} -+ - static void __init test_fill_set(void) - { - DECLARE_BITMAP(bmap, 1024); -@@ -557,8 +600,6 @@ static void __init test_bitmap_parse(void) - } - } - --#define EXP1_IN_BITS (sizeof(exp1) * 8) -- - static void __init test_bitmap_arr32(void) - { - unsigned int nbits, next_bit; -@@ -685,6 +726,239 @@ static void __init test_for_each_set_clump8(void) - expect_eq_clump8(start, CLUMP_EXP_NUMBITS, clump_exp, &clump); - } - -+static void __init test_for_each_set_bit_wrap(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int wr, bit; -+ -+ bitmap_zero(orig, 500); -+ -+ /* Set individual bits */ -+ for (bit = 0; bit < 500; bit += 10) -+ bitmap_set(orig, bit, 1); -+ -+ /* Set range of bits */ -+ bitmap_set(orig, 100, 50); -+ -+ for (wr = 0; wr < 500; wr++) { -+ bitmap_zero(copy, 500); -+ -+ for_each_set_bit_wrap(bit, orig, 500, wr) -+ bitmap_set(copy, bit, 1); -+ -+ expect_eq_bitmap(orig, copy, 500); -+ } -+} -+ -+static void __init test_for_each_set_bit(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int bit; -+ -+ bitmap_zero(orig, 500); -+ bitmap_zero(copy, 500); -+ -+ /* Set individual bits */ -+ for (bit = 0; bit < 500; bit += 10) -+ bitmap_set(orig, bit, 1); -+ -+ /* Set range of bits */ -+ bitmap_set(orig, 100, 50); -+ -+ for_each_set_bit(bit, orig, 500) -+ bitmap_set(copy, bit, 1); -+ -+ expect_eq_bitmap(orig, copy, 500); -+} -+ -+static void __init test_for_each_set_bit_from(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int wr, bit; -+ -+ bitmap_zero(orig, 500); -+ -+ /* Set individual bits */ -+ for (bit = 0; bit < 500; bit += 10) -+ bitmap_set(orig, bit, 1); -+ -+ /* Set range of bits */ -+ bitmap_set(orig, 100, 50); -+ -+ for (wr = 0; wr < 500; wr++) { -+ DECLARE_BITMAP(tmp, 500); -+ -+ bitmap_zero(copy, 500); -+ bit = wr; -+ -+ for_each_set_bit_from(bit, orig, 500) -+ bitmap_set(copy, bit, 1); -+ -+ bitmap_copy(tmp, orig, 500); -+ bitmap_clear(tmp, 0, wr); -+ expect_eq_bitmap(tmp, copy, 500); -+ } -+} -+ -+static void __init test_for_each_clear_bit(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int bit; -+ -+ bitmap_fill(orig, 500); -+ bitmap_fill(copy, 500); -+ -+ /* Set individual bits */ -+ for (bit = 0; bit < 500; bit += 10) -+ bitmap_clear(orig, bit, 1); -+ -+ /* Set range of bits */ -+ bitmap_clear(orig, 100, 50); -+ -+ for_each_clear_bit(bit, orig, 500) -+ bitmap_clear(copy, bit, 1); -+ -+ expect_eq_bitmap(orig, copy, 500); -+} -+ -+static void __init test_for_each_clear_bit_from(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int wr, bit; -+ -+ bitmap_fill(orig, 500); -+ -+ /* Set individual bits */ -+ for (bit = 0; bit < 500; bit += 10) -+ bitmap_clear(orig, bit, 1); -+ -+ /* Set range of bits */ -+ bitmap_clear(orig, 100, 50); -+ -+ for (wr = 0; wr < 500; wr++) { -+ DECLARE_BITMAP(tmp, 500); -+ -+ bitmap_fill(copy, 500); -+ bit = wr; -+ -+ for_each_clear_bit_from(bit, orig, 500) -+ bitmap_clear(copy, bit, 1); -+ -+ bitmap_copy(tmp, orig, 500); -+ bitmap_set(tmp, 0, wr); -+ expect_eq_bitmap(tmp, copy, 500); -+ } -+} -+ -+static void __init test_for_each_set_bitrange(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int s, e; -+ -+ bitmap_zero(orig, 500); -+ bitmap_zero(copy, 500); -+ -+ /* Set individual bits */ -+ for (s = 0; s < 500; s += 10) -+ bitmap_set(orig, s, 1); -+ -+ /* Set range of bits */ -+ bitmap_set(orig, 100, 50); -+ -+ for_each_set_bitrange(s, e, orig, 500) -+ bitmap_set(copy, s, e-s); -+ -+ expect_eq_bitmap(orig, copy, 500); -+} -+ -+static void __init test_for_each_clear_bitrange(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int s, e; -+ -+ bitmap_fill(orig, 500); -+ bitmap_fill(copy, 500); -+ -+ /* Set individual bits */ -+ for (s = 0; s < 500; s += 10) -+ bitmap_clear(orig, s, 1); -+ -+ /* Set range of bits */ -+ bitmap_clear(orig, 100, 50); -+ -+ for_each_clear_bitrange(s, e, orig, 500) -+ bitmap_clear(copy, s, e-s); -+ -+ expect_eq_bitmap(orig, copy, 500); -+} -+ -+static void __init test_for_each_set_bitrange_from(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int wr, s, e; -+ -+ bitmap_zero(orig, 500); -+ -+ /* Set individual bits */ -+ for (s = 0; s < 500; s += 10) -+ bitmap_set(orig, s, 1); -+ -+ /* Set range of bits */ -+ bitmap_set(orig, 100, 50); -+ -+ for (wr = 0; wr < 500; wr++) { -+ DECLARE_BITMAP(tmp, 500); -+ -+ bitmap_zero(copy, 500); -+ s = wr; -+ -+ for_each_set_bitrange_from(s, e, orig, 500) -+ bitmap_set(copy, s, e - s); -+ -+ bitmap_copy(tmp, orig, 500); -+ bitmap_clear(tmp, 0, wr); -+ expect_eq_bitmap(tmp, copy, 500); -+ } -+} -+ -+static void __init test_for_each_clear_bitrange_from(void) -+{ -+ DECLARE_BITMAP(orig, 500); -+ DECLARE_BITMAP(copy, 500); -+ unsigned int wr, s, e; -+ -+ bitmap_fill(orig, 500); -+ -+ /* Set individual bits */ -+ for (s = 0; s < 500; s += 10) -+ bitmap_clear(orig, s, 1); -+ -+ /* Set range of bits */ -+ bitmap_set(orig, 100, 50); -+ -+ for (wr = 0; wr < 500; wr++) { -+ DECLARE_BITMAP(tmp, 500); -+ -+ bitmap_fill(copy, 500); -+ s = wr; -+ -+ for_each_clear_bitrange_from(s, e, orig, 500) -+ bitmap_clear(copy, s, e - s); -+ -+ bitmap_copy(tmp, orig, 500); -+ bitmap_set(tmp, 0, wr); -+ expect_eq_bitmap(tmp, copy, 500); -+ } -+} -+ - struct test_bitmap_cut { - unsigned int first; - unsigned int cut; -@@ -948,10 +1222,21 @@ static void __init selftest(void) - test_bitmap_parselist(); - test_bitmap_printlist(); - test_mem_optimisations(); -- test_for_each_set_clump8(); - test_bitmap_cut(); - test_bitmap_print_buf(); - test_bitmap_const_eval(); -+ -+ test_find_nth_bit(); -+ test_for_each_set_bit(); -+ test_for_each_set_bit_from(); -+ test_for_each_clear_bit(); -+ test_for_each_clear_bit_from(); -+ test_for_each_set_bitrange(); -+ test_for_each_clear_bitrange(); -+ test_for_each_set_bitrange_from(); -+ test_for_each_clear_bitrange_from(); -+ test_for_each_set_clump8(); -+ test_for_each_set_bit_wrap(); - } - - KSTM_MODULE_LOADERS(test_bitmap); -diff --git a/tools/include/linux/find.h b/tools/include/linux/find.h -index 47e2bd6c5174..38c0a542b0e2 100644 ---- a/tools/include/linux/find.h -+++ b/tools/include/linux/find.h -@@ -8,21 +8,23 @@ - - #include - --extern unsigned long _find_next_bit(const unsigned long *addr1, -- const unsigned long *addr2, unsigned long nbits, -- unsigned long start, unsigned long invert, unsigned long le); -+unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, -+ unsigned long start); -+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long nbits, unsigned long start); -+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, -+ unsigned long start); - extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); - extern unsigned long _find_first_and_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long size); - extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); --extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); - - #ifndef find_next_bit - /** - * find_next_bit - find the next set bit in a memory region - * @addr: The address to base the search on -- * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits -+ * @offset: The bitnumber to start searching at - * - * Returns the bit number for the next set bit - * If no bits are set, returns @size. -@@ -41,7 +43,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, - return val ? __ffs(val) : size; - } - -- return _find_next_bit(addr, NULL, size, offset, 0UL, 0); -+ return _find_next_bit(addr, size, offset); - } - #endif - -@@ -50,8 +52,8 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, - * find_next_and_bit - find the next set bit in both memory regions - * @addr1: The first address to base the search on - * @addr2: The second address to base the search on -- * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits -+ * @offset: The bitnumber to start searching at - * - * Returns the bit number for the next set bit - * If no bits are set, returns @size. -@@ -71,7 +73,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, - return val ? __ffs(val) : size; - } - -- return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); -+ return _find_next_and_bit(addr1, addr2, size, offset); - } - #endif - -@@ -79,8 +81,8 @@ unsigned long find_next_and_bit(const unsigned long *addr1, - /** - * find_next_zero_bit - find the next cleared bit in a memory region - * @addr: The address to base the search on -- * @offset: The bitnumber to start searching at - * @size: The bitmap size in bits -+ * @offset: The bitnumber to start searching at - * - * Returns the bit number of the next zero bit - * If no bits are zero, returns @size. -@@ -99,7 +101,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, - return val == ~0UL ? size : ffz(val); - } - -- return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); -+ return _find_next_zero_bit(addr, size, offset); - } - #endif - -@@ -172,43 +174,4 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) - } - #endif - --#ifndef find_last_bit --/** -- * find_last_bit - find the last set bit in a memory region -- * @addr: The address to start the search at -- * @size: The number of bits to search -- * -- * Returns the bit number of the last set bit, or size. -- */ --static inline --unsigned long find_last_bit(const unsigned long *addr, unsigned long size) --{ -- if (small_const_nbits(size)) { -- unsigned long val = *addr & GENMASK(size - 1, 0); -- -- return val ? __fls(val) : size; -- } -- -- return _find_last_bit(addr, size); --} --#endif -- --/** -- * find_next_clump8 - find next 8-bit clump with set bits in a memory region -- * @clump: location to store copy of found clump -- * @addr: address to base the search on -- * @size: bitmap size in number of bits -- * @offset: bit offset at which to start searching -- * -- * Returns the bit offset for the next set clump; the found clump value is -- * copied to the location pointed by @clump. If no bits are set, returns @size. -- */ --extern unsigned long find_next_clump8(unsigned long *clump, -- const unsigned long *addr, -- unsigned long size, unsigned long offset); -- --#define find_first_clump8(clump, bits, size) \ -- find_next_clump8((clump), (bits), (size), 0) -- -- - #endif /*__LINUX_FIND_H_ */ -diff --git a/tools/lib/find_bit.c b/tools/lib/find_bit.c -index ba4b8d94e004..6a3dc167d30e 100644 ---- a/tools/lib/find_bit.c -+++ b/tools/lib/find_bit.c -@@ -18,66 +18,54 @@ - #include - #include - --#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ -- !defined(find_next_and_bit) -- - /* -- * This is a common helper function for find_next_bit, find_next_zero_bit, and -- * find_next_and_bit. The differences are: -- * - The "invert" argument, which is XORed with each fetched word before -- * searching it for one bits. -- * - The optional "addr2", which is anded with "addr1" if present. -+ * Common helper for find_bit() function family -+ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) -+ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) -+ * @size: The bitmap size in bits - */ --unsigned long _find_next_bit(const unsigned long *addr1, -- const unsigned long *addr2, unsigned long nbits, -- unsigned long start, unsigned long invert, unsigned long le) --{ -- unsigned long tmp, mask; -- (void) le; -- -- if (unlikely(start >= nbits)) -- return nbits; -- -- tmp = addr1[start / BITS_PER_LONG]; -- if (addr2) -- tmp &= addr2[start / BITS_PER_LONG]; -- tmp ^= invert; -- -- /* Handle 1st word. */ -- mask = BITMAP_FIRST_WORD_MASK(start); -- -- /* -- * Due to the lack of swab() in tools, and the fact that it doesn't -- * need little-endian support, just comment it out -- */ --#if (0) -- if (le) -- mask = swab(mask); --#endif -- -- tmp &= mask; -+#define FIND_FIRST_BIT(FETCH, MUNGE, size) \ -+({ \ -+ unsigned long idx, val, sz = (size); \ -+ \ -+ for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ -+ val = (FETCH); \ -+ if (val) { \ -+ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ -+ break; \ -+ } \ -+ } \ -+ \ -+ sz; \ -+}) - -- start = round_down(start, BITS_PER_LONG); -- -- while (!tmp) { -- start += BITS_PER_LONG; -- if (start >= nbits) -- return nbits; -- -- tmp = addr1[start / BITS_PER_LONG]; -- if (addr2) -- tmp &= addr2[start / BITS_PER_LONG]; -- tmp ^= invert; -- } -- --#if (0) -- if (le) -- tmp = swab(tmp); --#endif -- -- return min(start + __ffs(tmp), nbits); --} --#endif -+/* -+ * Common helper for find_next_bit() function family -+ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) -+ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) -+ * @size: The bitmap size in bits -+ * @start: The bitnumber to start searching at -+ */ -+#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ -+({ \ -+ unsigned long mask, idx, tmp, sz = (size), __start = (start); \ -+ \ -+ if (unlikely(__start >= sz)) \ -+ goto out; \ -+ \ -+ mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ -+ idx = __start / BITS_PER_LONG; \ -+ \ -+ for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ -+ if ((idx + 1) * BITS_PER_LONG >= sz) \ -+ goto out; \ -+ idx++; \ -+ } \ -+ \ -+ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ -+out: \ -+ sz; \ -+}) - - #ifndef find_first_bit - /* -@@ -85,14 +73,7 @@ unsigned long _find_next_bit(const unsigned long *addr1, - */ - unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) - { -- unsigned long idx; -- -- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { -- if (addr[idx]) -- return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); -- } -- -- return size; -+ return FIND_FIRST_BIT(addr[idx], /* nop */, size); - } - #endif - -@@ -104,15 +85,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, - const unsigned long *addr2, - unsigned long size) - { -- unsigned long idx, val; -- -- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { -- val = addr1[idx] & addr2[idx]; -- if (val) -- return min(idx * BITS_PER_LONG + __ffs(val), size); -- } -- -- return size; -+ return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); - } - #endif - -@@ -122,13 +95,29 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, - */ - unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) - { -- unsigned long idx; -+ return FIND_FIRST_BIT(~addr[idx], /* nop */, size); -+} -+#endif - -- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { -- if (addr[idx] != ~0UL) -- return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); -- } -+#ifndef find_next_bit -+unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) -+{ -+ return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); -+} -+#endif - -- return size; -+#ifndef find_next_and_bit -+unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, -+ unsigned long nbits, unsigned long start) -+{ -+ return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); -+} -+#endif -+ -+#ifndef find_next_zero_bit -+unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, -+ unsigned long start) -+{ -+ return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); - } - #endif --- -2.38.0.rc1.8.g2a7d63a245 - diff --git a/6.0/all/0001-cachyos-base-all.patch b/6.0/all/0001-cachyos-base-all.patch index 5780a61d..618403c4 100644 --- a/6.0/all/0001-cachyos-base-all.patch +++ b/6.0/all/0001-cachyos-base-all.patch @@ -1,3001 +1,8047 @@ -From 5162124c2fa507648ad2101787fca971ccc8175b Mon Sep 17 00:00:00 2001 +From 4ee5774d519ab3d21a214f4aa94e3f2ddc6ceb81 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Mon, 5 Sep 2022 08:34:43 +0200 -Subject: [PATCH 01/13] bbr2 +Date: Tue, 27 Sep 2022 15:12:20 +0200 +Subject: [PATCH 01/16] cachy Signed-off-by: Peter Jung --- - Documentation/networking/ip-sysctl.rst | 58 + - include/linux/tcp.h | 6 +- - include/net/inet_connection_sock.h | 3 +- - include/net/netns/ipv4.h | 5 + - include/net/tcp.h | 58 +- - include/uapi/linux/inet_diag.h | 33 + - include/uapi/linux/snmp.h | 1 + - net/ipv4/Kconfig | 22 + - net/ipv4/Makefile | 3 +- - net/ipv4/bpf_tcp_ca.c | 12 + - net/ipv4/proc.c | 1 + - net/ipv4/sysctl_net_ipv4.c | 43 + - net/ipv4/tcp.c | 1 + - net/ipv4/tcp_bbr.c | 38 +- - net/ipv4/tcp_bbr2.c | 2692 ++++++++++++++++++++++++ - net/ipv4/tcp_cong.c | 1 + - net/ipv4/tcp_input.c | 27 +- - net/ipv4/tcp_ipv4.c | 7 + - net/ipv4/tcp_output.c | 26 +- - net/ipv4/tcp_plb.c | 100 + - net/ipv4/tcp_rate.c | 30 +- - net/ipv4/tcp_timer.c | 1 + - 22 files changed, 3133 insertions(+), 35 deletions(-) - create mode 100644 net/ipv4/tcp_bbr2.c - create mode 100644 net/ipv4/tcp_plb.c + .gitignore | 1 + + Documentation/admin-guide/pm/amd-pstate.rst | 19 + + Makefile | 2 + + arch/arc/configs/axs101_defconfig | 1 + + arch/arc/configs/axs103_defconfig | 1 + + arch/arc/configs/axs103_smp_defconfig | 1 + + arch/arc/configs/haps_hs_defconfig | 1 + + arch/arc/configs/haps_hs_smp_defconfig | 1 + + arch/arc/configs/hsdk_defconfig | 1 + + arch/arc/configs/nsim_700_defconfig | 1 + + arch/arc/configs/nsimosci_defconfig | 1 + + arch/arc/configs/nsimosci_hs_defconfig | 1 + + arch/arc/configs/nsimosci_hs_smp_defconfig | 1 + + arch/arc/configs/tb10x_defconfig | 1 + + arch/arc/configs/vdk_hs38_defconfig | 1 + + arch/arc/configs/vdk_hs38_smp_defconfig | 1 + + arch/x86/Kconfig.cpu | 332 +++++- + arch/x86/Makefile | 40 +- + arch/x86/Makefile.postlink | 41 + + arch/x86/boot/compressed/Makefile | 10 +- + arch/x86/include/asm/msr-index.h | 7 + + arch/x86/include/asm/topology.h | 1 + + arch/x86/include/asm/vdso/processor.h | 2 +- + arch/x86/include/asm/vermagic.h | 66 ++ + arch/x86/kernel/alternative.c | 2 + + arch/x86/kernel/cpu/intel_epb.c | 4 + + arch/x86/kernel/cpu/microcode/core.c | 40 +- + arch/x86/kernel/cpu/microcode/intel.c | 14 +- + arch/x86/kernel/itmt.c | 29 +- + arch/x86/kernel/msr.c | 2 +- + arch/x86/kernel/tsc.c | 3 + + arch/x86/mm/fault.c | 4 +- + block/bfq-cgroup.c | 5 - + block/bfq-iosched.c | 20 +- + block/bfq-iosched.h | 18 +- + block/bfq-wf2q.c | 9 +- + block/blk-core.c | 3 + + block/elevator.c | 7 +- + drivers/Makefile | 15 +- + drivers/acpi/cppc_acpi.c | 131 ++- + drivers/ata/libahci.c | 4 +- + drivers/base/arch_topology.c | 2 +- + drivers/base/firmware_loader/main.c | 2 + + drivers/block/zram/Kconfig | 18 + + drivers/block/zram/zram_drv.c | 39 + + drivers/cpufreq/amd-pstate.c | 1003 ++++++++++++++++++- + drivers/cpufreq/cppc_cpufreq.c | 2 +- + drivers/cpufreq/intel_pstate.c | 7 + + drivers/idle/intel_idle.c | 44 +- + drivers/input/serio/i8042.c | 10 +- + drivers/md/dm-crypt.c | 5 + + drivers/net/dummy.c | 2 +- + drivers/nvme/host/core.c | 2 +- + drivers/pci/pci.c | 2 +- + drivers/powercap/intel_rapl_common.c | 2 +- + drivers/thermal/intel/intel_powerclamp.c | 10 + + fs/xattr.c | 15 +- + include/acpi/cppc_acpi.h | 17 + + include/linux/ipc_namespace.h | 5 +- + include/linux/jbd2.h | 2 +- + include/linux/page_counter.h | 1 + + include/linux/pagemap.h | 2 +- + include/linux/sched.h | 1 + + include/linux/user_namespace.h | 4 + + include/linux/wait.h | 2 + + include/uapi/linux/if_bonding.h | 2 +- + init/Kconfig | 26 + + init/do_mounts.c | 16 +- + kernel/Kconfig.hz | 24 + + kernel/fork.c | 14 + + kernel/locking/rwsem.c | 4 +- + kernel/module/internal.h | 2 + + kernel/module/main.c | 1 + + kernel/module/procfs.c | 13 + + kernel/module/signing.c | 4 + + kernel/sched/core.c | 19 +- + kernel/sched/debug.c | 1 + + kernel/sched/fair.c | 20 +- + kernel/sched/wait.c | 24 + + kernel/sysctl.c | 22 + + kernel/user_namespace.c | 7 + + kernel/watchdog.c | 2 +- + lib/Kconfig | 8 +- + lib/raid6/algos.c | 4 +- + lib/string.c | 62 +- + lib/zstd/Makefile | 16 +- + lib/zstd/common/entropy_common.c | 4 +- + lib/zstd/common/zstd_common.c | 7 + + lib/zstd/compress/zstd_double_fast.c | 61 +- + lib/zstd/compress/zstd_fast.c | 69 +- + lib/zstd/compress/zstd_lazy.c | 223 ++--- + mm/compaction.c | 6 +- + mm/ksm.c | 11 +- + mm/memcontrol.c | 2 +- + mm/page-writeback.c | 8 + + mm/page_alloc.c | 5 +- + mm/swap.c | 5 + + mm/vmpressure.c | 4 + + mm/vmscan.c | 4 + + net/ipv4/inet_connection_sock.c | 2 +- + net/ipv4/tcp.c | 4 +- + 101 files changed, 2400 insertions(+), 349 deletions(-) + create mode 100644 arch/x86/Makefile.postlink -diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst -index a759872a2883..f2372f8f860b 100644 ---- a/Documentation/networking/ip-sysctl.rst -+++ b/Documentation/networking/ip-sysctl.rst -@@ -1040,6 +1040,64 @@ tcp_challenge_ack_limit - INTEGER - TCP stack implements per TCP socket limits anyway. - Default: INT_MAX (unlimited) +diff --git a/.gitignore b/.gitignore +index 265959544978..cd4ef88584ea 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -37,6 +37,7 @@ + *.o + *.o.* + *.patch ++*.relocs + *.s + *.so + *.so.dbg +diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst +index 83b58eb4ab4d..d0f0e115013b 100644 +--- a/Documentation/admin-guide/pm/amd-pstate.rst ++++ b/Documentation/admin-guide/pm/amd-pstate.rst +@@ -261,6 +261,25 @@ lowest non-linear performance in `AMD CPPC Performance Capability + `_.) + This attribute is read-only. -+tcp_plb_enabled - BOOLEAN -+ If set, TCP PLB (Protective Load Balancing) is enabled. PLB is -+ described in the following paper: -+ https://doi.org/10.1145/3544216.3544226. Based on PLB parameters, -+ upon sensing sustained congestion, TCP triggers a change in -+ flow label field for outgoing IPv6 packets. A change in flow label -+ field potentially changes the path of outgoing packets for switches -+ that use ECMP/WCMP for routing. ++``energy_performance_available_preferences`` + -+ Default: 0 ++All the supported EPP preference could be selected, List of the strings that ++can be set to the ``energy_performance_preference`` attribute ++those different profiles represent different energy vs efficiency hints provided ++to low-level firmware ++however, the ``default`` represents the epp value is set by platform firmware ++This attribute is read-only. + -+tcp_plb_cong_thresh - INTEGER -+ Fraction of packets marked with congestion over a round (RTT) to -+ tag that round as congested. This is referred to as K in the PLB paper: -+ https://doi.org/10.1145/3544216.3544226. ++``energy_performance_preference`` + -+ The 0-1 fraction range is mapped to 0-256 range to avoid floating -+ point operations. For example, 128 means that if at least 50% of -+ the packets in a round were marked as congested then the round -+ will be tagged as congested. ++The current energy performance preference can be read from this attribute. ++and user can change current preference according to energy or performance needs ++Please get all support profiles list from ++``energy_performance_available_preferences`` attribute, all the profiles are ++integer values defined between 0 to 255 when EPP feature is enabled by platform ++firmware, if EPP feature is disabled, driver will ignore the written value ++This attribute is read-write. + -+ Possible Values: 0 - 256 + Other performance and frequency values can be read back from + ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. + +diff --git a/Makefile b/Makefile +index 647a42a1f800..5c327c29ef12 100644 +--- a/Makefile ++++ b/Makefile +@@ -758,6 +758,8 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) + + ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE + KBUILD_CFLAGS += -O2 ++else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 ++KBUILD_CFLAGS += -O3 + else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE + KBUILD_CFLAGS += -Os + endif +diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig +index e31a8ebc3ecc..0016149f9583 100644 +--- a/arch/arc/configs/axs101_defconfig ++++ b/arch/arc/configs/axs101_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig +index e0e8567f0d75..5b031582a1cf 100644 +--- a/arch/arc/configs/axs103_defconfig ++++ b/arch/arc/configs/axs103_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig +index fcbc952bc75b..d4eec39e0112 100644 +--- a/arch/arc/configs/axs103_smp_defconfig ++++ b/arch/arc/configs/axs103_smp_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig +index d87ad7e88d62..7337cdf4ffdd 100644 +--- a/arch/arc/configs/haps_hs_defconfig ++++ b/arch/arc/configs/haps_hs_defconfig +@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EXPERT=y + CONFIG_PERF_EVENTS=y + # CONFIG_COMPAT_BRK is not set +diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig +index 8d82cdb7f86a..bc927221afc0 100644 +--- a/arch/arc/configs/haps_hs_smp_defconfig ++++ b/arch/arc/configs/haps_hs_smp_defconfig +@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig +index f856b03e0fb5..aa000075a575 100644 +--- a/arch/arc/configs/hsdk_defconfig ++++ b/arch/arc/configs/hsdk_defconfig +@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y + CONFIG_BLK_DEV_RAM=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig +index a1ce12bf5b16..326f6cde7826 100644 +--- a/arch/arc/configs/nsim_700_defconfig ++++ b/arch/arc/configs/nsim_700_defconfig +@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y +diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig +index ca10f4a2c823..bf39a0091679 100644 +--- a/arch/arc/configs/nsimosci_defconfig ++++ b/arch/arc/configs/nsimosci_defconfig +@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y +diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig +index 31b6ec3683c6..7121bd71c543 100644 +--- a/arch/arc/configs/nsimosci_hs_defconfig ++++ b/arch/arc/configs/nsimosci_hs_defconfig +@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y +diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig +index 41a0037f48a5..f9863b294a70 100644 +--- a/arch/arc/configs/nsimosci_hs_smp_defconfig ++++ b/arch/arc/configs/nsimosci_hs_smp_defconfig +@@ -8,6 +8,7 @@ CONFIG_IKCONFIG_PROC=y + # CONFIG_UTS_NS is not set + # CONFIG_PID_NS is not set + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_PERF_EVENTS=y + # CONFIG_COMPAT_BRK is not set + CONFIG_KPROBES=y +diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig +index d93b65008d4a..a12656ec0072 100644 +--- a/arch/arc/configs/tb10x_defconfig ++++ b/arch/arc/configs/tb10x_defconfig +@@ -14,6 +14,7 @@ CONFIG_INITRAMFS_SOURCE="../tb10x-rootfs.cpio" + CONFIG_INITRAMFS_ROOT_UID=2100 + CONFIG_INITRAMFS_ROOT_GID=501 + # CONFIG_RD_GZIP is not set ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_KALLSYMS_ALL=y + # CONFIG_AIO is not set + CONFIG_EMBEDDED=y +diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig +index 0c3b21416819..d7c858df520c 100644 +--- a/arch/arc/configs/vdk_hs38_defconfig ++++ b/arch/arc/configs/vdk_hs38_defconfig +@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y + CONFIG_IKCONFIG=y + CONFIG_IKCONFIG_PROC=y + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig +index f9ad9d3ee702..015c1d43889e 100644 +--- a/arch/arc/configs/vdk_hs38_smp_defconfig ++++ b/arch/arc/configs/vdk_hs38_smp_defconfig +@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y + CONFIG_IKCONFIG=y + CONFIG_IKCONFIG_PROC=y + CONFIG_BLK_DEV_INITRD=y ++CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y + CONFIG_EMBEDDED=y + CONFIG_PERF_EVENTS=y + # CONFIG_VM_EVENT_COUNTERS is not set +diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu +index 542377cd419d..22b919cdb6d1 100644 +--- a/arch/x86/Kconfig.cpu ++++ b/arch/x86/Kconfig.cpu +@@ -157,7 +157,7 @@ config MPENTIUM4 + + + config MK6 +- bool "K6/K6-II/K6-III" ++ bool "AMD K6/K6-II/K6-III" + depends on X86_32 + help + Select this for an AMD K6-family processor. Enables use of +@@ -165,7 +165,7 @@ config MK6 + flags to GCC. + + config MK7 +- bool "Athlon/Duron/K7" ++ bool "AMD Athlon/Duron/K7" + depends on X86_32 + help + Select this for an AMD Athlon K7-family processor. Enables use of +@@ -173,12 +173,98 @@ config MK7 + flags to GCC. + + config MK8 +- bool "Opteron/Athlon64/Hammer/K8" ++ bool "AMD Opteron/Athlon64/Hammer/K8" + help + Select this for an AMD Opteron or Athlon64 Hammer-family processor. + Enables use of some extended instructions, and passes appropriate + optimization flags to GCC. + ++config MK8SSE3 ++ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" ++ help ++ Select this for improved AMD Opteron or Athlon64 Hammer-family processors. ++ Enables use of some extended instructions, and passes appropriate ++ optimization flags to GCC. + -+ Default: 128 ++config MK10 ++ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" ++ help ++ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, ++ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. ++ Enables use of some extended instructions, and passes appropriate ++ optimization flags to GCC. + -+tcp_plb_idle_rehash_rounds - INTEGER -+ Number of consecutive congested rounds (RTT) seen after which -+ a rehash can be performed, given there are no packets in flight. -+ This is referred to as M in PLB paper: -+ https://doi.org/10.1145/3544216.3544226. ++config MBARCELONA ++ bool "AMD Barcelona" ++ help ++ Select this for AMD Family 10h Barcelona processors. + -+ Possible Values: 0 - 31 ++ Enables -march=barcelona + -+ Default: 3 ++config MBOBCAT ++ bool "AMD Bobcat" ++ help ++ Select this for AMD Family 14h Bobcat processors. + -+tcp_plb_rehash_rounds - INTEGER -+ Number of consecutive congested rounds (RTT) seen after which -+ a forced rehash can be performed. Be careful when setting this -+ parameter, as a small value increases the risk of retransmissions. -+ This is referred to as N in PLB paper: -+ https://doi.org/10.1145/3544216.3544226. ++ Enables -march=btver1 + -+ Possible Values: 0 - 31 ++config MJAGUAR ++ bool "AMD Jaguar" ++ help ++ Select this for AMD Family 16h Jaguar processors. + -+ Default: 12 ++ Enables -march=btver2 + -+tcp_plb_suspend_rto_sec - INTEGER -+ Time, in seconds, to suspend PLB in event of an RTO. In order to avoid -+ having PLB repath onto a connectivity "black hole", after an RTO a TCP -+ connection suspends PLB repathing for a random duration between 1x and -+ 2x of this parameter. Randomness is added to avoid concurrent rehashing -+ of multiple TCP connections. This should be set corresponding to the -+ amount of time it takes to repair a failed link. ++config MBULLDOZER ++ bool "AMD Bulldozer" ++ help ++ Select this for AMD Family 15h Bulldozer processors. + -+ Possible Values: 0 - 255 ++ Enables -march=bdver1 + -+ Default: 60 ++config MPILEDRIVER ++ bool "AMD Piledriver" ++ help ++ Select this for AMD Family 15h Piledriver processors. + - UDP variables - ============= - -diff --git a/include/linux/tcp.h b/include/linux/tcp.h -index a9fbe22732c3..4034e46093b4 100644 ---- a/include/linux/tcp.h -+++ b/include/linux/tcp.h -@@ -255,7 +255,8 @@ struct tcp_sock { - u8 compressed_ack; - u8 dup_ack_counter:2, - tlp_retrans:1, /* TLP is a retransmission */ -- unused:5; -+ fast_ack_mode:2, /* which fast ack mode ? */ -+ unused:3; - u32 chrono_start; /* Start time in jiffies of a TCP chrono */ - u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ - u8 chrono_type:2, /* current chronograph type */ -@@ -443,6 +444,9 @@ struct tcp_sock { - */ - struct request_sock __rcu *fastopen_rsk; - struct saved_syn *saved_syn; ++ Enables -march=bdver2 + -+/* Rerouting information */ -+ u16 ecn_rehash; /* PLB triggered rehash attempts */ - }; - - enum tsq_enum { -diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h -index ee88f0f1350f..e3075b3f1ece 100644 ---- a/include/net/inet_connection_sock.h -+++ b/include/net/inet_connection_sock.h -@@ -132,7 +132,8 @@ struct inet_connection_sock { - u32 icsk_probes_tstamp; - u32 icsk_user_timeout; - -- u64 icsk_ca_priv[104 / sizeof(u64)]; -+/* XXX inflated by temporary internal debugging info */ -+ u64 icsk_ca_priv[224 / sizeof(u64)]; - #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) - }; ++config MSTEAMROLLER ++ bool "AMD Steamroller" ++ help ++ Select this for AMD Family 15h Steamroller processors. ++ ++ Enables -march=bdver3 ++ ++config MEXCAVATOR ++ bool "AMD Excavator" ++ help ++ Select this for AMD Family 15h Excavator processors. ++ ++ Enables -march=bdver4 ++ ++config MZEN ++ bool "AMD Zen" ++ help ++ Select this for AMD Family 17h Zen processors. ++ ++ Enables -march=znver1 ++ ++config MZEN2 ++ bool "AMD Zen 2" ++ help ++ Select this for AMD Family 17h Zen 2 processors. ++ ++ Enables -march=znver2 ++ ++config MZEN3 ++ bool "AMD Zen 3" ++ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ help ++ Select this for AMD Family 19h Zen 3 processors. ++ ++ Enables -march=znver3 ++ + config MCRUSOE + bool "Crusoe" + depends on X86_32 +@@ -270,7 +356,7 @@ config MPSC + in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. -diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h -index 6320a76cefdc..2e39e07ed41f 100644 ---- a/include/net/netns/ipv4.h -+++ b/include/net/netns/ipv4.h -@@ -181,6 +181,11 @@ struct netns_ipv4 { - unsigned long tfo_active_disable_stamp; - u32 tcp_challenge_timestamp; - u32 tcp_challenge_count; -+ u8 sysctl_tcp_plb_enabled; -+ int sysctl_tcp_plb_cong_thresh; -+ u8 sysctl_tcp_plb_idle_rehash_rounds; -+ u8 sysctl_tcp_plb_rehash_rounds; -+ u8 sysctl_tcp_plb_suspend_rto_sec; + config MCORE2 +- bool "Core 2/newer Xeon" ++ bool "Intel Core 2" + help - int sysctl_udp_wmem_min; - int sysctl_udp_rmem_min; -diff --git a/include/net/tcp.h b/include/net/tcp.h -index d10962b9f0d0..d3947345feaa 100644 ---- a/include/net/tcp.h -+++ b/include/net/tcp.h -@@ -372,6 +372,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk, - #define TCP_ECN_QUEUE_CWR 2 - #define TCP_ECN_DEMAND_CWR 4 - #define TCP_ECN_SEEN 8 -+#define TCP_ECN_ECT_PERMANENT 16 + Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and +@@ -278,6 +364,8 @@ config MCORE2 + family in /proc/cpuinfo. Newer ones have 6 and older ones 15 + (not a typo) - enum tcp_tw_status { - TCP_TW_SUCCESS = 0, -@@ -816,6 +817,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) - return max_t(s64, t1 - t0, 0); - } ++ Enables -march=core2 ++ + config MATOM + bool "Intel Atom" + help +@@ -287,6 +375,182 @@ config MATOM + accordingly optimized code. Use a recent GCC with specific Atom + support in order to fully benefit from selecting this option. -+static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) -+{ -+ return max_t(s32, t1 - t0, 0); -+} ++config MNEHALEM ++ bool "Intel Nehalem" ++ select X86_P6_NOP ++ help + - static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) - { - return tcp_ns_to_ts(skb->skb_mstamp_ns); -@@ -891,9 +897,14 @@ struct tcp_skb_cb { - /* pkts S/ACKed so far upon tx of skb, incl retrans: */ - __u32 delivered; - /* start of send pipeline phase */ -- u64 first_tx_mstamp; -+ u32 first_tx_mstamp; - /* when we reached the "delivered" count */ -- u64 delivered_mstamp; -+ u32 delivered_mstamp; -+#define TCPCB_IN_FLIGHT_BITS 20 -+#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) -+ u32 in_flight:20, /* packets in flight at transmit */ -+ unused2:12; -+ u32 lost; /* packets lost so far upon tx of skb */ - } tx; /* only used for outgoing skbs */ - union { - struct inet_skb_parm h4; -@@ -1019,7 +1030,11 @@ enum tcp_ca_ack_event_flags { - #define TCP_CONG_NON_RESTRICTED 0x1 - /* Requires ECN/ECT set on all packets */ - #define TCP_CONG_NEEDS_ECN 0x2 --#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) -+/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ -+#define TCP_CONG_WANTS_CE_EVENTS 0x4 -+#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ -+ TCP_CONG_NEEDS_ECN | \ -+ TCP_CONG_WANTS_CE_EVENTS) - - union tcp_cc_info; - -@@ -1039,8 +1054,11 @@ struct ack_sample { - */ - struct rate_sample { - u64 prior_mstamp; /* starting timestamp for interval */ -+ u32 prior_lost; /* tp->lost at "prior_mstamp" */ - u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ - u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ -+ u32 tx_in_flight; /* packets in flight at starting timestamp */ -+ s32 lost; /* number of packets lost over interval */ - s32 delivered; /* number of packets delivered over interval */ - s32 delivered_ce; /* number of packets delivered w/ CE marks*/ - long interval_us; /* time for tp->delivered to incr "delivered" */ -@@ -1054,6 +1072,7 @@ struct rate_sample { - bool is_app_limited; /* is sample from packet with bubble in pipe? */ - bool is_retrans; /* is sample from retransmission? */ - bool is_ack_delayed; /* is this (likely) a delayed ACK? */ -+ bool is_ece; /* did this ACK have ECN marked? */ - }; - - struct tcp_congestion_ops { -@@ -1077,8 +1096,11 @@ struct tcp_congestion_ops { - /* hook for packet ack accounting (optional) */ - void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - -- /* override sysctl_tcp_min_tso_segs */ -- u32 (*min_tso_segs)(struct sock *sk); -+ /* pick target number of segments per TSO/GSO skb (optional): */ -+ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); ++ Select this for 1st Gen Core processors in the Nehalem family. + -+ /* react to a specific lost skb (optional) */ -+ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); - - /* call when packets are delivered to update cwnd and pacing rate, - * after all the ca_state processing. (optional) -@@ -1141,6 +1163,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) - } - #endif - -+static inline bool tcp_ca_wants_ce_events(const struct sock *sk) -+{ -+ const struct inet_connection_sock *icsk = inet_csk(sk); ++ Enables -march=nehalem + -+ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | -+ TCP_CONG_WANTS_CE_EVENTS); -+} ++config MWESTMERE ++ bool "Intel Westmere" ++ select X86_P6_NOP ++ help + - static inline bool tcp_ca_needs_ecn(const struct sock *sk) - { - const struct inet_connection_sock *icsk = inet_csk(sk); -@@ -1160,6 +1190,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) - void tcp_set_ca_state(struct sock *sk, const u8 ca_state); - - /* From tcp_rate.c */ -+void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); - void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); - void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - struct rate_sample *rs); -@@ -2130,6 +2161,23 @@ extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, - extern void tcp_rack_reo_timeout(struct sock *sk); - extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); - -+/* tcp_plb.c */ ++ Select this for the Intel Westmere formerly Nehalem-C family. + -+#define TCP_PLB_SCALE 8 /* scaling factor for fractions in PLB (e.g. ce_ratio) */ ++ Enables -march=westmere + -+/* State for PLB (Protective Load Balancing) for a single TCP connection. */ -+struct tcp_plb_state { -+ u8 consec_cong_rounds:5, /* consecutive congested rounds */ -+ enabled:1, /* Check if PLB is enabled */ -+ unused:2; -+ u32 pause_until; /* jiffies32 when PLB can resume repathing */ -+}; ++config MSILVERMONT ++ bool "Intel Silvermont" ++ select X86_P6_NOP ++ help + -+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, -+ const int cong_ratio); -+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); -+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); ++ Select this for the Intel Silvermont platform. + - /* At how many usecs into the future should the RTO fire? */ - static inline s64 tcp_rto_delta_us(const struct sock *sk) - { -diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h -index 50655de04c9b..0e24f11627d5 100644 ---- a/include/uapi/linux/inet_diag.h -+++ b/include/uapi/linux/inet_diag.h -@@ -231,9 +231,42 @@ struct tcp_bbr_info { - __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ - }; - -+/* Phase as reported in netlink/ss stats. */ -+enum tcp_bbr2_phase { -+ BBR2_PHASE_INVALID = 0, -+ BBR2_PHASE_STARTUP = 1, -+ BBR2_PHASE_DRAIN = 2, -+ BBR2_PHASE_PROBE_RTT = 3, -+ BBR2_PHASE_PROBE_BW_UP = 4, -+ BBR2_PHASE_PROBE_BW_DOWN = 5, -+ BBR2_PHASE_PROBE_BW_CRUISE = 6, -+ BBR2_PHASE_PROBE_BW_REFILL = 7 -+}; ++ Enables -march=silvermont + -+struct tcp_bbr2_info { -+ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ -+ __u32 bbr_bw_lsb; /* lower 32 bits of bw */ -+ __u32 bbr_bw_msb; /* upper 32 bits of bw */ -+ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ -+ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ -+ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ -+ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ -+ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ -+ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ -+ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ -+ __u8 bbr_mode; /* current bbr_mode in state machine */ -+ __u8 bbr_phase; /* current state machine phase */ -+ __u8 unused1; /* alignment padding; not used yet */ -+ __u8 bbr_version; /* MUST be at this offset in struct */ -+ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ -+ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ -+ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ -+}; ++config MGOLDMONT ++ bool "Intel Goldmont" ++ select X86_P6_NOP ++ help + - union tcp_cc_info { - struct tcpvegas_info vegas; - struct tcp_dctcp_info dctcp; - struct tcp_bbr_info bbr; -+ struct tcp_bbr2_info bbr2; - }; - #endif /* _UAPI_INET_DIAG_H_ */ -diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h -index 4d7470036a8b..8ce035f1c874 100644 ---- a/include/uapi/linux/snmp.h -+++ b/include/uapi/linux/snmp.h -@@ -292,6 +292,7 @@ enum - LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */ - LINUX_MIB_TCPMIGRATEREQSUCCESS, /* TCPMigrateReqSuccess */ - LINUX_MIB_TCPMIGRATEREQFAILURE, /* TCPMigrateReqFailure */ -+ LINUX_MIB_TCPECNREHASH, /* TCPECNRehash */ - __LINUX_MIB_MAX - }; - -diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig -index e983bb0c5012..d172ffbdf27f 100644 ---- a/net/ipv4/Kconfig -+++ b/net/ipv4/Kconfig -@@ -668,6 +668,24 @@ config TCP_CONG_BBR - AQM schemes that do not provide a delay signal. It requires the fq - ("Fair Queue") pacing packet scheduler. - -+config TCP_CONG_BBR2 -+ tristate "BBR2 TCP" -+ default n ++ Select this for the Intel Goldmont platform including Apollo Lake and Denverton. ++ ++ Enables -march=goldmont ++ ++config MGOLDMONTPLUS ++ bool "Intel Goldmont Plus" ++ select X86_P6_NOP + help + -+ BBR2 TCP congestion control is a model-based congestion control -+ algorithm that aims to maximize network utilization, keep queues and -+ retransmit rates low, and to be able to coexist with Reno/CUBIC in -+ common scenarios. It builds an explicit model of the network path. It -+ tolerates a targeted degree of random packet loss and delay that are -+ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, -+ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can -+ coexist with flows that use loss-based congestion control, and can -+ operate with shallow buffers, deep buffers, bufferbloat, policers, or -+ AQM schemes that do not provide a delay signal. It requires pacing, -+ using either TCP internal pacing or the fq ("Fair Queue") pacing packet -+ scheduler. ++ Select this for the Intel Goldmont Plus platform including Gemini Lake. + - choice - prompt "Default TCP congestion control" - default DEFAULT_CUBIC -@@ -705,6 +723,9 @@ choice - config DEFAULT_BBR - bool "BBR" if TCP_CONG_BBR=y - -+ config DEFAULT_BBR2 -+ bool "BBR2" if TCP_CONG_BBR2=y ++ Enables -march=goldmont-plus + - config DEFAULT_RENO - bool "Reno" - endchoice -@@ -729,6 +750,7 @@ config DEFAULT_TCP_CONG - default "dctcp" if DEFAULT_DCTCP - default "cdg" if DEFAULT_CDG - default "bbr" if DEFAULT_BBR -+ default "bbr2" if DEFAULT_BBR2 - default "cubic" - - config TCP_MD5SIG -diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile -index bbdd9c44f14e..e7a86a50838a 100644 ---- a/net/ipv4/Makefile -+++ b/net/ipv4/Makefile -@@ -10,7 +10,7 @@ obj-y := route.o inetpeer.o protocol.o \ - tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \ - tcp_minisocks.o tcp_cong.o tcp_metrics.o tcp_fastopen.o \ - tcp_rate.o tcp_recovery.o tcp_ulp.o \ -- tcp_offload.o datagram.o raw.o udp.o udplite.o \ -+ tcp_offload.o tcp_plb.o datagram.o raw.o udp.o udplite.o \ - udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \ - fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \ - inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \ -@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o - obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o - obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o - obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o -+obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o - obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o - obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o - obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o -diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c -index 85a9e500c42d..24fcbac984fe 100644 ---- a/net/ipv4/bpf_tcp_ca.c -+++ b/net/ipv4/bpf_tcp_ca.c -@@ -14,6 +14,18 @@ - /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ - extern struct bpf_struct_ops bpf_tcp_congestion_ops; - -+static u32 optional_ops[] = { -+ offsetof(struct tcp_congestion_ops, init), -+ offsetof(struct tcp_congestion_ops, release), -+ offsetof(struct tcp_congestion_ops, set_state), -+ offsetof(struct tcp_congestion_ops, cwnd_event), -+ offsetof(struct tcp_congestion_ops, in_ack_event), -+ offsetof(struct tcp_congestion_ops, pkts_acked), -+ offsetof(struct tcp_congestion_ops, tso_segs), -+ offsetof(struct tcp_congestion_ops, sndbuf_expand), -+ offsetof(struct tcp_congestion_ops, cong_control), -+}; ++config MSANDYBRIDGE ++ bool "Intel Sandy Bridge" ++ select X86_P6_NOP ++ help + - static u32 unsupported_ops[] = { - offsetof(struct tcp_congestion_ops, get_info), - }; -diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c -index 0088a4c64d77..e0a664b467e0 100644 ---- a/net/ipv4/proc.c -+++ b/net/ipv4/proc.c -@@ -297,6 +297,7 @@ static const struct snmp_mib snmp4_net_list[] = { - SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS), - SNMP_MIB_ITEM("TCPMigrateReqSuccess", LINUX_MIB_TCPMIGRATEREQSUCCESS), - SNMP_MIB_ITEM("TCPMigrateReqFailure", LINUX_MIB_TCPMIGRATEREQFAILURE), -+ SNMP_MIB_ITEM("TCPECNRehash", LINUX_MIB_TCPECNREHASH), - SNMP_MIB_SENTINEL - }; - -diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c -index 5490c285668b..ed35a9485e71 100644 ---- a/net/ipv4/sysctl_net_ipv4.c -+++ b/net/ipv4/sysctl_net_ipv4.c -@@ -39,6 +39,8 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ; - static int one_day_secs = 24 * 3600; - static u32 fib_multipath_hash_fields_all_mask __maybe_unused = - FIB_MULTIPATH_HASH_FIELD_ALL_MASK; -+static int tcp_plb_max_rounds = 31; -+static int tcp_plb_max_cong_thresh = 256; - - /* obsolete */ - static int sysctl_tcp_low_latency __read_mostly; -@@ -1346,6 +1348,47 @@ static struct ctl_table ipv4_net_table[] = { - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_TWO, - }, -+ { -+ .procname = "tcp_plb_enabled", -+ .data = &init_net.ipv4.sysctl_tcp_plb_enabled, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "tcp_plb_cong_thresh", -+ .data = &init_net.ipv4.sysctl_tcp_plb_cong_thresh, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &tcp_plb_max_cong_thresh, -+ }, -+ { -+ .procname = "tcp_plb_idle_rehash_rounds", -+ .data = &init_net.ipv4.sysctl_tcp_plb_idle_rehash_rounds, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra2 = &tcp_plb_max_rounds, -+ }, -+ { -+ .procname = "tcp_plb_rehash_rounds", -+ .data = &init_net.ipv4.sysctl_tcp_plb_rehash_rounds, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra2 = &tcp_plb_max_rounds, -+ }, -+ { -+ .procname = "tcp_plb_suspend_rto_sec", -+ .data = &init_net.ipv4.sysctl_tcp_plb_suspend_rto_sec, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ }, - { } - }; - -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index e373dde1f46f..b735776c0225 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -3187,6 +3187,7 @@ int tcp_disconnect(struct sock *sk, int flags) - tp->rx_opt.dsack = 0; - tp->rx_opt.num_sacks = 0; - tp->rcv_ooopack = 0; -+ tp->fast_ack_mode = 0; - - - /* Clean up fastopen related fields */ -diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c -index 54eec33c6e1c..bfbf158c71f4 100644 ---- a/net/ipv4/tcp_bbr.c -+++ b/net/ipv4/tcp_bbr.c -@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) - sk->sk_pacing_rate = rate; - } - --/* override sysctl_tcp_min_tso_segs */ - static u32 bbr_min_tso_segs(struct sock *sk) - { - return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; - } - -+/* Return the number of segments BBR would like in a TSO/GSO skb, given -+ * a particular max gso size as a constraint. -+ */ -+static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, -+ u32 gso_max_size) -+{ -+ u32 segs; -+ u64 bytes; ++ Select this for 2nd Gen Core processors in the Sandy Bridge family. + -+ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ -+ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; ++ Enables -march=sandybridge + -+ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); -+ return segs; -+} ++config MIVYBRIDGE ++ bool "Intel Ivy Bridge" ++ select X86_P6_NOP ++ help + -+/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ -+static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) -+{ -+ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); -+} ++ Select this for 3rd Gen Core processors in the Ivy Bridge family. + -+/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ - static u32 bbr_tso_segs_goal(struct sock *sk) - { - struct tcp_sock *tp = tcp_sk(sk); -- u32 segs, bytes; -- -- /* Sort of tcp_tso_autosize() but ignoring -- * driver provided sk_gso_max_size. -- */ -- bytes = min_t(unsigned long, -- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), -- GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); -- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - -- return min(segs, 0x7FU); -+ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); - } - - /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { - .undo_cwnd = bbr_undo_cwnd, - .cwnd_event = bbr_cwnd_event, - .ssthresh = bbr_ssthresh, -- .min_tso_segs = bbr_min_tso_segs, -+ .tso_segs = bbr_tso_segs, - .get_info = bbr_get_info, - .set_state = bbr_set_state, - }; -diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c -new file mode 100644 -index 000000000000..2e39f7a353be ---- /dev/null -+++ b/net/ipv4/tcp_bbr2.c -@@ -0,0 +1,2692 @@ -+/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 -+ * -+ * BBRv2 is a model-based congestion control algorithm that aims for low -+ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model -+ * of the network path, it uses measurements of bandwidth and RTT, as well as -+ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that -+ * although it can use ECN or loss signals explicitly, it does not require -+ * either; it can bound its in-flight data based on its estimate of the BDP. -+ * -+ * The model has both higher and lower bounds for the operating range: -+ * lo: bw_lo, inflight_lo: conservative short-term lower bound -+ * hi: bw_hi, inflight_hi: robust long-term upper bound -+ * The bandwidth-probing time scale is (a) extended dynamically based on -+ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by -+ * an interactive wall-clock time-scale to be more scalable and responsive -+ * than Reno and CUBIC. -+ * -+ * Here is a state transition diagram for BBR: -+ * -+ * | -+ * V -+ * +---> STARTUP ----+ -+ * | | | -+ * | V | -+ * | DRAIN ----+ -+ * | | | -+ * | V | -+ * +---> PROBE_BW ----+ -+ * | ^ | | -+ * | | | | -+ * | +----+ | -+ * | | -+ * +---- PROBE_RTT <--+ -+ * -+ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. -+ * When it estimates the pipe is full, it enters DRAIN to drain the queue. -+ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. -+ * A long-lived BBR flow spends the vast majority of its time remaining -+ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth -+ * in a fair manner, with a small, bounded queue. *If* a flow has been -+ * continuously sending for the entire min_rtt window, and hasn't seen an RTT -+ * sample that matches or decreases its min_rtt estimate for 10 seconds, then -+ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe -+ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if -+ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; -+ * otherwise we enter STARTUP to try to fill the pipe. -+ * -+ * BBR is described in detail in: -+ * "BBR: Congestion-Based Congestion Control", -+ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, -+ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. -+ * -+ * There is a public e-mail list for discussing BBR development and testing: -+ * https://groups.google.com/forum/#!forum/bbr-dev -+ * -+ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, -+ * otherwise TCP stack falls back to an internal pacing using one high -+ * resolution timer per TCP socket and may use more resources. -+ */ -+#include -+#include -+#include -+#include -+#include ++ Enables -march=ivybridge + -+#include "tcp_dctcp.h" ++config MHASWELL ++ bool "Intel Haswell" ++ select X86_P6_NOP ++ help + -+/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth -+ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. -+ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. -+ * Since the minimum window is >=4 packets, the lower bound isn't -+ * an issue. The upper bound isn't an issue with existing technologies. -+ */ -+#define BW_SCALE 24 -+#define BW_UNIT (1 << BW_SCALE) ++ Select this for 4th Gen Core processors in the Haswell family. + -+#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ -+#define BBR_UNIT (1 << BBR_SCALE) ++ Enables -march=haswell + -+#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ -+#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ ++config MBROADWELL ++ bool "Intel Broadwell" ++ select X86_P6_NOP ++ help + -+#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ ++ Select this for 5th Gen Core processors in the Broadwell family. + -+/* BBR has the following modes for deciding how fast to send: */ -+enum bbr_mode { -+ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ -+ BBR_DRAIN, /* drain any queue created during startup */ -+ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ -+ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ -+}; ++ Enables -march=broadwell + -+/* How does the incoming ACK stream relate to our bandwidth probing? */ -+enum bbr_ack_phase { -+ BBR_ACKS_INIT, /* not probing; not getting probe feedback */ -+ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ -+ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ -+ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ -+ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ -+}; -+ -+/* BBR congestion control block */ -+struct bbr { -+ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ -+ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ -+ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ -+ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ -+ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ -+ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ -+ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ -+ u64 cycle_mstamp; /* time of this cycle phase start */ -+ u32 mode:3, /* current bbr_mode in state machine */ -+ prev_ca_state:3, /* CA state on previous ACK */ -+ packet_conservation:1, /* use packet conservation? */ -+ round_start:1, /* start of packet-timed tx->ack round? */ -+ ce_state:1, /* If most recent data has CE bit set */ -+ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ -+ try_fast_path:1, /* can we take fast path? */ -+ unused2:11, -+ idle_restart:1, /* restarting after idle? */ -+ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ -+ cycle_idx:3, /* current index in pacing_gain cycle array */ -+ has_seen_rtt:1; /* have we seen an RTT sample yet? */ -+ u32 pacing_gain:11, /* current gain for setting pacing rate */ -+ cwnd_gain:11, /* current gain for setting cwnd */ -+ full_bw_reached:1, /* reached full bw in Startup? */ -+ full_bw_cnt:2, /* number of rounds without large bw gains */ -+ init_cwnd:7; /* initial cwnd */ -+ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ -+ u32 full_bw; /* recent bw, to estimate if pipe is full */ -+ -+ /* For tracking ACK aggregation: */ -+ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ -+ u16 extra_acked[2]; /* max excess data ACKed in epoch */ -+ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ -+ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ -+ extra_acked_win_idx:1, /* current index in extra_acked array */ -+ /* BBR v2 state: */ -+ unused1:2, -+ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ -+ loss_in_cycle:1, /* packet loss in this cycle? */ -+ ecn_in_cycle:1; /* ECN in this cycle? */ -+ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ -+ u32 undo_bw_lo; /* bw_lo before latest losses */ -+ u32 undo_inflight_lo; /* inflight_lo before latest losses */ -+ u32 undo_inflight_hi; /* inflight_hi before latest losses */ -+ u32 bw_latest; /* max delivered bw in last round trip */ -+ u32 bw_lo; /* lower bound on sending bandwidth */ -+ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ -+ u32 inflight_latest; /* max delivered data in last round trip */ -+ u32 inflight_lo; /* lower bound of inflight data range */ -+ u32 inflight_hi; /* upper bound of inflight data range */ -+ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ -+ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ -+ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ -+ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ -+ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ -+ bw_probe_samples:1, /* rate samples reflect bw probing? */ -+ prev_probe_too_high:1, /* did last PROBE_UP go too high? */ -+ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ -+ rounds_since_probe:8, /* packet-timed rounds since probed bw */ -+ loss_round_start:1, /* loss_round_delivered round trip? */ -+ loss_in_round:1, /* loss marked in this round trip? */ -+ ecn_in_round:1, /* ECN marked in this round trip? */ -+ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ -+ loss_events_in_round:4,/* losses in STARTUP round */ -+ initialized:1; /* has bbr_init() been called? */ -+ u32 alpha_last_delivered; /* tp->delivered at alpha update */ -+ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ -+ struct tcp_plb_state plb; -+ -+ /* Params configurable using setsockopt. Refer to correspoding -+ * module param for detailed description of params. -+ */ -+ struct bbr_params { -+ u32 high_gain:11, /* max allowed value: 2047 */ -+ drain_gain:10, /* max allowed value: 1023 */ -+ cwnd_gain:11; /* max allowed value: 2047 */ -+ u32 cwnd_min_target:4, /* max allowed value: 15 */ -+ min_rtt_win_sec:5, /* max allowed value: 31 */ -+ probe_rtt_mode_ms:9, /* max allowed value: 511 */ -+ full_bw_cnt:3, /* max allowed value: 7 */ -+ cwnd_tso_budget:1, /* allowed values: {0, 1} */ -+ unused3:6, -+ drain_to_target:1, /* boolean */ -+ precise_ece_ack:1, /* boolean */ -+ extra_acked_in_startup:1, /* allowed values: {0, 1} */ -+ fast_path:1; /* boolean */ -+ u32 full_bw_thresh:10, /* max allowed value: 1023 */ -+ startup_cwnd_gain:11, /* max allowed value: 2047 */ -+ bw_probe_pif_gain:9, /* max allowed value: 511 */ -+ usage_based_cwnd:1, /* boolean */ -+ unused2:1; -+ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ -+ refill_add_inc:2; /* max allowed value: 3 */ -+ u16 extra_acked_gain:11, /* max allowed value: 2047 */ -+ extra_acked_win_rtts:5; /* max allowed value: 31*/ -+ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ -+ /* Mostly BBR v2 parameters below here: */ -+ u32 ecn_alpha_gain:8, /* max allowed value: 255 */ -+ ecn_factor:8, /* max allowed value: 255 */ -+ ecn_thresh:8, /* max allowed value: 255 */ -+ beta:8; /* max allowed value: 255 */ -+ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ -+ bw_probe_reno_gain:9, /* max allowed value: 511 */ -+ full_loss_cnt:4; /* max allowed value: 15 */ -+ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ -+ inflight_headroom:8, /* max allowed value: 255 */ -+ loss_thresh:8, /* max allowed value: 255 */ -+ bw_probe_max_rounds:8; /* max allowed value: 255 */ -+ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ -+ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ -+ full_ecn_cnt:2; /* max allowed value: 3 */ -+ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ -+ undo:1, /* boolean */ -+ tso_rtt_shift:4, /* max allowed value: 15 */ -+ unused5:1; -+ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ -+ unused1:14, -+ ecn_alpha_init:9; /* max allowed value: 256 */ -+ } params; -+ -+ struct { -+ u32 snd_isn; /* Initial sequence number */ -+ u32 rs_bw; /* last valid rate sample bw */ -+ u32 target_cwnd; /* target cwnd, based on BDP */ -+ u8 undo:1, /* Undo even happened but not yet logged */ -+ unused:7; -+ char event; /* single-letter event debug codes */ -+ u16 unused2; -+ } debug; -+}; -+ -+struct bbr_context { -+ u32 sample_bw; -+ u32 target_cwnd; -+ u32 log:1; -+}; -+ -+/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ -+static u32 bbr_min_rtt_win_sec = 10; -+/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. -+ * Max allowed value is 511 (0x1FF). -+ */ -+static u32 bbr_probe_rtt_mode_ms = 200; -+/* Window length of probe_rtt_min_us filter (in ms), and consequently the -+ * typical interval between PROBE_RTT mode entries. -+ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC -+ */ -+static u32 bbr_probe_rtt_win_ms = 5000; -+/* Skip TSO below the following bandwidth (bits/sec): */ -+static int bbr_min_tso_rate = 1200000; -+ -+/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting -+ * in bigger TSO bursts. By default we cut the RTT-based allowance in half -+ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance -+ * is below 1500 bytes after 6 * ~500 usec = 3ms. -+ */ -+static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ -+ -+/* Select cwnd TSO budget approach: -+ * 0: padding -+ * 1: flooring -+ */ -+static uint bbr_cwnd_tso_budget = 1; ++config MSKYLAKE ++ bool "Intel Skylake" ++ select X86_P6_NOP ++ help + -+/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. -+ * In order to help drive the network toward lower queues and low latency while -+ * maintaining high utilization, the average pacing rate aims to be slightly -+ * lower than the estimated bandwidth. This is an important aspect of the -+ * design. -+ */ -+static const int bbr_pacing_margin_percent = 1; ++ Select this for 6th Gen Core processors in the Skylake family. + -+/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain -+ * that will allow a smoothly increasing pacing rate that will double each RTT -+ * and send the same number of packets per RTT that an un-paced, slow-starting -+ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). -+ */ -+static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; -+/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ -+static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; -+/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain -+ * the queue created in BBR_STARTUP in a single round. Max allowed value -+ * is 1023 (0x3FF). -+ */ -+static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; -+/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. -+ * Max allowed value is 2047 (0x7FF). -+ */ -+static int bbr_cwnd_gain = BBR_UNIT * 2; -+/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. -+ * Max allowed value for each element is 1023 (0x3FF). -+ */ -+enum bbr_pacing_gain_phase { -+ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ -+ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ -+ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ -+ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ -+}; -+static int bbr_pacing_gain[] = { -+ BBR_UNIT * 5 / 4, /* probe for more available bw */ -+ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ -+ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ -+ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ -+}; ++ Enables -march=skylake + -+/* Try to keep at least this many packets in flight, if things go smoothly. For -+ * smooth functioning, a sliding window protocol ACKing every other packet -+ * needs at least 4 packets in flight. Max allowed value is 15 (0xF). -+ */ -+static u32 bbr_cwnd_min_target = 4; ++config MSKYLAKEX ++ bool "Intel Skylake X" ++ select X86_P6_NOP ++ help + -+/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. -+ * Use 0 to disable. Max allowed value is 255. -+ */ -+static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; ++ Select this for 6th Gen Core processors in the Skylake X family. + -+/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ -+/* If bw has increased significantly (1.25x), there may be more bw available. -+ * Max allowed value is 1023 (0x3FF). -+ */ -+static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; -+/* But after 3 rounds w/o significant bw growth, estimate pipe is full. -+ * Max allowed value is 7 (0x7). -+ */ -+static u32 bbr_full_bw_cnt = 3; ++ Enables -march=skylake-avx512 + -+static u32 bbr_flags; /* Debugging related stuff */ ++config MCANNONLAKE ++ bool "Intel Cannon Lake" ++ select X86_P6_NOP ++ help + -+/* Whether to debug using printk. -+ */ -+static bool bbr_debug_with_printk; ++ Select this for 8th Gen Core processors + -+/* Whether to debug using ftrace event tcp:tcp_bbr_event. -+ * Ignored when bbr_debug_with_printk is set. -+ */ -+static bool bbr_debug_ftrace; ++ Enables -march=cannonlake + -+/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ -+static bool bbr_drain_to_target = true; /* default: enabled */ ++config MICELAKE ++ bool "Intel Ice Lake" ++ select X86_P6_NOP ++ help + -+/* Experiment: Flags to control BBR with ECN behavior. -+ */ -+static bool bbr_precise_ece_ack = true; /* default: enabled */ ++ Select this for 10th Gen Core processors in the Ice Lake family. + -+/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is -+ * (2^(16+14) B)/(1024 B/packet) = 1M packets. -+ */ -+static u32 bbr_cwnd_warn_val = 1U << 20; ++ Enables -march=icelake-client + -+static u16 bbr_debug_port_mask; ++config MCASCADELAKE ++ bool "Intel Cascade Lake" ++ select X86_P6_NOP ++ help + -+/* BBR module parameters. These are module parameters only in Google prod. -+ * Upstream these are intentionally not module parameters. -+ */ -+static int bbr_pacing_gain_size = CYCLE_LEN; ++ Select this for Xeon processors in the Cascade Lake family. + -+/* Gain factor for adding extra_acked to target cwnd: */ -+static int bbr_extra_acked_gain = 256; ++ Enables -march=cascadelake + -+/* Window length of extra_acked window. Max allowed val is 31. */ -+static u32 bbr_extra_acked_win_rtts = 5; ++config MCOOPERLAKE ++ bool "Intel Cooper Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) ++ select X86_P6_NOP ++ help + -+/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ -+static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; ++ Select this for Xeon processors in the Cooper Lake family. + -+/* Time period for clamping cwnd increment due to ack aggregation */ -+static u32 bbr_extra_acked_max_us = 100 * 1000; ++ Enables -march=cooperlake + -+/* Use extra acked in startup ? -+ * 0: disabled -+ * 1: use latest extra_acked value from 1-2 rtt in startup -+ */ -+static int bbr_extra_acked_in_startup = 1; /* default: enabled */ ++config MTIGERLAKE ++ bool "Intel Tiger Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) ++ select X86_P6_NOP ++ help + -+/* Experiment: don't grow cwnd beyond twice of what we just probed. */ -+static bool bbr_usage_based_cwnd; /* default: disabled */ ++ Select this for third-generation 10 nm process processors in the Tiger Lake family. + -+/* For lab testing, researchers can enable BBRv2 ECN support with this flag, -+ * when they know that any ECN marks that the connections experience will be -+ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. -+ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on -+ * negotiation or configuration that is outside the scope of the BBRv2 -+ * alpha release. -+ */ -+static bool bbr_ecn_enable = false; ++ Enables -march=tigerlake + -+module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); -+module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); -+module_param_named(high_gain, bbr_high_gain, int, 0644); -+module_param_named(drain_gain, bbr_drain_gain, int, 0644); -+module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); -+module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); -+module_param_array_named(pacing_gain, bbr_pacing_gain, int, -+ &bbr_pacing_gain_size, 0644); -+module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); -+module_param_named(probe_rtt_cwnd_gain, -+ bbr_probe_rtt_cwnd_gain, uint, 0664); -+module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); -+module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); -+module_param_named(flags, bbr_flags, uint, 0644); -+module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); -+module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); -+module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); -+module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); -+module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); -+module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); -+module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); -+module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); -+module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); -+module_param_named(extra_acked_win_rtts, -+ bbr_extra_acked_win_rtts, uint, 0664); -+module_param_named(extra_acked_max_us, -+ bbr_extra_acked_max_us, uint, 0664); -+module_param_named(ack_epoch_acked_reset_thresh, -+ bbr_ack_epoch_acked_reset_thresh, uint, 0664); -+module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); -+module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); -+module_param_named(extra_acked_in_startup, -+ bbr_extra_acked_in_startup, int, 0664); -+module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); -+module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); ++config MSAPPHIRERAPIDS ++ bool "Intel Sapphire Rapids" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ select X86_P6_NOP ++ help + -+static void bbr2_exit_probe_rtt(struct sock *sk); -+static void bbr2_reset_congestion_signals(struct sock *sk); ++ Select this for third-generation 10 nm process processors in the Sapphire Rapids family. + -+static void bbr_check_probe_rtt_done(struct sock *sk); ++ Enables -march=sapphirerapids + -+/* Do we estimate that STARTUP filled the pipe? */ -+static bool bbr_full_bw_reached(const struct sock *sk) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); ++config MROCKETLAKE ++ bool "Intel Rocket Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ select X86_P6_NOP ++ help + -+ return bbr->full_bw_reached; -+} ++ Select this for eleventh-generation processors in the Rocket Lake family. + -+/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ -+static u32 bbr_max_bw(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ Enables -march=rocketlake + -+ return max(bbr->bw_hi[0], bbr->bw_hi[1]); -+} ++config MALDERLAKE ++ bool "Intel Alder Lake" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ select X86_P6_NOP ++ help + -+/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ -+static u32 bbr_bw(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ Select this for twelfth-generation processors in the Alder Lake family. + -+ return min(bbr_max_bw(sk), bbr->bw_lo); -+} ++ Enables -march=alderlake + -+/* Return maximum extra acked in past k-2k round trips, -+ * where k = bbr_extra_acked_win_rtts. -+ */ -+static u16 bbr_extra_acked(const struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); + config GENERIC_CPU + bool "Generic-x86-64" + depends on X86_64 +@@ -294,6 +558,50 @@ config GENERIC_CPU + Generic x86-64 CPU. + Run equally well on all x86-64 CPUs. + ++config GENERIC_CPU2 ++ bool "Generic-x86-64-v2" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ depends on X86_64 ++ help ++ Generic x86-64 CPU. ++ Run equally well on all x86-64 CPUs with min support of x86-64-v2. + -+ return max(bbr->extra_acked[0], bbr->extra_acked[1]); -+} ++config GENERIC_CPU3 ++ bool "Generic-x86-64-v3" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ depends on X86_64 ++ help ++ Generic x86-64-v3 CPU with v3 instructions. ++ Run equally well on all x86-64 CPUs with min support of x86-64-v3. + -+/* Return rate in bytes per second, optionally with a gain. -+ * The order here is chosen carefully to avoid overflow of u64. This should -+ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. -+ */ -+static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, -+ int margin) -+{ -+ unsigned int mss = tcp_sk(sk)->mss_cache; ++config GENERIC_CPU4 ++ bool "Generic-x86-64-v4" ++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) ++ depends on X86_64 ++ help ++ Generic x86-64 CPU with v4 instructions. ++ Run equally well on all x86-64 CPUs with min support of x86-64-v4. + -+ rate *= mss; -+ rate *= gain; -+ rate >>= BBR_SCALE; -+ rate *= USEC_PER_SEC / 100 * (100 - margin); -+ rate >>= BW_SCALE; -+ rate = max(rate, 1ULL); -+ return rate; -+} ++config MNATIVE_INTEL ++ bool "Intel-Native optimizations autodetected by the compiler" ++ help + -+static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) -+{ -+ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); -+} ++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects ++ the optimum settings to use based on your processor. Do NOT use this ++ for AMD CPUs. Intel Only! + -+static u64 bbr_rate_kbps(struct sock *sk, u64 rate) -+{ -+ rate = bbr_bw_bytes_per_sec(sk, rate); -+ rate *= 8; -+ do_div(rate, 1000); -+ return rate; -+} ++ Enables -march=native + -+static u32 bbr_tso_segs_goal(struct sock *sk); -+static void bbr_debug(struct sock *sk, u32 acked, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ static const char ca_states[] = { -+ [TCP_CA_Open] = 'O', -+ [TCP_CA_Disorder] = 'D', -+ [TCP_CA_CWR] = 'C', -+ [TCP_CA_Recovery] = 'R', -+ [TCP_CA_Loss] = 'L', -+ }; -+ static const char mode[] = { -+ 'G', /* Growing - BBR_STARTUP */ -+ 'D', /* Drain - BBR_DRAIN */ -+ 'W', /* Window - BBR_PROBE_BW */ -+ 'M', /* Min RTT - BBR_PROBE_RTT */ -+ }; -+ static const char ack_phase[] = { /* bbr_ack_phase strings */ -+ 'I', /* BBR_ACKS_INIT - 'Init' */ -+ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ -+ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ -+ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ -+ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ -+ }; -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ const u32 una = tp->snd_una - bbr->debug.snd_isn; -+ const u32 fack = tcp_highest_sack_seq(tp); -+ const u16 dport = ntohs(inet_sk(sk)->inet_dport); -+ bool is_port_match = (bbr_debug_port_mask && -+ ((dport & bbr_debug_port_mask) == 0)); -+ char debugmsg[320]; ++config MNATIVE_AMD ++ bool "AMD-Native optimizations autodetected by the compiler" ++ help + -+ if (sk->sk_state == TCP_SYN_SENT) -+ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ ++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects ++ the optimum settings to use based on your processor. Do NOT use this ++ for Intel CPUs. AMD Only! + -+ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { -+ char addr[INET6_ADDRSTRLEN + 10] = { 0 }; ++ Enables -march=native + -+ if (sk->sk_family == AF_INET) -+ snprintf(addr, sizeof(addr), "%pI4:%u", -+ &inet_sk(sk)->inet_daddr, dport); -+ else if (sk->sk_family == AF_INET6) -+ snprintf(addr, sizeof(addr), "%pI6:%u", -+ &sk->sk_v6_daddr, dport); -+ -+ WARN_ONCE(1, -+ "BBR %s cwnd alert: %u " -+ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " -+ "bw: %u rtt: %u min_rtt: %u " -+ "acked: %u tso_segs: %u " -+ "bw: %d %ld %d pif: %u\n", -+ addr, tp->snd_cwnd, -+ una, inet_csk(sk)->icsk_ca_state, -+ bbr->pacing_gain, bbr->cwnd_gain, -+ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, -+ acked, bbr_tso_segs_goal(sk), -+ rs->delivered, rs->interval_us, rs->is_retrans, -+ tcp_packets_in_flight(tp)); -+ } + endchoice + + config X86_GENERIC +@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT + config X86_L1_CACHE_SHIFT + int + default "7" if MPENTIUM4 || MPSC +- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU ++ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 + default "4" if MELAN || M486SX || M486 || MGEODEGX1 + default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX + +@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 + + config X86_INTEL_USERCOPY + def_bool y +- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 ++ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL + + config X86_USE_PPRO_CHECKSUM + def_bool y +- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM ++ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD + + # + # P6_NOPs are a relatively minor optimization that require a family >= +@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM + config X86_P6_NOP + def_bool y + depends on X86_64 +- depends on (MCORE2 || MPENTIUM4 || MPSC) ++ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) + + config X86_TSC + def_bool y +- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 ++ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 + + config X86_CMPXCHG64 + def_bool y +- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 ++ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD + + # this should be set for all -march=.. options where the compiler + # generates cmov. + config X86_CMOV + def_bool y +- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) ++ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) + + config X86_MINIMUM_CPU_FAMILY + int + default "64" if X86_64 +- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) ++ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) + default "5" if X86_32 && X86_CMPXCHG64 + default "4" + +diff --git a/arch/x86/Makefile b/arch/x86/Makefile +index bafbd905e6e7..476ab926c33e 100644 +--- a/arch/x86/Makefile ++++ b/arch/x86/Makefile +@@ -150,8 +150,44 @@ else + # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) + cflags-$(CONFIG_MK8) += -march=k8 + cflags-$(CONFIG_MPSC) += -march=nocona +- cflags-$(CONFIG_MCORE2) += -march=core2 +- cflags-$(CONFIG_MATOM) += -march=atom ++ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3 ++ cflags-$(CONFIG_MK10) += -march=amdfam10 ++ cflags-$(CONFIG_MBARCELONA) += -march=barcelona ++ cflags-$(CONFIG_MBOBCAT) += -march=btver1 ++ cflags-$(CONFIG_MJAGUAR) += -march=btver2 ++ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1 ++ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm ++ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm ++ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm ++ cflags-$(CONFIG_MZEN) += -march=znver1 ++ cflags-$(CONFIG_MZEN2) += -march=znver2 ++ cflags-$(CONFIG_MZEN3) += -march=znver3 ++ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native ++ cflags-$(CONFIG_MNATIVE_AMD) += -march=native ++ cflags-$(CONFIG_MATOM) += -march=bonnell ++ cflags-$(CONFIG_MCORE2) += -march=core2 ++ cflags-$(CONFIG_MNEHALEM) += -march=nehalem ++ cflags-$(CONFIG_MWESTMERE) += -march=westmere ++ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont ++ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont ++ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus ++ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge ++ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge ++ cflags-$(CONFIG_MHASWELL) += -march=haswell ++ cflags-$(CONFIG_MBROADWELL) += -march=broadwell ++ cflags-$(CONFIG_MSKYLAKE) += -march=skylake ++ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512 ++ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake ++ cflags-$(CONFIG_MICELAKE) += -march=icelake-client ++ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake ++ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake ++ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake ++ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids ++ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake ++ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake ++ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2 ++ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3 ++ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4 + cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic + KBUILD_CFLAGS += $(cflags-y) + +diff --git a/arch/x86/Makefile.postlink b/arch/x86/Makefile.postlink +new file mode 100644 +index 000000000000..b38ffa4defb3 +--- /dev/null ++++ b/arch/x86/Makefile.postlink +@@ -0,0 +1,41 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# =========================================================================== ++# Post-link x86 pass ++# =========================================================================== ++# ++# 1. Separate relocations from vmlinux into vmlinux.relocs. ++# 2. Strip relocations from vmlinux. + -+ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) -+ return; ++PHONY := __archpost ++__archpost: + -+ if (!sock_flag(sk, SOCK_DBG) && !is_port_match) -+ return; ++-include include/config/auto.conf ++include scripts/Kbuild.include + -+ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) -+ return; ++CMD_RELOCS = arch/x86/tools/relocs ++quiet_cmd_relocs = RELOCS $@.relocs ++ cmd_relocs = $(CMD_RELOCS) $@ > $@.relocs;$(CMD_RELOCS) --abs-relocs $@ + -+ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && -+ !(bbr_flags & FLAG_DEBUG_LOOPBACK)) -+ return; ++quiet_cmd_strip_relocs = RSTRIP $@ ++ cmd_strip_relocs = $(OBJCOPY) --remove-section='.rel.*' --remove-section='.rel__*' --remove-section='.rela.*' --remove-section='.rela__*' $@ + -+ snprintf(debugmsg, sizeof(debugmsg) - 1, -+ "BBR %pI4:%-5u %5u,%03u:%-7u %c " -+ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " -+ "bw %llu lb %llu ib %llu qb %llu " -+ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " -+ "lr %d er %d ea %d bwl %lld il %d ih %d c %d " -+ "v %d %c %u %c %s\n", -+ &inet_sk(sk)->inet_daddr, dport, -+ una / 1000, una % 1000, fack - tp->snd_una, -+ ca_states[inet_csk(sk)->icsk_ca_state], -+ bbr->debug.undo ? '@' : mode[bbr->mode], -+ tp->snd_cwnd, -+ bbr_extra_acked(sk), /* br (legacy): extra_acked */ -+ rs->tx_in_flight, /* cr (legacy): tx_inflight */ -+ rs->rtt_us, -+ rs->delivered, -+ rs->interval_us, -+ bbr->min_rtt_us, -+ rs->is_app_limited ? '_' : 'l', -+ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ -+ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ -+ 0ULL, /* lb: [obsolete] */ -+ 0ULL, /* ib: [obsolete] */ -+ div_u64((u64)sk->sk_pacing_rate * 8, 1000), -+ acked, -+ tcp_packets_in_flight(tp), -+ rs->is_ack_delayed ? 'd' : '.', -+ bbr->round_start ? '*' : '.', -+ tp->delivered, tp->lost, -+ tp->app_limited, -+ 0, /* #: [obsolete] */ -+ ctx->target_cwnd, -+ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ -+ ca_states[bbr->prev_ca_state], -+ (rs->lost + rs->delivered) > 0 ? -+ (1000 * rs->lost / -+ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ -+ (rs->delivered) > 0 ? -+ (1000 * rs->delivered_ce / -+ (rs->delivered)) : 0, /* er: ECN rate x1000 */ -+ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ -+ bbr->bw_lo == ~0U ? -+ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ -+ bbr->inflight_lo, /* il */ -+ bbr->inflight_hi, /* ih */ -+ bbr->bw_probe_up_cnt, /* c */ -+ 2, /* v: version */ -+ bbr->debug.event, -+ bbr->cycle_idx, -+ ack_phase[bbr->ack_phase], -+ bbr->bw_probe_samples ? "Y" : "N"); -+ debugmsg[sizeof(debugmsg) - 1] = 0; ++# `@true` prevents complaint when there is nothing to be done + -+ /* printk takes a higher precedence. */ -+ if (bbr_debug_with_printk) -+ printk(KERN_DEBUG "%s", debugmsg); ++vmlinux: FORCE ++ @true ++ifeq ($(CONFIG_X86_NEED_RELOCS),y) ++ $(call cmd,relocs) ++ $(call cmd,strip_relocs) ++endif + -+ if (unlikely(bbr->debug.undo)) -+ bbr->debug.undo = 0; -+} ++%.ko: FORCE ++ @true + -+/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ -+static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) -+{ -+ u64 rate = bw; ++clean: ++ @rm -f vmlinux.relocs + -+ rate = bbr_rate_bytes_per_sec(sk, rate, gain, -+ bbr_pacing_margin_percent); -+ rate = min_t(u64, rate, sk->sk_max_pacing_rate); -+ return rate; -+} ++PHONY += FORCE clean + -+/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ -+static void bbr_init_pacing_rate_from_rtt(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw; -+ u32 rtt_us; ++FORCE: + -+ if (tp->srtt_us) { /* any RTT sample yet? */ -+ rtt_us = max(tp->srtt_us >> 3, 1U); -+ bbr->has_seen_rtt = 1; -+ } else { /* no RTT sample yet */ -+ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ -+ } -+ bw = (u64)tp->snd_cwnd * BW_UNIT; -+ do_div(bw, rtt_us); -+ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); -+} ++.PHONY: $(PHONY) +diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile +index 35ce1a64068b..eba7709d75ae 100644 +--- a/arch/x86/boot/compressed/Makefile ++++ b/arch/x86/boot/compressed/Makefile +@@ -120,14 +120,12 @@ $(obj)/vmlinux.bin: vmlinux FORCE + + targets += $(patsubst $(obj)/%,%,$(vmlinux-objs-y)) vmlinux.bin.all vmlinux.relocs + +-CMD_RELOCS = arch/x86/tools/relocs +-quiet_cmd_relocs = RELOCS $@ +- cmd_relocs = $(CMD_RELOCS) $< > $@;$(CMD_RELOCS) --abs-relocs $< +-$(obj)/vmlinux.relocs: vmlinux FORCE +- $(call if_changed,relocs) ++# vmlinux.relocs is created by the vmlinux postlink step. ++vmlinux.relocs: vmlinux ++ @true + + vmlinux.bin.all-y := $(obj)/vmlinux.bin +-vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs ++vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += vmlinux.relocs + + $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE + $(call if_changed,gzip) +diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h +index 6674bdb096f3..e278872e9187 100644 +--- a/arch/x86/include/asm/msr-index.h ++++ b/arch/x86/include/asm/msr-index.h +@@ -569,6 +569,7 @@ + #define MSR_AMD_CPPC_CAP2 0xc00102b2 + #define MSR_AMD_CPPC_REQ 0xc00102b3 + #define MSR_AMD_CPPC_STATUS 0xc00102b4 ++#define MSR_AMD_CPPC_HW_CTL 0xc0010015 + + #define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) + #define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) +@@ -579,12 +580,18 @@ + #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) + #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) + #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) ++#define AMD_CPPC_PRECISION_BOOST_BIT 25 ++#define AMD_CPPC_PRECISION_BOOST_ENABLED BIT_ULL(AMD_CPPC_PRECISION_BOOST_BIT) + + /* AMD Performance Counter Global Status and Control MSRs */ + #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 + #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 + #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 + ++#define AMD_CPPC_EPP_PERFORMANCE 0x00 ++#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 ++#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF ++#define AMD_CPPC_EPP_POWERSAVE 0xFF + /* Fam 17h MSRs */ + #define MSR_F17H_IRPERF 0xc00000e9 + +diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h +index 458c891a8273..d86eb1ebf59f 100644 +--- a/arch/x86/include/asm/topology.h ++++ b/arch/x86/include/asm/topology.h +@@ -175,6 +175,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled; + + /* Interface to set priority of a cpu */ + void sched_set_itmt_core_prio(int prio, int core_cpu); ++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu); + + /* Interface to notify scheduler that system supports ITMT */ + int sched_set_itmt_support(void); +diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h +index 57b1a7034c64..e2c45674f989 100644 +--- a/arch/x86/include/asm/vdso/processor.h ++++ b/arch/x86/include/asm/vdso/processor.h +@@ -10,7 +10,7 @@ + /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ + static __always_inline void rep_nop(void) + { +- asm volatile("rep; nop" ::: "memory"); ++ asm volatile("lfence" ::: "memory"); + } + + static __always_inline void cpu_relax(void) +diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h +index 75884d2cdec3..4e6a08d4c7e5 100644 +--- a/arch/x86/include/asm/vermagic.h ++++ b/arch/x86/include/asm/vermagic.h +@@ -17,6 +17,48 @@ + #define MODULE_PROC_FAMILY "586MMX " + #elif defined CONFIG_MCORE2 + #define MODULE_PROC_FAMILY "CORE2 " ++#elif defined CONFIG_MNATIVE_INTEL ++#define MODULE_PROC_FAMILY "NATIVE_INTEL " ++#elif defined CONFIG_MNATIVE_AMD ++#define MODULE_PROC_FAMILY "NATIVE_AMD " ++#elif defined CONFIG_MNEHALEM ++#define MODULE_PROC_FAMILY "NEHALEM " ++#elif defined CONFIG_MWESTMERE ++#define MODULE_PROC_FAMILY "WESTMERE " ++#elif defined CONFIG_MSILVERMONT ++#define MODULE_PROC_FAMILY "SILVERMONT " ++#elif defined CONFIG_MGOLDMONT ++#define MODULE_PROC_FAMILY "GOLDMONT " ++#elif defined CONFIG_MGOLDMONTPLUS ++#define MODULE_PROC_FAMILY "GOLDMONTPLUS " ++#elif defined CONFIG_MSANDYBRIDGE ++#define MODULE_PROC_FAMILY "SANDYBRIDGE " ++#elif defined CONFIG_MIVYBRIDGE ++#define MODULE_PROC_FAMILY "IVYBRIDGE " ++#elif defined CONFIG_MHASWELL ++#define MODULE_PROC_FAMILY "HASWELL " ++#elif defined CONFIG_MBROADWELL ++#define MODULE_PROC_FAMILY "BROADWELL " ++#elif defined CONFIG_MSKYLAKE ++#define MODULE_PROC_FAMILY "SKYLAKE " ++#elif defined CONFIG_MSKYLAKEX ++#define MODULE_PROC_FAMILY "SKYLAKEX " ++#elif defined CONFIG_MCANNONLAKE ++#define MODULE_PROC_FAMILY "CANNONLAKE " ++#elif defined CONFIG_MICELAKE ++#define MODULE_PROC_FAMILY "ICELAKE " ++#elif defined CONFIG_MCASCADELAKE ++#define MODULE_PROC_FAMILY "CASCADELAKE " ++#elif defined CONFIG_MCOOPERLAKE ++#define MODULE_PROC_FAMILY "COOPERLAKE " ++#elif defined CONFIG_MTIGERLAKE ++#define MODULE_PROC_FAMILY "TIGERLAKE " ++#elif defined CONFIG_MSAPPHIRERAPIDS ++#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " ++#elif defined CONFIG_ROCKETLAKE ++#define MODULE_PROC_FAMILY "ROCKETLAKE " ++#elif defined CONFIG_MALDERLAKE ++#define MODULE_PROC_FAMILY "ALDERLAKE " + #elif defined CONFIG_MATOM + #define MODULE_PROC_FAMILY "ATOM " + #elif defined CONFIG_M686 +@@ -35,6 +77,30 @@ + #define MODULE_PROC_FAMILY "K7 " + #elif defined CONFIG_MK8 + #define MODULE_PROC_FAMILY "K8 " ++#elif defined CONFIG_MK8SSE3 ++#define MODULE_PROC_FAMILY "K8SSE3 " ++#elif defined CONFIG_MK10 ++#define MODULE_PROC_FAMILY "K10 " ++#elif defined CONFIG_MBARCELONA ++#define MODULE_PROC_FAMILY "BARCELONA " ++#elif defined CONFIG_MBOBCAT ++#define MODULE_PROC_FAMILY "BOBCAT " ++#elif defined CONFIG_MBULLDOZER ++#define MODULE_PROC_FAMILY "BULLDOZER " ++#elif defined CONFIG_MPILEDRIVER ++#define MODULE_PROC_FAMILY "PILEDRIVER " ++#elif defined CONFIG_MSTEAMROLLER ++#define MODULE_PROC_FAMILY "STEAMROLLER " ++#elif defined CONFIG_MJAGUAR ++#define MODULE_PROC_FAMILY "JAGUAR " ++#elif defined CONFIG_MEXCAVATOR ++#define MODULE_PROC_FAMILY "EXCAVATOR " ++#elif defined CONFIG_MZEN ++#define MODULE_PROC_FAMILY "ZEN " ++#elif defined CONFIG_MZEN2 ++#define MODULE_PROC_FAMILY "ZEN2 " ++#elif defined CONFIG_MZEN3 ++#define MODULE_PROC_FAMILY "ZEN3 " + #elif defined CONFIG_MELAN + #define MODULE_PROC_FAMILY "ELAN " + #elif defined CONFIG_MCRUSOE +diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c +index 62f6b8b7c4a5..f9c9b5850847 100644 +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -936,7 +936,9 @@ void __init alternative_instructions(void) + * Then patch alternatives, such that those paravirt calls that are in + * alternatives can be overwritten by their immediate fragments. + */ ++ printk("clr: Applying alternatives\n"); + apply_alternatives(__alt_instructions, __alt_instructions_end); ++ printk("clr: Applying alternatives done\n"); + + apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); + +diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c +index fbaf12e43f41..c8c2d6f1a8ac 100644 +--- a/arch/x86/kernel/cpu/intel_epb.c ++++ b/arch/x86/kernel/cpu/intel_epb.c +@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev, + if (ret < 0) + return ret; + ++ /* update the ITMT scheduler logic to use the power policy data */ ++ /* scale the val up by 2 so the range is 224 - 256 */ ++ sched_set_itmt_power_ratio(256 - val * 2, cpu); + -+/* Pace using current bw estimate and a gain factor. */ -+static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) + return count; + } + +diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c +index ad57e0e4d674..e2ed33c2bd4d 100644 +--- a/arch/x86/kernel/cpu/microcode/core.c ++++ b/arch/x86/kernel/cpu/microcode/core.c +@@ -44,6 +44,8 @@ + + static struct microcode_ops *microcode_ops; + static bool dis_ucode_ldr = true; ++bool ucode_rollback = false; ++int enable_rollback = 0; + + bool initrd_gone; + +@@ -80,6 +82,26 @@ static u32 final_levels[] = { + 0, /* T-101 terminator */ + }; + ++static int __init ucode_setup(char *str) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); -+ -+ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) -+ bbr_init_pacing_rate_from_rtt(sk); -+ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) -+ sk->sk_pacing_rate = rate; -+} ++ if (!str) ++ return -EINVAL; + -+static u32 bbr_min_tso_segs(struct sock *sk) -+{ -+ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; ++ while (*str) { ++ if (!strncmp(str, "rollback", 8)) { ++ enable_rollback = 1; ++ pr_info("Microcode Rollback Enabled\n"); ++ } ++ str += strcspn(str, ","); ++ while (*str == ',') ++ str++; ++ } ++ return 0; +} + -+/* Return the number of segments BBR would like in a TSO/GSO skb, given -+ * a particular max gso size as a constraint. -+ */ -+static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, -+ u32 gso_max_size) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 segs, r; -+ u64 bytes; -+ -+ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ -+ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; ++__setup("ucode=", ucode_setup); + -+ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every -+ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. -+ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) -+ */ -+ if (bbr->params.tso_rtt_shift) { -+ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; -+ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ -+ bytes += GSO_MAX_SIZE >> r; -+ } + -+ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); -+ return segs; -+} + /* + * Check the current patch level on this CPU. + * +@@ -512,6 +534,7 @@ static ssize_t reload_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) + { ++ struct cpuinfo_x86 *c = &boot_cpu_data; + enum ucode_state tmp_ret = UCODE_OK; + int bsp = boot_cpu_data.cpu_index; + unsigned long val; +@@ -521,7 +544,7 @@ static ssize_t reload_store(struct device *dev, + if (ret) + return ret; + +- if (val != 1) ++ if (!val || val > 2) + return size; + + cpus_read_lock(); +@@ -529,6 +552,20 @@ static ssize_t reload_store(struct device *dev, + ret = check_online_cpus(); + if (ret) + goto put; ++ /* ++ * Check if the vendor is Intel to permit reloading ++ * microcode even if the revision is unchanged. ++ * This is typically used during development of microcode ++ * and changing rev is a pain. ++ */ ++ if ((val == 2) && ((c->x86_vendor != X86_VENDOR_INTEL) || ++ !enable_rollback)) ++ return size; ++ else if (val == 2) { ++ mutex_lock(µcode_mutex); ++ ucode_rollback = true; ++ mutex_unlock(µcode_mutex); ++ } + + tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); + if (tmp_ret != UCODE_NEW) +@@ -539,6 +576,7 @@ static ssize_t reload_store(struct device *dev, + mutex_unlock(µcode_mutex); + + put: ++ ucode_rollback = false; + cpus_read_unlock(); + + if (ret == 0) +diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c +index 025c8f0cd948..60473f577a25 100644 +--- a/arch/x86/kernel/cpu/microcode/intel.c ++++ b/arch/x86/kernel/cpu/microcode/intel.c +@@ -44,6 +44,7 @@ static struct microcode_intel *intel_ucode_patch; + + /* last level cache size per core */ + static int llc_size_per_core; ++extern bool ucode_rollback; + + /* + * Returns 1 if update has been found, 0 otherwise. +@@ -80,7 +81,7 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev + { + struct microcode_header_intel *mc_hdr = mc; + +- if (mc_hdr->rev <= new_rev) ++ if (!ucode_rollback && mc_hdr->rev <= new_rev) + return 0; + + return find_matching_signature(mc, csig, cpf); +@@ -120,7 +121,7 @@ static void save_microcode_patch(struct ucode_cpu_info *uci, void *data, unsigne + if (find_matching_signature(data, sig, pf)) { + prev_found = true; + +- if (mc_hdr->rev <= mc_saved_hdr->rev) ++ if (!ucode_rollback && mc_hdr->rev <= mc_saved_hdr->rev) + continue; + + p = memdup_patch(data, size); +@@ -649,7 +650,7 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci) + + phdr = (struct microcode_header_intel *)iter->data; + +- if (phdr->rev <= uci->cpu_sig.rev) ++ if (!ucode_rollback && phdr->rev <= uci->cpu_sig.rev) + continue; + + if (!find_matching_signature(phdr, +@@ -734,10 +735,11 @@ static enum ucode_state apply_microcode_intel(int cpu) + * already. + */ + rev = intel_get_microcode_revision(); +- if (rev >= mc->hdr.rev) { ++ if (!ucode_rollback && rev >= mc->hdr.rev) { + ret = UCODE_OK; + goto out; +- } ++ } else if (ucode_rollback) ++ ret = UCODE_OK; + + /* + * Writeback and invalidate caches before updating microcode to avoid +@@ -756,7 +758,7 @@ static enum ucode_state apply_microcode_intel(int cpu) + return UCODE_ERROR; + } + +- if (bsp && rev != prev_rev) { ++ if (bsp && ((rev != prev_rev) || ucode_rollback)) { + pr_info("updated to revision 0x%x, date = %04x-%02x-%02x\n", + rev, + mc->hdr.date & 0xffff, +diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c +index 9ff480e94511..d4326e050fb7 100644 +--- a/arch/x86/kernel/itmt.c ++++ b/arch/x86/kernel/itmt.c +@@ -25,6 +25,7 @@ + + static DEFINE_MUTEX(itmt_update_mutex); + DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); ++DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio); + + /* Boolean to track if system has ITMT capabilities */ + static bool __read_mostly sched_itmt_capable; +@@ -169,7 +170,12 @@ void sched_clear_itmt_support(void) + + int arch_asym_cpu_priority(int cpu) + { +- return per_cpu(sched_core_priority, cpu); ++ int power_ratio = per_cpu(sched_power_ratio, cpu); + -+/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ -+static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) -+{ -+ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); -+} ++ /* a power ratio of 0 (uninitialized) is assumed to be maximum */ ++ if (power_ratio == 0) ++ power_ratio = 256 - 2 * 6; ++ return per_cpu(sched_core_priority, cpu) * power_ratio / 256; + } + + /** +@@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) + i++; + } + } + -+/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ -+static u32 bbr_tso_segs_goal(struct sock *sk) ++/** ++ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT ++ * @power_ratio: The power scaling ratio [1..256] for the core ++ * @core_cpu: The cpu number associated with the core ++ * ++ * Set a scaling to the cpu performance based on long term power ++ * settings (like EPB). ++ * ++ * Note this is for the policy not for the actual dynamic frequency; ++ * the frequency will increase itself as workloads run on a core. ++ */ ++ ++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu) +{ -+ struct tcp_sock *tp = tcp_sk(sk); ++ int cpu; + -+ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); ++ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) { ++ per_cpu(sched_power_ratio, cpu) = power_ratio; ++ } +} +diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c +index ed8ac6bcbafb..d6fc5bdb0246 100644 +--- a/arch/x86/kernel/msr.c ++++ b/arch/x86/kernel/msr.c +@@ -48,7 +48,7 @@ enum allow_write_msrs { + MSR_WRITES_DEFAULT, + }; + +-static enum allow_write_msrs allow_writes = MSR_WRITES_DEFAULT; ++static enum allow_write_msrs allow_writes = MSR_WRITES_ON; + + static ssize_t msr_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c +index cafacb2e58cc..c2f80184fd33 100644 +--- a/arch/x86/kernel/tsc.c ++++ b/arch/x86/kernel/tsc.c +@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void) + if (!constant_tsc || !mask) + return 0; + ++ if (cpu != 0) ++ return cpu_data(0).loops_per_jiffy; + -+/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -+static void bbr_save_cwnd(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); + sibling = cpumask_any_but(mask, cpu); + if (sibling < nr_cpu_ids) + return cpu_data(sibling).loops_per_jiffy; +diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c +index fa71a5d12e87..c4bbd8c66134 100644 +--- a/arch/x86/mm/fault.c ++++ b/arch/x86/mm/fault.c +@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, + if (!printk_ratelimit()) + return; + +- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx", ++ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i", + loglvl, tsk->comm, task_pid_nr(tsk), address, +- (void *)regs->ip, (void *)regs->sp, error_code); ++ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id()); + + print_vma_addr(KERN_CONT " in ", regs->ip); + +diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c +index 30b15a9a47c4..144bca006463 100644 +--- a/block/bfq-cgroup.c ++++ b/block/bfq-cgroup.c +@@ -254,17 +254,12 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, + + #else /* CONFIG_BFQ_CGROUP_DEBUG */ + +-void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, +- blk_opf_t opf) { } + void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { } + void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { } + void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, + u64 io_start_time_ns, blk_opf_t opf) { } + void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } +-void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } +-void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } + void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } +-void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } + + #endif /* CONFIG_BFQ_CGROUP_DEBUG */ + +diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c +index c740b41fe0a4..5ea6245f0208 100644 +--- a/block/bfq-iosched.c ++++ b/block/bfq-iosched.c +@@ -1925,7 +1925,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, + bfqq->service_from_backlogged = 0; + bfq_clear_bfqq_softrt_update(bfqq); + +- bfq_add_bfqq_busy(bfqd, bfqq); ++ bfq_add_bfqq_busy(bfqq); + + /* + * Expire in-service queue if preemption may be needed for +@@ -2419,7 +2419,7 @@ static void bfq_remove_request(struct request_queue *q, + bfqq->next_rq = NULL; + + if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { +- bfq_del_bfqq_busy(bfqd, bfqq, false); ++ bfq_del_bfqq_busy(bfqq, false); + /* + * bfqq emptied. In normal operation, when + * bfqq is empty, bfqq->entity.service and +@@ -3098,7 +3098,7 @@ void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) + */ + if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && + bfqq != bfqd->in_service_queue) +- bfq_del_bfqq_busy(bfqd, bfqq, false); ++ bfq_del_bfqq_busy(bfqq, false); + + bfq_reassign_last_bfqq(bfqq, NULL); + +@@ -3908,7 +3908,7 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, + */ + bfqq->budget_timeout = jiffies; + +- bfq_del_bfqq_busy(bfqd, bfqq, true); ++ bfq_del_bfqq_busy(bfqq, true); + } else { + bfq_requeue_bfqq(bfqd, bfqq, true); + /* +@@ -5255,9 +5255,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) + struct hlist_node *n; + struct bfq_group *bfqg = bfqq_group(bfqq); + +- if (bfqq->bfqd) +- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", +- bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); + + bfqq->ref--; + if (bfqq->ref) +@@ -5321,7 +5319,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) + hlist_del_init(&item->woken_list_node); + } + +- if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) ++ if (bfqq->bfqd->last_completed_rq_bfqq == bfqq) + bfqq->bfqd->last_completed_rq_bfqq = NULL; + + kmem_cache_free(bfq_pool, bfqq); +@@ -7463,6 +7461,7 @@ MODULE_ALIAS("bfq-iosched"); + static int __init bfq_init(void) + { + int ret; ++ char msg[60] = "BFQ I/O-scheduler: BFQ-CachyOS v5.19"; + + #ifdef CONFIG_BFQ_GROUP_IOSCHED + ret = blkcg_policy_register(&blkcg_policy_bfq); +@@ -7494,6 +7493,11 @@ static int __init bfq_init(void) + if (ret) + goto slab_kill; + ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ strcat(msg, " (with cgroups support)"); ++#endif ++ pr_info("%s", msg); + -+ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) -+ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ -+ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ -+ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); -+} + return 0; + + slab_kill: +diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h +index ad8e513d7e87..64ee618064ba 100644 +--- a/block/bfq-iosched.h ++++ b/block/bfq-iosched.h +@@ -993,20 +993,23 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); + /* ---------------- cgroups-support interface ---------------- */ + + void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq); +-void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, +- blk_opf_t opf); + void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf); + void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf); + void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, + u64 io_start_time_ns, blk_opf_t opf); + void bfqg_stats_update_dequeue(struct bfq_group *bfqg); +-void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); +-void bfqg_stats_update_idle_time(struct bfq_group *bfqg); + void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg); +-void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); + void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, + struct bfq_group *bfqg); + ++#ifdef CONFIG_BFQ_CGROUP_DEBUG ++void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, ++ blk_opf_t opf); ++void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); ++void bfqg_stats_update_idle_time(struct bfq_group *bfqg); ++void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); ++#endif + -+static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); + void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg); + void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio); + void bfq_end_wr_async(struct bfq_data *bfqd); +@@ -1077,9 +1080,8 @@ void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, + void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); + void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bool expiration); +-void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, +- bool expiration); +-void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); ++void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration); ++void bfq_add_bfqq_busy(struct bfq_queue *bfqq); + + /* --------------- end of interface of B-WF2Q+ ---------------- */ + +diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c +index 983413cdefad..8fc3da4c23bb 100644 +--- a/block/bfq-wf2q.c ++++ b/block/bfq-wf2q.c +@@ -1651,9 +1651,10 @@ void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * the service tree. As a special case, it can be invoked during an + * expiration. + */ +-void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, +- bool expiration) ++void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration) + { ++ struct bfq_data *bfqd = bfqq->bfqd; + -+ if (event == CA_EVENT_TX_START) { -+ tcp_plb_check_rehash(sk, &bbr->plb); + bfq_log_bfqq(bfqd, bfqq, "del from busy"); + + bfq_clear_bfqq_busy(bfqq); +@@ -1674,8 +1675,10 @@ void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, + /* + * Called when an inactive queue receives a new request. + */ +-void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++void bfq_add_bfqq_busy(struct bfq_queue *bfqq) + { ++ struct bfq_data *bfqd = bfqq->bfqd; + -+ if (!tp->app_limited) -+ return; -+ bbr->idle_restart = 1; -+ bbr->ack_epoch_mstamp = tp->tcp_mstamp; -+ bbr->ack_epoch_acked = 0; -+ /* Avoid pointless buffer overflows: pace at est. bw if we don't -+ * need more speed (we're restarting from idle and app-limited). -+ */ -+ if (bbr->mode == BBR_PROBE_BW) -+ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); -+ else if (bbr->mode == BBR_PROBE_RTT) -+ bbr_check_probe_rtt_done(sk); -+ } else if ((event == CA_EVENT_ECN_IS_CE || -+ event == CA_EVENT_ECN_NO_CE) && -+ bbr_ecn_enable && -+ bbr->params.precise_ece_ack) { -+ u32 state = bbr->ce_state; -+ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); -+ bbr->ce_state = state; -+ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) -+ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); -+ } -+} + bfq_log_bfqq(bfqd, bfqq, "add to busy"); + + bfq_activate_bfqq(bfqd, bfqq); +diff --git a/block/blk-core.c b/block/blk-core.c +index 651057c4146b..88873f267b2a 100644 +--- a/block/blk-core.c ++++ b/block/blk-core.c +@@ -742,6 +742,9 @@ void submit_bio_noacct(struct bio *bio) + status = BLK_STS_OK; + goto end_io; + } + -+/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: -+ * -+ * bdp = ceil(bw * min_rtt * gain) ++ if (bio->bi_opf & REQ_PREFLUSH) ++ current->fsync_count++; + } + + if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) +diff --git a/block/elevator.c b/block/elevator.c +index c319765892bb..b2687032a0e0 100644 +--- a/block/elevator.c ++++ b/block/elevator.c +@@ -640,8 +640,13 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) + + if (q->nr_hw_queues != 1 && + !blk_mq_is_shared_tags(q->tag_set->flags)) ++#if defined(CONFIG_CACHY) && defined(CONFIG_MQ_IOSCHED_KYBER) ++ return elevator_get(q, "kyber", false); ++#elif defined(CONFIG_CACHY) ++ return elevator_get(q, "mq-deadline", false); ++#else + return NULL; +- ++#endif + return elevator_get(q, "mq-deadline", false); + } + +diff --git a/drivers/Makefile b/drivers/Makefile +index 057857258bfd..643e2d3dee81 100644 +--- a/drivers/Makefile ++++ b/drivers/Makefile +@@ -59,15 +59,8 @@ obj-y += char/ + # iommu/ comes before gpu as gpu are using iommu controllers + obj-y += iommu/ + +-# gpu/ comes after char for AGP vs DRM startup and after iommu +-obj-y += gpu/ +- + obj-$(CONFIG_CONNECTOR) += connector/ + +-# i810fb and intelfb depend on char/agp/ +-obj-$(CONFIG_FB_I810) += video/fbdev/i810/ +-obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ +- + obj-$(CONFIG_PARPORT) += parport/ + obj-y += base/ block/ misc/ mfd/ nfc/ + obj-$(CONFIG_LIBNVDIMM) += nvdimm/ +@@ -79,6 +72,14 @@ obj-y += macintosh/ + obj-y += scsi/ + obj-y += nvme/ + obj-$(CONFIG_ATA) += ata/ ++ ++# gpu/ comes after char for AGP vs DRM startup and after iommu ++obj-y += gpu/ ++ ++# i810fb and intelfb depend on char/agp/ ++obj-$(CONFIG_FB_I810) += video/fbdev/i810/ ++obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/ ++ + obj-$(CONFIG_TARGET_CORE) += target/ + obj-$(CONFIG_MTD) += mtd/ + obj-$(CONFIG_SPI) += spi/ +diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c +index 1e15a9f25ae9..4801300d3c77 100644 +--- a/drivers/acpi/cppc_acpi.c ++++ b/drivers/acpi/cppc_acpi.c +@@ -424,6 +424,9 @@ bool acpi_cpc_valid(void) + struct cpc_desc *cpc_ptr; + int cpu; + ++ if (acpi_disabled) ++ return false; ++ + for_each_present_cpu(cpu) { + cpc_ptr = per_cpu(cpc_desc_ptr, cpu); + if (!cpc_ptr) +@@ -1320,6 +1323,132 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) + } + EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs); + ++/** ++ * cppc_get_epp_caps - Get the energy preference register value. ++ * @cpunum: CPU from which to get epp preference level. ++ * @perf_caps: Return address. + * -+ * The key factor, gain, controls the amount of queue. While a small gain -+ * builds a smaller queue, it becomes more vulnerable to noise in RTT -+ * measurements (e.g., delayed ACKs or other ACK compression effects). This -+ * noise may cause BBR to under-estimate the rate. ++ * Return: 0 for success, -EIO otherwise. + */ -+static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) ++int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 bdp; -+ u64 w; ++ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); ++ struct cpc_register_resource *energy_perf_reg; ++ u64 energy_perf; + -+ /* If we've never had a valid RTT sample, cap cwnd at the initial -+ * default. This should only happen when the connection is not using TCP -+ * timestamps and has retransmitted all of the SYN/SYNACK/data packets -+ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which -+ * case we need to slow-start up toward something safe: initial cwnd. -+ */ -+ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ -+ return bbr->init_cwnd; /* be safe: cap at initial cwnd */ ++ if (!cpc_desc) { ++ pr_warn("No CPC descriptor for CPU:%d\n", cpunum); ++ return -ENODEV; ++ } + -+ w = (u64)bw * bbr->min_rtt_us; ++ energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + -+ /* Apply a gain to the given value, remove the BW_SCALE shift, and -+ * round the value up to avoid a negative feedback loop. -+ */ -+ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ if (!CPC_SUPPORTED(energy_perf_reg)) ++ pr_warn("energy perf reg update is unsupported!\n"); + -+ return bdp; -+} ++ if (CPC_IN_PCC(energy_perf_reg)) { ++ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); ++ struct cppc_pcc_data *pcc_ss_data = NULL; ++ int ret = 0; + -+/* To achieve full performance in high-speed paths, we budget enough cwnd to -+ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: -+ * - one skb in sending host Qdisc, -+ * - one skb in sending host TSO/GSO engine -+ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine -+ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because -+ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, -+ * which allows 2 outstanding 2-packet sequences, to try to keep pipe -+ * full even with ACK-every-other-packet delayed ACKs. -+ */ -+static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 tso_segs_goal; ++ if (pcc_ss_id < 0) ++ return -EIO; + -+ tso_segs_goal = 3 * bbr_tso_segs_goal(sk); ++ pcc_ss_data = pcc_data[pcc_ss_id]; + -+ /* Allow enough full-sized skbs in flight to utilize end systems. */ -+ if (bbr->params.cwnd_tso_budget == 1) { -+ cwnd = max_t(u32, cwnd, tso_segs_goal); -+ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); -+ } else { -+ cwnd += tso_segs_goal; -+ cwnd = (cwnd + 1) & ~1U; -+ } -+ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) -+ cwnd += 2; ++ down_write(&pcc_ss_data->pcc_lock); + -+ return cwnd; -+} ++ if (send_pcc_cmd(pcc_ss_id, CMD_READ) >= 0) { ++ cpc_read(cpunum, energy_perf_reg, &energy_perf); ++ perf_caps->energy_perf = energy_perf; ++ } else { ++ ret = -EIO; ++ } + -+/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ -+static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) -+{ -+ u32 inflight; ++ up_write(&pcc_ss_data->pcc_lock); + -+ inflight = bbr_bdp(sk, bw, gain); -+ inflight = bbr_quantization_budget(sk, inflight); ++ return ret; ++ } + -+ return inflight; ++ return 0; +} ++EXPORT_SYMBOL_GPL(cppc_get_epp_caps); + -+/* With pacing at lower layers, there's often less data "in the network" than -+ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), -+ * we often have several skbs queued in the pacing layer with a pre-scheduled -+ * earliest departure time (EDT). BBR adapts its pacing rate based on the -+ * inflight level that it estimates has already been "baked in" by previous -+ * departure time decisions. We calculate a rough estimate of the number of our -+ * packets that might be in the network at the earliest departure time for the -+ * next skb scheduled: -+ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw -+ * If we're increasing inflight, then we want to know if the transmit of the -+ * EDT skb will push inflight above the target, so inflight_at_edt includes -+ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, -+ * then estimate if inflight will sink too low just before the EDT transmit. -+ */ -+static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) ++int cppc_set_auto_epp(int cpu, bool enable) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 now_ns, edt_ns, interval_us; -+ u32 interval_delivered, inflight_at_edt; ++ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); ++ struct cpc_register_resource *auto_sel_reg; ++ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); ++ struct cppc_pcc_data *pcc_ss_data = NULL; ++ int ret = -EINVAL; + -+ now_ns = tp->tcp_clock_cache; -+ edt_ns = max(tp->tcp_wstamp_ns, now_ns); -+ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); -+ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; -+ inflight_at_edt = inflight_now; -+ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ -+ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ -+ if (interval_delivered >= inflight_at_edt) -+ return 0; -+ return inflight_at_edt - interval_delivered; -+} ++ if (!cpc_desc) { ++ pr_warn("No CPC descriptor for CPU:%d\n", cpu); ++ return -EINVAL; ++ } + -+/* Find the cwnd increment based on estimate of ack aggregation */ -+static u32 bbr_ack_aggregation_cwnd(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 max_aggr_cwnd, aggr_cwnd = 0; ++ auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; + -+ if (bbr->params.extra_acked_gain && -+ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { -+ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) -+ / BW_UNIT; -+ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) -+ >> BBR_SCALE; -+ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); -+ } ++ if (CPC_IN_PCC(auto_sel_reg)) { ++ if (pcc_ss_id < 0) ++ return -EIO; + -+ return aggr_cwnd; -+} ++ ret = cpc_write(cpu, auto_sel_reg, enable); ++ if (ret) ++ return ret; + -+/* Returns the cwnd for PROBE_RTT mode. */ -+static u32 bbr_probe_rtt_cwnd(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ pcc_ss_data = pcc_data[pcc_ss_id]; + -+ if (bbr->params.probe_rtt_cwnd_gain == 0) -+ return bbr->params.cwnd_min_target; -+ return max_t(u32, bbr->params.cwnd_min_target, -+ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); ++ down_write(&pcc_ss_data->pcc_lock); ++ /* after writing CPC, transfer the ownership of PCC to platform */ ++ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); ++ up_write(&pcc_ss_data->pcc_lock); ++ return ret; ++ } ++ ++ return cpc_write(cpu, auto_sel_reg, enable); +} ++EXPORT_SYMBOL_GPL(cppc_set_auto_epp); + -+/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss -+ * has drawn us down below target), or snap down to target if we're above it. ++/* ++ * Set Energy Performance Preference Register value through ++ * Performance Controls Interface + */ -+static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, -+ u32 acked, u32 bw, int gain, u32 cwnd, -+ struct bbr_context *ctx) ++int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; ++ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); ++ struct cpc_register_resource *epp_set_reg; ++ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); ++ struct cppc_pcc_data *pcc_ss_data = NULL; ++ int ret = -EINVAL; + -+ if (!acked) -+ goto done; /* no packet fully ACKed; just apply caps */ ++ if (!cpc_desc) { ++ pr_warn("No CPC descriptor for CPU:%d\n", cpu); ++ return -EINVAL; ++ } + -+ target_cwnd = bbr_bdp(sk, bw, gain); ++ epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + -+ /* Increment the cwnd to account for excess ACKed data that seems -+ * due to aggregation (of data and/or ACKs) visible in the ACK stream. -+ */ -+ target_cwnd += bbr_ack_aggregation_cwnd(sk); -+ target_cwnd = bbr_quantization_budget(sk, target_cwnd); ++ if (CPC_IN_PCC(epp_set_reg)) { ++ if (pcc_ss_id < 0) ++ return -EIO; + -+ /* If we're below target cwnd, slow start cwnd toward target cwnd. */ -+ bbr->debug.target_cwnd = target_cwnd; ++ ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); ++ if (ret) ++ return ret; + -+ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ -+ bbr->try_fast_path = 0; -+ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ -+ cwnd += acked; -+ if (cwnd >= target_cwnd) { -+ cwnd = target_cwnd; -+ bbr->try_fast_path = 1; -+ } -+ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { -+ cwnd += acked; -+ } else { -+ bbr->try_fast_path = 1; -+ } ++ pcc_ss_data = pcc_data[pcc_ss_id]; + -+ /* When growing cwnd, don't grow beyond twice what we just probed. */ -+ if (bbr->params.usage_based_cwnd) { -+ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); -+ cwnd = min(cwnd, max_probe); ++ down_write(&pcc_ss_data->pcc_lock); ++ /* after writing CPC, transfer the ownership of PCC to platform */ ++ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); ++ up_write(&pcc_ss_data->pcc_lock); + } + -+ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); -+done: -+ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ -+ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ -+ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); -+ -+ ctx->target_cwnd = target_cwnd; -+ ctx->log = (tp->snd_cwnd != prev_cwnd); ++ return ret; +} ++EXPORT_SYMBOL_GPL(cppc_set_epp_perf); + -+/* See if we have reached next round trip */ -+static void bbr_update_round_start(struct sock *sk, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); + /** + * cppc_set_enable - Set to enable CPPC on the processor by writing the + * Continuous Performance Control package EnableRegister field. +@@ -1355,7 +1484,7 @@ int cppc_set_enable(int cpu, bool enable) + pcc_ss_data = pcc_data[pcc_ss_id]; + + down_write(&pcc_ss_data->pcc_lock); +- /* after writing CPC, transfer the ownership of PCC to platfrom */ ++ /* after writing CPC, transfer the ownership of PCC to platform */ + ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); + up_write(&pcc_ss_data->pcc_lock); + return ret; +diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c +index cf8c7fd59ada..ad9bb5353dd3 100644 +--- a/drivers/ata/libahci.c ++++ b/drivers/ata/libahci.c +@@ -33,14 +33,14 @@ + #include "libata.h" + + static int ahci_skip_host_reset; +-int ahci_ignore_sss; ++int ahci_ignore_sss=1; + EXPORT_SYMBOL_GPL(ahci_ignore_sss); + + module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444); + MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)"); + + module_param_named(ignore_sss, ahci_ignore_sss, int, 0444); +-MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)"); ++MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])"); + + static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy, + unsigned hints); +diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c +index 46cbe4471e78..dd90591e51ba 100644 +--- a/drivers/base/arch_topology.c ++++ b/drivers/base/arch_topology.c +@@ -353,7 +353,7 @@ void topology_init_cpu_capacity_cppc(void) + struct cppc_perf_caps perf_caps; + int cpu; + +- if (likely(acpi_disabled || !acpi_cpc_valid())) ++ if (likely(!acpi_cpc_valid())) + return; + + raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), +diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c +index 7c3590fd97c2..bb4880e10581 100644 +--- a/drivers/base/firmware_loader/main.c ++++ b/drivers/base/firmware_loader/main.c +@@ -470,6 +470,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, + static char fw_path_para[256]; + static const char * const fw_path[] = { + fw_path_para, ++ "/etc/firmware/" UTS_RELEASE, ++ "/etc/firmware", + "/lib/firmware/updates/" UTS_RELEASE, + "/lib/firmware/updates", + "/lib/firmware/" UTS_RELEASE, +diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig +index d4100b0c083e..8739ea13161f 100644 +--- a/drivers/block/zram/Kconfig ++++ b/drivers/block/zram/Kconfig +@@ -78,3 +78,21 @@ config ZRAM_MEMORY_TRACKING + /sys/kernel/debug/zram/zramX/block_state. + + See Documentation/admin-guide/blockdev/zram.rst for more information. + -+ bbr->round_start = 0; ++config ZRAM_ENTROPY ++ bool "Use entropy optimization for zram" ++ depends on ZRAM && ZRAM_DEF_COMP_ZSTD ++ help ++ With this feature, entropy will be calculated for each page. ++ Pages above ZRAM_ENTROPY_THRESHOLD entropy will be ++ stored uncompressed. Use this feature if you need a performance ++ boost and a small loss in compression. + -+ /* See if we've reached the next RTT */ -+ if (rs->interval_us > 0 && -+ !before(rs->prior_delivered, bbr->next_rtt_delivered)) { -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr->round_start = 1; -+ } -+} ++config ZRAM_ENTROPY_THRESHOLD ++ int ++ depends on ZRAM && ZRAM_ENTROPY ++ default 100000 if ZRAM_DEF_COMP_ZSTD ++ help ++ Pages with entropy above ZRAM_ENTROPY_THRESHOLD will be stored ++ uncompressed. The default value was chosen as a result a lot of ++ experiments. You can try set your own value. +diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c +index 226ea76cc819..d02e37e7afd9 100644 +--- a/drivers/block/zram/zram_drv.c ++++ b/drivers/block/zram/zram_drv.c +@@ -1347,6 +1347,35 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, + return ret; + } + + -+/* Calculate the bandwidth based on how fast packets are delivered */ -+static void bbr_calculate_bw_sample(struct sock *sk, -+ const struct rate_sample *rs, struct bbr_context *ctx) ++#ifdef CONFIG_ZRAM_ENTROPY ++static inline u32 ilog2_w(u64 n) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw = 0; -+ -+ /* Divide delivered by the interval to find a (lower bound) bottleneck -+ * bandwidth sample. Delivered is in packets and interval_us in uS and -+ * ratio will be <<1 for most connections. So delivered is first scaled. -+ * Round up to allow growth at low rates, even with integer division. -+ */ -+ if (rs->interval_us > 0) { -+ if (WARN_ONCE(rs->delivered < 0, -+ "negative delivered: %d interval_us: %ld\n", -+ rs->delivered, rs->interval_us)) -+ return; -+ -+ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); -+ } -+ -+ ctx->sample_bw = bw; -+ bbr->debug.rs_bw = bw; ++ return ilog2(n * n * n * n); +} + -+/* Estimates the windowed max degree of ack aggregation. -+ * This is used to provision extra in-flight data to keep sending during -+ * inter-ACK silences. -+ * -+ * Degree of ack aggregation is estimated as extra data acked beyond expected. -+ * -+ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" -+ * cwnd += max_extra_acked -+ * -+ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). -+ * Max filter is an approximate sliding window of 5-10 (packet timed) round -+ * trips for non-startup phase, and 1-2 round trips for startup. -+ */ -+static void bbr_update_ack_aggregation(struct sock *sk, -+ const struct rate_sample *rs) ++static inline s32 shannon_entropy(const u8 *src) +{ -+ u32 epoch_us, expected_acked, extra_acked; -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct tcp_sock *tp = tcp_sk(sk); -+ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; -+ -+ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || -+ rs->delivered < 0 || rs->interval_us <= 0) -+ return; ++ s32 entropy_sum = 0; ++ u32 sz_base, i; ++ u16 entropy_count[256] = { 0 }; + -+ if (bbr->round_start) { -+ bbr->extra_acked_win_rtts = min(0x1F, -+ bbr->extra_acked_win_rtts + 1); -+ if (bbr->params.extra_acked_in_startup && -+ !bbr_full_bw_reached(sk)) -+ extra_acked_win_rtts_thresh = 1; -+ if (bbr->extra_acked_win_rtts >= -+ extra_acked_win_rtts_thresh) { -+ bbr->extra_acked_win_rtts = 0; -+ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? -+ 0 : 1; -+ bbr->extra_acked[bbr->extra_acked_win_idx] = 0; -+ } -+ } ++ for (i = 0; i < PAGE_SIZE; ++i) ++ entropy_count[src[i]]++; + -+ /* Compute how many packets we expected to be delivered over epoch. */ -+ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, -+ bbr->ack_epoch_mstamp); -+ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; ++ sz_base = ilog2_w(PAGE_SIZE); ++ for (i = 0; i < ARRAY_SIZE(entropy_count); ++i) { ++ if (entropy_count[i] > 0) { ++ s32 p = entropy_count[i]; + -+ /* Reset the aggregation epoch if ACK rate is below expected rate or -+ * significantly large no. of ack received since epoch (potentially -+ * quite old epoch). -+ */ -+ if (bbr->ack_epoch_acked <= expected_acked || -+ (bbr->ack_epoch_acked + rs->acked_sacked >= -+ bbr_ack_epoch_acked_reset_thresh)) { -+ bbr->ack_epoch_acked = 0; -+ bbr->ack_epoch_mstamp = tp->delivered_mstamp; -+ expected_acked = 0; ++ entropy_sum += p * (sz_base - ilog2_w((u64)p)); ++ } + } + -+ /* Compute excess data delivered, beyond what was expected. */ -+ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, -+ bbr->ack_epoch_acked + rs->acked_sacked); -+ extra_acked = bbr->ack_epoch_acked - expected_acked; -+ extra_acked = min(extra_acked, tp->snd_cwnd); -+ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) -+ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; ++ return entropy_sum; +} ++#endif + -+/* Estimate when the pipe is full, using the change in delivery rate: BBR -+ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by -+ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited -+ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the -+ * higher rwin, 3: we get higher delivery rate samples. Or transient -+ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar -+ * design goal, but uses delay and inter-ACK spacing instead of bandwidth. -+ */ -+static void bbr_check_full_bw_reached(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 bw_thresh; -+ -+ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) -+ return; + static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, + u32 index, struct bio *bio) + { +@@ -1373,7 +1402,17 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, + compress_again: + zstrm = zcomp_stream_get(zram->comp); + src = kmap_atomic(page); + -+ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; -+ if (bbr_max_bw(sk) >= bw_thresh) { -+ bbr->full_bw = bbr_max_bw(sk); -+ bbr->full_bw_cnt = 0; -+ return; -+ } -+ ++bbr->full_bw_cnt; -+ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; -+} ++#ifdef CONFIG_ZRAM_ENTROPY ++ /* Just save this page uncompressible */ ++ if (shannon_entropy((const u8 *)src) > CONFIG_ZRAM_ENTROPY_THRESHOLD) ++ comp_len = PAGE_SIZE; ++ else ++ ret = zcomp_compress(zstrm, src, &comp_len); ++#else + ret = zcomp_compress(zstrm, src, &comp_len); ++#endif + -+/* If pipe is probably full, drain the queue and then enter steady-state. */ -+static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); + kunmap_atomic(src); + + if (unlikely(ret)) { +diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c +index 9ac75c1cde9c..e8cd2d2b7cf8 100644 +--- a/drivers/cpufreq/amd-pstate.c ++++ b/drivers/cpufreq/amd-pstate.c +@@ -31,23 +31,18 @@ + #include + #include + #include +-#include + #include + #include + #include + #include + +-#include + #include + + #include +-#include +-#include +-#include + #include "amd-pstate-trace.h" + +-#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 +-#define AMD_PSTATE_TRANSITION_DELAY 500 ++#define AMD_PSTATE_TRANSITION_LATENCY 20000 ++#define AMD_PSTATE_TRANSITION_DELAY 1000 + + /* + * TODO: We need more time to fine tune processors with shared memory solution +@@ -63,7 +58,13 @@ module_param(shared_mem, bool, 0444); + MODULE_PARM_DESC(shared_mem, + "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); + +-static struct cpufreq_driver amd_pstate_driver; ++static bool epp_enabled = true; ++module_param(epp_enabled, bool, 0444); ++MODULE_PARM_DESC(epp_enabled, ++ "load amd_pstate or amd_pstate_epp (true = amd_pstate_epp driver instance (default), false = amd_pstate driver instance)"); + -+ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { -+ bbr->mode = BBR_DRAIN; /* drain queue we created */ -+ tcp_sk(sk)->snd_ssthresh = -+ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); -+ bbr2_reset_congestion_signals(sk); -+ } /* fall through to check if in-flight is already small: */ -+ if (bbr->mode == BBR_DRAIN && -+ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= -+ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) -+ return true; /* exiting DRAIN now */ -+ return false; -+} -+ -+static void bbr_check_probe_rtt_done(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); ++static struct cpufreq_driver *default_pstate_driver; ++static struct amd_cpudata **all_cpu_data; + + /** + * struct amd_aperf_mperf +@@ -75,6 +76,7 @@ struct amd_aperf_mperf { + u64 aperf; + u64 mperf; + u64 tsc; ++ u64 time; + }; + + /** +@@ -97,6 +99,20 @@ struct amd_aperf_mperf { + * @prev: Last Aperf/Mperf/tsc count value read from register + * @freq: current cpu frequency value + * @boost_supported: check whether the Processor or SBIOS supports boost mode ++ * @precision_boost_off: the core performance boost disabled state ++ * @cppc_hw_conf_cached: the cached hardware configuration register ++ * @epp_powersave: Last saved CPPC energy performance preference ++ * when policy switched to performance ++ * @epp_policy: Last saved policy used to set energy-performance preference ++ * @epp_cached: Cached CPPC energy-performance preference value ++ * @policy: Cpufreq policy value ++ * @sched_flags: Store scheduler flags for possible cross CPU update ++ * @update_util_set: CPUFreq utility callback is set ++ * @last_update: Time stamp of the last performance state update ++ * @cppc_boost_min: Last CPPC boosted min performance state ++ * @cppc_cap1_cached: Cached value of the last CPPC Capabilities MSR ++ * @update_util: Cpufreq utility callback information ++ * @suspended: Whether or not the driver has been suspended. + * + * The amd_cpudata is key private data for each CPU thread in AMD P-State, and + * represents all the attributes and goals that AMD P-State requests at runtime. +@@ -120,10 +136,200 @@ struct amd_cpudata { + struct amd_aperf_mperf cur; + struct amd_aperf_mperf prev; + +- u64 freq; ++ u64 freq; + bool boost_supported; ++ bool precision_boost_off; ++ u64 cppc_hw_conf_cached; + -+ if (!(bbr->probe_rtt_done_stamp && -+ after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) -+ return; ++ /* EPP feature related attributes*/ ++ s16 epp_powersave; ++ s16 epp_policy; ++ s16 epp_cached; ++ u32 policy; ++ u32 sched_flags; ++ bool update_util_set; ++ u64 last_update; ++ u64 last_io_update; ++ u32 cppc_boost_min; ++ u64 cppc_cap1_cached; ++ struct update_util_data update_util; ++ struct amd_aperf_mperf sample; ++ bool suspended; ++}; + -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr2_exit_probe_rtt(sk); -+} ++/** ++ * struct amd_pstate_params - global parameters for the performance control ++ * @ cppc_boost_disabled Wheter or not the core performance boost disabled ++ */ ++struct amd_pstate_params { ++ bool cppc_boost_disabled; ++}; + -+/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and -+ * periodically drain the bottleneck queue, to converge to measure the true -+ * min_rtt (unloaded propagation delay). This allows the flows to keep queues -+ * small (reducing queuing delay and packet loss) and achieve fairness among -+ * BBR flows. -+ * -+ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, -+ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. -+ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed -+ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and -+ * re-enter the previous mode. BBR uses 200ms to approximately bound the -+ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). -+ * -+ * Note that flows need only pay 2% if they are busy sending over the last 10 -+ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have -+ * natural silences or low-rate periods within 10 seconds where the rate is low -+ * enough for long enough to drain its queue in the bottleneck. We pick up -+ * these min RTT measurements opportunistically with our min_rtt filter. :-) ++/* ++ * AMD Energy Preference Performance (EPP) ++ * The EPP is used in the CCLK DPM controller to drive ++ * the frequency that a core is going to operate during ++ * short periods of activity. EPP values will be utilized for ++ * different OS profiles (balanced, performance, power savings) ++ * display strings corresponding to EPP index in the ++ * energy_perf_strings[] ++ * index String ++ *------------------------------------- ++ * 0 default ++ * 1 performance ++ * 2 balance_performance ++ * 3 balance_power ++ * 4 power + */ -+static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool probe_rtt_expired, min_rtt_expired; -+ u32 expire; ++ enum energy_perf_value_index { ++ EPP_INDEX_DEFAULT = 0, ++ EPP_INDEX_PERFORMANCE, ++ EPP_INDEX_BALANCE_PERFORMANCE, ++ EPP_INDEX_BALANCE_POWERSAVE, ++ EPP_INDEX_POWERSAVE, ++}; + -+ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ -+ expire = bbr->probe_rtt_min_stamp + -+ msecs_to_jiffies(bbr->params.probe_rtt_win_ms); -+ probe_rtt_expired = after(tcp_jiffies32, expire); -+ if (rs->rtt_us >= 0 && -+ (rs->rtt_us <= bbr->probe_rtt_min_us || -+ (probe_rtt_expired && !rs->is_ack_delayed))) { -+ bbr->probe_rtt_min_us = rs->rtt_us; -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; -+ } -+ /* Track min RTT seen in the min_rtt_win_sec filter window: */ -+ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; -+ min_rtt_expired = after(tcp_jiffies32, expire); -+ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || -+ min_rtt_expired) { -+ bbr->min_rtt_us = bbr->probe_rtt_min_us; -+ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; -+ } ++static const char * const energy_perf_strings[] = { ++ [EPP_INDEX_DEFAULT] = "default", ++ [EPP_INDEX_PERFORMANCE] = "performance", ++ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", ++ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", ++ [EPP_INDEX_POWERSAVE] = "power", ++ NULL + }; + ++static unsigned int epp_values[] = { ++ [EPP_INDEX_DEFAULT] = 0, ++ [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, ++ [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, ++ [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, ++ [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, ++}; + -+ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && -+ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { -+ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ -+ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ -+ bbr->probe_rtt_done_stamp = 0; -+ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; -+ bbr->next_rtt_delivered = tp->delivered; -+ } ++static struct amd_pstate_params global; + -+ if (bbr->mode == BBR_PROBE_RTT) { -+ /* Ignore low rate samples during this mode. */ -+ tp->app_limited = -+ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; -+ /* Maintain min packets in flight for max(200 ms, 1 round). */ -+ if (!bbr->probe_rtt_done_stamp && -+ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { -+ bbr->probe_rtt_done_stamp = tcp_jiffies32 + -+ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); -+ bbr->probe_rtt_round_done = 0; -+ bbr->next_rtt_delivered = tp->delivered; -+ } else if (bbr->probe_rtt_done_stamp) { -+ if (bbr->round_start) -+ bbr->probe_rtt_round_done = 1; -+ if (bbr->probe_rtt_round_done) -+ bbr_check_probe_rtt_done(sk); ++static DEFINE_MUTEX(amd_pstate_limits_lock); ++static DEFINE_MUTEX(amd_pstate_driver_lock); ++static DEFINE_SPINLOCK(amd_pstate_cpu_lock); ++ ++static bool cppc_boost __read_mostly; ++struct kobject *amd_pstate_kobj; ++ ++#ifdef CONFIG_ACPI_CPPC_LIB ++static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) ++{ ++ s16 epp; ++ struct cppc_perf_caps perf_caps; ++ int ret; ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ if (!cppc_req_cached) { ++ epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, ++ &cppc_req_cached); ++ if (epp) ++ return epp; ++ } ++ epp = (cppc_req_cached >> 24) & 0xFF; ++ } else { ++ ret = cppc_get_epp_caps(cpudata->cpu, &perf_caps); ++ if (ret < 0) { ++ pr_debug("Could not retrieve energy perf value (%d)\n", ret); ++ return -EIO; + } ++ epp = (s16) perf_caps.energy_perf; + } -+ /* Restart after idle ends only once we process a new S/ACK for data */ -+ if (rs->delivered > 0) -+ bbr->idle_restart = 0; ++ ++ return epp; +} ++#endif + -+static void bbr_update_gains(struct sock *sk) ++static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata, int *raw_epp) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ s16 epp; ++ int index = -EINVAL; + -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ bbr->pacing_gain = bbr->params.high_gain; -+ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; ++ *raw_epp = 0; ++ epp = amd_pstate_get_epp(cpudata, 0); ++ if (epp < 0) ++ return epp; ++ ++ switch (epp) { ++ case AMD_CPPC_EPP_PERFORMANCE: ++ index = EPP_INDEX_PERFORMANCE; + break; -+ case BBR_DRAIN: -+ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ -+ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ ++ case AMD_CPPC_EPP_BALANCE_PERFORMANCE: ++ index = EPP_INDEX_BALANCE_PERFORMANCE; + break; -+ case BBR_PROBE_BW: -+ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; -+ bbr->cwnd_gain = bbr->params.cwnd_gain; ++ case AMD_CPPC_EPP_BALANCE_POWERSAVE: ++ index = EPP_INDEX_BALANCE_POWERSAVE; + break; -+ case BBR_PROBE_RTT: -+ bbr->pacing_gain = BBR_UNIT; -+ bbr->cwnd_gain = BBR_UNIT; ++ case AMD_CPPC_EPP_POWERSAVE: ++ index = EPP_INDEX_POWERSAVE; + break; + default: -+ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); -+ break; ++ *raw_epp = epp; ++ index = 0; + } -+} -+ -+static void bbr_init(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ int i; -+ -+ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); -+ -+ bbr->initialized = 1; -+ bbr->params.high_gain = min(0x7FF, bbr_high_gain); -+ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); -+ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); -+ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); -+ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); -+ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); -+ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); -+ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); -+ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); -+ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); -+ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); -+ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); -+ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; -+ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; -+ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; -+ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); -+ bbr->params.probe_rtt_win_ms = -+ min(0x3FFFU, -+ min_t(u32, bbr_probe_rtt_win_ms, -+ bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); -+ for (i = 0; i < CYCLE_LEN; i++) -+ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); -+ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; -+ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); -+ -+ bbr->debug.snd_isn = tp->snd_una; -+ bbr->debug.target_cwnd = 0; -+ bbr->debug.undo = 0; -+ -+ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); -+ bbr->prior_cwnd = tp->prior_cwnd; -+ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; -+ bbr->next_rtt_delivered = 0; -+ bbr->prev_ca_state = TCP_CA_Open; -+ bbr->packet_conservation = 0; -+ -+ bbr->probe_rtt_done_stamp = 0; -+ bbr->probe_rtt_round_done = 0; -+ bbr->probe_rtt_min_us = tcp_min_rtt(tp); -+ bbr->probe_rtt_min_stamp = tcp_jiffies32; -+ bbr->min_rtt_us = tcp_min_rtt(tp); -+ bbr->min_rtt_stamp = tcp_jiffies32; -+ -+ bbr->has_seen_rtt = 0; -+ bbr_init_pacing_rate_from_rtt(sk); -+ -+ bbr->round_start = 0; -+ bbr->idle_restart = 0; -+ bbr->full_bw_reached = 0; -+ bbr->full_bw = 0; -+ bbr->full_bw_cnt = 0; -+ bbr->cycle_mstamp = 0; -+ bbr->cycle_idx = 0; -+ bbr->mode = BBR_STARTUP; -+ bbr->debug.rs_bw = 0; -+ -+ bbr->ack_epoch_mstamp = tp->tcp_mstamp; -+ bbr->ack_epoch_acked = 0; -+ bbr->extra_acked_win_rtts = 0; -+ bbr->extra_acked_win_idx = 0; -+ bbr->extra_acked[0] = 0; -+ bbr->extra_acked[1] = 0; -+ -+ bbr->ce_state = 0; -+ bbr->prior_rcv_nxt = tp->rcv_nxt; -+ bbr->try_fast_path = 0; -+ -+ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); -+} -+ -+static u32 bbr_sndbuf_expand(struct sock *sk) -+{ -+ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ -+ return 3; -+} -+ -+/* __________________________________________________________________________ -+ * -+ * Functions new to BBR v2 ("bbr") congestion control are below here. -+ * __________________________________________________________________________ -+ */ + -+/* Incorporate a new bw sample into the current window of our max filter. */ -+static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); ++ return index; +} + -+/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ -+static void bbr2_advance_bw_hi_filter(struct sock *sk) ++#ifdef CONFIG_ACPI_CPPC_LIB ++static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (!bbr->bw_hi[1]) -+ return; /* no samples in this window; remember old window */ -+ bbr->bw_hi[0] = bbr->bw_hi[1]; -+ bbr->bw_hi[1] = 0; -+} ++ int ret; ++ struct cppc_perf_ctrls perf_ctrls; + -+/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ -+static u32 bbr2_target_inflight(struct sock *sk) -+{ -+ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ u64 value = READ_ONCE(cpudata->cppc_req_cached); + -+ return min(bdp, tcp_sk(sk)->snd_cwnd); ++ value &= ~GENMASK_ULL(31, 24); ++ value |= (u64)epp << 24; ++ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ if (!ret) { ++ cpudata->epp_cached = epp; ++ } ++ } else { ++ perf_ctrls.energy_perf = epp; ++ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls); ++ if (ret){ ++ pr_debug("failed to set energy perf value (%d)\n", ret); ++ return ret; ++ } ++ cpudata->epp_cached = epp; ++ } ++ return ret; +} + -+static bool bbr2_is_probing_bandwidth(struct sock *sk) ++static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, ++ int pref_index, bool use_raw, ++ u32 raw_epp) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ int epp = -EINVAL; ++ int ret; + -+ return (bbr->mode == BBR_STARTUP) || -+ (bbr->mode == BBR_PROBE_BW && -+ (bbr->cycle_idx == BBR_BW_PROBE_REFILL || -+ bbr->cycle_idx == BBR_BW_PROBE_UP)); -+} ++ if (!pref_index) { ++ pr_debug("EPP pref_index is invaid\n"); ++ return -EINVAL; ++ } + -+/* Has the given amount of time elapsed since we marked the phase start? */ -+static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) -+{ -+ const struct tcp_sock *tp = tcp_sk(sk); -+ const struct bbr *bbr = inet_csk_ca(sk); ++ if (use_raw) ++ epp = raw_epp; ++ else if (epp == -EINVAL) ++ epp = epp_values[pref_index]; + -+ return tcp_stamp_us_delta(tp->tcp_mstamp, -+ bbr->cycle_mstamp + interval_us) > 0; -+} ++ if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { ++ pr_debug("EPP cannot be set under performance policy\n"); ++ return -EBUSY; ++ } + -+static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ ret = amd_pstate_set_epp(cpudata, epp); + -+ bbr->full_bw_reached = 1; -+ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); ++ return ret; +} ++#endif + -+/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ -+static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); + static inline int pstate_enable(bool enable) + { + return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); +@@ -131,12 +337,26 @@ static inline int pstate_enable(bool enable) + + static int cppc_enable(bool enable) + { ++ struct cppc_perf_ctrls perf_ctrls; + int cpu, ret = 0; + + for_each_present_cpu(cpu) { + ret = cppc_set_enable(cpu, enable); + if (ret) + return ret; + -+ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || -+ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) -+ return; ++ if (epp_enabled) { ++ /* Enable autonomous mode for EPP */ ++ ret = cppc_set_auto_epp(cpu, enable); ++ if (ret) ++ return ret; + -+ if (ce_ratio >= bbr->params.ecn_thresh) -+ bbr->startup_ecn_rounds++; -+ else -+ bbr->startup_ecn_rounds = 0; ++ /* Set zero to desired perf to allow EPP firmware control*/ ++ perf_ctrls.desired_perf = 0; ++ ret = cppc_set_perf(cpu, &perf_ctrls); ++ if (ret) ++ return ret; ++ } + } + + return ret; +@@ -269,6 +489,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, + u64 prev = READ_ONCE(cpudata->cppc_req_cached); + u64 value = prev; + ++ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); + value &= ~AMD_CPPC_MIN_PERF(~0L); + value |= AMD_CPPC_MIN_PERF(min_perf); + +@@ -312,7 +533,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, + return -ENODEV; + + cap_perf = READ_ONCE(cpudata->highest_perf); +- min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); ++ min_perf = READ_ONCE(cpudata->lowest_perf); + max_perf = cap_perf; + + freqs.old = policy->cur; +@@ -357,8 +578,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, + if (max_perf < min_perf) + max_perf = min_perf; + +- des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); +- + amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); + } + +@@ -438,18 +657,27 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) + { + struct amd_cpudata *cpudata = policy->driver_data; + int ret; ++ u64 value; + + if (!cpudata->boost_supported) { + pr_err("Boost mode is not supported by this processor or SBIOS\n"); + return -EINVAL; + } + +- if (state) ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); ++ if (ret) ++ return ret; + -+ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { -+ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ -+ bbr2_handle_queue_too_high_in_startup(sk); -+ return; ++ if (state) { ++ value |= AMD_CPPC_PRECISION_BOOST_ENABLED; + policy->cpuinfo.max_freq = cpudata->max_freq; +- else ++ } else { ++ value &= ~AMD_CPPC_PRECISION_BOOST_ENABLED; + policy->cpuinfo.max_freq = cpudata->nominal_freq; +- ++ } + policy->max = policy->cpuinfo.max_freq; ++ WRITE_ONCE(cpudata->cppc_hw_conf_cached, value); ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, value); + + ret = freq_qos_update_request(&cpudata->req[1], + policy->cpuinfo.max_freq); +@@ -470,7 +698,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) + return; + + cpudata->boost_supported = true; +- amd_pstate_driver.boost_enabled = true; ++ default_pstate_driver->boost_enabled = true; + } + + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) +@@ -478,6 +706,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) + int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; + struct device *dev; + struct amd_cpudata *cpudata; ++ u64 value; + + dev = get_cpu_device(policy->cpu); + if (!dev) +@@ -541,7 +770,17 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) + cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; + + policy->driver_data = cpudata; ++ if (!shared_mem) { ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); ++ if (ret) ++ return ret; ++ cpudata->precision_boost_off = value & AMD_CPPC_PRECISION_BOOST_ENABLED; + ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); ++ if (ret) ++ return ret; ++ WRITE_ONCE(cpudata->cppc_req_cached, value); + } + amd_pstate_boost_init(cpudata); + + return 0; +@@ -555,9 +794,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) + + static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) + { +- struct amd_cpudata *cpudata; +- +- cpudata = policy->driver_data; ++ struct amd_cpudata *cpudata = policy->driver_data; + + freq_qos_remove_request(&cpudata->req[1]); + freq_qos_remove_request(&cpudata->req[0]); +@@ -599,9 +836,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, + char *buf) + { + int max_freq; +- struct amd_cpudata *cpudata; +- +- cpudata = policy->driver_data; ++ struct amd_cpudata *cpudata = policy->driver_data; + + max_freq = amd_get_max_freq(cpudata); + if (max_freq < 0) +@@ -614,9 +849,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli + char *buf) + { + int freq; +- struct amd_cpudata *cpudata; +- +- cpudata = policy->driver_data; ++ struct amd_cpudata *cpudata = policy->driver_data; + + freq = amd_get_lowest_nonlinear_freq(cpudata); + if (freq < 0) +@@ -640,10 +873,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, + return sprintf(&buf[0], "%u\n", perf); + } + ++static ssize_t show_energy_performance_available_preferences( ++ struct cpufreq_policy *policy, char *buf) ++{ ++ int i = 0; ++ int ret = 0; ++ ++ while (energy_perf_strings[i] != NULL) ++ ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); ++ ++ ret += sprintf(&buf[ret], "\n"); ++ ++ return ret; +} + -+static int bbr2_update_ecn_alpha(struct sock *sk) ++static ssize_t store_energy_performance_preference( ++ struct cpufreq_policy *policy, const char *buf, size_t count) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ s32 delivered, delivered_ce; -+ u64 alpha, ce_ratio; -+ u32 gain; -+ -+ if (bbr->params.ecn_factor == 0) -+ return -1; ++ struct amd_cpudata *cpudata = policy->driver_data; ++ char str_preference[21]; ++ bool raw = false; ++ ssize_t ret; ++ u32 epp = 0; + -+ delivered = tp->delivered - bbr->alpha_last_delivered; -+ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; ++ ret = sscanf(buf, "%20s", str_preference); ++ if (ret != 1) ++ return -EINVAL; + -+ if (delivered == 0 || /* avoid divide by zero */ -+ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ -+ return -1; ++ ret = match_string(energy_perf_strings, -1, str_preference); ++ if (ret < 0) { ++ ret = kstrtouint(buf, 10, &epp); ++ if (ret) ++ return ret; + -+ /* See if we should use ECN sender logic for this connection. */ -+ if (!bbr->ecn_eligible && bbr_ecn_enable && -+ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || -+ !bbr->params.ecn_max_rtt_us)) -+ bbr->ecn_eligible = 1; ++ if ((epp > 255) || (epp < 0)) ++ return -EINVAL; + -+ ce_ratio = (u64)delivered_ce << BBR_SCALE; -+ do_div(ce_ratio, delivered); -+ gain = bbr->params.ecn_alpha_gain; -+ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; -+ alpha += (gain * ce_ratio) >> BBR_SCALE; -+ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); ++ raw = true; ++ } + -+ bbr->alpha_last_delivered = tp->delivered; -+ bbr->alpha_last_delivered_ce = tp->delivered_ce; ++ mutex_lock(&amd_pstate_limits_lock); ++ ret = amd_pstate_set_energy_pref_index(cpudata, ret, raw, epp); ++ mutex_unlock(&amd_pstate_limits_lock); + -+ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); -+ return (int)ce_ratio; ++ return ret ?: count; +} + -+/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ -+static void bbr2_raise_inflight_hi_slope(struct sock *sk) ++static ssize_t show_energy_performance_preference( ++ struct cpufreq_policy *policy, char *buf) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 growth_this_round, cnt; ++ struct amd_cpudata *cpudata = policy->driver_data; ++ int preference, raw_epp; + -+ /* Calculate "slope": packets S/Acked per inflight_hi increment. */ -+ growth_this_round = 1 << bbr->bw_probe_up_rounds; -+ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); -+ cnt = tp->snd_cwnd / growth_this_round; -+ cnt = max(cnt, 1U); -+ bbr->bw_probe_up_cnt = cnt; -+ bbr->debug.event = 'G'; /* Grow inflight_hi slope */ ++ preference = amd_pstate_get_energy_pref_index(cpudata, &raw_epp); ++ if (preference < 0) ++ return preference; ++ ++ if (raw_epp) ++ return sprintf(buf, "%d\n", raw_epp); ++ else ++ return sprintf(buf, "%s\n", energy_perf_strings[preference]); +} + -+/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ -+static void bbr2_probe_inflight_hi_upward(struct sock *sk, -+ const struct rate_sample *rs) ++static void amd_pstate_update_policies(void) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 delta; -+ -+ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { -+ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ -+ return; /* not fully using inflight_hi, so don't grow it */ -+ } ++ int cpu; + -+ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ -+ bbr->bw_probe_up_acks += rs->acked_sacked; -+ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { -+ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; -+ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; -+ bbr->inflight_hi += delta; -+ bbr->debug.event = 'I'; /* Increment inflight_hi */ -+ } ++ for_each_possible_cpu(cpu) ++ cpufreq_update_policy(cpu); ++} + -+ if (bbr->round_start) -+ bbr2_raise_inflight_hi_slope(sk); ++static ssize_t show_pstate_dynamic_boost(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "%u\n", cppc_boost); +} + -+/* Does loss/ECN rate for this sample say inflight is "too high"? -+ * This is used by both the bbr_check_loss_too_high_in_startup() function, -+ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which -+ * uses it to notice when loss/ECN rates suggest inflight is too high. -+ */ -+static bool bbr2_is_inflight_too_high(const struct sock *sk, -+ const struct rate_sample *rs) ++static ssize_t store_pstate_dynamic_boost(struct kobject *a, ++ struct kobj_attribute *b, ++ const char *buf, size_t count) +{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ u32 loss_thresh, ecn_thresh; ++ unsigned int input; ++ int ret; + -+ if (rs->lost > 0 && rs->tx_in_flight) { -+ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> -+ BBR_SCALE; -+ if (rs->lost > loss_thresh) -+ return true; -+ } ++ ret = kstrtouint(buf, 10, &input); ++ if (ret) ++ return ret; + -+ if (rs->delivered_ce > 0 && rs->delivered > 0 && -+ bbr->ecn_eligible && bbr->params.ecn_thresh) { -+ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> -+ BBR_SCALE; -+ if (rs->delivered_ce >= ecn_thresh) -+ return true; -+ } ++ mutex_lock(&amd_pstate_driver_lock); ++ cppc_boost = !!input; ++ amd_pstate_update_policies(); ++ mutex_unlock(&amd_pstate_driver_lock); + -+ return false; ++ return count; +} + -+/* Calculate the tx_in_flight level that corresponded to excessive loss. -+ * We find "lost_prefix" segs of the skb where loss rate went too high, -+ * by solving for "lost_prefix" in the following equation: -+ * lost / inflight >= loss_thresh -+ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh -+ * Then we take that equation, convert it to fixed point, and -+ * round up to the nearest packet. -+ */ -+static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, -+ const struct rate_sample *rs, -+ const struct sk_buff *skb) -+{ -+ const struct bbr *bbr = inet_csk_ca(sk); -+ u32 loss_thresh = bbr->params.loss_thresh; -+ u32 pcount, divisor, inflight_hi; -+ s32 inflight_prev, lost_prev; -+ u64 loss_budget, lost_prefix; -+ -+ pcount = tcp_skb_pcount(skb); + cpufreq_freq_attr_ro(amd_pstate_max_freq); + cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); + + cpufreq_freq_attr_ro(amd_pstate_highest_perf); ++cpufreq_freq_attr_rw(energy_performance_preference); ++cpufreq_freq_attr_ro(energy_performance_available_preferences); ++define_one_global_rw(pstate_dynamic_boost); + + static struct freq_attr *amd_pstate_attr[] = { + &amd_pstate_max_freq, +@@ -652,6 +983,548 @@ static struct freq_attr *amd_pstate_attr[] = { + NULL, + }; + ++static struct freq_attr *amd_pstate_epp_attr[] = { ++ &amd_pstate_max_freq, ++ &amd_pstate_lowest_nonlinear_freq, ++ &amd_pstate_highest_perf, ++ &energy_performance_preference, ++ &energy_performance_available_preferences, ++ NULL, ++}; + -+ /* How much data was in flight before this skb? */ -+ inflight_prev = rs->tx_in_flight - pcount; -+ if (WARN_ONCE(inflight_prev < 0, -+ "tx_in_flight: %u pcount: %u reneg: %u", -+ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) -+ return ~0U; ++static struct attribute *pstate_global_attributes[] = { ++ &pstate_dynamic_boost.attr, ++ NULL ++}; + -+ /* How much inflight data was marked lost before this skb? */ -+ lost_prev = rs->lost - pcount; -+ if (WARN_ON_ONCE(lost_prev < 0)) -+ return ~0U; ++static const struct attribute_group amd_pstate_global_attr_group = { ++ .attrs = pstate_global_attributes, ++}; + -+ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ -+ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; -+ loss_budget >>= BBR_SCALE; -+ if (lost_prev >= loss_budget) { -+ lost_prefix = 0; /* previous losses crossed loss_thresh */ -+ } else { -+ lost_prefix = loss_budget - lost_prev; -+ lost_prefix <<= BBR_SCALE; -+ divisor = BBR_UNIT - loss_thresh; -+ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ -+ return ~0U; -+ do_div(lost_prefix, divisor); -+ } ++static inline void update_boost_state(void) ++{ ++ u64 misc_en; ++ struct amd_cpudata *cpudata; + -+ inflight_hi = inflight_prev + lost_prefix; -+ return inflight_hi; ++ cpudata = all_cpu_data[0]; ++ rdmsrl(MSR_AMD_CPPC_HW_CTL, misc_en); ++ global.cppc_boost_disabled = misc_en & AMD_CPPC_PRECISION_BOOST_ENABLED; +} + -+/* If loss/ECN rates during probing indicated we may have overfilled a -+ * buffer, return an operating point that tries to leave unutilized headroom in -+ * the path for other flows, for fairness convergence and lower RTTs and loss. -+ */ -+static u32 bbr2_inflight_with_headroom(const struct sock *sk) ++static int amd_pstate_init_cpu(unsigned int cpunum) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 headroom, headroom_fraction; ++ struct amd_cpudata *cpudata; + -+ if (bbr->inflight_hi == ~0U) -+ return ~0U; ++ cpudata = all_cpu_data[cpunum]; ++ if (!cpudata) { ++ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); ++ if (!cpudata) ++ return -ENOMEM; ++ WRITE_ONCE(all_cpu_data[cpunum], cpudata); + -+ headroom_fraction = bbr->params.inflight_headroom; -+ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; -+ headroom = max(headroom, 1U); -+ return max_t(s32, bbr->inflight_hi - headroom, -+ bbr->params.cwnd_min_target); ++ cpudata->cpu = cpunum; ++ } ++ cpudata->epp_powersave = -EINVAL; ++ cpudata->epp_policy = 0; ++ pr_debug("controlling: cpu %d\n", cpunum); ++ return 0; +} + -+/* Bound cwnd to a sensible level, based on our current probing state -+ * machine phase and model of a good inflight level (inflight_lo, inflight_hi). -+ */ -+static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) ++static int __amd_pstate_cpu_init(struct cpufreq_policy *policy) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 cap; ++ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; ++ struct amd_cpudata *cpudata; ++ struct device *dev; ++ int rc; ++ u64 value; + -+ /* tcp_rcv_synsent_state_process() currently calls tcp_ack() -+ * and thus cong_control() without first initializing us(!). -+ */ -+ if (!bbr->initialized) -+ return; ++ rc = amd_pstate_init_cpu(policy->cpu); ++ if (rc) ++ return rc; + -+ cap = ~0U; -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { -+ /* Probe to see if more packets fit in the path. */ -+ cap = bbr->inflight_hi; -+ } else { -+ if (bbr->mode == BBR_PROBE_RTT || -+ (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) -+ cap = bbr2_inflight_with_headroom(sk); ++ cpudata = all_cpu_data[policy->cpu]; ++ ++ dev = get_cpu_device(policy->cpu); ++ if (!dev) ++ goto free_cpudata1; ++ ++ rc = amd_pstate_init_perf(cpudata); ++ if (rc) ++ goto free_cpudata1; ++ ++ min_freq = amd_get_min_freq(cpudata); ++ max_freq = amd_get_max_freq(cpudata); ++ nominal_freq = amd_get_nominal_freq(cpudata); ++ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); ++ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { ++ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", ++ min_freq, max_freq); ++ ret = -EINVAL; ++ goto free_cpudata1; + } -+ /* Adapt to any loss/ECN since our last bw probe. */ -+ cap = min(cap, bbr->inflight_lo); + -+ cap = max_t(u32, cap, bbr->params.cwnd_min_target); -+ tp->snd_cwnd = min(cap, tp->snd_cwnd); -+} ++ policy->min = min_freq; ++ policy->max = max_freq; + -+/* Estimate a short-term lower bound on the capacity available now, based -+ * on measurements of the current delivery process and recent history. When we -+ * are seeing loss/ECN at times when we are not probing bw, then conservatively -+ * move toward flow balance by multiplicatively cutting our short-term -+ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a -+ * multiplicative decrease in order to converge to a lower capacity in time -+ * logarithmic in the magnitude of the decrease. -+ * -+ * However, we do not cut our short-term estimates lower than the current rate -+ * and volume of delivered data from this round trip, since from the current -+ * delivery process we can estimate the measured capacity available now. -+ * -+ * Anything faster than that approach would knowingly risk high loss, which can -+ * cause low bw for Reno/CUBIC and high loss recovery latency for -+ * request/response flows using any congestion control. -+ */ -+static void bbr2_adapt_lower_bounds(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 ecn_cut, ecn_inflight_lo, beta; ++ policy->cpuinfo.min_freq = min_freq; ++ policy->cpuinfo.max_freq = max_freq; ++ /* It will be updated by governor */ ++ policy->cur = policy->cpuinfo.min_freq; + -+ /* We only use lower-bound estimates when not probing bw. -+ * When probing we need to push inflight higher to probe bw. -+ */ -+ if (bbr2_is_probing_bandwidth(sk)) -+ return; ++ /* Initial processor data capability frequencies */ ++ cpudata->max_freq = max_freq; ++ cpudata->min_freq = min_freq; ++ cpudata->nominal_freq = nominal_freq; ++ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; + -+ /* ECN response. */ -+ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { -+ /* Reduce inflight to (1 - alpha*ecn_factor). */ -+ ecn_cut = (BBR_UNIT - -+ ((bbr->ecn_alpha * bbr->params.ecn_factor) >> -+ BBR_SCALE)); -+ if (bbr->inflight_lo == ~0U) -+ bbr->inflight_lo = tp->snd_cwnd; -+ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; -+ } else { -+ ecn_inflight_lo = ~0U; -+ } ++ policy->driver_data = cpudata; + -+ /* Loss response. */ -+ if (bbr->loss_in_round) { -+ /* Reduce bw and inflight to (1 - beta). */ -+ if (bbr->bw_lo == ~0U) -+ bbr->bw_lo = bbr_max_bw(sk); -+ if (bbr->inflight_lo == ~0U) -+ bbr->inflight_lo = tp->snd_cwnd; -+ beta = bbr->params.beta; -+ bbr->bw_lo = -+ max_t(u32, bbr->bw_latest, -+ (u64)bbr->bw_lo * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ bbr->inflight_lo = -+ max_t(u32, bbr->inflight_latest, -+ (u64)bbr->inflight_lo * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ } ++ update_boost_state(); ++ cpudata->epp_cached = amd_pstate_get_epp(cpudata, value); + -+ /* Adjust to the lower of the levels implied by loss or ECN. */ -+ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); -+} ++ policy->min = policy->cpuinfo.min_freq; ++ policy->max = policy->cpuinfo.max_freq; + -+/* Reset any short-term lower-bound adaptation to congestion, so that we can -+ * push our inflight up. -+ */ -+static void bbr2_reset_lower_bounds(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ if (boot_cpu_has(X86_FEATURE_CPPC)) ++ policy->fast_switch_possible = true; + -+ bbr->bw_lo = ~0U; -+ bbr->inflight_lo = ~0U; ++ if (!shared_mem && boot_cpu_has(X86_FEATURE_CPPC)) { ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); ++ if (ret) ++ return ret; ++ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ++ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); ++ if (ret) ++ return ret; ++ WRITE_ONCE(cpudata->cppc_cap1_cached, value); ++ } ++ amd_pstate_boost_init(cpudata); ++ ++ return 0; ++ ++free_cpudata1: ++ kfree(cpudata); ++ return ret; +} + -+/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state -+ * machine phase where we adapt our lower bound based on congestion signals. -+ */ -+static void bbr2_reset_congestion_signals(struct sock *sk) ++static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ int ret; + -+ bbr->loss_in_round = 0; -+ bbr->ecn_in_round = 0; -+ bbr->loss_in_cycle = 0; -+ bbr->ecn_in_cycle = 0; -+ bbr->bw_latest = 0; -+ bbr->inflight_latest = 0; ++ ret = __amd_pstate_cpu_init(policy); ++ if (ret) ++ return ret; ++ /* ++ * Set the policy to powersave to provide a valid fallback value in case ++ * the default cpufreq governor is neither powersave nor performance. ++ */ ++ policy->policy = CPUFREQ_POLICY_POWERSAVE; ++ ++ return 0; +} + -+/* Update (most of) our congestion signals: track the recent rate and volume of -+ * delivered data, presence of loss, and EWMA degree of ECN marking. -+ */ -+static void bbr2_update_congestion_signals( -+ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) ++static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw; ++ pr_debug("amd-pstate: CPU %d exiting\n", policy->cpu); ++ policy->fast_switch_possible = false; ++ return 0; ++} + -+ bbr->loss_round_start = 0; -+ if (rs->interval_us <= 0 || !rs->acked_sacked) -+ return; /* Not a valid observation */ -+ bw = ctx->sample_bw; ++static void amd_pstate_update_max_freq(unsigned int cpu) ++{ ++ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); + -+ if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) -+ bbr2_take_bw_hi_sample(sk, bw); ++ if (!policy) ++ return; + -+ bbr->loss_in_round |= (rs->losses > 0); ++ refresh_frequency_limits(policy); ++ cpufreq_cpu_release(policy); ++} + -+ /* Update rate and volume of delivered data from latest round trip: */ -+ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); -+ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); ++static void amd_pstate_epp_update_limits(unsigned int cpu) ++{ ++ mutex_lock(&amd_pstate_driver_lock); ++ update_boost_state(); ++ if (global.cppc_boost_disabled) { ++ for_each_possible_cpu(cpu) ++ amd_pstate_update_max_freq(cpu); ++ } else { ++ cpufreq_update_policy(cpu); ++ } ++ mutex_unlock(&amd_pstate_driver_lock); ++} + -+ if (before(rs->prior_delivered, bbr->loss_round_delivered)) -+ return; /* skip the per-round-trip updates */ -+ /* Now do per-round-trip updates. */ -+ bbr->loss_round_delivered = tp->delivered; /* mark round trip */ -+ bbr->loss_round_start = 1; -+ bbr2_adapt_lower_bounds(sk); ++static int cppc_boost_hold_time_ns = 3 * NSEC_PER_MSEC; + -+ /* Update windowed "latest" (single-round-trip) filters. */ -+ bbr->loss_in_round = 0; -+ bbr->ecn_in_round = 0; -+ bbr->bw_latest = ctx->sample_bw; -+ bbr->inflight_latest = rs->delivered; ++static inline void amd_pstate_boost_up(struct amd_cpudata *cpudata) ++{ ++ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); ++ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); ++ u32 max_limit = (hwp_req & 0xff); ++ u32 min_limit = (hwp_req & 0xff00) >> 8; ++ u32 boost_level1; ++ ++ /* If max and min are equal or already at max, nothing to boost */ ++ if (max_limit == min_limit) ++ return; ++ ++ /* Set boost max and min to initial value */ ++ if (!cpudata->cppc_boost_min) ++ cpudata->cppc_boost_min = min_limit; ++ ++ boost_level1 = ((AMD_CPPC_NOMINAL_PERF(hwp_cap) + min_limit) >> 1); ++ ++ if (cpudata->cppc_boost_min < boost_level1) ++ cpudata->cppc_boost_min = boost_level1; ++ else if (cpudata->cppc_boost_min < AMD_CPPC_NOMINAL_PERF(hwp_cap)) ++ cpudata->cppc_boost_min = AMD_CPPC_NOMINAL_PERF(hwp_cap); ++ else if (cpudata->cppc_boost_min == AMD_CPPC_NOMINAL_PERF(hwp_cap)) ++ cpudata->cppc_boost_min = max_limit; ++ else ++ return; ++ ++ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); ++ hwp_req |= AMD_CPPC_MIN_PERF(cpudata->cppc_boost_min); ++ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); ++ cpudata->last_update = cpudata->sample.time; +} + -+/* Bandwidth probing can cause loss. To help coexistence with loss-based -+ * congestion control we spread out our probing in a Reno-conscious way. Due to -+ * the shape of the Reno sawtooth, the time required between loss epochs for an -+ * idealized Reno flow is a number of round trips that is the BDP of that -+ * flow. We count packet-timed round trips directly, since measured RTT can -+ * vary widely, and Reno is driven by packet-timed round trips. -+ */ -+static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) ++static inline void amd_pstate_boost_down(struct amd_cpudata *cpudata) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 inflight, rounds, reno_gain, reno_rounds; ++ bool expired; + -+ /* Random loss can shave some small percentage off of our inflight -+ * in each round. To survive this, flows need robust periodic probes. -+ */ -+ rounds = bbr->params.bw_probe_max_rounds; ++ if (cpudata->cppc_boost_min) { ++ expired = time_after64(cpudata->sample.time, cpudata->last_update + ++ cppc_boost_hold_time_ns); + -+ reno_gain = bbr->params.bw_probe_reno_gain; -+ if (reno_gain) { -+ inflight = bbr2_target_inflight(sk); -+ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; -+ rounds = min(rounds, reno_rounds); ++ if (expired) { ++ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, cpudata->cppc_req_cached); ++ cpudata->cppc_boost_min = 0; ++ } + } -+ return bbr->rounds_since_probe >= rounds; ++ ++ cpudata->last_update = cpudata->sample.time; +} + -+/* How long do we want to wait before probing for bandwidth (and risking -+ * loss)? We randomize the wait, for better mixing and fairness convergence. -+ * -+ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. -+ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, -+ * (eg 4K video to a broadband user): -+ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets -+ * -+ * We bound the BBR-native inter-bw-probe wall clock time to be: -+ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time -+ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must -+ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs -+ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable -+ * amount of time to discover unutilized bw on human-scale interactive -+ * time-scales (e.g. perhaps traffic from a web page download that we -+ * were competing with is now complete). -+ */ -+static void bbr2_pick_probe_wait(struct sock *sk) ++static inline void amd_pstate_boost_update_util(struct amd_cpudata *cpudata, ++ u64 time) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ cpudata->sample.time = time; ++ if (smp_processor_id() != cpudata->cpu) ++ return; + -+ /* Decide the random round-trip bound for wait until probe: */ -+ bbr->rounds_since_probe = -+ prandom_u32_max(bbr->params.bw_probe_rand_rounds); -+ /* Decide the random wall clock bound for wait until probe: */ -+ bbr->probe_wait_us = bbr->params.bw_probe_base_us + -+ prandom_u32_max(bbr->params.bw_probe_rand_us); -+} ++ if (cpudata->sched_flags & SCHED_CPUFREQ_IOWAIT) { ++ bool do_io = false; + -+static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ cpudata->sched_flags = 0; ++ /* ++ * Set iowait_boost flag and update time. Since IO WAIT flag ++ * is set all the time, we can't just conclude that there is ++ * some IO bound activity is scheduled on this CPU with just ++ * one occurrence. If we receive at least two in two ++ * consecutive ticks, then we treat as boost candidate. ++ * This is leveraged from Intel Pstate driver. ++ */ ++ if (time_before64(time, cpudata->last_io_update + 2 * TICK_NSEC)) ++ do_io = true; + -+ bbr->cycle_idx = cycle_idx; -+ /* New phase, so need to update cwnd and pacing rate. */ -+ bbr->try_fast_path = 0; ++ cpudata->last_io_update = time; ++ ++ if (do_io) ++ amd_pstate_boost_up(cpudata); ++ ++ } else { ++ amd_pstate_boost_down(cpudata); ++ } +} + -+/* Send at estimated bw to fill the pipe, but not queue. We need this phase -+ * before PROBE_UP, because as soon as we send faster than the available bw -+ * we will start building a queue, and if the buffer is shallow we can cause -+ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and -+ * inflight_hi estimates will underestimate. -+ */ -+static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) ++static inline void amd_pstate_cppc_update_hook(struct update_util_data *data, ++ u64 time, unsigned int flags) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct amd_cpudata *cpudata = container_of(data, ++ struct amd_cpudata, update_util); + -+ bbr2_reset_lower_bounds(sk); -+ if (bbr->inflight_hi != ~0U) -+ bbr->inflight_hi += bbr->params.refill_add_inc; -+ bbr->bw_probe_up_rounds = bw_probe_up_rounds; -+ bbr->bw_probe_up_acks = 0; -+ bbr->stopped_risky_probe = 0; -+ bbr->ack_phase = BBR_ACKS_REFILLING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); ++ cpudata->sched_flags |= flags; ++ ++ if (smp_processor_id() == cpudata->cpu) ++ amd_pstate_boost_update_util(cpudata, time); +} + -+/* Now probe max deliverable data rate and volume. */ -+static void bbr2_start_bw_probe_up(struct sock *sk) ++static void amd_pstate_clear_update_util_hook(unsigned int cpu) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct amd_cpudata *cpudata = all_cpu_data[cpu]; + -+ bbr->ack_phase = BBR_ACKS_PROBE_STARTING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr->cycle_mstamp = tp->tcp_mstamp; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); -+ bbr2_raise_inflight_hi_slope(sk); ++ if (!cpudata->update_util_set) ++ return; ++ ++ cpufreq_remove_update_util_hook(cpu); ++ cpudata->update_util_set = false; ++ synchronize_rcu(); +} + -+/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall -+ * clock time at which to probe beyond an inflight that we think to be -+ * safe. This will knowingly risk packet loss, so we want to do this rarely, to -+ * keep packet loss rates low. Also start a round-trip counter, to probe faster -+ * if we estimate a Reno flow at our BDP would probe faster. -+ */ -+static void bbr2_start_bw_probe_down(struct sock *sk) ++static void amd_pstate_set_update_util_hook(unsigned int cpu_num) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct amd_cpudata *cpudata = all_cpu_data[cpu_num]; + -+ bbr2_reset_congestion_signals(sk); -+ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ -+ bbr2_pick_probe_wait(sk); -+ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ -+ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; -+ bbr->next_rtt_delivered = tp->delivered; -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); ++ if (!cppc_boost) { ++ if (cpudata->update_util_set) ++ amd_pstate_clear_update_util_hook(cpudata->cpu); ++ return; ++ } ++ ++ if (cpudata->update_util_set) ++ return; ++ ++ cpudata->sample.time = 0; ++ cpufreq_add_update_util_hook(cpu_num, &cpudata->update_util, ++ amd_pstate_cppc_update_hook); ++ cpudata->update_util_set = true; +} + -+/* Cruise: maintain what we estimate to be a neutral, conservative -+ * operating point, without attempting to probe up for bandwidth or down for -+ * RTT, and only reducing inflight in response to loss/ECN signals. -+ */ -+static void bbr2_start_bw_probe_cruise(struct sock *sk) ++static void amd_pstate_epp_init(unsigned int cpu) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct amd_cpudata *cpudata = all_cpu_data[cpu]; ++ u32 max_perf, min_perf; ++ u64 value; ++ s16 epp; ++ int ret; + -+ if (bbr->inflight_lo != ~0U) -+ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); ++ max_perf = READ_ONCE(cpudata->highest_perf); ++ min_perf = READ_ONCE(cpudata->lowest_perf); + -+ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); ++ value = READ_ONCE(cpudata->cppc_req_cached); ++ ++ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) ++ min_perf = max_perf; ++ ++ /* Initial min/max values for CPPC Performance Controls Register */ ++ value &= ~AMD_CPPC_MIN_PERF(~0L); ++ value |= AMD_CPPC_MIN_PERF(min_perf); ++ ++ value &= ~AMD_CPPC_MAX_PERF(~0L); ++ value |= AMD_CPPC_MAX_PERF(max_perf); ++ ++ /* CPPC EPP feature require to set zero to the desire perf bit */ ++ value &= ~AMD_CPPC_DES_PERF(~0L); ++ value |= AMD_CPPC_DES_PERF(0); ++ ++ if (cpudata->epp_policy == cpudata->policy) ++ goto skip_epp; ++ ++ cpudata->epp_policy = cpudata->policy; ++ ++ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { ++ epp = amd_pstate_get_epp(cpudata, value); ++ cpudata->epp_powersave = epp; ++ if (epp < 0) ++ goto skip_epp; ++ ++ epp = 0; ++ } else { ++ if (cpudata->epp_powersave < 0) ++ goto skip_epp; ++ /* Get BIOS pre-defined epp value */ ++ epp = amd_pstate_get_epp(cpudata, value); ++ if (epp) ++ goto skip_epp; ++ epp = cpudata->epp_powersave; ++ } ++ /* Set initial EPP value */ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ value &= ~GENMASK_ULL(31, 24); ++ value |= (u64)epp << 24; ++ } ++ ++skip_epp: ++ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ if (!ret) { ++ cpudata->epp_cached = epp; ++ } +} + -+/* Loss and/or ECN rate is too high while probing. -+ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. -+ */ -+static void bbr2_handle_inflight_too_high(struct sock *sk, -+ const struct rate_sample *rs) ++static void amd_pstate_set_max_limits(struct amd_cpudata *cpudata) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ const u32 beta = bbr->params.beta; ++ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); ++ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); ++ u32 max_limit = (hwp_cap >> 24) & 0xff; + -+ bbr->prev_probe_too_high = 1; -+ bbr->bw_probe_samples = 0; /* only react once per probe */ -+ bbr->debug.event = 'L'; /* Loss/ECN too high */ -+ /* If we are app-limited then we are not robustly -+ * probing the max volume of inflight data we think -+ * might be safe (analogous to how app-limited bw -+ * samples are not known to be robustly probing bw). -+ */ -+ if (!rs->is_app_limited) -+ bbr->inflight_hi = max_t(u32, rs->tx_in_flight, -+ (u64)bbr2_target_inflight(sk) * -+ (BBR_UNIT - beta) >> BBR_SCALE); -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) -+ bbr2_start_bw_probe_down(sk); ++ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); ++ hwp_req |= AMD_CPPC_MIN_PERF(max_limit); ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); +} + -+/* If we're seeing bw and loss samples reflecting our bw probing, adapt -+ * using the signals we see. If loss or ECN mark rate gets too high, then adapt -+ * inflight_hi downward. If we're able to push inflight higher without such -+ * signals, push higher: adapt inflight_hi upward. -+ */ -+static bool bbr2_adapt_upper_bounds(struct sock *sk, -+ const struct rate_sample *rs) ++static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct amd_cpudata *cpudata; + -+ /* Track when we'll see bw/loss samples resulting from our bw probes. */ -+ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) -+ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; -+ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { -+ /* End of samples from bw probing phase. */ -+ bbr->bw_probe_samples = 0; -+ bbr->ack_phase = BBR_ACKS_INIT; -+ /* At this point in the cycle, our current bw sample is also -+ * our best recent chance at finding the highest available bw -+ * for this flow. So now is the best time to forget the bw -+ * samples from the previous cycle, by advancing the window. -+ */ -+ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) -+ bbr2_advance_bw_hi_filter(sk); -+ /* If we had an inflight_hi, then probed and pushed inflight all -+ * the way up to hit that inflight_hi without seeing any -+ * high loss/ECN in all the resulting ACKs from that probing, -+ * then probe up again, this time letting inflight persist at -+ * inflight_hi for a round trip, then accelerating beyond. -+ */ -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { -+ bbr->debug.event = 'R'; /* reprobe */ -+ bbr2_start_bw_probe_refill(sk, 0); -+ return true; /* yes, decided state transition */ -+ } -+ } ++ if (!policy->cpuinfo.max_freq) ++ return -ENODEV; + -+ if (bbr2_is_inflight_too_high(sk, rs)) { -+ if (bbr->bw_probe_samples) /* sample is from bw probing? */ -+ bbr2_handle_inflight_too_high(sk, rs); -+ } else { -+ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ -+ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ -+ return false; ++ pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", ++ policy->cpuinfo.max_freq, policy->max); + -+ /* To be resilient to random loss, we must raise inflight_hi -+ * if we observe in any phase that a higher level is safe. -+ */ -+ if (rs->tx_in_flight > bbr->inflight_hi) { -+ bbr->inflight_hi = rs->tx_in_flight; -+ bbr->debug.event = 'U'; /* raise up inflight_hi */ ++ cpudata = all_cpu_data[policy->cpu]; ++ cpudata->policy = policy->policy; ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ mutex_lock(&amd_pstate_limits_lock); ++ ++ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { ++ amd_pstate_clear_update_util_hook(policy->cpu); ++ amd_pstate_set_max_limits(cpudata); ++ } else { ++ amd_pstate_set_update_util_hook(policy->cpu); + } + -+ if (bbr->mode == BBR_PROBE_BW && -+ bbr->cycle_idx == BBR_BW_PROBE_UP) -+ bbr2_probe_inflight_hi_upward(sk, rs); ++ if (boot_cpu_has(X86_FEATURE_CPPC)) ++ amd_pstate_epp_init(policy->cpu); ++ ++ mutex_unlock(&amd_pstate_limits_lock); + } + -+ return false; ++ return 0; +} + -+/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ -+static bool bbr2_check_time_to_probe_bw(struct sock *sk) ++static void amd_pstate_epp_reenable(struct amd_cpudata * cpudata) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 n; ++ struct cppc_perf_ctrls perf_ctrls; ++ u64 value, max_perf; ++ int ret; + -+ /* If we seem to be at an operating point where we are not seeing loss -+ * but we are seeing ECN marks, then when the ECN marks cease we reprobe -+ * quickly (in case a burst of cross-traffic has ceased and freed up bw, -+ * or in case we are sharing with multiplicatively probing traffic). -+ */ -+ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && -+ bbr->ecn_in_cycle && !bbr->loss_in_cycle && -+ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { -+ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ -+ /* Calculate n so that when bbr2_raise_inflight_hi_slope() -+ * computes growth_this_round as 2^n it will be roughly the -+ * desired volume of data (inflight_hi*ecn_reprobe_gain). -+ */ -+ n = ilog2((((u64)bbr->inflight_hi * -+ bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); -+ bbr2_start_bw_probe_refill(sk, n); -+ return true; -+ } ++ ret = amd_pstate_enable(true); ++ if (ret) ++ pr_err("failed to enable amd pstate during resume, return %d\n", ret); + -+ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || -+ bbr2_is_reno_coexistence_probe_time(sk)) { -+ bbr2_start_bw_probe_refill(sk, 0); -+ return true; ++ value = READ_ONCE(cpudata->cppc_req_cached); ++ max_perf = READ_ONCE(cpudata->highest_perf); ++ ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ } else { ++ perf_ctrls.max_perf = max_perf; ++ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); ++ cppc_set_perf(cpudata->cpu, &perf_ctrls); + } -+ return false; +} + -+/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ -+static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) ++static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool is_under_bdp, is_long_enough; ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; + -+ /* Always need to pull inflight down to leave headroom in queue. */ -+ if (inflight > bbr2_inflight_with_headroom(sk)) -+ return false; ++ pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); + -+ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); -+ if (bbr->params.drain_to_target) -+ return is_under_bdp; ++ if (epp_enabled) { ++ amd_pstate_epp_reenable(cpudata); ++ cpudata->suspended = false; ++ } + -+ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); -+ return is_under_bdp || is_long_enough; ++ return 0; +} + -+/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ -+static void bbr2_update_cycle_phase(struct sock *sk, -+ const struct rate_sample *rs) ++static void amd_pstate_epp_offline(struct cpufreq_policy *policy) +{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ bool is_risky = false, is_queuing = false; -+ u32 inflight, bw; ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ struct cppc_perf_ctrls perf_ctrls; ++ int min_perf; ++ u64 value; + -+ if (!bbr_full_bw_reached(sk)) -+ return; ++ min_perf = READ_ONCE(cpudata->lowest_perf); ++ value = READ_ONCE(cpudata->cppc_req_cached); + -+ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ -+ if (bbr2_adapt_upper_bounds(sk, rs)) -+ return; /* already decided state transition */ -+ -+ if (bbr->mode != BBR_PROBE_BW) -+ return; -+ -+ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); -+ bw = bbr_max_bw(sk); -+ -+ switch (bbr->cycle_idx) { -+ /* First we spend most of our time cruising with a pacing_gain of 1.0, -+ * which paces at the estimated bw, to try to fully use the pipe -+ * without building queue. If we encounter loss/ECN marks, we adapt -+ * by slowing down. -+ */ -+ case BBR_BW_PROBE_CRUISE: -+ if (bbr2_check_time_to_probe_bw(sk)) -+ return; /* already decided state transition */ -+ break; -+ -+ /* After cruising, when it's time to probe, we first "refill": we send -+ * at the estimated bw to fill the pipe, before probing higher and -+ * knowingly risking overflowing the bottleneck buffer (causing loss). -+ */ -+ case BBR_BW_PROBE_REFILL: -+ if (bbr->round_start) { -+ /* After one full round trip of sending in REFILL, we -+ * start to see bw samples reflecting our REFILL, which -+ * may be putting too much data in flight. -+ */ -+ bbr->bw_probe_samples = 1; -+ bbr2_start_bw_probe_up(sk); -+ } -+ break; -+ -+ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to -+ * probe for bw. If we have not seen loss/ECN, we try to raise inflight -+ * to at least pacing_gain*BDP; note that this may take more than -+ * min_rtt if min_rtt is small (e.g. on a LAN). -+ * -+ * We terminate PROBE_UP bandwidth probing upon any of the following: -+ * -+ * (1) We've pushed inflight up to hit the inflight_hi target set in the -+ * most recent previous bw probe phase. Thus we want to start -+ * draining the queue immediately because it's very likely the most -+ * recently sent packets will fill the queue and cause drops. -+ * (checked here) -+ * (2) We have probed for at least 1*min_rtt_us, and the -+ * estimated queue is high enough (inflight > 1.25 * estimated_bdp). -+ * (checked here) -+ * (3) Loss filter says loss rate is "too high". -+ * (checked in bbr_is_inflight_too_high()) -+ * (4) ECN filter says ECN mark rate is "too high". -+ * (checked in bbr_is_inflight_too_high()) -+ */ -+ case BBR_BW_PROBE_UP: -+ if (bbr->prev_probe_too_high && -+ inflight >= bbr->inflight_hi) { -+ bbr->stopped_risky_probe = 1; -+ is_risky = true; -+ bbr->debug.event = 'D'; /* D for danger */ -+ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && -+ inflight >= -+ bbr_inflight(sk, bw, -+ bbr->params.bw_probe_pif_gain)) { -+ is_queuing = true; -+ bbr->debug.event = 'Q'; /* building Queue */ -+ } -+ if (is_risky || is_queuing) { -+ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ -+ bbr2_start_bw_probe_down(sk); /* restart w/ down */ -+ } -+ break; -+ -+ /* After probing in PROBE_UP, we have usually accumulated some data in -+ * the bottleneck buffer (if bw probing didn't find more bw). We next -+ * enter PROBE_DOWN to try to drain any excess data from the queue. To -+ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until -+ * our inflight is less then that target cruising point, which is the -+ * minimum of (a) the amount needed to leave headroom, and (b) the -+ * estimated BDP. Once inflight falls to match the target, we estimate -+ * the queue is drained; persisting would underutilize the pipe. -+ */ -+ case BBR_BW_PROBE_DOWN: -+ if (bbr2_check_time_to_probe_bw(sk)) -+ return; /* already decided state transition */ -+ if (bbr2_check_time_to_cruise(sk, inflight, bw)) -+ bbr2_start_bw_probe_cruise(sk); -+ break; ++ mutex_lock(&amd_pstate_limits_lock); ++ if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; + -+ default: -+ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); ++ /* Set max perf same as min perf */ ++ value &= ~AMD_CPPC_MAX_PERF(~0L); ++ value |= AMD_CPPC_MAX_PERF(min_perf); ++ value &= ~AMD_CPPC_MIN_PERF(~0L); ++ value |= AMD_CPPC_MIN_PERF(min_perf); ++ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); ++ } else { ++ perf_ctrls.desired_perf = 0; ++ perf_ctrls.max_perf = min_perf; ++ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(AMD_CPPC_EPP_POWERSAVE); ++ cppc_set_perf(cpudata->cpu, &perf_ctrls); + } ++ mutex_unlock(&amd_pstate_limits_lock); +} + -+/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ -+static void bbr2_exit_probe_rtt(struct sock *sk) ++static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; + -+ bbr2_reset_lower_bounds(sk); -+ if (bbr_full_bw_reached(sk)) { -+ bbr->mode = BBR_PROBE_BW; -+ /* Raising inflight after PROBE_RTT may cause loss, so reset -+ * the PROBE_BW clock and schedule the next bandwidth probe for -+ * a friendly and randomized future point in time. -+ */ -+ bbr2_start_bw_probe_down(sk); -+ /* Since we are exiting PROBE_RTT, we know inflight is -+ * below our estimated BDP, so it is reasonable to cruise. -+ */ -+ bbr2_start_bw_probe_cruise(sk); -+ } else { -+ bbr->mode = BBR_STARTUP; -+ } -+} ++ pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); + -+/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until -+ * the end of the round in recovery to get a good estimate of how many packets -+ * have been lost, and how many we need to drain with a low pacing rate. -+ */ -+static void bbr2_check_loss_too_high_in_startup(struct sock *sk, -+ const struct rate_sample *rs) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ if (cpudata->suspended) ++ return 0; + -+ if (bbr_full_bw_reached(sk)) -+ return; ++ if (epp_enabled) ++ amd_pstate_epp_offline(policy); + -+ /* For STARTUP exit, check the loss rate at the end of each round trip -+ * of Recovery episodes in STARTUP. We check the loss rate at the end -+ * of the round trip to filter out noisy/low loss and have a better -+ * sense of inflight (extent of loss), so we can drain more accurately. -+ */ -+ if (rs->losses && bbr->loss_events_in_round < 0xf) -+ bbr->loss_events_in_round++; /* update saturating counter */ -+ if (bbr->params.full_loss_cnt && bbr->loss_round_start && -+ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && -+ bbr->loss_events_in_round >= bbr->params.full_loss_cnt && -+ bbr2_is_inflight_too_high(sk, rs)) { -+ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ -+ bbr2_handle_queue_too_high_in_startup(sk); -+ return; -+ } -+ if (bbr->loss_round_start) -+ bbr->loss_events_in_round = 0; ++ return 0; +} + -+/* If we are done draining, advance into steady state operation in PROBE_BW. */ -+static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) ++static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ amd_pstate_clear_update_util_hook(policy->cpu); + -+ if (bbr_check_drain(sk, rs, ctx)) { -+ bbr->mode = BBR_PROBE_BW; -+ bbr2_start_bw_probe_down(sk); -+ } ++ return amd_pstate_cpu_offline(policy); +} + -+static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, -+ struct bbr_context *ctx) ++static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) +{ -+ bbr2_update_congestion_signals(sk, rs, ctx); -+ bbr_update_ack_aggregation(sk, rs); -+ bbr2_check_loss_too_high_in_startup(sk, rs); -+ bbr_check_full_bw_reached(sk, rs); -+ bbr2_check_drain(sk, rs, ctx); -+ bbr2_update_cycle_phase(sk, rs); -+ bbr_update_min_rtt(sk, rs); -+} ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ int ret; + -+/* Fast path for app-limited case. -+ * -+ * On each ack, we execute bbr state machine, which primarily consists of: -+ * 1) update model based on new rate sample, and -+ * 2) update control based on updated model or state change. -+ * -+ * There are certain workload/scenarios, e.g. app-limited case, where -+ * either we can skip updating model or we can skip update of both model -+ * as well as control. This provides signifcant softirq cpu savings for -+ * processing incoming acks. -+ * -+ * In case of app-limited, if there is no congestion (loss/ecn) and -+ * if observed bw sample is less than current estimated bw, then we can -+ * skip some of the computation in bbr state processing: -+ * -+ * - if there is no rtt/mode/phase change: In this case, since all the -+ * parameters of the network model are constant, we can skip model -+ * as well control update. -+ * -+ * - else we can skip rest of the model update. But we still need to -+ * update the control to account for the new rtt/mode/phase. -+ * -+ * Returns whether we can take fast path or not. -+ */ -+static bool bbr2_fast_path(struct sock *sk, bool *update_model, -+ const struct rate_sample *rs, struct bbr_context *ctx) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ u32 prev_min_rtt_us, prev_mode; ++ pr_debug("AMD CPU Core %d suspending\n", cpudata->cpu); + -+ if (bbr->params.fast_path && bbr->try_fast_path && -+ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && -+ !bbr->loss_in_round && !bbr->ecn_in_round) { -+ prev_mode = bbr->mode; -+ prev_min_rtt_us = bbr->min_rtt_us; -+ bbr2_check_drain(sk, rs, ctx); -+ bbr2_update_cycle_phase(sk, rs); -+ bbr_update_min_rtt(sk, rs); ++ cpudata->suspended = true; + -+ if (bbr->mode == prev_mode && -+ bbr->min_rtt_us == prev_min_rtt_us && -+ bbr->try_fast_path) -+ return true; ++ /* disable CPPC in lowlevel firmware */ ++ ret = amd_pstate_enable(false); ++ if (ret) ++ pr_err("failed to disable amd pstate during suspend, return %d\n", ret); + -+ /* Skip model update, but control still needs to be updated */ -+ *update_model = false; -+ } -+ return false; ++ return 0; +} + -+static void bbr2_main(struct sock *sk, const struct rate_sample *rs) ++static int amd_pstate_epp_resume(struct cpufreq_policy *policy) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct bbr_context ctx = { 0 }; -+ bool update_model = true; -+ u32 bw; -+ int ce_ratio = -1; -+ -+ bbr->debug.event = '.'; /* init to default NOP (no event yet) */ -+ -+ bbr_update_round_start(sk, rs, &ctx); -+ if (bbr->round_start) { -+ bbr->rounds_since_probe = -+ min_t(s32, bbr->rounds_since_probe + 1, 0xFF); -+ ce_ratio = bbr2_update_ecn_alpha(sk); -+ tcp_plb_update_state(sk, &bbr->plb, ce_ratio); -+ tcp_plb_check_rehash(sk, &bbr->plb); -+ } ++ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; + -+ bbr->ecn_in_round |= rs->is_ece; -+ bbr_calculate_bw_sample(sk, rs, &ctx); ++ pr_debug("AMD CPU Core %d resuming\n", cpudata->cpu); + -+ if (bbr2_fast_path(sk, &update_model, rs, &ctx)) -+ goto out; ++ if (cpudata->suspended && epp_enabled) { ++ mutex_lock(&amd_pstate_limits_lock); + -+ if (update_model) -+ bbr2_update_model(sk, rs, &ctx); ++ /* enable amd pstate from suspend state*/ ++ amd_pstate_epp_reenable(cpudata); + -+ bbr_update_gains(sk); -+ bw = bbr_bw(sk); -+ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); -+ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, -+ tp->snd_cwnd, &ctx); -+ bbr2_bound_cwnd_for_inflight_model(sk); ++ mutex_unlock(&amd_pstate_limits_lock); ++ } + -+out: -+ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; -+ bbr->loss_in_cycle |= rs->lost > 0; -+ bbr->ecn_in_cycle |= rs->delivered_ce > 0; ++ cpudata->suspended = false; + -+ bbr_debug(sk, rs->acked_sacked, rs, &ctx); ++ return 0; +} + -+/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared -+ * down here, so that the algorithm functions that use the parameters must use -+ * the per-socket parameters; if they accidentally use the global version -+ * then there will be a compile error. -+ * TODO(ncardwell): move all per-socket parameters down to this section. -+ */ -+ -+/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. -+ * No loss response when 0. Max allwed value is 255. -+ */ -+static u32 bbr_beta = BBR_UNIT * 30 / 100; -+ -+/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. -+ * Max allowed value is 255. -+ */ -+static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ -+ -+/* The initial value for the ecn_alpha state variable. Default and max -+ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly -+ * to congestion if the bottleneck is congested when the flow starts up. -+ */ -+static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ -+ -+/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. -+ * No ECN based bounding when 0. Max allwed value is 255. -+ */ -+static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ -+ -+/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. -+ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. -+ */ -+static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ -+ -+/* Max RTT (in usec) at which to use sender-side ECN logic. -+ * Disabled when 0 (ECN allowed at any RTT). -+ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. -+ */ -+static u32 bbr_ecn_max_rtt_us = 5000; -+ -+/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN -+ * clears then use a multiplicative increase to quickly reprobe bw by -+ * starting inflight probing at the given multiple of inflight_hi. -+ * Default for this experimental knob is 0 (disabled). -+ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. -+ */ -+static u32 bbr_ecn_reprobe_gain; -+ -+/* Estimate bw probing has gone too far if loss rate exceeds this level. */ -+static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ -+ -+/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, -+ * and loss rate is higher than bbr_loss_thresh. -+ * Disabled if 0. Max allowed value is 15 (0xF). -+ */ -+static u32 bbr_full_loss_cnt = 8; ++static void amd_pstate_verify_cpu_policy(struct amd_cpudata *cpudata, ++ struct cpufreq_policy_data *policy) ++{ ++ update_boost_state(); ++ cpufreq_verify_within_cpu_limits(policy); ++} + -+/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh -+ * meets this count. Max allowed value is 3. -+ */ -+static u32 bbr_full_ecn_cnt = 2; ++static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) ++{ ++ amd_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); ++ pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min ); ++ return 0; ++} + -+/* Fraction of unutilized headroom to try to leave in path upon high loss. */ -+static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; + static struct cpufreq_driver amd_pstate_driver = { + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, + .verify = amd_pstate_verify, +@@ -662,18 +1535,34 @@ static struct cpufreq_driver amd_pstate_driver = { + .resume = amd_pstate_cpu_resume, + .set_boost = amd_pstate_set_boost, + .name = "amd-pstate", +- .attr = amd_pstate_attr, ++ .attr = amd_pstate_attr, ++}; + -+/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. -+ * Default is 1.25x, as in BBR v1. Max allowed is 511. -+ */ -+static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; ++static struct cpufreq_driver amd_pstate_epp_driver = { ++ .flags = CPUFREQ_CONST_LOOPS, ++ .verify = amd_pstate_epp_verify_policy, ++ .setpolicy = amd_pstate_epp_set_policy, ++ .init = amd_pstate_epp_cpu_init, ++ .exit = amd_pstate_epp_cpu_exit, ++ .update_limits = amd_pstate_epp_update_limits, ++ .offline = amd_pstate_epp_cpu_offline, ++ .online = amd_pstate_epp_cpu_online, ++ .suspend = amd_pstate_epp_suspend, ++ .resume = amd_pstate_epp_resume, ++ .name = "amd_pstate_epp", ++ .attr = amd_pstate_epp_attr, + }; + + static int __init amd_pstate_init(void) + { ++ static struct amd_cpudata **cpudata; + int ret; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) + return -ENODEV; + + if (!acpi_cpc_valid()) { +- pr_debug("the _CPC object is not present in SBIOS\n"); ++ pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); + return -ENODEV; + } + +@@ -681,10 +1570,25 @@ static int __init amd_pstate_init(void) + if (cpufreq_get_current_driver()) + return -EEXIST; + ++ cpudata = vzalloc(array_size(sizeof(void *), num_possible_cpus())); ++ if (!cpudata) ++ return -ENOMEM; ++ WRITE_ONCE(all_cpu_data, cpudata); + -+/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. -+ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. -+ * Max allowed is 511. -+ */ -+static u32 bbr_bw_probe_reno_gain = BBR_UNIT; ++ if (epp_enabled) { ++ pr_info("AMD CPPC loading with amd_pstate_epp driver instance.\n"); ++ default_pstate_driver = &amd_pstate_epp_driver; ++ } else { ++ pr_info("AMD CPPC loading with amd_pstate driver instance.\n"); ++ default_pstate_driver = &amd_pstate_driver; ++ } + -+/* Max number of packet-timed rounds to wait before probing for bandwidth. If -+ * we want to tolerate 1% random loss per round, and not have this cut our -+ * inflight too much, we must probe for bw periodically on roughly this scale. -+ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. -+ * We aim to be fair with Reno/CUBIC up to a BDP of at least: -+ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets -+ */ -+static u32 bbr_bw_probe_max_rounds = 63; + /* capability check */ + if (boot_cpu_has(X86_FEATURE_CPPC)) { ++ if (!epp_enabled) { ++ default_pstate_driver->adjust_perf = amd_pstate_adjust_perf; ++ } + pr_debug("AMD CPPC MSR based functionality is supported\n"); +- amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; + } else if (shared_mem) { + static_call_update(amd_pstate_enable, cppc_enable); + static_call_update(amd_pstate_init_perf, cppc_init_perf); +@@ -701,19 +1605,56 @@ static int __init amd_pstate_init(void) + return ret; + } + +- ret = cpufreq_register_driver(&amd_pstate_driver); ++ ret = cpufreq_register_driver(default_pstate_driver); + if (ret) +- pr_err("failed to register amd_pstate_driver with return %d\n", ++ pr_err("failed to register amd pstate driver with return %d\n", ++ ret); + -+/* Max amount of randomness to inject in round counting for Reno-coexistence. -+ * Max value is 15. -+ */ -+static u32 bbr_bw_probe_rand_rounds = 2; ++ amd_pstate_kobj = kobject_create_and_add("amd-pstate", &cpu_subsys.dev_root->kobj); ++ if (!amd_pstate_kobj) { ++ pr_err("amd-pstate: Global sysfs registration failed.\n"); ++ } + -+/* Use BBR-native probe time scale starting at this many usec. -+ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: -+ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs -+ */ -+static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ ++ ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); ++ if (ret) { ++ pr_err("amd-pstate: Sysfs attribute export failed with error %d.\n", + ret); ++ } + + return ret; + } + ++static inline void amd_pstate_kobj_cleanup(struct kobject *kobj) ++{ ++ kobject_del(kobj); ++ kobject_put(kobj); ++} + -+/* Use BBR-native probes spread over this many usec: */ -+static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ + static void __exit amd_pstate_exit(void) + { +- cpufreq_unregister_driver(&amd_pstate_driver); ++ unsigned int cpu; + -+/* Undo the model changes made in loss recovery if recovery was spurious? */ -+static bool bbr_undo = true; ++ cpufreq_unregister_driver(default_pstate_driver); + + amd_pstate_enable(false); + -+/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ -+static bool bbr_fast_path = true; /* default: enabled */ ++ sysfs_remove_group(amd_pstate_kobj, &amd_pstate_global_attr_group); ++ amd_pstate_kobj_cleanup(amd_pstate_kobj); + -+/* Use fast ack mode ? */ -+static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ ++ cpus_read_lock(); ++ for_each_online_cpu(cpu) { ++ if (all_cpu_data[cpu]) { ++ if (default_pstate_driver == &amd_pstate_epp_driver) ++ amd_pstate_clear_update_util_hook(cpu); + -+/* How much to additively increase inflight_hi when entering REFILL? */ -+static u32 bbr_refill_add_inc; /* default: disabled */ ++ spin_lock(&amd_pstate_cpu_lock); ++ kfree(all_cpu_data[cpu]); ++ WRITE_ONCE(all_cpu_data[cpu], NULL); ++ spin_unlock(&amd_pstate_cpu_lock); ++ } ++ } ++ cpus_read_unlock(); + -+module_param_named(beta, bbr_beta, uint, 0644); -+module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); -+module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); -+module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); -+module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); -+module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); -+module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); -+module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); + } + + module_init(amd_pstate_init); +diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c +index 24eaf0ec344d..9adb7612993e 100644 +--- a/drivers/cpufreq/cppc_cpufreq.c ++++ b/drivers/cpufreq/cppc_cpufreq.c +@@ -947,7 +947,7 @@ static int __init cppc_cpufreq_init(void) + { + int ret; + +- if ((acpi_disabled) || !acpi_cpc_valid()) ++ if (!acpi_cpc_valid()) + return -ENODEV; + + cppc_check_hisi_workaround(); +diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c +index 57cdb3679885..50881662b45b 100644 +--- a/drivers/cpufreq/intel_pstate.c ++++ b/drivers/cpufreq/intel_pstate.c +@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu) + * update them at any time after it has been called. + */ + sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); ++ /* ++ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff. ++ * In this case we can't use CPPC.highest_perf to enable ITMT. ++ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide. ++ */ ++ if (cppc_perf.highest_perf == 0xff) ++ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); + + if (max_highest_perf <= min_highest_perf) { + if (cppc_perf.highest_perf > max_highest_perf) +diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c +index 3e101719689a..746138abc3fe 100644 +--- a/drivers/idle/intel_idle.c ++++ b/drivers/idle/intel_idle.c +@@ -578,7 +578,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 120, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -586,7 +586,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 33, +- .target_residency = 100, ++ .target_residency = 900, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -594,7 +594,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 133, +- .target_residency = 400, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -602,7 +602,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x32", + .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 166, +- .target_residency = 500, ++ .target_residency = 1500, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -610,7 +610,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 300, +- .target_residency = 900, ++ .target_residency = 2000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -618,7 +618,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 600, +- .target_residency = 1800, ++ .target_residency = 5000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -626,7 +626,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 2600, +- .target_residency = 7700, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -646,7 +646,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 120, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -654,7 +654,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 40, +- .target_residency = 100, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -662,7 +662,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 133, +- .target_residency = 400, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -670,7 +670,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x32", + .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 166, +- .target_residency = 500, ++ .target_residency = 2000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -678,7 +678,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 300, +- .target_residency = 900, ++ .target_residency = 4000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -686,7 +686,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 600, +- .target_residency = 1800, ++ .target_residency = 7000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -694,7 +694,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 2600, +- .target_residency = 7700, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -715,7 +715,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 120, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -723,7 +723,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 70, +- .target_residency = 100, ++ .target_residency = 1000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -731,7 +731,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 85, +- .target_residency = 200, ++ .target_residency = 600, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -739,7 +739,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x33", + .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 124, +- .target_residency = 800, ++ .target_residency = 3000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -747,7 +747,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 200, +- .target_residency = 800, ++ .target_residency = 3200, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -755,7 +755,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 480, +- .target_residency = 5000, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -763,7 +763,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, + .exit_latency = 890, +- .target_residency = 5000, ++ .target_residency = 9000, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +@@ -784,7 +784,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 10, +- .target_residency = 20, ++ .target_residency = 300, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { +diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c +index 3fc0a89cc785..a7c103f9dfd3 100644 +--- a/drivers/input/serio/i8042.c ++++ b/drivers/input/serio/i8042.c +@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void) + if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { + i8042_ctr &= ~I8042_CTR_KBDINT; + i8042_ctr |= I8042_CTR_KBDDIS; +- pr_err("Failed to enable KBD port\n"); ++ pr_info("Failed to enable KBD port\n"); + return -EIO; + } + +@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void) + if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { + i8042_ctr &= ~I8042_CTR_AUXINT; + i8042_ctr |= I8042_CTR_AUXDIS; +- pr_err("Failed to enable AUX port\n"); ++ pr_info("Failed to enable AUX port\n"); + return -EIO; + } + +@@ -732,7 +732,7 @@ static int i8042_check_mux(void) + i8042_ctr &= ~I8042_CTR_AUXINT; + + if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { +- pr_err("Failed to disable AUX port, can't use MUX\n"); ++ pr_info("Failed to disable AUX port, can't use MUX\n"); + return -EIO; + } + +@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void) + do { + + if (i8042_command(¶m, I8042_CMD_CTL_TEST)) { +- pr_err("i8042 controller selftest timeout\n"); ++ pr_info("i8042 controller selftest timeout\n"); + return -ENODEV; + } + +@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void) + pr_info("giving up on controller selftest, continuing anyway...\n"); + return 0; + #else +- pr_err("i8042 controller selftest failed\n"); ++ pr_info("i8042 controller selftest failed\n"); + return -EIO; + #endif + } +diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c +index 159c6806c19b..a201f1c2fce5 100644 +--- a/drivers/md/dm-crypt.c ++++ b/drivers/md/dm-crypt.c +@@ -3137,6 +3137,11 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar + } + } + ++#ifdef CONFIG_CACHY ++ set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); ++ set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); ++#endif ++ + return 0; + } + +diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c +index f82ad7419508..5e8faa70aad6 100644 +--- a/drivers/net/dummy.c ++++ b/drivers/net/dummy.c +@@ -43,7 +43,7 @@ + + #define DRV_NAME "dummy" + +-static int numdummies = 1; ++static int numdummies = 0; + + /* fake multicast ability */ + static void set_multicast_list(struct net_device *dev) +diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c +index 66446f1e06cf..c65b03f91ecf 100644 +--- a/drivers/nvme/host/core.c ++++ b/drivers/nvme/host/core.c +@@ -58,7 +58,7 @@ static u8 nvme_max_retries = 5; + module_param_named(max_retries, nvme_max_retries, byte, 0644); + MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); + +-static unsigned long default_ps_max_latency_us = 100000; ++static unsigned long default_ps_max_latency_us = 200; + module_param(default_ps_max_latency_us, ulong, 0644); + MODULE_PARM_DESC(default_ps_max_latency_us, + "max power saving latency for new devices; use PM QOS to change per device"); +diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c +index 95bc329e74c0..73114509ddff 100644 +--- a/drivers/pci/pci.c ++++ b/drivers/pci/pci.c +@@ -62,7 +62,7 @@ struct pci_pme_device { + struct pci_dev *dev; + }; + +-#define PME_TIMEOUT 1000 /* How long between PME checks */ ++#define PME_TIMEOUT 4000 /* How long between PME checks */ + + static void pci_dev_d3_sleep(struct pci_dev *dev) + { +diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c +index 21d624f9f5fb..aad4e9578d89 100644 +--- a/drivers/powercap/intel_rapl_common.c ++++ b/drivers/powercap/intel_rapl_common.c +@@ -1515,7 +1515,7 @@ static int __init rapl_init(void) + + id = x86_match_cpu(rapl_ids); + if (!id) { +- pr_err("driver does not support CPU family %d model %d\n", ++ pr_info("driver does not support CPU family %d model %d\n", + boot_cpu_data.x86, boot_cpu_data.x86_model); + + return -ENODEV; +diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c +index c841ab37e7c6..fe3d190ab12b 100644 +--- a/drivers/thermal/intel/intel_powerclamp.c ++++ b/drivers/thermal/intel/intel_powerclamp.c +@@ -644,6 +644,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = { + .set_cur_state = powerclamp_set_cur_state, + }; + ++static const struct x86_cpu_id amd_cpu[] = { ++ { X86_VENDOR_AMD }, ++ {}, ++}; ++ + static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = { + X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL), + {} +@@ -653,6 +658,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); + static int __init powerclamp_probe(void) + { + ++ if (x86_match_cpu(amd_cpu)){ ++ pr_info("Intel PowerClamp does not support AMD CPUs\n"); ++ return -ENODEV; ++ } ++ + if (!x86_match_cpu(intel_powerclamp_ids)) { + pr_err("CPU does not support MWAIT\n"); + return -ENODEV; +diff --git a/fs/xattr.c b/fs/xattr.c +index a1f4998bc6be..020e0b5ea5c6 100644 +--- a/fs/xattr.c ++++ b/fs/xattr.c +@@ -122,16 +122,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, + } + + /* +- * In the user.* namespace, only regular files and directories can have +- * extended attributes. For sticky directories, only the owner and +- * privileged users can write attributes. ++ * In the user.* namespace, only regular files, symbolic links, and ++ * directories can have extended attributes. For symbolic links and ++ * sticky directories, only the owner and privileged users can write ++ * attributes. + */ + if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { +- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) ++ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode)) + return (mask & MAY_WRITE) ? -EPERM : -ENODATA; +- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && +- (mask & MAY_WRITE) && +- !inode_owner_or_capable(mnt_userns, inode)) ++ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX)) ++ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE) ++ && !inode_owner_or_capable(mnt_userns, inode)) + return -EPERM; + } + +diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h +index f73d357ecdf5..41d151a6d52e 100644 +--- a/include/acpi/cppc_acpi.h ++++ b/include/acpi/cppc_acpi.h +@@ -108,12 +108,14 @@ struct cppc_perf_caps { + u32 lowest_nonlinear_perf; + u32 lowest_freq; + u32 nominal_freq; ++ u32 energy_perf; + }; + + struct cppc_perf_ctrls { + u32 max_perf; + u32 min_perf; + u32 desired_perf; ++ u32 energy_perf; + }; + + struct cppc_perf_fb_ctrs { +@@ -148,6 +150,9 @@ extern bool cpc_ffh_supported(void); + extern bool cpc_supported_by_cpu(void); + extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); + extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); ++extern int cppc_set_auto_epp(int cpu, bool enable); ++extern int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps); ++extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); + #else /* !CONFIG_ACPI_CPPC_LIB */ + static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) + { +@@ -197,6 +202,18 @@ static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) + { + return -ENOTSUPP; + } ++static inline int cppc_set_auto_epp(int cpu, bool enable) ++{ ++ return -ENOTSUPP; ++} ++static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) ++{ ++ return -ENOTSUPP; ++} ++static inline int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) ++{ ++ return -ENOTSUPP; ++} + #endif /* !CONFIG_ACPI_CPPC_LIB */ + + #endif /* _CPPC_ACPI_H*/ +diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h +index e3e8c8662b49..9209856b6644 100644 +--- a/include/linux/ipc_namespace.h ++++ b/include/linux/ipc_namespace.h +@@ -36,8 +36,6 @@ struct ipc_namespace { + unsigned int msg_ctlmax; + unsigned int msg_ctlmnb; + unsigned int msg_ctlmni; +- atomic_t msg_bytes; +- atomic_t msg_hdrs; + + size_t shm_ctlmax; + size_t shm_ctlall; +@@ -77,6 +75,9 @@ struct ipc_namespace { + struct llist_node mnt_llist; + + struct ns_common ns; ++ int padding[16]; ++ atomic_t msg_bytes; ++ atomic_t msg_hdrs; + } __randomize_layout; + + extern struct ipc_namespace init_ipc_ns; +diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h +index 0b7242370b56..16b8fc483b3d 100644 +--- a/include/linux/jbd2.h ++++ b/include/linux/jbd2.h +@@ -45,7 +45,7 @@ + /* + * The default maximum commit age, in seconds. + */ +-#define JBD2_DEFAULT_MAX_COMMIT_AGE 5 ++#define JBD2_DEFAULT_MAX_COMMIT_AGE 30 + + #ifdef CONFIG_JBD2_DEBUG + /* +diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h +index 679591301994..685193470d64 100644 +--- a/include/linux/page_counter.h ++++ b/include/linux/page_counter.h +@@ -8,6 +8,7 @@ + + struct page_counter { + atomic_long_t usage; ++ unsigned long padding[7]; + unsigned long min; + unsigned long low; + unsigned long high; +diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h +index 0178b2040ea3..39f05e00dac4 100644 +--- a/include/linux/pagemap.h ++++ b/include/linux/pagemap.h +@@ -1183,7 +1183,7 @@ struct readahead_control { + ._index = i, \ + } + +-#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) ++#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) + + void page_cache_ra_unbounded(struct readahead_control *, + unsigned long nr_to_read, unsigned long lookahead_count); +diff --git a/include/linux/sched.h b/include/linux/sched.h +index e7b2f8a5c711..367b30ed77cb 100644 +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -1052,6 +1052,7 @@ struct task_struct { + /* Cached requested key. */ + struct key *cached_requested_key; + #endif ++ int fsync_count; + + /* + * executable name, excluding path. +diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h +index 33a4240e6a6f..82213f9c4c17 100644 +--- a/include/linux/user_namespace.h ++++ b/include/linux/user_namespace.h +@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns, + + #ifdef CONFIG_USER_NS + ++extern int unprivileged_userns_clone; ++ + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) + { + if (ns) +@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); + struct ns_common *ns_get_owner(struct ns_common *ns); + #else + ++#define unprivileged_userns_clone 0 ++ + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) + { + return &init_user_ns; +diff --git a/include/linux/wait.h b/include/linux/wait.h +index 58cfbf81447c..d5664b5ae3e1 100644 +--- a/include/linux/wait.h ++++ b/include/linux/wait.h +@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) + + extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); ++extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + +@@ -1164,6 +1165,7 @@ do { \ + */ + void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); + bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); ++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); + long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); + void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); + long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); +diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h +index d174914a837d..bf8e2af101a3 100644 +--- a/include/uapi/linux/if_bonding.h ++++ b/include/uapi/linux/if_bonding.h +@@ -82,7 +82,7 @@ + #define BOND_STATE_ACTIVE 0 /* link is active */ + #define BOND_STATE_BACKUP 1 /* link is backup */ + +-#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */ ++#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */ + + #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */ + +diff --git a/init/Kconfig b/init/Kconfig +index 532362fcfe31..442a945ca6ae 100644 +--- a/init/Kconfig ++++ b/init/Kconfig +@@ -112,6 +112,10 @@ config THREAD_INFO_IN_TASK + + menu "General setup" + ++config CACHY ++ bool "Some kernel tweaks by CachyOS" ++ default y ++ + config BROKEN + bool + +@@ -1241,6 +1245,22 @@ config USER_NS + + If unsure, say N. + ++config USER_NS_UNPRIVILEGED ++ bool "Allow unprivileged users to create namespaces" ++ default y ++ depends on USER_NS ++ help ++ When disabled, unprivileged users will not be able to create ++ new namespaces. Allowing users to create their own namespaces ++ has been part of several recent local privilege escalation ++ exploits, so if you need user namespaces but are ++ paranoid^Wsecurity-conscious you want to disable this. ++ ++ This setting can be overridden at runtime via the ++ kernel.unprivileged_userns_clone sysctl. ++ ++ If unsure, say Y. ++ + config PID_NS + bool "PID Namespaces" + default y +@@ -1407,6 +1427,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE + with the "-O2" compiler flag for best performance and most + helpful compile-time warnings. + ++config CC_OPTIMIZE_FOR_PERFORMANCE_O3 ++ bool "Optimize more for performance (-O3)" ++ help ++ Choosing this option will pass "-O3" to your compiler to optimize ++ the kernel yet more for performance. ++ + config CC_OPTIMIZE_FOR_SIZE + bool "Optimize for size (-Os)" + help +diff --git a/init/do_mounts.c b/init/do_mounts.c +index 7058e14ad5f7..ce4e841ea705 100644 +--- a/init/do_mounts.c ++++ b/init/do_mounts.c +@@ -283,8 +283,18 @@ dev_t name_to_dev_t(const char *name) + if (strcmp(name, "/dev/ram") == 0) + return Root_RAM0; + #ifdef CONFIG_BLOCK +- if (strncmp(name, "PARTUUID=", 9) == 0) +- return devt_from_partuuid(name + 9); ++ if (strncmp(name, "PARTUUID=", 9) == 0) { ++ dev_t res; ++ int needtowait = 40<<1; ++ res = devt_from_partuuid(name + 9); ++ while (!res && needtowait) { ++ /* waiting 0.5 sec */ ++ msleep(500); ++ res = devt_from_partuuid(name + 9); ++ needtowait--; ++ } ++ return res; ++ } + if (strncmp(name, "PARTLABEL=", 10) == 0) + return devt_from_partlabel(name + 10); + if (strncmp(name, "/dev/", 5) == 0) +@@ -612,7 +622,9 @@ void __init prepare_namespace(void) + * For example, it is not atypical to wait 5 seconds here + * for the touchpad of a laptop to initialize. + */ ++ async_synchronize_full(); + wait_for_device_probe(); ++ async_synchronize_full(); + + md_run_setup(); + +diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz +index 38ef6d06888e..0f78364efd4f 100644 +--- a/kernel/Kconfig.hz ++++ b/kernel/Kconfig.hz +@@ -40,6 +40,27 @@ choice + on SMP and NUMA systems and exactly dividing by both PAL and + NTSC frame rates for video and multimedia work. + ++ config HZ_500 ++ bool "500 HZ" ++ help ++ 500 Hz is a balanced timer frequency. Provides fast interactivity ++ on desktops with good smoothness without increasing CPU power ++ consumption and sacrificing the battery life on laptops. ++ ++ config HZ_600 ++ bool "600 HZ" ++ help ++ 600 Hz is a balanced timer frequency. Provides fast interactivity ++ on desktops with good smoothness without increasing CPU power ++ consumption and sacrificing the battery life on laptops. ++ ++ config HZ_750 ++ bool "750 HZ" ++ help ++ 750 Hz is a balanced timer frequency. Provides fast interactivity ++ on desktops with good smoothness without increasing CPU power ++ consumption and sacrificing the battery life on laptops. ++ + config HZ_1000 + bool "1000 HZ" + help +@@ -53,6 +74,9 @@ config HZ + default 100 if HZ_100 + default 250 if HZ_250 + default 300 if HZ_300 ++ default 500 if HZ_500 ++ default 600 if HZ_600 ++ default 750 if HZ_750 + default 1000 if HZ_1000 + + config SCHED_HRTICK +diff --git a/kernel/fork.c b/kernel/fork.c +index 2b6bd511c6ed..704fe6bc9cb4 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -99,6 +99,10 @@ + #include + #include + ++#ifdef CONFIG_USER_NS ++#include ++#endif ++ + #include + #include + #include +@@ -2009,6 +2013,10 @@ static __latent_entropy struct task_struct *copy_process( + if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) + return ERR_PTR(-EINVAL); + ++ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) ++ if (!capable(CAP_SYS_ADMIN)) ++ return ERR_PTR(-EPERM); ++ + /* + * Thread groups must share signals as well, and detached threads + * can only be started up within the thread group. +@@ -3159,6 +3167,12 @@ int ksys_unshare(unsigned long unshare_flags) + if (unshare_flags & CLONE_NEWNS) + unshare_flags |= CLONE_FS; + ++ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { ++ err = -EPERM; ++ if (!capable(CAP_SYS_ADMIN)) ++ goto bad_unshare_out; ++ } ++ + err = check_unshare_flags(unshare_flags); + if (err) + goto bad_unshare_out; +diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c +index 65f0262f635e..8a8f77930818 100644 +--- a/kernel/locking/rwsem.c ++++ b/kernel/locking/rwsem.c +@@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) + struct task_struct *new, *owner; + unsigned long flags, new_flags; + enum owner_state state; ++ int i = 0; + + lockdep_assert_preemption_disabled(); + +@@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) + break; + } + +- cpu_relax(); ++ if (i++ > 1000) ++ cpu_relax(); + } + + return state; +diff --git a/kernel/module/internal.h b/kernel/module/internal.h +index 680d980a4fb2..8a3abfff9fe9 100644 +--- a/kernel/module/internal.h ++++ b/kernel/module/internal.h +@@ -53,6 +53,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; + extern const s32 __start___kcrctab[]; + extern const s32 __start___kcrctab_gpl[]; + ++extern struct boot_params boot_params; ++ + struct load_info { + const char *name; + /* pointer to module in temporary copy, freed at end of load_module() */ +diff --git a/kernel/module/main.c b/kernel/module/main.c +index a4e4d84b6f4e..0e698f5bee86 100644 +--- a/kernel/module/main.c ++++ b/kernel/module/main.c +@@ -53,6 +53,7 @@ + #include + #include + #include ++#include + #include + #include "internal.h" + +diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c +index cf5b9f1e6ec4..80260fa4dac5 100644 +--- a/kernel/module/procfs.c ++++ b/kernel/module/procfs.c +@@ -141,6 +141,19 @@ static const struct proc_ops modules_proc_ops = { + static int __init proc_modules_init(void) + { + proc_create("modules", 0, NULL, &modules_proc_ops); ++ ++#ifdef CONFIG_MODULE_SIG_FORCE ++ switch (boot_params.secure_boot) { ++ case efi_secureboot_mode_unset: ++ case efi_secureboot_mode_unknown: ++ case efi_secureboot_mode_disabled: ++ /* ++ * sig_unenforce is only applied if SecureBoot is not ++ * enabled. ++ */ ++ sig_enforce = !sig_unenforce; ++ } ++#endif + return 0; + } + module_init(proc_modules_init); +diff --git a/kernel/module/signing.c b/kernel/module/signing.c +index a2ff4242e623..876e93758e91 100644 +--- a/kernel/module/signing.c ++++ b/kernel/module/signing.c +@@ -21,6 +21,10 @@ + + static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE); + module_param(sig_enforce, bool_enable_only, 0644); ++/* Allow disabling module signature requirement by adding boot param */ ++static bool sig_unenforce = false; ++module_param(sig_unenforce, bool_enable_only, 0644); ++ + + /* + * Export sig_enforce kernel cmdline parameter to allow other subsystems rely +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index ee28253c9ac0..f2534e712a89 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -138,6 +138,18 @@ __read_mostly int sysctl_resched_latency_warn_ms = 100; + __read_mostly int sysctl_resched_latency_warn_once = 1; + #endif /* CONFIG_SCHED_DEBUG */ + ++/* ++ * Choose the yield level that will perform. ++ * 0: No yield. ++ * 1: Yield only to better priority/deadline tasks. ++ * 2: Re-queue current tasks. (default CFS) ++ */ ++#ifdef CONFIG_CACHY ++__read_mostly int sysctl_sched_yield_type = 1; ++#else ++__read_mostly int sysctl_sched_yield_type = 0; ++#endif ++ + /* + * Number of tasks to iterate in a single balance run. + * Limited because this is done with IRQs disabled. +@@ -8266,10 +8278,15 @@ static void do_sched_yield(void) + struct rq_flags rf; + struct rq *rq; + ++ if (!sysctl_sched_yield_type) ++ return; ++ + rq = this_rq_lock_irq(&rf); + + schedstat_inc(rq->yld_count); +- current->sched_class->yield_task(rq); ++ ++ if (sysctl_sched_yield_type > 1) ++ current->sched_class->yield_task(rq); + + preempt_disable(); + rq_unlock_irq(rq, &rf); +diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c +index 667876da8382..13fbcd19b916 100644 +--- a/kernel/sched/debug.c ++++ b/kernel/sched/debug.c +@@ -959,6 +959,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, + PN(se.exec_start); + PN(se.vruntime); + PN(se.sum_exec_runtime); ++ P(fsync_count); + + nr_switches = p->nvcsw + p->nivcsw; + +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index 914096c5b1ae..74d773040b54 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -68,9 +68,13 @@ + * + * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) + */ ++#ifdef CONFIG_CACHY ++unsigned int sysctl_sched_latency = 3000000ULL; ++static unsigned int normalized_sysctl_sched_latency = 3000000ULL; ++#else + unsigned int sysctl_sched_latency = 6000000ULL; + static unsigned int normalized_sysctl_sched_latency = 6000000ULL; +- ++#endif + /* + * The initial- and re-scaling of tunables is configurable + * +@@ -89,8 +93,13 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; + * + * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) + */ ++#ifdef CONFIG_CACHY ++unsigned int sysctl_sched_min_granularity = 400000ULL; ++static unsigned int normalized_sysctl_sched_min_granularity = 400000ULL; ++#else + unsigned int sysctl_sched_min_granularity = 750000ULL; + static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; ++#endif + + /* + * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. +@@ -120,8 +129,13 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; + * + * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) + */ ++#ifdef CONFIG_CACHY ++unsigned int sysctl_sched_wakeup_granularity = 500000UL; ++static unsigned int normalized_sysctl_sched_wakeup_granularity = 500000UL; ++#else + unsigned int sysctl_sched_wakeup_granularity = 1000000UL; + static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; ++#endif + + const_debug unsigned int sysctl_sched_migration_cost = 500000UL; + +@@ -174,8 +188,12 @@ int __weak arch_asym_cpu_priority(int cpu) + * + * (default: 5 msec, units: microseconds) + */ ++#ifdef CONFIG_CACHY ++static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; ++#else + static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; + #endif ++#endif + + #ifdef CONFIG_SYSCTL + static struct ctl_table sched_fair_sysctls[] = { +diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c +index 9860bb9a847c..c7f045873a5f 100644 +--- a/kernel/sched/wait.c ++++ b/kernel/sched/wait.c +@@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ + } + EXPORT_SYMBOL_GPL(add_wait_queue_priority); + ++void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) ++{ ++ unsigned long flags; ++ ++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; ++ spin_lock_irqsave(&wq_head->lock, flags); ++ __add_wait_queue(wq_head, wq_entry); ++ spin_unlock_irqrestore(&wq_head->lock, flags); ++} ++EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); ++ + void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) + { + unsigned long flags; +@@ -289,6 +300,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent + } + EXPORT_SYMBOL(prepare_to_wait_exclusive); + ++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) ++{ ++ unsigned long flags; ++ ++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; ++ spin_lock_irqsave(&wq_head->lock, flags); ++ if (list_empty(&wq_entry->entry)) ++ __add_wait_queue(wq_head, wq_entry); ++ set_current_state(state); ++ spin_unlock_irqrestore(&wq_head->lock, flags); ++} ++EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); ++ + void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) + { + wq_entry->flags = flags; +diff --git a/kernel/sysctl.c b/kernel/sysctl.c +index 205d605cacc5..4432b6f15890 100644 +--- a/kernel/sysctl.c ++++ b/kernel/sysctl.c +@@ -89,8 +89,12 @@ + #ifdef CONFIG_PERF_EVENTS + static const int six_hundred_forty_kb = 640 * 1024; + #endif ++#ifdef CONFIG_USER_NS ++#include ++#endif + + ++extern int sysctl_sched_yield_type; + static const int ngroups_max = NGROUPS_MAX; + static const int cap_last_cap = CAP_LAST_CAP; + +@@ -1649,6 +1653,24 @@ static struct ctl_table kern_table[] = { + .mode = 0644, + .proc_handler = proc_dointvec, + }, ++ { ++ .procname = "yield_type", ++ .data = &sysctl_sched_yield_type, ++ .maxlen = sizeof (int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_TWO, ++ }, ++#ifdef CONFIG_USER_NS ++ { ++ .procname = "unprivileged_userns_clone", ++ .data = &unprivileged_userns_clone, ++ .maxlen = sizeof(int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec, ++ }, ++#endif + #ifdef CONFIG_PROC_SYSCTL + { + .procname = "tainted", +diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c +index 5481ba44a8d6..423ab2563ad7 100644 +--- a/kernel/user_namespace.c ++++ b/kernel/user_namespace.c +@@ -21,6 +21,13 @@ + #include + #include + ++/* sysctl */ ++#ifdef CONFIG_USER_NS_UNPRIVILEGED ++int unprivileged_userns_clone = 1; ++#else ++int unprivileged_userns_clone; ++#endif ++ + static struct kmem_cache *user_ns_cachep __read_mostly; + static DEFINE_MUTEX(userns_state_mutex); + +diff --git a/kernel/watchdog.c b/kernel/watchdog.c +index 8e61f21e7e33..be1439d38f26 100644 +--- a/kernel/watchdog.c ++++ b/kernel/watchdog.c +@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled; + int __read_mostly watchdog_user_enabled = 1; + int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT; + int __read_mostly soft_watchdog_user_enabled = 1; +-int __read_mostly watchdog_thresh = 10; ++int __read_mostly watchdog_thresh = 40; + static int __read_mostly nmi_watchdog_available; + + struct cpumask watchdog_cpumask __read_mostly; +diff --git a/lib/Kconfig b/lib/Kconfig +index dc1ab2ed1dc6..3ea8941ab18d 100644 +--- a/lib/Kconfig ++++ b/lib/Kconfig +@@ -343,12 +343,16 @@ config LZ4HC_COMPRESS + config LZ4_DECOMPRESS + tristate + +-config ZSTD_COMPRESS ++config ZSTD_COMMON + select XXHASH + tristate + ++config ZSTD_COMPRESS ++ select ZSTD_COMMON ++ tristate ++ + config ZSTD_DECOMPRESS +- select XXHASH ++ select ZSTD_COMMON + tristate + + source "lib/xz/Kconfig" +diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c +index 39b74221f4a7..ec3eab8cd6b1 100644 +--- a/lib/raid6/algos.c ++++ b/lib/raid6/algos.c +@@ -128,8 +128,10 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) + + for (best = NULL, algo = raid6_recov_algos; *algo; algo++) + if (!best || (*algo)->priority > best->priority) +- if (!(*algo)->valid || (*algo)->valid()) ++ if (!(*algo)->valid || (*algo)->valid()) { + best = *algo; ++ break; ++ } + + if (best) { + raid6_2data_recov = best->data2; +diff --git a/lib/string.c b/lib/string.c +index 6f334420f687..2e60e9e40535 100644 +--- a/lib/string.c ++++ b/lib/string.c +@@ -866,24 +866,61 @@ char *strnstr(const char *s1, const char *s2, size_t len) + EXPORT_SYMBOL(strnstr); + #endif + ++#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 ++ ++#define MEMCHR_MASK_GEN(mask) (mask *= 0x0101010101010101ULL) ++ ++#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) ++ ++#define MEMCHR_MASK_GEN(mask) \ ++ do { \ ++ mask *= 0x01010101; \ ++ mask |= mask << 32; \ ++ } while (0) ++ ++#else ++ ++#define MEMCHR_MASK_GEN(mask) \ ++ do { \ ++ mask |= mask << 8; \ ++ mask |= mask << 16; \ ++ mask |= mask << 32; \ ++ } while (0) ++ ++#endif ++ + #ifndef __HAVE_ARCH_MEMCHR + /** + * memchr - Find a character in an area of memory. +- * @s: The memory area ++ * @p: The memory area + * @c: The byte to search for +- * @n: The size of the area. ++ * @length: The size of the area. + * + * returns the address of the first occurrence of @c, or %NULL + * if @c is not found + */ +-void *memchr(const void *s, int c, size_t n) ++void *memchr(const void *p, int c, unsigned long length) + { +- const unsigned char *p = s; +- while (n-- != 0) { +- if ((unsigned char)c == *p++) { +- return (void *)(p - 1); ++ u64 mask, val; ++ const void *end = p + length; ++ ++ c &= 0xff; ++ if (p <= end - 8) { ++ mask = c; ++ MEMCHR_MASK_GEN(mask); ++ ++ for (; p <= end - 8; p += 8) { ++ val = *(u64 *)p ^ mask; ++ if ((val + 0xfefefefefefefeffu) & ++ (~val & 0x8080808080808080u)) ++ break; + } + } ++ ++ for (; p < end; p++) ++ if (*(unsigned char *)p == c) ++ return (void *)p; ++ + return NULL; + } + EXPORT_SYMBOL(memchr); +@@ -919,16 +956,7 @@ void *memchr_inv(const void *start, int c, size_t bytes) + return check_bytes8(start, value, bytes); + + value64 = value; +-#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 +- value64 *= 0x0101010101010101ULL; +-#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) +- value64 *= 0x01010101; +- value64 |= value64 << 32; +-#else +- value64 |= value64 << 8; +- value64 |= value64 << 16; +- value64 |= value64 << 32; +-#endif ++ MEMCHR_MASK_GEN(value64); + + prefix = (unsigned long)start % 8; + if (prefix) { +diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile +index fc45339fc3a3..440bd0007ae2 100644 +--- a/lib/zstd/Makefile ++++ b/lib/zstd/Makefile +@@ -10,14 +10,10 @@ + # ################################################################ + obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o + obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o ++obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o + + zstd_compress-y := \ + zstd_compress_module.o \ +- common/debug.o \ +- common/entropy_common.o \ +- common/error_private.o \ +- common/fse_decompress.o \ +- common/zstd_common.o \ + compress/fse_compress.o \ + compress/hist.o \ + compress/huf_compress.o \ +@@ -33,12 +29,14 @@ zstd_compress-y := \ + + zstd_decompress-y := \ + zstd_decompress_module.o \ ++ decompress/huf_decompress.o \ ++ decompress/zstd_ddict.o \ ++ decompress/zstd_decompress.o \ ++ decompress/zstd_decompress_block.o \ ++ ++zstd_common-y := \ + common/debug.o \ + common/entropy_common.o \ + common/error_private.o \ + common/fse_decompress.o \ + common/zstd_common.o \ +- decompress/huf_decompress.o \ +- decompress/zstd_ddict.o \ +- decompress/zstd_decompress.o \ +- decompress/zstd_decompress_block.o \ +diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c +index 53b47a2b52ff..f84612627471 100644 +--- a/lib/zstd/common/entropy_common.c ++++ b/lib/zstd/common/entropy_common.c +@@ -15,6 +15,7 @@ + /* ************************************* + * Dependencies + ***************************************/ ++#include + #include "mem.h" + #include "error_private.h" /* ERR_*, ERROR */ + #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +@@ -239,7 +240,7 @@ size_t FSE_readNCount( + { + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); + } +- ++EXPORT_SYMBOL_GPL(FSE_readNCount); + + /*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). +@@ -255,6 +256,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + } ++EXPORT_SYMBOL_GPL(HUF_readStats); + + FORCE_INLINE_TEMPLATE size_t + HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, +diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c +index 3d7e35b309b5..06f62b2026d5 100644 +--- a/lib/zstd/common/zstd_common.c ++++ b/lib/zstd/common/zstd_common.c +@@ -13,6 +13,7 @@ + /*-************************************* + * Dependencies + ***************************************/ ++#include + #define ZSTD_DEPS_NEED_MALLOC + #include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + #include "error_private.h" +@@ -59,6 +60,7 @@ void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); + } ++EXPORT_SYMBOL_GPL(ZSTD_customMalloc); + + void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) + { +@@ -71,6 +73,7 @@ void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) + } + return ZSTD_calloc(1, size); + } ++EXPORT_SYMBOL_GPL(ZSTD_customCalloc); + + void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) + { +@@ -81,3 +84,7 @@ void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) + ZSTD_free(ptr); + } + } ++EXPORT_SYMBOL_GPL(ZSTD_customFree); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("Zstd Common"); +diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c +index b0424d23ac57..fb941a5b70f5 100644 +--- a/lib/zstd/compress/zstd_double_fast.c ++++ b/lib/zstd/compress/zstd_double_fast.c +@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(dictMode, mls) \ ++ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \ ++ } ++ ++ZSTD_GEN_FN(noDict, 4) ++ZSTD_GEN_FN(noDict, 5) ++ZSTD_GEN_FN(noDict, 6) ++ZSTD_GEN_FN(noDict, 7) ++ ++ZSTD_GEN_FN(dictMatchState, 4) ++ZSTD_GEN_FN(dictMatchState, 5) ++ZSTD_GEN_FN(dictMatchState, 6) ++ZSTD_GEN_FN(dictMatchState, 7) ++ ++#undef ZSTD_GEN_FN ++ + + size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); ++ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) +- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); ++ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ +@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + } + + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN ++ + size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); + } + } +diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c +index 96b7d48e2868..e0652e31d421 100644 +--- a/lib/zstd/compress/zstd_fast.c ++++ b/lib/zstd/compress/zstd_fast.c +@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_fast_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN + + size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN ++ + size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } + } + +@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) +- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); ++ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ +@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( + return (size_t)(iend - anchor); + } + ++#define ZSTD_GEN_FN(mls) \ ++ static size_t ZSTD_compressBlock_fast_extDict_##mls( \ ++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ++ void const* src, size_t srcSize) \ ++ { \ ++ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ ++ } ++ ++ZSTD_GEN_FN(4) ++ZSTD_GEN_FN(5) ++ZSTD_GEN_FN(6) ++ZSTD_GEN_FN(7) ++ ++#undef ZSTD_GEN_FN + + size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], +@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict( + { + default: /* includes case 3 */ + case 4 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); ++ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); ++ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); ++ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : +- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); ++ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize); + } + } +diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c +index fb54d4e28a2b..1db22db5b1e9 100644 +--- a/lib/zstd/compress/zstd_lazy.c ++++ b/lib/zstd/compress/zstd_lazy.c +@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + } + + +-static size_t +-ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); +- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); +- case 7 : +- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); +- } +-} +- +- +-static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); +- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); +- case 7 : +- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); +- } +-} +- +- +-static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); +- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); +- case 7 : +- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); +- } +-} +- +- +- + /* ********************************* + * Hash Chain + ***********************************/ +@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B + + /* inlining is important to hardwire a hot branch (template emulation) */ + FORCE_INLINE_TEMPLATE +-size_t ZSTD_HcFindBestMatch_generic ( ++size_t ZSTD_HcFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, +@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic ( + return ml; + } + ++typedef size_t (*searchMax_f)( ++ ZSTD_matchState_t* ms, ++ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + +-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); +- } +-} ++/* ++ * This struct contains the functions necessary for lazy to search. ++ * Currently, that is only searchMax. However, it is still valuable to have the ++ * VTable because this makes it easier to add more functions to the VTable later. ++ */ ++typedef struct { ++ searchMax_f searchMax; ++} ZSTD_LazyVTable; ++ ++#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \ ++ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ ++ ZSTD_matchState_t* ms, \ ++ const BYTE* ip, const BYTE* const iLimit, \ ++ size_t* offsetPtr) \ ++ { \ ++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ ++ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ ++ } \ ++ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ ++ ZSTD_BtFindBestMatch_##dictMode##_##mls \ ++ }; + ++#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \ ++ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ ++ ZSTD_matchState_t* ms, \ ++ const BYTE* ip, const BYTE* const iLimit, \ ++ size_t* offsetPtr) \ ++ { \ ++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ ++ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ ++ } \ ++ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ ++ ZSTD_HcFindBestMatch_##dictMode##_##mls \ ++ }; + +-static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); ++#define ZSTD_FOR_EACH_MLS(X, dictMode) \ ++ X(dictMode, 4) \ ++ X(dictMode, 5) \ ++ X(dictMode, 6) ++ ++#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ ++ X(__VA_ARGS__, noDict) \ ++ X(__VA_ARGS__, extDict) \ ++ X(__VA_ARGS__, dictMatchState) \ ++ X(__VA_ARGS__, dedicatedDictSearch) ++ ++/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ ++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) ++/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ ++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) ++ ++#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ ++ { \ ++ &ZSTD_BtVTable_##dictMode##_4, \ ++ &ZSTD_BtVTable_##dictMode##_5, \ ++ &ZSTD_BtVTable_##dictMode##_6 \ + } +-} +- + +-static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); ++#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ ++ { \ ++ &ZSTD_HcVTable_##dictMode##_4, \ ++ &ZSTD_HcVTable_##dictMode##_5, \ ++ &ZSTD_HcVTable_##dictMode##_6 \ + } +-} + +- +-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* const iLimit, +- size_t* offsetPtr) +-{ +- switch(ms->cParams.minMatch) +- { +- default : /* includes case 3 */ +- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); +- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); +- case 7 : +- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); ++#define GEN_ZSTD_VTABLE_ARRAY(X) \ ++ { \ ++ X(noDict), \ ++ X(extDict), \ ++ X(dictMatchState), \ ++ X(dedicatedDictSearch) \ + } +-} +- + + /* ******************************* + * Common parser - lazy strategy + *********************************/ + typedef enum { search_hashChain, search_binaryTree } searchMethod_e; + ++static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) ++{ ++ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ ++ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); ++ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); ++ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ ++ ++ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); ++ switch (searchMethod) { ++ case search_hashChain: ++ return hcVTables[dictMode][mls - 4]; ++ case search_binaryTree: ++ return btVTables[dictMode][mls - 4]; ++ default: ++ return NULL; ++ } ++} ++ + FORCE_INLINE_TEMPLATE size_t + ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, +@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic( + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + +- typedef size_t (*searchMax_f)( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); +- + /* + * This table is indexed first by the four ZSTD_dictMode_e values, and then + * by the two searchMethod_e values. NULLs are placed for configurations + * that should never occur (extDict modes go to the other implementation + * below and there is no DDSS for binary tree search yet). + */ +- const searchMax_f searchFuncs[4][2] = { +- { +- ZSTD_HcFindBestMatch_selectMLS, +- ZSTD_BtFindBestMatch_selectMLS +- }, +- { +- NULL, +- NULL +- }, +- { +- ZSTD_HcFindBestMatch_dictMatchState_selectMLS, +- ZSTD_BtFindBestMatch_dictMatchState_selectMLS +- }, +- { +- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, +- NULL +- } +- }; +- +- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; ++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; + U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + + const int isDMS = dictMode == ZSTD_dictMatchState; +@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + +- typedef size_t (*searchMax_f)( +- ZSTD_matchState_t* ms, +- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); +- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; ++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; + + U32 offset_1 = rep[0], offset_2 = rep[1]; + +diff --git a/mm/compaction.c b/mm/compaction.c +index 640fa76228dd..01a661f5ecac 100644 +--- a/mm/compaction.c ++++ b/mm/compaction.c +@@ -1727,7 +1727,7 @@ typedef enum { + * Allow userspace to control policy on scanning the unevictable LRU for + * compactable pages. + */ +-#ifdef CONFIG_PREEMPT_RT ++#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) + int sysctl_compact_unevictable_allowed __read_mostly = 0; + #else + int sysctl_compact_unevictable_allowed __read_mostly = 1; +@@ -2719,7 +2719,11 @@ static void compact_nodes(void) + * aggressively the kernel should compact memory in the + * background. It takes values in the range [0, 100]. + */ ++#ifdef CONFIG_CACHY ++unsigned int __read_mostly sysctl_compaction_proactiveness; ++#else + unsigned int __read_mostly sysctl_compaction_proactiveness = 20; ++#endif + + int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, + void *buffer, size_t *length, loff_t *ppos) +diff --git a/mm/ksm.c b/mm/ksm.c +index 42ab153335a2..4e64adb9adee 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -2427,9 +2427,14 @@ static int ksm_scan_thread(void *nothing) + + if (ksmd_should_run()) { + sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs); +- wait_event_interruptible_timeout(ksm_iter_wait, +- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), +- msecs_to_jiffies(sleep_ms)); ++ if (sleep_ms >= 1000) ++ wait_event_interruptible_timeout(ksm_iter_wait, ++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), ++ msecs_to_jiffies(round_jiffies_relative(sleep_ms))); ++ else ++ wait_event_interruptible_timeout(ksm_iter_wait, ++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), ++ msecs_to_jiffies(sleep_ms)); + } else { + wait_event_freezable(ksm_thread_wait, + ksmd_should_run() || kthread_should_stop()); +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index b69979c9ced5..7eadbafc006b 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -625,7 +625,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) + cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); + + x = __this_cpu_add_return(stats_updates, abs(val)); +- if (x > MEMCG_CHARGE_BATCH) { ++ if (x > MEMCG_CHARGE_BATCH * 128) { + /* + * If stats_flush_threshold exceeds the threshold + * (>num_online_cpus()), cgroup stats update will be triggered +diff --git a/mm/page-writeback.c b/mm/page-writeback.c +index 032a7bf8d259..cbcecd565c16 100644 +--- a/mm/page-writeback.c ++++ b/mm/page-writeback.c +@@ -70,7 +70,11 @@ static long ratelimit_pages = 32; + /* + * Start background writeback (via writeback threads) at this percentage + */ ++#ifdef CONFIG_CACHY ++static int dirty_background_ratio = 5; ++#else + static int dirty_background_ratio = 10; ++#endif + + /* + * dirty_background_bytes starts at 0 (disabled) so that it is a function of +@@ -98,7 +102,11 @@ static unsigned long vm_dirty_bytes; + /* + * The interval between `kupdate'-style writebacks + */ ++#ifdef CONFIG_CACHY ++unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */ ++#else + unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ ++#endif + + EXPORT_SYMBOL_GPL(dirty_writeback_interval); + +diff --git a/mm/page_alloc.c b/mm/page_alloc.c +index e5486d47406e..cf131d6e08fb 100644 +--- a/mm/page_alloc.c ++++ b/mm/page_alloc.c +@@ -6982,11 +6982,11 @@ static int zone_batchsize(struct zone *zone) + + /* + * The number of pages to batch allocate is either ~0.1% +- * of the zone or 1MB, whichever is smaller. The batch ++ * of the zone or 4MB, whichever is smaller. The batch + * size is striking a balance between allocation latency + * and zone lock contention. + */ +- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); ++ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); + batch /= 4; /* We effectively *= 4 below */ + if (batch < 1) + batch = 1; +@@ -7064,6 +7064,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) + * historical relationship between high and batch. + */ + high = max(high, batch << 2); ++ high = max(high, 1024); + + return high; + #else +diff --git a/mm/swap.c b/mm/swap.c +index 9cee7f6a3809..186b4e5dcecf 100644 +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -1070,6 +1070,10 @@ EXPORT_SYMBOL(pagevec_lookup_range_tag); + */ + void __init swap_setup(void) + { ++#ifdef CONFIG_CACHY ++ /* Only swap-in pages requested, avoid readahead */ ++ page_cluster = 0; ++#else + unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); + + /* Use a smaller cluster for small-memory machines */ +@@ -1081,4 +1085,5 @@ void __init swap_setup(void) + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ ++#endif + } +diff --git a/mm/vmpressure.c b/mm/vmpressure.c +index b52644771cc4..11a4b0e3b583 100644 +--- a/mm/vmpressure.c ++++ b/mm/vmpressure.c +@@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; + * essence, they are percents: the higher the value, the more number + * unsuccessful reclaims there were. + */ ++#ifdef CONFIG_CACHY ++static const unsigned int vmpressure_level_med = 65; ++#else + static const unsigned int vmpressure_level_med = 60; ++#endif + static const unsigned int vmpressure_level_critical = 95; + + /* +diff --git a/mm/vmscan.c b/mm/vmscan.c +index b2b1431352dc..0fc65ace3a4e 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -178,7 +178,11 @@ struct scan_control { + /* + * From 0 .. 200. Higher means more swappy. + */ ++#ifdef CONFIG_CACHY ++int vm_swappiness = 20; ++#else + int vm_swappiness = 60; ++#endif + + static void set_task_reclaim_state(struct task_struct *task, + struct reclaim_state *rs) +diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c +index eb31c7158b39..8cefa0b62fb9 100644 +--- a/net/ipv4/inet_connection_sock.c ++++ b/net/ipv4/inet_connection_sock.c +@@ -445,7 +445,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) + * having to remove and re-insert us on the wait queue. + */ + for (;;) { +- prepare_to_wait_exclusive(sk_sleep(sk), &wait, ++ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, + TASK_INTERRUPTIBLE); + release_sock(sk); + if (reqsk_queue_empty(&icsk->icsk_accept_queue)) +diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c +index e373dde1f46f..3fc8ca6a264e 100644 +--- a/net/ipv4/tcp.c ++++ b/net/ipv4/tcp.c +@@ -4789,8 +4789,8 @@ void __init tcp_init(void) + tcp_init_mem(); + /* Set per-socket limits to no more than 1/128 the pressure threshold */ + limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); +- max_wshare = min(4UL*1024*1024, limit); +- max_rshare = min(6UL*1024*1024, limit); ++ max_wshare = min(16UL*1024*1024, limit); ++ max_rshare = min(16UL*1024*1024, limit); + + init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; + init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; +-- +2.38.0.rc1.8.g2a7d63a245 + +From 0feaada45827f920b03a53edea1d34597614db84 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 5 Sep 2022 08:34:43 +0200 +Subject: [PATCH 02/16] bbr2 + +Signed-off-by: Peter Jung +--- + Documentation/networking/ip-sysctl.rst | 58 + + include/linux/tcp.h | 6 +- + include/net/inet_connection_sock.h | 3 +- + include/net/netns/ipv4.h | 5 + + include/net/tcp.h | 58 +- + include/uapi/linux/inet_diag.h | 33 + + include/uapi/linux/snmp.h | 1 + + net/ipv4/Kconfig | 22 + + net/ipv4/Makefile | 3 +- + net/ipv4/bpf_tcp_ca.c | 12 + + net/ipv4/proc.c | 1 + + net/ipv4/sysctl_net_ipv4.c | 43 + + net/ipv4/tcp.c | 1 + + net/ipv4/tcp_bbr.c | 38 +- + net/ipv4/tcp_bbr2.c | 2692 ++++++++++++++++++++++++ + net/ipv4/tcp_cong.c | 1 + + net/ipv4/tcp_input.c | 27 +- + net/ipv4/tcp_ipv4.c | 7 + + net/ipv4/tcp_output.c | 26 +- + net/ipv4/tcp_plb.c | 100 + + net/ipv4/tcp_rate.c | 30 +- + net/ipv4/tcp_timer.c | 1 + + 22 files changed, 3133 insertions(+), 35 deletions(-) + create mode 100644 net/ipv4/tcp_bbr2.c + create mode 100644 net/ipv4/tcp_plb.c + +diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst +index a759872a2883..f2372f8f860b 100644 +--- a/Documentation/networking/ip-sysctl.rst ++++ b/Documentation/networking/ip-sysctl.rst +@@ -1040,6 +1040,64 @@ tcp_challenge_ack_limit - INTEGER + TCP stack implements per TCP socket limits anyway. + Default: INT_MAX (unlimited) + ++tcp_plb_enabled - BOOLEAN ++ If set, TCP PLB (Protective Load Balancing) is enabled. PLB is ++ described in the following paper: ++ https://doi.org/10.1145/3544216.3544226. Based on PLB parameters, ++ upon sensing sustained congestion, TCP triggers a change in ++ flow label field for outgoing IPv6 packets. A change in flow label ++ field potentially changes the path of outgoing packets for switches ++ that use ECMP/WCMP for routing. ++ ++ Default: 0 ++ ++tcp_plb_cong_thresh - INTEGER ++ Fraction of packets marked with congestion over a round (RTT) to ++ tag that round as congested. This is referred to as K in the PLB paper: ++ https://doi.org/10.1145/3544216.3544226. ++ ++ The 0-1 fraction range is mapped to 0-256 range to avoid floating ++ point operations. For example, 128 means that if at least 50% of ++ the packets in a round were marked as congested then the round ++ will be tagged as congested. ++ ++ Possible Values: 0 - 256 ++ ++ Default: 128 ++ ++tcp_plb_idle_rehash_rounds - INTEGER ++ Number of consecutive congested rounds (RTT) seen after which ++ a rehash can be performed, given there are no packets in flight. ++ This is referred to as M in PLB paper: ++ https://doi.org/10.1145/3544216.3544226. ++ ++ Possible Values: 0 - 31 ++ ++ Default: 3 ++ ++tcp_plb_rehash_rounds - INTEGER ++ Number of consecutive congested rounds (RTT) seen after which ++ a forced rehash can be performed. Be careful when setting this ++ parameter, as a small value increases the risk of retransmissions. ++ This is referred to as N in PLB paper: ++ https://doi.org/10.1145/3544216.3544226. ++ ++ Possible Values: 0 - 31 ++ ++ Default: 12 ++ ++tcp_plb_suspend_rto_sec - INTEGER ++ Time, in seconds, to suspend PLB in event of an RTO. In order to avoid ++ having PLB repath onto a connectivity "black hole", after an RTO a TCP ++ connection suspends PLB repathing for a random duration between 1x and ++ 2x of this parameter. Randomness is added to avoid concurrent rehashing ++ of multiple TCP connections. This should be set corresponding to the ++ amount of time it takes to repair a failed link. ++ ++ Possible Values: 0 - 255 ++ ++ Default: 60 ++ + UDP variables + ============= + +diff --git a/include/linux/tcp.h b/include/linux/tcp.h +index a9fbe22732c3..4034e46093b4 100644 +--- a/include/linux/tcp.h ++++ b/include/linux/tcp.h +@@ -255,7 +255,8 @@ struct tcp_sock { + u8 compressed_ack; + u8 dup_ack_counter:2, + tlp_retrans:1, /* TLP is a retransmission */ +- unused:5; ++ fast_ack_mode:2, /* which fast ack mode ? */ ++ unused:3; + u32 chrono_start; /* Start time in jiffies of a TCP chrono */ + u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ + u8 chrono_type:2, /* current chronograph type */ +@@ -443,6 +444,9 @@ struct tcp_sock { + */ + struct request_sock __rcu *fastopen_rsk; + struct saved_syn *saved_syn; ++ ++/* Rerouting information */ ++ u16 ecn_rehash; /* PLB triggered rehash attempts */ + }; + + enum tsq_enum { +diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h +index ee88f0f1350f..e3075b3f1ece 100644 +--- a/include/net/inet_connection_sock.h ++++ b/include/net/inet_connection_sock.h +@@ -132,7 +132,8 @@ struct inet_connection_sock { + u32 icsk_probes_tstamp; + u32 icsk_user_timeout; + +- u64 icsk_ca_priv[104 / sizeof(u64)]; ++/* XXX inflated by temporary internal debugging info */ ++ u64 icsk_ca_priv[224 / sizeof(u64)]; + #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) + }; + +diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h +index 6320a76cefdc..2e39e07ed41f 100644 +--- a/include/net/netns/ipv4.h ++++ b/include/net/netns/ipv4.h +@@ -181,6 +181,11 @@ struct netns_ipv4 { + unsigned long tfo_active_disable_stamp; + u32 tcp_challenge_timestamp; + u32 tcp_challenge_count; ++ u8 sysctl_tcp_plb_enabled; ++ int sysctl_tcp_plb_cong_thresh; ++ u8 sysctl_tcp_plb_idle_rehash_rounds; ++ u8 sysctl_tcp_plb_rehash_rounds; ++ u8 sysctl_tcp_plb_suspend_rto_sec; + + int sysctl_udp_wmem_min; + int sysctl_udp_rmem_min; +diff --git a/include/net/tcp.h b/include/net/tcp.h +index d10962b9f0d0..d3947345feaa 100644 +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -372,6 +372,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk, + #define TCP_ECN_QUEUE_CWR 2 + #define TCP_ECN_DEMAND_CWR 4 + #define TCP_ECN_SEEN 8 ++#define TCP_ECN_ECT_PERMANENT 16 + + enum tcp_tw_status { + TCP_TW_SUCCESS = 0, +@@ -816,6 +817,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) + return max_t(s64, t1 - t0, 0); + } + ++static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) ++{ ++ return max_t(s32, t1 - t0, 0); ++} ++ + static inline u32 tcp_skb_timestamp(const struct sk_buff *skb) + { + return tcp_ns_to_ts(skb->skb_mstamp_ns); +@@ -891,9 +897,14 @@ struct tcp_skb_cb { + /* pkts S/ACKed so far upon tx of skb, incl retrans: */ + __u32 delivered; + /* start of send pipeline phase */ +- u64 first_tx_mstamp; ++ u32 first_tx_mstamp; + /* when we reached the "delivered" count */ +- u64 delivered_mstamp; ++ u32 delivered_mstamp; ++#define TCPCB_IN_FLIGHT_BITS 20 ++#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) ++ u32 in_flight:20, /* packets in flight at transmit */ ++ unused2:12; ++ u32 lost; /* packets lost so far upon tx of skb */ + } tx; /* only used for outgoing skbs */ + union { + struct inet_skb_parm h4; +@@ -1019,7 +1030,11 @@ enum tcp_ca_ack_event_flags { + #define TCP_CONG_NON_RESTRICTED 0x1 + /* Requires ECN/ECT set on all packets */ + #define TCP_CONG_NEEDS_ECN 0x2 +-#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) ++/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ ++#define TCP_CONG_WANTS_CE_EVENTS 0x4 ++#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ ++ TCP_CONG_NEEDS_ECN | \ ++ TCP_CONG_WANTS_CE_EVENTS) + + union tcp_cc_info; + +@@ -1039,8 +1054,11 @@ struct ack_sample { + */ + struct rate_sample { + u64 prior_mstamp; /* starting timestamp for interval */ ++ u32 prior_lost; /* tp->lost at "prior_mstamp" */ + u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ + u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ ++ u32 tx_in_flight; /* packets in flight at starting timestamp */ ++ s32 lost; /* number of packets lost over interval */ + s32 delivered; /* number of packets delivered over interval */ + s32 delivered_ce; /* number of packets delivered w/ CE marks*/ + long interval_us; /* time for tp->delivered to incr "delivered" */ +@@ -1054,6 +1072,7 @@ struct rate_sample { + bool is_app_limited; /* is sample from packet with bubble in pipe? */ + bool is_retrans; /* is sample from retransmission? */ + bool is_ack_delayed; /* is this (likely) a delayed ACK? */ ++ bool is_ece; /* did this ACK have ECN marked? */ + }; + + struct tcp_congestion_ops { +@@ -1077,8 +1096,11 @@ struct tcp_congestion_ops { + /* hook for packet ack accounting (optional) */ + void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); + +- /* override sysctl_tcp_min_tso_segs */ +- u32 (*min_tso_segs)(struct sock *sk); ++ /* pick target number of segments per TSO/GSO skb (optional): */ ++ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); ++ ++ /* react to a specific lost skb (optional) */ ++ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); + + /* call when packets are delivered to update cwnd and pacing rate, + * after all the ca_state processing. (optional) +@@ -1141,6 +1163,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) + } + #endif + ++static inline bool tcp_ca_wants_ce_events(const struct sock *sk) ++{ ++ const struct inet_connection_sock *icsk = inet_csk(sk); ++ ++ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | ++ TCP_CONG_WANTS_CE_EVENTS); ++} ++ + static inline bool tcp_ca_needs_ecn(const struct sock *sk) + { + const struct inet_connection_sock *icsk = inet_csk(sk); +@@ -1160,6 +1190,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) + void tcp_set_ca_state(struct sock *sk, const u8 ca_state); + + /* From tcp_rate.c */ ++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); + void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); + void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, + struct rate_sample *rs); +@@ -2130,6 +2161,23 @@ extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, + extern void tcp_rack_reo_timeout(struct sock *sk); + extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); + ++/* tcp_plb.c */ ++ ++#define TCP_PLB_SCALE 8 /* scaling factor for fractions in PLB (e.g. ce_ratio) */ ++ ++/* State for PLB (Protective Load Balancing) for a single TCP connection. */ ++struct tcp_plb_state { ++ u8 consec_cong_rounds:5, /* consecutive congested rounds */ ++ enabled:1, /* Check if PLB is enabled */ ++ unused:2; ++ u32 pause_until; /* jiffies32 when PLB can resume repathing */ ++}; ++ ++void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, ++ const int cong_ratio); ++void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); ++void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); ++ + /* At how many usecs into the future should the RTO fire? */ + static inline s64 tcp_rto_delta_us(const struct sock *sk) + { +diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h +index 50655de04c9b..0e24f11627d5 100644 +--- a/include/uapi/linux/inet_diag.h ++++ b/include/uapi/linux/inet_diag.h +@@ -231,9 +231,42 @@ struct tcp_bbr_info { + __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + }; + ++/* Phase as reported in netlink/ss stats. */ ++enum tcp_bbr2_phase { ++ BBR2_PHASE_INVALID = 0, ++ BBR2_PHASE_STARTUP = 1, ++ BBR2_PHASE_DRAIN = 2, ++ BBR2_PHASE_PROBE_RTT = 3, ++ BBR2_PHASE_PROBE_BW_UP = 4, ++ BBR2_PHASE_PROBE_BW_DOWN = 5, ++ BBR2_PHASE_PROBE_BW_CRUISE = 6, ++ BBR2_PHASE_PROBE_BW_REFILL = 7 ++}; ++ ++struct tcp_bbr2_info { ++ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */ ++ __u32 bbr_bw_lsb; /* lower 32 bits of bw */ ++ __u32 bbr_bw_msb; /* upper 32 bits of bw */ ++ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ ++ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ ++ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ ++ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ ++ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ ++ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ ++ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ ++ __u8 bbr_mode; /* current bbr_mode in state machine */ ++ __u8 bbr_phase; /* current state machine phase */ ++ __u8 unused1; /* alignment padding; not used yet */ ++ __u8 bbr_version; /* MUST be at this offset in struct */ ++ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */ ++ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */ ++ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ ++}; ++ + union tcp_cc_info { + struct tcpvegas_info vegas; + struct tcp_dctcp_info dctcp; + struct tcp_bbr_info bbr; ++ struct tcp_bbr2_info bbr2; + }; + #endif /* _UAPI_INET_DIAG_H_ */ +diff --git a/include/uapi/linux/snmp.h b/include/uapi/linux/snmp.h +index 4d7470036a8b..8ce035f1c874 100644 +--- a/include/uapi/linux/snmp.h ++++ b/include/uapi/linux/snmp.h +@@ -292,6 +292,7 @@ enum + LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */ + LINUX_MIB_TCPMIGRATEREQSUCCESS, /* TCPMigrateReqSuccess */ + LINUX_MIB_TCPMIGRATEREQFAILURE, /* TCPMigrateReqFailure */ ++ LINUX_MIB_TCPECNREHASH, /* TCPECNRehash */ + __LINUX_MIB_MAX + }; + +diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig +index e983bb0c5012..d172ffbdf27f 100644 +--- a/net/ipv4/Kconfig ++++ b/net/ipv4/Kconfig +@@ -668,6 +668,24 @@ config TCP_CONG_BBR + AQM schemes that do not provide a delay signal. It requires the fq + ("Fair Queue") pacing packet scheduler. + ++config TCP_CONG_BBR2 ++ tristate "BBR2 TCP" ++ default n ++ help ++ ++ BBR2 TCP congestion control is a model-based congestion control ++ algorithm that aims to maximize network utilization, keep queues and ++ retransmit rates low, and to be able to coexist with Reno/CUBIC in ++ common scenarios. It builds an explicit model of the network path. It ++ tolerates a targeted degree of random packet loss and delay that are ++ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi, ++ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can ++ coexist with flows that use loss-based congestion control, and can ++ operate with shallow buffers, deep buffers, bufferbloat, policers, or ++ AQM schemes that do not provide a delay signal. It requires pacing, ++ using either TCP internal pacing or the fq ("Fair Queue") pacing packet ++ scheduler. ++ + choice + prompt "Default TCP congestion control" + default DEFAULT_CUBIC +@@ -705,6 +723,9 @@ choice + config DEFAULT_BBR + bool "BBR" if TCP_CONG_BBR=y + ++ config DEFAULT_BBR2 ++ bool "BBR2" if TCP_CONG_BBR2=y ++ + config DEFAULT_RENO + bool "Reno" + endchoice +@@ -729,6 +750,7 @@ config DEFAULT_TCP_CONG + default "dctcp" if DEFAULT_DCTCP + default "cdg" if DEFAULT_CDG + default "bbr" if DEFAULT_BBR ++ default "bbr2" if DEFAULT_BBR2 + default "cubic" + + config TCP_MD5SIG +diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile +index bbdd9c44f14e..e7a86a50838a 100644 +--- a/net/ipv4/Makefile ++++ b/net/ipv4/Makefile +@@ -10,7 +10,7 @@ obj-y := route.o inetpeer.o protocol.o \ + tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \ + tcp_minisocks.o tcp_cong.o tcp_metrics.o tcp_fastopen.o \ + tcp_rate.o tcp_recovery.o tcp_ulp.o \ +- tcp_offload.o datagram.o raw.o udp.o udplite.o \ ++ tcp_offload.o tcp_plb.o datagram.o raw.o udp.o udplite.o \ + udp_offload.o arp.o icmp.o devinet.o af_inet.o igmp.o \ + fib_frontend.o fib_semantics.o fib_trie.o fib_notifier.o \ + inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \ +@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o + obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o + obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o + obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o ++obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o + obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o + obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o + obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o +diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c +index 85a9e500c42d..24fcbac984fe 100644 +--- a/net/ipv4/bpf_tcp_ca.c ++++ b/net/ipv4/bpf_tcp_ca.c +@@ -14,6 +14,18 @@ + /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ + extern struct bpf_struct_ops bpf_tcp_congestion_ops; + ++static u32 optional_ops[] = { ++ offsetof(struct tcp_congestion_ops, init), ++ offsetof(struct tcp_congestion_ops, release), ++ offsetof(struct tcp_congestion_ops, set_state), ++ offsetof(struct tcp_congestion_ops, cwnd_event), ++ offsetof(struct tcp_congestion_ops, in_ack_event), ++ offsetof(struct tcp_congestion_ops, pkts_acked), ++ offsetof(struct tcp_congestion_ops, tso_segs), ++ offsetof(struct tcp_congestion_ops, sndbuf_expand), ++ offsetof(struct tcp_congestion_ops, cong_control), ++}; ++ + static u32 unsupported_ops[] = { + offsetof(struct tcp_congestion_ops, get_info), + }; +diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c +index 0088a4c64d77..e0a664b467e0 100644 +--- a/net/ipv4/proc.c ++++ b/net/ipv4/proc.c +@@ -297,6 +297,7 @@ static const struct snmp_mib snmp4_net_list[] = { + SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS), + SNMP_MIB_ITEM("TCPMigrateReqSuccess", LINUX_MIB_TCPMIGRATEREQSUCCESS), + SNMP_MIB_ITEM("TCPMigrateReqFailure", LINUX_MIB_TCPMIGRATEREQFAILURE), ++ SNMP_MIB_ITEM("TCPECNRehash", LINUX_MIB_TCPECNREHASH), + SNMP_MIB_SENTINEL + }; + +diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c +index 5490c285668b..ed35a9485e71 100644 +--- a/net/ipv4/sysctl_net_ipv4.c ++++ b/net/ipv4/sysctl_net_ipv4.c +@@ -39,6 +39,8 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ; + static int one_day_secs = 24 * 3600; + static u32 fib_multipath_hash_fields_all_mask __maybe_unused = + FIB_MULTIPATH_HASH_FIELD_ALL_MASK; ++static int tcp_plb_max_rounds = 31; ++static int tcp_plb_max_cong_thresh = 256; + + /* obsolete */ + static int sysctl_tcp_low_latency __read_mostly; +@@ -1346,6 +1348,47 @@ static struct ctl_table ipv4_net_table[] = { + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, ++ { ++ .procname = "tcp_plb_enabled", ++ .data = &init_net.ipv4.sysctl_tcp_plb_enabled, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "tcp_plb_cong_thresh", ++ .data = &init_net.ipv4.sysctl_tcp_plb_cong_thresh, ++ .maxlen = sizeof(int), ++ .mode = 0644, ++ .proc_handler = proc_dointvec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &tcp_plb_max_cong_thresh, ++ }, ++ { ++ .procname = "tcp_plb_idle_rehash_rounds", ++ .data = &init_net.ipv4.sysctl_tcp_plb_idle_rehash_rounds, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra2 = &tcp_plb_max_rounds, ++ }, ++ { ++ .procname = "tcp_plb_rehash_rounds", ++ .data = &init_net.ipv4.sysctl_tcp_plb_rehash_rounds, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra2 = &tcp_plb_max_rounds, ++ }, ++ { ++ .procname = "tcp_plb_suspend_rto_sec", ++ .data = &init_net.ipv4.sysctl_tcp_plb_suspend_rto_sec, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ }, + { } + }; + +diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c +index 3fc8ca6a264e..2fcd440df1f6 100644 +--- a/net/ipv4/tcp.c ++++ b/net/ipv4/tcp.c +@@ -3187,6 +3187,7 @@ int tcp_disconnect(struct sock *sk, int flags) + tp->rx_opt.dsack = 0; + tp->rx_opt.num_sacks = 0; + tp->rcv_ooopack = 0; ++ tp->fast_ack_mode = 0; + + + /* Clean up fastopen related fields */ +diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c +index 54eec33c6e1c..bfbf158c71f4 100644 +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) + sk->sk_pacing_rate = rate; + } + +-/* override sysctl_tcp_min_tso_segs */ + static u32 bbr_min_tso_segs(struct sock *sk) + { + return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; + } + ++/* Return the number of segments BBR would like in a TSO/GSO skb, given ++ * a particular max gso size as a constraint. ++ */ ++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, ++ u32 gso_max_size) ++{ ++ u32 segs; ++ u64 bytes; ++ ++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ ++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; ++ ++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); ++ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); ++ return segs; ++} ++ ++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ ++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) ++{ ++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); ++} ++ ++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ + static u32 bbr_tso_segs_goal(struct sock *sk) + { + struct tcp_sock *tp = tcp_sk(sk); +- u32 segs, bytes; +- +- /* Sort of tcp_tso_autosize() but ignoring +- * driver provided sk_gso_max_size. +- */ +- bytes = min_t(unsigned long, +- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift), +- GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); +- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); + +- return min(segs, 0x7FU); ++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); + } + + /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { + .undo_cwnd = bbr_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .ssthresh = bbr_ssthresh, +- .min_tso_segs = bbr_min_tso_segs, ++ .tso_segs = bbr_tso_segs, + .get_info = bbr_get_info, + .set_state = bbr_set_state, + }; +diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c +new file mode 100644 +index 000000000000..2e39f7a353be +--- /dev/null ++++ b/net/ipv4/tcp_bbr2.c +@@ -0,0 +1,2692 @@ ++/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2 ++ * ++ * BBRv2 is a model-based congestion control algorithm that aims for low ++ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model ++ * of the network path, it uses measurements of bandwidth and RTT, as well as ++ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that ++ * although it can use ECN or loss signals explicitly, it does not require ++ * either; it can bound its in-flight data based on its estimate of the BDP. ++ * ++ * The model has both higher and lower bounds for the operating range: ++ * lo: bw_lo, inflight_lo: conservative short-term lower bound ++ * hi: bw_hi, inflight_hi: robust long-term upper bound ++ * The bandwidth-probing time scale is (a) extended dynamically based on ++ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by ++ * an interactive wall-clock time-scale to be more scalable and responsive ++ * than Reno and CUBIC. ++ * ++ * Here is a state transition diagram for BBR: ++ * ++ * | ++ * V ++ * +---> STARTUP ----+ ++ * | | | ++ * | V | ++ * | DRAIN ----+ ++ * | | | ++ * | V | ++ * +---> PROBE_BW ----+ ++ * | ^ | | ++ * | | | | ++ * | +----+ | ++ * | | ++ * +---- PROBE_RTT <--+ ++ * ++ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly. ++ * When it estimates the pipe is full, it enters DRAIN to drain the queue. ++ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT. ++ * A long-lived BBR flow spends the vast majority of its time remaining ++ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth ++ * in a fair manner, with a small, bounded queue. *If* a flow has been ++ * continuously sending for the entire min_rtt window, and hasn't seen an RTT ++ * sample that matches or decreases its min_rtt estimate for 10 seconds, then ++ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe ++ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if ++ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW; ++ * otherwise we enter STARTUP to try to fill the pipe. ++ * ++ * BBR is described in detail in: ++ * "BBR: Congestion-Based Congestion Control", ++ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh, ++ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016. ++ * ++ * There is a public e-mail list for discussing BBR development and testing: ++ * https://groups.google.com/forum/#!forum/bbr-dev ++ * ++ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, ++ * otherwise TCP stack falls back to an internal pacing using one high ++ * resolution timer per TCP socket and may use more resources. ++ */ ++#include ++#include ++#include ++#include ++#include ++ ++#include "tcp_dctcp.h" ++ ++/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth ++ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. ++ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. ++ * Since the minimum window is >=4 packets, the lower bound isn't ++ * an issue. The upper bound isn't an issue with existing technologies. ++ */ ++#define BW_SCALE 24 ++#define BW_UNIT (1 << BW_SCALE) ++ ++#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */ ++#define BBR_UNIT (1 << BBR_SCALE) ++ ++#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */ ++#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */ ++ ++#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ ++ ++/* BBR has the following modes for deciding how fast to send: */ ++enum bbr_mode { ++ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */ ++ BBR_DRAIN, /* drain any queue created during startup */ ++ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */ ++ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ ++}; ++ ++/* How does the incoming ACK stream relate to our bandwidth probing? */ ++enum bbr_ack_phase { ++ BBR_ACKS_INIT, /* not probing; not getting probe feedback */ ++ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ ++ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ ++ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ ++ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ ++}; ++ ++/* BBR congestion control block */ ++struct bbr { ++ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ ++ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ ++ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ ++ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */ ++ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ ++ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ ++ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ ++ u64 cycle_mstamp; /* time of this cycle phase start */ ++ u32 mode:3, /* current bbr_mode in state machine */ ++ prev_ca_state:3, /* CA state on previous ACK */ ++ packet_conservation:1, /* use packet conservation? */ ++ round_start:1, /* start of packet-timed tx->ack round? */ ++ ce_state:1, /* If most recent data has CE bit set */ ++ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ ++ try_fast_path:1, /* can we take fast path? */ ++ unused2:11, ++ idle_restart:1, /* restarting after idle? */ ++ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ ++ cycle_idx:3, /* current index in pacing_gain cycle array */ ++ has_seen_rtt:1; /* have we seen an RTT sample yet? */ ++ u32 pacing_gain:11, /* current gain for setting pacing rate */ ++ cwnd_gain:11, /* current gain for setting cwnd */ ++ full_bw_reached:1, /* reached full bw in Startup? */ ++ full_bw_cnt:2, /* number of rounds without large bw gains */ ++ init_cwnd:7; /* initial cwnd */ ++ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ ++ u32 full_bw; /* recent bw, to estimate if pipe is full */ ++ ++ /* For tracking ACK aggregation: */ ++ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */ ++ u16 extra_acked[2]; /* max excess data ACKed in epoch */ ++ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ ++ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ ++ extra_acked_win_idx:1, /* current index in extra_acked array */ ++ /* BBR v2 state: */ ++ unused1:2, ++ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ ++ loss_in_cycle:1, /* packet loss in this cycle? */ ++ ecn_in_cycle:1; /* ECN in this cycle? */ ++ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ ++ u32 undo_bw_lo; /* bw_lo before latest losses */ ++ u32 undo_inflight_lo; /* inflight_lo before latest losses */ ++ u32 undo_inflight_hi; /* inflight_hi before latest losses */ ++ u32 bw_latest; /* max delivered bw in last round trip */ ++ u32 bw_lo; /* lower bound on sending bandwidth */ ++ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/ ++ u32 inflight_latest; /* max delivered data in last round trip */ ++ u32 inflight_lo; /* lower bound of inflight data range */ ++ u32 inflight_hi; /* upper bound of inflight data range */ ++ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ ++ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ ++ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ ++ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ ++ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ ++ bw_probe_samples:1, /* rate samples reflect bw probing? */ ++ prev_probe_too_high:1, /* did last PROBE_UP go too high? */ ++ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ ++ rounds_since_probe:8, /* packet-timed rounds since probed bw */ ++ loss_round_start:1, /* loss_round_delivered round trip? */ ++ loss_in_round:1, /* loss marked in this round trip? */ ++ ecn_in_round:1, /* ECN marked in this round trip? */ ++ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ ++ loss_events_in_round:4,/* losses in STARTUP round */ ++ initialized:1; /* has bbr_init() been called? */ ++ u32 alpha_last_delivered; /* tp->delivered at alpha update */ ++ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ ++ struct tcp_plb_state plb; ++ ++ /* Params configurable using setsockopt. Refer to correspoding ++ * module param for detailed description of params. ++ */ ++ struct bbr_params { ++ u32 high_gain:11, /* max allowed value: 2047 */ ++ drain_gain:10, /* max allowed value: 1023 */ ++ cwnd_gain:11; /* max allowed value: 2047 */ ++ u32 cwnd_min_target:4, /* max allowed value: 15 */ ++ min_rtt_win_sec:5, /* max allowed value: 31 */ ++ probe_rtt_mode_ms:9, /* max allowed value: 511 */ ++ full_bw_cnt:3, /* max allowed value: 7 */ ++ cwnd_tso_budget:1, /* allowed values: {0, 1} */ ++ unused3:6, ++ drain_to_target:1, /* boolean */ ++ precise_ece_ack:1, /* boolean */ ++ extra_acked_in_startup:1, /* allowed values: {0, 1} */ ++ fast_path:1; /* boolean */ ++ u32 full_bw_thresh:10, /* max allowed value: 1023 */ ++ startup_cwnd_gain:11, /* max allowed value: 2047 */ ++ bw_probe_pif_gain:9, /* max allowed value: 511 */ ++ usage_based_cwnd:1, /* boolean */ ++ unused2:1; ++ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */ ++ refill_add_inc:2; /* max allowed value: 3 */ ++ u16 extra_acked_gain:11, /* max allowed value: 2047 */ ++ extra_acked_win_rtts:5; /* max allowed value: 31*/ ++ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */ ++ /* Mostly BBR v2 parameters below here: */ ++ u32 ecn_alpha_gain:8, /* max allowed value: 255 */ ++ ecn_factor:8, /* max allowed value: 255 */ ++ ecn_thresh:8, /* max allowed value: 255 */ ++ beta:8; /* max allowed value: 255 */ ++ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */ ++ bw_probe_reno_gain:9, /* max allowed value: 511 */ ++ full_loss_cnt:4; /* max allowed value: 15 */ ++ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */ ++ inflight_headroom:8, /* max allowed value: 255 */ ++ loss_thresh:8, /* max allowed value: 255 */ ++ bw_probe_max_rounds:8; /* max allowed value: 255 */ ++ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */ ++ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */ ++ full_ecn_cnt:2; /* max allowed value: 3 */ ++ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */ ++ undo:1, /* boolean */ ++ tso_rtt_shift:4, /* max allowed value: 15 */ ++ unused5:1; ++ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */ ++ unused1:14, ++ ecn_alpha_init:9; /* max allowed value: 256 */ ++ } params; ++ ++ struct { ++ u32 snd_isn; /* Initial sequence number */ ++ u32 rs_bw; /* last valid rate sample bw */ ++ u32 target_cwnd; /* target cwnd, based on BDP */ ++ u8 undo:1, /* Undo even happened but not yet logged */ ++ unused:7; ++ char event; /* single-letter event debug codes */ ++ u16 unused2; ++ } debug; ++}; ++ ++struct bbr_context { ++ u32 sample_bw; ++ u32 target_cwnd; ++ u32 log:1; ++}; ++ ++/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */ ++static u32 bbr_min_rtt_win_sec = 10; ++/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode. ++ * Max allowed value is 511 (0x1FF). ++ */ ++static u32 bbr_probe_rtt_mode_ms = 200; ++/* Window length of probe_rtt_min_us filter (in ms), and consequently the ++ * typical interval between PROBE_RTT mode entries. ++ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC ++ */ ++static u32 bbr_probe_rtt_win_ms = 5000; ++/* Skip TSO below the following bandwidth (bits/sec): */ ++static int bbr_min_tso_rate = 1200000; ++ ++/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting ++ * in bigger TSO bursts. By default we cut the RTT-based allowance in half ++ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance ++ * is below 1500 bytes after 6 * ~500 usec = 3ms. ++ */ ++static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */ ++ ++/* Select cwnd TSO budget approach: ++ * 0: padding ++ * 1: flooring ++ */ ++static uint bbr_cwnd_tso_budget = 1; ++ ++/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. ++ * In order to help drive the network toward lower queues and low latency while ++ * maintaining high utilization, the average pacing rate aims to be slightly ++ * lower than the estimated bandwidth. This is an important aspect of the ++ * design. ++ */ ++static const int bbr_pacing_margin_percent = 1; ++ ++/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain ++ * that will allow a smoothly increasing pacing rate that will double each RTT ++ * and send the same number of packets per RTT that an un-paced, slow-starting ++ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF). ++ */ ++static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; ++/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */ ++static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1; ++/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain ++ * the queue created in BBR_STARTUP in a single round. Max allowed value ++ * is 1023 (0x3FF). ++ */ ++static int bbr_drain_gain = BBR_UNIT * 1000 / 2885; ++/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs. ++ * Max allowed value is 2047 (0x7FF). ++ */ ++static int bbr_cwnd_gain = BBR_UNIT * 2; ++/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw. ++ * Max allowed value for each element is 1023 (0x3FF). ++ */ ++enum bbr_pacing_gain_phase { ++ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ ++ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ ++ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ ++ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */ ++}; ++static int bbr_pacing_gain[] = { ++ BBR_UNIT * 5 / 4, /* probe for more available bw */ ++ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ ++ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ ++ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ ++}; ++ ++/* Try to keep at least this many packets in flight, if things go smoothly. For ++ * smooth functioning, a sliding window protocol ACKing every other packet ++ * needs at least 4 packets in flight. Max allowed value is 15 (0xF). ++ */ ++static u32 bbr_cwnd_min_target = 4; ++ ++/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%. ++ * Use 0 to disable. Max allowed value is 255. ++ */ ++static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; ++ ++/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ ++/* If bw has increased significantly (1.25x), there may be more bw available. ++ * Max allowed value is 1023 (0x3FF). ++ */ ++static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; ++/* But after 3 rounds w/o significant bw growth, estimate pipe is full. ++ * Max allowed value is 7 (0x7). ++ */ ++static u32 bbr_full_bw_cnt = 3; ++ ++static u32 bbr_flags; /* Debugging related stuff */ ++ ++/* Whether to debug using printk. ++ */ ++static bool bbr_debug_with_printk; ++ ++/* Whether to debug using ftrace event tcp:tcp_bbr_event. ++ * Ignored when bbr_debug_with_printk is set. ++ */ ++static bool bbr_debug_ftrace; ++ ++/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */ ++static bool bbr_drain_to_target = true; /* default: enabled */ ++ ++/* Experiment: Flags to control BBR with ECN behavior. ++ */ ++static bool bbr_precise_ece_ack = true; /* default: enabled */ ++ ++/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is ++ * (2^(16+14) B)/(1024 B/packet) = 1M packets. ++ */ ++static u32 bbr_cwnd_warn_val = 1U << 20; ++ ++static u16 bbr_debug_port_mask; ++ ++/* BBR module parameters. These are module parameters only in Google prod. ++ * Upstream these are intentionally not module parameters. ++ */ ++static int bbr_pacing_gain_size = CYCLE_LEN; ++ ++/* Gain factor for adding extra_acked to target cwnd: */ ++static int bbr_extra_acked_gain = 256; ++ ++/* Window length of extra_acked window. Max allowed val is 31. */ ++static u32 bbr_extra_acked_win_rtts = 5; ++ ++/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ ++static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; ++ ++/* Time period for clamping cwnd increment due to ack aggregation */ ++static u32 bbr_extra_acked_max_us = 100 * 1000; ++ ++/* Use extra acked in startup ? ++ * 0: disabled ++ * 1: use latest extra_acked value from 1-2 rtt in startup ++ */ ++static int bbr_extra_acked_in_startup = 1; /* default: enabled */ ++ ++/* Experiment: don't grow cwnd beyond twice of what we just probed. */ ++static bool bbr_usage_based_cwnd; /* default: disabled */ ++ ++/* For lab testing, researchers can enable BBRv2 ECN support with this flag, ++ * when they know that any ECN marks that the connections experience will be ++ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks. ++ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on ++ * negotiation or configuration that is outside the scope of the BBRv2 ++ * alpha release. ++ */ ++static bool bbr_ecn_enable = false; ++ ++module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644); ++module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644); ++module_param_named(high_gain, bbr_high_gain, int, 0644); ++module_param_named(drain_gain, bbr_drain_gain, int, 0644); ++module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644); ++module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644); ++module_param_array_named(pacing_gain, bbr_pacing_gain, int, ++ &bbr_pacing_gain_size, 0644); ++module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644); ++module_param_named(probe_rtt_cwnd_gain, ++ bbr_probe_rtt_cwnd_gain, uint, 0664); ++module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664); ++module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644); ++module_param_named(flags, bbr_flags, uint, 0644); ++module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644); ++module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644); ++module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644); ++module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644); ++module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644); ++module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644); ++module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644); ++module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664); ++module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); ++module_param_named(extra_acked_win_rtts, ++ bbr_extra_acked_win_rtts, uint, 0664); ++module_param_named(extra_acked_max_us, ++ bbr_extra_acked_max_us, uint, 0664); ++module_param_named(ack_epoch_acked_reset_thresh, ++ bbr_ack_epoch_acked_reset_thresh, uint, 0664); ++module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); ++module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664); ++module_param_named(extra_acked_in_startup, ++ bbr_extra_acked_in_startup, int, 0664); ++module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664); ++module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664); ++ ++static void bbr2_exit_probe_rtt(struct sock *sk); ++static void bbr2_reset_congestion_signals(struct sock *sk); ++ ++static void bbr_check_probe_rtt_done(struct sock *sk); ++ ++/* Do we estimate that STARTUP filled the pipe? */ ++static bool bbr_full_bw_reached(const struct sock *sk) ++{ ++ const struct bbr *bbr = inet_csk_ca(sk); ++ ++ return bbr->full_bw_reached; ++} ++ ++/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ ++static u32 bbr_max_bw(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return max(bbr->bw_hi[0], bbr->bw_hi[1]); ++} ++ ++/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ ++static u32 bbr_bw(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return min(bbr_max_bw(sk), bbr->bw_lo); ++} ++ ++/* Return maximum extra acked in past k-2k round trips, ++ * where k = bbr_extra_acked_win_rtts. ++ */ ++static u16 bbr_extra_acked(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return max(bbr->extra_acked[0], bbr->extra_acked[1]); ++} ++ ++/* Return rate in bytes per second, optionally with a gain. ++ * The order here is chosen carefully to avoid overflow of u64. This should ++ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. ++ */ ++static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, ++ int margin) ++{ ++ unsigned int mss = tcp_sk(sk)->mss_cache; ++ ++ rate *= mss; ++ rate *= gain; ++ rate >>= BBR_SCALE; ++ rate *= USEC_PER_SEC / 100 * (100 - margin); ++ rate >>= BW_SCALE; ++ rate = max(rate, 1ULL); ++ return rate; ++} ++ ++static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) ++{ ++ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); ++} ++ ++static u64 bbr_rate_kbps(struct sock *sk, u64 rate) ++{ ++ rate = bbr_bw_bytes_per_sec(sk, rate); ++ rate *= 8; ++ do_div(rate, 1000); ++ return rate; ++} ++ ++static u32 bbr_tso_segs_goal(struct sock *sk); ++static void bbr_debug(struct sock *sk, u32 acked, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ static const char ca_states[] = { ++ [TCP_CA_Open] = 'O', ++ [TCP_CA_Disorder] = 'D', ++ [TCP_CA_CWR] = 'C', ++ [TCP_CA_Recovery] = 'R', ++ [TCP_CA_Loss] = 'L', ++ }; ++ static const char mode[] = { ++ 'G', /* Growing - BBR_STARTUP */ ++ 'D', /* Drain - BBR_DRAIN */ ++ 'W', /* Window - BBR_PROBE_BW */ ++ 'M', /* Min RTT - BBR_PROBE_RTT */ ++ }; ++ static const char ack_phase[] = { /* bbr_ack_phase strings */ ++ 'I', /* BBR_ACKS_INIT - 'Init' */ ++ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */ ++ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */ ++ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */ ++ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */ ++ }; ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ const u32 una = tp->snd_una - bbr->debug.snd_isn; ++ const u32 fack = tcp_highest_sack_seq(tp); ++ const u16 dport = ntohs(inet_sk(sk)->inet_dport); ++ bool is_port_match = (bbr_debug_port_mask && ++ ((dport & bbr_debug_port_mask) == 0)); ++ char debugmsg[320]; ++ ++ if (sk->sk_state == TCP_SYN_SENT) ++ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */ ++ ++ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) { ++ char addr[INET6_ADDRSTRLEN + 10] = { 0 }; ++ ++ if (sk->sk_family == AF_INET) ++ snprintf(addr, sizeof(addr), "%pI4:%u", ++ &inet_sk(sk)->inet_daddr, dport); ++ else if (sk->sk_family == AF_INET6) ++ snprintf(addr, sizeof(addr), "%pI6:%u", ++ &sk->sk_v6_daddr, dport); ++ ++ WARN_ONCE(1, ++ "BBR %s cwnd alert: %u " ++ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u " ++ "bw: %u rtt: %u min_rtt: %u " ++ "acked: %u tso_segs: %u " ++ "bw: %d %ld %d pif: %u\n", ++ addr, tp->snd_cwnd, ++ una, inet_csk(sk)->icsk_ca_state, ++ bbr->pacing_gain, bbr->cwnd_gain, ++ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us, ++ acked, bbr_tso_segs_goal(sk), ++ rs->delivered, rs->interval_us, rs->is_retrans, ++ tcp_packets_in_flight(tp)); ++ } ++ ++ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace)) ++ return; ++ ++ if (!sock_flag(sk, SOCK_DBG) && !is_port_match) ++ return; ++ ++ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE)) ++ return; ++ ++ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) && ++ !(bbr_flags & FLAG_DEBUG_LOOPBACK)) ++ return; ++ ++ snprintf(debugmsg, sizeof(debugmsg) - 1, ++ "BBR %pI4:%-5u %5u,%03u:%-7u %c " ++ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu " ++ "bw %llu lb %llu ib %llu qb %llu " ++ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c " ++ "lr %d er %d ea %d bwl %lld il %d ih %d c %d " ++ "v %d %c %u %c %s\n", ++ &inet_sk(sk)->inet_daddr, dport, ++ una / 1000, una % 1000, fack - tp->snd_una, ++ ca_states[inet_csk(sk)->icsk_ca_state], ++ bbr->debug.undo ? '@' : mode[bbr->mode], ++ tp->snd_cwnd, ++ bbr_extra_acked(sk), /* br (legacy): extra_acked */ ++ rs->tx_in_flight, /* cr (legacy): tx_inflight */ ++ rs->rtt_us, ++ rs->delivered, ++ rs->interval_us, ++ bbr->min_rtt_us, ++ rs->is_app_limited ? '_' : 'l', ++ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */ ++ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */ ++ 0ULL, /* lb: [obsolete] */ ++ 0ULL, /* ib: [obsolete] */ ++ div_u64((u64)sk->sk_pacing_rate * 8, 1000), ++ acked, ++ tcp_packets_in_flight(tp), ++ rs->is_ack_delayed ? 'd' : '.', ++ bbr->round_start ? '*' : '.', ++ tp->delivered, tp->lost, ++ tp->app_limited, ++ 0, /* #: [obsolete] */ ++ ctx->target_cwnd, ++ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */ ++ ca_states[bbr->prev_ca_state], ++ (rs->lost + rs->delivered) > 0 ? ++ (1000 * rs->lost / ++ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */ ++ (rs->delivered) > 0 ? ++ (1000 * rs->delivered_ce / ++ (rs->delivered)) : 0, /* er: ECN rate x1000 */ ++ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */ ++ bbr->bw_lo == ~0U ? ++ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */ ++ bbr->inflight_lo, /* il */ ++ bbr->inflight_hi, /* ih */ ++ bbr->bw_probe_up_cnt, /* c */ ++ 2, /* v: version */ ++ bbr->debug.event, ++ bbr->cycle_idx, ++ ack_phase[bbr->ack_phase], ++ bbr->bw_probe_samples ? "Y" : "N"); ++ debugmsg[sizeof(debugmsg) - 1] = 0; ++ ++ /* printk takes a higher precedence. */ ++ if (bbr_debug_with_printk) ++ printk(KERN_DEBUG "%s", debugmsg); ++ ++ if (unlikely(bbr->debug.undo)) ++ bbr->debug.undo = 0; ++} ++ ++/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ ++static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) ++{ ++ u64 rate = bw; ++ ++ rate = bbr_rate_bytes_per_sec(sk, rate, gain, ++ bbr_pacing_margin_percent); ++ rate = min_t(u64, rate, sk->sk_max_pacing_rate); ++ return rate; ++} ++ ++/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ ++static void bbr_init_pacing_rate_from_rtt(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw; ++ u32 rtt_us; ++ ++ if (tp->srtt_us) { /* any RTT sample yet? */ ++ rtt_us = max(tp->srtt_us >> 3, 1U); ++ bbr->has_seen_rtt = 1; ++ } else { /* no RTT sample yet */ ++ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */ ++ } ++ bw = (u64)tp->snd_cwnd * BW_UNIT; ++ do_div(bw, rtt_us); ++ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain); ++} ++ ++/* Pace using current bw estimate and a gain factor. */ ++static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain); ++ ++ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) ++ bbr_init_pacing_rate_from_rtt(sk); ++ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate) ++ sk->sk_pacing_rate = rate; ++} ++ ++static u32 bbr_min_tso_segs(struct sock *sk) ++{ ++ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; ++} ++ ++/* Return the number of segments BBR would like in a TSO/GSO skb, given ++ * a particular max gso size as a constraint. ++ */ ++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, ++ u32 gso_max_size) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 segs, r; ++ u64 bytes; ++ ++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ ++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift; ++ ++ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every ++ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. ++ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) ++ */ ++ if (bbr->params.tso_rtt_shift) { ++ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift; ++ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ ++ bytes += GSO_MAX_SIZE >> r; ++ } ++ ++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); ++ segs = max_t(u32, div_u64(bytes, mss_now), bbr_min_tso_segs(sk)); ++ return segs; ++} ++ ++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ ++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) ++{ ++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); ++} ++ ++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ ++static u32 bbr_tso_segs_goal(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ ++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE); ++} ++ ++/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ ++static void bbr_save_cwnd(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) ++ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ ++ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */ ++ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); ++} ++ ++static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (event == CA_EVENT_TX_START) { ++ tcp_plb_check_rehash(sk, &bbr->plb); ++ ++ if (!tp->app_limited) ++ return; ++ bbr->idle_restart = 1; ++ bbr->ack_epoch_mstamp = tp->tcp_mstamp; ++ bbr->ack_epoch_acked = 0; ++ /* Avoid pointless buffer overflows: pace at est. bw if we don't ++ * need more speed (we're restarting from idle and app-limited). ++ */ ++ if (bbr->mode == BBR_PROBE_BW) ++ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); ++ else if (bbr->mode == BBR_PROBE_RTT) ++ bbr_check_probe_rtt_done(sk); ++ } else if ((event == CA_EVENT_ECN_IS_CE || ++ event == CA_EVENT_ECN_NO_CE) && ++ bbr_ecn_enable && ++ bbr->params.precise_ece_ack) { ++ u32 state = bbr->ce_state; ++ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); ++ bbr->ce_state = state; ++ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE) ++ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS); ++ } ++} ++ ++/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: ++ * ++ * bdp = ceil(bw * min_rtt * gain) ++ * ++ * The key factor, gain, controls the amount of queue. While a small gain ++ * builds a smaller queue, it becomes more vulnerable to noise in RTT ++ * measurements (e.g., delayed ACKs or other ACK compression effects). This ++ * noise may cause BBR to under-estimate the rate. ++ */ ++static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 bdp; ++ u64 w; ++ ++ /* If we've never had a valid RTT sample, cap cwnd at the initial ++ * default. This should only happen when the connection is not using TCP ++ * timestamps and has retransmitted all of the SYN/SYNACK/data packets ++ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which ++ * case we need to slow-start up toward something safe: initial cwnd. ++ */ ++ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ ++ return bbr->init_cwnd; /* be safe: cap at initial cwnd */ ++ ++ w = (u64)bw * bbr->min_rtt_us; ++ ++ /* Apply a gain to the given value, remove the BW_SCALE shift, and ++ * round the value up to avoid a negative feedback loop. ++ */ ++ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ ++ return bdp; ++} ++ ++/* To achieve full performance in high-speed paths, we budget enough cwnd to ++ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: ++ * - one skb in sending host Qdisc, ++ * - one skb in sending host TSO/GSO engine ++ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine ++ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because ++ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, ++ * which allows 2 outstanding 2-packet sequences, to try to keep pipe ++ * full even with ACK-every-other-packet delayed ACKs. ++ */ ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 tso_segs_goal; ++ ++ tso_segs_goal = 3 * bbr_tso_segs_goal(sk); ++ ++ /* Allow enough full-sized skbs in flight to utilize end systems. */ ++ if (bbr->params.cwnd_tso_budget == 1) { ++ cwnd = max_t(u32, cwnd, tso_segs_goal); ++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); ++ } else { ++ cwnd += tso_segs_goal; ++ cwnd = (cwnd + 1) & ~1U; ++ } ++ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ ++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) ++ cwnd += 2; ++ ++ return cwnd; ++} ++ ++/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ ++static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) ++{ ++ u32 inflight; ++ ++ inflight = bbr_bdp(sk, bw, gain); ++ inflight = bbr_quantization_budget(sk, inflight); ++ ++ return inflight; ++} ++ ++/* With pacing at lower layers, there's often less data "in the network" than ++ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq), ++ * we often have several skbs queued in the pacing layer with a pre-scheduled ++ * earliest departure time (EDT). BBR adapts its pacing rate based on the ++ * inflight level that it estimates has already been "baked in" by previous ++ * departure time decisions. We calculate a rough estimate of the number of our ++ * packets that might be in the network at the earliest departure time for the ++ * next skb scheduled: ++ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw ++ * If we're increasing inflight, then we want to know if the transmit of the ++ * EDT skb will push inflight above the target, so inflight_at_edt includes ++ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight, ++ * then estimate if inflight will sink too low just before the EDT transmit. ++ */ ++static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 now_ns, edt_ns, interval_us; ++ u32 interval_delivered, inflight_at_edt; ++ ++ now_ns = tp->tcp_clock_cache; ++ edt_ns = max(tp->tcp_wstamp_ns, now_ns); ++ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC); ++ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE; ++ inflight_at_edt = inflight_now; ++ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ ++ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */ ++ if (interval_delivered >= inflight_at_edt) ++ return 0; ++ return inflight_at_edt - interval_delivered; ++} ++ ++/* Find the cwnd increment based on estimate of ack aggregation */ ++static u32 bbr_ack_aggregation_cwnd(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 max_aggr_cwnd, aggr_cwnd = 0; ++ ++ if (bbr->params.extra_acked_gain && ++ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) { ++ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) ++ / BW_UNIT; ++ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk)) ++ >> BBR_SCALE; ++ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); ++ } ++ ++ return aggr_cwnd; ++} ++ ++/* Returns the cwnd for PROBE_RTT mode. */ ++static u32 bbr_probe_rtt_cwnd(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->params.probe_rtt_cwnd_gain == 0) ++ return bbr->params.cwnd_min_target; ++ return max_t(u32, bbr->params.cwnd_min_target, ++ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain)); ++} ++ ++/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss ++ * has drawn us down below target), or snap down to target if we're above it. ++ */ ++static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, ++ u32 acked, u32 bw, int gain, u32 cwnd, ++ struct bbr_context *ctx) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe; ++ ++ if (!acked) ++ goto done; /* no packet fully ACKed; just apply caps */ ++ ++ target_cwnd = bbr_bdp(sk, bw, gain); ++ ++ /* Increment the cwnd to account for excess ACKed data that seems ++ * due to aggregation (of data and/or ACKs) visible in the ACK stream. ++ */ ++ target_cwnd += bbr_ack_aggregation_cwnd(sk); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd); ++ ++ /* If we're below target cwnd, slow start cwnd toward target cwnd. */ ++ bbr->debug.target_cwnd = target_cwnd; ++ ++ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ ++ bbr->try_fast_path = 0; ++ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ ++ cwnd += acked; ++ if (cwnd >= target_cwnd) { ++ cwnd = target_cwnd; ++ bbr->try_fast_path = 1; ++ } ++ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { ++ cwnd += acked; ++ } else { ++ bbr->try_fast_path = 1; ++ } ++ ++ /* When growing cwnd, don't grow beyond twice what we just probed. */ ++ if (bbr->params.usage_based_cwnd) { ++ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd); ++ cwnd = min(cwnd, max_probe); ++ } ++ ++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target); ++done: ++ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */ ++ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ ++ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk)); ++ ++ ctx->target_cwnd = target_cwnd; ++ ctx->log = (tp->snd_cwnd != prev_cwnd); ++} ++ ++/* See if we have reached next round trip */ ++static void bbr_update_round_start(struct sock *sk, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->round_start = 0; ++ ++ /* See if we've reached the next RTT */ ++ if (rs->interval_us > 0 && ++ !before(rs->prior_delivered, bbr->next_rtt_delivered)) { ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr->round_start = 1; ++ } ++} ++ ++/* Calculate the bandwidth based on how fast packets are delivered */ ++static void bbr_calculate_bw_sample(struct sock *sk, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw = 0; ++ ++ /* Divide delivered by the interval to find a (lower bound) bottleneck ++ * bandwidth sample. Delivered is in packets and interval_us in uS and ++ * ratio will be <<1 for most connections. So delivered is first scaled. ++ * Round up to allow growth at low rates, even with integer division. ++ */ ++ if (rs->interval_us > 0) { ++ if (WARN_ONCE(rs->delivered < 0, ++ "negative delivered: %d interval_us: %ld\n", ++ rs->delivered, rs->interval_us)) ++ return; ++ ++ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); ++ } ++ ++ ctx->sample_bw = bw; ++ bbr->debug.rs_bw = bw; ++} ++ ++/* Estimates the windowed max degree of ack aggregation. ++ * This is used to provision extra in-flight data to keep sending during ++ * inter-ACK silences. ++ * ++ * Degree of ack aggregation is estimated as extra data acked beyond expected. ++ * ++ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval" ++ * cwnd += max_extra_acked ++ * ++ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). ++ * Max filter is an approximate sliding window of 5-10 (packet timed) round ++ * trips for non-startup phase, and 1-2 round trips for startup. ++ */ ++static void bbr_update_ack_aggregation(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ u32 epoch_us, expected_acked, extra_acked; ++ struct bbr *bbr = inet_csk_ca(sk); ++ struct tcp_sock *tp = tcp_sk(sk); ++ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts; ++ ++ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 || ++ rs->delivered < 0 || rs->interval_us <= 0) ++ return; ++ ++ if (bbr->round_start) { ++ bbr->extra_acked_win_rtts = min(0x1F, ++ bbr->extra_acked_win_rtts + 1); ++ if (bbr->params.extra_acked_in_startup && ++ !bbr_full_bw_reached(sk)) ++ extra_acked_win_rtts_thresh = 1; ++ if (bbr->extra_acked_win_rtts >= ++ extra_acked_win_rtts_thresh) { ++ bbr->extra_acked_win_rtts = 0; ++ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? ++ 0 : 1; ++ bbr->extra_acked[bbr->extra_acked_win_idx] = 0; ++ } ++ } ++ ++ /* Compute how many packets we expected to be delivered over epoch. */ ++ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp, ++ bbr->ack_epoch_mstamp); ++ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT; ++ ++ /* Reset the aggregation epoch if ACK rate is below expected rate or ++ * significantly large no. of ack received since epoch (potentially ++ * quite old epoch). ++ */ ++ if (bbr->ack_epoch_acked <= expected_acked || ++ (bbr->ack_epoch_acked + rs->acked_sacked >= ++ bbr_ack_epoch_acked_reset_thresh)) { ++ bbr->ack_epoch_acked = 0; ++ bbr->ack_epoch_mstamp = tp->delivered_mstamp; ++ expected_acked = 0; ++ } ++ ++ /* Compute excess data delivered, beyond what was expected. */ ++ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, ++ bbr->ack_epoch_acked + rs->acked_sacked); ++ extra_acked = bbr->ack_epoch_acked - expected_acked; ++ extra_acked = min(extra_acked, tp->snd_cwnd); ++ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) ++ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; ++} ++ ++/* Estimate when the pipe is full, using the change in delivery rate: BBR ++ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by ++ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited ++ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the ++ * higher rwin, 3: we get higher delivery rate samples. Or transient ++ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar ++ * design goal, but uses delay and inter-ACK spacing instead of bandwidth. ++ */ ++static void bbr_check_full_bw_reached(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 bw_thresh; ++ ++ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) ++ return; ++ ++ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE; ++ if (bbr_max_bw(sk) >= bw_thresh) { ++ bbr->full_bw = bbr_max_bw(sk); ++ bbr->full_bw_cnt = 0; ++ return; ++ } ++ ++bbr->full_bw_cnt; ++ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt; ++} ++ ++/* If pipe is probably full, drain the queue and then enter steady-state. */ ++static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs, ++ struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { ++ bbr->mode = BBR_DRAIN; /* drain queue we created */ ++ tcp_sk(sk)->snd_ssthresh = ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); ++ bbr2_reset_congestion_signals(sk); ++ } /* fall through to check if in-flight is already small: */ ++ if (bbr->mode == BBR_DRAIN && ++ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) ++ return true; /* exiting DRAIN now */ ++ return false; ++} ++ ++static void bbr_check_probe_rtt_done(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (!(bbr->probe_rtt_done_stamp && ++ after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) ++ return; ++ ++ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ ++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); ++ bbr2_exit_probe_rtt(sk); ++} ++ ++/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and ++ * periodically drain the bottleneck queue, to converge to measure the true ++ * min_rtt (unloaded propagation delay). This allows the flows to keep queues ++ * small (reducing queuing delay and packet loss) and achieve fairness among ++ * BBR flows. ++ * ++ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires, ++ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets. ++ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed ++ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and ++ * re-enter the previous mode. BBR uses 200ms to approximately bound the ++ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s). ++ * ++ * Note that flows need only pay 2% if they are busy sending over the last 10 ++ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have ++ * natural silences or low-rate periods within 10 seconds where the rate is low ++ * enough for long enough to drain its queue in the bottleneck. We pick up ++ * these min RTT measurements opportunistically with our min_rtt filter. :-) ++ */ ++static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ bool probe_rtt_expired, min_rtt_expired; ++ u32 expire; ++ ++ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ ++ expire = bbr->probe_rtt_min_stamp + ++ msecs_to_jiffies(bbr->params.probe_rtt_win_ms); ++ probe_rtt_expired = after(tcp_jiffies32, expire); ++ if (rs->rtt_us >= 0 && ++ (rs->rtt_us <= bbr->probe_rtt_min_us || ++ (probe_rtt_expired && !rs->is_ack_delayed))) { ++ bbr->probe_rtt_min_us = rs->rtt_us; ++ bbr->probe_rtt_min_stamp = tcp_jiffies32; ++ } ++ /* Track min RTT seen in the min_rtt_win_sec filter window: */ ++ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ; ++ min_rtt_expired = after(tcp_jiffies32, expire); ++ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || ++ min_rtt_expired) { ++ bbr->min_rtt_us = bbr->probe_rtt_min_us; ++ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; ++ } ++ ++ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired && ++ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { ++ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ ++ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ ++ bbr->probe_rtt_done_stamp = 0; ++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; ++ bbr->next_rtt_delivered = tp->delivered; ++ } ++ ++ if (bbr->mode == BBR_PROBE_RTT) { ++ /* Ignore low rate samples during this mode. */ ++ tp->app_limited = ++ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; ++ /* Maintain min packets in flight for max(200 ms, 1 round). */ ++ if (!bbr->probe_rtt_done_stamp && ++ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { ++ bbr->probe_rtt_done_stamp = tcp_jiffies32 + ++ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms); ++ bbr->probe_rtt_round_done = 0; ++ bbr->next_rtt_delivered = tp->delivered; ++ } else if (bbr->probe_rtt_done_stamp) { ++ if (bbr->round_start) ++ bbr->probe_rtt_round_done = 1; ++ if (bbr->probe_rtt_round_done) ++ bbr_check_probe_rtt_done(sk); ++ } ++ } ++ /* Restart after idle ends only once we process a new S/ACK for data */ ++ if (rs->delivered > 0) ++ bbr->idle_restart = 0; ++} ++ ++static void bbr_update_gains(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ switch (bbr->mode) { ++ case BBR_STARTUP: ++ bbr->pacing_gain = bbr->params.high_gain; ++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; ++ break; ++ case BBR_DRAIN: ++ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */ ++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */ ++ break; ++ case BBR_PROBE_BW: ++ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx]; ++ bbr->cwnd_gain = bbr->params.cwnd_gain; ++ break; ++ case BBR_PROBE_RTT: ++ bbr->pacing_gain = BBR_UNIT; ++ bbr->cwnd_gain = BBR_UNIT; ++ break; ++ default: ++ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); ++ break; ++ } ++} ++ ++static void bbr_init(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ int i; ++ ++ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val); ++ ++ bbr->initialized = 1; ++ bbr->params.high_gain = min(0x7FF, bbr_high_gain); ++ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain); ++ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain); ++ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain); ++ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget); ++ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target); ++ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec); ++ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms); ++ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt); ++ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh); ++ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain); ++ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts); ++ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0; ++ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0; ++ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0; ++ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain); ++ bbr->params.probe_rtt_win_ms = ++ min(0x3FFFU, ++ min_t(u32, bbr_probe_rtt_win_ms, ++ bbr->params.min_rtt_win_sec * MSEC_PER_SEC)); ++ for (i = 0; i < CYCLE_LEN; i++) ++ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]); ++ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0; ++ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift); ++ ++ bbr->debug.snd_isn = tp->snd_una; ++ bbr->debug.target_cwnd = 0; ++ bbr->debug.undo = 0; ++ ++ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd); ++ bbr->prior_cwnd = tp->prior_cwnd; ++ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; ++ bbr->next_rtt_delivered = 0; ++ bbr->prev_ca_state = TCP_CA_Open; ++ bbr->packet_conservation = 0; ++ ++ bbr->probe_rtt_done_stamp = 0; ++ bbr->probe_rtt_round_done = 0; ++ bbr->probe_rtt_min_us = tcp_min_rtt(tp); ++ bbr->probe_rtt_min_stamp = tcp_jiffies32; ++ bbr->min_rtt_us = tcp_min_rtt(tp); ++ bbr->min_rtt_stamp = tcp_jiffies32; ++ ++ bbr->has_seen_rtt = 0; ++ bbr_init_pacing_rate_from_rtt(sk); ++ ++ bbr->round_start = 0; ++ bbr->idle_restart = 0; ++ bbr->full_bw_reached = 0; ++ bbr->full_bw = 0; ++ bbr->full_bw_cnt = 0; ++ bbr->cycle_mstamp = 0; ++ bbr->cycle_idx = 0; ++ bbr->mode = BBR_STARTUP; ++ bbr->debug.rs_bw = 0; ++ ++ bbr->ack_epoch_mstamp = tp->tcp_mstamp; ++ bbr->ack_epoch_acked = 0; ++ bbr->extra_acked_win_rtts = 0; ++ bbr->extra_acked_win_idx = 0; ++ bbr->extra_acked[0] = 0; ++ bbr->extra_acked[1] = 0; ++ ++ bbr->ce_state = 0; ++ bbr->prior_rcv_nxt = tp->rcv_nxt; ++ bbr->try_fast_path = 0; ++ ++ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); ++} ++ ++static u32 bbr_sndbuf_expand(struct sock *sk) ++{ ++ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ ++ return 3; ++} ++ ++/* __________________________________________________________________________ ++ * ++ * Functions new to BBR v2 ("bbr") congestion control are below here. ++ * __________________________________________________________________________ ++ */ ++ ++/* Incorporate a new bw sample into the current window of our max filter. */ ++static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); ++} ++ ++/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ ++static void bbr2_advance_bw_hi_filter(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (!bbr->bw_hi[1]) ++ return; /* no samples in this window; remember old window */ ++ bbr->bw_hi[0] = bbr->bw_hi[1]; ++ bbr->bw_hi[1] = 0; ++} ++ ++/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ ++static u32 bbr2_target_inflight(struct sock *sk) ++{ ++ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); ++ ++ return min(bdp, tcp_sk(sk)->snd_cwnd); ++} ++ ++static bool bbr2_is_probing_bandwidth(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ return (bbr->mode == BBR_STARTUP) || ++ (bbr->mode == BBR_PROBE_BW && ++ (bbr->cycle_idx == BBR_BW_PROBE_REFILL || ++ bbr->cycle_idx == BBR_BW_PROBE_UP)); ++} ++ ++/* Has the given amount of time elapsed since we marked the phase start? */ ++static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) ++{ ++ const struct tcp_sock *tp = tcp_sk(sk); ++ const struct bbr *bbr = inet_csk_ca(sk); ++ ++ return tcp_stamp_us_delta(tp->tcp_mstamp, ++ bbr->cycle_mstamp + interval_us) > 0; ++} ++ ++static void bbr2_handle_queue_too_high_in_startup(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->full_bw_reached = 1; ++ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); ++} ++ ++/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ ++static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || ++ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh) ++ return; ++ ++ if (ce_ratio >= bbr->params.ecn_thresh) ++ bbr->startup_ecn_rounds++; ++ else ++ bbr->startup_ecn_rounds = 0; ++ ++ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) { ++ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */ ++ bbr2_handle_queue_too_high_in_startup(sk); ++ return; ++ } ++} ++ ++static int bbr2_update_ecn_alpha(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ s32 delivered, delivered_ce; ++ u64 alpha, ce_ratio; ++ u32 gain; ++ ++ if (bbr->params.ecn_factor == 0) ++ return -1; ++ ++ delivered = tp->delivered - bbr->alpha_last_delivered; ++ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; ++ ++ if (delivered == 0 || /* avoid divide by zero */ ++ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ ++ return -1; ++ ++ /* See if we should use ECN sender logic for this connection. */ ++ if (!bbr->ecn_eligible && bbr_ecn_enable && ++ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us || ++ !bbr->params.ecn_max_rtt_us)) ++ bbr->ecn_eligible = 1; ++ ++ ce_ratio = (u64)delivered_ce << BBR_SCALE; ++ do_div(ce_ratio, delivered); ++ gain = bbr->params.ecn_alpha_gain; ++ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; ++ alpha += (gain * ce_ratio) >> BBR_SCALE; ++ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); ++ ++ bbr->alpha_last_delivered = tp->delivered; ++ bbr->alpha_last_delivered_ce = tp->delivered_ce; ++ ++ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio); ++ return (int)ce_ratio; ++} ++ ++/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ ++static void bbr2_raise_inflight_hi_slope(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 growth_this_round, cnt; ++ ++ /* Calculate "slope": packets S/Acked per inflight_hi increment. */ ++ growth_this_round = 1 << bbr->bw_probe_up_rounds; ++ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); ++ cnt = tp->snd_cwnd / growth_this_round; ++ cnt = max(cnt, 1U); ++ bbr->bw_probe_up_cnt = cnt; ++ bbr->debug.event = 'G'; /* Grow inflight_hi slope */ ++} ++ ++/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ ++static void bbr2_probe_inflight_hi_upward(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 delta; ++ ++ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) { ++ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */ ++ return; /* not fully using inflight_hi, so don't grow it */ ++ } ++ ++ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ ++ bbr->bw_probe_up_acks += rs->acked_sacked; ++ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { ++ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; ++ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; ++ bbr->inflight_hi += delta; ++ bbr->debug.event = 'I'; /* Increment inflight_hi */ ++ } ++ ++ if (bbr->round_start) ++ bbr2_raise_inflight_hi_slope(sk); ++} ++ ++/* Does loss/ECN rate for this sample say inflight is "too high"? ++ * This is used by both the bbr_check_loss_too_high_in_startup() function, ++ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which ++ * uses it to notice when loss/ECN rates suggest inflight is too high. ++ */ ++static bool bbr2_is_inflight_too_high(const struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ const struct bbr *bbr = inet_csk_ca(sk); ++ u32 loss_thresh, ecn_thresh; ++ ++ if (rs->lost > 0 && rs->tx_in_flight) { ++ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >> ++ BBR_SCALE; ++ if (rs->lost > loss_thresh) ++ return true; ++ } ++ ++ if (rs->delivered_ce > 0 && rs->delivered > 0 && ++ bbr->ecn_eligible && bbr->params.ecn_thresh) { ++ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >> ++ BBR_SCALE; ++ if (rs->delivered_ce >= ecn_thresh) ++ return true; ++ } ++ ++ return false; ++} ++ ++/* Calculate the tx_in_flight level that corresponded to excessive loss. ++ * We find "lost_prefix" segs of the skb where loss rate went too high, ++ * by solving for "lost_prefix" in the following equation: ++ * lost / inflight >= loss_thresh ++ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh ++ * Then we take that equation, convert it to fixed point, and ++ * round up to the nearest packet. ++ */ ++static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk, ++ const struct rate_sample *rs, ++ const struct sk_buff *skb) ++{ ++ const struct bbr *bbr = inet_csk_ca(sk); ++ u32 loss_thresh = bbr->params.loss_thresh; ++ u32 pcount, divisor, inflight_hi; ++ s32 inflight_prev, lost_prev; ++ u64 loss_budget, lost_prefix; ++ ++ pcount = tcp_skb_pcount(skb); ++ ++ /* How much data was in flight before this skb? */ ++ inflight_prev = rs->tx_in_flight - pcount; ++ if (WARN_ONCE(inflight_prev < 0, ++ "tx_in_flight: %u pcount: %u reneg: %u", ++ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg)) ++ return ~0U; ++ ++ /* How much inflight data was marked lost before this skb? */ ++ lost_prev = rs->lost - pcount; ++ if (WARN_ON_ONCE(lost_prev < 0)) ++ return ~0U; ++ ++ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ ++ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; ++ loss_budget >>= BBR_SCALE; ++ if (lost_prev >= loss_budget) { ++ lost_prefix = 0; /* previous losses crossed loss_thresh */ ++ } else { ++ lost_prefix = loss_budget - lost_prev; ++ lost_prefix <<= BBR_SCALE; ++ divisor = BBR_UNIT - loss_thresh; ++ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ ++ return ~0U; ++ do_div(lost_prefix, divisor); ++ } ++ ++ inflight_hi = inflight_prev + lost_prefix; ++ return inflight_hi; ++} ++ ++/* If loss/ECN rates during probing indicated we may have overfilled a ++ * buffer, return an operating point that tries to leave unutilized headroom in ++ * the path for other flows, for fairness convergence and lower RTTs and loss. ++ */ ++static u32 bbr2_inflight_with_headroom(const struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 headroom, headroom_fraction; ++ ++ if (bbr->inflight_hi == ~0U) ++ return ~0U; ++ ++ headroom_fraction = bbr->params.inflight_headroom; ++ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; ++ headroom = max(headroom, 1U); ++ return max_t(s32, bbr->inflight_hi - headroom, ++ bbr->params.cwnd_min_target); ++} ++ ++/* Bound cwnd to a sensible level, based on our current probing state ++ * machine phase and model of a good inflight level (inflight_lo, inflight_hi). ++ */ ++static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 cap; ++ ++ /* tcp_rcv_synsent_state_process() currently calls tcp_ack() ++ * and thus cong_control() without first initializing us(!). ++ */ ++ if (!bbr->initialized) ++ return; ++ ++ cap = ~0U; ++ if (bbr->mode == BBR_PROBE_BW && ++ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { ++ /* Probe to see if more packets fit in the path. */ ++ cap = bbr->inflight_hi; ++ } else { ++ if (bbr->mode == BBR_PROBE_RTT || ++ (bbr->mode == BBR_PROBE_BW && ++ bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) ++ cap = bbr2_inflight_with_headroom(sk); ++ } ++ /* Adapt to any loss/ECN since our last bw probe. */ ++ cap = min(cap, bbr->inflight_lo); ++ ++ cap = max_t(u32, cap, bbr->params.cwnd_min_target); ++ tp->snd_cwnd = min(cap, tp->snd_cwnd); ++} ++ ++/* Estimate a short-term lower bound on the capacity available now, based ++ * on measurements of the current delivery process and recent history. When we ++ * are seeing loss/ECN at times when we are not probing bw, then conservatively ++ * move toward flow balance by multiplicatively cutting our short-term ++ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a ++ * multiplicative decrease in order to converge to a lower capacity in time ++ * logarithmic in the magnitude of the decrease. ++ * ++ * However, we do not cut our short-term estimates lower than the current rate ++ * and volume of delivered data from this round trip, since from the current ++ * delivery process we can estimate the measured capacity available now. ++ * ++ * Anything faster than that approach would knowingly risk high loss, which can ++ * cause low bw for Reno/CUBIC and high loss recovery latency for ++ * request/response flows using any congestion control. ++ */ ++static void bbr2_adapt_lower_bounds(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 ecn_cut, ecn_inflight_lo, beta; ++ ++ /* We only use lower-bound estimates when not probing bw. ++ * When probing we need to push inflight higher to probe bw. ++ */ ++ if (bbr2_is_probing_bandwidth(sk)) ++ return; ++ ++ /* ECN response. */ ++ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) { ++ /* Reduce inflight to (1 - alpha*ecn_factor). */ ++ ecn_cut = (BBR_UNIT - ++ ((bbr->ecn_alpha * bbr->params.ecn_factor) >> ++ BBR_SCALE)); ++ if (bbr->inflight_lo == ~0U) ++ bbr->inflight_lo = tp->snd_cwnd; ++ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; ++ } else { ++ ecn_inflight_lo = ~0U; ++ } ++ ++ /* Loss response. */ ++ if (bbr->loss_in_round) { ++ /* Reduce bw and inflight to (1 - beta). */ ++ if (bbr->bw_lo == ~0U) ++ bbr->bw_lo = bbr_max_bw(sk); ++ if (bbr->inflight_lo == ~0U) ++ bbr->inflight_lo = tp->snd_cwnd; ++ beta = bbr->params.beta; ++ bbr->bw_lo = ++ max_t(u32, bbr->bw_latest, ++ (u64)bbr->bw_lo * ++ (BBR_UNIT - beta) >> BBR_SCALE); ++ bbr->inflight_lo = ++ max_t(u32, bbr->inflight_latest, ++ (u64)bbr->inflight_lo * ++ (BBR_UNIT - beta) >> BBR_SCALE); ++ } ++ ++ /* Adjust to the lower of the levels implied by loss or ECN. */ ++ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); ++} ++ ++/* Reset any short-term lower-bound adaptation to congestion, so that we can ++ * push our inflight up. ++ */ ++static void bbr2_reset_lower_bounds(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->bw_lo = ~0U; ++ bbr->inflight_lo = ~0U; ++} ++ ++/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state ++ * machine phase where we adapt our lower bound based on congestion signals. ++ */ ++static void bbr2_reset_congestion_signals(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->loss_in_round = 0; ++ bbr->ecn_in_round = 0; ++ bbr->loss_in_cycle = 0; ++ bbr->ecn_in_cycle = 0; ++ bbr->bw_latest = 0; ++ bbr->inflight_latest = 0; ++} ++ ++/* Update (most of) our congestion signals: track the recent rate and volume of ++ * delivered data, presence of loss, and EWMA degree of ECN marking. ++ */ ++static void bbr2_update_congestion_signals( ++ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw; ++ ++ bbr->loss_round_start = 0; ++ if (rs->interval_us <= 0 || !rs->acked_sacked) ++ return; /* Not a valid observation */ ++ bw = ctx->sample_bw; ++ ++ if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) ++ bbr2_take_bw_hi_sample(sk, bw); ++ ++ bbr->loss_in_round |= (rs->losses > 0); ++ ++ /* Update rate and volume of delivered data from latest round trip: */ ++ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); ++ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); ++ ++ if (before(rs->prior_delivered, bbr->loss_round_delivered)) ++ return; /* skip the per-round-trip updates */ ++ /* Now do per-round-trip updates. */ ++ bbr->loss_round_delivered = tp->delivered; /* mark round trip */ ++ bbr->loss_round_start = 1; ++ bbr2_adapt_lower_bounds(sk); ++ ++ /* Update windowed "latest" (single-round-trip) filters. */ ++ bbr->loss_in_round = 0; ++ bbr->ecn_in_round = 0; ++ bbr->bw_latest = ctx->sample_bw; ++ bbr->inflight_latest = rs->delivered; ++} ++ ++/* Bandwidth probing can cause loss. To help coexistence with loss-based ++ * congestion control we spread out our probing in a Reno-conscious way. Due to ++ * the shape of the Reno sawtooth, the time required between loss epochs for an ++ * idealized Reno flow is a number of round trips that is the BDP of that ++ * flow. We count packet-timed round trips directly, since measured RTT can ++ * vary widely, and Reno is driven by packet-timed round trips. ++ */ ++static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 inflight, rounds, reno_gain, reno_rounds; ++ ++ /* Random loss can shave some small percentage off of our inflight ++ * in each round. To survive this, flows need robust periodic probes. ++ */ ++ rounds = bbr->params.bw_probe_max_rounds; ++ ++ reno_gain = bbr->params.bw_probe_reno_gain; ++ if (reno_gain) { ++ inflight = bbr2_target_inflight(sk); ++ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE; ++ rounds = min(rounds, reno_rounds); ++ } ++ return bbr->rounds_since_probe >= rounds; ++} ++ ++/* How long do we want to wait before probing for bandwidth (and risking ++ * loss)? We randomize the wait, for better mixing and fairness convergence. ++ * ++ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. ++ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, ++ * (eg 4K video to a broadband user): ++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets ++ * ++ * We bound the BBR-native inter-bw-probe wall clock time to be: ++ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time ++ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must ++ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs ++ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable ++ * amount of time to discover unutilized bw on human-scale interactive ++ * time-scales (e.g. perhaps traffic from a web page download that we ++ * were competing with is now complete). ++ */ ++static void bbr2_pick_probe_wait(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ /* Decide the random round-trip bound for wait until probe: */ ++ bbr->rounds_since_probe = ++ prandom_u32_max(bbr->params.bw_probe_rand_rounds); ++ /* Decide the random wall clock bound for wait until probe: */ ++ bbr->probe_wait_us = bbr->params.bw_probe_base_us + ++ prandom_u32_max(bbr->params.bw_probe_rand_us); ++} ++ ++static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->cycle_idx = cycle_idx; ++ /* New phase, so need to update cwnd and pacing rate. */ ++ bbr->try_fast_path = 0; ++} ++ ++/* Send at estimated bw to fill the pipe, but not queue. We need this phase ++ * before PROBE_UP, because as soon as we send faster than the available bw ++ * we will start building a queue, and if the buffer is shallow we can cause ++ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and ++ * inflight_hi estimates will underestimate. ++ */ ++static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr2_reset_lower_bounds(sk); ++ if (bbr->inflight_hi != ~0U) ++ bbr->inflight_hi += bbr->params.refill_add_inc; ++ bbr->bw_probe_up_rounds = bw_probe_up_rounds; ++ bbr->bw_probe_up_acks = 0; ++ bbr->stopped_risky_probe = 0; ++ bbr->ack_phase = BBR_ACKS_REFILLING; ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); ++} ++ ++/* Now probe max deliverable data rate and volume. */ ++static void bbr2_start_bw_probe_up(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->ack_phase = BBR_ACKS_PROBE_STARTING; ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr->cycle_mstamp = tp->tcp_mstamp; ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP); ++ bbr2_raise_inflight_hi_slope(sk); ++} ++ ++/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall ++ * clock time at which to probe beyond an inflight that we think to be ++ * safe. This will knowingly risk packet loss, so we want to do this rarely, to ++ * keep packet loss rates low. Also start a round-trip counter, to probe faster ++ * if we estimate a Reno flow at our BDP would probe faster. ++ */ ++static void bbr2_start_bw_probe_down(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr2_reset_congestion_signals(sk); ++ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ ++ bbr2_pick_probe_wait(sk); ++ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ ++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; ++ bbr->next_rtt_delivered = tp->delivered; ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); ++} ++ ++/* Cruise: maintain what we estimate to be a neutral, conservative ++ * operating point, without attempting to probe up for bandwidth or down for ++ * RTT, and only reducing inflight in response to loss/ECN signals. ++ */ ++static void bbr2_start_bw_probe_cruise(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr->inflight_lo != ~0U) ++ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); ++ ++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); ++} ++ ++/* Loss and/or ECN rate is too high while probing. ++ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. ++ */ ++static void bbr2_handle_inflight_too_high(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ const u32 beta = bbr->params.beta; ++ ++ bbr->prev_probe_too_high = 1; ++ bbr->bw_probe_samples = 0; /* only react once per probe */ ++ bbr->debug.event = 'L'; /* Loss/ECN too high */ ++ /* If we are app-limited then we are not robustly ++ * probing the max volume of inflight data we think ++ * might be safe (analogous to how app-limited bw ++ * samples are not known to be robustly probing bw). ++ */ ++ if (!rs->is_app_limited) ++ bbr->inflight_hi = max_t(u32, rs->tx_in_flight, ++ (u64)bbr2_target_inflight(sk) * ++ (BBR_UNIT - beta) >> BBR_SCALE); ++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) ++ bbr2_start_bw_probe_down(sk); ++} ++ ++/* If we're seeing bw and loss samples reflecting our bw probing, adapt ++ * using the signals we see. If loss or ECN mark rate gets too high, then adapt ++ * inflight_hi downward. If we're able to push inflight higher without such ++ * signals, push higher: adapt inflight_hi upward. ++ */ ++static bool bbr2_adapt_upper_bounds(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ /* Track when we'll see bw/loss samples resulting from our bw probes. */ ++ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) ++ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; ++ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { ++ /* End of samples from bw probing phase. */ ++ bbr->bw_probe_samples = 0; ++ bbr->ack_phase = BBR_ACKS_INIT; ++ /* At this point in the cycle, our current bw sample is also ++ * our best recent chance at finding the highest available bw ++ * for this flow. So now is the best time to forget the bw ++ * samples from the previous cycle, by advancing the window. ++ */ ++ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) ++ bbr2_advance_bw_hi_filter(sk); ++ /* If we had an inflight_hi, then probed and pushed inflight all ++ * the way up to hit that inflight_hi without seeing any ++ * high loss/ECN in all the resulting ACKs from that probing, ++ * then probe up again, this time letting inflight persist at ++ * inflight_hi for a round trip, then accelerating beyond. ++ */ ++ if (bbr->mode == BBR_PROBE_BW && ++ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { ++ bbr->debug.event = 'R'; /* reprobe */ ++ bbr2_start_bw_probe_refill(sk, 0); ++ return true; /* yes, decided state transition */ ++ } ++ } ++ ++ if (bbr2_is_inflight_too_high(sk, rs)) { ++ if (bbr->bw_probe_samples) /* sample is from bw probing? */ ++ bbr2_handle_inflight_too_high(sk, rs); ++ } else { ++ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ ++ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */ ++ return false; ++ ++ /* To be resilient to random loss, we must raise inflight_hi ++ * if we observe in any phase that a higher level is safe. ++ */ ++ if (rs->tx_in_flight > bbr->inflight_hi) { ++ bbr->inflight_hi = rs->tx_in_flight; ++ bbr->debug.event = 'U'; /* raise up inflight_hi */ ++ } ++ ++ if (bbr->mode == BBR_PROBE_BW && ++ bbr->cycle_idx == BBR_BW_PROBE_UP) ++ bbr2_probe_inflight_hi_upward(sk, rs); ++ } ++ ++ return false; ++} ++ ++/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ ++static bool bbr2_check_time_to_probe_bw(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 n; ++ ++ /* If we seem to be at an operating point where we are not seeing loss ++ * but we are seeing ECN marks, then when the ECN marks cease we reprobe ++ * quickly (in case a burst of cross-traffic has ceased and freed up bw, ++ * or in case we are sharing with multiplicatively probing traffic). ++ */ ++ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible && ++ bbr->ecn_in_cycle && !bbr->loss_in_cycle && ++ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { ++ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */ ++ /* Calculate n so that when bbr2_raise_inflight_hi_slope() ++ * computes growth_this_round as 2^n it will be roughly the ++ * desired volume of data (inflight_hi*ecn_reprobe_gain). ++ */ ++ n = ilog2((((u64)bbr->inflight_hi * ++ bbr->params.ecn_reprobe_gain) >> BBR_SCALE)); ++ bbr2_start_bw_probe_refill(sk, n); ++ return true; ++ } ++ ++ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) || ++ bbr2_is_reno_coexistence_probe_time(sk)) { ++ bbr2_start_bw_probe_refill(sk, 0); ++ return true; ++ } ++ return false; ++} ++ ++/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ ++static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ bool is_under_bdp, is_long_enough; ++ ++ /* Always need to pull inflight down to leave headroom in queue. */ ++ if (inflight > bbr2_inflight_with_headroom(sk)) ++ return false; ++ ++ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT); ++ if (bbr->params.drain_to_target) ++ return is_under_bdp; ++ ++ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us); ++ return is_under_bdp || is_long_enough; ++} ++ ++/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ ++static void bbr2_update_cycle_phase(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ bool is_risky = false, is_queuing = false; ++ u32 inflight, bw; ++ ++ if (!bbr_full_bw_reached(sk)) ++ return; ++ ++ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ ++ if (bbr2_adapt_upper_bounds(sk, rs)) ++ return; /* already decided state transition */ ++ ++ if (bbr->mode != BBR_PROBE_BW) ++ return; ++ ++ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); ++ bw = bbr_max_bw(sk); ++ ++ switch (bbr->cycle_idx) { ++ /* First we spend most of our time cruising with a pacing_gain of 1.0, ++ * which paces at the estimated bw, to try to fully use the pipe ++ * without building queue. If we encounter loss/ECN marks, we adapt ++ * by slowing down. ++ */ ++ case BBR_BW_PROBE_CRUISE: ++ if (bbr2_check_time_to_probe_bw(sk)) ++ return; /* already decided state transition */ ++ break; ++ ++ /* After cruising, when it's time to probe, we first "refill": we send ++ * at the estimated bw to fill the pipe, before probing higher and ++ * knowingly risking overflowing the bottleneck buffer (causing loss). ++ */ ++ case BBR_BW_PROBE_REFILL: ++ if (bbr->round_start) { ++ /* After one full round trip of sending in REFILL, we ++ * start to see bw samples reflecting our REFILL, which ++ * may be putting too much data in flight. ++ */ ++ bbr->bw_probe_samples = 1; ++ bbr2_start_bw_probe_up(sk); ++ } ++ break; ++ ++ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to ++ * probe for bw. If we have not seen loss/ECN, we try to raise inflight ++ * to at least pacing_gain*BDP; note that this may take more than ++ * min_rtt if min_rtt is small (e.g. on a LAN). ++ * ++ * We terminate PROBE_UP bandwidth probing upon any of the following: ++ * ++ * (1) We've pushed inflight up to hit the inflight_hi target set in the ++ * most recent previous bw probe phase. Thus we want to start ++ * draining the queue immediately because it's very likely the most ++ * recently sent packets will fill the queue and cause drops. ++ * (checked here) ++ * (2) We have probed for at least 1*min_rtt_us, and the ++ * estimated queue is high enough (inflight > 1.25 * estimated_bdp). ++ * (checked here) ++ * (3) Loss filter says loss rate is "too high". ++ * (checked in bbr_is_inflight_too_high()) ++ * (4) ECN filter says ECN mark rate is "too high". ++ * (checked in bbr_is_inflight_too_high()) ++ */ ++ case BBR_BW_PROBE_UP: ++ if (bbr->prev_probe_too_high && ++ inflight >= bbr->inflight_hi) { ++ bbr->stopped_risky_probe = 1; ++ is_risky = true; ++ bbr->debug.event = 'D'; /* D for danger */ ++ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) && ++ inflight >= ++ bbr_inflight(sk, bw, ++ bbr->params.bw_probe_pif_gain)) { ++ is_queuing = true; ++ bbr->debug.event = 'Q'; /* building Queue */ ++ } ++ if (is_risky || is_queuing) { ++ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ ++ bbr2_start_bw_probe_down(sk); /* restart w/ down */ ++ } ++ break; ++ ++ /* After probing in PROBE_UP, we have usually accumulated some data in ++ * the bottleneck buffer (if bw probing didn't find more bw). We next ++ * enter PROBE_DOWN to try to drain any excess data from the queue. To ++ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until ++ * our inflight is less then that target cruising point, which is the ++ * minimum of (a) the amount needed to leave headroom, and (b) the ++ * estimated BDP. Once inflight falls to match the target, we estimate ++ * the queue is drained; persisting would underutilize the pipe. ++ */ ++ case BBR_BW_PROBE_DOWN: ++ if (bbr2_check_time_to_probe_bw(sk)) ++ return; /* already decided state transition */ ++ if (bbr2_check_time_to_cruise(sk, inflight, bw)) ++ bbr2_start_bw_probe_cruise(sk); ++ break; ++ ++ default: ++ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); ++ } ++} ++ ++/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ ++static void bbr2_exit_probe_rtt(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr2_reset_lower_bounds(sk); ++ if (bbr_full_bw_reached(sk)) { ++ bbr->mode = BBR_PROBE_BW; ++ /* Raising inflight after PROBE_RTT may cause loss, so reset ++ * the PROBE_BW clock and schedule the next bandwidth probe for ++ * a friendly and randomized future point in time. ++ */ ++ bbr2_start_bw_probe_down(sk); ++ /* Since we are exiting PROBE_RTT, we know inflight is ++ * below our estimated BDP, so it is reasonable to cruise. ++ */ ++ bbr2_start_bw_probe_cruise(sk); ++ } else { ++ bbr->mode = BBR_STARTUP; ++ } ++} ++ ++/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until ++ * the end of the round in recovery to get a good estimate of how many packets ++ * have been lost, and how many we need to drain with a low pacing rate. ++ */ ++static void bbr2_check_loss_too_high_in_startup(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr_full_bw_reached(sk)) ++ return; ++ ++ /* For STARTUP exit, check the loss rate at the end of each round trip ++ * of Recovery episodes in STARTUP. We check the loss rate at the end ++ * of the round trip to filter out noisy/low loss and have a better ++ * sense of inflight (extent of loss), so we can drain more accurately. ++ */ ++ if (rs->losses && bbr->loss_events_in_round < 0xf) ++ bbr->loss_events_in_round++; /* update saturating counter */ ++ if (bbr->params.full_loss_cnt && bbr->loss_round_start && ++ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && ++ bbr->loss_events_in_round >= bbr->params.full_loss_cnt && ++ bbr2_is_inflight_too_high(sk, rs)) { ++ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */ ++ bbr2_handle_queue_too_high_in_startup(sk); ++ return; ++ } ++ if (bbr->loss_round_start) ++ bbr->loss_events_in_round = 0; ++} ++ ++/* If we are done draining, advance into steady state operation in PROBE_BW. */ ++static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs, ++ struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (bbr_check_drain(sk, rs, ctx)) { ++ bbr->mode = BBR_PROBE_BW; ++ bbr2_start_bw_probe_down(sk); ++ } ++} ++ ++static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs, ++ struct bbr_context *ctx) ++{ ++ bbr2_update_congestion_signals(sk, rs, ctx); ++ bbr_update_ack_aggregation(sk, rs); ++ bbr2_check_loss_too_high_in_startup(sk, rs); ++ bbr_check_full_bw_reached(sk, rs); ++ bbr2_check_drain(sk, rs, ctx); ++ bbr2_update_cycle_phase(sk, rs); ++ bbr_update_min_rtt(sk, rs); ++} ++ ++/* Fast path for app-limited case. ++ * ++ * On each ack, we execute bbr state machine, which primarily consists of: ++ * 1) update model based on new rate sample, and ++ * 2) update control based on updated model or state change. ++ * ++ * There are certain workload/scenarios, e.g. app-limited case, where ++ * either we can skip updating model or we can skip update of both model ++ * as well as control. This provides signifcant softirq cpu savings for ++ * processing incoming acks. ++ * ++ * In case of app-limited, if there is no congestion (loss/ecn) and ++ * if observed bw sample is less than current estimated bw, then we can ++ * skip some of the computation in bbr state processing: ++ * ++ * - if there is no rtt/mode/phase change: In this case, since all the ++ * parameters of the network model are constant, we can skip model ++ * as well control update. ++ * ++ * - else we can skip rest of the model update. But we still need to ++ * update the control to account for the new rtt/mode/phase. ++ * ++ * Returns whether we can take fast path or not. ++ */ ++static bool bbr2_fast_path(struct sock *sk, bool *update_model, ++ const struct rate_sample *rs, struct bbr_context *ctx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 prev_min_rtt_us, prev_mode; ++ ++ if (bbr->params.fast_path && bbr->try_fast_path && ++ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && ++ !bbr->loss_in_round && !bbr->ecn_in_round) { ++ prev_mode = bbr->mode; ++ prev_min_rtt_us = bbr->min_rtt_us; ++ bbr2_check_drain(sk, rs, ctx); ++ bbr2_update_cycle_phase(sk, rs); ++ bbr_update_min_rtt(sk, rs); ++ ++ if (bbr->mode == prev_mode && ++ bbr->min_rtt_us == prev_min_rtt_us && ++ bbr->try_fast_path) ++ return true; ++ ++ /* Skip model update, but control still needs to be updated */ ++ *update_model = false; ++ } ++ return false; ++} ++ ++static void bbr2_main(struct sock *sk, const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ struct bbr_context ctx = { 0 }; ++ bool update_model = true; ++ u32 bw; ++ int ce_ratio = -1; ++ ++ bbr->debug.event = '.'; /* init to default NOP (no event yet) */ ++ ++ bbr_update_round_start(sk, rs, &ctx); ++ if (bbr->round_start) { ++ bbr->rounds_since_probe = ++ min_t(s32, bbr->rounds_since_probe + 1, 0xFF); ++ ce_ratio = bbr2_update_ecn_alpha(sk); ++ tcp_plb_update_state(sk, &bbr->plb, ce_ratio); ++ tcp_plb_check_rehash(sk, &bbr->plb); ++ } ++ ++ bbr->ecn_in_round |= rs->is_ece; ++ bbr_calculate_bw_sample(sk, rs, &ctx); ++ ++ if (bbr2_fast_path(sk, &update_model, rs, &ctx)) ++ goto out; ++ ++ if (update_model) ++ bbr2_update_model(sk, rs, &ctx); ++ ++ bbr_update_gains(sk); ++ bw = bbr_bw(sk); ++ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); ++ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, ++ tp->snd_cwnd, &ctx); ++ bbr2_bound_cwnd_for_inflight_model(sk); ++ ++out: ++ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; ++ bbr->loss_in_cycle |= rs->lost > 0; ++ bbr->ecn_in_cycle |= rs->delivered_ce > 0; ++ ++ bbr_debug(sk, rs->acked_sacked, rs, &ctx); ++} ++ ++/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared ++ * down here, so that the algorithm functions that use the parameters must use ++ * the per-socket parameters; if they accidentally use the global version ++ * then there will be a compile error. ++ * TODO(ncardwell): move all per-socket parameters down to this section. ++ */ ++ ++/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. ++ * No loss response when 0. Max allwed value is 255. ++ */ ++static u32 bbr_beta = BBR_UNIT * 30 / 100; ++ ++/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE. ++ * Max allowed value is 255. ++ */ ++static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */ ++ ++/* The initial value for the ecn_alpha state variable. Default and max ++ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly ++ * to congestion if the bottleneck is congested when the flow starts up. ++ */ ++static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */ ++ ++/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. ++ * No ECN based bounding when 0. Max allwed value is 255. ++ */ ++static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ ++ ++/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. ++ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255. ++ */ ++static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ ++ ++/* Max RTT (in usec) at which to use sender-side ECN logic. ++ * Disabled when 0 (ECN allowed at any RTT). ++ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms. ++ */ ++static u32 bbr_ecn_max_rtt_us = 5000; ++ ++/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN ++ * clears then use a multiplicative increase to quickly reprobe bw by ++ * starting inflight probing at the given multiple of inflight_hi. ++ * Default for this experimental knob is 0 (disabled). ++ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5. ++ */ ++static u32 bbr_ecn_reprobe_gain; ++ ++/* Estimate bw probing has gone too far if loss rate exceeds this level. */ ++static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ ++ ++/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, ++ * and loss rate is higher than bbr_loss_thresh. ++ * Disabled if 0. Max allowed value is 15 (0xF). ++ */ ++static u32 bbr_full_loss_cnt = 8; ++ ++/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh ++ * meets this count. Max allowed value is 3. ++ */ ++static u32 bbr_full_ecn_cnt = 2; ++ ++/* Fraction of unutilized headroom to try to leave in path upon high loss. */ ++static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; ++ ++/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase. ++ * Default is 1.25x, as in BBR v1. Max allowed is 511. ++ */ ++static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4; ++ ++/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips. ++ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism. ++ * Max allowed is 511. ++ */ ++static u32 bbr_bw_probe_reno_gain = BBR_UNIT; ++ ++/* Max number of packet-timed rounds to wait before probing for bandwidth. If ++ * we want to tolerate 1% random loss per round, and not have this cut our ++ * inflight too much, we must probe for bw periodically on roughly this scale. ++ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. ++ * We aim to be fair with Reno/CUBIC up to a BDP of at least: ++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets ++ */ ++static u32 bbr_bw_probe_max_rounds = 63; ++ ++/* Max amount of randomness to inject in round counting for Reno-coexistence. ++ * Max value is 15. ++ */ ++static u32 bbr_bw_probe_rand_rounds = 2; ++ ++/* Use BBR-native probe time scale starting at this many usec. ++ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: ++ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs ++ */ ++static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ ++ ++/* Use BBR-native probes spread over this many usec: */ ++static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ ++ ++/* Undo the model changes made in loss recovery if recovery was spurious? */ ++static bool bbr_undo = true; ++ ++/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ ++static bool bbr_fast_path = true; /* default: enabled */ ++ ++/* Use fast ack mode ? */ ++static int bbr_fast_ack_mode = 1; /* default: rwnd check off */ ++ ++/* How much to additively increase inflight_hi when entering REFILL? */ ++static u32 bbr_refill_add_inc; /* default: disabled */ ++ ++module_param_named(beta, bbr_beta, uint, 0644); ++module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644); ++module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644); ++module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644); ++module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644); ++module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644); ++module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644); ++module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664); +module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664); +module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664); +module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664); @@ -3010,22286 +8056,74134 @@ index 000000000000..2e39f7a353be +module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664); +module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664); + -+static void bbr2_init(struct sock *sk) ++static void bbr2_init(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ const struct net *net = sock_net(sk); ++ ++ bbr_init(sk); /* run shared init code for v1 and v2 */ ++ ++ /* BBR v2 parameters: */ ++ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); ++ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); ++ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); ++ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); ++ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); ++ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); ++ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); ++ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); ++ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); ++ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); ++ bbr->params.inflight_headroom = ++ min_t(u32, 0xFFU, bbr_inflight_headroom); ++ bbr->params.bw_probe_pif_gain = ++ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); ++ bbr->params.bw_probe_reno_gain = ++ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); ++ bbr->params.bw_probe_max_rounds = ++ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); ++ bbr->params.bw_probe_rand_rounds = ++ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); ++ bbr->params.bw_probe_base_us = ++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); ++ bbr->params.bw_probe_rand_us = ++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); ++ bbr->params.undo = bbr_undo; ++ bbr->params.fast_path = bbr_fast_path ? 1 : 0; ++ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); ++ ++ /* BBR v2 state: */ ++ bbr->initialized = 1; ++ /* Start sampling ECN mark rate after first full flight is ACKed: */ ++ bbr->loss_round_delivered = tp->delivered + 1; ++ bbr->loss_round_start = 0; ++ bbr->undo_bw_lo = 0; ++ bbr->undo_inflight_lo = 0; ++ bbr->undo_inflight_hi = 0; ++ bbr->loss_events_in_round = 0; ++ bbr->startup_ecn_rounds = 0; ++ bbr2_reset_congestion_signals(sk); ++ bbr->bw_lo = ~0U; ++ bbr->bw_hi[0] = 0; ++ bbr->bw_hi[1] = 0; ++ bbr->inflight_lo = ~0U; ++ bbr->inflight_hi = ~0U; ++ bbr->bw_probe_up_cnt = ~0U; ++ bbr->bw_probe_up_acks = 0; ++ bbr->bw_probe_up_rounds = 0; ++ bbr->probe_wait_us = 0; ++ bbr->stopped_risky_probe = 0; ++ bbr->ack_phase = BBR_ACKS_INIT; ++ bbr->rounds_since_probe = 0; ++ bbr->bw_probe_samples = 0; ++ bbr->prev_probe_too_high = 0; ++ bbr->ecn_eligible = 0; ++ bbr->ecn_alpha = bbr->params.ecn_alpha_init; ++ bbr->alpha_last_delivered = 0; ++ bbr->alpha_last_delivered_ce = 0; ++ ++ bbr->plb.enabled = 0; ++ bbr->plb.consec_cong_rounds = 0; ++ bbr->plb.pause_until = 0; ++ if ((tp->ecn_flags & TCP_ECN_OK) && ++ net->ipv4.sysctl_tcp_plb_enabled) ++ bbr->plb.enabled = 1; ++ ++ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); ++ ++ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable) ++ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; ++} ++ ++/* Core TCP stack informs us that the given skb was just marked lost. */ ++static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ struct tcp_skb_cb *scb = TCP_SKB_CB(skb); ++ struct rate_sample rs; ++ ++ /* Capture "current" data over the full round trip of loss, ++ * to have a better chance to see the full capacity of the path. ++ */ ++ if (!bbr->loss_in_round) /* first loss in this round trip? */ ++ bbr->loss_round_delivered = tp->delivered; /* set round trip */ ++ bbr->loss_in_round = 1; ++ bbr->loss_in_cycle = 1; ++ ++ if (!bbr->bw_probe_samples) ++ return; /* not an skb sent while probing for bandwidth */ ++ if (unlikely(!scb->tx.delivered_mstamp)) ++ return; /* skb was SACKed, reneged, marked lost; ignore it */ ++ /* We are probing for bandwidth. Construct a rate sample that ++ * estimates what happened in the flight leading up to this lost skb, ++ * then see if the loss rate went too high, and if so at which packet. ++ */ ++ memset(&rs, 0, sizeof(rs)); ++ rs.tx_in_flight = scb->tx.in_flight; ++ rs.lost = tp->lost - scb->tx.lost; ++ rs.is_app_limited = scb->tx.is_app_limited; ++ if (bbr2_is_inflight_too_high(sk, &rs)) { ++ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); ++ bbr2_handle_inflight_too_high(sk, &rs); ++ } ++} ++ ++/* Revert short-term model if current loss recovery event was spurious. */ ++static u32 bbr2_undo_cwnd(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->debug.undo = 1; ++ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ ++ bbr->full_bw_cnt = 0; ++ bbr->loss_in_round = 0; ++ ++ if (!bbr->params.undo) ++ return tp->snd_cwnd; ++ ++ /* Revert to cwnd and other state saved before loss episode. */ ++ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); ++ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); ++ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); ++ return bbr->prior_cwnd; ++} ++ ++/* Entering loss recovery, so save state for when we undo recovery. */ ++static u32 bbr2_ssthresh(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr_save_cwnd(sk); ++ /* For undo, save state that adapts based on loss signal. */ ++ bbr->undo_bw_lo = bbr->bw_lo; ++ bbr->undo_inflight_lo = bbr->inflight_lo; ++ bbr->undo_inflight_hi = bbr->inflight_hi; ++ return tcp_sk(sk)->snd_ssthresh; ++} ++ ++static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) ++{ ++ switch (bbr->mode) { ++ case BBR_STARTUP: ++ return BBR2_PHASE_STARTUP; ++ case BBR_DRAIN: ++ return BBR2_PHASE_DRAIN; ++ case BBR_PROBE_BW: ++ break; ++ case BBR_PROBE_RTT: ++ return BBR2_PHASE_PROBE_RTT; ++ default: ++ return BBR2_PHASE_INVALID; ++ } ++ switch (bbr->cycle_idx) { ++ case BBR_BW_PROBE_UP: ++ return BBR2_PHASE_PROBE_BW_UP; ++ case BBR_BW_PROBE_DOWN: ++ return BBR2_PHASE_PROBE_BW_DOWN; ++ case BBR_BW_PROBE_CRUISE: ++ return BBR2_PHASE_PROBE_BW_CRUISE; ++ case BBR_BW_PROBE_REFILL: ++ return BBR2_PHASE_PROBE_BW_REFILL; ++ default: ++ return BBR2_PHASE_INVALID; ++ } ++} ++ ++static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, ++ union tcp_cc_info *info) ++{ ++ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || ++ ext & (1 << (INET_DIAG_VEGASINFO - 1))) { ++ struct bbr *bbr = inet_csk_ca(sk); ++ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); ++ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); ++ u64 bw_lo = bbr->bw_lo == ~0U ? ++ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); ++ ++ memset(&info->bbr2, 0, sizeof(info->bbr2)); ++ info->bbr2.bbr_bw_lsb = (u32)bw; ++ info->bbr2.bbr_bw_msb = (u32)(bw >> 32); ++ info->bbr2.bbr_min_rtt = bbr->min_rtt_us; ++ info->bbr2.bbr_pacing_gain = bbr->pacing_gain; ++ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; ++ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; ++ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); ++ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; ++ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); ++ info->bbr2.bbr_mode = bbr->mode; ++ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); ++ info->bbr2.bbr_version = (__u8)2; ++ info->bbr2.bbr_inflight_lo = bbr->inflight_lo; ++ info->bbr2.bbr_inflight_hi = bbr->inflight_hi; ++ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); ++ *attr = INET_DIAG_BBRINFO; ++ return sizeof(info->bbr2); ++ } ++ return 0; ++} ++ ++static void bbr2_set_state(struct sock *sk, u8 new_state) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (new_state == TCP_CA_Loss) { ++ struct rate_sample rs = { .losses = 1 }; ++ struct bbr_context ctx = { 0 }; ++ ++ tcp_plb_update_state_upon_rto(sk, &bbr->plb); ++ bbr->prev_ca_state = TCP_CA_Loss; ++ bbr->full_bw = 0; ++ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { ++ /* bbr_adapt_lower_bounds() needs cwnd before ++ * we suffered an RTO, to update inflight_lo: ++ */ ++ bbr->inflight_lo = ++ max(tp->snd_cwnd, bbr->prior_cwnd); ++ } ++ bbr_debug(sk, 0, &rs, &ctx); ++ } else if (bbr->prev_ca_state == TCP_CA_Loss && ++ new_state != TCP_CA_Loss) { ++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); ++ bbr->try_fast_path = 0; /* bound cwnd using latest model */ ++ } ++} ++ ++static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { ++ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, ++ .name = "bbr2", ++ .owner = THIS_MODULE, ++ .init = bbr2_init, ++ .cong_control = bbr2_main, ++ .sndbuf_expand = bbr_sndbuf_expand, ++ .skb_marked_lost = bbr2_skb_marked_lost, ++ .undo_cwnd = bbr2_undo_cwnd, ++ .cwnd_event = bbr_cwnd_event, ++ .ssthresh = bbr2_ssthresh, ++ .tso_segs = bbr_tso_segs, ++ .get_info = bbr2_get_info, ++ .set_state = bbr2_set_state, ++}; ++ ++static int __init bbr_register(void) ++{ ++ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); ++ return tcp_register_congestion_control(&tcp_bbr2_cong_ops); ++} ++ ++static void __exit bbr_unregister(void) ++{ ++ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); ++} ++ ++module_init(bbr_register); ++module_exit(bbr_unregister); ++ ++MODULE_AUTHOR("Van Jacobson "); ++MODULE_AUTHOR("Neal Cardwell "); ++MODULE_AUTHOR("Yuchung Cheng "); ++MODULE_AUTHOR("Soheil Hassas Yeganeh "); ++MODULE_AUTHOR("Priyaranjan Jha "); ++MODULE_AUTHOR("Yousuk Seung "); ++MODULE_AUTHOR("Kevin Yang "); ++MODULE_AUTHOR("Arjun Roy "); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); +diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c +index d3cae40749e8..0f268f2ff2e9 100644 +--- a/net/ipv4/tcp_cong.c ++++ b/net/ipv4/tcp_cong.c +@@ -189,6 +189,7 @@ void tcp_init_congestion_control(struct sock *sk) + struct inet_connection_sock *icsk = inet_csk(sk); + + tcp_sk(sk)->prior_ssthresh = 0; ++ tcp_sk(sk)->fast_ack_mode = 0; + if (icsk->icsk_ca_ops->init) + icsk->icsk_ca_ops->init(sk); + if (tcp_ca_needs_ecn(sk)) +diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c +index bc2ea12221f9..fa5c3e5d9c85 100644 +--- a/net/ipv4/tcp_input.c ++++ b/net/ipv4/tcp_input.c +@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) + tcp_enter_quickack_mode(sk, 2); + break; + case INET_ECN_CE: +- if (tcp_ca_needs_ecn(sk)) ++ if (tcp_ca_wants_ce_events(sk)) + tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); + + if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { +@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) + tp->ecn_flags |= TCP_ECN_SEEN; + break; + default: +- if (tcp_ca_needs_ecn(sk)) ++ if (tcp_ca_wants_ce_events(sk)) + tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); + tp->ecn_flags |= TCP_ECN_SEEN; + break; +@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) + */ + static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) + { ++ struct sock *sk = (struct sock *)tp; ++ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; ++ + tp->lost += tcp_skb_pcount(skb); ++ if (ca_ops->skb_marked_lost) ++ ca_ops->skb_marked_lost(sk, skb); + } + + void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) +@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, + WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); + tcp_skb_pcount_add(skb, -pcount); + ++ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ ++ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, ++ "prev in_flight: %u skb in_flight: %u pcount: %u", ++ TCP_SKB_CB(prev)->tx.in_flight, ++ TCP_SKB_CB(skb)->tx.in_flight, ++ pcount)) ++ TCP_SKB_CB(skb)->tx.in_flight = 0; ++ else ++ TCP_SKB_CB(skb)->tx.in_flight -= pcount; ++ TCP_SKB_CB(prev)->tx.in_flight += pcount; ++ + /* When we're adding to gso_segs == 1, gso_size will be zero, + * in theory this shouldn't be necessary but as long as DSACK + * code can come after this skb later on it's better to keep +@@ -3811,6 +3827,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) + + prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; + rs.prior_in_flight = tcp_packets_in_flight(tp); ++ tcp_rate_check_app_limited(sk); + + /* ts_recent update must be made after we are sure that the packet + * is in window. +@@ -3909,6 +3926,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) + delivered = tcp_newly_delivered(sk, delivered, flag); + lost = tp->lost - lost; /* freshly marked lost */ + rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); ++ rs.is_ece = !!(flag & FLAG_ECE); + tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); + tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); + tcp_xmit_recovery(sk, rexmit); +@@ -5508,13 +5526,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) + + /* More than one full frame received... */ + if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && ++ (tp->fast_ack_mode == 1 || + /* ... and right edge of window advances far enough. + * (tcp_recvmsg() will send ACK otherwise). + * If application uses SO_RCVLOWAT, we want send ack now if + * we have not received enough bytes to satisfy the condition. + */ +- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || +- __tcp_select_window(sk) >= tp->rcv_wnd)) || ++ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || ++ __tcp_select_window(sk) >= tp->rcv_wnd))) || + /* We ACK each frame or... */ + tcp_in_quickack_mode(sk) || + /* Protocol state mandates a one-time immediate ACK */ +diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c +index 5b019ba2b9d2..5884f4388c61 100644 +--- a/net/ipv4/tcp_ipv4.c ++++ b/net/ipv4/tcp_ipv4.c +@@ -3165,6 +3165,13 @@ static int __net_init tcp_sk_init(struct net *net) + net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0; + atomic_set(&net->ipv4.tfo_active_disable_times, 0); + ++ /* Set default values for PLB */ ++ net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */ ++ net->ipv4.sysctl_tcp_plb_cong_thresh = 128; /* 50% congestion */ ++ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3; ++ net->ipv4.sysctl_tcp_plb_rehash_rounds = 12; ++ net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60; ++ + /* Reno is always built in */ + if (!net_eq(net, &init_net) && + bpf_try_module_get(init_net.ipv4.tcp_congestion_control, +diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c +index 290019de766d..15e89addae08 100644 +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -375,7 +375,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, + th->cwr = 1; + skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; + } +- } else if (!tcp_ca_needs_ecn(sk)) { ++ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && ++ !tcp_ca_needs_ecn(sk)) { + /* ACK or retransmitted segment: clear ECT|CE */ + INET_ECN_dontxmit(sk); + } +@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, + { + struct tcp_sock *tp = tcp_sk(sk); + struct sk_buff *buff; +- int nsize, old_factor; ++ int nsize, old_factor, inflight_prev; + long limit; + int nlen; + u8 flags; +@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, + + if (diff) + tcp_adjust_pcount(sk, skb, diff); ++ ++ /* Set buff tx.in_flight as if buff were sent by itself. */ ++ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; ++ if (WARN_ONCE(inflight_prev < 0, ++ "inconsistent: tx.in_flight: %u old_factor: %d", ++ TCP_SKB_CB(skb)->tx.in_flight, old_factor)) ++ inflight_prev = 0; ++ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + ++ tcp_skb_pcount(buff); + } + + /* Link BUFF into the send queue. */ +@@ -1988,13 +1998,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, + static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) + { + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; +- u32 min_tso, tso_segs; +- +- min_tso = ca_ops->min_tso_segs ? +- ca_ops->min_tso_segs(sk) : +- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); ++ u32 tso_segs; + +- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); ++ tso_segs = ca_ops->tso_segs ? ++ ca_ops->tso_segs(sk, mss_now) : ++ tcp_tso_autosize(sk, mss_now, ++ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs)); + return min_t(u32, tso_segs, sk->sk_gso_max_segs); + } + +@@ -2630,6 +2639,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, + skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); + list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); + tcp_init_tso_segs(skb, mss_now); ++ tcp_set_tx_in_flight(sk, skb); + goto repair; /* Skip network transmission */ + } + +diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c +new file mode 100644 +index 000000000000..71b02c0404ce +--- /dev/null ++++ b/net/ipv4/tcp_plb.c +@@ -0,0 +1,100 @@ ++/* Protective Load Balancing (PLB) ++ * ++ * PLB was designed to reduce link load imbalance across datacenter ++ * switches. PLB is a host-based optimization; it leverages congestion ++ * signals from the transport layer to randomly change the path of the ++ * connection experiencing sustained congestion. PLB prefers to repath ++ * after idle periods to minimize packet reordering. It repaths by ++ * changing the IPv6 Flow Label on the packets of a connection, which ++ * datacenter switches include as part of ECMP/WCMP hashing. ++ * ++ * PLB is described in detail in: ++ * ++ * Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu, ++ * Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson, ++ * David Wetherall,Abdul Kabbani: ++ * "PLB: Congestion Signals are Simple and Effective for ++ * Network Load Balancing" ++ * In ACM SIGCOMM 2022, Amsterdam Netherlands. ++ * ++ */ ++ ++#include ++ ++/* Called once per round-trip to update PLB state for a connection. */ ++void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, ++ const int cong_ratio) ++{ ++ struct net *net = sock_net(sk); ++ ++ if (!plb->enabled) ++ return; ++ ++ if (cong_ratio >= 0) { ++ if (cong_ratio < net->ipv4.sysctl_tcp_plb_cong_thresh) ++ plb->consec_cong_rounds = 0; ++ else if (plb->consec_cong_rounds < ++ net->ipv4.sysctl_tcp_plb_rehash_rounds) ++ plb->consec_cong_rounds++; ++ } ++} ++EXPORT_SYMBOL_GPL(tcp_plb_update_state); ++ ++/* Check whether recent congestion has been persistent enough to warrant ++ * a load balancing decision that switches the connection to another path. ++ */ ++void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) ++{ ++ struct net *net = sock_net(sk); ++ bool can_idle_rehash, can_force_rehash; ++ ++ if (!plb->enabled) ++ return; ++ ++ /* Note that tcp_jiffies32 can wrap, so we clear pause_until ++ * to 0 to indicate there is no recent RTO event that constrains ++ * PLB rehashing. ++ */ ++ if (plb->pause_until && ++ !before(tcp_jiffies32, plb->pause_until)) ++ plb->pause_until = 0; ++ ++ can_idle_rehash = net->ipv4.sysctl_tcp_plb_idle_rehash_rounds && ++ !tcp_sk(sk)->packets_out && ++ plb->consec_cong_rounds >= ++ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds; ++ can_force_rehash = plb->consec_cong_rounds >= ++ net->ipv4.sysctl_tcp_plb_rehash_rounds; ++ ++ if (!plb->pause_until && (can_idle_rehash || can_force_rehash)) { ++ sk_rethink_txhash(sk); ++ plb->consec_cong_rounds = 0; ++ tcp_sk(sk)->ecn_rehash++; ++ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPECNREHASH); ++ } ++} ++EXPORT_SYMBOL_GPL(tcp_plb_check_rehash); ++ ++/* Upon RTO, disallow load balancing for a while, to avoid having load ++ * balancing decisions switch traffic to a black-holed path that was ++ * previously avoided with a sk_rethink_txhash() call at RTO time. ++ */ ++void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb) ++{ ++ struct net *net = sock_net(sk); ++ u32 pause; ++ ++ if (!plb->enabled) ++ return; ++ ++ pause = net->ipv4.sysctl_tcp_plb_suspend_rto_sec * HZ; ++ pause += prandom_u32_max(pause); ++ plb->pause_until = tcp_jiffies32 + pause; ++ ++ /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call ++ * that may switch this connection to a path with completely different ++ * congestion characteristics. ++ */ ++ plb->consec_cong_rounds = 0; ++} ++EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto); +diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c +index a8f6d9d06f2e..a8b4c9504570 100644 +--- a/net/ipv4/tcp_rate.c ++++ b/net/ipv4/tcp_rate.c +@@ -34,6 +34,24 @@ + * ready to send in the write queue. + */ + ++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ u32 in_flight; ++ ++ /* Check, sanitize, and record packets in flight after skb was sent. */ ++ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); ++ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, ++ "insane in_flight %u cc %s mss %u " ++ "cwnd %u pif %u %u %u %u\n", ++ in_flight, inet_csk(sk)->icsk_ca_ops->name, ++ tp->mss_cache, tp->snd_cwnd, ++ tp->packets_out, tp->retrans_out, ++ tp->sacked_out, tp->lost_out)) ++ in_flight = TCPCB_IN_FLIGHT_MAX; ++ TCP_SKB_CB(skb)->tx.in_flight = in_flight; ++} ++ + /* Snapshot the current delivery information in the skb, to generate + * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). + */ +@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) + TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; + TCP_SKB_CB(skb)->tx.delivered = tp->delivered; + TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; ++ TCP_SKB_CB(skb)->tx.lost = tp->lost; + TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; ++ tcp_set_tx_in_flight(sk, skb); + } + + /* When an skb is sacked or acked, we fill in the rate sample with the (prior) +@@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, + if (!rs->prior_delivered || + tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, + scb->end_seq, rs->last_end_seq)) { ++ rs->prior_lost = scb->tx.lost; + rs->prior_delivered_ce = scb->tx.delivered_ce; + rs->prior_delivered = scb->tx.delivered; + rs->prior_mstamp = scb->tx.delivered_mstamp; + rs->is_app_limited = scb->tx.is_app_limited; + rs->is_retrans = scb->sacked & TCPCB_RETRANS; + rs->last_end_seq = scb->end_seq; ++ rs->tx_in_flight = scb->tx.in_flight; + + /* Record send time of most recently ACKed packet: */ + tp->first_tx_mstamp = tx_tstamp; + /* Find the duration of the "send phase" of this window: */ +- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, +- scb->tx.first_tx_mstamp); ++ rs->interval_us = tcp_stamp32_us_delta( ++ tp->first_tx_mstamp, ++ scb->tx.first_tx_mstamp); + + } + /* Mark off the skb delivered once it's sacked to avoid being +@@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, + return; + } + rs->delivered = tp->delivered - rs->prior_delivered; ++ rs->lost = tp->lost - rs->prior_lost; + + rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; + /* delivered_ce occupies less than 32 bits in the skb control block */ +@@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, + * longer phase. + */ + snd_us = rs->interval_us; /* send phase */ +- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, ++ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, + rs->prior_mstamp); /* ack phase */ + rs->interval_us = max(snd_us, ack_us); + +diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c +index b4dfb82d6ecb..bb613fc8948a 100644 +--- a/net/ipv4/tcp_timer.c ++++ b/net/ipv4/tcp_timer.c +@@ -605,6 +605,7 @@ void tcp_write_timer_handler(struct sock *sk) + return; + } + ++ tcp_rate_check_app_limited(sk); + tcp_mstamp_refresh(tcp_sk(sk)); + event = icsk->icsk_pending; + +-- +2.38.0.rc1.8.g2a7d63a245 + +From 3a2a43e0dc41577b2d9262692c628362129d539d Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Sun, 25 Sep 2022 23:49:46 +0200 +Subject: [PATCH 03/16] futex-winesync + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/devices.txt | 3 +- + Documentation/userspace-api/index.rst | 1 + + .../userspace-api/ioctl/ioctl-number.rst | 2 + + Documentation/userspace-api/winesync.rst | 444 +++++ + MAINTAINERS | 9 + + drivers/misc/Kconfig | 11 + + drivers/misc/Makefile | 1 + + drivers/misc/winesync.c | 1212 ++++++++++++++ + include/linux/miscdevice.h | 1 + + include/uapi/linux/winesync.h | 71 + + tools/testing/selftests/Makefile | 1 + + .../selftests/drivers/winesync/Makefile | 8 + + .../testing/selftests/drivers/winesync/config | 1 + + .../selftests/drivers/winesync/winesync.c | 1479 +++++++++++++++++ + 14 files changed, 3243 insertions(+), 1 deletion(-) + create mode 100644 Documentation/userspace-api/winesync.rst + create mode 100644 drivers/misc/winesync.c + create mode 100644 include/uapi/linux/winesync.h + create mode 100644 tools/testing/selftests/drivers/winesync/Makefile + create mode 100644 tools/testing/selftests/drivers/winesync/config + create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c + +diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt +index 9764d6edb189..a4696d3b4a5a 100644 +--- a/Documentation/admin-guide/devices.txt ++++ b/Documentation/admin-guide/devices.txt +@@ -376,8 +376,9 @@ + 240 = /dev/userio Serio driver testing device + 241 = /dev/vhost-vsock Host kernel driver for virtio vsock + 242 = /dev/rfkill Turning off radio transmissions (rfkill) ++ 243 = /dev/winesync Wine synchronization primitive device + +- 243-254 Reserved for local use ++ 244-254 Reserved for local use + 255 Reserved for MISC_DYNAMIC_MINOR + + 11 char Raw keyboard device (Linux/SPARC only) +diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst +index a61eac0c73f8..0bf697ddcb09 100644 +--- a/Documentation/userspace-api/index.rst ++++ b/Documentation/userspace-api/index.rst +@@ -29,6 +29,7 @@ place where this information is gathered. + sysfs-platform_profile + vduse + futex2 ++ winesync + + .. only:: subproject and html + +diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst +index 3b985b19f39d..3f313fd4338c 100644 +--- a/Documentation/userspace-api/ioctl/ioctl-number.rst ++++ b/Documentation/userspace-api/ioctl/ioctl-number.rst +@@ -375,6 +375,8 @@ Code Seq# Include File Comments + + 0xF6 all LTTng Linux Trace Toolkit Next Generation + ++0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives ++ + 0xF8 all arch/x86/include/uapi/asm/amd_hsmp.h AMD HSMP EPYC system management interface driver + + 0xFD all linux/dm-ioctl.h +diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst +new file mode 100644 +index 000000000000..f0110d2744c7 +--- /dev/null ++++ b/Documentation/userspace-api/winesync.rst +@@ -0,0 +1,444 @@ ++===================================== ++Wine synchronization primitive driver ++===================================== ++ ++This page documents the user-space API for the winesync driver. ++ ++winesync is a support driver for emulation of NT synchronization ++primitives by the Wine project or other NT emulators. It exists ++because implementation in user-space, using existing tools, cannot ++simultaneously satisfy performance, correctness, and security ++constraints. It is implemented entirely in software, and does not ++drive any hardware device. ++ ++This interface is meant as a compatibility tool only, and should not ++be used for general synchronization. Instead use generic, versatile ++interfaces such as futex(2) and poll(2). ++ ++Synchronization primitives ++========================== ++ ++The winesync driver exposes three types of synchronization primitives: ++semaphores, mutexes, and events. ++ ++A semaphore holds a single volatile 32-bit counter, and a static ++32-bit integer denoting the maximum value. It is considered signaled ++when the counter is nonzero. The counter is decremented by one when a ++wait is satisfied. Both the initial and maximum count are established ++when the semaphore is created. ++ ++A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit ++identifier denoting its owner. A mutex is considered signaled when its ++owner is zero (indicating that it is not owned). The recursion count ++is incremented when a wait is satisfied, and ownership is set to the ++given identifier. ++ ++A mutex also holds an internal flag denoting whether its previous ++owner has died; such a mutex is said to be inconsistent. Owner death ++is not tracked automatically based on thread death, but rather must be ++communicated using ``WINESYNC_IOC_KILL_OWNER``. An inconsistent mutex ++is inherently considered unowned. ++ ++Except for the "unowned" semantics of zero, the actual value of the ++owner identifier is not interpreted by the winesync driver at all. The ++intended use is to store a thread identifier; however, the winesync ++driver does not actually validate that a calling thread provides ++consistent or unique identifiers. ++ ++An event holds a volatile boolean state denoting whether it is ++signaled or not. There are two types of events, auto-reset and ++manual-reset. An auto-reset event is designaled when a wait is ++satisfied; a manual-reset event is not. The event type is specified ++when the event is created. ++ ++Unless specified otherwise, all operations on an object are atomic and ++totally ordered with respect to other operations on the same object. ++ ++Objects are represented by unsigned 32-bit integers. ++ ++Char device ++=========== ++ ++The winesync driver creates a single char device /dev/winesync. Each ++file description opened on the device represents a unique namespace. ++That is, objects created on one open file description are shared ++across all its individual descriptors, but are not shared with other ++open() calls on the same device. The same file description may be ++shared across multiple processes. ++ ++ioctl reference ++=============== ++ ++All operations on the device are done through ioctls. There are three ++structures used in ioctl calls:: ++ ++ struct winesync_sem_args { ++ __u32 sem; ++ __u32 count; ++ __u32 max; ++ }; ++ ++ struct winesync_mutex_args { ++ __u32 mutex; ++ __u32 owner; ++ __u32 count; ++ }; ++ ++ struct winesync_event_args { ++ __u32 event; ++ __u32 signaled; ++ __u32 manual; ++ }; ++ ++ struct winesync_wait_args { ++ __u64 timeout; ++ __u64 objs; ++ __u32 count; ++ __u32 owner; ++ __u32 index; ++ __u32 pad; ++ }; ++ ++Depending on the ioctl, members of the structure may be used as input, ++output, or not at all. All ioctls return 0 on success. ++ ++The ioctls are as follows: ++ ++.. c:macro:: WINESYNC_IOC_CREATE_SEM ++ ++ Create a semaphore object. Takes a pointer to struct ++ :c:type:`winesync_sem_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``sem`` ++ - On output, contains the identifier of the created semaphore. ++ * - ``count`` ++ - Initial count of the semaphore. ++ * - ``max`` ++ - Maximum count of the semaphore. ++ ++ Fails with ``EINVAL`` if ``count`` is greater than ``max``. ++ ++.. c:macro:: WINESYNC_IOC_CREATE_MUTEX ++ ++ Create a mutex object. Takes a pointer to struct ++ :c:type:`winesync_mutex_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``mutex`` ++ - On output, contains the identifier of the created mutex. ++ * - ``count`` ++ - Initial recursion count of the mutex. ++ * - ``owner`` ++ - Initial owner of the mutex. ++ ++ If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is ++ zero and ``count`` is nonzero, the function fails with ``EINVAL``. ++ ++.. c:macro:: WINESYNC_IOC_CREATE_EVENT ++ ++ Create an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - On output, contains the identifier of the created event. ++ * - ``signaled`` ++ - If nonzero, the event is initially signaled, otherwise ++ nonsignaled. ++ * - ``manual`` ++ - If nonzero, the event is a manual-reset event, otherwise ++ auto-reset. ++ ++.. c:macro:: WINESYNC_IOC_DELETE ++ ++ Delete an object of any type. Takes an input-only pointer to a ++ 32-bit integer denoting the object to delete. ++ ++ Wait ioctls currently in progress are not interrupted, and behave as ++ if the object remains valid. ++ ++.. c:macro:: WINESYNC_IOC_PUT_SEM ++ ++ Post to a semaphore object. Takes a pointer to struct ++ :c:type:`winesync_sem_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``sem`` ++ - Semaphore object to post to. ++ * - ``count`` ++ - Count to add to the semaphore. On output, contains the ++ previous count of the semaphore. ++ * - ``max`` ++ - Not used. ++ ++ If adding ``count`` to the semaphore's current count would raise the ++ latter past the semaphore's maximum count, the ioctl fails with ++ ``EOVERFLOW`` and the semaphore is not affected. If raising the ++ semaphore's count causes it to become signaled, eligible threads ++ waiting on this semaphore will be woken and the semaphore's count ++ decremented appropriately. ++ ++.. c:macro:: WINESYNC_IOC_PUT_MUTEX ++ ++ Release a mutex object. Takes a pointer to struct ++ :c:type:`winesync_mutex_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``mutex`` ++ - Mutex object to release. ++ * - ``owner`` ++ - Mutex owner identifier. ++ * - ``count`` ++ - On output, contains the previous recursion count. ++ ++ If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` ++ is not the current owner of the mutex, the ioctl fails with ++ ``EPERM``. ++ ++ The mutex's count will be decremented by one. If decrementing the ++ mutex's count causes it to become zero, the mutex is marked as ++ unowned and signaled, and eligible threads waiting on it will be ++ woken as appropriate. ++ ++.. c:macro:: WINESYNC_IOC_SET_EVENT ++ ++ Signal an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object to set. ++ * - ``signaled`` ++ - On output, contains the previous state of the event. ++ * - ``manual`` ++ - Unused. ++ ++ Eligible threads will be woken, and auto-reset events will be ++ designaled appropriately. ++ ++.. c:macro:: WINESYNC_IOC_RESET_EVENT ++ ++ Designal an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object to reset. ++ * - ``signaled`` ++ - On output, contains the previous state of the event. ++ * - ``manual`` ++ - Unused. ++ ++.. c:macro:: WINESYNC_IOC_PULSE_EVENT ++ ++ Wake threads waiting on an event object without leaving it in a ++ signaled state. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object to pulse. ++ * - ``signaled`` ++ - On output, contains the previous state of the event. ++ * - ``manual`` ++ - Unused. ++ ++ A pulse operation can be thought of as a set followed by a reset, ++ performed as a single atomic operation. If two threads are waiting ++ on an auto-reset event which is pulsed, only one will be woken. If ++ two threads are waiting a manual-reset event which is pulsed, both ++ will be woken. However, in both cases, the event will be unsignaled ++ afterwards, and a simultaneous read operation will always report the ++ event as unsignaled. ++ ++.. c:macro:: WINESYNC_IOC_READ_SEM ++ ++ Read the current state of a semaphore object. Takes a pointer to ++ struct :c:type:`winesync_sem_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``sem`` ++ - Semaphore object to read. ++ * - ``count`` ++ - On output, contains the current count of the semaphore. ++ * - ``max`` ++ - On output, contains the maximum count of the semaphore. ++ ++.. c:macro:: WINESYNC_IOC_READ_MUTEX ++ ++ Read the current state of a mutex object. Takes a pointer to struct ++ :c:type:`winesync_mutex_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``mutex`` ++ - Mutex object to read. ++ * - ``owner`` ++ - On output, contains the current owner of the mutex, or zero ++ if the mutex is not currently owned. ++ * - ``count`` ++ - On output, contains the current recursion count of the mutex. ++ ++ If the mutex is marked as inconsistent, the function fails with ++ ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to ++ zero. ++ ++.. c:macro:: WINESYNC_IOC_READ_EVENT ++ ++ Read the current state of an event object. Takes a pointer to struct ++ :c:type:`winesync_event_args`, which is used as follows: ++ ++ .. list-table:: ++ ++ * - ``event`` ++ - Event object. ++ * - ``signaled`` ++ - On output, contains the current state of the event. ++ * - ``manual`` ++ - On output, contains 1 if the event is a manual-reset event, ++ and 0 otherwise. ++ ++.. c:macro:: WINESYNC_IOC_KILL_OWNER ++ ++ Mark any mutexes owned by the given owner as unowned and ++ inconsistent. Takes an input-only pointer to a 32-bit integer ++ denoting the owner. If the owner is zero, the ioctl fails with ++ ``EINVAL``. ++ ++ For each mutex currently owned by the given owner, eligible threads ++ waiting on said mutex will be woken as appropriate (and such waits ++ will fail with ``EOWNERDEAD``, as described below). ++ ++ The operation as a whole is not atomic; however, the modification of ++ each mutex is atomic and totally ordered with respect to other ++ operations on the same mutex. ++ ++.. c:macro:: WINESYNC_IOC_WAIT_ANY ++ ++ Poll on any of a list of objects, atomically acquiring at most one. ++ Takes a pointer to struct :c:type:`winesync_wait_args`, which is ++ used as follows: ++ ++ .. list-table:: ++ ++ * - ``timeout`` ++ - Optional pointer to a 64-bit struct :c:type:`timespec` ++ (specified as an integer so that the structure has the same ++ size regardless of architecture). The timeout is specified in ++ absolute format, as measured against the MONOTONIC clock. If ++ the timeout is equal to or earlier than the current time, the ++ function returns immediately without sleeping. If ``timeout`` ++ is zero, i.e. NULL, the function will sleep until an object ++ is signaled, and will not fail with ``ETIMEDOUT``. ++ * - ``objs`` ++ - Pointer to an array of ``count`` 32-bit object identifiers ++ (specified as an integer so that the structure has the same ++ size regardless of architecture). If any identifier is ++ invalid, the function fails with ``EINVAL``. ++ * - ``count`` ++ - Number of object identifiers specified in the ``objs`` array. ++ * - ``owner`` ++ - Mutex owner identifier. If any object in ``objs`` is a mutex, ++ the ioctl will attempt to acquire that mutex on behalf of ++ ``owner``. If ``owner`` is zero, the ioctl fails with ++ ``EINVAL``. ++ * - ``index`` ++ - On success, contains the index (into ``objs``) of the object ++ which was signaled. If ``alert`` was signaled instead, ++ this contains ``count``. ++ * - ``alert`` ++ - Optional event object identifier. If nonzero, this specifies ++ an "alert" event object which, if signaled, will terminate ++ the wait. If nonzero, the identifier must point to a valid ++ event. ++ ++ This function attempts to acquire one of the given objects. If ++ unable to do so, it sleeps until an object becomes signaled, ++ subsequently acquiring it, or the timeout expires. In the latter ++ case the ioctl fails with ``ETIMEDOUT``. The function only acquires ++ one object, even if multiple objects are signaled. ++ ++ A semaphore is considered to be signaled if its count is nonzero, ++ and is acquired by decrementing its count by one. A mutex is ++ considered to be signaled if it is unowned or if its owner matches ++ the ``owner`` argument, and is acquired by incrementing its ++ recursion count by one and setting its owner to the ``owner`` ++ argument. An auto-reset event is acquired by designaling it; a ++ manual-reset event is not affected by acquisition. ++ ++ Acquisition is atomic and totally ordered with respect to other ++ operations on the same object. If two wait operations (with ++ different ``owner`` identifiers) are queued on the same mutex, only ++ one is signaled. If two wait operations are queued on the same ++ semaphore, and a value of one is posted to it, only one is signaled. ++ The order in which threads are signaled is not specified. ++ ++ If an inconsistent mutex is acquired, the ioctl fails with ++ ``EOWNERDEAD``. Although this is a failure return, the function may ++ otherwise be considered successful. The mutex is marked as owned by ++ the given owner (with a recursion count of 1) and as no longer ++ inconsistent, and ``index`` is still set to the index of the mutex. ++ ++ The ``alert`` argument is an "extra" event which can terminate the ++ wait, independently of all other objects. If members of ``objs`` and ++ ``alert`` are both simultaneously signaled, a member of ``objs`` ++ will always be given priority and acquired first. Aside from this, ++ for "any" waits, there is no difference between passing an event as ++ this parameter, and passing it as an additional object at the end of ++ the ``objs`` array. For "all" waits, there is an additional ++ difference, as described below. ++ ++ It is valid to pass the same object more than once, including by ++ passing the same event in the ``objs`` array and in ``alert``. If a ++ wakeup occurs due to that object being signaled, ``index`` is set to ++ the lowest index corresponding to that object. ++ ++ The function may fail with ``EINTR`` if a signal is received. ++ ++.. c:macro:: WINESYNC_IOC_WAIT_ALL ++ ++ Poll on a list of objects, atomically acquiring all of them. Takes a ++ pointer to struct :c:type:`winesync_wait_args`, which is used ++ identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is ++ always filled with zero on success if not woken via alert. ++ ++ This function attempts to simultaneously acquire all of the given ++ objects. If unable to do so, it sleeps until all objects become ++ simultaneously signaled, subsequently acquiring them, or the timeout ++ expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and ++ no objects are modified. ++ ++ Objects may become signaled and subsequently designaled (through ++ acquisition by other threads) while this thread is sleeping. Only ++ once all objects are simultaneously signaled does the ioctl acquire ++ them and return. The entire acquisition is atomic and totally ++ ordered with respect to other operations on any of the given ++ objects. ++ ++ If an inconsistent mutex is acquired, the ioctl fails with ++ ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects ++ are nevertheless marked as acquired. Note that if multiple mutex ++ objects are specified, there is no way to know which were marked as ++ inconsistent. ++ ++ As with "any" waits, the ``alert`` argument is an "extra" event ++ which can terminate the wait. Critically, however, an "all" wait ++ will succeed if all members in ``objs`` are signaled, *or* if ++ ``alert`` is signaled. In the latter case ``index`` will be set to ++ ``count``. As with "any" waits, if both conditions are filled, the ++ former takes priority, and objects in ``objs`` will be acquired. ++ ++ Unlike ``WINESYNC_IOC_WAIT_ANY``, it is not valid to pass the same ++ object more than once, nor is it valid to pass the same object in ++ ``objs`` and in ``alert`` If this is attempted, the function fails ++ with ``EINVAL``. +diff --git a/MAINTAINERS b/MAINTAINERS +index f5ca4aefd184..31a7aa60cdc3 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -21921,6 +21921,15 @@ M: David Härdeman + S: Maintained + F: drivers/media/rc/winbond-cir.c + ++WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER ++M: Zebediah Figura ++L: wine-devel@winehq.org ++S: Supported ++F: Documentation/userspace-api/winesync.rst ++F: drivers/misc/winesync.c ++F: include/uapi/linux/winesync.h ++F: tools/testing/selftests/drivers/winesync/ ++ + WINSYSTEMS EBC-C384 WATCHDOG DRIVER + M: William Breathitt Gray + L: linux-watchdog@vger.kernel.org +diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig +index 94e9fb4cdd76..4f9e3d80a6e8 100644 +--- a/drivers/misc/Kconfig ++++ b/drivers/misc/Kconfig +@@ -496,6 +496,17 @@ config VCPU_STALL_DETECTOR + + If you do not intend to run this kernel as a guest, say N. + ++config WINESYNC ++ tristate "Synchronization primitives for Wine" ++ help ++ This module provides kernel support for synchronization primitives ++ used by Wine. It is not a hardware driver. ++ ++ To compile this driver as a module, choose M here: the ++ module will be called winesync. ++ ++ If unsure, say N. ++ + source "drivers/misc/c2port/Kconfig" + source "drivers/misc/eeprom/Kconfig" + source "drivers/misc/cb710/Kconfig" +diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile +index 2be8542616dd..d061fe45407b 100644 +--- a/drivers/misc/Makefile ++++ b/drivers/misc/Makefile +@@ -58,6 +58,7 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ + obj-$(CONFIG_UACCE) += uacce/ + obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o + obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o ++obj-$(CONFIG_WINESYNC) += winesync.o + obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o + obj-$(CONFIG_OPEN_DICE) += open-dice.o + obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o +\ No newline at end of file +diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c +new file mode 100644 +index 000000000000..7a28f58dbbf2 +--- /dev/null ++++ b/drivers/misc/winesync.c +@@ -0,0 +1,1212 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * winesync.c - Kernel driver for Wine synchronization primitives ++ * ++ * Copyright (C) 2021 Zebediah Figura ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define WINESYNC_NAME "winesync" ++ ++enum winesync_type { ++ WINESYNC_TYPE_SEM, ++ WINESYNC_TYPE_MUTEX, ++ WINESYNC_TYPE_EVENT, ++}; ++ ++struct winesync_obj { ++ struct rcu_head rhead; ++ struct kref refcount; ++ spinlock_t lock; ++ ++ /* ++ * any_waiters is protected by the object lock, but all_waiters is ++ * protected by the device wait_all_lock. ++ */ ++ struct list_head any_waiters; ++ struct list_head all_waiters; ++ ++ /* ++ * Hint describing how many tasks are queued on this object in a ++ * wait-all operation. ++ * ++ * Any time we do a wake, we may need to wake "all" waiters as well as ++ * "any" waiters. In order to atomically wake "all" waiters, we must ++ * lock all of the objects, and that means grabbing the wait_all_lock ++ * below (and, due to lock ordering rules, before locking this object). ++ * However, wait-all is a rare operation, and grabbing the wait-all ++ * lock for every wake would create unnecessary contention. Therefore we ++ * first check whether all_hint is zero, and, if it is, we skip trying ++ * to wake "all" waiters. ++ * ++ * This hint isn't protected by any lock. It might change during the ++ * course of a wake, but there's no meaningful race there; it's only a ++ * hint. ++ * ++ * Since wait requests must originate from user-space threads, we're ++ * limited here by PID_MAX_LIMIT, so there's no risk of saturation. ++ */ ++ atomic_t all_hint; ++ ++ enum winesync_type type; ++ ++ /* The following fields are protected by the object lock. */ ++ union { ++ struct { ++ __u32 count; ++ __u32 max; ++ } sem; ++ struct { ++ __u32 count; ++ __u32 owner; ++ bool ownerdead; ++ } mutex; ++ struct { ++ bool manual; ++ bool signaled; ++ } event; ++ } u; ++}; ++ ++struct winesync_q_entry { ++ struct list_head node; ++ struct winesync_q *q; ++ struct winesync_obj *obj; ++ __u32 index; ++}; ++ ++struct winesync_q { ++ struct task_struct *task; ++ __u32 owner; ++ ++ /* ++ * Protected via atomic_cmpxchg(). Only the thread that wins the ++ * compare-and-swap may actually change object states and wake this ++ * task. ++ */ ++ atomic_t signaled; ++ ++ bool all; ++ bool ownerdead; ++ __u32 count; ++ struct winesync_q_entry entries[]; ++}; ++ ++struct winesync_device { ++ /* ++ * Wait-all operations must atomically grab all objects, and be totally ++ * ordered with respect to each other and wait-any operations. If one ++ * thread is trying to acquire several objects, another thread cannot ++ * touch the object at the same time. ++ * ++ * We achieve this by grabbing multiple object locks at the same time. ++ * However, this creates a lock ordering problem. To solve that problem, ++ * wait_all_lock is taken first whenever multiple objects must be locked ++ * at the same time. ++ */ ++ spinlock_t wait_all_lock; ++ ++ struct xarray objects; ++}; ++ ++static struct winesync_obj *get_obj(struct winesync_device *dev, __u32 id) ++{ ++ struct winesync_obj *obj; ++ ++ rcu_read_lock(); ++ obj = xa_load(&dev->objects, id); ++ if (obj && !kref_get_unless_zero(&obj->refcount)) ++ obj = NULL; ++ rcu_read_unlock(); ++ ++ return obj; ++} ++ ++static void destroy_obj(struct kref *ref) ++{ ++ struct winesync_obj *obj = container_of(ref, struct winesync_obj, refcount); ++ ++ kfree_rcu(obj, rhead); ++} ++ ++static void put_obj(struct winesync_obj *obj) ++{ ++ kref_put(&obj->refcount, destroy_obj); ++} ++ ++static struct winesync_obj *get_obj_typed(struct winesync_device *dev, __u32 id, ++ enum winesync_type type) ++{ ++ struct winesync_obj *obj = get_obj(dev, id); ++ ++ if (obj && obj->type != type) { ++ put_obj(obj); ++ return NULL; ++ } ++ return obj; ++} ++ ++static int winesync_char_open(struct inode *inode, struct file *file) ++{ ++ struct winesync_device *dev; ++ ++ dev = kzalloc(sizeof(*dev), GFP_KERNEL); ++ if (!dev) ++ return -ENOMEM; ++ ++ spin_lock_init(&dev->wait_all_lock); ++ ++ xa_init_flags(&dev->objects, XA_FLAGS_ALLOC); ++ ++ file->private_data = dev; ++ return nonseekable_open(inode, file); ++} ++ ++static int winesync_char_release(struct inode *inode, struct file *file) ++{ ++ struct winesync_device *dev = file->private_data; ++ struct winesync_obj *obj; ++ unsigned long id; ++ ++ xa_for_each(&dev->objects, id, obj) ++ put_obj(obj); ++ ++ xa_destroy(&dev->objects); ++ ++ kfree(dev); ++ ++ return 0; ++} ++ ++static void init_obj(struct winesync_obj *obj) ++{ ++ kref_init(&obj->refcount); ++ atomic_set(&obj->all_hint, 0); ++ spin_lock_init(&obj->lock); ++ INIT_LIST_HEAD(&obj->any_waiters); ++ INIT_LIST_HEAD(&obj->all_waiters); ++} ++ ++static bool is_signaled(struct winesync_obj *obj, __u32 owner) ++{ ++ lockdep_assert_held(&obj->lock); ++ ++ switch (obj->type) { ++ case WINESYNC_TYPE_SEM: ++ return !!obj->u.sem.count; ++ case WINESYNC_TYPE_MUTEX: ++ if (obj->u.mutex.owner && obj->u.mutex.owner != owner) ++ return false; ++ return obj->u.mutex.count < UINT_MAX; ++ case WINESYNC_TYPE_EVENT: ++ return obj->u.event.signaled; ++ } ++ ++ WARN(1, "bad object type %#x\n", obj->type); ++ return false; ++} ++ ++/* ++ * "locked_obj" is an optional pointer to an object which is already locked and ++ * should not be locked again. This is necessary so that changing an object's ++ * state and waking it can be a single atomic operation. ++ */ ++static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, ++ struct winesync_obj *locked_obj) ++{ ++ __u32 count = q->count; ++ bool can_wake = true; ++ __u32 i; ++ ++ lockdep_assert_held(&dev->wait_all_lock); ++ if (locked_obj) ++ lockdep_assert_held(&locked_obj->lock); ++ ++ for (i = 0; i < count; i++) { ++ if (q->entries[i].obj != locked_obj) ++ spin_lock(&q->entries[i].obj->lock); ++ } ++ ++ for (i = 0; i < count; i++) { ++ if (!is_signaled(q->entries[i].obj, q->owner)) { ++ can_wake = false; ++ break; ++ } ++ } ++ ++ if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { ++ for (i = 0; i < count; i++) { ++ struct winesync_obj *obj = q->entries[i].obj; ++ ++ switch (obj->type) { ++ case WINESYNC_TYPE_SEM: ++ obj->u.sem.count--; ++ break; ++ case WINESYNC_TYPE_MUTEX: ++ if (obj->u.mutex.ownerdead) ++ q->ownerdead = true; ++ obj->u.mutex.ownerdead = false; ++ obj->u.mutex.count++; ++ obj->u.mutex.owner = q->owner; ++ break; ++ case WINESYNC_TYPE_EVENT: ++ if (!obj->u.event.manual) ++ obj->u.event.signaled = false; ++ break; ++ } ++ } ++ wake_up_process(q->task); ++ } ++ ++ for (i = 0; i < count; i++) { ++ if (q->entries[i].obj != locked_obj) ++ spin_unlock(&q->entries[i].obj->lock); ++ } ++} ++ ++static void try_wake_all_obj(struct winesync_device *dev, ++ struct winesync_obj *obj) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&dev->wait_all_lock); ++ lockdep_assert_held(&obj->lock); ++ ++ list_for_each_entry(entry, &obj->all_waiters, node) ++ try_wake_all(dev, entry->q, obj); ++} ++ ++static void try_wake_any_sem(struct winesync_obj *sem) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&sem->lock); ++ ++ list_for_each_entry(entry, &sem->any_waiters, node) { ++ struct winesync_q *q = entry->q; ++ ++ if (!sem->u.sem.count) ++ break; ++ ++ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { ++ sem->u.sem.count--; ++ wake_up_process(q->task); ++ } ++ } ++} ++ ++static void try_wake_any_mutex(struct winesync_obj *mutex) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&mutex->lock); ++ ++ list_for_each_entry(entry, &mutex->any_waiters, node) { ++ struct winesync_q *q = entry->q; ++ ++ if (mutex->u.mutex.count == UINT_MAX) ++ break; ++ if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) ++ continue; ++ ++ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { ++ if (mutex->u.mutex.ownerdead) ++ q->ownerdead = true; ++ mutex->u.mutex.ownerdead = false; ++ mutex->u.mutex.count++; ++ mutex->u.mutex.owner = q->owner; ++ wake_up_process(q->task); ++ } ++ } ++} ++ ++static void try_wake_any_event(struct winesync_obj *event) ++{ ++ struct winesync_q_entry *entry; ++ ++ lockdep_assert_held(&event->lock); ++ ++ list_for_each_entry(entry, &event->any_waiters, node) { ++ struct winesync_q *q = entry->q; ++ ++ if (!event->u.event.signaled) ++ break; ++ ++ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { ++ if (!event->u.event.manual) ++ event->u.event.signaled = false; ++ wake_up_process(q->task); ++ } ++ } ++} ++ ++static int winesync_create_sem(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_sem_args __user *user_args = argp; ++ struct winesync_sem_args args; ++ struct winesync_obj *sem; ++ __u32 id; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ if (args.count > args.max) ++ return -EINVAL; ++ ++ sem = kzalloc(sizeof(*sem), GFP_KERNEL); ++ if (!sem) ++ return -ENOMEM; ++ ++ init_obj(sem); ++ sem->type = WINESYNC_TYPE_SEM; ++ sem->u.sem.count = args.count; ++ sem->u.sem.max = args.max; ++ ++ ret = xa_alloc(&dev->objects, &id, sem, xa_limit_32b, GFP_KERNEL); ++ if (ret < 0) { ++ kfree(sem); ++ return ret; ++ } ++ ++ return put_user(id, &user_args->sem); ++} ++ ++static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_mutex_args __user *user_args = argp; ++ struct winesync_mutex_args args; ++ struct winesync_obj *mutex; ++ __u32 id; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ if (!args.owner != !args.count) ++ return -EINVAL; ++ ++ mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); ++ if (!mutex) ++ return -ENOMEM; ++ ++ init_obj(mutex); ++ mutex->type = WINESYNC_TYPE_MUTEX; ++ mutex->u.mutex.count = args.count; ++ mutex->u.mutex.owner = args.owner; ++ ++ ret = xa_alloc(&dev->objects, &id, mutex, xa_limit_32b, GFP_KERNEL); ++ if (ret < 0) { ++ kfree(mutex); ++ return ret; ++ } ++ ++ return put_user(id, &user_args->mutex); ++} ++ ++static int winesync_create_event(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ __u32 id; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ event = kzalloc(sizeof(*event), GFP_KERNEL); ++ if (!event) ++ return -ENOMEM; ++ ++ init_obj(event); ++ event->type = WINESYNC_TYPE_EVENT; ++ event->u.event.manual = args.manual; ++ event->u.event.signaled = args.signaled; ++ ++ ret = xa_alloc(&dev->objects, &id, event, xa_limit_32b, GFP_KERNEL); ++ if (ret < 0) { ++ kfree(event); ++ return ret; ++ } ++ ++ return put_user(id, &user_args->event); ++} ++ ++static int winesync_delete(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_obj *obj; ++ __u32 id; ++ ++ if (get_user(id, (__u32 __user *)argp)) ++ return -EFAULT; ++ ++ obj = xa_erase(&dev->objects, id); ++ if (!obj) ++ return -EINVAL; ++ ++ put_obj(obj); ++ return 0; ++} ++ ++/* ++ * Actually change the semaphore state, returning -EOVERFLOW if it is made ++ * invalid. ++ */ ++static int put_sem_state(struct winesync_obj *sem, __u32 count) ++{ ++ lockdep_assert_held(&sem->lock); ++ ++ if (sem->u.sem.count + count < sem->u.sem.count || ++ sem->u.sem.count + count > sem->u.sem.max) ++ return -EOVERFLOW; ++ ++ sem->u.sem.count += count; ++ return 0; ++} ++ ++static int winesync_put_sem(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_sem_args __user *user_args = argp; ++ struct winesync_sem_args args; ++ struct winesync_obj *sem; ++ __u32 prev_count; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ sem = get_obj_typed(dev, args.sem, WINESYNC_TYPE_SEM); ++ if (!sem) ++ return -EINVAL; ++ ++ if (atomic_read(&sem->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&sem->lock); ++ ++ prev_count = sem->u.sem.count; ++ ret = put_sem_state(sem, args.count); ++ if (!ret) { ++ try_wake_all_obj(dev, sem); ++ try_wake_any_sem(sem); ++ } ++ ++ spin_unlock(&sem->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&sem->lock); ++ ++ prev_count = sem->u.sem.count; ++ ret = put_sem_state(sem, args.count); ++ if (!ret) ++ try_wake_any_sem(sem); ++ ++ spin_unlock(&sem->lock); ++ } ++ ++ put_obj(sem); ++ ++ if (!ret && put_user(prev_count, &user_args->count)) ++ ret = -EFAULT; ++ ++ return ret; ++} ++ ++/* ++ * Actually change the mutex state, returning -EPERM if not the owner. ++ */ ++static int put_mutex_state(struct winesync_obj *mutex, ++ const struct winesync_mutex_args *args) ++{ ++ lockdep_assert_held(&mutex->lock); ++ ++ if (mutex->u.mutex.owner != args->owner) ++ return -EPERM; ++ ++ if (!--mutex->u.mutex.count) ++ mutex->u.mutex.owner = 0; ++ return 0; ++} ++ ++static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_mutex_args __user *user_args = argp; ++ struct winesync_mutex_args args; ++ struct winesync_obj *mutex; ++ __u32 prev_count; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ if (!args.owner) ++ return -EINVAL; ++ ++ mutex = get_obj_typed(dev, args.mutex, WINESYNC_TYPE_MUTEX); ++ if (!mutex) ++ return -EINVAL; ++ ++ if (atomic_read(&mutex->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&mutex->lock); ++ ++ prev_count = mutex->u.mutex.count; ++ ret = put_mutex_state(mutex, &args); ++ if (!ret) { ++ try_wake_all_obj(dev, mutex); ++ try_wake_any_mutex(mutex); ++ } ++ ++ spin_unlock(&mutex->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&mutex->lock); ++ ++ prev_count = mutex->u.mutex.count; ++ ret = put_mutex_state(mutex, &args); ++ if (!ret) ++ try_wake_any_mutex(mutex); ++ ++ spin_unlock(&mutex->lock); ++ } ++ ++ put_obj(mutex); ++ ++ if (!ret && put_user(prev_count, &user_args->count)) ++ ret = -EFAULT; ++ ++ return ret; ++} ++ ++static int winesync_read_sem(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_sem_args __user *user_args = argp; ++ struct winesync_sem_args args; ++ struct winesync_obj *sem; ++ __u32 id; ++ ++ if (get_user(id, &user_args->sem)) ++ return -EFAULT; ++ ++ sem = get_obj_typed(dev, id, WINESYNC_TYPE_SEM); ++ if (!sem) ++ return -EINVAL; ++ ++ args.sem = id; ++ spin_lock(&sem->lock); ++ args.count = sem->u.sem.count; ++ args.max = sem->u.sem.max; ++ spin_unlock(&sem->lock); ++ ++ put_obj(sem); ++ ++ if (copy_to_user(user_args, &args, sizeof(args))) ++ return -EFAULT; ++ return 0; ++} ++ ++static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_mutex_args __user *user_args = argp; ++ struct winesync_mutex_args args; ++ struct winesync_obj *mutex; ++ __u32 id; ++ int ret; ++ ++ if (get_user(id, &user_args->mutex)) ++ return -EFAULT; ++ ++ mutex = get_obj_typed(dev, id, WINESYNC_TYPE_MUTEX); ++ if (!mutex) ++ return -EINVAL; ++ ++ args.mutex = id; ++ spin_lock(&mutex->lock); ++ args.count = mutex->u.mutex.count; ++ args.owner = mutex->u.mutex.owner; ++ ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; ++ spin_unlock(&mutex->lock); ++ ++ put_obj(mutex); ++ ++ if (copy_to_user(user_args, &args, sizeof(args))) ++ return -EFAULT; ++ return ret; ++} ++ ++static int winesync_read_event(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ __u32 id; ++ ++ if (get_user(id, &user_args->event)) ++ return -EFAULT; ++ ++ event = get_obj_typed(dev, id, WINESYNC_TYPE_EVENT); ++ if (!event) ++ return -EINVAL; ++ ++ args.event = id; ++ spin_lock(&event->lock); ++ args.manual = event->u.event.manual; ++ args.signaled = event->u.event.signaled; ++ spin_unlock(&event->lock); ++ ++ put_obj(event); ++ ++ if (copy_to_user(user_args, &args, sizeof(args))) ++ return -EFAULT; ++ return 0; ++} ++ ++/* ++ * Actually change the mutex state to mark its owner as dead. ++ */ ++static void put_mutex_ownerdead_state(struct winesync_obj *mutex) ++{ ++ lockdep_assert_held(&mutex->lock); ++ ++ mutex->u.mutex.ownerdead = true; ++ mutex->u.mutex.owner = 0; ++ mutex->u.mutex.count = 0; ++} ++ ++static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_obj *obj; ++ unsigned long id; ++ __u32 owner; ++ ++ if (get_user(owner, (__u32 __user *)argp)) ++ return -EFAULT; ++ if (!owner) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ ++ xa_for_each(&dev->objects, id, obj) { ++ if (!kref_get_unless_zero(&obj->refcount)) ++ continue; ++ ++ if (obj->type != WINESYNC_TYPE_MUTEX) { ++ put_obj(obj); ++ continue; ++ } ++ ++ if (atomic_read(&obj->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&obj->lock); ++ ++ if (obj->u.mutex.owner == owner) { ++ put_mutex_ownerdead_state(obj); ++ try_wake_all_obj(dev, obj); ++ try_wake_any_mutex(obj); ++ } ++ ++ spin_unlock(&obj->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&obj->lock); ++ ++ if (obj->u.mutex.owner == owner) { ++ put_mutex_ownerdead_state(obj); ++ try_wake_any_mutex(obj); ++ } ++ ++ spin_unlock(&obj->lock); ++ } ++ ++ put_obj(obj); ++ } ++ ++ rcu_read_unlock(); ++ ++ return 0; ++} ++ ++static int winesync_set_event(struct winesync_device *dev, void __user *argp, ++ bool pulse) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ bool prev_state; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); ++ if (!event) ++ return -EINVAL; ++ ++ if (atomic_read(&event->all_hint) > 0) { ++ spin_lock(&dev->wait_all_lock); ++ spin_lock(&event->lock); ++ ++ prev_state = event->u.event.signaled; ++ event->u.event.signaled = true; ++ try_wake_all_obj(dev, event); ++ try_wake_any_event(event); ++ if (pulse) ++ event->u.event.signaled = false; ++ ++ spin_unlock(&event->lock); ++ spin_unlock(&dev->wait_all_lock); ++ } else { ++ spin_lock(&event->lock); ++ ++ prev_state = event->u.event.signaled; ++ event->u.event.signaled = true; ++ try_wake_any_event(event); ++ if (pulse) ++ event->u.event.signaled = false; ++ ++ spin_unlock(&event->lock); ++ } ++ ++ put_obj(event); ++ ++ if (put_user(prev_state, &user_args->signaled)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int winesync_reset_event(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_event_args __user *user_args = argp; ++ struct winesync_event_args args; ++ struct winesync_obj *event; ++ bool prev_state; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); ++ if (!event) ++ return -EINVAL; ++ ++ spin_lock(&event->lock); ++ ++ prev_state = event->u.event.signaled; ++ event->u.event.signaled = false; ++ ++ spin_unlock(&event->lock); ++ ++ put_obj(event); ++ ++ if (put_user(prev_state, &user_args->signaled)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) ++{ ++ int ret = 0; ++ ++ do { ++ if (signal_pending(current)) { ++ ret = -ERESTARTSYS; ++ break; ++ } ++ ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (atomic_read(&q->signaled) != -1) { ++ ret = 0; ++ break; ++ } ++ ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); ++ } while (ret < 0); ++ __set_current_state(TASK_RUNNING); ++ ++ return ret; ++} ++ ++/* ++ * Allocate and initialize the winesync_q structure, but do not queue us yet. ++ * Also, calculate the relative timeout. ++ */ ++static int setup_wait(struct winesync_device *dev, ++ const struct winesync_wait_args *args, bool all, ++ ktime_t *ret_timeout, struct winesync_q **ret_q) ++{ ++ const __u32 count = args->count; ++ struct winesync_q *q; ++ ktime_t timeout = 0; ++ __u32 total_count; ++ __u32 *ids; ++ __u32 i, j; ++ ++ if (!args->owner) ++ return -EINVAL; ++ ++ if (args->timeout) { ++ struct timespec64 to; ++ ++ if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) ++ return -EFAULT; ++ if (!timespec64_valid(&to)) ++ return -EINVAL; ++ ++ timeout = timespec64_to_ns(&to); ++ } ++ ++ total_count = count; ++ if (args->alert) ++ total_count++; ++ ++ ids = kmalloc_array(total_count, sizeof(*ids), GFP_KERNEL); ++ if (!ids) ++ return -ENOMEM; ++ if (copy_from_user(ids, u64_to_user_ptr(args->objs), ++ array_size(count, sizeof(*ids)))) { ++ kfree(ids); ++ return -EFAULT; ++ } ++ if (args->alert) ++ ids[count] = args->alert; ++ ++ q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); ++ if (!q) { ++ kfree(ids); ++ return -ENOMEM; ++ } ++ q->task = current; ++ q->owner = args->owner; ++ atomic_set(&q->signaled, -1); ++ q->all = all; ++ q->ownerdead = false; ++ q->count = count; ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = get_obj(dev, ids[i]); ++ ++ if (!obj) ++ goto err; ++ ++ if (all) { ++ /* Check that the objects are all distinct. */ ++ for (j = 0; j < i; j++) { ++ if (obj == q->entries[j].obj) { ++ put_obj(obj); ++ goto err; ++ } ++ } ++ } ++ ++ entry->obj = obj; ++ entry->q = q; ++ entry->index = i; ++ } ++ ++ kfree(ids); ++ ++ *ret_q = q; ++ *ret_timeout = timeout; ++ return 0; ++ ++err: ++ for (j = 0; j < i; j++) ++ put_obj(q->entries[j].obj); ++ kfree(ids); ++ kfree(q); ++ return -EINVAL; ++} ++ ++static void try_wake_any_obj(struct winesync_obj *obj) ++{ ++ switch (obj->type) { ++ case WINESYNC_TYPE_SEM: ++ try_wake_any_sem(obj); ++ break; ++ case WINESYNC_TYPE_MUTEX: ++ try_wake_any_mutex(obj); ++ break; ++ case WINESYNC_TYPE_EVENT: ++ try_wake_any_event(obj); ++ break; ++ } ++} ++ ++static int winesync_wait_any(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_wait_args args; ++ struct winesync_q *q; ++ __u32 i, total_count; ++ ktime_t timeout; ++ int signaled; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ ret = setup_wait(dev, &args, false, &timeout, &q); ++ if (ret < 0) ++ return ret; ++ ++ total_count = args.count; ++ if (args.alert) ++ total_count++; ++ ++ /* queue ourselves */ ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_add_tail(&entry->node, &obj->any_waiters); ++ spin_unlock(&obj->lock); ++ } ++ ++ /* ++ * Check if we are already signaled. ++ * ++ * Note that the API requires that normal objects are checked before ++ * the alert event. Hence we queue the alert event last, and check ++ * objects in order. ++ */ ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_obj *obj = q->entries[i].obj; ++ ++ if (atomic_read(&q->signaled) != -1) ++ break; ++ ++ spin_lock(&obj->lock); ++ try_wake_any_obj(obj); ++ spin_unlock(&obj->lock); ++ } ++ ++ /* sleep */ ++ ++ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); ++ ++ /* and finally, unqueue */ ++ ++ for (i = 0; i < total_count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_del(&entry->node); ++ spin_unlock(&obj->lock); ++ ++ put_obj(obj); ++ } ++ ++ signaled = atomic_read(&q->signaled); ++ if (signaled != -1) { ++ struct winesync_wait_args __user *user_args = argp; ++ ++ /* even if we caught a signal, we need to communicate success */ ++ ret = q->ownerdead ? -EOWNERDEAD : 0; ++ ++ if (put_user(signaled, &user_args->index)) ++ ret = -EFAULT; ++ } else if (!ret) { ++ ret = -ETIMEDOUT; ++ } ++ ++ kfree(q); ++ return ret; ++} ++ ++static int winesync_wait_all(struct winesync_device *dev, void __user *argp) ++{ ++ struct winesync_wait_args args; ++ struct winesync_q *q; ++ ktime_t timeout; ++ int signaled; ++ __u32 i; ++ int ret; ++ ++ if (copy_from_user(&args, argp, sizeof(args))) ++ return -EFAULT; ++ ++ ret = setup_wait(dev, &args, true, &timeout, &q); ++ if (ret < 0) ++ return ret; ++ ++ /* queue ourselves */ ++ ++ spin_lock(&dev->wait_all_lock); ++ ++ for (i = 0; i < args.count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ atomic_inc(&obj->all_hint); ++ ++ /* ++ * obj->all_waiters is protected by dev->wait_all_lock rather ++ * than obj->lock, so there is no need to acquire it here. ++ */ ++ list_add_tail(&entry->node, &obj->all_waiters); ++ } ++ if (args.alert) { ++ struct winesync_q_entry *entry = &q->entries[args.count]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_add_tail(&entry->node, &obj->any_waiters); ++ spin_unlock(&obj->lock); ++ } ++ ++ /* check if we are already signaled */ ++ ++ try_wake_all(dev, q, NULL); ++ ++ spin_unlock(&dev->wait_all_lock); ++ ++ /* ++ * Check if the alert event is signaled, making sure to do so only ++ * after checking if the other objects are signaled. ++ */ ++ ++ if (args.alert) { ++ struct winesync_obj *obj = q->entries[args.count].obj; ++ ++ if (atomic_read(&q->signaled) == -1) { ++ spin_lock(&obj->lock); ++ try_wake_any_obj(obj); ++ spin_unlock(&obj->lock); ++ } ++ } ++ ++ /* sleep */ ++ ++ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); ++ ++ /* and finally, unqueue */ ++ ++ spin_lock(&dev->wait_all_lock); ++ ++ for (i = 0; i < args.count; i++) { ++ struct winesync_q_entry *entry = &q->entries[i]; ++ struct winesync_obj *obj = entry->obj; ++ ++ /* ++ * obj->all_waiters is protected by dev->wait_all_lock rather ++ * than obj->lock, so there is no need to acquire it here. ++ */ ++ list_del(&entry->node); ++ ++ atomic_dec(&obj->all_hint); ++ ++ put_obj(obj); ++ } ++ if (args.alert) { ++ struct winesync_q_entry *entry = &q->entries[args.count]; ++ struct winesync_obj *obj = entry->obj; ++ ++ spin_lock(&obj->lock); ++ list_del(&entry->node); ++ spin_unlock(&obj->lock); ++ ++ put_obj(obj); ++ } ++ ++ spin_unlock(&dev->wait_all_lock); ++ ++ signaled = atomic_read(&q->signaled); ++ if (signaled != -1) { ++ struct winesync_wait_args __user *user_args = argp; ++ ++ /* even if we caught a signal, we need to communicate success */ ++ ret = q->ownerdead ? -EOWNERDEAD : 0; ++ ++ if (put_user(signaled, &user_args->index)) ++ ret = -EFAULT; ++ } else if (!ret) { ++ ret = -ETIMEDOUT; ++ } ++ ++ kfree(q); ++ return ret; ++} ++ ++static long winesync_char_ioctl(struct file *file, unsigned int cmd, ++ unsigned long parm) ++{ ++ struct winesync_device *dev = file->private_data; ++ void __user *argp = (void __user *)parm; ++ ++ switch (cmd) { ++ case WINESYNC_IOC_CREATE_EVENT: ++ return winesync_create_event(dev, argp); ++ case WINESYNC_IOC_CREATE_MUTEX: ++ return winesync_create_mutex(dev, argp); ++ case WINESYNC_IOC_CREATE_SEM: ++ return winesync_create_sem(dev, argp); ++ case WINESYNC_IOC_DELETE: ++ return winesync_delete(dev, argp); ++ case WINESYNC_IOC_KILL_OWNER: ++ return winesync_kill_owner(dev, argp); ++ case WINESYNC_IOC_PULSE_EVENT: ++ return winesync_set_event(dev, argp, true); ++ case WINESYNC_IOC_PUT_MUTEX: ++ return winesync_put_mutex(dev, argp); ++ case WINESYNC_IOC_PUT_SEM: ++ return winesync_put_sem(dev, argp); ++ case WINESYNC_IOC_READ_EVENT: ++ return winesync_read_event(dev, argp); ++ case WINESYNC_IOC_READ_MUTEX: ++ return winesync_read_mutex(dev, argp); ++ case WINESYNC_IOC_READ_SEM: ++ return winesync_read_sem(dev, argp); ++ case WINESYNC_IOC_RESET_EVENT: ++ return winesync_reset_event(dev, argp); ++ case WINESYNC_IOC_SET_EVENT: ++ return winesync_set_event(dev, argp, false); ++ case WINESYNC_IOC_WAIT_ALL: ++ return winesync_wait_all(dev, argp); ++ case WINESYNC_IOC_WAIT_ANY: ++ return winesync_wait_any(dev, argp); ++ default: ++ return -ENOSYS; ++ } ++} ++ ++static const struct file_operations winesync_fops = { ++ .owner = THIS_MODULE, ++ .open = winesync_char_open, ++ .release = winesync_char_release, ++ .unlocked_ioctl = winesync_char_ioctl, ++ .compat_ioctl = winesync_char_ioctl, ++ .llseek = no_llseek, ++}; ++ ++static struct miscdevice winesync_misc = { ++ .minor = WINESYNC_MINOR, ++ .name = WINESYNC_NAME, ++ .fops = &winesync_fops, ++}; ++ ++static int __init winesync_init(void) ++{ ++ return misc_register(&winesync_misc); ++} ++ ++static void __exit winesync_exit(void) ++{ ++ misc_deregister(&winesync_misc); ++} ++ ++module_init(winesync_init); ++module_exit(winesync_exit); ++ ++MODULE_AUTHOR("Zebediah Figura"); ++MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); ++MODULE_LICENSE("GPL"); ++MODULE_ALIAS("devname:" WINESYNC_NAME); ++MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); +diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h +index c0fea6ca5076..36fc5d5315a4 100644 +--- a/include/linux/miscdevice.h ++++ b/include/linux/miscdevice.h +@@ -71,6 +71,7 @@ + #define USERIO_MINOR 240 + #define VHOST_VSOCK_MINOR 241 + #define RFKILL_MINOR 242 ++#define WINESYNC_MINOR 243 + #define MISC_DYNAMIC_MINOR 255 + + struct device; +diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h +new file mode 100644 +index 000000000000..5b4e369f7469 +--- /dev/null ++++ b/include/uapi/linux/winesync.h +@@ -0,0 +1,71 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* ++ * Kernel support for Wine synchronization primitives ++ * ++ * Copyright (C) 2021 Zebediah Figura ++ */ ++ ++#ifndef __LINUX_WINESYNC_H ++#define __LINUX_WINESYNC_H ++ ++#include ++ ++struct winesync_sem_args { ++ __u32 sem; ++ __u32 count; ++ __u32 max; ++}; ++ ++struct winesync_mutex_args { ++ __u32 mutex; ++ __u32 owner; ++ __u32 count; ++}; ++ ++struct winesync_event_args { ++ __u32 event; ++ __u32 manual; ++ __u32 signaled; ++}; ++ ++struct winesync_wait_args { ++ __u64 timeout; ++ __u64 objs; ++ __u32 count; ++ __u32 owner; ++ __u32 index; ++ __u32 alert; ++}; ++ ++#define WINESYNC_IOC_BASE 0xf7 ++ ++#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ ++ struct winesync_sem_args) ++#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 1, __u32) ++#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 2, \ ++ struct winesync_sem_args) ++#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 3, \ ++ struct winesync_wait_args) ++#define WINESYNC_IOC_WAIT_ALL _IOWR(WINESYNC_IOC_BASE, 4, \ ++ struct winesync_wait_args) ++#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 5, \ ++ struct winesync_mutex_args) ++#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ ++ struct winesync_mutex_args) ++#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 7, __u32) ++#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 8, \ ++ struct winesync_sem_args) ++#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 9, \ ++ struct winesync_mutex_args) ++#define WINESYNC_IOC_CREATE_EVENT _IOWR(WINESYNC_IOC_BASE, 10, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_SET_EVENT _IOWR(WINESYNC_IOC_BASE, 11, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_RESET_EVENT _IOWR(WINESYNC_IOC_BASE, 12, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_PULSE_EVENT _IOWR(WINESYNC_IOC_BASE, 13, \ ++ struct winesync_event_args) ++#define WINESYNC_IOC_READ_EVENT _IOWR(WINESYNC_IOC_BASE, 14, \ ++ struct winesync_event_args) ++ ++#endif +diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile +index 1fc89b8ef433..c7d3d9f5e0a0 100644 +--- a/tools/testing/selftests/Makefile ++++ b/tools/testing/selftests/Makefile +@@ -14,6 +14,7 @@ TARGETS += drivers/dma-buf + TARGETS += drivers/s390x/uvdevice + TARGETS += drivers/net/bonding + TARGETS += drivers/net/team ++TARGETS += drivers/winesync + TARGETS += efivarfs + TARGETS += exec + TARGETS += filesystems +diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile +new file mode 100644 +index 000000000000..43b39fdeea10 +--- /dev/null ++++ b/tools/testing/selftests/drivers/winesync/Makefile +@@ -0,0 +1,8 @@ ++# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only ++TEST_GEN_PROGS := winesync ++ ++top_srcdir =../../../../.. ++CFLAGS += -I$(top_srcdir)/usr/include ++LDLIBS += -lpthread ++ ++include ../../lib.mk +diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config +new file mode 100644 +index 000000000000..60539c826d06 +--- /dev/null ++++ b/tools/testing/selftests/drivers/winesync/config +@@ -0,0 +1 @@ ++CONFIG_WINESYNC=y +diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c +new file mode 100644 +index 000000000000..169e922484b0 +--- /dev/null ++++ b/tools/testing/selftests/drivers/winesync/winesync.c +@@ -0,0 +1,1479 @@ ++// SPDX-License-Identifier: GPL-2.0-or-later ++/* ++ * Various unit tests for the "winesync" synchronization primitive driver. ++ * ++ * Copyright (C) 2021 Zebediah Figura ++ */ ++ ++#define _GNU_SOURCE ++#include ++#include ++#include ++#include ++#include ++#include ++#include "../../kselftest_harness.h" ++ ++static int read_sem_state(int fd, __u32 sem, __u32 *count, __u32 *max) ++{ ++ struct winesync_sem_args args; ++ int ret; ++ ++ args.sem = sem; ++ args.count = 0xdeadbeef; ++ args.max = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &args); ++ *count = args.count; ++ *max = args.max; ++ return ret; ++} ++ ++#define check_sem_state(fd, sem, count, max) \ ++ ({ \ ++ __u32 __count, __max; \ ++ int ret = read_sem_state((fd), (sem), &__count, &__max); \ ++ EXPECT_EQ(0, ret); \ ++ EXPECT_EQ((count), __count); \ ++ EXPECT_EQ((max), __max); \ ++ }) ++ ++static int put_sem(int fd, __u32 sem, __u32 *count) ++{ ++ struct winesync_sem_args args; ++ int ret; ++ ++ args.sem = sem; ++ args.count = *count; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &args); ++ *count = args.count; ++ return ret; ++} ++ ++static int read_mutex_state(int fd, __u32 mutex, __u32 *count, __u32 *owner) ++{ ++ struct winesync_mutex_args args; ++ int ret; ++ ++ args.mutex = mutex; ++ args.count = 0xdeadbeef; ++ args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &args); ++ *count = args.count; ++ *owner = args.owner; ++ return ret; ++} ++ ++#define check_mutex_state(fd, mutex, count, owner) \ ++ ({ \ ++ __u32 __count, __owner; \ ++ int ret = read_mutex_state((fd), (mutex), &__count, &__owner); \ ++ EXPECT_EQ(0, ret); \ ++ EXPECT_EQ((count), __count); \ ++ EXPECT_EQ((owner), __owner); \ ++ }) ++ ++static int put_mutex(int fd, __u32 mutex, __u32 owner, __u32 *count) ++{ ++ struct winesync_mutex_args args; ++ int ret; ++ ++ args.mutex = mutex; ++ args.owner = owner; ++ args.count = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &args); ++ *count = args.count; ++ return ret; ++} ++ ++static int read_event_state(int fd, __u32 event, __u32 *signaled, __u32 *manual) ++{ ++ struct winesync_event_args args; ++ int ret; ++ ++ args.event = event; ++ args.signaled = 0xdeadbeef; ++ args.manual = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &args); ++ *signaled = args.signaled; ++ *manual = args.manual; ++ return ret; ++} ++ ++#define check_event_state(fd, event, signaled, manual) \ ++ ({ \ ++ __u32 __signaled, __manual; \ ++ int ret = read_event_state((fd), (event), \ ++ &__signaled, &__manual); \ ++ EXPECT_EQ(0, ret); \ ++ EXPECT_EQ((signaled), __signaled); \ ++ EXPECT_EQ((manual), __manual); \ ++ }) ++ ++static int wait_objs(int fd, unsigned long request, __u32 count, ++ const __u32 *objs, __u32 owner, __u32 alert, __u32 *index) ++{ ++ struct winesync_wait_args args = {0}; ++ struct timespec timeout; ++ int ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ args.timeout = (uintptr_t)&timeout; ++ args.count = count; ++ args.objs = (uintptr_t)objs; ++ args.owner = owner; ++ args.index = 0xdeadbeef; ++ args.alert = alert; ++ ret = ioctl(fd, request, &args); ++ *index = args.index; ++ return ret; ++} ++ ++static int wait_any(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, ++ count, objs, owner, 0, index); ++} ++ ++static int wait_all(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, ++ count, objs, owner, 0, index); ++} ++ ++static int wait_any_alert(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 alert, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, ++ count, objs, owner, alert, index); ++} ++ ++static int wait_all_alert(int fd, __u32 count, const __u32 *objs, ++ __u32 owner, __u32 alert, __u32 *index) ++{ ++ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, ++ count, objs, owner, alert, index); ++} ++ ++TEST(semaphore_state) ++{ ++ struct winesync_sem_args sem_args; ++ struct timespec timeout; ++ __u32 sem, count, index; ++ int fd, ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 3; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ sem_args.count = 2; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ check_sem_state(fd, sem, 2, 2); ++ ++ count = 0; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ check_sem_state(fd, sem, 2, 2); ++ ++ count = 1; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOVERFLOW, errno); ++ check_sem_state(fd, sem, 2, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem, 1, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem, 0, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ count = 3; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOVERFLOW, errno); ++ check_sem_state(fd, sem, 0, 2); ++ ++ count = 2; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, count); ++ check_sem_state(fd, sem, 2, 2); ++ ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ ret = wait_any(fd, 1, &sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ ++ count = 1; ++ ret = put_sem(fd, sem, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, count); ++ check_sem_state(fd, sem, 1, 2); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(mutex_state) ++{ ++ struct winesync_mutex_args mutex_args; ++ __u32 mutex, owner, count, index; ++ struct timespec timeout; ++ int fd, ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 0; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 2; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ mutex = mutex_args.mutex; ++ check_mutex_state(fd, mutex, 2, 123); ++ ++ ret = put_mutex(fd, mutex, 0, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = put_mutex(fd, mutex, 456, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EPERM, errno); ++ check_mutex_state(fd, mutex, 2, 123); ++ ++ ret = put_mutex(fd, mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ ret = put_mutex(fd, mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, count); ++ check_mutex_state(fd, mutex, 0, 0); ++ ++ ret = put_mutex(fd, mutex, 123, &count); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EPERM, errno); ++ ++ ret = wait_any(fd, 1, &mutex, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 456); ++ ++ ret = wait_any(fd, 1, &mutex, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 2, 456); ++ ++ ret = put_mutex(fd, mutex, 456, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ check_mutex_state(fd, mutex, 1, 456); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ owner = 0; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ check_mutex_state(fd, mutex, 1, 456); ++ ++ owner = 456; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.count = 0xdeadbeef; ++ mutex_args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, mutex_args.count); ++ EXPECT_EQ(0, mutex_args.owner); ++ ++ mutex_args.count = 0xdeadbeef; ++ mutex_args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, mutex_args.count); ++ EXPECT_EQ(0, mutex_args.owner); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.count = 0xdeadbeef; ++ mutex_args.owner = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, mutex_args.count); ++ EXPECT_EQ(0, mutex_args.owner); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 0; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ mutex = mutex_args.mutex; ++ check_mutex_state(fd, mutex, 0, 0); ++ ++ ret = wait_any(fd, 1, &mutex, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_mutex_state(fd, mutex, 1, 123); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(manual_event_state) ++{ ++ struct winesync_event_args event_args; ++ __u32 index; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ event_args.manual = 1; ++ event_args.signaled = 0; ++ event_args.event = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, event_args.event); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(auto_event_state) ++{ ++ struct winesync_event_args event_args; ++ __u32 index; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ event_args.manual = 0; ++ event_args.signaled = 1; ++ event_args.event = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, event_args.event); ++ ++ check_event_state(fd, event_args.event, 1, 0); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 0); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ event_args.signaled = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = wait_any(fd, 1, &event_args.event, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(test_wait_any) ++{ ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ __u32 objs[2], owner, index; ++ struct timespec timeout; ++ int fd, ret; ++ ++ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 2; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 0; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ ret = wait_any(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++ ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ EXPECT_EQ(1, index); ++ ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ ++ /* test waiting on the same object twice */ ++ sem_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ objs[0] = objs[1] = sem_args.sem; ++ ret = wait_any(fd, 2, objs, 456, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, wait_args.index); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ ++ ret = wait_any(fd, 0, NULL, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(test_wait_all) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ __u32 objs[2], owner, index; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 2; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 0; ++ mutex_args.count = 0; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ event_args.manual = true; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ ret = wait_all(fd, 2, objs, 456, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++ ++ sem_args.count = 3; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 2, 3); ++ check_mutex_state(fd, mutex_args.mutex, 3, 123); ++ ++ owner = 123; ++ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EOWNERDEAD, errno); ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = event_args.event; ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ /* test waiting on the same object twice */ ++ objs[0] = objs[1] = sem_args.sem; ++ ret = wait_all(fd, 2, objs, 123, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(invalid_objects) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ __u32 objs[2] = {0}; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ sem_args.max = 1; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ ++ mutex_args.mutex = sem_args.sem; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ event_args.event = sem_args.sem; ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = sem_args.sem + 1; ++ wait_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ objs[0] = sem_args.sem + 1; ++ objs[1] = sem_args.sem; ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ ++ sem_args.sem = mutex_args.mutex; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(EINVAL, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++struct wake_args ++{ ++ int fd; ++ __u32 obj; ++}; ++ ++struct wait_args ++{ ++ int fd; ++ unsigned long request; ++ struct winesync_wait_args *args; ++ int ret; ++ int err; ++}; ++ ++static void *wait_thread(void *arg) ++{ ++ struct wait_args *args = arg; ++ ++ args->ret = ioctl(args->fd, args->request, args->args); ++ args->err = errno; ++ return NULL; ++} ++ ++static void get_abs_timeout(struct timespec *timeout, clockid_t clock, ++ unsigned int ms) ++{ ++ clock_gettime(clock, timeout); ++ timeout->tv_nsec += ms * 1000000; ++ timeout->tv_sec += (timeout->tv_nsec / 1000000000); ++ timeout->tv_nsec %= 1000000000; ++} ++ ++static int wait_for_thread(pthread_t thread, unsigned int ms) ++{ ++ struct timespec timeout; ++ get_abs_timeout(&timeout, CLOCK_REALTIME, ms); ++ return pthread_timedjoin_np(thread, NULL, &timeout); ++} ++ ++TEST(wake_any) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ __u32 objs[2], count, index; ++ struct timespec timeout; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 0; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 1; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ ++ /* test waking the semaphore */ ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 2; ++ wait_args.owner = 456; ++ wait_args.index = 0xdeadbeef; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ANY; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ check_sem_state(fd, sem_args.sem, 0, 3); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(0, wait_args.index); ++ ++ /* test waking the mutex */ ++ ++ /* first grab it again for owner 123 */ ++ ret = wait_any(fd, 1, &mutex_args.mutex, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.owner = 456; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = put_mutex(fd, mutex_args.mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, count); ++ ++ ret = pthread_tryjoin_np(thread, NULL); ++ EXPECT_EQ(EBUSY, ret); ++ ++ ret = put_mutex(fd, mutex_args.mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, mutex_args.count); ++ check_mutex_state(fd, mutex_args.mutex, 1, 456); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ /* test waking events */ ++ ++ event_args.manual = false; ++ event_args.signaled = false; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[1] = event_args.event; ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 0); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ event_args.manual = true; ++ event_args.signaled = false; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[1] = event_args.event; ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 1, 1); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, event_args.signaled); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, event_args.signaled); ++ check_event_state(fd, event_args.event, 0, 1); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(1, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ /* delete an object while it's being waited on */ ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); ++ wait_args.owner = 123; ++ objs[1] = mutex_args.mutex; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 200); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(-1, thread_args.ret); ++ EXPECT_EQ(ETIMEDOUT, thread_args.err); ++ ++ close(fd); ++} ++ ++TEST(wake_all) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ const struct net *net = sock_net(sk); ++ struct winesync_event_args manual_event_args = {0}; ++ struct winesync_event_args auto_event_args = {0}; ++ struct winesync_mutex_args mutex_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ __u32 objs[4], count, index; ++ struct timespec timeout; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 0; ++ sem_args.max = 3; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ ++ mutex_args.owner = 123; ++ mutex_args.count = 1; ++ mutex_args.mutex = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++ ++ manual_event_args.manual = true; ++ manual_event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &manual_event_args); ++ EXPECT_EQ(0, ret); ++ ++ auto_event_args.manual = false; ++ auto_event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &auto_event_args); ++ EXPECT_EQ(0, ret); ++ ++ objs[0] = sem_args.sem; ++ objs[1] = mutex_args.mutex; ++ objs[2] = manual_event_args.event; ++ objs[3] = auto_event_args.event; ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 4; ++ wait_args.owner = 456; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ALL; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ ++ ret = pthread_tryjoin_np(thread, NULL); ++ EXPECT_EQ(EBUSY, ret); ++ ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ ++ ret = wait_any(fd, 1, &sem_args.sem, 123, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = put_mutex(fd, mutex_args.mutex, 123, &count); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, count); ++ ++ ret = pthread_tryjoin_np(thread, NULL); ++ EXPECT_EQ(EBUSY, ret); ++ ++ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &manual_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, manual_event_args.signaled); ++ ++ sem_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, sem_args.count); ++ check_sem_state(fd, sem_args.sem, 2, 3); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &auto_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, auto_event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &manual_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, manual_event_args.signaled); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &auto_event_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, auto_event_args.signaled); ++ ++ check_sem_state(fd, sem_args.sem, 1, 3); ++ check_mutex_state(fd, mutex_args.mutex, 1, 456); ++ check_event_state(fd, manual_event_args.event, 1, 1); ++ check_event_state(fd, auto_event_args.event, 0, 0); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ ++ /* delete an object while it's being waited on */ ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); ++ wait_args.owner = 123; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &manual_event_args.event); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &auto_event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 200); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(-1, thread_args.ret); ++ EXPECT_EQ(ETIMEDOUT, thread_args.err); ++ ++ close(fd); ++} ++ ++TEST(alert_any) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ struct timespec timeout; ++ __u32 objs[2], index; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 0; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[0] = sem_args.sem; ++ ++ sem_args.count = 1; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[1] = sem_args.sem; ++ ++ event_args.manual = true; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(1, index); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ /* test wakeup via alert */ ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 2; ++ wait_args.owner = 123; ++ wait_args.index = 0xdeadbeef; ++ wait_args.alert = event_args.event; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ANY; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(2, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ /* test with an auto-reset event */ ++ ++ event_args.manual = false; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ sem_args.sem = objs[0]; ++ sem_args.count = 1; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST(alert_all) ++{ ++ struct winesync_event_args event_args = {0}; ++ struct winesync_wait_args wait_args = {0}; ++ struct winesync_sem_args sem_args = {0}; ++ struct wait_args thread_args; ++ struct timespec timeout; ++ __u32 objs[2], index; ++ pthread_t thread; ++ int fd, ret; ++ ++ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); ++ ASSERT_LE(0, fd); ++ ++ sem_args.count = 2; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[0] = sem_args.sem; ++ ++ sem_args.count = 1; ++ sem_args.max = 2; ++ sem_args.sem = 0xdeadbeef; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ objs[1] = sem_args.sem; ++ ++ event_args.manual = true; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ /* test wakeup via alert */ ++ ++ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); ++ wait_args.timeout = (uintptr_t)&timeout; ++ wait_args.objs = (uintptr_t)objs; ++ wait_args.count = 2; ++ wait_args.owner = 123; ++ wait_args.index = 0xdeadbeef; ++ wait_args.alert = event_args.event; ++ thread_args.fd = fd; ++ thread_args.args = &wait_args; ++ thread_args.request = WINESYNC_IOC_WAIT_ALL; ++ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(ETIMEDOUT, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_for_thread(thread, 100); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, thread_args.ret); ++ EXPECT_EQ(2, wait_args.index); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ /* test with an auto-reset event */ ++ ++ event_args.manual = false; ++ event_args.signaled = true; ++ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); ++ EXPECT_EQ(0, ret); ++ ++ sem_args.sem = objs[1]; ++ sem_args.count = 2; ++ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); ++ EXPECT_EQ(0, ret); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(0, index); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(0, ret); ++ EXPECT_EQ(2, index); ++ ++ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); ++ EXPECT_EQ(-1, ret); ++ EXPECT_EQ(ETIMEDOUT, errno); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); ++ EXPECT_EQ(0, ret); ++ ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); ++ EXPECT_EQ(0, ret); ++ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); ++ EXPECT_EQ(0, ret); ++ ++ close(fd); ++} ++ ++TEST_HARNESS_MAIN +-- +2.38.0.rc1.8.g2a7d63a245 + +From 0905ce4d17bc19b8ec54ef87ed8f42e365a2bcc2 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Fri, 5 Aug 2022 19:33:47 +0200 +Subject: [PATCH 04/16] Introducing-OpenVPN-Data-Channel-Offload + +Signed-off-by: Peter Jung +--- + MAINTAINERS | 8 + + drivers/net/Kconfig | 19 + + drivers/net/Makefile | 1 + + drivers/net/ovpn-dco/Makefile | 21 + + drivers/net/ovpn-dco/addr.h | 41 + + drivers/net/ovpn-dco/bind.c | 62 ++ + drivers/net/ovpn-dco/bind.h | 67 ++ + drivers/net/ovpn-dco/crypto.c | 154 ++++ + drivers/net/ovpn-dco/crypto.h | 144 ++++ + drivers/net/ovpn-dco/crypto_aead.c | 367 +++++++++ + drivers/net/ovpn-dco/crypto_aead.h | 27 + + drivers/net/ovpn-dco/main.c | 271 +++++++ + drivers/net/ovpn-dco/main.h | 32 + + drivers/net/ovpn-dco/netlink.c | 1143 ++++++++++++++++++++++++++++ + drivers/net/ovpn-dco/netlink.h | 22 + + drivers/net/ovpn-dco/ovpn.c | 600 +++++++++++++++ + drivers/net/ovpn-dco/ovpn.h | 43 ++ + drivers/net/ovpn-dco/ovpnstruct.h | 59 ++ + drivers/net/ovpn-dco/peer.c | 906 ++++++++++++++++++++++ + drivers/net/ovpn-dco/peer.h | 168 ++++ + drivers/net/ovpn-dco/pktid.c | 127 ++++ + drivers/net/ovpn-dco/pktid.h | 116 +++ + drivers/net/ovpn-dco/proto.h | 101 +++ + drivers/net/ovpn-dco/rcu.h | 21 + + drivers/net/ovpn-dco/skb.h | 54 ++ + drivers/net/ovpn-dco/sock.c | 134 ++++ + drivers/net/ovpn-dco/sock.h | 54 ++ + drivers/net/ovpn-dco/stats.c | 20 + + drivers/net/ovpn-dco/stats.h | 67 ++ + drivers/net/ovpn-dco/tcp.c | 326 ++++++++ + drivers/net/ovpn-dco/tcp.h | 38 + + drivers/net/ovpn-dco/udp.c | 343 +++++++++ + drivers/net/ovpn-dco/udp.h | 25 + + include/net/netlink.h | 1 + + include/uapi/linux/ovpn_dco.h | 265 +++++++ + include/uapi/linux/udp.h | 1 + + 36 files changed, 5848 insertions(+) + create mode 100644 drivers/net/ovpn-dco/Makefile + create mode 100644 drivers/net/ovpn-dco/addr.h + create mode 100644 drivers/net/ovpn-dco/bind.c + create mode 100644 drivers/net/ovpn-dco/bind.h + create mode 100644 drivers/net/ovpn-dco/crypto.c + create mode 100644 drivers/net/ovpn-dco/crypto.h + create mode 100644 drivers/net/ovpn-dco/crypto_aead.c + create mode 100644 drivers/net/ovpn-dco/crypto_aead.h + create mode 100644 drivers/net/ovpn-dco/main.c + create mode 100644 drivers/net/ovpn-dco/main.h + create mode 100644 drivers/net/ovpn-dco/netlink.c + create mode 100644 drivers/net/ovpn-dco/netlink.h + create mode 100644 drivers/net/ovpn-dco/ovpn.c + create mode 100644 drivers/net/ovpn-dco/ovpn.h + create mode 100644 drivers/net/ovpn-dco/ovpnstruct.h + create mode 100644 drivers/net/ovpn-dco/peer.c + create mode 100644 drivers/net/ovpn-dco/peer.h + create mode 100644 drivers/net/ovpn-dco/pktid.c + create mode 100644 drivers/net/ovpn-dco/pktid.h + create mode 100644 drivers/net/ovpn-dco/proto.h + create mode 100644 drivers/net/ovpn-dco/rcu.h + create mode 100644 drivers/net/ovpn-dco/skb.h + create mode 100644 drivers/net/ovpn-dco/sock.c + create mode 100644 drivers/net/ovpn-dco/sock.h + create mode 100644 drivers/net/ovpn-dco/stats.c + create mode 100644 drivers/net/ovpn-dco/stats.h + create mode 100644 drivers/net/ovpn-dco/tcp.c + create mode 100644 drivers/net/ovpn-dco/tcp.h + create mode 100644 drivers/net/ovpn-dco/udp.c + create mode 100644 drivers/net/ovpn-dco/udp.h + create mode 100644 include/uapi/linux/ovpn_dco.h + +diff --git a/MAINTAINERS b/MAINTAINERS +index 31a7aa60cdc3..a29c9731350c 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -15320,6 +15320,14 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git + F: Documentation/filesystems/overlayfs.rst + F: fs/overlayfs/ + ++OVPN-DCO NETWORK DRIVER ++M: Antonio Quartulli ++L: openvpn-devel@lists.sourceforge.net (moderated for non-subscribers) ++L: netdev@vger.kernel.org ++S: Maintained ++F: drivers/net/ovpn-dco/ ++F: include/uapi/linux/ovpn_dco.h + -+ bbr_init(sk); /* run shared init code for v1 and v2 */ + P54 WIRELESS DRIVER + M: Christian Lamparter + L: linux-wireless@vger.kernel.org +diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig +index 94c889802566..349866bd4448 100644 +--- a/drivers/net/Kconfig ++++ b/drivers/net/Kconfig +@@ -116,6 +116,25 @@ config WIREGUARD_DEBUG + + Say N here unless you know what you're doing. + ++config OVPN_DCO ++ tristate "OpenVPN data channel offload" ++ depends on NET && INET ++ select NET_UDP_TUNNEL ++ select DST_CACHE ++ select CRYPTO ++ select CRYPTO_AES ++ select CRYPTO_GCM ++ select CRYPTO_CHACHA20POLY1305 ++ help ++ This module enhances the performance of an OpenVPN connection by ++ allowing the user to offload the data channel processing to ++ kernelspace. ++ Connection handshake, parameters negotiation and other non-data ++ related mechanisms are still performed in userspace. + -+ /* BBR v2 parameters: */ -+ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta); -+ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain); -+ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init); -+ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor); -+ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh); -+ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us); -+ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain); -+ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh); -+ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt); -+ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt); -+ bbr->params.inflight_headroom = -+ min_t(u32, 0xFFU, bbr_inflight_headroom); -+ bbr->params.bw_probe_pif_gain = -+ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain); -+ bbr->params.bw_probe_reno_gain = -+ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain); -+ bbr->params.bw_probe_max_rounds = -+ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds); -+ bbr->params.bw_probe_rand_rounds = -+ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds); -+ bbr->params.bw_probe_base_us = -+ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us); -+ bbr->params.bw_probe_rand_us = -+ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us); -+ bbr->params.undo = bbr_undo; -+ bbr->params.fast_path = bbr_fast_path ? 1 : 0; -+ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc); ++ The OpenVPN userspace software at version 2.6 or higher is required ++ to use this functionality. + -+ /* BBR v2 state: */ -+ bbr->initialized = 1; -+ /* Start sampling ECN mark rate after first full flight is ACKed: */ -+ bbr->loss_round_delivered = tp->delivered + 1; -+ bbr->loss_round_start = 0; -+ bbr->undo_bw_lo = 0; -+ bbr->undo_inflight_lo = 0; -+ bbr->undo_inflight_hi = 0; -+ bbr->loss_events_in_round = 0; -+ bbr->startup_ecn_rounds = 0; -+ bbr2_reset_congestion_signals(sk); -+ bbr->bw_lo = ~0U; -+ bbr->bw_hi[0] = 0; -+ bbr->bw_hi[1] = 0; -+ bbr->inflight_lo = ~0U; -+ bbr->inflight_hi = ~0U; -+ bbr->bw_probe_up_cnt = ~0U; -+ bbr->bw_probe_up_acks = 0; -+ bbr->bw_probe_up_rounds = 0; -+ bbr->probe_wait_us = 0; -+ bbr->stopped_risky_probe = 0; -+ bbr->ack_phase = BBR_ACKS_INIT; -+ bbr->rounds_since_probe = 0; -+ bbr->bw_probe_samples = 0; -+ bbr->prev_probe_too_high = 0; -+ bbr->ecn_eligible = 0; -+ bbr->ecn_alpha = bbr->params.ecn_alpha_init; -+ bbr->alpha_last_delivered = 0; -+ bbr->alpha_last_delivered_ce = 0; + config EQUALIZER + tristate "EQL (serial line load balancing) support" + help +diff --git a/drivers/net/Makefile b/drivers/net/Makefile +index 3f1192d3c52d..8ed151e8d233 100644 +--- a/drivers/net/Makefile ++++ b/drivers/net/Makefile +@@ -11,6 +11,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/ + obj-$(CONFIG_IPVTAP) += ipvlan/ + obj-$(CONFIG_DUMMY) += dummy.o + obj-$(CONFIG_WIREGUARD) += wireguard/ ++obj-$(CONFIG_OVPN_DCO) += ovpn-dco/ + obj-$(CONFIG_EQUALIZER) += eql.o + obj-$(CONFIG_IFB) += ifb.o + obj-$(CONFIG_MACSEC) += macsec.o +diff --git a/drivers/net/ovpn-dco/Makefile b/drivers/net/ovpn-dco/Makefile +new file mode 100644 +index 000000000000..7efefe8f13a9 +--- /dev/null ++++ b/drivers/net/ovpn-dco/Makefile +@@ -0,0 +1,21 @@ ++# SPDX-License-Identifier: GPL-2.0 ++# ++# ovpn-dco -- OpenVPN data channel offload ++# ++# Copyright (C) 2020-2022 OpenVPN, Inc. ++# ++# Author: Antonio Quartulli + -+ bbr->plb.enabled = 0; -+ bbr->plb.consec_cong_rounds = 0; -+ bbr->plb.pause_until = 0; -+ if ((tp->ecn_flags & TCP_ECN_OK) && -+ net->ipv4.sysctl_tcp_plb_enabled) -+ bbr->plb.enabled = 1; ++obj-$(CONFIG_OVPN_DCO) += ovpn-dco.o ++ovpn-dco-y += main.o ++ovpn-dco-y += bind.o ++ovpn-dco-y += crypto.o ++ovpn-dco-y += ovpn.o ++ovpn-dco-y += peer.o ++ovpn-dco-y += sock.o ++ovpn-dco-y += stats.o ++ovpn-dco-y += netlink.o ++ovpn-dco-y += crypto_aead.o ++ovpn-dco-y += pktid.o ++ovpn-dco-y += tcp.o ++ovpn-dco-y += udp.o +diff --git a/drivers/net/ovpn-dco/addr.h b/drivers/net/ovpn-dco/addr.h +new file mode 100644 +index 000000000000..3d6ad0fc15af +--- /dev/null ++++ b/drivers/net/ovpn-dco/addr.h +@@ -0,0 +1,41 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode); ++#ifndef _NET_OVPN_DCO_OVPNADDR_H_ ++#define _NET_OVPN_DCO_OVPNADDR_H_ + -+ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable) -+ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; -+} ++#include "crypto.h" + -+/* Core TCP stack informs us that the given skb was just marked lost. */ -+static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ struct tcp_skb_cb *scb = TCP_SKB_CB(skb); -+ struct rate_sample rs; ++#include ++#include ++#include ++#include + -+ /* Capture "current" data over the full round trip of loss, -+ * to have a better chance to see the full capacity of the path. -+ */ -+ if (!bbr->loss_in_round) /* first loss in this round trip? */ -+ bbr->loss_round_delivered = tp->delivered; /* set round trip */ -+ bbr->loss_in_round = 1; -+ bbr->loss_in_cycle = 1; ++/* our basic transport layer address */ ++struct ovpn_sockaddr { ++ union { ++ struct sockaddr_in in4; ++ struct sockaddr_in6 in6; ++ }; ++}; + -+ if (!bbr->bw_probe_samples) -+ return; /* not an skb sent while probing for bandwidth */ -+ if (unlikely(!scb->tx.delivered_mstamp)) -+ return; /* skb was SACKed, reneged, marked lost; ignore it */ -+ /* We are probing for bandwidth. Construct a rate sample that -+ * estimates what happened in the flight leading up to this lost skb, -+ * then see if the loss rate went too high, and if so at which packet. -+ */ -+ memset(&rs, 0, sizeof(rs)); -+ rs.tx_in_flight = scb->tx.in_flight; -+ rs.lost = tp->lost - scb->tx.lost; -+ rs.is_app_limited = scb->tx.is_app_limited; -+ if (bbr2_is_inflight_too_high(sk, &rs)) { -+ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb); -+ bbr2_handle_inflight_too_high(sk, &rs); ++/* Translate skb->protocol value to AF_INET or AF_INET6 */ ++static inline unsigned short skb_protocol_to_family(const struct sk_buff *skb) ++{ ++ switch (skb->protocol) { ++ case htons(ETH_P_IP): ++ return AF_INET; ++ case htons(ETH_P_IPV6): ++ return AF_INET6; ++ default: ++ return 0; + } +} + -+/* Revert short-term model if current loss recovery event was spurious. */ -+static u32 bbr2_undo_cwnd(struct sock *sk) ++#endif /* _NET_OVPN_DCO_OVPNADDR_H_ */ +diff --git a/drivers/net/ovpn-dco/bind.c b/drivers/net/ovpn-dco/bind.c +new file mode 100644 +index 000000000000..107697ea983e +--- /dev/null ++++ b/drivers/net/ovpn-dco/bind.c +@@ -0,0 +1,62 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2012-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "ovpn.h" ++#include "bind.h" ++#include "peer.h" ++ ++#include ++#include ++#include ++#include ++ ++/* Given a remote sockaddr, compute the skb hash ++ * and get a dst_entry so we can send packets to the remote. ++ * Called from process context or softirq (must be indicated with ++ * process_context bool). ++ */ ++struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *ss) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct ovpn_bind *bind; ++ size_t sa_len; + -+ bbr->debug.undo = 1; -+ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ -+ bbr->full_bw_cnt = 0; -+ bbr->loss_in_round = 0; ++ if (ss->ss_family == AF_INET) ++ sa_len = sizeof(struct sockaddr_in); ++ else if (ss->ss_family == AF_INET6) ++ sa_len = sizeof(struct sockaddr_in6); ++ else ++ return ERR_PTR(-EAFNOSUPPORT); + -+ if (!bbr->params.undo) -+ return tp->snd_cwnd; ++ bind = kzalloc(sizeof(*bind), GFP_ATOMIC); ++ if (unlikely(!bind)) ++ return ERR_PTR(-ENOMEM); + -+ /* Revert to cwnd and other state saved before loss episode. */ -+ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); -+ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); -+ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); -+ return bbr->prior_cwnd; ++ memcpy(&bind->sa, ss, sa_len); ++ ++ return bind; +} + -+/* Entering loss recovery, so save state for when we undo recovery. */ -+static u32 bbr2_ssthresh(struct sock *sk) ++static void ovpn_bind_release_rcu(struct rcu_head *head) +{ -+ struct bbr *bbr = inet_csk_ca(sk); ++ struct ovpn_bind *bind = container_of(head, struct ovpn_bind, rcu); + -+ bbr_save_cwnd(sk); -+ /* For undo, save state that adapts based on loss signal. */ -+ bbr->undo_bw_lo = bbr->bw_lo; -+ bbr->undo_inflight_lo = bbr->inflight_lo; -+ bbr->undo_inflight_hi = bbr->inflight_hi; -+ return tcp_sk(sk)->snd_ssthresh; ++ kfree(bind); +} + -+static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr) ++void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *new) +{ -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ return BBR2_PHASE_STARTUP; -+ case BBR_DRAIN: -+ return BBR2_PHASE_DRAIN; -+ case BBR_PROBE_BW: -+ break; -+ case BBR_PROBE_RTT: -+ return BBR2_PHASE_PROBE_RTT; -+ default: -+ return BBR2_PHASE_INVALID; -+ } -+ switch (bbr->cycle_idx) { -+ case BBR_BW_PROBE_UP: -+ return BBR2_PHASE_PROBE_BW_UP; -+ case BBR_BW_PROBE_DOWN: -+ return BBR2_PHASE_PROBE_BW_DOWN; -+ case BBR_BW_PROBE_CRUISE: -+ return BBR2_PHASE_PROBE_BW_CRUISE; -+ case BBR_BW_PROBE_REFILL: -+ return BBR2_PHASE_PROBE_BW_REFILL; -+ default: -+ return BBR2_PHASE_INVALID; -+ } -+} ++ struct ovpn_bind *old; + -+static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr, -+ union tcp_cc_info *info) -+{ -+ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || -+ ext & (1 << (INET_DIAG_VEGASINFO - 1))) { -+ struct bbr *bbr = inet_csk_ca(sk); -+ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); -+ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); -+ u64 bw_lo = bbr->bw_lo == ~0U ? -+ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); ++ spin_lock_bh(&peer->lock); ++ old = rcu_replace_pointer(peer->bind, new, true); ++ spin_unlock_bh(&peer->lock); + -+ memset(&info->bbr2, 0, sizeof(info->bbr2)); -+ info->bbr2.bbr_bw_lsb = (u32)bw; -+ info->bbr2.bbr_bw_msb = (u32)(bw >> 32); -+ info->bbr2.bbr_min_rtt = bbr->min_rtt_us; -+ info->bbr2.bbr_pacing_gain = bbr->pacing_gain; -+ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain; -+ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi; -+ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32); -+ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo; -+ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32); -+ info->bbr2.bbr_mode = bbr->mode; -+ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr); -+ info->bbr2.bbr_version = (__u8)2; -+ info->bbr2.bbr_inflight_lo = bbr->inflight_lo; -+ info->bbr2.bbr_inflight_hi = bbr->inflight_hi; -+ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk); -+ *attr = INET_DIAG_BBRINFO; -+ return sizeof(info->bbr2); -+ } -+ return 0; ++ if (old) ++ call_rcu(&old->rcu, ovpn_bind_release_rcu); +} +diff --git a/drivers/net/ovpn-dco/bind.h b/drivers/net/ovpn-dco/bind.h +new file mode 100644 +index 000000000000..a562e471acae +--- /dev/null ++++ b/drivers/net/ovpn-dco/bind.h +@@ -0,0 +1,67 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OVPN -- OpenVPN protocol accelerator for Linux ++ * Copyright (C) 2012-2022 OpenVPN, Inc. ++ * All rights reserved. ++ * Author: James Yonan ++ */ + -+static void bbr2_set_state(struct sock *sk, u8 new_state) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); ++#ifndef _NET_OVPN_DCO_OVPNBIND_H_ ++#define _NET_OVPN_DCO_OVPNBIND_H_ + -+ if (new_state == TCP_CA_Loss) { -+ struct rate_sample rs = { .losses = 1 }; -+ struct bbr_context ctx = { 0 }; ++#include "addr.h" ++#include "rcu.h" + -+ tcp_plb_update_state_upon_rto(sk, &bbr->plb); -+ bbr->prev_ca_state = TCP_CA_Loss; -+ bbr->full_bw = 0; -+ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { -+ /* bbr_adapt_lower_bounds() needs cwnd before -+ * we suffered an RTO, to update inflight_lo: -+ */ -+ bbr->inflight_lo = -+ max(tp->snd_cwnd, bbr->prior_cwnd); -+ } -+ bbr_debug(sk, 0, &rs, &ctx); -+ } else if (bbr->prev_ca_state == TCP_CA_Loss && -+ new_state != TCP_CA_Loss) { -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr->try_fast_path = 0; /* bound cwnd using latest model */ -+ } -+} ++#include ++#include ++#include + -+static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = { -+ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, -+ .name = "bbr2", -+ .owner = THIS_MODULE, -+ .init = bbr2_init, -+ .cong_control = bbr2_main, -+ .sndbuf_expand = bbr_sndbuf_expand, -+ .skb_marked_lost = bbr2_skb_marked_lost, -+ .undo_cwnd = bbr2_undo_cwnd, -+ .cwnd_event = bbr_cwnd_event, -+ .ssthresh = bbr2_ssthresh, -+ .tso_segs = bbr_tso_segs, -+ .get_info = bbr2_get_info, -+ .set_state = bbr2_set_state, -+}; ++struct ovpn_peer; + -+static int __init bbr_register(void) -+{ -+ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); -+ return tcp_register_congestion_control(&tcp_bbr2_cong_ops); -+} ++struct ovpn_bind { ++ struct ovpn_sockaddr sa; /* remote sockaddr */ + -+static void __exit bbr_unregister(void) ++ union { ++ struct in_addr ipv4; ++ struct in6_addr ipv6; ++ } local; ++ ++ struct rcu_head rcu; ++}; ++ ++static inline bool ovpn_bind_skb_src_match(const struct ovpn_bind *bind, struct sk_buff *skb) +{ -+ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops); -+} ++ const unsigned short family = skb_protocol_to_family(skb); ++ const struct ovpn_sockaddr *sa = &bind->sa; + -+module_init(bbr_register); -+module_exit(bbr_unregister); ++ if (unlikely(!bind)) ++ return false; + -+MODULE_AUTHOR("Van Jacobson "); -+MODULE_AUTHOR("Neal Cardwell "); -+MODULE_AUTHOR("Yuchung Cheng "); -+MODULE_AUTHOR("Soheil Hassas Yeganeh "); -+MODULE_AUTHOR("Priyaranjan Jha "); -+MODULE_AUTHOR("Yousuk Seung "); -+MODULE_AUTHOR("Kevin Yang "); -+MODULE_AUTHOR("Arjun Roy "); ++ if (unlikely(sa->in4.sin_family != family)) ++ return false; + -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); -diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c -index d3cae40749e8..0f268f2ff2e9 100644 ---- a/net/ipv4/tcp_cong.c -+++ b/net/ipv4/tcp_cong.c -@@ -189,6 +189,7 @@ void tcp_init_congestion_control(struct sock *sk) - struct inet_connection_sock *icsk = inet_csk(sk); - - tcp_sk(sk)->prior_ssthresh = 0; -+ tcp_sk(sk)->fast_ack_mode = 0; - if (icsk->icsk_ca_ops->init) - icsk->icsk_ca_ops->init(sk); - if (tcp_ca_needs_ecn(sk)) -diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c -index bc2ea12221f9..fa5c3e5d9c85 100644 ---- a/net/ipv4/tcp_input.c -+++ b/net/ipv4/tcp_input.c -@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) - tcp_enter_quickack_mode(sk, 2); - break; - case INET_ECN_CE: -- if (tcp_ca_needs_ecn(sk)) -+ if (tcp_ca_wants_ce_events(sk)) - tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); - - if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { -@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb) - tp->ecn_flags |= TCP_ECN_SEEN; - break; - default: -- if (tcp_ca_needs_ecn(sk)) -+ if (tcp_ca_wants_ce_events(sk)) - tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); - tp->ecn_flags |= TCP_ECN_SEEN; - break; -@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) - */ - static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) - { -+ struct sock *sk = (struct sock *)tp; -+ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; ++ switch (family) { ++ case AF_INET: ++ if (unlikely(sa->in4.sin_addr.s_addr != ip_hdr(skb)->saddr)) ++ return false; + - tp->lost += tcp_skb_pcount(skb); -+ if (ca_ops->skb_marked_lost) -+ ca_ops->skb_marked_lost(sk, skb); - } - - void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) -@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, - WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); - tcp_skb_pcount_add(skb, -pcount); - -+ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ -+ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, -+ "prev in_flight: %u skb in_flight: %u pcount: %u", -+ TCP_SKB_CB(prev)->tx.in_flight, -+ TCP_SKB_CB(skb)->tx.in_flight, -+ pcount)) -+ TCP_SKB_CB(skb)->tx.in_flight = 0; -+ else -+ TCP_SKB_CB(skb)->tx.in_flight -= pcount; -+ TCP_SKB_CB(prev)->tx.in_flight += pcount; ++ if (unlikely(sa->in4.sin_port != udp_hdr(skb)->source)) ++ return false; ++ break; ++ case AF_INET6: ++ if (unlikely(!ipv6_addr_equal(&sa->in6.sin6_addr, &ipv6_hdr(skb)->saddr))) ++ return false; + - /* When we're adding to gso_segs == 1, gso_size will be zero, - * in theory this shouldn't be necessary but as long as DSACK - * code can come after this skb later on it's better to keep -@@ -3811,6 +3827,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) - - prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; - rs.prior_in_flight = tcp_packets_in_flight(tp); -+ tcp_rate_check_app_limited(sk); - - /* ts_recent update must be made after we are sure that the packet - * is in window. -@@ -3909,6 +3926,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) - delivered = tcp_newly_delivered(sk, delivered, flag); - lost = tp->lost - lost; /* freshly marked lost */ - rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); -+ rs.is_ece = !!(flag & FLAG_ECE); - tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); - tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); - tcp_xmit_recovery(sk, rexmit); -@@ -5508,13 +5526,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) - - /* More than one full frame received... */ - if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && -+ (tp->fast_ack_mode == 1 || - /* ... and right edge of window advances far enough. - * (tcp_recvmsg() will send ACK otherwise). - * If application uses SO_RCVLOWAT, we want send ack now if - * we have not received enough bytes to satisfy the condition. - */ -- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || -- __tcp_select_window(sk) >= tp->rcv_wnd)) || -+ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || -+ __tcp_select_window(sk) >= tp->rcv_wnd))) || - /* We ACK each frame or... */ - tcp_in_quickack_mode(sk) || - /* Protocol state mandates a one-time immediate ACK */ -diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c -index 5b019ba2b9d2..5884f4388c61 100644 ---- a/net/ipv4/tcp_ipv4.c -+++ b/net/ipv4/tcp_ipv4.c -@@ -3165,6 +3165,13 @@ static int __net_init tcp_sk_init(struct net *net) - net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0; - atomic_set(&net->ipv4.tfo_active_disable_times, 0); - -+ /* Set default values for PLB */ -+ net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */ -+ net->ipv4.sysctl_tcp_plb_cong_thresh = 128; /* 50% congestion */ -+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3; -+ net->ipv4.sysctl_tcp_plb_rehash_rounds = 12; -+ net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60; ++ if (unlikely(sa->in6.sin6_port != udp_hdr(skb)->source)) ++ return false; ++ break; ++ default: ++ return false; ++ } + - /* Reno is always built in */ - if (!net_eq(net, &init_net) && - bpf_try_module_get(init_net.ipv4.tcp_congestion_control, -diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c -index 290019de766d..15e89addae08 100644 ---- a/net/ipv4/tcp_output.c -+++ b/net/ipv4/tcp_output.c -@@ -375,7 +375,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, - th->cwr = 1; - skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; - } -- } else if (!tcp_ca_needs_ecn(sk)) { -+ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && -+ !tcp_ca_needs_ecn(sk)) { - /* ACK or retransmitted segment: clear ECT|CE */ - INET_ECN_dontxmit(sk); - } -@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, - { - struct tcp_sock *tp = tcp_sk(sk); - struct sk_buff *buff; -- int nsize, old_factor; -+ int nsize, old_factor, inflight_prev; - long limit; - int nlen; - u8 flags; -@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, - - if (diff) - tcp_adjust_pcount(sk, skb, diff); ++ return true; ++} + -+ /* Set buff tx.in_flight as if buff were sent by itself. */ -+ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; -+ if (WARN_ONCE(inflight_prev < 0, -+ "inconsistent: tx.in_flight: %u old_factor: %d", -+ TCP_SKB_CB(skb)->tx.in_flight, old_factor)) -+ inflight_prev = 0; -+ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + -+ tcp_skb_pcount(buff); - } - - /* Link BUFF into the send queue. */ -@@ -1988,13 +1998,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, - static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) - { - const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; -- u32 min_tso, tso_segs; -- -- min_tso = ca_ops->min_tso_segs ? -- ca_ops->min_tso_segs(sk) : -- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); -+ u32 tso_segs; - -- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); -+ tso_segs = ca_ops->tso_segs ? -+ ca_ops->tso_segs(sk, mss_now) : -+ tcp_tso_autosize(sk, mss_now, -+ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs)); - return min_t(u32, tso_segs, sk->sk_gso_max_segs); - } - -@@ -2630,6 +2639,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, - skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true); - list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); - tcp_init_tso_segs(skb, mss_now); -+ tcp_set_tx_in_flight(sk, skb); - goto repair; /* Skip network transmission */ - } - -diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c ++struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *sa); ++void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *bind); ++ ++#endif /* _NET_OVPN_DCO_OVPNBIND_H_ */ +diff --git a/drivers/net/ovpn-dco/crypto.c b/drivers/net/ovpn-dco/crypto.c new file mode 100644 -index 000000000000..71b02c0404ce +index 000000000000..fcc3a351ba9d --- /dev/null -+++ b/net/ipv4/tcp_plb.c -@@ -0,0 +1,100 @@ -+/* Protective Load Balancing (PLB) -+ * -+ * PLB was designed to reduce link load imbalance across datacenter -+ * switches. PLB is a host-based optimization; it leverages congestion -+ * signals from the transport layer to randomly change the path of the -+ * connection experiencing sustained congestion. PLB prefers to repath -+ * after idle periods to minimize packet reordering. It repaths by -+ * changing the IPv6 Flow Label on the packets of a connection, which -+ * datacenter switches include as part of ECMP/WCMP hashing. -+ * -+ * PLB is described in detail in: ++++ b/drivers/net/ovpn-dco/crypto.c +@@ -0,0 +1,154 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator + * -+ * Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu, -+ * Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson, -+ * David Wetherall,Abdul Kabbani: -+ * "PLB: Congestion Signals are Simple and Effective for -+ * Network Load Balancing" -+ * In ACM SIGCOMM 2022, Amsterdam Netherlands. ++ * Copyright (C) 2020-2022 OpenVPN, Inc. + * ++ * Author: James Yonan ++ * Antonio Quartulli + */ + -+#include ++#include "main.h" ++#include "crypto_aead.h" ++#include "crypto.h" + -+/* Called once per round-trip to update PLB state for a connection. */ -+void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, -+ const int cong_ratio) ++#include ++ ++static void ovpn_ks_destroy_rcu(struct rcu_head *head) +{ -+ struct net *net = sock_net(sk); ++ struct ovpn_crypto_key_slot *ks; + -+ if (!plb->enabled) -+ return; ++ ks = container_of(head, struct ovpn_crypto_key_slot, rcu); ++ ovpn_aead_crypto_key_slot_destroy(ks); ++} + -+ if (cong_ratio >= 0) { -+ if (cong_ratio < net->ipv4.sysctl_tcp_plb_cong_thresh) -+ plb->consec_cong_rounds = 0; -+ else if (plb->consec_cong_rounds < -+ net->ipv4.sysctl_tcp_plb_rehash_rounds) -+ plb->consec_cong_rounds++; -+ } ++void ovpn_crypto_key_slot_release(struct kref *kref) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ ks = container_of(kref, struct ovpn_crypto_key_slot, refcount); ++ call_rcu(&ks->rcu, ovpn_ks_destroy_rcu); +} -+EXPORT_SYMBOL_GPL(tcp_plb_update_state); + -+/* Check whether recent congestion has been persistent enough to warrant -+ * a load balancing decision that switches the connection to another path. ++/* can only be invoked when all peer references have been dropped (i.e. RCU ++ * release routine) + */ -+void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) ++void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) +{ -+ struct net *net = sock_net(sk); -+ bool can_idle_rehash, can_force_rehash; ++ struct ovpn_crypto_key_slot *ks; + -+ if (!plb->enabled) -+ return; ++ ks = rcu_access_pointer(cs->primary); ++ if (ks) { ++ RCU_INIT_POINTER(cs->primary, NULL); ++ ovpn_crypto_key_slot_put(ks); ++ } + -+ /* Note that tcp_jiffies32 can wrap, so we clear pause_until -+ * to 0 to indicate there is no recent RTO event that constrains -+ * PLB rehashing. -+ */ -+ if (plb->pause_until && -+ !before(tcp_jiffies32, plb->pause_until)) -+ plb->pause_until = 0; ++ ks = rcu_access_pointer(cs->secondary); ++ if (ks) { ++ RCU_INIT_POINTER(cs->secondary, NULL); ++ ovpn_crypto_key_slot_put(ks); ++ } + -+ can_idle_rehash = net->ipv4.sysctl_tcp_plb_idle_rehash_rounds && -+ !tcp_sk(sk)->packets_out && -+ plb->consec_cong_rounds >= -+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds; -+ can_force_rehash = plb->consec_cong_rounds >= -+ net->ipv4.sysctl_tcp_plb_rehash_rounds; ++ mutex_destroy(&cs->mutex); ++} + -+ if (!plb->pause_until && (can_idle_rehash || can_force_rehash)) { -+ sk_rethink_txhash(sk); -+ plb->consec_cong_rounds = 0; -+ tcp_sk(sk)->ecn_rehash++; -+ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPECNREHASH); -+ } ++/* removes the primary key from the crypto context */ ++void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ ++ mutex_lock(&cs->mutex); ++ ks = rcu_replace_pointer(cs->primary, NULL, lockdep_is_held(&cs->mutex)); ++ ovpn_crypto_key_slot_put(ks); ++ mutex_unlock(&cs->mutex); +} -+EXPORT_SYMBOL_GPL(tcp_plb_check_rehash); + -+/* Upon RTO, disallow load balancing for a while, to avoid having load -+ * balancing decisions switch traffic to a black-holed path that was -+ * previously avoided with a sk_rethink_txhash() call at RTO time. ++/* Reset the ovpn_crypto_state object in a way that is atomic ++ * to RCU readers. + */ -+void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb) ++int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, ++ const struct ovpn_peer_key_reset *pkr) ++ __must_hold(cs->mutex) +{ -+ struct net *net = sock_net(sk); -+ u32 pause; ++ struct ovpn_crypto_key_slot *old = NULL; ++ struct ovpn_crypto_key_slot *new; + -+ if (!plb->enabled) -+ return; ++ lockdep_assert_held(&cs->mutex); + -+ pause = net->ipv4.sysctl_tcp_plb_suspend_rto_sec * HZ; -+ pause += prandom_u32_max(pause); -+ plb->pause_until = tcp_jiffies32 + pause; ++ new = ovpn_aead_crypto_key_slot_new(&pkr->key); ++ if (IS_ERR(new)) ++ return PTR_ERR(new); + -+ /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call -+ * that may switch this connection to a path with completely different -+ * congestion characteristics. -+ */ -+ plb->consec_cong_rounds = 0; ++ switch (pkr->slot) { ++ case OVPN_KEY_SLOT_PRIMARY: ++ old = rcu_replace_pointer(cs->primary, new, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ case OVPN_KEY_SLOT_SECONDARY: ++ old = rcu_replace_pointer(cs->secondary, new, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ default: ++ goto free_key; ++ } ++ ++ if (old) ++ ovpn_crypto_key_slot_put(old); ++ ++ return 0; ++free_key: ++ ovpn_crypto_key_slot_put(new); ++ return -EINVAL; +} -+EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto); -diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c -index a8f6d9d06f2e..a8b4c9504570 100644 ---- a/net/ipv4/tcp_rate.c -+++ b/net/ipv4/tcp_rate.c -@@ -34,6 +34,24 @@ - * ready to send in the write queue. - */ - -+void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) ++ ++void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, ++ enum ovpn_key_slot slot) +{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ u32 in_flight; ++ struct ovpn_crypto_key_slot *ks = NULL; + -+ /* Check, sanitize, and record packets in flight after skb was sent. */ -+ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); -+ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, -+ "insane in_flight %u cc %s mss %u " -+ "cwnd %u pif %u %u %u %u\n", -+ in_flight, inet_csk(sk)->icsk_ca_ops->name, -+ tp->mss_cache, tp->snd_cwnd, -+ tp->packets_out, tp->retrans_out, -+ tp->sacked_out, tp->lost_out)) -+ in_flight = TCPCB_IN_FLIGHT_MAX; -+ TCP_SKB_CB(skb)->tx.in_flight = in_flight; -+} ++ mutex_lock(&cs->mutex); ++ switch (slot) { ++ case OVPN_KEY_SLOT_PRIMARY: ++ ks = rcu_replace_pointer(cs->primary, NULL, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ case OVPN_KEY_SLOT_SECONDARY: ++ ks = rcu_replace_pointer(cs->secondary, NULL, ++ lockdep_is_held(&cs->mutex)); ++ break; ++ default: ++ pr_warn("Invalid slot to release: %u\n", slot); ++ break; ++ } ++ mutex_unlock(&cs->mutex); + - /* Snapshot the current delivery information in the skb, to generate - * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). - */ -@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) - TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; - TCP_SKB_CB(skb)->tx.delivered = tp->delivered; - TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; -+ TCP_SKB_CB(skb)->tx.lost = tp->lost; - TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; -+ tcp_set_tx_in_flight(sk, skb); - } - - /* When an skb is sacked or acked, we fill in the rate sample with the (prior) -@@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - if (!rs->prior_delivered || - tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, - scb->end_seq, rs->last_end_seq)) { -+ rs->prior_lost = scb->tx.lost; - rs->prior_delivered_ce = scb->tx.delivered_ce; - rs->prior_delivered = scb->tx.delivered; - rs->prior_mstamp = scb->tx.delivered_mstamp; - rs->is_app_limited = scb->tx.is_app_limited; - rs->is_retrans = scb->sacked & TCPCB_RETRANS; - rs->last_end_seq = scb->end_seq; -+ rs->tx_in_flight = scb->tx.in_flight; - - /* Record send time of most recently ACKed packet: */ - tp->first_tx_mstamp = tx_tstamp; - /* Find the duration of the "send phase" of this window: */ -- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, -- scb->tx.first_tx_mstamp); -+ rs->interval_us = tcp_stamp32_us_delta( -+ tp->first_tx_mstamp, -+ scb->tx.first_tx_mstamp); - - } - /* Mark off the skb delivered once it's sacked to avoid being -@@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - return; - } - rs->delivered = tp->delivered - rs->prior_delivered; -+ rs->lost = tp->lost - rs->prior_lost; - - rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; - /* delivered_ce occupies less than 32 bits in the skb control block */ -@@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - * longer phase. - */ - snd_us = rs->interval_us; /* send phase */ -- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, -+ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, - rs->prior_mstamp); /* ack phase */ - rs->interval_us = max(snd_us, ack_us); - -diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c -index b4dfb82d6ecb..bb613fc8948a 100644 ---- a/net/ipv4/tcp_timer.c -+++ b/net/ipv4/tcp_timer.c -@@ -605,6 +605,7 @@ void tcp_write_timer_handler(struct sock *sk) - return; - } - -+ tcp_rate_check_app_limited(sk); - tcp_mstamp_refresh(tcp_sk(sk)); - event = icsk->icsk_pending; - --- -2.37.3 - -From 4ce1238c4f6da720e90328477839811957f4f32f Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:15:39 +0200 -Subject: [PATCH 02/13] cachy - -Signed-off-by: Peter Jung ---- - .gitignore | 1 + - Documentation/admin-guide/mm/damon/index.rst | 6 +- - Documentation/admin-guide/mm/damon/start.rst | 13 +- - Documentation/admin-guide/mm/damon/usage.rst | 5 + - Documentation/admin-guide/mm/ksm.rst | 36 + - Documentation/admin-guide/pm/amd-pstate.rst | 19 + - Makefile | 2 + - arch/alpha/kernel/syscalls/syscall.tbl | 1 + - arch/arc/configs/axs101_defconfig | 1 + - arch/arc/configs/axs103_defconfig | 1 + - arch/arc/configs/axs103_smp_defconfig | 1 + - arch/arc/configs/haps_hs_defconfig | 1 + - arch/arc/configs/haps_hs_smp_defconfig | 1 + - arch/arc/configs/hsdk_defconfig | 1 + - arch/arc/configs/nsim_700_defconfig | 1 + - arch/arc/configs/nsimosci_defconfig | 1 + - arch/arc/configs/nsimosci_hs_defconfig | 1 + - arch/arc/configs/nsimosci_hs_smp_defconfig | 1 + - arch/arc/configs/tb10x_defconfig | 1 + - arch/arc/configs/vdk_hs38_defconfig | 1 + - arch/arc/configs/vdk_hs38_smp_defconfig | 1 + - arch/arm/tools/syscall.tbl | 1 + - arch/arm64/include/asm/unistd.h | 2 +- - arch/arm64/include/asm/unistd32.h | 2 + - arch/ia64/kernel/syscalls/syscall.tbl | 1 + - arch/m68k/kernel/syscalls/syscall.tbl | 1 + - arch/microblaze/kernel/syscalls/syscall.tbl | 1 + - arch/mips/kernel/syscalls/syscall_n32.tbl | 1 + - arch/mips/kernel/syscalls/syscall_n64.tbl | 1 + - arch/mips/kernel/syscalls/syscall_o32.tbl | 1 + - arch/parisc/kernel/syscalls/syscall.tbl | 1 + - arch/powerpc/kernel/syscalls/syscall.tbl | 1 + - arch/s390/kernel/syscalls/syscall.tbl | 1 + - arch/sh/kernel/syscalls/syscall.tbl | 1 + - arch/sparc/kernel/syscalls/syscall.tbl | 1 + - arch/x86/Kconfig.cpu | 332 +++++- - arch/x86/Makefile | 40 +- - arch/x86/Makefile.postlink | 41 + - arch/x86/boot/compressed/Makefile | 10 +- - arch/x86/entry/syscalls/syscall_32.tbl | 1 + - arch/x86/entry/syscalls/syscall_64.tbl | 1 + - arch/x86/include/asm/msr-index.h | 7 + - arch/x86/include/asm/topology.h | 1 + - arch/x86/include/asm/vdso/processor.h | 2 +- - arch/x86/include/asm/vermagic.h | 66 ++ - arch/x86/kernel/alternative.c | 2 + - arch/x86/kernel/cpu/intel_epb.c | 4 + - arch/x86/kernel/cpu/microcode/core.c | 40 +- - arch/x86/kernel/cpu/microcode/intel.c | 14 +- - arch/x86/kernel/itmt.c | 29 +- - arch/x86/kernel/msr.c | 2 +- - arch/x86/kernel/tsc.c | 3 + - arch/x86/mm/fault.c | 4 +- - arch/xtensa/kernel/syscalls/syscall.tbl | 1 + - block/bfq-cgroup.c | 5 - - block/bfq-iosched.c | 20 +- - block/bfq-iosched.h | 18 +- - block/bfq-wf2q.c | 9 +- - block/blk-core.c | 3 + - block/elevator.c | 7 +- - drivers/acpi/cppc_acpi.c | 131 ++- - drivers/ata/libahci.c | 4 +- - drivers/base/arch_topology.c | 2 +- - drivers/base/firmware_loader/main.c | 2 + - drivers/block/zram/Kconfig | 18 + - drivers/block/zram/zram_drv.c | 39 + - drivers/cpufreq/amd-pstate.c | 1003 ++++++++++++++++- - drivers/cpufreq/cppc_cpufreq.c | 2 +- - drivers/cpufreq/intel_pstate.c | 7 + - drivers/idle/intel_idle.c | 44 +- - drivers/input/serio/i8042.c | 10 +- - drivers/md/dm-crypt.c | 5 + - drivers/net/dummy.c | 2 +- - drivers/nvme/host/core.c | 2 +- - drivers/pci/pci.c | 2 +- - drivers/powercap/intel_rapl_common.c | 2 +- - drivers/soundwire/bus.h | 2 +- - drivers/thermal/intel/intel_powerclamp.c | 10 + - fs/proc/base.c | 15 + - fs/xattr.c | 15 +- - include/acpi/cppc_acpi.h | 17 + - include/linux/ipc_namespace.h | 5 +- - include/linux/jbd2.h | 2 +- - include/linux/ksm.h | 4 + - include/linux/mm_types.h | 5 + - include/linux/page_counter.h | 1 + - include/linux/pagemap.h | 2 +- - include/linux/sched.h | 1 + - include/linux/syscalls.h | 1 + - include/linux/user_namespace.h | 4 + - include/linux/wait.h | 2 + - include/uapi/asm-generic/unistd.h | 5 +- - include/uapi/linux/if_bonding.h | 2 +- - init/Kconfig | 26 + - init/do_mounts.c | 16 +- - kernel/Kconfig.hz | 24 + - kernel/fork.c | 14 + - kernel/locking/rwsem.c | 4 +- - kernel/module/internal.h | 2 + - kernel/module/main.c | 1 + - kernel/module/procfs.c | 13 + - kernel/module/signing.c | 4 + - kernel/sched/core.c | 19 +- - kernel/sched/debug.c | 1 + - kernel/sched/fair.c | 20 +- - kernel/sched/wait.c | 24 + - kernel/sys_ni.c | 1 + - kernel/sysctl.c | 22 + - kernel/user_namespace.c | 7 + - kernel/watchdog.c | 2 +- - lib/raid6/algos.c | 4 +- - lib/string.c | 62 +- - lib/zstd/compress/zstd_compress.c | 2 +- - lib/zstd/compress/zstd_double_fast.c | 61 +- - lib/zstd/compress/zstd_fast.c | 69 +- - lib/zstd/compress/zstd_lazy.c | 223 ++-- - mm/compaction.c | 6 +- - mm/damon/Kconfig | 3 + - mm/damon/core-test.h | 29 +- - mm/damon/core.c | 45 +- - mm/damon/dbgfs.c | 2 +- - mm/damon/lru_sort.c | 41 +- - mm/damon/vaddr.c | 18 +- - mm/ksm.c | 351 +++++- - mm/madvise.c | 115 ++ - mm/memcontrol.c | 2 +- - mm/page-writeback.c | 8 + - mm/page_alloc.c | 5 +- - mm/swap.c | 5 + - mm/vmpressure.c | 4 + - mm/vmscan.c | 4 + - net/ipv4/inet_connection_sock.c | 2 +- - net/ipv4/tcp.c | 4 +- - tools/testing/selftests/damon/Makefile | 1 + - .../debugfs_duplicate_context_creation.sh | 27 + - tools/testing/selftests/vm/.gitignore | 1 + - tools/testing/selftests/vm/Makefile | 1 + - tools/testing/selftests/vm/test-ksm-auto.c | 273 +++++ - 138 files changed, 3281 insertions(+), 429 deletions(-) - create mode 100644 arch/x86/Makefile.postlink - create mode 100644 tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh - create mode 100644 tools/testing/selftests/vm/test-ksm-auto.c - -diff --git a/.gitignore b/.gitignore -index 265959544978..cd4ef88584ea 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -37,6 +37,7 @@ - *.o - *.o.* - *.patch -+*.relocs - *.s - *.so - *.so.dbg -diff --git a/Documentation/admin-guide/mm/damon/index.rst b/Documentation/admin-guide/mm/damon/index.rst -index 05500042f777..33d37bb2fb4e 100644 ---- a/Documentation/admin-guide/mm/damon/index.rst -+++ b/Documentation/admin-guide/mm/damon/index.rst -@@ -1,8 +1,8 @@ - .. SPDX-License-Identifier: GPL-2.0 - --======================== --Monitoring Data Accesses --======================== -+========================== -+DAMON: Data Access MONitor -+========================== - - :doc:`DAMON ` allows light-weight data access monitoring. - Using DAMON, users can analyze the memory access patterns of their systems and -diff --git a/Documentation/admin-guide/mm/damon/start.rst b/Documentation/admin-guide/mm/damon/start.rst -index 4d5ca2c46288..9f88afc734da 100644 ---- a/Documentation/admin-guide/mm/damon/start.rst -+++ b/Documentation/admin-guide/mm/damon/start.rst -@@ -29,16 +29,9 @@ called DAMON Operator (DAMO). It is available at - https://github.com/awslabs/damo. The examples below assume that ``damo`` is on - your ``$PATH``. It's not mandatory, though. - --Because DAMO is using the debugfs interface (refer to :doc:`usage` for the --detail) of DAMON, you should ensure debugfs is mounted. Mount it manually as --below:: -- -- # mount -t debugfs none /sys/kernel/debug/ -- --or append the following line to your ``/etc/fstab`` file so that your system --can automatically mount debugfs upon booting:: -- -- debugfs /sys/kernel/debug debugfs defaults 0 0 -+Because DAMO is using the sysfs interface (refer to :doc:`usage` for the -+detail) of DAMON, you should ensure :doc:`sysfs ` is -+mounted. - - - Recording Data Access Patterns -diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst -index ca91ecc29078..b47b0cbbd491 100644 ---- a/Documentation/admin-guide/mm/damon/usage.rst -+++ b/Documentation/admin-guide/mm/damon/usage.rst -@@ -393,6 +393,11 @@ the files as above. Above is only for an example. - debugfs Interface - ================= - -+.. note:: ++ if (!ks) { ++ pr_debug("Key slot already released: %u\n", slot); ++ return; ++ } ++ pr_debug("deleting key slot %u, key_id=%u\n", slot, ks->key_id); + -+ DAMON debugfs interface will be removed after next LTS kernel is released, so -+ users should move to the :ref:`sysfs interface `. -+ - DAMON exports eight files, ``attrs``, ``target_ids``, ``init_regions``, - ``schemes``, ``monitor_on``, ``kdamond_pid``, ``mk_contexts`` and - ``rm_contexts`` under its debugfs directory, ``/damon/``. -diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst -index b244f0202a03..fb6ba2002a4b 100644 ---- a/Documentation/admin-guide/mm/ksm.rst -+++ b/Documentation/admin-guide/mm/ksm.rst -@@ -184,6 +184,42 @@ The maximum possible ``pages_sharing/pages_shared`` ratio is limited by the - ``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must - be increased accordingly. - -+Monitoring KSM profit -+===================== ++ ovpn_crypto_key_slot_put(ks); ++} ++ ++/* this swap is not atomic, but there will be a very short time frame where the ++ * old_secondary key won't be available. This should not be a big deal as most ++ * likely both peers are already using the new primary at this point. ++ */ ++void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs) ++{ ++ const struct ovpn_crypto_key_slot *old_primary, *old_secondary; + -+KSM can save memory by merging identical pages, but also can consume -+additional memory, because it needs to generate a number of rmap_items to -+save each scanned page's brief rmap information. Some of these pages may -+be merged, but some may not be abled to be merged after being checked -+several times, which are unprofitable memory consumed. ++ mutex_lock(&cs->mutex); + -+1) How to determine whether KSM save memory or consume memory in system-wide -+ range? Here is a simple approximate calculation for reference:: ++ old_secondary = rcu_dereference_protected(cs->secondary, ++ lockdep_is_held(&cs->mutex)); ++ old_primary = rcu_replace_pointer(cs->primary, old_secondary, ++ lockdep_is_held(&cs->mutex)); ++ rcu_assign_pointer(cs->secondary, old_primary); + -+ general_profit =~ pages_sharing * sizeof(page) - (all_rmap_items) * -+ sizeof(rmap_item); ++ pr_debug("key swapped: %u <-> %u\n", ++ old_primary ? old_primary->key_id : 0, ++ old_secondary ? old_secondary->key_id : 0); + -+ where all_rmap_items can be easily obtained by summing ``pages_sharing``, -+ ``pages_shared``, ``pages_unshared`` and ``pages_volatile``. ++ mutex_unlock(&cs->mutex); ++} +diff --git a/drivers/net/ovpn-dco/crypto.h b/drivers/net/ovpn-dco/crypto.h +new file mode 100644 +index 000000000000..79f580e54a63 +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+2) The KSM profit inner a single process can be similarly obtained by the -+ following approximate calculation:: ++#ifndef _NET_OVPN_DCO_OVPNCRYPTO_H_ ++#define _NET_OVPN_DCO_OVPNCRYPTO_H_ + -+ process_profit =~ ksm_merging_pages * sizeof(page) - -+ ksm_rmap_items * sizeof(rmap_item). ++#include "main.h" ++#include "pktid.h" + -+ where ksm_merging_pages is shown under the directory ``/proc//``, -+ and ksm_rmap_items is shown in ``/proc//ksm_stat``. ++#include ++#include + -+From the perspective of application, a high ratio of ``ksm_rmap_items`` to -+``ksm_merging_pages`` means a bad madvise-applied policy, so developers or -+administrators have to rethink how to change madvise policy. Giving an example -+for reference, a page's size is usually 4K, and the rmap_item's size is -+separately 32B on 32-bit CPU architecture and 64B on 64-bit CPU architecture. -+so if the ``ksm_rmap_items/ksm_merging_pages`` ratio exceeds 64 on 64-bit CPU -+or exceeds 128 on 32-bit CPU, then the app's madvise policy should be dropped, -+because the ksm profit is approximately zero or negative. ++struct ovpn_peer; ++struct ovpn_crypto_key_slot; + - Monitoring KSM events - ===================== - -diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst -index 83b58eb4ab4d..d0f0e115013b 100644 ---- a/Documentation/admin-guide/pm/amd-pstate.rst -+++ b/Documentation/admin-guide/pm/amd-pstate.rst -@@ -261,6 +261,25 @@ lowest non-linear performance in `AMD CPPC Performance Capability - `_.) - This attribute is read-only. - -+``energy_performance_available_preferences`` ++/* info needed for both encrypt and decrypt directions */ ++struct ovpn_key_direction { ++ const u8 *cipher_key; ++ size_t cipher_key_size; ++ const u8 *nonce_tail; /* only needed for GCM modes */ ++ size_t nonce_tail_size; /* only needed for GCM modes */ ++}; + -+All the supported EPP preference could be selected, List of the strings that -+can be set to the ``energy_performance_preference`` attribute -+those different profiles represent different energy vs efficiency hints provided -+to low-level firmware -+however, the ``default`` represents the epp value is set by platform firmware -+This attribute is read-only. ++/* all info for a particular symmetric key (primary or secondary) */ ++struct ovpn_key_config { ++ enum ovpn_cipher_alg cipher_alg; ++ u8 key_id; ++ struct ovpn_key_direction encrypt; ++ struct ovpn_key_direction decrypt; ++}; + -+``energy_performance_preference`` ++/* used to pass settings from netlink to the crypto engine */ ++struct ovpn_peer_key_reset { ++ enum ovpn_key_slot slot; ++ struct ovpn_key_config key; ++}; + -+The current energy performance preference can be read from this attribute. -+and user can change current preference according to energy or performance needs -+Please get all support profiles list from -+``energy_performance_available_preferences`` attribute, all the profiles are -+integer values defined between 0 to 255 when EPP feature is enabled by platform -+firmware, if EPP feature is disabled, driver will ignore the written value -+This attribute is read-write. ++struct ovpn_crypto_key_slot { ++ u8 key_id; + - Other performance and frequency values can be read back from - ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. - -diff --git a/Makefile b/Makefile -index 647a42a1f800..5c327c29ef12 100644 ---- a/Makefile -+++ b/Makefile -@@ -758,6 +758,8 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) - - ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE - KBUILD_CFLAGS += -O2 -+else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 -+KBUILD_CFLAGS += -O3 - else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE - KBUILD_CFLAGS += -Os - endif -diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl -index 3515bc4f16a4..00ff721da300 100644 ---- a/arch/alpha/kernel/syscalls/syscall.tbl -+++ b/arch/alpha/kernel/syscalls/syscall.tbl -@@ -490,3 +490,4 @@ - 558 common process_mrelease sys_process_mrelease - 559 common futex_waitv sys_futex_waitv - 560 common set_mempolicy_home_node sys_ni_syscall -+561 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig -index e31a8ebc3ecc..0016149f9583 100644 ---- a/arch/arc/configs/axs101_defconfig -+++ b/arch/arc/configs/axs101_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig -index e0e8567f0d75..5b031582a1cf 100644 ---- a/arch/arc/configs/axs103_defconfig -+++ b/arch/arc/configs/axs103_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig -index fcbc952bc75b..d4eec39e0112 100644 ---- a/arch/arc/configs/axs103_smp_defconfig -+++ b/arch/arc/configs/axs103_smp_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig -index d87ad7e88d62..7337cdf4ffdd 100644 ---- a/arch/arc/configs/haps_hs_defconfig -+++ b/arch/arc/configs/haps_hs_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EXPERT=y - CONFIG_PERF_EVENTS=y - # CONFIG_COMPAT_BRK is not set -diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig -index 8d82cdb7f86a..bc927221afc0 100644 ---- a/arch/arc/configs/haps_hs_smp_defconfig -+++ b/arch/arc/configs/haps_hs_smp_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig -index f856b03e0fb5..aa000075a575 100644 ---- a/arch/arc/configs/hsdk_defconfig -+++ b/arch/arc/configs/hsdk_defconfig -@@ -9,6 +9,7 @@ CONFIG_NAMESPACES=y - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y - CONFIG_BLK_DEV_RAM=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig -index a1ce12bf5b16..326f6cde7826 100644 ---- a/arch/arc/configs/nsim_700_defconfig -+++ b/arch/arc/configs/nsim_700_defconfig -@@ -11,6 +11,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig -index ca10f4a2c823..bf39a0091679 100644 ---- a/arch/arc/configs/nsimosci_defconfig -+++ b/arch/arc/configs/nsimosci_defconfig -@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig -index 31b6ec3683c6..7121bd71c543 100644 ---- a/arch/arc/configs/nsimosci_hs_defconfig -+++ b/arch/arc/configs/nsimosci_hs_defconfig -@@ -10,6 +10,7 @@ CONFIG_NAMESPACES=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y -diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig -index 41a0037f48a5..f9863b294a70 100644 ---- a/arch/arc/configs/nsimosci_hs_smp_defconfig -+++ b/arch/arc/configs/nsimosci_hs_smp_defconfig -@@ -8,6 +8,7 @@ CONFIG_IKCONFIG_PROC=y - # CONFIG_UTS_NS is not set - # CONFIG_PID_NS is not set - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_PERF_EVENTS=y - # CONFIG_COMPAT_BRK is not set - CONFIG_KPROBES=y -diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig -index d93b65008d4a..a12656ec0072 100644 ---- a/arch/arc/configs/tb10x_defconfig -+++ b/arch/arc/configs/tb10x_defconfig -@@ -14,6 +14,7 @@ CONFIG_INITRAMFS_SOURCE="../tb10x-rootfs.cpio" - CONFIG_INITRAMFS_ROOT_UID=2100 - CONFIG_INITRAMFS_ROOT_GID=501 - # CONFIG_RD_GZIP is not set -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_KALLSYMS_ALL=y - # CONFIG_AIO is not set - CONFIG_EMBEDDED=y -diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig -index 0c3b21416819..d7c858df520c 100644 ---- a/arch/arc/configs/vdk_hs38_defconfig -+++ b/arch/arc/configs/vdk_hs38_defconfig -@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_IKCONFIG=y - CONFIG_IKCONFIG_PROC=y - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig -index f9ad9d3ee702..015c1d43889e 100644 ---- a/arch/arc/configs/vdk_hs38_smp_defconfig -+++ b/arch/arc/configs/vdk_hs38_smp_defconfig -@@ -4,6 +4,7 @@ CONFIG_HIGH_RES_TIMERS=y - CONFIG_IKCONFIG=y - CONFIG_IKCONFIG_PROC=y - CONFIG_BLK_DEV_INITRD=y -+CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y - CONFIG_EMBEDDED=y - CONFIG_PERF_EVENTS=y - # CONFIG_VM_EVENT_COUNTERS is not set -diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl -index ac964612d8b0..90933eabe115 100644 ---- a/arch/arm/tools/syscall.tbl -+++ b/arch/arm/tools/syscall.tbl -@@ -464,3 +464,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h -index 037feba03a51..64a514f90131 100644 ---- a/arch/arm64/include/asm/unistd.h -+++ b/arch/arm64/include/asm/unistd.h -@@ -39,7 +39,7 @@ - #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) - #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) - --#define __NR_compat_syscalls 451 -+#define __NR_compat_syscalls 452 - #endif - - #define __ARCH_WANT_SYS_CLONE -diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h -index 604a2053d006..91f2bb7199af 100644 ---- a/arch/arm64/include/asm/unistd32.h -+++ b/arch/arm64/include/asm/unistd32.h -@@ -907,6 +907,8 @@ __SYSCALL(__NR_process_mrelease, sys_process_mrelease) - __SYSCALL(__NR_futex_waitv, sys_futex_waitv) - #define __NR_set_mempolicy_home_node 450 - __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) -+#define __NR_pmadv_ksm 451 -+__SYSCALL(__NR_pmadv_ksm, sys_pmadv_ksm) - - /* - * Please add new compat syscalls above this comment and update -diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl -index 78b1d03e86e1..79ad5a5682b3 100644 ---- a/arch/ia64/kernel/syscalls/syscall.tbl -+++ b/arch/ia64/kernel/syscalls/syscall.tbl -@@ -371,3 +371,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl -index b1f3940bc298..5ccf925567da 100644 ---- a/arch/m68k/kernel/syscalls/syscall.tbl -+++ b/arch/m68k/kernel/syscalls/syscall.tbl -@@ -450,3 +450,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl -index 820145e47350..6b76208597f3 100644 ---- a/arch/microblaze/kernel/syscalls/syscall.tbl -+++ b/arch/microblaze/kernel/syscalls/syscall.tbl -@@ -456,3 +456,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl -index 253ff994ed2e..e4aeedb17c38 100644 ---- a/arch/mips/kernel/syscalls/syscall_n32.tbl -+++ b/arch/mips/kernel/syscalls/syscall_n32.tbl -@@ -389,3 +389,4 @@ - 448 n32 process_mrelease sys_process_mrelease - 449 n32 futex_waitv sys_futex_waitv - 450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node -+451 n32 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl -index 3f1886ad9d80..fe88db51efa0 100644 ---- a/arch/mips/kernel/syscalls/syscall_n64.tbl -+++ b/arch/mips/kernel/syscalls/syscall_n64.tbl -@@ -365,3 +365,4 @@ - 448 n64 process_mrelease sys_process_mrelease - 449 n64 futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 n64 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl -index 8f243e35a7b2..674cb940bd15 100644 ---- a/arch/mips/kernel/syscalls/syscall_o32.tbl -+++ b/arch/mips/kernel/syscalls/syscall_o32.tbl -@@ -438,3 +438,4 @@ - 448 o32 process_mrelease sys_process_mrelease - 449 o32 futex_waitv sys_futex_waitv - 450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node -+451 o32 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl -index 8a99c998da9b..429b129d5d46 100644 ---- a/arch/parisc/kernel/syscalls/syscall.tbl -+++ b/arch/parisc/kernel/syscalls/syscall.tbl -@@ -448,3 +448,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl -index 2600b4237292..bb2f71a36941 100644 ---- a/arch/powerpc/kernel/syscalls/syscall.tbl -+++ b/arch/powerpc/kernel/syscalls/syscall.tbl -@@ -530,3 +530,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl -index 799147658dee..1cd523748bd2 100644 ---- a/arch/s390/kernel/syscalls/syscall.tbl -+++ b/arch/s390/kernel/syscalls/syscall.tbl -@@ -453,3 +453,4 @@ - 448 common process_mrelease sys_process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm sys_pmadv_ksm -diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl -index 2de85c977f54..cfc75fa43eae 100644 ---- a/arch/sh/kernel/syscalls/syscall.tbl -+++ b/arch/sh/kernel/syscalls/syscall.tbl -@@ -453,3 +453,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl -index 4398cc6fb68d..d2c0a6426f6b 100644 ---- a/arch/sparc/kernel/syscalls/syscall.tbl -+++ b/arch/sparc/kernel/syscalls/syscall.tbl -@@ -496,3 +496,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu -index 542377cd419d..22b919cdb6d1 100644 ---- a/arch/x86/Kconfig.cpu -+++ b/arch/x86/Kconfig.cpu -@@ -157,7 +157,7 @@ config MPENTIUM4 - - - config MK6 -- bool "K6/K6-II/K6-III" -+ bool "AMD K6/K6-II/K6-III" - depends on X86_32 - help - Select this for an AMD K6-family processor. Enables use of -@@ -165,7 +165,7 @@ config MK6 - flags to GCC. - - config MK7 -- bool "Athlon/Duron/K7" -+ bool "AMD Athlon/Duron/K7" - depends on X86_32 - help - Select this for an AMD Athlon K7-family processor. Enables use of -@@ -173,12 +173,98 @@ config MK7 - flags to GCC. - - config MK8 -- bool "Opteron/Athlon64/Hammer/K8" -+ bool "AMD Opteron/Athlon64/Hammer/K8" - help - Select this for an AMD Opteron or Athlon64 Hammer-family processor. - Enables use of some extended instructions, and passes appropriate - optimization flags to GCC. - -+config MK8SSE3 -+ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3" -+ help -+ Select this for improved AMD Opteron or Athlon64 Hammer-family processors. -+ Enables use of some extended instructions, and passes appropriate -+ optimization flags to GCC. ++ struct crypto_aead *encrypt; ++ struct crypto_aead *decrypt; ++ struct ovpn_nonce_tail nonce_tail_xmit; ++ struct ovpn_nonce_tail nonce_tail_recv; + -+config MK10 -+ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10" -+ help -+ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50, -+ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor. -+ Enables use of some extended instructions, and passes appropriate -+ optimization flags to GCC. ++ struct ovpn_pktid_recv pid_recv ____cacheline_aligned_in_smp; ++ struct ovpn_pktid_xmit pid_xmit ____cacheline_aligned_in_smp; ++ struct kref refcount; ++ struct rcu_head rcu; ++}; + -+config MBARCELONA -+ bool "AMD Barcelona" -+ help -+ Select this for AMD Family 10h Barcelona processors. ++struct ovpn_crypto_state { ++ struct ovpn_crypto_key_slot __rcu *primary; ++ struct ovpn_crypto_key_slot __rcu *secondary; + -+ Enables -march=barcelona ++ /* protects primary and secondary slots */ ++ struct mutex mutex; ++}; + -+config MBOBCAT -+ bool "AMD Bobcat" -+ help -+ Select this for AMD Family 14h Bobcat processors. ++static inline bool ovpn_crypto_key_slot_hold(struct ovpn_crypto_key_slot *ks) ++{ ++ return kref_get_unless_zero(&ks->refcount); ++} + -+ Enables -march=btver1 ++static inline void ovpn_crypto_state_init(struct ovpn_crypto_state *cs) ++{ ++ RCU_INIT_POINTER(cs->primary, NULL); ++ RCU_INIT_POINTER(cs->secondary, NULL); ++ mutex_init(&cs->mutex); ++} + -+config MJAGUAR -+ bool "AMD Jaguar" -+ help -+ Select this for AMD Family 16h Jaguar processors. ++static inline struct ovpn_crypto_key_slot * ++ovpn_crypto_key_id_to_slot(const struct ovpn_crypto_state *cs, u8 key_id) ++{ ++ struct ovpn_crypto_key_slot *ks; + -+ Enables -march=btver2 ++ if (unlikely(!cs)) ++ return NULL; + -+config MBULLDOZER -+ bool "AMD Bulldozer" -+ help -+ Select this for AMD Family 15h Bulldozer processors. ++ rcu_read_lock(); ++ ks = rcu_dereference(cs->primary); ++ if (ks && ks->key_id == key_id) { ++ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) ++ ks = NULL; ++ goto out; ++ } + -+ Enables -march=bdver1 ++ ks = rcu_dereference(cs->secondary); ++ if (ks && ks->key_id == key_id) { ++ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) ++ ks = NULL; ++ goto out; ++ } + -+config MPILEDRIVER -+ bool "AMD Piledriver" -+ help -+ Select this for AMD Family 15h Piledriver processors. ++ /* when both key slots are occupied but no matching key ID is found, ks has to be reset to ++ * NULL to avoid carrying a stale pointer ++ */ ++ ks = NULL; ++out: ++ rcu_read_unlock(); + -+ Enables -march=bdver2 ++ return ks; ++} + -+config MSTEAMROLLER -+ bool "AMD Steamroller" -+ help -+ Select this for AMD Family 15h Steamroller processors. ++static inline struct ovpn_crypto_key_slot * ++ovpn_crypto_key_slot_primary(const struct ovpn_crypto_state *cs) ++{ ++ struct ovpn_crypto_key_slot *ks; + -+ Enables -march=bdver3 ++ rcu_read_lock(); ++ ks = rcu_dereference(cs->primary); ++ if (unlikely(ks && !ovpn_crypto_key_slot_hold(ks))) ++ ks = NULL; ++ rcu_read_unlock(); + -+config MEXCAVATOR -+ bool "AMD Excavator" -+ help -+ Select this for AMD Family 15h Excavator processors. ++ return ks; ++} + -+ Enables -march=bdver4 ++void ovpn_crypto_key_slot_release(struct kref *kref); + -+config MZEN -+ bool "AMD Zen" -+ help -+ Select this for AMD Family 17h Zen processors. ++static inline void ovpn_crypto_key_slot_put(struct ovpn_crypto_key_slot *ks) ++{ ++ kref_put(&ks->refcount, ovpn_crypto_key_slot_release); ++} + -+ Enables -march=znver1 ++int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, ++ const struct ovpn_peer_key_reset *pkr); + -+config MZEN2 -+ bool "AMD Zen 2" -+ help -+ Select this for AMD Family 17h Zen 2 processors. ++void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, ++ enum ovpn_key_slot slot); + -+ Enables -march=znver2 ++void ovpn_crypto_state_release(struct ovpn_crypto_state *cs); + -+config MZEN3 -+ bool "AMD Zen 3" -+ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ help -+ Select this for AMD Family 19h Zen 3 processors. ++void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs); + -+ Enables -march=znver3 ++void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs); + - config MCRUSOE - bool "Crusoe" - depends on X86_32 -@@ -270,7 +356,7 @@ config MPSC - in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one. - - config MCORE2 -- bool "Core 2/newer Xeon" -+ bool "Intel Core 2" - help - - Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and -@@ -278,6 +364,8 @@ config MCORE2 - family in /proc/cpuinfo. Newer ones have 6 and older ones 15 - (not a typo) - -+ Enables -march=core2 ++#endif /* _NET_OVPN_DCO_OVPNCRYPTO_H_ */ +diff --git a/drivers/net/ovpn-dco/crypto_aead.c b/drivers/net/ovpn-dco/crypto_aead.c +new file mode 100644 +index 000000000000..c21bff90d748 +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto_aead.c +@@ -0,0 +1,367 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + - config MATOM - bool "Intel Atom" - help -@@ -287,6 +375,182 @@ config MATOM - accordingly optimized code. Use a recent GCC with specific Atom - support in order to fully benefit from selecting this option. - -+config MNEHALEM -+ bool "Intel Nehalem" -+ select X86_P6_NOP -+ help ++#include "crypto_aead.h" ++#include "crypto.h" ++#include "pktid.h" ++#include "proto.h" ++#include "skb.h" + -+ Select this for 1st Gen Core processors in the Nehalem family. ++#include ++#include ++#include + -+ Enables -march=nehalem ++#define AUTH_TAG_SIZE 16 + -+config MWESTMERE -+ bool "Intel Westmere" -+ select X86_P6_NOP -+ help ++static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) ++{ ++ return OVPN_OP_SIZE_V2 + /* OP header size */ ++ 4 + /* Packet ID */ ++ crypto_aead_authsize(ks->encrypt); /* Auth Tag */ ++} + -+ Select this for the Intel Westmere formerly Nehalem-C family. ++int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id) ++{ ++ const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); ++ const unsigned int head_size = ovpn_aead_encap_overhead(ks); ++ struct scatterlist sg[MAX_SKB_FRAGS + 2]; ++ DECLARE_CRYPTO_WAIT(wait); ++ struct aead_request *req; ++ struct sk_buff *trailer; ++ u8 iv[NONCE_SIZE]; ++ int nfrags, ret; ++ u32 pktid, op; + -+ Enables -march=westmere ++ /* Sample AEAD header format: ++ * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... ++ * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] ++ * [4-byte ++ * IV head] ++ */ + -+config MSILVERMONT -+ bool "Intel Silvermont" -+ select X86_P6_NOP -+ help ++ /* check that there's enough headroom in the skb for packet ++ * encapsulation, after adding network header and encryption overhead ++ */ ++ if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM + head_size))) ++ return -ENOBUFS; + -+ Select this for the Intel Silvermont platform. ++ /* get number of skb frags and ensure that packet data is writable */ ++ nfrags = skb_cow_data(skb, 0, &trailer); ++ if (unlikely(nfrags < 0)) ++ return nfrags; + -+ Enables -march=silvermont ++ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) ++ return -ENOSPC; + -+config MGOLDMONT -+ bool "Intel Goldmont" -+ select X86_P6_NOP -+ help ++ req = aead_request_alloc(ks->encrypt, GFP_KERNEL); ++ if (unlikely(!req)) ++ return -ENOMEM; + -+ Select this for the Intel Goldmont platform including Apollo Lake and Denverton. ++ /* sg table: ++ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), ++ * 1, 2, 3, ..., n: payload, ++ * n+1: auth_tag (len=tag_size) ++ */ ++ sg_init_table(sg, nfrags + 2); + -+ Enables -march=goldmont ++ /* build scatterlist to encrypt packet payload */ ++ ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); ++ if (unlikely(nfrags != ret)) { ++ ret = -EINVAL; ++ goto free_req; ++ } + -+config MGOLDMONTPLUS -+ bool "Intel Goldmont Plus" -+ select X86_P6_NOP -+ help ++ /* append auth_tag onto scatterlist */ ++ __skb_push(skb, tag_size); ++ sg_set_buf(sg + nfrags + 1, skb->data, tag_size); + -+ Select this for the Intel Goldmont Plus platform including Gemini Lake. ++ /* obtain packet ID, which is used both as a first ++ * 4 bytes of nonce and last 4 bytes of associated data. ++ */ ++ ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); ++ if (unlikely(ret < 0)) ++ goto free_req; + -+ Enables -march=goldmont-plus ++ /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes nonce */ ++ ovpn_pktid_aead_write(pktid, &ks->nonce_tail_xmit, iv); + -+config MSANDYBRIDGE -+ bool "Intel Sandy Bridge" -+ select X86_P6_NOP -+ help ++ /* make space for packet id and push it to the front */ ++ __skb_push(skb, NONCE_WIRE_SIZE); ++ memcpy(skb->data, iv, NONCE_WIRE_SIZE); ++ ++ /* add packet op as head of additional data */ ++ op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer_id); ++ __skb_push(skb, OVPN_OP_SIZE_V2); ++ BUILD_BUG_ON(sizeof(op) != OVPN_OP_SIZE_V2); ++ *((__force __be32 *)skb->data) = htonl(op); + -+ Select this for 2nd Gen Core processors in the Sandy Bridge family. ++ /* AEAD Additional data */ ++ sg_set_buf(sg, skb->data, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); + -+ Enables -march=sandybridge ++ /* setup async crypto operation */ ++ aead_request_set_tfm(req, ks->encrypt); ++ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | ++ CRYPTO_TFM_REQ_MAY_SLEEP, ++ crypto_req_done, &wait); ++ aead_request_set_crypt(req, sg, sg, skb->len - head_size, iv); ++ aead_request_set_ad(req, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); + -+config MIVYBRIDGE -+ bool "Intel Ivy Bridge" -+ select X86_P6_NOP -+ help ++ /* encrypt it */ ++ ret = crypto_wait_req(crypto_aead_encrypt(req), &wait); ++ if (ret < 0) ++ net_err_ratelimited("%s: encrypt failed: %d\n", __func__, ret); + -+ Select this for 3rd Gen Core processors in the Ivy Bridge family. ++free_req: ++ aead_request_free(req); ++ return ret; ++} + -+ Enables -march=ivybridge ++int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) ++{ ++ const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); ++ struct scatterlist sg[MAX_SKB_FRAGS + 2]; ++ int ret, payload_len, nfrags; ++ u8 *sg_data, iv[NONCE_SIZE]; ++ unsigned int payload_offset; ++ DECLARE_CRYPTO_WAIT(wait); ++ struct aead_request *req; ++ struct sk_buff *trailer; ++ unsigned int sg_len; ++ __be32 *pid; + -+config MHASWELL -+ bool "Intel Haswell" -+ select X86_P6_NOP -+ help ++ payload_offset = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE + tag_size; ++ payload_len = skb->len - payload_offset; + -+ Select this for 4th Gen Core processors in the Haswell family. ++ /* sanity check on packet size, payload size must be >= 0 */ ++ if (unlikely(payload_len < 0)) ++ return -EINVAL; + -+ Enables -march=haswell ++ /* Prepare the skb data buffer to be accessed up until the auth tag. ++ * This is required because this area is directly mapped into the sg list. ++ */ ++ if (unlikely(!pskb_may_pull(skb, payload_offset))) ++ return -ENODATA; + -+config MBROADWELL -+ bool "Intel Broadwell" -+ select X86_P6_NOP -+ help ++ /* get number of skb frags and ensure that packet data is writable */ ++ nfrags = skb_cow_data(skb, 0, &trailer); ++ if (unlikely(nfrags < 0)) ++ return nfrags; + -+ Select this for 5th Gen Core processors in the Broadwell family. ++ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) ++ return -ENOSPC; + -+ Enables -march=broadwell ++ req = aead_request_alloc(ks->decrypt, GFP_KERNEL); ++ if (unlikely(!req)) ++ return -ENOMEM; + -+config MSKYLAKE -+ bool "Intel Skylake" -+ select X86_P6_NOP -+ help ++ /* sg table: ++ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), ++ * 1, 2, 3, ..., n: payload, ++ * n+1: auth_tag (len=tag_size) ++ */ ++ sg_init_table(sg, nfrags + 2); + -+ Select this for 6th Gen Core processors in the Skylake family. ++ /* packet op is head of additional data */ ++ sg_data = skb->data; ++ sg_len = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE; ++ sg_set_buf(sg, sg_data, sg_len); + -+ Enables -march=skylake ++ /* build scatterlist to decrypt packet payload */ ++ ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); ++ if (unlikely(nfrags != ret)) { ++ ret = -EINVAL; ++ goto free_req; ++ } + -+config MSKYLAKEX -+ bool "Intel Skylake X" -+ select X86_P6_NOP -+ help ++ /* append auth_tag onto scatterlist */ ++ sg_set_buf(sg + nfrags + 1, skb->data + sg_len, tag_size); + -+ Select this for 6th Gen Core processors in the Skylake X family. ++ /* copy nonce into IV buffer */ ++ memcpy(iv, skb->data + OVPN_OP_SIZE_V2, NONCE_WIRE_SIZE); ++ memcpy(iv + NONCE_WIRE_SIZE, ks->nonce_tail_recv.u8, ++ sizeof(struct ovpn_nonce_tail)); + -+ Enables -march=skylake-avx512 ++ /* setup async crypto operation */ ++ aead_request_set_tfm(req, ks->decrypt); ++ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | ++ CRYPTO_TFM_REQ_MAY_SLEEP, ++ crypto_req_done, &wait); ++ aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); + -+config MCANNONLAKE -+ bool "Intel Cannon Lake" -+ select X86_P6_NOP -+ help ++ aead_request_set_ad(req, NONCE_WIRE_SIZE + OVPN_OP_SIZE_V2); + -+ Select this for 8th Gen Core processors ++ /* decrypt it */ ++ ret = crypto_wait_req(crypto_aead_decrypt(req), &wait); ++ if (ret < 0) { ++ net_err_ratelimited("%s: decrypt failed: %d\n", __func__, ret); ++ goto free_req; ++ } + -+ Enables -march=cannonlake ++ /* PID sits after the op */ ++ pid = (__force __be32 *)(skb->data + OVPN_OP_SIZE_V2); ++ ret = ovpn_pktid_recv(&ks->pid_recv, ntohl(*pid), 0); ++ if (unlikely(ret < 0)) ++ goto free_req; + -+config MICELAKE -+ bool "Intel Ice Lake" -+ select X86_P6_NOP -+ help ++ /* point to encapsulated IP packet */ ++ __skb_pull(skb, payload_offset); + -+ Select this for 10th Gen Core processors in the Ice Lake family. ++free_req: ++ aead_request_free(req); ++ return ret; ++} + -+ Enables -march=icelake-client ++/* Initialize a struct crypto_aead object */ ++struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, ++ const unsigned char *key, unsigned int keylen) ++{ ++ struct crypto_aead *aead; ++ int ret; + -+config MCASCADELAKE -+ bool "Intel Cascade Lake" -+ select X86_P6_NOP -+ help ++ aead = crypto_alloc_aead(alg_name, 0, 0); ++ if (IS_ERR(aead)) { ++ ret = PTR_ERR(aead); ++ pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); ++ aead = NULL; ++ goto error; ++ } + -+ Select this for Xeon processors in the Cascade Lake family. ++ ret = crypto_aead_setkey(aead, key, keylen); ++ if (ret) { ++ pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, keylen, ret); ++ goto error; ++ } + -+ Enables -march=cascadelake ++ ret = crypto_aead_setauthsize(aead, AUTH_TAG_SIZE); ++ if (ret) { ++ pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, ret); ++ goto error; ++ } + -+config MCOOPERLAKE -+ bool "Intel Cooper Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) -+ select X86_P6_NOP -+ help ++ /* basic AEAD assumption */ ++ if (crypto_aead_ivsize(aead) != NONCE_SIZE) { ++ pr_err("%s IV size must be %d\n", title, NONCE_SIZE); ++ ret = -EINVAL; ++ goto error; ++ } + -+ Select this for Xeon processors in the Cooper Lake family. ++ pr_debug("********* Cipher %s (%s)\n", alg_name, title); ++ pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); ++ pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); ++ pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); ++ pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); ++ pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); + -+ Enables -march=cooperlake ++ return aead; + -+config MTIGERLAKE -+ bool "Intel Tiger Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000) -+ select X86_P6_NOP -+ help ++error: ++ crypto_free_aead(aead); ++ return ERR_PTR(ret); ++} + -+ Select this for third-generation 10 nm process processors in the Tiger Lake family. ++void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks) ++{ ++ if (!ks) ++ return; + -+ Enables -march=tigerlake ++ crypto_free_aead(ks->encrypt); ++ crypto_free_aead(ks->decrypt); ++ kfree(ks); ++} + -+config MSAPPHIRERAPIDS -+ bool "Intel Sapphire Rapids" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help ++static struct ovpn_crypto_key_slot * ++ovpn_aead_crypto_key_slot_init(enum ovpn_cipher_alg alg, ++ const unsigned char *encrypt_key, ++ unsigned int encrypt_keylen, ++ const unsigned char *decrypt_key, ++ unsigned int decrypt_keylen, ++ const unsigned char *encrypt_nonce_tail, ++ unsigned int encrypt_nonce_tail_len, ++ const unsigned char *decrypt_nonce_tail, ++ unsigned int decrypt_nonce_tail_len, ++ u16 key_id) ++{ ++ struct ovpn_crypto_key_slot *ks = NULL; ++ const char *alg_name; ++ int ret; + -+ Select this for third-generation 10 nm process processors in the Sapphire Rapids family. ++ /* validate crypto alg */ ++ switch (alg) { ++ case OVPN_CIPHER_ALG_AES_GCM: ++ alg_name = "gcm(aes)"; ++ break; ++ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: ++ alg_name = "rfc7539(chacha20,poly1305)"; ++ break; ++ default: ++ return ERR_PTR(-EOPNOTSUPP); ++ } + -+ Enables -march=sapphirerapids ++ /* build the key slot */ ++ ks = kmalloc(sizeof(*ks), GFP_KERNEL); ++ if (!ks) ++ return ERR_PTR(-ENOMEM); + -+config MROCKETLAKE -+ bool "Intel Rocket Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help ++ ks->encrypt = NULL; ++ ks->decrypt = NULL; ++ kref_init(&ks->refcount); ++ ks->key_id = key_id; + -+ Select this for eleventh-generation processors in the Rocket Lake family. ++ ks->encrypt = ovpn_aead_init("encrypt", alg_name, encrypt_key, ++ encrypt_keylen); ++ if (IS_ERR(ks->encrypt)) { ++ ret = PTR_ERR(ks->encrypt); ++ ks->encrypt = NULL; ++ goto destroy_ks; ++ } + -+ Enables -march=rocketlake ++ ks->decrypt = ovpn_aead_init("decrypt", alg_name, decrypt_key, ++ decrypt_keylen); ++ if (IS_ERR(ks->decrypt)) { ++ ret = PTR_ERR(ks->decrypt); ++ ks->decrypt = NULL; ++ goto destroy_ks; ++ } + -+config MALDERLAKE -+ bool "Intel Alder Lake" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ select X86_P6_NOP -+ help ++ if (sizeof(struct ovpn_nonce_tail) != encrypt_nonce_tail_len || ++ sizeof(struct ovpn_nonce_tail) != decrypt_nonce_tail_len) { ++ ret = -EINVAL; ++ goto destroy_ks; ++ } + -+ Select this for twelfth-generation processors in the Alder Lake family. ++ memcpy(ks->nonce_tail_xmit.u8, encrypt_nonce_tail, ++ sizeof(struct ovpn_nonce_tail)); ++ memcpy(ks->nonce_tail_recv.u8, decrypt_nonce_tail, ++ sizeof(struct ovpn_nonce_tail)); + -+ Enables -march=alderlake ++ /* init packet ID generation/validation */ ++ ovpn_pktid_xmit_init(&ks->pid_xmit); ++ ovpn_pktid_recv_init(&ks->pid_recv); + - config GENERIC_CPU - bool "Generic-x86-64" - depends on X86_64 -@@ -294,6 +558,50 @@ config GENERIC_CPU - Generic x86-64 CPU. - Run equally well on all x86-64 CPUs. - -+config GENERIC_CPU2 -+ bool "Generic-x86-64-v2" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64 CPU. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v2. ++ return ks; + -+config GENERIC_CPU3 -+ bool "Generic-x86-64-v3" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64-v3 CPU with v3 instructions. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v3. ++destroy_ks: ++ ovpn_aead_crypto_key_slot_destroy(ks); ++ return ERR_PTR(ret); ++} + -+config GENERIC_CPU4 -+ bool "Generic-x86-64-v4" -+ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) -+ depends on X86_64 -+ help -+ Generic x86-64 CPU with v4 instructions. -+ Run equally well on all x86-64 CPUs with min support of x86-64-v4. ++struct ovpn_crypto_key_slot * ++ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) ++{ ++ return ovpn_aead_crypto_key_slot_init(kc->cipher_alg, ++ kc->encrypt.cipher_key, ++ kc->encrypt.cipher_key_size, ++ kc->decrypt.cipher_key, ++ kc->decrypt.cipher_key_size, ++ kc->encrypt.nonce_tail, ++ kc->encrypt.nonce_tail_size, ++ kc->decrypt.nonce_tail, ++ kc->decrypt.nonce_tail_size, ++ kc->key_id); ++} +diff --git a/drivers/net/ovpn-dco/crypto_aead.h b/drivers/net/ovpn-dco/crypto_aead.h +new file mode 100644 +index 000000000000..1e3054e7d5a4 +--- /dev/null ++++ b/drivers/net/ovpn-dco/crypto_aead.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+config MNATIVE_INTEL -+ bool "Intel-Native optimizations autodetected by the compiler" -+ help ++#ifndef _NET_OVPN_DCO_OVPNAEAD_H_ ++#define _NET_OVPN_DCO_OVPNAEAD_H_ + -+ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects -+ the optimum settings to use based on your processor. Do NOT use this -+ for AMD CPUs. Intel Only! ++#include "crypto.h" + -+ Enables -march=native ++#include ++#include + -+config MNATIVE_AMD -+ bool "AMD-Native optimizations autodetected by the compiler" -+ help ++struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, ++ const unsigned char *key, unsigned int keylen); + -+ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects -+ the optimum settings to use based on your processor. Do NOT use this -+ for Intel CPUs. AMD Only! ++int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id); ++int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); + -+ Enables -march=native ++struct ovpn_crypto_key_slot *ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc); ++void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks); + - endchoice - - config X86_GENERIC -@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT - config X86_L1_CACHE_SHIFT - int - default "7" if MPENTIUM4 || MPSC -- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU -+ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4 - default "4" if MELAN || M486SX || M486 || MGEODEGX1 - default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX - -@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16 - - config X86_INTEL_USERCOPY - def_bool y -- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 -+ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL - - config X86_USE_PPRO_CHECKSUM - def_bool y -- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM -+ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD - - # - # P6_NOPs are a relatively minor optimization that require a family >= -@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM - config X86_P6_NOP - def_bool y - depends on X86_64 -- depends on (MCORE2 || MPENTIUM4 || MPSC) -+ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL) - - config X86_TSC - def_bool y -- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64 -+ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64 - - config X86_CMPXCHG64 - def_bool y -- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 -+ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD - - # this should be set for all -march=.. options where the compiler - # generates cmov. - config X86_CMOV - def_bool y -- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX) -+ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) - - config X86_MINIMUM_CPU_FAMILY - int - default "64" if X86_64 -- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8) -+ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) - default "5" if X86_32 && X86_CMPXCHG64 - default "4" - -diff --git a/arch/x86/Makefile b/arch/x86/Makefile -index bafbd905e6e7..476ab926c33e 100644 ---- a/arch/x86/Makefile -+++ b/arch/x86/Makefile -@@ -150,8 +150,44 @@ else - # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu) - cflags-$(CONFIG_MK8) += -march=k8 - cflags-$(CONFIG_MPSC) += -march=nocona -- cflags-$(CONFIG_MCORE2) += -march=core2 -- cflags-$(CONFIG_MATOM) += -march=atom -+ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3 -+ cflags-$(CONFIG_MK10) += -march=amdfam10 -+ cflags-$(CONFIG_MBARCELONA) += -march=barcelona -+ cflags-$(CONFIG_MBOBCAT) += -march=btver1 -+ cflags-$(CONFIG_MJAGUAR) += -march=btver2 -+ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1 -+ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm -+ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm -+ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm -+ cflags-$(CONFIG_MZEN) += -march=znver1 -+ cflags-$(CONFIG_MZEN2) += -march=znver2 -+ cflags-$(CONFIG_MZEN3) += -march=znver3 -+ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native -+ cflags-$(CONFIG_MNATIVE_AMD) += -march=native -+ cflags-$(CONFIG_MATOM) += -march=bonnell -+ cflags-$(CONFIG_MCORE2) += -march=core2 -+ cflags-$(CONFIG_MNEHALEM) += -march=nehalem -+ cflags-$(CONFIG_MWESTMERE) += -march=westmere -+ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont -+ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont -+ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus -+ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge -+ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge -+ cflags-$(CONFIG_MHASWELL) += -march=haswell -+ cflags-$(CONFIG_MBROADWELL) += -march=broadwell -+ cflags-$(CONFIG_MSKYLAKE) += -march=skylake -+ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512 -+ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake -+ cflags-$(CONFIG_MICELAKE) += -march=icelake-client -+ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake -+ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake -+ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake -+ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids -+ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake -+ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake -+ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2 -+ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3 -+ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4 - cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic - KBUILD_CFLAGS += $(cflags-y) - -diff --git a/arch/x86/Makefile.postlink b/arch/x86/Makefile.postlink ++#endif /* _NET_OVPN_DCO_OVPNAEAD_H_ */ +diff --git a/drivers/net/ovpn-dco/main.c b/drivers/net/ovpn-dco/main.c new file mode 100644 -index 000000000000..4650aaf6d8b3 +index 000000000000..4eb90ea7a500 --- /dev/null -+++ b/arch/x86/Makefile.postlink -@@ -0,0 +1,41 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# =========================================================================== -+# Post-link x86 pass -+# =========================================================================== -+# -+# 1. Separate relocations from vmlinux into vmlinux.relocs. -+# 2. Strip relocations from vmlinux. ++++ b/drivers/net/ovpn-dco/main.c +@@ -0,0 +1,271 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ + -+PHONY := __archpost -+__archpost: ++#include "main.h" ++ ++#include "ovpn.h" ++#include "ovpnstruct.h" ++#include "netlink.h" + -+-include include/config/auto.conf -+include scripts/Kbuild.include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+CMD_RELOCS = arch/x86/tools/relocs -+quiet_cmd_relocs = RELOCS $@.relocs -+ cmd_relocs = $(CMD_RELOCS) $@ > $@.relocs;$(CMD_RELOCS) --abs-relocs $@ ++#include + -+quiet_cmd_strip_relocs = RSTRIP $@ -+ cmd_strip_relocs = objcopy --remove-relocations='*' $@ ++/* Driver info */ ++#define DRV_NAME "ovpn-dco" ++#define DRV_VERSION OVPN_DCO_VERSION ++#define DRV_DESCRIPTION "OpenVPN data channel offload (ovpn-dco)" ++#define DRV_COPYRIGHT "(C) 2020-2022 OpenVPN, Inc." + -+# `@true` prevents complaint when there is nothing to be done ++static void ovpn_struct_free(struct net_device *net) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(net); + -+vmlinux: FORCE -+ @true -+ifeq ($(CONFIG_X86_NEED_RELOCS),y) -+ $(call cmd,relocs) -+ $(call cmd,strip_relocs) -+endif ++ security_tun_dev_free_security(ovpn->security); ++ free_percpu(net->tstats); ++ flush_workqueue(ovpn->crypto_wq); ++ flush_workqueue(ovpn->events_wq); ++ destroy_workqueue(ovpn->crypto_wq); ++ destroy_workqueue(ovpn->events_wq); ++ rcu_barrier(); ++} + -+%.ko: FORCE -+ @true ++/* Net device open */ ++static int ovpn_net_open(struct net_device *dev) ++{ ++ struct in_device *dev_v4 = __in_dev_get_rtnl(dev); + -+clean: -+ @rm -f vmlinux.relocs ++ if (dev_v4) { ++ /* disable redirects as Linux gets confused by ovpn-dco handling same-LAN routing */ ++ IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); ++ IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; ++ } + -+PHONY += FORCE clean ++ netif_tx_start_all_queues(dev); ++ return 0; ++} + -+FORCE: ++/* Net device stop -- called prior to device unload */ ++static int ovpn_net_stop(struct net_device *dev) ++{ ++ netif_tx_stop_all_queues(dev); ++ return 0; ++} + -+.PHONY: $(PHONY) -diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile -index 35ce1a64068b..eba7709d75ae 100644 ---- a/arch/x86/boot/compressed/Makefile -+++ b/arch/x86/boot/compressed/Makefile -@@ -120,14 +120,12 @@ $(obj)/vmlinux.bin: vmlinux FORCE - - targets += $(patsubst $(obj)/%,%,$(vmlinux-objs-y)) vmlinux.bin.all vmlinux.relocs - --CMD_RELOCS = arch/x86/tools/relocs --quiet_cmd_relocs = RELOCS $@ -- cmd_relocs = $(CMD_RELOCS) $< > $@;$(CMD_RELOCS) --abs-relocs $< --$(obj)/vmlinux.relocs: vmlinux FORCE -- $(call if_changed,relocs) -+# vmlinux.relocs is created by the vmlinux postlink step. -+vmlinux.relocs: vmlinux -+ @true - - vmlinux.bin.all-y := $(obj)/vmlinux.bin --vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs -+vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += vmlinux.relocs - - $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,gzip) -diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl -index 320480a8db4f..331aaf1a782f 100644 ---- a/arch/x86/entry/syscalls/syscall_32.tbl -+++ b/arch/x86/entry/syscalls/syscall_32.tbl -@@ -455,3 +455,4 @@ - 448 i386 process_mrelease sys_process_mrelease - 449 i386 futex_waitv sys_futex_waitv - 450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node -+451 i386 pmadv_ksm sys_pmadv_ksm -diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl -index c84d12608cd2..14902db4c01f 100644 ---- a/arch/x86/entry/syscalls/syscall_64.tbl -+++ b/arch/x86/entry/syscalls/syscall_64.tbl -@@ -372,6 +372,7 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm - - # - # Due to a historical design error, certain syscalls are numbered differently -diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h -index 6674bdb096f3..e278872e9187 100644 ---- a/arch/x86/include/asm/msr-index.h -+++ b/arch/x86/include/asm/msr-index.h -@@ -569,6 +569,7 @@ - #define MSR_AMD_CPPC_CAP2 0xc00102b2 - #define MSR_AMD_CPPC_REQ 0xc00102b3 - #define MSR_AMD_CPPC_STATUS 0xc00102b4 -+#define MSR_AMD_CPPC_HW_CTL 0xc0010015 - - #define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) - #define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) -@@ -579,12 +580,18 @@ - #define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) - #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) - #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) -+#define AMD_CPPC_PRECISION_BOOST_BIT 25 -+#define AMD_CPPC_PRECISION_BOOST_ENABLED BIT_ULL(AMD_CPPC_PRECISION_BOOST_BIT) - - /* AMD Performance Counter Global Status and Control MSRs */ - #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 - #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 - #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 - -+#define AMD_CPPC_EPP_PERFORMANCE 0x00 -+#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 -+#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF -+#define AMD_CPPC_EPP_POWERSAVE 0xFF - /* Fam 17h MSRs */ - #define MSR_F17H_IRPERF 0xc00000e9 - -diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h -index 458c891a8273..d86eb1ebf59f 100644 ---- a/arch/x86/include/asm/topology.h -+++ b/arch/x86/include/asm/topology.h -@@ -175,6 +175,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled; - - /* Interface to set priority of a cpu */ - void sched_set_itmt_core_prio(int prio, int core_cpu); -+void sched_set_itmt_power_ratio(int power_ratio, int core_cpu); - - /* Interface to notify scheduler that system supports ITMT */ - int sched_set_itmt_support(void); -diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h -index 57b1a7034c64..e2c45674f989 100644 ---- a/arch/x86/include/asm/vdso/processor.h -+++ b/arch/x86/include/asm/vdso/processor.h -@@ -10,7 +10,7 @@ - /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ - static __always_inline void rep_nop(void) - { -- asm volatile("rep; nop" ::: "memory"); -+ asm volatile("lfence" ::: "memory"); - } - - static __always_inline void cpu_relax(void) -diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h -index 75884d2cdec3..4e6a08d4c7e5 100644 ---- a/arch/x86/include/asm/vermagic.h -+++ b/arch/x86/include/asm/vermagic.h -@@ -17,6 +17,48 @@ - #define MODULE_PROC_FAMILY "586MMX " - #elif defined CONFIG_MCORE2 - #define MODULE_PROC_FAMILY "CORE2 " -+#elif defined CONFIG_MNATIVE_INTEL -+#define MODULE_PROC_FAMILY "NATIVE_INTEL " -+#elif defined CONFIG_MNATIVE_AMD -+#define MODULE_PROC_FAMILY "NATIVE_AMD " -+#elif defined CONFIG_MNEHALEM -+#define MODULE_PROC_FAMILY "NEHALEM " -+#elif defined CONFIG_MWESTMERE -+#define MODULE_PROC_FAMILY "WESTMERE " -+#elif defined CONFIG_MSILVERMONT -+#define MODULE_PROC_FAMILY "SILVERMONT " -+#elif defined CONFIG_MGOLDMONT -+#define MODULE_PROC_FAMILY "GOLDMONT " -+#elif defined CONFIG_MGOLDMONTPLUS -+#define MODULE_PROC_FAMILY "GOLDMONTPLUS " -+#elif defined CONFIG_MSANDYBRIDGE -+#define MODULE_PROC_FAMILY "SANDYBRIDGE " -+#elif defined CONFIG_MIVYBRIDGE -+#define MODULE_PROC_FAMILY "IVYBRIDGE " -+#elif defined CONFIG_MHASWELL -+#define MODULE_PROC_FAMILY "HASWELL " -+#elif defined CONFIG_MBROADWELL -+#define MODULE_PROC_FAMILY "BROADWELL " -+#elif defined CONFIG_MSKYLAKE -+#define MODULE_PROC_FAMILY "SKYLAKE " -+#elif defined CONFIG_MSKYLAKEX -+#define MODULE_PROC_FAMILY "SKYLAKEX " -+#elif defined CONFIG_MCANNONLAKE -+#define MODULE_PROC_FAMILY "CANNONLAKE " -+#elif defined CONFIG_MICELAKE -+#define MODULE_PROC_FAMILY "ICELAKE " -+#elif defined CONFIG_MCASCADELAKE -+#define MODULE_PROC_FAMILY "CASCADELAKE " -+#elif defined CONFIG_MCOOPERLAKE -+#define MODULE_PROC_FAMILY "COOPERLAKE " -+#elif defined CONFIG_MTIGERLAKE -+#define MODULE_PROC_FAMILY "TIGERLAKE " -+#elif defined CONFIG_MSAPPHIRERAPIDS -+#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS " -+#elif defined CONFIG_ROCKETLAKE -+#define MODULE_PROC_FAMILY "ROCKETLAKE " -+#elif defined CONFIG_MALDERLAKE -+#define MODULE_PROC_FAMILY "ALDERLAKE " - #elif defined CONFIG_MATOM - #define MODULE_PROC_FAMILY "ATOM " - #elif defined CONFIG_M686 -@@ -35,6 +77,30 @@ - #define MODULE_PROC_FAMILY "K7 " - #elif defined CONFIG_MK8 - #define MODULE_PROC_FAMILY "K8 " -+#elif defined CONFIG_MK8SSE3 -+#define MODULE_PROC_FAMILY "K8SSE3 " -+#elif defined CONFIG_MK10 -+#define MODULE_PROC_FAMILY "K10 " -+#elif defined CONFIG_MBARCELONA -+#define MODULE_PROC_FAMILY "BARCELONA " -+#elif defined CONFIG_MBOBCAT -+#define MODULE_PROC_FAMILY "BOBCAT " -+#elif defined CONFIG_MBULLDOZER -+#define MODULE_PROC_FAMILY "BULLDOZER " -+#elif defined CONFIG_MPILEDRIVER -+#define MODULE_PROC_FAMILY "PILEDRIVER " -+#elif defined CONFIG_MSTEAMROLLER -+#define MODULE_PROC_FAMILY "STEAMROLLER " -+#elif defined CONFIG_MJAGUAR -+#define MODULE_PROC_FAMILY "JAGUAR " -+#elif defined CONFIG_MEXCAVATOR -+#define MODULE_PROC_FAMILY "EXCAVATOR " -+#elif defined CONFIG_MZEN -+#define MODULE_PROC_FAMILY "ZEN " -+#elif defined CONFIG_MZEN2 -+#define MODULE_PROC_FAMILY "ZEN2 " -+#elif defined CONFIG_MZEN3 -+#define MODULE_PROC_FAMILY "ZEN3 " - #elif defined CONFIG_MELAN - #define MODULE_PROC_FAMILY "ELAN " - #elif defined CONFIG_MCRUSOE -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index 62f6b8b7c4a5..f9c9b5850847 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -936,7 +936,9 @@ void __init alternative_instructions(void) - * Then patch alternatives, such that those paravirt calls that are in - * alternatives can be overwritten by their immediate fragments. - */ -+ printk("clr: Applying alternatives\n"); - apply_alternatives(__alt_instructions, __alt_instructions_end); -+ printk("clr: Applying alternatives done\n"); - - apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end); - -diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c -index fbaf12e43f41..c8c2d6f1a8ac 100644 ---- a/arch/x86/kernel/cpu/intel_epb.c -+++ b/arch/x86/kernel/cpu/intel_epb.c -@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev, - if (ret < 0) - return ret; - -+ /* update the ITMT scheduler logic to use the power policy data */ -+ /* scale the val up by 2 so the range is 224 - 256 */ -+ sched_set_itmt_power_ratio(256 - val * 2, cpu); ++/******************************************* ++ * ovpn ethtool ops ++ *******************************************/ + - return count; - } - -diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c -index ad57e0e4d674..e2ed33c2bd4d 100644 ---- a/arch/x86/kernel/cpu/microcode/core.c -+++ b/arch/x86/kernel/cpu/microcode/core.c -@@ -44,6 +44,8 @@ - - static struct microcode_ops *microcode_ops; - static bool dis_ucode_ldr = true; -+bool ucode_rollback = false; -+int enable_rollback = 0; - - bool initrd_gone; - -@@ -80,6 +82,26 @@ static u32 final_levels[] = { - 0, /* T-101 terminator */ - }; - -+static int __init ucode_setup(char *str) ++static int ovpn_get_link_ksettings(struct net_device *dev, ++ struct ethtool_link_ksettings *cmd) +{ -+ if (!str) -+ return -EINVAL; ++ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 0); ++ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, 0); ++ cmd->base.speed = SPEED_1000; ++ cmd->base.duplex = DUPLEX_FULL; ++ cmd->base.port = PORT_TP; ++ cmd->base.phy_address = 0; ++ cmd->base.transceiver = XCVR_INTERNAL; ++ cmd->base.autoneg = AUTONEG_DISABLE; + -+ while (*str) { -+ if (!strncmp(str, "rollback", 8)) { -+ enable_rollback = 1; -+ pr_info("Microcode Rollback Enabled\n"); -+ } -+ str += strcspn(str, ","); -+ while (*str == ',') -+ str++; -+ } + return 0; +} + -+__setup("ucode=", ucode_setup); ++static void ovpn_get_drvinfo(struct net_device *dev, ++ struct ethtool_drvinfo *info) ++{ ++ strscpy(info->driver, DRV_NAME, sizeof(info->driver)); ++ strscpy(info->version, DRV_VERSION, sizeof(info->version)); ++ strscpy(info->bus_info, "ovpn", sizeof(info->bus_info)); ++} + ++bool ovpn_dev_is_valid(const struct net_device *dev) ++{ ++ return dev->netdev_ops->ndo_start_xmit == ovpn_net_xmit; ++} + - /* - * Check the current patch level on this CPU. - * -@@ -512,6 +534,7 @@ static ssize_t reload_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t size) - { -+ struct cpuinfo_x86 *c = &boot_cpu_data; - enum ucode_state tmp_ret = UCODE_OK; - int bsp = boot_cpu_data.cpu_index; - unsigned long val; -@@ -521,7 +544,7 @@ static ssize_t reload_store(struct device *dev, - if (ret) - return ret; - -- if (val != 1) -+ if (!val || val > 2) - return size; - - cpus_read_lock(); -@@ -529,6 +552,20 @@ static ssize_t reload_store(struct device *dev, - ret = check_online_cpus(); - if (ret) - goto put; -+ /* -+ * Check if the vendor is Intel to permit reloading -+ * microcode even if the revision is unchanged. -+ * This is typically used during development of microcode -+ * and changing rev is a pain. -+ */ -+ if ((val == 2) && ((c->x86_vendor != X86_VENDOR_INTEL) || -+ !enable_rollback)) -+ return size; -+ else if (val == 2) { -+ mutex_lock(µcode_mutex); -+ ucode_rollback = true; -+ mutex_unlock(µcode_mutex); -+ } - - tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev, true); - if (tmp_ret != UCODE_NEW) -@@ -539,6 +576,7 @@ static ssize_t reload_store(struct device *dev, - mutex_unlock(µcode_mutex); - - put: -+ ucode_rollback = false; - cpus_read_unlock(); - - if (ret == 0) -diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c -index 025c8f0cd948..60473f577a25 100644 ---- a/arch/x86/kernel/cpu/microcode/intel.c -+++ b/arch/x86/kernel/cpu/microcode/intel.c -@@ -44,6 +44,7 @@ static struct microcode_intel *intel_ucode_patch; - - /* last level cache size per core */ - static int llc_size_per_core; -+extern bool ucode_rollback; - - /* - * Returns 1 if update has been found, 0 otherwise. -@@ -80,7 +81,7 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev - { - struct microcode_header_intel *mc_hdr = mc; - -- if (mc_hdr->rev <= new_rev) -+ if (!ucode_rollback && mc_hdr->rev <= new_rev) - return 0; - - return find_matching_signature(mc, csig, cpf); -@@ -120,7 +121,7 @@ static void save_microcode_patch(struct ucode_cpu_info *uci, void *data, unsigne - if (find_matching_signature(data, sig, pf)) { - prev_found = true; - -- if (mc_hdr->rev <= mc_saved_hdr->rev) -+ if (!ucode_rollback && mc_hdr->rev <= mc_saved_hdr->rev) - continue; - - p = memdup_patch(data, size); -@@ -649,7 +650,7 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci) - - phdr = (struct microcode_header_intel *)iter->data; - -- if (phdr->rev <= uci->cpu_sig.rev) -+ if (!ucode_rollback && phdr->rev <= uci->cpu_sig.rev) - continue; - - if (!find_matching_signature(phdr, -@@ -734,10 +735,11 @@ static enum ucode_state apply_microcode_intel(int cpu) - * already. - */ - rev = intel_get_microcode_revision(); -- if (rev >= mc->hdr.rev) { -+ if (!ucode_rollback && rev >= mc->hdr.rev) { - ret = UCODE_OK; - goto out; -- } -+ } else if (ucode_rollback) -+ ret = UCODE_OK; - - /* - * Writeback and invalidate caches before updating microcode to avoid -@@ -756,7 +758,7 @@ static enum ucode_state apply_microcode_intel(int cpu) - return UCODE_ERROR; - } - -- if (bsp && rev != prev_rev) { -+ if (bsp && ((rev != prev_rev) || ucode_rollback)) { - pr_info("updated to revision 0x%x, date = %04x-%02x-%02x\n", - rev, - mc->hdr.date & 0xffff, -diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c -index 9ff480e94511..d4326e050fb7 100644 ---- a/arch/x86/kernel/itmt.c -+++ b/arch/x86/kernel/itmt.c -@@ -25,6 +25,7 @@ - - static DEFINE_MUTEX(itmt_update_mutex); - DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); -+DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio); - - /* Boolean to track if system has ITMT capabilities */ - static bool __read_mostly sched_itmt_capable; -@@ -169,7 +170,12 @@ void sched_clear_itmt_support(void) - - int arch_asym_cpu_priority(int cpu) - { -- return per_cpu(sched_core_priority, cpu); -+ int power_ratio = per_cpu(sched_power_ratio, cpu); ++/******************************************* ++ * ovpn exported methods ++ *******************************************/ ++ ++static const struct net_device_ops ovpn_netdev_ops = { ++ .ndo_open = ovpn_net_open, ++ .ndo_stop = ovpn_net_stop, ++ .ndo_start_xmit = ovpn_net_xmit, ++ .ndo_get_stats64 = dev_get_tstats64, ++}; ++ ++static const struct ethtool_ops ovpn_ethtool_ops = { ++ .get_link_ksettings = ovpn_get_link_ksettings, ++ .get_drvinfo = ovpn_get_drvinfo, ++ .get_link = ethtool_op_get_link, ++ .get_ts_info = ethtool_op_get_ts_info, ++}; ++ ++static void ovpn_setup(struct net_device *dev) ++{ ++ /* compute the overhead considering AEAD encryption */ ++ const int overhead = sizeof(u32) + NONCE_WIRE_SIZE + 16 + sizeof(struct udphdr) + ++ max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); ++ ++ netdev_features_t feat = NETIF_F_SG | NETIF_F_LLTX | ++ NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_GSO | ++ NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA; ++ ++ dev->ethtool_ops = &ovpn_ethtool_ops; ++ dev->needs_free_netdev = true; ++ ++ dev->netdev_ops = &ovpn_netdev_ops; ++ ++ dev->priv_destructor = ovpn_struct_free; ++ ++ /* Point-to-Point TUN Device */ ++ dev->hard_header_len = 0; ++ dev->addr_len = 0; ++ dev->mtu = ETH_DATA_LEN - overhead; ++ dev->min_mtu = IPV4_MIN_MTU; ++ dev->max_mtu = IP_MAX_MTU - overhead; ++ ++ /* Zero header length */ ++ dev->type = ARPHRD_NONE; ++ dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; ++ ++ dev->features |= feat; ++ dev->hw_features |= feat; ++ dev->hw_enc_features |= feat; ++ ++ dev->needed_headroom = OVPN_HEAD_ROOM; ++ dev->needed_tailroom = OVPN_MAX_PADDING; ++} ++ ++static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = { ++ [IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, __OVPN_MODE_FIRST, ++ __OVPN_MODE_AFTER_LAST - 1), ++}; ++ ++static int ovpn_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], ++ struct nlattr *data[], struct netlink_ext_ack *extack) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ int ret; ++ ++ ret = security_tun_dev_create(); ++ if (ret < 0) ++ return ret; + -+ /* a power ratio of 0 (uninitialized) is assumed to be maximum */ -+ if (power_ratio == 0) -+ power_ratio = 256 - 2 * 6; -+ return per_cpu(sched_core_priority, cpu) * power_ratio / 256; - } - - /** -@@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu) - i++; - } - } ++ ret = ovpn_struct_init(dev); ++ if (ret < 0) ++ return ret; ++ ++ ovpn->mode = OVPN_MODE_P2P; ++ if (data && data[IFLA_OVPN_MODE]) { ++ ovpn->mode = nla_get_u8(data[IFLA_OVPN_MODE]); ++ netdev_dbg(dev, "%s: setting device (%s) mode: %u\n", __func__, dev->name, ++ ovpn->mode); ++ } ++ ++ return register_netdevice(dev); ++} ++ ++static void ovpn_dellink(struct net_device *dev, struct list_head *head) ++{ ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ ++ switch (ovpn->mode) { ++ case OVPN_MODE_P2P: ++ ovpn_peer_release_p2p(ovpn); ++ break; ++ default: ++ ovpn_peers_free(ovpn); ++ break; ++ } ++ ++ unregister_netdevice_queue(dev, head); ++} + +/** -+ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT -+ * @power_ratio: The power scaling ratio [1..256] for the core -+ * @core_cpu: The cpu number associated with the core ++ * ovpn_num_queues - define number of queues to allocate per device + * -+ * Set a scaling to the cpu performance based on long term power -+ * settings (like EPB). ++ * The value returned by this function is used to decide how many RX and TX ++ * queues to allocate when creating the netdev object + * -+ * Note this is for the policy not for the actual dynamic frequency; -+ * the frequency will increase itself as workloads run on a core. ++ * Return the number of queues to allocate + */ ++static unsigned int ovpn_num_queues(void) ++{ ++ return num_online_cpus(); ++} + -+void sched_set_itmt_power_ratio(int power_ratio, int core_cpu) ++static struct rtnl_link_ops ovpn_link_ops __read_mostly = { ++ .kind = DRV_NAME, ++ .priv_size = sizeof(struct ovpn_struct), ++ .setup = ovpn_setup, ++ .policy = ovpn_policy, ++ .maxtype = IFLA_OVPN_MAX, ++ .newlink = ovpn_newlink, ++ .dellink = ovpn_dellink, ++ .get_num_tx_queues = ovpn_num_queues, ++ .get_num_rx_queues = ovpn_num_queues, ++}; ++ ++static int __init ovpn_init(void) +{ -+ int cpu; ++ int err = 0; + -+ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) { -+ per_cpu(sched_power_ratio, cpu) = power_ratio; ++ pr_info("%s %s -- %s\n", DRV_DESCRIPTION, DRV_VERSION, DRV_COPYRIGHT); ++ ++ /* init RTNL link ops */ ++ err = rtnl_link_register(&ovpn_link_ops); ++ if (err) { ++ pr_err("ovpn: can't register RTNL link ops\n"); ++ goto err; ++ } ++ ++ err = ovpn_netlink_register(); ++ if (err) { ++ pr_err("ovpn: can't register netlink family\n"); ++ goto err_rtnl_unregister; + } ++ ++ return 0; ++ ++err_rtnl_unregister: ++ rtnl_link_unregister(&ovpn_link_ops); ++err: ++ pr_err("ovpn: initialization failed, error status=%d\n", err); ++ return err; +} -diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c -index ed8ac6bcbafb..d6fc5bdb0246 100644 ---- a/arch/x86/kernel/msr.c -+++ b/arch/x86/kernel/msr.c -@@ -48,7 +48,7 @@ enum allow_write_msrs { - MSR_WRITES_DEFAULT, - }; - --static enum allow_write_msrs allow_writes = MSR_WRITES_DEFAULT; -+static enum allow_write_msrs allow_writes = MSR_WRITES_ON; - - static ssize_t msr_read(struct file *file, char __user *buf, - size_t count, loff_t *ppos) -diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c -index cafacb2e58cc..c2f80184fd33 100644 ---- a/arch/x86/kernel/tsc.c -+++ b/arch/x86/kernel/tsc.c -@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void) - if (!constant_tsc || !mask) - return 0; - -+ if (cpu != 0) -+ return cpu_data(0).loops_per_jiffy; + - sibling = cpumask_any_but(mask, cpu); - if (sibling < nr_cpu_ids) - return cpu_data(sibling).loops_per_jiffy; -diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c -index fa71a5d12e87..c4bbd8c66134 100644 ---- a/arch/x86/mm/fault.c -+++ b/arch/x86/mm/fault.c -@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, - if (!printk_ratelimit()) - return; - -- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx", -+ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i", - loglvl, tsk->comm, task_pid_nr(tsk), address, -- (void *)regs->ip, (void *)regs->sp, error_code); -+ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id()); - - print_vma_addr(KERN_CONT " in ", regs->ip); - -diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl -index 52c94ab5c205..1518e261d882 100644 ---- a/arch/xtensa/kernel/syscalls/syscall.tbl -+++ b/arch/xtensa/kernel/syscalls/syscall.tbl -@@ -421,3 +421,4 @@ - 448 common process_mrelease sys_process_mrelease - 449 common futex_waitv sys_futex_waitv - 450 common set_mempolicy_home_node sys_set_mempolicy_home_node -+451 common pmadv_ksm sys_pmadv_ksm -diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c -index 30b15a9a47c4..144bca006463 100644 ---- a/block/bfq-cgroup.c -+++ b/block/bfq-cgroup.c -@@ -254,17 +254,12 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - - #else /* CONFIG_BFQ_CGROUP_DEBUG */ - --void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -- blk_opf_t opf) { } - void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { } - void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { } - void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - u64 io_start_time_ns, blk_opf_t opf) { } - void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } --void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } --void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } - void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } --void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } - - #endif /* CONFIG_BFQ_CGROUP_DEBUG */ - -diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c -index c740b41fe0a4..5ea6245f0208 100644 ---- a/block/bfq-iosched.c -+++ b/block/bfq-iosched.c -@@ -1925,7 +1925,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, - bfqq->service_from_backlogged = 0; - bfq_clear_bfqq_softrt_update(bfqq); - -- bfq_add_bfqq_busy(bfqd, bfqq); -+ bfq_add_bfqq_busy(bfqq); - - /* - * Expire in-service queue if preemption may be needed for -@@ -2419,7 +2419,7 @@ static void bfq_remove_request(struct request_queue *q, - bfqq->next_rq = NULL; - - if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { -- bfq_del_bfqq_busy(bfqd, bfqq, false); -+ bfq_del_bfqq_busy(bfqq, false); - /* - * bfqq emptied. In normal operation, when - * bfqq is empty, bfqq->entity.service and -@@ -3098,7 +3098,7 @@ void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) - */ - if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && - bfqq != bfqd->in_service_queue) -- bfq_del_bfqq_busy(bfqd, bfqq, false); -+ bfq_del_bfqq_busy(bfqq, false); - - bfq_reassign_last_bfqq(bfqq, NULL); - -@@ -3908,7 +3908,7 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, - */ - bfqq->budget_timeout = jiffies; - -- bfq_del_bfqq_busy(bfqd, bfqq, true); -+ bfq_del_bfqq_busy(bfqq, true); - } else { - bfq_requeue_bfqq(bfqd, bfqq, true); - /* -@@ -5255,9 +5255,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) - struct hlist_node *n; - struct bfq_group *bfqg = bfqq_group(bfqq); - -- if (bfqq->bfqd) -- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", -- bfqq, bfqq->ref); -+ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); - - bfqq->ref--; - if (bfqq->ref) -@@ -5321,7 +5319,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) - hlist_del_init(&item->woken_list_node); - } - -- if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) -+ if (bfqq->bfqd->last_completed_rq_bfqq == bfqq) - bfqq->bfqd->last_completed_rq_bfqq = NULL; - - kmem_cache_free(bfq_pool, bfqq); -@@ -7463,6 +7461,7 @@ MODULE_ALIAS("bfq-iosched"); - static int __init bfq_init(void) - { - int ret; -+ char msg[60] = "BFQ I/O-scheduler: BFQ-CachyOS v5.19"; - - #ifdef CONFIG_BFQ_GROUP_IOSCHED - ret = blkcg_policy_register(&blkcg_policy_bfq); -@@ -7494,6 +7493,11 @@ static int __init bfq_init(void) - if (ret) - goto slab_kill; - -+#ifdef CONFIG_BFQ_GROUP_IOSCHED -+ strcat(msg, " (with cgroups support)"); -+#endif -+ pr_info("%s", msg); ++static __exit void ovpn_cleanup(void) ++{ ++ rtnl_link_unregister(&ovpn_link_ops); ++ ovpn_netlink_unregister(); ++ rcu_barrier(); /* because we use call_rcu */ ++} + - return 0; - - slab_kill: -diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h -index ad8e513d7e87..64ee618064ba 100644 ---- a/block/bfq-iosched.h -+++ b/block/bfq-iosched.h -@@ -993,20 +993,23 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); - /* ---------------- cgroups-support interface ---------------- */ - - void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq); --void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -- blk_opf_t opf); - void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf); - void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf); - void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, - u64 io_start_time_ns, blk_opf_t opf); - void bfqg_stats_update_dequeue(struct bfq_group *bfqg); --void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); --void bfqg_stats_update_idle_time(struct bfq_group *bfqg); - void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg); --void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); - void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, - struct bfq_group *bfqg); - -+#ifdef CONFIG_BFQ_CGROUP_DEBUG -+void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, -+ blk_opf_t opf); -+void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); -+void bfqg_stats_update_idle_time(struct bfq_group *bfqg); -+void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); -+#endif ++module_init(ovpn_init); ++module_exit(ovpn_cleanup); + - void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg); - void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio); - void bfq_end_wr_async(struct bfq_data *bfqd); -@@ -1077,9 +1080,8 @@ void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); - void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - bool expiration); --void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -- bool expiration); --void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); -+void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration); -+void bfq_add_bfqq_busy(struct bfq_queue *bfqq); - - /* --------------- end of interface of B-WF2Q+ ---------------- */ - -diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c -index 983413cdefad..8fc3da4c23bb 100644 ---- a/block/bfq-wf2q.c -+++ b/block/bfq-wf2q.c -@@ -1651,9 +1651,10 @@ void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, - * the service tree. As a special case, it can be invoked during an - * expiration. - */ --void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, -- bool expiration) -+void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration) - { -+ struct bfq_data *bfqd = bfqq->bfqd; ++MODULE_DESCRIPTION(DRV_DESCRIPTION); ++MODULE_AUTHOR(DRV_COPYRIGHT); ++MODULE_LICENSE("GPL"); ++MODULE_VERSION(DRV_VERSION); ++MODULE_ALIAS_RTNL_LINK(DRV_NAME); ++MODULE_ALIAS_GENL_FAMILY(OVPN_NL_NAME); +diff --git a/drivers/net/ovpn-dco/main.h b/drivers/net/ovpn-dco/main.h +new file mode 100644 +index 000000000000..c4ef200b30f4 +--- /dev/null ++++ b/drivers/net/ovpn-dco/main.h +@@ -0,0 +1,32 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + - bfq_log_bfqq(bfqd, bfqq, "del from busy"); - - bfq_clear_bfqq_busy(bfqq); -@@ -1674,8 +1675,10 @@ void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, - /* - * Called when an inactive queue receives a new request. - */ --void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) -+void bfq_add_bfqq_busy(struct bfq_queue *bfqq) - { -+ struct bfq_data *bfqd = bfqq->bfqd; ++#ifndef _NET_OVPN_DCO_MAIN_H_ ++#define _NET_OVPN_DCO_MAIN_H_ + - bfq_log_bfqq(bfqd, bfqq, "add to busy"); - - bfq_activate_bfqq(bfqd, bfqq); -diff --git a/block/blk-core.c b/block/blk-core.c -index 651057c4146b..88873f267b2a 100644 ---- a/block/blk-core.c -+++ b/block/blk-core.c -@@ -742,6 +742,9 @@ void submit_bio_noacct(struct bio *bio) - status = BLK_STS_OK; - goto end_io; - } ++#include ++#include ++#include ++#include + -+ if (bio->bi_opf & REQ_PREFLUSH) -+ current->fsync_count++; - } - - if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) -diff --git a/block/elevator.c b/block/elevator.c -index c319765892bb..b2687032a0e0 100644 ---- a/block/elevator.c -+++ b/block/elevator.c -@@ -640,8 +640,13 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) - - if (q->nr_hw_queues != 1 && - !blk_mq_is_shared_tags(q->tag_set->flags)) -+#if defined(CONFIG_CACHY) && defined(CONFIG_MQ_IOSCHED_KYBER) -+ return elevator_get(q, "kyber", false); -+#elif defined(CONFIG_CACHY) -+ return elevator_get(q, "mq-deadline", false); -+#else - return NULL; -- -+#endif - return elevator_get(q, "mq-deadline", false); - } - -diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c -index 1e15a9f25ae9..4801300d3c77 100644 ---- a/drivers/acpi/cppc_acpi.c -+++ b/drivers/acpi/cppc_acpi.c -@@ -424,6 +424,9 @@ bool acpi_cpc_valid(void) - struct cpc_desc *cpc_ptr; - int cpu; - -+ if (acpi_disabled) -+ return false; ++#define OVPN_DCO_VERSION "2.0.0" + - for_each_present_cpu(cpu) { - cpc_ptr = per_cpu(cpc_desc_ptr, cpu); - if (!cpc_ptr) -@@ -1320,6 +1323,132 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) - } - EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs); - -+/** -+ * cppc_get_epp_caps - Get the energy preference register value. -+ * @cpunum: CPU from which to get epp preference level. -+ * @perf_caps: Return address. ++struct net_device; ++bool ovpn_dev_is_valid(const struct net_device *dev); ++ ++#define SKB_HEADER_LEN \ ++ (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ ++ sizeof(struct udphdr) + NET_SKB_PAD) ++ ++#define OVPN_HEAD_ROOM ALIGN(16 + SKB_HEADER_LEN, 4) ++#define OVPN_MAX_PADDING 16 ++#define OVPN_QUEUE_LEN 1024 ++#define OVPN_MAX_TUN_QUEUE_LEN 0x10000 ++ ++#endif /* _NET_OVPN_DCO_OVPN_DCO_H_ */ +diff --git a/drivers/net/ovpn-dco/netlink.c b/drivers/net/ovpn-dco/netlink.c +new file mode 100644 +index 000000000000..ee5c943e7db4 +--- /dev/null ++++ b/drivers/net/ovpn-dco/netlink.c +@@ -0,0 +1,1143 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator + * -+ * Return: 0 for success, -EIO otherwise. ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli + */ -+int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) -+{ -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); -+ struct cpc_register_resource *energy_perf_reg; -+ u64 energy_perf; + -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpunum); -+ return -ENODEV; -+ } ++#include "main.h" ++#include "ovpn.h" ++#include "peer.h" ++#include "proto.h" ++#include "netlink.h" ++#include "ovpnstruct.h" ++#include "udp.h" + -+ energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; ++#include + -+ if (!CPC_SUPPORTED(energy_perf_reg)) -+ pr_warn("energy perf reg update is unsupported!\n"); ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+ if (CPC_IN_PCC(energy_perf_reg)) { -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = 0; ++/** The ovpn-dco netlink family */ ++static struct genl_family ovpn_netlink_family; + -+ if (pcc_ss_id < 0) -+ return -EIO; ++enum ovpn_netlink_multicast_groups { ++ OVPN_MCGRP_PEERS, ++}; + -+ pcc_ss_data = pcc_data[pcc_ss_id]; ++static const struct genl_multicast_group ovpn_netlink_mcgrps[] = { ++ [OVPN_MCGRP_PEERS] = { .name = OVPN_NL_MULTICAST_GROUP_PEERS }, ++}; + -+ down_write(&pcc_ss_data->pcc_lock); ++/** Key direction policy. Can be used for configuring an encryption and a decryption key */ ++static const struct nla_policy ovpn_netlink_policy_key_dir[OVPN_KEY_DIR_ATTR_MAX + 1] = { ++ [OVPN_KEY_DIR_ATTR_CIPHER_KEY] = NLA_POLICY_MAX_LEN(U8_MAX), ++ [OVPN_KEY_DIR_ATTR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(NONCE_TAIL_SIZE), ++}; ++ ++/** CMD_NEW_KEY policy */ ++static const struct nla_policy ovpn_netlink_policy_new_key[OVPN_NEW_KEY_ATTR_MAX + 1] = { ++ [OVPN_NEW_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_NEW_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, ++ __OVPN_KEY_SLOT_AFTER_LAST - 1), ++ [OVPN_NEW_KEY_ATTR_KEY_ID] = { .type = NLA_U8 }, ++ [OVPN_NEW_KEY_ATTR_CIPHER_ALG] = { .type = NLA_U16 }, ++ [OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), ++ [OVPN_NEW_KEY_ATTR_DECRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), ++}; ++ ++/** CMD_DEL_KEY policy */ ++static const struct nla_policy ovpn_netlink_policy_del_key[OVPN_DEL_KEY_ATTR_MAX + 1] = { ++ [OVPN_DEL_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_DEL_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, ++ __OVPN_KEY_SLOT_AFTER_LAST - 1), ++}; ++ ++/** CMD_SWAP_KEYS policy */ ++static const struct nla_policy ovpn_netlink_policy_swap_keys[OVPN_SWAP_KEYS_ATTR_MAX + 1] = { ++ [OVPN_SWAP_KEYS_ATTR_PEER_ID] = { .type = NLA_U32 }, ++}; ++ ++/** CMD_NEW_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_new_peer[OVPN_NEW_PEER_ATTR_MAX + 1] = { ++ [OVPN_NEW_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE] = NLA_POLICY_MIN_LEN(sizeof(struct sockaddr)), ++ [OVPN_NEW_PEER_ATTR_SOCKET] = { .type = NLA_U32 }, ++ [OVPN_NEW_PEER_ATTR_IPV4] = { .type = NLA_U32 }, ++ [OVPN_NEW_PEER_ATTR_IPV6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), ++ [OVPN_NEW_PEER_ATTR_LOCAL_IP] = NLA_POLICY_MAX_LEN(sizeof(struct in6_addr)), ++}; ++ ++/** CMD_SET_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_set_peer[OVPN_SET_PEER_ATTR_MAX + 1] = { ++ [OVPN_SET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, ++ [OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT] = { .type = NLA_U32 }, ++}; + -+ if (send_pcc_cmd(pcc_ss_id, CMD_READ) >= 0) { -+ cpc_read(cpunum, energy_perf_reg, &energy_perf); -+ perf_caps->energy_perf = energy_perf; -+ } else { -+ ret = -EIO; -+ } ++/** CMD_DEL_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_del_peer[OVPN_DEL_PEER_ATTR_MAX + 1] = { ++ [OVPN_DEL_PEER_ATTR_REASON] = NLA_POLICY_RANGE(NLA_U8, __OVPN_DEL_PEER_REASON_FIRST, ++ __OVPN_DEL_PEER_REASON_AFTER_LAST - 1), ++ [OVPN_DEL_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++}; + -+ up_write(&pcc_ss_data->pcc_lock); ++/** CMD_GET_PEER policy */ ++static const struct nla_policy ovpn_netlink_policy_get_peer[OVPN_GET_PEER_ATTR_MAX + 1] = { ++ [OVPN_GET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, ++}; + -+ return ret; -+ } ++/** CMD_PACKET polocy */ ++static const struct nla_policy ovpn_netlink_policy_packet[OVPN_PACKET_ATTR_MAX + 1] = { ++ [OVPN_PACKET_ATTR_PEER_ID] = { .type = NLA_U32 }, ++ [OVPN_PACKET_ATTR_PACKET] = NLA_POLICY_MAX_LEN(U16_MAX), ++}; + -+ return 0; -+} -+EXPORT_SYMBOL_GPL(cppc_get_epp_caps); ++/** Generic message container policy */ ++static const struct nla_policy ovpn_netlink_policy[OVPN_ATTR_MAX + 1] = { ++ [OVPN_ATTR_IFINDEX] = { .type = NLA_U32 }, ++ [OVPN_ATTR_NEW_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_peer), ++ [OVPN_ATTR_SET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_set_peer), ++ [OVPN_ATTR_DEL_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_peer), ++ [OVPN_ATTR_GET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_get_peer), ++ [OVPN_ATTR_NEW_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_key), ++ [OVPN_ATTR_SWAP_KEYS] = NLA_POLICY_NESTED(ovpn_netlink_policy_swap_keys), ++ [OVPN_ATTR_DEL_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_key), ++ [OVPN_ATTR_PACKET] = NLA_POLICY_NESTED(ovpn_netlink_policy_packet), ++}; + -+int cppc_set_auto_epp(int cpu, bool enable) ++static struct net_device * ++ovpn_get_dev_from_attrs(struct net *net, struct nlattr **attrs) +{ -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); -+ struct cpc_register_resource *auto_sel_reg; -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = -EINVAL; ++ struct net_device *dev; ++ int ifindex; + -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpu); -+ return -EINVAL; -+ } ++ if (!attrs[OVPN_ATTR_IFINDEX]) ++ return ERR_PTR(-EINVAL); + -+ auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; ++ ifindex = nla_get_u32(attrs[OVPN_ATTR_IFINDEX]); + -+ if (CPC_IN_PCC(auto_sel_reg)) { -+ if (pcc_ss_id < 0) -+ return -EIO; ++ dev = dev_get_by_index(net, ifindex); ++ if (!dev) ++ return ERR_PTR(-ENODEV); + -+ ret = cpc_write(cpu, auto_sel_reg, enable); -+ if (ret) -+ return ret; ++ if (!ovpn_dev_is_valid(dev)) ++ goto err_put_dev; + -+ pcc_ss_data = pcc_data[pcc_ss_id]; ++ return dev; + -+ down_write(&pcc_ss_data->pcc_lock); -+ /* after writing CPC, transfer the ownership of PCC to platform */ -+ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); -+ up_write(&pcc_ss_data->pcc_lock); -+ return ret; -+ } ++err_put_dev: ++ dev_put(dev); + -+ return cpc_write(cpu, auto_sel_reg, enable); ++ return ERR_PTR(-EINVAL); +} -+EXPORT_SYMBOL_GPL(cppc_set_auto_epp); + -+/* -+ * Set Energy Performance Preference Register value through -+ * Performance Controls Interface ++/** ++ * ovpn_pre_doit() - Prepare ovpn genl doit request ++ * @ops: requested netlink operation ++ * @skb: Netlink message with request data ++ * @info: receiver information ++ * ++ * Return: 0 on success or negative error number in case of failure + */ -+int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) ++static int ovpn_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, ++ struct genl_info *info) +{ -+ int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); -+ struct cpc_register_resource *epp_set_reg; -+ struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); -+ struct cppc_pcc_data *pcc_ss_data = NULL; -+ int ret = -EINVAL; ++ struct net *net = genl_info_net(info); ++ struct net_device *dev; + -+ if (!cpc_desc) { -+ pr_warn("No CPC descriptor for CPU:%d\n", cpu); -+ return -EINVAL; -+ } ++ dev = ovpn_get_dev_from_attrs(net, info->attrs); ++ if (IS_ERR(dev)) ++ return PTR_ERR(dev); + -+ epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; ++ info->user_ptr[0] = netdev_priv(dev); + -+ if (CPC_IN_PCC(epp_set_reg)) { -+ if (pcc_ss_id < 0) -+ return -EIO; ++ return 0; ++} + -+ ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); -+ if (ret) -+ return ret; ++/** ++ * ovpn_post_doit() - complete ovpn genl doit request ++ * @ops: requested netlink operation ++ * @skb: Netlink message with request data ++ * @info: receiver information ++ */ ++static void ovpn_post_doit(const struct genl_ops *ops, struct sk_buff *skb, ++ struct genl_info *info) ++{ ++ struct ovpn_struct *ovpn; + -+ pcc_ss_data = pcc_data[pcc_ss_id]; ++ ovpn = info->user_ptr[0]; ++ dev_put(ovpn->dev); ++} + -+ down_write(&pcc_ss_data->pcc_lock); -+ /* after writing CPC, transfer the ownership of PCC to platform */ -+ ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); -+ up_write(&pcc_ss_data->pcc_lock); -+ } ++static int ovpn_netlink_get_key_dir(struct genl_info *info, struct nlattr *key, ++ enum ovpn_cipher_alg cipher, ++ struct ovpn_key_direction *dir) ++{ ++ struct nlattr *attr, *attrs[OVPN_KEY_DIR_ATTR_MAX + 1]; ++ int ret; + -+ return ret; -+} -+EXPORT_SYMBOL_GPL(cppc_set_epp_perf); ++ ret = nla_parse_nested(attrs, OVPN_KEY_DIR_ATTR_MAX, key, NULL, info->extack); ++ if (ret) ++ return ret; + - /** - * cppc_set_enable - Set to enable CPPC on the processor by writing the - * Continuous Performance Control package EnableRegister field. -@@ -1355,7 +1484,7 @@ int cppc_set_enable(int cpu, bool enable) - pcc_ss_data = pcc_data[pcc_ss_id]; - - down_write(&pcc_ss_data->pcc_lock); -- /* after writing CPC, transfer the ownership of PCC to platfrom */ -+ /* after writing CPC, transfer the ownership of PCC to platform */ - ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); - up_write(&pcc_ss_data->pcc_lock); - return ret; -diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c -index cf8c7fd59ada..ad9bb5353dd3 100644 ---- a/drivers/ata/libahci.c -+++ b/drivers/ata/libahci.c -@@ -33,14 +33,14 @@ - #include "libata.h" - - static int ahci_skip_host_reset; --int ahci_ignore_sss; -+int ahci_ignore_sss=1; - EXPORT_SYMBOL_GPL(ahci_ignore_sss); - - module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444); - MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)"); - - module_param_named(ignore_sss, ahci_ignore_sss, int, 0444); --MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)"); -+MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])"); - - static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy, - unsigned hints); -diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c -index 46cbe4471e78..dd90591e51ba 100644 ---- a/drivers/base/arch_topology.c -+++ b/drivers/base/arch_topology.c -@@ -353,7 +353,7 @@ void topology_init_cpu_capacity_cppc(void) - struct cppc_perf_caps perf_caps; - int cpu; - -- if (likely(acpi_disabled || !acpi_cpc_valid())) -+ if (likely(!acpi_cpc_valid())) - return; - - raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), -diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c -index 7c3590fd97c2..bb4880e10581 100644 ---- a/drivers/base/firmware_loader/main.c -+++ b/drivers/base/firmware_loader/main.c -@@ -470,6 +470,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, - static char fw_path_para[256]; - static const char * const fw_path[] = { - fw_path_para, -+ "/etc/firmware/" UTS_RELEASE, -+ "/etc/firmware", - "/lib/firmware/updates/" UTS_RELEASE, - "/lib/firmware/updates", - "/lib/firmware/" UTS_RELEASE, -diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig -index d4100b0c083e..8739ea13161f 100644 ---- a/drivers/block/zram/Kconfig -+++ b/drivers/block/zram/Kconfig -@@ -78,3 +78,21 @@ config ZRAM_MEMORY_TRACKING - /sys/kernel/debug/zram/zramX/block_state. - - See Documentation/admin-guide/blockdev/zram.rst for more information. ++ switch (cipher) { ++ case OVPN_CIPHER_ALG_AES_GCM: ++ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: ++ attr = attrs[OVPN_KEY_DIR_ATTR_CIPHER_KEY]; ++ if (!attr) ++ return -EINVAL; + -+config ZRAM_ENTROPY -+ bool "Use entropy optimization for zram" -+ depends on ZRAM && ZRAM_DEF_COMP_ZSTD -+ help -+ With this feature, entropy will be calculated for each page. -+ Pages above ZRAM_ENTROPY_THRESHOLD entropy will be -+ stored uncompressed. Use this feature if you need a performance -+ boost and a small loss in compression. ++ dir->cipher_key = nla_data(attr); ++ dir->cipher_key_size = nla_len(attr); + -+config ZRAM_ENTROPY_THRESHOLD -+ int -+ depends on ZRAM && ZRAM_ENTROPY -+ default 100000 if ZRAM_DEF_COMP_ZSTD -+ help -+ Pages with entropy above ZRAM_ENTROPY_THRESHOLD will be stored -+ uncompressed. The default value was chosen as a result a lot of -+ experiments. You can try set your own value. -diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c -index 226ea76cc819..d02e37e7afd9 100644 ---- a/drivers/block/zram/zram_drv.c -+++ b/drivers/block/zram/zram_drv.c -@@ -1347,6 +1347,35 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, - return ret; - } - ++ attr = attrs[OVPN_KEY_DIR_ATTR_NONCE_TAIL]; ++ /* These algorithms require a 96bit nonce, ++ * Construct it by combining 4-bytes packet id and ++ * 8-bytes nonce-tail from userspace ++ */ ++ if (!attr) ++ return -EINVAL; + -+#ifdef CONFIG_ZRAM_ENTROPY -+static inline u32 ilog2_w(u64 n) -+{ -+ return ilog2(n * n * n * n); ++ dir->nonce_tail = nla_data(attr); ++ dir->nonce_tail_size = nla_len(attr); ++ break; ++ default: ++ return -EINVAL; ++ } ++ ++ return 0; +} + -+static inline s32 shannon_entropy(const u8 *src) ++static int ovpn_netlink_new_key(struct sk_buff *skb, struct genl_info *info) +{ -+ s32 entropy_sum = 0; -+ u32 sz_base, i; -+ u16 entropy_count[256] = { 0 }; ++ struct nlattr *attrs[OVPN_NEW_KEY_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer_key_reset pkr; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; + -+ for (i = 0; i < PAGE_SIZE; ++i) -+ entropy_count[src[i]]++; ++ if (!info->attrs[OVPN_ATTR_NEW_KEY]) ++ return -EINVAL; + -+ sz_base = ilog2_w(PAGE_SIZE); -+ for (i = 0; i < ARRAY_SIZE(entropy_count); ++i) { -+ if (entropy_count[i] > 0) { -+ s32 p = entropy_count[i]; ++ ret = nla_parse_nested(attrs, OVPN_NEW_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_KEY], ++ NULL, info->extack); ++ if (ret) ++ return ret; + -+ entropy_sum += p * (sz_base - ilog2_w((u64)p)); -+ } -+ } ++ if (!attrs[OVPN_NEW_KEY_ATTR_PEER_ID] || ++ !attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT] || ++ !attrs[OVPN_NEW_KEY_ATTR_KEY_ID] || ++ !attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG] || ++ !attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] || ++ !attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY]) ++ return -EINVAL; + -+ return entropy_sum; -+} -+#endif ++ peer_id = nla_get_u32(attrs[OVPN_NEW_KEY_ATTR_PEER_ID]); ++ pkr.slot = nla_get_u8(attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT]); ++ pkr.key.key_id = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_KEY_ID]); + - static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - u32 index, struct bio *bio) - { -@@ -1373,7 +1402,17 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - compress_again: - zstrm = zcomp_stream_get(zram->comp); - src = kmap_atomic(page); ++ pkr.key.cipher_alg = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG]); + -+#ifdef CONFIG_ZRAM_ENTROPY -+ /* Just save this page uncompressible */ -+ if (shannon_entropy((const u8 *)src) > CONFIG_ZRAM_ENTROPY_THRESHOLD) -+ comp_len = PAGE_SIZE; -+ else -+ ret = zcomp_compress(zstrm, src, &comp_len); -+#else - ret = zcomp_compress(zstrm, src, &comp_len); -+#endif ++ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY], ++ pkr.key.cipher_alg, &pkr.key.encrypt); ++ if (ret < 0) ++ return ret; + - kunmap_atomic(src); - - if (unlikely(ret)) { -diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c -index 9ac75c1cde9c..e8cd2d2b7cf8 100644 ---- a/drivers/cpufreq/amd-pstate.c -+++ b/drivers/cpufreq/amd-pstate.c -@@ -31,23 +31,18 @@ - #include - #include - #include --#include - #include - #include - #include - #include - --#include - #include - - #include --#include --#include --#include - #include "amd-pstate-trace.h" - --#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 --#define AMD_PSTATE_TRANSITION_DELAY 500 -+#define AMD_PSTATE_TRANSITION_LATENCY 20000 -+#define AMD_PSTATE_TRANSITION_DELAY 1000 - - /* - * TODO: We need more time to fine tune processors with shared memory solution -@@ -63,7 +58,13 @@ module_param(shared_mem, bool, 0444); - MODULE_PARM_DESC(shared_mem, - "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); - --static struct cpufreq_driver amd_pstate_driver; -+static bool epp_enabled = true; -+module_param(epp_enabled, bool, 0444); -+MODULE_PARM_DESC(epp_enabled, -+ "load amd_pstate or amd_pstate_epp (true = amd_pstate_epp driver instance (default), false = amd_pstate driver instance)"); ++ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY], ++ pkr.key.cipher_alg, &pkr.key.decrypt); ++ if (ret < 0) ++ return ret; + -+static struct cpufreq_driver *default_pstate_driver; -+static struct amd_cpudata **all_cpu_data; - - /** - * struct amd_aperf_mperf -@@ -75,6 +76,7 @@ struct amd_aperf_mperf { - u64 aperf; - u64 mperf; - u64 tsc; -+ u64 time; - }; - - /** -@@ -97,6 +99,20 @@ struct amd_aperf_mperf { - * @prev: Last Aperf/Mperf/tsc count value read from register - * @freq: current cpu frequency value - * @boost_supported: check whether the Processor or SBIOS supports boost mode -+ * @precision_boost_off: the core performance boost disabled state -+ * @cppc_hw_conf_cached: the cached hardware configuration register -+ * @epp_powersave: Last saved CPPC energy performance preference -+ * when policy switched to performance -+ * @epp_policy: Last saved policy used to set energy-performance preference -+ * @epp_cached: Cached CPPC energy-performance preference value -+ * @policy: Cpufreq policy value -+ * @sched_flags: Store scheduler flags for possible cross CPU update -+ * @update_util_set: CPUFreq utility callback is set -+ * @last_update: Time stamp of the last performance state update -+ * @cppc_boost_min: Last CPPC boosted min performance state -+ * @cppc_cap1_cached: Cached value of the last CPPC Capabilities MSR -+ * @update_util: Cpufreq utility callback information -+ * @suspended: Whether or not the driver has been suspended. - * - * The amd_cpudata is key private data for each CPU thread in AMD P-State, and - * represents all the attributes and goals that AMD P-State requests at runtime. -@@ -120,10 +136,200 @@ struct amd_cpudata { - struct amd_aperf_mperf cur; - struct amd_aperf_mperf prev; - -- u64 freq; -+ u64 freq; - bool boost_supported; -+ bool precision_boost_off; -+ u64 cppc_hw_conf_cached; ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) { ++ netdev_dbg(ovpn->dev, "%s: no peer with id %u to set key for\n", __func__, peer_id); ++ return -ENOENT; ++ } + -+ /* EPP feature related attributes*/ -+ s16 epp_powersave; -+ s16 epp_policy; -+ s16 epp_cached; -+ u32 policy; -+ u32 sched_flags; -+ bool update_util_set; -+ u64 last_update; -+ u64 last_io_update; -+ u32 cppc_boost_min; -+ u64 cppc_cap1_cached; -+ struct update_util_data update_util; -+ struct amd_aperf_mperf sample; -+ bool suspended; -+}; ++ mutex_lock(&peer->crypto.mutex); ++ ret = ovpn_crypto_state_reset(&peer->crypto, &pkr); ++ if (ret < 0) { ++ netdev_dbg(ovpn->dev, "%s: cannot install new key for peer %u\n", __func__, ++ peer_id); ++ goto unlock; ++ } + -+/** -+ * struct amd_pstate_params - global parameters for the performance control -+ * @ cppc_boost_disabled Wheter or not the core performance boost disabled -+ */ -+struct amd_pstate_params { -+ bool cppc_boost_disabled; -+}; ++ netdev_dbg(ovpn->dev, "%s: new key installed (id=%u) for peer %u\n", __func__, ++ pkr.key.key_id, peer_id); ++unlock: ++ mutex_unlock(&peer->crypto.mutex); ++ ovpn_peer_put(peer); ++ return ret; ++} + -+/* -+ * AMD Energy Preference Performance (EPP) -+ * The EPP is used in the CCLK DPM controller to drive -+ * the frequency that a core is going to operate during -+ * short periods of activity. EPP values will be utilized for -+ * different OS profiles (balanced, performance, power savings) -+ * display strings corresponding to EPP index in the -+ * energy_perf_strings[] -+ * index String -+ *------------------------------------- -+ * 0 default -+ * 1 performance -+ * 2 balance_performance -+ * 3 balance_power -+ * 4 power -+ */ -+ enum energy_perf_value_index { -+ EPP_INDEX_DEFAULT = 0, -+ EPP_INDEX_PERFORMANCE, -+ EPP_INDEX_BALANCE_PERFORMANCE, -+ EPP_INDEX_BALANCE_POWERSAVE, -+ EPP_INDEX_POWERSAVE, -+}; ++static int ovpn_netlink_del_key(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_DEL_KEY_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ enum ovpn_key_slot slot; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; + -+static const char * const energy_perf_strings[] = { -+ [EPP_INDEX_DEFAULT] = "default", -+ [EPP_INDEX_PERFORMANCE] = "performance", -+ [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", -+ [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", -+ [EPP_INDEX_POWERSAVE] = "power", -+ NULL - }; - -+static unsigned int epp_values[] = { -+ [EPP_INDEX_DEFAULT] = 0, -+ [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, -+ [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, -+ [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, -+ [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, -+}; ++ if (!info->attrs[OVPN_ATTR_DEL_KEY]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_DEL_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_KEY], NULL, ++ info->extack); ++ if (ret) ++ return ret; + -+static struct amd_pstate_params global; ++ if (!attrs[OVPN_DEL_KEY_ATTR_PEER_ID] || !attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]) ++ return -EINVAL; + -+static DEFINE_MUTEX(amd_pstate_limits_lock); -+static DEFINE_MUTEX(amd_pstate_driver_lock); -+static DEFINE_SPINLOCK(amd_pstate_cpu_lock); ++ peer_id = nla_get_u32(attrs[OVPN_DEL_KEY_ATTR_PEER_ID]); ++ slot = nla_get_u8(attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]); + -+static bool cppc_boost __read_mostly; -+struct kobject *amd_pstate_kobj; ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; + -+#ifdef CONFIG_ACPI_CPPC_LIB -+static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) ++ ovpn_crypto_key_slot_delete(&peer->crypto, slot); ++ ovpn_peer_put(peer); ++ ++ return 0; ++} ++ ++static int ovpn_netlink_swap_keys(struct sk_buff *skb, struct genl_info *info) +{ -+ s16 epp; -+ struct cppc_perf_caps perf_caps; ++ struct nlattr *attrs[OVPN_SWAP_KEYS_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer *peer; ++ u32 peer_id; + int ret; + -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!cppc_req_cached) { -+ epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, -+ &cppc_req_cached); -+ if (epp) -+ return epp; -+ } -+ epp = (cppc_req_cached >> 24) & 0xFF; -+ } else { -+ ret = cppc_get_epp_caps(cpudata->cpu, &perf_caps); -+ if (ret < 0) { -+ pr_debug("Could not retrieve energy perf value (%d)\n", ret); -+ return -EIO; -+ } -+ epp = (s16) perf_caps.energy_perf; -+ } ++ if (!info->attrs[OVPN_ATTR_SWAP_KEYS]) ++ return -EINVAL; + -+ return epp; -+} -+#endif ++ ret = nla_parse_nested(attrs, OVPN_SWAP_KEYS_ATTR_MAX, info->attrs[OVPN_ATTR_SWAP_KEYS], ++ NULL, info->extack); ++ if (ret) ++ return ret; + -+static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata, int *raw_epp) -+{ -+ s16 epp; -+ int index = -EINVAL; ++ if (!attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]) ++ return -EINVAL; + -+ *raw_epp = 0; -+ epp = amd_pstate_get_epp(cpudata, 0); -+ if (epp < 0) -+ return epp; ++ peer_id = nla_get_u32(attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]); + -+ switch (epp) { -+ case AMD_CPPC_EPP_PERFORMANCE: -+ index = EPP_INDEX_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_PERFORMANCE: -+ index = EPP_INDEX_BALANCE_PERFORMANCE; -+ break; -+ case AMD_CPPC_EPP_BALANCE_POWERSAVE: -+ index = EPP_INDEX_BALANCE_POWERSAVE; -+ break; -+ case AMD_CPPC_EPP_POWERSAVE: -+ index = EPP_INDEX_POWERSAVE; -+ break; -+ default: -+ *raw_epp = epp; -+ index = 0; -+ } ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; + -+ return index; ++ ovpn_crypto_key_slots_swap(&peer->crypto); ++ ovpn_peer_put(peer); ++ ++ return 0; +} + -+#ifdef CONFIG_ACPI_CPPC_LIB -+static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) ++static int ovpn_netlink_new_peer(struct sk_buff *skb, struct genl_info *info) +{ ++ struct nlattr *attrs[OVPN_NEW_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct sockaddr_storage *ss = NULL; ++ struct sockaddr_in mapped; ++ struct sockaddr_in6 *in6; ++ struct ovpn_peer *peer; ++ size_t sa_len, ip_len; ++ struct socket *sock; ++ u8 *local_ip = NULL; ++ u32 sockfd, id; + int ret; -+ struct cppc_perf_ctrls perf_ctrls; + -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ u64 value = READ_ONCE(cpudata->cppc_req_cached); ++ if (!info->attrs[OVPN_ATTR_NEW_PEER]) ++ return -EINVAL; + -+ value &= ~GENMASK_ULL(31, 24); -+ value |= (u64)epp << 24; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (!ret) { -+ cpudata->epp_cached = epp; ++ ret = nla_parse_nested(attrs, OVPN_NEW_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_NEW_PEER_ATTR_PEER_ID] || !attrs[OVPN_NEW_PEER_ATTR_SOCKET] || ++ (!attrs[OVPN_NEW_PEER_ATTR_IPV4] && !attrs[OVPN_NEW_PEER_ATTR_IPV6])) { ++ netdev_dbg(ovpn->dev, "%s: basic attributes missing\n", __func__); ++ return -EINVAL; ++ } ++ ++ /* lookup the fd in the kernel table and extract the socket object */ ++ sockfd = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_SOCKET]); ++ /* sockfd_lookup() increases sock's refcounter */ ++ sock = sockfd_lookup(sockfd, &ret); ++ if (!sock) { ++ netdev_dbg(ovpn->dev, "%s: cannot lookup peer socket (fd=%u): %d\n", __func__, ++ sockfd, ret); ++ return -ENOTSOCK; ++ } ++ ++ /* Only when using UDP as transport protocol the remote endpoint must be configured ++ * so that ovpn-dco knows where to send packets to. ++ * ++ * In case of TCP, the socket is connected to the peer and ovpn-dco will just send bytes ++ * over it, without the need to specify a destination. ++ */ ++ if (sock->sk->sk_protocol == IPPROTO_UDP) { ++ ret = -EINVAL; ++ ++ if (!attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]) { ++ netdev_err(ovpn->dev, "%s: cannot add UDP peer with no remote endpoint\n", ++ __func__); ++ goto sockfd_release; + } -+ } else { -+ perf_ctrls.energy_perf = epp; -+ ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls); -+ if (ret){ -+ pr_debug("failed to set energy perf value (%d)\n", ret); -+ return ret; -+ } -+ cpudata->epp_cached = epp; ++ ++ ss = nla_data(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); ++ sa_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); ++ switch (sa_len) { ++ case sizeof(struct sockaddr_in): ++ if (ss->ss_family == AF_INET) ++ /* valid sockaddr */ ++ break; ++ ++ netdev_err(ovpn->dev, "%s: remote sockaddr_in has invalid family\n", ++ __func__); ++ goto sockfd_release; ++ case sizeof(struct sockaddr_in6): ++ if (ss->ss_family == AF_INET6) ++ /* valid sockaddr */ ++ break; ++ ++ netdev_err(ovpn->dev, "%s: remote sockaddr_in6 has invalid family\n", ++ __func__); ++ goto sockfd_release; ++ default: ++ netdev_err(ovpn->dev, "%s: invalid size for sockaddr\n", __func__); ++ goto sockfd_release; ++ } ++ ++ if (ss->ss_family == AF_INET6) { ++ in6 = (struct sockaddr_in6 *)ss; ++ ++ if (ipv6_addr_type(&in6->sin6_addr) & IPV6_ADDR_MAPPED) { ++ mapped.sin_family = AF_INET; ++ mapped.sin_addr.s_addr = in6->sin6_addr.s6_addr32[3]; ++ mapped.sin_port = in6->sin6_port; ++ ss = (struct sockaddr_storage *)&mapped; ++ } ++ } ++ ++ /* When using UDP we may be talking over socket bound to 0.0.0.0/::. ++ * In this case, if the host has multiple IPs, we need to make sure ++ * that outgoing traffic has as source IP the same address that the ++ * peer is using to reach us. ++ * ++ * Since early control packets were all forwarded to userspace, we ++ * need the latter to tell us what IP has to be used. ++ */ ++ if (attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]) { ++ ip_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); ++ local_ip = nla_data(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); ++ ++ if (ip_len == sizeof(struct in_addr)) { ++ if (ss->ss_family != AF_INET) { ++ netdev_dbg(ovpn->dev, ++ "%s: the specified local IP is IPv4, but the peer endpoint is not\n", ++ __func__); ++ goto sockfd_release; ++ } ++ } else if (ip_len == sizeof(struct in6_addr)) { ++ bool is_mapped = ipv6_addr_type((struct in6_addr *)local_ip) & ++ IPV6_ADDR_MAPPED; ++ ++ if (ss->ss_family != AF_INET6 && !is_mapped) { ++ netdev_dbg(ovpn->dev, ++ "%s: the specified local IP is IPv6, but the peer endpoint is not\n", ++ __func__); ++ goto sockfd_release; ++ } ++ ++ if (is_mapped) ++ /* this is an IPv6-mapped IPv4 address, therefore extract ++ * the actual v4 address from the last 4 bytes ++ */ ++ local_ip += 12; ++ } else { ++ netdev_dbg(ovpn->dev, ++ "%s: invalid length %zu for local IP\n", __func__, ++ ip_len); ++ goto sockfd_release; ++ } ++ } ++ ++ /* sanity checks passed */ ++ ret = 0; + } -+ return ret; -+} + -+static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, -+ int pref_index, bool use_raw, -+ u32 raw_epp) -+{ -+ int epp = -EINVAL; -+ int ret; ++ id = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_PEER_ID]); ++ peer = ovpn_peer_new(ovpn, ss, sock, id, local_ip); ++ if (IS_ERR(peer)) { ++ netdev_err(ovpn->dev, "%s: cannot create new peer object for peer %u %pIScp\n", ++ __func__, id, ss); ++ ret = PTR_ERR(peer); ++ goto sockfd_release; ++ } + -+ if (!pref_index) { -+ pr_debug("EPP pref_index is invaid\n"); -+ return -EINVAL; ++ if (attrs[OVPN_NEW_PEER_ATTR_IPV4]) { ++ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV4]) != sizeof(struct in_addr)) { ++ ret = -EINVAL; ++ goto peer_release; ++ } ++ ++ peer->vpn_addrs.ipv4.s_addr = nla_get_be32(attrs[OVPN_NEW_PEER_ATTR_IPV4]); + } + -+ if (use_raw) -+ epp = raw_epp; -+ else if (epp == -EINVAL) -+ epp = epp_values[pref_index]; ++ if (attrs[OVPN_NEW_PEER_ATTR_IPV6]) { ++ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV6]) != sizeof(struct in6_addr)) { ++ ret = -EINVAL; ++ goto peer_release; ++ } + -+ if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ pr_debug("EPP cannot be set under performance policy\n"); -+ return -EBUSY; ++ memcpy(&peer->vpn_addrs.ipv6, nla_data(attrs[OVPN_NEW_PEER_ATTR_IPV6]), ++ sizeof(struct in6_addr)); + } + -+ ret = amd_pstate_set_epp(cpudata, epp); ++ netdev_dbg(ovpn->dev, ++ "%s: adding peer with endpoint=%pIScp/%s id=%u VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n", ++ __func__, ss, sock->sk->sk_prot_creator->name, peer->id, ++ &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6); ++ ++ ret = ovpn_peer_add(ovpn, peer); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot add new peer (id=%u) to hashtable: %d\n", ++ __func__, peer->id, ret); ++ goto peer_release; ++ } + ++ return 0; ++ ++peer_release: ++ /* release right away because peer is not really used in any context */ ++ ovpn_peer_release(peer); ++ return ret; ++ ++sockfd_release: ++ sockfd_put(sock); + return ret; +} -+#endif + - static inline int pstate_enable(bool enable) - { - return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); -@@ -131,12 +337,26 @@ static inline int pstate_enable(bool enable) - - static int cppc_enable(bool enable) - { -+ struct cppc_perf_ctrls perf_ctrls; - int cpu, ret = 0; - - for_each_present_cpu(cpu) { - ret = cppc_set_enable(cpu, enable); - if (ret) - return ret; ++static int ovpn_netlink_set_peer(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ u32 peer_id, interv, timeout; ++ bool keepalive_set = false; ++ struct ovpn_peer *peer; ++ int ret; + -+ if (epp_enabled) { -+ /* Enable autonomous mode for EPP */ -+ ret = cppc_set_auto_epp(cpu, enable); -+ if (ret) -+ return ret; ++ if (!info->attrs[OVPN_ATTR_SET_PEER]) ++ return -EINVAL; + -+ /* Set zero to desired perf to allow EPP firmware control*/ -+ perf_ctrls.desired_perf = 0; -+ ret = cppc_set_perf(cpu, &perf_ctrls); -+ if (ret) -+ return ret; -+ } - } - - return ret; -@@ -269,6 +489,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, - u64 prev = READ_ONCE(cpudata->cppc_req_cached); - u64 value = prev; - -+ des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - value &= ~AMD_CPPC_MIN_PERF(~0L); - value |= AMD_CPPC_MIN_PERF(min_perf); - -@@ -312,7 +533,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, - return -ENODEV; - - cap_perf = READ_ONCE(cpudata->highest_perf); -- min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); -+ min_perf = READ_ONCE(cpudata->lowest_perf); - max_perf = cap_perf; - - freqs.old = policy->cur; -@@ -357,8 +578,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, - if (max_perf < min_perf) - max_perf = min_perf; - -- des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); -- - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); - } - -@@ -438,18 +657,27 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) - { - struct amd_cpudata *cpudata = policy->driver_data; - int ret; -+ u64 value; - - if (!cpudata->boost_supported) { - pr_err("Boost mode is not supported by this processor or SBIOS\n"); - return -EINVAL; - } - -- if (state) -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); ++ ret = nla_parse_nested(attrs, OVPN_SET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_SET_PEER], NULL, ++ info->extack); + if (ret) + return ret; + -+ if (state) { -+ value |= AMD_CPPC_PRECISION_BOOST_ENABLED; - policy->cpuinfo.max_freq = cpudata->max_freq; -- else -+ } else { -+ value &= ~AMD_CPPC_PRECISION_BOOST_ENABLED; - policy->cpuinfo.max_freq = cpudata->nominal_freq; -- -+ } - policy->max = policy->cpuinfo.max_freq; -+ WRITE_ONCE(cpudata->cppc_hw_conf_cached, value); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, value); - - ret = freq_qos_update_request(&cpudata->req[1], - policy->cpuinfo.max_freq); -@@ -470,7 +698,7 @@ static void amd_pstate_boost_init(struct amd_cpudata *cpudata) - return; - - cpudata->boost_supported = true; -- amd_pstate_driver.boost_enabled = true; -+ default_pstate_driver->boost_enabled = true; - } - - static int amd_pstate_cpu_init(struct cpufreq_policy *policy) -@@ -478,6 +706,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; - struct device *dev; - struct amd_cpudata *cpudata; -+ u64 value; - - dev = get_cpu_device(policy->cpu); - if (!dev) -@@ -541,7 +770,17 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; - - policy->driver_data = cpudata; -+ if (!shared_mem) { -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_HW_CTL, &value); -+ if (ret) -+ return ret; -+ cpudata->precision_boost_off = value & AMD_CPPC_PRECISION_BOOST_ENABLED; - -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ if (!attrs[OVPN_SET_PEER_ATTR_PEER_ID]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_PEER_ID]); ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ /* when setting the keepalive, both parameters have to be configured */ ++ if (attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] && ++ attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]) { ++ keepalive_set = true; ++ interv = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL]); ++ timeout = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]); + } - amd_pstate_boost_init(cpudata); - - return 0; -@@ -555,9 +794,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) - - static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) - { -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - freq_qos_remove_request(&cpudata->req[1]); - freq_qos_remove_request(&cpudata->req[0]); -@@ -599,9 +836,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, - char *buf) - { - int max_freq; -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - max_freq = amd_get_max_freq(cpudata); - if (max_freq < 0) -@@ -614,9 +849,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli - char *buf) - { - int freq; -- struct amd_cpudata *cpudata; -- -- cpudata = policy->driver_data; -+ struct amd_cpudata *cpudata = policy->driver_data; - - freq = amd_get_lowest_nonlinear_freq(cpudata); - if (freq < 0) -@@ -640,10 +873,108 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, - return sprintf(&buf[0], "%u\n", perf); - } - -+static ssize_t show_energy_performance_available_preferences( -+ struct cpufreq_policy *policy, char *buf) ++ ++ if (keepalive_set) ++ ovpn_peer_keepalive_set(peer, interv, timeout); ++ ++ ovpn_peer_put(peer); ++ return 0; ++} ++ ++static int ovpn_netlink_send_peer(struct sk_buff *skb, const struct ovpn_peer *peer, u32 portid, ++ u32 seq, int flags) +{ -+ int i = 0; -+ int ret = 0; ++ const struct ovpn_bind *bind; ++ struct nlattr *attr; ++ void *hdr; ++ ++ hdr = genlmsg_put(skb, portid, seq, &ovpn_netlink_family, flags, OVPN_CMD_GET_PEER); ++ if (!hdr) { ++ netdev_dbg(peer->ovpn->dev, "%s: cannot create message header\n", __func__); ++ return -EMSGSIZE; ++ } ++ ++ attr = nla_nest_start(skb, OVPN_ATTR_GET_PEER); ++ if (!attr) { ++ netdev_dbg(peer->ovpn->dev, "%s: cannot create submessage\n", __func__); ++ goto err; ++ } ++ ++ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_PEER_ID, peer->id)) ++ goto err; ++ ++ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV4, sizeof(peer->vpn_addrs.ipv4), ++ &peer->vpn_addrs.ipv4)) ++ goto err; + -+ while (energy_perf_strings[i] != NULL) -+ ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); ++ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV6, sizeof(peer->vpn_addrs.ipv6), ++ &peer->vpn_addrs.ipv6)) ++ goto err; + -+ ret += sprintf(&buf[ret], "\n"); ++ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, ++ peer->keepalive_interval) || ++ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, ++ peer->keepalive_timeout)) ++ goto err; + -+ return ret; ++ rcu_read_lock(); ++ bind = rcu_dereference(peer->bind); ++ if (bind) { ++ if (bind->sa.in4.sin_family == AF_INET) { ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, ++ sizeof(bind->sa.in4), &bind->sa.in4) || ++ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, ++ sizeof(bind->local.ipv4), &bind->local.ipv4)) ++ goto err_unlock; ++ } else if (bind->sa.in4.sin_family == AF_INET6) { ++ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, ++ sizeof(bind->sa.in6), &bind->sa.in6) || ++ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, ++ sizeof(bind->local.ipv6), &bind->local.ipv6)) ++ goto err_unlock; ++ } ++ } ++ rcu_read_unlock(); ++ ++ if (nla_put_net16(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, ++ inet_sk(peer->sock->sock->sk)->inet_sport) || ++ /* RX stats */ ++ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_RX_BYTES, ++ atomic64_read(&peer->stats.rx.bytes), ++ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || ++ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, ++ atomic_read(&peer->stats.rx.packets)) || ++ /* TX stats */ ++ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_TX_BYTES, ++ atomic64_read(&peer->stats.tx.bytes), ++ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || ++ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, ++ atomic_read(&peer->stats.tx.packets))) ++ goto err; ++ ++ nla_nest_end(skb, attr); ++ genlmsg_end(skb, hdr); ++ ++ return 0; ++err_unlock: ++ rcu_read_unlock(); ++err: ++ genlmsg_cancel(skb, hdr); ++ return -EMSGSIZE; +} + -+static ssize_t store_energy_performance_preference( -+ struct cpufreq_policy *policy, const char *buf, size_t count) ++static int ovpn_netlink_get_peer(struct sk_buff *skb, struct genl_info *info) +{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ char str_preference[21]; -+ bool raw = false; -+ ssize_t ret; -+ u32 epp = 0; ++ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer *peer; ++ struct sk_buff *msg; ++ u32 peer_id; ++ int ret; + -+ ret = sscanf(buf, "%20s", str_preference); -+ if (ret != 1) ++ if (!info->attrs[OVPN_ATTR_GET_PEER]) + return -EINVAL; + -+ ret = match_string(energy_perf_strings, -1, str_preference); -+ if (ret < 0) { -+ ret = kstrtouint(buf, 10, &epp); -+ if (ret) -+ return ret; ++ ret = nla_parse_nested(attrs, OVPN_GET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_GET_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; + -+ if ((epp > 255) || (epp < 0)) -+ return -EINVAL; ++ if (!attrs[OVPN_GET_PEER_ATTR_PEER_ID]) ++ return -EINVAL; + -+ raw = true; ++ peer_id = nla_get_u32(attrs[OVPN_GET_PEER_ATTR_PEER_ID]); ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); ++ if (!msg) ++ return -ENOMEM; ++ ++ ret = ovpn_netlink_send_peer(msg, peer, info->snd_portid, info->snd_seq, 0); ++ if (ret < 0) { ++ nlmsg_free(msg); ++ goto err; + } + -+ mutex_lock(&amd_pstate_limits_lock); -+ ret = amd_pstate_set_energy_pref_index(cpudata, ret, raw, epp); -+ mutex_unlock(&amd_pstate_limits_lock); ++ ret = genlmsg_reply(msg, info); ++err: ++ ovpn_peer_put(peer); ++ return ret; ++} + -+ return ret ?: count; ++static int ovpn_netlink_dump_done(struct netlink_callback *cb) ++{ ++ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; ++ ++ dev_put(ovpn->dev); ++ return 0; +} + -+static ssize_t show_energy_performance_preference( -+ struct cpufreq_policy *policy, char *buf) ++static int ovpn_netlink_dump_prepare(struct netlink_callback *cb) +{ -+ struct amd_cpudata *cpudata = policy->driver_data; -+ int preference, raw_epp; ++ struct net *netns = sock_net(cb->skb->sk); ++ struct nlattr **attrbuf; ++ struct net_device *dev; ++ int ret; + -+ preference = amd_pstate_get_energy_pref_index(cpudata, &raw_epp); -+ if (preference < 0) -+ return preference; ++ attrbuf = kcalloc(OVPN_ATTR_MAX + 1, sizeof(*attrbuf), GFP_KERNEL); ++ if (!attrbuf) ++ return -ENOMEM; + -+ if (raw_epp) -+ return sprintf(buf, "%d\n", raw_epp); -+ else -+ return sprintf(buf, "%s\n", energy_perf_strings[preference]); ++ ret = nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrbuf, OVPN_ATTR_MAX, ++ ovpn_netlink_policy, NULL); ++ if (ret < 0) ++ goto err; ++ ++ dev = ovpn_get_dev_from_attrs(netns, attrbuf); ++ if (IS_ERR(dev)) { ++ ret = PTR_ERR(dev); ++ goto err; ++ } ++ ++ cb->args[0] = (long)netdev_priv(dev); ++ ret = 0; ++err: ++ kfree(attrbuf); ++ return ret; +} + -+static void amd_pstate_update_policies(void) ++static int ovpn_netlink_dump_peers(struct sk_buff *skb, struct netlink_callback *cb) +{ -+ int cpu; ++ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; ++ int ret, bkt, last_idx = cb->args[1], dumped = 0; ++ struct ovpn_peer *peer; + -+ for_each_possible_cpu(cpu) -+ cpufreq_update_policy(cpu); ++ if (!ovpn) { ++ ret = ovpn_netlink_dump_prepare(cb); ++ if (ret < 0) { ++ netdev_dbg(ovpn->dev, "%s: cannot prepare for dump: %d\n", __func__, ret); ++ return ret; ++ } ++ ++ ovpn = (struct ovpn_struct *)cb->args[0]; ++ } ++ ++ rcu_read_lock(); ++ hash_for_each_rcu(ovpn->peers.by_id, bkt, peer, hash_entry_id) { ++ /* skip already dumped peers that were dumped by previous invocations */ ++ if (last_idx > 0) { ++ last_idx--; ++ continue; ++ } ++ ++ if (ovpn_netlink_send_peer(skb, peer, NETLINK_CB(cb->skb).portid, ++ cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) ++ break; ++ ++ /* count peers being dumped during this invocation */ ++ dumped++; ++ } ++ rcu_read_unlock(); ++ ++ /* sum up peers dumped in this message, so that at the next invocation ++ * we can continue from where we left ++ */ ++ cb->args[1] += dumped; ++ ++ return skb->len; +} + -+static ssize_t show_pstate_dynamic_boost(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) ++static int ovpn_netlink_del_peer(struct sk_buff *skb, struct genl_info *info) +{ -+ return sprintf(buf, "%u\n", cppc_boost); ++ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ struct ovpn_peer *peer; ++ u32 peer_id; ++ int ret; ++ ++ if (!info->attrs[OVPN_ATTR_DEL_PEER]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_DEL_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_PEER], NULL, ++ info->extack); ++ if (ret) ++ return ret; ++ ++ if (!attrs[OVPN_DEL_PEER_ATTR_PEER_ID]) ++ return -EINVAL; ++ ++ peer_id = nla_get_u32(attrs[OVPN_DEL_PEER_ATTR_PEER_ID]); ++ ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) ++ return -ENOENT; ++ ++ netdev_dbg(ovpn->dev, "%s: peer id=%u\n", __func__, peer->id); ++ ret = ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_USERSPACE); ++ ovpn_peer_put(peer); ++ ++ return ret; +} + -+static ssize_t store_pstate_dynamic_boost(struct kobject *a, -+ struct kobj_attribute *b, -+ const char *buf, size_t count) ++static int ovpn_netlink_register_packet(struct sk_buff *skb, ++ struct genl_info *info) +{ -+ unsigned int input; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ ++ /* only one registered process per interface is allowed for now */ ++ if (ovpn->registered_nl_portid_set) { ++ netdev_dbg(ovpn->dev, "%s: userspace listener already registered\n", __func__); ++ return -EBUSY; ++ } ++ ++ netdev_dbg(ovpn->dev, "%s: registering userspace at %u\n", __func__, info->snd_portid); ++ ++ ovpn->registered_nl_portid = info->snd_portid; ++ ovpn->registered_nl_portid_set = true; ++ ++ return 0; ++} ++ ++static int ovpn_netlink_packet(struct sk_buff *skb, struct genl_info *info) ++{ ++ struct nlattr *attrs[OVPN_PACKET_ATTR_MAX + 1]; ++ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ const u8 *packet; ++ u32 peer_id; ++ size_t len; ++ u8 opcode; + int ret; + -+ ret = kstrtouint(buf, 10, &input); ++ if (!info->attrs[OVPN_ATTR_PACKET]) ++ return -EINVAL; ++ ++ ret = nla_parse_nested(attrs, OVPN_PACKET_ATTR_MAX, info->attrs[OVPN_ATTR_PACKET], ++ NULL, info->extack); + if (ret) + return ret; + -+ mutex_lock(&amd_pstate_driver_lock); -+ cppc_boost = !!input; -+ amd_pstate_update_policies(); -+ mutex_unlock(&amd_pstate_driver_lock); ++ if (!attrs[OVPN_PACKET_ATTR_PACKET] || !attrs[OVPN_PACKET_ATTR_PEER_ID]) { ++ netdev_dbg(ovpn->dev, "received netlink packet with no payload\n"); ++ return -EINVAL; ++ } + -+ return count; -+} ++ peer_id = nla_get_u32(attrs[OVPN_PACKET_ATTR_PEER_ID]); + - cpufreq_freq_attr_ro(amd_pstate_max_freq); - cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); - - cpufreq_freq_attr_ro(amd_pstate_highest_perf); -+cpufreq_freq_attr_rw(energy_performance_preference); -+cpufreq_freq_attr_ro(energy_performance_available_preferences); -+define_one_global_rw(pstate_dynamic_boost); - - static struct freq_attr *amd_pstate_attr[] = { - &amd_pstate_max_freq, -@@ -652,6 +983,548 @@ static struct freq_attr *amd_pstate_attr[] = { - NULL, - }; - -+static struct freq_attr *amd_pstate_epp_attr[] = { -+ &amd_pstate_max_freq, -+ &amd_pstate_lowest_nonlinear_freq, -+ &amd_pstate_highest_perf, -+ &energy_performance_preference, -+ &energy_performance_available_preferences, -+ NULL, -+}; ++ len = nla_len(attrs[OVPN_PACKET_ATTR_PACKET]); + -+static struct attribute *pstate_global_attributes[] = { -+ &pstate_dynamic_boost.attr, -+ NULL ++ if (len < 4 || len > ovpn->dev->mtu) { ++ netdev_dbg(ovpn->dev, "%s: invalid packet size %zu (min is 4, max is MTU: %u)\n", ++ __func__, len, ovpn->dev->mtu); ++ return -EINVAL; ++ } ++ ++ packet = nla_data(attrs[OVPN_PACKET_ATTR_PACKET]); ++ opcode = ovpn_opcode_from_byte(packet[0]); ++ ++ /* reject data packets from userspace as they could lead to IV reuse */ ++ if (opcode == OVPN_DATA_V1 || opcode == OVPN_DATA_V2) { ++ netdev_dbg(ovpn->dev, "%s: rejecting data packet from userspace (opcode=%u)\n", ++ __func__, opcode); ++ return -EINVAL; ++ } ++ ++ netdev_dbg(ovpn->dev, "%s: sending userspace packet to peer %u...\n", __func__, peer_id); ++ ++ return ovpn_send_data(ovpn, peer_id, packet, len); ++} ++ ++static const struct genl_ops ovpn_netlink_ops[] = { ++ { ++ .cmd = OVPN_CMD_NEW_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_new_peer, ++ }, ++ { ++ .cmd = OVPN_CMD_SET_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_set_peer, ++ }, ++ { ++ .cmd = OVPN_CMD_DEL_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_del_peer, ++ }, ++ { ++ .cmd = OVPN_CMD_GET_PEER, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_get_peer, ++ .dumpit = ovpn_netlink_dump_peers, ++ .done = ovpn_netlink_dump_done, ++ }, ++ { ++ .cmd = OVPN_CMD_NEW_KEY, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_new_key, ++ }, ++ { ++ .cmd = OVPN_CMD_DEL_KEY, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_del_key, ++ }, ++ { ++ .cmd = OVPN_CMD_SWAP_KEYS, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_swap_keys, ++ }, ++ { ++ .cmd = OVPN_CMD_REGISTER_PACKET, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_register_packet, ++ }, ++ { ++ .cmd = OVPN_CMD_PACKET, ++ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, ++ .flags = GENL_ADMIN_PERM, ++ .doit = ovpn_netlink_packet, ++ }, +}; + -+static const struct attribute_group amd_pstate_global_attr_group = { -+ .attrs = pstate_global_attributes, ++static struct genl_family ovpn_netlink_family __ro_after_init = { ++ .hdrsize = 0, ++ .name = OVPN_NL_NAME, ++ .version = 1, ++ .maxattr = OVPN_ATTR_MAX, ++ .policy = ovpn_netlink_policy, ++ .netnsok = true, ++ .pre_doit = ovpn_pre_doit, ++ .post_doit = ovpn_post_doit, ++ .module = THIS_MODULE, ++ .ops = ovpn_netlink_ops, ++ .n_ops = ARRAY_SIZE(ovpn_netlink_ops), ++ .mcgrps = ovpn_netlink_mcgrps, ++ .n_mcgrps = ARRAY_SIZE(ovpn_netlink_mcgrps), +}; + -+static inline void update_boost_state(void) ++int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer) +{ -+ u64 misc_en; -+ struct amd_cpudata *cpudata; ++ struct sk_buff *msg; ++ struct nlattr *attr; ++ void *hdr; ++ int ret; + -+ cpudata = all_cpu_data[0]; -+ rdmsrl(MSR_AMD_CPPC_HW_CTL, misc_en); -+ global.cppc_boost_disabled = misc_en & AMD_CPPC_PRECISION_BOOST_ENABLED; -+} ++ netdev_info(peer->ovpn->dev, "%s: deleting peer with id %u, reason %d\n", ++ peer->ovpn->dev->name, peer->id, peer->delete_reason); + -+static int amd_pstate_init_cpu(unsigned int cpunum) -+{ -+ struct amd_cpudata *cpudata; ++ msg = nlmsg_new(100, GFP_KERNEL); ++ if (!msg) ++ return -ENOMEM; + -+ cpudata = all_cpu_data[cpunum]; -+ if (!cpudata) { -+ cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); -+ if (!cpudata) -+ return -ENOMEM; -+ WRITE_ONCE(all_cpu_data[cpunum], cpudata); ++ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, ++ OVPN_CMD_DEL_PEER); ++ if (!hdr) { ++ ret = -ENOBUFS; ++ goto err_free_msg; ++ } + -+ cpudata->cpu = cpunum; ++ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, peer->ovpn->dev->ifindex)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; + } -+ cpudata->epp_powersave = -EINVAL; -+ cpudata->epp_policy = 0; -+ pr_debug("controlling: cpu %d\n", cpunum); ++ ++ attr = nla_nest_start(msg, OVPN_ATTR_DEL_PEER); ++ if (!attr) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u8(msg, OVPN_DEL_PEER_ATTR_REASON, peer->delete_reason)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u32(msg, OVPN_DEL_PEER_ATTR_PEER_ID, peer->id)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ nla_nest_end(msg, attr); ++ ++ genlmsg_end(msg, hdr); ++ ++ genlmsg_multicast_netns(&ovpn_netlink_family, dev_net(peer->ovpn->dev), ++ msg, 0, OVPN_MCGRP_PEERS, GFP_KERNEL); ++ + return 0; ++ ++err_free_msg: ++ nlmsg_free(msg); ++ return ret; +} + -+static int __amd_pstate_cpu_init(struct cpufreq_policy *policy) ++int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, ++ const u8 *buf, size_t len) +{ -+ int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; -+ struct amd_cpudata *cpudata; -+ struct device *dev; -+ int rc; -+ u64 value; ++ struct nlattr *attr; ++ struct sk_buff *msg; ++ void *hdr; ++ int ret; + -+ rc = amd_pstate_init_cpu(policy->cpu); -+ if (rc) -+ return rc; ++ if (!ovpn->registered_nl_portid_set) { ++ net_warn_ratelimited("%s: no userspace listener\n", __func__); ++ return 0; ++ } + -+ cpudata = all_cpu_data[policy->cpu]; ++ netdev_dbg(ovpn->dev, "%s: sending packet to userspace, len: %zd\n", __func__, len); + -+ dev = get_cpu_device(policy->cpu); -+ if (!dev) -+ goto free_cpudata1; ++ msg = nlmsg_new(100 + len, GFP_ATOMIC); ++ if (!msg) ++ return -ENOMEM; + -+ rc = amd_pstate_init_perf(cpudata); -+ if (rc) -+ goto free_cpudata1; ++ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, ++ OVPN_CMD_PACKET); ++ if (!hdr) { ++ ret = -ENOBUFS; ++ goto err_free_msg; ++ } ++ ++ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, ovpn->dev->ifindex)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ attr = nla_nest_start(msg, OVPN_ATTR_PACKET); ++ if (!attr) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } ++ ++ if (nla_put(msg, OVPN_PACKET_ATTR_PACKET, len, buf)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; ++ } + -+ min_freq = amd_get_min_freq(cpudata); -+ max_freq = amd_get_max_freq(cpudata); -+ nominal_freq = amd_get_nominal_freq(cpudata); -+ lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); -+ if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { -+ dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", -+ min_freq, max_freq); -+ ret = -EINVAL; -+ goto free_cpudata1; ++ if (nla_put_u32(msg, OVPN_PACKET_ATTR_PEER_ID, peer->id)) { ++ ret = -EMSGSIZE; ++ goto err_free_msg; + } + -+ policy->min = min_freq; -+ policy->max = max_freq; ++ nla_nest_end(msg, attr); + -+ policy->cpuinfo.min_freq = min_freq; -+ policy->cpuinfo.max_freq = max_freq; -+ /* It will be updated by governor */ -+ policy->cur = policy->cpuinfo.min_freq; ++ genlmsg_end(msg, hdr); + -+ /* Initial processor data capability frequencies */ -+ cpudata->max_freq = max_freq; -+ cpudata->min_freq = min_freq; -+ cpudata->nominal_freq = nominal_freq; -+ cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; ++ return genlmsg_unicast(dev_net(ovpn->dev), msg, ++ ovpn->registered_nl_portid); + -+ policy->driver_data = cpudata; ++err_free_msg: ++ nlmsg_free(msg); ++ return ret; ++} + -+ update_boost_state(); -+ cpudata->epp_cached = amd_pstate_get_epp(cpudata, value); ++static int ovpn_netlink_notify(struct notifier_block *nb, unsigned long state, ++ void *_notify) ++{ ++ struct netlink_notify *notify = _notify; ++ struct ovpn_struct *ovpn; ++ struct net_device *dev; ++ struct net *netns; ++ bool found = false; + -+ policy->min = policy->cpuinfo.min_freq; -+ policy->max = policy->cpuinfo.max_freq; ++ if (state != NETLINK_URELEASE || notify->protocol != NETLINK_GENERIC) ++ return NOTIFY_DONE; + -+ if (boot_cpu_has(X86_FEATURE_CPPC)) -+ policy->fast_switch_possible = true; ++ rcu_read_lock(); ++ for_each_net_rcu(netns) { ++ for_each_netdev_rcu(netns, dev) { ++ if (!ovpn_dev_is_valid(dev)) ++ continue; + -+ if (!shared_mem && boot_cpu_has(X86_FEATURE_CPPC)) { -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_req_cached, value); ++ ovpn = netdev_priv(dev); ++ if (notify->portid != ovpn->registered_nl_portid) ++ continue; + -+ ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); -+ if (ret) -+ return ret; -+ WRITE_ONCE(cpudata->cppc_cap1_cached, value); ++ found = true; ++ netdev_dbg(ovpn->dev, "%s: deregistering userspace listener\n", __func__); ++ ovpn->registered_nl_portid_set = false; ++ break; ++ } + } -+ amd_pstate_boost_init(cpudata); ++ rcu_read_unlock(); + -+ return 0; ++ /* if no interface matched our purposes, pass the notification along */ ++ if (!found) ++ return NOTIFY_DONE; + -+free_cpudata1: -+ kfree(cpudata); -+ return ret; ++ return NOTIFY_OK; +} + -+static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) ++static struct notifier_block ovpn_netlink_notifier = { ++ .notifier_call = ovpn_netlink_notify, ++}; ++ ++int ovpn_netlink_init(struct ovpn_struct *ovpn) ++{ ++ ovpn->registered_nl_portid_set = false; ++ ++ return 0; ++} ++ ++/** ++ * ovpn_netlink_register() - register the ovpn genl netlink family ++ */ ++int __init ovpn_netlink_register(void) +{ + int ret; + -+ ret = __amd_pstate_cpu_init(policy); ++ ret = genl_register_family(&ovpn_netlink_family); + if (ret) + return ret; -+ /* -+ * Set the policy to powersave to provide a valid fallback value in case -+ * the default cpufreq governor is neither powersave nor performance. -+ */ -+ policy->policy = CPUFREQ_POLICY_POWERSAVE; ++ ++ ret = netlink_register_notifier(&ovpn_netlink_notifier); ++ if (ret) ++ goto err; + + return 0; ++err: ++ genl_unregister_family(&ovpn_netlink_family); ++ return ret; +} + -+static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) ++/** ++ * ovpn_netlink_unregister() - unregister the ovpn genl netlink family ++ */ ++void __exit ovpn_netlink_unregister(void) +{ -+ pr_debug("amd-pstate: CPU %d exiting\n", policy->cpu); -+ policy->fast_switch_possible = false; -+ return 0; ++ netlink_unregister_notifier(&ovpn_netlink_notifier); ++ genl_unregister_family(&ovpn_netlink_family); +} +diff --git a/drivers/net/ovpn-dco/netlink.h b/drivers/net/ovpn-dco/netlink.h +new file mode 100644 +index 000000000000..843daf052c03 +--- /dev/null ++++ b/drivers/net/ovpn-dco/netlink.h +@@ -0,0 +1,22 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ + -+static void amd_pstate_update_max_freq(unsigned int cpu) -+{ -+ struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); ++#ifndef _NET_OVPN_DCO_NETLINK_H_ ++#define _NET_OVPN_DCO_NETLINK_H_ + -+ if (!policy) -+ return; ++struct ovpn_struct; ++struct ovpn_peer; + -+ refresh_frequency_limits(policy); -+ cpufreq_cpu_release(policy); -+} ++int ovpn_netlink_init(struct ovpn_struct *ovpn); ++int ovpn_netlink_register(void); ++void ovpn_netlink_unregister(void); ++int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, ++ const u8 *buf, size_t len); ++int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer); + -+static void amd_pstate_epp_update_limits(unsigned int cpu) -+{ -+ mutex_lock(&amd_pstate_driver_lock); -+ update_boost_state(); -+ if (global.cppc_boost_disabled) { -+ for_each_possible_cpu(cpu) -+ amd_pstate_update_max_freq(cpu); -+ } else { -+ cpufreq_update_policy(cpu); -+ } -+ mutex_unlock(&amd_pstate_driver_lock); -+} ++#endif /* _NET_OVPN_DCO_NETLINK_H_ */ +diff --git a/drivers/net/ovpn-dco/ovpn.c b/drivers/net/ovpn-dco/ovpn.c +new file mode 100644 +index 000000000000..66c019174f5e +--- /dev/null ++++ b/drivers/net/ovpn-dco/ovpn.c +@@ -0,0 +1,600 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+static int cppc_boost_hold_time_ns = 3 * NSEC_PER_MSEC; ++#include "main.h" ++#include "bind.h" ++#include "netlink.h" ++#include "ovpn.h" ++#include "sock.h" ++#include "peer.h" ++#include "stats.h" ++#include "proto.h" ++#include "crypto.h" ++#include "crypto_aead.h" ++#include "skb.h" ++#include "tcp.h" ++#include "udp.h" + -+static inline void amd_pstate_boost_up(struct amd_cpudata *cpudata) -+{ -+ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); -+ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); -+ u32 max_limit = (hwp_req & 0xff); -+ u32 min_limit = (hwp_req & 0xff00) >> 8; -+ u32 boost_level1; ++#include ++#include + -+ /* If max and min are equal or already at max, nothing to boost */ -+ if (max_limit == min_limit) -+ return; ++static const unsigned char ovpn_keepalive_message[] = { ++ 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, ++ 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 ++}; + -+ /* Set boost max and min to initial value */ -+ if (!cpudata->cppc_boost_min) -+ cpudata->cppc_boost_min = min_limit; ++static const unsigned char ovpn_explicit_exit_notify_message[] = { ++ 0x28, 0x7f, 0x34, 0x6b, 0xd4, 0xef, 0x7a, 0x81, ++ 0x2d, 0x56, 0xb8, 0xd3, 0xaf, 0xc5, 0x45, 0x9c, ++ 6 // OCC_EXIT ++}; + -+ boost_level1 = ((AMD_CPPC_NOMINAL_PERF(hwp_cap) + min_limit) >> 1); ++/* Is keepalive message? ++ * Assumes that single byte at skb->data is defined. ++ */ ++static bool ovpn_is_keepalive(struct sk_buff *skb) ++{ ++ if (*skb->data != OVPN_KEEPALIVE_FIRST_BYTE) ++ return false; + -+ if (cpudata->cppc_boost_min < boost_level1) -+ cpudata->cppc_boost_min = boost_level1; -+ else if (cpudata->cppc_boost_min < AMD_CPPC_NOMINAL_PERF(hwp_cap)) -+ cpudata->cppc_boost_min = AMD_CPPC_NOMINAL_PERF(hwp_cap); -+ else if (cpudata->cppc_boost_min == AMD_CPPC_NOMINAL_PERF(hwp_cap)) -+ cpudata->cppc_boost_min = max_limit; -+ else -+ return; ++ if (!pskb_may_pull(skb, sizeof(ovpn_keepalive_message))) ++ return false; + -+ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); -+ hwp_req |= AMD_CPPC_MIN_PERF(cpudata->cppc_boost_min); -+ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); -+ cpudata->last_update = cpudata->sample.time; ++ return !memcmp(skb->data, ovpn_keepalive_message, ++ sizeof(ovpn_keepalive_message)); +} + -+static inline void amd_pstate_boost_down(struct amd_cpudata *cpudata) ++int ovpn_struct_init(struct net_device *dev) +{ -+ bool expired; ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ int err; + -+ if (cpudata->cppc_boost_min) { -+ expired = time_after64(cpudata->sample.time, cpudata->last_update + -+ cppc_boost_hold_time_ns); ++ memset(ovpn, 0, sizeof(*ovpn)); + -+ if (expired) { -+ wrmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, cpudata->cppc_req_cached); -+ cpudata->cppc_boost_min = 0; -+ } -+ } ++ ovpn->dev = dev; + -+ cpudata->last_update = cpudata->sample.time; ++ err = ovpn_netlink_init(ovpn); ++ if (err < 0) ++ return err; ++ ++ spin_lock_init(&ovpn->lock); ++ spin_lock_init(&ovpn->peers.lock); ++ ++ ovpn->crypto_wq = alloc_workqueue("ovpn-crypto-wq-%s", ++ WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, ++ dev->name); ++ if (!ovpn->crypto_wq) ++ return -ENOMEM; ++ ++ ovpn->events_wq = alloc_workqueue("ovpn-event-wq-%s", WQ_MEM_RECLAIM, 0, dev->name); ++ if (!ovpn->events_wq) ++ return -ENOMEM; ++ ++ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); ++ if (!dev->tstats) ++ return -ENOMEM; ++ ++ err = security_tun_dev_alloc_security(&ovpn->security); ++ if (err < 0) ++ return err; ++ ++ /* kernel -> userspace tun queue length */ ++ ovpn->max_tun_queue_len = OVPN_MAX_TUN_QUEUE_LEN; ++ ++ return 0; +} + -+static inline void amd_pstate_boost_update_util(struct amd_cpudata *cpudata, -+ u64 time) ++/* Called after decrypt to write IP packet to tun netdev. ++ * This method is expected to manage/free skb. ++ */ ++static void tun_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) +{ -+ cpudata->sample.time = time; -+ if (smp_processor_id() != cpudata->cpu) -+ return; ++ /* packet integrity was verified on the VPN layer - no need to perform ++ * any additional check along the stack ++ */ ++ skb->ip_summed = CHECKSUM_UNNECESSARY; ++ skb->csum_level = ~0; + -+ if (cpudata->sched_flags & SCHED_CPUFREQ_IOWAIT) { -+ bool do_io = false; ++ /* skb hash for transport packet no longer valid after decapsulation */ ++ skb_clear_hash(skb); + -+ cpudata->sched_flags = 0; -+ /* -+ * Set iowait_boost flag and update time. Since IO WAIT flag -+ * is set all the time, we can't just conclude that there is -+ * some IO bound activity is scheduled on this CPU with just -+ * one occurrence. If we receive at least two in two -+ * consecutive ticks, then we treat as boost candidate. -+ * This is leveraged from Intel Pstate driver. -+ */ -+ if (time_before64(time, cpudata->last_io_update + 2 * TICK_NSEC)) -+ do_io = true; ++ /* post-decrypt scrub -- prepare to inject encapsulated packet onto tun ++ * interface, based on __skb_tunnel_rx() in dst.h ++ */ ++ skb->dev = peer->ovpn->dev; ++ skb_set_queue_mapping(skb, 0); ++ skb_scrub_packet(skb, true); + -+ cpudata->last_io_update = time; ++ skb_reset_network_header(skb); ++ skb_reset_transport_header(skb); ++ skb_probe_transport_header(skb); ++ skb_reset_inner_headers(skb); + -+ if (do_io) -+ amd_pstate_boost_up(cpudata); ++ /* update per-cpu RX stats with the stored size of encrypted packet */ + -+ } else { -+ amd_pstate_boost_down(cpudata); -+ } ++ /* we are in softirq context - hence no locking nor disable preemption needed */ ++ dev_sw_netstats_rx_add(peer->ovpn->dev, OVPN_SKB_CB(skb)->rx_stats_size); ++ ++ /* cause packet to be "received" by tun interface */ ++ napi_gro_receive(&peer->napi, skb); +} + -+static inline void amd_pstate_cppc_update_hook(struct update_util_data *data, -+ u64 time, unsigned int flags) ++int ovpn_napi_poll(struct napi_struct *napi, int budget) +{ -+ struct amd_cpudata *cpudata = container_of(data, -+ struct amd_cpudata, update_util); ++ struct ovpn_peer *peer = container_of(napi, struct ovpn_peer, napi); ++ struct sk_buff *skb; ++ int work_done = 0; + -+ cpudata->sched_flags |= flags; ++ if (unlikely(budget <= 0)) ++ return 0; ++ /* this function should schedule at most 'budget' number of ++ * packets for delivery to the tun interface. ++ * If in the queue we have more packets than what allowed by the ++ * budget, the next polling will take care of those ++ */ ++ while ((work_done < budget) && ++ (skb = ptr_ring_consume_bh(&peer->netif_rx_ring))) { ++ tun_netdev_write(peer, skb); ++ work_done++; ++ } + -+ if (smp_processor_id() == cpudata->cpu) -+ amd_pstate_boost_update_util(cpudata, time); ++ if (work_done < budget) ++ napi_complete_done(napi, work_done); ++ ++ return work_done; +} + -+static void amd_pstate_clear_update_util_hook(unsigned int cpu) ++static int ovpn_transport_to_userspace(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, ++ struct sk_buff *skb) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu]; ++ int ret; + -+ if (!cpudata->update_util_set) -+ return; ++ ret = skb_linearize(skb); ++ if (ret < 0) ++ return ret; + -+ cpufreq_remove_update_util_hook(cpu); -+ cpudata->update_util_set = false; -+ synchronize_rcu(); ++ ret = ovpn_netlink_send_packet(ovpn, peer, skb->data, skb->len); ++ if (ret < 0) ++ return ret; ++ ++ consume_skb(skb); ++ return 0; +} + -+static void amd_pstate_set_update_util_hook(unsigned int cpu_num) ++/* Entry point for processing an incoming packet (in skb form) ++ * ++ * Enqueue the packet and schedule RX consumer. ++ * Reference to peer is dropped only in case of success. ++ * ++ * Return 0 if the packet was handled (and consumed) ++ * Return <0 in case of error (return value is error code) ++ */ ++int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu_num]; ++ int ret; + -+ if (!cppc_boost) { -+ if (cpudata->update_util_set) -+ amd_pstate_clear_update_util_hook(cpudata->cpu); -+ return; ++ /* At this point we know the packet is from a configured peer. ++ * DATA_V2 packets are handled in kernel space, the rest goes to user space. ++ * ++ * Packets are sent to userspace via netlink API in order to be consistenbt across ++ * UDP and TCP. ++ */ ++ if (unlikely(ovpn_opcode_from_skb(skb, 0) != OVPN_DATA_V2)) { ++ ret = ovpn_transport_to_userspace(ovpn, peer, skb); ++ if (ret < 0) ++ return ret; ++ ++ ovpn_peer_put(peer); ++ return 0; + } + -+ if (cpudata->update_util_set) -+ return; ++ ret = ptr_ring_produce_bh(&peer->rx_ring, skb); ++ if (unlikely(ret < 0)) ++ return -ENOSPC; + -+ cpudata->sample.time = 0; -+ cpufreq_add_update_util_hook(cpu_num, &cpudata->update_util, -+ amd_pstate_cppc_update_hook); -+ cpudata->update_util_set = true; ++ if (!queue_work(ovpn->crypto_wq, &peer->decrypt_work)) ++ ovpn_peer_put(peer); ++ ++ return 0; +} + -+static void amd_pstate_epp_init(unsigned int cpu) ++static int ovpn_decrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[cpu]; -+ u32 max_perf, min_perf; -+ u64 value; -+ s16 epp; -+ int ret; ++ struct ovpn_peer *allowed_peer = NULL; ++ struct ovpn_crypto_key_slot *ks; ++ unsigned int rx_stats_size; ++ __be16 proto; ++ int ret = -1; ++ u8 key_id; + -+ max_perf = READ_ONCE(cpudata->highest_perf); -+ min_perf = READ_ONCE(cpudata->lowest_perf); ++ /* save original packet size for stats accounting */ ++ OVPN_SKB_CB(skb)->rx_stats_size = skb->len; + -+ value = READ_ONCE(cpudata->cppc_req_cached); ++ /* get the key slot matching the key Id in the received packet */ ++ key_id = ovpn_key_id_from_skb(skb); ++ ks = ovpn_crypto_key_id_to_slot(&peer->crypto, key_id); ++ if (unlikely(!ks)) { ++ net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n", __func__, ++ peer->id, key_id); ++ goto drop; ++ } + -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) -+ min_perf = max_perf; ++ /* decrypt */ ++ ret = ovpn_aead_decrypt(ks, skb); + -+ /* Initial min/max values for CPPC Performance Controls Register */ -+ value &= ~AMD_CPPC_MIN_PERF(~0L); -+ value |= AMD_CPPC_MIN_PERF(min_perf); ++ ovpn_crypto_key_slot_put(ks); + -+ value &= ~AMD_CPPC_MAX_PERF(~0L); -+ value |= AMD_CPPC_MAX_PERF(max_perf); ++ if (unlikely(ret < 0)) { ++ net_err_ratelimited("%s: error during decryption for peer %u, key-id %u: %d\n", ++ __func__, peer->id, key_id, ret); ++ goto drop; ++ } + -+ /* CPPC EPP feature require to set zero to the desire perf bit */ -+ value &= ~AMD_CPPC_DES_PERF(~0L); -+ value |= AMD_CPPC_DES_PERF(0); ++ /* note event of authenticated packet received for keepalive */ ++ ovpn_peer_keepalive_recv_reset(peer); + -+ if (cpudata->epp_policy == cpudata->policy) -+ goto skip_epp; ++ /* update source and destination endpoint for this peer */ ++ if (peer->sock->sock->sk->sk_protocol == IPPROTO_UDP) ++ ovpn_peer_update_local_endpoint(peer, skb); + -+ cpudata->epp_policy = cpudata->policy; ++ /* increment RX stats */ ++ rx_stats_size = OVPN_SKB_CB(skb)->rx_stats_size; ++ ovpn_peer_stats_increment_rx(&peer->stats, rx_stats_size); + -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ epp = amd_pstate_get_epp(cpudata, value); -+ cpudata->epp_powersave = epp; -+ if (epp < 0) -+ goto skip_epp; ++ /* check if this is a valid datapacket that has to be delivered to the ++ * tun interface ++ */ ++ skb_reset_network_header(skb); ++ proto = ovpn_ip_check_protocol(skb); ++ if (unlikely(!proto)) { ++ /* check if null packet */ ++ if (unlikely(!pskb_may_pull(skb, 1))) { ++ ret = -EINVAL; ++ goto drop; ++ } + -+ epp = 0; -+ } else { -+ if (cpudata->epp_powersave < 0) -+ goto skip_epp; -+ /* Get BIOS pre-defined epp value */ -+ epp = amd_pstate_get_epp(cpudata, value); -+ if (epp) -+ goto skip_epp; -+ epp = cpudata->epp_powersave; -+ } -+ /* Set initial EPP value */ -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ value &= ~GENMASK_ULL(31, 24); -+ value |= (u64)epp << 24; ++ /* check if special OpenVPN message */ ++ if (ovpn_is_keepalive(skb)) { ++ netdev_dbg(peer->ovpn->dev, "%s: ping received from peer with id %u\n", ++ __func__, peer->id); ++ /* not an error */ ++ consume_skb(skb); ++ /* inform the caller that NAPI should not be scheduled ++ * for this packet ++ */ ++ return -1; ++ } ++ ++ ret = -EPROTONOSUPPORT; ++ goto drop; + } ++ skb->protocol = proto; + -+skip_epp: -+ WRITE_ONCE(cpudata->cppc_req_cached, value); -+ ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ if (!ret) { -+ cpudata->epp_cached = epp; ++ /* perform Reverse Path Filtering (RPF) */ ++ allowed_peer = ovpn_peer_lookup_vpn_addr(peer->ovpn, skb, true); ++ if (unlikely(allowed_peer != peer)) { ++ ret = -EPERM; ++ goto drop; + } -+} + -+static void amd_pstate_set_max_limits(struct amd_cpudata *cpudata) -+{ -+ u64 hwp_cap = READ_ONCE(cpudata->cppc_cap1_cached); -+ u64 hwp_req = READ_ONCE(cpudata->cppc_req_cached); -+ u32 max_limit = (hwp_cap >> 24) & 0xff; ++ ret = ptr_ring_produce_bh(&peer->netif_rx_ring, skb); ++drop: ++ if (likely(allowed_peer)) ++ ovpn_peer_put(allowed_peer); + -+ hwp_req &= ~AMD_CPPC_MIN_PERF(~0L); -+ hwp_req |= AMD_CPPC_MIN_PERF(max_limit); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, hwp_req); ++ if (unlikely(ret < 0)) ++ kfree_skb(skb); ++ ++ return ret; +} + -+static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) ++/* pick packet from RX queue, decrypt and forward it to the tun device */ ++void ovpn_decrypt_work(struct work_struct *work) +{ -+ struct amd_cpudata *cpudata; ++ struct ovpn_peer *peer; ++ struct sk_buff *skb; + -+ if (!policy->cpuinfo.max_freq) -+ return -ENODEV; ++ peer = container_of(work, struct ovpn_peer, decrypt_work); ++ while ((skb = ptr_ring_consume_bh(&peer->rx_ring))) { ++ if (likely(ovpn_decrypt_one(peer, skb) == 0)) { ++ /* if a packet has been enqueued for NAPI, signal ++ * availability to the networking stack ++ */ ++ local_bh_disable(); ++ napi_schedule(&peer->napi); ++ local_bh_enable(); ++ } + -+ pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", -+ policy->cpuinfo.max_freq, policy->max); ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); ++ } ++ ovpn_peer_put(peer); ++} + -+ cpudata = all_cpu_data[policy->cpu]; -+ cpudata->policy = policy->policy; ++static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct ovpn_crypto_key_slot *ks; ++ bool success = false; ++ int ret; + -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ mutex_lock(&amd_pstate_limits_lock); ++ /* get primary key to be used for encrypting data */ ++ ks = ovpn_crypto_key_slot_primary(&peer->crypto); ++ if (unlikely(!ks)) { ++ net_info_ratelimited("%s: error while retrieving primary key slot\n", __func__); ++ return false; ++ } + -+ if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { -+ amd_pstate_clear_update_util_hook(policy->cpu); -+ amd_pstate_set_max_limits(cpudata); -+ } else { -+ amd_pstate_set_update_util_hook(policy->cpu); -+ } ++ if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && ++ skb_checksum_help(skb))) { ++ net_err_ratelimited("%s: cannot compute checksum for outgoing packet\n", __func__); ++ goto err; ++ } + -+ if (boot_cpu_has(X86_FEATURE_CPPC)) -+ amd_pstate_epp_init(policy->cpu); ++ ovpn_peer_stats_increment_tx(&peer->stats, skb->len); + -+ mutex_unlock(&amd_pstate_limits_lock); ++ /* encrypt */ ++ ret = ovpn_aead_encrypt(ks, skb, peer->id); ++ if (unlikely(ret < 0)) { ++ /* if we ran out of IVs we must kill the key as it can't be used anymore */ ++ if (ret == -ERANGE) { ++ netdev_warn(peer->ovpn->dev, ++ "%s: killing primary key as we ran out of IVs\n", __func__); ++ ovpn_crypto_kill_primary(&peer->crypto); ++ goto err; ++ } ++ net_err_ratelimited("%s: error during encryption for peer %u, key-id %u: %d\n", ++ __func__, peer->id, ks->key_id, ret); ++ goto err; + } + -+ return 0; ++ success = true; ++err: ++ ovpn_crypto_key_slot_put(ks); ++ return success; +} + -+static void amd_pstate_epp_reenable(struct amd_cpudata * cpudata) ++/* Process packets in TX queue in a transport-specific way. ++ * ++ * UDP transport - encrypt and send across the tunnel. ++ * TCP transport - encrypt and put into TCP TX queue. ++ */ ++void ovpn_encrypt_work(struct work_struct *work) +{ -+ struct cppc_perf_ctrls perf_ctrls; -+ u64 value, max_perf; -+ int ret; ++ struct sk_buff *skb, *curr, *next; ++ struct ovpn_peer *peer; + -+ ret = amd_pstate_enable(true); -+ if (ret) -+ pr_err("failed to enable amd pstate during resume, return %d\n", ret); ++ peer = container_of(work, struct ovpn_peer, encrypt_work); ++ while ((skb = ptr_ring_consume_bh(&peer->tx_ring))) { ++ /* this might be a GSO-segmented skb list: process each skb ++ * independently ++ */ ++ skb_list_walk_safe(skb, curr, next) { ++ /* if one segment fails encryption, we drop the entire ++ * packet, because it does not really make sense to send ++ * only part of it at this point ++ */ ++ if (unlikely(!ovpn_encrypt_one(peer, curr))) { ++ kfree_skb_list(skb); ++ skb = NULL; ++ break; ++ } ++ } + -+ value = READ_ONCE(cpudata->cppc_req_cached); -+ max_perf = READ_ONCE(cpudata->highest_perf); ++ /* successful encryption */ ++ if (skb) { ++ skb_list_walk_safe(skb, curr, next) { ++ skb_mark_not_on_list(curr); + -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ } else { -+ perf_ctrls.max_perf = max_perf; -+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); -+ cppc_set_perf(cpudata->cpu, &perf_ctrls); ++ switch (peer->sock->sock->sk->sk_protocol) { ++ case IPPROTO_UDP: ++ ovpn_udp_send_skb(peer->ovpn, peer, curr); ++ break; ++ case IPPROTO_TCP: ++ ovpn_tcp_send_skb(peer, curr); ++ break; ++ default: ++ /* no transport configured yet */ ++ consume_skb(skb); ++ break; ++ } ++ } ++ ++ /* note event of authenticated packet xmit for keepalive */ ++ ovpn_peer_keepalive_xmit_reset(peer); ++ } ++ ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); + } ++ ovpn_peer_put(peer); +} + -+static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) ++/* Put skb into TX queue and schedule a consumer */ ++static void ovpn_queue_skb(struct ovpn_struct *ovpn, struct sk_buff *skb, struct ovpn_peer *peer) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ int ret; + -+ pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); ++ if (likely(!peer)) ++ peer = ovpn_peer_lookup_vpn_addr(ovpn, skb, false); ++ if (unlikely(!peer)) { ++ net_dbg_ratelimited("%s: no peer to send data to\n", ovpn->dev->name); ++ goto drop; ++ } + -+ if (epp_enabled) { -+ amd_pstate_epp_reenable(cpudata); -+ cpudata->suspended = false; ++ ret = ptr_ring_produce_bh(&peer->tx_ring, skb); ++ if (unlikely(ret < 0)) { ++ net_err_ratelimited("%s: cannot queue packet to TX ring\n", __func__); ++ goto drop; + } + -+ return 0; ++ if (!queue_work(ovpn->crypto_wq, &peer->encrypt_work)) ++ ovpn_peer_put(peer); ++ ++ return; ++drop: ++ if (peer) ++ ovpn_peer_put(peer); ++ kfree_skb_list(skb); +} + -+static void amd_pstate_epp_offline(struct cpufreq_policy *policy) ++/* Net device start xmit ++ */ ++netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ struct cppc_perf_ctrls perf_ctrls; -+ int min_perf; -+ u64 value; ++ struct ovpn_struct *ovpn = netdev_priv(dev); ++ struct sk_buff *segments, *tmp, *curr, *next; ++ struct sk_buff_head skb_list; ++ __be16 proto; ++ int ret; + -+ min_perf = READ_ONCE(cpudata->lowest_perf); -+ value = READ_ONCE(cpudata->cppc_req_cached); ++ /* reset netfilter state */ ++ nf_reset_ct(skb); + -+ mutex_lock(&amd_pstate_limits_lock); -+ if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; ++ /* verify IP header size in network packet */ ++ proto = ovpn_ip_check_protocol(skb); ++ if (unlikely(!proto || skb->protocol != proto)) { ++ net_dbg_ratelimited("%s: dropping malformed payload packet\n", ++ dev->name); ++ goto drop; ++ } + -+ /* Set max perf same as min perf */ -+ value &= ~AMD_CPPC_MAX_PERF(~0L); -+ value |= AMD_CPPC_MAX_PERF(min_perf); -+ value &= ~AMD_CPPC_MIN_PERF(~0L); -+ value |= AMD_CPPC_MIN_PERF(min_perf); -+ wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); -+ } else { -+ perf_ctrls.desired_perf = 0; -+ perf_ctrls.max_perf = min_perf; -+ perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(AMD_CPPC_EPP_POWERSAVE); -+ cppc_set_perf(cpudata->cpu, &perf_ctrls); ++ if (skb_is_gso(skb)) { ++ segments = skb_gso_segment(skb, 0); ++ if (IS_ERR(segments)) { ++ ret = PTR_ERR(segments); ++ net_dbg_ratelimited("%s: cannot segment packet: %d\n", dev->name, ret); ++ goto drop; ++ } ++ ++ consume_skb(skb); ++ skb = segments; + } -+ mutex_unlock(&amd_pstate_limits_lock); -+} + -+static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) -+{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ /* from this moment on, "skb" might be a list */ + -+ pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); ++ __skb_queue_head_init(&skb_list); ++ skb_list_walk_safe(skb, curr, next) { ++ skb_mark_not_on_list(curr); + -+ if (cpudata->suspended) -+ return 0; ++ tmp = skb_share_check(curr, GFP_ATOMIC); ++ if (unlikely(!tmp)) { ++ kfree_skb_list(next); ++ net_dbg_ratelimited("%s: skb_share_check failed\n", dev->name); ++ goto drop_list; ++ } + -+ if (epp_enabled) -+ amd_pstate_epp_offline(policy); ++ __skb_queue_tail(&skb_list, tmp); ++ } ++ skb_list.prev->next = NULL; + -+ return 0; -+} ++ ovpn_queue_skb(ovpn, skb_list.next, NULL); + -+static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) -+{ -+ amd_pstate_clear_update_util_hook(policy->cpu); ++ return NETDEV_TX_OK; + -+ return amd_pstate_cpu_offline(policy); ++drop_list: ++ skb_queue_walk_safe(&skb_list, curr, next) ++ kfree_skb(curr); ++drop: ++ skb_tx_error(skb); ++ kfree_skb_list(skb); ++ return NET_XMIT_DROP; +} + -+static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) ++/* Encrypt and transmit a special message to peer, such as keepalive ++ * or explicit-exit-notify. Called from softirq context. ++ * Assumes that caller holds a reference to peer. ++ */ ++static void ovpn_xmit_special(struct ovpn_peer *peer, const void *data, ++ const unsigned int len) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; -+ int ret; ++ struct ovpn_struct *ovpn; ++ struct sk_buff *skb; + -+ pr_debug("AMD CPU Core %d suspending\n", cpudata->cpu); ++ ovpn = peer->ovpn; ++ if (unlikely(!ovpn)) ++ return; + -+ cpudata->suspended = true; ++ skb = alloc_skb(256 + len, GFP_ATOMIC); ++ if (unlikely(!skb)) ++ return; + -+ /* disable CPPC in lowlevel firmware */ -+ ret = amd_pstate_enable(false); -+ if (ret) -+ pr_err("failed to disable amd pstate during suspend, return %d\n", ret); ++ skb_reserve(skb, 128); ++ skb->priority = TC_PRIO_BESTEFFORT; ++ memcpy(__skb_put(skb, len), data, len); + -+ return 0; ++ /* increase reference counter when passing peer to sending queue */ ++ if (!ovpn_peer_hold(peer)) { ++ netdev_dbg(ovpn->dev, "%s: cannot hold peer reference for sending special packet\n", ++ __func__); ++ kfree_skb(skb); ++ return; ++ } ++ ++ ovpn_queue_skb(ovpn, skb, peer); +} + -+static int amd_pstate_epp_resume(struct cpufreq_policy *policy) ++void ovpn_keepalive_xmit(struct ovpn_peer *peer) +{ -+ struct amd_cpudata *cpudata = all_cpu_data[policy->cpu]; ++ ovpn_xmit_special(peer, ovpn_keepalive_message, ++ sizeof(ovpn_keepalive_message)); ++} + -+ pr_debug("AMD CPU Core %d resuming\n", cpudata->cpu); ++/* Transmit explicit exit notification. ++ * Called from process context. ++ */ ++void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer) ++{ ++ ovpn_xmit_special(peer, ovpn_explicit_exit_notify_message, ++ sizeof(ovpn_explicit_exit_notify_message)); ++} + -+ if (cpudata->suspended && epp_enabled) { -+ mutex_lock(&amd_pstate_limits_lock); ++/* Copy buffer into skb and send it across the tunnel. ++ * ++ * For UDP transport: just sent the skb to peer ++ * For TCP transport: put skb into TX queue ++ */ ++int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len) ++{ ++ u16 skb_len = SKB_HEADER_LEN + len; ++ struct ovpn_peer *peer; ++ struct sk_buff *skb; ++ bool tcp = false; ++ int ret = 0; + -+ /* enable amd pstate from suspend state*/ -+ amd_pstate_epp_reenable(cpudata); ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (unlikely(!peer)) { ++ netdev_dbg(ovpn->dev, "no peer to send data to\n"); ++ return -EHOSTUNREACH; ++ } + -+ mutex_unlock(&amd_pstate_limits_lock); ++ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) { ++ skb_len += sizeof(u16); ++ tcp = true; + } + -+ cpudata->suspended = false; ++ skb = alloc_skb(skb_len, GFP_ATOMIC); ++ if (unlikely(!skb)) { ++ ret = -ENOMEM; ++ goto out; ++ } + -+ return 0; -+} ++ skb_reserve(skb, SKB_HEADER_LEN); ++ skb_put_data(skb, data, len); + -+static void amd_pstate_verify_cpu_policy(struct amd_cpudata *cpudata, -+ struct cpufreq_policy_data *policy) -+{ -+ update_boost_state(); -+ cpufreq_verify_within_cpu_limits(policy); ++ /* prepend TCP packet with size, as required by OpenVPN protocol */ ++ if (tcp) { ++ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); ++ ovpn_queue_tcp_skb(peer, skb); ++ } else { ++ ovpn_udp_send_skb(ovpn, peer, skb); ++ } ++out: ++ ovpn_peer_put(peer); ++ return ret; +} +diff --git a/drivers/net/ovpn-dco/ovpn.h b/drivers/net/ovpn-dco/ovpn.h +new file mode 100644 +index 000000000000..9364fd5dd309 +--- /dev/null ++++ b/drivers/net/ovpn-dco/ovpn.h +@@ -0,0 +1,43 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) -+{ -+ amd_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy); -+ pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min ); -+ return 0; -+} ++#ifndef _NET_OVPN_DCO_OVPN_H_ ++#define _NET_OVPN_DCO_OVPN_H_ + - static struct cpufreq_driver amd_pstate_driver = { - .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, - .verify = amd_pstate_verify, -@@ -662,18 +1535,34 @@ static struct cpufreq_driver amd_pstate_driver = { - .resume = amd_pstate_cpu_resume, - .set_boost = amd_pstate_set_boost, - .name = "amd-pstate", -- .attr = amd_pstate_attr, -+ .attr = amd_pstate_attr, -+}; ++#include "main.h" ++#include "peer.h" ++#include "sock.h" ++#include "ovpnstruct.h" + -+static struct cpufreq_driver amd_pstate_epp_driver = { -+ .flags = CPUFREQ_CONST_LOOPS, -+ .verify = amd_pstate_epp_verify_policy, -+ .setpolicy = amd_pstate_epp_set_policy, -+ .init = amd_pstate_epp_cpu_init, -+ .exit = amd_pstate_epp_cpu_exit, -+ .update_limits = amd_pstate_epp_update_limits, -+ .offline = amd_pstate_epp_cpu_offline, -+ .online = amd_pstate_epp_cpu_online, -+ .suspend = amd_pstate_epp_suspend, -+ .resume = amd_pstate_epp_resume, -+ .name = "amd_pstate_epp", -+ .attr = amd_pstate_epp_attr, - }; - - static int __init amd_pstate_init(void) - { -+ static struct amd_cpudata **cpudata; - int ret; - - if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) - return -ENODEV; - - if (!acpi_cpc_valid()) { -- pr_debug("the _CPC object is not present in SBIOS\n"); -+ pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); - return -ENODEV; - } - -@@ -681,10 +1570,25 @@ static int __init amd_pstate_init(void) - if (cpufreq_get_current_driver()) - return -EEXIST; - -+ cpudata = vzalloc(array_size(sizeof(void *), num_possible_cpus())); -+ if (!cpudata) -+ return -ENOMEM; -+ WRITE_ONCE(all_cpu_data, cpudata); ++#include ++#include ++#include + -+ if (epp_enabled) { -+ pr_info("AMD CPPC loading with amd_pstate_epp driver instance.\n"); -+ default_pstate_driver = &amd_pstate_epp_driver; -+ } else { -+ pr_info("AMD CPPC loading with amd_pstate driver instance.\n"); -+ default_pstate_driver = &amd_pstate_driver; -+ } ++struct ovpn_struct; ++struct net_device; + - /* capability check */ - if (boot_cpu_has(X86_FEATURE_CPPC)) { -+ if (!epp_enabled) { -+ default_pstate_driver->adjust_perf = amd_pstate_adjust_perf; -+ } - pr_debug("AMD CPPC MSR based functionality is supported\n"); -- amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; - } else if (shared_mem) { - static_call_update(amd_pstate_enable, cppc_enable); - static_call_update(amd_pstate_init_perf, cppc_init_perf); -@@ -701,19 +1605,56 @@ static int __init amd_pstate_init(void) - return ret; - } - -- ret = cpufreq_register_driver(&amd_pstate_driver); -+ ret = cpufreq_register_driver(default_pstate_driver); - if (ret) -- pr_err("failed to register amd_pstate_driver with return %d\n", -+ pr_err("failed to register amd pstate driver with return %d\n", -+ ret); ++int ovpn_struct_init(struct net_device *dev); ++ ++u16 ovpn_select_queue(struct net_device *dev, struct sk_buff *skb, ++ struct net_device *sb_dev); ++ ++void ovpn_keepalive_xmit(struct ovpn_peer *peer); ++void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer); ++ ++netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev); ++ ++int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb); ++ ++void ovpn_encrypt_work(struct work_struct *work); ++void ovpn_decrypt_work(struct work_struct *work); ++int ovpn_napi_poll(struct napi_struct *napi, int budget); + -+ amd_pstate_kobj = kobject_create_and_add("amd-pstate", &cpu_subsys.dev_root->kobj); -+ if (!amd_pstate_kobj) { -+ pr_err("amd-pstate: Global sysfs registration failed.\n"); -+ } ++int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len); + -+ ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); -+ if (ret) { -+ pr_err("amd-pstate: Sysfs attribute export failed with error %d.\n", - ret); -+ } - - return ret; - } - -+static inline void amd_pstate_kobj_cleanup(struct kobject *kobj) -+{ -+ kobject_del(kobj); -+ kobject_put(kobj); -+} ++#endif /* _NET_OVPN_DCO_OVPN_H_ */ +diff --git a/drivers/net/ovpn-dco/ovpnstruct.h b/drivers/net/ovpn-dco/ovpnstruct.h +new file mode 100644 +index 000000000000..f9bc559609cd +--- /dev/null ++++ b/drivers/net/ovpn-dco/ovpnstruct.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + - static void __exit amd_pstate_exit(void) - { -- cpufreq_unregister_driver(&amd_pstate_driver); -+ unsigned int cpu; ++#ifndef _NET_OVPN_DCO_OVPNSTRUCT_H_ ++#define _NET_OVPN_DCO_OVPNSTRUCT_H_ + -+ cpufreq_unregister_driver(default_pstate_driver); - - amd_pstate_enable(false); ++#include "peer.h" + -+ sysfs_remove_group(amd_pstate_kobj, &amd_pstate_global_attr_group); -+ amd_pstate_kobj_cleanup(amd_pstate_kobj); ++#include ++#include ++#include + -+ cpus_read_lock(); -+ for_each_online_cpu(cpu) { -+ if (all_cpu_data[cpu]) { -+ if (default_pstate_driver == &amd_pstate_epp_driver) -+ amd_pstate_clear_update_util_hook(cpu); ++/* Our state per ovpn interface */ ++struct ovpn_struct { ++ /* read-mostly objects in this section */ ++ struct net_device *dev; + -+ spin_lock(&amd_pstate_cpu_lock); -+ kfree(all_cpu_data[cpu]); -+ WRITE_ONCE(all_cpu_data[cpu], NULL); -+ spin_unlock(&amd_pstate_cpu_lock); -+ } -+ } -+ cpus_read_unlock(); ++ /* device operation mode (i.e. P2P, MP) */ ++ enum ovpn_mode mode; + - } - - module_init(amd_pstate_init); -diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c -index 24eaf0ec344d..9adb7612993e 100644 ---- a/drivers/cpufreq/cppc_cpufreq.c -+++ b/drivers/cpufreq/cppc_cpufreq.c -@@ -947,7 +947,7 @@ static int __init cppc_cpufreq_init(void) - { - int ret; - -- if ((acpi_disabled) || !acpi_cpc_valid()) -+ if (!acpi_cpc_valid()) - return -ENODEV; - - cppc_check_hisi_workaround(); -diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c -index 57cdb3679885..50881662b45b 100644 ---- a/drivers/cpufreq/intel_pstate.c -+++ b/drivers/cpufreq/intel_pstate.c -@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu) - * update them at any time after it has been called. - */ - sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu); -+ /* -+ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff. -+ * In this case we can't use CPPC.highest_perf to enable ITMT. -+ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide. ++ /* protect writing to the ovpn_struct object */ ++ spinlock_t lock; ++ ++ /* workqueue used to schedule crypto work that may sleep */ ++ struct workqueue_struct *crypto_wq; ++ /* workqueue used to schedule generic event that may sleep or that need ++ * to be performed out of softirq context + */ -+ if (cppc_perf.highest_perf == 0xff) -+ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached)); - - if (max_highest_perf <= min_highest_perf) { - if (cppc_perf.highest_perf > max_highest_perf) -diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c -index 3e101719689a..746138abc3fe 100644 ---- a/drivers/idle/intel_idle.c -+++ b/drivers/idle/intel_idle.c -@@ -578,7 +578,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -586,7 +586,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 33, -- .target_residency = 100, -+ .target_residency = 900, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -594,7 +594,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 133, -- .target_residency = 400, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -602,7 +602,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 166, -- .target_residency = 500, -+ .target_residency = 1500, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -610,7 +610,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 300, -- .target_residency = 900, -+ .target_residency = 2000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -618,7 +618,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 600, -- .target_residency = 1800, -+ .target_residency = 5000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -626,7 +626,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 2600, -- .target_residency = 7700, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -646,7 +646,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -654,7 +654,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 40, -- .target_residency = 100, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -662,7 +662,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 133, -- .target_residency = 400, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -670,7 +670,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 166, -- .target_residency = 500, -+ .target_residency = 2000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -678,7 +678,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 300, -- .target_residency = 900, -+ .target_residency = 4000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -686,7 +686,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 600, -- .target_residency = 1800, -+ .target_residency = 7000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -694,7 +694,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 2600, -- .target_residency = 7700, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -715,7 +715,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 120, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -723,7 +723,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, - .exit_latency = 70, -- .target_residency = 100, -+ .target_residency = 1000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -731,7 +731,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 85, -- .target_residency = 200, -+ .target_residency = 600, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -739,7 +739,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x33", - .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 124, -- .target_residency = 800, -+ .target_residency = 3000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -747,7 +747,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 200, -- .target_residency = 800, -+ .target_residency = 3200, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -755,7 +755,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 480, -- .target_residency = 5000, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -763,7 +763,7 @@ static struct cpuidle_state skl_cstates[] __initdata = { - .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, - .exit_latency = 890, -- .target_residency = 5000, -+ .target_residency = 9000, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -@@ -784,7 +784,7 @@ static struct cpuidle_state skx_cstates[] __initdata = { - .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, - .exit_latency = 10, -- .target_residency = 20, -+ .target_residency = 300, - .enter = &intel_idle, - .enter_s2idle = intel_idle_s2idle, }, - { -diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c -index 3fc0a89cc785..a7c103f9dfd3 100644 ---- a/drivers/input/serio/i8042.c -+++ b/drivers/input/serio/i8042.c -@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void) - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - i8042_ctr &= ~I8042_CTR_KBDINT; - i8042_ctr |= I8042_CTR_KBDDIS; -- pr_err("Failed to enable KBD port\n"); -+ pr_info("Failed to enable KBD port\n"); - return -EIO; - } - -@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void) - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { - i8042_ctr &= ~I8042_CTR_AUXINT; - i8042_ctr |= I8042_CTR_AUXDIS; -- pr_err("Failed to enable AUX port\n"); -+ pr_info("Failed to enable AUX port\n"); - return -EIO; - } - -@@ -732,7 +732,7 @@ static int i8042_check_mux(void) - i8042_ctr &= ~I8042_CTR_AUXINT; - - if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) { -- pr_err("Failed to disable AUX port, can't use MUX\n"); -+ pr_info("Failed to disable AUX port, can't use MUX\n"); - return -EIO; - } - -@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void) - do { - - if (i8042_command(¶m, I8042_CMD_CTL_TEST)) { -- pr_err("i8042 controller selftest timeout\n"); -+ pr_info("i8042 controller selftest timeout\n"); - return -ENODEV; - } - -@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void) - pr_info("giving up on controller selftest, continuing anyway...\n"); - return 0; - #else -- pr_err("i8042 controller selftest failed\n"); -+ pr_info("i8042 controller selftest failed\n"); - return -EIO; - #endif - } -diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c -index 159c6806c19b..a201f1c2fce5 100644 ---- a/drivers/md/dm-crypt.c -+++ b/drivers/md/dm-crypt.c -@@ -3137,6 +3137,11 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar - } - } - -+#ifdef CONFIG_CACHY -+ set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); -+ set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); -+#endif ++ struct workqueue_struct *events_wq; + - return 0; - } - -diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c -index f82ad7419508..5e8faa70aad6 100644 ---- a/drivers/net/dummy.c -+++ b/drivers/net/dummy.c -@@ -43,7 +43,7 @@ - - #define DRV_NAME "dummy" - --static int numdummies = 1; -+static int numdummies = 0; - - /* fake multicast ability */ - static void set_multicast_list(struct net_device *dev) -diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c -index 66446f1e06cf..c65b03f91ecf 100644 ---- a/drivers/nvme/host/core.c -+++ b/drivers/nvme/host/core.c -@@ -58,7 +58,7 @@ static u8 nvme_max_retries = 5; - module_param_named(max_retries, nvme_max_retries, byte, 0644); - MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); - --static unsigned long default_ps_max_latency_us = 100000; -+static unsigned long default_ps_max_latency_us = 200; - module_param(default_ps_max_latency_us, ulong, 0644); - MODULE_PARM_DESC(default_ps_max_latency_us, - "max power saving latency for new devices; use PM QOS to change per device"); -diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c -index 95bc329e74c0..73114509ddff 100644 ---- a/drivers/pci/pci.c -+++ b/drivers/pci/pci.c -@@ -62,7 +62,7 @@ struct pci_pme_device { - struct pci_dev *dev; - }; - --#define PME_TIMEOUT 1000 /* How long between PME checks */ -+#define PME_TIMEOUT 4000 /* How long between PME checks */ - - static void pci_dev_d3_sleep(struct pci_dev *dev) - { -diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c -index 21d624f9f5fb..aad4e9578d89 100644 ---- a/drivers/powercap/intel_rapl_common.c -+++ b/drivers/powercap/intel_rapl_common.c -@@ -1515,7 +1515,7 @@ static int __init rapl_init(void) - - id = x86_match_cpu(rapl_ids); - if (!id) { -- pr_err("driver does not support CPU family %d model %d\n", -+ pr_info("driver does not support CPU family %d model %d\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); - - return -ENODEV; -diff --git a/drivers/soundwire/bus.h b/drivers/soundwire/bus.h -index 7631ef5e71fb..d3ed828daac0 100644 ---- a/drivers/soundwire/bus.h -+++ b/drivers/soundwire/bus.h -@@ -5,7 +5,7 @@ - #define __SDW_BUS_H - - #define DEFAULT_BANK_SWITCH_TIMEOUT 3000 --#define DEFAULT_PROBE_TIMEOUT 2000 -+#define DEFAULT_PROBE_TIMEOUT 10000 - - u64 sdw_dmi_override_adr(struct sdw_bus *bus, u64 addr); - -diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c -index c841ab37e7c6..fe3d190ab12b 100644 ---- a/drivers/thermal/intel/intel_powerclamp.c -+++ b/drivers/thermal/intel/intel_powerclamp.c -@@ -644,6 +644,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = { - .set_cur_state = powerclamp_set_cur_state, - }; - -+static const struct x86_cpu_id amd_cpu[] = { -+ { X86_VENDOR_AMD }, -+ {}, ++ /* list of known peers */ ++ struct { ++ DECLARE_HASHTABLE(by_id, 12); ++ DECLARE_HASHTABLE(by_transp_addr, 12); ++ DECLARE_HASHTABLE(by_vpn_addr, 12); ++ /* protects write access to any of the hashtables above */ ++ spinlock_t lock; ++ } peers; ++ ++ /* for p2p mode */ ++ struct ovpn_peer __rcu *peer; ++ ++ unsigned int max_tun_queue_len; ++ ++ netdev_features_t set_features; ++ ++ void *security; ++ ++ u32 registered_nl_portid; ++ bool registered_nl_portid_set; +}; + - static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = { - X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL), - {} -@@ -653,6 +658,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); - static int __init powerclamp_probe(void) - { - -+ if (x86_match_cpu(amd_cpu)){ -+ pr_info("Intel PowerClamp does not support AMD CPUs\n"); -+ return -ENODEV; ++#endif /* _NET_OVPN_DCO_OVPNSTRUCT_H_ */ +diff --git a/drivers/net/ovpn-dco/peer.c b/drivers/net/ovpn-dco/peer.c +new file mode 100644 +index 000000000000..87d3f1b34c4d +--- /dev/null ++++ b/drivers/net/ovpn-dco/peer.c +@@ -0,0 +1,906 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ ++ ++#include "ovpn.h" ++#include "bind.h" ++#include "crypto.h" ++#include "peer.h" ++#include "netlink.h" ++#include "tcp.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++static void ovpn_peer_ping(struct timer_list *t) ++{ ++ struct ovpn_peer *peer = from_timer(peer, t, keepalive_xmit); ++ ++ netdev_dbg(peer->ovpn->dev, "%s: sending ping to peer %u\n", __func__, peer->id); ++ ovpn_keepalive_xmit(peer); ++} ++ ++static void ovpn_peer_expire(struct timer_list *t) ++{ ++ struct ovpn_peer *peer = from_timer(peer, t, keepalive_recv); ++ ++ netdev_dbg(peer->ovpn->dev, "%s: peer %u expired\n", __func__, peer->id); ++ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_EXPIRED); ++} ++ ++/* Construct a new peer */ ++static struct ovpn_peer *ovpn_peer_create(struct ovpn_struct *ovpn, u32 id) ++{ ++ struct ovpn_peer *peer; ++ int ret; ++ ++ /* alloc and init peer object */ ++ peer = kzalloc(sizeof(*peer), GFP_KERNEL); ++ if (!peer) ++ return ERR_PTR(-ENOMEM); ++ ++ peer->id = id; ++ peer->halt = false; ++ peer->ovpn = ovpn; ++ ++ peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY); ++ peer->vpn_addrs.ipv6 = in6addr_any; ++ ++ RCU_INIT_POINTER(peer->bind, NULL); ++ ovpn_crypto_state_init(&peer->crypto); ++ spin_lock_init(&peer->lock); ++ kref_init(&peer->refcount); ++ ovpn_peer_stats_init(&peer->stats); ++ ++ INIT_WORK(&peer->encrypt_work, ovpn_encrypt_work); ++ INIT_WORK(&peer->decrypt_work, ovpn_decrypt_work); ++ ++ ret = dst_cache_init(&peer->dst_cache, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot initialize dst cache\n", __func__); ++ goto err; + } + - if (!x86_match_cpu(intel_powerclamp_ids)) { - pr_err("CPU does not support MWAIT\n"); - return -ENODEV; -diff --git a/fs/proc/base.c b/fs/proc/base.c -index 93f7e3d971e4..812a42947afd 100644 ---- a/fs/proc/base.c -+++ b/fs/proc/base.c -@@ -3196,6 +3196,19 @@ static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace * - - return 0; - } -+static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, -+ struct pid *pid, struct task_struct *task) -+{ -+ struct mm_struct *mm; ++ ret = ptr_ring_init(&peer->tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot allocate TX ring\n", __func__); ++ goto err_dst_cache; ++ } + -+ mm = get_task_mm(task); -+ if (mm) { -+ seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); -+ mmput(mm); ++ ret = ptr_ring_init(&peer->rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot allocate RX ring\n", __func__); ++ goto err_tx_ring; + } + -+ return 0; ++ ret = ptr_ring_init(&peer->netif_rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(ovpn->dev, "%s: cannot allocate NETIF RX ring\n", __func__); ++ goto err_rx_ring; ++ } ++ ++ /* configure and start NAPI */ ++ netif_tx_napi_add(ovpn->dev, &peer->napi, ovpn_napi_poll, ++ NAPI_POLL_WEIGHT); ++ napi_enable(&peer->napi); ++ ++ dev_hold(ovpn->dev); ++ ++ timer_setup(&peer->keepalive_xmit, ovpn_peer_ping, 0); ++ timer_setup(&peer->keepalive_recv, ovpn_peer_expire, 0); ++ ++ return peer; ++err_rx_ring: ++ ptr_ring_cleanup(&peer->rx_ring, NULL); ++err_tx_ring: ++ ptr_ring_cleanup(&peer->tx_ring, NULL); ++err_dst_cache: ++ dst_cache_destroy(&peer->dst_cache); ++err: ++ kfree(peer); ++ return ERR_PTR(ret); +} - #endif /* CONFIG_KSM */ - - #ifdef CONFIG_STACKLEAK_METRICS -@@ -3331,6 +3344,7 @@ static const struct pid_entry tgid_base_stuff[] = { - #endif - #ifdef CONFIG_KSM - ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), -+ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), - #endif - }; - -@@ -3668,6 +3682,7 @@ static const struct pid_entry tid_base_stuff[] = { - #endif - #ifdef CONFIG_KSM - ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), -+ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), - #endif - }; - -diff --git a/fs/xattr.c b/fs/xattr.c -index a1f4998bc6be..020e0b5ea5c6 100644 ---- a/fs/xattr.c -+++ b/fs/xattr.c -@@ -122,16 +122,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode, - } - - /* -- * In the user.* namespace, only regular files and directories can have -- * extended attributes. For sticky directories, only the owner and -- * privileged users can write attributes. -+ * In the user.* namespace, only regular files, symbolic links, and -+ * directories can have extended attributes. For symbolic links and -+ * sticky directories, only the owner and privileged users can write -+ * attributes. - */ - if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { -- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) -+ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode)) - return (mask & MAY_WRITE) ? -EPERM : -ENODATA; -- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) && -- (mask & MAY_WRITE) && -- !inode_owner_or_capable(mnt_userns, inode)) -+ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX)) -+ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE) -+ && !inode_owner_or_capable(mnt_userns, inode)) - return -EPERM; - } - -diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h -index f73d357ecdf5..41d151a6d52e 100644 ---- a/include/acpi/cppc_acpi.h -+++ b/include/acpi/cppc_acpi.h -@@ -108,12 +108,14 @@ struct cppc_perf_caps { - u32 lowest_nonlinear_perf; - u32 lowest_freq; - u32 nominal_freq; -+ u32 energy_perf; - }; - - struct cppc_perf_ctrls { - u32 max_perf; - u32 min_perf; - u32 desired_perf; -+ u32 energy_perf; - }; - - struct cppc_perf_fb_ctrs { -@@ -148,6 +150,9 @@ extern bool cpc_ffh_supported(void); - extern bool cpc_supported_by_cpu(void); - extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); - extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); -+extern int cppc_set_auto_epp(int cpu, bool enable); -+extern int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps); -+extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); - #else /* !CONFIG_ACPI_CPPC_LIB */ - static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) - { -@@ -197,6 +202,18 @@ static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) - { - return -ENOTSUPP; - } -+static inline int cppc_set_auto_epp(int cpu, bool enable) ++ ++/* Reset the ovpn_sockaddr associated with a peer */ ++static int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, const struct sockaddr_storage *ss, ++ const u8 *local_ip) +{ -+ return -ENOTSUPP; ++ struct ovpn_bind *bind; ++ size_t ip_len; ++ ++ /* create new ovpn_bind object */ ++ bind = ovpn_bind_from_sockaddr(ss); ++ if (IS_ERR(bind)) ++ return PTR_ERR(bind); ++ ++ if (local_ip) { ++ if (ss->ss_family == AF_INET) { ++ ip_len = sizeof(struct in_addr); ++ } else if (ss->ss_family == AF_INET6) { ++ ip_len = sizeof(struct in6_addr); ++ } else { ++ netdev_dbg(peer->ovpn->dev, "%s: invalid family for remote endpoint\n", ++ __func__); ++ kfree(bind); ++ return -EINVAL; ++ } ++ ++ memcpy(&bind->local, local_ip, ip_len); ++ } ++ ++ /* set binding */ ++ ovpn_bind_reset(peer, bind); ++ ++ return 0; +} -+static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) ++ ++void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb) +{ -+ return -ENOTSUPP; ++ struct sockaddr_storage ss; ++ const u8 *local_ip = NULL; ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa; ++ struct ovpn_bind *bind; ++ sa_family_t family; ++ ++ rcu_read_lock(); ++ bind = rcu_dereference(peer->bind); ++ if (unlikely(!bind)) ++ goto unlock; ++ ++ if (likely(ovpn_bind_skb_src_match(bind, skb))) ++ goto unlock; ++ ++ family = skb_protocol_to_family(skb); ++ ++ if (bind->sa.in4.sin_family == family) ++ local_ip = (u8 *)&bind->local; ++ ++ switch (family) { ++ case AF_INET: ++ sa = (struct sockaddr_in *)&ss; ++ sa->sin_family = AF_INET; ++ sa->sin_addr.s_addr = ip_hdr(skb)->saddr; ++ sa->sin_port = udp_hdr(skb)->source; ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)&ss; ++ sa6->sin6_family = AF_INET6; ++ sa6->sin6_addr = ipv6_hdr(skb)->saddr; ++ sa6->sin6_port = udp_hdr(skb)->source; ++ sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); ++ break; ++ default: ++ goto unlock; ++ } ++ ++ netdev_dbg(peer->ovpn->dev, "%s: peer %d floated to %pIScp", __func__, peer->id, &ss); ++ ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip); ++unlock: ++ rcu_read_unlock(); +} -+static inline int cppc_get_epp_caps(int cpunum, struct cppc_perf_caps *perf_caps) ++ ++static void ovpn_peer_timer_delete_all(struct ovpn_peer *peer) +{ -+ return -ENOTSUPP; ++ del_timer_sync(&peer->keepalive_xmit); ++ del_timer_sync(&peer->keepalive_recv); +} - #endif /* !CONFIG_ACPI_CPPC_LIB */ - - #endif /* _CPPC_ACPI_H*/ -diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h -index e3e8c8662b49..9209856b6644 100644 ---- a/include/linux/ipc_namespace.h -+++ b/include/linux/ipc_namespace.h -@@ -36,8 +36,6 @@ struct ipc_namespace { - unsigned int msg_ctlmax; - unsigned int msg_ctlmnb; - unsigned int msg_ctlmni; -- atomic_t msg_bytes; -- atomic_t msg_hdrs; - - size_t shm_ctlmax; - size_t shm_ctlall; -@@ -77,6 +75,9 @@ struct ipc_namespace { - struct llist_node mnt_llist; - - struct ns_common ns; -+ int padding[16]; -+ atomic_t msg_bytes; -+ atomic_t msg_hdrs; - } __randomize_layout; - - extern struct ipc_namespace init_ipc_ns; -diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h -index 0b7242370b56..16b8fc483b3d 100644 ---- a/include/linux/jbd2.h -+++ b/include/linux/jbd2.h -@@ -45,7 +45,7 @@ - /* - * The default maximum commit age, in seconds. - */ --#define JBD2_DEFAULT_MAX_COMMIT_AGE 5 -+#define JBD2_DEFAULT_MAX_COMMIT_AGE 30 - - #ifdef CONFIG_JBD2_DEBUG - /* -diff --git a/include/linux/ksm.h b/include/linux/ksm.h -index 0b4f17418f64..e24cf9144b24 100644 ---- a/include/linux/ksm.h -+++ b/include/linux/ksm.h -@@ -19,6 +19,10 @@ struct stable_node; - struct mem_cgroup; - - #ifdef CONFIG_KSM -+int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, -+ unsigned long *vm_flags); -+int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, -+ unsigned long end, unsigned long *vm_flags); - int ksm_madvise(struct vm_area_struct *vma, unsigned long start, - unsigned long end, int advice, unsigned long *vm_flags); - int __ksm_enter(struct mm_struct *mm); -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index cf97f3884fda..8ac6d7e53b17 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -671,6 +671,11 @@ struct mm_struct { - * merging. - */ - unsigned long ksm_merging_pages; -+ /* -+ * Represent how many pages are checked for ksm merging -+ * including merged and not merged. -+ */ -+ unsigned long ksm_rmap_items; - #endif - } __randomize_layout; - -diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h -index 679591301994..685193470d64 100644 ---- a/include/linux/page_counter.h -+++ b/include/linux/page_counter.h -@@ -8,6 +8,7 @@ - - struct page_counter { - atomic_long_t usage; -+ unsigned long padding[7]; - unsigned long min; - unsigned long low; - unsigned long high; -diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index 0178b2040ea3..39f05e00dac4 100644 ---- a/include/linux/pagemap.h -+++ b/include/linux/pagemap.h -@@ -1183,7 +1183,7 @@ struct readahead_control { - ._index = i, \ - } - --#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) -+#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) - - void page_cache_ra_unbounded(struct readahead_control *, - unsigned long nr_to_read, unsigned long lookahead_count); -diff --git a/include/linux/sched.h b/include/linux/sched.h -index e7b2f8a5c711..367b30ed77cb 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -1052,6 +1052,7 @@ struct task_struct { - /* Cached requested key. */ - struct key *cached_requested_key; - #endif -+ int fsync_count; - - /* - * executable name, excluding path. -diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h -index a34b0f9a9972..82afad91de9f 100644 ---- a/include/linux/syscalls.h -+++ b/include/linux/syscalls.h -@@ -917,6 +917,7 @@ asmlinkage long sys_madvise(unsigned long start, size_t len, int behavior); - asmlinkage long sys_process_madvise(int pidfd, const struct iovec __user *vec, - size_t vlen, int behavior, unsigned int flags); - asmlinkage long sys_process_mrelease(int pidfd, unsigned int flags); -+asmlinkage long sys_pmadv_ksm(int pidfd, int behavior, unsigned int flags); - asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size, - unsigned long prot, unsigned long pgoff, - unsigned long flags); -diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h -index 33a4240e6a6f..82213f9c4c17 100644 ---- a/include/linux/user_namespace.h -+++ b/include/linux/user_namespace.h -@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns, - - #ifdef CONFIG_USER_NS - -+extern int unprivileged_userns_clone; + - static inline struct user_namespace *get_user_ns(struct user_namespace *ns) - { - if (ns) -@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); - struct ns_common *ns_get_owner(struct ns_common *ns); - #else - -+#define unprivileged_userns_clone 0 ++static void ovpn_peer_free(struct ovpn_peer *peer) ++{ ++ ovpn_bind_reset(peer, NULL); ++ ovpn_peer_timer_delete_all(peer); + - static inline struct user_namespace *get_user_ns(struct user_namespace *ns) - { - return &init_user_ns; -diff --git a/include/linux/wait.h b/include/linux/wait.h -index 58cfbf81447c..d5664b5ae3e1 100644 ---- a/include/linux/wait.h -+++ b/include/linux/wait.h -@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) - - extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); -+extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - -@@ -1164,6 +1165,7 @@ do { \ - */ - void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); -+void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); - void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); - long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); -diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h -index 45fa180cc56a..40f7e6d04af0 100644 ---- a/include/uapi/asm-generic/unistd.h -+++ b/include/uapi/asm-generic/unistd.h -@@ -886,8 +886,11 @@ __SYSCALL(__NR_futex_waitv, sys_futex_waitv) - #define __NR_set_mempolicy_home_node 450 - __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) - -+#define __NR_pmadv_ksm 451 -+__SYSCALL(__NR_pmadv_ksm, sys_pmadv_ksm) ++ WARN_ON(!__ptr_ring_empty(&peer->tx_ring)); ++ ptr_ring_cleanup(&peer->tx_ring, NULL); ++ WARN_ON(!__ptr_ring_empty(&peer->rx_ring)); ++ ptr_ring_cleanup(&peer->rx_ring, NULL); ++ WARN_ON(!__ptr_ring_empty(&peer->netif_rx_ring)); ++ ptr_ring_cleanup(&peer->netif_rx_ring, NULL); + - #undef __NR_syscalls --#define __NR_syscalls 451 -+#define __NR_syscalls 452 - - /* - * 32 bit systems traditionally used different -diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h -index d174914a837d..bf8e2af101a3 100644 ---- a/include/uapi/linux/if_bonding.h -+++ b/include/uapi/linux/if_bonding.h -@@ -82,7 +82,7 @@ - #define BOND_STATE_ACTIVE 0 /* link is active */ - #define BOND_STATE_BACKUP 1 /* link is backup */ - --#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */ -+#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */ - - #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */ - -diff --git a/init/Kconfig b/init/Kconfig -index 532362fcfe31..442a945ca6ae 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -112,6 +112,10 @@ config THREAD_INFO_IN_TASK - - menu "General setup" - -+config CACHY -+ bool "Some kernel tweaks by CachyOS" -+ default y ++ dst_cache_destroy(&peer->dst_cache); + - config BROKEN - bool - -@@ -1241,6 +1245,22 @@ config USER_NS - - If unsure, say N. - -+config USER_NS_UNPRIVILEGED -+ bool "Allow unprivileged users to create namespaces" -+ default y -+ depends on USER_NS -+ help -+ When disabled, unprivileged users will not be able to create -+ new namespaces. Allowing users to create their own namespaces -+ has been part of several recent local privilege escalation -+ exploits, so if you need user namespaces but are -+ paranoid^Wsecurity-conscious you want to disable this. ++ dev_put(peer->ovpn->dev); + -+ This setting can be overridden at runtime via the -+ kernel.unprivileged_userns_clone sysctl. ++ kfree(peer); ++} + -+ If unsure, say Y. ++static void ovpn_peer_release_rcu(struct rcu_head *head) ++{ ++ struct ovpn_peer *peer = container_of(head, struct ovpn_peer, rcu); + - config PID_NS - bool "PID Namespaces" - default y -@@ -1407,6 +1427,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE - with the "-O2" compiler flag for best performance and most - helpful compile-time warnings. - -+config CC_OPTIMIZE_FOR_PERFORMANCE_O3 -+ bool "Optimize more for performance (-O3)" -+ help -+ Choosing this option will pass "-O3" to your compiler to optimize -+ the kernel yet more for performance. ++ ovpn_crypto_state_release(&peer->crypto); ++ ovpn_peer_free(peer); ++} + - config CC_OPTIMIZE_FOR_SIZE - bool "Optimize for size (-Os)" - help -diff --git a/init/do_mounts.c b/init/do_mounts.c -index 7058e14ad5f7..ce4e841ea705 100644 ---- a/init/do_mounts.c -+++ b/init/do_mounts.c -@@ -283,8 +283,18 @@ dev_t name_to_dev_t(const char *name) - if (strcmp(name, "/dev/ram") == 0) - return Root_RAM0; - #ifdef CONFIG_BLOCK -- if (strncmp(name, "PARTUUID=", 9) == 0) -- return devt_from_partuuid(name + 9); -+ if (strncmp(name, "PARTUUID=", 9) == 0) { -+ dev_t res; -+ int needtowait = 40<<1; -+ res = devt_from_partuuid(name + 9); -+ while (!res && needtowait) { -+ /* waiting 0.5 sec */ -+ msleep(500); -+ res = devt_from_partuuid(name + 9); -+ needtowait--; -+ } -+ return res; -+ } - if (strncmp(name, "PARTLABEL=", 10) == 0) - return devt_from_partlabel(name + 10); - if (strncmp(name, "/dev/", 5) == 0) -@@ -612,7 +622,9 @@ void __init prepare_namespace(void) - * For example, it is not atypical to wait 5 seconds here - * for the touchpad of a laptop to initialize. - */ -+ async_synchronize_full(); - wait_for_device_probe(); -+ async_synchronize_full(); - - md_run_setup(); - -diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz -index 38ef6d06888e..0f78364efd4f 100644 ---- a/kernel/Kconfig.hz -+++ b/kernel/Kconfig.hz -@@ -40,6 +40,27 @@ choice - on SMP and NUMA systems and exactly dividing by both PAL and - NTSC frame rates for video and multimedia work. - -+ config HZ_500 -+ bool "500 HZ" -+ help -+ 500 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. ++void ovpn_peer_release(struct ovpn_peer *peer) ++{ ++ napi_disable(&peer->napi); ++ netif_napi_del(&peer->napi); + -+ config HZ_600 -+ bool "600 HZ" -+ help -+ 600 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. ++ if (peer->sock) ++ ovpn_socket_put(peer->sock); + -+ config HZ_750 -+ bool "750 HZ" -+ help -+ 750 Hz is a balanced timer frequency. Provides fast interactivity -+ on desktops with good smoothness without increasing CPU power -+ consumption and sacrificing the battery life on laptops. ++ call_rcu(&peer->rcu, ovpn_peer_release_rcu); ++} + - config HZ_1000 - bool "1000 HZ" - help -@@ -53,6 +74,9 @@ config HZ - default 100 if HZ_100 - default 250 if HZ_250 - default 300 if HZ_300 -+ default 500 if HZ_500 -+ default 600 if HZ_600 -+ default 750 if HZ_750 - default 1000 if HZ_1000 - - config SCHED_HRTICK -diff --git a/kernel/fork.c b/kernel/fork.c -index 2b6bd511c6ed..704fe6bc9cb4 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -99,6 +99,10 @@ - #include - #include - -+#ifdef CONFIG_USER_NS -+#include -+#endif ++static void ovpn_peer_delete_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, ++ delete_work); ++ ovpn_peer_release(peer); ++ ovpn_netlink_notify_del_peer(peer); ++} + - #include - #include - #include -@@ -2009,6 +2013,10 @@ static __latent_entropy struct task_struct *copy_process( - if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) - return ERR_PTR(-EINVAL); - -+ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) -+ if (!capable(CAP_SYS_ADMIN)) -+ return ERR_PTR(-EPERM); ++/* Use with kref_put calls, when releasing refcount ++ * on ovpn_peer objects. This method should only ++ * be called from process context with config_mutex held. ++ */ ++void ovpn_peer_release_kref(struct kref *kref) ++{ ++ struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount); + - /* - * Thread groups must share signals as well, and detached threads - * can only be started up within the thread group. -@@ -3159,6 +3167,12 @@ int ksys_unshare(unsigned long unshare_flags) - if (unshare_flags & CLONE_NEWNS) - unshare_flags |= CLONE_FS; - -+ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { -+ err = -EPERM; -+ if (!capable(CAP_SYS_ADMIN)) -+ goto bad_unshare_out; -+ } ++ INIT_WORK(&peer->delete_work, ovpn_peer_delete_work); ++ queue_work(peer->ovpn->events_wq, &peer->delete_work); ++} + - err = check_unshare_flags(unshare_flags); - if (err) - goto bad_unshare_out; -diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c -index 65f0262f635e..8a8f77930818 100644 ---- a/kernel/locking/rwsem.c -+++ b/kernel/locking/rwsem.c -@@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) - struct task_struct *new, *owner; - unsigned long flags, new_flags; - enum owner_state state; -+ int i = 0; - - lockdep_assert_preemption_disabled(); - -@@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) - break; - } - -- cpu_relax(); -+ if (i++ > 1000) -+ cpu_relax(); - } - - return state; -diff --git a/kernel/module/internal.h b/kernel/module/internal.h -index 680d980a4fb2..8a3abfff9fe9 100644 ---- a/kernel/module/internal.h -+++ b/kernel/module/internal.h -@@ -53,6 +53,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; - extern const s32 __start___kcrctab[]; - extern const s32 __start___kcrctab_gpl[]; - -+extern struct boot_params boot_params; ++struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, ++ struct socket *sock, u32 id, uint8_t *local_ip) ++{ ++ struct ovpn_peer *peer; ++ int ret; + - struct load_info { - const char *name; - /* pointer to module in temporary copy, freed at end of load_module() */ -diff --git a/kernel/module/main.c b/kernel/module/main.c -index a4e4d84b6f4e..0e698f5bee86 100644 ---- a/kernel/module/main.c -+++ b/kernel/module/main.c -@@ -53,6 +53,7 @@ - #include - #include - #include -+#include - #include - #include "internal.h" - -diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c -index cf5b9f1e6ec4..80260fa4dac5 100644 ---- a/kernel/module/procfs.c -+++ b/kernel/module/procfs.c -@@ -141,6 +141,19 @@ static const struct proc_ops modules_proc_ops = { - static int __init proc_modules_init(void) - { - proc_create("modules", 0, NULL, &modules_proc_ops); ++ /* create new peer */ ++ peer = ovpn_peer_create(ovpn, id); ++ if (IS_ERR(peer)) ++ return peer; + -+#ifdef CONFIG_MODULE_SIG_FORCE -+ switch (boot_params.secure_boot) { -+ case efi_secureboot_mode_unset: -+ case efi_secureboot_mode_unknown: -+ case efi_secureboot_mode_disabled: -+ /* -+ * sig_unenforce is only applied if SecureBoot is not -+ * enabled. -+ */ -+ sig_enforce = !sig_unenforce; ++ if (sock->sk->sk_protocol == IPPROTO_UDP) { ++ /* a UDP peer must have a remote endpoint */ ++ if (!sa) { ++ ovpn_peer_release(peer); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ /* set peer sockaddr */ ++ ret = ovpn_peer_reset_sockaddr(peer, sa, local_ip); ++ if (ret < 0) { ++ ovpn_peer_release(peer); ++ return ERR_PTR(ret); ++ } + } -+#endif - return 0; - } - module_init(proc_modules_init); -diff --git a/kernel/module/signing.c b/kernel/module/signing.c -index a2ff4242e623..876e93758e91 100644 ---- a/kernel/module/signing.c -+++ b/kernel/module/signing.c -@@ -21,6 +21,10 @@ - - static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE); - module_param(sig_enforce, bool_enable_only, 0644); -+/* Allow disabling module signature requirement by adding boot param */ -+static bool sig_unenforce = false; -+module_param(sig_unenforce, bool_enable_only, 0644); + - - /* - * Export sig_enforce kernel cmdline parameter to allow other subsystems rely -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index ee28253c9ac0..f2534e712a89 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -138,6 +138,18 @@ __read_mostly int sysctl_resched_latency_warn_ms = 100; - __read_mostly int sysctl_resched_latency_warn_once = 1; - #endif /* CONFIG_SCHED_DEBUG */ - -+/* -+ * Choose the yield level that will perform. -+ * 0: No yield. -+ * 1: Yield only to better priority/deadline tasks. -+ * 2: Re-queue current tasks. (default CFS) -+ */ -+#ifdef CONFIG_CACHY -+__read_mostly int sysctl_sched_yield_type = 1; -+#else -+__read_mostly int sysctl_sched_yield_type = 0; -+#endif ++ peer->sock = ovpn_socket_new(sock, peer); ++ if (IS_ERR(peer->sock)) { ++ peer->sock = NULL; ++ ovpn_peer_release(peer); ++ return ERR_PTR(-ENOTSOCK); ++ } + - /* - * Number of tasks to iterate in a single balance run. - * Limited because this is done with IRQs disabled. -@@ -8266,10 +8278,15 @@ static void do_sched_yield(void) - struct rq_flags rf; - struct rq *rq; - -+ if (!sysctl_sched_yield_type) -+ return; ++ /* schedule initial TCP RX work only after having assigned peer->sock */ ++ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) ++ queue_work(peer->ovpn->events_wq, &peer->tcp.rx_work); + - rq = this_rq_lock_irq(&rf); - - schedstat_inc(rq->yld_count); -- current->sched_class->yield_task(rq); ++ return peer; ++} + -+ if (sysctl_sched_yield_type > 1) -+ current->sched_class->yield_task(rq); - - preempt_disable(); - rq_unlock_irq(rq, &rf); -diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index 667876da8382..13fbcd19b916 100644 ---- a/kernel/sched/debug.c -+++ b/kernel/sched/debug.c -@@ -959,6 +959,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, - PN(se.exec_start); - PN(se.vruntime); - PN(se.sum_exec_runtime); -+ P(fsync_count); - - nr_switches = p->nvcsw + p->nivcsw; - -diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 914096c5b1ae..74d773040b54 100644 ---- a/kernel/sched/fair.c -+++ b/kernel/sched/fair.c -@@ -68,9 +68,13 @@ - * - * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_latency = 3000000ULL; -+static unsigned int normalized_sysctl_sched_latency = 3000000ULL; -+#else - unsigned int sysctl_sched_latency = 6000000ULL; - static unsigned int normalized_sysctl_sched_latency = 6000000ULL; -- -+#endif - /* - * The initial- and re-scaling of tunables is configurable - * -@@ -89,8 +93,13 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; - * - * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_min_granularity = 400000ULL; -+static unsigned int normalized_sysctl_sched_min_granularity = 400000ULL; -+#else - unsigned int sysctl_sched_min_granularity = 750000ULL; - static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; -+#endif - - /* - * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks. -@@ -120,8 +129,13 @@ unsigned int sysctl_sched_child_runs_first __read_mostly; - * - * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) - */ -+#ifdef CONFIG_CACHY -+unsigned int sysctl_sched_wakeup_granularity = 500000UL; -+static unsigned int normalized_sysctl_sched_wakeup_granularity = 500000UL; -+#else - unsigned int sysctl_sched_wakeup_granularity = 1000000UL; - static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; -+#endif - - const_debug unsigned int sysctl_sched_migration_cost = 500000UL; - -@@ -174,8 +188,12 @@ int __weak arch_asym_cpu_priority(int cpu) - * - * (default: 5 msec, units: microseconds) - */ -+#ifdef CONFIG_CACHY -+static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; -+#else - static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; - #endif -+#endif - - #ifdef CONFIG_SYSCTL - static struct ctl_table sched_fair_sysctls[] = { -diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c -index 9860bb9a847c..c7f045873a5f 100644 ---- a/kernel/sched/wait.c -+++ b/kernel/sched/wait.c -@@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ - } - EXPORT_SYMBOL_GPL(add_wait_queue_priority); - -+void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) ++/* Configure keepalive parameters */ ++void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) +{ -+ unsigned long flags; ++ u32 delta; + -+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; -+ spin_lock_irqsave(&wq_head->lock, flags); -+ __add_wait_queue(wq_head, wq_entry); -+ spin_unlock_irqrestore(&wq_head->lock, flags); ++ netdev_dbg(peer->ovpn->dev, ++ "%s: scheduling keepalive for peer %u: interval=%u timeout=%u\n", __func__, ++ peer->id, interval, timeout); ++ ++ peer->keepalive_interval = interval; ++ if (interval > 0) { ++ delta = msecs_to_jiffies(interval * MSEC_PER_SEC); ++ mod_timer(&peer->keepalive_xmit, jiffies + delta); ++ } else { ++ del_timer(&peer->keepalive_xmit); ++ } ++ ++ peer->keepalive_timeout = timeout; ++ if (timeout) { ++ delta = msecs_to_jiffies(timeout * MSEC_PER_SEC); ++ mod_timer(&peer->keepalive_recv, jiffies + delta); ++ } else { ++ del_timer(&peer->keepalive_recv); ++ } +} -+EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); + - void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) - { - unsigned long flags; -@@ -289,6 +300,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent - } - EXPORT_SYMBOL(prepare_to_wait_exclusive); - -+void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) ++#define ovpn_peer_index(_tbl, _key, _key_len) \ ++ (jhash(_key, _key_len, 0) % HASH_SIZE(_tbl)) \ ++ ++static struct ovpn_peer *ovpn_peer_lookup_vpn_addr4(struct hlist_head *head, __be32 *addr) +{ -+ unsigned long flags; ++ struct ovpn_peer *tmp, *peer = NULL; + -+ wq_entry->flags |= WQ_FLAG_EXCLUSIVE; -+ spin_lock_irqsave(&wq_head->lock, flags); -+ if (list_empty(&wq_entry->entry)) -+ __add_wait_queue(wq_head, wq_entry); -+ set_current_state(state); -+ spin_unlock_irqrestore(&wq_head->lock, flags); -+} -+EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr4) { ++ if (*addr != tmp->vpn_addrs.ipv4.s_addr) ++ continue; + - void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) - { - wq_entry->flags = flags; -diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c -index 860b2dcf3ac4..810e1fcaff94 100644 ---- a/kernel/sys_ni.c -+++ b/kernel/sys_ni.c -@@ -292,6 +292,7 @@ COND_SYSCALL(mincore); - COND_SYSCALL(madvise); - COND_SYSCALL(process_madvise); - COND_SYSCALL(process_mrelease); -+COND_SYSCALL(pmadv_ksm); - COND_SYSCALL(remap_file_pages); - COND_SYSCALL(mbind); - COND_SYSCALL(get_mempolicy); -diff --git a/kernel/sysctl.c b/kernel/sysctl.c -index 205d605cacc5..4432b6f15890 100644 ---- a/kernel/sysctl.c -+++ b/kernel/sysctl.c -@@ -89,8 +89,12 @@ - #ifdef CONFIG_PERF_EVENTS - static const int six_hundred_forty_kb = 640 * 1024; - #endif -+#ifdef CONFIG_USER_NS -+#include -+#endif - - -+extern int sysctl_sched_yield_type; - static const int ngroups_max = NGROUPS_MAX; - static const int cap_last_cap = CAP_LAST_CAP; - -@@ -1649,6 +1653,24 @@ static struct ctl_table kern_table[] = { - .mode = 0644, - .proc_handler = proc_dointvec, - }, -+ { -+ .procname = "yield_type", -+ .data = &sysctl_sched_yield_type, -+ .maxlen = sizeof (int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_TWO, -+ }, -+#ifdef CONFIG_USER_NS -+ { -+ .procname = "unprivileged_userns_clone", -+ .data = &unprivileged_userns_clone, -+ .maxlen = sizeof(int), -+ .mode = 0644, -+ .proc_handler = proc_dointvec, -+ }, -+#endif - #ifdef CONFIG_PROC_SYSCTL - { - .procname = "tainted", -diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c -index 5481ba44a8d6..423ab2563ad7 100644 ---- a/kernel/user_namespace.c -+++ b/kernel/user_namespace.c -@@ -21,6 +21,13 @@ - #include - #include - -+/* sysctl */ -+#ifdef CONFIG_USER_NS_UNPRIVILEGED -+int unprivileged_userns_clone = 1; -+#else -+int unprivileged_userns_clone; -+#endif ++ if (!ovpn_peer_hold(tmp)) ++ continue; + - static struct kmem_cache *user_ns_cachep __read_mostly; - static DEFINE_MUTEX(userns_state_mutex); - -diff --git a/kernel/watchdog.c b/kernel/watchdog.c -index 8e61f21e7e33..be1439d38f26 100644 ---- a/kernel/watchdog.c -+++ b/kernel/watchdog.c -@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled; - int __read_mostly watchdog_user_enabled = 1; - int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT; - int __read_mostly soft_watchdog_user_enabled = 1; --int __read_mostly watchdog_thresh = 10; -+int __read_mostly watchdog_thresh = 40; - static int __read_mostly nmi_watchdog_available; - - struct cpumask watchdog_cpumask __read_mostly; -diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c -index 39b74221f4a7..ec3eab8cd6b1 100644 ---- a/lib/raid6/algos.c -+++ b/lib/raid6/algos.c -@@ -128,8 +128,10 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) -- if (!(*algo)->valid || (*algo)->valid()) -+ if (!(*algo)->valid || (*algo)->valid()) { - best = *algo; -+ break; -+ } - - if (best) { - raid6_2data_recov = best->data2; -diff --git a/lib/string.c b/lib/string.c -index 6f334420f687..2e60e9e40535 100644 ---- a/lib/string.c -+++ b/lib/string.c -@@ -866,24 +866,61 @@ char *strnstr(const char *s1, const char *s2, size_t len) - EXPORT_SYMBOL(strnstr); - #endif - -+#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); + -+#define MEMCHR_MASK_GEN(mask) (mask *= 0x0101010101010101ULL) ++ return peer; ++} + -+#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) ++static struct ovpn_peer *ovpn_peer_lookup_vpn_addr6(struct hlist_head *head, struct in6_addr *addr) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ int i; + -+#define MEMCHR_MASK_GEN(mask) \ -+ do { \ -+ mask *= 0x01010101; \ -+ mask |= mask << 32; \ -+ } while (0) ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr6) { ++ for (i = 0; i < 4; i++) { ++ if (addr->s6_addr32[i] != tmp->vpn_addrs.ipv6.s6_addr32[i]) ++ continue; ++ } + -+#else ++ if (!ovpn_peer_hold(tmp)) ++ continue; + -+#define MEMCHR_MASK_GEN(mask) \ -+ do { \ -+ mask |= mask << 8; \ -+ mask |= mask << 16; \ -+ mask |= mask << 32; \ -+ } while (0) ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); ++ ++ return peer; ++} ++ ++/** ++ * ovpn_nexthop4() - looks up the IP of the nexthop for the given destination ++ * ++ * Looks up in the IPv4 system routing table the IO of the nexthop to be used ++ * to reach the destination passed as argument. IF no nexthop can be found, the ++ * destination itself is returned as it probably has to be used as nexthop. ++ * ++ * @ovpn: the private data representing the current VPN session ++ * @dst: the destination to be looked up ++ * ++ * Return the IP of the next hop if found or the dst itself otherwise ++ */ ++static __be32 ovpn_nexthop4(struct ovpn_struct *ovpn, __be32 dst) ++{ ++ struct rtable *rt; ++ struct flowi4 fl = { ++ .daddr = dst ++ }; + -+#endif ++ rt = ip_route_output_flow(dev_net(ovpn->dev), &fl, NULL); ++ if (IS_ERR(rt)) { ++ net_dbg_ratelimited("%s: no route to host %pI4\n", __func__, &dst); ++ /* if we end up here this packet is probably going to be thrown away later */ ++ return dst; ++ } + - #ifndef __HAVE_ARCH_MEMCHR - /** - * memchr - Find a character in an area of memory. -- * @s: The memory area -+ * @p: The memory area - * @c: The byte to search for -- * @n: The size of the area. -+ * @length: The size of the area. - * - * returns the address of the first occurrence of @c, or %NULL - * if @c is not found - */ --void *memchr(const void *s, int c, size_t n) -+void *memchr(const void *p, int c, unsigned long length) - { -- const unsigned char *p = s; -- while (n-- != 0) { -- if ((unsigned char)c == *p++) { -- return (void *)(p - 1); -+ u64 mask, val; -+ const void *end = p + length; ++ if (!rt->rt_uses_gateway) ++ goto out; + -+ c &= 0xff; -+ if (p <= end - 8) { -+ mask = c; -+ MEMCHR_MASK_GEN(mask); ++ dst = rt->rt_gw4; ++out: ++ ip_rt_put(rt); ++ return dst; ++} + -+ for (; p <= end - 8; p += 8) { -+ val = *(u64 *)p ^ mask; -+ if ((val + 0xfefefefefefefeffu) & -+ (~val & 0x8080808080808080u)) -+ break; - } - } ++/** ++ * ovpn_nexthop6() - looks up the IPv6 of the nexthop for the given destination ++ * ++ * Looks up in the IPv6 system routing table the IO of the nexthop to be used ++ * to reach the destination passed as argument. IF no nexthop can be found, the ++ * destination itself is returned as it probably has to be used as nexthop. ++ * ++ * @ovpn: the private data representing the current VPN session ++ * @dst: the destination to be looked up ++ * ++ * Return the IP of the next hop if found or the dst itself otherwise ++ */ ++static struct in6_addr ovpn_nexthop6(struct ovpn_struct *ovpn, struct in6_addr dst) ++{ ++#if IS_ENABLED(CONFIG_IPV6) ++ struct rt6_info *rt; ++ struct flowi6 fl = { ++ .daddr = dst, ++ }; + -+ for (; p < end; p++) -+ if (*(unsigned char *)p == c) -+ return (void *)p; ++ rt = (struct rt6_info *)ipv6_stub->ipv6_dst_lookup_flow(dev_net(ovpn->dev), NULL, &fl, ++ NULL); ++ if (IS_ERR(rt)) { ++ net_dbg_ratelimited("%s: no route to host %pI6\n", __func__, &dst); ++ /* if we end up here this packet is probably going to be thrown away later */ ++ return dst; ++ } + - return NULL; - } - EXPORT_SYMBOL(memchr); -@@ -919,16 +956,7 @@ void *memchr_inv(const void *start, int c, size_t bytes) - return check_bytes8(start, value, bytes); - - value64 = value; --#if defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) && BITS_PER_LONG == 64 -- value64 *= 0x0101010101010101ULL; --#elif defined(CONFIG_ARCH_HAS_FAST_MULTIPLIER) -- value64 *= 0x01010101; -- value64 |= value64 << 32; --#else -- value64 |= value64 << 8; -- value64 |= value64 << 16; -- value64 |= value64 << 32; --#endif -+ MEMCHR_MASK_GEN(value64); - - prefix = (unsigned long)start % 8; - if (prefix) { -diff --git a/lib/zstd/compress/zstd_compress.c b/lib/zstd/compress/zstd_compress.c -index a4e916008b3a..73fff4c60149 100644 ---- a/lib/zstd/compress/zstd_compress.c -+++ b/lib/zstd/compress/zstd_compress.c -@@ -4441,7 +4441,7 @@ static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, - size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { - size_t offsetBound; - U32 windowSize = 1 << windowLog; -- /* posInSrc represents the amount of data the the decoder would decode up to this point. -+ /* posInSrc represents the amount of data the decoder would decode up to this point. - * As long as the amount of data decoded is less than or equal to window size, offsets may be - * larger than the total length of output decoded in order to reference the dict, even larger than - * window size. After output surpasses windowSize, we're limited to windowSize offsets again. -diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c -index b0424d23ac57..fb941a5b70f5 100644 ---- a/lib/zstd/compress/zstd_double_fast.c -+++ b/lib/zstd/compress/zstd_double_fast.c -@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(dictMode, mls) \ -+ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \ -+ } ++ if (!(rt->rt6i_flags & RTF_GATEWAY)) ++ goto out; + -+ZSTD_GEN_FN(noDict, 4) -+ZSTD_GEN_FN(noDict, 5) -+ZSTD_GEN_FN(noDict, 6) -+ZSTD_GEN_FN(noDict, 7) ++ dst = rt->rt6i_gateway; ++out: ++ dst_release((struct dst_entry *)rt); ++#endif ++ return dst; ++} + -+ZSTD_GEN_FN(dictMatchState, 4) -+ZSTD_GEN_FN(dictMatchState, 5) -+ZSTD_GEN_FN(dictMatchState, 6) -+ZSTD_GEN_FN(dictMatchState, 7) ++/** ++ * ovpn_peer_lookup_vpn_addr() - Lookup peer to send skb to ++ * ++ * This function takes a tunnel packet and looks up the peer to send it to ++ * after encapsulation. The skb is expected to be the in-tunnel packet, without ++ * any OpenVPN related header. ++ * ++ * Assume that the IP header is accessible in the skb data. ++ * ++ * @ovpn: the private data representing the current VPN session ++ * @skb: the skb to extract the destination address from ++ * ++ * Return the peer if found or NULL otherwise. ++ */ ++struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, ++ bool use_src) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ struct hlist_head *head; ++ struct rt6_info *rt6i = NULL; ++ struct rtable *rt = NULL; ++ sa_family_t sa_fam; ++ struct in6_addr addr6; ++ __be32 addr4; ++ u32 index; + -+#undef ZSTD_GEN_FN ++ /* in P2P mode, no matter the destination, packets are always sent to the single peer ++ * listening on the other side ++ */ ++ if (ovpn->mode == OVPN_MODE_P2P) { ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (likely(tmp && ovpn_peer_hold(tmp))) ++ peer = tmp; ++ rcu_read_unlock(); ++ return peer; ++ } + - - size_t ZSTD_compressBlock_doubleFast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); -+ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - - /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ - if (prefixStartIndex == dictStartIndex) -- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); -+ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( - } - - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } ++ sa_fam = skb_protocol_to_family(skb); + -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) ++ switch (sa_fam) { ++ case AF_INET: ++ if (use_src) ++ addr4 = ip_hdr(skb)->saddr; ++ else ++ addr4 = ip_hdr(skb)->daddr; ++ addr4 = ovpn_nexthop4(ovpn, addr4); + -+#undef ZSTD_GEN_FN ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr4, sizeof(addr4)); ++ head = &ovpn->peers.by_vpn_addr[index]; + - size_t ZSTD_compressBlock_doubleFast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); - } - } -diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c -index 96b7d48e2868..e0652e31d421 100644 ---- a/lib/zstd/compress/zstd_fast.c -+++ b/lib/zstd/compress/zstd_fast.c -@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } ++ peer = ovpn_peer_lookup_vpn_addr4(head, &addr4); ++ break; ++ case AF_INET6: ++ if (use_src) ++ addr6 = ipv6_hdr(skb)->saddr; ++ else ++ addr6 = ipv6_hdr(skb)->daddr; ++ addr6 = ovpn_nexthop6(ovpn, addr6); + -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr6, sizeof(addr6)); ++ head = &ovpn->peers.by_vpn_addr[index]; + -+#undef ZSTD_GEN_FN - - size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } ++ peer = ovpn_peer_lookup_vpn_addr6(head, &addr6); ++ break; ++ } + -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) ++ if (rt) ++ ip_rt_put(rt); ++ if (rt6i) ++ dst_release((struct dst_entry *)rt6i); + -+#undef ZSTD_GEN_FN ++ return peer; ++} + - size_t ZSTD_compressBlock_fast_dictMatchState( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } - } - -@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( - - /* switch to "regular" variant if extDict is invalidated due to maxDistance */ - if (prefixStartIndex == dictStartIndex) -- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); -+ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ -@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( - return (size_t)(iend - anchor); - } - -+#define ZSTD_GEN_FN(mls) \ -+ static size_t ZSTD_compressBlock_fast_extDict_##mls( \ -+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ -+ void const* src, size_t srcSize) \ -+ { \ -+ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \ -+ } ++static bool ovpn_peer_transp_match(struct ovpn_peer *peer, struct sockaddr_storage *ss) ++{ ++ struct ovpn_bind *bind = rcu_dereference(peer->bind); ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa4; + -+ZSTD_GEN_FN(4) -+ZSTD_GEN_FN(5) -+ZSTD_GEN_FN(6) -+ZSTD_GEN_FN(7) ++ if (unlikely(!bind)) ++ return false; + -+#undef ZSTD_GEN_FN - - size_t ZSTD_compressBlock_fast_extDict( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], -@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict( - { - default: /* includes case 3 */ - case 4 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); -+ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); -+ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); -+ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : -- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); -+ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize); - } - } -diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c -index fb54d4e28a2b..1db22db5b1e9 100644 ---- a/lib/zstd/compress/zstd_lazy.c -+++ b/lib/zstd/compress/zstd_lazy.c -@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, - } - - --static size_t --ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); -- } --} -- -- --static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); -- } --} -- -- --static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); -- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); -- case 7 : -- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); -- } --} -- -- -- - /* ********************************* - * Hash Chain - ***********************************/ -@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B - - /* inlining is important to hardwire a hot branch (template emulation) */ - FORCE_INLINE_TEMPLATE --size_t ZSTD_HcFindBestMatch_generic ( -+size_t ZSTD_HcFindBestMatch( - ZSTD_matchState_t* ms, - const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, -@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic ( - return ml; - } - -+typedef size_t (*searchMax_f)( -+ ZSTD_matchState_t* ms, -+ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - --FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); -- } --} -+/* -+ * This struct contains the functions necessary for lazy to search. -+ * Currently, that is only searchMax. However, it is still valuable to have the -+ * VTable because this makes it easier to add more functions to the VTable later. -+ */ -+typedef struct { -+ searchMax_f searchMax; -+} ZSTD_LazyVTable; ++ if (ss->ss_family != bind->sa.in4.sin_family) ++ return false; ++ ++ switch (ss->ss_family) { ++ case AF_INET: ++ sa4 = (struct sockaddr_in *)ss; ++ if (sa4->sin_addr.s_addr != bind->sa.in4.sin_addr.s_addr) ++ return false; ++ if (sa4->sin_port != bind->sa.in4.sin_port) ++ return false; ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)ss; ++ if (memcmp(&sa6->sin6_addr, &bind->sa.in6.sin6_addr, sizeof(struct in6_addr))) ++ return false; ++ if (sa6->sin6_port != bind->sa.in6.sin6_port) ++ return false; ++ break; ++ default: ++ return false; ++ } + -+#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \ -+ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, \ -+ const BYTE* ip, const BYTE* const iLimit, \ -+ size_t* offsetPtr) \ -+ { \ -+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ -+ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ -+ } \ -+ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ -+ ZSTD_BtFindBestMatch_##dictMode##_##mls \ -+ }; - -+#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \ -+ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ -+ ZSTD_matchState_t* ms, \ -+ const BYTE* ip, const BYTE* const iLimit, \ -+ size_t* offsetPtr) \ -+ { \ -+ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ -+ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ -+ } \ -+ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ -+ ZSTD_HcFindBestMatch_##dictMode##_##mls \ -+ }; - --static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); -+#define ZSTD_FOR_EACH_MLS(X, dictMode) \ -+ X(dictMode, 4) \ -+ X(dictMode, 5) \ -+ X(dictMode, 6) ++ return true; ++} + -+#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ -+ X(__VA_ARGS__, noDict) \ -+ X(__VA_ARGS__, extDict) \ -+ X(__VA_ARGS__, dictMatchState) \ -+ X(__VA_ARGS__, dedicatedDictSearch) ++static bool ovpn_peer_skb_to_sockaddr(struct sk_buff *skb, struct sockaddr_storage *ss) ++{ ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa4; + -+/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ -+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) -+/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ -+ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) ++ ss->ss_family = skb_protocol_to_family(skb); ++ switch (ss->ss_family) { ++ case AF_INET: ++ sa4 = (struct sockaddr_in *)ss; ++ sa4->sin_family = AF_INET; ++ sa4->sin_addr.s_addr = ip_hdr(skb)->saddr; ++ sa4->sin_port = udp_hdr(skb)->source; ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)ss; ++ sa6->sin6_family = AF_INET6; ++ sa6->sin6_addr = ipv6_hdr(skb)->saddr; ++ sa6->sin6_port = udp_hdr(skb)->source; ++ break; ++ default: ++ return false; ++ } + -+#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ -+ { \ -+ &ZSTD_BtVTable_##dictMode##_4, \ -+ &ZSTD_BtVTable_##dictMode##_5, \ -+ &ZSTD_BtVTable_##dictMode##_6 \ - } --} -- - --static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch); -+#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ -+ { \ -+ &ZSTD_HcVTable_##dictMode##_4, \ -+ &ZSTD_HcVTable_##dictMode##_5, \ -+ &ZSTD_HcVTable_##dictMode##_6 \ - } --} - -- --FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* const iLimit, -- size_t* offsetPtr) --{ -- switch(ms->cParams.minMatch) -- { -- default : /* includes case 3 */ -- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); -- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); -- case 7 : -- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); -+#define GEN_ZSTD_VTABLE_ARRAY(X) \ -+ { \ -+ X(noDict), \ -+ X(extDict), \ -+ X(dictMatchState), \ -+ X(dedicatedDictSearch) \ - } --} -- - - /* ******************************* - * Common parser - lazy strategy - *********************************/ - typedef enum { search_hashChain, search_binaryTree } searchMethod_e; - -+static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) ++ return true; ++} ++ ++static struct ovpn_peer *ovpn_peer_lookup_transp_addr_p2p(struct ovpn_struct *ovpn, ++ struct sockaddr_storage *ss) +{ -+ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ -+ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); -+ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); -+ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ ++ struct ovpn_peer *tmp, *peer = NULL; + -+ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); -+ switch (searchMethod) { -+ case search_hashChain: -+ return hcVTables[dictMode][mls - 4]; -+ case search_binaryTree: -+ return btVTables[dictMode][mls - 4]; -+ default: -+ return NULL; -+ } ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (likely(tmp && ovpn_peer_transp_match(tmp, ss) && ovpn_peer_hold(tmp))) ++ peer = tmp; ++ rcu_read_unlock(); ++ ++ return peer; +} + - FORCE_INLINE_TEMPLATE size_t - ZSTD_compressBlock_lazy_generic( - ZSTD_matchState_t* ms, seqStore_t* seqStore, -@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic( - const U32 prefixLowestIndex = ms->window.dictLimit; - const BYTE* const prefixLowest = base + prefixLowestIndex; - -- typedef size_t (*searchMax_f)( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -- - /* - * This table is indexed first by the four ZSTD_dictMode_e values, and then - * by the two searchMethod_e values. NULLs are placed for configurations - * that should never occur (extDict modes go to the other implementation - * below and there is no DDSS for binary tree search yet). - */ -- const searchMax_f searchFuncs[4][2] = { -- { -- ZSTD_HcFindBestMatch_selectMLS, -- ZSTD_BtFindBestMatch_selectMLS -- }, -- { -- NULL, -- NULL -- }, -- { -- ZSTD_HcFindBestMatch_dictMatchState_selectMLS, -- ZSTD_BtFindBestMatch_dictMatchState_selectMLS -- }, -- { -- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS, -- NULL -- } -- }; -- -- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree]; -+ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; - U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; - - const int isDMS = dictMode == ZSTD_dictMatchState; -@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( - const BYTE* const dictStart = dictBase + ms->window.lowLimit; - const U32 windowLog = ms->cParams.windowLog; - -- typedef size_t (*searchMax_f)( -- ZSTD_matchState_t* ms, -- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); -- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; -+ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; - - U32 offset_1 = rep[0], offset_2 = rep[1]; - -diff --git a/mm/compaction.c b/mm/compaction.c -index 640fa76228dd..01a661f5ecac 100644 ---- a/mm/compaction.c -+++ b/mm/compaction.c -@@ -1727,7 +1727,7 @@ typedef enum { - * Allow userspace to control policy on scanning the unevictable LRU for - * compactable pages. - */ --#ifdef CONFIG_PREEMPT_RT -+#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) - int sysctl_compact_unevictable_allowed __read_mostly = 0; - #else - int sysctl_compact_unevictable_allowed __read_mostly = 1; -@@ -2719,7 +2719,11 @@ static void compact_nodes(void) - * aggressively the kernel should compact memory in the - * background. It takes values in the range [0, 100]. - */ -+#ifdef CONFIG_CACHY -+unsigned int __read_mostly sysctl_compaction_proactiveness; -+#else - unsigned int __read_mostly sysctl_compaction_proactiveness = 20; -+#endif - - int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, - void *buffer, size_t *length, loff_t *ppos) -diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig -index 66265e3a9c65..7821fcb3f258 100644 ---- a/mm/damon/Kconfig -+++ b/mm/damon/Kconfig -@@ -68,6 +68,9 @@ config DAMON_DBGFS - - If unsure, say N. - -+ This will be removed after >5.15.y LTS kernel is released, so users -+ should move to the sysfs interface (DAMON_SYSFS). -+ - config DAMON_DBGFS_KUNIT_TEST - bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS - depends on DAMON_DBGFS && KUNIT=y -diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h -index 573669566f84..3db9b7368756 100644 ---- a/mm/damon/core-test.h -+++ b/mm/damon/core-test.h -@@ -126,7 +126,7 @@ static void damon_test_split_at(struct kunit *test) - t = damon_new_target(); - r = damon_new_region(0, 100); - damon_add_region(r, t); -- damon_split_region_at(c, t, r, 25); -+ damon_split_region_at(t, r, 25); - KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); - KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); - -@@ -219,14 +219,14 @@ static void damon_test_split_regions_of(struct kunit *test) - t = damon_new_target(); - r = damon_new_region(0, 22); - damon_add_region(r, t); -- damon_split_regions_of(c, t, 2); -+ damon_split_regions_of(t, 2); - KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u); - damon_free_target(t); - - t = damon_new_target(); - r = damon_new_region(0, 220); - damon_add_region(r, t); -- damon_split_regions_of(c, t, 4); -+ damon_split_regions_of(t, 4); - KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u); - damon_free_target(t); - damon_destroy_ctx(c); -@@ -267,6 +267,28 @@ static void damon_test_ops_registration(struct kunit *test) - KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL); - } - -+static void damon_test_set_regions(struct kunit *test) -+{ -+ struct damon_target *t = damon_new_target(); -+ struct damon_region *r1 = damon_new_region(4, 16); -+ struct damon_region *r2 = damon_new_region(24, 32); -+ struct damon_addr_range range = {.start = 8, .end = 28}; -+ unsigned long expects[] = {8, 16, 16, 24, 24, 28}; -+ int expect_idx = 0; -+ struct damon_region *r; -+ -+ damon_add_region(r1, t); -+ damon_add_region(r2, t); -+ damon_set_regions(t, &range, 1); -+ -+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); -+ damon_for_each_region(r, t) { -+ KUNIT_EXPECT_EQ(test, r->ar.start, expects[expect_idx++]); -+ KUNIT_EXPECT_EQ(test, r->ar.end, expects[expect_idx++]); -+ } -+ damon_destroy_target(t); -+} -+ - static struct kunit_case damon_test_cases[] = { - KUNIT_CASE(damon_test_target), - KUNIT_CASE(damon_test_regions), -@@ -276,6 +298,7 @@ static struct kunit_case damon_test_cases[] = { - KUNIT_CASE(damon_test_merge_regions_of), - KUNIT_CASE(damon_test_split_regions_of), - KUNIT_CASE(damon_test_ops_registration), -+ KUNIT_CASE(damon_test_set_regions), - {}, - }; - -diff --git a/mm/damon/core.c b/mm/damon/core.c -index 7d25dc582fe3..1c7366bf477d 100644 ---- a/mm/damon/core.c -+++ b/mm/damon/core.c -@@ -168,6 +168,27 @@ static bool damon_intersect(struct damon_region *r, - return !(r->ar.end <= re->start || re->end <= r->ar.start); - } - -+/* -+ * Fill holes in regions with new regions. -+ */ -+static void damon_fill_regions_holes(struct damon_region *first, -+ struct damon_region *last, struct damon_target *t) ++struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb) +{ -+ struct damon_region *r = first; ++ struct ovpn_peer *peer = NULL, *tmp; ++ struct sockaddr_storage ss = { 0 }; ++ struct hlist_head *head; ++ size_t sa_len; ++ bool found; ++ u32 index; + -+ damon_for_each_region_from(r, t) { -+ struct damon_region *next, *newr; ++ if (unlikely(!ovpn_peer_skb_to_sockaddr(skb, &ss))) ++ return NULL; + -+ if (r == last) -+ break; -+ next = damon_next_region(r); -+ if (r->ar.end != next->ar.start) { -+ newr = damon_new_region(r->ar.end, next->ar.start); -+ damon_insert_region(newr, r, next, t); -+ } ++ if (ovpn->mode == OVPN_MODE_P2P) ++ return ovpn_peer_lookup_transp_addr_p2p(ovpn, &ss); ++ ++ switch (ss.ss_family) { ++ case AF_INET: ++ sa_len = sizeof(struct sockaddr_in); ++ break; ++ case AF_INET6: ++ sa_len = sizeof(struct sockaddr_in6); ++ break; ++ default: ++ return NULL; ++ } ++ ++ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &ss, sa_len); ++ head = &ovpn->peers.by_transp_addr[index]; ++ ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_transp_addr) { ++ found = ovpn_peer_transp_match(tmp, &ss); ++ if (!found) ++ continue; ++ ++ if (!ovpn_peer_hold(tmp)) ++ continue; ++ ++ peer = tmp; ++ break; + } ++ rcu_read_unlock(); ++ ++ return peer; +} + - /* - * damon_set_regions() - Set regions of a target for given address ranges. - * @t: the given target. -@@ -225,6 +246,9 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, - first->ar.start = ALIGN_DOWN(range->start, - DAMON_MIN_REGION); - last->ar.end = ALIGN(range->end, DAMON_MIN_REGION); -+ -+ /* fill possible holes in the range */ -+ damon_fill_regions_holes(first, last, t); - } - } - return 0; -@@ -658,9 +682,8 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) - } - } - --static void damon_split_region_at(struct damon_ctx *ctx, -- struct damon_target *t, struct damon_region *r, -- unsigned long sz_r); -+static void damon_split_region_at(struct damon_target *t, -+ struct damon_region *r, unsigned long sz_r); - - static bool __damos_valid_target(struct damon_region *r, struct damos *s) - { -@@ -726,7 +749,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, - continue; - sz = DAMON_MIN_REGION; - } -- damon_split_region_at(c, t, r, sz); -+ damon_split_region_at(t, r, sz); - r = damon_next_region(r); - sz = r->ar.end - r->ar.start; - } -@@ -745,7 +768,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, - DAMON_MIN_REGION); - if (!sz) - goto update_stat; -- damon_split_region_at(c, t, r, sz); -+ damon_split_region_at(t, r, sz); - } - ktime_get_coarse_ts64(&begin); - sz_applied = c->ops.apply_scheme(c, t, r, s); -@@ -928,9 +951,8 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, - * r the region to be split - * sz_r size of the first sub-region that will be made - */ --static void damon_split_region_at(struct damon_ctx *ctx, -- struct damon_target *t, struct damon_region *r, -- unsigned long sz_r) -+static void damon_split_region_at(struct damon_target *t, -+ struct damon_region *r, unsigned long sz_r) - { - struct damon_region *new; - -@@ -947,8 +969,7 @@ static void damon_split_region_at(struct damon_ctx *ctx, - } - - /* Split every region in the given target into 'nr_subs' regions */ --static void damon_split_regions_of(struct damon_ctx *ctx, -- struct damon_target *t, int nr_subs) -+static void damon_split_regions_of(struct damon_target *t, int nr_subs) - { - struct damon_region *r, *next; - unsigned long sz_region, sz_sub = 0; -@@ -969,7 +990,7 @@ static void damon_split_regions_of(struct damon_ctx *ctx, - if (sz_sub == 0 || sz_sub >= sz_region) - continue; - -- damon_split_region_at(ctx, t, r, sz_sub); -+ damon_split_region_at(t, r, sz_sub); - sz_region = sz_sub; - } - } -@@ -1004,7 +1025,7 @@ static void kdamond_split_regions(struct damon_ctx *ctx) - nr_subregions = 3; - - damon_for_each_target(t, ctx) -- damon_split_regions_of(ctx, t, nr_subregions); -+ damon_split_regions_of(t, nr_subregions); - - last_nr_regions = nr_regions; - } -diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c -index cfdf63132d5a..3b55a1b219b5 100644 ---- a/mm/damon/dbgfs.c -+++ b/mm/damon/dbgfs.c -@@ -1044,7 +1044,7 @@ static int __init __damon_dbgfs_init(void) - fops[i]); - dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]); - -- dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL); -+ dbgfs_dirs = kmalloc(sizeof(dbgfs_root), GFP_KERNEL); - if (!dbgfs_dirs) { - debugfs_remove(dbgfs_root); - return -ENOMEM; -diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c -index 9de6f00a71c5..a3674532fa67 100644 ---- a/mm/damon/lru_sort.c -+++ b/mm/damon/lru_sort.c -@@ -257,18 +257,13 @@ module_param(nr_cold_quota_exceeds, ulong, 0400); - static struct damon_ctx *ctx; - static struct damon_target *target; - --struct damon_lru_sort_ram_walk_arg { -- unsigned long start; -- unsigned long end; --}; -- - static int walk_system_ram(struct resource *res, void *arg) - { -- struct damon_lru_sort_ram_walk_arg *a = arg; -+ struct damon_addr_range *r = arg; - -- if (a->end - a->start < resource_size(res)) { -- a->start = res->start; -- a->end = res->end; -+ if (r->end - r->start < resource_size(res)) { -+ r->start = res->start; -+ r->end = res->end; - } - return 0; - } -@@ -277,16 +272,12 @@ static int walk_system_ram(struct resource *res, void *arg) - * Find biggest 'System RAM' resource and store its start and end address in - * @start and @end, respectively. If no System RAM is found, returns false. - */ --static bool get_monitoring_region(unsigned long *start, unsigned long *end) -+static bool get_monitoring_region(struct damon_addr_range *range) - { -- struct damon_lru_sort_ram_walk_arg arg = {}; -- -- walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); -- if (arg.end <= arg.start) -+ walk_system_ram_res(0, ULONG_MAX, range, walk_system_ram); -+ if (range->end <= range->start) - return false; - -- *start = arg.start; -- *end = arg.end; - return true; - } - -@@ -378,6 +369,16 @@ static int damon_lru_sort_apply_parameters(void) - unsigned int hot_thres, cold_thres; - int err = 0; - -+ if (monitor_region_start > monitor_region_end) -+ return -EINVAL; -+ if (!monitor_region_end) -+ return -EINVAL; ++static struct ovpn_peer *ovpn_peer_lookup_id_p2p(struct ovpn_struct *ovpn, u32 peer_id) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; + -+ addr_range.start = monitor_region_start; -+ addr_range.end = monitor_region_end; -+ if (!get_monitoring_region(&addr_range)) -+ return -EINVAL; ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp))) ++ peer = tmp; ++ rcu_read_unlock(); + - err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, - min_nr_regions, max_nr_regions); - if (err) -@@ -401,14 +402,6 @@ static int damon_lru_sort_apply_parameters(void) - return -ENOMEM; - damon_add_scheme(ctx, scheme); - -- if (monitor_region_start > monitor_region_end) -- return -EINVAL; -- if (!monitor_region_start && !monitor_region_end && -- !get_monitoring_region(&monitor_region_start, -- &monitor_region_end)) -- return -EINVAL; -- addr_range.start = monitor_region_start; -- addr_range.end = monitor_region_end; - return damon_set_regions(target, &addr_range, 1); - } - -diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c -index 3c7b9d6dca95..cc04d467ba23 100644 ---- a/mm/damon/vaddr.c -+++ b/mm/damon/vaddr.c -@@ -302,9 +302,14 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, - pte_t *pte; - spinlock_t *ptl; - -- if (pmd_huge(*pmd)) { -+ if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(walk->mm, pmd); -- if (pmd_huge(*pmd)) { -+ if (!pmd_present(*pmd)) { -+ spin_unlock(ptl); -+ return 0; -+ } ++ return peer; ++} + -+ if (pmd_trans_huge(*pmd)) { - damon_pmdp_mkold(pmd, walk->mm, addr); - spin_unlock(ptl); - return 0; -@@ -429,9 +434,14 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, - struct damon_young_walk_private *priv = walk->private; - - #ifdef CONFIG_TRANSPARENT_HUGEPAGE -- if (pmd_huge(*pmd)) { -+ if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(walk->mm, pmd); -- if (!pmd_huge(*pmd)) { -+ if (!pmd_present(*pmd)) { -+ spin_unlock(ptl); -+ return 0; -+ } ++struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id) ++{ ++ struct ovpn_peer *tmp, *peer = NULL; ++ struct hlist_head *head; ++ u32 index; + -+ if (!pmd_trans_huge(*pmd)) { - spin_unlock(ptl); - goto regular_page; - } -diff --git a/mm/ksm.c b/mm/ksm.c -index 42ab153335a2..f3e6eaf437ae 100644 ---- a/mm/ksm.c -+++ b/mm/ksm.c -@@ -131,6 +131,10 @@ struct mm_slot { - * @address: the next address inside that to be scanned - * @rmap_list: link to the next rmap to be scanned in the rmap_list - * @seqnr: count of completed full scans (needed when removing unstable node) -+ * @new_ksmpages: count of the new merged KSM pages in the current scanning -+ * of mm_lists (cleared after every turn of ksm_do_scan() ends) -+ * @prev_ksmpages: the record of the new merged KSM pages in the last turn of -+ * scanning by ksm_do_scan(). - * - * There is only the one ksm_scan instance of this cursor structure. - */ -@@ -139,6 +143,8 @@ struct ksm_scan { - unsigned long address; - struct rmap_item **rmap_list; - unsigned long seqnr; -+ unsigned long new_ksmpages; -+ unsigned long prev_ksmpages; - }; - - /** -@@ -277,6 +283,33 @@ static unsigned int zero_checksum __read_mostly; - /* Whether to merge empty (zeroed) pages with actual zero pages */ - static bool ksm_use_zero_pages __read_mostly; - -+/* -+ * Work in auto-mode. -+ * The multiplicative factor of pages_to_scan. -+ * Real pages to scan equals to the product of scanning_factor -+ * and pages_to_scan -+ */ -+#define INIT_SCANNING_FACTOR 1 -+static unsigned int scanning_factor = INIT_SCANNING_FACTOR; ++ if (ovpn->mode == OVPN_MODE_P2P) ++ return ovpn_peer_lookup_id_p2p(ovpn, peer_id); + -+/* The upper limit of scanning_factor */ -+#define DEFAULT_MAX_SCANNING_FACTOR 16 -+static unsigned int max_scanning_factor = DEFAULT_MAX_SCANNING_FACTOR; ++ index = ovpn_peer_index(ovpn->peers.by_id, &peer_id, sizeof(peer_id)); ++ head = &ovpn->peers.by_id[index]; + -+/* -+ * Work in auto mode. -+ * Value: 0~100. Default 20 means "20%". When free memory is lower -+ * than this total memory * ksm_auto_threshold/100, auto_triggered -+ * will be set true. -+ */ -+unsigned int ksm_auto_threshold = 20; ++ rcu_read_lock(); ++ hlist_for_each_entry_rcu(tmp, head, hash_entry_id) { ++ if (tmp->id != peer_id) ++ continue; + -+/* Work in auto-mode. Whether trigger ksmd to compare and merge pages */ -+static bool auto_triggered; ++ if (!ovpn_peer_hold(tmp)) ++ continue; + -+/* Count of times that ksmd is triggered due to low free memory */ -+static unsigned long triggered_times; ++ peer = tmp; ++ break; ++ } ++ rcu_read_unlock(); + - #ifdef CONFIG_NUMA - /* Zeroed when merging across nodes is not allowed */ - static unsigned int ksm_merge_across_nodes = 1; -@@ -290,6 +323,7 @@ static int ksm_nr_node_ids = 1; - #define KSM_RUN_MERGE 1 - #define KSM_RUN_UNMERGE 2 - #define KSM_RUN_OFFLINE 4 -+#define KSM_RUN_AUTO 8 - static unsigned long ksm_run = KSM_RUN_STOP; - static void wait_while_offlining(void); - -@@ -387,6 +421,7 @@ static inline struct rmap_item *alloc_rmap_item(void) - static inline void free_rmap_item(struct rmap_item *rmap_item) - { - ksm_rmap_items--; -+ rmap_item->mm->ksm_rmap_items--; - rmap_item->mm = NULL; /* debug safety */ - kmem_cache_free(rmap_item_cache, rmap_item); - } -@@ -2020,6 +2055,8 @@ static void stable_tree_append(struct rmap_item *rmap_item, - rmap_item->address |= STABLE_FLAG; - hlist_add_head(&rmap_item->hlist, &stable_node->hlist); - -+ ksm_scan.new_ksmpages++; -+ - if (rmap_item->hlist.next) - ksm_pages_sharing++; - else -@@ -2219,6 +2256,7 @@ static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, - if (rmap_item) { - /* It has already been zeroed */ - rmap_item->mm = mm_slot->mm; -+ rmap_item->mm->ksm_rmap_items++; - rmap_item->address = addr; - rmap_item->rmap_list = *rmap_list; - *rmap_list = rmap_item; -@@ -2399,16 +2437,107 @@ static void ksm_do_scan(unsigned int scan_npages) - rmap_item = scan_get_next_rmap_item(&page); - if (!rmap_item) - return; -- cmp_and_merge_page(page, rmap_item); -+ if (ksm_run & KSM_RUN_AUTO && !auto_triggered) { -+ /* -+ * This should happens only when ksm_run is KSM_RUN_AUTO -+ * and free memory threshold still not reached. -+ * The reason to calculate it's checksum is to reduce the -+ * waiting time the rmap_item is added to unstable tree. -+ */ -+ rmap_item->oldchecksum = calc_checksum(page); -+ } else -+ cmp_and_merge_page(page, rmap_item); ++ return peer; ++} + - put_page(page); - } - } - -+#define RIGHT_SHIFT_FOUR_BIT 4 -+/* Work in auto mode, should reset auto_triggered ? */ -+static bool should_stop_ksmd_to_merge(void) ++void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct ovpn_bind *bind; ++ ++ rcu_read_lock(); ++ bind = rcu_dereference(peer->bind); ++ if (unlikely(!bind)) ++ goto unlock; ++ ++ switch (skb_protocol_to_family(skb)) { ++ case AF_INET: ++ if (unlikely(bind->local.ipv4.s_addr != ip_hdr(skb)->daddr)) { ++ netdev_dbg(peer->ovpn->dev, ++ "%s: learning local IPv4 for peer %d (%pI4 -> %pI4)\n", __func__, ++ peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); ++ bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; ++ } ++ break; ++ case AF_INET6: ++ if (unlikely(memcmp(&bind->local.ipv6, &ipv6_hdr(skb)->daddr, ++ sizeof(bind->local.ipv6)))) { ++ netdev_dbg(peer->ovpn->dev, ++ "%s: learning local IPv6 for peer %d (%pI6c -> %pI6c\n", ++ __func__, peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); ++ bind->local.ipv6 = ipv6_hdr(skb)->daddr; ++ } ++ break; ++ default: ++ break; ++ } ++unlock: ++ rcu_read_unlock(); ++} ++ ++static int ovpn_peer_add_mp(struct ovpn_struct *ovpn, struct ovpn_peer *peer) +{ -+ unsigned long total_ram_pages, free_pages; -+ unsigned int threshold; ++ struct sockaddr_storage sa = { 0 }; ++ struct sockaddr_in6 *sa6; ++ struct sockaddr_in *sa4; ++ struct ovpn_bind *bind; ++ struct ovpn_peer *tmp; ++ size_t salen; ++ int ret = 0; ++ u32 index; ++ ++ spin_lock_bh(&ovpn->peers.lock); ++ /* do not add duplicates */ ++ tmp = ovpn_peer_lookup_id(ovpn, peer->id); ++ if (tmp) { ++ ovpn_peer_put(tmp); ++ ret = -EEXIST; ++ goto unlock; ++ } ++ ++ hlist_del_init_rcu(&peer->hash_entry_transp_addr); ++ bind = rcu_dereference_protected(peer->bind, true); ++ /* peers connected via UDP have bind == NULL */ ++ if (bind) { ++ switch (bind->sa.in4.sin_family) { ++ case AF_INET: ++ sa4 = (struct sockaddr_in *)&sa; ++ ++ sa4->sin_family = AF_INET; ++ sa4->sin_addr.s_addr = bind->sa.in4.sin_addr.s_addr; ++ sa4->sin_port = bind->sa.in4.sin_port; ++ salen = sizeof(*sa4); ++ break; ++ case AF_INET6: ++ sa6 = (struct sockaddr_in6 *)&sa; ++ ++ sa6->sin6_family = AF_INET6; ++ sa6->sin6_addr = bind->sa.in6.sin6_addr; ++ sa6->sin6_port = bind->sa.in6.sin6_port; ++ salen = sizeof(*sa6); ++ break; ++ default: ++ ret = -EPROTONOSUPPORT; ++ goto unlock; ++ } ++ ++ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &sa, salen); ++ hlist_add_head_rcu(&peer->hash_entry_transp_addr, ++ &ovpn->peers.by_transp_addr[index]); ++ } ++ ++ index = ovpn_peer_index(ovpn->peers.by_id, &peer->id, sizeof(peer->id)); ++ hlist_add_head_rcu(&peer->hash_entry_id, &ovpn->peers.by_id[index]); ++ ++ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv4, ++ sizeof(peer->vpn_addrs.ipv4)); ++ hlist_add_head_rcu(&peer->hash_entry_addr4, &ovpn->peers.by_vpn_addr[index]); ++ } ++ ++ hlist_del_init_rcu(&peer->hash_entry_addr6); ++ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) { ++ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv6, ++ sizeof(peer->vpn_addrs.ipv6)); ++ hlist_add_head_rcu(&peer->hash_entry_addr6, &ovpn->peers.by_vpn_addr[index]); ++ } + -+ total_ram_pages = totalram_pages(); -+ free_pages = global_zone_page_state(NR_FREE_PAGES); -+ threshold = READ_ONCE(ksm_auto_threshold); ++unlock: ++ spin_unlock_bh(&ovpn->peers.lock); + -+ return free_pages > (total_ram_pages * threshold / 100) + -+ (total_ram_pages >> RIGHT_SHIFT_FOUR_BIT); ++ return ret; +} + -+/* Work in auto mode, should ksmd start to merge ? */ -+static bool should_trigger_ksmd_to_merge(void) ++static int ovpn_peer_add_p2p(struct ovpn_struct *ovpn, struct ovpn_peer *peer) +{ -+ unsigned long total_ram_pages, free_pages; -+ unsigned int threshold; ++ struct ovpn_peer *tmp; + -+ total_ram_pages = totalram_pages(); -+ free_pages = global_zone_page_state(NR_FREE_PAGES); -+ threshold = READ_ONCE(ksm_auto_threshold); ++ spin_lock_bh(&ovpn->lock); ++ /* in p2p mode it is possible to have a single peer only, therefore the ++ * old one is released and substituted by the new one ++ */ ++ tmp = rcu_dereference(ovpn->peer); ++ if (tmp) { ++ tmp->delete_reason = OVPN_DEL_PEER_REASON_TEARDOWN; ++ ovpn_peer_put(tmp); ++ } + -+ return free_pages < (total_ram_pages * threshold / 100); -+} ++ rcu_assign_pointer(ovpn->peer, peer); ++ spin_unlock_bh(&ovpn->lock); + -+static inline void trigger_ksmd_to_merge(void) -+{ -+ if (!auto_triggered) { -+ triggered_times++; -+ auto_triggered = true; -+ } ++ return 0; +} + -+static inline void stop_ksmd_to_merge(void) ++/* assume refcounter was increased by caller */ ++int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer) +{ -+ if (auto_triggered) -+ auto_triggered = false; ++ switch (ovpn->mode) { ++ case OVPN_MODE_MP: ++ return ovpn_peer_add_mp(ovpn, peer); ++ case OVPN_MODE_P2P: ++ return ovpn_peer_add_p2p(ovpn, peer); ++ default: ++ return -EOPNOTSUPP; ++ } +} + - static int ksmd_should_run(void) - { -- return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list); -+ if (!list_empty(&ksm_mm_head.mm_list)) -+ return ksm_run & KSM_RUN_AUTO || ksm_run & KSM_RUN_MERGE; -+ return 0; - } - -+/* -+ * Work in auto mode, the scan-enhanced algorithm. -+ * current_factor: the current scanning_factor. -+ * return: the scanning_factor caculated by scan-enhanced algorithm. -+ */ -+static unsigned int scan_enhanced_algorithm(unsigned int current_factor) ++static void ovpn_peer_unhash(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ -+ unsigned int next_factor; -+ unsigned int max, min; -+ -+ /* -+ * The calculation is divied into three cases as follows: -+ * -+ * Case 1: when new_ksmpages > prev_ksmpages * 1/2, get the -+ * next factor by double the current factor. -+ * Case 2: when 0 < new_ksmpages < prev_ksmpages * 1/2, keep -+ * the factor unchanged. -+ * Case 3: when new_ksmpages equals 0, then get the next -+ * factor by halfing the current factor. -+ */ -+ max = READ_ONCE(max_scanning_factor); -+ min = INIT_SCANNING_FACTOR; -+ if (ksm_scan.new_ksmpages * 2 > ksm_scan.prev_ksmpages) { -+ next_factor = current_factor << 1; /* Doubling */ -+ if (next_factor > max) -+ next_factor = max; -+ } else if (ksm_scan.new_ksmpages == 0) { -+ next_factor = current_factor >> 1; /* Halfing */ -+ next_factor = next_factor < min ? min : next_factor; -+ } else -+ next_factor = current_factor; ++ hlist_del_init_rcu(&peer->hash_entry_id); ++ hlist_del_init_rcu(&peer->hash_entry_addr4); ++ hlist_del_init_rcu(&peer->hash_entry_addr6); ++ hlist_del_init_rcu(&peer->hash_entry_transp_addr); + -+ return next_factor; ++ ovpn_peer_put(peer); ++ peer->delete_reason = reason; +} + -+#define SLOW_SCAN_PAGES 5 /* Used when ksmd is not triggered to merge*/ -+ - static int ksm_scan_thread(void *nothing) - { - unsigned int sleep_ms; -@@ -2419,17 +2548,41 @@ static int ksm_scan_thread(void *nothing) - while (!kthread_should_stop()) { - mutex_lock(&ksm_thread_mutex); - wait_while_offlining(); -- if (ksmd_should_run()) -- ksm_do_scan(ksm_thread_pages_to_scan); -+ if (ksmd_should_run()) { -+ if (ksm_run & KSM_RUN_AUTO) { -+ if (!auto_triggered) -+ ksm_do_scan(SLOW_SCAN_PAGES); -+ else -+ ksm_do_scan(ksm_thread_pages_to_scan * scanning_factor); -+ -+ scanning_factor = scan_enhanced_algorithm(scanning_factor); -+ /* -+ * Reset ksm_scan.new_ksmpages after -+ * updating scanning_factor by scan_enhanced_algorithm. -+ */ -+ ksm_scan.new_ksmpages = 0; -+ -+ if (should_trigger_ksmd_to_merge()) -+ trigger_ksmd_to_merge(); -+ else if (should_stop_ksmd_to_merge()) -+ stop_ksmd_to_merge(); -+ } else -+ ksm_do_scan(ksm_thread_pages_to_scan); -+ } - mutex_unlock(&ksm_thread_mutex); - - try_to_freeze(); - - if (ksmd_should_run()) { - sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs); -- wait_event_interruptible_timeout(ksm_iter_wait, -- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -- msecs_to_jiffies(sleep_ms)); -+ if (sleep_ms >= 1000) -+ wait_event_interruptible_timeout(ksm_iter_wait, -+ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -+ msecs_to_jiffies(round_jiffies_relative(sleep_ms))); -+ else -+ wait_event_interruptible_timeout(ksm_iter_wait, -+ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs), -+ msecs_to_jiffies(sleep_ms)); - } else { - wait_event_freezable(ksm_thread_wait, - ksmd_should_run() || kthread_should_stop()); -@@ -2438,54 +2591,78 @@ static int ksm_scan_thread(void *nothing) - return 0; - } - --int ksm_madvise(struct vm_area_struct *vma, unsigned long start, -- unsigned long end, int advice, unsigned long *vm_flags) -+int ksm_madvise_merge(struct mm_struct *mm, struct vm_area_struct *vma, -+ unsigned long *vm_flags) - { -- struct mm_struct *mm = vma->vm_mm; - int err; - -- switch (advice) { -- case MADV_MERGEABLE: -- /* -- * Be somewhat over-protective for now! -- */ -- if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | -- VM_PFNMAP | VM_IO | VM_DONTEXPAND | -- VM_HUGETLB | VM_MIXEDMAP)) -- return 0; /* just ignore the advice */ -+ /* -+ * Be somewhat over-protective for now! -+ */ -+ if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | -+ VM_PFNMAP | VM_IO | VM_DONTEXPAND | -+ VM_HUGETLB | VM_MIXEDMAP)) -+ return 0; /* just ignore the advice */ - -- if (vma_is_dax(vma)) -- return 0; -+ if (vma_is_dax(vma)) -+ return 0; - - #ifdef VM_SAO - if (*vm_flags & VM_SAO) - return 0; - #endif - #ifdef VM_SPARC_ADI -- if (*vm_flags & VM_SPARC_ADI) -- return 0; -+ if (*vm_flags & VM_SPARC_ADI) -+ return 0; - #endif - -- if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { -- err = __ksm_enter(mm); -- if (err) -- return err; -- } -+ if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { -+ err = __ksm_enter(mm); -+ if (err) -+ return err; -+ } - -- *vm_flags |= VM_MERGEABLE; -- break; -+ *vm_flags |= VM_MERGEABLE; - -- case MADV_UNMERGEABLE: -- if (!(*vm_flags & VM_MERGEABLE)) -- return 0; /* just ignore the advice */ -+ return 0; -+} - -- if (vma->anon_vma) { -- err = unmerge_ksm_pages(vma, start, end); -- if (err) -- return err; -- } -+int ksm_madvise_unmerge(struct vm_area_struct *vma, unsigned long start, -+ unsigned long end, unsigned long *vm_flags) ++static int ovpn_peer_del_mp(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ -+ int err; -+ -+ if (!(*vm_flags & VM_MERGEABLE)) -+ return 0; /* just ignore the advice */ ++ struct ovpn_peer *tmp; ++ int ret = 0; + -+ if (vma->anon_vma) { -+ err = unmerge_ksm_pages(vma, start, end); -+ if (err) -+ return err; ++ spin_lock_bh(&peer->ovpn->peers.lock); ++ tmp = ovpn_peer_lookup_id(peer->ovpn, peer->id); ++ if (tmp != peer) { ++ ret = -ENOENT; ++ goto unlock; + } ++ ovpn_peer_unhash(peer, reason); + -+ *vm_flags &= ~VM_MERGEABLE; -+ -+ return 0; -+} ++unlock: ++ spin_unlock_bh(&peer->ovpn->peers.lock); + -+int ksm_madvise(struct vm_area_struct *vma, unsigned long start, -+ unsigned long end, int advice, unsigned long *vm_flags) -+{ -+ struct mm_struct *mm = vma->vm_mm; -+ int err; - -- *vm_flags &= ~VM_MERGEABLE; -+ switch (advice) { -+ case MADV_MERGEABLE: -+ err = ksm_madvise_merge(mm, vma, vm_flags); -+ if (err) -+ return err; -+ break; ++ if (tmp) ++ ovpn_peer_put(tmp); + -+ case MADV_UNMERGEABLE: -+ err = ksm_madvise_unmerge(vma, start, end, vm_flags); -+ if (err) -+ return err; - break; - } - -@@ -2891,6 +3068,34 @@ static ssize_t pages_to_scan_store(struct kobject *kobj, - } - KSM_ATTR(pages_to_scan); - -+static ssize_t max_scanning_factor_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sysfs_emit(buf, "%u\n", max_scanning_factor); ++ return ret; +} + -+static ssize_t max_scanning_factor_store(struct kobject *kobj, -+ struct kobj_attribute *attr, -+ const char *buf, size_t count) ++static int ovpn_peer_del_p2p(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ -+ unsigned int value, max; -+ int err; -+ -+ err = kstrtouint(buf, 10, &value); -+ if (err) -+ return -EINVAL; ++ struct ovpn_peer *tmp; ++ int ret = -ENOENT; + -+ max = totalram_pages() / ksm_thread_pages_to_scan; ++ spin_lock_bh(&peer->ovpn->lock); ++ tmp = rcu_dereference(peer->ovpn->peer); ++ if (tmp != peer) ++ goto unlock; + -+ if (value < 1 && value > max) -+ return -EINVAL; ++ ovpn_peer_put(tmp); ++ tmp->delete_reason = reason; ++ RCU_INIT_POINTER(peer->ovpn->peer, NULL); ++ ret = 0; + -+ max_scanning_factor = value; ++unlock: ++ spin_unlock_bh(&peer->ovpn->lock); + -+ return count; ++ return ret; +} -+KSM_ATTR(max_scanning_factor); + - static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr, - char *buf) - { -@@ -2906,7 +3111,7 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, - err = kstrtouint(buf, 10, &flags); - if (err) - return -EINVAL; -- if (flags > KSM_RUN_UNMERGE) -+ if (flags > KSM_RUN_UNMERGE && flags != KSM_RUN_AUTO) - return -EINVAL; - - /* -@@ -2932,13 +3137,73 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, - } - mutex_unlock(&ksm_thread_mutex); - -- if (flags & KSM_RUN_MERGE) -+ if (flags & KSM_RUN_MERGE || flags & KSM_RUN_AUTO) - wake_up_interruptible(&ksm_thread_wait); - - return count; - } - KSM_ATTR(run); - -+static ssize_t ksmd_status_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) ++void ovpn_peer_release_p2p(struct ovpn_struct *ovpn) +{ -+ int len = 0; -+ unsigned int mergeable_mms = 0; -+ struct list_head *pos; -+ -+ list_for_each(pos, &ksm_mm_head.mm_list) -+ mergeable_mms++; -+ -+ if (ksm_run & KSM_RUN_AUTO) { -+ len += sysfs_emit_at(buf, len, "mode: auto\n"); -+ len += sysfs_emit_at(buf, len, "auto_triggered: %d\n", -+ auto_triggered); -+ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", -+ mergeable_mms); -+ len += sysfs_emit_at(buf, len, "scanning_factor: %u\n", -+ scanning_factor); -+ len += sysfs_emit_at(buf, len, "triggered_times: %lu\n", -+ triggered_times); -+ } else if (ksm_run & KSM_RUN_MERGE) { -+ len += sysfs_emit_at(buf, len, "mode: on\n"); -+ len += sysfs_emit_at(buf, len, "mergeable_mms: %u\n", -+ mergeable_mms); -+ } else if (ksm_run & KSM_RUN_UNMERGE) -+ len += sysfs_emit_at(buf, len, "mode: unmerge\n"); -+ else -+ len += sysfs_emit_at(buf, len, "mode: off\n"); ++ struct ovpn_peer *tmp; + ++ rcu_read_lock(); ++ tmp = rcu_dereference(ovpn->peer); ++ if (!tmp) ++ goto unlock; + -+ return len; ++ ovpn_peer_del_p2p(tmp, OVPN_DEL_PEER_REASON_TEARDOWN); ++unlock: ++ rcu_read_unlock(); +} -+KSM_ATTR_RO(ksmd_status); + -+static ssize_t auto_threshold_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) ++int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) +{ -+ return sysfs_emit(buf, "%u\n", ksm_auto_threshold); ++ switch (peer->ovpn->mode) { ++ case OVPN_MODE_MP: ++ return ovpn_peer_del_mp(peer, reason); ++ case OVPN_MODE_P2P: ++ return ovpn_peer_del_p2p(peer, reason); ++ default: ++ return -EOPNOTSUPP; ++ } +} + -+static ssize_t auto_threshold_store(struct kobject *kobj, -+ struct kobj_attribute *attr, -+ const char *buf, size_t count) ++void ovpn_peers_free(struct ovpn_struct *ovpn) +{ -+ unsigned int value; -+ int err; ++ struct hlist_node *tmp; ++ struct ovpn_peer *peer; ++ int bkt; + -+ err = kstrtouint(buf, 10, &value); -+ if (err) -+ return -EINVAL; ++ spin_lock_bh(&ovpn->peers.lock); ++ hash_for_each_safe(ovpn->peers.by_id, bkt, tmp, peer, hash_entry_id) ++ ovpn_peer_unhash(peer, OVPN_DEL_PEER_REASON_TEARDOWN); ++ spin_unlock_bh(&ovpn->peers.lock); ++} +diff --git a/drivers/net/ovpn-dco/peer.h b/drivers/net/ovpn-dco/peer.h +new file mode 100644 +index 000000000000..b759c2c9da48 +--- /dev/null ++++ b/drivers/net/ovpn-dco/peer.h +@@ -0,0 +1,168 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+ if (value > 100) -+ return -EINVAL; ++#ifndef _NET_OVPN_DCO_OVPNPEER_H_ ++#define _NET_OVPN_DCO_OVPNPEER_H_ + -+ ksm_auto_threshold = value; ++#include "addr.h" ++#include "bind.h" ++#include "sock.h" ++#include "stats.h" + -+ return count; -+} -+KSM_ATTR(auto_threshold); ++#include ++#include ++#include + - #ifdef CONFIG_NUMA - static ssize_t merge_across_nodes_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -@@ -3148,7 +3413,10 @@ KSM_ATTR_RO(full_scans); - static struct attribute *ksm_attrs[] = { - &sleep_millisecs_attr.attr, - &pages_to_scan_attr.attr, -+ &max_scanning_factor_attr.attr, - &run_attr.attr, -+ &ksmd_status_attr.attr, -+ &auto_threshold_attr.attr, - &pages_shared_attr.attr, - &pages_sharing_attr.attr, - &pages_unshared_attr.attr, -@@ -3180,6 +3448,7 @@ static int __init ksm_init(void) - zero_checksum = calc_checksum(ZERO_PAGE(0)); - /* Default to false for backwards compatibility */ - ksm_use_zero_pages = false; -+ auto_triggered = false; - - err = ksm_slab_init(); - if (err) -diff --git a/mm/madvise.c b/mm/madvise.c -index 5f0f0948a50e..82f6ea8c493d 100644 ---- a/mm/madvise.c -+++ b/mm/madvise.c -@@ -1173,6 +1173,10 @@ process_madvise_behavior_valid(int behavior) - case MADV_COLD: - case MADV_PAGEOUT: - case MADV_WILLNEED: -+#ifdef CONFIG_KSM -+ case MADV_MERGEABLE: -+ case MADV_UNMERGEABLE: -+#endif - return true; - default: - return false; -@@ -1493,3 +1497,114 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, - out: - return ret; - } ++struct ovpn_peer { ++ struct ovpn_struct *ovpn; + -+SYSCALL_DEFINE3(pmadv_ksm, int, pidfd, int, behaviour, unsigned int, flags) -+{ -+#ifdef CONFIG_KSM -+ ssize_t ret; -+ struct pid *pid; -+ struct task_struct *task; -+ struct mm_struct *mm; -+ unsigned int f_flags; -+ struct vm_area_struct *vma; ++ u32 id; + -+ if (flags != 0) { -+ ret = -EINVAL; -+ goto out; -+ } ++ struct { ++ struct in_addr ipv4; ++ struct in6_addr ipv6; ++ } vpn_addrs; + -+ switch (behaviour) { -+ case MADV_MERGEABLE: -+ case MADV_UNMERGEABLE: -+ break; -+ default: -+ ret = -EINVAL; -+ goto out; -+ break; -+ } ++ struct hlist_node hash_entry_id; ++ struct hlist_node hash_entry_addr4; ++ struct hlist_node hash_entry_addr6; ++ struct hlist_node hash_entry_transp_addr; + -+ pid = pidfd_get_pid(pidfd, &f_flags); -+ if (IS_ERR(pid)) { -+ ret = PTR_ERR(pid); -+ goto out; -+ } ++ /* work objects to handle encryption/decryption of packets. ++ * these works are queued on the ovpn->crypt_wq workqueue. ++ */ ++ struct work_struct encrypt_work; ++ struct work_struct decrypt_work; + -+ task = get_pid_task(pid, PIDTYPE_PID); -+ if (!task) { -+ ret = -ESRCH; -+ goto put_pid; -+ } ++ struct ptr_ring tx_ring; ++ struct ptr_ring rx_ring; ++ struct ptr_ring netif_rx_ring; + -+ /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */ -+ mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); -+ if (IS_ERR_OR_NULL(mm)) { -+ ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH; -+ goto release_task; -+ } ++ struct napi_struct napi; + -+ /* Require CAP_SYS_NICE for influencing process performance. */ -+ if (!capable(CAP_SYS_NICE)) { -+ ret = -EPERM; -+ goto release_mm; -+ } ++ struct ovpn_socket *sock; + -+ if (mmap_write_lock_killable(mm)) { -+ ret = -EINTR; -+ goto release_mm; -+ } ++ /* state of the TCP reading. Needed to keep track of how much of a single packet has already ++ * been read from the stream and how much is missing ++ */ ++ struct { ++ struct ptr_ring tx_ring; ++ struct work_struct tx_work; ++ struct work_struct rx_work; + -+ for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ switch (behaviour) { -+ case MADV_MERGEABLE: -+ ret = ksm_madvise_merge(vma->vm_mm, vma, &vma->vm_flags); -+ break; -+ case MADV_UNMERGEABLE: -+ ret = ksm_madvise_unmerge(vma, vma->vm_start, vma->vm_end, &vma->vm_flags); -+ break; -+ default: -+ /* look, ma, no brain */ -+ break; -+ } -+ if (ret) -+ break; -+ } ++ u8 raw_len[sizeof(u16)]; ++ struct sk_buff *skb; ++ u16 offset; ++ u16 data_len; ++ struct { ++ void (*sk_state_change)(struct sock *sk); ++ void (*sk_data_ready)(struct sock *sk); ++ void (*sk_write_space)(struct sock *sk); ++ } sk_cb; ++ } tcp; + -+ mmap_write_unlock(mm); ++ struct dst_cache dst_cache; + -+release_mm: -+ mmput(mm); -+release_task: -+ put_task_struct(task); -+put_pid: -+ put_pid(pid); -+out: -+ return ret; -+#else /* CONFIG_KSM */ -+ return -ENOSYS; -+#endif /* CONFIG_KSM */ -+} ++ /* our crypto state */ ++ struct ovpn_crypto_state crypto; + -+#ifdef CONFIG_KSM -+static ssize_t ksm_show(struct kobject *kobj, struct kobj_attribute *attr, -+ char *buf) -+{ -+ return sprintf(buf, "%u\n", __NR_pmadv_ksm); -+} -+static struct kobj_attribute pmadv_ksm_attr = __ATTR_RO(ksm); ++ /* our binding to peer, protected by spinlock */ ++ struct ovpn_bind __rcu *bind; + -+static struct attribute *pmadv_sysfs_attrs[] = { -+ &pmadv_ksm_attr.attr, -+ NULL, -+}; ++ /* timer used to send periodic ping messages to the other peer, if no ++ * other data was sent within the past keepalive_interval seconds ++ */ ++ struct timer_list keepalive_xmit; ++ /* keepalive interval in seconds */ ++ unsigned long keepalive_interval; + -+static const struct attribute_group pmadv_sysfs_attr_group = { -+ .attrs = pmadv_sysfs_attrs, -+ .name = "pmadv", -+}; ++ /* timer used to mark a peer as expired when no data is received for ++ * keepalive_timeout seconds ++ */ ++ struct timer_list keepalive_recv; ++ /* keepalive timeout in seconds */ ++ unsigned long keepalive_timeout; + -+static int __init pmadv_sysfs_init(void) -+{ -+ return sysfs_create_group(kernel_kobj, &pmadv_sysfs_attr_group); -+} -+subsys_initcall(pmadv_sysfs_init); -+#endif /* CONFIG_KSM */ -diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index b69979c9ced5..7eadbafc006b 100644 ---- a/mm/memcontrol.c -+++ b/mm/memcontrol.c -@@ -625,7 +625,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) - cgroup_rstat_updated(memcg->css.cgroup, smp_processor_id()); - - x = __this_cpu_add_return(stats_updates, abs(val)); -- if (x > MEMCG_CHARGE_BATCH) { -+ if (x > MEMCG_CHARGE_BATCH * 128) { - /* - * If stats_flush_threshold exceeds the threshold - * (>num_online_cpus()), cgroup stats update will be triggered -diff --git a/mm/page-writeback.c b/mm/page-writeback.c -index 032a7bf8d259..cbcecd565c16 100644 ---- a/mm/page-writeback.c -+++ b/mm/page-writeback.c -@@ -70,7 +70,11 @@ static long ratelimit_pages = 32; - /* - * Start background writeback (via writeback threads) at this percentage - */ -+#ifdef CONFIG_CACHY -+static int dirty_background_ratio = 5; -+#else - static int dirty_background_ratio = 10; -+#endif - - /* - * dirty_background_bytes starts at 0 (disabled) so that it is a function of -@@ -98,7 +102,11 @@ static unsigned long vm_dirty_bytes; - /* - * The interval between `kupdate'-style writebacks - */ -+#ifdef CONFIG_CACHY -+unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */ -+#else - unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ -+#endif - - EXPORT_SYMBOL_GPL(dirty_writeback_interval); - -diff --git a/mm/page_alloc.c b/mm/page_alloc.c -index e5486d47406e..cf131d6e08fb 100644 ---- a/mm/page_alloc.c -+++ b/mm/page_alloc.c -@@ -6982,11 +6982,11 @@ static int zone_batchsize(struct zone *zone) - - /* - * The number of pages to batch allocate is either ~0.1% -- * of the zone or 1MB, whichever is smaller. The batch -+ * of the zone or 4MB, whichever is smaller. The batch - * size is striking a balance between allocation latency - * and zone lock contention. - */ -- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); -+ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE); - batch /= 4; /* We effectively *= 4 below */ - if (batch < 1) - batch = 1; -@@ -7064,6 +7064,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online) - * historical relationship between high and batch. - */ - high = max(high, batch << 2); -+ high = max(high, 1024); - - return high; - #else -diff --git a/mm/swap.c b/mm/swap.c -index 9cee7f6a3809..186b4e5dcecf 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -1070,6 +1070,10 @@ EXPORT_SYMBOL(pagevec_lookup_range_tag); - */ - void __init swap_setup(void) - { -+#ifdef CONFIG_CACHY -+ /* Only swap-in pages requested, avoid readahead */ -+ page_cluster = 0; -+#else - unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); - - /* Use a smaller cluster for small-memory machines */ -@@ -1081,4 +1085,5 @@ void __init swap_setup(void) - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ -+#endif - } -diff --git a/mm/vmpressure.c b/mm/vmpressure.c -index b52644771cc4..11a4b0e3b583 100644 ---- a/mm/vmpressure.c -+++ b/mm/vmpressure.c -@@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; - * essence, they are percents: the higher the value, the more number - * unsuccessful reclaims there were. - */ -+#ifdef CONFIG_CACHY -+static const unsigned int vmpressure_level_med = 65; -+#else - static const unsigned int vmpressure_level_med = 60; -+#endif - static const unsigned int vmpressure_level_critical = 95; - - /* -diff --git a/mm/vmscan.c b/mm/vmscan.c -index b2b1431352dc..0fc65ace3a4e 100644 ---- a/mm/vmscan.c -+++ b/mm/vmscan.c -@@ -178,7 +178,11 @@ struct scan_control { - /* - * From 0 .. 200. Higher means more swappy. - */ -+#ifdef CONFIG_CACHY -+int vm_swappiness = 20; -+#else - int vm_swappiness = 60; -+#endif - - static void set_task_reclaim_state(struct task_struct *task, - struct reclaim_state *rs) -diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c -index eb31c7158b39..8cefa0b62fb9 100644 ---- a/net/ipv4/inet_connection_sock.c -+++ b/net/ipv4/inet_connection_sock.c -@@ -445,7 +445,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) - * having to remove and re-insert us on the wait queue. - */ - for (;;) { -- prepare_to_wait_exclusive(sk_sleep(sk), &wait, -+ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, - TASK_INTERRUPTIBLE); - release_sock(sk); - if (reqsk_queue_empty(&icsk->icsk_accept_queue)) -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index b735776c0225..2fcd440df1f6 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -4790,8 +4790,8 @@ void __init tcp_init(void) - tcp_init_mem(); - /* Set per-socket limits to no more than 1/128 the pressure threshold */ - limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7); -- max_wshare = min(4UL*1024*1024, limit); -- max_rshare = min(6UL*1024*1024, limit); -+ max_wshare = min(16UL*1024*1024, limit); -+ max_rshare = min(16UL*1024*1024, limit); - - init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE; - init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024; -diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile -index 0470c5f3e690..a1fa2eff8192 100644 ---- a/tools/testing/selftests/damon/Makefile -+++ b/tools/testing/selftests/damon/Makefile -@@ -6,6 +6,7 @@ TEST_GEN_FILES += huge_count_read_write - TEST_FILES = _chk_dependency.sh _debugfs_common.sh - TEST_PROGS = debugfs_attrs.sh debugfs_schemes.sh debugfs_target_ids.sh - TEST_PROGS += debugfs_empty_targets.sh debugfs_huge_count_read_write.sh -+TEST_PROGS += debugfs_duplicate_context_creation.sh - TEST_PROGS += sysfs.sh - - include ../lib.mk -diff --git a/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh b/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh -new file mode 100644 -index 000000000000..4a76e37ef16b ---- /dev/null -+++ b/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh -@@ -0,0 +1,27 @@ -+#!/bin/bash -+# SPDX-License-Identifier: GPL-2.0 ++ /* true if ovpn_peer_mark_delete was called */ ++ bool halt; + -+source _debugfs_common.sh -+ -+# Test duplicated context creation -+# ================================ -+ -+if ! echo foo > "$DBGFS/mk_contexts" -+then -+ echo "context creation failed" -+ exit 1 -+fi -+ -+if echo foo > "$DBGFS/mk_contexts" -+then -+ echo "duplicate context creation success" -+ exit 1 -+fi -+ -+if ! echo foo > "$DBGFS/rm_contexts" -+then -+ echo "context deletion failed" -+ exit 1 -+fi -+ -+exit 0 -diff --git a/tools/testing/selftests/vm/.gitignore b/tools/testing/selftests/vm/.gitignore -index 31e5eea2a9b9..1cd8816c055d 100644 ---- a/tools/testing/selftests/vm/.gitignore -+++ b/tools/testing/selftests/vm/.gitignore -@@ -34,3 +34,4 @@ local_config.* - soft-dirty - split_huge_page_test - ksm_tests -+test-ksm-auto -diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile -index d9fa6a9ea584..002d9482c37f 100644 ---- a/tools/testing/selftests/vm/Makefile -+++ b/tools/testing/selftests/vm/Makefile -@@ -54,6 +54,7 @@ TEST_GEN_FILES += userfaultfd - TEST_GEN_PROGS += soft-dirty - TEST_GEN_PROGS += split_huge_page_test - TEST_GEN_FILES += ksm_tests -+TEST_GEN_FILES += test-ksm-auto - - ifeq ($(MACHINE),x86_64) - CAN_BUILD_I386 := $(shell ./../x86/check_cc.sh "$(CC)" ../x86/trivial_32bit_program.c -m32) -diff --git a/tools/testing/selftests/vm/test-ksm-auto.c b/tools/testing/selftests/vm/test-ksm-auto.c -new file mode 100644 -index 000000000000..0d71593e008e ---- /dev/null -+++ b/tools/testing/selftests/vm/test-ksm-auto.c -@@ -0,0 +1,273 @@ -+// SPDX-License-Identifier: GPL-2.0 -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++ /* per-peer rx/tx stats */ ++ struct ovpn_peer_stats stats; ++ ++ /* why peer was deleted - keepalive timeout, module removed etc */ ++ enum ovpn_del_peer_reason delete_reason; ++ ++ /* protects binding to peer (bind) and timers ++ * (keepalive_xmit, keepalive_expire) ++ */ ++ spinlock_t lock; + -+#define KSM_CLEAR_MODE "2\n" -+#define KSM_NORMAL_MODE "1\n" -+#define KSM_AUTO_MODE "8\n" ++ /* needed because crypto methods can go async */ ++ struct kref refcount; + -+#define PAGESIZE (4*1024) -+/* Don't change the value, it will afffect the result */ -+#define TOTAL_MADVISE_SIZE (300*1024*1024) ++ /* needed to free a peer in an RCU safe way */ ++ struct rcu_head rcu; + -+char *ksm_run_file = "/sys/kernel/mm/ksm/run"; -+char *ksm_auto_threshold_file = "/sys/kernel/mm/ksm/auto_threshold"; -+char *ksm_pages_volatile_file = "/sys/kernel/mm/ksm/pages_volatile"; -+char *ksm_pages_sharing_file = "/sys/kernel/mm/ksm/pages_sharing"; ++ /* needed to notify userspace about deletion */ ++ struct work_struct delete_work; ++}; + -+#define SHAPE_FULL 1 -+#define SHAPE_SPARSE 2 -+/* They are related to the shape of memory */ -+int final_pages[3] = {0, 76500, 42}; ++void ovpn_peer_release_kref(struct kref *kref); ++void ovpn_peer_release(struct ovpn_peer *peer); + -+static char *mmap_and_madvise(long long size, int advise) ++static inline bool ovpn_peer_hold(struct ovpn_peer *peer) +{ -+ char *ptr; -+ int err; -+ -+ err = 0; -+ -+ ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); -+ if (!ptr) -+ return NULL; -+ -+ err = madvise(ptr, size, advise); -+ if (err) { -+ perror("Madvise failed\n"); -+ free(ptr); -+ return NULL; -+ } ++ return kref_get_unless_zero(&peer->refcount); ++} + -+ return ptr; ++static inline void ovpn_peer_put(struct ovpn_peer *peer) ++{ ++ kref_put(&peer->refcount, ovpn_peer_release_kref); +} + -+void make_samepage_ares(char *ptr, int size, int shape_type) ++static inline void ovpn_peer_keepalive_recv_reset(struct ovpn_peer *peer) +{ -+ int i, j; -+ char rnd_num; ++ u32 delta = msecs_to_jiffies(peer->keepalive_timeout * MSEC_PER_SEC); + -+ switch (shape_type) { -+ case SHAPE_FULL: -+ for (i = 0; i < (size / PAGESIZE); i++) -+ memset(ptr + (i * PAGESIZE), 0x1, PAGESIZE); -+ break; -+ case SHAPE_SPARSE: -+ /* Make pages different */ -+ j = 0; -+ for (i = 1; i < (size / PAGESIZE); i++) { -+ ptr[i * PAGESIZE + (j%PAGESIZE)] = j%127 + 1; -+ j++; -+ } -+ for (i = 0; i < (size / PAGESIZE); i += 1800) -+ memset(ptr + (i * PAGESIZE), -1, PAGESIZE); -+ } ++ if (unlikely(!delta)) ++ return; + -+ return; ++ mod_timer(&peer->keepalive_recv, jiffies + delta); +} + -+int read_file(char *file, char *buffer, int buf_len) ++static inline void ovpn_peer_keepalive_xmit_reset(struct ovpn_peer *peer) +{ -+ FILE *fp; -+ size_t result; -+ long lSize; ++ u32 delta = msecs_to_jiffies(peer->keepalive_interval * MSEC_PER_SEC); + -+ fp = fopen(file, "r"); -+ if (!fp) -+ return -1; ++ if (unlikely(!delta)) ++ return; + -+ fseek(fp, 0, SEEK_END); -+ lSize = ftell(fp); -+ rewind(fp); ++ mod_timer(&peer->keepalive_xmit, jiffies + delta); ++} + -+ memset(buffer, 0, buf_len); -+ result = fread(buffer, 1, buf_len, fp); -+ if (result == 0) -+ return -1; ++struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, ++ struct socket *sock, u32 id, uint8_t *local_ip); + -+ fclose(fp); ++void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout); + -+ return 0; -+} ++int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer); ++int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason); ++struct ovpn_peer *ovpn_peer_find(struct ovpn_struct *ovpn, u32 peer_id); ++void ovpn_peer_release_p2p(struct ovpn_struct *ovpn); ++void ovpn_peers_free(struct ovpn_struct *ovpn); + -+int write_file(char *file, const char *buffer, int len) -+{ -+ FILE *fp; -+ size_t result; ++struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb); ++struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, ++ bool use_src); ++struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id); + -+ fp = fopen(file, "w+"); -+ if (!fp) -+ return -1; ++void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb); ++void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb); + -+ result = fwrite(buffer, len, 1, fp); -+ if (result == 0) -+ return -1; ++#endif /* _NET_OVPN_DCO_OVPNPEER_H_ */ +diff --git a/drivers/net/ovpn-dco/pktid.c b/drivers/net/ovpn-dco/pktid.c +new file mode 100644 +index 000000000000..fcde8fba5156 +--- /dev/null ++++ b/drivers/net/ovpn-dco/pktid.c +@@ -0,0 +1,127 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ + -+ fclose(fp); ++#include "pktid.h" + -+ return 0; -+} ++#include ++#include + -+static inline void get_orig_info(int *run, int *auto_threshold) ++void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid) +{ -+ char buffer[50]; -+ -+ /* Read the original state of ksm/run */ -+ if (read_file(ksm_run_file, buffer, sizeof(buffer))) { -+ printf("read file %s failed\n", ksm_run_file); -+ exit(1); -+ } -+ *run = atoi(buffer); ++ atomic64_set(&pid->seq_num, 1); ++ pid->tcp_linear = NULL; ++} + -+ if (read_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { -+ printf("read file: %s failed\n", ksm_auto_threshold_file); -+ exit(1); -+ } -+ *auto_threshold = atoi(buffer); ++void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr) ++{ ++ memset(pr, 0, sizeof(*pr)); ++ spin_lock_init(&pr->lock); +} + -+static inline void restore_orig_state(int run, int auto_threshold) ++/* Packet replay detection. ++ * Allows ID backtrack of up to REPLAY_WINDOW_SIZE - 1. ++ */ ++int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time) +{ -+ char buffer[50]; ++ const unsigned long now = jiffies; ++ int ret; + -+ /* restore the original state */ -+ memset(buffer, 0, sizeof(buffer)); -+ snprintf(buffer, sizeof(buffer) - 1, "%d\n", run); -+ if (write_file(ksm_run_file, buffer, sizeof(buffer))) { -+ printf("write file %s failed\n", ksm_run_file); -+ exit(1); -+ } ++ spin_lock(&pr->lock); ++ ++ /* expire backtracks at or below pr->id after PKTID_RECV_EXPIRE time */ ++ if (unlikely(time_after_eq(now, pr->expire))) ++ pr->id_floor = pr->id; + -+ memset(buffer, 0, sizeof(buffer)); -+ snprintf(buffer, sizeof(buffer) - 1, "%d\n", auto_threshold); -+ if (write_file(ksm_auto_threshold_file, buffer, sizeof(buffer))) { -+ printf("write file %s failed\n", ksm_run_file); -+ exit(1); ++ /* ID must not be zero */ ++ if (unlikely(pkt_id == 0)) { ++ ret = -EINVAL; ++ goto out; + } -+} + -+void set_ksmd_run_mode(char *mode) -+{ -+ if (write_file(ksm_run_file, mode, 2)) { -+ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); -+ exit(1); ++ /* time changed? */ ++ if (unlikely(pkt_time != pr->time)) { ++ if (pkt_time > pr->time) { ++ /* time moved forward, accept */ ++ pr->base = 0; ++ pr->extent = 0; ++ pr->id = 0; ++ pr->time = pkt_time; ++ pr->id_floor = 0; ++ } else { ++ /* time moved backward, reject */ ++ ret = -ETIME; ++ goto out; ++ } + } -+} + -+static inline void wait_ksmpages_converged(int final_pages) -+{ -+ int pages_sharing; -+ char buffer[50]; ++ if (likely(pkt_id == pr->id + 1)) { ++ /* well-formed ID sequence (incremented by 1) */ ++ pr->base = REPLAY_INDEX(pr->base, -1); ++ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); ++ if (pr->extent < REPLAY_WINDOW_SIZE) ++ ++pr->extent; ++ pr->id = pkt_id; ++ } else if (pkt_id > pr->id) { ++ /* ID jumped forward by more than one */ ++ const unsigned int delta = pkt_id - pr->id; + -+ for (;;) { -+ if (read_file(ksm_pages_sharing_file, buffer, sizeof(buffer))) { -+ printf("read file %s failed\n", ksm_pages_sharing_file); -+ exit(1); ++ if (delta < REPLAY_WINDOW_SIZE) { ++ unsigned int i; ++ ++ pr->base = REPLAY_INDEX(pr->base, -delta); ++ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); ++ pr->extent += delta; ++ if (pr->extent > REPLAY_WINDOW_SIZE) ++ pr->extent = REPLAY_WINDOW_SIZE; ++ for (i = 1; i < delta; ++i) { ++ unsigned int newb = REPLAY_INDEX(pr->base, i); ++ ++ pr->history[newb / 8] &= ~BIT(newb % 8); ++ } ++ } else { ++ pr->base = 0; ++ pr->extent = REPLAY_WINDOW_SIZE; ++ memset(pr->history, 0, sizeof(pr->history)); ++ pr->history[0] = 1; + } ++ pr->id = pkt_id; ++ } else { ++ /* ID backtrack */ ++ const unsigned int delta = pr->id - pkt_id; + -+ pages_sharing = atoi(buffer); -+ if (pages_sharing >= final_pages) -+ break; ++ if (delta > pr->max_backtrack) ++ pr->max_backtrack = delta; ++ if (delta < pr->extent) { ++ if (pkt_id > pr->id_floor) { ++ const unsigned int ri = REPLAY_INDEX(pr->base, ++ delta); ++ u8 *p = &pr->history[ri / 8]; ++ const u8 mask = (1 << (ri % 8)); ++ ++ if (*p & mask) { ++ ret = -EINVAL; ++ goto out; ++ } ++ *p |= mask; ++ } else { ++ ret = -EINVAL; ++ goto out; ++ } ++ } else { ++ ret = -EINVAL; ++ goto out; ++ } + } ++ ++ pr->expire = now + PKTID_RECV_EXPIRE; ++ ret = 0; ++out: ++ spin_unlock(&pr->lock); ++ return ret; +} +diff --git a/drivers/net/ovpn-dco/pktid.h b/drivers/net/ovpn-dco/pktid.h +new file mode 100644 +index 000000000000..2447bb37ba55 +--- /dev/null ++++ b/drivers/net/ovpn-dco/pktid.h +@@ -0,0 +1,116 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ ++ ++#ifndef _NET_OVPN_DCO_OVPNPKTID_H_ ++#define _NET_OVPN_DCO_OVPNPKTID_H_ ++ ++#include "main.h" ++ ++/* When the OpenVPN protocol is run in AEAD mode, use ++ * the OpenVPN packet ID as the AEAD nonce: ++ * ++ * 00000005 521c3b01 4308c041 ++ * [seq # ] [ nonce_tail ] ++ * [ 12-byte full IV ] -> NONCE_SIZE ++ * [4-bytes -> NONCE_WIRE_SIZE ++ * on wire] ++ */ ++ ++/* OpenVPN nonce size */ ++#define NONCE_SIZE 12 ++/* amount of bytes of the nonce received from user space */ ++#define NONCE_TAIL_SIZE 8 ++ ++/* OpenVPN nonce size reduced by 8-byte nonce tail -- this is the ++ * size of the AEAD Associated Data (AD) sent over the wire ++ * and is normally the head of the IV ++ */ ++#define NONCE_WIRE_SIZE (NONCE_SIZE - sizeof(struct ovpn_nonce_tail)) ++ ++/* If no packets received for this length of time, set a backtrack floor ++ * at highest received packet ID thus far. ++ */ ++#define PKTID_RECV_EXPIRE (30 * HZ) ++ ++/* Last 8 bytes of AEAD nonce ++ * Provided by userspace and usually derived from ++ * key material generated during TLS handshake ++ */ ++struct ovpn_nonce_tail { ++ u8 u8[NONCE_TAIL_SIZE]; ++}; ++ ++/* Packet-ID state for transmitter */ ++struct ovpn_pktid_xmit { ++ atomic64_t seq_num; ++ struct ovpn_tcp_linear *tcp_linear; ++}; ++ ++/* replay window sizing in bytes = 2^REPLAY_WINDOW_ORDER */ ++#define REPLAY_WINDOW_ORDER 8 ++ ++#define REPLAY_WINDOW_BYTES BIT(REPLAY_WINDOW_ORDER) ++#define REPLAY_WINDOW_SIZE (REPLAY_WINDOW_BYTES * 8) ++#define REPLAY_INDEX(base, i) (((base) + (i)) & (REPLAY_WINDOW_SIZE - 1)) ++ ++/* Packet-ID state for receiver. ++ * Other than lock member, can be zeroed to initialize. ++ */ ++struct ovpn_pktid_recv { ++ /* "sliding window" bitmask of recent packet IDs received */ ++ u8 history[REPLAY_WINDOW_BYTES]; ++ /* bit position of deque base in history */ ++ unsigned int base; ++ /* extent (in bits) of deque in history */ ++ unsigned int extent; ++ /* expiration of history in jiffies */ ++ unsigned long expire; ++ /* highest sequence number received */ ++ u32 id; ++ /* highest time stamp received */ ++ u32 time; ++ /* we will only accept backtrack IDs > id_floor */ ++ u32 id_floor; ++ unsigned int max_backtrack; ++ /* protects entire pktd ID state */ ++ spinlock_t lock; ++}; + -+void print_shape(int shape_type) ++/* Get the next packet ID for xmit */ ++static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) +{ -+ switch (shape_type) { -+ case SHAPE_FULL: -+ printf("Now the shape of memory area is full-samepages:\n"); -+ printf("[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n\n"); -+ break; -+ case SHAPE_SPARSE: -+ printf("Now the shape of memory area is sparse-samepages:\n"); -+ printf("[xx] [xx] [xx] \n\n"); -+ break; -+ } ++ const s64 seq_num = atomic64_fetch_add_unless(&pid->seq_num, 1, ++ 0x100000000LL); ++ /* when the 32bit space is over, we return an error because the packet ID is used to create ++ * the cipher IV and we do not want to re-use the same value more than once ++ */ ++ if (unlikely(seq_num == 0x100000000LL)) ++ return -ERANGE; ++ ++ *pktid = (u32)seq_num; ++ ++ return 0; +} + -+void print_ksmd_cpu_comsuption(void) ++/* Write 12-byte AEAD IV to dest */ ++static inline void ovpn_pktid_aead_write(const u32 pktid, ++ const struct ovpn_nonce_tail *nt, ++ unsigned char *dest) +{ -+ system("(ps x| grep \"ksmd\" | grep -v grep | awk \'{print $1}\' |" -+ " xargs -i cat /proc/{}/stat) | awk \'{print \"ksm current " -+ "cpu total slice: \" $14+$15+$16+$17}\'"); ++ *(__force __be32 *)(dest) = htonl(pktid); ++ BUILD_BUG_ON(4 + sizeof(struct ovpn_nonce_tail) != NONCE_SIZE); ++ memcpy(dest + 4, nt->u8, sizeof(struct ovpn_nonce_tail)); +} + -+void test_ksmd_performance(char *madvise_area, int shape_type) -+{ -+ struct timeval tv_start, tv_end; ++void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid); ++void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr); + -+ make_samepage_ares(madvise_area, TOTAL_MADVISE_SIZE, shape_type); -+ print_shape(shape_type); ++int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time); + -+ /********* Start to time ksmd's normal-run mode **********/ -+ printf("Start to test normal-run ksmd...\n"); ++#endif /* _NET_OVPN_DCO_OVPNPKTID_H_ */ +diff --git a/drivers/net/ovpn-dco/proto.h b/drivers/net/ovpn-dco/proto.h +new file mode 100644 +index 000000000000..875529021c1b +--- /dev/null ++++ b/drivers/net/ovpn-dco/proto.h +@@ -0,0 +1,101 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ + -+ print_ksmd_cpu_comsuption(); ++#ifndef _NET_OVPN_DCO_OVPNPROTO_H_ ++#define _NET_OVPN_DCO_OVPNPROTO_H_ + -+ set_ksmd_run_mode(KSM_CLEAR_MODE); -+ set_ksmd_run_mode(KSM_NORMAL_MODE); ++#include "main.h" + -+ gettimeofday(&tv_start, NULL); ++#include + -+ wait_ksmpages_converged(final_pages[shape_type]); ++/* Methods for operating on the initial command ++ * byte of the OpenVPN protocol. ++ */ + -+ gettimeofday(&tv_end, NULL); -+ printf("ksm normal-run's merging time: %lf seconds\n", -+ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - -+ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); ++/* packet opcode (high 5 bits) and key-id (low 3 bits) are combined in ++ * one byte ++ */ ++#define OVPN_KEY_ID_MASK 0x07 ++#define OVPN_OPCODE_SHIFT 3 ++#define OVPN_OPCODE_MASK 0x1F ++/* upper bounds on opcode and key ID */ ++#define OVPN_KEY_ID_MAX (OVPN_KEY_ID_MASK + 1) ++#define OVPN_OPCODE_MAX (OVPN_OPCODE_MASK + 1) ++/* packet opcodes of interest to us */ ++#define OVPN_DATA_V1 6 /* data channel V1 packet */ ++#define OVPN_DATA_V2 9 /* data channel V2 packet */ ++/* size of initial packet opcode */ ++#define OVPN_OP_SIZE_V1 1 ++#define OVPN_OP_SIZE_V2 4 ++#define OVPN_PEER_ID_MASK 0x00FFFFFF ++#define OVPN_PEER_ID_UNDEF 0x00FFFFFF ++/* first byte of keepalive message */ ++#define OVPN_KEEPALIVE_FIRST_BYTE 0x2a ++/* first byte of exit message */ ++#define OVPN_EXPLICIT_EXIT_NOTIFY_FIRST_BYTE 0x28 + -+ /******* Start to time ksmd's auto-run mode **********/ -+ print_ksmd_cpu_comsuption(); ++/** ++ * Extract the OP code from the specified byte ++ * ++ * Return the OP code ++ */ ++static inline u8 ovpn_opcode_from_byte(u8 byte) ++{ ++ return byte >> OVPN_OPCODE_SHIFT; ++} + -+ printf("Start to test auto-run ksmd...\n"); -+ set_ksmd_run_mode(KSM_CLEAR_MODE); -+ set_ksmd_run_mode(KSM_AUTO_MODE); -+ if (write_file(ksm_auto_threshold_file, "99\n", 2)) -+ printf("Failed: write 1 to %s\n", ksm_auto_threshold_file); ++/** ++ * Extract the OP code from the skb head. ++ * ++ * Note: this function assumes that the skb head was pulled enough ++ * to access the first byte. ++ * ++ * Return the OP code ++ */ ++static inline u8 ovpn_opcode_from_skb(const struct sk_buff *skb, u16 offset) ++{ ++ return ovpn_opcode_from_byte(*(skb->data + offset)); ++} + -+ gettimeofday(&tv_start, NULL); ++/** ++ * Extract the key ID from the skb head. ++ * ++ * Note: this function assumes that the skb head was pulled enough ++ * to access the first byte. ++ * ++ * Return the key ID ++ */ + -+ wait_ksmpages_converged(shape_type); ++static inline u8 ovpn_key_id_from_skb(const struct sk_buff *skb) ++{ ++ return *skb->data & OVPN_KEY_ID_MASK; ++} + -+ gettimeofday(&tv_end, NULL); -+ printf("ksm auto-run's merging time: %lf seconds\n", -+ ((tv_end.tv_sec * 1000000 + tv_end.tv_usec) - -+ (tv_start.tv_sec * 1000000 + tv_start.tv_usec))/1000000.0); ++/** ++ * Extract the peer ID from the skb head. ++ * ++ * Note: this function assumes that the skb head was pulled enough ++ * to access the first 4 bytes. ++ * ++ * Return the peer ID. ++ */ + -+ print_ksmd_cpu_comsuption(); ++static inline u32 ovpn_peer_id_from_skb(const struct sk_buff *skb, u16 offset) ++{ ++ return ntohl(*(__be32 *)(skb->data + offset)) & OVPN_PEER_ID_MASK; +} + -+int main(int argc, char **argv) ++static inline u32 ovpn_opcode_compose(u8 opcode, u8 key_id, u32 peer_id) +{ -+ char *madvise_area; -+ int orig_run, orig_auto_threshold; ++ const u8 op = (opcode << OVPN_OPCODE_SHIFT) | (key_id & OVPN_KEY_ID_MASK); + -+ /* Get the original state of ksm */ -+ get_orig_info(&orig_run, &orig_auto_threshold); -+ printf("Now we mmap 300MB anouymous memory for testing.\n" -+ "There are two type of TEST which have different shape of\n" -+ "samepage areas.\n" -+ "Note: the test requires no other MERGEABLE-madvised vm areas\n" -+ "in system than the areas our testing process allocs.\n"); -+ madvise_area = mmap_and_madvise(TOTAL_MADVISE_SIZE, MADV_MERGEABLE); -+ if (!madvise_area) { -+ printf("madvise failed\n"); -+ exit(1); -+ } ++ return (op << 24) | (peer_id & OVPN_PEER_ID_MASK); ++} + -+ printf("\n****************** TEST 1 ******************\n"); -+ test_ksmd_performance(madvise_area, SHAPE_FULL); -+ printf("\n****************** TEST 2 ******************\n"); -+ test_ksmd_performance(madvise_area, SHAPE_SPARSE); ++#endif /* _NET_OVPN_DCO_OVPNPROTO_H_ */ +diff --git a/drivers/net/ovpn-dco/rcu.h b/drivers/net/ovpn-dco/rcu.h +new file mode 100644 +index 000000000000..02a50f49ba2e +--- /dev/null ++++ b/drivers/net/ovpn-dco/rcu.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+ /* Restore the original state */ -+ restore_orig_state(orig_run, orig_auto_threshold); ++#ifndef _NET_OVPN_DCO_OVPNRCU_H_ ++#define _NET_OVPN_DCO_OVPNRCU_H_ + -+ return 0; ++static inline void ovpn_rcu_lockdep_assert_held(void) ++{ ++#ifdef CONFIG_PROVE_RCU ++ RCU_LOCKDEP_WARN(!rcu_read_lock_held(), ++ "ovpn-dco RCU read lock not held"); ++#endif +} --- -2.37.3 - -From 862003890a3fa4ec3ebea18593a68d14abcb8bb9 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 25 Sep 2022 23:49:46 +0200 -Subject: [PATCH 03/13] futex-winesync - -Signed-off-by: Peter Jung ---- - Documentation/admin-guide/devices.txt | 3 +- - Documentation/userspace-api/index.rst | 1 + - .../userspace-api/ioctl/ioctl-number.rst | 2 + - Documentation/userspace-api/winesync.rst | 444 +++++ - MAINTAINERS | 9 + - drivers/misc/Kconfig | 11 + - drivers/misc/Makefile | 1 + - drivers/misc/winesync.c | 1212 ++++++++++++++ - include/linux/miscdevice.h | 1 + - include/uapi/linux/winesync.h | 71 + - tools/testing/selftests/Makefile | 1 + - .../selftests/drivers/winesync/Makefile | 8 + - .../testing/selftests/drivers/winesync/config | 1 + - .../selftests/drivers/winesync/winesync.c | 1479 +++++++++++++++++ - 14 files changed, 3243 insertions(+), 1 deletion(-) - create mode 100644 Documentation/userspace-api/winesync.rst - create mode 100644 drivers/misc/winesync.c - create mode 100644 include/uapi/linux/winesync.h - create mode 100644 tools/testing/selftests/drivers/winesync/Makefile - create mode 100644 tools/testing/selftests/drivers/winesync/config - create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c - -diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt -index 9764d6edb189..a4696d3b4a5a 100644 ---- a/Documentation/admin-guide/devices.txt -+++ b/Documentation/admin-guide/devices.txt -@@ -376,8 +376,9 @@ - 240 = /dev/userio Serio driver testing device - 241 = /dev/vhost-vsock Host kernel driver for virtio vsock - 242 = /dev/rfkill Turning off radio transmissions (rfkill) -+ 243 = /dev/winesync Wine synchronization primitive device - -- 243-254 Reserved for local use -+ 244-254 Reserved for local use - 255 Reserved for MISC_DYNAMIC_MINOR - - 11 char Raw keyboard device (Linux/SPARC only) -diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst -index a61eac0c73f8..0bf697ddcb09 100644 ---- a/Documentation/userspace-api/index.rst -+++ b/Documentation/userspace-api/index.rst -@@ -29,6 +29,7 @@ place where this information is gathered. - sysfs-platform_profile - vduse - futex2 -+ winesync - - .. only:: subproject and html - -diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst -index 3b985b19f39d..3f313fd4338c 100644 ---- a/Documentation/userspace-api/ioctl/ioctl-number.rst -+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst -@@ -375,6 +375,8 @@ Code Seq# Include File Comments - - 0xF6 all LTTng Linux Trace Toolkit Next Generation - -+0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives -+ - 0xF8 all arch/x86/include/uapi/asm/amd_hsmp.h AMD HSMP EPYC system management interface driver - - 0xFD all linux/dm-ioctl.h -diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst ++ ++#endif /* _NET_OVPN_DCO_OVPNRCU_H_ */ +diff --git a/drivers/net/ovpn-dco/skb.h b/drivers/net/ovpn-dco/skb.h new file mode 100644 -index 000000000000..f0110d2744c7 +index 000000000000..d38dc2da01df --- /dev/null -+++ b/Documentation/userspace-api/winesync.rst -@@ -0,0 +1,444 @@ -+===================================== -+Wine synchronization primitive driver -+===================================== -+ -+This page documents the user-space API for the winesync driver. -+ -+winesync is a support driver for emulation of NT synchronization -+primitives by the Wine project or other NT emulators. It exists -+because implementation in user-space, using existing tools, cannot -+simultaneously satisfy performance, correctness, and security -+constraints. It is implemented entirely in software, and does not -+drive any hardware device. ++++ b/drivers/net/ovpn-dco/skb.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ * James Yonan ++ */ + -+This interface is meant as a compatibility tool only, and should not -+be used for general synchronization. Instead use generic, versatile -+interfaces such as futex(2) and poll(2). ++#ifndef _NET_OVPN_DCO_SKB_H_ ++#define _NET_OVPN_DCO_SKB_H_ + -+Synchronization primitives -+========================== ++#include ++#include ++#include ++#include ++#include ++#include + -+The winesync driver exposes three types of synchronization primitives: -+semaphores, mutexes, and events. ++#define OVPN_SKB_CB(skb) ((struct ovpn_skb_cb *)&((skb)->cb)) + -+A semaphore holds a single volatile 32-bit counter, and a static -+32-bit integer denoting the maximum value. It is considered signaled -+when the counter is nonzero. The counter is decremented by one when a -+wait is satisfied. Both the initial and maximum count are established -+when the semaphore is created. ++struct ovpn_skb_cb { ++ /* original recv packet size for stats accounting */ ++ unsigned int rx_stats_size; + -+A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit -+identifier denoting its owner. A mutex is considered signaled when its -+owner is zero (indicating that it is not owned). The recursion count -+is incremented when a wait is satisfied, and ownership is set to the -+given identifier. ++ union { ++ struct in_addr ipv4; ++ struct in6_addr ipv6; ++ } local; ++ sa_family_t sa_fam; ++}; + -+A mutex also holds an internal flag denoting whether its previous -+owner has died; such a mutex is said to be inconsistent. Owner death -+is not tracked automatically based on thread death, but rather must be -+communicated using ``WINESYNC_IOC_KILL_OWNER``. An inconsistent mutex -+is inherently considered unowned. ++/* Return IP protocol version from skb header. ++ * Return 0 if protocol is not IPv4/IPv6 or cannot be read. ++ */ ++static inline __be16 ovpn_ip_check_protocol(struct sk_buff *skb) ++{ ++ __be16 proto = 0; + -+Except for the "unowned" semantics of zero, the actual value of the -+owner identifier is not interpreted by the winesync driver at all. The -+intended use is to store a thread identifier; however, the winesync -+driver does not actually validate that a calling thread provides -+consistent or unique identifiers. ++ /* skb could be non-linear, ++ * make sure IP header is in non-fragmented part ++ */ ++ if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) ++ return 0; + -+An event holds a volatile boolean state denoting whether it is -+signaled or not. There are two types of events, auto-reset and -+manual-reset. An auto-reset event is designaled when a wait is -+satisfied; a manual-reset event is not. The event type is specified -+when the event is created. ++ if (ip_hdr(skb)->version == 4) ++ proto = htons(ETH_P_IP); ++ else if (ip_hdr(skb)->version == 6) ++ proto = htons(ETH_P_IPV6); + -+Unless specified otherwise, all operations on an object are atomic and -+totally ordered with respect to other operations on the same object. ++ return proto; ++} + -+Objects are represented by unsigned 32-bit integers. ++#endif /* _NET_OVPN_DCO_SKB_H_ */ +diff --git a/drivers/net/ovpn-dco/sock.c b/drivers/net/ovpn-dco/sock.c +new file mode 100644 +index 000000000000..e92a4a9b952e +--- /dev/null ++++ b/drivers/net/ovpn-dco/sock.c +@@ -0,0 +1,134 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+Char device -+=========== ++#include "main.h" ++#include "ovpn.h" ++#include "peer.h" ++#include "sock.h" ++#include "rcu.h" ++#include "tcp.h" ++#include "udp.h" + -+The winesync driver creates a single char device /dev/winesync. Each -+file description opened on the device represents a unique namespace. -+That is, objects created on one open file description are shared -+across all its individual descriptors, but are not shared with other -+open() calls on the same device. The same file description may be -+shared across multiple processes. ++#include ++#include + -+ioctl reference -+=============== ++/* Finalize release of socket, called after RCU grace period */ ++static void ovpn_socket_detach(struct socket *sock) ++{ ++ if (!sock) ++ return; + -+All operations on the device are done through ioctls. There are three -+structures used in ioctl calls:: ++ if (sock->sk->sk_protocol == IPPROTO_UDP) ++ ovpn_udp_socket_detach(sock); ++ else if (sock->sk->sk_protocol == IPPROTO_TCP) ++ ovpn_tcp_socket_detach(sock); + -+ struct winesync_sem_args { -+ __u32 sem; -+ __u32 count; -+ __u32 max; -+ }; ++ sockfd_put(sock); ++} + -+ struct winesync_mutex_args { -+ __u32 mutex; -+ __u32 owner; -+ __u32 count; -+ }; ++void ovpn_socket_release_kref(struct kref *kref) ++{ ++ struct ovpn_socket *sock = container_of(kref, struct ovpn_socket, refcount); + -+ struct winesync_event_args { -+ __u32 event; -+ __u32 signaled; -+ __u32 manual; -+ }; ++ ovpn_socket_detach(sock->sock); ++ kfree_rcu(sock, rcu); ++} + -+ struct winesync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 pad; -+ }; ++static bool ovpn_socket_hold(struct ovpn_socket *sock) ++{ ++ return kref_get_unless_zero(&sock->refcount); ++} + -+Depending on the ioctl, members of the structure may be used as input, -+output, or not at all. All ioctls return 0 on success. ++static struct ovpn_socket *ovpn_socket_get(struct socket *sock) ++{ ++ struct ovpn_socket *ovpn_sock; + -+The ioctls are as follows: ++ rcu_read_lock(); ++ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); ++ if (!ovpn_socket_hold(ovpn_sock)) { ++ pr_warn("%s: found ovpn_socket with ref = 0\n", __func__); ++ ovpn_sock = NULL; ++ } ++ rcu_read_unlock(); + -+.. c:macro:: WINESYNC_IOC_CREATE_SEM ++ return ovpn_sock; ++} + -+ Create a semaphore object. Takes a pointer to struct -+ :c:type:`winesync_sem_args`, which is used as follows: ++/* Finalize release of socket, called after RCU grace period */ ++static int ovpn_socket_attach(struct socket *sock, struct ovpn_peer *peer) ++{ ++ int ret = -EOPNOTSUPP; + -+ .. list-table:: ++ if (!sock || !peer) ++ return -EINVAL; + -+ * - ``sem`` -+ - On output, contains the identifier of the created semaphore. -+ * - ``count`` -+ - Initial count of the semaphore. -+ * - ``max`` -+ - Maximum count of the semaphore. ++ if (sock->sk->sk_protocol == IPPROTO_UDP) ++ ret = ovpn_udp_socket_attach(sock, peer->ovpn); ++ else if (sock->sk->sk_protocol == IPPROTO_TCP) ++ ret = ovpn_tcp_socket_attach(sock, peer); + -+ Fails with ``EINVAL`` if ``count`` is greater than ``max``. ++ return ret; ++} + -+.. c:macro:: WINESYNC_IOC_CREATE_MUTEX ++struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk) ++{ ++ struct ovpn_socket *ovpn_sock; + -+ Create a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: ++ ovpn_rcu_lockdep_assert_held(); + -+ .. list-table:: ++ if (unlikely(READ_ONCE(udp_sk(sk)->encap_type) != UDP_ENCAP_OVPNINUDP)) ++ return NULL; + -+ * - ``mutex`` -+ - On output, contains the identifier of the created mutex. -+ * - ``count`` -+ - Initial recursion count of the mutex. -+ * - ``owner`` -+ - Initial owner of the mutex. ++ ovpn_sock = rcu_dereference_sk_user_data(sk); ++ if (unlikely(!ovpn_sock)) ++ return NULL; + -+ If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is -+ zero and ``count`` is nonzero, the function fails with ``EINVAL``. ++ /* make sure that sk matches our stored transport socket */ ++ if (unlikely(!ovpn_sock->sock || sk != ovpn_sock->sock->sk)) ++ return NULL; + -+.. c:macro:: WINESYNC_IOC_CREATE_EVENT ++ return ovpn_sock->ovpn; ++} + -+ Create an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: ++struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ++{ ++ struct ovpn_socket *ovpn_sock; ++ int ret; + -+ .. list-table:: ++ ret = ovpn_socket_attach(sock, peer); ++ if (ret < 0 && ret != -EALREADY) ++ return ERR_PTR(ret); + -+ * - ``event`` -+ - On output, contains the identifier of the created event. -+ * - ``signaled`` -+ - If nonzero, the event is initially signaled, otherwise -+ nonsignaled. -+ * - ``manual`` -+ - If nonzero, the event is a manual-reset event, otherwise -+ auto-reset. ++ /* if this socket is already owned by this interface, just increase the refcounter */ ++ if (ret == -EALREADY) { ++ /* caller is expected to increase the sock refcounter before passing it to this ++ * function. For this reason we drop it if not needed, like when this socket is ++ * already owned. ++ */ ++ ovpn_sock = ovpn_socket_get(sock); ++ sockfd_put(sock); ++ return ovpn_sock; ++ } + -+.. c:macro:: WINESYNC_IOC_DELETE ++ ovpn_sock = kzalloc(sizeof(*ovpn_sock), GFP_KERNEL); ++ if (!ovpn_sock) ++ return ERR_PTR(-ENOMEM); + -+ Delete an object of any type. Takes an input-only pointer to a -+ 32-bit integer denoting the object to delete. ++ ovpn_sock->ovpn = peer->ovpn; ++ ovpn_sock->sock = sock; ++ kref_init(&ovpn_sock->refcount); + -+ Wait ioctls currently in progress are not interrupted, and behave as -+ if the object remains valid. ++ /* TCP sockets are per-peer, therefore they are linked to their unique peer */ ++ if (sock->sk->sk_protocol == IPPROTO_TCP) ++ ovpn_sock->peer = peer; + -+.. c:macro:: WINESYNC_IOC_PUT_SEM ++ rcu_assign_sk_user_data(sock->sk, ovpn_sock); + -+ Post to a semaphore object. Takes a pointer to struct -+ :c:type:`winesync_sem_args`, which is used as follows: ++ return ovpn_sock; ++} +diff --git a/drivers/net/ovpn-dco/sock.h b/drivers/net/ovpn-dco/sock.h +new file mode 100644 +index 000000000000..9e79c1b5fe04 +--- /dev/null ++++ b/drivers/net/ovpn-dco/sock.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+ .. list-table:: ++#ifndef _NET_OVPN_DCO_SOCK_H_ ++#define _NET_OVPN_DCO_SOCK_H_ + -+ * - ``sem`` -+ - Semaphore object to post to. -+ * - ``count`` -+ - Count to add to the semaphore. On output, contains the -+ previous count of the semaphore. -+ * - ``max`` -+ - Not used. ++#include ++#include ++#include + -+ If adding ``count`` to the semaphore's current count would raise the -+ latter past the semaphore's maximum count, the ioctl fails with -+ ``EOVERFLOW`` and the semaphore is not affected. If raising the -+ semaphore's count causes it to become signaled, eligible threads -+ waiting on this semaphore will be woken and the semaphore's count -+ decremented appropriately. ++#include "peer.h" + -+.. c:macro:: WINESYNC_IOC_PUT_MUTEX ++struct ovpn_struct; + -+ Release a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: ++/** ++ * struct ovpn_socket - a kernel socket referenced in the ovpn-dco code ++ */ ++struct ovpn_socket { ++ union { ++ /** @ovpn: the VPN session object owning this socket (UDP only) */ ++ struct ovpn_struct *ovpn; + -+ .. list-table:: ++ /** @peer: the unique peer transmitting over this socket (TCP only) */ ++ struct ovpn_peer *peer; ++ }; + -+ * - ``mutex`` -+ - Mutex object to release. -+ * - ``owner`` -+ - Mutex owner identifier. -+ * - ``count`` -+ - On output, contains the previous recursion count. ++ /** @sock: the kernel socket */ ++ struct socket *sock; + -+ If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` -+ is not the current owner of the mutex, the ioctl fails with -+ ``EPERM``. ++ /** @refcount: amount of contexts currently referencing this object */ ++ struct kref refcount; + -+ The mutex's count will be decremented by one. If decrementing the -+ mutex's count causes it to become zero, the mutex is marked as -+ unowned and signaled, and eligible threads waiting on it will be -+ woken as appropriate. ++ /** @rcu: member used to schedule RCU destructor callback */ ++ struct rcu_head rcu; ++}; + -+.. c:macro:: WINESYNC_IOC_SET_EVENT ++struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk); + -+ Signal an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: ++void ovpn_socket_release_kref(struct kref *kref); + -+ .. list-table:: ++static inline void ovpn_socket_put(struct ovpn_socket *sock) ++{ ++ kref_put(&sock->refcount, ovpn_socket_release_kref); ++} + -+ * - ``event`` -+ - Event object to set. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. ++struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer); + -+ Eligible threads will be woken, and auto-reset events will be -+ designaled appropriately. ++#endif /* _NET_OVPN_DCO_SOCK_H_ */ +diff --git a/drivers/net/ovpn-dco/stats.c b/drivers/net/ovpn-dco/stats.c +new file mode 100644 +index 000000000000..ee000b2a2177 +--- /dev/null ++++ b/drivers/net/ovpn-dco/stats.c +@@ -0,0 +1,20 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+.. c:macro:: WINESYNC_IOC_RESET_EVENT ++#include "main.h" ++#include "stats.h" + -+ Designal an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: ++void ovpn_peer_stats_init(struct ovpn_peer_stats *ps) ++{ ++ atomic64_set(&ps->rx.bytes, 0); ++ atomic_set(&ps->rx.packets, 0); + -+ .. list-table:: ++ atomic64_set(&ps->tx.bytes, 0); ++ atomic_set(&ps->tx.packets, 0); ++} +diff --git a/drivers/net/ovpn-dco/stats.h b/drivers/net/ovpn-dco/stats.h +new file mode 100644 +index 000000000000..3aa6bdc049c6 +--- /dev/null ++++ b/drivers/net/ovpn-dco/stats.h +@@ -0,0 +1,67 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ * Lev Stipakov ++ */ + -+ * - ``event`` -+ - Event object to reset. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. ++#ifndef _NET_OVPN_DCO_OVPNSTATS_H_ ++#define _NET_OVPN_DCO_OVPNSTATS_H_ + -+.. c:macro:: WINESYNC_IOC_PULSE_EVENT ++#include ++#include + -+ Wake threads waiting on an event object without leaving it in a -+ signaled state. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: ++struct ovpn_struct; + -+ .. list-table:: ++/* per-peer stats, measured on transport layer */ + -+ * - ``event`` -+ - Event object to pulse. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. ++/* one stat */ ++struct ovpn_peer_stat { ++ atomic64_t bytes; ++ atomic_t packets; ++}; + -+ A pulse operation can be thought of as a set followed by a reset, -+ performed as a single atomic operation. If two threads are waiting -+ on an auto-reset event which is pulsed, only one will be woken. If -+ two threads are waiting a manual-reset event which is pulsed, both -+ will be woken. However, in both cases, the event will be unsignaled -+ afterwards, and a simultaneous read operation will always report the -+ event as unsignaled. ++/* rx and tx stats, enabled by notify_per != 0 or period != 0 */ ++struct ovpn_peer_stats { ++ struct ovpn_peer_stat rx; ++ struct ovpn_peer_stat tx; ++}; + -+.. c:macro:: WINESYNC_IOC_READ_SEM ++/* struct for OVPN_ERR_STATS */ + -+ Read the current state of a semaphore object. Takes a pointer to -+ struct :c:type:`winesync_sem_args`, which is used as follows: ++struct ovpn_err_stat { ++ unsigned int category; ++ int errcode; ++ u64 count; ++}; + -+ .. list-table:: ++struct ovpn_err_stats { ++ /* total stats, returned by kovpn */ ++ unsigned int total_stats; ++ /* number of stats dimensioned below */ ++ unsigned int n_stats; ++ struct ovpn_err_stat stats[]; ++}; + -+ * - ``sem`` -+ - Semaphore object to read. -+ * - ``count`` -+ - On output, contains the current count of the semaphore. -+ * - ``max`` -+ - On output, contains the maximum count of the semaphore. ++void ovpn_peer_stats_init(struct ovpn_peer_stats *ps); + -+.. c:macro:: WINESYNC_IOC_READ_MUTEX ++static inline void ovpn_peer_stats_increment(struct ovpn_peer_stat *stat, const unsigned int n) ++{ ++ atomic64_add(n, &stat->bytes); ++ atomic_inc(&stat->packets); ++} + -+ Read the current state of a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: ++static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats, const unsigned int n) ++{ ++ ovpn_peer_stats_increment(&stats->rx, n); ++} + -+ .. list-table:: ++static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats, const unsigned int n) ++{ ++ ovpn_peer_stats_increment(&stats->tx, n); ++} + -+ * - ``mutex`` -+ - Mutex object to read. -+ * - ``owner`` -+ - On output, contains the current owner of the mutex, or zero -+ if the mutex is not currently owned. -+ * - ``count`` -+ - On output, contains the current recursion count of the mutex. ++#endif /* _NET_OVPN_DCO_OVPNSTATS_H_ */ +diff --git a/drivers/net/ovpn-dco/tcp.c b/drivers/net/ovpn-dco/tcp.c +new file mode 100644 +index 000000000000..7e6690fee6e7 +--- /dev/null ++++ b/drivers/net/ovpn-dco/tcp.c +@@ -0,0 +1,326 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ + -+ If the mutex is marked as inconsistent, the function fails with -+ ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to -+ zero. ++#include "main.h" ++#include "ovpnstruct.h" ++#include "ovpn.h" ++#include "peer.h" ++#include "skb.h" ++#include "tcp.h" + -+.. c:macro:: WINESYNC_IOC_READ_EVENT ++#include ++#include ++#include + -+ Read the current state of an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: ++static void ovpn_tcp_state_change(struct sock *sk) ++{ ++} + -+ .. list-table:: ++static void ovpn_tcp_data_ready(struct sock *sk) ++{ ++ struct ovpn_socket *sock; + -+ * - ``event`` -+ - Event object. -+ * - ``signaled`` -+ - On output, contains the current state of the event. -+ * - ``manual`` -+ - On output, contains 1 if the event is a manual-reset event, -+ and 0 otherwise. ++ rcu_read_lock(); ++ sock = rcu_dereference_sk_user_data(sk); ++ rcu_read_unlock(); + -+.. c:macro:: WINESYNC_IOC_KILL_OWNER ++ if (!sock || !sock->peer) ++ return; + -+ Mark any mutexes owned by the given owner as unowned and -+ inconsistent. Takes an input-only pointer to a 32-bit integer -+ denoting the owner. If the owner is zero, the ioctl fails with -+ ``EINVAL``. ++ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.rx_work); ++} + -+ For each mutex currently owned by the given owner, eligible threads -+ waiting on said mutex will be woken as appropriate (and such waits -+ will fail with ``EOWNERDEAD``, as described below). ++static void ovpn_tcp_write_space(struct sock *sk) ++{ ++ struct ovpn_socket *sock; + -+ The operation as a whole is not atomic; however, the modification of -+ each mutex is atomic and totally ordered with respect to other -+ operations on the same mutex. ++ rcu_read_lock(); ++ sock = rcu_dereference_sk_user_data(sk); ++ rcu_read_unlock(); + -+.. c:macro:: WINESYNC_IOC_WAIT_ANY ++ if (!sock || !sock->peer) ++ return; + -+ Poll on any of a list of objects, atomically acquiring at most one. -+ Takes a pointer to struct :c:type:`winesync_wait_args`, which is -+ used as follows: ++ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.tx_work); ++} + -+ .. list-table:: ++static void ovpn_destroy_skb(void *skb) ++{ ++ consume_skb(skb); ++} + -+ * - ``timeout`` -+ - Optional pointer to a 64-bit struct :c:type:`timespec` -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). The timeout is specified in -+ absolute format, as measured against the MONOTONIC clock. If -+ the timeout is equal to or earlier than the current time, the -+ function returns immediately without sleeping. If ``timeout`` -+ is zero, i.e. NULL, the function will sleep until an object -+ is signaled, and will not fail with ``ETIMEDOUT``. -+ * - ``objs`` -+ - Pointer to an array of ``count`` 32-bit object identifiers -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). If any identifier is -+ invalid, the function fails with ``EINVAL``. -+ * - ``count`` -+ - Number of object identifiers specified in the ``objs`` array. -+ * - ``owner`` -+ - Mutex owner identifier. If any object in ``objs`` is a mutex, -+ the ioctl will attempt to acquire that mutex on behalf of -+ ``owner``. If ``owner`` is zero, the ioctl fails with -+ ``EINVAL``. -+ * - ``index`` -+ - On success, contains the index (into ``objs``) of the object -+ which was signaled. If ``alert`` was signaled instead, -+ this contains ``count``. -+ * - ``alert`` -+ - Optional event object identifier. If nonzero, this specifies -+ an "alert" event object which, if signaled, will terminate -+ the wait. If nonzero, the identifier must point to a valid -+ event. ++void ovpn_tcp_socket_detach(struct socket *sock) ++{ ++ struct ovpn_socket *ovpn_sock; ++ struct ovpn_peer *peer; + -+ This function attempts to acquire one of the given objects. If -+ unable to do so, it sleeps until an object becomes signaled, -+ subsequently acquiring it, or the timeout expires. In the latter -+ case the ioctl fails with ``ETIMEDOUT``. The function only acquires -+ one object, even if multiple objects are signaled. ++ if (!sock) ++ return; + -+ A semaphore is considered to be signaled if its count is nonzero, -+ and is acquired by decrementing its count by one. A mutex is -+ considered to be signaled if it is unowned or if its owner matches -+ the ``owner`` argument, and is acquired by incrementing its -+ recursion count by one and setting its owner to the ``owner`` -+ argument. An auto-reset event is acquired by designaling it; a -+ manual-reset event is not affected by acquisition. ++ rcu_read_lock(); ++ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); ++ rcu_read_unlock(); + -+ Acquisition is atomic and totally ordered with respect to other -+ operations on the same object. If two wait operations (with -+ different ``owner`` identifiers) are queued on the same mutex, only -+ one is signaled. If two wait operations are queued on the same -+ semaphore, and a value of one is posted to it, only one is signaled. -+ The order in which threads are signaled is not specified. ++ if (!ovpn_sock->peer) ++ return; + -+ If an inconsistent mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Although this is a failure return, the function may -+ otherwise be considered successful. The mutex is marked as owned by -+ the given owner (with a recursion count of 1) and as no longer -+ inconsistent, and ``index`` is still set to the index of the mutex. ++ peer = ovpn_sock->peer; + -+ The ``alert`` argument is an "extra" event which can terminate the -+ wait, independently of all other objects. If members of ``objs`` and -+ ``alert`` are both simultaneously signaled, a member of ``objs`` -+ will always be given priority and acquired first. Aside from this, -+ for "any" waits, there is no difference between passing an event as -+ this parameter, and passing it as an additional object at the end of -+ the ``objs`` array. For "all" waits, there is an additional -+ difference, as described below. ++ /* restore CBs that were saved in ovpn_sock_set_tcp_cb() */ ++ write_lock_bh(&sock->sk->sk_callback_lock); ++ sock->sk->sk_state_change = peer->tcp.sk_cb.sk_state_change; ++ sock->sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; ++ sock->sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; ++ rcu_assign_sk_user_data(sock->sk, NULL); ++ write_unlock_bh(&sock->sk->sk_callback_lock); + -+ It is valid to pass the same object more than once, including by -+ passing the same event in the ``objs`` array and in ``alert``. If a -+ wakeup occurs due to that object being signaled, ``index`` is set to -+ the lowest index corresponding to that object. ++ /* cancel any ongoing work. Done after removing the CBs so that these workers cannot be ++ * re-armed ++ */ ++ cancel_work_sync(&peer->tcp.tx_work); ++ cancel_work_sync(&peer->tcp.rx_work); + -+ The function may fail with ``EINTR`` if a signal is received. ++ ptr_ring_cleanup(&peer->tcp.tx_ring, ovpn_destroy_skb); ++} + -+.. c:macro:: WINESYNC_IOC_WAIT_ALL ++/* Try to send one skb (or part of it) over the TCP stream. ++ * ++ * Return 0 on success or a negative error code otherwise. ++ * ++ * Note that the skb is modified by putting away the data being sent, therefore ++ * the caller should check if skb->len is zero to understand if the full skb was ++ * sent or not. ++ */ ++static int ovpn_tcp_send_one(struct ovpn_peer *peer, struct sk_buff *skb) ++{ ++ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; ++ struct kvec iv = { 0 }; ++ int ret; + -+ Poll on a list of objects, atomically acquiring all of them. Takes a -+ pointer to struct :c:type:`winesync_wait_args`, which is used -+ identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is -+ always filled with zero on success if not woken via alert. ++ if (skb_linearize(skb) < 0) { ++ net_err_ratelimited("%s: can't linearize packet\n", __func__); ++ return -ENOMEM; ++ } + -+ This function attempts to simultaneously acquire all of the given -+ objects. If unable to do so, it sleeps until all objects become -+ simultaneously signaled, subsequently acquiring them, or the timeout -+ expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and -+ no objects are modified. ++ /* initialize iv structure now as skb_linearize() may have changed skb->data */ ++ iv.iov_base = skb->data; ++ iv.iov_len = skb->len; + -+ Objects may become signaled and subsequently designaled (through -+ acquisition by other threads) while this thread is sleeping. Only -+ once all objects are simultaneously signaled does the ioctl acquire -+ them and return. The entire acquisition is atomic and totally -+ ordered with respect to other operations on any of the given -+ objects. ++ ret = kernel_sendmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len); ++ if (ret > 0) { ++ __skb_pull(skb, ret); + -+ If an inconsistent mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects -+ are nevertheless marked as acquired. Note that if multiple mutex -+ objects are specified, there is no way to know which were marked as -+ inconsistent. ++ /* since we update per-cpu stats in process context, ++ * we need to disable softirqs ++ */ ++ local_bh_disable(); ++ dev_sw_netstats_tx_add(peer->ovpn->dev, 1, ret); ++ local_bh_enable(); + -+ As with "any" waits, the ``alert`` argument is an "extra" event -+ which can terminate the wait. Critically, however, an "all" wait -+ will succeed if all members in ``objs`` are signaled, *or* if -+ ``alert`` is signaled. In the latter case ``index`` will be set to -+ ``count``. As with "any" waits, if both conditions are filled, the -+ former takes priority, and objects in ``objs`` will be acquired. ++ return 0; ++ } + -+ Unlike ``WINESYNC_IOC_WAIT_ANY``, it is not valid to pass the same -+ object more than once, nor is it valid to pass the same object in -+ ``objs`` and in ``alert`` If this is attempted, the function fails -+ with ``EINVAL``. -diff --git a/MAINTAINERS b/MAINTAINERS -index f5ca4aefd184..31a7aa60cdc3 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -21921,6 +21921,15 @@ M: David Härdeman - S: Maintained - F: drivers/media/rc/winbond-cir.c - -+WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER -+M: Zebediah Figura -+L: wine-devel@winehq.org -+S: Supported -+F: Documentation/userspace-api/winesync.rst -+F: drivers/misc/winesync.c -+F: include/uapi/linux/winesync.h -+F: tools/testing/selftests/drivers/winesync/ ++ return ret; ++} + - WINSYSTEMS EBC-C384 WATCHDOG DRIVER - M: William Breathitt Gray - L: linux-watchdog@vger.kernel.org -diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig -index 94e9fb4cdd76..4f9e3d80a6e8 100644 ---- a/drivers/misc/Kconfig -+++ b/drivers/misc/Kconfig -@@ -496,6 +496,17 @@ config VCPU_STALL_DETECTOR - - If you do not intend to run this kernel as a guest, say N. - -+config WINESYNC -+ tristate "Synchronization primitives for Wine" -+ help -+ This module provides kernel support for synchronization primitives -+ used by Wine. It is not a hardware driver. ++/* Process packets in TCP TX queue */ ++static void ovpn_tcp_tx_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer; ++ struct sk_buff *skb; ++ int ret; + -+ To compile this driver as a module, choose M here: the -+ module will be called winesync. ++ peer = container_of(work, struct ovpn_peer, tcp.tx_work); ++ while ((skb = __ptr_ring_peek(&peer->tcp.tx_ring))) { ++ ret = ovpn_tcp_send_one(peer, skb); ++ if (ret < 0 && ret != -EAGAIN) { ++ net_warn_ratelimited("%s: cannot send TCP packet to peer %u: %d\n", __func__, ++ peer->id, ret); ++ /* in case of TCP error stop sending loop and delete peer */ ++ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_ERROR); ++ break; ++ } else if (!skb->len) { ++ /* skb was entirely consumed and can now be removed from the ring */ ++ __ptr_ring_discard_one(&peer->tcp.tx_ring); ++ consume_skb(skb); ++ } + -+ If unsure, say N. ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); ++ } ++} + - source "drivers/misc/c2port/Kconfig" - source "drivers/misc/eeprom/Kconfig" - source "drivers/misc/cb710/Kconfig" -diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile -index 2be8542616dd..d061fe45407b 100644 ---- a/drivers/misc/Makefile -+++ b/drivers/misc/Makefile -@@ -58,6 +58,7 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ - obj-$(CONFIG_UACCE) += uacce/ - obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o - obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o -+obj-$(CONFIG_WINESYNC) += winesync.o - obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o - obj-$(CONFIG_OPEN_DICE) += open-dice.o - obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o -\ No newline at end of file -diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c -new file mode 100644 -index 000000000000..7a28f58dbbf2 ---- /dev/null -+++ b/drivers/misc/winesync.c -@@ -0,0 +1,1212 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * winesync.c - Kernel driver for Wine synchronization primitives -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ ++static int ovpn_tcp_rx_one(struct ovpn_peer *peer) ++{ ++ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; ++ struct ovpn_skb_cb *cb; ++ int status, ret; + -+#include -+#include -+#include -+#include -+#include -+#include ++ /* no skb allocated means that we have to read (or finish reading) the 2 bytes prefix ++ * containing the actual packet size. ++ */ ++ if (!peer->tcp.skb) { ++ struct kvec iv = { ++ .iov_base = peer->tcp.raw_len + peer->tcp.offset, ++ .iov_len = sizeof(u16) - peer->tcp.offset, ++ }; + -+#define WINESYNC_NAME "winesync" ++ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); ++ if (ret <= 0) ++ return ret; + -+enum winesync_type { -+ WINESYNC_TYPE_SEM, -+ WINESYNC_TYPE_MUTEX, -+ WINESYNC_TYPE_EVENT, -+}; ++ peer->tcp.offset += ret; ++ /* the entire packet size was read, prepare skb for reading data */ ++ if (peer->tcp.offset == sizeof(u16)) { ++ u16 len = ntohs(*(__be16 *)peer->tcp.raw_len); ++ /* invalid packet length: this is a fatal TCP error */ ++ if (!len) { ++ netdev_err(peer->ovpn->dev, "%s: received invalid packet length\n", ++ __func__); ++ return -EINVAL; ++ } + -+struct winesync_obj { -+ struct rcu_head rhead; -+ struct kref refcount; -+ spinlock_t lock; ++ peer->tcp.skb = netdev_alloc_skb_ip_align(peer->ovpn->dev, len); ++ peer->tcp.offset = 0; ++ peer->tcp.data_len = len; ++ } ++ } else { ++ struct kvec iv = { ++ .iov_base = peer->tcp.skb->data + peer->tcp.offset, ++ .iov_len = peer->tcp.data_len - peer->tcp.offset, ++ }; + -+ /* -+ * any_waiters is protected by the object lock, but all_waiters is -+ * protected by the device wait_all_lock. -+ */ -+ struct list_head any_waiters; -+ struct list_head all_waiters; ++ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); ++ if (ret <= 0) ++ return ret; + -+ /* -+ * Hint describing how many tasks are queued on this object in a -+ * wait-all operation. -+ * -+ * Any time we do a wake, we may need to wake "all" waiters as well as -+ * "any" waiters. In order to atomically wake "all" waiters, we must -+ * lock all of the objects, and that means grabbing the wait_all_lock -+ * below (and, due to lock ordering rules, before locking this object). -+ * However, wait-all is a rare operation, and grabbing the wait-all -+ * lock for every wake would create unnecessary contention. Therefore we -+ * first check whether all_hint is zero, and, if it is, we skip trying -+ * to wake "all" waiters. -+ * -+ * This hint isn't protected by any lock. It might change during the -+ * course of a wake, but there's no meaningful race there; it's only a -+ * hint. -+ * -+ * Since wait requests must originate from user-space threads, we're -+ * limited here by PID_MAX_LIMIT, so there's no risk of saturation. -+ */ -+ atomic_t all_hint; ++ peer->tcp.offset += ret; ++ /* full packet received, send it up for processing */ ++ if (peer->tcp.offset == peer->tcp.data_len) { ++ /* update the skb data structure with the amount of data written by ++ * kernel_recvmsg() ++ */ ++ skb_put(peer->tcp.skb, peer->tcp.data_len); + -+ enum winesync_type type; ++ /* do not perform IP caching for TCP connections */ ++ cb = OVPN_SKB_CB(peer->tcp.skb); ++ cb->sa_fam = AF_UNSPEC; + -+ /* The following fields are protected by the object lock. */ -+ union { -+ struct { -+ __u32 count; -+ __u32 max; -+ } sem; -+ struct { -+ __u32 count; -+ __u32 owner; -+ bool ownerdead; -+ } mutex; -+ struct { -+ bool manual; -+ bool signaled; -+ } event; -+ } u; -+}; ++ /* hold reference to peer as requird by ovpn_recv() */ ++ ovpn_peer_hold(peer); ++ status = ovpn_recv(peer->ovpn, peer, peer->tcp.skb); ++ /* skb not consumed - free it now */ ++ if (unlikely(status < 0)) ++ kfree_skb(peer->tcp.skb); + -+struct winesync_q_entry { -+ struct list_head node; -+ struct winesync_q *q; -+ struct winesync_obj *obj; -+ __u32 index; -+}; ++ peer->tcp.skb = NULL; ++ peer->tcp.offset = 0; ++ peer->tcp.data_len = 0; ++ } ++ } + -+struct winesync_q { -+ struct task_struct *task; -+ __u32 owner; ++ return ret; ++} + -+ /* -+ * Protected via atomic_cmpxchg(). Only the thread that wins the -+ * compare-and-swap may actually change object states and wake this -+ * task. -+ */ -+ atomic_t signaled; ++static void ovpn_tcp_rx_work(struct work_struct *work) ++{ ++ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, tcp.rx_work); ++ int ret; + -+ bool all; -+ bool ownerdead; -+ __u32 count; -+ struct winesync_q_entry entries[]; -+}; ++ while (true) { ++ /* give a chance to be rescheduled if needed */ ++ cond_resched(); + -+struct winesync_device { -+ /* -+ * Wait-all operations must atomically grab all objects, and be totally -+ * ordered with respect to each other and wait-any operations. If one -+ * thread is trying to acquire several objects, another thread cannot -+ * touch the object at the same time. -+ * -+ * We achieve this by grabbing multiple object locks at the same time. -+ * However, this creates a lock ordering problem. To solve that problem, -+ * wait_all_lock is taken first whenever multiple objects must be locked -+ * at the same time. -+ */ -+ spinlock_t wait_all_lock; ++ ret = ovpn_tcp_rx_one(peer); ++ if (ret <= 0) ++ break; ++ } + -+ struct xarray objects; -+}; ++ if (ret < 0 && ret != -EAGAIN) ++ netdev_err(peer->ovpn->dev, "%s: TCP socket error: %d\n", __func__, ret); ++} + -+static struct winesync_obj *get_obj(struct winesync_device *dev, __u32 id) ++/* Put packet into TCP TX queue and schedule a consumer */ ++void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb) +{ -+ struct winesync_obj *obj; ++ int ret; + -+ rcu_read_lock(); -+ obj = xa_load(&dev->objects, id); -+ if (obj && !kref_get_unless_zero(&obj->refcount)) -+ obj = NULL; -+ rcu_read_unlock(); ++ ret = ptr_ring_produce_bh(&peer->tcp.tx_ring, skb); ++ if (ret < 0) { ++ kfree_skb_list(skb); ++ return; ++ } + -+ return obj; ++ queue_work(peer->ovpn->events_wq, &peer->tcp.tx_work); +} + -+static void destroy_obj(struct kref *ref) ++/* Set TCP encapsulation callbacks */ ++int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer) +{ -+ struct winesync_obj *obj = container_of(ref, struct winesync_obj, refcount); ++ void *old_data; ++ int ret; + -+ kfree_rcu(obj, rhead); -+} ++ INIT_WORK(&peer->tcp.tx_work, ovpn_tcp_tx_work); ++ INIT_WORK(&peer->tcp.rx_work, ovpn_tcp_rx_work); ++ ++ ret = ptr_ring_init(&peer->tcp.tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); ++ if (ret < 0) { ++ netdev_err(peer->ovpn->dev, "cannot allocate TCP TX ring\n"); ++ return ret; ++ } ++ ++ peer->tcp.skb = NULL; ++ peer->tcp.offset = 0; ++ peer->tcp.data_len = 0; ++ ++ write_lock_bh(&sock->sk->sk_callback_lock); + -+static void put_obj(struct winesync_obj *obj) -+{ -+ kref_put(&obj->refcount, destroy_obj); -+} ++ /* make sure no pre-existing encapsulation handler exists */ ++ rcu_read_lock(); ++ old_data = rcu_dereference_sk_user_data(sock->sk); ++ rcu_read_unlock(); ++ if (old_data) { ++ netdev_err(peer->ovpn->dev, "provided socket already taken by other user\n"); ++ ret = -EBUSY; ++ goto err; ++ } + -+static struct winesync_obj *get_obj_typed(struct winesync_device *dev, __u32 id, -+ enum winesync_type type) -+{ -+ struct winesync_obj *obj = get_obj(dev, id); ++ /* sanity check */ ++ if (sock->sk->sk_protocol != IPPROTO_TCP) { ++ netdev_err(peer->ovpn->dev, "expected TCP socket\n"); ++ ret = -EINVAL; ++ goto err; ++ } + -+ if (obj && obj->type != type) { -+ put_obj(obj); -+ return NULL; ++ /* only a fully connected socket are expected. Connection should be handled in userspace */ ++ if (sock->sk->sk_state != TCP_ESTABLISHED) { ++ netdev_err(peer->ovpn->dev, "unexpected state for TCP socket: %d\n", ++ sock->sk->sk_state); ++ ret = -EINVAL; ++ goto err; + } -+ return obj; -+} + -+static int winesync_char_open(struct inode *inode, struct file *file) -+{ -+ struct winesync_device *dev; ++ /* save current CBs so that they can be restored upon socket release */ ++ peer->tcp.sk_cb.sk_state_change = sock->sk->sk_state_change; ++ peer->tcp.sk_cb.sk_data_ready = sock->sk->sk_data_ready; ++ peer->tcp.sk_cb.sk_write_space = sock->sk->sk_write_space; + -+ dev = kzalloc(sizeof(*dev), GFP_KERNEL); -+ if (!dev) -+ return -ENOMEM; ++ /* assign our static CBs */ ++ sock->sk->sk_state_change = ovpn_tcp_state_change; ++ sock->sk->sk_data_ready = ovpn_tcp_data_ready; ++ sock->sk->sk_write_space = ovpn_tcp_write_space; + -+ spin_lock_init(&dev->wait_all_lock); ++ write_unlock_bh(&sock->sk->sk_callback_lock); + -+ xa_init_flags(&dev->objects, XA_FLAGS_ALLOC); ++ return 0; ++err: ++ write_unlock_bh(&sock->sk->sk_callback_lock); ++ ptr_ring_cleanup(&peer->tcp.tx_ring, NULL); + -+ file->private_data = dev; -+ return nonseekable_open(inode, file); ++ return ret; +} +diff --git a/drivers/net/ovpn-dco/tcp.h b/drivers/net/ovpn-dco/tcp.h +new file mode 100644 +index 000000000000..d243a8e1c34e +--- /dev/null ++++ b/drivers/net/ovpn-dco/tcp.h +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ + -+static int winesync_char_release(struct inode *inode, struct file *file) -+{ -+ struct winesync_device *dev = file->private_data; -+ struct winesync_obj *obj; -+ unsigned long id; ++#ifndef _NET_OVPN_DCO_TCP_H_ ++#define _NET_OVPN_DCO_TCP_H_ + -+ xa_for_each(&dev->objects, id, obj) -+ put_obj(obj); ++#include "peer.h" + -+ xa_destroy(&dev->objects); ++#include ++#include ++#include ++#include + -+ kfree(dev); ++void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb); + -+ return 0; -+} ++int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer); ++void ovpn_tcp_socket_detach(struct socket *sock); + -+static void init_obj(struct winesync_obj *obj) ++/* Prepare skb and enqueue it for sending to peer. ++ * ++ * Preparation consist in prepending the skb payload with its size. ++ * Required by the OpenVPN protocol in order to extract packets from ++ * the TCP stream on the receiver side. ++ */ ++static inline void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sk_buff *skb) +{ -+ kref_init(&obj->refcount); -+ atomic_set(&obj->all_hint, 0); -+ spin_lock_init(&obj->lock); -+ INIT_LIST_HEAD(&obj->any_waiters); -+ INIT_LIST_HEAD(&obj->all_waiters); ++ u16 len = skb->len; ++ ++ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); ++ ovpn_queue_tcp_skb(peer, skb); +} + -+static bool is_signaled(struct winesync_obj *obj, __u32 owner) -+{ -+ lockdep_assert_held(&obj->lock); ++#endif /* _NET_OVPN_DCO_TCP_H_ */ +diff --git a/drivers/net/ovpn-dco/udp.c b/drivers/net/ovpn-dco/udp.c +new file mode 100644 +index 000000000000..afa236d1f15c +--- /dev/null ++++ b/drivers/net/ovpn-dco/udp.c +@@ -0,0 +1,343 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ + -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ return !!obj->u.sem.count; -+ case WINESYNC_TYPE_MUTEX: -+ if (obj->u.mutex.owner && obj->u.mutex.owner != owner) -+ return false; -+ return obj->u.mutex.count < UINT_MAX; -+ case WINESYNC_TYPE_EVENT: -+ return obj->u.event.signaled; -+ } ++#include "main.h" ++#include "bind.h" ++#include "ovpn.h" ++#include "ovpnstruct.h" ++#include "peer.h" ++#include "proto.h" ++#include "skb.h" ++#include "udp.h" + -+ WARN(1, "bad object type %#x\n", obj->type); -+ return false; -+} ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+/* -+ * "locked_obj" is an optional pointer to an object which is already locked and -+ * should not be locked again. This is necessary so that changing an object's -+ * state and waking it can be a single atomic operation. ++/** ++ * ovpn_udp_encap_recv() - Start processing a received UDP packet. ++ * If the first byte of the payload is DATA_V2, the packet is further processed, ++ * otherwise it is forwarded to the UDP stack for delivery to user space. ++ * ++ * @sk: the socket the packet was received on ++ * @skb: the sk_buff containing the actual packet ++ * ++ * Return codes: ++ * 0 : we consumed or dropped packet ++ * >0 : skb should be passed up to userspace as UDP (packet not consumed) ++ * <0 : skb should be resubmitted as proto -N (packet not consumed) + */ -+static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, -+ struct winesync_obj *locked_obj) ++static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) +{ -+ __u32 count = q->count; -+ bool can_wake = true; -+ __u32 i; ++ struct ovpn_peer *peer = NULL; ++ struct ovpn_struct *ovpn; ++ u32 peer_id; ++ u8 opcode; ++ int ret; + -+ lockdep_assert_held(&dev->wait_all_lock); -+ if (locked_obj) -+ lockdep_assert_held(&locked_obj->lock); ++ ovpn = ovpn_from_udp_sock(sk); ++ if (unlikely(!ovpn)) { ++ net_err_ratelimited("%s: cannot obtain ovpn object from UDP socket\n", __func__); ++ goto drop; ++ } + -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ spin_lock(&q->entries[i].obj->lock); ++ /* Make sure the first 4 bytes of the skb data buffer after the UDP header are accessible. ++ * They are required to fetch the OP code, the key ID and the peer ID. ++ */ ++ if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) + 4))) { ++ net_dbg_ratelimited("%s: packet too small\n", __func__); ++ goto drop; + } + -+ for (i = 0; i < count; i++) { -+ if (!is_signaled(q->entries[i].obj, q->owner)) { -+ can_wake = false; -+ break; ++ opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr)); ++ if (likely(opcode == OVPN_DATA_V2)) { ++ peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr)); ++ /* some OpenVPN server implementations send data packets with the peer-id set to ++ * undef. In this case we skip the peer lookup by peer-id and we try with the ++ * transport address ++ */ ++ if (peer_id != OVPN_PEER_ID_UNDEF) { ++ peer = ovpn_peer_lookup_id(ovpn, peer_id); ++ if (!peer) { ++ net_err_ratelimited("%s: received data from unknown peer (id: %d)\n", ++ __func__, peer_id); ++ goto drop; ++ } ++ ++ /* check if this peer changed it's IP address and update state */ ++ ovpn_peer_float(peer, skb); + } + } + -+ if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { -+ for (i = 0; i < count; i++) { -+ struct winesync_obj *obj = q->entries[i].obj; -+ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ obj->u.sem.count--; -+ break; -+ case WINESYNC_TYPE_MUTEX: -+ if (obj->u.mutex.ownerdead) -+ q->ownerdead = true; -+ obj->u.mutex.ownerdead = false; -+ obj->u.mutex.count++; -+ obj->u.mutex.owner = q->owner; -+ break; -+ case WINESYNC_TYPE_EVENT: -+ if (!obj->u.event.manual) -+ obj->u.event.signaled = false; -+ break; ++ if (!peer) { ++ /* might be a control packet or a data packet with undef peer-id */ ++ peer = ovpn_peer_lookup_transp_addr(ovpn, skb); ++ if (unlikely(!peer)) { ++ if (opcode != OVPN_DATA_V2) { ++ netdev_dbg(ovpn->dev, ++ "%s: control packet from unknown peer, sending to userspace", ++ __func__); ++ return 1; + } ++ ++ netdev_dbg(ovpn->dev, ++ "%s: received data with undef peer-id from unknown source\n", ++ __func__); ++ goto drop; + } -+ wake_up_process(q->task); + } + -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ spin_unlock(&q->entries[i].obj->lock); ++ /* pop off outer UDP header */ ++ __skb_pull(skb, sizeof(struct udphdr)); ++ ++ ret = ovpn_recv(ovpn, peer, skb); ++ if (unlikely(ret < 0)) { ++ net_err_ratelimited("%s: cannot handle incoming packet: %d\n", __func__, ret); ++ goto drop; + } ++ ++ /* should this be a non DATA_V2 packet, ret will be >0 and this will instruct the UDP ++ * stack to continue processing this packet as usual (i.e. deliver to user space) ++ */ ++ return ret; ++ ++drop: ++ if (peer) ++ ovpn_peer_put(peer); ++ kfree_skb(skb); ++ return 0; +} + -+static void try_wake_all_obj(struct winesync_device *dev, -+ struct winesync_obj *obj) ++static int ovpn_udp4_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, ++ struct dst_cache *cache, struct sock *sk, ++ struct sk_buff *skb) +{ -+ struct winesync_q_entry *entry; ++ struct rtable *rt; ++ struct flowi4 fl = { ++ .saddr = bind->local.ipv4.s_addr, ++ .daddr = bind->sa.in4.sin_addr.s_addr, ++ .fl4_sport = inet_sk(sk)->inet_sport, ++ .fl4_dport = bind->sa.in4.sin_port, ++ .flowi4_proto = sk->sk_protocol, ++ .flowi4_mark = sk->sk_mark, ++ }; ++ int ret; + -+ lockdep_assert_held(&dev->wait_all_lock); -+ lockdep_assert_held(&obj->lock); ++ local_bh_disable(); ++ rt = dst_cache_get_ip4(cache, &fl.saddr); ++ if (rt) ++ goto transmit; + -+ list_for_each_entry(entry, &obj->all_waiters, node) -+ try_wake_all(dev, entry->q, obj); ++ if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { ++ /* we may end up here when the cached address is not usable anymore. ++ * In this case we reset address/cache and perform a new look up ++ */ ++ fl.saddr = 0; ++ bind->local.ipv4.s_addr = 0; ++ dst_cache_reset(cache); ++ } ++ ++ rt = ip_route_output_flow(sock_net(sk), &fl, sk); ++ if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { ++ fl.saddr = 0; ++ bind->local.ipv4.s_addr = 0; ++ dst_cache_reset(cache); ++ ++ rt = ip_route_output_flow(sock_net(sk), &fl, sk); ++ } ++ ++ if (IS_ERR(rt)) { ++ ret = PTR_ERR(rt); ++ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, ++ &bind->sa.in4, ret); ++ goto err; ++ } ++ dst_cache_set_ip4(cache, &rt->dst, fl.saddr); ++ ++transmit: ++ udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, ++ ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, ++ fl.fl4_dport, false, sk->sk_no_check_tx); ++ ret = 0; ++err: ++ local_bh_enable(); ++ return ret; +} + -+static void try_wake_any_sem(struct winesync_obj *sem) ++#if IS_ENABLED(CONFIG_IPV6) ++static int ovpn_udp6_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, ++ struct dst_cache *cache, struct sock *sk, ++ struct sk_buff *skb) +{ -+ struct winesync_q_entry *entry; ++ struct dst_entry *dst; ++ int ret; + -+ lockdep_assert_held(&sem->lock); ++ struct flowi6 fl = { ++ .saddr = bind->local.ipv6, ++ .daddr = bind->sa.in6.sin6_addr, ++ .fl6_sport = inet_sk(sk)->inet_sport, ++ .fl6_dport = bind->sa.in6.sin6_port, ++ .flowi6_proto = sk->sk_protocol, ++ .flowi6_mark = sk->sk_mark, ++ .flowi6_oif = bind->sa.in6.sin6_scope_id, ++ }; + -+ list_for_each_entry(entry, &sem->any_waiters, node) { -+ struct winesync_q *q = entry->q; ++ local_bh_disable(); ++ dst = dst_cache_get_ip6(cache, &fl.saddr); ++ if (dst) ++ goto transmit; + -+ if (!sem->u.sem.count) -+ break; ++ if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { ++ /* we may end up here when the cached address is not usable anymore. ++ * In this case we reset address/cache and perform a new look up ++ */ ++ fl.saddr = in6addr_any; ++ bind->local.ipv6 = in6addr_any; ++ dst_cache_reset(cache); ++ } + -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ sem->u.sem.count--; -+ wake_up_process(q->task); -+ } ++ dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); ++ if (IS_ERR(dst)) { ++ ret = PTR_ERR(dst); ++ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, ++ &bind->sa.in6, ret); ++ goto err; + } ++ dst_cache_set_ip6(cache, dst, &fl.saddr); ++ ++transmit: ++ udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0, ++ ip6_dst_hoplimit(dst), 0, fl.fl6_sport, ++ fl.fl6_dport, udp_get_no_check6_tx(sk)); ++ ret = 0; ++err: ++ local_bh_enable(); ++ return ret; +} ++#endif + -+static void try_wake_any_mutex(struct winesync_obj *mutex) ++/* Transmit skb utilizing kernel-provided UDP tunneling framework. ++ * ++ * rcu_read_lock should be held on entry. ++ * On return, the skb is consumed. ++ */ ++static int ovpn_udp_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, ++ struct dst_cache *cache, struct sock *sk, ++ struct sk_buff *skb) +{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&mutex->lock); ++ int ret; + -+ list_for_each_entry(entry, &mutex->any_waiters, node) { -+ struct winesync_q *q = entry->q; ++ ovpn_rcu_lockdep_assert_held(); + -+ if (mutex->u.mutex.count == UINT_MAX) -+ break; -+ if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) -+ continue; ++ /* set sk to null if skb is already orphaned */ ++ if (!skb->destructor) ++ skb->sk = NULL; + -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ if (mutex->u.mutex.ownerdead) -+ q->ownerdead = true; -+ mutex->u.mutex.ownerdead = false; -+ mutex->u.mutex.count++; -+ mutex->u.mutex.owner = q->owner; -+ wake_up_process(q->task); -+ } ++ switch (bind->sa.in4.sin_family) { ++ case AF_INET: ++ ret = ovpn_udp4_output(ovpn, bind, cache, sk, skb); ++ break; ++#if IS_ENABLED(CONFIG_IPV6) ++ case AF_INET6: ++ ret = ovpn_udp6_output(ovpn, bind, cache, sk, skb); ++ break; ++#endif ++ default: ++ ret = -EAFNOSUPPORT; ++ break; + } ++ ++ return ret; +} + -+static void try_wake_any_event(struct winesync_obj *event) ++void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, ++ struct sk_buff *skb) +{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&event->lock); ++ struct ovpn_bind *bind; ++ struct socket *sock; ++ int ret = -1; + -+ list_for_each_entry(entry, &event->any_waiters, node) { -+ struct winesync_q *q = entry->q; ++ skb->dev = ovpn->dev; ++ /* no checksum performed at this layer */ ++ skb->ip_summed = CHECKSUM_NONE; + -+ if (!event->u.event.signaled) -+ break; ++ /* get socket info */ ++ sock = peer->sock->sock; ++ if (unlikely(!sock)) { ++ net_dbg_ratelimited("%s: no sock for remote peer\n", __func__); ++ goto out; ++ } + -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ if (!event->u.event.manual) -+ event->u.event.signaled = false; -+ wake_up_process(q->task); -+ } ++ rcu_read_lock(); ++ /* get binding */ ++ bind = rcu_dereference(peer->bind); ++ if (unlikely(!bind)) { ++ net_dbg_ratelimited("%s: no bind for remote peer\n", __func__); ++ goto out_unlock; + } ++ ++ /* crypto layer -> transport (UDP) */ ++ ret = ovpn_udp_output(ovpn, bind, &peer->dst_cache, sock->sk, skb); ++ ++out_unlock: ++ rcu_read_unlock(); ++out: ++ if (ret < 0) ++ kfree_skb(skb); +} + -+static int winesync_create_sem(struct winesync_device *dev, void __user *argp) ++/* Set UDP encapsulation callbacks */ ++int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn) +{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; ++ struct udp_tunnel_sock_cfg cfg = { ++ .sk_user_data = ovpn, ++ .encap_type = UDP_ENCAP_OVPNINUDP, ++ .encap_rcv = ovpn_udp_encap_recv, ++ }; ++ struct ovpn_socket *old_data; + -+ if (args.count > args.max) ++ /* sanity check */ ++ if (sock->sk->sk_protocol != IPPROTO_UDP) { ++ netdev_err(ovpn->dev, "%s: expected UDP socket\n", __func__); + return -EINVAL; ++ } + -+ sem = kzalloc(sizeof(*sem), GFP_KERNEL); -+ if (!sem) -+ return -ENOMEM; -+ -+ init_obj(sem); -+ sem->type = WINESYNC_TYPE_SEM; -+ sem->u.sem.count = args.count; -+ sem->u.sem.max = args.max; ++ /* make sure no pre-existing encapsulation handler exists */ ++ rcu_read_lock(); ++ old_data = rcu_dereference_sk_user_data(sock->sk); ++ rcu_read_unlock(); ++ if (old_data) { ++ if (old_data->ovpn == ovpn) { ++ netdev_dbg(ovpn->dev, ++ "%s: provided socket already owned by this interface\n", ++ __func__); ++ return -EALREADY; ++ } + -+ ret = xa_alloc(&dev->objects, &id, sem, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(sem); -+ return ret; ++ netdev_err(ovpn->dev, "%s: provided socket already taken by other user\n", ++ __func__); ++ return -EBUSY; + } + -+ return put_user(id, &user_args->sem); ++ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); ++ ++ return 0; +} + -+static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) ++/* Detach socket from encapsulation handler and/or other callbacks */ ++void ovpn_udp_socket_detach(struct socket *sock) +{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 id; -+ int ret; ++ struct udp_tunnel_sock_cfg cfg = { }; + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; ++ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); ++} +diff --git a/drivers/net/ovpn-dco/udp.h b/drivers/net/ovpn-dco/udp.h +new file mode 100644 +index 000000000000..be94fb74669b +--- /dev/null ++++ b/drivers/net/ovpn-dco/udp.h +@@ -0,0 +1,25 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: Antonio Quartulli ++ */ + -+ if (!args.owner != !args.count) -+ return -EINVAL; ++#ifndef _NET_OVPN_DCO_UDP_H_ ++#define _NET_OVPN_DCO_UDP_H_ + -+ mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); -+ if (!mutex) -+ return -ENOMEM; ++#include "peer.h" ++#include "ovpnstruct.h" + -+ init_obj(mutex); -+ mutex->type = WINESYNC_TYPE_MUTEX; -+ mutex->u.mutex.count = args.count; -+ mutex->u.mutex.owner = args.owner; ++#include ++#include ++#include ++#include + -+ ret = xa_alloc(&dev->objects, &id, mutex, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(mutex); -+ return ret; -+ } ++int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn); ++void ovpn_udp_socket_detach(struct socket *sock); ++void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, ++ struct sk_buff *skb); + -+ return put_user(id, &user_args->mutex); -+} ++#endif /* _NET_OVPN_DCO_UDP_H_ */ +diff --git a/include/net/netlink.h b/include/net/netlink.h +index 7a2a9d3144ba..335f44871529 100644 +--- a/include/net/netlink.h ++++ b/include/net/netlink.h +@@ -441,6 +441,7 @@ struct nla_policy { + .max = _len \ + } + #define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len) ++#define NLA_POLICY_MAX_LEN(_len) NLA_POLICY_MAX(NLA_BINARY, _len) + + /** + * struct nl_info - netlink source information +diff --git a/include/uapi/linux/ovpn_dco.h b/include/uapi/linux/ovpn_dco.h +new file mode 100644 +index 000000000000..6afee8b3fedd +--- /dev/null ++++ b/include/uapi/linux/ovpn_dco.h +@@ -0,0 +1,265 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* ++ * OpenVPN data channel accelerator ++ * ++ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * ++ * Author: James Yonan ++ * Antonio Quartulli ++ */ + -+static int winesync_create_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ __u32 id; -+ int ret; ++#ifndef _UAPI_LINUX_OVPN_DCO_H_ ++#define _UAPI_LINUX_OVPN_DCO_H_ + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; ++#define OVPN_NL_NAME "ovpn-dco" + -+ event = kzalloc(sizeof(*event), GFP_KERNEL); -+ if (!event) -+ return -ENOMEM; ++#define OVPN_NL_MULTICAST_GROUP_PEERS "peers" + -+ init_obj(event); -+ event->type = WINESYNC_TYPE_EVENT; -+ event->u.event.manual = args.manual; -+ event->u.event.signaled = args.signaled; ++/** ++ * enum ovpn_nl_commands - supported netlink commands ++ */ ++enum ovpn_nl_commands { ++ /** ++ * @OVPN_CMD_UNSPEC: unspecified command to catch errors ++ */ ++ OVPN_CMD_UNSPEC = 0, + -+ ret = xa_alloc(&dev->objects, &id, event, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(event); -+ return ret; -+ } ++ /** ++ * @OVPN_CMD_NEW_PEER: Configure peer with its crypto keys ++ */ ++ OVPN_CMD_NEW_PEER, + -+ return put_user(id, &user_args->event); -+} ++ /** ++ * @OVPN_CMD_SET_PEER: Tweak parameters for an existing peer ++ */ ++ OVPN_CMD_SET_PEER, + -+static int winesync_delete(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_obj *obj; -+ __u32 id; ++ /** ++ * @OVPN_CMD_DEL_PEER: Remove peer from internal table ++ */ ++ OVPN_CMD_DEL_PEER, + -+ if (get_user(id, (__u32 __user *)argp)) -+ return -EFAULT; ++ OVPN_CMD_NEW_KEY, + -+ obj = xa_erase(&dev->objects, id); -+ if (!obj) -+ return -EINVAL; ++ OVPN_CMD_SWAP_KEYS, + -+ put_obj(obj); -+ return 0; -+} ++ OVPN_CMD_DEL_KEY, + -+/* -+ * Actually change the semaphore state, returning -EOVERFLOW if it is made -+ * invalid. -+ */ -+static int put_sem_state(struct winesync_obj *sem, __u32 count) -+{ -+ lockdep_assert_held(&sem->lock); ++ /** ++ * @OVPN_CMD_REGISTER_PACKET: Register for specific packet types to be ++ * forwarded to userspace ++ */ ++ OVPN_CMD_REGISTER_PACKET, + -+ if (sem->u.sem.count + count < sem->u.sem.count || -+ sem->u.sem.count + count > sem->u.sem.max) -+ return -EOVERFLOW; ++ /** ++ * @OVPN_CMD_PACKET: Send a packet from userspace to kernelspace. Also ++ * used to send to userspace packets for which a process had registered ++ * with OVPN_CMD_REGISTER_PACKET ++ */ ++ OVPN_CMD_PACKET, + -+ sem->u.sem.count += count; -+ return 0; -+} ++ /** ++ * @OVPN_CMD_GET_PEER: Retrieve the status of a peer or all peers ++ */ ++ OVPN_CMD_GET_PEER, ++}; + -+static int winesync_put_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 prev_count; -+ int ret; ++enum ovpn_cipher_alg { ++ /** ++ * @OVPN_CIPHER_ALG_NONE: No encryption - reserved for debugging only ++ */ ++ OVPN_CIPHER_ALG_NONE = 0, ++ /** ++ * @OVPN_CIPHER_ALG_AES_GCM: AES-GCM AEAD cipher with any allowed key size ++ */ ++ OVPN_CIPHER_ALG_AES_GCM, ++ /** ++ * @OVPN_CIPHER_ALG_CHACHA20_POLY1305: ChaCha20Poly1305 AEAD cipher ++ */ ++ OVPN_CIPHER_ALG_CHACHA20_POLY1305, ++}; + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; ++enum ovpn_del_peer_reason { ++ __OVPN_DEL_PEER_REASON_FIRST, ++ OVPN_DEL_PEER_REASON_TEARDOWN = __OVPN_DEL_PEER_REASON_FIRST, ++ OVPN_DEL_PEER_REASON_USERSPACE, ++ OVPN_DEL_PEER_REASON_EXPIRED, ++ OVPN_DEL_PEER_REASON_TRANSPORT_ERROR, ++ __OVPN_DEL_PEER_REASON_AFTER_LAST ++}; + -+ sem = get_obj_typed(dev, args.sem, WINESYNC_TYPE_SEM); -+ if (!sem) -+ return -EINVAL; ++enum ovpn_key_slot { ++ __OVPN_KEY_SLOT_FIRST, ++ OVPN_KEY_SLOT_PRIMARY = __OVPN_KEY_SLOT_FIRST, ++ OVPN_KEY_SLOT_SECONDARY, ++ __OVPN_KEY_SLOT_AFTER_LAST, ++}; + -+ if (atomic_read(&sem->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&sem->lock); ++enum ovpn_netlink_attrs { ++ OVPN_ATTR_UNSPEC = 0, ++ OVPN_ATTR_IFINDEX, ++ OVPN_ATTR_NEW_PEER, ++ OVPN_ATTR_SET_PEER, ++ OVPN_ATTR_DEL_PEER, ++ OVPN_ATTR_NEW_KEY, ++ OVPN_ATTR_SWAP_KEYS, ++ OVPN_ATTR_DEL_KEY, ++ OVPN_ATTR_PACKET, ++ OVPN_ATTR_GET_PEER, + -+ prev_count = sem->u.sem.count; -+ ret = put_sem_state(sem, args.count); -+ if (!ret) { -+ try_wake_all_obj(dev, sem); -+ try_wake_any_sem(sem); -+ } ++ __OVPN_ATTR_AFTER_LAST, ++ OVPN_ATTR_MAX = __OVPN_ATTR_AFTER_LAST - 1, ++}; + -+ spin_unlock(&sem->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&sem->lock); ++enum ovpn_netlink_key_dir_attrs { ++ OVPN_KEY_DIR_ATTR_UNSPEC = 0, ++ OVPN_KEY_DIR_ATTR_CIPHER_KEY, ++ OVPN_KEY_DIR_ATTR_NONCE_TAIL, + -+ prev_count = sem->u.sem.count; -+ ret = put_sem_state(sem, args.count); -+ if (!ret) -+ try_wake_any_sem(sem); ++ __OVPN_KEY_DIR_ATTR_AFTER_LAST, ++ OVPN_KEY_DIR_ATTR_MAX = __OVPN_KEY_DIR_ATTR_AFTER_LAST - 1, ++}; + -+ spin_unlock(&sem->lock); -+ } ++enum ovpn_netlink_new_key_attrs { ++ OVPN_NEW_KEY_ATTR_UNSPEC = 0, ++ OVPN_NEW_KEY_ATTR_PEER_ID, ++ OVPN_NEW_KEY_ATTR_KEY_SLOT, ++ OVPN_NEW_KEY_ATTR_KEY_ID, ++ OVPN_NEW_KEY_ATTR_CIPHER_ALG, ++ OVPN_NEW_KEY_ATTR_ENCRYPT_KEY, ++ OVPN_NEW_KEY_ATTR_DECRYPT_KEY, + -+ put_obj(sem); ++ __OVPN_NEW_KEY_ATTR_AFTER_LAST, ++ OVPN_NEW_KEY_ATTR_MAX = __OVPN_NEW_KEY_ATTR_AFTER_LAST - 1, ++}; + -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; ++enum ovpn_netlink_del_key_attrs { ++ OVPN_DEL_KEY_ATTR_UNSPEC = 0, ++ OVPN_DEL_KEY_ATTR_PEER_ID, ++ OVPN_DEL_KEY_ATTR_KEY_SLOT, + -+ return ret; -+} ++ __OVPN_DEL_KEY_ATTR_AFTER_LAST, ++ OVPN_DEL_KEY_ATTR_MAX = __OVPN_DEL_KEY_ATTR_AFTER_LAST - 1, ++}; + -+/* -+ * Actually change the mutex state, returning -EPERM if not the owner. -+ */ -+static int put_mutex_state(struct winesync_obj *mutex, -+ const struct winesync_mutex_args *args) -+{ -+ lockdep_assert_held(&mutex->lock); ++enum ovpn_netlink_swap_keys_attrs { ++ OVPN_SWAP_KEYS_ATTR_UNSPEC = 0, ++ OVPN_SWAP_KEYS_ATTR_PEER_ID, + -+ if (mutex->u.mutex.owner != args->owner) -+ return -EPERM; ++ __OVPN_SWAP_KEYS_ATTR_AFTER_LAST, ++ OVPN_SWAP_KEYS_ATTR_MAX = __OVPN_SWAP_KEYS_ATTR_AFTER_LAST - 1, + -+ if (!--mutex->u.mutex.count) -+ mutex->u.mutex.owner = 0; -+ return 0; -+} ++}; + -+static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 prev_count; -+ int ret; ++enum ovpn_netlink_new_peer_attrs { ++ OVPN_NEW_PEER_ATTR_UNSPEC = 0, ++ OVPN_NEW_PEER_ATTR_PEER_ID, ++ OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE, ++ OVPN_NEW_PEER_ATTR_SOCKET, ++ OVPN_NEW_PEER_ATTR_IPV4, ++ OVPN_NEW_PEER_ATTR_IPV6, ++ OVPN_NEW_PEER_ATTR_LOCAL_IP, + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ if (!args.owner) -+ return -EINVAL; ++ __OVPN_NEW_PEER_ATTR_AFTER_LAST, ++ OVPN_NEW_PEER_ATTR_MAX = __OVPN_NEW_PEER_ATTR_AFTER_LAST - 1, ++}; + -+ mutex = get_obj_typed(dev, args.mutex, WINESYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -EINVAL; ++enum ovpn_netlink_set_peer_attrs { ++ OVPN_SET_PEER_ATTR_UNSPEC = 0, ++ OVPN_SET_PEER_ATTR_PEER_ID, ++ OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL, ++ OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT, + -+ if (atomic_read(&mutex->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&mutex->lock); ++ __OVPN_SET_PEER_ATTR_AFTER_LAST, ++ OVPN_SET_PEER_ATTR_MAX = __OVPN_SET_PEER_ATTR_AFTER_LAST - 1, ++}; + -+ prev_count = mutex->u.mutex.count; -+ ret = put_mutex_state(mutex, &args); -+ if (!ret) { -+ try_wake_all_obj(dev, mutex); -+ try_wake_any_mutex(mutex); -+ } ++enum ovpn_netlink_del_peer_attrs { ++ OVPN_DEL_PEER_ATTR_UNSPEC = 0, ++ OVPN_DEL_PEER_ATTR_REASON, ++ OVPN_DEL_PEER_ATTR_PEER_ID, + -+ spin_unlock(&mutex->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&mutex->lock); ++ __OVPN_DEL_PEER_ATTR_AFTER_LAST, ++ OVPN_DEL_PEER_ATTR_MAX = __OVPN_DEL_PEER_ATTR_AFTER_LAST - 1, ++}; + -+ prev_count = mutex->u.mutex.count; -+ ret = put_mutex_state(mutex, &args); -+ if (!ret) -+ try_wake_any_mutex(mutex); ++enum ovpn_netlink_get_peer_attrs { ++ OVPN_GET_PEER_ATTR_UNSPEC = 0, ++ OVPN_GET_PEER_ATTR_PEER_ID, + -+ spin_unlock(&mutex->lock); -+ } ++ __OVPN_GET_PEER_ATTR_AFTER_LAST, ++ OVPN_GET_PEER_ATTR_MAX = __OVPN_GET_PEER_ATTR_AFTER_LAST - 1, ++}; + -+ put_obj(mutex); ++enum ovpn_netlink_get_peer_response_attrs { ++ OVPN_GET_PEER_RESP_ATTR_UNSPEC = 0, ++ OVPN_GET_PEER_RESP_ATTR_PEER_ID, ++ OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, ++ OVPN_GET_PEER_RESP_ATTR_IPV4, ++ OVPN_GET_PEER_RESP_ATTR_IPV6, ++ OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, ++ OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, ++ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, ++ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, ++ OVPN_GET_PEER_RESP_ATTR_RX_BYTES, ++ OVPN_GET_PEER_RESP_ATTR_TX_BYTES, ++ OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, ++ OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, + -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; ++ __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST, ++ OVPN_GET_PEER_RESP_ATTR_MAX = __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST - 1, ++}; + -+ return ret; -+} ++enum ovpn_netlink_peer_stats_attrs { ++ OVPN_PEER_STATS_ATTR_UNSPEC = 0, ++ OVPN_PEER_STATS_BYTES, ++ OVPN_PEER_STATS_PACKETS, + -+static int winesync_read_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 id; ++ __OVPN_PEER_STATS_ATTR_AFTER_LAST, ++ OVPN_PEER_STATS_ATTR_MAX = __OVPN_PEER_STATS_ATTR_AFTER_LAST - 1, ++}; + -+ if (get_user(id, &user_args->sem)) -+ return -EFAULT; ++enum ovpn_netlink_peer_attrs { ++ OVPN_PEER_ATTR_UNSPEC = 0, ++ OVPN_PEER_ATTR_PEER_ID, ++ OVPN_PEER_ATTR_SOCKADDR_REMOTE, ++ OVPN_PEER_ATTR_IPV4, ++ OVPN_PEER_ATTR_IPV6, ++ OVPN_PEER_ATTR_LOCAL_IP, ++ OVPN_PEER_ATTR_KEEPALIVE_INTERVAL, ++ OVPN_PEER_ATTR_KEEPALIVE_TIMEOUT, ++ OVPN_PEER_ATTR_ENCRYPT_KEY, ++ OVPN_PEER_ATTR_DECRYPT_KEY, ++ OVPN_PEER_ATTR_RX_STATS, ++ OVPN_PEER_ATTR_TX_STATS, + -+ sem = get_obj_typed(dev, id, WINESYNC_TYPE_SEM); -+ if (!sem) -+ return -EINVAL; ++ __OVPN_PEER_ATTR_AFTER_LAST, ++ OVPN_PEER_ATTR_MAX = __OVPN_PEER_ATTR_AFTER_LAST - 1, ++}; + -+ args.sem = id; -+ spin_lock(&sem->lock); -+ args.count = sem->u.sem.count; -+ args.max = sem->u.sem.max; -+ spin_unlock(&sem->lock); ++enum ovpn_netlink_packet_attrs { ++ OVPN_PACKET_ATTR_UNSPEC = 0, ++ OVPN_PACKET_ATTR_PACKET, ++ OVPN_PACKET_ATTR_PEER_ID, + -+ put_obj(sem); ++ __OVPN_PACKET_ATTR_AFTER_LAST, ++ OVPN_PACKET_ATTR_MAX = __OVPN_PACKET_ATTR_AFTER_LAST - 1, ++}; + -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} ++enum ovpn_ifla_attrs { ++ IFLA_OVPN_UNSPEC = 0, ++ IFLA_OVPN_MODE, + -+static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 id; -+ int ret; ++ __IFLA_OVPN_AFTER_LAST, ++ IFLA_OVPN_MAX = __IFLA_OVPN_AFTER_LAST - 1, ++}; ++ ++enum ovpn_mode { ++ __OVPN_MODE_FIRST = 0, ++ OVPN_MODE_P2P = __OVPN_MODE_FIRST, ++ OVPN_MODE_MP, ++ ++ __OVPN_MODE_AFTER_LAST, ++}; ++ ++#endif /* _UAPI_LINUX_OVPN_DCO_H_ */ +diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h +index 4828794efcf8..8008c762e6b8 100644 +--- a/include/uapi/linux/udp.h ++++ b/include/uapi/linux/udp.h +@@ -43,5 +43,6 @@ struct udphdr { + #define UDP_ENCAP_GTP1U 5 /* 3GPP TS 29.060 */ + #define UDP_ENCAP_RXRPC 6 + #define TCP_ENCAP_ESPINTCP 7 /* Yikes, this is really xfrm encap types. */ ++#define UDP_ENCAP_OVPNINUDP 8 /* OpenVPN over UDP connection */ + + #endif /* _UAPI_LINUX_UDP_H */ +-- +2.38.0.rc1.8.g2a7d63a245 + +From 14903eee0b5577711272732705260cb83e5e0777 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:26:01 +0200 +Subject: [PATCH 05/16] mm/demotion: Memory tiers and demotion + +The current kernel has the basic memory tiering support: Inactive pages on +a higher tier NUMA node can be migrated (demoted) to a lower tier NUMA +node to make room for new allocations on the higher tier NUMA node. +Frequently accessed pages on a lower tier NUMA node can be migrated +(promoted) to a higher tier NUMA node to improve the performance. + +In the current kernel, memory tiers are defined implicitly via a demotion +path relationship between NUMA nodes, which is created during the kernel +initialization and updated when a NUMA node is hot-added or hot-removed. +The current implementation puts all nodes with CPU into the highest tier, +and builds the tier hierarchy tier-by-tier by establishing the per-node +demotion targets based on the distances between nodes. + +This current memory tier kernel implementation needs to be improved for +several important use cases: + +* The current tier initialization code always initializes each + memory-only NUMA node into a lower tier. But a memory-only NUMA node + may have a high performance memory device (e.g. a DRAM-backed + memory-only node on a virtual machine) and that should be put into a + higher tier. + +* The current tier hierarchy always puts CPU nodes into the top tier. + But on a system with HBM (e.g. GPU memory) devices, these memory-only + HBM NUMA nodes should be in the top tier, and DRAM nodes with CPUs are + better to be placed into the next lower tier. + +* Also because the current tier hierarchy always puts CPU nodes into the + top tier, when a CPU is hot-added (or hot-removed) and triggers a memory + node from CPU-less into a CPU node (or vice versa), the memory tier + hierarchy gets changed, even though no memory node is added or removed. + This can make the tier hierarchy unstable and make it difficult to + support tier-based memory accounting. + +* A higher tier node can only be demoted to nodes with shortest distance + on the next lower tier as defined by the demotion path, not any other + node from any lower tier. This strict, demotion order does not work in + all use cases (e.g. some use cases may want to allow cross-socket + demotion to another node in the same demotion tier as a fallback when + the preferred demotion node is out of space), and has resulted in the + feature request for an interface to override the system-wide, per-node + demotion order from the userspace. This demotion order is also + inconsistent with the page allocation fallback order when all the nodes + in a higher tier are out of space: The page allocation can fall back to + any node from any lower tier, whereas the demotion order doesn't allow + that. + +This patch series make the creation of memory tiers explicit under the +control of device driver. + +Memory Tier Initialization +========================== + +Linux kernel presents memory devices as NUMA nodes and each memory device +is of a specific type. The memory type of a device is represented by its +abstract distance. A memory tier corresponds to a range of abstract +distance. This allows for classifying memory devices with a specific +performance range into a memory tier. + +By default, all memory nodes are assigned to the default tier with +abstract distance 512. + +A device driver can move its memory nodes from the default tier. For +example, PMEM can move its memory nodes below the default tier, whereas +GPU can move its memory nodes above the default tier. + +The kernel initialization code makes the decision on which exact tier a +memory node should be assigned to based on the requests from the device +drivers as well as the memory device hardware information provided by the +firmware. + +Hot-adding/removing CPUs doesn't affect memory tier hierarchy. + +This patch (of 10): + +In the current kernel, memory tiers are defined implicitly via a demotion +path relationship between NUMA nodes, which is created during the kernel +initialization and updated when a NUMA node is hot-added or hot-removed. +The current implementation puts all nodes with CPU into the highest tier, +and builds the tier hierarchy by establishing the per-node demotion +targets based on the distances between nodes. + +This current memory tier kernel implementation needs to be improved for +several important use cases, + +The current tier initialization code always initializes each memory-only +NUMA node into a lower tier. But a memory-only NUMA node may have a high +performance memory device (e.g. a DRAM-backed memory-only node on a +virtual machine) that should be put into a higher tier. + +The current tier hierarchy always puts CPU nodes into the top tier. But +on a system with HBM or GPU devices, the memory-only NUMA nodes mapping +these devices should be in the top tier, and DRAM nodes with CPUs are +better to be placed into the next lower tier. + +With current kernel higher tier node can only be demoted to nodes with +shortest distance on the next lower tier as defined by the demotion path, +not any other node from any lower tier. This strict, demotion order does +not work in all use cases (e.g. some use cases may want to allow +cross-socket demotion to another node in the same demotion tier as a +fallback when the preferred demotion node is out of space), This demotion +order is also inconsistent with the page allocation fallback order when +all the nodes in a higher tier are out of space: The page allocation can +fall back to any node from any lower tier, whereas the demotion order +doesn't allow that. + +This patch series address the above by defining memory tiers explicitly. + +Linux kernel presents memory devices as NUMA nodes and each memory device +is of a specific type. The memory type of a device is represented by its +abstract distance. A memory tier corresponds to a range of abstract +distance. This allows for classifying memory devices with a specific +performance range into a memory tier. + +This patch configures the range/chunk size to be 128. The default DRAM +abstract distance is 512. We can have 4 memory tiers below the default +DRAM with abstract distance range 0 - 127, 127 - 255, 256- 383, 384 - 511. +Faster memory devices can be placed in these faster(higher) memory tiers. +Slower memory devices like persistent memory will have abstract distance +higher than the default DRAM level. + +Signed-off-by: Peter Jung +--- + .../ABI/testing/sysfs-kernel-mm-memory-tiers | 25 + + drivers/dax/kmem.c | 42 +- + include/linux/memory-tiers.h | 102 +++ + include/linux/migrate.h | 15 - + include/linux/mmzone.h | 3 + + include/linux/node.h | 5 - + include/linux/nodemask.h | 15 +- + kernel/sched/fair.c | 1 + + mm/Makefile | 1 + + mm/huge_memory.c | 1 + + mm/memory-tiers.c | 732 ++++++++++++++++++ + mm/memory.c | 1 + + mm/migrate.c | 453 +---------- + mm/mprotect.c | 1 + + mm/vmscan.c | 59 +- + mm/vmstat.c | 4 - + 16 files changed, 963 insertions(+), 497 deletions(-) + create mode 100644 Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers + create mode 100644 include/linux/memory-tiers.h + create mode 100644 mm/memory-tiers.c + +diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +new file mode 100644 +index 000000000000..45985e411f13 +--- /dev/null ++++ b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +@@ -0,0 +1,25 @@ ++What: /sys/devices/virtual/memory_tiering/ ++Date: August 2022 ++Contact: Linux memory management mailing list ++Description: A collection of all the memory tiers allocated. + -+ if (get_user(id, &user_args->mutex)) -+ return -EFAULT; ++ Individual memory tier details are contained in subdirectories ++ named by the abstract distance of the memory tier. + -+ mutex = get_obj_typed(dev, id, WINESYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -EINVAL; ++ /sys/devices/virtual/memory_tiering/memory_tierN/ + -+ args.mutex = id; -+ spin_lock(&mutex->lock); -+ args.count = mutex->u.mutex.count; -+ args.owner = mutex->u.mutex.owner; -+ ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; -+ spin_unlock(&mutex->lock); + -+ put_obj(mutex); ++What: /sys/devices/virtual/memory_tiering/memory_tierN/ ++ /sys/devices/virtual/memory_tiering/memory_tierN/nodes ++Date: August 2022 ++Contact: Linux memory management mailing list ++Description: Directory with details of a specific memory tier + -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return ret; -+} ++ This is the directory containing information about a particular ++ memory tier, memtierN, where N is derived based on abstract distance. + -+static int winesync_read_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ __u32 id; ++ A smaller value of N implies a higher (faster) memory tier in the ++ hierarchy. + -+ if (get_user(id, &user_args->event)) -+ return -EFAULT; ++ nodes: NUMA nodes that are part of this memory tier. + -+ event = get_obj_typed(dev, id, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; +diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c +index a37622060fff..4852a2dbdb27 100644 +--- a/drivers/dax/kmem.c ++++ b/drivers/dax/kmem.c +@@ -11,9 +11,17 @@ + #include + #include + #include ++#include + #include "dax-private.h" + #include "bus.h" + ++/* ++ * Default abstract distance assigned to the NUMA node onlined ++ * by DAX/kmem if the low level platform driver didn't initialize ++ * one for this NUMA node. ++ */ ++#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) + -+ args.event = id; -+ spin_lock(&event->lock); -+ args.manual = event->u.event.manual; -+ args.signaled = event->u.event.signaled; -+ spin_unlock(&event->lock); + /* Memory resource name used for add_memory_driver_managed(). */ + static const char *kmem_name; + /* Set if any memory will remain added when the driver will be unloaded. */ +@@ -41,6 +49,7 @@ struct dax_kmem_data { + struct resource *res[]; + }; + ++static struct memory_dev_type *dax_slowmem_type; + static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + { + struct device *dev = &dev_dax->dev; +@@ -79,11 +88,13 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + return -EINVAL; + } + ++ init_node_memory_type(numa_node, dax_slowmem_type); ++ ++ rc = -ENOMEM; + data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL); + if (!data) +- return -ENOMEM; ++ goto err_dax_kmem_data; + +- rc = -ENOMEM; + data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); + if (!data->res_name) + goto err_res_name; +@@ -155,6 +166,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + kfree(data->res_name); + err_res_name: + kfree(data); ++err_dax_kmem_data: ++ clear_node_memory_type(numa_node, dax_slowmem_type); + return rc; + } + +@@ -162,6 +175,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) + static void dev_dax_kmem_remove(struct dev_dax *dev_dax) + { + int i, success = 0; ++ int node = dev_dax->target_node; + struct device *dev = &dev_dax->dev; + struct dax_kmem_data *data = dev_get_drvdata(dev); + +@@ -198,6 +212,14 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) + kfree(data->res_name); + kfree(data); + dev_set_drvdata(dev, NULL); ++ /* ++ * Clear the memtype association on successful unplug. ++ * If not, we have memory blocks left which can be ++ * offlined/onlined later. We need to keep memory_dev_type ++ * for that. This implies this reference will be around ++ * till next reboot. ++ */ ++ clear_node_memory_type(node, dax_slowmem_type); + } + } + #else +@@ -228,9 +250,22 @@ static int __init dax_kmem_init(void) + if (!kmem_name) + return -ENOMEM; + ++ dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE); ++ if (IS_ERR(dax_slowmem_type)) { ++ rc = PTR_ERR(dax_slowmem_type); ++ goto err_dax_slowmem_type; ++ } + -+ put_obj(event); + rc = dax_driver_register(&device_dax_kmem_driver); + if (rc) +- kfree_const(kmem_name); ++ goto error_dax_driver; + -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} ++ return rc; ++ ++error_dax_driver: ++ destroy_memory_type(dax_slowmem_type); ++err_dax_slowmem_type: ++ kfree_const(kmem_name); + return rc; + } + +@@ -239,6 +274,7 @@ static void __exit dax_kmem_exit(void) + dax_driver_unregister(&device_dax_kmem_driver); + if (!any_hotremove_failed) + kfree_const(kmem_name); ++ destroy_memory_type(dax_slowmem_type); + } + + MODULE_AUTHOR("Intel Corporation"); +diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h +new file mode 100644 +index 000000000000..965009aa01d7 +--- /dev/null ++++ b/include/linux/memory-tiers.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_MEMORY_TIERS_H ++#define _LINUX_MEMORY_TIERS_H + ++#include ++#include ++#include ++#include +/* -+ * Actually change the mutex state to mark its owner as dead. ++ * Each tier cover a abstrace distance chunk size of 128 + */ -+static void put_mutex_ownerdead_state(struct winesync_obj *mutex) -+{ -+ lockdep_assert_held(&mutex->lock); ++#define MEMTIER_CHUNK_BITS 7 ++#define MEMTIER_CHUNK_SIZE (1 << MEMTIER_CHUNK_BITS) ++/* ++ * Smaller abstract distance values imply faster (higher) memory tiers. Offset ++ * the DRAM adistance so that we can accommodate devices with a slightly lower ++ * adistance value (slightly faster) than default DRAM adistance to be part of ++ * the same memory tier. ++ */ ++#define MEMTIER_ADISTANCE_DRAM ((4 * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1)) ++#define MEMTIER_HOTPLUG_PRIO 100 ++ ++struct memory_tier; ++struct memory_dev_type { ++ /* list of memory types that are part of same tier as this type */ ++ struct list_head tier_sibiling; ++ /* abstract distance for this specific memory type */ ++ int adistance; ++ /* Nodes of same abstract distance */ ++ nodemask_t nodes; ++ struct kref kref; ++}; + -+ mutex->u.mutex.ownerdead = true; -+ mutex->u.mutex.owner = 0; -+ mutex->u.mutex.count = 0; ++#ifdef CONFIG_NUMA ++extern bool numa_demotion_enabled; ++struct memory_dev_type *alloc_memory_type(int adistance); ++void destroy_memory_type(struct memory_dev_type *memtype); ++void init_node_memory_type(int node, struct memory_dev_type *default_type); ++void clear_node_memory_type(int node, struct memory_dev_type *memtype); ++#ifdef CONFIG_MIGRATION ++int next_demotion_node(int node); ++void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); ++bool node_is_toptier(int node); ++#else ++static inline int next_demotion_node(int node) ++{ ++ return NUMA_NO_NODE; +} + -+static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) ++static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) +{ -+ struct winesync_obj *obj; -+ unsigned long id; -+ __u32 owner; -+ -+ if (get_user(owner, (__u32 __user *)argp)) -+ return -EFAULT; -+ if (!owner) -+ return -EINVAL; -+ -+ rcu_read_lock(); -+ -+ xa_for_each(&dev->objects, id, obj) { -+ if (!kref_get_unless_zero(&obj->refcount)) -+ continue; ++ *targets = NODE_MASK_NONE; ++} + -+ if (obj->type != WINESYNC_TYPE_MUTEX) { -+ put_obj(obj); -+ continue; -+ } ++static inline bool node_is_toptier(int node) ++{ ++ return true; ++} ++#endif + -+ if (atomic_read(&obj->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&obj->lock); ++#else + -+ if (obj->u.mutex.owner == owner) { -+ put_mutex_ownerdead_state(obj); -+ try_wake_all_obj(dev, obj); -+ try_wake_any_mutex(obj); -+ } ++#define numa_demotion_enabled false ++/* ++ * CONFIG_NUMA implementation returns non NULL error. ++ */ ++static inline struct memory_dev_type *alloc_memory_type(int adistance) ++{ ++ return NULL; ++} + -+ spin_unlock(&obj->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&obj->lock); ++static inline void destroy_memory_type(struct memory_dev_type *memtype) ++{ + -+ if (obj->u.mutex.owner == owner) { -+ put_mutex_ownerdead_state(obj); -+ try_wake_any_mutex(obj); -+ } ++} + -+ spin_unlock(&obj->lock); -+ } ++static inline void init_node_memory_type(int node, struct memory_dev_type *default_type) ++{ + -+ put_obj(obj); -+ } ++} + -+ rcu_read_unlock(); ++static inline void clear_node_memory_type(int node, struct memory_dev_type *memtype) ++{ + -+ return 0; +} + -+static int winesync_set_event(struct winesync_device *dev, void __user *argp, -+ bool pulse) ++static inline int next_demotion_node(int node) +{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ bool prev_state; ++ return NUMA_NO_NODE; ++} + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; ++static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) ++{ ++ *targets = NODE_MASK_NONE; ++} + -+ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; ++static inline bool node_is_toptier(int node) ++{ ++ return true; ++} ++#endif /* CONFIG_NUMA */ ++#endif /* _LINUX_MEMORY_TIERS_H */ +diff --git a/include/linux/migrate.h b/include/linux/migrate.h +index 22c0a0cf5e0c..704a04f5a074 100644 +--- a/include/linux/migrate.h ++++ b/include/linux/migrate.h +@@ -100,21 +100,6 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping, + + #endif /* CONFIG_MIGRATION */ + +-#if defined(CONFIG_MIGRATION) && defined(CONFIG_NUMA) +-extern void set_migration_target_nodes(void); +-extern void migrate_on_reclaim_init(void); +-extern bool numa_demotion_enabled; +-extern int next_demotion_node(int node); +-#else +-static inline void set_migration_target_nodes(void) {} +-static inline void migrate_on_reclaim_init(void) {} +-static inline int next_demotion_node(int node) +-{ +- return NUMA_NO_NODE; +-} +-#define numa_demotion_enabled false +-#endif +- + #ifdef CONFIG_COMPACTION + bool PageMovable(struct page *page); + void __SetPageMovable(struct page *page, const struct movable_operations *ops); +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index e24b40c52468..7d78133fe8dd 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -1012,6 +1012,9 @@ typedef struct pglist_data { + /* Per-node vmstats */ + struct per_cpu_nodestat __percpu *per_cpu_nodestats; + atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; ++#ifdef CONFIG_NUMA ++ struct memory_tier __rcu *memtier; ++#endif + } pg_data_t; + + #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) +diff --git a/include/linux/node.h b/include/linux/node.h +index 40d641a8bfb0..9ec680dd607f 100644 +--- a/include/linux/node.h ++++ b/include/linux/node.h +@@ -185,9 +185,4 @@ static inline void register_hugetlbfs_with_node(node_registration_func_t reg, + + #define to_node(device) container_of(device, struct node, dev) + +-static inline bool node_is_toptier(int node) +-{ +- return node_state(node, N_CPU); +-} +- + #endif /* _LINUX_NODE_H_ */ +diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h +index 4b71a96190a8..ac5b6a371be5 100644 +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -504,12 +504,21 @@ static inline int num_node_state(enum node_states state) + static inline int node_random(const nodemask_t *maskp) + { + #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) +- int w, bit = NUMA_NO_NODE; ++ int w, bit; + + w = nodes_weight(*maskp); +- if (w) ++ switch (w) { ++ case 0: ++ bit = NUMA_NO_NODE; ++ break; ++ case 1: ++ bit = first_node(*maskp); ++ break; ++ default: + bit = bitmap_ord_to_pos(maskp->bits, +- get_random_int() % w, MAX_NUMNODES); ++ get_random_int() % w, MAX_NUMNODES); ++ break; ++ } + return bit; + #else + return 0; +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index 74d773040b54..1d3e53fed1a4 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -40,6 +40,7 @@ + + #include + #include ++#include + #include + #include + #include +diff --git a/mm/Makefile b/mm/Makefile +index 9a564f836403..488f604e77e0 100644 +--- a/mm/Makefile ++++ b/mm/Makefile +@@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/ + obj-$(CONFIG_FAILSLAB) += failslab.o + obj-$(CONFIG_MEMTEST) += memtest.o + obj-$(CONFIG_MIGRATION) += migrate.o ++obj-$(CONFIG_NUMA) += memory-tiers.o + obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o + obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o + obj-$(CONFIG_PAGE_COUNTER) += page_counter.o +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index e9414ee57c5b..6eb4b1799b79 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include + + #include + #include +diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c +new file mode 100644 +index 000000000000..f116b7b6333e +--- /dev/null ++++ b/mm/memory-tiers.c +@@ -0,0 +1,732 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include + -+ if (atomic_read(&event->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&event->lock); ++#include "internal.h" + -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ try_wake_all_obj(dev, event); -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; ++struct memory_tier { ++ /* hierarchy of memory tiers */ ++ struct list_head list; ++ /* list of all memory types part of this tier */ ++ struct list_head memory_types; ++ /* ++ * start value of abstract distance. memory tier maps ++ * an abstract distance range, ++ * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE ++ */ ++ int adistance_start; ++ struct device dev; ++ /* All the nodes that are part of all the lower memory tiers. */ ++ nodemask_t lower_tier_mask; ++}; + -+ spin_unlock(&event->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&event->lock); ++struct demotion_nodes { ++ nodemask_t preferred; ++}; + -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; ++struct node_memory_type_map { ++ struct memory_dev_type *memtype; ++ int map_count; ++}; + -+ spin_unlock(&event->lock); -+ } ++static DEFINE_MUTEX(memory_tier_lock); ++static LIST_HEAD(memory_tiers); ++static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; ++static struct memory_dev_type *default_dram_type; + -+ put_obj(event); ++static struct bus_type memory_tier_subsys = { ++ .name = "memory_tiering", ++ .dev_name = "memory_tier", ++}; + -+ if (put_user(prev_state, &user_args->signaled)) -+ return -EFAULT; ++#ifdef CONFIG_MIGRATION ++static int top_tier_adistance; ++/* ++ * node_demotion[] examples: ++ * ++ * Example 1: ++ * ++ * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes. ++ * ++ * node distances: ++ * node 0 1 2 3 ++ * 0 10 20 30 40 ++ * 1 20 10 40 30 ++ * 2 30 40 10 40 ++ * 3 40 30 40 10 ++ * ++ * memory_tiers0 = 0-1 ++ * memory_tiers1 = 2-3 ++ * ++ * node_demotion[0].preferred = 2 ++ * node_demotion[1].preferred = 3 ++ * node_demotion[2].preferred = ++ * node_demotion[3].preferred = ++ * ++ * Example 2: ++ * ++ * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node. ++ * ++ * node distances: ++ * node 0 1 2 ++ * 0 10 20 30 ++ * 1 20 10 30 ++ * 2 30 30 10 ++ * ++ * memory_tiers0 = 0-2 ++ * ++ * node_demotion[0].preferred = ++ * node_demotion[1].preferred = ++ * node_demotion[2].preferred = ++ * ++ * Example 3: ++ * ++ * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node. ++ * ++ * node distances: ++ * node 0 1 2 ++ * 0 10 20 30 ++ * 1 20 10 40 ++ * 2 30 40 10 ++ * ++ * memory_tiers0 = 1 ++ * memory_tiers1 = 0 ++ * memory_tiers2 = 2 ++ * ++ * node_demotion[0].preferred = 2 ++ * node_demotion[1].preferred = 0 ++ * node_demotion[2].preferred = ++ * ++ */ ++static struct demotion_nodes *node_demotion __read_mostly; ++#endif /* CONFIG_MIGRATION */ + -+ return 0; ++static inline struct memory_tier *to_memory_tier(struct device *device) ++{ ++ return container_of(device, struct memory_tier, dev); +} + -+static int winesync_reset_event(struct winesync_device *dev, void __user *argp) ++static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier) +{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ bool prev_state; ++ nodemask_t nodes = NODE_MASK_NONE; ++ struct memory_dev_type *memtype; + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ -+ put_obj(event); -+ -+ if (put_user(prev_state, &user_args->signaled)) -+ return -EFAULT; ++ list_for_each_entry(memtype, &memtier->memory_types, tier_sibiling) ++ nodes_or(nodes, nodes, memtype->nodes); + -+ return 0; ++ return nodes; +} + -+static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) ++static void memory_tier_device_release(struct device *dev) +{ -+ int ret = 0; -+ -+ do { -+ if (signal_pending(current)) { -+ ret = -ERESTARTSYS; -+ break; -+ } ++ struct memory_tier *tier = to_memory_tier(dev); ++ /* ++ * synchronize_rcu in clear_node_memory_tier makes sure ++ * we don't have rcu access to this memory tier. ++ */ ++ kfree(tier); ++} + -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (atomic_read(&q->signaled) != -1) { -+ ret = 0; -+ break; -+ } -+ ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); -+ } while (ret < 0); -+ __set_current_state(TASK_RUNNING); ++static ssize_t nodes_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ int ret; ++ nodemask_t nmask; + ++ mutex_lock(&memory_tier_lock); ++ nmask = get_memtier_nodemask(to_memory_tier(dev)); ++ ret = sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&nmask)); ++ mutex_unlock(&memory_tier_lock); + return ret; +} ++static DEVICE_ATTR_RO(nodes); + -+/* -+ * Allocate and initialize the winesync_q structure, but do not queue us yet. -+ * Also, calculate the relative timeout. -+ */ -+static int setup_wait(struct winesync_device *dev, -+ const struct winesync_wait_args *args, bool all, -+ ktime_t *ret_timeout, struct winesync_q **ret_q) -+{ -+ const __u32 count = args->count; -+ struct winesync_q *q; -+ ktime_t timeout = 0; -+ __u32 total_count; -+ __u32 *ids; -+ __u32 i, j; -+ -+ if (!args->owner) -+ return -EINVAL; ++static struct attribute *memtier_dev_attrs[] = { ++ &dev_attr_nodes.attr, ++ NULL ++}; + -+ if (args->timeout) { -+ struct timespec64 to; ++static const struct attribute_group memtier_dev_group = { ++ .attrs = memtier_dev_attrs, ++}; + -+ if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) -+ return -EFAULT; -+ if (!timespec64_valid(&to)) -+ return -EINVAL; ++static const struct attribute_group *memtier_dev_groups[] = { ++ &memtier_dev_group, ++ NULL ++}; + -+ timeout = timespec64_to_ns(&to); -+ } ++static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype) ++{ ++ int ret; ++ bool found_slot = false; ++ struct memory_tier *memtier, *new_memtier; ++ int adistance = memtype->adistance; ++ unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE; + -+ total_count = count; -+ if (args->alert) -+ total_count++; ++ lockdep_assert_held_once(&memory_tier_lock); + -+ ids = kmalloc_array(total_count, sizeof(*ids), GFP_KERNEL); -+ if (!ids) -+ return -ENOMEM; -+ if (copy_from_user(ids, u64_to_user_ptr(args->objs), -+ array_size(count, sizeof(*ids)))) { -+ kfree(ids); -+ return -EFAULT; ++ adistance = round_down(adistance, memtier_adistance_chunk_size); ++ /* ++ * If the memtype is already part of a memory tier, ++ * just return that. ++ */ ++ if (!list_empty(&memtype->tier_sibiling)) { ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ if (adistance == memtier->adistance_start) ++ return memtier; ++ } ++ WARN_ON(1); ++ return ERR_PTR(-EINVAL); + } -+ if (args->alert) -+ ids[count] = args->alert; + -+ q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); -+ if (!q) { -+ kfree(ids); -+ return -ENOMEM; ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ if (adistance == memtier->adistance_start) { ++ goto link_memtype; ++ } else if (adistance < memtier->adistance_start) { ++ found_slot = true; ++ break; ++ } + } -+ q->task = current; -+ q->owner = args->owner; -+ atomic_set(&q->signaled, -1); -+ q->all = all; -+ q->ownerdead = false; -+ q->count = count; + -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = get_obj(dev, ids[i]); ++ new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL); ++ if (!new_memtier) ++ return ERR_PTR(-ENOMEM); + -+ if (!obj) -+ goto err; ++ new_memtier->adistance_start = adistance; ++ INIT_LIST_HEAD(&new_memtier->list); ++ INIT_LIST_HEAD(&new_memtier->memory_types); ++ if (found_slot) ++ list_add_tail(&new_memtier->list, &memtier->list); ++ else ++ list_add_tail(&new_memtier->list, &memory_tiers); + -+ if (all) { -+ /* Check that the objects are all distinct. */ -+ for (j = 0; j < i; j++) { -+ if (obj == q->entries[j].obj) { -+ put_obj(obj); -+ goto err; -+ } -+ } -+ } ++ new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS; ++ new_memtier->dev.bus = &memory_tier_subsys; ++ new_memtier->dev.release = memory_tier_device_release; ++ new_memtier->dev.groups = memtier_dev_groups; + -+ entry->obj = obj; -+ entry->q = q; -+ entry->index = i; ++ ret = device_register(&new_memtier->dev); ++ if (ret) { ++ list_del(&memtier->list); ++ put_device(&memtier->dev); ++ return ERR_PTR(ret); + } ++ memtier = new_memtier; + -+ kfree(ids); ++link_memtype: ++ list_add(&memtype->tier_sibiling, &memtier->memory_types); ++ return memtier; ++} + -+ *ret_q = q; -+ *ret_timeout = timeout; -+ return 0; ++static struct memory_tier *__node_get_memory_tier(int node) ++{ ++ pg_data_t *pgdat; + -+err: -+ for (j = 0; j < i; j++) -+ put_obj(q->entries[j].obj); -+ kfree(ids); -+ kfree(q); -+ return -EINVAL; ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return NULL; ++ /* ++ * Since we hold memory_tier_lock, we can avoid ++ * RCU read locks when accessing the details. No ++ * parallel updates are possible here. ++ */ ++ return rcu_dereference_check(pgdat->memtier, ++ lockdep_is_held(&memory_tier_lock)); +} + -+static void try_wake_any_obj(struct winesync_obj *obj) ++#ifdef CONFIG_MIGRATION ++bool node_is_toptier(int node) +{ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ try_wake_any_sem(obj); -+ break; -+ case WINESYNC_TYPE_MUTEX: -+ try_wake_any_mutex(obj); -+ break; -+ case WINESYNC_TYPE_EVENT: -+ try_wake_any_event(obj); -+ break; ++ bool toptier; ++ pg_data_t *pgdat; ++ struct memory_tier *memtier; ++ ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return false; ++ ++ rcu_read_lock(); ++ memtier = rcu_dereference(pgdat->memtier); ++ if (!memtier) { ++ toptier = true; ++ goto out; + } ++ if (memtier->adistance_start <= top_tier_adistance) ++ toptier = true; ++ else ++ toptier = false; ++out: ++ rcu_read_unlock(); ++ return toptier; +} + -+static int winesync_wait_any(struct winesync_device *dev, void __user *argp) ++void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) +{ -+ struct winesync_wait_args args; -+ struct winesync_q *q; -+ __u32 i, total_count; -+ ktime_t timeout; -+ int signaled; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, false, &timeout, &q); -+ if (ret < 0) -+ return ret; ++ struct memory_tier *memtier; + -+ total_count = args.count; -+ if (args.alert) -+ total_count++; ++ /* ++ * pg_data_t.memtier updates includes a synchronize_rcu() ++ * which ensures that we either find NULL or a valid memtier ++ * in NODE_DATA. protect the access via rcu_read_lock(); ++ */ ++ rcu_read_lock(); ++ memtier = rcu_dereference(pgdat->memtier); ++ if (memtier) ++ *targets = memtier->lower_tier_mask; ++ else ++ *targets = NODE_MASK_NONE; ++ rcu_read_unlock(); ++} + -+ /* queue ourselves */ ++/** ++ * next_demotion_node() - Get the next node in the demotion path ++ * @node: The starting node to lookup the next node ++ * ++ * Return: node id for next memory node in the demotion path hierarchy ++ * from @node; NUMA_NO_NODE if @node is terminal. This does not keep ++ * @node online or guarantee that it *continues* to be the next demotion ++ * target. ++ */ ++int next_demotion_node(int node) ++{ ++ struct demotion_nodes *nd; ++ int target; + -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; ++ if (!node_demotion) ++ return NUMA_NO_NODE; + -+ spin_lock(&obj->lock); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ spin_unlock(&obj->lock); -+ } ++ nd = &node_demotion[node]; + + /* -+ * Check if we are already signaled. ++ * node_demotion[] is updated without excluding this ++ * function from running. + * -+ * Note that the API requires that normal objects are checked before -+ * the alert event. Hence we queue the alert event last, and check -+ * objects in order. ++ * Make sure to use RCU over entire code blocks if ++ * node_demotion[] reads need to be consistent. + */ ++ rcu_read_lock(); ++ /* ++ * If there are multiple target nodes, just select one ++ * target node randomly. ++ * ++ * In addition, we can also use round-robin to select ++ * target node, but we should introduce another variable ++ * for node_demotion[] to record last selected target node, ++ * that may cause cache ping-pong due to the changing of ++ * last target node. Or introducing per-cpu data to avoid ++ * caching issue, which seems more complicated. So selecting ++ * target node randomly seems better until now. ++ */ ++ target = node_random(&nd->preferred); ++ rcu_read_unlock(); + -+ for (i = 0; i < total_count; i++) { -+ struct winesync_obj *obj = q->entries[i].obj; -+ -+ if (atomic_read(&q->signaled) != -1) -+ break; -+ -+ spin_lock(&obj->lock); -+ try_wake_any_obj(obj); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* sleep */ -+ -+ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); -+ -+ /* and finally, unqueue */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_del(&entry->node); -+ spin_unlock(&obj->lock); -+ -+ put_obj(obj); -+ } -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct winesync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; ++ return target; +} + -+static int winesync_wait_all(struct winesync_device *dev, void __user *argp) ++static void disable_all_demotion_targets(void) +{ -+ struct winesync_wait_args args; -+ struct winesync_q *q; -+ ktime_t timeout; -+ int signaled; -+ __u32 i; -+ int ret; ++ struct memory_tier *memtier; ++ int node; ++ ++ for_each_node_state(node, N_MEMORY) { ++ node_demotion[node].preferred = NODE_MASK_NONE; ++ /* ++ * We are holding memory_tier_lock, it is safe ++ * to access pgda->memtier. ++ */ ++ memtier = __node_get_memory_tier(node); ++ if (memtier) ++ memtier->lower_tier_mask = NODE_MASK_NONE; ++ } ++ /* ++ * Ensure that the "disable" is visible across the system. ++ * Readers will see either a combination of before+disable ++ * state or disable+after. They will never see before and ++ * after state together. ++ */ ++ synchronize_rcu(); ++} + -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; ++/* ++ * Find an automatic demotion target for all memory ++ * nodes. Failing here is OK. It might just indicate ++ * being at the end of a chain. ++ */ ++static void establish_demotion_targets(void) ++{ ++ struct memory_tier *memtier; ++ struct demotion_nodes *nd; ++ int target = NUMA_NO_NODE, node; ++ int distance, best_distance; ++ nodemask_t tier_nodes, lower_tier; + -+ ret = setup_wait(dev, &args, true, &timeout, &q); -+ if (ret < 0) -+ return ret; ++ lockdep_assert_held_once(&memory_tier_lock); + -+ /* queue ourselves */ ++ if (!node_demotion || !IS_ENABLED(CONFIG_MIGRATION)) ++ return; + -+ spin_lock(&dev->wait_all_lock); ++ disable_all_demotion_targets(); + -+ for (i = 0; i < args.count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; ++ for_each_node_state(node, N_MEMORY) { ++ best_distance = -1; ++ nd = &node_demotion[node]; + -+ atomic_inc(&obj->all_hint); ++ memtier = __node_get_memory_tier(node); ++ if (!memtier || list_is_last(&memtier->list, &memory_tiers)) ++ continue; ++ /* ++ * Get the lower memtier to find the demotion node list. ++ */ ++ memtier = list_next_entry(memtier, list); ++ tier_nodes = get_memtier_nodemask(memtier); ++ /* ++ * find_next_best_node, use 'used' nodemask as a skip list. ++ * Add all memory nodes except the selected memory tier ++ * nodelist to skip list so that we find the best node from the ++ * memtier nodelist. ++ */ ++ nodes_andnot(tier_nodes, node_states[N_MEMORY], tier_nodes); + + /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. ++ * Find all the nodes in the memory tier node list of same best distance. ++ * add them to the preferred mask. We randomly select between nodes ++ * in the preferred mask when allocating pages during demotion. + */ -+ list_add_tail(&entry->node, &obj->all_waiters); -+ } -+ if (args.alert) { -+ struct winesync_q_entry *entry = &q->entries[args.count]; -+ struct winesync_obj *obj = entry->obj; ++ do { ++ target = find_next_best_node(node, &tier_nodes); ++ if (target == NUMA_NO_NODE) ++ break; + -+ spin_lock(&obj->lock); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ spin_unlock(&obj->lock); ++ distance = node_distance(node, target); ++ if (distance == best_distance || best_distance == -1) { ++ best_distance = distance; ++ node_set(target, nd->preferred); ++ } else { ++ break; ++ } ++ } while (1); + } ++ /* ++ * Promotion is allowed from a memory tier to higher ++ * memory tier only if the memory tier doesn't include ++ * compute. We want to skip promotion from a memory tier, ++ * if any node that is part of the memory tier have CPUs. ++ * Once we detect such a memory tier, we consider that tier ++ * as top tiper from which promotion is not allowed. ++ */ ++ list_for_each_entry_reverse(memtier, &memory_tiers, list) { ++ tier_nodes = get_memtier_nodemask(memtier); ++ nodes_and(tier_nodes, node_states[N_CPU], tier_nodes); ++ if (!nodes_empty(tier_nodes)) { ++ /* ++ * abstract distance below the max value of this memtier ++ * is considered toptier. ++ */ ++ top_tier_adistance = memtier->adistance_start + ++ MEMTIER_CHUNK_SIZE - 1; ++ break; ++ } ++ } ++ /* ++ * Now build the lower_tier mask for each node collecting node mask from ++ * all memory tier below it. This allows us to fallback demotion page ++ * allocation to a set of nodes that is closer the above selected ++ * perferred node. ++ */ ++ lower_tier = node_states[N_MEMORY]; ++ list_for_each_entry(memtier, &memory_tiers, list) { ++ /* ++ * Keep removing current tier from lower_tier nodes, ++ * This will remove all nodes in current and above ++ * memory tier from the lower_tier mask. ++ */ ++ tier_nodes = get_memtier_nodemask(memtier); ++ nodes_andnot(lower_tier, lower_tier, tier_nodes); ++ memtier->lower_tier_mask = lower_tier; ++ } ++} + -+ /* check if we are already signaled */ -+ -+ try_wake_all(dev, q, NULL); -+ -+ spin_unlock(&dev->wait_all_lock); ++#else ++static inline void disable_all_demotion_targets(void) {} ++static inline void establish_demotion_targets(void) {} ++#endif /* CONFIG_MIGRATION */ + ++static inline void __init_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ if (!node_memory_types[node].memtype) ++ node_memory_types[node].memtype = memtype; + /* -+ * Check if the alert event is signaled, making sure to do so only -+ * after checking if the other objects are signaled. ++ * for each device getting added in the same NUMA node ++ * with this specific memtype, bump the map count. We ++ * Only take memtype device reference once, so that ++ * changing a node memtype can be done by droping the ++ * only reference count taken here. + */ + -+ if (args.alert) { -+ struct winesync_obj *obj = q->entries[args.count].obj; -+ -+ if (atomic_read(&q->signaled) == -1) { -+ spin_lock(&obj->lock); -+ try_wake_any_obj(obj); -+ spin_unlock(&obj->lock); -+ } ++ if (node_memory_types[node].memtype == memtype) { ++ if (!node_memory_types[node].map_count++) ++ kref_get(&memtype->kref); + } ++} + -+ /* sleep */ ++static struct memory_tier *set_node_memory_tier(int node) ++{ ++ struct memory_tier *memtier; ++ struct memory_dev_type *memtype; ++ pg_data_t *pgdat = NODE_DATA(node); + -+ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); + -+ /* and finally, unqueue */ ++ lockdep_assert_held_once(&memory_tier_lock); + -+ spin_lock(&dev->wait_all_lock); ++ if (!node_state(node, N_MEMORY)) ++ return ERR_PTR(-EINVAL); + -+ for (i = 0; i < args.count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; ++ __init_node_memory_type(node, default_dram_type); + -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. -+ */ -+ list_del(&entry->node); ++ memtype = node_memory_types[node].memtype; ++ node_set(node, memtype->nodes); ++ memtier = find_create_memory_tier(memtype); ++ if (!IS_ERR(memtier)) ++ rcu_assign_pointer(pgdat->memtier, memtier); ++ return memtier; ++} + -+ atomic_dec(&obj->all_hint); ++static void destroy_memory_tier(struct memory_tier *memtier) ++{ ++ list_del(&memtier->list); ++ device_unregister(&memtier->dev); ++} + -+ put_obj(obj); -+ } -+ if (args.alert) { -+ struct winesync_q_entry *entry = &q->entries[args.count]; -+ struct winesync_obj *obj = entry->obj; ++static bool clear_node_memory_tier(int node) ++{ ++ bool cleared = false; ++ pg_data_t *pgdat; ++ struct memory_tier *memtier; + -+ spin_lock(&obj->lock); -+ list_del(&entry->node); -+ spin_unlock(&obj->lock); ++ pgdat = NODE_DATA(node); ++ if (!pgdat) ++ return false; + -+ put_obj(obj); ++ /* ++ * Make sure that anybody looking at NODE_DATA who finds ++ * a valid memtier finds memory_dev_types with nodes still ++ * linked to the memtier. We achieve this by waiting for ++ * rcu read section to finish using synchronize_rcu. ++ * This also enables us to free the destroyed memory tier ++ * with kfree instead of kfree_rcu ++ */ ++ memtier = __node_get_memory_tier(node); ++ if (memtier) { ++ struct memory_dev_type *memtype; ++ ++ rcu_assign_pointer(pgdat->memtier, NULL); ++ synchronize_rcu(); ++ memtype = node_memory_types[node].memtype; ++ node_clear(node, memtype->nodes); ++ if (nodes_empty(memtype->nodes)) { ++ list_del_init(&memtype->tier_sibiling); ++ if (list_empty(&memtier->memory_types)) ++ destroy_memory_tier(memtier); ++ } ++ cleared = true; + } ++ return cleared; ++} + -+ spin_unlock(&dev->wait_all_lock); ++static void release_memtype(struct kref *kref) ++{ ++ struct memory_dev_type *memtype; + -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct winesync_wait_args __user *user_args = argp; ++ memtype = container_of(kref, struct memory_dev_type, kref); ++ kfree(memtype); ++} + -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; ++struct memory_dev_type *alloc_memory_type(int adistance) ++{ ++ struct memory_dev_type *memtype; + -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } ++ memtype = kmalloc(sizeof(*memtype), GFP_KERNEL); ++ if (!memtype) ++ return ERR_PTR(-ENOMEM); + -+ kfree(q); -+ return ret; ++ memtype->adistance = adistance; ++ INIT_LIST_HEAD(&memtype->tier_sibiling); ++ memtype->nodes = NODE_MASK_NONE; ++ kref_init(&memtype->kref); ++ return memtype; +} ++EXPORT_SYMBOL_GPL(alloc_memory_type); + -+static long winesync_char_ioctl(struct file *file, unsigned int cmd, -+ unsigned long parm) ++void destroy_memory_type(struct memory_dev_type *memtype) +{ -+ struct winesync_device *dev = file->private_data; -+ void __user *argp = (void __user *)parm; ++ kref_put(&memtype->kref, release_memtype); ++} ++EXPORT_SYMBOL_GPL(destroy_memory_type); + -+ switch (cmd) { -+ case WINESYNC_IOC_CREATE_EVENT: -+ return winesync_create_event(dev, argp); -+ case WINESYNC_IOC_CREATE_MUTEX: -+ return winesync_create_mutex(dev, argp); -+ case WINESYNC_IOC_CREATE_SEM: -+ return winesync_create_sem(dev, argp); -+ case WINESYNC_IOC_DELETE: -+ return winesync_delete(dev, argp); -+ case WINESYNC_IOC_KILL_OWNER: -+ return winesync_kill_owner(dev, argp); -+ case WINESYNC_IOC_PULSE_EVENT: -+ return winesync_set_event(dev, argp, true); -+ case WINESYNC_IOC_PUT_MUTEX: -+ return winesync_put_mutex(dev, argp); -+ case WINESYNC_IOC_PUT_SEM: -+ return winesync_put_sem(dev, argp); -+ case WINESYNC_IOC_READ_EVENT: -+ return winesync_read_event(dev, argp); -+ case WINESYNC_IOC_READ_MUTEX: -+ return winesync_read_mutex(dev, argp); -+ case WINESYNC_IOC_READ_SEM: -+ return winesync_read_sem(dev, argp); -+ case WINESYNC_IOC_RESET_EVENT: -+ return winesync_reset_event(dev, argp); -+ case WINESYNC_IOC_SET_EVENT: -+ return winesync_set_event(dev, argp, false); -+ case WINESYNC_IOC_WAIT_ALL: -+ return winesync_wait_all(dev, argp); -+ case WINESYNC_IOC_WAIT_ANY: -+ return winesync_wait_any(dev, argp); -+ default: -+ return -ENOSYS; ++void init_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ ++ mutex_lock(&memory_tier_lock); ++ __init_node_memory_type(node, memtype); ++ mutex_unlock(&memory_tier_lock); ++} ++EXPORT_SYMBOL_GPL(init_node_memory_type); ++ ++void clear_node_memory_type(int node, struct memory_dev_type *memtype) ++{ ++ mutex_lock(&memory_tier_lock); ++ if (node_memory_types[node].memtype == memtype) ++ node_memory_types[node].map_count--; ++ /* ++ * If we umapped all the attached devices to this node, ++ * clear the node memory type. ++ */ ++ if (!node_memory_types[node].map_count) { ++ node_memory_types[node].memtype = NULL; ++ kref_put(&memtype->kref, release_memtype); + } ++ mutex_unlock(&memory_tier_lock); +} ++EXPORT_SYMBOL_GPL(clear_node_memory_type); + -+static const struct file_operations winesync_fops = { -+ .owner = THIS_MODULE, -+ .open = winesync_char_open, -+ .release = winesync_char_release, -+ .unlocked_ioctl = winesync_char_ioctl, -+ .compat_ioctl = winesync_char_ioctl, -+ .llseek = no_llseek, -+}; ++static int __meminit memtier_hotplug_callback(struct notifier_block *self, ++ unsigned long action, void *_arg) ++{ ++ struct memory_tier *memtier; ++ struct memory_notify *arg = _arg; + -+static struct miscdevice winesync_misc = { -+ .minor = WINESYNC_MINOR, -+ .name = WINESYNC_NAME, -+ .fops = &winesync_fops, -+}; ++ /* ++ * Only update the node migration order when a node is ++ * changing status, like online->offline. ++ */ ++ if (arg->status_change_nid < 0) ++ return notifier_from_errno(0); ++ ++ switch (action) { ++ case MEM_OFFLINE: ++ mutex_lock(&memory_tier_lock); ++ if (clear_node_memory_tier(arg->status_change_nid)) ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ break; ++ case MEM_ONLINE: ++ mutex_lock(&memory_tier_lock); ++ memtier = set_node_memory_tier(arg->status_change_nid); ++ if (!IS_ERR(memtier)) ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ break; ++ } + -+static int __init winesync_init(void) -+{ -+ return misc_register(&winesync_misc); ++ return notifier_from_errno(0); +} + -+static void __exit winesync_exit(void) ++static int __init memory_tier_init(void) +{ -+ misc_deregister(&winesync_misc); ++ int ret, node; ++ struct memory_tier *memtier; ++ ++ ret = subsys_virtual_register(&memory_tier_subsys, NULL); ++ if (ret) ++ panic("%s() failed to register memory tier subsystem\n", __func__); ++ ++#ifdef CONFIG_MIGRATION ++ node_demotion = kcalloc(nr_node_ids, sizeof(struct demotion_nodes), ++ GFP_KERNEL); ++ WARN_ON(!node_demotion); ++#endif ++ mutex_lock(&memory_tier_lock); ++ /* ++ * For now we can have 4 faster memory tiers with smaller adistance ++ * than default DRAM tier. ++ */ ++ default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM); ++ if (!default_dram_type) ++ panic("%s() failed to allocate default DRAM tier\n", __func__); ++ ++ /* ++ * Look at all the existing N_MEMORY nodes and add them to ++ * default memory tier or to a tier if we already have memory ++ * types assigned. ++ */ ++ for_each_node_state(node, N_MEMORY) { ++ memtier = set_node_memory_tier(node); ++ if (IS_ERR(memtier)) ++ /* ++ * Continue with memtiers we are able to setup ++ */ ++ break; ++ } ++ establish_demotion_targets(); ++ mutex_unlock(&memory_tier_lock); ++ ++ hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRIO); ++ return 0; +} ++subsys_initcall(memory_tier_init); + -+module_init(winesync_init); -+module_exit(winesync_exit); ++bool numa_demotion_enabled = false; + -+MODULE_AUTHOR("Zebediah Figura"); -+MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); -+MODULE_LICENSE("GPL"); -+MODULE_ALIAS("devname:" WINESYNC_NAME); -+MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); -diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h -index c0fea6ca5076..36fc5d5315a4 100644 ---- a/include/linux/miscdevice.h -+++ b/include/linux/miscdevice.h -@@ -71,6 +71,7 @@ - #define USERIO_MINOR 240 - #define VHOST_VSOCK_MINOR 241 - #define RFKILL_MINOR 242 -+#define WINESYNC_MINOR 243 - #define MISC_DYNAMIC_MINOR 255 - - struct device; -diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h -new file mode 100644 -index 000000000000..5b4e369f7469 ---- /dev/null -+++ b/include/uapi/linux/winesync.h -@@ -0,0 +1,71 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * Kernel support for Wine synchronization primitives -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ ++#ifdef CONFIG_MIGRATION ++#ifdef CONFIG_SYSFS ++static ssize_t numa_demotion_enabled_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sysfs_emit(buf, "%s\n", ++ numa_demotion_enabled ? "true" : "false"); ++} + -+#ifndef __LINUX_WINESYNC_H -+#define __LINUX_WINESYNC_H ++static ssize_t numa_demotion_enabled_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ ssize_t ret; + -+#include ++ ret = kstrtobool(buf, &numa_demotion_enabled); ++ if (ret) ++ return ret; + -+struct winesync_sem_args { -+ __u32 sem; -+ __u32 count; -+ __u32 max; -+}; ++ return count; ++} + -+struct winesync_mutex_args { -+ __u32 mutex; -+ __u32 owner; -+ __u32 count; -+}; ++static struct kobj_attribute numa_demotion_enabled_attr = ++ __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, ++ numa_demotion_enabled_store); + -+struct winesync_event_args { -+ __u32 event; -+ __u32 manual; -+ __u32 signaled; ++static struct attribute *numa_attrs[] = { ++ &numa_demotion_enabled_attr.attr, ++ NULL, +}; + -+struct winesync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 alert; ++static const struct attribute_group numa_attr_group = { ++ .attrs = numa_attrs, +}; + -+#define WINESYNC_IOC_BASE 0xf7 ++static int __init numa_init_sysfs(void) ++{ ++ int err; ++ struct kobject *numa_kobj; + -+#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 1, __u32) -+#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 2, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 3, \ -+ struct winesync_wait_args) -+#define WINESYNC_IOC_WAIT_ALL _IOWR(WINESYNC_IOC_BASE, 4, \ -+ struct winesync_wait_args) -+#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 5, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 7, __u32) -+#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 8, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 9, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_CREATE_EVENT _IOWR(WINESYNC_IOC_BASE, 10, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_SET_EVENT _IOWR(WINESYNC_IOC_BASE, 11, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_RESET_EVENT _IOWR(WINESYNC_IOC_BASE, 12, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_PULSE_EVENT _IOWR(WINESYNC_IOC_BASE, 13, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_READ_EVENT _IOWR(WINESYNC_IOC_BASE, 14, \ -+ struct winesync_event_args) ++ numa_kobj = kobject_create_and_add("numa", mm_kobj); ++ if (!numa_kobj) { ++ pr_err("failed to create numa kobject\n"); ++ return -ENOMEM; ++ } ++ err = sysfs_create_group(numa_kobj, &numa_attr_group); ++ if (err) { ++ pr_err("failed to register numa group\n"); ++ goto delete_obj; ++ } ++ return 0; + ++delete_obj: ++ kobject_put(numa_kobj); ++ return err; ++} ++subsys_initcall(numa_init_sysfs); ++#endif /* CONFIG_SYSFS */ +#endif -diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile -index 1fc89b8ef433..c7d3d9f5e0a0 100644 ---- a/tools/testing/selftests/Makefile -+++ b/tools/testing/selftests/Makefile -@@ -14,6 +14,7 @@ TARGETS += drivers/dma-buf - TARGETS += drivers/s390x/uvdevice - TARGETS += drivers/net/bonding - TARGETS += drivers/net/team -+TARGETS += drivers/winesync - TARGETS += efivarfs - TARGETS += exec - TARGETS += filesystems -diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile -new file mode 100644 -index 000000000000..43b39fdeea10 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/Makefile -@@ -0,0 +1,8 @@ -+# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only -+TEST_GEN_PROGS := winesync -+ -+top_srcdir =../../../../.. -+CFLAGS += -I$(top_srcdir)/usr/include -+LDLIBS += -lpthread -+ -+include ../../lib.mk -diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config -new file mode 100644 -index 000000000000..60539c826d06 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/config -@@ -0,0 +1 @@ -+CONFIG_WINESYNC=y -diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c -new file mode 100644 -index 000000000000..169e922484b0 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/winesync.c -@@ -0,0 +1,1479 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Various unit tests for the "winesync" synchronization primitive driver. -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ +diff --git a/mm/memory.c b/mm/memory.c +index 4ba73f5aa8bb..3a3d8721bf4c 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -66,6 +66,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/mm/migrate.c b/mm/migrate.c +index 6a1597c92261..55e7718cfe45 100644 +--- a/mm/migrate.c ++++ b/mm/migrate.c +@@ -50,6 +50,7 @@ + #include + #include + #include ++#include + + #include + +@@ -2170,456 +2171,4 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, + return 0; + } + #endif /* CONFIG_NUMA_BALANCING */ +- +-/* +- * node_demotion[] example: +- * +- * Consider a system with two sockets. Each socket has +- * three classes of memory attached: fast, medium and slow. +- * Each memory class is placed in its own NUMA node. The +- * CPUs are placed in the node with the "fast" memory. The +- * 6 NUMA nodes (0-5) might be split among the sockets like +- * this: +- * +- * Socket A: 0, 1, 2 +- * Socket B: 3, 4, 5 +- * +- * When Node 0 fills up, its memory should be migrated to +- * Node 1. When Node 1 fills up, it should be migrated to +- * Node 2. The migration path start on the nodes with the +- * processors (since allocations default to this node) and +- * fast memory, progress through medium and end with the +- * slow memory: +- * +- * 0 -> 1 -> 2 -> stop +- * 3 -> 4 -> 5 -> stop +- * +- * This is represented in the node_demotion[] like this: +- * +- * { nr=1, nodes[0]=1 }, // Node 0 migrates to 1 +- * { nr=1, nodes[0]=2 }, // Node 1 migrates to 2 +- * { nr=0, nodes[0]=-1 }, // Node 2 does not migrate +- * { nr=1, nodes[0]=4 }, // Node 3 migrates to 4 +- * { nr=1, nodes[0]=5 }, // Node 4 migrates to 5 +- * { nr=0, nodes[0]=-1 }, // Node 5 does not migrate +- * +- * Moreover some systems may have multiple slow memory nodes. +- * Suppose a system has one socket with 3 memory nodes, node 0 +- * is fast memory type, and node 1/2 both are slow memory +- * type, and the distance between fast memory node and slow +- * memory node is same. So the migration path should be: +- * +- * 0 -> 1/2 -> stop +- * +- * This is represented in the node_demotion[] like this: +- * { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2 +- * { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate +- * { nr=0, nodes[0]=-1, }, // Node 2 does not migrate +- */ +- +-/* +- * Writes to this array occur without locking. Cycles are +- * not allowed: Node X demotes to Y which demotes to X... +- * +- * If multiple reads are performed, a single rcu_read_lock() +- * must be held over all reads to ensure that no cycles are +- * observed. +- */ +-#define DEFAULT_DEMOTION_TARGET_NODES 15 +- +-#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES +-#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1) +-#else +-#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES +-#endif +- +-struct demotion_nodes { +- unsigned short nr; +- short nodes[DEMOTION_TARGET_NODES]; +-}; +- +-static struct demotion_nodes *node_demotion __read_mostly; +- +-/** +- * next_demotion_node() - Get the next node in the demotion path +- * @node: The starting node to lookup the next node +- * +- * Return: node id for next memory node in the demotion path hierarchy +- * from @node; NUMA_NO_NODE if @node is terminal. This does not keep +- * @node online or guarantee that it *continues* to be the next demotion +- * target. +- */ +-int next_demotion_node(int node) +-{ +- struct demotion_nodes *nd; +- unsigned short target_nr, index; +- int target; +- +- if (!node_demotion) +- return NUMA_NO_NODE; +- +- nd = &node_demotion[node]; +- +- /* +- * node_demotion[] is updated without excluding this +- * function from running. RCU doesn't provide any +- * compiler barriers, so the READ_ONCE() is required +- * to avoid compiler reordering or read merging. +- * +- * Make sure to use RCU over entire code blocks if +- * node_demotion[] reads need to be consistent. +- */ +- rcu_read_lock(); +- target_nr = READ_ONCE(nd->nr); +- +- switch (target_nr) { +- case 0: +- target = NUMA_NO_NODE; +- goto out; +- case 1: +- index = 0; +- break; +- default: +- /* +- * If there are multiple target nodes, just select one +- * target node randomly. +- * +- * In addition, we can also use round-robin to select +- * target node, but we should introduce another variable +- * for node_demotion[] to record last selected target node, +- * that may cause cache ping-pong due to the changing of +- * last target node. Or introducing per-cpu data to avoid +- * caching issue, which seems more complicated. So selecting +- * target node randomly seems better until now. +- */ +- index = get_random_int() % target_nr; +- break; +- } +- +- target = READ_ONCE(nd->nodes[index]); +- +-out: +- rcu_read_unlock(); +- return target; +-} +- +-/* Disable reclaim-based migration. */ +-static void __disable_all_migrate_targets(void) +-{ +- int node, i; +- +- if (!node_demotion) +- return; +- +- for_each_online_node(node) { +- node_demotion[node].nr = 0; +- for (i = 0; i < DEMOTION_TARGET_NODES; i++) +- node_demotion[node].nodes[i] = NUMA_NO_NODE; +- } +-} +- +-static void disable_all_migrate_targets(void) +-{ +- __disable_all_migrate_targets(); +- +- /* +- * Ensure that the "disable" is visible across the system. +- * Readers will see either a combination of before+disable +- * state or disable+after. They will never see before and +- * after state together. +- * +- * The before+after state together might have cycles and +- * could cause readers to do things like loop until this +- * function finishes. This ensures they can only see a +- * single "bad" read and would, for instance, only loop +- * once. +- */ +- synchronize_rcu(); +-} +- +-/* +- * Find an automatic demotion target for 'node'. +- * Failing here is OK. It might just indicate +- * being at the end of a chain. +- */ +-static int establish_migrate_target(int node, nodemask_t *used, +- int best_distance) +-{ +- int migration_target, index, val; +- struct demotion_nodes *nd; +- +- if (!node_demotion) +- return NUMA_NO_NODE; +- +- nd = &node_demotion[node]; +- +- migration_target = find_next_best_node(node, used); +- if (migration_target == NUMA_NO_NODE) +- return NUMA_NO_NODE; +- +- /* +- * If the node has been set a migration target node before, +- * which means it's the best distance between them. Still +- * check if this node can be demoted to other target nodes +- * if they have a same best distance. +- */ +- if (best_distance != -1) { +- val = node_distance(node, migration_target); +- if (val > best_distance) +- goto out_clear; +- } +- +- index = nd->nr; +- if (WARN_ONCE(index >= DEMOTION_TARGET_NODES, +- "Exceeds maximum demotion target nodes\n")) +- goto out_clear; +- +- nd->nodes[index] = migration_target; +- nd->nr++; +- +- return migration_target; +-out_clear: +- node_clear(migration_target, *used); +- return NUMA_NO_NODE; +-} +- +-/* +- * When memory fills up on a node, memory contents can be +- * automatically migrated to another node instead of +- * discarded at reclaim. +- * +- * Establish a "migration path" which will start at nodes +- * with CPUs and will follow the priorities used to build the +- * page allocator zonelists. +- * +- * The difference here is that cycles must be avoided. If +- * node0 migrates to node1, then neither node1, nor anything +- * node1 migrates to can migrate to node0. Also one node can +- * be migrated to multiple nodes if the target nodes all have +- * a same best-distance against the source node. +- * +- * This function can run simultaneously with readers of +- * node_demotion[]. However, it can not run simultaneously +- * with itself. Exclusion is provided by memory hotplug events +- * being single-threaded. +- */ +-static void __set_migration_target_nodes(void) +-{ +- nodemask_t next_pass; +- nodemask_t this_pass; +- nodemask_t used_targets = NODE_MASK_NONE; +- int node, best_distance; +- +- /* +- * Avoid any oddities like cycles that could occur +- * from changes in the topology. This will leave +- * a momentary gap when migration is disabled. +- */ +- disable_all_migrate_targets(); +- +- /* +- * Allocations go close to CPUs, first. Assume that +- * the migration path starts at the nodes with CPUs. +- */ +- next_pass = node_states[N_CPU]; +-again: +- this_pass = next_pass; +- next_pass = NODE_MASK_NONE; +- /* +- * To avoid cycles in the migration "graph", ensure +- * that migration sources are not future targets by +- * setting them in 'used_targets'. Do this only +- * once per pass so that multiple source nodes can +- * share a target node. +- * +- * 'used_targets' will become unavailable in future +- * passes. This limits some opportunities for +- * multiple source nodes to share a destination. +- */ +- nodes_or(used_targets, used_targets, this_pass); +- +- for_each_node_mask(node, this_pass) { +- best_distance = -1; +- +- /* +- * Try to set up the migration path for the node, and the target +- * migration nodes can be multiple, so doing a loop to find all +- * the target nodes if they all have a best node distance. +- */ +- do { +- int target_node = +- establish_migrate_target(node, &used_targets, +- best_distance); +- +- if (target_node == NUMA_NO_NODE) +- break; +- +- if (best_distance == -1) +- best_distance = node_distance(node, target_node); +- +- /* +- * Visit targets from this pass in the next pass. +- * Eventually, every node will have been part of +- * a pass, and will become set in 'used_targets'. +- */ +- node_set(target_node, next_pass); +- } while (1); +- } +- /* +- * 'next_pass' contains nodes which became migration +- * targets in this pass. Make additional passes until +- * no more migrations targets are available. +- */ +- if (!nodes_empty(next_pass)) +- goto again; +-} +- +-/* +- * For callers that do not hold get_online_mems() already. +- */ +-void set_migration_target_nodes(void) +-{ +- get_online_mems(); +- __set_migration_target_nodes(); +- put_online_mems(); +-} +- +-/* +- * This leaves migrate-on-reclaim transiently disabled between +- * the MEM_GOING_OFFLINE and MEM_OFFLINE events. This runs +- * whether reclaim-based migration is enabled or not, which +- * ensures that the user can turn reclaim-based migration at +- * any time without needing to recalculate migration targets. +- * +- * These callbacks already hold get_online_mems(). That is why +- * __set_migration_target_nodes() can be used as opposed to +- * set_migration_target_nodes(). +- */ +-#ifdef CONFIG_MEMORY_HOTPLUG +-static int __meminit migrate_on_reclaim_callback(struct notifier_block *self, +- unsigned long action, void *_arg) +-{ +- struct memory_notify *arg = _arg; +- +- /* +- * Only update the node migration order when a node is +- * changing status, like online->offline. This avoids +- * the overhead of synchronize_rcu() in most cases. +- */ +- if (arg->status_change_nid < 0) +- return notifier_from_errno(0); +- +- switch (action) { +- case MEM_GOING_OFFLINE: +- /* +- * Make sure there are not transient states where +- * an offline node is a migration target. This +- * will leave migration disabled until the offline +- * completes and the MEM_OFFLINE case below runs. +- */ +- disable_all_migrate_targets(); +- break; +- case MEM_OFFLINE: +- case MEM_ONLINE: +- /* +- * Recalculate the target nodes once the node +- * reaches its final state (online or offline). +- */ +- __set_migration_target_nodes(); +- break; +- case MEM_CANCEL_OFFLINE: +- /* +- * MEM_GOING_OFFLINE disabled all the migration +- * targets. Reenable them. +- */ +- __set_migration_target_nodes(); +- break; +- case MEM_GOING_ONLINE: +- case MEM_CANCEL_ONLINE: +- break; +- } +- +- return notifier_from_errno(0); +-} +-#endif +- +-void __init migrate_on_reclaim_init(void) +-{ +- node_demotion = kcalloc(nr_node_ids, +- sizeof(struct demotion_nodes), +- GFP_KERNEL); +- WARN_ON(!node_demotion); +-#ifdef CONFIG_MEMORY_HOTPLUG +- hotplug_memory_notifier(migrate_on_reclaim_callback, 100); +-#endif +- /* +- * At this point, all numa nodes with memory/CPus have their state +- * properly set, so we can build the demotion order now. +- * Let us hold the cpu_hotplug lock just, as we could possibily have +- * CPU hotplug events during boot. +- */ +- cpus_read_lock(); +- set_migration_target_nodes(); +- cpus_read_unlock(); +-} +- +-bool numa_demotion_enabled = false; +- +-#ifdef CONFIG_SYSFS +-static ssize_t numa_demotion_enabled_show(struct kobject *kobj, +- struct kobj_attribute *attr, char *buf) +-{ +- return sysfs_emit(buf, "%s\n", +- numa_demotion_enabled ? "true" : "false"); +-} +- +-static ssize_t numa_demotion_enabled_store(struct kobject *kobj, +- struct kobj_attribute *attr, +- const char *buf, size_t count) +-{ +- ssize_t ret; +- +- ret = kstrtobool(buf, &numa_demotion_enabled); +- if (ret) +- return ret; +- +- return count; +-} +- +-static struct kobj_attribute numa_demotion_enabled_attr = +- __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, +- numa_demotion_enabled_store); +- +-static struct attribute *numa_attrs[] = { +- &numa_demotion_enabled_attr.attr, +- NULL, +-}; +- +-static const struct attribute_group numa_attr_group = { +- .attrs = numa_attrs, +-}; +- +-static int __init numa_init_sysfs(void) +-{ +- int err; +- struct kobject *numa_kobj; +- +- numa_kobj = kobject_create_and_add("numa", mm_kobj); +- if (!numa_kobj) { +- pr_err("failed to create numa kobject\n"); +- return -ENOMEM; +- } +- err = sysfs_create_group(numa_kobj, &numa_attr_group); +- if (err) { +- pr_err("failed to register numa group\n"); +- goto delete_obj; +- } +- return 0; +- +-delete_obj: +- kobject_put(numa_kobj); +- return err; +-} +-subsys_initcall(numa_init_sysfs); +-#endif /* CONFIG_SYSFS */ + #endif /* CONFIG_NUMA */ +diff --git a/mm/mprotect.c b/mm/mprotect.c +index bc6bddd156ca..eb8982bde7ee 100644 +--- a/mm/mprotect.c ++++ b/mm/mprotect.c +@@ -31,6 +31,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/mm/vmscan.c b/mm/vmscan.c +index 0fc65ace3a4e..e673be68cea3 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -43,6 +43,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -1524,21 +1525,34 @@ static void folio_check_dirty_writeback(struct folio *folio, + mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); + } + +-static struct page *alloc_demote_page(struct page *page, unsigned long node) ++static struct page *alloc_demote_page(struct page *page, unsigned long private) + { +- struct migration_target_control mtc = { +- /* +- * Allocate from 'node', or fail quickly and quietly. +- * When this happens, 'page' will likely just be discarded +- * instead of migrated. +- */ +- .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | +- __GFP_THISNODE | __GFP_NOWARN | +- __GFP_NOMEMALLOC | GFP_NOWAIT, +- .nid = node +- }; ++ struct page *target_page; ++ nodemask_t *allowed_mask; ++ struct migration_target_control *mtc; ++ ++ mtc = (struct migration_target_control *)private; ++ ++ allowed_mask = mtc->nmask; ++ /* ++ * make sure we allocate from the target node first also trying to ++ * demote or reclaim pages from the target node via kswapd if we are ++ * low on free memory on target node. If we don't do this and if ++ * we have free memory on the slower(lower) memtier, we would start ++ * allocating pages from slower(lower) memory tiers without even forcing ++ * a demotion of cold pages from the target memtier. This can result ++ * in the kernel placing hot pages in slower(lower) memory tiers. ++ */ ++ mtc->nmask = NULL; ++ mtc->gfp_mask |= __GFP_THISNODE; ++ target_page = alloc_migration_target(page, (unsigned long)mtc); ++ if (target_page) ++ return target_page; + +- return alloc_migration_target(page, (unsigned long)&mtc); ++ mtc->gfp_mask &= ~__GFP_THISNODE; ++ mtc->nmask = allowed_mask; + -+#define _GNU_SOURCE -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../kselftest_harness.h" ++ return alloc_migration_target(page, (unsigned long)mtc); + } + + /* +@@ -1551,6 +1565,19 @@ static unsigned int demote_page_list(struct list_head *demote_pages, + { + int target_nid = next_demotion_node(pgdat->node_id); + unsigned int nr_succeeded; ++ nodemask_t allowed_mask; + -+static int read_sem_state(int fd, __u32 sem, __u32 *count, __u32 *max) -+{ -+ struct winesync_sem_args args; -+ int ret; ++ struct migration_target_control mtc = { ++ /* ++ * Allocate from 'node', or fail quickly and quietly. ++ * When this happens, 'page' will likely just be discarded ++ * instead of migrated. ++ */ ++ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | ++ __GFP_NOMEMALLOC | GFP_NOWAIT, ++ .nid = target_nid, ++ .nmask = &allowed_mask ++ }; + + if (list_empty(demote_pages)) + return 0; +@@ -1558,10 +1585,12 @@ static unsigned int demote_page_list(struct list_head *demote_pages, + if (target_nid == NUMA_NO_NODE) + return 0; + ++ node_get_allowed_targets(pgdat, &allowed_mask); + -+ args.sem = sem; -+ args.count = 0xdeadbeef; -+ args.max = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &args); -+ *count = args.count; -+ *max = args.max; -+ return ret; -+} + /* Demotion ignores all cpuset and mempolicy settings */ + migrate_pages(demote_pages, alloc_demote_page, NULL, +- target_nid, MIGRATE_ASYNC, MR_DEMOTION, +- &nr_succeeded); ++ (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, ++ &nr_succeeded); + + if (current_is_kswapd()) + __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded); +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 90af9a8572f5..0b8098e8296b 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -28,7 +28,6 @@ + #include + #include + #include +-#include + + #include "internal.h" + +@@ -2067,7 +2066,6 @@ static int vmstat_cpu_online(unsigned int cpu) + + if (!node_state(cpu_to_node(cpu), N_CPU)) { + node_set_state(cpu_to_node(cpu), N_CPU); +- set_migration_target_nodes(); + } + + return 0; +@@ -2092,7 +2090,6 @@ static int vmstat_cpu_dead(unsigned int cpu) + return 0; + + node_clear_state(node, N_CPU); +- set_migration_target_nodes(); + + return 0; + } +@@ -2125,7 +2122,6 @@ void __init init_mm_internals(void) + + start_shepherd_timer(); + #endif +- migrate_on_reclaim_init(); + #ifdef CONFIG_PROC_FS + proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); + proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); +-- +2.38.0.rc1.8.g2a7d63a245 + +From 30817d963bfdddf095e330e41317c9efceec642a Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:26:29 +0200 +Subject: [PATCH 06/16] mm/khugepaged: add struct collapse_control + +Signed-off-by: Peter Jung +--- + arch/alpha/include/uapi/asm/mman.h | 2 + + arch/mips/include/uapi/asm/mman.h | 2 + + arch/parisc/include/uapi/asm/mman.h | 2 + + arch/xtensa/include/uapi/asm/mman.h | 2 + + fs/proc/task_mmu.c | 2 +- + include/linux/huge_mm.h | 23 +- + include/trace/events/huge_memory.h | 1 + + include/uapi/asm-generic/mman-common.h | 2 + + mm/huge_memory.c | 32 +- + mm/internal.h | 2 +- + mm/khugepaged.c | 763 +++++++++++-------- + mm/ksm.c | 10 + + mm/madvise.c | 9 +- + mm/memory.c | 4 +- + mm/rmap.c | 15 +- + tools/include/uapi/asm-generic/mman-common.h | 2 + + tools/testing/selftests/vm/khugepaged.c | 563 ++++++++------ + 17 files changed, 825 insertions(+), 611 deletions(-) + +diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h +index 4aa996423b0d..763929e814e9 100644 +--- a/arch/alpha/include/uapi/asm/mman.h ++++ b/arch/alpha/include/uapi/asm/mman.h +@@ -76,6 +76,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + -+#define check_sem_state(fd, sem, count, max) \ -+ ({ \ -+ __u32 __count, __max; \ -+ int ret = read_sem_state((fd), (sem), &__count, &__max); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((max), __max); \ -+ }) + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h +index 1be428663c10..c6e1fc77c996 100644 +--- a/arch/mips/include/uapi/asm/mman.h ++++ b/arch/mips/include/uapi/asm/mman.h +@@ -103,6 +103,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + -+static int put_sem(int fd, __u32 sem, __u32 *count) -+{ -+ struct winesync_sem_args args; -+ int ret; + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h +index a7ea3204a5fa..22133a6a506e 100644 +--- a/arch/parisc/include/uapi/asm/mman.h ++++ b/arch/parisc/include/uapi/asm/mman.h +@@ -70,6 +70,8 @@ + #define MADV_WIPEONFORK 71 /* Zero memory on fork, child only */ + #define MADV_KEEPONFORK 72 /* Undo MADV_WIPEONFORK */ + ++#define MADV_COLLAPSE 73 /* Synchronous hugepage collapse */ + -+ args.sem = sem; -+ args.count = *count; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &args); -+ *count = args.count; -+ return ret; -+} + #define MADV_HWPOISON 100 /* poison a page for testing */ + #define MADV_SOFT_OFFLINE 101 /* soft offline page for testing */ + +diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h +index 7966a58af472..1ff0c858544f 100644 +--- a/arch/xtensa/include/uapi/asm/mman.h ++++ b/arch/xtensa/include/uapi/asm/mman.h +@@ -111,6 +111,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + -+static int read_mutex_state(int fd, __u32 mutex, __u32 *count, __u32 *owner) -+{ -+ struct winesync_mutex_args args; -+ int ret; + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c +index 4e0023643f8b..482f91577f8c 100644 +--- a/fs/proc/task_mmu.c ++++ b/fs/proc/task_mmu.c +@@ -864,7 +864,7 @@ static int show_smap(struct seq_file *m, void *v) + __show_smap(m, &mss, false); + + seq_printf(m, "THPeligible: %d\n", +- hugepage_vma_check(vma, vma->vm_flags, true, false)); ++ hugepage_vma_check(vma, vma->vm_flags, true, false, true)); + + if (arch_pkeys_enabled()) + seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); +diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h +index 768e5261fdae..38265f9f782e 100644 +--- a/include/linux/huge_mm.h ++++ b/include/linux/huge_mm.h +@@ -168,9 +168,8 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) + !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); + } + +-bool hugepage_vma_check(struct vm_area_struct *vma, +- unsigned long vm_flags, +- bool smaps, bool in_pf); ++bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, ++ bool smaps, bool in_pf, bool enforce_sysfs); + + #define transparent_hugepage_use_zero_page() \ + (transparent_hugepage_flags & \ +@@ -219,6 +218,9 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, + + int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, + int advice); ++int madvise_collapse(struct vm_area_struct *vma, ++ struct vm_area_struct **prev, ++ unsigned long start, unsigned long end); + void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, + unsigned long end, long adjust_next); + spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma); +@@ -321,8 +323,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, + } + + static inline bool hugepage_vma_check(struct vm_area_struct *vma, +- unsigned long vm_flags, +- bool smaps, bool in_pf) ++ unsigned long vm_flags, bool smaps, ++ bool in_pf, bool enforce_sysfs) + { + return false; + } +@@ -362,9 +364,16 @@ static inline void split_huge_pmd_address(struct vm_area_struct *vma, + static inline int hugepage_madvise(struct vm_area_struct *vma, + unsigned long *vm_flags, int advice) + { +- BUG(); +- return 0; ++ return -EINVAL; + } + -+ args.mutex = mutex; -+ args.count = 0xdeadbeef; -+ args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &args); -+ *count = args.count; -+ *owner = args.owner; -+ return ret; ++static inline int madvise_collapse(struct vm_area_struct *vma, ++ struct vm_area_struct **prev, ++ unsigned long start, unsigned long end) ++{ ++ return -EINVAL; +} + -+#define check_mutex_state(fd, mutex, count, owner) \ -+ ({ \ -+ __u32 __count, __owner; \ -+ int ret = read_mutex_state((fd), (mutex), &__count, &__owner); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((owner), __owner); \ -+ }) -+ -+static int put_mutex(int fd, __u32 mutex, __u32 owner, __u32 *count) -+{ -+ struct winesync_mutex_args args; -+ int ret; + static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, + unsigned long start, + unsigned long end, +diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h +index d651f3437367..55392bf30a03 100644 +--- a/include/trace/events/huge_memory.h ++++ b/include/trace/events/huge_memory.h +@@ -11,6 +11,7 @@ + EM( SCAN_FAIL, "failed") \ + EM( SCAN_SUCCEED, "succeeded") \ + EM( SCAN_PMD_NULL, "pmd_null") \ ++ EM( SCAN_PMD_MAPPED, "page_pmd_mapped") \ + EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \ + EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ + EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ +diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h +index 6c1aa92a92e4..6ce1f1ceb432 100644 +--- a/include/uapi/asm-generic/mman-common.h ++++ b/include/uapi/asm-generic/mman-common.h +@@ -77,6 +77,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 6eb4b1799b79..42cdc3338adc 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -71,9 +71,8 @@ static atomic_t huge_zero_refcount; + struct page *huge_zero_page __read_mostly; + unsigned long huge_zero_pfn __read_mostly = ~0UL; + +-bool hugepage_vma_check(struct vm_area_struct *vma, +- unsigned long vm_flags, +- bool smaps, bool in_pf) ++bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, ++ bool smaps, bool in_pf, bool enforce_sysfs) + { + if (!vma->vm_mm) /* vdso */ + return false; +@@ -122,11 +121,10 @@ bool hugepage_vma_check(struct vm_area_struct *vma, + if (!in_pf && shmem_file(vma->vm_file)) + return shmem_huge_enabled(vma); + +- if (!hugepage_flags_enabled()) +- return false; +- +- /* THP settings require madvise. */ +- if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always()) ++ /* Enforce sysfs THP requirements as necessary */ ++ if (enforce_sysfs && ++ (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) && ++ !hugepage_flags_always()))) + return false; + + /* Only regular file is valid */ +@@ -2289,25 +2287,11 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, + void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, + bool freeze, struct folio *folio) + { +- pgd_t *pgd; +- p4d_t *p4d; +- pud_t *pud; +- pmd_t *pmd; +- +- pgd = pgd_offset(vma->vm_mm, address); +- if (!pgd_present(*pgd)) +- return; ++ pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); + +- p4d = p4d_offset(pgd, address); +- if (!p4d_present(*p4d)) ++ if (!pmd) + return; + +- pud = pud_offset(p4d, address); +- if (!pud_present(*pud)) +- return; +- +- pmd = pmd_offset(pud, address); +- + __split_huge_pmd(vma, pmd, address, freeze, folio); + } + +diff --git a/mm/internal.h b/mm/internal.h +index 785409805ed7..55ce10e4d0c0 100644 +--- a/mm/internal.h ++++ b/mm/internal.h +@@ -187,7 +187,7 @@ extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason + /* + * in mm/rmap.c: + */ +-extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); ++pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); + + /* + * in mm/page_alloc.c +diff --git a/mm/khugepaged.c b/mm/khugepaged.c +index 01f71786d530..5f7c60b8b269 100644 +--- a/mm/khugepaged.c ++++ b/mm/khugepaged.c +@@ -28,6 +28,7 @@ enum scan_result { + SCAN_FAIL, + SCAN_SUCCEED, + SCAN_PMD_NULL, ++ SCAN_PMD_MAPPED, + SCAN_EXCEED_NONE_PTE, + SCAN_EXCEED_SWAP_PTE, + SCAN_EXCEED_SHARED_PTE, +@@ -73,6 +74,8 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); + * default collapse hugepages if there is at least one pte mapped like + * it would have happened if the vma was large enough during page + * fault. ++ * ++ * Note that these are only respected if collapse was initiated by khugepaged. + */ + static unsigned int khugepaged_max_ptes_none __read_mostly; + static unsigned int khugepaged_max_ptes_swap __read_mostly; +@@ -85,6 +88,16 @@ static struct kmem_cache *mm_slot_cache __read_mostly; + + #define MAX_PTE_MAPPED_THP 8 + ++struct collapse_control { ++ bool is_khugepaged; + -+ args.mutex = mutex; -+ args.owner = owner; -+ args.count = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &args); -+ *count = args.count; -+ return ret; -+} ++ /* Num pages scanned per node */ ++ u32 node_load[MAX_NUMNODES]; + -+static int read_event_state(int fd, __u32 event, __u32 *signaled, __u32 *manual) -+{ -+ struct winesync_event_args args; -+ int ret; ++ /* Last target selected in hpage_collapse_find_target_node() */ ++ int last_target_node; ++}; + -+ args.event = event; -+ args.signaled = 0xdeadbeef; -+ args.manual = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &args); -+ *signaled = args.signaled; -+ *manual = args.manual; -+ return ret; + /** + * struct mm_slot - hash lookup from mm to mm_slot + * @hash: hash collision list +@@ -425,7 +438,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm, + hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); + } + +-static inline int khugepaged_test_exit(struct mm_struct *mm) ++static inline int hpage_collapse_test_exit(struct mm_struct *mm) + { + return atomic_read(&mm->mm_users) == 0; + } +@@ -440,7 +453,7 @@ void __khugepaged_enter(struct mm_struct *mm) + return; + + /* __khugepaged_exit() must not run from under us */ +- VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); ++ VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); + if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { + free_mm_slot(mm_slot); + return; +@@ -466,7 +479,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, + { + if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && + hugepage_flags_enabled()) { +- if (hugepage_vma_check(vma, vm_flags, false, false)) ++ if (hugepage_vma_check(vma, vm_flags, false, false, true)) + __khugepaged_enter(vma->vm_mm); + } + } +@@ -492,11 +505,10 @@ void __khugepaged_exit(struct mm_struct *mm) + } else if (mm_slot) { + /* + * This is required to serialize against +- * khugepaged_test_exit() (which is guaranteed to run +- * under mmap sem read mode). Stop here (after we +- * return all pagetables will be destroyed) until +- * khugepaged has finished working on the pagetables +- * under the mmap_lock. ++ * hpage_collapse_test_exit() (which is guaranteed to run ++ * under mmap sem read mode). Stop here (after we return all ++ * pagetables will be destroyed) until khugepaged has finished ++ * working on the pagetables under the mmap_lock. + */ + mmap_write_lock(mm); + mmap_write_unlock(mm); +@@ -546,11 +558,12 @@ static bool is_refcount_suitable(struct page *page) + static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + unsigned long address, + pte_t *pte, ++ struct collapse_control *cc, + struct list_head *compound_pagelist) + { + struct page *page = NULL; + pte_t *_pte; +- int none_or_zero = 0, shared = 0, result = 0, referenced = 0; ++ int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; + bool writable = false; + + for (_pte = pte; _pte < pte + HPAGE_PMD_NR; +@@ -558,8 +571,10 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + pte_t pteval = *_pte; + if (pte_none(pteval) || (pte_present(pteval) && + is_zero_pfn(pte_pfn(pteval)))) { ++ ++none_or_zero; + if (!userfaultfd_armed(vma) && +- ++none_or_zero <= khugepaged_max_ptes_none) { ++ (!cc->is_khugepaged || ++ none_or_zero <= khugepaged_max_ptes_none)) { + continue; + } else { + result = SCAN_EXCEED_NONE_PTE; +@@ -579,11 +594,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + + VM_BUG_ON_PAGE(!PageAnon(page), page); + +- if (page_mapcount(page) > 1 && +- ++shared > khugepaged_max_ptes_shared) { +- result = SCAN_EXCEED_SHARED_PTE; +- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); +- goto out; ++ if (page_mapcount(page) > 1) { ++ ++shared; ++ if (cc->is_khugepaged && ++ shared > khugepaged_max_ptes_shared) { ++ result = SCAN_EXCEED_SHARED_PTE; ++ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); ++ goto out; ++ } + } + + if (PageCompound(page)) { +@@ -646,10 +664,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + if (PageCompound(page)) + list_add_tail(&page->lru, compound_pagelist); + next: +- /* There should be enough young pte to collapse the page */ +- if (pte_young(pteval) || +- page_is_young(page) || PageReferenced(page) || +- mmu_notifier_test_young(vma->vm_mm, address)) ++ /* ++ * If collapse was initiated by khugepaged, check that there is ++ * enough young pte to justify collapsing the page ++ */ ++ if (cc->is_khugepaged && ++ (pte_young(pteval) || page_is_young(page) || ++ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, ++ address))) + referenced++; + + if (pte_write(pteval)) +@@ -658,19 +680,19 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, + + if (unlikely(!writable)) { + result = SCAN_PAGE_RO; +- } else if (unlikely(!referenced)) { ++ } else if (unlikely(cc->is_khugepaged && !referenced)) { + result = SCAN_LACK_REFERENCED_PAGE; + } else { + result = SCAN_SUCCEED; + trace_mm_collapse_huge_page_isolate(page, none_or_zero, + referenced, writable, result); +- return 1; ++ return result; + } + out: + release_pte_pages(pte, _pte, compound_pagelist); + trace_mm_collapse_huge_page_isolate(page, none_or_zero, + referenced, writable, result); +- return 0; ++ return result; + } + + static void __collapse_huge_page_copy(pte_t *pte, struct page *page, +@@ -735,9 +757,12 @@ static void khugepaged_alloc_sleep(void) + remove_wait_queue(&khugepaged_wait, &wait); + } + +-static int khugepaged_node_load[MAX_NUMNODES]; ++struct collapse_control khugepaged_collapse_control = { ++ .is_khugepaged = true, ++ .last_target_node = NUMA_NO_NODE, ++}; + +-static bool khugepaged_scan_abort(int nid) ++static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) + { + int i; + +@@ -749,11 +774,11 @@ static bool khugepaged_scan_abort(int nid) + return false; + + /* If there is a count for this node already, it must be acceptable */ +- if (khugepaged_node_load[nid]) ++ if (cc->node_load[nid]) + return false; + + for (i = 0; i < MAX_NUMNODES; i++) { +- if (!khugepaged_node_load[i]) ++ if (!cc->node_load[i]) + continue; + if (node_distance(nid, i) > node_reclaim_distance) + return true; +@@ -772,146 +797,62 @@ static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) + } + + #ifdef CONFIG_NUMA +-static int khugepaged_find_target_node(void) ++static int hpage_collapse_find_target_node(struct collapse_control *cc) + { +- static int last_khugepaged_target_node = NUMA_NO_NODE; + int nid, target_node = 0, max_value = 0; + + /* find first node with max normal pages hit */ + for (nid = 0; nid < MAX_NUMNODES; nid++) +- if (khugepaged_node_load[nid] > max_value) { +- max_value = khugepaged_node_load[nid]; ++ if (cc->node_load[nid] > max_value) { ++ max_value = cc->node_load[nid]; + target_node = nid; + } + + /* do some balance if several nodes have the same hit record */ +- if (target_node <= last_khugepaged_target_node) +- for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; +- nid++) +- if (max_value == khugepaged_node_load[nid]) { ++ if (target_node <= cc->last_target_node) ++ for (nid = cc->last_target_node + 1; nid < MAX_NUMNODES; ++ nid++) ++ if (max_value == cc->node_load[nid]) { + target_node = nid; + break; + } + +- last_khugepaged_target_node = target_node; ++ cc->last_target_node = target_node; + return target_node; + } +- +-static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) ++#else ++static int hpage_collapse_find_target_node(struct collapse_control *cc) + { +- if (IS_ERR(*hpage)) { +- if (!*wait) +- return false; +- +- *wait = false; +- *hpage = NULL; +- khugepaged_alloc_sleep(); +- } else if (*hpage) { +- put_page(*hpage); +- *hpage = NULL; +- } +- +- return true; ++ return 0; + } ++#endif + +-static struct page * +-khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) ++static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node) + { +- VM_BUG_ON_PAGE(*hpage, *hpage); +- + *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); + if (unlikely(!*hpage)) { + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); +- *hpage = ERR_PTR(-ENOMEM); +- return NULL; ++ return false; + } + + prep_transhuge_page(*hpage); + count_vm_event(THP_COLLAPSE_ALLOC); +- return *hpage; +-} +-#else +-static int khugepaged_find_target_node(void) +-{ +- return 0; +-} +- +-static inline struct page *alloc_khugepaged_hugepage(void) +-{ +- struct page *page; +- +- page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), +- HPAGE_PMD_ORDER); +- if (page) +- prep_transhuge_page(page); +- return page; +-} +- +-static struct page *khugepaged_alloc_hugepage(bool *wait) +-{ +- struct page *hpage; +- +- do { +- hpage = alloc_khugepaged_hugepage(); +- if (!hpage) { +- count_vm_event(THP_COLLAPSE_ALLOC_FAILED); +- if (!*wait) +- return NULL; +- +- *wait = false; +- khugepaged_alloc_sleep(); +- } else +- count_vm_event(THP_COLLAPSE_ALLOC); +- } while (unlikely(!hpage) && likely(hugepage_flags_enabled())); +- +- return hpage; +-} +- +-static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) +-{ +- /* +- * If the hpage allocated earlier was briefly exposed in page cache +- * before collapse_file() failed, it is possible that racing lookups +- * have not yet completed, and would then be unpleasantly surprised by +- * finding the hpage reused for the same mapping at a different offset. +- * Just release the previous allocation if there is any danger of that. +- */ +- if (*hpage && page_count(*hpage) > 1) { +- put_page(*hpage); +- *hpage = NULL; +- } +- +- if (!*hpage) +- *hpage = khugepaged_alloc_hugepage(wait); +- +- if (unlikely(!*hpage)) +- return false; +- + return true; + } + +-static struct page * +-khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) +-{ +- VM_BUG_ON(!*hpage); +- +- return *hpage; +-} +-#endif +- + /* + * If mmap_lock temporarily dropped, revalidate vma + * before taking mmap_lock. +- * Return 0 if succeeds, otherwise return none-zero +- * value (scan code). ++ * Returns enum scan_result value. + */ + + static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, +- struct vm_area_struct **vmap) ++ struct vm_area_struct **vmap, ++ struct collapse_control *cc) + { + struct vm_area_struct *vma; + +- if (unlikely(khugepaged_test_exit(mm))) ++ if (unlikely(hpage_collapse_test_exit(mm))) + return SCAN_ANY_PROCESS; + + *vmap = vma = find_vma(mm, address); +@@ -920,7 +861,8 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, + + if (!transhuge_vma_suitable(vma, address)) + return SCAN_ADDRESS_RANGE; +- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, ++ cc->is_khugepaged)) + return SCAN_VMA_CHECK; + /* + * Anon VMA expected, the address may be unmapped then +@@ -931,21 +873,60 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, + */ + if (!vma->anon_vma || !vma_is_anonymous(vma)) + return SCAN_VMA_CHECK; +- return 0; ++ return SCAN_SUCCEED; +} + -+#define check_event_state(fd, event, signaled, manual) \ -+ ({ \ -+ __u32 __signaled, __manual; \ -+ int ret = read_event_state((fd), (event), \ -+ &__signaled, &__manual); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((signaled), __signaled); \ -+ EXPECT_EQ((manual), __manual); \ -+ }) -+ -+static int wait_objs(int fd, unsigned long request, __u32 count, -+ const __u32 *objs, __u32 owner, __u32 alert, __u32 *index) ++static int find_pmd_or_thp_or_none(struct mm_struct *mm, ++ unsigned long address, ++ pmd_t **pmd) +{ -+ struct winesync_wait_args args = {0}; -+ struct timespec timeout; -+ int ret; ++ pmd_t pmde; + -+ clock_gettime(CLOCK_MONOTONIC, &timeout); ++ *pmd = mm_find_pmd(mm, address); ++ if (!*pmd) ++ return SCAN_PMD_NULL; + -+ args.timeout = (uintptr_t)&timeout; -+ args.count = count; -+ args.objs = (uintptr_t)objs; -+ args.owner = owner; -+ args.index = 0xdeadbeef; -+ args.alert = alert; -+ ret = ioctl(fd, request, &args); -+ *index = args.index; -+ return ret; -+} ++ pmde = pmd_read_atomic(*pmd); + -+static int wait_any(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, -+ count, objs, owner, 0, index); -+} ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ ++ barrier(); ++#endif ++ if (!pmd_present(pmde)) ++ return SCAN_PMD_NULL; ++ if (pmd_trans_huge(pmde)) ++ return SCAN_PMD_MAPPED; ++ if (pmd_bad(pmde)) ++ return SCAN_PMD_NULL; ++ return SCAN_SUCCEED; ++} ++ ++static int check_pmd_still_valid(struct mm_struct *mm, ++ unsigned long address, ++ pmd_t *pmd) ++{ ++ pmd_t *new_pmd; ++ int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); ++ ++ if (result != SCAN_SUCCEED) ++ return result; ++ if (new_pmd != pmd) ++ return SCAN_FAIL; ++ return SCAN_SUCCEED; + } + + /* + * Bring missing pages in from swap, to complete THP collapse. +- * Only done if khugepaged_scan_pmd believes it is worthwhile. ++ * Only done if hpage_collapse_scan_pmd believes it is worthwhile. + * + * Called and returns without pte mapped or spinlocks held. + * Note that if false is returned, mmap_lock will be released. + */ + +-static bool __collapse_huge_page_swapin(struct mm_struct *mm, +- struct vm_area_struct *vma, +- unsigned long haddr, pmd_t *pmd, +- int referenced) ++static int __collapse_huge_page_swapin(struct mm_struct *mm, ++ struct vm_area_struct *vma, ++ unsigned long haddr, pmd_t *pmd, ++ int referenced) + { + int swapped_in = 0; + vm_fault_t ret = 0; +@@ -976,12 +957,13 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, + */ + if (ret & VM_FAULT_RETRY) { + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); +- return false; ++ /* Likely, but not guaranteed, that page lock failed */ ++ return SCAN_PAGE_LOCK; + } + if (ret & VM_FAULT_ERROR) { + mmap_read_unlock(mm); + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); +- return false; ++ return SCAN_FAIL; + } + swapped_in++; + } +@@ -991,30 +973,41 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, + lru_add_drain(); + + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); +- return true; ++ return SCAN_SUCCEED; + } + +-static void collapse_huge_page(struct mm_struct *mm, +- unsigned long address, +- struct page **hpage, +- int node, int referenced, int unmapped) ++static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, ++ struct collapse_control *cc) ++{ ++ /* Only allocate from the target node */ ++ gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : ++ GFP_TRANSHUGE) | __GFP_THISNODE; ++ int node = hpage_collapse_find_target_node(cc); ++ ++ if (!hpage_collapse_alloc_page(hpage, gfp, node)) ++ return SCAN_ALLOC_HUGE_PAGE_FAIL; ++ if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) ++ return SCAN_CGROUP_CHARGE_FAIL; ++ count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); ++ return SCAN_SUCCEED; ++} ++ ++static int collapse_huge_page(struct mm_struct *mm, unsigned long address, ++ int referenced, int unmapped, ++ struct collapse_control *cc) + { + LIST_HEAD(compound_pagelist); + pmd_t *pmd, _pmd; + pte_t *pte; + pgtable_t pgtable; +- struct page *new_page; ++ struct page *hpage; + spinlock_t *pmd_ptl, *pte_ptl; +- int isolated = 0, result = 0; ++ int result = SCAN_FAIL; + struct vm_area_struct *vma; + struct mmu_notifier_range range; +- gfp_t gfp; + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + +- /* Only allocate from the target node */ +- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; +- + /* + * Before allocating the hugepage, release the mmap_lock read lock. + * The allocation can take potentially a long time if it involves +@@ -1022,40 +1015,34 @@ static void collapse_huge_page(struct mm_struct *mm, + * that. We will recheck the vma after taking it again in write mode. + */ + mmap_read_unlock(mm); +- new_page = khugepaged_alloc_page(hpage, gfp, node); +- if (!new_page) { +- result = SCAN_ALLOC_HUGE_PAGE_FAIL; +- goto out_nolock; +- } + +- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { +- result = SCAN_CGROUP_CHARGE_FAIL; ++ result = alloc_charge_hpage(&hpage, mm, cc); ++ if (result != SCAN_SUCCEED) + goto out_nolock; +- } +- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); + + mmap_read_lock(mm); +- result = hugepage_vma_revalidate(mm, address, &vma); +- if (result) { ++ result = hugepage_vma_revalidate(mm, address, &vma, cc); ++ if (result != SCAN_SUCCEED) { + mmap_read_unlock(mm); + goto out_nolock; + } + +- pmd = mm_find_pmd(mm, address); +- if (!pmd) { +- result = SCAN_PMD_NULL; ++ result = find_pmd_or_thp_or_none(mm, address, &pmd); ++ if (result != SCAN_SUCCEED) { + mmap_read_unlock(mm); + goto out_nolock; + } + +- /* +- * __collapse_huge_page_swapin will return with mmap_lock released +- * when it fails. So we jump out_nolock directly in that case. +- * Continuing to collapse causes inconsistency. +- */ +- if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, +- pmd, referenced)) { +- goto out_nolock; ++ if (unmapped) { ++ /* ++ * __collapse_huge_page_swapin will return with mmap_lock ++ * released when it fails. So we jump out_nolock directly in ++ * that case. Continuing to collapse causes inconsistency. ++ */ ++ result = __collapse_huge_page_swapin(mm, vma, address, pmd, ++ referenced); ++ if (result != SCAN_SUCCEED) ++ goto out_nolock; + } + + mmap_read_unlock(mm); +@@ -1065,11 +1052,12 @@ static void collapse_huge_page(struct mm_struct *mm, + * handled by the anon_vma lock + PG_lock. + */ + mmap_write_lock(mm); +- result = hugepage_vma_revalidate(mm, address, &vma); +- if (result) ++ result = hugepage_vma_revalidate(mm, address, &vma, cc); ++ if (result != SCAN_SUCCEED) + goto out_up_write; + /* check if the pmd is still valid */ +- if (mm_find_pmd(mm, address) != pmd) ++ result = check_pmd_still_valid(mm, address, pmd); ++ if (result != SCAN_SUCCEED) + goto out_up_write; + + anon_vma_lock_write(vma->anon_vma); +@@ -1093,11 +1081,11 @@ static void collapse_huge_page(struct mm_struct *mm, + mmu_notifier_invalidate_range_end(&range); + + spin_lock(pte_ptl); +- isolated = __collapse_huge_page_isolate(vma, address, pte, +- &compound_pagelist); ++ result = __collapse_huge_page_isolate(vma, address, pte, cc, ++ &compound_pagelist); + spin_unlock(pte_ptl); + +- if (unlikely(!isolated)) { ++ if (unlikely(result != SCAN_SUCCEED)) { + pte_unmap(pte); + spin_lock(pmd_ptl); + BUG_ON(!pmd_none(*pmd)); +@@ -1109,7 +1097,6 @@ static void collapse_huge_page(struct mm_struct *mm, + pmd_populate(mm, pmd, pmd_pgtable(_pmd)); + spin_unlock(pmd_ptl); + anon_vma_unlock_write(vma->anon_vma); +- result = SCAN_FAIL; + goto out_up_write; + } + +@@ -1119,8 +1106,8 @@ static void collapse_huge_page(struct mm_struct *mm, + */ + anon_vma_unlock_write(vma->anon_vma); + +- __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, +- &compound_pagelist); ++ __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, ++ &compound_pagelist); + pte_unmap(pte); + /* + * spin_lock() below is not the equivalent of smp_wmb(), but +@@ -1128,42 +1115,43 @@ static void collapse_huge_page(struct mm_struct *mm, + * avoid the copy_huge_page writes to become visible after + * the set_pmd_at() write. + */ +- __SetPageUptodate(new_page); ++ __SetPageUptodate(hpage); + pgtable = pmd_pgtable(_pmd); + +- _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); ++ _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); + _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); + + spin_lock(pmd_ptl); + BUG_ON(!pmd_none(*pmd)); +- page_add_new_anon_rmap(new_page, vma, address); +- lru_cache_add_inactive_or_unevictable(new_page, vma); ++ page_add_new_anon_rmap(hpage, vma, address); ++ lru_cache_add_inactive_or_unevictable(hpage, vma); + pgtable_trans_huge_deposit(mm, pmd, pgtable); + set_pmd_at(mm, address, pmd, _pmd); + update_mmu_cache_pmd(vma, address, pmd); + spin_unlock(pmd_ptl); + +- *hpage = NULL; ++ hpage = NULL; + +- khugepaged_pages_collapsed++; + result = SCAN_SUCCEED; + out_up_write: + mmap_write_unlock(mm); + out_nolock: +- if (!IS_ERR_OR_NULL(*hpage)) +- mem_cgroup_uncharge(page_folio(*hpage)); +- trace_mm_collapse_huge_page(mm, isolated, result); +- return; ++ if (hpage) { ++ mem_cgroup_uncharge(page_folio(hpage)); ++ put_page(hpage); ++ } ++ trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); ++ return result; + } + +-static int khugepaged_scan_pmd(struct mm_struct *mm, +- struct vm_area_struct *vma, +- unsigned long address, +- struct page **hpage) ++static int hpage_collapse_scan_pmd(struct mm_struct *mm, ++ struct vm_area_struct *vma, ++ unsigned long address, bool *mmap_locked, ++ struct collapse_control *cc) + { + pmd_t *pmd; + pte_t *pte, *_pte; +- int ret = 0, result = 0, referenced = 0; ++ int result = SCAN_FAIL, referenced = 0; + int none_or_zero = 0, shared = 0; + struct page *page = NULL; + unsigned long _address; +@@ -1173,19 +1161,19 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + +- pmd = mm_find_pmd(mm, address); +- if (!pmd) { +- result = SCAN_PMD_NULL; ++ result = find_pmd_or_thp_or_none(mm, address, &pmd); ++ if (result != SCAN_SUCCEED) + goto out; +- } + +- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); ++ memset(cc->node_load, 0, sizeof(cc->node_load)); + pte = pte_offset_map_lock(mm, pmd, address, &ptl); + for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; + _pte++, _address += PAGE_SIZE) { + pte_t pteval = *_pte; + if (is_swap_pte(pteval)) { +- if (++unmapped <= khugepaged_max_ptes_swap) { ++ ++unmapped; ++ if (!cc->is_khugepaged || ++ unmapped <= khugepaged_max_ptes_swap) { + /* + * Always be strict with uffd-wp + * enabled swap entries. Please see +@@ -1203,8 +1191,10 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + } + } + if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { ++ ++none_or_zero; + if (!userfaultfd_armed(vma) && +- ++none_or_zero <= khugepaged_max_ptes_none) { ++ (!cc->is_khugepaged || ++ none_or_zero <= khugepaged_max_ptes_none)) { + continue; + } else { + result = SCAN_EXCEED_NONE_PTE; +@@ -1234,27 +1224,30 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + goto out_unmap; + } + +- if (page_mapcount(page) > 1 && +- ++shared > khugepaged_max_ptes_shared) { +- result = SCAN_EXCEED_SHARED_PTE; +- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); +- goto out_unmap; ++ if (page_mapcount(page) > 1) { ++ ++shared; ++ if (cc->is_khugepaged && ++ shared > khugepaged_max_ptes_shared) { ++ result = SCAN_EXCEED_SHARED_PTE; ++ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); ++ goto out_unmap; ++ } + } + + page = compound_head(page); + + /* + * Record which node the original page is from and save this +- * information to khugepaged_node_load[]. ++ * information to cc->node_load[]. + * Khugepaged will allocate hugepage from the node has the max + * hit record. + */ + node = page_to_nid(page); +- if (khugepaged_scan_abort(node)) { ++ if (hpage_collapse_scan_abort(node, cc)) { + result = SCAN_SCAN_ABORT; + goto out_unmap; + } +- khugepaged_node_load[node]++; ++ cc->node_load[node]++; + if (!PageLRU(page)) { + result = SCAN_PAGE_LRU; + goto out_unmap; +@@ -1289,31 +1282,38 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, + result = SCAN_PAGE_COUNT; + goto out_unmap; + } +- if (pte_young(pteval) || +- page_is_young(page) || PageReferenced(page) || +- mmu_notifier_test_young(vma->vm_mm, address)) + -+static int wait_all(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, -+ count, objs, owner, 0, index); -+} ++ /* ++ * If collapse was initiated by khugepaged, check that there is ++ * enough young pte to justify collapsing the page ++ */ ++ if (cc->is_khugepaged && ++ (pte_young(pteval) || page_is_young(page) || ++ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, ++ address))) + referenced++; + } + if (!writable) { + result = SCAN_PAGE_RO; +- } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { ++ } else if (cc->is_khugepaged && ++ (!referenced || ++ (unmapped && referenced < HPAGE_PMD_NR / 2))) { + result = SCAN_LACK_REFERENCED_PAGE; + } else { + result = SCAN_SUCCEED; +- ret = 1; + } + out_unmap: + pte_unmap_unlock(pte, ptl); +- if (ret) { +- node = khugepaged_find_target_node(); ++ if (result == SCAN_SUCCEED) { ++ result = collapse_huge_page(mm, address, referenced, ++ unmapped, cc); + /* collapse_huge_page will return with the mmap_lock released */ +- collapse_huge_page(mm, address, hpage, node, +- referenced, unmapped); ++ *mmap_locked = false; + } + out: + trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, + none_or_zero, result, unmapped); +- return ret; ++ return result; + } + + static void collect_mm_slot(struct mm_slot *mm_slot) +@@ -1322,7 +1322,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) + + lockdep_assert_held(&khugepaged_mm_lock); + +- if (khugepaged_test_exit(mm)) { ++ if (hpage_collapse_test_exit(mm)) { + /* free mm_slot */ + hash_del(&mm_slot->hash); + list_del(&mm_slot->mm_node); +@@ -1400,12 +1400,13 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) + return; + + /* +- * This vm_flags may not have VM_HUGEPAGE if the page was not +- * collapsed by this mm. But we can still collapse if the page is +- * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() +- * will not fail the vma for missing VM_HUGEPAGE ++ * If we are here, we've succeeded in replacing all the native pages ++ * in the page cache with a single hugepage. If a mm were to fault-in ++ * this memory (mapped by a suitably aligned VMA), we'd get the hugepage ++ * and map it by a PMD, regardless of sysfs THP settings. As such, let's ++ * analogously elide sysfs THP settings here. + */ +- if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false)) ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) + return; + + /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ +@@ -1420,8 +1421,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) + if (!PageHead(hpage)) + goto drop_hpage; + +- pmd = mm_find_pmd(mm, haddr); +- if (!pmd) ++ if (find_pmd_or_thp_or_none(mm, haddr, &pmd) != SCAN_SUCCEED) + goto drop_hpage; + + start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); +@@ -1495,7 +1495,7 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) + if (!mmap_write_trylock(mm)) + return; + +- if (unlikely(khugepaged_test_exit(mm))) ++ if (unlikely(hpage_collapse_test_exit(mm))) + goto out; + + for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) +@@ -1539,8 +1539,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + if (vma->vm_end < addr + HPAGE_PMD_SIZE) + continue; + mm = vma->vm_mm; +- pmd = mm_find_pmd(mm, addr); +- if (!pmd) ++ if (find_pmd_or_thp_or_none(mm, addr, &pmd) != SCAN_SUCCEED) + continue; + /* + * We need exclusive mmap_lock to retract page table. +@@ -1558,7 +1557,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + * it'll always mapped in small page size for uffd-wp + * registered ranges. + */ +- if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) ++ if (!hpage_collapse_test_exit(mm) && ++ !userfaultfd_wp(vma)) + collapse_and_free_pmd(mm, vma, addr, pmd); + mmap_write_unlock(mm); + } else { +@@ -1575,8 +1575,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + * @mm: process address space where collapse happens + * @file: file that collapse on + * @start: collapse start address +- * @hpage: new allocated huge page for collapse +- * @node: appointed node the new huge page allocate from ++ * @cc: collapse context and scratchpad + * + * Basic scheme is simple, details are more complex: + * - allocate and lock a new huge page; +@@ -1593,13 +1592,11 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) + * + restore gaps in the page cache; + * + unlock and free huge page; + */ +-static void collapse_file(struct mm_struct *mm, +- struct file *file, pgoff_t start, +- struct page **hpage, int node) ++static int collapse_file(struct mm_struct *mm, struct file *file, ++ pgoff_t start, struct collapse_control *cc) + { + struct address_space *mapping = file->f_mapping; +- gfp_t gfp; +- struct page *new_page; ++ struct page *hpage; + pgoff_t index, end = start + HPAGE_PMD_NR; + LIST_HEAD(pagelist); + XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); +@@ -1610,20 +1607,9 @@ static void collapse_file(struct mm_struct *mm, + VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); + VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + +- /* Only allocate from the target node */ +- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; +- +- new_page = khugepaged_alloc_page(hpage, gfp, node); +- if (!new_page) { +- result = SCAN_ALLOC_HUGE_PAGE_FAIL; +- goto out; +- } +- +- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { +- result = SCAN_CGROUP_CHARGE_FAIL; ++ result = alloc_charge_hpage(&hpage, mm, cc); ++ if (result != SCAN_SUCCEED) + goto out; +- } +- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); + + /* + * Ensure we have slots for all the pages in the range. This is +@@ -1641,14 +1627,14 @@ static void collapse_file(struct mm_struct *mm, + } + } while (1); + +- __SetPageLocked(new_page); ++ __SetPageLocked(hpage); + if (is_shmem) +- __SetPageSwapBacked(new_page); +- new_page->index = start; +- new_page->mapping = mapping; ++ __SetPageSwapBacked(hpage); ++ hpage->index = start; ++ hpage->mapping = mapping; + + /* +- * At this point the new_page is locked and not up-to-date. ++ * At this point the hpage is locked and not up-to-date. + * It's safe to insert it into the page cache, because nobody would + * be able to map it or use it in another way until we unlock it. + */ +@@ -1676,7 +1662,7 @@ static void collapse_file(struct mm_struct *mm, + result = SCAN_FAIL; + goto xa_locked; + } +- xas_store(&xas, new_page); ++ xas_store(&xas, hpage); + nr_none++; + continue; + } +@@ -1818,19 +1804,19 @@ static void collapse_file(struct mm_struct *mm, + list_add_tail(&page->lru, &pagelist); + + /* Finally, replace with the new page. */ +- xas_store(&xas, new_page); ++ xas_store(&xas, hpage); + continue; + out_unlock: + unlock_page(page); + put_page(page); + goto xa_unlocked; + } +- nr = thp_nr_pages(new_page); ++ nr = thp_nr_pages(hpage); + + if (is_shmem) +- __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); ++ __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); + else { +- __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); ++ __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); + filemap_nr_thps_inc(mapping); + /* + * Paired with smp_mb() in do_dentry_open() to ensure +@@ -1841,21 +1827,21 @@ static void collapse_file(struct mm_struct *mm, + smp_mb(); + if (inode_is_open_for_write(mapping->host)) { + result = SCAN_FAIL; +- __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); ++ __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); + filemap_nr_thps_dec(mapping); + goto xa_locked; + } + } + + if (nr_none) { +- __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); ++ __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); + /* nr_none is always 0 for non-shmem. */ +- __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); ++ __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); + } + + /* Join all the small entries into a single multi-index entry */ + xas_set_order(&xas, start, HPAGE_PMD_ORDER); +- xas_store(&xas, new_page); ++ xas_store(&xas, hpage); + xa_locked: + xas_unlock_irq(&xas); + xa_unlocked: +@@ -1877,11 +1863,11 @@ static void collapse_file(struct mm_struct *mm, + index = start; + list_for_each_entry_safe(page, tmp, &pagelist, lru) { + while (index < page->index) { +- clear_highpage(new_page + (index % HPAGE_PMD_NR)); ++ clear_highpage(hpage + (index % HPAGE_PMD_NR)); + index++; + } +- copy_highpage(new_page + (page->index % HPAGE_PMD_NR), +- page); ++ copy_highpage(hpage + (page->index % HPAGE_PMD_NR), ++ page); + list_del(&page->lru); + page->mapping = NULL; + page_ref_unfreeze(page, 1); +@@ -1892,23 +1878,22 @@ static void collapse_file(struct mm_struct *mm, + index++; + } + while (index < end) { +- clear_highpage(new_page + (index % HPAGE_PMD_NR)); ++ clear_highpage(hpage + (index % HPAGE_PMD_NR)); + index++; + } + +- SetPageUptodate(new_page); +- page_ref_add(new_page, HPAGE_PMD_NR - 1); ++ SetPageUptodate(hpage); ++ page_ref_add(hpage, HPAGE_PMD_NR - 1); + if (is_shmem) +- set_page_dirty(new_page); +- lru_cache_add(new_page); ++ set_page_dirty(hpage); ++ lru_cache_add(hpage); + + /* + * Remove pte page tables, so we can re-fault the page as huge. + */ + retract_page_tables(mapping, start); +- *hpage = NULL; +- +- khugepaged_pages_collapsed++; ++ unlock_page(hpage); ++ hpage = NULL; + } else { + struct page *page; + +@@ -1947,19 +1932,23 @@ static void collapse_file(struct mm_struct *mm, + VM_BUG_ON(nr_none); + xas_unlock_irq(&xas); + +- new_page->mapping = NULL; ++ hpage->mapping = NULL; + } + +- unlock_page(new_page); ++ if (hpage) ++ unlock_page(hpage); + out: + VM_BUG_ON(!list_empty(&pagelist)); +- if (!IS_ERR_OR_NULL(*hpage)) +- mem_cgroup_uncharge(page_folio(*hpage)); ++ if (hpage) { ++ mem_cgroup_uncharge(page_folio(hpage)); ++ put_page(hpage); ++ } + /* TODO: tracepoints */ ++ return result; + } + +-static void khugepaged_scan_file(struct mm_struct *mm, +- struct file *file, pgoff_t start, struct page **hpage) ++static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, ++ pgoff_t start, struct collapse_control *cc) + { + struct page *page = NULL; + struct address_space *mapping = file->f_mapping; +@@ -1970,14 +1959,16 @@ static void khugepaged_scan_file(struct mm_struct *mm, + + present = 0; + swap = 0; +- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); ++ memset(cc->node_load, 0, sizeof(cc->node_load)); + rcu_read_lock(); + xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { + if (xas_retry(&xas, page)) + continue; + + if (xa_is_value(page)) { +- if (++swap > khugepaged_max_ptes_swap) { ++ ++swap; ++ if (cc->is_khugepaged && ++ swap > khugepaged_max_ptes_swap) { + result = SCAN_EXCEED_SWAP_PTE; + count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); + break; +@@ -1995,11 +1986,11 @@ static void khugepaged_scan_file(struct mm_struct *mm, + } + + node = page_to_nid(page); +- if (khugepaged_scan_abort(node)) { ++ if (hpage_collapse_scan_abort(node, cc)) { + result = SCAN_SCAN_ABORT; + break; + } +- khugepaged_node_load[node]++; ++ cc->node_load[node]++; + + if (!PageLRU(page)) { + result = SCAN_PAGE_LRU; +@@ -2028,20 +2019,21 @@ static void khugepaged_scan_file(struct mm_struct *mm, + rcu_read_unlock(); + + if (result == SCAN_SUCCEED) { +- if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { ++ if (cc->is_khugepaged && ++ present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { + result = SCAN_EXCEED_NONE_PTE; + count_vm_event(THP_SCAN_EXCEED_NONE_PTE); + } else { +- node = khugepaged_find_target_node(); +- collapse_file(mm, file, start, hpage, node); ++ result = collapse_file(mm, file, start, cc); + } + } + + /* TODO: tracepoints */ ++ return result; + } + #else +-static void khugepaged_scan_file(struct mm_struct *mm, +- struct file *file, pgoff_t start, struct page **hpage) ++static int khugepaged_scan_file(struct mm_struct *mm, struct file *file, ++ pgoff_t start, struct collapse_control *cc) + { + BUILD_BUG(); + } +@@ -2051,8 +2043,8 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) + } + #endif + +-static unsigned int khugepaged_scan_mm_slot(unsigned int pages, +- struct page **hpage) ++static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, ++ struct collapse_control *cc) + __releases(&khugepaged_mm_lock) + __acquires(&khugepaged_mm_lock) + { +@@ -2063,6 +2055,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + + VM_BUG_ON(!pages); + lockdep_assert_held(&khugepaged_mm_lock); ++ *result = SCAN_FAIL; + + if (khugepaged_scan.mm_slot) + mm_slot = khugepaged_scan.mm_slot; +@@ -2083,7 +2076,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + vma = NULL; + if (unlikely(!mmap_read_trylock(mm))) + goto breakouterloop_mmap_lock; +- if (likely(!khugepaged_test_exit(mm))) ++ if (likely(!hpage_collapse_test_exit(mm))) + vma = find_vma(mm, khugepaged_scan.address); + + progress++; +@@ -2091,11 +2084,11 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + unsigned long hstart, hend; + + cond_resched(); +- if (unlikely(khugepaged_test_exit(mm))) { ++ if (unlikely(hpage_collapse_test_exit(mm))) { + progress++; + break; + } +- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) { ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { + skip: + progress++; + continue; +@@ -2109,9 +2102,10 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); + + while (khugepaged_scan.address < hend) { +- int ret; ++ bool mmap_locked = true; ++ + cond_resched(); +- if (unlikely(khugepaged_test_exit(mm))) ++ if (unlikely(hpage_collapse_test_exit(mm))) + goto breakouterloop; + + VM_BUG_ON(khugepaged_scan.address < hstart || +@@ -2123,19 +2117,29 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + khugepaged_scan.address); + + mmap_read_unlock(mm); +- ret = 1; +- khugepaged_scan_file(mm, file, pgoff, hpage); ++ *result = khugepaged_scan_file(mm, file, pgoff, ++ cc); ++ mmap_locked = false; + fput(file); + } else { +- ret = khugepaged_scan_pmd(mm, vma, +- khugepaged_scan.address, +- hpage); ++ *result = hpage_collapse_scan_pmd(mm, vma, ++ khugepaged_scan.address, ++ &mmap_locked, ++ cc); + } ++ if (*result == SCAN_SUCCEED) ++ ++khugepaged_pages_collapsed; + /* move to next address */ + khugepaged_scan.address += HPAGE_PMD_SIZE; + progress += HPAGE_PMD_NR; +- if (ret) +- /* we released mmap_lock so break loop */ ++ if (!mmap_locked) ++ /* ++ * We released mmap_lock so break loop. Note ++ * that we drop mmap_lock before all hugepage ++ * allocations, so if allocation fails, we are ++ * guaranteed to break here and report the ++ * correct result back to caller. ++ */ + goto breakouterloop_mmap_lock; + if (progress >= pages) + goto breakouterloop; +@@ -2151,7 +2155,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, + * Release the current mm_slot if this mm is about to die, or + * if we scanned all vmas of this mm. + */ +- if (khugepaged_test_exit(mm) || !vma) { ++ if (hpage_collapse_test_exit(mm) || !vma) { + /* + * Make sure that if mm_users is reaching zero while + * khugepaged runs here, khugepaged_exit will find +@@ -2185,19 +2189,16 @@ static int khugepaged_wait_event(void) + kthread_should_stop(); + } + +-static void khugepaged_do_scan(void) ++static void khugepaged_do_scan(struct collapse_control *cc) + { +- struct page *hpage = NULL; + unsigned int progress = 0, pass_through_head = 0; + unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); + bool wait = true; ++ int result = SCAN_SUCCEED; + + lru_add_drain_all(); + +- while (progress < pages) { +- if (!khugepaged_prealloc_page(&hpage, &wait)) +- break; +- ++ while (true) { + cond_resched(); + + if (unlikely(kthread_should_stop() || try_to_freeze())) +@@ -2209,14 +2210,25 @@ static void khugepaged_do_scan(void) + if (khugepaged_has_work() && + pass_through_head < 2) + progress += khugepaged_scan_mm_slot(pages - progress, +- &hpage); ++ &result, cc); + else + progress = pages; + spin_unlock(&khugepaged_mm_lock); +- } + +- if (!IS_ERR_OR_NULL(hpage)) +- put_page(hpage); ++ if (progress >= pages) ++ break; + -+static int wait_any_alert(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 alert, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, -+ count, objs, owner, alert, index); -+} ++ if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { ++ /* ++ * If fail to allocate the first time, try to sleep for ++ * a while. When hit again, cancel the scan. ++ */ ++ if (!wait) ++ break; ++ wait = false; ++ khugepaged_alloc_sleep(); ++ } ++ } + } + + static bool khugepaged_should_wakeup(void) +@@ -2253,7 +2265,7 @@ static int khugepaged(void *none) + set_user_nice(current, MAX_NICE); + + while (!kthread_should_stop()) { +- khugepaged_do_scan(); ++ khugepaged_do_scan(&khugepaged_collapse_control); + khugepaged_wait_work(); + } + +@@ -2352,3 +2364,120 @@ void khugepaged_min_free_kbytes_update(void) + set_recommended_min_free_kbytes(); + mutex_unlock(&khugepaged_mutex); + } + -+static int wait_all_alert(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 alert, __u32 *index) ++static int madvise_collapse_errno(enum scan_result r) +{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, -+ count, objs, owner, alert, index); ++ /* ++ * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide ++ * actionable feedback to caller, so they may take an appropriate ++ * fallback measure depending on the nature of the failure. ++ */ ++ switch (r) { ++ case SCAN_ALLOC_HUGE_PAGE_FAIL: ++ return -ENOMEM; ++ case SCAN_CGROUP_CHARGE_FAIL: ++ return -EBUSY; ++ /* Resource temporary unavailable - trying again might succeed */ ++ case SCAN_PAGE_LOCK: ++ case SCAN_PAGE_LRU: ++ return -EAGAIN; ++ /* ++ * Other: Trying again likely not to succeed / error intrinsic to ++ * specified memory range. khugepaged likely won't be able to collapse ++ * either. ++ */ ++ default: ++ return -EINVAL; ++ } +} + -+TEST(semaphore_state) ++int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, ++ unsigned long start, unsigned long end) +{ -+ struct winesync_sem_args sem_args; -+ struct timespec timeout; -+ __u32 sem, count, index; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++ struct collapse_control *cc; ++ struct mm_struct *mm = vma->vm_mm; ++ unsigned long hstart, hend, addr; ++ int thps = 0, last_fail = SCAN_FAIL; ++ bool mmap_locked = true; + -+ sem_args.count = 3; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++ BUG_ON(vma->vm_start > start); ++ BUG_ON(vma->vm_end < end); + -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ check_sem_state(fd, sem, 2, 2); ++ *prev = vma; + -+ count = 0; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_sem_state(fd, sem, 2, 2); ++ /* TODO: Support file/shmem */ ++ if (!vma->anon_vma || !vma_is_anonymous(vma)) ++ return -EINVAL; + -+ count = 1; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(fd, sem, 2, 2); ++ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) ++ return -EINVAL; + -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem, 1, 2); ++ cc = kmalloc(sizeof(*cc), GFP_KERNEL); ++ if (!cc) ++ return -ENOMEM; ++ cc->is_khugepaged = false; ++ cc->last_target_node = NUMA_NO_NODE; + -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem, 0, 2); ++ mmgrab(mm); ++ lru_add_drain_all(); + -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); ++ hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; ++ hend = end & HPAGE_PMD_MASK; + -+ count = 3; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(fd, sem, 0, 2); ++ for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { ++ int result = SCAN_FAIL; + -+ count = 2; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(fd, sem, 2, 2); ++ if (!mmap_locked) { ++ cond_resched(); ++ mmap_read_lock(mm); ++ mmap_locked = true; ++ result = hugepage_vma_revalidate(mm, addr, &vma, cc); ++ if (result != SCAN_SUCCEED) { ++ last_fail = result; ++ goto out_nolock; ++ } ++ } ++ mmap_assert_locked(mm); ++ memset(cc->node_load, 0, sizeof(cc->node_load)); ++ result = hpage_collapse_scan_pmd(mm, vma, addr, &mmap_locked, ++ cc); ++ if (!mmap_locked) ++ *prev = NULL; /* Tell caller we dropped mmap_lock */ ++ ++ switch (result) { ++ case SCAN_SUCCEED: ++ case SCAN_PMD_MAPPED: ++ ++thps; ++ break; ++ /* Whitelisted set of results where continuing OK */ ++ case SCAN_PMD_NULL: ++ case SCAN_PTE_NON_PRESENT: ++ case SCAN_PTE_UFFD_WP: ++ case SCAN_PAGE_RO: ++ case SCAN_LACK_REFERENCED_PAGE: ++ case SCAN_PAGE_NULL: ++ case SCAN_PAGE_COUNT: ++ case SCAN_PAGE_LOCK: ++ case SCAN_PAGE_COMPOUND: ++ case SCAN_PAGE_LRU: ++ last_fail = result; ++ break; ++ default: ++ last_fail = result; ++ /* Other error, exit */ ++ goto out_maybelock; ++ } ++ } + -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); ++out_maybelock: ++ /* Caller expects us to hold mmap_lock on return */ ++ if (!mmap_locked) ++ mmap_read_lock(mm); ++out_nolock: ++ mmap_assert_locked(mm); ++ mmdrop(mm); ++ kfree(cc); + -+ count = 1; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(fd, sem, 1, 2); ++ return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 ++ : madvise_collapse_errno(last_fail); ++} +diff --git a/mm/ksm.c b/mm/ksm.c +index 4e64adb9adee..dde27d7e92d4 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -1134,6 +1134,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, + { + struct mm_struct *mm = vma->vm_mm; + pmd_t *pmd; ++ pmd_t pmde; + pte_t *ptep; + pte_t newpte; + spinlock_t *ptl; +@@ -1148,6 +1149,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, + pmd = mm_find_pmd(mm, addr); + if (!pmd) + goto out; ++ /* ++ * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() ++ * without holding anon_vma lock for write. So when looking for a ++ * genuine pmde (in which to find pte), test present and !THP together. ++ */ ++ pmde = *pmd; ++ barrier(); ++ if (!pmd_present(pmde) || pmd_trans_huge(pmde)) ++ goto out; + + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, + addr + PAGE_SIZE); +diff --git a/mm/madvise.c b/mm/madvise.c +index 5f0f0948a50e..af97100a0727 100644 +--- a/mm/madvise.c ++++ b/mm/madvise.c +@@ -59,6 +59,7 @@ static int madvise_need_mmap_write(int behavior) + case MADV_FREE: + case MADV_POPULATE_READ: + case MADV_POPULATE_WRITE: ++ case MADV_COLLAPSE: + return 0; + default: + /* be safe, default to 1. list exceptions explicitly */ +@@ -1057,6 +1058,8 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, + if (error) + goto out; + break; ++ case MADV_COLLAPSE: ++ return madvise_collapse(vma, prev, start, end); + } + + anon_name = anon_vma_name(vma); +@@ -1150,6 +1153,7 @@ madvise_behavior_valid(int behavior) + #ifdef CONFIG_TRANSPARENT_HUGEPAGE + case MADV_HUGEPAGE: + case MADV_NOHUGEPAGE: ++ case MADV_COLLAPSE: + #endif + case MADV_DONTDUMP: + case MADV_DODUMP: +@@ -1166,13 +1170,13 @@ madvise_behavior_valid(int behavior) + } + } + +-static bool +-process_madvise_behavior_valid(int behavior) ++static bool process_madvise_behavior_valid(int behavior) + { + switch (behavior) { + case MADV_COLD: + case MADV_PAGEOUT: + case MADV_WILLNEED: ++ case MADV_COLLAPSE: + return true; + default: + return false; +@@ -1339,6 +1343,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, + * MADV_NOHUGEPAGE - mark the given range as not worth being backed by + * transparent huge pages so the existing pages will not be + * coalesced into THP and new pages will not be allocated as THP. ++ * MADV_COLLAPSE - synchronously coalesce pages into new THP. + * MADV_DONTDUMP - the application wants to prevent pages in the given range + * from being included in its core dump. + * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. +diff --git a/mm/memory.c b/mm/memory.c +index 3a3d8721bf4c..e58d5d522467 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -4986,7 +4986,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, + return VM_FAULT_OOM; + retry_pud: + if (pud_none(*vmf.pud) && +- hugepage_vma_check(vma, vm_flags, false, true)) { ++ hugepage_vma_check(vma, vm_flags, false, true, true)) { + ret = create_huge_pud(&vmf); + if (!(ret & VM_FAULT_FALLBACK)) + return ret; +@@ -5020,7 +5020,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, + goto retry_pud; + + if (pmd_none(*vmf.pmd) && +- hugepage_vma_check(vma, vm_flags, false, true)) { ++ hugepage_vma_check(vma, vm_flags, false, true, true)) { + ret = create_huge_pmd(&vmf); + if (!(ret & VM_FAULT_FALLBACK)) + return ret; +diff --git a/mm/rmap.c b/mm/rmap.c +index 93d5a6f793d2..9af08343ce55 100644 +--- a/mm/rmap.c ++++ b/mm/rmap.c +@@ -770,13 +770,17 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) + return vma_address(page, vma); + } + ++/* ++ * Returns the actual pmd_t* where we expect 'address' to be mapped from, or ++ * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* ++ * represents. ++ */ + pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) + { + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; +- pmd_t pmde; + + pgd = pgd_offset(mm, address); + if (!pgd_present(*pgd)) +@@ -791,15 +795,6 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) + goto out; + + pmd = pmd_offset(pud, address); +- /* +- * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() +- * without holding anon_vma lock for write. So when looking for a +- * genuine pmde (in which to find pte), test present and !THP together. +- */ +- pmde = *pmd; +- barrier(); +- if (!pmd_present(pmde) || pmd_trans_huge(pmde)) +- pmd = NULL; + out: + return pmd; + } +diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h +index 6c1aa92a92e4..6ce1f1ceb432 100644 +--- a/tools/include/uapi/asm-generic/mman-common.h ++++ b/tools/include/uapi/asm-generic/mman-common.h +@@ -77,6 +77,8 @@ + + #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + ++#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ ++ + /* compatibility flags */ + #define MAP_FILE 0 + +diff --git a/tools/testing/selftests/vm/khugepaged.c b/tools/testing/selftests/vm/khugepaged.c +index 155120b67a16..b77b1e28cdb3 100644 +--- a/tools/testing/selftests/vm/khugepaged.c ++++ b/tools/testing/selftests/vm/khugepaged.c +@@ -14,6 +14,9 @@ + #ifndef MADV_PAGEOUT + #define MADV_PAGEOUT 21 + #endif ++#ifndef MADV_COLLAPSE ++#define MADV_COLLAPSE 25 ++#endif + + #define BASE_ADDR ((void *)(1UL << 30)) + static unsigned long hpage_pmd_size; +@@ -23,6 +26,11 @@ static int hpage_pmd_nr; + #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" + #define PID_SMAPS "/proc/self/smaps" + ++struct collapse_context { ++ void (*collapse)(const char *msg, char *p, int nr_hpages, bool expect); ++ bool enforce_pte_scan_limits; ++}; + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem); -+ EXPECT_EQ(0, ret); + enum thp_enabled { + THP_ALWAYS, + THP_MADVISE, +@@ -90,18 +98,6 @@ struct settings { + struct khugepaged_settings khugepaged; + }; + +-static struct settings default_settings = { +- .thp_enabled = THP_MADVISE, +- .thp_defrag = THP_DEFRAG_ALWAYS, +- .shmem_enabled = SHMEM_NEVER, +- .use_zero_page = 0, +- .khugepaged = { +- .defrag = 1, +- .alloc_sleep_millisecs = 10, +- .scan_sleep_millisecs = 10, +- }, +-}; +- + static struct settings saved_settings; + static bool skip_settings_restore; + +@@ -279,6 +275,39 @@ static void write_settings(struct settings *settings) + write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); + } + ++#define MAX_SETTINGS_DEPTH 4 ++static struct settings settings_stack[MAX_SETTINGS_DEPTH]; ++static int settings_index; + -+ close(fd); ++static struct settings *current_settings(void) ++{ ++ if (!settings_index) { ++ printf("Fail: No settings set"); ++ exit(EXIT_FAILURE); ++ } ++ return settings_stack + settings_index - 1; +} + -+TEST(mutex_state) ++static void push_settings(struct settings *settings) +{ -+ struct winesync_mutex_args mutex_args; -+ __u32 mutex, owner, count, index; -+ struct timespec timeout; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 0; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 2; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ mutex = mutex_args.mutex; -+ check_mutex_state(fd, mutex, 2, 123); -+ -+ ret = put_mutex(fd, mutex, 0, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = put_mutex(fd, mutex, 456, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ check_mutex_state(fd, mutex, 2, 123); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ check_mutex_state(fd, mutex, 0, 0); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 2, 456); -+ -+ ret = put_mutex(fd, mutex, 456, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 0; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ owner = 456; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ mutex = mutex_args.mutex; -+ check_mutex_state(fd, mutex, 0, 0); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); ++ if (settings_index >= MAX_SETTINGS_DEPTH) { ++ printf("Fail: Settings stack exceeded"); ++ exit(EXIT_FAILURE); ++ } ++ settings_stack[settings_index++] = *settings; ++ write_settings(current_settings()); +} + -+TEST(manual_event_state) ++static void pop_settings(void) +{ -+ struct winesync_event_args event_args; -+ __u32 index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 1; -+ event_args.signaled = 0; -+ event_args.event = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, event_args.event); -+ check_event_state(fd, event_args.event, 0, 1); ++ if (settings_index <= 0) { ++ printf("Fail: Settings stack empty"); ++ exit(EXIT_FAILURE); ++ } ++ --settings_index; ++ write_settings(current_settings()); ++} + -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); + static void restore_settings(int sig) + { + if (skip_settings_restore) +@@ -322,14 +351,6 @@ static void save_settings(void) + signal(SIGQUIT, restore_settings); + } + +-static void adjust_settings(void) +-{ +- +- printf("Adjust settings..."); +- write_settings(&default_settings); +- success("OK"); +-} +- + #define MAX_LINE_LENGTH 500 + + static bool check_for_pattern(FILE *fp, char *pattern, char *buf) +@@ -341,7 +362,7 @@ static bool check_for_pattern(FILE *fp, char *pattern, char *buf) + return false; + } + +-static bool check_huge(void *addr) ++static bool check_huge(void *addr, int nr_hpages) + { + bool thp = false; + int ret; +@@ -366,7 +387,7 @@ static bool check_huge(void *addr) + goto err_out; + + ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB", +- hpage_pmd_size >> 10); ++ nr_hpages * (hpage_pmd_size >> 10)); + if (ret >= MAX_LINE_LENGTH) { + printf("%s: Pattern is too long\n", __func__); + exit(EXIT_FAILURE); +@@ -434,12 +455,12 @@ static bool check_swap(void *addr, unsigned long size) + return swap; + } + +-static void *alloc_mapping(void) ++static void *alloc_mapping(int nr) + { + void *p; + +- p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, +- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); ++ p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, ++ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (p != BASE_ADDR) { + printf("Failed to allocate VMA at %p\n", BASE_ADDR); + exit(EXIT_FAILURE); +@@ -456,6 +477,25 @@ static void fill_memory(int *p, unsigned long start, unsigned long end) + p[i * page_size / sizeof(*p)] = i + 0xdead0000; + } + ++/* ++ * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with ++ * validate_memory()'able contents. ++ */ ++static void *alloc_hpage(void) ++{ ++ void *p; + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); ++ p = alloc_mapping(1); ++ printf("Allocate huge page..."); ++ madvise(p, hpage_pmd_size, MADV_HUGEPAGE); ++ fill_memory(p, 0, hpage_pmd_size); ++ if (check_huge(p, 1)) ++ success("OK"); ++ else ++ fail("Fail"); ++ return p; ++} + -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(fd, event_args.event, 1, 1); + static void validate_memory(int *p, unsigned long start, unsigned long end) + { + int i; +@@ -469,26 +509,59 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) + } + } + ++static void madvise_collapse(const char *msg, char *p, int nr_hpages, ++ bool expect) ++{ ++ int ret; ++ struct settings settings = *current_settings(); + -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); ++ printf("%s...", msg); ++ /* Sanity check */ ++ if (!check_huge(p, 0)) { ++ printf("Unexpected huge page\n"); ++ exit(EXIT_FAILURE); ++ } + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); ++ /* ++ * Prevent khugepaged interference and tests that MADV_COLLAPSE ++ * ignores /sys/kernel/mm/transparent_hugepage/enabled ++ */ ++ settings.thp_enabled = THP_NEVER; ++ push_settings(&settings); ++ ++ /* Clear VM_NOHUGEPAGE */ ++ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); ++ ret = madvise(p, nr_hpages * hpage_pmd_size, MADV_COLLAPSE); ++ if (((bool)ret) == expect) ++ fail("Fail: Bad return value"); ++ else if (check_huge(p, nr_hpages) != expect) ++ fail("Fail: check_huge()"); ++ else ++ success("OK"); + -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); ++ pop_settings(); ++} + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); + #define TICK 500000 +-static bool wait_for_scan(const char *msg, char *p) ++static bool wait_for_scan(const char *msg, char *p, int nr_hpages) + { + int full_scans; + int timeout = 6; /* 3 seconds */ + + /* Sanity check */ +- if (check_huge(p)) { ++ if (!check_huge(p, 0)) { + printf("Unexpected huge page\n"); + exit(EXIT_FAILURE); + } + +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); ++ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); + + /* Wait until the second full_scan completed */ + full_scans = read_num("khugepaged/full_scans") + 2; + + printf("%s...", msg); + while (timeout--) { +- if (check_huge(p)) ++ if (check_huge(p, nr_hpages)) + break; + if (read_num("khugepaged/full_scans") >= full_scans) + break; +@@ -496,121 +569,121 @@ static bool wait_for_scan(const char *msg, char *p) + usleep(TICK); + } + +- madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); ++ madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); + + return timeout == -1; + } + ++static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, ++ bool expect) ++{ ++ if (wait_for_scan(msg, p, nr_hpages)) { ++ if (expect) ++ fail("Timeout"); ++ else ++ success("OK"); ++ return; ++ } else if (check_huge(p, nr_hpages) == expect) { ++ success("OK"); ++ } else { ++ fail("Fail"); ++ } ++} + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); + static void alloc_at_fault(void) + { +- struct settings settings = default_settings; ++ struct settings settings = *current_settings(); + char *p; + + settings.thp_enabled = THP_ALWAYS; +- write_settings(&settings); ++ push_settings(&settings); + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + *p = 1; + printf("Allocate huge page on fault..."); +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); + +- write_settings(&default_settings); ++ pop_settings(); + + madvise(p, page_size, MADV_DONTNEED); + printf("Split huge PMD on MADV_DONTNEED..."); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + munmap(p, hpage_pmd_size); + } + +-static void collapse_full(void) ++static void collapse_full(struct collapse_context *c) + { + void *p; +- +- p = alloc_mapping(); +- fill_memory(p, 0, hpage_pmd_size); +- if (wait_for_scan("Collapse fully populated PTE table", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- validate_memory(p, 0, hpage_pmd_size); +- munmap(p, hpage_pmd_size); ++ int nr_hpages = 4; ++ unsigned long size = nr_hpages * hpage_pmd_size; ++ ++ p = alloc_mapping(nr_hpages); ++ fill_memory(p, 0, size); ++ c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, ++ true); ++ validate_memory(p, 0, size); ++ munmap(p, size); + } + +-static void collapse_empty(void) ++static void collapse_empty(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); +- if (wait_for_scan("Do not collapse empty PTE table", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- fail("Fail"); +- else +- success("OK"); ++ p = alloc_mapping(1); ++ c->collapse("Do not collapse empty PTE table", p, 1, false); + munmap(p, hpage_pmd_size); + } + +-static void collapse_single_pte_entry(void) ++static void collapse_single_pte_entry(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + fill_memory(p, 0, page_size); +- if (wait_for_scan("Collapse PTE table with single PTE entry present", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table with single PTE entry present", p, ++ 1, true); + validate_memory(p, 0, page_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_max_ptes_none(void) ++static void collapse_max_ptes_none(struct collapse_context *c) + { + int max_ptes_none = hpage_pmd_nr / 2; +- struct settings settings = default_settings; ++ struct settings settings = *current_settings(); + void *p; + + settings.khugepaged.max_ptes_none = max_ptes_none; +- write_settings(&settings); ++ push_settings(&settings); + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + + fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); +- if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- fail("Fail"); +- else +- success("OK"); ++ c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, ++ !c->enforce_pte_scan_limits); + validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); + +- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); +- if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); ++ if (c->enforce_pte_scan_limits) { ++ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); ++ c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ++ true); ++ validate_memory(p, 0, ++ (hpage_pmd_nr - max_ptes_none) * page_size); ++ } + + munmap(p, hpage_pmd_size); +- write_settings(&default_settings); ++ pop_settings(); + } + +-static void collapse_swapin_single_pte(void) ++static void collapse_swapin_single_pte(struct collapse_context *c) + { + void *p; +- p = alloc_mapping(); ++ p = alloc_mapping(1); + fill_memory(p, 0, hpage_pmd_size); + + printf("Swapout one page..."); +@@ -625,23 +698,18 @@ static void collapse_swapin_single_pte(void) + goto out; + } + +- if (wait_for_scan("Collapse with swapping in single PTE entry", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse with swapping in single PTE entry", p, 1, true); + validate_memory(p, 0, hpage_pmd_size); + out: + munmap(p, hpage_pmd_size); + } + +-static void collapse_max_ptes_swap(void) ++static void collapse_max_ptes_swap(struct collapse_context *c) + { + int max_ptes_swap = read_num("khugepaged/max_ptes_swap"); + void *p; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + + fill_memory(p, 0, hpage_pmd_size); + printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); +@@ -656,115 +724,83 @@ static void collapse_max_ptes_swap(void) + goto out; + } + +- if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- fail("Fail"); +- else +- success("OK"); ++ c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ++ !c->enforce_pte_scan_limits); + validate_memory(p, 0, hpage_pmd_size); + +- fill_memory(p, 0, hpage_pmd_size); +- printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); +- if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { +- perror("madvise(MADV_PAGEOUT)"); +- exit(EXIT_FAILURE); +- } +- if (check_swap(p, max_ptes_swap * page_size)) { +- success("OK"); +- } else { +- fail("Fail"); +- goto out; +- } ++ if (c->enforce_pte_scan_limits) { ++ fill_memory(p, 0, hpage_pmd_size); ++ printf("Swapout %d of %d pages...", max_ptes_swap, ++ hpage_pmd_nr); ++ if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { ++ perror("madvise(MADV_PAGEOUT)"); ++ exit(EXIT_FAILURE); ++ } ++ if (check_swap(p, max_ptes_swap * page_size)) { ++ success("OK"); ++ } else { ++ fail("Fail"); ++ goto out; ++ } + +- if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- validate_memory(p, 0, hpage_pmd_size); ++ c->collapse("Collapse with max_ptes_swap pages swapped out", p, ++ 1, true); ++ validate_memory(p, 0, hpage_pmd_size); ++ } + out: + munmap(p, hpage_pmd_size); + } + +-static void collapse_single_pte_entry_compound(void) ++static void collapse_single_pte_entry_compound(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ p = alloc_hpage(); + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); +- + printf("Split huge page leaving single PTE mapping compound page..."); + madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table with single PTE mapping compound page", ++ p, 1, true); + validate_memory(p, 0, page_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_full_of_compound(void) ++static void collapse_full_of_compound(struct collapse_context *c) + { + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- ++ p = alloc_hpage(); + printf("Split huge page leaving single PTE page table full of compound pages..."); + madvise(p, page_size, MADV_NOHUGEPAGE); + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Collapse PTE table full of compound pages", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table full of compound pages", p, 1, true); + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_compound_extreme(void) ++static void collapse_compound_extreme(struct collapse_context *c) + { + void *p; + int i; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + for (i = 0; i < hpage_pmd_nr; i++) { + printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", + i + 1, hpage_pmd_nr); + + madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); + fill_memory(BASE_ADDR, 0, hpage_pmd_size); +- if (!check_huge(BASE_ADDR)) { ++ if (!check_huge(BASE_ADDR, 1)) { + printf("Failed to allocate huge page\n"); + exit(EXIT_FAILURE); + } +@@ -793,32 +829,28 @@ static void collapse_compound_extreme(void) + + munmap(BASE_ADDR, hpage_pmd_size); + fill_memory(p, 0, hpage_pmd_size); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Collapse PTE table full of different compound pages", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table full of different compound pages", p, 1, ++ true); + + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); + } + +-static void collapse_fork(void) ++static void collapse_fork(struct collapse_context *c) + { + int wstatus; + void *p; + +- p = alloc_mapping(); ++ p = alloc_mapping(1); + + printf("Allocate small page..."); + fill_memory(p, 0, page_size); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); +@@ -829,19 +861,14 @@ static void collapse_fork(void) + skip_settings_restore = true; + exit_status = 0; + +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + + fill_memory(p, page_size, 2 * page_size); +- +- if (wait_for_scan("Collapse PTE table with single page shared with parent process", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table with single page shared with parent process", ++ p, 1, true); + + validate_memory(p, 0, page_size); + munmap(p, hpage_pmd_size); +@@ -852,7 +879,7 @@ static void collapse_fork(void) + exit_status += WEXITSTATUS(wstatus); + + printf("Check if parent still has small page..."); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); +@@ -860,28 +887,19 @@ static void collapse_fork(void) + munmap(p, hpage_pmd_size); + } + +-static void collapse_fork_compound(void) ++static void collapse_fork_compound(struct collapse_context *c) + { + int wstatus; + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- ++ p = alloc_hpage(); + printf("Share huge page over fork()..."); + if (!fork()) { + /* Do not touch settings on child exit */ + skip_settings_restore = true; + exit_status = 0; + +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -889,21 +907,17 @@ static void collapse_fork_compound(void) + printf("Split huge page PMD in child process..."); + madvise(p, page_size, MADV_NOHUGEPAGE); + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + fill_memory(p, 0, page_size); + + write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); +- if (wait_for_scan("Collapse PTE table full of compound pages in child", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Collapse PTE table full of compound pages in child", ++ p, 1, true); + write_num("khugepaged/max_ptes_shared", +- default_settings.khugepaged.max_ptes_shared); ++ current_settings()->khugepaged.max_ptes_shared); + + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); +@@ -914,7 +928,7 @@ static void collapse_fork_compound(void) + exit_status += WEXITSTATUS(wstatus); + + printf("Check if parent still has huge page..."); +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -922,29 +936,20 @@ static void collapse_fork_compound(void) + munmap(p, hpage_pmd_size); + } + +-static void collapse_max_ptes_shared() ++static void collapse_max_ptes_shared(struct collapse_context *c) + { + int max_ptes_shared = read_num("khugepaged/max_ptes_shared"); + int wstatus; + void *p; + +- p = alloc_mapping(); +- +- printf("Allocate huge page..."); +- madvise(p, hpage_pmd_size, MADV_HUGEPAGE); +- fill_memory(p, 0, hpage_pmd_size); +- if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- ++ p = alloc_hpage(); + printf("Share huge page over fork()..."); + if (!fork()) { + /* Do not touch settings on child exit */ + skip_settings_restore = true; + exit_status = 0; + +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -952,33 +957,27 @@ static void collapse_max_ptes_shared() + printf("Trigger CoW on page %d of %d...", + hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); + fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); +- if (!check_huge(p)) ++ if (check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + +- if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p)) +- fail("Timeout"); +- else if (!check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- +- printf("Trigger CoW on page %d of %d...", +- hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); +- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); +- if (!check_huge(p)) +- success("OK"); +- else +- fail("Fail"); +- +- +- if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p)) +- fail("Timeout"); +- else if (check_huge(p)) +- success("OK"); +- else +- fail("Fail"); ++ c->collapse("Maybe collapse with max_ptes_shared exceeded", p, ++ 1, !c->enforce_pte_scan_limits); ++ ++ if (c->enforce_pte_scan_limits) { ++ printf("Trigger CoW on page %d of %d...", ++ hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); ++ fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * ++ page_size); ++ if (check_huge(p, 0)) ++ success("OK"); ++ else ++ fail("Fail"); + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); ++ c->collapse("Collapse with max_ptes_shared PTEs shared", ++ p, 1, true); ++ } + + validate_memory(p, 0, hpage_pmd_size); + munmap(p, hpage_pmd_size); +@@ -989,7 +988,7 @@ static void collapse_max_ptes_shared() + exit_status += WEXITSTATUS(wstatus); + + printf("Check if parent still has huge page..."); +- if (check_huge(p)) ++ if (check_huge(p, 1)) + success("OK"); + else + fail("Fail"); +@@ -997,8 +996,52 @@ static void collapse_max_ptes_shared() + munmap(p, hpage_pmd_size); + } + +-int main(void) ++static void madvise_collapse_existing_thps(void) + { ++ void *p; ++ int err; + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++ p = alloc_mapping(1); ++ fill_memory(p, 0, hpage_pmd_size); + -+ close(fd); ++ printf("Collapse fully populated PTE table..."); ++ /* ++ * Note that we don't set MADV_HUGEPAGE here, which ++ * also tests that VM_HUGEPAGE isn't required for ++ * MADV_COLLAPSE in "madvise" mode. ++ */ ++ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); ++ if (err == 0 && check_huge(p, 1)) { ++ success("OK"); ++ printf("Re-collapse PMD-mapped hugepage"); ++ err = madvise(p, hpage_pmd_size, MADV_COLLAPSE); ++ if (err == 0 && check_huge(p, 1)) ++ success("OK"); ++ else ++ fail("Fail"); ++ } else { ++ fail("Fail"); ++ } ++ validate_memory(p, 0, hpage_pmd_size); ++ munmap(p, hpage_pmd_size); +} + -+TEST(auto_event_state) ++int main(int argc, const char **argv) +{ -+ struct winesync_event_args event_args; -+ __u32 index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 0; -+ event_args.signaled = 1; -+ event_args.event = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, event_args.event); -+ -+ check_event_state(fd, event_args.event, 1, 0); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 0); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); ++ struct collapse_context c; ++ struct settings default_settings = { ++ .thp_enabled = THP_MADVISE, ++ .thp_defrag = THP_DEFRAG_ALWAYS, ++ .shmem_enabled = SHMEM_NEVER, ++ .use_zero_page = 0, ++ .khugepaged = { ++ .defrag = 1, ++ .alloc_sleep_millisecs = 10, ++ .scan_sleep_millisecs = 10, ++ }, ++ }; ++ const char *tests = argc == 1 ? "all" : argv[1]; ++ + setbuf(stdout, NULL); + + page_size = getpagesize(); +@@ -1011,21 +1054,47 @@ int main(void) + default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8; + + save_settings(); +- adjust_settings(); ++ push_settings(&default_settings); + + alloc_at_fault(); +- collapse_full(); +- collapse_empty(); +- collapse_single_pte_entry(); +- collapse_max_ptes_none(); +- collapse_swapin_single_pte(); +- collapse_max_ptes_swap(); +- collapse_single_pte_entry_compound(); +- collapse_full_of_compound(); +- collapse_compound_extreme(); +- collapse_fork(); +- collapse_fork_compound(); +- collapse_max_ptes_shared(); ++ ++ if (!strcmp(tests, "khugepaged") || !strcmp(tests, "all")) { ++ printf("\n*** Testing context: khugepaged ***\n"); ++ c.collapse = &khugepaged_collapse; ++ c.enforce_pte_scan_limits = true; ++ ++ collapse_full(&c); ++ collapse_empty(&c); ++ collapse_single_pte_entry(&c); ++ collapse_max_ptes_none(&c); ++ collapse_swapin_single_pte(&c); ++ collapse_max_ptes_swap(&c); ++ collapse_single_pte_entry_compound(&c); ++ collapse_full_of_compound(&c); ++ collapse_compound_extreme(&c); ++ collapse_fork(&c); ++ collapse_fork_compound(&c); ++ collapse_max_ptes_shared(&c); ++ } ++ if (!strcmp(tests, "madvise") || !strcmp(tests, "all")) { ++ printf("\n*** Testing context: madvise ***\n"); ++ c.collapse = &madvise_collapse; ++ c.enforce_pte_scan_limits = false; ++ ++ collapse_full(&c); ++ collapse_empty(&c); ++ collapse_single_pte_entry(&c); ++ collapse_max_ptes_none(&c); ++ collapse_swapin_single_pte(&c); ++ collapse_max_ptes_swap(&c); ++ collapse_single_pte_entry_compound(&c); ++ collapse_full_of_compound(&c); ++ collapse_compound_extreme(&c); ++ collapse_fork(&c); ++ collapse_fork_compound(&c); ++ collapse_max_ptes_shared(&c); ++ madvise_collapse_existing_thps(); ++ } + + restore_settings(0); + } +-- +2.38.0.rc1.8.g2a7d63a245 + +From 3430d4868012555c67c2ec34b073b0e4ecda986d Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:26:48 +0200 +Subject: [PATCH 07/16] mm: multi-gen LRU + +Signed-off-by: Peter Jung +--- + Documentation/admin-guide/mm/index.rst | 1 + + Documentation/admin-guide/mm/multigen_lru.rst | 162 + + Documentation/mm/index.rst | 1 + + Documentation/mm/multigen_lru.rst | 159 + + arch/Kconfig | 8 + + arch/arm64/include/asm/pgtable.h | 15 +- + arch/x86/Kconfig | 1 + + arch/x86/include/asm/pgtable.h | 9 +- + arch/x86/mm/pgtable.c | 5 +- + fs/exec.c | 2 + + fs/fuse/dev.c | 3 +- + include/linux/cgroup.h | 15 +- + include/linux/memcontrol.h | 36 + + include/linux/mm.h | 5 + + include/linux/mm_inline.h | 231 +- + include/linux/mm_types.h | 76 + + include/linux/mmzone.h | 216 ++ + include/linux/nodemask.h | 1 + + include/linux/page-flags-layout.h | 16 +- + include/linux/page-flags.h | 4 +- + include/linux/pgtable.h | 17 +- + include/linux/sched.h | 4 + + include/linux/swap.h | 4 + + kernel/bounds.c | 7 + + kernel/cgroup/cgroup-internal.h | 1 - + kernel/exit.c | 1 + + kernel/fork.c | 9 + + kernel/sched/core.c | 1 + + mm/Kconfig | 26 + + mm/huge_memory.c | 3 +- + mm/internal.h | 1 + + mm/memcontrol.c | 28 + + mm/memory.c | 39 +- + mm/mm_init.c | 6 +- + mm/mmzone.c | 2 + + mm/rmap.c | 6 + + mm/swap.c | 54 +- + mm/vmscan.c | 3253 +++++++++++++++-- + mm/workingset.c | 110 +- + 39 files changed, 4252 insertions(+), 286 deletions(-) + create mode 100644 Documentation/admin-guide/mm/multigen_lru.rst + create mode 100644 Documentation/mm/multigen_lru.rst + +diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst +index 1bd11118dfb1..d1064e0ba34a 100644 +--- a/Documentation/admin-guide/mm/index.rst ++++ b/Documentation/admin-guide/mm/index.rst +@@ -32,6 +32,7 @@ the Linux memory management. + idle_page_tracking + ksm + memory-hotplug ++ multigen_lru + nommu-mmap + numa_memory_policy + numaperf +diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst +new file mode 100644 +index 000000000000..33e068830497 +--- /dev/null ++++ b/Documentation/admin-guide/mm/multigen_lru.rst +@@ -0,0 +1,162 @@ ++.. SPDX-License-Identifier: GPL-2.0 + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); ++============= ++Multi-Gen LRU ++============= ++The multi-gen LRU is an alternative LRU implementation that optimizes ++page reclaim and improves performance under memory pressure. Page ++reclaim decides the kernel's caching policy and ability to overcommit ++memory. It directly impacts the kswapd CPU usage and RAM efficiency. + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++Quick start ++=========== ++Build the kernel with the following configurations. + -+ close(fd); -+} ++* ``CONFIG_LRU_GEN=y`` ++* ``CONFIG_LRU_GEN_ENABLED=y`` + -+TEST(test_wait_any) -+{ -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2], owner, index; -+ struct timespec timeout; -+ int fd, ret; ++All set! + -+ clock_gettime(CLOCK_MONOTONIC, &timeout); ++Runtime options ++=============== ++``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the ++following subsections. + -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++Kill switch ++----------- ++``enabled`` accepts different values to enable or disable the ++following components. Its default value depends on ++``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled ++unless some of them have unforeseen side effects. Writing to ++``enabled`` has no effect when a component is not supported by the ++hardware, and valid values will be accepted even when the main switch ++is off. + -+ sem_args.count = 2; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); ++====== =============================================================== ++Values Components ++====== =============================================================== ++0x0001 The main switch for the multi-gen LRU. ++0x0002 Clearing the accessed bit in leaf page table entries in large ++ batches, when MMU sets it (e.g., on x86). This behavior can ++ theoretically worsen lock contention (mmap_lock). If it is ++ disabled, the multi-gen LRU will suffer a minor performance ++ degradation for workloads that contiguously map hot pages, ++ whose accessed bits can be otherwise cleared by fewer larger ++ batches. ++0x0004 Clearing the accessed bit in non-leaf page table entries as ++ well, when MMU sets it (e.g., on x86). This behavior was not ++ verified on x86 varieties other than Intel and AMD. If it is ++ disabled, the multi-gen LRU will suffer a negligible ++ performance degradation. ++[yYnN] Apply to all the components above. ++====== =============================================================== + -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++E.g., ++:: + -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; ++ echo y >/sys/kernel/mm/lru_gen/enabled ++ cat /sys/kernel/mm/lru_gen/enabled ++ 0x0007 ++ echo 5 >/sys/kernel/mm/lru_gen/enabled ++ cat /sys/kernel/mm/lru_gen/enabled ++ 0x0005 + -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++Thrashing prevention ++-------------------- ++Personal computers are more sensitive to thrashing because it can ++cause janks (lags when rendering UI) and negatively impact user ++experience. The multi-gen LRU offers thrashing prevention to the ++majority of laptop and desktop users who do not have ``oomd``. + -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of ++``N`` milliseconds from getting evicted. The OOM killer is triggered ++if this working set cannot be kept in memory. In other words, this ++option works as an adjustable pressure relief valve, and when open, it ++terminates applications that are hopefully not being used. + -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++Based on the average human detectable lag (~100ms), ``N=1000`` usually ++eliminates intolerable janks due to thrashing. Larger values like ++``N=3000`` make janks less noticeable at the risk of premature OOM ++kills. + -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); ++The default value ``0`` means disabled. + -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++Experimental features ++===================== ++``/sys/kernel/debug/lru_gen`` accepts commands described in the ++following subsections. Multiple command lines are supported, so does ++concatenation with delimiters ``,`` and ``;``. + -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++``/sys/kernel/debug/lru_gen_full`` provides additional stats for ++debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from ++evicted generations in this file. + -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); ++Working set estimation ++---------------------- ++Working set estimation measures how much memory an application needs ++in a given time interval, and it is usually done with little impact on ++the performance of the application. E.g., data centers want to ++optimize job scheduling (bin packing) to improve memory utilizations. ++When a new job comes in, the job scheduler needs to find out whether ++each server it manages can allocate a certain amount of memory for ++this new job before it can pick a candidate. To do so, the job ++scheduler needs to estimate the working sets of the existing jobs. + -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); ++When it is read, ``lru_gen`` returns a histogram of numbers of pages ++accessed over different time intervals for each memcg and node. ++``MAX_NR_GENS`` decides the number of bins for each histogram. The ++histograms are noncumulative. ++:: + -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(1, index); ++ memcg memcg_id memcg_path ++ node node_id ++ min_gen_nr age_in_ms nr_anon_pages nr_file_pages ++ ... ++ max_gen_nr age_in_ms nr_anon_pages nr_file_pages + -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); ++Each bin contains an estimated number of pages that have been accessed ++within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages ++and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of ++the former is the largest and that of the latter is the smallest. + -+ /* test waiting on the same object twice */ -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); ++Users can write the following command to ``lru_gen`` to create a new ++generation ``max_gen_nr+1``: + -+ objs[0] = objs[1] = sem_args.sem; -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, wait_args.index); -+ check_sem_state(fd, sem_args.sem, 1, 3); ++ ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` + -+ ret = wait_any(fd, 0, NULL, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); ++``can_swap`` defaults to the swap setting and, if it is set to ``1``, ++it forces the scan of anon pages when swap is off, and vice versa. ++``force_scan`` defaults to ``1`` and, if it is set to ``0``, it ++employs heuristics to reduce the overhead, which is likely to reduce ++the coverage as well. + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); ++A typical use case is that a job scheduler runs this command at a ++certain time interval to create new generations, and it ranks the ++servers it manages based on the sizes of their cold pages defined by ++this time interval. + -+ close(fd); -+} ++Proactive reclaim ++----------------- ++Proactive reclaim induces page reclaim when there is no memory ++pressure. It usually targets cold pages only. E.g., when a new job ++comes in, the job scheduler wants to proactively reclaim cold pages on ++the server it selected, to improve the chance of successfully landing ++this new job. + -+TEST(test_wait_all) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2], owner, index; -+ int fd, ret; ++Users can write the following command to ``lru_gen`` to evict ++generations less than or equal to ``min_gen_nr``. + -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++ ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` + -+ sem_args.count = 2; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); ++``min_gen_nr`` should be less than ``max_gen_nr-1``, since ++``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to ++the active list) and therefore cannot be evicted. ``swappiness`` ++overrides the default value in ``/proc/sys/vm/swappiness``. ++``nr_to_reclaim`` limits the number of pages to evict. + -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++A typical use case is that a job scheduler runs this command before it ++tries to land a new job on a server. If it fails to materialize enough ++cold pages because of the overestimation, it retries on the next ++server according to the ranking result obtained from the working set ++estimation step. This less forceful approach limits the impacts on the ++existing jobs. +diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst +index 575ccd40e30c..4aa12b8be278 100644 +--- a/Documentation/mm/index.rst ++++ b/Documentation/mm/index.rst +@@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. + ksm + memory-model + mmu_notifier ++ multigen_lru + numa + overcommit-accounting + page_migration +diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst +new file mode 100644 +index 000000000000..d7062c6a8946 +--- /dev/null ++++ b/Documentation/mm/multigen_lru.rst +@@ -0,0 +1,159 @@ ++.. SPDX-License-Identifier: GPL-2.0 + -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++============= ++Multi-Gen LRU ++============= ++The multi-gen LRU is an alternative LRU implementation that optimizes ++page reclaim and improves performance under memory pressure. Page ++reclaim decides the kernel's caching policy and ability to overcommit ++memory. It directly impacts the kswapd CPU usage and RAM efficiency. + -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; ++Design overview ++=============== ++Objectives ++---------- ++The design objectives are: + -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++* Good representation of access recency ++* Try to profit from spatial locality ++* Fast paths to make obvious choices ++* Simple self-correcting heuristics + -+ ret = wait_all(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++The representation of access recency is at the core of all LRU ++implementations. In the multi-gen LRU, each generation represents a ++group of pages with similar access recency. Generations establish a ++(time-based) common frame of reference and therefore help make better ++choices, e.g., between different memcgs on a computer or different ++computers in a data center (for job scheduling). + -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++Exploiting spatial locality improves efficiency when gathering the ++accessed bit. A rmap walk targets a single page and does not try to ++profit from discovering a young PTE. A page table walk can sweep all ++the young PTEs in an address space, but the address space can be too ++sparse to make a profit. The key is to optimize both methods and use ++them in combination. + -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); ++Fast paths reduce code complexity and runtime overhead. Unmapped pages ++do not require TLB flushes; clean pages do not require writeback. ++These facts are only helpful when other conditions, e.g., access ++recency, are similar. With generations as a common frame of reference, ++additional factors stand out. But obvious choices might not be good ++choices; thus self-correction is necessary. + -+ sem_args.count = 3; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); ++The benefits of simple self-correcting heuristics are self-evident. ++Again, with generations as a common frame of reference, this becomes ++attainable. Specifically, pages in the same generation can be ++categorized based on additional factors, and a feedback loop can ++statistically compare the refault percentages across those categories ++and infer which of them are better choices. + -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 2, 3); -+ check_mutex_state(fd, mutex_args.mutex, 3, 123); ++Assumptions ++----------- ++The protection of hot pages and the selection of cold pages are based ++on page access channels and patterns. There are two access channels: + -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); ++* Accesses through page tables ++* Accesses through file descriptors + -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); ++The protection of the former channel is by design stronger because: + -+ objs[0] = sem_args.sem; -+ objs[1] = event_args.event; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_event_state(fd, event_args.event, 1, 1); ++1. The uncertainty in determining the access patterns of the former ++ channel is higher due to the approximation of the accessed bit. ++2. The cost of evicting the former channel is higher due to the TLB ++ flushes required and the likelihood of encountering the dirty bit. ++3. The penalty of underprotecting the former channel is higher because ++ applications usually do not prepare themselves for major page ++ faults like they do for blocked I/O. E.g., GUI applications ++ commonly use dedicated I/O threads to avoid blocking rendering ++ threads. + -+ /* test waiting on the same object twice */ -+ objs[0] = objs[1] = sem_args.sem; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++There are also two access patterns: + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++* Accesses exhibiting temporal locality ++* Accesses not exhibiting temporal locality + -+ close(fd); -+} ++For the reasons listed above, the former channel is assumed to follow ++the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is ++present, and the latter channel is assumed to follow the latter ++pattern unless outlying refaults have been observed. + -+TEST(invalid_objects) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2] = {0}; -+ int fd, ret; ++Workflow overview ++================= ++Evictable pages are divided into multiple generations for each ++``lruvec``. The youngest generation number is stored in ++``lrugen->max_seq`` for both anon and file types as they are aged on ++an equal footing. The oldest generation numbers are stored in ++``lrugen->min_seq[]`` separately for anon and file types as clean file ++pages can be evicted regardless of swap constraints. These three ++variables are monotonically increasing. + -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` ++bits in order to fit into the gen counter in ``folio->flags``. Each ++truncated generation number is an index to ``lrugen->lists[]``. The ++sliding window technique is used to track at least ``MIN_NR_GENS`` and ++at most ``MAX_NR_GENS`` generations. The gen counter stores a value ++within ``[1, MAX_NR_GENS]`` while a page is on one of ++``lrugen->lists[]``; otherwise it stores zero. + -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++Each generation is divided into multiple tiers. A page accessed ``N`` ++times through file descriptors is in tier ``order_base_2(N)``. Unlike ++generations, tiers do not have dedicated ``lrugen->lists[]``. In ++contrast to moving across generations, which requires the LRU lock, ++moving across tiers only involves atomic operations on ++``folio->flags`` and therefore has a negligible cost. A feedback loop ++modeled after the PID controller monitors refaults over all the tiers ++from anon and file types and decides which tiers from which types to ++evict or protect. + -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++There are two conceptually independent procedures: the aging and the ++eviction. They form a closed-loop system, i.e., the page reclaim. + -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++Aging ++----- ++The aging produces young generations. Given an ``lruvec``, it ++increments ``max_seq`` when ``max_seq-min_seq+1`` approaches ++``MIN_NR_GENS``. The aging promotes hot pages to the youngest ++generation when it finds them accessed through page tables; the ++demotion of cold pages happens consequently when it increments ++``max_seq``. The aging uses page table walks and rmap walks to find ++young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` ++and calls ``walk_page_range()`` with each ``mm_struct`` on this list ++to scan PTEs, and after each iteration, it increments ``max_seq``. For ++the latter, when the eviction walks the rmap and finds a young PTE, ++the aging scans the adjacent PTEs. For both, on finding a young PTE, ++the aging clears the accessed bit and updates the gen counter of the ++page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. + -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++Eviction ++-------- ++The eviction consumes old generations. Given an ``lruvec``, it ++increments ``min_seq`` when ``lrugen->lists[]`` indexed by ++``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to ++evict from, it first compares ``min_seq[]`` to select the older type. ++If both types are equally old, it selects the one whose first tier has ++a lower refault percentage. The first tier contains single-use ++unmapped clean pages, which are the best bet. The eviction sorts a ++page according to its gen counter if the aging has found this page ++accessed through page tables and updated its gen counter. It also ++moves a page to the next generation, i.e., ``min_seq+1``, if this page ++was accessed multiple times through file descriptors and the feedback ++loop has detected outlying refaults from the tier this page is in. To ++this end, the feedback loop uses the first tier as the baseline, for ++the reason stated earlier. + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++Summary ++------- ++The multi-gen LRU can be disassembled into the following parts: + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++* Generations ++* Rmap walks ++* Page table walks ++* Bloom filters ++* PID controller + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++The aging and the eviction form a producer-consumer model; ++specifically, the latter drives the former by the sliding window over ++generations. Within the aging, rmap walks drive page table walks by ++inserting hot densely populated page tables to the Bloom filters. ++Within the eviction, the PID controller uses refaults as the feedback ++to select types to evict and tiers to protect. +diff --git a/arch/Kconfig b/arch/Kconfig +index 8b311e400ec1..bf19a84fffa2 100644 +--- a/arch/Kconfig ++++ b/arch/Kconfig +@@ -1418,6 +1418,14 @@ config DYNAMIC_SIGFRAME + config HAVE_ARCH_NODE_DEV_GROUP + bool + ++config ARCH_HAS_NONLEAF_PMD_YOUNG ++ bool ++ help ++ Architectures that select this option are capable of setting the ++ accessed bit in non-leaf PMD entries when using them as part of linear ++ address translations. Page table walkers that clear the accessed bit ++ may use this capability to reduce their search space. + -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); + source "kernel/gcov/Kconfig" + + source "scripts/gcc-plugins/Kconfig" +diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h +index b5df82aa99e6..71a1af42f0e8 100644 +--- a/arch/arm64/include/asm/pgtable.h ++++ b/arch/arm64/include/asm/pgtable.h +@@ -1082,24 +1082,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, + * page after fork() + CoW for pfn mappings. We don't always have a + * hardware-managed access flag on arm64. + */ +-static inline bool arch_faults_on_old_pte(void) +-{ +- /* The register read below requires a stable CPU to make any sense */ +- cant_migrate(); +- +- return !cpu_has_hw_af(); +-} +-#define arch_faults_on_old_pte arch_faults_on_old_pte ++#define arch_has_hw_pte_young cpu_has_hw_af + + /* + * Experimentally, it's cheap to set the access flag in hardware and we + * benefit from prefaulting mappings as 'old' to start with. + */ +-static inline bool arch_wants_old_prefaulted_pte(void) +-{ +- return !arch_faults_on_old_pte(); +-} +-#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte ++#define arch_wants_old_prefaulted_pte cpu_has_hw_af + + static inline bool pud_sect_supported(void) + { +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index f9920f1341c8..674d694a665e 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -85,6 +85,7 @@ config X86 + select ARCH_HAS_PMEM_API if X86_64 + select ARCH_HAS_PTE_DEVMAP if X86_64 + select ARCH_HAS_PTE_SPECIAL ++ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 + select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 + select ARCH_HAS_COPY_MC if X86_64 + select ARCH_HAS_SET_MEMORY +diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h +index 44e2d6f1dbaa..5059799bebe3 100644 +--- a/arch/x86/include/asm/pgtable.h ++++ b/arch/x86/include/asm/pgtable.h +@@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) + + static inline int pmd_bad(pmd_t pmd) + { +- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; ++ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != ++ (_KERNPG_TABLE & ~_PAGE_ACCESSED); + } + + static inline unsigned long pages_to_mb(unsigned long npg) +@@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) + return boot_cpu_has_bug(X86_BUG_L1TF); + } + +-#define arch_faults_on_old_pte arch_faults_on_old_pte +-static inline bool arch_faults_on_old_pte(void) ++#define arch_has_hw_pte_young arch_has_hw_pte_young ++static inline bool arch_has_hw_pte_young(void) + { +- return false; ++ return true; + } + + #ifdef CONFIG_PAGE_TABLE_CHECK +diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c +index a932d7712d85..8525f2876fb4 100644 +--- a/arch/x86/mm/pgtable.c ++++ b/arch/x86/mm/pgtable.c +@@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, + return ret; + } + +-#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) + int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) + { +@@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, + + return ret; + } ++#endif + -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE + int pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pud_t *pudp) + { +diff --git a/fs/exec.c b/fs/exec.c +index d046dbb9cbd0..c67b12f0f577 100644 +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -1011,6 +1011,7 @@ static int exec_mmap(struct mm_struct *mm) + active_mm = tsk->active_mm; + tsk->active_mm = mm; + tsk->mm = mm; ++ lru_gen_add_mm(mm); + /* + * This prevents preemption while active_mm is being loaded and + * it and mm are being updated, which could cause problems for +@@ -1026,6 +1027,7 @@ static int exec_mmap(struct mm_struct *mm) + tsk->mm->vmacache_seqnum = 0; + vmacache_flush(tsk); + task_unlock(tsk); ++ lru_gen_use_mm(mm); + if (old_mm) { + mmap_read_unlock(old_mm); + BUG_ON(active_mm != old_mm); +diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c +index 51897427a534..b4a6e0a1b945 100644 +--- a/fs/fuse/dev.c ++++ b/fs/fuse/dev.c +@@ -776,7 +776,8 @@ static int fuse_check_page(struct page *page) + 1 << PG_active | + 1 << PG_workingset | + 1 << PG_reclaim | +- 1 << PG_waiters))) { ++ 1 << PG_waiters | ++ LRU_GEN_MASK | LRU_REFS_MASK))) { + dump_page(page, "fuse: trying to steal weird page"); + return 1; + } +diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h +index ac5d0515680e..9179463c3c9f 100644 +--- a/include/linux/cgroup.h ++++ b/include/linux/cgroup.h +@@ -432,6 +432,18 @@ static inline void cgroup_put(struct cgroup *cgrp) + css_put(&cgrp->self); + } + ++extern struct mutex cgroup_mutex; + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++static inline void cgroup_lock(void) ++{ ++ mutex_lock(&cgroup_mutex); ++} + -+ sem_args.max = 1; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); ++static inline void cgroup_unlock(void) ++{ ++ mutex_unlock(&cgroup_mutex); ++} + -+ mutex_args.mutex = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); + /** + * task_css_set_check - obtain a task's css_set with extra access conditions + * @task: the task to obtain css_set for +@@ -446,7 +458,6 @@ static inline void cgroup_put(struct cgroup *cgrp) + * as locks used during the cgroup_subsys::attach() methods. + */ + #ifdef CONFIG_PROVE_RCU +-extern struct mutex cgroup_mutex; + extern spinlock_t css_set_lock; + #define task_css_set_check(task, __c) \ + rcu_dereference_check((task)->cgroups, \ +@@ -708,6 +719,8 @@ struct cgroup; + static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } + static inline void css_get(struct cgroup_subsys_state *css) {} + static inline void css_put(struct cgroup_subsys_state *css) {} ++static inline void cgroup_lock(void) {} ++static inline void cgroup_unlock(void) {} + static inline int cgroup_attach_task_all(struct task_struct *from, + struct task_struct *t) { return 0; } + static inline int cgroupstats_build(struct cgroupstats *stats, +diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h +index 6257867fbf95..207cfd3b42e5 100644 +--- a/include/linux/memcontrol.h ++++ b/include/linux/memcontrol.h +@@ -350,6 +350,11 @@ struct mem_cgroup { + struct deferred_split deferred_split_queue; + #endif + ++#ifdef CONFIG_LRU_GEN ++ /* per-memcg mm_struct list */ ++ struct lru_gen_mm_list mm_list; ++#endif + -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); + struct mem_cgroup_per_node *nodeinfo[]; + }; + +@@ -444,6 +449,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) + * - LRU isolation + * - lock_page_memcg() + * - exclusive reference ++ * - mem_cgroup_trylock_pages() + * + * For a kmem folio a caller should hold an rcu read lock to protect memcg + * associated with a kmem folio from being released. +@@ -505,6 +511,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) + * - LRU isolation + * - lock_page_memcg() + * - exclusive reference ++ * - mem_cgroup_trylock_pages() + * + * For a kmem page a caller should hold an rcu read lock to protect memcg + * associated with a kmem page from being released. +@@ -959,6 +966,23 @@ void unlock_page_memcg(struct page *page); + + void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); + ++/* try to stablize folio_memcg() for all the pages in a memcg */ ++static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) ++{ ++ rcu_read_lock(); + -+ event_args.event = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++ if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) ++ return true; + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++ rcu_read_unlock(); ++ return false; ++} + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++static inline void mem_cgroup_unlock_pages(void) ++{ ++ rcu_read_unlock(); ++} + -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); + /* idx can be of type enum memcg_stat_item or node_stat_item */ + static inline void mod_memcg_state(struct mem_cgroup *memcg, + int idx, int val) +@@ -1433,6 +1457,18 @@ static inline void folio_memcg_unlock(struct folio *folio) + { + } + ++static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) ++{ ++ /* to match folio_memcg_rcu() */ ++ rcu_read_lock(); ++ return true; ++} + -+ objs[0] = sem_args.sem; -+ objs[1] = sem_args.sem + 1; -+ wait_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++static inline void mem_cgroup_unlock_pages(void) ++{ ++ rcu_read_unlock(); ++} + -+ objs[0] = sem_args.sem + 1; -+ objs[1] = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); + static inline void mem_cgroup_handle_over_high(void) + { + } +diff --git a/include/linux/mm.h b/include/linux/mm.h +index 21f8b27bd9fd..88976a521ef5 100644 +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -1465,6 +1465,11 @@ static inline unsigned long folio_pfn(struct folio *folio) + return page_to_pfn(&folio->page); + } + ++static inline struct folio *pfn_folio(unsigned long pfn) ++{ ++ return page_folio(pfn_to_page(pfn)); ++} ++ + static inline atomic_t *folio_pincount_ptr(struct folio *folio) + { + return &folio_page(folio, 1)->compound_pincount; +diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h +index 7b25b53c474a..4949eda9a9a2 100644 +--- a/include/linux/mm_inline.h ++++ b/include/linux/mm_inline.h +@@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) + return folio_is_file_lru(page_folio(page)); + } + +-static __always_inline void update_lru_size(struct lruvec *lruvec, ++static __always_inline void __update_lru_size(struct lruvec *lruvec, + enum lru_list lru, enum zone_type zid, + long nr_pages) + { + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + ++ lockdep_assert_held(&lruvec->lru_lock); ++ WARN_ON_ONCE(nr_pages != (int)nr_pages); + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); + __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); + __mod_zone_page_state(&pgdat->node_zones[zid], + NR_ZONE_LRU_BASE + lru, nr_pages); ++} + -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); ++static __always_inline void update_lru_size(struct lruvec *lruvec, ++ enum lru_list lru, enum zone_type zid, ++ long nr_pages) ++{ ++ __update_lru_size(lruvec, lru, zid, nr_pages); + #ifdef CONFIG_MEMCG + mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); + #endif +@@ -94,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) + return lru; + } + ++#ifdef CONFIG_LRU_GEN + -+ sem_args.sem = mutex_args.mutex; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++#ifdef CONFIG_LRU_GEN_ENABLED ++static inline bool lru_gen_enabled(void) ++{ ++ DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); + -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); ++ return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); ++} ++#else ++static inline bool lru_gen_enabled(void) ++{ ++ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); ++ return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); ++} ++#endif + -+ close(fd); ++static inline bool lru_gen_in_fault(void) ++{ ++ return current->in_lru_fault; +} + -+struct wake_args ++static inline int lru_gen_from_seq(unsigned long seq) +{ -+ int fd; -+ __u32 obj; -+}; ++ return seq % MAX_NR_GENS; ++} + -+struct wait_args ++static inline int lru_hist_from_seq(unsigned long seq) +{ -+ int fd; -+ unsigned long request; -+ struct winesync_wait_args *args; -+ int ret; -+ int err; -+}; ++ return seq % NR_HIST_GENS; ++} + -+static void *wait_thread(void *arg) ++static inline int lru_tier_from_refs(int refs) +{ -+ struct wait_args *args = arg; ++ VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); + -+ args->ret = ioctl(args->fd, args->request, args->args); -+ args->err = errno; -+ return NULL; ++ /* see the comment in folio_lru_refs() */ ++ return order_base_2(refs + 1); +} + -+static void get_abs_timeout(struct timespec *timeout, clockid_t clock, -+ unsigned int ms) ++static inline int folio_lru_refs(struct folio *folio) +{ -+ clock_gettime(clock, timeout); -+ timeout->tv_nsec += ms * 1000000; -+ timeout->tv_sec += (timeout->tv_nsec / 1000000000); -+ timeout->tv_nsec %= 1000000000; ++ unsigned long flags = READ_ONCE(folio->flags); ++ bool workingset = flags & BIT(PG_workingset); ++ ++ /* ++ * Return the number of accesses beyond PG_referenced, i.e., N-1 if the ++ * total number of accesses is N>1, since N=0,1 both map to the first ++ * tier. lru_tier_from_refs() will account for this off-by-one. Also see ++ * the comment on MAX_NR_TIERS. ++ */ ++ return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; +} + -+static int wait_for_thread(pthread_t thread, unsigned int ms) ++static inline int folio_lru_gen(struct folio *folio) +{ -+ struct timespec timeout; -+ get_abs_timeout(&timeout, CLOCK_REALTIME, ms); -+ return pthread_timedjoin_np(thread, NULL, &timeout); ++ unsigned long flags = READ_ONCE(folio->flags); ++ ++ return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +} + -+TEST(wake_any) ++static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) +{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 objs[2], count, index; -+ struct timespec timeout; -+ pthread_t thread; -+ int fd, ret; ++ unsigned long max_seq = lruvec->lrugen.max_seq; + -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + -+ sem_args.count = 0; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); ++ /* see the comment on MIN_NR_GENS */ ++ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); ++} + -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, ++ int old_gen, int new_gen) ++{ ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ int delta = folio_nr_pages(folio); ++ enum lru_list lru = type * LRU_INACTIVE_FILE; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; ++ VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); + -+ /* test waking the semaphore */ ++ if (old_gen >= 0) ++ WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], ++ lrugen->nr_pages[old_gen][type][zone] - delta); ++ if (new_gen >= 0) ++ WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], ++ lrugen->nr_pages[new_gen][type][zone] + delta); + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 456; -+ wait_args.index = 0xdeadbeef; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++ /* addition */ ++ if (old_gen < 0) { ++ if (lru_gen_is_active(lruvec, new_gen)) ++ lru += LRU_ACTIVE; ++ __update_lru_size(lruvec, lru, zone, delta); ++ return; ++ } + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++ /* deletion */ ++ if (new_gen < 0) { ++ if (lru_gen_is_active(lruvec, old_gen)) ++ lru += LRU_ACTIVE; ++ __update_lru_size(lruvec, lru, zone, -delta); ++ return; ++ } + -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ check_sem_state(fd, sem_args.sem, 0, 3); ++ /* promotion */ ++ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { ++ __update_lru_size(lruvec, lru, zone, -delta); ++ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); ++ } + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(0, wait_args.index); ++ /* demotion requires isolation, e.g., lru_deactivate_fn() */ ++ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); ++} + -+ /* test waking the mutex */ ++static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ unsigned long seq; ++ unsigned long flags; ++ int gen = folio_lru_gen(folio); ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ /* first grab it again for owner 123 */ -+ ret = wait_any(fd, 1, &mutex_args.mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); ++ VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.owner = 456; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++ if (folio_test_unevictable(folio) || !lrugen->enabled) ++ return false; ++ /* ++ * There are three common cases for this page: ++ * 1. If it's hot, e.g., freshly faulted in or previously hot and ++ * migrated, add it to the youngest generation. ++ * 2. If it's cold but can't be evicted immediately, i.e., an anon page ++ * not in swapcache or a dirty page pending writeback, add it to the ++ * second oldest generation. ++ * 3. Everything else (clean, cold) is added to the oldest generation. ++ */ ++ if (folio_test_active(folio)) ++ seq = lrugen->max_seq; ++ else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || ++ (folio_test_reclaim(folio) && ++ (folio_test_dirty(folio) || folio_test_writeback(folio)))) ++ seq = lrugen->min_seq[type] + 1; ++ else ++ seq = lrugen->min_seq[type]; + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++ gen = lru_gen_from_seq(seq); ++ flags = (gen + 1UL) << LRU_GEN_PGOFF; ++ /* see the comment on MIN_NR_GENS about PG_active */ ++ set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); + -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); ++ lru_gen_update_size(lruvec, folio, -1, gen); ++ /* for folio_rotate_reclaimable() */ ++ if (reclaiming) ++ list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); ++ else ++ list_add(&folio->lru, &lrugen->lists[gen][type][zone]); + -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); ++ return true; ++} + -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, mutex_args.count); -+ check_mutex_state(fd, mutex_args.mutex, 1, 456); ++static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ unsigned long flags; ++ int gen = folio_lru_gen(folio); + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); ++ if (gen < 0) ++ return false; + -+ /* test waking events */ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + -+ event_args.manual = false; -+ event_args.signaled = false; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++ /* for folio_migrate_flags() */ ++ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; ++ flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); ++ gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + -+ objs[1] = event_args.event; -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++ lru_gen_update_size(lruvec, folio, gen, -1); ++ list_del(&folio->lru); + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++ return true; ++} + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); ++#else /* !CONFIG_LRU_GEN */ + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); ++static inline bool lru_gen_enabled(void) ++{ ++ return false; ++} + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++static inline bool lru_gen_in_fault(void) ++{ ++ return false; ++} + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ return false; ++} + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); ++static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ return false; ++} + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); ++#endif /* CONFIG_LRU_GEN */ + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); + static __always_inline + void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) + { + enum lru_list lru = folio_lru_list(folio); + ++ if (lru_gen_add_folio(lruvec, folio, false)) ++ return; + -+ event_args.manual = true; -+ event_args.signaled = false; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); + update_lru_size(lruvec, lru, folio_zonenum(folio), + folio_nr_pages(folio)); + if (lru != LRU_UNEVICTABLE) +@@ -116,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) + { + enum lru_list lru = folio_lru_list(folio); + ++ if (lru_gen_add_folio(lruvec, folio, true)) ++ return; + -+ objs[1] = event_args.event; -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); + update_lru_size(lruvec, lru, folio_zonenum(folio), + folio_nr_pages(folio)); + /* This is not expected to be used on LRU_UNEVICTABLE */ +@@ -133,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) + { + enum lru_list lru = folio_lru_list(folio); + ++ if (lru_gen_del_folio(lruvec, folio, false)) ++ return; + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); + if (lru != LRU_UNEVICTABLE) + list_del(&folio->lru); + update_lru_size(lruvec, lru, folio_zonenum(folio), +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index cf97f3884fda..e1797813cc2c 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -672,6 +672,22 @@ struct mm_struct { + */ + unsigned long ksm_merging_pages; + #endif ++#ifdef CONFIG_LRU_GEN ++ struct { ++ /* this mm_struct is on lru_gen_mm_list */ ++ struct list_head list; ++ /* ++ * Set when switching to this mm_struct, as a hint of ++ * whether it has been used since the last time per-node ++ * page table walkers cleared the corresponding bits. ++ */ ++ unsigned long bitmap; ++#ifdef CONFIG_MEMCG ++ /* points to the memcg of "owner" above */ ++ struct mem_cgroup *memcg; ++#endif ++ } lru_gen; ++#endif /* CONFIG_LRU_GEN */ + } __randomize_layout; + + /* +@@ -698,6 +714,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) + return (struct cpumask *)&mm->cpu_bitmap; + } + ++#ifdef CONFIG_LRU_GEN + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); ++struct lru_gen_mm_list { ++ /* mm_struct list for page table walkers */ ++ struct list_head fifo; ++ /* protects the list above */ ++ spinlock_t lock; ++}; + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); ++void lru_gen_add_mm(struct mm_struct *mm); ++void lru_gen_del_mm(struct mm_struct *mm); ++#ifdef CONFIG_MEMCG ++void lru_gen_migrate_mm(struct mm_struct *mm); ++#endif + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); ++static inline void lru_gen_init_mm(struct mm_struct *mm) ++{ ++ INIT_LIST_HEAD(&mm->lru_gen.list); ++ mm->lru_gen.bitmap = 0; ++#ifdef CONFIG_MEMCG ++ mm->lru_gen.memcg = NULL; ++#endif ++} + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++static inline void lru_gen_use_mm(struct mm_struct *mm) ++{ ++ /* ++ * When the bitmap is set, page reclaim knows this mm_struct has been ++ * used since the last time it cleared the bitmap. So it might be worth ++ * walking the page tables of this mm_struct to clear the accessed bit. ++ */ ++ WRITE_ONCE(mm->lru_gen.bitmap, -1); ++} + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++#else /* !CONFIG_LRU_GEN */ + -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); ++static inline void lru_gen_add_mm(struct mm_struct *mm) ++{ ++} + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); ++static inline void lru_gen_del_mm(struct mm_struct *mm) ++{ ++} + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++#ifdef CONFIG_MEMCG ++static inline void lru_gen_migrate_mm(struct mm_struct *mm) ++{ ++} ++#endif + -+ /* delete an object while it's being waited on */ ++static inline void lru_gen_init_mm(struct mm_struct *mm) ++{ ++} + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); -+ wait_args.owner = 123; -+ objs[1] = mutex_args.mutex; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++static inline void lru_gen_use_mm(struct mm_struct *mm) ++{ ++} + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++#endif /* CONFIG_LRU_GEN */ + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); + struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index 7d78133fe8dd..e5c240eed6af 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -306,6 +306,8 @@ static inline bool is_active_lru(enum lru_list lru) + return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); + } + ++#define WORKINGSET_ANON 0 ++#define WORKINGSET_FILE 1 + #define ANON_AND_FILE 2 + + enum lruvec_flags { +@@ -314,6 +316,207 @@ enum lruvec_flags { + */ + }; + ++#endif /* !__GENERATING_BOUNDS_H */ + -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); ++/* ++ * Evictable pages are divided into multiple generations. The youngest and the ++ * oldest generation numbers, max_seq and min_seq, are monotonically increasing. ++ * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An ++ * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the ++ * corresponding generation. The gen counter in folio->flags stores gen+1 while ++ * a page is on one of lrugen->lists[]. Otherwise it stores 0. ++ * ++ * A page is added to the youngest generation on faulting. The aging needs to ++ * check the accessed bit at least twice before handing this page over to the ++ * eviction. The first check takes care of the accessed bit set on the initial ++ * fault; the second check makes sure this page hasn't been used since then. ++ * This process, AKA second chance, requires a minimum of two generations, ++ * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive ++ * LRU, e.g., /proc/vmstat, these two generations are considered active; the ++ * rest of generations, if they exist, are considered inactive. See ++ * lru_gen_is_active(). ++ * ++ * PG_active is always cleared while a page is on one of lrugen->lists[] so that ++ * the aging needs not to worry about it. And it's set again when a page ++ * considered active is isolated for non-reclaiming purposes, e.g., migration. ++ * See lru_gen_add_folio() and lru_gen_del_folio(). ++ * ++ * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the ++ * number of categories of the active/inactive LRU when keeping track of ++ * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits ++ * in folio->flags. ++ */ ++#define MIN_NR_GENS 2U ++#define MAX_NR_GENS 4U + -+ close(fd); -+} ++/* ++ * Each generation is divided into multiple tiers. A page accessed N times ++ * through file descriptors is in tier order_base_2(N). A page in the first tier ++ * (N=0,1) is marked by PG_referenced unless it was faulted in through page ++ * tables or read ahead. A page in any other tier (N>1) is marked by ++ * PG_referenced and PG_workingset. This implies a minimum of two tiers is ++ * supported without using additional bits in folio->flags. ++ * ++ * In contrast to moving across generations which requires the LRU lock, moving ++ * across tiers only involves atomic operations on folio->flags and therefore ++ * has a negligible cost in the buffered access path. In the eviction path, ++ * comparisons of refaulted/(evicted+protected) from the first tier and the ++ * rest infer whether pages accessed multiple times through file descriptors ++ * are statistically hot and thus worth protecting. ++ * ++ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the ++ * number of categories of the active/inactive LRU when keeping track of ++ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in ++ * folio->flags. ++ */ ++#define MAX_NR_TIERS 4U + -+TEST(wake_all) -+{ -+ struct winesync_event_args manual_event_args = {0}; -+ struct winesync_event_args auto_event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 objs[4], count, index; -+ struct timespec timeout; -+ pthread_t thread; -+ int fd, ret; ++#ifndef __GENERATING_BOUNDS_H + -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++struct lruvec; ++struct page_vma_mapped_walk; + -+ sem_args.count = 0; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); ++#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) ++#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) + -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); ++#ifdef CONFIG_LRU_GEN + -+ manual_event_args.manual = true; -+ manual_event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); ++enum { ++ LRU_GEN_ANON, ++ LRU_GEN_FILE, ++}; + -+ auto_event_args.manual = false; -+ auto_event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); ++enum { ++ LRU_GEN_CORE, ++ LRU_GEN_MM_WALK, ++ LRU_GEN_NONLEAF_YOUNG, ++ NR_LRU_GEN_CAPS ++}; + -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ objs[2] = manual_event_args.event; -+ objs[3] = auto_event_args.event; ++#define MIN_LRU_BATCH BITS_PER_LONG ++#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 4; -+ wait_args.owner = 456; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++/* whether to keep historical stats from evicted generations */ ++#ifdef CONFIG_LRU_GEN_STATS ++#define NR_HIST_GENS MAX_NR_GENS ++#else ++#define NR_HIST_GENS 1U ++#endif + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++/* ++ * The youngest generation number is stored in max_seq for both anon and file ++ * types as they are aged on an equal footing. The oldest generation numbers are ++ * stored in min_seq[] separately for anon and file types as clean file pages ++ * can be evicted regardless of swap constraints. ++ * ++ * Normally anon and file min_seq are in sync. But if swapping is constrained, ++ * e.g., out of swap space, file min_seq is allowed to advance and leave anon ++ * min_seq behind. ++ * ++ * The number of pages in each generation is eventually consistent and therefore ++ * can be transiently negative when reset_batch_size() is pending. ++ */ ++struct lru_gen_struct { ++ /* the aging increments the youngest generation number */ ++ unsigned long max_seq; ++ /* the eviction increments the oldest generation numbers */ ++ unsigned long min_seq[ANON_AND_FILE]; ++ /* the birth time of each generation in jiffies */ ++ unsigned long timestamps[MAX_NR_GENS]; ++ /* the multi-gen LRU lists, lazily sorted on eviction */ ++ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* the multi-gen LRU sizes, eventually consistent */ ++ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* the exponential moving average of refaulted */ ++ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; ++ /* the exponential moving average of evicted+protected */ ++ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; ++ /* the first tier doesn't need protection, hence the minus one */ ++ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; ++ /* can be modified without holding the LRU lock */ ++ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; ++ atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; ++ /* whether the multi-gen LRU is enabled */ ++ bool enabled; ++}; + -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); ++enum { ++ MM_LEAF_TOTAL, /* total leaf entries */ ++ MM_LEAF_OLD, /* old leaf entries */ ++ MM_LEAF_YOUNG, /* young leaf entries */ ++ MM_NONLEAF_TOTAL, /* total non-leaf entries */ ++ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ ++ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ ++ NR_MM_STATS ++}; + -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); ++/* double-buffering Bloom filters */ ++#define NR_BLOOM_FILTERS 2 + -+ check_sem_state(fd, sem_args.sem, 1, 3); ++struct lru_gen_mm_state { ++ /* set to max_seq after each iteration */ ++ unsigned long seq; ++ /* where the current iteration continues (inclusive) */ ++ struct list_head *head; ++ /* where the last iteration ended (exclusive) */ ++ struct list_head *tail; ++ /* to wait for the last page table walker to finish */ ++ struct wait_queue_head wait; ++ /* Bloom filters flip after each iteration */ ++ unsigned long *filters[NR_BLOOM_FILTERS]; ++ /* the mm stats for debugging */ ++ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; ++ /* the number of concurrent page table walkers */ ++ int nr_walkers; ++}; + -+ ret = wait_any(fd, 1, &sem_args.sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); ++struct lru_gen_mm_walk { ++ /* the lruvec under reclaim */ ++ struct lruvec *lruvec; ++ /* unstable max_seq from lru_gen_struct */ ++ unsigned long max_seq; ++ /* the next address within an mm to scan */ ++ unsigned long next_addr; ++ /* to batch promoted pages */ ++ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; ++ /* to batch the mm stats */ ++ int mm_stats[NR_MM_STATS]; ++ /* total batched items */ ++ int batched; ++ bool can_swap; ++ bool force_scan; ++}; + -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); ++void lru_gen_init_lruvec(struct lruvec *lruvec); ++void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); + -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); ++#ifdef CONFIG_MEMCG ++void lru_gen_init_memcg(struct mem_cgroup *memcg); ++void lru_gen_exit_memcg(struct mem_cgroup *memcg); ++#endif + -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); ++#else /* !CONFIG_LRU_GEN */ + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, manual_event_args.signaled); ++static inline void lru_gen_init_lruvec(struct lruvec *lruvec) ++{ ++} + -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ check_sem_state(fd, sem_args.sem, 2, 3); ++static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) ++{ ++} + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, auto_event_args.signaled); ++#ifdef CONFIG_MEMCG ++static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) ++{ ++} + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, manual_event_args.signaled); ++static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) ++{ ++} ++#endif + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, auto_event_args.signaled); ++#endif /* CONFIG_LRU_GEN */ + -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 456); -+ check_event_state(fd, manual_event_args.event, 1, 1); -+ check_event_state(fd, auto_event_args.event, 0, 0); + struct lruvec { + struct list_head lists[NR_LRU_LISTS]; + /* per lruvec lru_lock for memcg */ +@@ -331,6 +534,12 @@ struct lruvec { + unsigned long refaults[ANON_AND_FILE]; + /* Various lruvec state flags (enum lruvec_flags) */ + unsigned long flags; ++#ifdef CONFIG_LRU_GEN ++ /* evictable pages divided into generations */ ++ struct lru_gen_struct lrugen; ++ /* to concurrently iterate lru_gen_mm_list */ ++ struct lru_gen_mm_state mm_state; ++#endif + #ifdef CONFIG_MEMCG + struct pglist_data *pgdat; + #endif +@@ -746,6 +955,8 @@ static inline bool zone_is_empty(struct zone *zone) + #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) + #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) + #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) ++#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) ++#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) + + /* + * Define the bit shifts to access each section. For non-existent +@@ -1007,6 +1218,11 @@ typedef struct pglist_data { + + unsigned long flags; + ++#ifdef CONFIG_LRU_GEN ++ /* kswap mm walk data */ ++ struct lru_gen_mm_walk mm_walk; ++#endif + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); + ZONE_PADDING(_pad2_) + + /* Per-node vmstats */ +diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h +index ac5b6a371be5..e66742db741c 100644 +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) + #define first_online_node 0 + #define first_memory_node 0 + #define next_online_node(nid) (MAX_NUMNODES) ++#define next_memory_node(nid) (MAX_NUMNODES) + #define nr_node_ids 1U + #define nr_online_nodes 1U + +diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h +index ef1e3e736e14..7d79818dc065 100644 +--- a/include/linux/page-flags-layout.h ++++ b/include/linux/page-flags-layout.h +@@ -55,7 +55,8 @@ + #define SECTIONS_WIDTH 0 + #endif + +-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS ++#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ ++ <= BITS_PER_LONG - NR_PAGEFLAGS + #define NODES_WIDTH NODES_SHIFT + #elif defined(CONFIG_SPARSEMEM_VMEMMAP) + #error "Vmemmap: No space for nodes field in page flags" +@@ -89,8 +90,8 @@ + #define LAST_CPUPID_SHIFT 0 + #endif + +-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ +- <= BITS_PER_LONG - NR_PAGEFLAGS ++#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ ++ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS + #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT + #else + #define LAST_CPUPID_WIDTH 0 +@@ -100,10 +101,15 @@ + #define LAST_CPUPID_NOT_IN_PAGE_FLAGS + #endif + +-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ +- > BITS_PER_LONG - NR_PAGEFLAGS ++#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ ++ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS + #error "Not enough bits in page flags" + #endif + ++/* see the comment on MAX_NR_TIERS */ ++#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ ++ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ ++ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) + -+ /* delete an object while it's being waited on */ + #endif + #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ +diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h +index 465ff35a8c00..0b0ae5084e60 100644 +--- a/include/linux/page-flags.h ++++ b/include/linux/page-flags.h +@@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) + 1UL << PG_private | 1UL << PG_private_2 | \ + 1UL << PG_writeback | 1UL << PG_reserved | \ + 1UL << PG_slab | 1UL << PG_active | \ +- 1UL << PG_unevictable | __PG_MLOCKED) ++ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) + + /* + * Flags checked when a page is prepped for return by the page allocator. +@@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) + * alloc-free cycle to prevent from reusing the page. + */ + #define PAGE_FLAGS_CHECK_AT_PREP \ +- (PAGEFLAGS_MASK & ~__PG_HWPOISON) ++ ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) + + #define PAGE_FLAGS_PRIVATE \ + (1UL << PG_private | 1UL << PG_private_2) +diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h +index 014ee8f0fbaa..d9095251bffd 100644 +--- a/include/linux/pgtable.h ++++ b/include/linux/pgtable.h +@@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + #endif + + #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +-#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) + static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +@@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + BUILD_BUG(); + return 0; + } +-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ ++#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ + #endif + + #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +@@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, + #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + #endif + ++#ifndef arch_has_hw_pte_young ++/* ++ * Return whether the accessed bit is supported on the local CPU. ++ * ++ * This stub assumes accessing through an old PTE triggers a page fault. ++ * Architectures that automatically set the access bit should overwrite it. ++ */ ++static inline bool arch_has_hw_pte_young(void) ++{ ++ return false; ++} ++#endif + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); -+ wait_args.owner = 123; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); + #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR + static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, +diff --git a/include/linux/sched.h b/include/linux/sched.h +index 367b30ed77cb..070b15ba2e1e 100644 +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -914,6 +914,10 @@ struct task_struct { + #ifdef CONFIG_MEMCG + unsigned in_user_fault:1; + #endif ++#ifdef CONFIG_LRU_GEN ++ /* whether the LRU algorithm may apply to this access */ ++ unsigned in_lru_fault:1; ++#endif + #ifdef CONFIG_COMPAT_BRK + unsigned brk_randomized:1; + #endif +diff --git a/include/linux/swap.h b/include/linux/swap.h +index 43150b9bbc5c..6308150b234a 100644 +--- a/include/linux/swap.h ++++ b/include/linux/swap.h +@@ -162,6 +162,10 @@ union swap_header { + */ + struct reclaim_state { + unsigned long reclaimed_slab; ++#ifdef CONFIG_LRU_GEN ++ /* per-thread mm walk data */ ++ struct lru_gen_mm_walk *mm_walk; ++#endif + }; + + #ifdef __KERNEL__ +diff --git a/kernel/bounds.c b/kernel/bounds.c +index 9795d75b09b2..b529182e8b04 100644 +--- a/kernel/bounds.c ++++ b/kernel/bounds.c +@@ -22,6 +22,13 @@ int main(void) + DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); + #endif + DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); ++#ifdef CONFIG_LRU_GEN ++ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); ++ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); ++#else ++ DEFINE(LRU_GEN_WIDTH, 0); ++ DEFINE(__LRU_REFS_WIDTH, 0); ++#endif + /* End of constants */ + + return 0; +diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h +index 36b740cb3d59..63dc3e82be4f 100644 +--- a/kernel/cgroup/cgroup-internal.h ++++ b/kernel/cgroup/cgroup-internal.h +@@ -164,7 +164,6 @@ struct cgroup_mgctx { + #define DEFINE_CGROUP_MGCTX(name) \ + struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) + +-extern struct mutex cgroup_mutex; + extern spinlock_t css_set_lock; + extern struct cgroup_subsys *cgroup_subsys[]; + extern struct list_head cgroup_roots; +diff --git a/kernel/exit.c b/kernel/exit.c +index 84021b24f79e..98a33bd7c25c 100644 +--- a/kernel/exit.c ++++ b/kernel/exit.c +@@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm) + goto retry; + } + WRITE_ONCE(mm->owner, c); ++ lru_gen_migrate_mm(mm); + task_unlock(c); + put_task_struct(c); + } +diff --git a/kernel/fork.c b/kernel/fork.c +index 704fe6bc9cb4..b15aaae04403 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -1156,6 +1156,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, + goto fail_nocontext; + + mm->user_ns = get_user_ns(user_ns); ++ lru_gen_init_mm(mm); + return mm; + + fail_nocontext: +@@ -1198,6 +1199,7 @@ static inline void __mmput(struct mm_struct *mm) + } + if (mm->binfmt) + module_put(mm->binfmt->module); ++ lru_gen_del_mm(mm); + mmdrop(mm); + } + +@@ -2700,6 +2702,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) + get_task_struct(p); + } + ++ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { ++ /* lock the task to synchronize with memcg migration */ ++ task_lock(p); ++ lru_gen_add_mm(p->mm); ++ task_unlock(p); ++ } + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); + wake_up_new_task(p); + + /* forking complete and child started to run, tell ptracer */ +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index f2534e712a89..265944f14948 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -5178,6 +5178,7 @@ context_switch(struct rq *rq, struct task_struct *prev, + * finish_task_switch()'s mmdrop(). + */ + switch_mm_irqs_off(prev->active_mm, next->mm, next); ++ lru_gen_use_mm(next->mm); + + if (!prev->mm) { // from kernel + /* will mmdrop() in finish_task_switch(). */ +diff --git a/mm/Kconfig b/mm/Kconfig +index 0331f1461f81..96cd3ae25c6f 100644 +--- a/mm/Kconfig ++++ b/mm/Kconfig +@@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP + purposes. It is required to enable userfaultfd write protection on + file-backed memory types like shmem and hugetlbfs. + ++# multi-gen LRU { ++config LRU_GEN ++ bool "Multi-Gen LRU" ++ depends on MMU ++ # make sure folio->flags has enough spare bits ++ depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP ++ help ++ A high performance LRU implementation to overcommit memory. See ++ Documentation/admin-guide/mm/multigen_lru.rst for details. + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &manual_event_args.event); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &auto_event_args.event); -+ EXPECT_EQ(0, ret); ++config LRU_GEN_ENABLED ++ bool "Enable by default" ++ depends on LRU_GEN ++ help ++ This option enables the multi-gen LRU by default. + -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); ++config LRU_GEN_STATS ++ bool "Full stats for debugging" ++ depends on LRU_GEN ++ help ++ Do not enable this option unless you plan to look at historical stats ++ from evicted generations for debugging purpose. + -+ close(fd); -+} ++ This option has a per-memcg and per-node memory overhead. ++# } + -+TEST(alert_any) + source "mm/damon/Kconfig" + + endmenu +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 42cdc3338adc..786497dd5f26 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2423,7 +2423,8 @@ static void __split_huge_page_tail(struct page *head, int tail, + #ifdef CONFIG_64BIT + (1L << PG_arch_2) | + #endif +- (1L << PG_dirty))); ++ (1L << PG_dirty) | ++ LRU_GEN_MASK | LRU_REFS_MASK)); + + /* ->mapping in first tail page is compound_mapcount */ + VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, +diff --git a/mm/internal.h b/mm/internal.h +index 55ce10e4d0c0..cf134d58fd6d 100644 +--- a/mm/internal.h ++++ b/mm/internal.h +@@ -83,6 +83,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); + void folio_rotate_reclaimable(struct folio *folio); + bool __folio_end_writeback(struct folio *folio); + void deactivate_file_folio(struct folio *folio); ++void folio_activate(struct folio *folio); + + void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, + unsigned long floor, unsigned long ceiling); +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index 7eadbafc006b..48588e89867a 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -2789,6 +2789,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) + * - LRU isolation + * - lock_page_memcg() + * - exclusive reference ++ * - mem_cgroup_trylock_pages() + */ + folio->memcg_data = (unsigned long)memcg; + } +@@ -5170,6 +5171,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) + + static void mem_cgroup_free(struct mem_cgroup *memcg) + { ++ lru_gen_exit_memcg(memcg); + memcg_wb_domain_exit(memcg); + __mem_cgroup_free(memcg); + } +@@ -5228,6 +5230,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) + memcg->deferred_split_queue.split_queue_len = 0; + #endif + idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); ++ lru_gen_init_memcg(memcg); + return memcg; + fail: + mem_cgroup_id_remove(memcg); +@@ -6196,6 +6199,30 @@ static void mem_cgroup_move_task(void) + } + #endif + ++#ifdef CONFIG_LRU_GEN ++static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ struct timespec timeout; -+ __u32 objs[2], index; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[0] = sem_args.sem; -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[1] = sem_args.sem; -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); ++ struct task_struct *task; ++ struct cgroup_subsys_state *css; + -+ /* test wakeup via alert */ ++ /* find the first leader if there is any */ ++ cgroup_taskset_for_each_leader(task, css, tset) ++ break; + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++ if (!task) ++ return; + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event_args.event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++ task_lock(task); ++ if (task->mm && READ_ONCE(task->mm->owner) == task) ++ lru_gen_migrate_mm(task->mm); ++ task_unlock(task); ++} ++#else ++static void mem_cgroup_attach(struct cgroup_taskset *tset) ++{ ++} ++#endif /* CONFIG_LRU_GEN */ + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); + static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) + { + if (value == PAGE_COUNTER_MAX) +@@ -6601,6 +6628,7 @@ struct cgroup_subsys memory_cgrp_subsys = { + .css_reset = mem_cgroup_css_reset, + .css_rstat_flush = mem_cgroup_css_rstat_flush, + .can_attach = mem_cgroup_can_attach, ++ .attach = mem_cgroup_attach, + .cancel_attach = mem_cgroup_cancel_attach, + .post_attach = mem_cgroup_move_task, + .dfl_cftypes = memory_files, +diff --git a/mm/memory.c b/mm/memory.c +index e58d5d522467..bc4dc2e45dcc 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -126,18 +126,6 @@ int randomize_va_space __read_mostly = + 2; + #endif + +-#ifndef arch_faults_on_old_pte +-static inline bool arch_faults_on_old_pte(void) +-{ +- /* +- * Those arches which don't have hw access flag feature need to +- * implement their own helper. By default, "true" means pagefault +- * will be hit on old pte. +- */ +- return true; +-} +-#endif +- + #ifndef arch_wants_old_prefaulted_pte + static inline bool arch_wants_old_prefaulted_pte(void) + { +@@ -2871,7 +2859,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, + * On architectures with software "accessed" bits, we would + * take a double page fault, so mark it accessed here. + */ +- if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { ++ if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { + pte_t entry; + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); +@@ -5115,6 +5103,27 @@ static inline void mm_account_fault(struct pt_regs *regs, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); + } + ++#ifdef CONFIG_LRU_GEN ++static void lru_gen_enter_fault(struct vm_area_struct *vma) ++{ ++ /* the LRU algorithm doesn't apply to sequential or random reads */ ++ current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); ++} + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++static void lru_gen_exit_fault(void) ++{ ++ current->in_lru_fault = false; ++} ++#else ++static void lru_gen_enter_fault(struct vm_area_struct *vma) ++{ ++} + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); ++static void lru_gen_exit_fault(void) ++{ ++} ++#endif /* CONFIG_LRU_GEN */ + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); + /* + * By the time we get here, we already hold the mm semaphore + * +@@ -5146,11 +5155,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, + if (flags & FAULT_FLAG_USER) + mem_cgroup_enter_user_fault(); + ++ lru_gen_enter_fault(vma); + -+ /* test with an auto-reset event */ + if (unlikely(is_vm_hugetlb_page(vma))) + ret = hugetlb_fault(vma->vm_mm, vma, address, flags); + else + ret = __handle_mm_fault(vma, address, flags); + ++ lru_gen_exit_fault(); + -+ event_args.manual = false; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); + if (flags & FAULT_FLAG_USER) { + mem_cgroup_exit_user_fault(); + /* +diff --git a/mm/mm_init.c b/mm/mm_init.c +index 9ddaf0e1b0ab..0d7b2bd2454a 100644 +--- a/mm/mm_init.c ++++ b/mm/mm_init.c +@@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) + + shift = 8 * sizeof(unsigned long); + width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH +- - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; ++ - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; + mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", +- "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", ++ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", + SECTIONS_WIDTH, + NODES_WIDTH, + ZONES_WIDTH, + LAST_CPUPID_WIDTH, + KASAN_TAG_WIDTH, ++ LRU_GEN_WIDTH, ++ LRU_REFS_WIDTH, + NR_PAGEFLAGS); + mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", +diff --git a/mm/mmzone.c b/mm/mmzone.c +index 0ae7571e35ab..68e1511be12d 100644 +--- a/mm/mmzone.c ++++ b/mm/mmzone.c +@@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) + * Poison its list head, so that any operations on it would crash. + */ + list_del(&lruvec->lists[LRU_UNEVICTABLE]); + -+ sem_args.sem = objs[0]; -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); ++ lru_gen_init_lruvec(lruvec); + } + + #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) +diff --git a/mm/rmap.c b/mm/rmap.c +index 9af08343ce55..3080edeaa3ad 100644 +--- a/mm/rmap.c ++++ b/mm/rmap.c +@@ -828,6 +828,12 @@ static bool folio_referenced_one(struct folio *folio, + } + + if (pvmw.pte) { ++ if (lru_gen_enabled() && pte_young(*pvmw.pte) && ++ !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { ++ lru_gen_look_around(&pvmw); ++ referenced++; ++ } + -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); + if (ptep_clear_flush_young_notify(vma, address, + pvmw.pte)) { + /* +diff --git a/mm/swap.c b/mm/swap.c +index 186b4e5dcecf..027943b341a0 100644 +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu) + folio_batch_move_lru(fbatch, folio_activate_fn); + } + +-static void folio_activate(struct folio *folio) ++void folio_activate(struct folio *folio) + { + if (folio_test_lru(folio) && !folio_test_active(folio) && + !folio_test_unevictable(folio)) { +@@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu) + { + } + +-static void folio_activate(struct folio *folio) ++void folio_activate(struct folio *folio) + { + struct lruvec *lruvec; + +@@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio) + local_unlock(&cpu_fbatches.lock); + } + ++#ifdef CONFIG_LRU_GEN ++static void folio_inc_refs(struct folio *folio) ++{ ++ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); ++ if (folio_test_unevictable(folio)) ++ return; + -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); ++ if (!folio_test_referenced(folio)) { ++ folio_set_referenced(folio); ++ return; ++ } + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++ if (!folio_test_workingset(folio)) { ++ folio_set_workingset(folio); ++ return; ++ } + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); -+ EXPECT_EQ(0, ret); ++ /* see the comment on MAX_NR_TIERS */ ++ do { ++ new_flags = old_flags & LRU_REFS_MASK; ++ if (new_flags == LRU_REFS_MASK) ++ break; + -+ close(fd); ++ new_flags += BIT(LRU_REFS_PGOFF); ++ new_flags |= old_flags & ~LRU_REFS_MASK; ++ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); +} -+ -+TEST(alert_all) ++#else ++static void folio_inc_refs(struct folio *folio) +{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ struct timespec timeout; -+ __u32 objs[2], index; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); ++} ++#endif /* CONFIG_LRU_GEN */ + -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[0] = sem_args.sem; + /* + * Mark a page as having seen activity. + * +@@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio) + */ + void folio_mark_accessed(struct folio *folio) + { ++ if (lru_gen_enabled()) { ++ folio_inc_refs(folio); ++ return; ++ } + -+ sem_args.count = 1; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[1] = sem_args.sem; + if (!folio_test_referenced(folio)) { + folio_set_referenced(folio); + } else if (folio_test_unevictable(folio)) { +@@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio) + folio_test_unevictable(folio), folio); + VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + ++ /* see the comment in lru_gen_add_folio() */ ++ if (lru_gen_enabled() && !folio_test_unevictable(folio) && ++ lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) ++ folio_set_active(folio); + -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); + folio_get(folio); + local_lock(&cpu_fbatches.lock); + fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); +@@ -575,7 +619,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) + + static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) + { +- if (folio_test_active(folio) && !folio_test_unevictable(folio)) { ++ if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { + long nr_pages = folio_nr_pages(folio); + + lruvec_del_folio(lruvec, folio); +@@ -688,8 +732,8 @@ void deactivate_page(struct page *page) + { + struct folio *folio = page_folio(page); + +- if (folio_test_lru(folio) && folio_test_active(folio) && +- !folio_test_unevictable(folio)) { ++ if (folio_test_lru(folio) && !folio_test_unevictable(folio) && ++ (folio_test_active(folio) || lru_gen_enabled())) { + struct folio_batch *fbatch; + + folio_get(folio); +diff --git a/mm/vmscan.c b/mm/vmscan.c +index e673be68cea3..feb8416d8edd 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -50,6 +50,10 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + + #include + #include +@@ -130,6 +134,12 @@ struct scan_control { + /* Always discard instead of demoting to lower tier memory */ + unsigned int no_demotion:1; + ++#ifdef CONFIG_LRU_GEN ++ /* help kswapd make better choices among multiple memcgs */ ++ unsigned int memcgs_need_aging:1; ++ unsigned long last_reclaimed; ++#endif + -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); + /* Allocation order */ + s8 order; + +@@ -1339,9 +1349,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, + + if (folio_test_swapcache(folio)) { + swp_entry_t swap = folio_swap_entry(folio); +- mem_cgroup_swapout(folio, swap); + -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); ++ /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ + if (reclaimed && !mapping_exiting(mapping)) + shadow = workingset_eviction(folio, target_memcg); ++ mem_cgroup_swapout(folio, swap); + __delete_from_swap_cache(folio, swap, shadow); + xa_unlock_irq(&mapping->i_pages); + put_swap_page(&folio->page, swap); +@@ -1666,6 +1678,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, + if (!sc->may_unmap && folio_mapped(folio)) + goto keep_locked; + ++ /* folio_update_gen() tried to promote this page? */ ++ if (lru_gen_enabled() && !ignore_references && ++ folio_mapped(folio) && folio_test_referenced(folio)) ++ goto keep_locked; + -+ /* test wakeup via alert */ + /* + * The number of dirty pages determines if a node is marked + * reclaim_congested. kswapd will stall and start writing +@@ -2761,6 +2778,112 @@ enum scan_balance { + SCAN_FILE, + }; + ++static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) ++{ ++ unsigned long file; ++ struct lruvec *target_lruvec; + -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++ if (lru_gen_enabled()) ++ return; + -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event_args.event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); ++ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); ++ /* ++ * Flush the memory cgroup stats, so that we read accurate per-memcg ++ * lruvec stats for heuristics. ++ */ ++ mem_cgroup_flush_stats(); + -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++ /* ++ * Determine the scan balance between anon and file LRUs. ++ */ ++ spin_lock_irq(&target_lruvec->lru_lock); ++ sc->anon_cost = target_lruvec->anon_cost; ++ sc->file_cost = target_lruvec->file_cost; ++ spin_unlock_irq(&target_lruvec->lru_lock); + -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); ++ /* ++ * Target desirable inactive:active list ratios for the anon ++ * and file LRU lists. ++ */ ++ if (!sc->force_deactivate) { ++ unsigned long refaults; + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++ /* ++ * When refaults are being observed, it means a new ++ * workingset is being established. Deactivate to get ++ * rid of any stale active pages quickly. ++ */ ++ refaults = lruvec_page_state(target_lruvec, ++ WORKINGSET_ACTIVATE_ANON); ++ if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || ++ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) ++ sc->may_deactivate |= DEACTIVATE_ANON; ++ else ++ sc->may_deactivate &= ~DEACTIVATE_ANON; + -+ /* test with an auto-reset event */ ++ refaults = lruvec_page_state(target_lruvec, ++ WORKINGSET_ACTIVATE_FILE); ++ if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || ++ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) ++ sc->may_deactivate |= DEACTIVATE_FILE; ++ else ++ sc->may_deactivate &= ~DEACTIVATE_FILE; ++ } else ++ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; + -+ event_args.manual = false; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); ++ /* ++ * If we have plenty of inactive file pages that aren't ++ * thrashing, try to reclaim those first before touching ++ * anonymous pages. ++ */ ++ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); ++ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) ++ sc->cache_trim_mode = 1; ++ else ++ sc->cache_trim_mode = 0; + -+ sem_args.sem = objs[1]; -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); ++ /* ++ * Prevent the reclaimer from falling into the cache trap: as ++ * cache pages start out inactive, every cache fault will tip ++ * the scan balance towards the file LRU. And as the file LRU ++ * shrinks, so does the window for rotation from references. ++ * This means we have a runaway feedback loop where a tiny ++ * thrashing file LRU becomes infinitely more attractive than ++ * anon pages. Try to detect this based on file LRU size. ++ */ ++ if (!cgroup_reclaim(sc)) { ++ unsigned long total_high_wmark = 0; ++ unsigned long free, anon; ++ int z; + -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); ++ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); ++ file = node_page_state(pgdat, NR_ACTIVE_FILE) + ++ node_page_state(pgdat, NR_INACTIVE_FILE); + -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); ++ for (z = 0; z < MAX_NR_ZONES; z++) { ++ struct zone *zone = &pgdat->node_zones[z]; + -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); ++ if (!managed_zone(zone)) ++ continue; + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); ++ total_high_wmark += high_wmark_pages(zone); ++ } + -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); -+ EXPECT_EQ(0, ret); ++ /* ++ * Consider anon: if that's low too, this isn't a ++ * runaway file reclaim problem, but rather just ++ * extreme pressure. Reclaim as per usual then. ++ */ ++ anon = node_page_state(pgdat, NR_INACTIVE_ANON); + -+ close(fd); ++ sc->file_is_tiny = ++ file + free <= total_high_wmark && ++ !(sc->may_deactivate & DEACTIVATE_ANON) && ++ anon >> sc->priority; ++ } +} + -+TEST_HARNESS_MAIN --- -2.37.3 - -From f56b86ff7aa71d9627939c0cbfa928f23042d902 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Fri, 5 Aug 2022 19:33:47 +0200 -Subject: [PATCH 04/13] Introducing-OpenVPN-Data-Channel-Offload - -Signed-off-by: Peter Jung ---- - MAINTAINERS | 8 + - drivers/net/Kconfig | 19 + - drivers/net/Makefile | 1 + - drivers/net/ovpn-dco/Makefile | 21 + - drivers/net/ovpn-dco/addr.h | 41 + - drivers/net/ovpn-dco/bind.c | 62 ++ - drivers/net/ovpn-dco/bind.h | 67 ++ - drivers/net/ovpn-dco/crypto.c | 154 ++++ - drivers/net/ovpn-dco/crypto.h | 144 ++++ - drivers/net/ovpn-dco/crypto_aead.c | 367 +++++++++ - drivers/net/ovpn-dco/crypto_aead.h | 27 + - drivers/net/ovpn-dco/main.c | 271 +++++++ - drivers/net/ovpn-dco/main.h | 32 + - drivers/net/ovpn-dco/netlink.c | 1143 ++++++++++++++++++++++++++++ - drivers/net/ovpn-dco/netlink.h | 22 + - drivers/net/ovpn-dco/ovpn.c | 600 +++++++++++++++ - drivers/net/ovpn-dco/ovpn.h | 43 ++ - drivers/net/ovpn-dco/ovpnstruct.h | 59 ++ - drivers/net/ovpn-dco/peer.c | 906 ++++++++++++++++++++++ - drivers/net/ovpn-dco/peer.h | 168 ++++ - drivers/net/ovpn-dco/pktid.c | 127 ++++ - drivers/net/ovpn-dco/pktid.h | 116 +++ - drivers/net/ovpn-dco/proto.h | 101 +++ - drivers/net/ovpn-dco/rcu.h | 21 + - drivers/net/ovpn-dco/skb.h | 54 ++ - drivers/net/ovpn-dco/sock.c | 134 ++++ - drivers/net/ovpn-dco/sock.h | 54 ++ - drivers/net/ovpn-dco/stats.c | 20 + - drivers/net/ovpn-dco/stats.h | 67 ++ - drivers/net/ovpn-dco/tcp.c | 326 ++++++++ - drivers/net/ovpn-dco/tcp.h | 38 + - drivers/net/ovpn-dco/udp.c | 343 +++++++++ - drivers/net/ovpn-dco/udp.h | 25 + - include/net/netlink.h | 1 + - include/uapi/linux/ovpn_dco.h | 265 +++++++ - include/uapi/linux/udp.h | 1 + - 36 files changed, 5848 insertions(+) - create mode 100644 drivers/net/ovpn-dco/Makefile - create mode 100644 drivers/net/ovpn-dco/addr.h - create mode 100644 drivers/net/ovpn-dco/bind.c - create mode 100644 drivers/net/ovpn-dco/bind.h - create mode 100644 drivers/net/ovpn-dco/crypto.c - create mode 100644 drivers/net/ovpn-dco/crypto.h - create mode 100644 drivers/net/ovpn-dco/crypto_aead.c - create mode 100644 drivers/net/ovpn-dco/crypto_aead.h - create mode 100644 drivers/net/ovpn-dco/main.c - create mode 100644 drivers/net/ovpn-dco/main.h - create mode 100644 drivers/net/ovpn-dco/netlink.c - create mode 100644 drivers/net/ovpn-dco/netlink.h - create mode 100644 drivers/net/ovpn-dco/ovpn.c - create mode 100644 drivers/net/ovpn-dco/ovpn.h - create mode 100644 drivers/net/ovpn-dco/ovpnstruct.h - create mode 100644 drivers/net/ovpn-dco/peer.c - create mode 100644 drivers/net/ovpn-dco/peer.h - create mode 100644 drivers/net/ovpn-dco/pktid.c - create mode 100644 drivers/net/ovpn-dco/pktid.h - create mode 100644 drivers/net/ovpn-dco/proto.h - create mode 100644 drivers/net/ovpn-dco/rcu.h - create mode 100644 drivers/net/ovpn-dco/skb.h - create mode 100644 drivers/net/ovpn-dco/sock.c - create mode 100644 drivers/net/ovpn-dco/sock.h - create mode 100644 drivers/net/ovpn-dco/stats.c - create mode 100644 drivers/net/ovpn-dco/stats.h - create mode 100644 drivers/net/ovpn-dco/tcp.c - create mode 100644 drivers/net/ovpn-dco/tcp.h - create mode 100644 drivers/net/ovpn-dco/udp.c - create mode 100644 drivers/net/ovpn-dco/udp.h - create mode 100644 include/uapi/linux/ovpn_dco.h - -diff --git a/MAINTAINERS b/MAINTAINERS -index 31a7aa60cdc3..a29c9731350c 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -15320,6 +15320,14 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git - F: Documentation/filesystems/overlayfs.rst - F: fs/overlayfs/ + /* + * Determine how aggressively the anon and file LRU lists should be + * scanned. +@@ -2980,159 +3103,2912 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, + return can_demote(pgdat->node_id, sc); + } -+OVPN-DCO NETWORK DRIVER -+M: Antonio Quartulli -+L: openvpn-devel@lists.sourceforge.net (moderated for non-subscribers) -+L: netdev@vger.kernel.org -+S: Maintained -+F: drivers/net/ovpn-dco/ -+F: include/uapi/linux/ovpn_dco.h -+ - P54 WIRELESS DRIVER - M: Christian Lamparter - L: linux-wireless@vger.kernel.org -diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig -index 94c889802566..349866bd4448 100644 ---- a/drivers/net/Kconfig -+++ b/drivers/net/Kconfig -@@ -116,6 +116,25 @@ config WIREGUARD_DEBUG +-static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +-{ +- unsigned long nr[NR_LRU_LISTS]; +- unsigned long targets[NR_LRU_LISTS]; +- unsigned long nr_to_scan; +- enum lru_list lru; +- unsigned long nr_reclaimed = 0; +- unsigned long nr_to_reclaim = sc->nr_to_reclaim; +- struct blk_plug plug; +- bool scan_adjusted; ++#ifdef CONFIG_LRU_GEN - Say N here unless you know what you're doing. +- get_scan_count(lruvec, sc, nr); ++#ifdef CONFIG_LRU_GEN_ENABLED ++DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); ++#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) ++#else ++DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); ++#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) ++#endif -+config OVPN_DCO -+ tristate "OpenVPN data channel offload" -+ depends on NET && INET -+ select NET_UDP_TUNNEL -+ select DST_CACHE -+ select CRYPTO -+ select CRYPTO_AES -+ select CRYPTO_GCM -+ select CRYPTO_CHACHA20POLY1305 -+ help -+ This module enhances the performance of an OpenVPN connection by -+ allowing the user to offload the data channel processing to -+ kernelspace. -+ Connection handshake, parameters negotiation and other non-data -+ related mechanisms are still performed in userspace. -+ -+ The OpenVPN userspace software at version 2.6 or higher is required -+ to use this functionality. -+ - config EQUALIZER - tristate "EQL (serial line load balancing) support" - help -diff --git a/drivers/net/Makefile b/drivers/net/Makefile -index 3f1192d3c52d..8ed151e8d233 100644 ---- a/drivers/net/Makefile -+++ b/drivers/net/Makefile -@@ -11,6 +11,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/ - obj-$(CONFIG_IPVTAP) += ipvlan/ - obj-$(CONFIG_DUMMY) += dummy.o - obj-$(CONFIG_WIREGUARD) += wireguard/ -+obj-$(CONFIG_OVPN_DCO) += ovpn-dco/ - obj-$(CONFIG_EQUALIZER) += eql.o - obj-$(CONFIG_IFB) += ifb.o - obj-$(CONFIG_MACSEC) += macsec.o -diff --git a/drivers/net/ovpn-dco/Makefile b/drivers/net/ovpn-dco/Makefile -new file mode 100644 -index 000000000000..7efefe8f13a9 ---- /dev/null -+++ b/drivers/net/ovpn-dco/Makefile -@@ -0,0 +1,21 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# ovpn-dco -- OpenVPN data channel offload -+# -+# Copyright (C) 2020-2022 OpenVPN, Inc. -+# -+# Author: Antonio Quartulli -+ -+obj-$(CONFIG_OVPN_DCO) += ovpn-dco.o -+ovpn-dco-y += main.o -+ovpn-dco-y += bind.o -+ovpn-dco-y += crypto.o -+ovpn-dco-y += ovpn.o -+ovpn-dco-y += peer.o -+ovpn-dco-y += sock.o -+ovpn-dco-y += stats.o -+ovpn-dco-y += netlink.o -+ovpn-dco-y += crypto_aead.o -+ovpn-dco-y += pktid.o -+ovpn-dco-y += tcp.o -+ovpn-dco-y += udp.o -diff --git a/drivers/net/ovpn-dco/addr.h b/drivers/net/ovpn-dco/addr.h -new file mode 100644 -index 000000000000..3d6ad0fc15af ---- /dev/null -+++ b/drivers/net/ovpn-dco/addr.h -@@ -0,0 +1,41 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNADDR_H_ -+#define _NET_OVPN_DCO_OVPNADDR_H_ -+ -+#include "crypto.h" -+ -+#include -+#include -+#include -+#include -+ -+/* our basic transport layer address */ -+struct ovpn_sockaddr { -+ union { -+ struct sockaddr_in in4; -+ struct sockaddr_in6 in6; -+ }; -+}; -+ -+/* Translate skb->protocol value to AF_INET or AF_INET6 */ -+static inline unsigned short skb_protocol_to_family(const struct sk_buff *skb) +- /* Record the original scan target for proportional adjustments later */ +- memcpy(targets, nr, sizeof(nr)); ++/****************************************************************************** ++ * shorthand helpers ++ ******************************************************************************/ + +- /* +- * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal +- * event that can occur when there is little memory pressure e.g. +- * multiple streaming readers/writers. Hence, we do not abort scanning +- * when the requested number of pages are reclaimed when scanning at +- * DEF_PRIORITY on the assumption that the fact we are direct +- * reclaiming implies that kswapd is not keeping up and it is best to +- * do a batch of work at once. For memcg reclaim one check is made to +- * abort proportional reclaim if either the file or anon lru has already +- * dropped to zero at the first pass. +- */ +- scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && +- sc->priority == DEF_PRIORITY); ++#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) + +- blk_start_plug(&plug); +- while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || +- nr[LRU_INACTIVE_FILE]) { +- unsigned long nr_anon, nr_file, percentage; +- unsigned long nr_scanned; ++#define DEFINE_MAX_SEQ(lruvec) \ ++ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) + +- for_each_evictable_lru(lru) { +- if (nr[lru]) { +- nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); +- nr[lru] -= nr_to_scan; ++#define DEFINE_MIN_SEQ(lruvec) \ ++ unsigned long min_seq[ANON_AND_FILE] = { \ ++ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ ++ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ ++ } + +- nr_reclaimed += shrink_list(lru, nr_to_scan, +- lruvec, sc); +- } +- } ++#define for_each_gen_type_zone(gen, type, zone) \ ++ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ ++ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ ++ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) + +- cond_resched(); ++static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) ++{ ++ struct pglist_data *pgdat = NODE_DATA(nid); + +- if (nr_reclaimed < nr_to_reclaim || scan_adjusted) +- continue; ++#ifdef CONFIG_MEMCG ++ if (memcg) { ++ struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; + +- /* +- * For kswapd and memcg, reclaim at least the number of pages +- * requested. Ensure that the anon and file LRUs are scanned +- * proportionally what was requested by get_scan_count(). We +- * stop reclaiming one LRU and reduce the amount scanning +- * proportional to the original scan target. +- */ +- nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; +- nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; ++ /* for hotadd_new_pgdat() */ ++ if (!lruvec->pgdat) ++ lruvec->pgdat = pgdat; + +- /* +- * It's just vindictive to attack the larger once the smaller +- * has gone to zero. And given the way we stop scanning the +- * smaller below, this makes sure that we only make one nudge +- * towards proportionality once we've got nr_to_reclaim. +- */ +- if (!nr_file || !nr_anon) +- break; ++ return lruvec; ++ } ++#endif ++ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); + +- if (nr_file > nr_anon) { +- unsigned long scan_target = targets[LRU_INACTIVE_ANON] + +- targets[LRU_ACTIVE_ANON] + 1; +- lru = LRU_BASE; +- percentage = nr_anon * 100 / scan_target; +- } else { +- unsigned long scan_target = targets[LRU_INACTIVE_FILE] + +- targets[LRU_ACTIVE_FILE] + 1; +- lru = LRU_FILE; +- percentage = nr_file * 100 / scan_target; +- } ++ return pgdat ? &pgdat->__lruvec : NULL; ++} + +- /* Stop scanning the smaller of the LRU */ +- nr[lru] = 0; +- nr[lru + LRU_ACTIVE] = 0; ++static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) +{ -+ switch (skb->protocol) { -+ case htons(ETH_P_IP): -+ return AF_INET; -+ case htons(ETH_P_IPV6): -+ return AF_INET6; -+ default: ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); + +- /* +- * Recalculate the other LRU scan count based on its original +- * scan target and the percentage scanning already complete +- */ +- lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; +- nr_scanned = targets[lru] - nr[lru]; +- nr[lru] = targets[lru] * (100 - percentage) / 100; +- nr[lru] -= min(nr[lru], nr_scanned); ++ if (!can_demote(pgdat->node_id, sc) && ++ mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) + return 0; -+ } + +- lru += LRU_ACTIVE; +- nr_scanned = targets[lru] - nr[lru]; +- nr[lru] = targets[lru] * (100 - percentage) / 100; +- nr[lru] -= min(nr[lru], nr_scanned); ++ return mem_cgroup_swappiness(memcg); +} + +- scan_adjusted = true; +- } +- blk_finish_plug(&plug); +- sc->nr_reclaimed += nr_reclaimed; ++static int get_nr_gens(struct lruvec *lruvec, int type) ++{ ++ return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; ++} + +- /* +- * Even if we did not try to evict anon pages at all, we want to +- * rebalance the anon lru active/inactive ratio. +- */ +- if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && +- inactive_is_low(lruvec, LRU_INACTIVE_ANON)) +- shrink_active_list(SWAP_CLUSTER_MAX, lruvec, +- sc, LRU_ACTIVE_ANON); ++static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) ++{ ++ /* see the comment on lru_gen_struct */ ++ return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && ++ get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && ++ get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; + } + +-/* Use reclaim/compaction for costly allocs or under memory pressure */ +-static bool in_reclaim_compaction(struct scan_control *sc) ++/****************************************************************************** ++ * mm_struct list ++ ******************************************************************************/ + -+#endif /* _NET_OVPN_DCO_OVPNADDR_H_ */ -diff --git a/drivers/net/ovpn-dco/bind.c b/drivers/net/ovpn-dco/bind.c -new file mode 100644 -index 000000000000..107697ea983e ---- /dev/null -+++ b/drivers/net/ovpn-dco/bind.c -@@ -0,0 +1,62 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2012-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "ovpn.h" -+#include "bind.h" -+#include "peer.h" -+ -+#include -+#include -+#include -+#include ++static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) + { +- if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && +- (sc->order > PAGE_ALLOC_COSTLY_ORDER || +- sc->priority < DEF_PRIORITY - 2)) +- return true; ++ static struct lru_gen_mm_list mm_list = { ++ .fifo = LIST_HEAD_INIT(mm_list.fifo), ++ .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), ++ }; + +- return false; ++#ifdef CONFIG_MEMCG ++ if (memcg) ++ return &memcg->mm_list; ++#endif ++ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); + -+/* Given a remote sockaddr, compute the skb hash -+ * and get a dst_entry so we can send packets to the remote. -+ * Called from process context or softirq (must be indicated with -+ * process_context bool). -+ */ -+struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *ss) -+{ -+ struct ovpn_bind *bind; -+ size_t sa_len; ++ return &mm_list; + } + +-/* +- * Reclaim/compaction is used for high-order allocation requests. It reclaims +- * order-0 pages before compacting the zone. should_continue_reclaim() returns +- * true if more pages should be reclaimed such that when the page allocator +- * calls try_to_compact_pages() that it will have enough free pages to succeed. +- * It will give up earlier than that if there is difficulty reclaiming pages. +- */ +-static inline bool should_continue_reclaim(struct pglist_data *pgdat, +- unsigned long nr_reclaimed, +- struct scan_control *sc) ++void lru_gen_add_mm(struct mm_struct *mm) + { +- unsigned long pages_for_compaction; +- unsigned long inactive_lru_pages; +- int z; ++ int nid; ++ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + +- /* If not in reclaim/compaction mode, stop */ +- if (!in_reclaim_compaction(sc)) +- return false; ++ VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); ++#ifdef CONFIG_MEMCG ++ VM_WARN_ON_ONCE(mm->lru_gen.memcg); ++ mm->lru_gen.memcg = memcg; ++#endif ++ spin_lock(&mm_list->lock); + +- /* +- * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX +- * number of pages that were scanned. This will return to the caller +- * with the risk reclaim/compaction and the resulting allocation attempt +- * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL +- * allocations through requiring that the full LRU list has been scanned +- * first, by assuming that zero delta of sc->nr_scanned means full LRU +- * scan, but that approximation was wrong, and there were corner cases ++ for_each_node_state(nid, N_MEMORY) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); + -+ if (ss->ss_family == AF_INET) -+ sa_len = sizeof(struct sockaddr_in); -+ else if (ss->ss_family == AF_INET6) -+ sa_len = sizeof(struct sockaddr_in6); -+ else -+ return ERR_PTR(-EAFNOSUPPORT); ++ if (!lruvec) ++ continue; + -+ bind = kzalloc(sizeof(*bind), GFP_ATOMIC); -+ if (unlikely(!bind)) -+ return ERR_PTR(-ENOMEM); ++ /* the first addition since the last iteration */ ++ if (lruvec->mm_state.tail == &mm_list->fifo) ++ lruvec->mm_state.tail = &mm->lru_gen.list; ++ } + -+ memcpy(&bind->sa, ss, sa_len); ++ list_add_tail(&mm->lru_gen.list, &mm_list->fifo); + -+ return bind; ++ spin_unlock(&mm_list->lock); +} + -+static void ovpn_bind_release_rcu(struct rcu_head *head) ++void lru_gen_del_mm(struct mm_struct *mm) +{ -+ struct ovpn_bind *bind = container_of(head, struct ovpn_bind, rcu); ++ int nid; ++ struct lru_gen_mm_list *mm_list; ++ struct mem_cgroup *memcg = NULL; + -+ kfree(bind); -+} ++ if (list_empty(&mm->lru_gen.list)) ++ return; + -+void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *new) -+{ -+ struct ovpn_bind *old; ++#ifdef CONFIG_MEMCG ++ memcg = mm->lru_gen.memcg; ++#endif ++ mm_list = get_mm_list(memcg); + -+ spin_lock_bh(&peer->lock); -+ old = rcu_replace_pointer(peer->bind, new, true); -+ spin_unlock_bh(&peer->lock); ++ spin_lock(&mm_list->lock); + -+ if (old) -+ call_rcu(&old->rcu, ovpn_bind_release_rcu); -+} -diff --git a/drivers/net/ovpn-dco/bind.h b/drivers/net/ovpn-dco/bind.h -new file mode 100644 -index 000000000000..a562e471acae ---- /dev/null -+++ b/drivers/net/ovpn-dco/bind.h -@@ -0,0 +1,67 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OVPN -- OpenVPN protocol accelerator for Linux -+ * Copyright (C) 2012-2022 OpenVPN, Inc. -+ * All rights reserved. -+ * Author: James Yonan -+ */ ++ for_each_node(nid) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); + -+#ifndef _NET_OVPN_DCO_OVPNBIND_H_ -+#define _NET_OVPN_DCO_OVPNBIND_H_ ++ if (!lruvec) ++ continue; + -+#include "addr.h" -+#include "rcu.h" ++ /* where the last iteration ended (exclusive) */ ++ if (lruvec->mm_state.tail == &mm->lru_gen.list) ++ lruvec->mm_state.tail = lruvec->mm_state.tail->next; + -+#include -+#include -+#include ++ /* where the current iteration continues (inclusive) */ ++ if (lruvec->mm_state.head != &mm->lru_gen.list) ++ continue; + -+struct ovpn_peer; ++ lruvec->mm_state.head = lruvec->mm_state.head->next; ++ /* the deletion ends the current iteration */ ++ if (lruvec->mm_state.head == &mm_list->fifo) ++ WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); ++ } + -+struct ovpn_bind { -+ struct ovpn_sockaddr sa; /* remote sockaddr */ ++ list_del_init(&mm->lru_gen.list); + -+ union { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } local; ++ spin_unlock(&mm_list->lock); + -+ struct rcu_head rcu; -+}; ++#ifdef CONFIG_MEMCG ++ mem_cgroup_put(mm->lru_gen.memcg); ++ mm->lru_gen.memcg = NULL; ++#endif ++} + -+static inline bool ovpn_bind_skb_src_match(const struct ovpn_bind *bind, struct sk_buff *skb) ++#ifdef CONFIG_MEMCG ++void lru_gen_migrate_mm(struct mm_struct *mm) +{ -+ const unsigned short family = skb_protocol_to_family(skb); -+ const struct ovpn_sockaddr *sa = &bind->sa; -+ -+ if (unlikely(!bind)) -+ return false; ++ struct mem_cgroup *memcg; ++ struct task_struct *task = rcu_dereference_protected(mm->owner, true); + -+ if (unlikely(sa->in4.sin_family != family)) -+ return false; ++ VM_WARN_ON_ONCE(task->mm != mm); ++ lockdep_assert_held(&task->alloc_lock); + -+ switch (family) { -+ case AF_INET: -+ if (unlikely(sa->in4.sin_addr.s_addr != ip_hdr(skb)->saddr)) -+ return false; ++ /* for mm_update_next_owner() */ ++ if (mem_cgroup_disabled()) ++ return; + -+ if (unlikely(sa->in4.sin_port != udp_hdr(skb)->source)) -+ return false; -+ break; -+ case AF_INET6: -+ if (unlikely(!ipv6_addr_equal(&sa->in6.sin6_addr, &ipv6_hdr(skb)->saddr))) -+ return false; ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_task(task); ++ rcu_read_unlock(); ++ if (memcg == mm->lru_gen.memcg) ++ return; + -+ if (unlikely(sa->in6.sin6_port != udp_hdr(skb)->source)) -+ return false; -+ break; -+ default: -+ return false; -+ } ++ VM_WARN_ON_ONCE(!mm->lru_gen.memcg); ++ VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); + -+ return true; ++ lru_gen_del_mm(mm); ++ lru_gen_add_mm(mm); +} ++#endif + -+struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *sa); -+void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *bind); -+ -+#endif /* _NET_OVPN_DCO_OVPNBIND_H_ */ -diff --git a/drivers/net/ovpn-dco/crypto.c b/drivers/net/ovpn-dco/crypto.c -new file mode 100644 -index 000000000000..fcc3a351ba9d ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto.c -@@ -0,0 +1,154 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator ++/* ++ * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when ++ * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of ++ * bits in a bitmap, k is the number of hash functions and n is the number of ++ * inserted items. + * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * Page table walkers use one of the two filters to reduce their search space. ++ * To get rid of non-leaf entries that no longer have enough leaf entries, the ++ * aging uses the double-buffering technique to flip to the other filter each ++ * time it produces a new generation. For non-leaf entries that have enough ++ * leaf entries, the aging carries them over to the next generation in ++ * walk_pmd_range(); the eviction also report them when walking the rmap ++ * in lru_gen_look_around(). + * -+ * Author: James Yonan -+ * Antonio Quartulli ++ * For future optimizations: ++ * 1. It's not necessary to keep both filters all the time. The spare one can be ++ * freed after the RCU grace period and reallocated if needed again. ++ * 2. And when reallocating, it's worth scaling its size according to the number ++ * of inserted entries in the other filter, to reduce the memory overhead on ++ * small systems and false positives on large systems. ++ * 3. Jenkins' hash function is an alternative to Knuth's. + */ ++#define BLOOM_FILTER_SHIFT 15 + -+#include "main.h" -+#include "crypto_aead.h" -+#include "crypto.h" ++static inline int filter_gen_from_seq(unsigned long seq) ++{ ++ return seq % NR_BLOOM_FILTERS; ++} + -+#include ++static void get_item_key(void *item, int *key) ++{ ++ u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); + -+static void ovpn_ks_destroy_rcu(struct rcu_head *head) ++ BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); ++ ++ key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); ++ key[1] = hash >> BLOOM_FILTER_SHIFT; ++} ++ ++static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) +{ -+ struct ovpn_crypto_key_slot *ks; ++ unsigned long *filter; ++ int gen = filter_gen_from_seq(seq); + -+ ks = container_of(head, struct ovpn_crypto_key_slot, rcu); -+ ovpn_aead_crypto_key_slot_destroy(ks); ++ filter = lruvec->mm_state.filters[gen]; ++ if (filter) { ++ bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); ++ return; ++ } ++ ++ filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), ++ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); ++ WRITE_ONCE(lruvec->mm_state.filters[gen], filter); +} + -+void ovpn_crypto_key_slot_release(struct kref *kref) ++static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) +{ -+ struct ovpn_crypto_key_slot *ks; ++ int key[2]; ++ unsigned long *filter; ++ int gen = filter_gen_from_seq(seq); + -+ ks = container_of(kref, struct ovpn_crypto_key_slot, refcount); -+ call_rcu(&ks->rcu, ovpn_ks_destroy_rcu); ++ filter = READ_ONCE(lruvec->mm_state.filters[gen]); ++ if (!filter) ++ return; ++ ++ get_item_key(item, key); ++ ++ if (!test_bit(key[0], filter)) ++ set_bit(key[0], filter); ++ if (!test_bit(key[1], filter)) ++ set_bit(key[1], filter); +} + -+/* can only be invoked when all peer references have been dropped (i.e. RCU -+ * release routine) -+ */ -+void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) ++static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) +{ -+ struct ovpn_crypto_key_slot *ks; ++ int key[2]; ++ unsigned long *filter; ++ int gen = filter_gen_from_seq(seq); + -+ ks = rcu_access_pointer(cs->primary); -+ if (ks) { -+ RCU_INIT_POINTER(cs->primary, NULL); -+ ovpn_crypto_key_slot_put(ks); -+ } ++ filter = READ_ONCE(lruvec->mm_state.filters[gen]); ++ if (!filter) ++ return true; + -+ ks = rcu_access_pointer(cs->secondary); -+ if (ks) { -+ RCU_INIT_POINTER(cs->secondary, NULL); -+ ovpn_crypto_key_slot_put(ks); -+ } ++ get_item_key(item, key); + -+ mutex_destroy(&cs->mutex); ++ return test_bit(key[0], filter) && test_bit(key[1], filter); +} + -+/* removes the primary key from the crypto context */ -+void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs) ++static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) +{ -+ struct ovpn_crypto_key_slot *ks; ++ int i; ++ int hist; + -+ mutex_lock(&cs->mutex); -+ ks = rcu_replace_pointer(cs->primary, NULL, lockdep_is_held(&cs->mutex)); -+ ovpn_crypto_key_slot_put(ks); -+ mutex_unlock(&cs->mutex); ++ lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); ++ ++ if (walk) { ++ hist = lru_hist_from_seq(walk->max_seq); ++ ++ for (i = 0; i < NR_MM_STATS; i++) { ++ WRITE_ONCE(lruvec->mm_state.stats[hist][i], ++ lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); ++ walk->mm_stats[i] = 0; ++ } ++ } ++ ++ if (NR_HIST_GENS > 1 && last) { ++ hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); ++ ++ for (i = 0; i < NR_MM_STATS; i++) ++ WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); ++ } +} + -+/* Reset the ovpn_crypto_state object in a way that is atomic -+ * to RCU readers. -+ */ -+int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, -+ const struct ovpn_peer_key_reset *pkr) -+ __must_hold(cs->mutex) ++static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) +{ -+ struct ovpn_crypto_key_slot *old = NULL; -+ struct ovpn_crypto_key_slot *new; ++ int type; ++ unsigned long size = 0; ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); + -+ lockdep_assert_held(&cs->mutex); ++ if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) ++ return true; + -+ new = ovpn_aead_crypto_key_slot_new(&pkr->key); -+ if (IS_ERR(new)) -+ return PTR_ERR(new); ++ clear_bit(key, &mm->lru_gen.bitmap); + -+ switch (pkr->slot) { -+ case OVPN_KEY_SLOT_PRIMARY: -+ old = rcu_replace_pointer(cs->primary, new, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ case OVPN_KEY_SLOT_SECONDARY: -+ old = rcu_replace_pointer(cs->secondary, new, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ default: -+ goto free_key; ++ for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { ++ size += type ? get_mm_counter(mm, MM_FILEPAGES) : ++ get_mm_counter(mm, MM_ANONPAGES) + ++ get_mm_counter(mm, MM_SHMEMPAGES); + } + -+ if (old) -+ ovpn_crypto_key_slot_put(old); ++ if (size < MIN_LRU_BATCH) ++ return true; + -+ return 0; -+free_key: -+ ovpn_crypto_key_slot_put(new); -+ return -EINVAL; ++ return !mmget_not_zero(mm); +} + -+void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, -+ enum ovpn_key_slot slot) ++static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, ++ struct mm_struct **iter) +{ -+ struct ovpn_crypto_key_slot *ks = NULL; ++ bool first = false; ++ bool last = true; ++ struct mm_struct *mm = NULL; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; + -+ mutex_lock(&cs->mutex); -+ switch (slot) { -+ case OVPN_KEY_SLOT_PRIMARY: -+ ks = rcu_replace_pointer(cs->primary, NULL, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ case OVPN_KEY_SLOT_SECONDARY: -+ ks = rcu_replace_pointer(cs->secondary, NULL, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ default: -+ pr_warn("Invalid slot to release: %u\n", slot); -+ break; ++ /* ++ * There are four interesting cases for this page table walker: ++ * 1. It tries to start a new iteration of mm_list with a stale max_seq; ++ * there is nothing left to do. ++ * 2. It's the first of the current generation, and it needs to reset ++ * the Bloom filter for the next generation. ++ * 3. It reaches the end of mm_list, and it needs to increment ++ * mm_state->seq; the iteration is done. ++ * 4. It's the last of the current generation, and it needs to reset the ++ * mm stats counters for the next generation. ++ */ ++ spin_lock(&mm_list->lock); ++ ++ VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); ++ VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); ++ VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); ++ ++ if (walk->max_seq <= mm_state->seq) { ++ if (!*iter) ++ last = false; ++ goto done; + } -+ mutex_unlock(&cs->mutex); + -+ if (!ks) { -+ pr_debug("Key slot already released: %u\n", slot); -+ return; ++ if (!mm_state->nr_walkers) { ++ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); ++ ++ mm_state->head = mm_list->fifo.next; ++ first = true; + } -+ pr_debug("deleting key slot %u, key_id=%u\n", slot, ks->key_id); + -+ ovpn_crypto_key_slot_put(ks); -+} ++ while (!mm && mm_state->head != &mm_list->fifo) { ++ mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); + -+/* this swap is not atomic, but there will be a very short time frame where the -+ * old_secondary key won't be available. This should not be a big deal as most -+ * likely both peers are already using the new primary at this point. -+ */ -+void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs) -+{ -+ const struct ovpn_crypto_key_slot *old_primary, *old_secondary; ++ mm_state->head = mm_state->head->next; + -+ mutex_lock(&cs->mutex); ++ /* force scan for those added after the last iteration */ ++ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { ++ mm_state->tail = mm_state->head; ++ walk->force_scan = true; ++ } + -+ old_secondary = rcu_dereference_protected(cs->secondary, -+ lockdep_is_held(&cs->mutex)); -+ old_primary = rcu_replace_pointer(cs->primary, old_secondary, -+ lockdep_is_held(&cs->mutex)); -+ rcu_assign_pointer(cs->secondary, old_primary); ++ if (should_skip_mm(mm, walk)) ++ mm = NULL; ++ } + -+ pr_debug("key swapped: %u <-> %u\n", -+ old_primary ? old_primary->key_id : 0, -+ old_secondary ? old_secondary->key_id : 0); ++ if (mm_state->head == &mm_list->fifo) ++ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); ++done: ++ if (*iter && !mm) ++ mm_state->nr_walkers--; ++ if (!*iter && mm) ++ mm_state->nr_walkers++; + -+ mutex_unlock(&cs->mutex); -+} -diff --git a/drivers/net/ovpn-dco/crypto.h b/drivers/net/ovpn-dco/crypto.h -new file mode 100644 -index 000000000000..79f580e54a63 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto.h -@@ -0,0 +1,144 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++ if (mm_state->nr_walkers) ++ last = false; + -+#ifndef _NET_OVPN_DCO_OVPNCRYPTO_H_ -+#define _NET_OVPN_DCO_OVPNCRYPTO_H_ ++ if (*iter || last) ++ reset_mm_stats(lruvec, walk, last); + -+#include "main.h" -+#include "pktid.h" ++ spin_unlock(&mm_list->lock); + -+#include -+#include ++ if (mm && first) ++ reset_bloom_filter(lruvec, walk->max_seq + 1); + -+struct ovpn_peer; -+struct ovpn_crypto_key_slot; ++ if (*iter) ++ mmput_async(*iter); + -+/* info needed for both encrypt and decrypt directions */ -+struct ovpn_key_direction { -+ const u8 *cipher_key; -+ size_t cipher_key_size; -+ const u8 *nonce_tail; /* only needed for GCM modes */ -+ size_t nonce_tail_size; /* only needed for GCM modes */ -+}; ++ *iter = mm; + -+/* all info for a particular symmetric key (primary or secondary) */ -+struct ovpn_key_config { -+ enum ovpn_cipher_alg cipher_alg; -+ u8 key_id; -+ struct ovpn_key_direction encrypt; -+ struct ovpn_key_direction decrypt; -+}; ++ return last; ++} + -+/* used to pass settings from netlink to the crypto engine */ -+struct ovpn_peer_key_reset { -+ enum ovpn_key_slot slot; -+ struct ovpn_key_config key; -+}; ++static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) ++{ ++ bool success = false; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); ++ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; + -+struct ovpn_crypto_key_slot { -+ u8 key_id; ++ spin_lock(&mm_list->lock); + -+ struct crypto_aead *encrypt; -+ struct crypto_aead *decrypt; -+ struct ovpn_nonce_tail nonce_tail_xmit; -+ struct ovpn_nonce_tail nonce_tail_recv; ++ VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); + -+ struct ovpn_pktid_recv pid_recv ____cacheline_aligned_in_smp; -+ struct ovpn_pktid_xmit pid_xmit ____cacheline_aligned_in_smp; -+ struct kref refcount; -+ struct rcu_head rcu; -+}; ++ if (max_seq > mm_state->seq && !mm_state->nr_walkers) { ++ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); + -+struct ovpn_crypto_state { -+ struct ovpn_crypto_key_slot __rcu *primary; -+ struct ovpn_crypto_key_slot __rcu *secondary; ++ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); ++ reset_mm_stats(lruvec, NULL, true); ++ success = true; ++ } + -+ /* protects primary and secondary slots */ -+ struct mutex mutex; -+}; ++ spin_unlock(&mm_list->lock); + -+static inline bool ovpn_crypto_key_slot_hold(struct ovpn_crypto_key_slot *ks) -+{ -+ return kref_get_unless_zero(&ks->refcount); ++ return success; +} + -+static inline void ovpn_crypto_state_init(struct ovpn_crypto_state *cs) ++/****************************************************************************** ++ * refault feedback loop ++ ******************************************************************************/ ++ ++/* ++ * A feedback loop based on Proportional-Integral-Derivative (PID) controller. ++ * ++ * The P term is refaulted/(evicted+protected) from a tier in the generation ++ * currently being evicted; the I term is the exponential moving average of the ++ * P term over the generations previously evicted, using the smoothing factor ++ * 1/2; the D term isn't supported. ++ * ++ * The setpoint (SP) is always the first tier of one type; the process variable ++ * (PV) is either any tier of the other type or any other tier of the same ++ * type. ++ * ++ * The error is the difference between the SP and the PV; the correction is to ++ * turn off protection when SP>PV or turn on protection when SPprimary, NULL); -+ RCU_INIT_POINTER(cs->secondary, NULL); -+ mutex_init(&cs->mutex); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int hist = lru_hist_from_seq(lrugen->min_seq[type]); ++ ++ pos->refaulted = lrugen->avg_refaulted[type][tier] + ++ atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ pos->total = lrugen->avg_total[type][tier] + ++ atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ pos->total += lrugen->protected[hist][type][tier - 1]; ++ pos->gain = gain; +} + -+static inline struct ovpn_crypto_key_slot * -+ovpn_crypto_key_id_to_slot(const struct ovpn_crypto_state *cs, u8 key_id) ++static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) +{ -+ struct ovpn_crypto_key_slot *ks; ++ int hist, tier; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; ++ unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; + -+ if (unlikely(!cs)) -+ return NULL; ++ lockdep_assert_held(&lruvec->lru_lock); + -+ rcu_read_lock(); -+ ks = rcu_dereference(cs->primary); -+ if (ks && ks->key_id == key_id) { -+ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ goto out; -+ } ++ if (!carryover && !clear) ++ return; + -+ ks = rcu_dereference(cs->secondary); -+ if (ks && ks->key_id == key_id) { -+ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ goto out; -+ } ++ hist = lru_hist_from_seq(seq); + -+ /* when both key slots are occupied but no matching key ID is found, ks has to be reset to -+ * NULL to avoid carrying a stale pointer -+ */ -+ ks = NULL; -+out: -+ rcu_read_unlock(); ++ for (tier = 0; tier < MAX_NR_TIERS; tier++) { ++ if (carryover) { ++ unsigned long sum; + -+ return ks; -+} ++ sum = lrugen->avg_refaulted[type][tier] + ++ atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); + -+static inline struct ovpn_crypto_key_slot * -+ovpn_crypto_key_slot_primary(const struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; ++ sum = lrugen->avg_total[type][tier] + ++ atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ sum += lrugen->protected[hist][type][tier - 1]; ++ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); ++ } + -+ rcu_read_lock(); -+ ks = rcu_dereference(cs->primary); -+ if (unlikely(ks && !ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ rcu_read_unlock(); ++ if (clear) { ++ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); ++ atomic_long_set(&lrugen->evicted[hist][type][tier], 0); ++ if (tier) ++ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); ++ } ++ } ++} + -+ return ks; ++static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ++{ ++ /* ++ * Return true if the PV has a limited number of refaults or a lower ++ * refaulted/total than the SP. ++ */ ++ return pv->refaulted < MIN_LRU_BATCH || ++ pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= ++ (sp->refaulted + 1) * pv->total * pv->gain; +} + -+void ovpn_crypto_key_slot_release(struct kref *kref); ++/****************************************************************************** ++ * the aging ++ ******************************************************************************/ + -+static inline void ovpn_crypto_key_slot_put(struct ovpn_crypto_key_slot *ks) ++/* promote pages accessed through page tables */ ++static int folio_update_gen(struct folio *folio, int gen) +{ -+ kref_put(&ks->refcount, ovpn_crypto_key_slot_release); -+} ++ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + -+int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, -+ const struct ovpn_peer_key_reset *pkr); ++ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(!rcu_read_lock_held()); + -+void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, -+ enum ovpn_key_slot slot); ++ do { ++ /* lru_gen_del_folio() has isolated this page? */ ++ if (!(old_flags & LRU_GEN_MASK)) { ++ /* for shrink_page_list() */ ++ new_flags = old_flags | BIT(PG_referenced); ++ continue; ++ } + -+void ovpn_crypto_state_release(struct ovpn_crypto_state *cs); ++ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); ++ new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; ++ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); + -+void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs); ++ return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++} + -+void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs); ++/* protect pages accessed multiple times through file descriptors */ ++static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++{ ++ int type = folio_is_file_lru(folio); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + -+#endif /* _NET_OVPN_DCO_OVPNCRYPTO_H_ */ -diff --git a/drivers/net/ovpn-dco/crypto_aead.c b/drivers/net/ovpn-dco/crypto_aead.c -new file mode 100644 -index 000000000000..c21bff90d748 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto_aead.c -@@ -0,0 +1,367 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++ VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); + -+#include "crypto_aead.h" -+#include "crypto.h" -+#include "pktid.h" -+#include "proto.h" -+#include "skb.h" ++ do { ++ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ /* folio_update_gen() has promoted this page? */ ++ if (new_gen >= 0 && new_gen != old_gen) ++ return new_gen; + -+#include -+#include -+#include ++ new_gen = (old_gen + 1) % MAX_NR_GENS; + -+#define AUTH_TAG_SIZE 16 ++ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); ++ new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; ++ /* for folio_end_writeback() */ ++ if (reclaiming) ++ new_flags |= BIT(PG_reclaim); ++ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); + -+static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) -+{ -+ return OVPN_OP_SIZE_V2 + /* OP header size */ -+ 4 + /* Packet ID */ -+ crypto_aead_authsize(ks->encrypt); /* Auth Tag */ ++ lru_gen_update_size(lruvec, folio, old_gen, new_gen); ++ ++ return new_gen; +} + -+int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id) ++static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, ++ int old_gen, int new_gen) +{ -+ const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); -+ const unsigned int head_size = ovpn_aead_encap_overhead(ks); -+ struct scatterlist sg[MAX_SKB_FRAGS + 2]; -+ DECLARE_CRYPTO_WAIT(wait); -+ struct aead_request *req; -+ struct sk_buff *trailer; -+ u8 iv[NONCE_SIZE]; -+ int nfrags, ret; -+ u32 pktid, op; ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ int delta = folio_nr_pages(folio); + -+ /* Sample AEAD header format: -+ * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... -+ * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] -+ * [4-byte -+ * IV head] -+ */ ++ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); ++ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); + -+ /* check that there's enough headroom in the skb for packet -+ * encapsulation, after adding network header and encryption overhead -+ */ -+ if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM + head_size))) -+ return -ENOBUFS; ++ walk->batched++; + -+ /* get number of skb frags and ensure that packet data is writable */ -+ nfrags = skb_cow_data(skb, 0, &trailer); -+ if (unlikely(nfrags < 0)) -+ return nfrags; ++ walk->nr_pages[old_gen][type][zone] -= delta; ++ walk->nr_pages[new_gen][type][zone] += delta; ++} + -+ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) -+ return -ENOSPC; ++static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) ++{ ++ int gen, type, zone; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ req = aead_request_alloc(ks->encrypt, GFP_KERNEL); -+ if (unlikely(!req)) -+ return -ENOMEM; ++ walk->batched = 0; + -+ /* sg table: -+ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), -+ * 1, 2, 3, ..., n: payload, -+ * n+1: auth_tag (len=tag_size) -+ */ -+ sg_init_table(sg, nfrags + 2); ++ for_each_gen_type_zone(gen, type, zone) { ++ enum lru_list lru = type * LRU_INACTIVE_FILE; ++ int delta = walk->nr_pages[gen][type][zone]; + -+ /* build scatterlist to encrypt packet payload */ -+ ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); -+ if (unlikely(nfrags != ret)) { -+ ret = -EINVAL; -+ goto free_req; ++ if (!delta) ++ continue; ++ ++ walk->nr_pages[gen][type][zone] = 0; ++ WRITE_ONCE(lrugen->nr_pages[gen][type][zone], ++ lrugen->nr_pages[gen][type][zone] + delta); ++ ++ if (lru_gen_is_active(lruvec, gen)) ++ lru += LRU_ACTIVE; ++ __update_lru_size(lruvec, lru, zone, delta); + } ++} + -+ /* append auth_tag onto scatterlist */ -+ __skb_push(skb, tag_size); -+ sg_set_buf(sg + nfrags + 1, skb->data, tag_size); ++static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) ++{ ++ struct address_space *mapping; ++ struct vm_area_struct *vma = args->vma; ++ struct lru_gen_mm_walk *walk = args->private; + -+ /* obtain packet ID, which is used both as a first -+ * 4 bytes of nonce and last 4 bytes of associated data. -+ */ -+ ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); -+ if (unlikely(ret < 0)) -+ goto free_req; ++ if (!vma_is_accessible(vma)) ++ return true; + -+ /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes nonce */ -+ ovpn_pktid_aead_write(pktid, &ks->nonce_tail_xmit, iv); ++ if (is_vm_hugetlb_page(vma)) ++ return true; + -+ /* make space for packet id and push it to the front */ -+ __skb_push(skb, NONCE_WIRE_SIZE); -+ memcpy(skb->data, iv, NONCE_WIRE_SIZE); ++ if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) ++ return true; + -+ /* add packet op as head of additional data */ -+ op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer_id); -+ __skb_push(skb, OVPN_OP_SIZE_V2); -+ BUILD_BUG_ON(sizeof(op) != OVPN_OP_SIZE_V2); -+ *((__force __be32 *)skb->data) = htonl(op); ++ if (vma == get_gate_vma(vma->vm_mm)) ++ return true; + -+ /* AEAD Additional data */ -+ sg_set_buf(sg, skb->data, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); ++ if (vma_is_anonymous(vma)) ++ return !walk->can_swap; + -+ /* setup async crypto operation */ -+ aead_request_set_tfm(req, ks->encrypt); -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP, -+ crypto_req_done, &wait); -+ aead_request_set_crypt(req, sg, sg, skb->len - head_size, iv); -+ aead_request_set_ad(req, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); ++ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) ++ return true; + -+ /* encrypt it */ -+ ret = crypto_wait_req(crypto_aead_encrypt(req), &wait); -+ if (ret < 0) -+ net_err_ratelimited("%s: encrypt failed: %d\n", __func__, ret); ++ mapping = vma->vm_file->f_mapping; ++ if (mapping_unevictable(mapping)) ++ return true; + -+free_req: -+ aead_request_free(req); -+ return ret; ++ if (shmem_mapping(mapping)) ++ return !walk->can_swap; ++ ++ /* to exclude special mappings like dax, etc. */ ++ return !mapping->a_ops->read_folio; +} + -+int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) ++/* ++ * Some userspace memory allocators map many single-page VMAs. Instead of ++ * returning back to the PGD table for each of such VMAs, finish an entire PMD ++ * table to reduce zigzags and improve cache performance. ++ */ ++static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, ++ unsigned long *vm_start, unsigned long *vm_end) +{ -+ const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); -+ struct scatterlist sg[MAX_SKB_FRAGS + 2]; -+ int ret, payload_len, nfrags; -+ u8 *sg_data, iv[NONCE_SIZE]; -+ unsigned int payload_offset; -+ DECLARE_CRYPTO_WAIT(wait); -+ struct aead_request *req; -+ struct sk_buff *trailer; -+ unsigned int sg_len; -+ __be32 *pid; ++ unsigned long start = round_up(*vm_end, size); ++ unsigned long end = (start | ~mask) + 1; + -+ payload_offset = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE + tag_size; -+ payload_len = skb->len - payload_offset; ++ VM_WARN_ON_ONCE(mask & size); ++ VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); + -+ /* sanity check on packet size, payload size must be >= 0 */ -+ if (unlikely(payload_len < 0)) -+ return -EINVAL; ++ while (args->vma) { ++ if (start >= args->vma->vm_end) { ++ args->vma = args->vma->vm_next; ++ continue; ++ } ++ ++ if (end && end <= args->vma->vm_start) ++ return false; ++ ++ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { ++ args->vma = args->vma->vm_next; ++ continue; ++ } + -+ /* Prepare the skb data buffer to be accessed up until the auth tag. -+ * This is required because this area is directly mapped into the sg list. -+ */ -+ if (unlikely(!pskb_may_pull(skb, payload_offset))) -+ return -ENODATA; ++ *vm_start = max(start, args->vma->vm_start); ++ *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; + -+ /* get number of skb frags and ensure that packet data is writable */ -+ nfrags = skb_cow_data(skb, 0, &trailer); -+ if (unlikely(nfrags < 0)) -+ return nfrags; ++ return true; ++ } + -+ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) -+ return -ENOSPC; ++ return false; ++} + -+ req = aead_request_alloc(ks->decrypt, GFP_KERNEL); -+ if (unlikely(!req)) -+ return -ENOMEM; ++static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) ++{ ++ unsigned long pfn = pte_pfn(pte); + -+ /* sg table: -+ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), -+ * 1, 2, 3, ..., n: payload, -+ * n+1: auth_tag (len=tag_size) -+ */ -+ sg_init_table(sg, nfrags + 2); ++ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + -+ /* packet op is head of additional data */ -+ sg_data = skb->data; -+ sg_len = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE; -+ sg_set_buf(sg, sg_data, sg_len); ++ if (!pte_present(pte) || is_zero_pfn(pfn)) ++ return -1; + -+ /* build scatterlist to decrypt packet payload */ -+ ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); -+ if (unlikely(nfrags != ret)) { -+ ret = -EINVAL; -+ goto free_req; -+ } ++ if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) ++ return -1; + -+ /* append auth_tag onto scatterlist */ -+ sg_set_buf(sg + nfrags + 1, skb->data + sg_len, tag_size); ++ if (WARN_ON_ONCE(!pfn_valid(pfn))) ++ return -1; + -+ /* copy nonce into IV buffer */ -+ memcpy(iv, skb->data + OVPN_OP_SIZE_V2, NONCE_WIRE_SIZE); -+ memcpy(iv + NONCE_WIRE_SIZE, ks->nonce_tail_recv.u8, -+ sizeof(struct ovpn_nonce_tail)); ++ return pfn; ++} + -+ /* setup async crypto operation */ -+ aead_request_set_tfm(req, ks->decrypt); -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP, -+ crypto_req_done, &wait); -+ aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) ++static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) ++{ ++ unsigned long pfn = pmd_pfn(pmd); + -+ aead_request_set_ad(req, NONCE_WIRE_SIZE + OVPN_OP_SIZE_V2); ++ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + -+ /* decrypt it */ -+ ret = crypto_wait_req(crypto_aead_decrypt(req), &wait); -+ if (ret < 0) { -+ net_err_ratelimited("%s: decrypt failed: %d\n", __func__, ret); -+ goto free_req; -+ } ++ if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) ++ return -1; + -+ /* PID sits after the op */ -+ pid = (__force __be32 *)(skb->data + OVPN_OP_SIZE_V2); -+ ret = ovpn_pktid_recv(&ks->pid_recv, ntohl(*pid), 0); -+ if (unlikely(ret < 0)) -+ goto free_req; ++ if (WARN_ON_ONCE(pmd_devmap(pmd))) ++ return -1; + -+ /* point to encapsulated IP packet */ -+ __skb_pull(skb, payload_offset); ++ if (WARN_ON_ONCE(!pfn_valid(pfn))) ++ return -1; + -+free_req: -+ aead_request_free(req); -+ return ret; ++ return pfn; +} ++#endif + -+/* Initialize a struct crypto_aead object */ -+struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, -+ const unsigned char *key, unsigned int keylen) ++static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, ++ struct pglist_data *pgdat, bool can_swap) +{ -+ struct crypto_aead *aead; -+ int ret; -+ -+ aead = crypto_alloc_aead(alg_name, 0, 0); -+ if (IS_ERR(aead)) { -+ ret = PTR_ERR(aead); -+ pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); -+ aead = NULL; -+ goto error; -+ } -+ -+ ret = crypto_aead_setkey(aead, key, keylen); -+ if (ret) { -+ pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, keylen, ret); -+ goto error; -+ } ++ struct folio *folio; + -+ ret = crypto_aead_setauthsize(aead, AUTH_TAG_SIZE); -+ if (ret) { -+ pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, ret); -+ goto error; -+ } ++ /* try to avoid unnecessary memory loads */ ++ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) ++ return NULL; + -+ /* basic AEAD assumption */ -+ if (crypto_aead_ivsize(aead) != NONCE_SIZE) { -+ pr_err("%s IV size must be %d\n", title, NONCE_SIZE); -+ ret = -EINVAL; -+ goto error; -+ } ++ folio = pfn_folio(pfn); ++ if (folio_nid(folio) != pgdat->node_id) ++ return NULL; + -+ pr_debug("********* Cipher %s (%s)\n", alg_name, title); -+ pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); -+ pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); -+ pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); -+ pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); -+ pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); ++ if (folio_memcg_rcu(folio) != memcg) ++ return NULL; + -+ return aead; ++ /* file VMAs can contain anon pages from COW */ ++ if (!folio_is_file_lru(folio) && !can_swap) ++ return NULL; + -+error: -+ crypto_free_aead(aead); -+ return ERR_PTR(ret); ++ return folio; +} + -+void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks) ++static bool suitable_to_scan(int total, int young) +{ -+ if (!ks) -+ return; ++ int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); + -+ crypto_free_aead(ks->encrypt); -+ crypto_free_aead(ks->decrypt); -+ kfree(ks); ++ /* suitable if the average number of young PTEs per cacheline is >=1 */ ++ return young * n >= total; +} + -+static struct ovpn_crypto_key_slot * -+ovpn_aead_crypto_key_slot_init(enum ovpn_cipher_alg alg, -+ const unsigned char *encrypt_key, -+ unsigned int encrypt_keylen, -+ const unsigned char *decrypt_key, -+ unsigned int decrypt_keylen, -+ const unsigned char *encrypt_nonce_tail, -+ unsigned int encrypt_nonce_tail_len, -+ const unsigned char *decrypt_nonce_tail, -+ unsigned int decrypt_nonce_tail_len, -+ u16 key_id) ++static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, ++ struct mm_walk *args) +{ -+ struct ovpn_crypto_key_slot *ks = NULL; -+ const char *alg_name; -+ int ret; ++ int i; ++ pte_t *pte; ++ spinlock_t *ptl; ++ unsigned long addr; ++ int total = 0; ++ int young = 0; ++ struct lru_gen_mm_walk *walk = args->private; ++ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + -+ /* validate crypto alg */ -+ switch (alg) { -+ case OVPN_CIPHER_ALG_AES_GCM: -+ alg_name = "gcm(aes)"; -+ break; -+ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: -+ alg_name = "rfc7539(chacha20,poly1305)"; -+ break; -+ default: -+ return ERR_PTR(-EOPNOTSUPP); -+ } ++ VM_WARN_ON_ONCE(pmd_leaf(*pmd)); + -+ /* build the key slot */ -+ ks = kmalloc(sizeof(*ks), GFP_KERNEL); -+ if (!ks) -+ return ERR_PTR(-ENOMEM); ++ ptl = pte_lockptr(args->mm, pmd); ++ if (!spin_trylock(ptl)) ++ return false; + -+ ks->encrypt = NULL; -+ ks->decrypt = NULL; -+ kref_init(&ks->refcount); -+ ks->key_id = key_id; ++ arch_enter_lazy_mmu_mode(); + -+ ks->encrypt = ovpn_aead_init("encrypt", alg_name, encrypt_key, -+ encrypt_keylen); -+ if (IS_ERR(ks->encrypt)) { -+ ret = PTR_ERR(ks->encrypt); -+ ks->encrypt = NULL; -+ goto destroy_ks; -+ } ++ pte = pte_offset_map(pmd, start & PMD_MASK); ++restart: ++ for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { ++ unsigned long pfn; ++ struct folio *folio; + -+ ks->decrypt = ovpn_aead_init("decrypt", alg_name, decrypt_key, -+ decrypt_keylen); -+ if (IS_ERR(ks->decrypt)) { -+ ret = PTR_ERR(ks->decrypt); -+ ks->decrypt = NULL; -+ goto destroy_ks; -+ } ++ total++; ++ walk->mm_stats[MM_LEAF_TOTAL]++; + -+ if (sizeof(struct ovpn_nonce_tail) != encrypt_nonce_tail_len || -+ sizeof(struct ovpn_nonce_tail) != decrypt_nonce_tail_len) { -+ ret = -EINVAL; -+ goto destroy_ks; -+ } ++ pfn = get_pte_pfn(pte[i], args->vma, addr); ++ if (pfn == -1) ++ continue; + -+ memcpy(ks->nonce_tail_xmit.u8, encrypt_nonce_tail, -+ sizeof(struct ovpn_nonce_tail)); -+ memcpy(ks->nonce_tail_recv.u8, decrypt_nonce_tail, -+ sizeof(struct ovpn_nonce_tail)); ++ if (!pte_young(pte[i])) { ++ walk->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } + -+ /* init packet ID generation/validation */ -+ ovpn_pktid_xmit_init(&ks->pid_xmit); -+ ovpn_pktid_recv_init(&ks->pid_recv); ++ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); ++ if (!folio) ++ continue; + -+ return ks; ++ if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) ++ VM_WARN_ON_ONCE(true); + -+destroy_ks: -+ ovpn_aead_crypto_key_slot_destroy(ks); -+ return ERR_PTR(ret); -+} ++ young++; ++ walk->mm_stats[MM_LEAF_YOUNG]++; + -+struct ovpn_crypto_key_slot * -+ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) -+{ -+ return ovpn_aead_crypto_key_slot_init(kc->cipher_alg, -+ kc->encrypt.cipher_key, -+ kc->encrypt.cipher_key_size, -+ kc->decrypt.cipher_key, -+ kc->decrypt.cipher_key_size, -+ kc->encrypt.nonce_tail, -+ kc->encrypt.nonce_tail_size, -+ kc->decrypt.nonce_tail, -+ kc->decrypt.nonce_tail_size, -+ kc->key_id); -+} -diff --git a/drivers/net/ovpn-dco/crypto_aead.h b/drivers/net/ovpn-dco/crypto_aead.h -new file mode 100644 -index 000000000000..1e3054e7d5a4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto_aead.h -@@ -0,0 +1,27 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && ++ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && ++ !folio_test_swapcache(folio))) ++ folio_mark_dirty(folio); + -+#ifndef _NET_OVPN_DCO_OVPNAEAD_H_ -+#define _NET_OVPN_DCO_OVPNAEAD_H_ ++ old_gen = folio_update_gen(folio, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ update_batch_size(walk, folio, old_gen, new_gen); ++ } + -+#include "crypto.h" ++ if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) ++ goto restart; + -+#include -+#include ++ pte_unmap(pte); + -+struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, -+ const unsigned char *key, unsigned int keylen); ++ arch_leave_lazy_mmu_mode(); ++ spin_unlock(ptl); + -+int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id); -+int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); ++ return suitable_to_scan(total, young); ++} + -+struct ovpn_crypto_key_slot *ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc); -+void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks); ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) ++static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, ++ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) ++{ ++ int i; ++ pmd_t *pmd; ++ spinlock_t *ptl; ++ struct lru_gen_mm_walk *walk = args->private; ++ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + -+#endif /* _NET_OVPN_DCO_OVPNAEAD_H_ */ -diff --git a/drivers/net/ovpn-dco/main.c b/drivers/net/ovpn-dco/main.c -new file mode 100644 -index 000000000000..4eb90ea7a500 ---- /dev/null -+++ b/drivers/net/ovpn-dco/main.c -@@ -0,0 +1,271 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ ++ VM_WARN_ON_ONCE(pud_leaf(*pud)); + -+#include "main.h" ++ /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ ++ if (*start == -1) { ++ *start = next; ++ return; ++ } + -+#include "ovpn.h" -+#include "ovpnstruct.h" -+#include "netlink.h" ++ i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); ++ if (i && i <= MIN_LRU_BATCH) { ++ __set_bit(i - 1, bitmap); ++ return; ++ } + -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++ pmd = pmd_offset(pud, *start); + -+#include ++ ptl = pmd_lockptr(args->mm, pmd); ++ if (!spin_trylock(ptl)) ++ goto done; + -+/* Driver info */ -+#define DRV_NAME "ovpn-dco" -+#define DRV_VERSION OVPN_DCO_VERSION -+#define DRV_DESCRIPTION "OpenVPN data channel offload (ovpn-dco)" -+#define DRV_COPYRIGHT "(C) 2020-2022 OpenVPN, Inc." ++ arch_enter_lazy_mmu_mode(); + -+static void ovpn_struct_free(struct net_device *net) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(net); ++ do { ++ unsigned long pfn; ++ struct folio *folio; ++ unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; + -+ security_tun_dev_free_security(ovpn->security); -+ free_percpu(net->tstats); -+ flush_workqueue(ovpn->crypto_wq); -+ flush_workqueue(ovpn->events_wq); -+ destroy_workqueue(ovpn->crypto_wq); -+ destroy_workqueue(ovpn->events_wq); -+ rcu_barrier(); -+} ++ pfn = get_pmd_pfn(pmd[i], vma, addr); ++ if (pfn == -1) ++ goto next; + -+/* Net device open */ -+static int ovpn_net_open(struct net_device *dev) -+{ -+ struct in_device *dev_v4 = __in_dev_get_rtnl(dev); ++ if (!pmd_trans_huge(pmd[i])) { ++ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && ++ get_cap(LRU_GEN_NONLEAF_YOUNG)) ++ pmdp_test_and_clear_young(vma, addr, pmd + i); ++ goto next; ++ } + -+ if (dev_v4) { -+ /* disable redirects as Linux gets confused by ovpn-dco handling same-LAN routing */ -+ IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); -+ IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; -+ } ++ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); ++ if (!folio) ++ goto next; + -+ netif_tx_start_all_queues(dev); -+ return 0; -+} ++ if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) ++ goto next; + -+/* Net device stop -- called prior to device unload */ -+static int ovpn_net_stop(struct net_device *dev) -+{ -+ netif_tx_stop_all_queues(dev); -+ return 0; -+} ++ walk->mm_stats[MM_LEAF_YOUNG]++; + -+/******************************************* -+ * ovpn ethtool ops -+ *******************************************/ ++ if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && ++ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && ++ !folio_test_swapcache(folio))) ++ folio_mark_dirty(folio); + -+static int ovpn_get_link_ksettings(struct net_device *dev, -+ struct ethtool_link_ksettings *cmd) -+{ -+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 0); -+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, 0); -+ cmd->base.speed = SPEED_1000; -+ cmd->base.duplex = DUPLEX_FULL; -+ cmd->base.port = PORT_TP; -+ cmd->base.phy_address = 0; -+ cmd->base.transceiver = XCVR_INTERNAL; -+ cmd->base.autoneg = AUTONEG_DISABLE; ++ old_gen = folio_update_gen(folio, new_gen); ++ if (old_gen >= 0 && old_gen != new_gen) ++ update_batch_size(walk, folio, old_gen, new_gen); ++next: ++ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; ++ } while (i <= MIN_LRU_BATCH); + -+ return 0; ++ arch_leave_lazy_mmu_mode(); ++ spin_unlock(ptl); ++done: ++ *start = -1; ++ bitmap_zero(bitmap, MIN_LRU_BATCH); +} -+ -+static void ovpn_get_drvinfo(struct net_device *dev, -+ struct ethtool_drvinfo *info) ++#else ++static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, ++ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) +{ -+ strscpy(info->driver, DRV_NAME, sizeof(info->driver)); -+ strscpy(info->version, DRV_VERSION, sizeof(info->version)); -+ strscpy(info->bus_info, "ovpn", sizeof(info->bus_info)); +} ++#endif + -+bool ovpn_dev_is_valid(const struct net_device *dev) ++static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, ++ struct mm_walk *args) +{ -+ return dev->netdev_ops->ndo_start_xmit == ovpn_net_xmit; -+} -+ -+/******************************************* -+ * ovpn exported methods -+ *******************************************/ ++ int i; ++ pmd_t *pmd; ++ unsigned long next; ++ unsigned long addr; ++ struct vm_area_struct *vma; ++ unsigned long pos = -1; ++ struct lru_gen_mm_walk *walk = args->private; ++ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; + -+static const struct net_device_ops ovpn_netdev_ops = { -+ .ndo_open = ovpn_net_open, -+ .ndo_stop = ovpn_net_stop, -+ .ndo_start_xmit = ovpn_net_xmit, -+ .ndo_get_stats64 = dev_get_tstats64, -+}; ++ VM_WARN_ON_ONCE(pud_leaf(*pud)); + -+static const struct ethtool_ops ovpn_ethtool_ops = { -+ .get_link_ksettings = ovpn_get_link_ksettings, -+ .get_drvinfo = ovpn_get_drvinfo, -+ .get_link = ethtool_op_get_link, -+ .get_ts_info = ethtool_op_get_ts_info, -+}; ++ /* ++ * Finish an entire PMD in two passes: the first only reaches to PTE ++ * tables to avoid taking the PMD lock; the second, if necessary, takes ++ * the PMD lock to clear the accessed bit in PMD entries. ++ */ ++ pmd = pmd_offset(pud, start & PUD_MASK); ++restart: ++ /* walk_pte_range() may call get_next_vma() */ ++ vma = args->vma; ++ for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { ++ pmd_t val = pmd_read_atomic(pmd + i); + -+static void ovpn_setup(struct net_device *dev) -+{ -+ /* compute the overhead considering AEAD encryption */ -+ const int overhead = sizeof(u32) + NONCE_WIRE_SIZE + 16 + sizeof(struct udphdr) + -+ max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); ++ /* for pmd_read_atomic() */ ++ barrier(); + -+ netdev_features_t feat = NETIF_F_SG | NETIF_F_LLTX | -+ NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_GSO | -+ NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA; ++ next = pmd_addr_end(addr, end); + -+ dev->ethtool_ops = &ovpn_ethtool_ops; -+ dev->needs_free_netdev = true; ++ if (!pmd_present(val) || is_huge_zero_pmd(val)) { ++ walk->mm_stats[MM_LEAF_TOTAL]++; ++ continue; ++ } + -+ dev->netdev_ops = &ovpn_netdev_ops; ++#ifdef CONFIG_TRANSPARENT_HUGEPAGE ++ if (pmd_trans_huge(val)) { ++ unsigned long pfn = pmd_pfn(val); ++ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + -+ dev->priv_destructor = ovpn_struct_free; ++ walk->mm_stats[MM_LEAF_TOTAL]++; + -+ /* Point-to-Point TUN Device */ -+ dev->hard_header_len = 0; -+ dev->addr_len = 0; -+ dev->mtu = ETH_DATA_LEN - overhead; -+ dev->min_mtu = IPV4_MIN_MTU; -+ dev->max_mtu = IP_MAX_MTU - overhead; ++ if (!pmd_young(val)) { ++ walk->mm_stats[MM_LEAF_OLD]++; ++ continue; ++ } + -+ /* Zero header length */ -+ dev->type = ARPHRD_NONE; -+ dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; ++ /* try to avoid unnecessary memory loads */ ++ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) ++ continue; + -+ dev->features |= feat; -+ dev->hw_features |= feat; -+ dev->hw_enc_features |= feat; ++ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); ++ continue; ++ } ++#endif ++ walk->mm_stats[MM_NONLEAF_TOTAL]++; + -+ dev->needed_headroom = OVPN_HEAD_ROOM; -+ dev->needed_tailroom = OVPN_MAX_PADDING; -+} ++#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG ++ if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { ++ if (!pmd_young(val)) ++ continue; + -+static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = { -+ [IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, __OVPN_MODE_FIRST, -+ __OVPN_MODE_AFTER_LAST - 1), -+}; ++ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); ++ } ++#endif ++ if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) ++ continue; + -+static int ovpn_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], -+ struct nlattr *data[], struct netlink_ext_ack *extack) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ int ret; ++ walk->mm_stats[MM_NONLEAF_FOUND]++; + -+ ret = security_tun_dev_create(); -+ if (ret < 0) -+ return ret; ++ if (!walk_pte_range(&val, addr, next, args)) ++ continue; + -+ ret = ovpn_struct_init(dev); -+ if (ret < 0) -+ return ret; ++ walk->mm_stats[MM_NONLEAF_ADDED]++; + -+ ovpn->mode = OVPN_MODE_P2P; -+ if (data && data[IFLA_OVPN_MODE]) { -+ ovpn->mode = nla_get_u8(data[IFLA_OVPN_MODE]); -+ netdev_dbg(dev, "%s: setting device (%s) mode: %u\n", __func__, dev->name, -+ ovpn->mode); ++ /* carry over to the next generation */ ++ update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); + } + -+ return register_netdevice(dev); ++ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); ++ ++ if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) ++ goto restart; +} + -+static void ovpn_dellink(struct net_device *dev, struct list_head *head) ++static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, ++ struct mm_walk *args) +{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); ++ int i; ++ pud_t *pud; ++ unsigned long addr; ++ unsigned long next; ++ struct lru_gen_mm_walk *walk = args->private; + -+ switch (ovpn->mode) { -+ case OVPN_MODE_P2P: -+ ovpn_peer_release_p2p(ovpn); -+ break; -+ default: -+ ovpn_peers_free(ovpn); -+ break; -+ } ++ VM_WARN_ON_ONCE(p4d_leaf(*p4d)); + -+ unregister_netdevice_queue(dev, head); -+} ++ pud = pud_offset(p4d, start & P4D_MASK); ++restart: ++ for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { ++ pud_t val = READ_ONCE(pud[i]); + -+/** -+ * ovpn_num_queues - define number of queues to allocate per device -+ * -+ * The value returned by this function is used to decide how many RX and TX -+ * queues to allocate when creating the netdev object -+ * -+ * Return the number of queues to allocate -+ */ -+static unsigned int ovpn_num_queues(void) -+{ -+ return num_online_cpus(); -+} ++ next = pud_addr_end(addr, end); + -+static struct rtnl_link_ops ovpn_link_ops __read_mostly = { -+ .kind = DRV_NAME, -+ .priv_size = sizeof(struct ovpn_struct), -+ .setup = ovpn_setup, -+ .policy = ovpn_policy, -+ .maxtype = IFLA_OVPN_MAX, -+ .newlink = ovpn_newlink, -+ .dellink = ovpn_dellink, -+ .get_num_tx_queues = ovpn_num_queues, -+ .get_num_rx_queues = ovpn_num_queues, -+}; ++ if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) ++ continue; + -+static int __init ovpn_init(void) -+{ -+ int err = 0; ++ walk_pmd_range(&val, addr, next, args); + -+ pr_info("%s %s -- %s\n", DRV_DESCRIPTION, DRV_VERSION, DRV_COPYRIGHT); ++ /* a racy check to curtail the waiting time */ ++ if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) ++ return 1; + -+ /* init RTNL link ops */ -+ err = rtnl_link_register(&ovpn_link_ops); -+ if (err) { -+ pr_err("ovpn: can't register RTNL link ops\n"); -+ goto err; ++ if (need_resched() || walk->batched >= MAX_LRU_BATCH) { ++ end = (addr | ~PUD_MASK) + 1; ++ goto done; ++ } + } + -+ err = ovpn_netlink_register(); -+ if (err) { -+ pr_err("ovpn: can't register netlink family\n"); -+ goto err_rtnl_unregister; -+ } ++ if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) ++ goto restart; + -+ return 0; ++ end = round_up(end, P4D_SIZE); ++done: ++ if (!end || !args->vma) ++ return 1; + -+err_rtnl_unregister: -+ rtnl_link_unregister(&ovpn_link_ops); -+err: -+ pr_err("ovpn: initialization failed, error status=%d\n", err); -+ return err; -+} ++ walk->next_addr = max(end, args->vma->vm_start); + -+static __exit void ovpn_cleanup(void) -+{ -+ rtnl_link_unregister(&ovpn_link_ops); -+ ovpn_netlink_unregister(); -+ rcu_barrier(); /* because we use call_rcu */ ++ return -EAGAIN; +} + -+module_init(ovpn_init); -+module_exit(ovpn_cleanup); ++static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) ++{ ++ static const struct mm_walk_ops mm_walk_ops = { ++ .test_walk = should_skip_vma, ++ .p4d_entry = walk_pud_range, ++ }; + -+MODULE_DESCRIPTION(DRV_DESCRIPTION); -+MODULE_AUTHOR(DRV_COPYRIGHT); -+MODULE_LICENSE("GPL"); -+MODULE_VERSION(DRV_VERSION); -+MODULE_ALIAS_RTNL_LINK(DRV_NAME); -+MODULE_ALIAS_GENL_FAMILY(OVPN_NL_NAME); -diff --git a/drivers/net/ovpn-dco/main.h b/drivers/net/ovpn-dco/main.h -new file mode 100644 -index 000000000000..c4ef200b30f4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/main.h -@@ -0,0 +1,32 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++ int err; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); + -+#ifndef _NET_OVPN_DCO_MAIN_H_ -+#define _NET_OVPN_DCO_MAIN_H_ ++ walk->next_addr = FIRST_USER_ADDRESS; + -+#include -+#include -+#include -+#include ++ do { ++ err = -EBUSY; + -+#define OVPN_DCO_VERSION "2.0.0" ++ /* folio_update_gen() requires stable folio_memcg() */ ++ if (!mem_cgroup_trylock_pages(memcg)) ++ break; + -+struct net_device; -+bool ovpn_dev_is_valid(const struct net_device *dev); ++ /* the caller might be holding the lock for write */ ++ if (mmap_read_trylock(mm)) { ++ err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); + -+#define SKB_HEADER_LEN \ -+ (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ -+ sizeof(struct udphdr) + NET_SKB_PAD) ++ mmap_read_unlock(mm); ++ } + -+#define OVPN_HEAD_ROOM ALIGN(16 + SKB_HEADER_LEN, 4) -+#define OVPN_MAX_PADDING 16 -+#define OVPN_QUEUE_LEN 1024 -+#define OVPN_MAX_TUN_QUEUE_LEN 0x10000 ++ mem_cgroup_unlock_pages(); + -+#endif /* _NET_OVPN_DCO_OVPN_DCO_H_ */ -diff --git a/drivers/net/ovpn-dco/netlink.c b/drivers/net/ovpn-dco/netlink.c -new file mode 100644 -index 000000000000..ee5c943e7db4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/netlink.c -@@ -0,0 +1,1143 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ ++ if (walk->batched) { ++ spin_lock_irq(&lruvec->lru_lock); ++ reset_batch_size(lruvec, walk); ++ spin_unlock_irq(&lruvec->lru_lock); ++ } + -+#include "main.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "proto.h" -+#include "netlink.h" -+#include "ovpnstruct.h" -+#include "udp.h" ++ cond_resched(); ++ } while (err == -EAGAIN); ++} + -+#include ++static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) ++{ ++ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++ if (pgdat && current_is_kswapd()) { ++ VM_WARN_ON_ONCE(walk); + -+/** The ovpn-dco netlink family */ -+static struct genl_family ovpn_netlink_family; ++ walk = &pgdat->mm_walk; ++ } else if (!pgdat && !walk) { ++ VM_WARN_ON_ONCE(current_is_kswapd()); + -+enum ovpn_netlink_multicast_groups { -+ OVPN_MCGRP_PEERS, -+}; ++ walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); ++ } + -+static const struct genl_multicast_group ovpn_netlink_mcgrps[] = { -+ [OVPN_MCGRP_PEERS] = { .name = OVPN_NL_MULTICAST_GROUP_PEERS }, -+}; ++ current->reclaim_state->mm_walk = walk; + -+/** Key direction policy. Can be used for configuring an encryption and a decryption key */ -+static const struct nla_policy ovpn_netlink_policy_key_dir[OVPN_KEY_DIR_ATTR_MAX + 1] = { -+ [OVPN_KEY_DIR_ATTR_CIPHER_KEY] = NLA_POLICY_MAX_LEN(U8_MAX), -+ [OVPN_KEY_DIR_ATTR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(NONCE_TAIL_SIZE), -+}; ++ return walk; ++} + -+/** CMD_NEW_KEY policy */ -+static const struct nla_policy ovpn_netlink_policy_new_key[OVPN_NEW_KEY_ATTR_MAX + 1] = { -+ [OVPN_NEW_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_NEW_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, -+ __OVPN_KEY_SLOT_AFTER_LAST - 1), -+ [OVPN_NEW_KEY_ATTR_KEY_ID] = { .type = NLA_U8 }, -+ [OVPN_NEW_KEY_ATTR_CIPHER_ALG] = { .type = NLA_U16 }, -+ [OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), -+ [OVPN_NEW_KEY_ATTR_DECRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), -+}; ++static void clear_mm_walk(void) ++{ ++ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + -+/** CMD_DEL_KEY policy */ -+static const struct nla_policy ovpn_netlink_policy_del_key[OVPN_DEL_KEY_ATTR_MAX + 1] = { -+ [OVPN_DEL_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_DEL_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, -+ __OVPN_KEY_SLOT_AFTER_LAST - 1), -+}; ++ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); ++ VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); + -+/** CMD_SWAP_KEYS policy */ -+static const struct nla_policy ovpn_netlink_policy_swap_keys[OVPN_SWAP_KEYS_ATTR_MAX + 1] = { -+ [OVPN_SWAP_KEYS_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; ++ current->reclaim_state->mm_walk = NULL; + -+/** CMD_NEW_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_new_peer[OVPN_NEW_PEER_ATTR_MAX + 1] = { -+ [OVPN_NEW_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE] = NLA_POLICY_MIN_LEN(sizeof(struct sockaddr)), -+ [OVPN_NEW_PEER_ATTR_SOCKET] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_IPV4] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_IPV6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), -+ [OVPN_NEW_PEER_ATTR_LOCAL_IP] = NLA_POLICY_MAX_LEN(sizeof(struct in6_addr)), -+}; ++ if (!current_is_kswapd()) ++ kfree(walk); ++} + -+/** CMD_SET_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_set_peer[OVPN_SET_PEER_ATTR_MAX + 1] = { -+ [OVPN_SET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, -+ [OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT] = { .type = NLA_U32 }, -+}; ++static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) ++{ ++ int zone; ++ int remaining = MAX_LRU_BATCH; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + -+/** CMD_DEL_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_del_peer[OVPN_DEL_PEER_ATTR_MAX + 1] = { -+ [OVPN_DEL_PEER_ATTR_REASON] = NLA_POLICY_RANGE(NLA_U8, __OVPN_DEL_PEER_REASON_FIRST, -+ __OVPN_DEL_PEER_REASON_AFTER_LAST - 1), -+ [OVPN_DEL_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; ++ if (type == LRU_GEN_ANON && !can_swap) ++ goto done; + -+/** CMD_GET_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_get_peer[OVPN_GET_PEER_ATTR_MAX + 1] = { -+ [OVPN_GET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; ++ /* prevent cold/hot inversion if force_scan is true */ ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ struct list_head *head = &lrugen->lists[old_gen][type][zone]; + -+/** CMD_PACKET polocy */ -+static const struct nla_policy ovpn_netlink_policy_packet[OVPN_PACKET_ATTR_MAX + 1] = { -+ [OVPN_PACKET_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_PACKET_ATTR_PACKET] = NLA_POLICY_MAX_LEN(U16_MAX), -+}; ++ while (!list_empty(head)) { ++ struct folio *folio = lru_to_folio(head); + -+/** Generic message container policy */ -+static const struct nla_policy ovpn_netlink_policy[OVPN_ATTR_MAX + 1] = { -+ [OVPN_ATTR_IFINDEX] = { .type = NLA_U32 }, -+ [OVPN_ATTR_NEW_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_peer), -+ [OVPN_ATTR_SET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_set_peer), -+ [OVPN_ATTR_DEL_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_peer), -+ [OVPN_ATTR_GET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_get_peer), -+ [OVPN_ATTR_NEW_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_key), -+ [OVPN_ATTR_SWAP_KEYS] = NLA_POLICY_NESTED(ovpn_netlink_policy_swap_keys), -+ [OVPN_ATTR_DEL_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_key), -+ [OVPN_ATTR_PACKET] = NLA_POLICY_NESTED(ovpn_netlink_policy_packet), -+}; ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + -+static struct net_device * -+ovpn_get_dev_from_attrs(struct net *net, struct nlattr **attrs) -+{ -+ struct net_device *dev; -+ int ifindex; ++ new_gen = folio_inc_gen(lruvec, folio, false); ++ list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); + -+ if (!attrs[OVPN_ATTR_IFINDEX]) -+ return ERR_PTR(-EINVAL); ++ if (!--remaining) ++ return false; ++ } ++ } ++done: ++ reset_ctrl_pos(lruvec, type, true); ++ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); + -+ ifindex = nla_get_u32(attrs[OVPN_ATTR_IFINDEX]); ++ return true; ++} + -+ dev = dev_get_by_index(net, ifindex); -+ if (!dev) -+ return ERR_PTR(-ENODEV); ++static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) ++{ ++ int gen, type, zone; ++ bool success = false; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ DEFINE_MIN_SEQ(lruvec); + -+ if (!ovpn_dev_is_valid(dev)) -+ goto err_put_dev; ++ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + -+ return dev; ++ /* find the oldest populated generation */ ++ for (type = !can_swap; type < ANON_AND_FILE; type++) { ++ while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { ++ gen = lru_gen_from_seq(min_seq[type]); + -+err_put_dev: -+ dev_put(dev); ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ if (!list_empty(&lrugen->lists[gen][type][zone])) ++ goto next; ++ } + -+ return ERR_PTR(-EINVAL); -+} ++ min_seq[type]++; ++ } ++next: ++ ; ++ } + -+/** -+ * ovpn_pre_doit() - Prepare ovpn genl doit request -+ * @ops: requested netlink operation -+ * @skb: Netlink message with request data -+ * @info: receiver information -+ * -+ * Return: 0 on success or negative error number in case of failure -+ */ -+static int ovpn_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct net *net = genl_info_net(info); -+ struct net_device *dev; ++ /* see the comment on lru_gen_struct */ ++ if (can_swap) { ++ min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); ++ min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); ++ } + -+ dev = ovpn_get_dev_from_attrs(net, info->attrs); -+ if (IS_ERR(dev)) -+ return PTR_ERR(dev); ++ for (type = !can_swap; type < ANON_AND_FILE; type++) { ++ if (min_seq[type] == lrugen->min_seq[type]) ++ continue; + -+ info->user_ptr[0] = netdev_priv(dev); ++ reset_ctrl_pos(lruvec, type, true); ++ WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); ++ success = true; ++ } + -+ return 0; ++ return success; +} + -+/** -+ * ovpn_post_doit() - complete ovpn genl doit request -+ * @ops: requested netlink operation -+ * @skb: Netlink message with request data -+ * @info: receiver information -+ */ -+static void ovpn_post_doit(const struct genl_ops *ops, struct sk_buff *skb, -+ struct genl_info *info) ++static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) +{ -+ struct ovpn_struct *ovpn; ++ int prev, next; ++ int type, zone; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ ovpn = info->user_ptr[0]; -+ dev_put(ovpn->dev); -+} ++ spin_lock_irq(&lruvec->lru_lock); + -+static int ovpn_netlink_get_key_dir(struct genl_info *info, struct nlattr *key, -+ enum ovpn_cipher_alg cipher, -+ struct ovpn_key_direction *dir) -+{ -+ struct nlattr *attr, *attrs[OVPN_KEY_DIR_ATTR_MAX + 1]; -+ int ret; ++ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + -+ ret = nla_parse_nested(attrs, OVPN_KEY_DIR_ATTR_MAX, key, NULL, info->extack); -+ if (ret) -+ return ret; ++ for (type = ANON_AND_FILE - 1; type >= 0; type--) { ++ if (get_nr_gens(lruvec, type) != MAX_NR_GENS) ++ continue; + -+ switch (cipher) { -+ case OVPN_CIPHER_ALG_AES_GCM: -+ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: -+ attr = attrs[OVPN_KEY_DIR_ATTR_CIPHER_KEY]; -+ if (!attr) -+ return -EINVAL; ++ VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); ++ ++ while (!inc_min_seq(lruvec, type, can_swap)) { ++ spin_unlock_irq(&lruvec->lru_lock); ++ cond_resched(); ++ spin_lock_irq(&lruvec->lru_lock); ++ } ++ } ++ ++ /* ++ * Update the active/inactive LRU sizes for compatibility. Both sides of ++ * the current max_seq need to be covered, since max_seq+1 can overlap ++ * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do ++ * overlap, cold/hot inversion happens. ++ */ ++ prev = lru_gen_from_seq(lrugen->max_seq - 1); ++ next = lru_gen_from_seq(lrugen->max_seq + 1); + -+ dir->cipher_key = nla_data(attr); -+ dir->cipher_key_size = nla_len(attr); ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) { ++ enum lru_list lru = type * LRU_INACTIVE_FILE; ++ long delta = lrugen->nr_pages[prev][type][zone] - ++ lrugen->nr_pages[next][type][zone]; + -+ attr = attrs[OVPN_KEY_DIR_ATTR_NONCE_TAIL]; -+ /* These algorithms require a 96bit nonce, -+ * Construct it by combining 4-bytes packet id and -+ * 8-bytes nonce-tail from userspace -+ */ -+ if (!attr) -+ return -EINVAL; ++ if (!delta) ++ continue; + -+ dir->nonce_tail = nla_data(attr); -+ dir->nonce_tail_size = nla_len(attr); -+ break; -+ default: -+ return -EINVAL; ++ __update_lru_size(lruvec, lru, zone, delta); ++ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); ++ } + } + -+ return 0; ++ for (type = 0; type < ANON_AND_FILE; type++) ++ reset_ctrl_pos(lruvec, type, false); ++ ++ WRITE_ONCE(lrugen->timestamps[next], jiffies); ++ /* make sure preceding modifications appear */ ++ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); ++ ++ spin_unlock_irq(&lruvec->lru_lock); +} + -+static int ovpn_netlink_new_key(struct sk_buff *skb, struct genl_info *info) ++static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, ++ struct scan_control *sc, bool can_swap, bool force_scan) +{ -+ struct nlattr *attrs[OVPN_NEW_KEY_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer_key_reset pkr; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; ++ bool success; ++ struct lru_gen_mm_walk *walk; ++ struct mm_struct *mm = NULL; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ if (!info->attrs[OVPN_ATTR_NEW_KEY]) -+ return -EINVAL; ++ VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); + -+ ret = nla_parse_nested(attrs, OVPN_NEW_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_KEY], -+ NULL, info->extack); -+ if (ret) -+ return ret; ++ /* see the comment in iterate_mm_list() */ ++ if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { ++ success = false; ++ goto done; ++ } + -+ if (!attrs[OVPN_NEW_KEY_ATTR_PEER_ID] || -+ !attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT] || -+ !attrs[OVPN_NEW_KEY_ATTR_KEY_ID] || -+ !attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG] || -+ !attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] || -+ !attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY]) -+ return -EINVAL; ++ /* ++ * If the hardware doesn't automatically set the accessed bit, fallback ++ * to lru_gen_look_around(), which only clears the accessed bit in a ++ * handful of PTEs. Spreading the work out over a period of time usually ++ * is less efficient, but it avoids bursty page faults. ++ */ ++ if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { ++ success = iterate_mm_list_nowalk(lruvec, max_seq); ++ goto done; ++ } + -+ peer_id = nla_get_u32(attrs[OVPN_NEW_KEY_ATTR_PEER_ID]); -+ pkr.slot = nla_get_u8(attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT]); -+ pkr.key.key_id = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_KEY_ID]); ++ walk = set_mm_walk(NULL); ++ if (!walk) { ++ success = iterate_mm_list_nowalk(lruvec, max_seq); ++ goto done; ++ } + -+ pkr.key.cipher_alg = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG]); ++ walk->lruvec = lruvec; ++ walk->max_seq = max_seq; ++ walk->can_swap = can_swap; ++ walk->force_scan = force_scan; + -+ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY], -+ pkr.key.cipher_alg, &pkr.key.encrypt); -+ if (ret < 0) -+ return ret; ++ do { ++ success = iterate_mm_list(lruvec, walk, &mm); ++ if (mm) ++ walk_mm(lruvec, mm, walk); + -+ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY], -+ pkr.key.cipher_alg, &pkr.key.decrypt); -+ if (ret < 0) -+ return ret; ++ cond_resched(); ++ } while (mm); ++done: ++ if (!success) { ++ if (sc->priority <= DEF_PRIORITY - 2) ++ wait_event_killable(lruvec->mm_state.wait, ++ max_seq < READ_ONCE(lrugen->max_seq)); + -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) { -+ netdev_dbg(ovpn->dev, "%s: no peer with id %u to set key for\n", __func__, peer_id); -+ return -ENOENT; ++ return max_seq < READ_ONCE(lrugen->max_seq); + } + -+ mutex_lock(&peer->crypto.mutex); -+ ret = ovpn_crypto_state_reset(&peer->crypto, &pkr); -+ if (ret < 0) { -+ netdev_dbg(ovpn->dev, "%s: cannot install new key for peer %u\n", __func__, -+ peer_id); -+ goto unlock; -+ } ++ VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); + -+ netdev_dbg(ovpn->dev, "%s: new key installed (id=%u) for peer %u\n", __func__, -+ pkr.key.key_id, peer_id); -+unlock: -+ mutex_unlock(&peer->crypto.mutex); -+ ovpn_peer_put(peer); -+ return ret; ++ inc_max_seq(lruvec, can_swap, force_scan); ++ /* either this sees any waiters or they will see updated max_seq */ ++ if (wq_has_sleeper(&lruvec->mm_state.wait)) ++ wake_up_all(&lruvec->mm_state.wait); ++ ++ wakeup_flusher_threads(WB_REASON_VMSCAN); ++ ++ return true; +} + -+static int ovpn_netlink_del_key(struct sk_buff *skb, struct genl_info *info) ++static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, ++ struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) +{ -+ struct nlattr *attrs[OVPN_DEL_KEY_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ enum ovpn_key_slot slot; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; ++ int gen, type, zone; ++ unsigned long old = 0; ++ unsigned long young = 0; ++ unsigned long total = 0; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); + -+ if (!info->attrs[OVPN_ATTR_DEL_KEY]) -+ return -EINVAL; ++ for (type = !can_swap; type < ANON_AND_FILE; type++) { ++ unsigned long seq; + -+ ret = nla_parse_nested(attrs, OVPN_DEL_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_KEY], NULL, -+ info->extack); -+ if (ret) -+ return ret; ++ for (seq = min_seq[type]; seq <= max_seq; seq++) { ++ unsigned long size = 0; + -+ if (!attrs[OVPN_DEL_KEY_ATTR_PEER_ID] || !attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]) -+ return -EINVAL; ++ gen = lru_gen_from_seq(seq); + -+ peer_id = nla_get_u32(attrs[OVPN_DEL_KEY_ATTR_PEER_ID]); -+ slot = nla_get_u8(attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]); ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) ++ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; ++ total += size; ++ if (seq == max_seq) ++ young += size; ++ else if (seq + MIN_NR_GENS == max_seq) ++ old += size; ++ } ++ } + -+ ovpn_crypto_key_slot_delete(&peer->crypto, slot); -+ ovpn_peer_put(peer); ++ /* try to scrape all its memory if this memcg was deleted */ ++ *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; + -+ return 0; ++ /* ++ * The aging tries to be lazy to reduce the overhead, while the eviction ++ * stalls when the number of generations reaches MIN_NR_GENS. Hence, the ++ * ideal number of generations is MIN_NR_GENS+1. ++ */ ++ if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) ++ return true; ++ if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) ++ return false; ++ ++ /* ++ * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) ++ * of the total number of pages for each generation. A reasonable range ++ * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The ++ * aging cares about the upper bound of hot pages, while the eviction ++ * cares about the lower bound of cold pages. ++ */ ++ if (young * MIN_NR_GENS > total) ++ return true; ++ if (old * (MIN_NR_GENS + 2) < total) ++ return true; ++ ++ return false; +} + -+static int ovpn_netlink_swap_keys(struct sk_buff *skb, struct genl_info *info) ++static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) +{ -+ struct nlattr *attrs[OVPN_SWAP_KEYS_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; ++ bool need_aging; ++ unsigned long nr_to_scan; ++ int swappiness = get_swappiness(lruvec, sc); ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); + -+ if (!info->attrs[OVPN_ATTR_SWAP_KEYS]) -+ return -EINVAL; ++ VM_WARN_ON_ONCE(sc->memcg_low_reclaim); + -+ ret = nla_parse_nested(attrs, OVPN_SWAP_KEYS_ATTR_MAX, info->attrs[OVPN_ATTR_SWAP_KEYS], -+ NULL, info->extack); -+ if (ret) -+ return ret; ++ mem_cgroup_calculate_protection(NULL, memcg); + -+ if (!attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]) -+ return -EINVAL; ++ if (mem_cgroup_below_min(memcg)) ++ return false; + -+ peer_id = nla_get_u32(attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]); ++ need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); + -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; ++ if (min_ttl) { ++ int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); ++ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); + -+ ovpn_crypto_key_slots_swap(&peer->crypto); -+ ovpn_peer_put(peer); ++ if (time_is_after_jiffies(birth + min_ttl)) ++ return false; + -+ return 0; ++ /* the size is likely too small to be helpful */ ++ if (!nr_to_scan && sc->priority != DEF_PRIORITY) ++ return false; ++ } ++ ++ if (need_aging) ++ try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); ++ ++ return true; +} + -+static int ovpn_netlink_new_peer(struct sk_buff *skb, struct genl_info *info) ++/* to protect the working set of the last N jiffies */ ++static unsigned long lru_gen_min_ttl __read_mostly; ++ ++static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +{ -+ struct nlattr *attrs[OVPN_NEW_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct sockaddr_storage *ss = NULL; -+ struct sockaddr_in mapped; -+ struct sockaddr_in6 *in6; -+ struct ovpn_peer *peer; -+ size_t sa_len, ip_len; -+ struct socket *sock; -+ u8 *local_ip = NULL; -+ u32 sockfd, id; -+ int ret; ++ struct mem_cgroup *memcg; ++ bool success = false; ++ unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); + -+ if (!info->attrs[OVPN_ATTR_NEW_PEER]) -+ return -EINVAL; ++ VM_WARN_ON_ONCE(!current_is_kswapd()); + -+ ret = nla_parse_nested(attrs, OVPN_NEW_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; ++ sc->last_reclaimed = sc->nr_reclaimed; + -+ if (!attrs[OVPN_NEW_PEER_ATTR_PEER_ID] || !attrs[OVPN_NEW_PEER_ATTR_SOCKET] || -+ (!attrs[OVPN_NEW_PEER_ATTR_IPV4] && !attrs[OVPN_NEW_PEER_ATTR_IPV6])) { -+ netdev_dbg(ovpn->dev, "%s: basic attributes missing\n", __func__); -+ return -EINVAL; ++ /* ++ * To reduce the chance of going into the aging path, which can be ++ * costly, optimistically skip it if the flag below was cleared in the ++ * eviction path. This improves the overall performance when multiple ++ * memcgs are available. ++ */ ++ if (!sc->memcgs_need_aging) { ++ sc->memcgs_need_aging = true; ++ return; + } + -+ /* lookup the fd in the kernel table and extract the socket object */ -+ sockfd = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_SOCKET]); -+ /* sockfd_lookup() increases sock's refcounter */ -+ sock = sockfd_lookup(sockfd, &ret); -+ if (!sock) { -+ netdev_dbg(ovpn->dev, "%s: cannot lookup peer socket (fd=%u): %d\n", __func__, -+ sockfd, ret); -+ return -ENOTSOCK; -+ } ++ set_mm_walk(pgdat); + -+ /* Only when using UDP as transport protocol the remote endpoint must be configured -+ * so that ovpn-dco knows where to send packets to. -+ * -+ * In case of TCP, the socket is connected to the peer and ovpn-dco will just send bytes -+ * over it, without the need to specify a destination. -+ */ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) { -+ ret = -EINVAL; ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + -+ if (!attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]) { -+ netdev_err(ovpn->dev, "%s: cannot add UDP peer with no remote endpoint\n", -+ __func__); -+ goto sockfd_release; -+ } ++ if (age_lruvec(lruvec, sc, min_ttl)) ++ success = true; + -+ ss = nla_data(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); -+ sa_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); -+ switch (sa_len) { -+ case sizeof(struct sockaddr_in): -+ if (ss->ss_family == AF_INET) -+ /* valid sockaddr */ -+ break; ++ cond_resched(); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + -+ netdev_err(ovpn->dev, "%s: remote sockaddr_in has invalid family\n", -+ __func__); -+ goto sockfd_release; -+ case sizeof(struct sockaddr_in6): -+ if (ss->ss_family == AF_INET6) -+ /* valid sockaddr */ -+ break; ++ clear_mm_walk(); + -+ netdev_err(ovpn->dev, "%s: remote sockaddr_in6 has invalid family\n", -+ __func__); -+ goto sockfd_release; -+ default: -+ netdev_err(ovpn->dev, "%s: invalid size for sockaddr\n", __func__); -+ goto sockfd_release; -+ } ++ /* check the order to exclude compaction-induced reclaim */ ++ if (success || !min_ttl || sc->order) ++ return; + -+ if (ss->ss_family == AF_INET6) { -+ in6 = (struct sockaddr_in6 *)ss; ++ /* ++ * The main goal is to OOM kill if every generation from all memcgs is ++ * younger than min_ttl. However, another possibility is all memcgs are ++ * either below min or empty. ++ */ ++ if (mutex_trylock(&oom_lock)) { ++ struct oom_control oc = { ++ .gfp_mask = sc->gfp_mask, ++ }; + -+ if (ipv6_addr_type(&in6->sin6_addr) & IPV6_ADDR_MAPPED) { -+ mapped.sin_family = AF_INET; -+ mapped.sin_addr.s_addr = in6->sin6_addr.s6_addr32[3]; -+ mapped.sin_port = in6->sin6_port; -+ ss = (struct sockaddr_storage *)&mapped; -+ } -+ } ++ out_of_memory(&oc); + -+ /* When using UDP we may be talking over socket bound to 0.0.0.0/::. -+ * In this case, if the host has multiple IPs, we need to make sure -+ * that outgoing traffic has as source IP the same address that the -+ * peer is using to reach us. -+ * -+ * Since early control packets were all forwarded to userspace, we -+ * need the latter to tell us what IP has to be used. -+ */ -+ if (attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]) { -+ ip_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); -+ local_ip = nla_data(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); ++ mutex_unlock(&oom_lock); ++ } ++} + -+ if (ip_len == sizeof(struct in_addr)) { -+ if (ss->ss_family != AF_INET) { -+ netdev_dbg(ovpn->dev, -+ "%s: the specified local IP is IPv4, but the peer endpoint is not\n", -+ __func__); -+ goto sockfd_release; -+ } -+ } else if (ip_len == sizeof(struct in6_addr)) { -+ bool is_mapped = ipv6_addr_type((struct in6_addr *)local_ip) & -+ IPV6_ADDR_MAPPED; ++/* ++ * This function exploits spatial locality when shrink_page_list() walks the ++ * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If ++ * the scan was done cacheline efficiently, it adds the PMD entry pointing to ++ * the PTE table to the Bloom filter. This forms a feedback loop between the ++ * eviction and the aging. ++ */ ++void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) ++{ ++ int i; ++ pte_t *pte; ++ unsigned long start; ++ unsigned long end; ++ unsigned long addr; ++ struct lru_gen_mm_walk *walk; ++ int young = 0; ++ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; ++ struct folio *folio = pfn_folio(pvmw->pfn); ++ struct mem_cgroup *memcg = folio_memcg(folio); ++ struct pglist_data *pgdat = folio_pgdat(folio); ++ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ DEFINE_MAX_SEQ(lruvec); ++ int old_gen, new_gen = lru_gen_from_seq(max_seq); + -+ if (ss->ss_family != AF_INET6 && !is_mapped) { -+ netdev_dbg(ovpn->dev, -+ "%s: the specified local IP is IPv6, but the peer endpoint is not\n", -+ __func__); -+ goto sockfd_release; -+ } ++ lockdep_assert_held(pvmw->ptl); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); + -+ if (is_mapped) -+ /* this is an IPv6-mapped IPv4 address, therefore extract -+ * the actual v4 address from the last 4 bytes -+ */ -+ local_ip += 12; -+ } else { -+ netdev_dbg(ovpn->dev, -+ "%s: invalid length %zu for local IP\n", __func__, -+ ip_len); -+ goto sockfd_release; -+ } -+ } ++ if (spin_is_contended(pvmw->ptl)) ++ return; + -+ /* sanity checks passed */ -+ ret = 0; -+ } ++ /* avoid taking the LRU lock under the PTL when possible */ ++ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; + -+ id = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_PEER_ID]); -+ peer = ovpn_peer_new(ovpn, ss, sock, id, local_ip); -+ if (IS_ERR(peer)) { -+ netdev_err(ovpn->dev, "%s: cannot create new peer object for peer %u %pIScp\n", -+ __func__, id, ss); -+ ret = PTR_ERR(peer); -+ goto sockfd_release; -+ } ++ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); ++ end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; + -+ if (attrs[OVPN_NEW_PEER_ATTR_IPV4]) { -+ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV4]) != sizeof(struct in_addr)) { -+ ret = -EINVAL; -+ goto peer_release; ++ if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { ++ if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) ++ end = start + MIN_LRU_BATCH * PAGE_SIZE; ++ else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) ++ start = end - MIN_LRU_BATCH * PAGE_SIZE; ++ else { ++ start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; ++ end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; + } -+ -+ peer->vpn_addrs.ipv4.s_addr = nla_get_be32(attrs[OVPN_NEW_PEER_ATTR_IPV4]); + } + -+ if (attrs[OVPN_NEW_PEER_ATTR_IPV6]) { -+ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV6]) != sizeof(struct in6_addr)) { -+ ret = -EINVAL; -+ goto peer_release; -+ } ++ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; + -+ memcpy(&peer->vpn_addrs.ipv6, nla_data(attrs[OVPN_NEW_PEER_ATTR_IPV6]), -+ sizeof(struct in6_addr)); -+ } ++ rcu_read_lock(); ++ arch_enter_lazy_mmu_mode(); + -+ netdev_dbg(ovpn->dev, -+ "%s: adding peer with endpoint=%pIScp/%s id=%u VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n", -+ __func__, ss, sock->sk->sk_prot_creator->name, peer->id, -+ &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6); ++ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { ++ unsigned long pfn; + -+ ret = ovpn_peer_add(ovpn, peer); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot add new peer (id=%u) to hashtable: %d\n", -+ __func__, peer->id, ret); -+ goto peer_release; -+ } ++ pfn = get_pte_pfn(pte[i], pvmw->vma, addr); ++ if (pfn == -1) ++ continue; + -+ return 0; ++ if (!pte_young(pte[i])) ++ continue; ++ ++ folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); ++ if (!folio) ++ continue; ++ ++ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) ++ VM_WARN_ON_ONCE(true); ++ ++ young++; ++ ++ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && ++ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && ++ !folio_test_swapcache(folio))) ++ folio_mark_dirty(folio); + -+peer_release: -+ /* release right away because peer is not really used in any context */ -+ ovpn_peer_release(peer); -+ return ret; ++ old_gen = folio_lru_gen(folio); ++ if (old_gen < 0) ++ folio_set_referenced(folio); ++ else if (old_gen != new_gen) ++ __set_bit(i, bitmap); ++ } + -+sockfd_release: -+ sockfd_put(sock); -+ return ret; -+} ++ arch_leave_lazy_mmu_mode(); ++ rcu_read_unlock(); + -+static int ovpn_netlink_set_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ u32 peer_id, interv, timeout; -+ bool keepalive_set = false; -+ struct ovpn_peer *peer; -+ int ret; ++ /* feedback from rmap walkers to page table walkers */ ++ if (suitable_to_scan(i, young)) ++ update_bloom_filter(lruvec, max_seq, pvmw->pmd); + -+ if (!info->attrs[OVPN_ATTR_SET_PEER]) -+ return -EINVAL; ++ if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { ++ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { ++ folio = pfn_folio(pte_pfn(pte[i])); ++ folio_activate(folio); ++ } ++ return; ++ } + -+ ret = nla_parse_nested(attrs, OVPN_SET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_SET_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; ++ /* folio_update_gen() requires stable folio_memcg() */ ++ if (!mem_cgroup_trylock_pages(memcg)) ++ return; + -+ if (!attrs[OVPN_SET_PEER_ATTR_PEER_ID]) -+ return -EINVAL; ++ if (!walk) { ++ spin_lock_irq(&lruvec->lru_lock); ++ new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); ++ } + -+ peer_id = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_PEER_ID]); ++ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { ++ folio = pfn_folio(pte_pfn(pte[i])); ++ if (folio_memcg_rcu(folio) != memcg) ++ continue; + -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; ++ old_gen = folio_update_gen(folio, new_gen); ++ if (old_gen < 0 || old_gen == new_gen) ++ continue; + -+ /* when setting the keepalive, both parameters have to be configured */ -+ if (attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] && -+ attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]) { -+ keepalive_set = true; -+ interv = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL]); -+ timeout = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]); ++ if (walk) ++ update_batch_size(walk, folio, old_gen, new_gen); ++ else ++ lru_gen_update_size(lruvec, folio, old_gen, new_gen); + } + -+ if (keepalive_set) -+ ovpn_peer_keepalive_set(peer, interv, timeout); ++ if (!walk) ++ spin_unlock_irq(&lruvec->lru_lock); + -+ ovpn_peer_put(peer); -+ return 0; ++ mem_cgroup_unlock_pages(); +} + -+static int ovpn_netlink_send_peer(struct sk_buff *skb, const struct ovpn_peer *peer, u32 portid, -+ u32 seq, int flags) ++/****************************************************************************** ++ * the eviction ++ ******************************************************************************/ ++ ++static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) +{ -+ const struct ovpn_bind *bind; -+ struct nlattr *attr; -+ void *hdr; ++ bool success; ++ int gen = folio_lru_gen(folio); ++ int type = folio_is_file_lru(folio); ++ int zone = folio_zonenum(folio); ++ int delta = folio_nr_pages(folio); ++ int refs = folio_lru_refs(folio); ++ int tier = lru_tier_from_refs(refs); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ hdr = genlmsg_put(skb, portid, seq, &ovpn_netlink_family, flags, OVPN_CMD_GET_PEER); -+ if (!hdr) { -+ netdev_dbg(peer->ovpn->dev, "%s: cannot create message header\n", __func__); -+ return -EMSGSIZE; -+ } ++ VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); + -+ attr = nla_nest_start(skb, OVPN_ATTR_GET_PEER); -+ if (!attr) { -+ netdev_dbg(peer->ovpn->dev, "%s: cannot create submessage\n", __func__); -+ goto err; ++ /* unevictable */ ++ if (!folio_evictable(folio)) { ++ success = lru_gen_del_folio(lruvec, folio, true); ++ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ folio_set_unevictable(folio); ++ lruvec_add_folio(lruvec, folio); ++ __count_vm_events(UNEVICTABLE_PGCULLED, delta); ++ return true; + } + -+ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_PEER_ID, peer->id)) -+ goto err; ++ /* dirty lazyfree */ ++ if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { ++ success = lru_gen_del_folio(lruvec, folio, true); ++ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ folio_set_swapbacked(folio); ++ lruvec_add_folio_tail(lruvec, folio); ++ return true; ++ } + -+ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV4, sizeof(peer->vpn_addrs.ipv4), -+ &peer->vpn_addrs.ipv4)) -+ goto err; ++ /* promoted */ ++ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { ++ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); ++ return true; ++ } + -+ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV6, sizeof(peer->vpn_addrs.ipv6), -+ &peer->vpn_addrs.ipv6)) -+ goto err; ++ /* protected */ ++ if (tier > tier_idx) { ++ int hist = lru_hist_from_seq(lrugen->min_seq[type]); + -+ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, -+ peer->keepalive_interval) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, -+ peer->keepalive_timeout)) -+ goto err; ++ gen = folio_inc_gen(lruvec, folio, false); ++ list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); + -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (bind) { -+ if (bind->sa.in4.sin_family == AF_INET) { -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ sizeof(bind->sa.in4), &bind->sa.in4) || -+ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ sizeof(bind->local.ipv4), &bind->local.ipv4)) -+ goto err_unlock; -+ } else if (bind->sa.in4.sin_family == AF_INET6) { -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ sizeof(bind->sa.in6), &bind->sa.in6) || -+ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ sizeof(bind->local.ipv6), &bind->local.ipv6)) -+ goto err_unlock; -+ } ++ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], ++ lrugen->protected[hist][type][tier - 1] + delta); ++ __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); ++ return true; + } -+ rcu_read_unlock(); -+ -+ if (nla_put_net16(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, -+ inet_sk(peer->sock->sock->sk)->inet_sport) || -+ /* RX stats */ -+ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_RX_BYTES, -+ atomic64_read(&peer->stats.rx.bytes), -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, -+ atomic_read(&peer->stats.rx.packets)) || -+ /* TX stats */ -+ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_TX_BYTES, -+ atomic64_read(&peer->stats.tx.bytes), -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, -+ atomic_read(&peer->stats.tx.packets))) -+ goto err; + -+ nla_nest_end(skb, attr); -+ genlmsg_end(skb, hdr); ++ /* waiting for writeback */ ++ if (folio_test_locked(folio) || folio_test_writeback(folio) || ++ (type == LRU_GEN_FILE && folio_test_dirty(folio))) { ++ gen = folio_inc_gen(lruvec, folio, true); ++ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); ++ return true; ++ } + -+ return 0; -+err_unlock: -+ rcu_read_unlock(); -+err: -+ genlmsg_cancel(skb, hdr); -+ return -EMSGSIZE; ++ return false; +} + -+static int ovpn_netlink_get_peer(struct sk_buff *skb, struct genl_info *info) ++static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) +{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ struct sk_buff *msg; -+ u32 peer_id; -+ int ret; ++ bool success; + -+ if (!info->attrs[OVPN_ATTR_GET_PEER]) -+ return -EINVAL; ++ /* unmapping inhibited */ ++ if (!sc->may_unmap && folio_mapped(folio)) ++ return false; + -+ ret = nla_parse_nested(attrs, OVPN_GET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_GET_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; ++ /* swapping inhibited */ ++ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && ++ (folio_test_dirty(folio) || ++ (folio_test_anon(folio) && !folio_test_swapcache(folio)))) ++ return false; + -+ if (!attrs[OVPN_GET_PEER_ATTR_PEER_ID]) -+ return -EINVAL; ++ /* raced with release_pages() */ ++ if (!folio_try_get(folio)) ++ return false; + -+ peer_id = nla_get_u32(attrs[OVPN_GET_PEER_ATTR_PEER_ID]); -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; ++ /* raced with another isolation */ ++ if (!folio_test_clear_lru(folio)) { ++ folio_put(folio); ++ return false; ++ } + -+ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); -+ if (!msg) -+ return -ENOMEM; ++ /* see the comment on MAX_NR_TIERS */ ++ if (!folio_test_referenced(folio)) ++ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); + -+ ret = ovpn_netlink_send_peer(msg, peer, info->snd_portid, info->snd_seq, 0); -+ if (ret < 0) { -+ nlmsg_free(msg); -+ goto err; -+ } ++ /* for shrink_page_list() */ ++ folio_clear_reclaim(folio); ++ folio_clear_referenced(folio); + -+ ret = genlmsg_reply(msg, info); -+err: -+ ovpn_peer_put(peer); -+ return ret; ++ success = lru_gen_del_folio(lruvec, folio, true); ++ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ ++ return true; +} + -+static int ovpn_netlink_dump_done(struct netlink_callback *cb) ++static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, ++ int type, int tier, struct list_head *list) +{ -+ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; ++ int gen, zone; ++ enum vm_event_item item; ++ int sorted = 0; ++ int scanned = 0; ++ int isolated = 0; ++ int remaining = MAX_LRU_BATCH; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); + -+ dev_put(ovpn->dev); -+ return 0; -+} ++ VM_WARN_ON_ONCE(!list_empty(list)); + -+static int ovpn_netlink_dump_prepare(struct netlink_callback *cb) -+{ -+ struct net *netns = sock_net(cb->skb->sk); -+ struct nlattr **attrbuf; -+ struct net_device *dev; -+ int ret; ++ if (get_nr_gens(lruvec, type) == MIN_NR_GENS) ++ return 0; + -+ attrbuf = kcalloc(OVPN_ATTR_MAX + 1, sizeof(*attrbuf), GFP_KERNEL); -+ if (!attrbuf) -+ return -ENOMEM; ++ gen = lru_gen_from_seq(lrugen->min_seq[type]); + -+ ret = nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrbuf, OVPN_ATTR_MAX, -+ ovpn_netlink_policy, NULL); -+ if (ret < 0) -+ goto err; ++ for (zone = sc->reclaim_idx; zone >= 0; zone--) { ++ LIST_HEAD(moved); ++ int skipped = 0; ++ struct list_head *head = &lrugen->lists[gen][type][zone]; + -+ dev = ovpn_get_dev_from_attrs(netns, attrbuf); -+ if (IS_ERR(dev)) { -+ ret = PTR_ERR(dev); -+ goto err; -+ } ++ while (!list_empty(head)) { ++ struct folio *folio = lru_to_folio(head); ++ int delta = folio_nr_pages(folio); + -+ cb->args[0] = (long)netdev_priv(dev); -+ ret = 0; -+err: -+ kfree(attrbuf); -+ return ret; -+} ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + -+static int ovpn_netlink_dump_peers(struct sk_buff *skb, struct netlink_callback *cb) -+{ -+ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; -+ int ret, bkt, last_idx = cb->args[1], dumped = 0; -+ struct ovpn_peer *peer; ++ scanned += delta; + -+ if (!ovpn) { -+ ret = ovpn_netlink_dump_prepare(cb); -+ if (ret < 0) { -+ netdev_dbg(ovpn->dev, "%s: cannot prepare for dump: %d\n", __func__, ret); -+ return ret; -+ } ++ if (sort_folio(lruvec, folio, tier)) ++ sorted += delta; ++ else if (isolate_folio(lruvec, folio, sc)) { ++ list_add(&folio->lru, list); ++ isolated += delta; ++ } else { ++ list_move(&folio->lru, &moved); ++ skipped += delta; ++ } + -+ ovpn = (struct ovpn_struct *)cb->args[0]; -+ } ++ if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) ++ break; ++ } + -+ rcu_read_lock(); -+ hash_for_each_rcu(ovpn->peers.by_id, bkt, peer, hash_entry_id) { -+ /* skip already dumped peers that were dumped by previous invocations */ -+ if (last_idx > 0) { -+ last_idx--; -+ continue; ++ if (skipped) { ++ list_splice(&moved, head); ++ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); + } + -+ if (ovpn_netlink_send_peer(skb, peer, NETLINK_CB(cb->skb).portid, -+ cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) ++ if (!remaining || isolated >= MIN_LRU_BATCH) + break; ++ } + -+ /* count peers being dumped during this invocation */ -+ dumped++; ++ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; ++ if (!cgroup_reclaim(sc)) { ++ __count_vm_events(item, isolated); ++ __count_vm_events(PGREFILL, sorted); + } -+ rcu_read_unlock(); ++ __count_memcg_events(memcg, item, isolated); ++ __count_memcg_events(memcg, PGREFILL, sorted); ++ __count_vm_events(PGSCAN_ANON + type, isolated); + -+ /* sum up peers dumped in this message, so that at the next invocation -+ * we can continue from where we left ++ /* ++ * There might not be eligible pages due to reclaim_idx, may_unmap and ++ * may_writepage. Check the remaining to prevent livelock if it's not ++ * making progress. + */ -+ cb->args[1] += dumped; -+ -+ return skb->len; ++ return isolated || !remaining ? scanned : 0; +} + -+static int ovpn_netlink_del_peer(struct sk_buff *skb, struct genl_info *info) ++static int get_tier_idx(struct lruvec *lruvec, int type) +{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; ++ int tier; ++ struct ctrl_pos sp, pv; + -+ if (!info->attrs[OVPN_ATTR_DEL_PEER]) -+ return -EINVAL; ++ /* ++ * To leave a margin for fluctuations, use a larger gain factor (1:2). ++ * This value is chosen because any other tier would have at least twice ++ * as many refaults as the first tier. ++ */ ++ read_ctrl_pos(lruvec, type, 0, 1, &sp); ++ for (tier = 1; tier < MAX_NR_TIERS; tier++) { ++ read_ctrl_pos(lruvec, type, tier, 2, &pv); ++ if (!positive_ctrl_err(&sp, &pv)) ++ break; ++ } + -+ ret = nla_parse_nested(attrs, OVPN_DEL_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; ++ return tier - 1; ++} + -+ if (!attrs[OVPN_DEL_PEER_ATTR_PEER_ID]) -+ return -EINVAL; ++static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) ++{ ++ int type, tier; ++ struct ctrl_pos sp, pv; ++ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; + -+ peer_id = nla_get_u32(attrs[OVPN_DEL_PEER_ATTR_PEER_ID]); ++ /* ++ * Compare the first tier of anon with that of file to determine which ++ * type to scan. Also need to compare other tiers of the selected type ++ * with the first tier of the other type to determine the last tier (of ++ * the selected type) to evict. ++ */ ++ read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); ++ read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); ++ type = positive_ctrl_err(&sp, &pv); + -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; ++ read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); ++ for (tier = 1; tier < MAX_NR_TIERS; tier++) { ++ read_ctrl_pos(lruvec, type, tier, gain[type], &pv); ++ if (!positive_ctrl_err(&sp, &pv)) ++ break; ++ } + -+ netdev_dbg(ovpn->dev, "%s: peer id=%u\n", __func__, peer->id); -+ ret = ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_USERSPACE); -+ ovpn_peer_put(peer); ++ *tier_idx = tier - 1; + -+ return ret; ++ return type; +} + -+static int ovpn_netlink_register_packet(struct sk_buff *skb, -+ struct genl_info *info) ++static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, ++ int *type_scanned, struct list_head *list) +{ -+ struct ovpn_struct *ovpn = info->user_ptr[0]; ++ int i; ++ int type; ++ int scanned; ++ int tier = -1; ++ DEFINE_MIN_SEQ(lruvec); + -+ /* only one registered process per interface is allowed for now */ -+ if (ovpn->registered_nl_portid_set) { -+ netdev_dbg(ovpn->dev, "%s: userspace listener already registered\n", __func__); -+ return -EBUSY; -+ } ++ /* ++ * Try to make the obvious choice first. When anon and file are both ++ * available from the same generation, interpret swappiness 1 as file ++ * first and 200 as anon first. ++ */ ++ if (!swappiness) ++ type = LRU_GEN_FILE; ++ else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) ++ type = LRU_GEN_ANON; ++ else if (swappiness == 1) ++ type = LRU_GEN_FILE; ++ else if (swappiness == 200) ++ type = LRU_GEN_ANON; ++ else ++ type = get_type_to_scan(lruvec, swappiness, &tier); + -+ netdev_dbg(ovpn->dev, "%s: registering userspace at %u\n", __func__, info->snd_portid); ++ for (i = !swappiness; i < ANON_AND_FILE; i++) { ++ if (tier < 0) ++ tier = get_tier_idx(lruvec, type); + -+ ovpn->registered_nl_portid = info->snd_portid; -+ ovpn->registered_nl_portid_set = true; ++ scanned = scan_folios(lruvec, sc, type, tier, list); ++ if (scanned) ++ break; + -+ return 0; -+} ++ type = !type; ++ tier = -1; ++ } + -+static int ovpn_netlink_packet(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_PACKET_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ const u8 *packet; -+ u32 peer_id; -+ size_t len; -+ u8 opcode; -+ int ret; ++ *type_scanned = type; + -+ if (!info->attrs[OVPN_ATTR_PACKET]) -+ return -EINVAL; ++ return scanned; ++} + -+ ret = nla_parse_nested(attrs, OVPN_PACKET_ATTR_MAX, info->attrs[OVPN_ATTR_PACKET], -+ NULL, info->extack); -+ if (ret) -+ return ret; ++static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, ++ bool *need_swapping) ++{ ++ int type; ++ int scanned; ++ int reclaimed; ++ LIST_HEAD(list); ++ struct folio *folio; ++ enum vm_event_item item; ++ struct reclaim_stat stat; ++ struct lru_gen_mm_walk *walk; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ struct pglist_data *pgdat = lruvec_pgdat(lruvec); + -+ if (!attrs[OVPN_PACKET_ATTR_PACKET] || !attrs[OVPN_PACKET_ATTR_PEER_ID]) { -+ netdev_dbg(ovpn->dev, "received netlink packet with no payload\n"); -+ return -EINVAL; -+ } ++ spin_lock_irq(&lruvec->lru_lock); + -+ peer_id = nla_get_u32(attrs[OVPN_PACKET_ATTR_PEER_ID]); ++ scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); + -+ len = nla_len(attrs[OVPN_PACKET_ATTR_PACKET]); ++ scanned += try_to_inc_min_seq(lruvec, swappiness); + -+ if (len < 4 || len > ovpn->dev->mtu) { -+ netdev_dbg(ovpn->dev, "%s: invalid packet size %zu (min is 4, max is MTU: %u)\n", -+ __func__, len, ovpn->dev->mtu); -+ return -EINVAL; -+ } ++ if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) ++ scanned = 0; + -+ packet = nla_data(attrs[OVPN_PACKET_ATTR_PACKET]); -+ opcode = ovpn_opcode_from_byte(packet[0]); ++ spin_unlock_irq(&lruvec->lru_lock); + -+ /* reject data packets from userspace as they could lead to IV reuse */ -+ if (opcode == OVPN_DATA_V1 || opcode == OVPN_DATA_V2) { -+ netdev_dbg(ovpn->dev, "%s: rejecting data packet from userspace (opcode=%u)\n", -+ __func__, opcode); -+ return -EINVAL; -+ } ++ if (list_empty(&list)) ++ return scanned; + -+ netdev_dbg(ovpn->dev, "%s: sending userspace packet to peer %u...\n", __func__, peer_id); ++ reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); + -+ return ovpn_send_data(ovpn, peer_id, packet, len); -+} ++ list_for_each_entry(folio, &list, lru) { ++ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ ++ if (folio_test_workingset(folio)) ++ folio_set_referenced(folio); + -+static const struct genl_ops ovpn_netlink_ops[] = { -+ { -+ .cmd = OVPN_CMD_NEW_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_new_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_SET_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_set_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_DEL_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_del_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_GET_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_get_peer, -+ .dumpit = ovpn_netlink_dump_peers, -+ .done = ovpn_netlink_dump_done, -+ }, -+ { -+ .cmd = OVPN_CMD_NEW_KEY, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_new_key, -+ }, -+ { -+ .cmd = OVPN_CMD_DEL_KEY, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_del_key, -+ }, -+ { -+ .cmd = OVPN_CMD_SWAP_KEYS, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_swap_keys, -+ }, -+ { -+ .cmd = OVPN_CMD_REGISTER_PACKET, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_register_packet, -+ }, -+ { -+ .cmd = OVPN_CMD_PACKET, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_packet, -+ }, -+}; ++ /* don't add rejected pages to the oldest generation */ ++ if (folio_test_reclaim(folio) && ++ (folio_test_dirty(folio) || folio_test_writeback(folio))) ++ folio_clear_active(folio); ++ else ++ folio_set_active(folio); ++ } + -+static struct genl_family ovpn_netlink_family __ro_after_init = { -+ .hdrsize = 0, -+ .name = OVPN_NL_NAME, -+ .version = 1, -+ .maxattr = OVPN_ATTR_MAX, -+ .policy = ovpn_netlink_policy, -+ .netnsok = true, -+ .pre_doit = ovpn_pre_doit, -+ .post_doit = ovpn_post_doit, -+ .module = THIS_MODULE, -+ .ops = ovpn_netlink_ops, -+ .n_ops = ARRAY_SIZE(ovpn_netlink_ops), -+ .mcgrps = ovpn_netlink_mcgrps, -+ .n_mcgrps = ARRAY_SIZE(ovpn_netlink_mcgrps), -+}; ++ spin_lock_irq(&lruvec->lru_lock); + -+int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer) -+{ -+ struct sk_buff *msg; -+ struct nlattr *attr; -+ void *hdr; -+ int ret; ++ move_pages_to_lru(lruvec, &list); + -+ netdev_info(peer->ovpn->dev, "%s: deleting peer with id %u, reason %d\n", -+ peer->ovpn->dev->name, peer->id, peer->delete_reason); ++ walk = current->reclaim_state->mm_walk; ++ if (walk && walk->batched) ++ reset_batch_size(lruvec, walk); + -+ msg = nlmsg_new(100, GFP_KERNEL); -+ if (!msg) -+ return -ENOMEM; ++ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; ++ if (!cgroup_reclaim(sc)) ++ __count_vm_events(item, reclaimed); ++ __count_memcg_events(memcg, item, reclaimed); ++ __count_vm_events(PGSTEAL_ANON + type, reclaimed); + -+ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, -+ OVPN_CMD_DEL_PEER); -+ if (!hdr) { -+ ret = -ENOBUFS; -+ goto err_free_msg; -+ } ++ spin_unlock_irq(&lruvec->lru_lock); + -+ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, peer->ovpn->dev->ifindex)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ mem_cgroup_uncharge_list(&list); ++ free_unref_page_list(&list); + -+ attr = nla_nest_start(msg, OVPN_ATTR_DEL_PEER); -+ if (!attr) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ sc->nr_reclaimed += reclaimed; + -+ if (nla_put_u8(msg, OVPN_DEL_PEER_ATTR_REASON, peer->delete_reason)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ if (need_swapping && type == LRU_GEN_ANON) ++ *need_swapping = true; + -+ if (nla_put_u32(msg, OVPN_DEL_PEER_ATTR_PEER_ID, peer->id)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ return scanned; ++} + -+ nla_nest_end(msg, attr); ++/* ++ * For future optimizations: ++ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg ++ * reclaim. ++ */ ++static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, ++ bool can_swap, bool *need_aging) ++{ ++ unsigned long nr_to_scan; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); + -+ genlmsg_end(msg, hdr); ++ if (mem_cgroup_below_min(memcg) || ++ (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) ++ return 0; + -+ genlmsg_multicast_netns(&ovpn_netlink_family, dev_net(peer->ovpn->dev), -+ msg, 0, OVPN_MCGRP_PEERS, GFP_KERNEL); ++ *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); ++ if (!*need_aging) ++ return nr_to_scan; + -+ return 0; ++ /* skip the aging path at the default priority */ ++ if (sc->priority == DEF_PRIORITY) ++ goto done; + -+err_free_msg: -+ nlmsg_free(msg); -+ return ret; ++ /* leave the work to lru_gen_age_node() */ ++ if (current_is_kswapd()) ++ return 0; ++ ++ if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) ++ return nr_to_scan; ++done: ++ return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; +} + -+int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ const u8 *buf, size_t len) ++static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, ++ struct scan_control *sc, bool need_swapping) +{ -+ struct nlattr *attr; -+ struct sk_buff *msg; -+ void *hdr; -+ int ret; ++ int i; ++ DEFINE_MAX_SEQ(lruvec); + -+ if (!ovpn->registered_nl_portid_set) { -+ net_warn_ratelimited("%s: no userspace listener\n", __func__); -+ return 0; -+ } ++ if (!current_is_kswapd()) { ++ /* age each memcg once to ensure fairness */ ++ if (max_seq - seq > 1) ++ return true; + -+ netdev_dbg(ovpn->dev, "%s: sending packet to userspace, len: %zd\n", __func__, len); ++ /* over-swapping can increase allocation latency */ ++ if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) ++ return true; + -+ msg = nlmsg_new(100 + len, GFP_ATOMIC); -+ if (!msg) -+ return -ENOMEM; ++ /* give this thread a chance to exit and free its memory */ ++ if (fatal_signal_pending(current)) { ++ sc->nr_reclaimed += MIN_LRU_BATCH; ++ return true; ++ } + -+ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, -+ OVPN_CMD_PACKET); -+ if (!hdr) { -+ ret = -ENOBUFS; -+ goto err_free_msg; -+ } ++ if (cgroup_reclaim(sc)) ++ return false; ++ } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) ++ return false; + -+ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, ovpn->dev->ifindex)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ /* keep scanning at low priorities to ensure fairness */ ++ if (sc->priority > DEF_PRIORITY - 2) ++ return false; + -+ attr = nla_nest_start(msg, OVPN_ATTR_PACKET); -+ if (!attr) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ /* ++ * A minimum amount of work was done under global memory pressure. For ++ * kswapd, it may be overshooting. For direct reclaim, the target isn't ++ * met, and yet the allocation may still succeed, since kswapd may have ++ * caught up. In either case, it's better to stop now, and restart if ++ * necessary. ++ */ ++ for (i = 0; i <= sc->reclaim_idx; i++) { ++ unsigned long wmark; ++ struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; + -+ if (nla_put(msg, OVPN_PACKET_ATTR_PACKET, len, buf)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } ++ if (!managed_zone(zone)) ++ continue; + -+ if (nla_put_u32(msg, OVPN_PACKET_ATTR_PEER_ID, peer->id)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; ++ wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); ++ if (wmark > zone_page_state(zone, NR_FREE_PAGES)) ++ return false; + } + -+ nla_nest_end(msg, attr); -+ -+ genlmsg_end(msg, hdr); -+ -+ return genlmsg_unicast(dev_net(ovpn->dev), msg, -+ ovpn->registered_nl_portid); ++ sc->nr_reclaimed += MIN_LRU_BATCH; + -+err_free_msg: -+ nlmsg_free(msg); -+ return ret; ++ return true; +} + -+static int ovpn_netlink_notify(struct notifier_block *nb, unsigned long state, -+ void *_notify) ++static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ -+ struct netlink_notify *notify = _notify; -+ struct ovpn_struct *ovpn; -+ struct net_device *dev; -+ struct net *netns; -+ bool found = false; ++ struct blk_plug plug; ++ bool need_aging = false; ++ bool need_swapping = false; ++ unsigned long scanned = 0; ++ unsigned long reclaimed = sc->nr_reclaimed; ++ DEFINE_MAX_SEQ(lruvec); + -+ if (state != NETLINK_URELEASE || notify->protocol != NETLINK_GENERIC) -+ return NOTIFY_DONE; ++ lru_add_drain(); + -+ rcu_read_lock(); -+ for_each_net_rcu(netns) { -+ for_each_netdev_rcu(netns, dev) { -+ if (!ovpn_dev_is_valid(dev)) -+ continue; ++ blk_start_plug(&plug); + -+ ovpn = netdev_priv(dev); -+ if (notify->portid != ovpn->registered_nl_portid) -+ continue; ++ set_mm_walk(lruvec_pgdat(lruvec)); + -+ found = true; -+ netdev_dbg(ovpn->dev, "%s: deregistering userspace listener\n", __func__); -+ ovpn->registered_nl_portid_set = false; ++ while (true) { ++ int delta; ++ int swappiness; ++ unsigned long nr_to_scan; ++ ++ if (sc->may_swap) ++ swappiness = get_swappiness(lruvec, sc); ++ else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) ++ swappiness = 1; ++ else ++ swappiness = 0; ++ ++ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); ++ if (!nr_to_scan) ++ goto done; ++ ++ delta = evict_folios(lruvec, sc, swappiness, &need_swapping); ++ if (!delta) ++ goto done; ++ ++ scanned += delta; ++ if (scanned >= nr_to_scan) + break; -+ } ++ ++ if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) ++ break; ++ ++ cond_resched(); + } -+ rcu_read_unlock(); + -+ /* if no interface matched our purposes, pass the notification along */ -+ if (!found) -+ return NOTIFY_DONE; ++ /* see the comment in lru_gen_age_node() */ ++ if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) ++ sc->memcgs_need_aging = false; ++done: ++ clear_mm_walk(); + -+ return NOTIFY_OK; ++ blk_finish_plug(&plug); +} + -+static struct notifier_block ovpn_netlink_notifier = { -+ .notifier_call = ovpn_netlink_notify, -+}; ++/****************************************************************************** ++ * state change ++ ******************************************************************************/ + -+int ovpn_netlink_init(struct ovpn_struct *ovpn) ++static bool __maybe_unused state_is_valid(struct lruvec *lruvec) +{ -+ ovpn->registered_nl_portid_set = false; -+ -+ return 0; -+} ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+/** -+ * ovpn_netlink_register() - register the ovpn genl netlink family -+ */ -+int __init ovpn_netlink_register(void) -+{ -+ int ret; ++ if (lrugen->enabled) { ++ enum lru_list lru; + -+ ret = genl_register_family(&ovpn_netlink_family); -+ if (ret) -+ return ret; ++ for_each_evictable_lru(lru) { ++ if (!list_empty(&lruvec->lists[lru])) ++ return false; ++ } ++ } else { ++ int gen, type, zone; + -+ ret = netlink_register_notifier(&ovpn_netlink_notifier); -+ if (ret) -+ goto err; ++ for_each_gen_type_zone(gen, type, zone) { ++ if (!list_empty(&lrugen->lists[gen][type][zone])) ++ return false; ++ } ++ } + -+ return 0; -+err: -+ genl_unregister_family(&ovpn_netlink_family); -+ return ret; ++ return true; +} + -+/** -+ * ovpn_netlink_unregister() - unregister the ovpn genl netlink family -+ */ -+void __exit ovpn_netlink_unregister(void) ++static bool fill_evictable(struct lruvec *lruvec) +{ -+ netlink_unregister_notifier(&ovpn_netlink_notifier); -+ genl_unregister_family(&ovpn_netlink_family); -+} -diff --git a/drivers/net/ovpn-dco/netlink.h b/drivers/net/ovpn-dco/netlink.h -new file mode 100644 -index 000000000000..843daf052c03 ---- /dev/null -+++ b/drivers/net/ovpn-dco/netlink.h -@@ -0,0 +1,22 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ ++ enum lru_list lru; ++ int remaining = MAX_LRU_BATCH; + -+#ifndef _NET_OVPN_DCO_NETLINK_H_ -+#define _NET_OVPN_DCO_NETLINK_H_ ++ for_each_evictable_lru(lru) { ++ int type = is_file_lru(lru); ++ bool active = is_active_lru(lru); ++ struct list_head *head = &lruvec->lists[lru]; + -+struct ovpn_struct; -+struct ovpn_peer; ++ while (!list_empty(head)) { ++ bool success; ++ struct folio *folio = lru_to_folio(head); + -+int ovpn_netlink_init(struct ovpn_struct *ovpn); -+int ovpn_netlink_register(void); -+void ovpn_netlink_unregister(void); -+int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ const u8 *buf, size_t len); -+int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); + -+#endif /* _NET_OVPN_DCO_NETLINK_H_ */ -diff --git a/drivers/net/ovpn-dco/ovpn.c b/drivers/net/ovpn-dco/ovpn.c -new file mode 100644 -index 000000000000..66c019174f5e ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpn.c -@@ -0,0 +1,600 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++ lruvec_del_folio(lruvec, folio); ++ success = lru_gen_add_folio(lruvec, folio, false); ++ VM_WARN_ON_ONCE(!success); + -+#include "main.h" -+#include "bind.h" -+#include "netlink.h" -+#include "ovpn.h" -+#include "sock.h" -+#include "peer.h" -+#include "stats.h" -+#include "proto.h" -+#include "crypto.h" -+#include "crypto_aead.h" -+#include "skb.h" -+#include "tcp.h" -+#include "udp.h" ++ if (!--remaining) ++ return false; ++ } ++ } + -+#include -+#include ++ return true; ++} + -+static const unsigned char ovpn_keepalive_message[] = { -+ 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, -+ 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 -+}; ++static bool drain_evictable(struct lruvec *lruvec) ++{ ++ int gen, type, zone; ++ int remaining = MAX_LRU_BATCH; + -+static const unsigned char ovpn_explicit_exit_notify_message[] = { -+ 0x28, 0x7f, 0x34, 0x6b, 0xd4, 0xef, 0x7a, 0x81, -+ 0x2d, 0x56, 0xb8, 0xd3, 0xaf, 0xc5, 0x45, 0x9c, -+ 6 // OCC_EXIT -+}; ++ for_each_gen_type_zone(gen, type, zone) { ++ struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; + -+/* Is keepalive message? -+ * Assumes that single byte at skb->data is defined. -+ */ -+static bool ovpn_is_keepalive(struct sk_buff *skb) -+{ -+ if (*skb->data != OVPN_KEEPALIVE_FIRST_BYTE) -+ return false; ++ while (!list_empty(head)) { ++ bool success; ++ struct folio *folio = lru_to_folio(head); ++ ++ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); ++ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); ++ ++ success = lru_gen_del_folio(lruvec, folio, false); ++ VM_WARN_ON_ONCE(!success); ++ lruvec_add_folio(lruvec, folio); + -+ if (!pskb_may_pull(skb, sizeof(ovpn_keepalive_message))) -+ return false; ++ if (!--remaining) ++ return false; ++ } ++ } + -+ return !memcmp(skb->data, ovpn_keepalive_message, -+ sizeof(ovpn_keepalive_message)); ++ return true; +} + -+int ovpn_struct_init(struct net_device *dev) ++static void lru_gen_change_state(bool enabled) +{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ int err; -+ -+ memset(ovpn, 0, sizeof(*ovpn)); -+ -+ ovpn->dev = dev; ++ static DEFINE_MUTEX(state_mutex); + -+ err = ovpn_netlink_init(ovpn); -+ if (err < 0) -+ return err; ++ struct mem_cgroup *memcg; + -+ spin_lock_init(&ovpn->lock); -+ spin_lock_init(&ovpn->peers.lock); ++ cgroup_lock(); ++ cpus_read_lock(); ++ get_online_mems(); ++ mutex_lock(&state_mutex); + -+ ovpn->crypto_wq = alloc_workqueue("ovpn-crypto-wq-%s", -+ WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, -+ dev->name); -+ if (!ovpn->crypto_wq) -+ return -ENOMEM; ++ if (enabled == lru_gen_enabled()) ++ goto unlock; + -+ ovpn->events_wq = alloc_workqueue("ovpn-event-wq-%s", WQ_MEM_RECLAIM, 0, dev->name); -+ if (!ovpn->events_wq) -+ return -ENOMEM; ++ if (enabled) ++ static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); ++ else ++ static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); + -+ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); -+ if (!dev->tstats) -+ return -ENOMEM; ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ int nid; + -+ err = security_tun_dev_alloc_security(&ovpn->security); -+ if (err < 0) -+ return err; ++ for_each_node(nid) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); + -+ /* kernel -> userspace tun queue length */ -+ ovpn->max_tun_queue_len = OVPN_MAX_TUN_QUEUE_LEN; ++ if (!lruvec) ++ continue; + -+ return 0; -+} ++ spin_lock_irq(&lruvec->lru_lock); + -+/* Called after decrypt to write IP packet to tun netdev. -+ * This method is expected to manage/free skb. -+ */ -+static void tun_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ /* packet integrity was verified on the VPN layer - no need to perform -+ * any additional check along the stack -+ */ -+ skb->ip_summed = CHECKSUM_UNNECESSARY; -+ skb->csum_level = ~0; ++ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); ++ VM_WARN_ON_ONCE(!state_is_valid(lruvec)); + -+ /* skb hash for transport packet no longer valid after decapsulation */ -+ skb_clear_hash(skb); ++ lruvec->lrugen.enabled = enabled; + -+ /* post-decrypt scrub -- prepare to inject encapsulated packet onto tun -+ * interface, based on __skb_tunnel_rx() in dst.h -+ */ -+ skb->dev = peer->ovpn->dev; -+ skb_set_queue_mapping(skb, 0); -+ skb_scrub_packet(skb, true); ++ while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { ++ spin_unlock_irq(&lruvec->lru_lock); ++ cond_resched(); ++ spin_lock_irq(&lruvec->lru_lock); ++ } + -+ skb_reset_network_header(skb); -+ skb_reset_transport_header(skb); -+ skb_probe_transport_header(skb); -+ skb_reset_inner_headers(skb); ++ spin_unlock_irq(&lruvec->lru_lock); ++ } + -+ /* update per-cpu RX stats with the stored size of encrypted packet */ ++ cond_resched(); ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++unlock: ++ mutex_unlock(&state_mutex); ++ put_online_mems(); ++ cpus_read_unlock(); ++ cgroup_unlock(); ++} + -+ /* we are in softirq context - hence no locking nor disable preemption needed */ -+ dev_sw_netstats_rx_add(peer->ovpn->dev, OVPN_SKB_CB(skb)->rx_stats_size); ++/****************************************************************************** ++ * sysfs interface ++ ******************************************************************************/ + -+ /* cause packet to be "received" by tun interface */ -+ napi_gro_receive(&peer->napi, skb); ++static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); +} + -+int ovpn_napi_poll(struct napi_struct *napi, int budget) ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, ++ const char *buf, size_t len) +{ -+ struct ovpn_peer *peer = container_of(napi, struct ovpn_peer, napi); -+ struct sk_buff *skb; -+ int work_done = 0; ++ unsigned int msecs; + -+ if (unlikely(budget <= 0)) -+ return 0; -+ /* this function should schedule at most 'budget' number of -+ * packets for delivery to the tun interface. -+ * If in the queue we have more packets than what allowed by the -+ * budget, the next polling will take care of those -+ */ -+ while ((work_done < budget) && -+ (skb = ptr_ring_consume_bh(&peer->netif_rx_ring))) { -+ tun_netdev_write(peer, skb); -+ work_done++; -+ } ++ if (kstrtouint(buf, 0, &msecs)) ++ return -EINVAL; + -+ if (work_done < budget) -+ napi_complete_done(napi, work_done); ++ WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); + -+ return work_done; ++ return len; +} + -+static int ovpn_transport_to_userspace(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ struct sk_buff *skb) ++static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( ++ min_ttl_ms, 0644, show_min_ttl, store_min_ttl ++); ++ ++static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ -+ int ret; ++ unsigned int caps = 0; + -+ ret = skb_linearize(skb); -+ if (ret < 0) -+ return ret; ++ if (get_cap(LRU_GEN_CORE)) ++ caps |= BIT(LRU_GEN_CORE); + -+ ret = ovpn_netlink_send_packet(ovpn, peer, skb->data, skb->len); -+ if (ret < 0) -+ return ret; ++ if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) ++ caps |= BIT(LRU_GEN_MM_WALK); + -+ consume_skb(skb); -+ return 0; ++ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) ++ caps |= BIT(LRU_GEN_NONLEAF_YOUNG); ++ ++ return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); +} + -+/* Entry point for processing an incoming packet (in skb form) -+ * -+ * Enqueue the packet and schedule RX consumer. -+ * Reference to peer is dropped only in case of success. -+ * -+ * Return 0 if the packet was handled (and consumed) -+ * Return <0 in case of error (return value is error code) -+ */ -+int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb) ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, ++ const char *buf, size_t len) +{ -+ int ret; -+ -+ /* At this point we know the packet is from a configured peer. -+ * DATA_V2 packets are handled in kernel space, the rest goes to user space. -+ * -+ * Packets are sent to userspace via netlink API in order to be consistenbt across -+ * UDP and TCP. -+ */ -+ if (unlikely(ovpn_opcode_from_skb(skb, 0) != OVPN_DATA_V2)) { -+ ret = ovpn_transport_to_userspace(ovpn, peer, skb); -+ if (ret < 0) -+ return ret; ++ int i; ++ unsigned int caps; + -+ ovpn_peer_put(peer); -+ return 0; -+ } ++ if (tolower(*buf) == 'n') ++ caps = 0; ++ else if (tolower(*buf) == 'y') ++ caps = -1; ++ else if (kstrtouint(buf, 0, &caps)) ++ return -EINVAL; + -+ ret = ptr_ring_produce_bh(&peer->rx_ring, skb); -+ if (unlikely(ret < 0)) -+ return -ENOSPC; ++ for (i = 0; i < NR_LRU_GEN_CAPS; i++) { ++ bool enabled = caps & BIT(i); + -+ if (!queue_work(ovpn->crypto_wq, &peer->decrypt_work)) -+ ovpn_peer_put(peer); ++ if (i == LRU_GEN_CORE) ++ lru_gen_change_state(enabled); ++ else if (enabled) ++ static_branch_enable(&lru_gen_caps[i]); ++ else ++ static_branch_disable(&lru_gen_caps[i]); ++ } + -+ return 0; ++ return len; +} + -+static int ovpn_decrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_peer *allowed_peer = NULL; -+ struct ovpn_crypto_key_slot *ks; -+ unsigned int rx_stats_size; -+ __be16 proto; -+ int ret = -1; -+ u8 key_id; ++static struct kobj_attribute lru_gen_enabled_attr = __ATTR( ++ enabled, 0644, show_enabled, store_enabled ++); + -+ /* save original packet size for stats accounting */ -+ OVPN_SKB_CB(skb)->rx_stats_size = skb->len; ++static struct attribute *lru_gen_attrs[] = { ++ &lru_gen_min_ttl_attr.attr, ++ &lru_gen_enabled_attr.attr, ++ NULL ++}; + -+ /* get the key slot matching the key Id in the received packet */ -+ key_id = ovpn_key_id_from_skb(skb); -+ ks = ovpn_crypto_key_id_to_slot(&peer->crypto, key_id); -+ if (unlikely(!ks)) { -+ net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n", __func__, -+ peer->id, key_id); -+ goto drop; -+ } ++static struct attribute_group lru_gen_attr_group = { ++ .name = "lru_gen", ++ .attrs = lru_gen_attrs, ++}; + -+ /* decrypt */ -+ ret = ovpn_aead_decrypt(ks, skb); ++/****************************************************************************** ++ * debugfs interface ++ ******************************************************************************/ + -+ ovpn_crypto_key_slot_put(ks); ++static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) ++{ ++ struct mem_cgroup *memcg; ++ loff_t nr_to_skip = *pos; + -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: error during decryption for peer %u, key-id %u: %d\n", -+ __func__, peer->id, key_id, ret); -+ goto drop; -+ } ++ m->private = kvmalloc(PATH_MAX, GFP_KERNEL); ++ if (!m->private) ++ return ERR_PTR(-ENOMEM); + -+ /* note event of authenticated packet received for keepalive */ -+ ovpn_peer_keepalive_recv_reset(peer); ++ memcg = mem_cgroup_iter(NULL, NULL, NULL); ++ do { ++ int nid; + -+ /* update source and destination endpoint for this peer */ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_UDP) -+ ovpn_peer_update_local_endpoint(peer, skb); ++ for_each_node_state(nid, N_MEMORY) { ++ if (!nr_to_skip--) ++ return get_lruvec(memcg, nid); ++ } ++ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + -+ /* increment RX stats */ -+ rx_stats_size = OVPN_SKB_CB(skb)->rx_stats_size; -+ ovpn_peer_stats_increment_rx(&peer->stats, rx_stats_size); ++ return NULL; ++} + -+ /* check if this is a valid datapacket that has to be delivered to the -+ * tun interface -+ */ -+ skb_reset_network_header(skb); -+ proto = ovpn_ip_check_protocol(skb); -+ if (unlikely(!proto)) { -+ /* check if null packet */ -+ if (unlikely(!pskb_may_pull(skb, 1))) { -+ ret = -EINVAL; -+ goto drop; -+ } ++static void lru_gen_seq_stop(struct seq_file *m, void *v) ++{ ++ if (!IS_ERR_OR_NULL(v)) ++ mem_cgroup_iter_break(NULL, lruvec_memcg(v)); + -+ /* check if special OpenVPN message */ -+ if (ovpn_is_keepalive(skb)) { -+ netdev_dbg(peer->ovpn->dev, "%s: ping received from peer with id %u\n", -+ __func__, peer->id); -+ /* not an error */ -+ consume_skb(skb); -+ /* inform the caller that NAPI should not be scheduled -+ * for this packet -+ */ -+ return -1; -+ } ++ kvfree(m->private); ++ m->private = NULL; ++} + -+ ret = -EPROTONOSUPPORT; -+ goto drop; -+ } -+ skb->protocol = proto; ++static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) ++{ ++ int nid = lruvec_pgdat(v)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(v); + -+ /* perform Reverse Path Filtering (RPF) */ -+ allowed_peer = ovpn_peer_lookup_vpn_addr(peer->ovpn, skb, true); -+ if (unlikely(allowed_peer != peer)) { -+ ret = -EPERM; -+ goto drop; -+ } ++ ++*pos; + -+ ret = ptr_ring_produce_bh(&peer->netif_rx_ring, skb); -+drop: -+ if (likely(allowed_peer)) -+ ovpn_peer_put(allowed_peer); ++ nid = next_memory_node(nid); ++ if (nid == MAX_NUMNODES) { ++ memcg = mem_cgroup_iter(NULL, memcg, NULL); ++ if (!memcg) ++ return NULL; + -+ if (unlikely(ret < 0)) -+ kfree_skb(skb); ++ nid = first_memory_node; ++ } + -+ return ret; ++ return get_lruvec(memcg, nid); +} + -+/* pick packet from RX queue, decrypt and forward it to the tun device */ -+void ovpn_decrypt_work(struct work_struct *work) ++static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, ++ unsigned long max_seq, unsigned long *min_seq, ++ unsigned long seq) +{ -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; ++ int i; ++ int type, tier; ++ int hist = lru_hist_from_seq(seq); ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ peer = container_of(work, struct ovpn_peer, decrypt_work); -+ while ((skb = ptr_ring_consume_bh(&peer->rx_ring))) { -+ if (likely(ovpn_decrypt_one(peer, skb) == 0)) { -+ /* if a packet has been enqueued for NAPI, signal -+ * availability to the networking stack -+ */ -+ local_bh_disable(); -+ napi_schedule(&peer->napi); -+ local_bh_enable(); ++ for (tier = 0; tier < MAX_NR_TIERS; tier++) { ++ seq_printf(m, " %10d", tier); ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ const char *s = " "; ++ unsigned long n[3] = {}; ++ ++ if (seq == max_seq) { ++ s = "RT "; ++ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); ++ n[1] = READ_ONCE(lrugen->avg_total[type][tier]); ++ } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { ++ s = "rep"; ++ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); ++ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); ++ if (tier) ++ n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); ++ } ++ ++ for (i = 0; i < 3; i++) ++ seq_printf(m, " %10lu%c", n[i], s[i]); + } ++ seq_putc(m, '\n'); ++ } + -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); ++ seq_puts(m, " "); ++ for (i = 0; i < NR_MM_STATS; i++) { ++ const char *s = " "; ++ unsigned long n = 0; ++ ++ if (seq == max_seq && NR_HIST_GENS == 1) { ++ s = "LOYNFA"; ++ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); ++ } else if (seq != max_seq && NR_HIST_GENS > 1) { ++ s = "loynfa"; ++ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); ++ } ++ ++ seq_printf(m, " %10lu%c", n, s[i]); + } -+ ovpn_peer_put(peer); ++ seq_putc(m, '\n'); +} + -+static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static int lru_gen_seq_show(struct seq_file *m, void *v) +{ -+ struct ovpn_crypto_key_slot *ks; -+ bool success = false; -+ int ret; ++ unsigned long seq; ++ bool full = !debugfs_real_fops(m->file)->write; ++ struct lruvec *lruvec = v; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ int nid = lruvec_pgdat(lruvec)->node_id; ++ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); + -+ /* get primary key to be used for encrypting data */ -+ ks = ovpn_crypto_key_slot_primary(&peer->crypto); -+ if (unlikely(!ks)) { -+ net_info_ratelimited("%s: error while retrieving primary key slot\n", __func__); -+ return false; -+ } ++ if (nid == first_memory_node) { ++ const char *path = memcg ? m->private : ""; + -+ if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && -+ skb_checksum_help(skb))) { -+ net_err_ratelimited("%s: cannot compute checksum for outgoing packet\n", __func__); -+ goto err; ++#ifdef CONFIG_MEMCG ++ if (memcg) ++ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); ++#endif ++ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); + } + -+ ovpn_peer_stats_increment_tx(&peer->stats, skb->len); -+ -+ /* encrypt */ -+ ret = ovpn_aead_encrypt(ks, skb, peer->id); -+ if (unlikely(ret < 0)) { -+ /* if we ran out of IVs we must kill the key as it can't be used anymore */ -+ if (ret == -ERANGE) { -+ netdev_warn(peer->ovpn->dev, -+ "%s: killing primary key as we ran out of IVs\n", __func__); -+ ovpn_crypto_kill_primary(&peer->crypto); -+ goto err; -+ } -+ net_err_ratelimited("%s: error during encryption for peer %u, key-id %u: %d\n", -+ __func__, peer->id, ks->key_id, ret); -+ goto err; -+ } ++ seq_printf(m, " node %5d\n", nid); + -+ success = true; -+err: -+ ovpn_crypto_key_slot_put(ks); -+ return success; -+} ++ if (!full) ++ seq = min_seq[LRU_GEN_ANON]; ++ else if (max_seq >= MAX_NR_GENS) ++ seq = max_seq - MAX_NR_GENS + 1; ++ else ++ seq = 0; + -+/* Process packets in TX queue in a transport-specific way. -+ * -+ * UDP transport - encrypt and send across the tunnel. -+ * TCP transport - encrypt and put into TCP TX queue. -+ */ -+void ovpn_encrypt_work(struct work_struct *work) -+{ -+ struct sk_buff *skb, *curr, *next; -+ struct ovpn_peer *peer; ++ for (; seq <= max_seq; seq++) { ++ int type, zone; ++ int gen = lru_gen_from_seq(seq); ++ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); + -+ peer = container_of(work, struct ovpn_peer, encrypt_work); -+ while ((skb = ptr_ring_consume_bh(&peer->tx_ring))) { -+ /* this might be a GSO-segmented skb list: process each skb -+ * independently -+ */ -+ skb_list_walk_safe(skb, curr, next) { -+ /* if one segment fails encryption, we drop the entire -+ * packet, because it does not really make sense to send -+ * only part of it at this point -+ */ -+ if (unlikely(!ovpn_encrypt_one(peer, curr))) { -+ kfree_skb_list(skb); -+ skb = NULL; -+ break; -+ } -+ } ++ seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); + -+ /* successful encryption */ -+ if (skb) { -+ skb_list_walk_safe(skb, curr, next) { -+ skb_mark_not_on_list(curr); ++ for (type = 0; type < ANON_AND_FILE; type++) { ++ unsigned long size = 0; ++ char mark = full && seq < min_seq[type] ? 'x' : ' '; + -+ switch (peer->sock->sock->sk->sk_protocol) { -+ case IPPROTO_UDP: -+ ovpn_udp_send_skb(peer->ovpn, peer, curr); -+ break; -+ case IPPROTO_TCP: -+ ovpn_tcp_send_skb(peer, curr); -+ break; -+ default: -+ /* no transport configured yet */ -+ consume_skb(skb); -+ break; -+ } -+ } ++ for (zone = 0; zone < MAX_NR_ZONES; zone++) ++ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + -+ /* note event of authenticated packet xmit for keepalive */ -+ ovpn_peer_keepalive_xmit_reset(peer); ++ seq_printf(m, " %10lu%c", size, mark); + } + -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); ++ seq_putc(m, '\n'); ++ ++ if (full) ++ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); + } -+ ovpn_peer_put(peer); ++ ++ return 0; +} + -+/* Put skb into TX queue and schedule a consumer */ -+static void ovpn_queue_skb(struct ovpn_struct *ovpn, struct sk_buff *skb, struct ovpn_peer *peer) ++static const struct seq_operations lru_gen_seq_ops = { ++ .start = lru_gen_seq_start, ++ .stop = lru_gen_seq_stop, ++ .next = lru_gen_seq_next, ++ .show = lru_gen_seq_show, ++}; ++ ++static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, ++ bool can_swap, bool force_scan) +{ -+ int ret; ++ DEFINE_MAX_SEQ(lruvec); ++ DEFINE_MIN_SEQ(lruvec); + -+ if (likely(!peer)) -+ peer = ovpn_peer_lookup_vpn_addr(ovpn, skb, false); -+ if (unlikely(!peer)) { -+ net_dbg_ratelimited("%s: no peer to send data to\n", ovpn->dev->name); -+ goto drop; -+ } ++ if (seq < max_seq) ++ return 0; + -+ ret = ptr_ring_produce_bh(&peer->tx_ring, skb); -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: cannot queue packet to TX ring\n", __func__); -+ goto drop; -+ } ++ if (seq > max_seq) ++ return -EINVAL; + -+ if (!queue_work(ovpn->crypto_wq, &peer->encrypt_work)) -+ ovpn_peer_put(peer); ++ if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) ++ return -ERANGE; + -+ return; -+drop: -+ if (peer) -+ ovpn_peer_put(peer); -+ kfree_skb_list(skb); ++ try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); ++ ++ return 0; +} + -+/* Net device start xmit -+ */ -+netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) ++static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, ++ int swappiness, unsigned long nr_to_reclaim) +{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ struct sk_buff *segments, *tmp, *curr, *next; -+ struct sk_buff_head skb_list; -+ __be16 proto; -+ int ret; -+ -+ /* reset netfilter state */ -+ nf_reset_ct(skb); ++ DEFINE_MAX_SEQ(lruvec); + -+ /* verify IP header size in network packet */ -+ proto = ovpn_ip_check_protocol(skb); -+ if (unlikely(!proto || skb->protocol != proto)) { -+ net_dbg_ratelimited("%s: dropping malformed payload packet\n", -+ dev->name); -+ goto drop; -+ } ++ if (seq + MIN_NR_GENS > max_seq) ++ return -EINVAL; + -+ if (skb_is_gso(skb)) { -+ segments = skb_gso_segment(skb, 0); -+ if (IS_ERR(segments)) { -+ ret = PTR_ERR(segments); -+ net_dbg_ratelimited("%s: cannot segment packet: %d\n", dev->name, ret); -+ goto drop; -+ } ++ sc->nr_reclaimed = 0; + -+ consume_skb(skb); -+ skb = segments; -+ } ++ while (!signal_pending(current)) { ++ DEFINE_MIN_SEQ(lruvec); + -+ /* from this moment on, "skb" might be a list */ ++ if (seq < min_seq[!swappiness]) ++ return 0; + -+ __skb_queue_head_init(&skb_list); -+ skb_list_walk_safe(skb, curr, next) { -+ skb_mark_not_on_list(curr); ++ if (sc->nr_reclaimed >= nr_to_reclaim) ++ return 0; + -+ tmp = skb_share_check(curr, GFP_ATOMIC); -+ if (unlikely(!tmp)) { -+ kfree_skb_list(next); -+ net_dbg_ratelimited("%s: skb_share_check failed\n", dev->name); -+ goto drop_list; -+ } ++ if (!evict_folios(lruvec, sc, swappiness, NULL)) ++ return 0; + -+ __skb_queue_tail(&skb_list, tmp); ++ cond_resched(); + } -+ skb_list.prev->next = NULL; -+ -+ ovpn_queue_skb(ovpn, skb_list.next, NULL); -+ -+ return NETDEV_TX_OK; + -+drop_list: -+ skb_queue_walk_safe(&skb_list, curr, next) -+ kfree_skb(curr); -+drop: -+ skb_tx_error(skb); -+ kfree_skb_list(skb); -+ return NET_XMIT_DROP; ++ return -EINTR; +} + -+/* Encrypt and transmit a special message to peer, such as keepalive -+ * or explicit-exit-notify. Called from softirq context. -+ * Assumes that caller holds a reference to peer. -+ */ -+static void ovpn_xmit_special(struct ovpn_peer *peer, const void *data, -+ const unsigned int len) ++static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, ++ struct scan_control *sc, int swappiness, unsigned long opt) +{ -+ struct ovpn_struct *ovpn; -+ struct sk_buff *skb; -+ -+ ovpn = peer->ovpn; -+ if (unlikely(!ovpn)) -+ return; ++ struct lruvec *lruvec; ++ int err = -EINVAL; ++ struct mem_cgroup *memcg = NULL; + -+ skb = alloc_skb(256 + len, GFP_ATOMIC); -+ if (unlikely(!skb)) -+ return; ++ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) ++ return -EINVAL; + -+ skb_reserve(skb, 128); -+ skb->priority = TC_PRIO_BESTEFFORT; -+ memcpy(__skb_put(skb, len), data, len); ++ if (!mem_cgroup_disabled()) { ++ rcu_read_lock(); ++ memcg = mem_cgroup_from_id(memcg_id); ++#ifdef CONFIG_MEMCG ++ if (memcg && !css_tryget(&memcg->css)) ++ memcg = NULL; ++#endif ++ rcu_read_unlock(); + -+ /* increase reference counter when passing peer to sending queue */ -+ if (!ovpn_peer_hold(peer)) { -+ netdev_dbg(ovpn->dev, "%s: cannot hold peer reference for sending special packet\n", -+ __func__); -+ kfree_skb(skb); -+ return; ++ if (!memcg) ++ return -EINVAL; + } + -+ ovpn_queue_skb(ovpn, skb, peer); -+} ++ if (memcg_id != mem_cgroup_id(memcg)) ++ goto done; + -+void ovpn_keepalive_xmit(struct ovpn_peer *peer) -+{ -+ ovpn_xmit_special(peer, ovpn_keepalive_message, -+ sizeof(ovpn_keepalive_message)); -+} ++ lruvec = get_lruvec(memcg, nid); + -+/* Transmit explicit exit notification. -+ * Called from process context. -+ */ -+void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer) -+{ -+ ovpn_xmit_special(peer, ovpn_explicit_exit_notify_message, -+ sizeof(ovpn_explicit_exit_notify_message)); ++ if (swappiness < 0) ++ swappiness = get_swappiness(lruvec, sc); ++ else if (swappiness > 200) ++ goto done; ++ ++ switch (cmd) { ++ case '+': ++ err = run_aging(lruvec, seq, sc, swappiness, opt); ++ break; ++ case '-': ++ err = run_eviction(lruvec, seq, sc, swappiness, opt); ++ break; ++ } ++done: ++ mem_cgroup_put(memcg); ++ ++ return err; +} + -+/* Copy buffer into skb and send it across the tunnel. -+ * -+ * For UDP transport: just sent the skb to peer -+ * For TCP transport: put skb into TX queue -+ */ -+int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len) ++/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ ++static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, ++ size_t len, loff_t *pos) +{ -+ u16 skb_len = SKB_HEADER_LEN + len; -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ bool tcp = false; -+ int ret = 0; -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (unlikely(!peer)) { -+ netdev_dbg(ovpn->dev, "no peer to send data to\n"); -+ return -EHOSTUNREACH; -+ } ++ void *buf; ++ char *cur, *next; ++ unsigned int flags; ++ struct blk_plug plug; ++ int err = -EINVAL; ++ struct scan_control sc = { ++ .may_writepage = true, ++ .may_unmap = true, ++ .may_swap = true, ++ .reclaim_idx = MAX_NR_ZONES - 1, ++ .gfp_mask = GFP_KERNEL, ++ }; + -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) { -+ skb_len += sizeof(u16); -+ tcp = true; -+ } ++ buf = kvmalloc(len + 1, GFP_KERNEL); ++ if (!buf) ++ return -ENOMEM; + -+ skb = alloc_skb(skb_len, GFP_ATOMIC); -+ if (unlikely(!skb)) { -+ ret = -ENOMEM; -+ goto out; ++ if (copy_from_user(buf, src, len)) { ++ kvfree(buf); ++ return -EFAULT; + } + -+ skb_reserve(skb, SKB_HEADER_LEN); -+ skb_put_data(skb, data, len); -+ -+ /* prepend TCP packet with size, as required by OpenVPN protocol */ -+ if (tcp) { -+ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); -+ ovpn_queue_tcp_skb(peer, skb); -+ } else { -+ ovpn_udp_send_skb(ovpn, peer, skb); ++ set_task_reclaim_state(current, &sc.reclaim_state); ++ flags = memalloc_noreclaim_save(); ++ blk_start_plug(&plug); ++ if (!set_mm_walk(NULL)) { ++ err = -ENOMEM; ++ goto done; + } -+out: -+ ovpn_peer_put(peer); -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/ovpn.h b/drivers/net/ovpn-dco/ovpn.h -new file mode 100644 -index 000000000000..9364fd5dd309 ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpn.h -@@ -0,0 +1,43 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ + -+#ifndef _NET_OVPN_DCO_OVPN_H_ -+#define _NET_OVPN_DCO_OVPN_H_ ++ next = buf; ++ next[len] = '\0'; + -+#include "main.h" -+#include "peer.h" -+#include "sock.h" -+#include "ovpnstruct.h" ++ while ((cur = strsep(&next, ",;\n"))) { ++ int n; ++ int end; ++ char cmd; ++ unsigned int memcg_id; ++ unsigned int nid; ++ unsigned long seq; ++ unsigned int swappiness = -1; ++ unsigned long opt = -1; + -+#include -+#include -+#include ++ cur = skip_spaces(cur); ++ if (!*cur) ++ continue; + -+struct ovpn_struct; -+struct net_device; ++ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, ++ &seq, &end, &swappiness, &end, &opt, &end); ++ if (n < 4 || cur[end]) { ++ err = -EINVAL; ++ break; ++ } + -+int ovpn_struct_init(struct net_device *dev); ++ err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); ++ if (err) ++ break; ++ } ++done: ++ clear_mm_walk(); ++ blk_finish_plug(&plug); ++ memalloc_noreclaim_restore(flags); ++ set_task_reclaim_state(current, NULL); + -+u16 ovpn_select_queue(struct net_device *dev, struct sk_buff *skb, -+ struct net_device *sb_dev); ++ kvfree(buf); + -+void ovpn_keepalive_xmit(struct ovpn_peer *peer); -+void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer); ++ return err ? : len; ++} + -+netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev); ++static int lru_gen_seq_open(struct inode *inode, struct file *file) ++{ ++ return seq_open(file, &lru_gen_seq_ops); ++} + -+int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb); ++static const struct file_operations lru_gen_rw_fops = { ++ .open = lru_gen_seq_open, ++ .read = seq_read, ++ .write = lru_gen_seq_write, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; + -+void ovpn_encrypt_work(struct work_struct *work); -+void ovpn_decrypt_work(struct work_struct *work); -+int ovpn_napi_poll(struct napi_struct *napi, int budget); ++static const struct file_operations lru_gen_ro_fops = { ++ .open = lru_gen_seq_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; + -+int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len); ++/****************************************************************************** ++ * initialization ++ ******************************************************************************/ + -+#endif /* _NET_OVPN_DCO_OVPN_H_ */ -diff --git a/drivers/net/ovpn-dco/ovpnstruct.h b/drivers/net/ovpn-dco/ovpnstruct.h -new file mode 100644 -index 000000000000..f9bc559609cd ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpnstruct.h -@@ -0,0 +1,59 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++void lru_gen_init_lruvec(struct lruvec *lruvec) ++{ ++ int i; ++ int gen, type, zone; ++ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+#ifndef _NET_OVPN_DCO_OVPNSTRUCT_H_ -+#define _NET_OVPN_DCO_OVPNSTRUCT_H_ ++ lrugen->max_seq = MIN_NR_GENS + 1; ++ lrugen->enabled = lru_gen_enabled(); + -+#include "peer.h" ++ for (i = 0; i <= MIN_NR_GENS + 1; i++) ++ lrugen->timestamps[i] = jiffies; + -+#include -+#include -+#include ++ for_each_gen_type_zone(gen, type, zone) ++ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); + -+/* Our state per ovpn interface */ -+struct ovpn_struct { -+ /* read-mostly objects in this section */ -+ struct net_device *dev; ++ lruvec->mm_state.seq = MIN_NR_GENS; ++ init_waitqueue_head(&lruvec->mm_state.wait); ++} + -+ /* device operation mode (i.e. P2P, MP) */ -+ enum ovpn_mode mode; ++#ifdef CONFIG_MEMCG ++void lru_gen_init_memcg(struct mem_cgroup *memcg) ++{ ++ INIT_LIST_HEAD(&memcg->mm_list.fifo); ++ spin_lock_init(&memcg->mm_list.lock); ++} + -+ /* protect writing to the ovpn_struct object */ -+ spinlock_t lock; ++void lru_gen_exit_memcg(struct mem_cgroup *memcg) ++{ ++ int i; ++ int nid; + -+ /* workqueue used to schedule crypto work that may sleep */ -+ struct workqueue_struct *crypto_wq; -+ /* workqueue used to schedule generic event that may sleep or that need -+ * to be performed out of softirq context -+ */ -+ struct workqueue_struct *events_wq; ++ for_each_node(nid) { ++ struct lruvec *lruvec = get_lruvec(memcg, nid); + -+ /* list of known peers */ -+ struct { -+ DECLARE_HASHTABLE(by_id, 12); -+ DECLARE_HASHTABLE(by_transp_addr, 12); -+ DECLARE_HASHTABLE(by_vpn_addr, 12); -+ /* protects write access to any of the hashtables above */ -+ spinlock_t lock; -+ } peers; ++ VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, ++ sizeof(lruvec->lrugen.nr_pages))); + -+ /* for p2p mode */ -+ struct ovpn_peer __rcu *peer; ++ for (i = 0; i < NR_BLOOM_FILTERS; i++) { ++ bitmap_free(lruvec->mm_state.filters[i]); ++ lruvec->mm_state.filters[i] = NULL; ++ } ++ } ++} ++#endif + -+ unsigned int max_tun_queue_len; ++static int __init init_lru_gen(void) ++{ ++ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); ++ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); + -+ netdev_features_t set_features; ++ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) ++ pr_err("lru_gen: failed to create sysfs group\n"); + -+ void *security; ++ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); ++ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); + -+ u32 registered_nl_portid; -+ bool registered_nl_portid_set; ++ return 0; +}; ++late_initcall(init_lru_gen); + -+#endif /* _NET_OVPN_DCO_OVPNSTRUCT_H_ */ -diff --git a/drivers/net/ovpn-dco/peer.c b/drivers/net/ovpn-dco/peer.c -new file mode 100644 -index 000000000000..87d3f1b34c4d ---- /dev/null -+++ b/drivers/net/ovpn-dco/peer.c -@@ -0,0 +1,906 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "ovpn.h" -+#include "bind.h" -+#include "crypto.h" -+#include "peer.h" -+#include "netlink.h" -+#include "tcp.h" -+ -+#include -+#include -+#include -+#include -+#include ++#else /* !CONFIG_LRU_GEN */ + -+static void ovpn_peer_ping(struct timer_list *t) ++static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +{ -+ struct ovpn_peer *peer = from_timer(peer, t, keepalive_xmit); ++} + -+ netdev_dbg(peer->ovpn->dev, "%s: sending ping to peer %u\n", __func__, peer->id); -+ ovpn_keepalive_xmit(peer); ++static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++{ +} + -+static void ovpn_peer_expire(struct timer_list *t) ++#endif /* CONFIG_LRU_GEN */ ++ ++static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ -+ struct ovpn_peer *peer = from_timer(peer, t, keepalive_recv); ++ unsigned long nr[NR_LRU_LISTS]; ++ unsigned long targets[NR_LRU_LISTS]; ++ unsigned long nr_to_scan; ++ enum lru_list lru; ++ unsigned long nr_reclaimed = 0; ++ unsigned long nr_to_reclaim = sc->nr_to_reclaim; ++ bool proportional_reclaim; ++ struct blk_plug plug; ++ ++ if (lru_gen_enabled()) { ++ lru_gen_shrink_lruvec(lruvec, sc); ++ return; ++ } ++ ++ get_scan_count(lruvec, sc, nr); ++ ++ /* Record the original scan target for proportional adjustments later */ ++ memcpy(targets, nr, sizeof(nr)); ++ ++ /* ++ * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal ++ * event that can occur when there is little memory pressure e.g. ++ * multiple streaming readers/writers. Hence, we do not abort scanning ++ * when the requested number of pages are reclaimed when scanning at ++ * DEF_PRIORITY on the assumption that the fact we are direct ++ * reclaiming implies that kswapd is not keeping up and it is best to ++ * do a batch of work at once. For memcg reclaim one check is made to ++ * abort proportional reclaim if either the file or anon lru has already ++ * dropped to zero at the first pass. ++ */ ++ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && ++ sc->priority == DEF_PRIORITY); + -+ netdev_dbg(peer->ovpn->dev, "%s: peer %u expired\n", __func__, peer->id); -+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_EXPIRED); -+} ++ blk_start_plug(&plug); ++ while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || ++ nr[LRU_INACTIVE_FILE]) { ++ unsigned long nr_anon, nr_file, percentage; ++ unsigned long nr_scanned; + -+/* Construct a new peer */ -+static struct ovpn_peer *ovpn_peer_create(struct ovpn_struct *ovpn, u32 id) -+{ -+ struct ovpn_peer *peer; -+ int ret; ++ for_each_evictable_lru(lru) { ++ if (nr[lru]) { ++ nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); ++ nr[lru] -= nr_to_scan; + -+ /* alloc and init peer object */ -+ peer = kzalloc(sizeof(*peer), GFP_KERNEL); -+ if (!peer) -+ return ERR_PTR(-ENOMEM); ++ nr_reclaimed += shrink_list(lru, nr_to_scan, ++ lruvec, sc); ++ } ++ } + -+ peer->id = id; -+ peer->halt = false; -+ peer->ovpn = ovpn; ++ cond_resched(); + -+ peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY); -+ peer->vpn_addrs.ipv6 = in6addr_any; ++ if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) ++ continue; + -+ RCU_INIT_POINTER(peer->bind, NULL); -+ ovpn_crypto_state_init(&peer->crypto); -+ spin_lock_init(&peer->lock); -+ kref_init(&peer->refcount); -+ ovpn_peer_stats_init(&peer->stats); ++ /* ++ * For kswapd and memcg, reclaim at least the number of pages ++ * requested. Ensure that the anon and file LRUs are scanned ++ * proportionally what was requested by get_scan_count(). We ++ * stop reclaiming one LRU and reduce the amount scanning ++ * proportional to the original scan target. ++ */ ++ nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; ++ nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; + -+ INIT_WORK(&peer->encrypt_work, ovpn_encrypt_work); -+ INIT_WORK(&peer->decrypt_work, ovpn_decrypt_work); ++ /* ++ * It's just vindictive to attack the larger once the smaller ++ * has gone to zero. And given the way we stop scanning the ++ * smaller below, this makes sure that we only make one nudge ++ * towards proportionality once we've got nr_to_reclaim. ++ */ ++ if (!nr_file || !nr_anon) ++ break; + -+ ret = dst_cache_init(&peer->dst_cache, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot initialize dst cache\n", __func__); -+ goto err; -+ } ++ if (nr_file > nr_anon) { ++ unsigned long scan_target = targets[LRU_INACTIVE_ANON] + ++ targets[LRU_ACTIVE_ANON] + 1; ++ lru = LRU_BASE; ++ percentage = nr_anon * 100 / scan_target; ++ } else { ++ unsigned long scan_target = targets[LRU_INACTIVE_FILE] + ++ targets[LRU_ACTIVE_FILE] + 1; ++ lru = LRU_FILE; ++ percentage = nr_file * 100 / scan_target; ++ } + -+ ret = ptr_ring_init(&peer->tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate TX ring\n", __func__); -+ goto err_dst_cache; -+ } ++ /* Stop scanning the smaller of the LRU */ ++ nr[lru] = 0; ++ nr[lru + LRU_ACTIVE] = 0; + -+ ret = ptr_ring_init(&peer->rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate RX ring\n", __func__); -+ goto err_tx_ring; -+ } ++ /* ++ * Recalculate the other LRU scan count based on its original ++ * scan target and the percentage scanning already complete ++ */ ++ lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; ++ nr_scanned = targets[lru] - nr[lru]; ++ nr[lru] = targets[lru] * (100 - percentage) / 100; ++ nr[lru] -= min(nr[lru], nr_scanned); + -+ ret = ptr_ring_init(&peer->netif_rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate NETIF RX ring\n", __func__); -+ goto err_rx_ring; ++ lru += LRU_ACTIVE; ++ nr_scanned = targets[lru] - nr[lru]; ++ nr[lru] = targets[lru] * (100 - percentage) / 100; ++ nr[lru] -= min(nr[lru], nr_scanned); + } ++ blk_finish_plug(&plug); ++ sc->nr_reclaimed += nr_reclaimed; + -+ /* configure and start NAPI */ -+ netif_tx_napi_add(ovpn->dev, &peer->napi, ovpn_napi_poll, -+ NAPI_POLL_WEIGHT); -+ napi_enable(&peer->napi); -+ -+ dev_hold(ovpn->dev); ++ /* ++ * Even if we did not try to evict anon pages at all, we want to ++ * rebalance the anon lru active/inactive ratio. ++ */ ++ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && ++ inactive_is_low(lruvec, LRU_INACTIVE_ANON)) ++ shrink_active_list(SWAP_CLUSTER_MAX, lruvec, ++ sc, LRU_ACTIVE_ANON); ++} + -+ timer_setup(&peer->keepalive_xmit, ovpn_peer_ping, 0); -+ timer_setup(&peer->keepalive_recv, ovpn_peer_expire, 0); ++/* Use reclaim/compaction for costly allocs or under memory pressure */ ++static bool in_reclaim_compaction(struct scan_control *sc) ++{ ++ if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && ++ (sc->order > PAGE_ALLOC_COSTLY_ORDER || ++ sc->priority < DEF_PRIORITY - 2)) ++ return true; + -+ return peer; -+err_rx_ring: -+ ptr_ring_cleanup(&peer->rx_ring, NULL); -+err_tx_ring: -+ ptr_ring_cleanup(&peer->tx_ring, NULL); -+err_dst_cache: -+ dst_cache_destroy(&peer->dst_cache); -+err: -+ kfree(peer); -+ return ERR_PTR(ret); ++ return false; +} + -+/* Reset the ovpn_sockaddr associated with a peer */ -+static int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, const struct sockaddr_storage *ss, -+ const u8 *local_ip) ++/* ++ * Reclaim/compaction is used for high-order allocation requests. It reclaims ++ * order-0 pages before compacting the zone. should_continue_reclaim() returns ++ * true if more pages should be reclaimed such that when the page allocator ++ * calls try_to_compact_pages() that it will have enough free pages to succeed. ++ * It will give up earlier than that if there is difficulty reclaiming pages. ++ */ ++static inline bool should_continue_reclaim(struct pglist_data *pgdat, ++ unsigned long nr_reclaimed, ++ struct scan_control *sc) +{ -+ struct ovpn_bind *bind; -+ size_t ip_len; ++ unsigned long pages_for_compaction; ++ unsigned long inactive_lru_pages; ++ int z; + -+ /* create new ovpn_bind object */ -+ bind = ovpn_bind_from_sockaddr(ss); -+ if (IS_ERR(bind)) -+ return PTR_ERR(bind); ++ /* If not in reclaim/compaction mode, stop */ ++ if (!in_reclaim_compaction(sc)) ++ return false; + -+ if (local_ip) { -+ if (ss->ss_family == AF_INET) { -+ ip_len = sizeof(struct in_addr); -+ } else if (ss->ss_family == AF_INET6) { -+ ip_len = sizeof(struct in6_addr); -+ } else { -+ netdev_dbg(peer->ovpn->dev, "%s: invalid family for remote endpoint\n", -+ __func__); -+ kfree(bind); -+ return -EINVAL; -+ } ++ /* ++ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX ++ * number of pages that were scanned. This will return to the caller ++ * with the risk reclaim/compaction and the resulting allocation attempt ++ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL ++ * allocations through requiring that the full LRU list has been scanned ++ * first, by assuming that zero delta of sc->nr_scanned means full LRU ++ * scan, but that approximation was wrong, and there were corner cases + * where always a non-zero amount of pages were scanned. + */ + if (!nr_reclaimed) +@@ -3230,109 +6106,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) + unsigned long nr_reclaimed, nr_scanned; + struct lruvec *target_lruvec; + bool reclaimable = false; +- unsigned long file; + + target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); + + again: +- /* +- * Flush the memory cgroup stats, so that we read accurate per-memcg +- * lruvec stats for heuristics. +- */ +- mem_cgroup_flush_stats(); +- + memset(&sc->nr, 0, sizeof(sc->nr)); + + nr_reclaimed = sc->nr_reclaimed; + nr_scanned = sc->nr_scanned; + +- /* +- * Determine the scan balance between anon and file LRUs. +- */ +- spin_lock_irq(&target_lruvec->lru_lock); +- sc->anon_cost = target_lruvec->anon_cost; +- sc->file_cost = target_lruvec->file_cost; +- spin_unlock_irq(&target_lruvec->lru_lock); +- +- /* +- * Target desirable inactive:active list ratios for the anon +- * and file LRU lists. +- */ +- if (!sc->force_deactivate) { +- unsigned long refaults; +- +- refaults = lruvec_page_state(target_lruvec, +- WORKINGSET_ACTIVATE_ANON); +- if (refaults != target_lruvec->refaults[0] || +- inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) +- sc->may_deactivate |= DEACTIVATE_ANON; +- else +- sc->may_deactivate &= ~DEACTIVATE_ANON; +- +- /* +- * When refaults are being observed, it means a new +- * workingset is being established. Deactivate to get +- * rid of any stale active pages quickly. +- */ +- refaults = lruvec_page_state(target_lruvec, +- WORKINGSET_ACTIVATE_FILE); +- if (refaults != target_lruvec->refaults[1] || +- inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) +- sc->may_deactivate |= DEACTIVATE_FILE; +- else +- sc->may_deactivate &= ~DEACTIVATE_FILE; +- } else +- sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; +- +- /* +- * If we have plenty of inactive file pages that aren't +- * thrashing, try to reclaim those first before touching +- * anonymous pages. +- */ +- file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); +- if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) +- sc->cache_trim_mode = 1; +- else +- sc->cache_trim_mode = 0; +- +- /* +- * Prevent the reclaimer from falling into the cache trap: as +- * cache pages start out inactive, every cache fault will tip +- * the scan balance towards the file LRU. And as the file LRU +- * shrinks, so does the window for rotation from references. +- * This means we have a runaway feedback loop where a tiny +- * thrashing file LRU becomes infinitely more attractive than +- * anon pages. Try to detect this based on file LRU size. +- */ +- if (!cgroup_reclaim(sc)) { +- unsigned long total_high_wmark = 0; +- unsigned long free, anon; +- int z; +- +- free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); +- file = node_page_state(pgdat, NR_ACTIVE_FILE) + +- node_page_state(pgdat, NR_INACTIVE_FILE); +- +- for (z = 0; z < MAX_NR_ZONES; z++) { +- struct zone *zone = &pgdat->node_zones[z]; +- if (!managed_zone(zone)) +- continue; +- +- total_high_wmark += high_wmark_pages(zone); +- } +- +- /* +- * Consider anon: if that's low too, this isn't a +- * runaway file reclaim problem, but rather just +- * extreme pressure. Reclaim as per usual then. +- */ +- anon = node_page_state(pgdat, NR_INACTIVE_ANON); +- +- sc->file_is_tiny = +- file + free <= total_high_wmark && +- !(sc->may_deactivate & DEACTIVATE_ANON) && +- anon >> sc->priority; +- } ++ prepare_scan_count(pgdat, sc); + + shrink_node_memcgs(pgdat, sc); + +@@ -3590,11 +6373,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) + struct lruvec *target_lruvec; + unsigned long refaults; + ++ if (lru_gen_enabled()) ++ return; + -+ memcpy(&bind->local, local_ip, ip_len); + target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); + refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); +- target_lruvec->refaults[0] = refaults; ++ target_lruvec->refaults[WORKINGSET_ANON] = refaults; + refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); +- target_lruvec->refaults[1] = refaults; ++ target_lruvec->refaults[WORKINGSET_FILE] = refaults; + } + + /* +@@ -3956,12 +6742,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + } + #endif + +-static void age_active_anon(struct pglist_data *pgdat, +- struct scan_control *sc) ++static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) + { + struct mem_cgroup *memcg; + struct lruvec *lruvec; + ++ if (lru_gen_enabled()) { ++ lru_gen_age_node(pgdat, sc); ++ return; + } + -+ /* set binding */ -+ ovpn_bind_reset(peer, bind); + if (!can_age_anon_pages(pgdat, sc)) + return; + +@@ -4281,12 +7071,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) + sc.may_swap = !nr_boost_reclaim; + + /* +- * Do some background aging of the anon list, to give +- * pages a chance to be referenced before reclaiming. All +- * pages are rotated regardless of classzone as this is +- * about consistent aging. ++ * Do some background aging, to give pages a chance to be ++ * referenced before reclaiming. All pages are rotated ++ * regardless of classzone as this is about consistent aging. + */ +- age_active_anon(pgdat, &sc); ++ kswapd_age_node(pgdat, &sc); + + /* + * If we're getting trouble reclaiming, start doing writepage +diff --git a/mm/workingset.c b/mm/workingset.c +index a5e84862fc86..ae7e984b23c6 100644 +--- a/mm/workingset.c ++++ b/mm/workingset.c +@@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; + static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, + bool workingset) + { +- eviction >>= bucket_order; + eviction &= EVICTION_MASK; + eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; + eviction = (eviction << NODES_SHIFT) | pgdat->node_id; +@@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, + + *memcgidp = memcgid; + *pgdat = NODE_DATA(nid); +- *evictionp = entry << bucket_order; ++ *evictionp = entry; + *workingsetp = workingset; + } + ++#ifdef CONFIG_LRU_GEN + -+ return 0; ++static void *lru_gen_eviction(struct folio *folio) ++{ ++ int hist; ++ unsigned long token; ++ unsigned long min_seq; ++ struct lruvec *lruvec; ++ struct lru_gen_struct *lrugen; ++ int type = folio_is_file_lru(folio); ++ int delta = folio_nr_pages(folio); ++ int refs = folio_lru_refs(folio); ++ int tier = lru_tier_from_refs(refs); ++ struct mem_cgroup *memcg = folio_memcg(folio); ++ struct pglist_data *pgdat = folio_pgdat(folio); ++ ++ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); ++ ++ lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ lrugen = &lruvec->lrugen; ++ min_seq = READ_ONCE(lrugen->min_seq[type]); ++ token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); ++ ++ hist = lru_hist_from_seq(min_seq); ++ atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); ++ ++ return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); +} + -+void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb) ++static void lru_gen_refault(struct folio *folio, void *shadow) +{ -+ struct sockaddr_storage ss; -+ const u8 *local_ip = NULL; -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa; -+ struct ovpn_bind *bind; -+ sa_family_t family; ++ int hist, tier, refs; ++ int memcg_id; ++ bool workingset; ++ unsigned long token; ++ unsigned long min_seq; ++ struct lruvec *lruvec; ++ struct lru_gen_struct *lrugen; ++ struct mem_cgroup *memcg; ++ struct pglist_data *pgdat; ++ int type = folio_is_file_lru(folio); ++ int delta = folio_nr_pages(folio); ++ ++ unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); ++ ++ if (pgdat != folio_pgdat(folio)) ++ return; + + rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) ++ ++ memcg = folio_memcg_rcu(folio); ++ if (memcg_id != mem_cgroup_id(memcg)) + goto unlock; + -+ if (likely(ovpn_bind_skb_src_match(bind, skb))) ++ lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ lrugen = &lruvec->lrugen; ++ ++ min_seq = READ_ONCE(lrugen->min_seq[type]); ++ if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) + goto unlock; + -+ family = skb_protocol_to_family(skb); ++ hist = lru_hist_from_seq(min_seq); ++ /* see the comment in folio_lru_refs() */ ++ refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; ++ tier = lru_tier_from_refs(refs); + -+ if (bind->sa.in4.sin_family == family) -+ local_ip = (u8 *)&bind->local; ++ atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); ++ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); + -+ switch (family) { -+ case AF_INET: -+ sa = (struct sockaddr_in *)&ss; -+ sa->sin_family = AF_INET; -+ sa->sin_addr.s_addr = ip_hdr(skb)->saddr; -+ sa->sin_port = udp_hdr(skb)->source; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)&ss; -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = ipv6_hdr(skb)->saddr; -+ sa6->sin6_port = udp_hdr(skb)->source; -+ sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); -+ break; -+ default: -+ goto unlock; ++ /* ++ * Count the following two cases as stalls: ++ * 1. For pages accessed through page tables, hotter pages pushed out ++ * hot pages which refaulted immediately. ++ * 2. For pages accessed multiple times through file descriptors, ++ * numbers of accesses might have been out of the range. ++ */ ++ if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { ++ folio_set_workingset(folio); ++ mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); + } -+ -+ netdev_dbg(peer->ovpn->dev, "%s: peer %d floated to %pIScp", __func__, peer->id, &ss); -+ ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip); +unlock: + rcu_read_unlock(); +} + -+static void ovpn_peer_timer_delete_all(struct ovpn_peer *peer) ++#else /* !CONFIG_LRU_GEN */ ++ ++static void *lru_gen_eviction(struct folio *folio) +{ -+ del_timer_sync(&peer->keepalive_xmit); -+ del_timer_sync(&peer->keepalive_recv); ++ return NULL; +} + -+static void ovpn_peer_free(struct ovpn_peer *peer) ++static void lru_gen_refault(struct folio *folio, void *shadow) +{ -+ ovpn_bind_reset(peer, NULL); -+ ovpn_peer_timer_delete_all(peer); ++} + -+ WARN_ON(!__ptr_ring_empty(&peer->tx_ring)); -+ ptr_ring_cleanup(&peer->tx_ring, NULL); -+ WARN_ON(!__ptr_ring_empty(&peer->rx_ring)); -+ ptr_ring_cleanup(&peer->rx_ring, NULL); -+ WARN_ON(!__ptr_ring_empty(&peer->netif_rx_ring)); -+ ptr_ring_cleanup(&peer->netif_rx_ring, NULL); ++#endif /* CONFIG_LRU_GEN */ + -+ dst_cache_destroy(&peer->dst_cache); + /** + * workingset_age_nonresident - age non-resident entries as LRU ages + * @lruvec: the lruvec that was aged +@@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) + VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + ++ if (lru_gen_enabled()) ++ return lru_gen_eviction(folio); + -+ dev_put(peer->ovpn->dev); + lruvec = mem_cgroup_lruvec(target_memcg, pgdat); + /* XXX: target_memcg can be NULL, go through lruvec */ + memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); + eviction = atomic_long_read(&lruvec->nonresident_age); ++ eviction >>= bucket_order; + workingset_age_nonresident(lruvec, folio_nr_pages(folio)); + return pack_shadow(memcgid, pgdat, eviction, + folio_test_workingset(folio)); +@@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) + int memcgid; + long nr; + ++ if (lru_gen_enabled()) { ++ lru_gen_refault(folio, shadow); ++ return; ++ } + -+ kfree(peer); -+} + unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); ++ eviction <<= bucket_order; + + rcu_read_lock(); + /* +-- +2.38.0.rc1.8.g2a7d63a245 + +From f7046da0d2b40d6725122f9d3ed897a12a8fda63 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Wed, 28 Sep 2022 00:27:32 +0200 +Subject: [PATCH 08/16] Introducing the Maple Tree + +The maple tree is an RCU-safe range based B-tree designed to use modern +processor cache efficiently. There are a number of places in the kernel +that a non-overlapping range-based tree would be beneficial, especially +one with a simple interface. If you use an rbtree with other data +structures to improve performance or an interval tree to track +non-overlapping ranges, then this is for you. + +The tree has a branching factor of 10 for non-leaf nodes and 16 for leaf +nodes. With the increased branching factor, it is significantly shorter +than the rbtree so it has fewer cache misses. The removal of the linked +list between subsequent entries also reduces the cache misses and the need +to pull in the previous and next VMA during many tree alterations. + +The first user that is covered in this patch set is the vm_area_struct, +where three data structures are replaced by the maple tree: the augmented +rbtree, the vma cache, and the linked list of VMAs in the mm_struct. The +long term goal is to reduce or remove the mmap_lock contention. + +The plan is to get to the point where we use the maple tree in RCU mode. +Readers will not block for writers. A single write operation will be +allowed at a time. A reader re-walks if stale data is encountered. VMAs +would be RCU enabled and this mode would be entered once multiple tasks +are using the mm_struct. + +Davidlor said + +: Yes I like the maple tree, and at this stage I don't think we can ask for +: more from this series wrt the MM - albeit there seems to still be some +: folks reporting breakage. Fundamentally I see Liam's work to (re)move +: complexity out of the MM (not to say that the actual maple tree is not +: complex) by consolidating the three complimentary data structures very +: much worth it considering performance does not take a hit. This was very +: much a turn off with the range locking approach, which worst case scenario +: incurred in prohibitive overhead. Also as Liam and Matthew have +: mentioned, RCU opens up a lot of nice performance opportunities, and in +: addition academia[1] has shown outstanding scalability of address spaces +: with the foundation of replacing the locked rbtree with RCU aware trees. + +A similar work has been discovered in the academic press + + https://pdos.csail.mit.edu/papers/rcuvm:asplos12.pdf + +Sheer coincidence. We designed our tree with the intention of solving the +hardest problem first. Upon settling on a b-tree variant and a rough +outline, we researched ranged based b-trees and RCU b-trees and did find +that article. So it was nice to find reassurances that we were on the +right path, but our design choice of using ranges made that paper unusable +for us. + +This patch (of 70): + +The maple tree is an RCU-safe range based B-tree designed to use modern +processor cache efficiently. There are a number of places in the kernel +that a non-overlapping range-based tree would be beneficial, especially +one with a simple interface. If you use an rbtree with other data +structures to improve performance or an interval tree to track +non-overlapping ranges, then this is for you. + +The tree has a branching factor of 10 for non-leaf nodes and 16 for leaf +nodes. With the increased branching factor, it is significantly shorter +than the rbtree so it has fewer cache misses. The removal of the linked +list between subsequent entries also reduces the cache misses and the need +to pull in the previous and next VMA during many tree alterations. + +The first user that is covered in this patch set is the vm_area_struct, +where three data structures are replaced by the maple tree: the augmented +rbtree, the vma cache, and the linked list of VMAs in the mm_struct. The +long term goal is to reduce or remove the mmap_lock contention. + +The plan is to get to the point where we use the maple tree in RCU mode. +Readers will not block for writers. A single write operation will be +allowed at a time. A reader re-walks if stale data is encountered. VMAs +would be RCU enabled and this mode would be entered once multiple tasks +are using the mm_struct. + +There is additional BUG_ON() calls added within the tree, most of which +are in debug code. These will be replaced with a WARN_ON() call in the +future. There is also additional BUG_ON() calls within the code which +will also be reduced in number at a later date. These exist to catch +things such as out-of-range accesses which would crash anyways. + +Signed-off-by: Peter Jung +--- + Documentation/core-api/index.rst | 1 + + Documentation/core-api/maple_tree.rst | 217 + + MAINTAINERS | 12 + + arch/arm64/kernel/elfcore.c | 16 +- + arch/arm64/kernel/vdso.c | 3 +- + arch/parisc/kernel/cache.c | 9 +- + arch/powerpc/kernel/vdso.c | 6 +- + arch/powerpc/mm/book3s32/tlb.c | 11 +- + arch/powerpc/mm/book3s64/subpage_prot.c | 13 +- + arch/riscv/kernel/vdso.c | 3 +- + arch/s390/kernel/vdso.c | 3 +- + arch/s390/mm/gmap.c | 6 +- + arch/um/kernel/tlb.c | 14 +- + arch/x86/entry/vdso/vma.c | 9 +- + arch/x86/kernel/tboot.c | 2 +- + arch/xtensa/kernel/syscall.c | 18 +- + drivers/firmware/efi/efi.c | 2 +- + drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 14 +- + drivers/misc/cxl/fault.c | 45 +- + drivers/tee/optee/call.c | 18 +- + drivers/xen/privcmd.c | 2 +- + fs/coredump.c | 34 +- + fs/exec.c | 12 +- + fs/proc/base.c | 5 +- + fs/proc/internal.h | 2 +- + fs/proc/task_mmu.c | 74 +- + fs/proc/task_nommu.c | 45 +- + fs/userfaultfd.c | 62 +- + include/linux/maple_tree.h | 685 + + include/linux/mm.h | 78 +- + include/linux/mm_types.h | 43 +- + include/linux/mm_types_task.h | 12 - + include/linux/sched.h | 1 - + include/linux/userfaultfd_k.h | 7 +- + include/linux/vm_event_item.h | 4 - + include/linux/vmacache.h | 28 - + include/linux/vmstat.h | 6 - + include/trace/events/maple_tree.h | 123 + + include/trace/events/mmap.h | 73 + + init/main.c | 2 + + ipc/shm.c | 21 +- + kernel/acct.c | 11 +- + kernel/bpf/task_iter.c | 10 +- + kernel/debug/debug_core.c | 12 - + kernel/events/core.c | 3 +- + kernel/events/uprobes.c | 9 +- + kernel/fork.c | 62 +- + kernel/sched/fair.c | 10 +- + lib/Kconfig.debug | 17 +- + lib/Makefile | 2 +- + lib/maple_tree.c | 7130 +++ + lib/test_maple_tree.c | 38307 ++++++++++++++++ + mm/Makefile | 2 +- + mm/damon/vaddr-test.h | 36 +- + mm/damon/vaddr.c | 53 +- + mm/debug.c | 14 +- + mm/gup.c | 7 +- + mm/huge_memory.c | 4 +- + mm/init-mm.c | 4 +- + mm/internal.h | 8 +- + mm/khugepaged.c | 11 +- + mm/ksm.c | 18 +- + mm/madvise.c | 2 +- + mm/memcontrol.c | 6 +- + mm/memory.c | 34 +- + mm/mempolicy.c | 56 +- + mm/mlock.c | 37 +- + mm/mmap.c | 2154 +- + mm/mprotect.c | 8 +- + mm/mremap.c | 22 +- + mm/msync.c | 2 +- + mm/nommu.c | 260 +- + mm/oom_kill.c | 3 +- + mm/pagewalk.c | 2 +- + mm/swapfile.c | 4 +- + mm/util.c | 32 - + mm/vmacache.c | 117 - + mm/vmscan.c | 12 +- + mm/vmstat.c | 4 - + tools/include/linux/slab.h | 4 + + tools/testing/radix-tree/.gitignore | 2 + + tools/testing/radix-tree/Makefile | 9 +- + tools/testing/radix-tree/generated/autoconf.h | 1 + + tools/testing/radix-tree/linux.c | 160 +- + tools/testing/radix-tree/linux/kernel.h | 1 + + tools/testing/radix-tree/linux/lockdep.h | 2 + + tools/testing/radix-tree/linux/maple_tree.h | 7 + + tools/testing/radix-tree/maple.c | 59 + + .../radix-tree/trace/events/maple_tree.h | 5 + + 89 files changed, 48582 insertions(+), 1894 deletions(-) + create mode 100644 Documentation/core-api/maple_tree.rst + create mode 100644 include/linux/maple_tree.h + delete mode 100644 include/linux/vmacache.h + create mode 100644 include/trace/events/maple_tree.h + create mode 100644 lib/maple_tree.c + create mode 100644 lib/test_maple_tree.c + delete mode 100644 mm/vmacache.c + create mode 100644 tools/testing/radix-tree/linux/maple_tree.h + create mode 100644 tools/testing/radix-tree/maple.c + create mode 100644 tools/testing/radix-tree/trace/events/maple_tree.h + +diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst +index dc95df462eea..1da6a4fac664 100644 +--- a/Documentation/core-api/index.rst ++++ b/Documentation/core-api/index.rst +@@ -36,6 +36,7 @@ Library functionality that is used throughout the kernel. + kref + assoc_array + xarray ++ maple_tree + idr + circular-buffers + rbtree +diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst +new file mode 100644 +index 000000000000..45defcf15da7 +--- /dev/null ++++ b/Documentation/core-api/maple_tree.rst +@@ -0,0 +1,217 @@ ++.. SPDX-License-Identifier: GPL-2.0+ + -+static void ovpn_peer_release_rcu(struct rcu_head *head) -+{ -+ struct ovpn_peer *peer = container_of(head, struct ovpn_peer, rcu); + -+ ovpn_crypto_state_release(&peer->crypto); -+ ovpn_peer_free(peer); -+} ++========== ++Maple Tree ++========== + -+void ovpn_peer_release(struct ovpn_peer *peer) -+{ -+ napi_disable(&peer->napi); -+ netif_napi_del(&peer->napi); ++:Author: Liam R. Howlett + -+ if (peer->sock) -+ ovpn_socket_put(peer->sock); ++Overview ++======== + -+ call_rcu(&peer->rcu, ovpn_peer_release_rcu); -+} ++The Maple Tree is a B-Tree data type which is optimized for storing ++non-overlapping ranges, including ranges of size 1. The tree was designed to ++be simple to use and does not require a user written search method. It ++supports iterating over a range of entries and going to the previous or next ++entry in a cache-efficient manner. The tree can also be put into an RCU-safe ++mode of operation which allows reading and writing concurrently. Writers must ++synchronize on a lock, which can be the default spinlock, or the user can set ++the lock to an external lock of a different type. ++ ++The Maple Tree maintains a small memory footprint and was designed to use ++modern processor cache efficiently. The majority of the users will be able to ++use the normal API. An :ref:`maple-tree-advanced-api` exists for more complex ++scenarios. The most important usage of the Maple Tree is the tracking of the ++virtual memory areas. ++ ++The Maple Tree can store values between ``0`` and ``ULONG_MAX``. The Maple ++Tree reserves values with the bottom two bits set to '10' which are below 4096 ++(ie 2, 6, 10 .. 4094) for internal use. If the entries may use reserved ++entries then the users can convert the entries using xa_mk_value() and convert ++them back by calling xa_to_value(). If the user needs to use a reserved ++value, then the user can convert the value when using the ++:ref:`maple-tree-advanced-api`, but are blocked by the normal API. ++ ++The Maple Tree can also be configured to support searching for a gap of a given ++size (or larger). ++ ++Pre-allocating of nodes is also supported using the ++:ref:`maple-tree-advanced-api`. This is useful for users who must guarantee a ++successful store operation within a given ++code segment when allocating cannot be done. Allocations of nodes are ++relatively small at around 256 bytes. ++ ++.. _maple-tree-normal-api: ++ ++Normal API ++========== ++ ++Start by initialising a maple tree, either with DEFINE_MTREE() for statically ++allocated maple trees or mt_init() for dynamically allocated ones. A ++freshly-initialised maple tree contains a ``NULL`` pointer for the range ``0`` ++- ``ULONG_MAX``. There are currently two types of maple trees supported: the ++allocation tree and the regular tree. The regular tree has a higher branching ++factor for internal nodes. The allocation tree has a lower branching factor ++but allows the user to search for a gap of a given size or larger from either ++``0`` upwards or ``ULONG_MAX`` down. An allocation tree can be used by ++passing in the ``MT_FLAGS_ALLOC_RANGE`` flag when initialising the tree. ++ ++You can then set entries using mtree_store() or mtree_store_range(). ++mtree_store() will overwrite any entry with the new entry and return 0 on ++success or an error code otherwise. mtree_store_range() works in the same way ++but takes a range. mtree_load() is used to retrieve the entry stored at a ++given index. You can use mtree_erase() to erase an entire range by only ++knowing one value within that range, or mtree_store() call with an entry of ++NULL may be used to partially erase a range or many ranges at once. ++ ++If you want to only store a new entry to a range (or index) if that range is ++currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which ++return -EEXIST if the range is not empty. ++ ++You can search for an entry from an index upwards by using mt_find(). ++ ++You can walk each entry within a range by calling mt_for_each(). You must ++provide a temporary variable to store a cursor. If you want to walk each ++element of the tree then ``0`` and ``ULONG_MAX`` may be used as the range. If ++the caller is going to hold the lock for the duration of the walk then it is ++worth looking at the mas_for_each() API in the :ref:`maple-tree-advanced-api` ++section. ++ ++Sometimes it is necessary to ensure the next call to store to a maple tree does ++not allocate memory, please see :ref:`maple-tree-advanced-api` for this use case. ++ ++Finally, you can remove all entries from a maple tree by calling ++mtree_destroy(). If the maple tree entries are pointers, you may wish to free ++the entries first. ++ ++Allocating Nodes ++---------------- ++ ++The allocations are handled by the internal tree code. See ++:ref:`maple-tree-advanced-alloc` for other options. ++ ++Locking ++------- + -+static void ovpn_peer_delete_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, -+ delete_work); -+ ovpn_peer_release(peer); -+ ovpn_netlink_notify_del_peer(peer); -+} ++You do not have to worry about locking. See :ref:`maple-tree-advanced-locks` ++for other options. ++ ++The Maple Tree uses RCU and an internal spinlock to synchronise access: ++ ++Takes RCU read lock: ++ * mtree_load() ++ * mt_find() ++ * mt_for_each() ++ * mt_next() ++ * mt_prev() ++ ++Takes ma_lock internally: ++ * mtree_store() ++ * mtree_store_range() ++ * mtree_insert() ++ * mtree_insert_range() ++ * mtree_erase() ++ * mtree_destroy() ++ * mt_set_in_rcu() ++ * mt_clear_in_rcu() ++ ++If you want to take advantage of the internal lock to protect the data ++structures that you are storing in the Maple Tree, you can call mtree_lock() ++before calling mtree_load(), then take a reference count on the object you ++have found before calling mtree_unlock(). This will prevent stores from ++removing the object from the tree between looking up the object and ++incrementing the refcount. You can also use RCU to avoid dereferencing ++freed memory, but an explanation of that is beyond the scope of this ++document. ++ ++.. _maple-tree-advanced-api: ++ ++Advanced API ++============ ++ ++The advanced API offers more flexibility and better performance at the ++cost of an interface which can be harder to use and has fewer safeguards. ++You must take care of your own locking while using the advanced API. ++You can use the ma_lock, RCU or an external lock for protection. ++You can mix advanced and normal operations on the same array, as long ++as the locking is compatible. The :ref:`maple-tree-normal-api` is implemented ++in terms of the advanced API. ++ ++The advanced API is based around the ma_state, this is where the 'mas' ++prefix originates. The ma_state struct keeps track of tree operations to make ++life easier for both internal and external tree users. ++ ++Initialising the maple tree is the same as in the :ref:`maple-tree-normal-api`. ++Please see above. ++ ++The maple state keeps track of the range start and end in mas->index and ++mas->last, respectively. ++ ++mas_walk() will walk the tree to the location of mas->index and set the ++mas->index and mas->last according to the range for the entry. ++ ++You can set entries using mas_store(). mas_store() will overwrite any entry ++with the new entry and return the first existing entry that is overwritten. ++The range is passed in as members of the maple state: index and last. ++ ++You can use mas_erase() to erase an entire range by setting index and ++last of the maple state to the desired range to erase. This will erase ++the first range that is found in that range, set the maple state index ++and last as the range that was erased and return the entry that existed ++at that location. ++ ++You can walk each entry within a range by using mas_for_each(). If you want ++to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as ++the range. If the lock needs to be periodically dropped, see the locking ++section mas_pause(). ++ ++Using a maple state allows mas_next() and mas_prev() to function as if the ++tree was a linked list. With such a high branching factor the amortized ++performance penalty is outweighed by cache optimization. mas_next() will ++return the next entry which occurs after the entry at index. mas_prev() ++will return the previous entry which occurs before the entry at index. ++ ++mas_find() will find the first entry which exists at or above index on ++the first call, and the next entry from every subsequent calls. ++ ++mas_find_rev() will find the fist entry which exists at or below the last on ++the first call, and the previous entry from every subsequent calls. ++ ++If the user needs to yield the lock during an operation, then the maple state ++must be paused using mas_pause(). ++ ++There are a few extra interfaces provided when using an allocation tree. ++If you wish to search for a gap within a range, then mas_empty_area() ++or mas_empty_area_rev() can be used. mas_empty_area() searches for a gap ++starting at the lowest index given up to the maximum of the range. ++mas_empty_area_rev() searches for a gap starting at the highest index given ++and continues downward to the lower bound of the range. ++ ++.. _maple-tree-advanced-alloc: ++ ++Advanced Allocating Nodes ++------------------------- ++ ++Allocations are usually handled internally to the tree, however if allocations ++need to occur before a write occurs then calling mas_expected_entries() will ++allocate the worst-case number of needed nodes to insert the provided number of ++ranges. This also causes the tree to enter mass insertion mode. Once ++insertions are complete calling mas_destroy() on the maple state will free the ++unused allocations. ++ ++.. _maple-tree-advanced-locks: ++ ++Advanced Locking ++---------------- ++ ++The maple tree uses a spinlock by default, but external locks can be used for ++tree updates as well. To use an external lock, the tree must be initialized ++with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the ++MTREE_INIT_EXT() #define, which takes an external lock as an argument. ++ ++Functions and structures ++======================== ++ ++.. kernel-doc:: include/linux/maple_tree.h ++.. kernel-doc:: lib/maple_tree.c +diff --git a/MAINTAINERS b/MAINTAINERS +index a29c9731350c..96a09757feb3 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -12094,6 +12094,18 @@ L: linux-man@vger.kernel.org + S: Maintained + W: http://www.kernel.org/doc/man-pages + ++MAPLE TREE ++M: Liam R. Howlett ++L: linux-mm@kvack.org ++S: Supported ++F: Documentation/core-api/maple_tree.rst ++F: include/linux/maple_tree.h ++F: include/trace/events/maple_tree.h ++F: lib/maple_tree.c ++F: lib/test_maple_tree.c ++F: tools/testing/radix-tree/linux/maple_tree.h ++F: tools/testing/radix-tree/maple.c ++ + MARDUK (CREATOR CI40) DEVICE TREE SUPPORT + M: Rahul Bedarkar + L: linux-mips@vger.kernel.org +diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c +index 98d67444a5b6..27ef7ad3ffd2 100644 +--- a/arch/arm64/kernel/elfcore.c ++++ b/arch/arm64/kernel/elfcore.c +@@ -8,9 +8,9 @@ + #include + #include + +-#define for_each_mte_vma(tsk, vma) \ ++#define for_each_mte_vma(vmi, vma) \ + if (system_supports_mte()) \ +- for (vma = tsk->mm->mmap; vma; vma = vma->vm_next) \ ++ for_each_vma(vmi, vma) \ + if (vma->vm_flags & VM_MTE) + + static unsigned long mte_vma_tag_dump_size(struct vm_area_struct *vma) +@@ -81,8 +81,9 @@ Elf_Half elf_core_extra_phdrs(void) + { + struct vm_area_struct *vma; + int vma_count = 0; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) ++ for_each_mte_vma(vmi, vma) + vma_count++; + + return vma_count; +@@ -91,8 +92,9 @@ Elf_Half elf_core_extra_phdrs(void) + int elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) { ++ for_each_mte_vma(vmi, vma) { + struct elf_phdr phdr; + + phdr.p_type = PT_AARCH64_MEMTAG_MTE; +@@ -116,8 +118,9 @@ size_t elf_core_extra_data_size(void) + { + struct vm_area_struct *vma; + size_t data_size = 0; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) ++ for_each_mte_vma(vmi, vma) + data_size += mte_vma_tag_dump_size(vma); + + return data_size; +@@ -126,8 +129,9 @@ size_t elf_core_extra_data_size(void) + int elf_core_write_extra_data(struct coredump_params *cprm) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, current->mm, 0); + +- for_each_mte_vma(current, vma) { ++ for_each_mte_vma(vmi, vma) { + if (vma->vm_flags & VM_DONTDUMP) + continue; + +diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c +index a61fc4f989b3..a8388af62b99 100644 +--- a/arch/arm64/kernel/vdso.c ++++ b/arch/arm64/kernel/vdso.c +@@ -136,10 +136,11 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) +diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c +index 3feb7694e0ca..1d3b8bc8a623 100644 +--- a/arch/parisc/kernel/cache.c ++++ b/arch/parisc/kernel/cache.c +@@ -657,15 +657,20 @@ static inline unsigned long mm_total_size(struct mm_struct *mm) + { + struct vm_area_struct *vma; + unsigned long usize = 0; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma && usize < parisc_cache_flush_threshold; vma = vma->vm_next) ++ for_each_vma(vmi, vma) { ++ if (usize >= parisc_cache_flush_threshold) ++ break; + usize += vma->vm_end - vma->vm_start; ++ } + return usize; + } + + void flush_cache_mm(struct mm_struct *mm) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + /* + * Flushing the whole cache on each cpu takes forever on +@@ -685,7 +690,7 @@ void flush_cache_mm(struct mm_struct *mm) + } + + /* Flush mm */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) ++ for_each_vma(vmi, vma) + flush_cache_pages(vma, vma->vm_start, vma->vm_end); + } + +diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c +index 0da287544054..94a8fa5017c3 100644 +--- a/arch/powerpc/kernel/vdso.c ++++ b/arch/powerpc/kernel/vdso.c +@@ -113,18 +113,18 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page) + int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + + mmap_read_lock(mm); +- +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, &vvar_spec)) + zap_page_range(vma, vma->vm_start, size); + } +- + mmap_read_unlock(mm); + -+/* Use with kref_put calls, when releasing refcount -+ * on ovpn_peer objects. This method should only -+ * be called from process context with config_mutex held. -+ */ -+void ovpn_peer_release_kref(struct kref *kref) -+{ -+ struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount); + return 0; + } + +diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c +index 19f0ef950d77..9ad6b56bfec9 100644 +--- a/arch/powerpc/mm/book3s32/tlb.c ++++ b/arch/powerpc/mm/book3s32/tlb.c +@@ -81,14 +81,15 @@ EXPORT_SYMBOL(hash__flush_range); + void hash__flush_tlb_mm(struct mm_struct *mm) + { + struct vm_area_struct *mp; ++ VMA_ITERATOR(vmi, mm, 0); + + /* +- * It is safe to go down the mm's list of vmas when called +- * from dup_mmap, holding mmap_lock. It would also be safe from +- * unmap_region or exit_mmap, but not from vmtruncate on SMP - +- * but it seems dup_mmap is the only SMP case which gets here. ++ * It is safe to iterate the vmas when called from dup_mmap, ++ * holding mmap_lock. It would also be safe from unmap_region ++ * or exit_mmap, but not from vmtruncate on SMP - but it seems ++ * dup_mmap is the only SMP case which gets here. + */ +- for (mp = mm->mmap; mp != NULL; mp = mp->vm_next) ++ for_each_vma(vmi, mp) + hash__flush_range(mp->vm_mm, mp->vm_start, mp->vm_end); + } + EXPORT_SYMBOL(hash__flush_tlb_mm); +diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c +index 60c6ea16a972..d73b3b4176e8 100644 +--- a/arch/powerpc/mm/book3s64/subpage_prot.c ++++ b/arch/powerpc/mm/book3s64/subpage_prot.c +@@ -149,24 +149,15 @@ static void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, + unsigned long len) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, addr); + + /* + * We don't try too hard, we just mark all the vma in that range + * VM_NOHUGEPAGE and split them. + */ +- vma = find_vma(mm, addr); +- /* +- * If the range is in unmapped range, just return +- */ +- if (vma && ((addr + len) <= vma->vm_start)) +- return; +- +- while (vma) { +- if (vma->vm_start >= (addr + len)) +- break; ++ for_each_vma_range(vmi, vma, addr + len) { + vma->vm_flags |= VM_NOHUGEPAGE; + walk_page_vma(vma, &subpage_walk_ops, NULL); +- vma = vma->vm_next; + } + } + #else +diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c +index 69b05b6c181b..692e7ae3dcb8 100644 +--- a/arch/riscv/kernel/vdso.c ++++ b/arch/riscv/kernel/vdso.c +@@ -114,11 +114,12 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + struct __vdso_info *vdso_info = mm->context.vdso_info; + + mmap_read_lock(mm); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, vdso_info->dm)) +diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c +index 5075cde77b29..535099f2736d 100644 +--- a/arch/s390/kernel/vdso.c ++++ b/arch/s390/kernel/vdso.c +@@ -69,10 +69,11 @@ static struct page *find_timens_vvar_page(struct vm_area_struct *vma) + int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (!vma_is_special_mapping(vma, &vvar_mapping)) +diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c +index 62758cb5872f..02d15c8dc92e 100644 +--- a/arch/s390/mm/gmap.c ++++ b/arch/s390/mm/gmap.c +@@ -2515,8 +2515,9 @@ static const struct mm_walk_ops thp_split_walk_ops = { + static inline void thp_split_mm(struct mm_struct *mm) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + vma->vm_flags &= ~VM_HUGEPAGE; + vma->vm_flags |= VM_NOHUGEPAGE; + walk_page_vma(vma, &thp_split_walk_ops, NULL); +@@ -2584,8 +2585,9 @@ int gmap_mark_unmergeable(void) + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + int ret; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, + MADV_UNMERGEABLE, &vma->vm_flags); + if (ret) +diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c +index bc38f79ca3a3..ad449173a1a1 100644 +--- a/arch/um/kernel/tlb.c ++++ b/arch/um/kernel/tlb.c +@@ -584,21 +584,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + + void flush_tlb_mm(struct mm_struct *mm) + { +- struct vm_area_struct *vma = mm->mmap; ++ struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- while (vma != NULL) { ++ for_each_vma(vmi, vma) + fix_range(mm, vma->vm_start, vma->vm_end, 0); +- vma = vma->vm_next; +- } + } + + void force_flush_all(void) + { + struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma = mm->mmap; ++ struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- while (vma != NULL) { ++ for_each_vma(vmi, vma) + fix_range(mm, vma->vm_start, vma->vm_end, 1); +- vma = vma->vm_next; +- } + } +diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c +index 1000d457c332..6292b960037b 100644 +--- a/arch/x86/entry/vdso/vma.c ++++ b/arch/x86/entry/vdso/vma.c +@@ -127,17 +127,17 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) + { + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); +- +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long size = vma->vm_end - vma->vm_start; + + if (vma_is_special_mapping(vma, &vvar_mapping)) + zap_page_range(vma, vma->vm_start, size); + } +- + mmap_read_unlock(mm); + -+ INIT_WORK(&peer->delete_work, ovpn_peer_delete_work); -+ queue_work(peer->ovpn->events_wq, &peer->delete_work); -+} + return 0; + } + #else +@@ -354,6 +354,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) + { + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_write_lock(mm); + /* +@@ -363,7 +364,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) + * We could search vma near context.vdso, but it's a slowpath, + * so let's explicitly check all VMAs to be completely sure. + */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (vma_is_special_mapping(vma, &vdso_mapping) || + vma_is_special_mapping(vma, &vvar_mapping)) { + mmap_write_unlock(mm); +diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c +index 3bacd935f840..4c1bcb6053fc 100644 +--- a/arch/x86/kernel/tboot.c ++++ b/arch/x86/kernel/tboot.c +@@ -95,7 +95,7 @@ void __init tboot_probe(void) + + static pgd_t *tboot_pg_dir; + static struct mm_struct tboot_mm = { +- .mm_rb = RB_ROOT, ++ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, tboot_mm.mmap_lock), + .pgd = swapper_pg_dir, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), +diff --git a/arch/xtensa/kernel/syscall.c b/arch/xtensa/kernel/syscall.c +index 201356faa7e6..b3c2450d6f23 100644 +--- a/arch/xtensa/kernel/syscall.c ++++ b/arch/xtensa/kernel/syscall.c +@@ -58,6 +58,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) + { + struct vm_area_struct *vmm; ++ struct vma_iterator vmi; + + if (flags & MAP_FIXED) { + /* We do not accept a shared mapping if it would violate +@@ -79,15 +80,20 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, + else + addr = PAGE_ALIGN(addr); + +- for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { +- /* At this point: (!vmm || addr < vmm->vm_end). */ +- if (TASK_SIZE - len < addr) +- return -ENOMEM; +- if (!vmm || addr + len <= vm_start_gap(vmm)) +- return addr; ++ vma_iter_init(&vmi, current->mm, addr); ++ for_each_vma(vmi, vmm) { ++ /* At this point: (addr < vmm->vm_end). */ ++ if (addr + len <= vm_start_gap(vmm)) ++ break; + -+struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, -+ struct socket *sock, u32 id, uint8_t *local_ip) -+{ -+ struct ovpn_peer *peer; -+ int ret; + addr = vmm->vm_end; + if (flags & MAP_SHARED) + addr = COLOUR_ALIGN(addr, pgoff); + } + -+ /* create new peer */ -+ peer = ovpn_peer_create(ovpn, id); -+ if (IS_ERR(peer)) -+ return peer; ++ if (TASK_SIZE - len < addr) ++ return -ENOMEM; + -+ if (sock->sk->sk_protocol == IPPROTO_UDP) { -+ /* a UDP peer must have a remote endpoint */ -+ if (!sa) { -+ ovpn_peer_release(peer); -+ return ERR_PTR(-EINVAL); -+ } ++ return addr; + } + #endif +diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c +index e4080ad96089..042a3ef4db1c 100644 +--- a/drivers/firmware/efi/efi.c ++++ b/drivers/firmware/efi/efi.c +@@ -57,7 +57,7 @@ static unsigned long __initdata mem_reserve = EFI_INVALID_TABLE_ADDR; + static unsigned long __initdata rt_prop = EFI_INVALID_TABLE_ADDR; + + struct mm_struct efi_mm = { +- .mm_rb = RB_ROOT, ++ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, efi_mm.mmap_lock), + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), + .write_protect_seq = SEQCNT_ZERO(efi_mm.write_protect_seq), +diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +index 8423df021b71..d4398948f016 100644 +--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c ++++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +@@ -426,12 +426,11 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { + static int + probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) + { +- const unsigned long end = addr + len; ++ VMA_ITERATOR(vmi, mm, addr); + struct vm_area_struct *vma; +- int ret = -EFAULT; + + mmap_read_lock(mm); +- for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { ++ for_each_vma_range(vmi, vma, addr + len) { + /* Check for holes, note that we also update the addr below */ + if (vma->vm_start > addr) + break; +@@ -439,16 +438,13 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) + if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + break; + +- if (vma->vm_end >= end) { +- ret = 0; +- break; +- } +- + addr = vma->vm_end; + } + mmap_read_unlock(mm); + +- return ret; ++ if (vma) ++ return -EFAULT; ++ return 0; + } + + /* +diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c +index 60c829113299..2c64f55cf01f 100644 +--- a/drivers/misc/cxl/fault.c ++++ b/drivers/misc/cxl/fault.c +@@ -280,22 +280,6 @@ void cxl_handle_fault(struct work_struct *fault_work) + mmput(mm); + } + +-static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) +-{ +- struct mm_struct *mm; +- +- mm = get_mem_context(ctx); +- if (mm == NULL) { +- pr_devel("cxl_prefault_one unable to get mm %i\n", +- pid_nr(ctx->pid)); +- return; +- } +- +- cxl_fault_segment(ctx, mm, ea); +- +- mmput(mm); +-} +- + static u64 next_segment(u64 ea, u64 vsid) + { + if (vsid & SLB_VSID_B_1T) +@@ -306,23 +290,16 @@ static u64 next_segment(u64 ea, u64 vsid) + return ea + 1; + } + +-static void cxl_prefault_vma(struct cxl_context *ctx) ++static void cxl_prefault_vma(struct cxl_context *ctx, struct mm_struct *mm) + { + u64 ea, last_esid = 0; + struct copro_slb slb; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + int rc; +- struct mm_struct *mm; +- +- mm = get_mem_context(ctx); +- if (mm == NULL) { +- pr_devel("cxl_prefault_vm unable to get mm %i\n", +- pid_nr(ctx->pid)); +- return; +- } + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + for (ea = vma->vm_start; ea < vma->vm_end; + ea = next_segment(ea, slb.vsid)) { + rc = copro_calculate_slb(mm, ea, &slb); +@@ -337,20 +314,28 @@ static void cxl_prefault_vma(struct cxl_context *ctx) + } + } + mmap_read_unlock(mm); +- +- mmput(mm); + } + + void cxl_prefault(struct cxl_context *ctx, u64 wed) + { ++ struct mm_struct *mm = get_mem_context(ctx); + -+ /* set peer sockaddr */ -+ ret = ovpn_peer_reset_sockaddr(peer, sa, local_ip); -+ if (ret < 0) { -+ ovpn_peer_release(peer); -+ return ERR_PTR(ret); -+ } ++ if (mm == NULL) { ++ pr_devel("cxl_prefault unable to get mm %i\n", ++ pid_nr(ctx->pid)); ++ return; + } + -+ peer->sock = ovpn_socket_new(sock, peer); -+ if (IS_ERR(peer->sock)) { -+ peer->sock = NULL; -+ ovpn_peer_release(peer); -+ return ERR_PTR(-ENOTSOCK); -+ } + switch (ctx->afu->prefault_mode) { + case CXL_PREFAULT_WED: +- cxl_prefault_one(ctx, wed); ++ cxl_fault_segment(ctx, mm, wed); + break; + case CXL_PREFAULT_ALL: +- cxl_prefault_vma(ctx); ++ cxl_prefault_vma(ctx, mm); + break; + default: + break; + } + -+ /* schedule initial TCP RX work only after having assigned peer->sock */ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) -+ queue_work(peer->ovpn->events_wq, &peer->tcp.rx_work); ++ mmput(mm); + } +diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c +index 28f87cd8b3ed..290b1bb0e9cd 100644 +--- a/drivers/tee/optee/call.c ++++ b/drivers/tee/optee/call.c +@@ -492,15 +492,18 @@ static bool is_normal_memory(pgprot_t p) + #endif + } + +-static int __check_mem_type(struct vm_area_struct *vma, unsigned long end) ++static int __check_mem_type(struct mm_struct *mm, unsigned long start, ++ unsigned long end) + { +- while (vma && is_normal_memory(vma->vm_page_prot)) { +- if (vma->vm_end >= end) +- return 0; +- vma = vma->vm_next; ++ struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, start); + -+ return peer; -+} ++ for_each_vma_range(vmi, vma, end) { ++ if (!is_normal_memory(vma->vm_page_prot)) ++ return -EINVAL; + } + +- return -EINVAL; ++ return 0; + } + + int optee_check_mem_type(unsigned long start, size_t num_pages) +@@ -516,8 +519,7 @@ int optee_check_mem_type(unsigned long start, size_t num_pages) + return 0; + + mmap_read_lock(mm); +- rc = __check_mem_type(find_vma(mm, start), +- start + num_pages * PAGE_SIZE); ++ rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE); + mmap_read_unlock(mm); + + return rc; +diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c +index e88e8f6f0a33..fae50a24630b 100644 +--- a/drivers/xen/privcmd.c ++++ b/drivers/xen/privcmd.c +@@ -282,7 +282,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata) + struct page, lru); + struct privcmd_mmap_entry *msg = page_address(page); + +- vma = find_vma(mm, msg->va); ++ vma = vma_lookup(mm, msg->va); + rc = -EINVAL; + + if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data) +diff --git a/fs/coredump.c b/fs/coredump.c +index 9f4aae202109..35f2af85b9bc 100644 +--- a/fs/coredump.c ++++ b/fs/coredump.c +@@ -1072,30 +1072,20 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, + return vma->vm_end - vma->vm_start; + } + +-static struct vm_area_struct *first_vma(struct task_struct *tsk, +- struct vm_area_struct *gate_vma) +-{ +- struct vm_area_struct *ret = tsk->mm->mmap; +- +- if (ret) +- return ret; +- return gate_vma; +-} +- + /* + * Helper function for iterating across a vma list. It ensures that the caller + * will visit `gate_vma' prior to terminating the search. + */ +-static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, ++static struct vm_area_struct *coredump_next_vma(struct ma_state *mas, ++ struct vm_area_struct *vma, + struct vm_area_struct *gate_vma) + { +- struct vm_area_struct *ret; +- +- ret = this_vma->vm_next; +- if (ret) +- return ret; +- if (this_vma == gate_vma) ++ if (gate_vma && (vma == gate_vma)) + return NULL; + -+/* Configure keepalive parameters */ -+void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) ++ vma = mas_next(mas, ULONG_MAX); ++ if (vma) ++ return vma; + return gate_vma; + } + +@@ -1119,9 +1109,10 @@ static void free_vma_snapshot(struct coredump_params *cprm) + */ + static bool dump_vma_snapshot(struct coredump_params *cprm) + { +- struct vm_area_struct *vma, *gate_vma; ++ struct vm_area_struct *gate_vma, *vma = NULL; + struct mm_struct *mm = current->mm; +- int i; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ int i = 0; + + /* + * Once the stack expansion code is fixed to not change VMA bounds +@@ -1141,8 +1132,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) + return false; + } + +- for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; +- vma = next_vma(vma, gate_vma), i++) { ++ while ((vma = coredump_next_vma(&mas, vma, gate_vma)) != NULL) { + struct core_vma_metadata *m = cprm->vma_meta + i; + + m->start = vma->vm_start; +@@ -1150,10 +1140,10 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) + m->flags = vma->vm_flags; + m->dump_size = vma_dump_size(vma, cprm->mm_flags); + m->pgoff = vma->vm_pgoff; +- + m->file = vma->vm_file; + if (m->file) + get_file(m->file); ++ i++; + } + + mmap_write_unlock(mm); +diff --git a/fs/exec.c b/fs/exec.c +index c67b12f0f577..3a7bae4e16e5 100644 +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -28,7 +28,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -683,6 +682,8 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + unsigned long length = old_end - old_start; + unsigned long new_start = old_start - shift; + unsigned long new_end = old_end - shift; ++ VMA_ITERATOR(vmi, mm, new_start); ++ struct vm_area_struct *next; + struct mmu_gather tlb; + + BUG_ON(new_start > new_end); +@@ -691,7 +692,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + * ensure there are no vmas between where we want to go + * and where we are + */ +- if (vma != find_vma(mm, new_start)) ++ if (vma != vma_next(&vmi)) + return -EFAULT; + + /* +@@ -710,12 +711,13 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + + lru_add_drain(); + tlb_gather_mmu(&tlb, mm); ++ next = vma_next(&vmi); + if (new_end > old_start) { + /* + * when the old and new regions overlap clear from new_end. + */ + free_pgd_range(&tlb, new_end, old_end, new_end, +- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); ++ next ? next->vm_start : USER_PGTABLES_CEILING); + } else { + /* + * otherwise, clean from old_start; this is done to not touch +@@ -724,7 +726,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) + * for the others its just a little faster. + */ + free_pgd_range(&tlb, old_start, old_end, new_end, +- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); ++ next ? next->vm_start : USER_PGTABLES_CEILING); + } + tlb_finish_mmu(&tlb); + +@@ -1024,8 +1026,6 @@ static int exec_mmap(struct mm_struct *mm) + activate_mm(active_mm, mm); + if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); +- tsk->mm->vmacache_seqnum = 0; +- vmacache_flush(tsk); + task_unlock(tsk); + lru_gen_use_mm(mm); + if (old_mm) { +diff --git a/fs/proc/base.c b/fs/proc/base.c +index 93f7e3d971e4..12885a75913f 100644 +--- a/fs/proc/base.c ++++ b/fs/proc/base.c +@@ -2350,6 +2350,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) + GENRADIX(struct map_files_info) fa; + struct map_files_info *p; + int ret; ++ struct vma_iterator vmi; + + genradix_init(&fa); + +@@ -2388,7 +2389,9 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) + * routine might require mmap_lock taken in might_fault(). + */ + +- for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { ++ pos = 2; ++ vma_iter_init(&vmi, mm, 0); ++ for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + if (++pos <= ctx->pos) +diff --git a/fs/proc/internal.h b/fs/proc/internal.h +index 06a80f78433d..f03000764ce5 100644 +--- a/fs/proc/internal.h ++++ b/fs/proc/internal.h +@@ -285,7 +285,7 @@ struct proc_maps_private { + struct task_struct *task; + struct mm_struct *mm; + #ifdef CONFIG_MMU +- struct vm_area_struct *tail_vma; ++ struct vma_iterator iter; + #endif + #ifdef CONFIG_NUMA + struct mempolicy *task_mempolicy; +diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c +index 482f91577f8c..7a10fe08beb3 100644 +--- a/fs/proc/task_mmu.c ++++ b/fs/proc/task_mmu.c +@@ -1,6 +1,5 @@ + // SPDX-License-Identifier: GPL-2.0 + #include +-#include + #include + #include + #include +@@ -124,12 +123,26 @@ static void release_task_mempolicy(struct proc_maps_private *priv) + } + #endif + ++static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv, ++ loff_t *ppos) +{ -+ u32 delta; -+ -+ netdev_dbg(peer->ovpn->dev, -+ "%s: scheduling keepalive for peer %u: interval=%u timeout=%u\n", __func__, -+ peer->id, interval, timeout); ++ struct vm_area_struct *vma = vma_next(&priv->iter); + -+ peer->keepalive_interval = interval; -+ if (interval > 0) { -+ delta = msecs_to_jiffies(interval * MSEC_PER_SEC); -+ mod_timer(&peer->keepalive_xmit, jiffies + delta); ++ if (vma) { ++ *ppos = vma->vm_start; + } else { -+ del_timer(&peer->keepalive_xmit); ++ *ppos = -2UL; ++ vma = get_gate_vma(priv->mm); + } + -+ peer->keepalive_timeout = timeout; -+ if (timeout) { -+ delta = msecs_to_jiffies(timeout * MSEC_PER_SEC); -+ mod_timer(&peer->keepalive_recv, jiffies + delta); -+ } else { -+ del_timer(&peer->keepalive_recv); -+ } ++ return vma; +} + -+#define ovpn_peer_index(_tbl, _key, _key_len) \ -+ (jhash(_key, _key_len, 0) % HASH_SIZE(_tbl)) \ -+ -+static struct ovpn_peer *ovpn_peer_lookup_vpn_addr4(struct hlist_head *head, __be32 *addr) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr4) { -+ if (*addr != tmp->vpn_addrs.ipv4.s_addr) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; + static void *m_start(struct seq_file *m, loff_t *ppos) + { + struct proc_maps_private *priv = m->private; + unsigned long last_addr = *ppos; + struct mm_struct *mm; +- struct vm_area_struct *vma; + + /* See m_next(). Zero at the start or after lseek. */ + if (last_addr == -1UL) +@@ -153,31 +166,21 @@ static void *m_start(struct seq_file *m, loff_t *ppos) + return ERR_PTR(-EINTR); + } + ++ vma_iter_init(&priv->iter, mm, last_addr); + hold_task_mempolicy(priv); +- priv->tail_vma = get_gate_vma(mm); +- +- vma = find_vma(mm, last_addr); +- if (vma) +- return vma; ++ if (last_addr == -2UL) ++ return get_gate_vma(mm); + +- return priv->tail_vma; ++ return proc_get_vma(priv, ppos); + } + + static void *m_next(struct seq_file *m, void *v, loff_t *ppos) + { +- struct proc_maps_private *priv = m->private; +- struct vm_area_struct *next, *vma = v; +- +- if (vma == priv->tail_vma) +- next = NULL; +- else if (vma->vm_next) +- next = vma->vm_next; +- else +- next = priv->tail_vma; +- +- *ppos = next ? next->vm_start : -1UL; +- +- return next; ++ if (*ppos == -2UL) { ++ *ppos = -1UL; ++ return NULL; + } -+ rcu_read_unlock(); ++ return proc_get_vma(m->private, ppos); + } + + static void m_stop(struct seq_file *m, void *v) +@@ -877,16 +880,16 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + { + struct proc_maps_private *priv = m->private; + struct mem_size_stats mss; +- struct mm_struct *mm; ++ struct mm_struct *mm = priv->mm; + struct vm_area_struct *vma; +- unsigned long last_vma_end = 0; ++ unsigned long vma_start = 0, last_vma_end = 0; + int ret = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + priv->task = get_proc_task(priv->inode); + if (!priv->task) + return -ESRCH; + +- mm = priv->mm; + if (!mm || !mmget_not_zero(mm)) { + ret = -ESRCH; + goto out_put_task; +@@ -899,8 +902,13 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + goto out_put_mm; + + hold_task_mempolicy(priv); ++ vma = mas_find(&mas, 0); + -+ return peer; -+} ++ if (unlikely(!vma)) ++ goto empty_set; + +- for (vma = priv->mm->mmap; vma;) { ++ vma_start = vma->vm_start; ++ do { + smap_gather_stats(vma, &mss, 0); + last_vma_end = vma->vm_end; + +@@ -909,6 +917,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + * access it for write request. + */ + if (mmap_lock_is_contended(mm)) { ++ mas_pause(&mas); + mmap_read_unlock(mm); + ret = mmap_read_lock_killable(mm); + if (ret) { +@@ -952,7 +961,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + * contains last_vma_end. + * Iterate VMA' from last_vma_end. + */ +- vma = find_vma(mm, last_vma_end - 1); ++ vma = mas_find(&mas, ULONG_MAX); + /* Case 3 above */ + if (!vma) + break; +@@ -966,11 +975,10 @@ static int show_smaps_rollup(struct seq_file *m, void *v) + smap_gather_stats(vma, &mss, last_vma_end); + } + /* Case 2 above */ +- vma = vma->vm_next; +- } ++ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); + +- show_vma_header_prefix(m, priv->mm->mmap->vm_start, +- last_vma_end, 0, 0, 0, 0); ++empty_set: ++ show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); + seq_pad(m, ' '); + seq_puts(m, "[rollup]\n"); + +@@ -1263,6 +1271,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, + return -ESRCH; + mm = get_task_mm(task); + if (mm) { ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + struct mmu_notifier_range range; + struct clear_refs_private cp = { + .type = type, +@@ -1282,7 +1291,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, + } + + if (type == CLEAR_REFS_SOFT_DIRTY) { +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (!(vma->vm_flags & VM_SOFTDIRTY)) + continue; + vma->vm_flags &= ~VM_SOFTDIRTY; +@@ -1294,8 +1303,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, + 0, NULL, mm, 0, -1UL); + mmu_notifier_invalidate_range_start(&range); + } +- walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, +- &cp); ++ walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); + if (type == CLEAR_REFS_SOFT_DIRTY) { + mmu_notifier_invalidate_range_end(&range); + flush_tlb_mm(mm); +diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c +index a6d21fc0033c..2fd06f52b6a4 100644 +--- a/fs/proc/task_nommu.c ++++ b/fs/proc/task_nommu.c +@@ -20,15 +20,13 @@ + */ + void task_mem(struct seq_file *m, struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + struct vm_region *region; +- struct rb_node *p; + unsigned long bytes = 0, sbytes = 0, slack = 0, size; +- +- mmap_read_lock(mm); +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { +- vma = rb_entry(p, struct vm_area_struct, vm_rb); + ++ mmap_read_lock(mm); ++ for_each_vma(vmi, vma) { + bytes += kobjsize(vma); + + region = vma->vm_region; +@@ -82,15 +80,13 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) + + unsigned long task_vsize(struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; +- struct rb_node *p; + unsigned long vsize = 0; + + mmap_read_lock(mm); +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { +- vma = rb_entry(p, struct vm_area_struct, vm_rb); ++ for_each_vma(vmi, vma) + vsize += vma->vm_end - vma->vm_start; +- } + mmap_read_unlock(mm); + return vsize; + } +@@ -99,14 +95,13 @@ unsigned long task_statm(struct mm_struct *mm, + unsigned long *shared, unsigned long *text, + unsigned long *data, unsigned long *resident) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + struct vm_region *region; +- struct rb_node *p; + unsigned long size = kobjsize(mm); + + mmap_read_lock(mm); +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { +- vma = rb_entry(p, struct vm_area_struct, vm_rb); ++ for_each_vma(vmi, vma) { + size += kobjsize(vma); + region = vma->vm_region; + if (region) { +@@ -190,17 +185,19 @@ static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) + */ + static int show_map(struct seq_file *m, void *_p) + { +- struct rb_node *p = _p; +- +- return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb)); ++ return nommu_vma_show(m, _p); + } + + static void *m_start(struct seq_file *m, loff_t *pos) + { + struct proc_maps_private *priv = m->private; + struct mm_struct *mm; +- struct rb_node *p; +- loff_t n = *pos; ++ struct vm_area_struct *vma; ++ unsigned long addr = *pos; + -+static struct ovpn_peer *ovpn_peer_lookup_vpn_addr6(struct hlist_head *head, struct in6_addr *addr) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ int i; ++ /* See m_next(). Zero at the start or after lseek. */ ++ if (addr == -1UL) ++ return NULL; + + /* pin the task and mm whilst we play with them */ + priv->task = get_proc_task(priv->inode); +@@ -216,10 +213,10 @@ static void *m_start(struct seq_file *m, loff_t *pos) + return ERR_PTR(-EINTR); + } + +- /* start from the Nth VMA */ +- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) +- if (n-- == 0) +- return p; ++ /* start the next element from addr */ ++ vma = find_vma(mm, addr); ++ if (vma) ++ return vma; + + mmap_read_unlock(mm); + mmput(mm); +@@ -242,10 +239,10 @@ static void m_stop(struct seq_file *m, void *_vml) + + static void *m_next(struct seq_file *m, void *_p, loff_t *pos) + { +- struct rb_node *p = _p; ++ struct vm_area_struct *vma = _p; + +- (*pos)++; +- return p ? rb_next(p) : NULL; ++ *pos = vma->vm_end; ++ return find_vma(vma->vm_mm, vma->vm_end); + } + + static const struct seq_operations proc_pid_maps_ops = { +diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c +index 175de70e3adf..f662cede4488 100644 +--- a/fs/userfaultfd.c ++++ b/fs/userfaultfd.c +@@ -615,14 +615,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, + if (release_new_ctx) { + struct vm_area_struct *vma; + struct mm_struct *mm = release_new_ctx->mm; ++ VMA_ITERATOR(vmi, mm, 0); + + /* the various vma->vm_userfaultfd_ctx still points to it */ + mmap_write_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) ++ for_each_vma(vmi, vma) { + if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; + vma->vm_flags &= ~__VM_UFFD_FLAGS; + } ++ } + mmap_write_unlock(mm); + + userfaultfd_ctx_put(release_new_ctx); +@@ -803,11 +805,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, + return false; + } + +-int userfaultfd_unmap_prep(struct vm_area_struct *vma, +- unsigned long start, unsigned long end, +- struct list_head *unmaps) ++int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, ++ unsigned long end, struct list_head *unmaps) + { +- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { ++ VMA_ITERATOR(vmi, mm, start); ++ struct vm_area_struct *vma; + -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr6) { -+ for (i = 0; i < 4; i++) { -+ if (addr->s6_addr32[i] != tmp->vpn_addrs.ipv6.s6_addr32[i]) -+ continue; ++ for_each_vma_range(vmi, vma, end) { + struct userfaultfd_unmap_ctx *unmap_ctx; + struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; + +@@ -857,6 +861,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) + /* len == 0 means wake all */ + struct userfaultfd_wake_range range = { .len = 0, }; + unsigned long new_flags; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + WRITE_ONCE(ctx->released, true); + +@@ -873,7 +878,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) + */ + mmap_write_lock(mm); + prev = NULL; +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + cond_resched(); + BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ + !!(vma->vm_flags & __VM_UFFD_FLAGS)); +@@ -887,10 +892,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file) + vma->vm_file, vma->vm_pgoff, + vma_policy(vma), + NULL_VM_UFFD_CTX, anon_vma_name(vma)); +- if (prev) ++ if (prev) { ++ mas_pause(&mas); + vma = prev; +- else ++ } else { + prev = vma; + } + -+ if (!ovpn_peer_hold(tmp)) -+ continue; + vma->vm_flags = new_flags; + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; + } +@@ -1272,6 +1280,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + bool found; + bool basic_ioctls; + unsigned long start, end, vma_end; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + user_uffdio_register = (struct uffdio_register __user *) arg; + +@@ -1314,7 +1323,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + goto out; + + mmap_write_lock(mm); +- vma = find_vma_prev(mm, start, &prev); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) + goto out_unlock; + +@@ -1339,7 +1349,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + */ + found = false; + basic_ioctls = false; +- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { ++ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { + cond_resched(); + + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ +@@ -1399,8 +1409,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + } + BUG_ON(!found); + +- if (vma->vm_start < start) +- prev = vma; ++ mas_set(&mas, start); ++ prev = mas_prev(&mas, 0); ++ if (prev != vma) ++ mas_next(&mas, ULONG_MAX); + + ret = 0; + do { +@@ -1430,6 +1442,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + ((struct vm_userfaultfd_ctx){ ctx }), + anon_vma_name(vma)); + if (prev) { ++ /* vma_merge() invalidated the mas */ ++ mas_pause(&mas); + vma = prev; + goto next; + } +@@ -1437,11 +1451,15 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + ret = split_vma(mm, vma, start, 1); + if (ret) + break; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } + if (vma->vm_end > end) { + ret = split_vma(mm, vma, end, 0); + if (ret) + break; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } + next: + /* +@@ -1458,8 +1476,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, + skip: + prev = vma; + start = vma->vm_end; +- vma = vma->vm_next; +- } while (vma && vma->vm_start < end); ++ vma = mas_next(&mas, end - 1); ++ } while (vma); + out_unlock: + mmap_write_unlock(mm); + mmput(mm); +@@ -1503,6 +1521,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + bool found; + unsigned long start, end, vma_end; + const void __user *buf = (void __user *)arg; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + ret = -EFAULT; + if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) +@@ -1521,7 +1540,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + goto out; + + mmap_write_lock(mm); +- vma = find_vma_prev(mm, start, &prev); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) + goto out_unlock; + +@@ -1546,7 +1566,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + */ + found = false; + ret = -EINVAL; +- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { ++ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { + cond_resched(); + + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ +@@ -1566,8 +1586,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + } + BUG_ON(!found); + +- if (vma->vm_start < start) +- prev = vma; ++ mas_set(&mas, start); ++ prev = mas_prev(&mas, 0); ++ if (prev != vma) ++ mas_next(&mas, ULONG_MAX); + + ret = 0; + do { +@@ -1636,8 +1658,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, + skip: + prev = vma; + start = vma->vm_end; +- vma = vma->vm_next; +- } while (vma && vma->vm_start < end); ++ vma = mas_next(&mas, end - 1); ++ } while (vma); + out_unlock: + mmap_write_unlock(mm); + mmput(mm); +diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h +new file mode 100644 +index 000000000000..2effab72add1 +--- /dev/null ++++ b/include/linux/maple_tree.h +@@ -0,0 +1,685 @@ ++/* SPDX-License-Identifier: GPL-2.0+ */ ++#ifndef _LINUX_MAPLE_TREE_H ++#define _LINUX_MAPLE_TREE_H ++/* ++ * Maple Tree - An RCU-safe adaptive tree for storing ranges ++ * Copyright (c) 2018-2022 Oracle ++ * Authors: Liam R. Howlett ++ * Matthew Wilcox ++ */ ++ ++#include ++#include ++#include ++/* #define CONFIG_MAPLE_RCU_DISABLED */ ++/* #define CONFIG_DEBUG_MAPLE_TREE_VERBOSE */ + -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); ++/* ++ * Allocated nodes are mutable until they have been inserted into the tree, ++ * at which time they cannot change their type until they have been removed ++ * from the tree and an RCU grace period has passed. ++ * ++ * Removed nodes have their ->parent set to point to themselves. RCU readers ++ * check ->parent before relying on the value that they loaded from the ++ * slots array. This lets us reuse the slots array for the RCU head. ++ * ++ * Nodes in the tree point to their parent unless bit 0 is set. ++ */ ++#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) ++/* 64bit sizes */ ++#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ ++#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ ++#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ ++#define MAPLE_ARANGE64_META_MAX 15 /* Out of range for metadata */ ++#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) ++#else ++/* 32bit sizes */ ++#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ ++#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ ++#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ ++#define MAPLE_ARANGE64_META_MAX 31 /* Out of range for metadata */ ++#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) ++#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ + -+ return peer; -+} ++#define MAPLE_NODE_MASK 255UL + -+/** -+ * ovpn_nexthop4() - looks up the IP of the nexthop for the given destination ++/* ++ * The node->parent of the root node has bit 0 set and the rest of the pointer ++ * is a pointer to the tree itself. No more bits are available in this pointer ++ * (on m68k, the data structure may only be 2-byte aligned). + * -+ * Looks up in the IPv4 system routing table the IO of the nexthop to be used -+ * to reach the destination passed as argument. IF no nexthop can be found, the -+ * destination itself is returned as it probably has to be used as nexthop. ++ * Internal non-root nodes can only have maple_range_* nodes as parents. The ++ * parent pointer is 256B aligned like all other tree nodes. When storing a 32 ++ * or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an ++ * extra bit to store the offset. This extra bit comes from a reuse of the last ++ * bit in the node type. This is possible by using bit 1 to indicate if bit 2 ++ * is part of the type or the slot. + * -+ * @ovpn: the private data representing the current VPN session -+ * @dst: the destination to be looked up ++ * Once the type is decided, the decision of an allocation range type or a range ++ * type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE ++ * flag. + * -+ * Return the IP of the next hop if found or the dst itself otherwise ++ * Node types: ++ * 0x??1 = Root ++ * 0x?00 = 16 bit nodes ++ * 0x010 = 32 bit nodes ++ * 0x110 = 64 bit nodes ++ * ++ * Slot size and location in the parent pointer: ++ * type : slot location ++ * 0x??1 : Root ++ * 0x?00 : 16 bit values, type in 0-1, slot in 2-6 ++ * 0x010 : 32 bit values, type in 0-2, slot in 3-6 ++ * 0x110 : 64 bit values, type in 0-2, slot in 3-6 + */ -+static __be32 ovpn_nexthop4(struct ovpn_struct *ovpn, __be32 dst) -+{ -+ struct rtable *rt; -+ struct flowi4 fl = { -+ .daddr = dst -+ }; -+ -+ rt = ip_route_output_flow(dev_net(ovpn->dev), &fl, NULL); -+ if (IS_ERR(rt)) { -+ net_dbg_ratelimited("%s: no route to host %pI4\n", __func__, &dst); -+ /* if we end up here this packet is probably going to be thrown away later */ -+ return dst; -+ } -+ -+ if (!rt->rt_uses_gateway) -+ goto out; + -+ dst = rt->rt_gw4; -+out: -+ ip_rt_put(rt); -+ return dst; -+} ++/* ++ * This metadata is used to optimize the gap updating code and in reverse ++ * searching for gaps or any other code that needs to find the end of the data. ++ */ ++struct maple_metadata { ++ unsigned char end; ++ unsigned char gap; ++}; + -+/** -+ * ovpn_nexthop6() - looks up the IPv6 of the nexthop for the given destination -+ * -+ * Looks up in the IPv6 system routing table the IO of the nexthop to be used -+ * to reach the destination passed as argument. IF no nexthop can be found, the -+ * destination itself is returned as it probably has to be used as nexthop. ++/* ++ * Leaf nodes do not store pointers to nodes, they store user data. Users may ++ * store almost any bit pattern. As noted above, the optimisation of storing an ++ * entry at 0 in the root pointer cannot be done for data which have the bottom ++ * two bits set to '10'. We also reserve values with the bottom two bits set to ++ * '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs ++ * return errnos as a negative errno shifted right by two bits and the bottom ++ * two bits set to '10', and while choosing to store these values in the array ++ * is not an error, it may lead to confusion if you're testing for an error with ++ * mas_is_err(). + * -+ * @ovpn: the private data representing the current VPN session -+ * @dst: the destination to be looked up ++ * Non-leaf nodes store the type of the node pointed to (enum maple_type in bits ++ * 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now. + * -+ * Return the IP of the next hop if found or the dst itself otherwise ++ * In regular B-Tree terms, pivots are called keys. The term pivot is used to ++ * indicate that the tree is specifying ranges, Pivots may appear in the ++ * subtree with an entry attached to the value whereas keys are unique to a ++ * specific position of a B-tree. Pivot values are inclusive of the slot with ++ * the same index. + */ -+static struct in6_addr ovpn_nexthop6(struct ovpn_struct *ovpn, struct in6_addr dst) -+{ -+#if IS_ENABLED(CONFIG_IPV6) -+ struct rt6_info *rt; -+ struct flowi6 fl = { -+ .daddr = dst, ++ ++struct maple_range_64 { ++ struct maple_pnode *parent; ++ unsigned long pivot[MAPLE_RANGE64_SLOTS - 1]; ++ union { ++ void __rcu *slot[MAPLE_RANGE64_SLOTS]; ++ struct { ++ void __rcu *pad[MAPLE_RANGE64_SLOTS - 1]; ++ struct maple_metadata meta; ++ }; + }; ++}; + -+ rt = (struct rt6_info *)ipv6_stub->ipv6_dst_lookup_flow(dev_net(ovpn->dev), NULL, &fl, -+ NULL); -+ if (IS_ERR(rt)) { -+ net_dbg_ratelimited("%s: no route to host %pI6\n", __func__, &dst); -+ /* if we end up here this packet is probably going to be thrown away later */ -+ return dst; -+ } ++/* ++ * At tree creation time, the user can specify that they're willing to trade off ++ * storing fewer entries in a tree in return for storing more information in ++ * each node. ++ * ++ * The maple tree supports recording the largest range of NULL entries available ++ * in this node, also called gaps. This optimises the tree for allocating a ++ * range. ++ */ ++struct maple_arange_64 { ++ struct maple_pnode *parent; ++ unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1]; ++ void __rcu *slot[MAPLE_ARANGE64_SLOTS]; ++ unsigned long gap[MAPLE_ARANGE64_SLOTS]; ++ struct maple_metadata meta; ++}; + -+ if (!(rt->rt6i_flags & RTF_GATEWAY)) -+ goto out; ++struct maple_alloc { ++ unsigned long total; ++ unsigned char node_count; ++ unsigned int request_count; ++ struct maple_alloc *slot[MAPLE_ALLOC_SLOTS]; ++}; ++ ++struct maple_topiary { ++ struct maple_pnode *parent; ++ struct maple_enode *next; /* Overlaps the pivot */ ++}; ++ ++enum maple_type { ++ maple_dense, ++ maple_leaf_64, ++ maple_range_64, ++ maple_arange_64, ++}; + -+ dst = rt->rt6i_gateway; -+out: -+ dst_release((struct dst_entry *)rt); -+#endif -+ return dst; -+} + +/** -+ * ovpn_peer_lookup_vpn_addr() - Lookup peer to send skb to -+ * -+ * This function takes a tunnel packet and looks up the peer to send it to -+ * after encapsulation. The skb is expected to be the in-tunnel packet, without -+ * any OpenVPN related header. ++ * DOC: Maple tree flags + * -+ * Assume that the IP header is accessible in the skb data. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @skb: the skb to extract the destination address from ++ * * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree ++ * * MT_FLAGS_USE_RCU - Operate in RCU mode ++ * * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags ++ * * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value ++ * * MT_FLAGS_LOCK_MASK - How the mt_lock is used ++ * * MT_FLAGS_LOCK_IRQ - Acquired irq-safe ++ * * MT_FLAGS_LOCK_BH - Acquired bh-safe ++ * * MT_FLAGS_LOCK_EXTERN - mt_lock is not used + * -+ * Return the peer if found or NULL otherwise. ++ * MAPLE_HEIGHT_MAX The largest height that can be stored + */ -+struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, -+ bool use_src) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ struct hlist_head *head; -+ struct rt6_info *rt6i = NULL; -+ struct rtable *rt = NULL; -+ sa_family_t sa_fam; -+ struct in6_addr addr6; -+ __be32 addr4; -+ u32 index; -+ -+ /* in P2P mode, no matter the destination, packets are always sent to the single peer -+ * listening on the other side -+ */ -+ if (ovpn->mode == OVPN_MODE_P2P) { -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ return peer; -+ } ++#define MT_FLAGS_ALLOC_RANGE 0x01 ++#define MT_FLAGS_USE_RCU 0x02 ++#define MT_FLAGS_HEIGHT_OFFSET 0x02 ++#define MT_FLAGS_HEIGHT_MASK 0x7C ++#define MT_FLAGS_LOCK_MASK 0x300 ++#define MT_FLAGS_LOCK_IRQ 0x100 ++#define MT_FLAGS_LOCK_BH 0x200 ++#define MT_FLAGS_LOCK_EXTERN 0x300 + -+ sa_fam = skb_protocol_to_family(skb); -+ -+ switch (sa_fam) { -+ case AF_INET: -+ if (use_src) -+ addr4 = ip_hdr(skb)->saddr; -+ else -+ addr4 = ip_hdr(skb)->daddr; -+ addr4 = ovpn_nexthop4(ovpn, addr4); ++#define MAPLE_HEIGHT_MAX 31 + -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr4, sizeof(addr4)); -+ head = &ovpn->peers.by_vpn_addr[index]; + -+ peer = ovpn_peer_lookup_vpn_addr4(head, &addr4); -+ break; -+ case AF_INET6: -+ if (use_src) -+ addr6 = ipv6_hdr(skb)->saddr; -+ else -+ addr6 = ipv6_hdr(skb)->daddr; -+ addr6 = ovpn_nexthop6(ovpn, addr6); ++#define MAPLE_NODE_TYPE_MASK 0x0F ++#define MAPLE_NODE_TYPE_SHIFT 0x03 + -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr6, sizeof(addr6)); -+ head = &ovpn->peers.by_vpn_addr[index]; ++#define MAPLE_RESERVED_RANGE 4096 + -+ peer = ovpn_peer_lookup_vpn_addr6(head, &addr6); -+ break; -+ } ++#ifdef CONFIG_LOCKDEP ++typedef struct lockdep_map *lockdep_map_p; ++#define mt_lock_is_held(mt) lock_is_held(mt->ma_external_lock) ++#define mt_set_external_lock(mt, lock) \ ++ (mt)->ma_external_lock = &(lock)->dep_map ++#else ++typedef struct { /* nothing */ } lockdep_map_p; ++#define mt_lock_is_held(mt) 1 ++#define mt_set_external_lock(mt, lock) do { } while (0) ++#endif + -+ if (rt) -+ ip_rt_put(rt); -+ if (rt6i) -+ dst_release((struct dst_entry *)rt6i); ++/* ++ * If the tree contains a single entry at index 0, it is usually stored in ++ * tree->ma_root. To optimise for the page cache, an entry which ends in '00', ++ * '01' or '11' is stored in the root, but an entry which ends in '10' will be ++ * stored in a node. Bits 3-6 are used to store enum maple_type. ++ * ++ * The flags are used both to store some immutable information about this tree ++ * (set at tree creation time) and dynamic information set under the spinlock. ++ * ++ * Another use of flags are to indicate global states of the tree. This is the ++ * case with the MAPLE_USE_RCU flag, which indicates the tree is currently in ++ * RCU mode. This mode was added to allow the tree to reuse nodes instead of ++ * re-allocating and RCU freeing nodes when there is a single user. ++ */ ++struct maple_tree { ++ union { ++ spinlock_t ma_lock; ++ lockdep_map_p ma_external_lock; ++ }; ++ void __rcu *ma_root; ++ unsigned int ma_flags; ++}; + -+ return peer; ++/** ++ * MTREE_INIT() - Initialize a maple tree ++ * @name: The maple tree name ++ * @__flags: The maple tree flags ++ * ++ */ ++#define MTREE_INIT(name, __flags) { \ ++ .ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), \ ++ .ma_flags = __flags, \ ++ .ma_root = NULL, \ +} + -+static bool ovpn_peer_transp_match(struct ovpn_peer *peer, struct sockaddr_storage *ss) -+{ -+ struct ovpn_bind *bind = rcu_dereference(peer->bind); -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; ++/** ++ * MTREE_INIT_EXT() - Initialize a maple tree with an external lock. ++ * @name: The tree name ++ * @__flags: The maple tree flags ++ * @__lock: The external lock ++ */ ++#ifdef CONFIG_LOCKDEP ++#define MTREE_INIT_EXT(name, __flags, __lock) { \ ++ .ma_external_lock = &(__lock).dep_map, \ ++ .ma_flags = (__flags), \ ++ .ma_root = NULL, \ ++} ++#else ++#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags) ++#endif + -+ if (unlikely(!bind)) -+ return false; ++#define DEFINE_MTREE(name) \ ++ struct maple_tree name = MTREE_INIT(name, 0) + -+ if (ss->ss_family != bind->sa.in4.sin_family) -+ return false; ++#define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) ++#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock)) + -+ switch (ss->ss_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)ss; -+ if (sa4->sin_addr.s_addr != bind->sa.in4.sin_addr.s_addr) -+ return false; -+ if (sa4->sin_port != bind->sa.in4.sin_port) -+ return false; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)ss; -+ if (memcmp(&sa6->sin6_addr, &bind->sa.in6.sin6_addr, sizeof(struct in6_addr))) -+ return false; -+ if (sa6->sin6_port != bind->sa.in6.sin6_port) -+ return false; -+ break; -+ default: -+ return false; -+ } ++/* ++ * The Maple Tree squeezes various bits in at various points which aren't ++ * necessarily obvious. Usually, this is done by observing that pointers are ++ * N-byte aligned and thus the bottom log_2(N) bits are available for use. We ++ * don't use the high bits of pointers to store additional information because ++ * we don't know what bits are unused on any given architecture. ++ * ++ * Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8 ++ * low bits for our own purposes. Nodes are currently of 4 types: ++ * 1. Single pointer (Range is 0-0) ++ * 2. Non-leaf Allocation Range nodes ++ * 3. Non-leaf Range nodes ++ * 4. Leaf Range nodes All nodes consist of a number of node slots, ++ * pivots, and a parent pointer. ++ */ + -+ return true; -+} ++struct maple_node { ++ union { ++ struct { ++ struct maple_pnode *parent; ++ void __rcu *slot[MAPLE_NODE_SLOTS]; ++ }; ++ struct { ++ void *pad; ++ struct rcu_head rcu; ++ struct maple_enode *piv_parent; ++ unsigned char parent_slot; ++ enum maple_type type; ++ unsigned char slot_len; ++ unsigned int ma_flags; ++ }; ++ struct maple_range_64 mr64; ++ struct maple_arange_64 ma64; ++ struct maple_alloc alloc; ++ }; ++}; + -+static bool ovpn_peer_skb_to_sockaddr(struct sk_buff *skb, struct sockaddr_storage *ss) -+{ -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; ++/* ++ * More complicated stores can cause two nodes to become one or three and ++ * potentially alter the height of the tree. Either half of the tree may need ++ * to be rebalanced against the other. The ma_topiary struct is used to track ++ * which nodes have been 'cut' from the tree so that the change can be done ++ * safely at a later date. This is done to support RCU. ++ */ ++struct ma_topiary { ++ struct maple_enode *head; ++ struct maple_enode *tail; ++ struct maple_tree *mtree; ++}; + -+ ss->ss_family = skb_protocol_to_family(skb); -+ switch (ss->ss_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)ss; -+ sa4->sin_family = AF_INET; -+ sa4->sin_addr.s_addr = ip_hdr(skb)->saddr; -+ sa4->sin_port = udp_hdr(skb)->source; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)ss; -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = ipv6_hdr(skb)->saddr; -+ sa6->sin6_port = udp_hdr(skb)->source; -+ break; -+ default: -+ return false; -+ } ++void *mtree_load(struct maple_tree *mt, unsigned long index); + -+ return true; -+} ++int mtree_insert(struct maple_tree *mt, unsigned long index, ++ void *entry, gfp_t gfp); ++int mtree_insert_range(struct maple_tree *mt, unsigned long first, ++ unsigned long last, void *entry, gfp_t gfp); ++int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp); ++int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp); + -+static struct ovpn_peer *ovpn_peer_lookup_transp_addr_p2p(struct ovpn_struct *ovpn, -+ struct sockaddr_storage *ss) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; ++int mtree_store_range(struct maple_tree *mt, unsigned long first, ++ unsigned long last, void *entry, gfp_t gfp); ++int mtree_store(struct maple_tree *mt, unsigned long index, ++ void *entry, gfp_t gfp); ++void *mtree_erase(struct maple_tree *mt, unsigned long index); + -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && ovpn_peer_transp_match(tmp, ss) && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); ++void mtree_destroy(struct maple_tree *mt); ++void __mt_destroy(struct maple_tree *mt); + -+ return peer; ++/** ++ * mtree_empty() - Determine if a tree has any present entries. ++ * @mt: Maple Tree. ++ * ++ * Context: Any context. ++ * Return: %true if the tree contains only NULL pointers. ++ */ ++static inline bool mtree_empty(const struct maple_tree *mt) ++{ ++ return mt->ma_root == NULL; +} + -+struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb) -+{ -+ struct ovpn_peer *peer = NULL, *tmp; -+ struct sockaddr_storage ss = { 0 }; -+ struct hlist_head *head; -+ size_t sa_len; -+ bool found; -+ u32 index; ++/* Advanced API */ + -+ if (unlikely(!ovpn_peer_skb_to_sockaddr(skb, &ss))) -+ return NULL; ++/* ++ * The maple state is defined in the struct ma_state and is used to keep track ++ * of information during operations, and even between operations when using the ++ * advanced API. ++ * ++ * If state->node has bit 0 set then it references a tree location which is not ++ * a node (eg the root). If bit 1 is set, the rest of the bits are a negative ++ * errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the ++ * node type. ++ * ++ * state->alloc either has a request number of nodes or an allocated node. If ++ * stat->alloc has a requested number of nodes, the first bit will be set (0x1) ++ * and the remaining bits are the value. If state->alloc is a node, then the ++ * node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for ++ * storing more allocated nodes, a total number of nodes allocated, and the ++ * node_count in this node. node_count is the number of allocated nodes in this ++ * node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further ++ * nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc ++ * by removing a node from the state->alloc node until state->alloc->node_count ++ * is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted ++ * to state->alloc. Nodes are pushed onto state->alloc by putting the current ++ * state->alloc into the pushed node's slot[0]. ++ * ++ * The state also contains the implied min/max of the state->node, the depth of ++ * this search, and the offset. The implied min/max are either from the parent ++ * node or are 0-oo for the root node. The depth is incremented or decremented ++ * every time a node is walked down or up. The offset is the slot/pivot of ++ * interest in the node - either for reading or writing. ++ * ++ * When returning a value the maple state index and last respectively contain ++ * the start and end of the range for the entry. Ranges are inclusive in the ++ * Maple Tree. ++ */ ++struct ma_state { ++ struct maple_tree *tree; /* The tree we're operating in */ ++ unsigned long index; /* The index we're operating on - range start */ ++ unsigned long last; /* The last index we're operating on - range end */ ++ struct maple_enode *node; /* The node containing this entry */ ++ unsigned long min; /* The minimum index of this node - implied pivot min */ ++ unsigned long max; /* The maximum index of this node - implied pivot max */ ++ struct maple_alloc *alloc; /* Allocated nodes for this operation */ ++ unsigned char depth; /* depth of tree descent during write */ ++ unsigned char offset; ++ unsigned char mas_flags; ++}; + -+ if (ovpn->mode == OVPN_MODE_P2P) -+ return ovpn_peer_lookup_transp_addr_p2p(ovpn, &ss); ++struct ma_wr_state { ++ struct ma_state *mas; ++ struct maple_node *node; /* Decoded mas->node */ ++ unsigned long r_min; /* range min */ ++ unsigned long r_max; /* range max */ ++ enum maple_type type; /* mas->node type */ ++ unsigned char offset_end; /* The offset where the write ends */ ++ unsigned char node_end; /* mas->node end */ ++ unsigned long *pivots; /* mas->node->pivots pointer */ ++ unsigned long end_piv; /* The pivot at the offset end */ ++ void __rcu **slots; /* mas->node->slots pointer */ ++ void *entry; /* The entry to write */ ++ void *content; /* The existing entry that is being overwritten */ ++}; + -+ switch (ss.ss_family) { -+ case AF_INET: -+ sa_len = sizeof(struct sockaddr_in); -+ break; -+ case AF_INET6: -+ sa_len = sizeof(struct sockaddr_in6); -+ break; -+ default: -+ return NULL; -+ } ++#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock)) ++#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock)) + -+ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &ss, sa_len); -+ head = &ovpn->peers.by_transp_addr[index]; + -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_transp_addr) { -+ found = ovpn_peer_transp_match(tmp, &ss); -+ if (!found) -+ continue; ++/* ++ * Special values for ma_state.node. ++ * MAS_START means we have not searched the tree. ++ * MAS_ROOT means we have searched the tree and the entry we found lives in ++ * the root of the tree (ie it has index 0, length 1 and is the only entry in ++ * the tree). ++ * MAS_NONE means we have searched the tree and there is no node in the ++ * tree for this entry. For example, we searched for index 1 in an empty ++ * tree. Or we have a tree which points to a full leaf node and we ++ * searched for an entry which is larger than can be contained in that ++ * leaf node. ++ * MA_ERROR represents an errno. After dropping the lock and attempting ++ * to resolve the error, the walk would have to be restarted from the ++ * top of the tree as the tree may have been modified. ++ */ ++#define MAS_START ((struct maple_enode *)1UL) ++#define MAS_ROOT ((struct maple_enode *)5UL) ++#define MAS_NONE ((struct maple_enode *)9UL) ++#define MAS_PAUSE ((struct maple_enode *)17UL) ++#define MA_ERROR(err) \ ++ ((struct maple_enode *)(((unsigned long)err << 2) | 2UL)) + -+ if (!ovpn_peer_hold(tmp)) -+ continue; ++#define MA_STATE(name, mt, first, end) \ ++ struct ma_state name = { \ ++ .tree = mt, \ ++ .index = first, \ ++ .last = end, \ ++ .node = MAS_START, \ ++ .min = 0, \ ++ .max = ULONG_MAX, \ ++ .alloc = NULL, \ ++ } + -+ peer = tmp; -+ break; ++#define MA_WR_STATE(name, ma_state, wr_entry) \ ++ struct ma_wr_state name = { \ ++ .mas = ma_state, \ ++ .content = NULL, \ ++ .entry = wr_entry, \ + } -+ rcu_read_unlock(); + -+ return peer; -+} ++#define MA_TOPIARY(name, tree) \ ++ struct ma_topiary name = { \ ++ .head = NULL, \ ++ .tail = NULL, \ ++ .mtree = tree, \ ++ } + -+static struct ovpn_peer *ovpn_peer_lookup_id_p2p(struct ovpn_struct *ovpn, u32 peer_id) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; ++void *mas_walk(struct ma_state *mas); ++void *mas_store(struct ma_state *mas, void *entry); ++void *mas_erase(struct ma_state *mas); ++int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp); ++void mas_store_prealloc(struct ma_state *mas, void *entry); ++void *mas_find(struct ma_state *mas, unsigned long max); ++void *mas_find_rev(struct ma_state *mas, unsigned long min); ++int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp); ++bool mas_is_err(struct ma_state *mas); + -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); ++bool mas_nomem(struct ma_state *mas, gfp_t gfp); ++void mas_pause(struct ma_state *mas); ++void maple_tree_init(void); ++void mas_destroy(struct ma_state *mas); ++int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); + -+ return peer; -+} ++void *mas_prev(struct ma_state *mas, unsigned long min); ++void *mas_next(struct ma_state *mas, unsigned long max); + -+struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id) ++int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, ++ unsigned long size); ++ ++/* Checks if a mas has not found anything */ ++static inline bool mas_is_none(struct ma_state *mas) +{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ struct hlist_head *head; -+ u32 index; ++ return mas->node == MAS_NONE; ++} + -+ if (ovpn->mode == OVPN_MODE_P2P) -+ return ovpn_peer_lookup_id_p2p(ovpn, peer_id); ++/* Checks if a mas has been paused */ ++static inline bool mas_is_paused(struct ma_state *mas) ++{ ++ return mas->node == MAS_PAUSE; ++} + -+ index = ovpn_peer_index(ovpn->peers.by_id, &peer_id, sizeof(peer_id)); -+ head = &ovpn->peers.by_id[index]; ++void mas_dup_tree(struct ma_state *oldmas, struct ma_state *mas); ++void mas_dup_store(struct ma_state *mas, void *entry); + -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_id) { -+ if (tmp->id != peer_id) -+ continue; ++/* ++ * This finds an empty area from the highest address to the lowest. ++ * AKA "Topdown" version, ++ */ ++int mas_empty_area_rev(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size); ++/** ++ * mas_reset() - Reset a Maple Tree operation state. ++ * @mas: Maple Tree operation state. ++ * ++ * Resets the error or walk state of the @mas so future walks of the ++ * array will start from the root. Use this if you have dropped the ++ * lock and want to reuse the ma_state. ++ * ++ * Context: Any context. ++ */ ++static inline void mas_reset(struct ma_state *mas) ++{ ++ mas->node = MAS_START; ++} + -+ if (!ovpn_peer_hold(tmp)) -+ continue; ++/** ++ * mas_for_each() - Iterate over a range of the maple tree. ++ * @__mas: Maple Tree operation state (maple_state) ++ * @__entry: Entry retrieved from the tree ++ * @__max: maximum index to retrieve from the tree ++ * ++ * When returned, mas->index and mas->last will hold the entire range for the ++ * entry. ++ * ++ * Note: may return the zero entry. ++ * ++ */ ++#define mas_for_each(__mas, __entry, __max) \ ++ while (((__entry) = mas_find((__mas), (__max))) != NULL) + -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); + -+ return peer; ++/** ++ * mas_set_range() - Set up Maple Tree operation state for a different index. ++ * @mas: Maple Tree operation state. ++ * @start: New start of range in the Maple Tree. ++ * @last: New end of range in the Maple Tree. ++ * ++ * Move the operation state to refer to a different range. This will ++ * have the effect of starting a walk from the top; see mas_next() ++ * to move to an adjacent index. ++ */ ++static inline ++void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last) ++{ ++ mas->index = start; ++ mas->last = last; ++ mas->node = MAS_START; +} + -+void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb) ++/** ++ * mas_set() - Set up Maple Tree operation state for a different index. ++ * @mas: Maple Tree operation state. ++ * @index: New index into the Maple Tree. ++ * ++ * Move the operation state to refer to a different index. This will ++ * have the effect of starting a walk from the top; see mas_next() ++ * to move to an adjacent index. ++ */ ++static inline void mas_set(struct ma_state *mas, unsigned long index) +{ -+ struct ovpn_bind *bind; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) -+ goto unlock; + -+ switch (skb_protocol_to_family(skb)) { -+ case AF_INET: -+ if (unlikely(bind->local.ipv4.s_addr != ip_hdr(skb)->daddr)) { -+ netdev_dbg(peer->ovpn->dev, -+ "%s: learning local IPv4 for peer %d (%pI4 -> %pI4)\n", __func__, -+ peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); -+ bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; -+ } -+ break; -+ case AF_INET6: -+ if (unlikely(memcmp(&bind->local.ipv6, &ipv6_hdr(skb)->daddr, -+ sizeof(bind->local.ipv6)))) { -+ netdev_dbg(peer->ovpn->dev, -+ "%s: learning local IPv6 for peer %d (%pI6c -> %pI6c\n", -+ __func__, peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); -+ bind->local.ipv6 = ipv6_hdr(skb)->daddr; -+ } -+ break; -+ default: -+ break; -+ } -+unlock: -+ rcu_read_unlock(); ++ mas_set_range(mas, index, index); +} + -+static int ovpn_peer_add_mp(struct ovpn_struct *ovpn, struct ovpn_peer *peer) ++static inline bool mt_external_lock(const struct maple_tree *mt) +{ -+ struct sockaddr_storage sa = { 0 }; -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ struct ovpn_bind *bind; -+ struct ovpn_peer *tmp; -+ size_t salen; -+ int ret = 0; -+ u32 index; -+ -+ spin_lock_bh(&ovpn->peers.lock); -+ /* do not add duplicates */ -+ tmp = ovpn_peer_lookup_id(ovpn, peer->id); -+ if (tmp) { -+ ovpn_peer_put(tmp); -+ ret = -EEXIST; -+ goto unlock; -+ } -+ -+ hlist_del_init_rcu(&peer->hash_entry_transp_addr); -+ bind = rcu_dereference_protected(peer->bind, true); -+ /* peers connected via UDP have bind == NULL */ -+ if (bind) { -+ switch (bind->sa.in4.sin_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)&sa; -+ -+ sa4->sin_family = AF_INET; -+ sa4->sin_addr.s_addr = bind->sa.in4.sin_addr.s_addr; -+ sa4->sin_port = bind->sa.in4.sin_port; -+ salen = sizeof(*sa4); -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)&sa; -+ -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = bind->sa.in6.sin6_addr; -+ sa6->sin6_port = bind->sa.in6.sin6_port; -+ salen = sizeof(*sa6); -+ break; -+ default: -+ ret = -EPROTONOSUPPORT; -+ goto unlock; -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &sa, salen); -+ hlist_add_head_rcu(&peer->hash_entry_transp_addr, -+ &ovpn->peers.by_transp_addr[index]); -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_id, &peer->id, sizeof(peer->id)); -+ hlist_add_head_rcu(&peer->hash_entry_id, &ovpn->peers.by_id[index]); -+ -+ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv4, -+ sizeof(peer->vpn_addrs.ipv4)); -+ hlist_add_head_rcu(&peer->hash_entry_addr4, &ovpn->peers.by_vpn_addr[index]); -+ } -+ -+ hlist_del_init_rcu(&peer->hash_entry_addr6); -+ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) { -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv6, -+ sizeof(peer->vpn_addrs.ipv6)); -+ hlist_add_head_rcu(&peer->hash_entry_addr6, &ovpn->peers.by_vpn_addr[index]); -+ } -+ -+unlock: -+ spin_unlock_bh(&ovpn->peers.lock); -+ -+ return ret; ++ return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN; +} + -+static int ovpn_peer_add_p2p(struct ovpn_struct *ovpn, struct ovpn_peer *peer) ++/** ++ * mt_init_flags() - Initialise an empty maple tree with flags. ++ * @mt: Maple Tree ++ * @flags: maple tree flags. ++ * ++ * If you need to initialise a Maple Tree with special flags (eg, an ++ * allocation tree), use this function. ++ * ++ * Context: Any context. ++ */ ++static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags) +{ -+ struct ovpn_peer *tmp; -+ -+ spin_lock_bh(&ovpn->lock); -+ /* in p2p mode it is possible to have a single peer only, therefore the -+ * old one is released and substituted by the new one -+ */ -+ tmp = rcu_dereference(ovpn->peer); -+ if (tmp) { -+ tmp->delete_reason = OVPN_DEL_PEER_REASON_TEARDOWN; -+ ovpn_peer_put(tmp); -+ } -+ -+ rcu_assign_pointer(ovpn->peer, peer); -+ spin_unlock_bh(&ovpn->lock); -+ -+ return 0; ++ mt->ma_flags = flags; ++ if (!mt_external_lock(mt)) ++ spin_lock_init(&mt->ma_lock); ++ rcu_assign_pointer(mt->ma_root, NULL); +} + -+/* assume refcounter was increased by caller */ -+int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer) ++/** ++ * mt_init() - Initialise an empty maple tree. ++ * @mt: Maple Tree ++ * ++ * An empty Maple Tree. ++ * ++ * Context: Any context. ++ */ ++static inline void mt_init(struct maple_tree *mt) +{ -+ switch (ovpn->mode) { -+ case OVPN_MODE_MP: -+ return ovpn_peer_add_mp(ovpn, peer); -+ case OVPN_MODE_P2P: -+ return ovpn_peer_add_p2p(ovpn, peer); -+ default: -+ return -EOPNOTSUPP; -+ } ++ mt_init_flags(mt, 0); ++} ++ ++static inline bool mt_in_rcu(struct maple_tree *mt) ++{ ++#ifdef CONFIG_MAPLE_RCU_DISABLED ++ return false; ++#endif ++ return mt->ma_flags & MT_FLAGS_USE_RCU; +} + -+static void ovpn_peer_unhash(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++/** ++ * mt_clear_in_rcu() - Switch the tree to non-RCU mode. ++ * @mt: The Maple Tree ++ */ ++static inline void mt_clear_in_rcu(struct maple_tree *mt) +{ -+ hlist_del_init_rcu(&peer->hash_entry_id); -+ hlist_del_init_rcu(&peer->hash_entry_addr4); -+ hlist_del_init_rcu(&peer->hash_entry_addr6); -+ hlist_del_init_rcu(&peer->hash_entry_transp_addr); ++ if (!mt_in_rcu(mt)) ++ return; + -+ ovpn_peer_put(peer); -+ peer->delete_reason = reason; ++ if (mt_external_lock(mt)) { ++ BUG_ON(!mt_lock_is_held(mt)); ++ mt->ma_flags &= ~MT_FLAGS_USE_RCU; ++ } else { ++ mtree_lock(mt); ++ mt->ma_flags &= ~MT_FLAGS_USE_RCU; ++ mtree_unlock(mt); ++ } +} + -+static int ovpn_peer_del_mp(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++/** ++ * mt_set_in_rcu() - Switch the tree to RCU safe mode. ++ * @mt: The Maple Tree ++ */ ++static inline void mt_set_in_rcu(struct maple_tree *mt) +{ -+ struct ovpn_peer *tmp; -+ int ret = 0; ++ if (mt_in_rcu(mt)) ++ return; + -+ spin_lock_bh(&peer->ovpn->peers.lock); -+ tmp = ovpn_peer_lookup_id(peer->ovpn, peer->id); -+ if (tmp != peer) { -+ ret = -ENOENT; -+ goto unlock; ++ if (mt_external_lock(mt)) { ++ BUG_ON(!mt_lock_is_held(mt)); ++ mt->ma_flags |= MT_FLAGS_USE_RCU; ++ } else { ++ mtree_lock(mt); ++ mt->ma_flags |= MT_FLAGS_USE_RCU; ++ mtree_unlock(mt); + } -+ ovpn_peer_unhash(peer, reason); ++} + -+unlock: -+ spin_unlock_bh(&peer->ovpn->peers.lock); ++void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max); ++void *mt_find_after(struct maple_tree *mt, unsigned long *index, ++ unsigned long max); ++void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min); ++void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max); + -+ if (tmp) -+ ovpn_peer_put(tmp); ++/** ++ * mt_for_each - Iterate over each entry starting at index until max. ++ * @__tree: The Maple Tree ++ * @__entry: The current entry ++ * @__index: The index to update to track the location in the tree ++ * @__max: The maximum limit for @index ++ * ++ * Note: Will not return the zero entry. ++ */ ++#define mt_for_each(__tree, __entry, __index, __max) \ ++ for (__entry = mt_find(__tree, &(__index), __max); \ ++ __entry; __entry = mt_find_after(__tree, &(__index), __max)) ++ ++ ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++extern atomic_t maple_tree_tests_run; ++extern atomic_t maple_tree_tests_passed; ++ ++void mt_dump(const struct maple_tree *mt); ++void mt_validate(struct maple_tree *mt); ++#define MT_BUG_ON(__tree, __x) do { \ ++ atomic_inc(&maple_tree_tests_run); \ ++ if (__x) { \ ++ pr_info("BUG at %s:%d (%u)\n", \ ++ __func__, __LINE__, __x); \ ++ mt_dump(__tree); \ ++ pr_info("Pass: %u Run:%u\n", \ ++ atomic_read(&maple_tree_tests_passed), \ ++ atomic_read(&maple_tree_tests_run)); \ ++ dump_stack(); \ ++ } else { \ ++ atomic_inc(&maple_tree_tests_passed); \ ++ } \ ++} while (0) ++#else ++#define MT_BUG_ON(__tree, __x) BUG_ON(__x) ++#endif /* CONFIG_DEBUG_MAPLE_TREE */ + -+ return ret; ++#endif /*_LINUX_MAPLE_TREE_H */ +diff --git a/include/linux/mm.h b/include/linux/mm.h +index 88976a521ef5..9ac0e02e2238 100644 +--- a/include/linux/mm.h ++++ b/include/linux/mm.h +@@ -661,6 +661,38 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) + return vma->vm_flags & VM_ACCESS_FLAGS; + } + ++static inline ++struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) ++{ ++ return mas_find(&vmi->mas, max); +} + -+static int ovpn_peer_del_p2p(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) ++static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) +{ -+ struct ovpn_peer *tmp; -+ int ret = -ENOENT; -+ -+ spin_lock_bh(&peer->ovpn->lock); -+ tmp = rcu_dereference(peer->ovpn->peer); -+ if (tmp != peer) -+ goto unlock; -+ -+ ovpn_peer_put(tmp); -+ tmp->delete_reason = reason; -+ RCU_INIT_POINTER(peer->ovpn->peer, NULL); -+ ret = 0; -+ -+unlock: -+ spin_unlock_bh(&peer->ovpn->lock); ++ /* ++ * Uses vma_find() to get the first VMA when the iterator starts. ++ * Calling mas_next() could skip the first entry. ++ */ ++ return vma_find(vmi, ULONG_MAX); ++} + -+ return ret; ++static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) ++{ ++ return mas_prev(&vmi->mas, 0); +} + -+void ovpn_peer_release_p2p(struct ovpn_struct *ovpn) ++static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) +{ -+ struct ovpn_peer *tmp; ++ return vmi->mas.index; ++} + -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (!tmp) -+ goto unlock; ++#define for_each_vma(__vmi, __vma) \ ++ while (((__vma) = vma_next(&(__vmi))) != NULL) + -+ ovpn_peer_del_p2p(tmp, OVPN_DEL_PEER_REASON_TEARDOWN); -+unlock: -+ rcu_read_unlock(); -+} ++/* The MM code likes to work with exclusive end addresses */ ++#define for_each_vma_range(__vmi, __vma, __end) \ ++ while (((__vma) = vma_find(&(__vmi), (__end) - 1)) != NULL) + -+int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ switch (peer->ovpn->mode) { -+ case OVPN_MODE_MP: -+ return ovpn_peer_del_mp(peer, reason); -+ case OVPN_MODE_P2P: -+ return ovpn_peer_del_p2p(peer, reason); -+ default: -+ return -EOPNOTSUPP; + #ifdef CONFIG_SHMEM + /* + * The vma_is_shmem is not inline because it is used only by slow +@@ -1800,8 +1832,9 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, + unsigned long size); + void zap_page_range(struct vm_area_struct *vma, unsigned long address, + unsigned long size); +-void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, +- unsigned long start, unsigned long end); ++void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, ++ struct vm_area_struct *start_vma, unsigned long start, ++ unsigned long end); + + struct mmu_notifier_range; + +@@ -2598,14 +2631,15 @@ extern int __split_vma(struct mm_struct *, struct vm_area_struct *, + extern int split_vma(struct mm_struct *, struct vm_area_struct *, + unsigned long addr, int new_below); + extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); +-extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, +- struct rb_node **, struct rb_node *); + extern void unlink_file_vma(struct vm_area_struct *); + extern struct vm_area_struct *copy_vma(struct vm_area_struct **, + unsigned long addr, unsigned long len, pgoff_t pgoff, + bool *need_rmap_locks); + extern void exit_mmap(struct mm_struct *); + ++void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas); ++void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas); ++ + static inline int check_data_rlimit(unsigned long rlim, + unsigned long new, + unsigned long start, +@@ -2653,8 +2687,9 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, + extern unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + unsigned long pgoff, unsigned long *populate, struct list_head *uf); +-extern int __do_munmap(struct mm_struct *, unsigned long, size_t, +- struct list_head *uf, bool downgrade); ++extern int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, ++ unsigned long start, size_t len, struct list_head *uf, ++ bool downgrade); + extern int do_munmap(struct mm_struct *, unsigned long, size_t, + struct list_head *uf); + extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); +@@ -2721,26 +2756,12 @@ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long add + extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, + struct vm_area_struct **pprev); + +-/** +- * find_vma_intersection() - Look up the first VMA which intersects the interval +- * @mm: The process address space. +- * @start_addr: The inclusive start user address. +- * @end_addr: The exclusive end user address. +- * +- * Returns: The first VMA within the provided range, %NULL otherwise. Assumes +- * start_addr < end_addr. ++/* ++ * Look up the first VMA which intersects the interval [start_addr, end_addr) ++ * NULL if none. Assume start_addr < end_addr. + */ +-static inline + struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, +- unsigned long start_addr, +- unsigned long end_addr) +-{ +- struct vm_area_struct *vma = find_vma(mm, start_addr); +- +- if (vma && end_addr <= vma->vm_start) +- vma = NULL; +- return vma; +-} ++ unsigned long start_addr, unsigned long end_addr); + + /** + * vma_lookup() - Find a VMA at a specific address +@@ -2752,12 +2773,7 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, + static inline + struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) + { +- struct vm_area_struct *vma = find_vma(mm, addr); +- +- if (vma && addr < vma->vm_start) +- vma = NULL; +- +- return vma; ++ return mtree_load(&mm->mm_mt, addr); + } + + static inline unsigned long vm_start_gap(struct vm_area_struct *vma) +@@ -2793,7 +2809,7 @@ static inline unsigned long vma_pages(struct vm_area_struct *vma) + static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, + unsigned long vm_start, unsigned long vm_end) + { +- struct vm_area_struct *vma = find_vma(mm, vm_start); ++ struct vm_area_struct *vma = vma_lookup(mm, vm_start); + + if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) + vma = NULL; +diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h +index e1797813cc2c..5e32211cb5a9 100644 +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -407,21 +408,6 @@ struct vm_area_struct { + unsigned long vm_end; /* The first byte after our end address + within vm_mm. */ + +- /* linked list of VM areas per task, sorted by address */ +- struct vm_area_struct *vm_next, *vm_prev; +- +- struct rb_node vm_rb; +- +- /* +- * Largest free memory gap in bytes to the left of this VMA. +- * Either between this VMA and vma->vm_prev, or between one of the +- * VMAs below us in the VMA rbtree and its ->vm_prev. This helps +- * get_unmapped_area find a free area of the right size. +- */ +- unsigned long rb_subtree_gap; +- +- /* Second cache line starts here. */ +- + struct mm_struct *vm_mm; /* The address space we belong to. */ + + /* +@@ -485,9 +471,7 @@ struct vm_area_struct { + struct kioctx_table; + struct mm_struct { + struct { +- struct vm_area_struct *mmap; /* list of VMAs */ +- struct rb_root mm_rb; +- u64 vmacache_seqnum; /* per-thread vmacache */ ++ struct maple_tree mm_mt; + #ifdef CONFIG_MMU + unsigned long (*get_unmapped_area) (struct file *filp, + unsigned long addr, unsigned long len, +@@ -501,7 +485,6 @@ struct mm_struct { + unsigned long mmap_compat_legacy_base; + #endif + unsigned long task_size; /* size of task vm space */ +- unsigned long highest_vm_end; /* highest vma end address */ + pgd_t * pgd; + + #ifdef CONFIG_MEMBARRIER +@@ -697,6 +680,7 @@ struct mm_struct { + unsigned long cpu_bitmap[]; + }; + ++#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN) + extern struct mm_struct init_mm; + + /* Pointer magic because the dynamic array size confuses some compilers. */ +@@ -774,6 +758,27 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) + + #endif /* CONFIG_LRU_GEN */ + ++struct vma_iterator { ++ struct ma_state mas; ++}; ++ ++#define VMA_ITERATOR(name, __mm, __addr) \ ++ struct vma_iterator name = { \ ++ .mas = { \ ++ .tree = &(__mm)->mm_mt, \ ++ .index = __addr, \ ++ .node = MAS_START, \ ++ }, \ + } -+} + -+void ovpn_peers_free(struct ovpn_struct *ovpn) ++static inline void vma_iter_init(struct vma_iterator *vmi, ++ struct mm_struct *mm, unsigned long addr) +{ -+ struct hlist_node *tmp; -+ struct ovpn_peer *peer; -+ int bkt; -+ -+ spin_lock_bh(&ovpn->peers.lock); -+ hash_for_each_safe(ovpn->peers.by_id, bkt, tmp, peer, hash_entry_id) -+ ovpn_peer_unhash(peer, OVPN_DEL_PEER_REASON_TEARDOWN); -+ spin_unlock_bh(&ovpn->peers.lock); ++ vmi->mas.tree = &mm->mm_mt; ++ vmi->mas.index = addr; ++ vmi->mas.node = MAS_START; +} -diff --git a/drivers/net/ovpn-dco/peer.h b/drivers/net/ovpn-dco/peer.h ++ + struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); +diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h +index c1bc6731125c..0bb4b6da9993 100644 +--- a/include/linux/mm_types_task.h ++++ b/include/linux/mm_types_task.h +@@ -24,18 +24,6 @@ + IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) + #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) + +-/* +- * The per task VMA cache array: +- */ +-#define VMACACHE_BITS 2 +-#define VMACACHE_SIZE (1U << VMACACHE_BITS) +-#define VMACACHE_MASK (VMACACHE_SIZE - 1) +- +-struct vmacache { +- u64 seqnum; +- struct vm_area_struct *vmas[VMACACHE_SIZE]; +-}; +- + /* + * When updating this, please also update struct resident_page_types[] in + * kernel/fork.c +diff --git a/include/linux/sched.h b/include/linux/sched.h +index 070b15ba2e1e..2cc9429d3585 100644 +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -861,7 +861,6 @@ struct task_struct { + struct mm_struct *active_mm; + + /* Per-thread vma caching: */ +- struct vmacache vmacache; + + #ifdef SPLIT_RSS_COUNTING + struct task_rss_stat rss_stat; +diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h +index e1b8a915e9e9..f07e6998bb68 100644 +--- a/include/linux/userfaultfd_k.h ++++ b/include/linux/userfaultfd_k.h +@@ -175,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma, + unsigned long start, + unsigned long end); + +-extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, +- unsigned long start, unsigned long end, +- struct list_head *uf); ++extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, ++ unsigned long end, struct list_head *uf); + extern void userfaultfd_unmap_complete(struct mm_struct *mm, + struct list_head *uf); + +@@ -258,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma, + return true; + } + +-static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, ++static inline int userfaultfd_unmap_prep(struct mm_struct *mm, + unsigned long start, unsigned long end, + struct list_head *uf) + { +diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h +index f3fc36cd2276..3518dba1e02f 100644 +--- a/include/linux/vm_event_item.h ++++ b/include/linux/vm_event_item.h +@@ -129,10 +129,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, + NR_TLB_LOCAL_FLUSH_ALL, + NR_TLB_LOCAL_FLUSH_ONE, + #endif /* CONFIG_DEBUG_TLBFLUSH */ +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- VMACACHE_FIND_CALLS, +- VMACACHE_FIND_HITS, +-#endif + #ifdef CONFIG_SWAP + SWAP_RA, + SWAP_RA_HIT, +diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h +deleted file mode 100644 +index 6fce268a4588..000000000000 +--- a/include/linux/vmacache.h ++++ /dev/null +@@ -1,28 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0 */ +-#ifndef __LINUX_VMACACHE_H +-#define __LINUX_VMACACHE_H +- +-#include +-#include +- +-static inline void vmacache_flush(struct task_struct *tsk) +-{ +- memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas)); +-} +- +-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma); +-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm, +- unsigned long addr); +- +-#ifndef CONFIG_MMU +-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, +- unsigned long start, +- unsigned long end); +-#endif +- +-static inline void vmacache_invalidate(struct mm_struct *mm) +-{ +- mm->vmacache_seqnum++; +-} +- +-#endif /* __LINUX_VMACACHE_H */ +diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h +index bfe38869498d..19cf5b6892ce 100644 +--- a/include/linux/vmstat.h ++++ b/include/linux/vmstat.h +@@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu) + #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) + #endif + +-#ifdef CONFIG_DEBUG_VM_VMACACHE +-#define count_vm_vmacache_event(x) count_vm_event(x) +-#else +-#define count_vm_vmacache_event(x) do {} while (0) +-#endif +- + #define __count_zid_vm_events(item, zid, delta) \ + __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) + +diff --git a/include/trace/events/maple_tree.h b/include/trace/events/maple_tree.h new file mode 100644 -index 000000000000..b759c2c9da48 +index 000000000000..2be403bdc2bd --- /dev/null -+++ b/drivers/net/ovpn-dco/peer.h -@@ -0,0 +1,168 @@ ++++ b/include/trace/events/maple_tree.h +@@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM maple_tree ++ ++#if !defined(_TRACE_MM_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_MM_H ++ ++ ++#include ++ ++struct ma_state; ++ ++TRACE_EVENT(ma_op, ++ ++ TP_PROTO(const char *fn, struct ma_state *mas), ++ ++ TP_ARGS(fn, mas), ++ ++ TP_STRUCT__entry( ++ __field(const char *, fn) ++ __field(unsigned long, min) ++ __field(unsigned long, max) ++ __field(unsigned long, index) ++ __field(unsigned long, last) ++ __field(void *, node) ++ ), ++ ++ TP_fast_assign( ++ __entry->fn = fn; ++ __entry->min = mas->min; ++ __entry->max = mas->max; ++ __entry->index = mas->index; ++ __entry->last = mas->last; ++ __entry->node = mas->node; ++ ), ++ ++ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", ++ __entry->fn, ++ (void *) __entry->node, ++ (unsigned long) __entry->min, ++ (unsigned long) __entry->max, ++ (unsigned long) __entry->index, ++ (unsigned long) __entry->last ++ ) ++) ++TRACE_EVENT(ma_read, ++ ++ TP_PROTO(const char *fn, struct ma_state *mas), ++ ++ TP_ARGS(fn, mas), ++ ++ TP_STRUCT__entry( ++ __field(const char *, fn) ++ __field(unsigned long, min) ++ __field(unsigned long, max) ++ __field(unsigned long, index) ++ __field(unsigned long, last) ++ __field(void *, node) ++ ), ++ ++ TP_fast_assign( ++ __entry->fn = fn; ++ __entry->min = mas->min; ++ __entry->max = mas->max; ++ __entry->index = mas->index; ++ __entry->last = mas->last; ++ __entry->node = mas->node; ++ ), ++ ++ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", ++ __entry->fn, ++ (void *) __entry->node, ++ (unsigned long) __entry->min, ++ (unsigned long) __entry->max, ++ (unsigned long) __entry->index, ++ (unsigned long) __entry->last ++ ) ++) ++ ++TRACE_EVENT(ma_write, ++ ++ TP_PROTO(const char *fn, struct ma_state *mas, unsigned long piv, ++ void *val), ++ ++ TP_ARGS(fn, mas, piv, val), ++ ++ TP_STRUCT__entry( ++ __field(const char *, fn) ++ __field(unsigned long, min) ++ __field(unsigned long, max) ++ __field(unsigned long, index) ++ __field(unsigned long, last) ++ __field(unsigned long, piv) ++ __field(void *, val) ++ __field(void *, node) ++ ), ++ ++ TP_fast_assign( ++ __entry->fn = fn; ++ __entry->min = mas->min; ++ __entry->max = mas->max; ++ __entry->index = mas->index; ++ __entry->last = mas->last; ++ __entry->piv = piv; ++ __entry->val = val; ++ __entry->node = mas->node; ++ ), ++ ++ TP_printk("%s\tNode %p (%lu %lu) range:%lu-%lu piv (%lu) val %p", ++ __entry->fn, ++ (void *) __entry->node, ++ (unsigned long) __entry->min, ++ (unsigned long) __entry->max, ++ (unsigned long) __entry->index, ++ (unsigned long) __entry->last, ++ (unsigned long) __entry->piv, ++ (void *) __entry->val ++ ) ++) ++#endif /* _TRACE_MM_H */ ++ ++/* This part must be outside protection */ ++#include +diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h +index 4661f7ba07c0..216de5f03621 100644 +--- a/include/trace/events/mmap.h ++++ b/include/trace/events/mmap.h +@@ -42,6 +42,79 @@ TRACE_EVENT(vm_unmapped_area, + __entry->low_limit, __entry->high_limit, __entry->align_mask, + __entry->align_offset) + ); ++ ++TRACE_EVENT(vma_mas_szero, ++ TP_PROTO(struct maple_tree *mt, unsigned long start, ++ unsigned long end), ++ ++ TP_ARGS(mt, start, end), ++ ++ TP_STRUCT__entry( ++ __field(struct maple_tree *, mt) ++ __field(unsigned long, start) ++ __field(unsigned long, end) ++ ), ++ ++ TP_fast_assign( ++ __entry->mt = mt; ++ __entry->start = start; ++ __entry->end = end; ++ ), ++ ++ TP_printk("mt_mod %p, (NULL), SNULL, %lu, %lu,", ++ __entry->mt, ++ (unsigned long) __entry->start, ++ (unsigned long) __entry->end ++ ) ++); + -+#ifndef _NET_OVPN_DCO_OVPNPEER_H_ -+#define _NET_OVPN_DCO_OVPNPEER_H_ ++TRACE_EVENT(vma_store, ++ TP_PROTO(struct maple_tree *mt, struct vm_area_struct *vma), ++ ++ TP_ARGS(mt, vma), ++ ++ TP_STRUCT__entry( ++ __field(struct maple_tree *, mt) ++ __field(struct vm_area_struct *, vma) ++ __field(unsigned long, vm_start) ++ __field(unsigned long, vm_end) ++ ), ++ ++ TP_fast_assign( ++ __entry->mt = mt; ++ __entry->vma = vma; ++ __entry->vm_start = vma->vm_start; ++ __entry->vm_end = vma->vm_end - 1; ++ ), ++ ++ TP_printk("mt_mod %p, (%p), STORE, %lu, %lu,", ++ __entry->mt, __entry->vma, ++ (unsigned long) __entry->vm_start, ++ (unsigned long) __entry->vm_end ++ ) ++); + -+#include "addr.h" -+#include "bind.h" -+#include "sock.h" -+#include "stats.h" + -+#include -+#include -+#include ++TRACE_EVENT(exit_mmap, ++ TP_PROTO(struct mm_struct *mm), + -+struct ovpn_peer { -+ struct ovpn_struct *ovpn; ++ TP_ARGS(mm), + -+ u32 id; ++ TP_STRUCT__entry( ++ __field(struct mm_struct *, mm) ++ __field(struct maple_tree *, mt) ++ ), + -+ struct { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } vpn_addrs; ++ TP_fast_assign( ++ __entry->mm = mm; ++ __entry->mt = &mm->mm_mt; ++ ), + -+ struct hlist_node hash_entry_id; -+ struct hlist_node hash_entry_addr4; -+ struct hlist_node hash_entry_addr6; -+ struct hlist_node hash_entry_transp_addr; ++ TP_printk("mt_mod %p, DESTROY\n", ++ __entry->mt ++ ) ++); + -+ /* work objects to handle encryption/decryption of packets. -+ * these works are queued on the ovpn->crypt_wq workqueue. -+ */ -+ struct work_struct encrypt_work; -+ struct work_struct decrypt_work; + #endif + + /* This part must be outside protection */ +diff --git a/init/main.c b/init/main.c +index 1fe7942f5d4a..df800fc61b2a 100644 +--- a/init/main.c ++++ b/init/main.c +@@ -117,6 +117,7 @@ static int kernel_init(void *); + + extern void init_IRQ(void); + extern void radix_tree_init(void); ++extern void maple_tree_init(void); + + /* + * Debug helper: via this flag we know that we are in 'early bootup code' +@@ -1002,6 +1003,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) + "Interrupts were enabled *very* early, fixing it\n")) + local_irq_disable(); + radix_tree_init(); ++ maple_tree_init(); + + /* + * Set up housekeeping before setting up workqueues to allow the unbound +diff --git a/ipc/shm.c b/ipc/shm.c +index b3048ebd5c31..7d86f058fb86 100644 +--- a/ipc/shm.c ++++ b/ipc/shm.c +@@ -1721,7 +1721,7 @@ long ksys_shmdt(char __user *shmaddr) + #ifdef CONFIG_MMU + loff_t size = 0; + struct file *file; +- struct vm_area_struct *next; ++ VMA_ITERATOR(vmi, mm, addr); + #endif + + if (addr & ~PAGE_MASK) +@@ -1751,12 +1751,9 @@ long ksys_shmdt(char __user *shmaddr) + * match the usual checks anyway. So assume all vma's are + * above the starting address given. + */ +- vma = find_vma(mm, addr); + + #ifdef CONFIG_MMU +- while (vma) { +- next = vma->vm_next; +- ++ for_each_vma(vmi, vma) { + /* + * Check if the starting address would match, i.e. it's + * a fragment created by mprotect() and/or munmap(), or it +@@ -1774,6 +1771,7 @@ long ksys_shmdt(char __user *shmaddr) + file = vma->vm_file; + size = i_size_read(file_inode(vma->vm_file)); + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); ++ mas_pause(&vmi.mas); + /* + * We discovered the size of the shm segment, so + * break out of here and fall through to the next +@@ -1781,10 +1779,9 @@ long ksys_shmdt(char __user *shmaddr) + * searching for matching vma's. + */ + retval = 0; +- vma = next; ++ vma = vma_next(&vmi); + break; + } +- vma = next; + } + + /* +@@ -1794,17 +1791,19 @@ long ksys_shmdt(char __user *shmaddr) + */ + size = PAGE_ALIGN(size); + while (vma && (loff_t)(vma->vm_end - addr) <= size) { +- next = vma->vm_next; +- + /* finding a matching vma now does not alter retval */ + if ((vma->vm_ops == &shm_vm_ops) && + ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && +- (vma->vm_file == file)) ++ (vma->vm_file == file)) { + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); +- vma = next; ++ mas_pause(&vmi.mas); ++ } + -+ struct ptr_ring tx_ring; -+ struct ptr_ring rx_ring; -+ struct ptr_ring netif_rx_ring; ++ vma = vma_next(&vmi); + } + + #else /* CONFIG_MMU */ ++ vma = vma_lookup(mm, addr); + /* under NOMMU conditions, the exact address to be destroyed must be + * given + */ +diff --git a/kernel/acct.c b/kernel/acct.c +index 13706356ec54..62200d799b9b 100644 +--- a/kernel/acct.c ++++ b/kernel/acct.c +@@ -555,15 +555,14 @@ void acct_collect(long exitcode, int group_dead) + unsigned long vsize = 0; + + if (group_dead && current->mm) { ++ struct mm_struct *mm = current->mm; ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + +- mmap_read_lock(current->mm); +- vma = current->mm->mmap; +- while (vma) { ++ mmap_read_lock(mm); ++ for_each_vma(vmi, vma) + vsize += vma->vm_end - vma->vm_start; +- vma = vma->vm_next; +- } +- mmap_read_unlock(current->mm); ++ mmap_read_unlock(mm); + } + + spin_lock_irq(¤t->sighand->siglock); +diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c +index 8c921799def4..1c8debd42dc9 100644 +--- a/kernel/bpf/task_iter.c ++++ b/kernel/bpf/task_iter.c +@@ -299,8 +299,8 @@ struct bpf_iter_seq_task_vma_info { + }; + + enum bpf_task_vma_iter_find_op { +- task_vma_iter_first_vma, /* use mm->mmap */ +- task_vma_iter_next_vma, /* use curr_vma->vm_next */ ++ task_vma_iter_first_vma, /* use find_vma() with addr 0 */ ++ task_vma_iter_next_vma, /* use vma_next() with curr_vma */ + task_vma_iter_find_vma, /* use find_vma() to find next vma */ + }; + +@@ -400,10 +400,10 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) + + switch (op) { + case task_vma_iter_first_vma: +- curr_vma = curr_task->mm->mmap; ++ curr_vma = find_vma(curr_task->mm, 0); + break; + case task_vma_iter_next_vma: +- curr_vma = curr_vma->vm_next; ++ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); + break; + case task_vma_iter_find_vma: + /* We dropped mmap_lock so it is necessary to use find_vma +@@ -417,7 +417,7 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) + if (curr_vma && + curr_vma->vm_start == info->prev_vm_start && + curr_vma->vm_end == info->prev_vm_end) +- curr_vma = curr_vma->vm_next; ++ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); + break; + } + if (!curr_vma) { +diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c +index 7beceb447211..d5e9ccde3ab8 100644 +--- a/kernel/debug/debug_core.c ++++ b/kernel/debug/debug_core.c +@@ -50,7 +50,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -283,17 +282,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr) + if (!CACHE_FLUSH_IS_SAFE) + return; + +- if (current->mm) { +- int i; +- +- for (i = 0; i < VMACACHE_SIZE; i++) { +- if (!current->vmacache.vmas[i]) +- continue; +- flush_cache_range(current->vmacache.vmas[i], +- addr, addr + BREAK_INSTR_SIZE); +- } +- } +- + /* Force flush instruction cache if it was outside the mm */ + flush_icache_range(addr, addr + BREAK_INSTR_SIZE); + } +diff --git a/kernel/events/core.c b/kernel/events/core.c +index 2621fd24ad26..101c5912c3fc 100644 +--- a/kernel/events/core.c ++++ b/kernel/events/core.c +@@ -10229,8 +10229,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, + struct perf_addr_filter_range *fr) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + +diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c +index 2eaa327f8158..401bc2d24ce0 100644 +--- a/kernel/events/uprobes.c ++++ b/kernel/events/uprobes.c +@@ -349,9 +349,10 @@ static bool valid_ref_ctr_vma(struct uprobe *uprobe, + static struct vm_area_struct * + find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *tmp; + +- for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) ++ for_each_vma(vmi, tmp) + if (valid_ref_ctr_vma(uprobe, tmp)) + return tmp; + +@@ -1231,11 +1232,12 @@ int uprobe_apply(struct inode *inode, loff_t offset, + + static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + int err = 0; + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + unsigned long vaddr; + loff_t offset; + +@@ -1983,9 +1985,10 @@ bool uprobe_deny_signal(void) + + static void mmf_recalc_uprobes(struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!valid_vma(vma, false)) + continue; + /* +diff --git a/kernel/fork.c b/kernel/fork.c +index b15aaae04403..1add76edca04 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -43,7 +43,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -479,7 +478,6 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) + */ + *new = data_race(*orig); + INIT_LIST_HEAD(&new->anon_vma_chain); +- new->vm_next = new->vm_prev = NULL; + dup_anon_vma_name(orig, new); + } + return new; +@@ -584,11 +582,12 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) + static __latent_entropy int dup_mmap(struct mm_struct *mm, + struct mm_struct *oldmm) + { +- struct vm_area_struct *mpnt, *tmp, *prev, **pprev; +- struct rb_node **rb_link, *rb_parent; ++ struct vm_area_struct *mpnt, *tmp; + int retval; +- unsigned long charge; ++ unsigned long charge = 0; + LIST_HEAD(uf); ++ MA_STATE(old_mas, &oldmm->mm_mt, 0, 0); ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + uprobe_start_dup_mmap(); + if (mmap_write_lock_killable(oldmm)) { +@@ -610,16 +609,16 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + mm->exec_vm = oldmm->exec_vm; + mm->stack_vm = oldmm->stack_vm; + +- rb_link = &mm->mm_rb.rb_node; +- rb_parent = NULL; +- pprev = &mm->mmap; + retval = ksm_fork(mm, oldmm); + if (retval) + goto out; + khugepaged_fork(mm, oldmm); + +- prev = NULL; +- for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { ++ retval = mas_expected_entries(&mas, oldmm->map_count); ++ if (retval) ++ goto out; + -+ struct napi_struct napi; ++ mas_for_each(&old_mas, mpnt, ULONG_MAX) { + struct file *file; + + if (mpnt->vm_flags & VM_DONTCOPY) { +@@ -633,7 +632,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + */ + if (fatal_signal_pending(current)) { + retval = -EINTR; +- goto out; ++ goto loop_out; + } + if (mpnt->vm_flags & VM_ACCOUNT) { + unsigned long len = vma_pages(mpnt); +@@ -686,17 +685,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + if (is_vm_hugetlb_page(tmp)) + reset_vma_resv_huge_pages(tmp); + +- /* +- * Link in the new vma and copy the page table entries. +- */ +- *pprev = tmp; +- pprev = &tmp->vm_next; +- tmp->vm_prev = prev; +- prev = tmp; +- +- __vma_link_rb(mm, tmp, rb_link, rb_parent); +- rb_link = &tmp->vm_rb.rb_right; +- rb_parent = &tmp->vm_rb; ++ /* Link the vma into the MT */ ++ mas.index = tmp->vm_start; ++ mas.last = tmp->vm_end - 1; ++ mas_store(&mas, tmp); ++ if (mas_is_err(&mas)) ++ goto fail_nomem_mas_store; + + mm->map_count++; + if (!(tmp->vm_flags & VM_WIPEONFORK)) +@@ -706,10 +700,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + tmp->vm_ops->open(tmp); + + if (retval) +- goto out; ++ goto loop_out; + } + /* a new mm has just been created */ + retval = arch_dup_mmap(oldmm, mm); ++loop_out: ++ mas_destroy(&mas); + out: + mmap_write_unlock(mm); + flush_tlb_mm(oldmm); +@@ -718,6 +714,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + fail_uprobe_end: + uprobe_end_dup_mmap(); + return retval; ++ ++fail_nomem_mas_store: ++ unlink_anon_vmas(tmp); + fail_nomem_anon_vma_fork: + mpol_put(vma_policy(tmp)); + fail_nomem_policy: +@@ -725,7 +724,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + fail_nomem: + retval = -ENOMEM; + vm_unacct_memory(charge); +- goto out; ++ goto loop_out; + } + + static inline int mm_alloc_pgd(struct mm_struct *mm) +@@ -1113,9 +1112,8 @@ static void mm_init_uprobes_state(struct mm_struct *mm) + static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, + struct user_namespace *user_ns) + { +- mm->mmap = NULL; +- mm->mm_rb = RB_ROOT; +- mm->vmacache_seqnum = 0; ++ mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); ++ mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); + atomic_set(&mm->mm_users, 1); + atomic_set(&mm->mm_count, 1); + seqcount_init(&mm->write_protect_seq); +@@ -1291,13 +1289,16 @@ int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) + /* Forbid mm->exe_file change if old file still mapped. */ + old_exe_file = get_mm_exe_file(mm); + if (old_exe_file) { ++ VMA_ITERATOR(vmi, mm, 0); + mmap_read_lock(mm); +- for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + if (path_equal(&vma->vm_file->f_path, +- &old_exe_file->f_path)) ++ &old_exe_file->f_path)) { + ret = -EBUSY; ++ break; ++ } + } + mmap_read_unlock(mm); + fput(old_exe_file); +@@ -1573,9 +1574,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) + if (!oldmm) + return 0; + +- /* initialize the new vmacache entries */ +- vmacache_flush(tsk); +- + if (clone_flags & CLONE_VM) { + mmget(oldmm); + mm = oldmm; +diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c +index 1d3e53fed1a4..4a6e22f96ede 100644 +--- a/kernel/sched/fair.c ++++ b/kernel/sched/fair.c +@@ -2784,6 +2784,7 @@ static void task_numa_work(struct callback_head *work) + struct task_struct *p = current; + struct mm_struct *mm = p->mm; + u64 runtime = p->se.sum_exec_runtime; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + struct vm_area_struct *vma; + unsigned long start, end; + unsigned long nr_pte_updates = 0; +@@ -2840,13 +2841,16 @@ static void task_numa_work(struct callback_head *work) + + if (!mmap_read_trylock(mm)) + return; +- vma = find_vma(mm, start); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) { + reset_ptenuma_scan(p); + start = 0; +- vma = mm->mmap; ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + } +- for (; vma; vma = vma->vm_next) { + -+ struct ovpn_socket *sock; ++ for (; vma; vma = mas_find(&mas, ULONG_MAX)) { + if (!vma_migratable(vma) || !vma_policy_mof(vma) || + is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + continue; +diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug +index d3e5f36bb01e..72cef567aae6 100644 +--- a/lib/Kconfig.debug ++++ b/lib/Kconfig.debug +@@ -814,13 +814,12 @@ config DEBUG_VM + + If unsure, say N. + +-config DEBUG_VM_VMACACHE +- bool "Debug VMA caching" ++config DEBUG_VM_MAPLE_TREE ++ bool "Debug VM maple trees" + depends on DEBUG_VM ++ select DEBUG_MAPLE_TREE + help +- Enable this to turn on VMA caching debug information. Doing so +- can cause significant overhead, so only enable it in non-production +- environments. ++ Enable VM maple tree debugging information and extra validations. + + If unsure, say N. + +@@ -1637,6 +1636,14 @@ config BUG_ON_DATA_CORRUPTION + + If unsure, say N. + ++config DEBUG_MAPLE_TREE ++ bool "Debug maple trees" ++ depends on DEBUG_KERNEL ++ help ++ Enable maple tree debugging information and extra validations. + -+ /* state of the TCP reading. Needed to keep track of how much of a single packet has already -+ * been read from the stream and how much is missing -+ */ -+ struct { -+ struct ptr_ring tx_ring; -+ struct work_struct tx_work; -+ struct work_struct rx_work; ++ If unsure, say N. + -+ u8 raw_len[sizeof(u16)]; -+ struct sk_buff *skb; -+ u16 offset; -+ u16 data_len; -+ struct { -+ void (*sk_state_change)(struct sock *sk); -+ void (*sk_data_ready)(struct sock *sk); -+ void (*sk_write_space)(struct sock *sk); -+ } sk_cb; -+ } tcp; + endmenu + + config DEBUG_CREDENTIALS +diff --git a/lib/Makefile b/lib/Makefile +index ffabc30a27d4..6dc0d6f8e57d 100644 +--- a/lib/Makefile ++++ b/lib/Makefile +@@ -29,7 +29,7 @@ endif + + lib-y := ctype.o string.o vsprintf.o cmdline.o \ + rbtree.o radix-tree.o timerqueue.o xarray.o \ +- idr.o extable.o irq_regs.o argv_split.o \ ++ maple_tree.o idr.o extable.o irq_regs.o argv_split.o \ + flex_proportions.o ratelimit.o show_mem.o \ + is_single_threaded.o plist.o decompress.o kobject_uevent.o \ + earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ +diff --git a/lib/maple_tree.c b/lib/maple_tree.c +new file mode 100644 +index 000000000000..e1743803c851 +--- /dev/null ++++ b/lib/maple_tree.c +@@ -0,0 +1,7130 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++/* ++ * Maple Tree implementation ++ * Copyright (c) 2018-2022 Oracle Corporation ++ * Authors: Liam R. Howlett ++ * Matthew Wilcox ++ */ + -+ struct dst_cache dst_cache; ++/* ++ * DOC: Interesting implementation details of the Maple Tree ++ * ++ * Each node type has a number of slots for entries and a number of slots for ++ * pivots. In the case of dense nodes, the pivots are implied by the position ++ * and are simply the slot index + the minimum of the node. ++ * ++ * In regular B-Tree terms, pivots are called keys. The term pivot is used to ++ * indicate that the tree is specifying ranges, Pivots may appear in the ++ * subtree with an entry attached to the value where as keys are unique to a ++ * specific position of a B-tree. Pivot values are inclusive of the slot with ++ * the same index. ++ * ++ * ++ * The following illustrates the layout of a range64 nodes slots and pivots. ++ * ++ * ++ * Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 | ++ * ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ++ * │ │ │ │ │ │ │ │ └─ Implied maximum ++ * │ │ │ │ │ │ │ └─ Pivot 14 ++ * │ │ │ │ │ │ └─ Pivot 13 ++ * │ │ │ │ │ └─ Pivot 12 ++ * │ │ │ │ └─ Pivot 11 ++ * │ │ │ └─ Pivot 2 ++ * │ │ └─ Pivot 1 ++ * │ └─ Pivot 0 ++ * └─ Implied minimum ++ * ++ * Slot contents: ++ * Internal (non-leaf) nodes contain pointers to other nodes. ++ * Leaf nodes contain entries. ++ * ++ * The location of interest is often referred to as an offset. All offsets have ++ * a slot, but the last offset has an implied pivot from the node above (or ++ * UINT_MAX for the root node. ++ * ++ * Ranges complicate certain write activities. When modifying any of ++ * the B-tree variants, it is known that one entry will either be added or ++ * deleted. When modifying the Maple Tree, one store operation may overwrite ++ * the entire data set, or one half of the tree, or the middle half of the tree. ++ * ++ */ + -+ /* our crypto state */ -+ struct ovpn_crypto_state crypto; + -+ /* our binding to peer, protected by spinlock */ -+ struct ovpn_bind __rcu *bind; ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+ /* timer used to send periodic ping messages to the other peer, if no -+ * other data was sent within the past keepalive_interval seconds -+ */ -+ struct timer_list keepalive_xmit; -+ /* keepalive interval in seconds */ -+ unsigned long keepalive_interval; ++#define CREATE_TRACE_POINTS ++#include + -+ /* timer used to mark a peer as expired when no data is received for -+ * keepalive_timeout seconds -+ */ -+ struct timer_list keepalive_recv; -+ /* keepalive timeout in seconds */ -+ unsigned long keepalive_timeout; ++#define MA_ROOT_PARENT 1 + -+ /* true if ovpn_peer_mark_delete was called */ -+ bool halt; ++/* ++ * Maple state flags ++ * * MA_STATE_BULK - Bulk insert mode ++ * * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert ++ * * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation ++ */ ++#define MA_STATE_BULK 1 ++#define MA_STATE_REBALANCE 2 ++#define MA_STATE_PREALLOC 4 ++ ++#define ma_parent_ptr(x) ((struct maple_pnode *)(x)) ++#define ma_mnode_ptr(x) ((struct maple_node *)(x)) ++#define ma_enode_ptr(x) ((struct maple_enode *)(x)) ++static struct kmem_cache *maple_node_cache; ++ ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++static const unsigned long mt_max[] = { ++ [maple_dense] = MAPLE_NODE_SLOTS, ++ [maple_leaf_64] = ULONG_MAX, ++ [maple_range_64] = ULONG_MAX, ++ [maple_arange_64] = ULONG_MAX, ++}; ++#define mt_node_max(x) mt_max[mte_node_type(x)] ++#endif + -+ /* per-peer rx/tx stats */ -+ struct ovpn_peer_stats stats; ++static const unsigned char mt_slots[] = { ++ [maple_dense] = MAPLE_NODE_SLOTS, ++ [maple_leaf_64] = MAPLE_RANGE64_SLOTS, ++ [maple_range_64] = MAPLE_RANGE64_SLOTS, ++ [maple_arange_64] = MAPLE_ARANGE64_SLOTS, ++}; ++#define mt_slot_count(x) mt_slots[mte_node_type(x)] + -+ /* why peer was deleted - keepalive timeout, module removed etc */ -+ enum ovpn_del_peer_reason delete_reason; ++static const unsigned char mt_pivots[] = { ++ [maple_dense] = 0, ++ [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1, ++ [maple_range_64] = MAPLE_RANGE64_SLOTS - 1, ++ [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1, ++}; ++#define mt_pivot_count(x) mt_pivots[mte_node_type(x)] + -+ /* protects binding to peer (bind) and timers -+ * (keepalive_xmit, keepalive_expire) -+ */ -+ spinlock_t lock; ++static const unsigned char mt_min_slots[] = { ++ [maple_dense] = MAPLE_NODE_SLOTS / 2, ++ [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, ++ [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, ++ [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1, ++}; ++#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] + -+ /* needed because crypto methods can go async */ -+ struct kref refcount; ++#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) ++#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) + -+ /* needed to free a peer in an RCU safe way */ -+ struct rcu_head rcu; ++struct maple_big_node { ++ struct maple_pnode *parent; ++ unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; ++ union { ++ struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; ++ struct { ++ unsigned long padding[MAPLE_BIG_NODE_GAPS]; ++ unsigned long gap[MAPLE_BIG_NODE_GAPS]; ++ }; ++ }; ++ unsigned char b_end; ++ enum maple_type type; ++}; + -+ /* needed to notify userspace about deletion */ -+ struct work_struct delete_work; ++/* ++ * The maple_subtree_state is used to build a tree to replace a segment of an ++ * existing tree in a more atomic way. Any walkers of the older tree will hit a ++ * dead node and restart on updates. ++ */ ++struct maple_subtree_state { ++ struct ma_state *orig_l; /* Original left side of subtree */ ++ struct ma_state *orig_r; /* Original right side of subtree */ ++ struct ma_state *l; /* New left side of subtree */ ++ struct ma_state *m; /* New middle of subtree (rare) */ ++ struct ma_state *r; /* New right side of subtree */ ++ struct ma_topiary *free; /* nodes to be freed */ ++ struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ ++ struct maple_big_node *bn; +}; + -+void ovpn_peer_release_kref(struct kref *kref); -+void ovpn_peer_release(struct ovpn_peer *peer); ++/* Functions */ ++static inline struct maple_node *mt_alloc_one(gfp_t gfp) ++{ ++ return kmem_cache_alloc(maple_node_cache, gfp | __GFP_ZERO); ++} + -+static inline bool ovpn_peer_hold(struct ovpn_peer *peer) ++static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes) +{ -+ return kref_get_unless_zero(&peer->refcount); ++ return kmem_cache_alloc_bulk(maple_node_cache, gfp | __GFP_ZERO, size, ++ nodes); +} + -+static inline void ovpn_peer_put(struct ovpn_peer *peer) ++static inline void mt_free_bulk(size_t size, void __rcu **nodes) +{ -+ kref_put(&peer->refcount, ovpn_peer_release_kref); ++ kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes); +} + -+static inline void ovpn_peer_keepalive_recv_reset(struct ovpn_peer *peer) ++static void mt_free_rcu(struct rcu_head *head) +{ -+ u32 delta = msecs_to_jiffies(peer->keepalive_timeout * MSEC_PER_SEC); ++ struct maple_node *node = container_of(head, struct maple_node, rcu); + -+ if (unlikely(!delta)) -+ return; ++ kmem_cache_free(maple_node_cache, node); ++} + -+ mod_timer(&peer->keepalive_recv, jiffies + delta); ++/* ++ * ma_free_rcu() - Use rcu callback to free a maple node ++ * @node: The node to free ++ * ++ * The maple tree uses the parent pointer to indicate this node is no longer in ++ * use and will be freed. ++ */ ++static void ma_free_rcu(struct maple_node *node) ++{ ++ node->parent = ma_parent_ptr(node); ++ call_rcu(&node->rcu, mt_free_rcu); +} + -+static inline void ovpn_peer_keepalive_xmit_reset(struct ovpn_peer *peer) ++static unsigned int mt_height(const struct maple_tree *mt) +{ -+ u32 delta = msecs_to_jiffies(peer->keepalive_interval * MSEC_PER_SEC); ++ return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; ++} + -+ if (unlikely(!delta)) -+ return; ++static void mas_set_height(struct ma_state *mas) ++{ ++ unsigned int new_flags = mas->tree->ma_flags; + -+ mod_timer(&peer->keepalive_xmit, jiffies + delta); ++ new_flags &= ~MT_FLAGS_HEIGHT_MASK; ++ BUG_ON(mas->depth > MAPLE_HEIGHT_MAX); ++ new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET; ++ mas->tree->ma_flags = new_flags; +} + -+struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, -+ struct socket *sock, u32 id, uint8_t *local_ip); ++static unsigned int mas_mt_height(struct ma_state *mas) ++{ ++ return mt_height(mas->tree); ++} + -+void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout); ++static inline enum maple_type mte_node_type(const struct maple_enode *entry) ++{ ++ return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) & ++ MAPLE_NODE_TYPE_MASK; ++} + -+int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer); -+int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason); -+struct ovpn_peer *ovpn_peer_find(struct ovpn_struct *ovpn, u32 peer_id); -+void ovpn_peer_release_p2p(struct ovpn_struct *ovpn); -+void ovpn_peers_free(struct ovpn_struct *ovpn); ++static inline bool ma_is_dense(const enum maple_type type) ++{ ++ return type < maple_leaf_64; ++} + -+struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb); -+struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, -+ bool use_src); -+struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id); ++static inline bool ma_is_leaf(const enum maple_type type) ++{ ++ return type < maple_range_64; ++} + -+void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb); -+void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb); ++static inline bool mte_is_leaf(const struct maple_enode *entry) ++{ ++ return ma_is_leaf(mte_node_type(entry)); ++} + -+#endif /* _NET_OVPN_DCO_OVPNPEER_H_ */ -diff --git a/drivers/net/ovpn-dco/pktid.c b/drivers/net/ovpn-dco/pktid.c -new file mode 100644 -index 000000000000..fcde8fba5156 ---- /dev/null -+++ b/drivers/net/ovpn-dco/pktid.c -@@ -0,0 +1,127 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan ++/* ++ * We also reserve values with the bottom two bits set to '10' which are ++ * below 4096 + */ ++static inline bool mt_is_reserved(const void *entry) ++{ ++ return ((unsigned long)entry < MAPLE_RESERVED_RANGE) && ++ xa_is_internal(entry); ++} ++ ++static inline void mas_set_err(struct ma_state *mas, long err) ++{ ++ mas->node = MA_ERROR(err); ++} + -+#include "pktid.h" ++static inline bool mas_is_ptr(struct ma_state *mas) ++{ ++ return mas->node == MAS_ROOT; ++} + -+#include -+#include ++static inline bool mas_is_start(struct ma_state *mas) ++{ ++ return mas->node == MAS_START; ++} + -+void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid) ++bool mas_is_err(struct ma_state *mas) +{ -+ atomic64_set(&pid->seq_num, 1); -+ pid->tcp_linear = NULL; ++ return xa_is_err(mas->node); +} + -+void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr) ++static inline bool mas_searchable(struct ma_state *mas) +{ -+ memset(pr, 0, sizeof(*pr)); -+ spin_lock_init(&pr->lock); ++ if (mas_is_none(mas)) ++ return false; ++ ++ if (mas_is_ptr(mas)) ++ return false; ++ ++ return true; +} + -+/* Packet replay detection. -+ * Allows ID backtrack of up to REPLAY_WINDOW_SIZE - 1. ++static inline struct maple_node *mte_to_node(const struct maple_enode *entry) ++{ ++ return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK); ++} ++ ++/* ++ * mte_to_mat() - Convert a maple encoded node to a maple topiary node. ++ * @entry: The maple encoded node ++ * ++ * Return: a maple topiary pointer + */ -+int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time) ++static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) +{ -+ const unsigned long now = jiffies; -+ int ret; ++ return (struct maple_topiary *) ++ ((unsigned long)entry & ~MAPLE_NODE_MASK); ++} + -+ spin_lock(&pr->lock); ++/* ++ * mas_mn() - Get the maple state node. ++ * @mas: The maple state ++ * ++ * Return: the maple node (not encoded - bare pointer). ++ */ ++static inline struct maple_node *mas_mn(const struct ma_state *mas) ++{ ++ return mte_to_node(mas->node); ++} + -+ /* expire backtracks at or below pr->id after PKTID_RECV_EXPIRE time */ -+ if (unlikely(time_after_eq(now, pr->expire))) -+ pr->id_floor = pr->id; ++/* ++ * mte_set_node_dead() - Set a maple encoded node as dead. ++ * @mn: The maple encoded node. ++ */ ++static inline void mte_set_node_dead(struct maple_enode *mn) ++{ ++ mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn)); ++ smp_wmb(); /* Needed for RCU */ ++} + -+ /* ID must not be zero */ -+ if (unlikely(pkt_id == 0)) { -+ ret = -EINVAL; -+ goto out; -+ } ++/* Bit 1 indicates the root is a node */ ++#define MAPLE_ROOT_NODE 0x02 ++/* maple_type stored bit 3-6 */ ++#define MAPLE_ENODE_TYPE_SHIFT 0x03 ++/* Bit 2 means a NULL somewhere below */ ++#define MAPLE_ENODE_NULL 0x04 + -+ /* time changed? */ -+ if (unlikely(pkt_time != pr->time)) { -+ if (pkt_time > pr->time) { -+ /* time moved forward, accept */ -+ pr->base = 0; -+ pr->extent = 0; -+ pr->id = 0; -+ pr->time = pkt_time; -+ pr->id_floor = 0; -+ } else { -+ /* time moved backward, reject */ -+ ret = -ETIME; -+ goto out; -+ } -+ } ++static inline struct maple_enode *mt_mk_node(const struct maple_node *node, ++ enum maple_type type) ++{ ++ return (void *)((unsigned long)node | ++ (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); ++} + -+ if (likely(pkt_id == pr->id + 1)) { -+ /* well-formed ID sequence (incremented by 1) */ -+ pr->base = REPLAY_INDEX(pr->base, -1); -+ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); -+ if (pr->extent < REPLAY_WINDOW_SIZE) -+ ++pr->extent; -+ pr->id = pkt_id; -+ } else if (pkt_id > pr->id) { -+ /* ID jumped forward by more than one */ -+ const unsigned int delta = pkt_id - pr->id; ++static inline void *mte_mk_root(const struct maple_enode *node) ++{ ++ return (void *)((unsigned long)node | MAPLE_ROOT_NODE); ++} + -+ if (delta < REPLAY_WINDOW_SIZE) { -+ unsigned int i; ++static inline void *mte_safe_root(const struct maple_enode *node) ++{ ++ return (void *)((unsigned long)node & ~MAPLE_ROOT_NODE); ++} + -+ pr->base = REPLAY_INDEX(pr->base, -delta); -+ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); -+ pr->extent += delta; -+ if (pr->extent > REPLAY_WINDOW_SIZE) -+ pr->extent = REPLAY_WINDOW_SIZE; -+ for (i = 1; i < delta; ++i) { -+ unsigned int newb = REPLAY_INDEX(pr->base, i); ++static inline void mte_set_full(const struct maple_enode *node) ++{ ++ node = (void *)((unsigned long)node & ~MAPLE_ENODE_NULL); ++} + -+ pr->history[newb / 8] &= ~BIT(newb % 8); -+ } -+ } else { -+ pr->base = 0; -+ pr->extent = REPLAY_WINDOW_SIZE; -+ memset(pr->history, 0, sizeof(pr->history)); -+ pr->history[0] = 1; -+ } -+ pr->id = pkt_id; -+ } else { -+ /* ID backtrack */ -+ const unsigned int delta = pr->id - pkt_id; ++static inline void mte_clear_full(const struct maple_enode *node) ++{ ++ node = (void *)((unsigned long)node | MAPLE_ENODE_NULL); ++} + -+ if (delta > pr->max_backtrack) -+ pr->max_backtrack = delta; -+ if (delta < pr->extent) { -+ if (pkt_id > pr->id_floor) { -+ const unsigned int ri = REPLAY_INDEX(pr->base, -+ delta); -+ u8 *p = &pr->history[ri / 8]; -+ const u8 mask = (1 << (ri % 8)); ++static inline bool ma_is_root(struct maple_node *node) ++{ ++ return ((unsigned long)node->parent & MA_ROOT_PARENT); ++} + -+ if (*p & mask) { -+ ret = -EINVAL; -+ goto out; -+ } -+ *p |= mask; -+ } else { -+ ret = -EINVAL; -+ goto out; -+ } -+ } else { -+ ret = -EINVAL; -+ goto out; -+ } -+ } ++static inline bool mte_is_root(const struct maple_enode *node) ++{ ++ return ma_is_root(mte_to_node(node)); ++} + -+ pr->expire = now + PKTID_RECV_EXPIRE; -+ ret = 0; -+out: -+ spin_unlock(&pr->lock); -+ return ret; ++static inline bool mas_is_root_limits(const struct ma_state *mas) ++{ ++ return !mas->min && mas->max == ULONG_MAX; +} -diff --git a/drivers/net/ovpn-dco/pktid.h b/drivers/net/ovpn-dco/pktid.h -new file mode 100644 -index 000000000000..2447bb37ba55 ---- /dev/null -+++ b/drivers/net/ovpn-dco/pktid.h -@@ -0,0 +1,116 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator ++ ++static inline bool mt_is_alloc(struct maple_tree *mt) ++{ ++ return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); ++} ++ ++/* ++ * The Parent Pointer ++ * Excluding root, the parent pointer is 256B aligned like all other tree nodes. ++ * When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16 ++ * bit values need an extra bit to store the offset. This extra bit comes from ++ * a reuse of the last bit in the node type. This is possible by using bit 1 to ++ * indicate if bit 2 is part of the type or the slot. + * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ * Note types: ++ * 0x??1 = Root ++ * 0x?00 = 16 bit nodes ++ * 0x010 = 32 bit nodes ++ * 0x110 = 64 bit nodes + * -+ * Author: Antonio Quartulli -+ * James Yonan ++ * Slot size and alignment ++ * 0b??1 : Root ++ * 0b?00 : 16 bit values, type in 0-1, slot in 2-7 ++ * 0b010 : 32 bit values, type in 0-2, slot in 3-7 ++ * 0b110 : 64 bit values, type in 0-2, slot in 3-7 + */ + -+#ifndef _NET_OVPN_DCO_OVPNPKTID_H_ -+#define _NET_OVPN_DCO_OVPNPKTID_H_ ++#define MAPLE_PARENT_ROOT 0x01 + -+#include "main.h" ++#define MAPLE_PARENT_SLOT_SHIFT 0x03 ++#define MAPLE_PARENT_SLOT_MASK 0xF8 + -+/* When the OpenVPN protocol is run in AEAD mode, use -+ * the OpenVPN packet ID as the AEAD nonce: -+ * -+ * 00000005 521c3b01 4308c041 -+ * [seq # ] [ nonce_tail ] -+ * [ 12-byte full IV ] -> NONCE_SIZE -+ * [4-bytes -> NONCE_WIRE_SIZE -+ * on wire] -+ */ ++#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02 ++#define MAPLE_PARENT_16B_SLOT_MASK 0xFC + -+/* OpenVPN nonce size */ -+#define NONCE_SIZE 12 -+/* amount of bytes of the nonce received from user space */ -+#define NONCE_TAIL_SIZE 8 ++#define MAPLE_PARENT_RANGE64 0x06 ++#define MAPLE_PARENT_RANGE32 0x04 ++#define MAPLE_PARENT_NOT_RANGE16 0x02 + -+/* OpenVPN nonce size reduced by 8-byte nonce tail -- this is the -+ * size of the AEAD Associated Data (AD) sent over the wire -+ * and is normally the head of the IV ++/* ++ * mte_parent_shift() - Get the parent shift for the slot storage. ++ * @parent: The parent pointer cast as an unsigned long ++ * Return: The shift into that pointer to the star to of the slot + */ -+#define NONCE_WIRE_SIZE (NONCE_SIZE - sizeof(struct ovpn_nonce_tail)) ++static inline unsigned long mte_parent_shift(unsigned long parent) ++{ ++ /* Note bit 1 == 0 means 16B */ ++ if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) ++ return MAPLE_PARENT_SLOT_SHIFT; + -+/* If no packets received for this length of time, set a backtrack floor -+ * at highest received packet ID thus far. -+ */ -+#define PKTID_RECV_EXPIRE (30 * HZ) ++ return MAPLE_PARENT_16B_SLOT_SHIFT; ++} + -+/* Last 8 bytes of AEAD nonce -+ * Provided by userspace and usually derived from -+ * key material generated during TLS handshake ++/* ++ * mte_parent_slot_mask() - Get the slot mask for the parent. ++ * @parent: The parent pointer cast as an unsigned long. ++ * Return: The slot mask for that parent. + */ -+struct ovpn_nonce_tail { -+ u8 u8[NONCE_TAIL_SIZE]; -+}; -+ -+/* Packet-ID state for transmitter */ -+struct ovpn_pktid_xmit { -+ atomic64_t seq_num; -+ struct ovpn_tcp_linear *tcp_linear; -+}; -+ -+/* replay window sizing in bytes = 2^REPLAY_WINDOW_ORDER */ -+#define REPLAY_WINDOW_ORDER 8 ++static inline unsigned long mte_parent_slot_mask(unsigned long parent) ++{ ++ /* Note bit 1 == 0 means 16B */ ++ if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) ++ return MAPLE_PARENT_SLOT_MASK; + -+#define REPLAY_WINDOW_BYTES BIT(REPLAY_WINDOW_ORDER) -+#define REPLAY_WINDOW_SIZE (REPLAY_WINDOW_BYTES * 8) -+#define REPLAY_INDEX(base, i) (((base) + (i)) & (REPLAY_WINDOW_SIZE - 1)) ++ return MAPLE_PARENT_16B_SLOT_MASK; ++} + -+/* Packet-ID state for receiver. -+ * Other than lock member, can be zeroed to initialize. ++/* ++ * mas_parent_enum() - Return the maple_type of the parent from the stored ++ * parent type. ++ * @mas: The maple state ++ * @node: The maple_enode to extract the parent's enum ++ * Return: The node->parent maple_type + */ -+struct ovpn_pktid_recv { -+ /* "sliding window" bitmask of recent packet IDs received */ -+ u8 history[REPLAY_WINDOW_BYTES]; -+ /* bit position of deque base in history */ -+ unsigned int base; -+ /* extent (in bits) of deque in history */ -+ unsigned int extent; -+ /* expiration of history in jiffies */ -+ unsigned long expire; -+ /* highest sequence number received */ -+ u32 id; -+ /* highest time stamp received */ -+ u32 time; -+ /* we will only accept backtrack IDs > id_floor */ -+ u32 id_floor; -+ unsigned int max_backtrack; -+ /* protects entire pktd ID state */ -+ spinlock_t lock; -+}; -+ -+/* Get the next packet ID for xmit */ -+static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) ++static inline ++enum maple_type mte_parent_enum(struct maple_enode *p_enode, ++ struct maple_tree *mt) +{ -+ const s64 seq_num = atomic64_fetch_add_unless(&pid->seq_num, 1, -+ 0x100000000LL); -+ /* when the 32bit space is over, we return an error because the packet ID is used to create -+ * the cipher IV and we do not want to re-use the same value more than once -+ */ -+ if (unlikely(seq_num == 0x100000000LL)) -+ return -ERANGE; ++ unsigned long p_type; + -+ *pktid = (u32)seq_num; ++ p_type = (unsigned long)p_enode; ++ if (p_type & MAPLE_PARENT_ROOT) ++ return 0; /* Validated in the caller. */ ++ ++ p_type &= MAPLE_NODE_MASK; ++ p_type = p_type & ~(MAPLE_PARENT_ROOT | mte_parent_slot_mask(p_type)); ++ ++ switch (p_type) { ++ case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */ ++ if (mt_is_alloc(mt)) ++ return maple_arange_64; ++ return maple_range_64; ++ } + + return 0; +} + -+/* Write 12-byte AEAD IV to dest */ -+static inline void ovpn_pktid_aead_write(const u32 pktid, -+ const struct ovpn_nonce_tail *nt, -+ unsigned char *dest) ++static inline ++enum maple_type mas_parent_enum(struct ma_state *mas, struct maple_enode *enode) +{ -+ *(__force __be32 *)(dest) = htonl(pktid); -+ BUILD_BUG_ON(4 + sizeof(struct ovpn_nonce_tail) != NONCE_SIZE); -+ memcpy(dest + 4, nt->u8, sizeof(struct ovpn_nonce_tail)); ++ return mte_parent_enum(ma_enode_ptr(mte_to_node(enode)->parent), mas->tree); +} + -+void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid); -+void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr); ++/* ++ * mte_set_parent() - Set the parent node and encode the slot ++ * @enode: The encoded maple node. ++ * @parent: The encoded maple node that is the parent of @enode. ++ * @slot: The slot that @enode resides in @parent. ++ * ++ * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the ++ * parent type. ++ */ ++static inline ++void mte_set_parent(struct maple_enode *enode, const struct maple_enode *parent, ++ unsigned char slot) ++{ ++ unsigned long val = (unsigned long) parent; ++ unsigned long shift; ++ unsigned long type; ++ enum maple_type p_type = mte_node_type(parent); ++ ++ BUG_ON(p_type == maple_dense); ++ BUG_ON(p_type == maple_leaf_64); ++ ++ switch (p_type) { ++ case maple_range_64: ++ case maple_arange_64: ++ shift = MAPLE_PARENT_SLOT_SHIFT; ++ type = MAPLE_PARENT_RANGE64; ++ break; ++ default: ++ case maple_dense: ++ case maple_leaf_64: ++ shift = type = 0; ++ break; ++ } + -+int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time); ++ val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ ++ val |= (slot << shift) | type; ++ mte_to_node(enode)->parent = ma_parent_ptr(val); ++} + -+#endif /* _NET_OVPN_DCO_OVPNPKTID_H_ */ -diff --git a/drivers/net/ovpn-dco/proto.h b/drivers/net/ovpn-dco/proto.h -new file mode 100644 -index 000000000000..875529021c1b ---- /dev/null -+++ b/drivers/net/ovpn-dco/proto.h -@@ -0,0 +1,101 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++/* ++ * mte_parent_slot() - get the parent slot of @enode. ++ * @enode: The encoded maple node. + * -+ * Author: Antonio Quartulli -+ * James Yonan ++ * Return: The slot in the parent node where @enode resides. + */ ++static inline unsigned int mte_parent_slot(const struct maple_enode *enode) ++{ ++ unsigned long val = (unsigned long) mte_to_node(enode)->parent; + -+#ifndef _NET_OVPN_DCO_OVPNPROTO_H_ -+#define _NET_OVPN_DCO_OVPNPROTO_H_ -+ -+#include "main.h" ++ /* Root. */ ++ if (val & 1) ++ return 0; + -+#include ++ /* ++ * Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost ++ * by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT ++ */ ++ return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val); ++} + -+/* Methods for operating on the initial command -+ * byte of the OpenVPN protocol. ++/* ++ * mte_parent() - Get the parent of @node. ++ * @node: The encoded maple node. ++ * ++ * Return: The parent maple node. + */ ++static inline struct maple_node *mte_parent(const struct maple_enode *enode) ++{ ++ return (void *)((unsigned long) ++ (mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK); ++} + -+/* packet opcode (high 5 bits) and key-id (low 3 bits) are combined in -+ * one byte ++/* ++ * ma_dead_node() - check if the @enode is dead. ++ * @enode: The encoded maple node ++ * ++ * Return: true if dead, false otherwise. + */ -+#define OVPN_KEY_ID_MASK 0x07 -+#define OVPN_OPCODE_SHIFT 3 -+#define OVPN_OPCODE_MASK 0x1F -+/* upper bounds on opcode and key ID */ -+#define OVPN_KEY_ID_MAX (OVPN_KEY_ID_MASK + 1) -+#define OVPN_OPCODE_MAX (OVPN_OPCODE_MASK + 1) -+/* packet opcodes of interest to us */ -+#define OVPN_DATA_V1 6 /* data channel V1 packet */ -+#define OVPN_DATA_V2 9 /* data channel V2 packet */ -+/* size of initial packet opcode */ -+#define OVPN_OP_SIZE_V1 1 -+#define OVPN_OP_SIZE_V2 4 -+#define OVPN_PEER_ID_MASK 0x00FFFFFF -+#define OVPN_PEER_ID_UNDEF 0x00FFFFFF -+/* first byte of keepalive message */ -+#define OVPN_KEEPALIVE_FIRST_BYTE 0x2a -+/* first byte of exit message */ -+#define OVPN_EXPLICIT_EXIT_NOTIFY_FIRST_BYTE 0x28 ++static inline bool ma_dead_node(const struct maple_node *node) ++{ ++ struct maple_node *parent = (void *)((unsigned long) ++ node->parent & ~MAPLE_NODE_MASK); + -+/** -+ * Extract the OP code from the specified byte ++ return (parent == node); ++} ++/* ++ * mte_dead_node() - check if the @enode is dead. ++ * @enode: The encoded maple node + * -+ * Return the OP code ++ * Return: true if dead, false otherwise. + */ -+static inline u8 ovpn_opcode_from_byte(u8 byte) ++static inline bool mte_dead_node(const struct maple_enode *enode) +{ -+ return byte >> OVPN_OPCODE_SHIFT; ++ struct maple_node *parent, *node; ++ ++ node = mte_to_node(enode); ++ parent = mte_parent(enode); ++ return (parent == node); +} + -+/** -+ * Extract the OP code from the skb head. ++/* ++ * mas_allocated() - Get the number of nodes allocated in a maple state. ++ * @mas: The maple state + * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first byte. ++ * The ma_state alloc member is overloaded to hold a pointer to the first ++ * allocated node or to the number of requested nodes to allocate. If bit 0 is ++ * set, then the alloc contains the number of requested nodes. If there is an ++ * allocated node, then the total allocated nodes is in that node. + * -+ * Return the OP code ++ * Return: The total number of nodes allocated + */ -+static inline u8 ovpn_opcode_from_skb(const struct sk_buff *skb, u16 offset) ++static inline unsigned long mas_allocated(const struct ma_state *mas) +{ -+ return ovpn_opcode_from_byte(*(skb->data + offset)); ++ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) ++ return 0; ++ ++ return mas->alloc->total; +} + -+/** -+ * Extract the key ID from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first byte. ++/* ++ * mas_set_alloc_req() - Set the requested number of allocations. ++ * @mas: the maple state ++ * @count: the number of allocations. + * -+ * Return the key ID ++ * The requested number of allocations is either in the first allocated node, ++ * located in @mas->alloc->request_count, or directly in @mas->alloc if there is ++ * no allocated node. Set the request either in the node or do the necessary ++ * encoding to store in @mas->alloc directly. + */ -+ -+static inline u8 ovpn_key_id_from_skb(const struct sk_buff *skb) ++static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) +{ -+ return *skb->data & OVPN_KEY_ID_MASK; ++ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) { ++ if (!count) ++ mas->alloc = NULL; ++ else ++ mas->alloc = (struct maple_alloc *)(((count) << 1U) | 1U); ++ return; ++ } ++ ++ mas->alloc->request_count = count; +} + -+/** -+ * Extract the peer ID from the skb head. ++/* ++ * mas_alloc_req() - get the requested number of allocations. ++ * @mas: The maple state + * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first 4 bytes. ++ * The alloc count is either stored directly in @mas, or in ++ * @mas->alloc->request_count if there is at least one node allocated. Decode ++ * the request count if it's stored directly in @mas->alloc. + * -+ * Return the peer ID. ++ * Return: The allocation request count. + */ -+ -+static inline u32 ovpn_peer_id_from_skb(const struct sk_buff *skb, u16 offset) ++static inline unsigned int mas_alloc_req(const struct ma_state *mas) +{ -+ return ntohl(*(__be32 *)(skb->data + offset)) & OVPN_PEER_ID_MASK; ++ if ((unsigned long)mas->alloc & 0x1) ++ return (unsigned long)(mas->alloc) >> 1; ++ else if (mas->alloc) ++ return mas->alloc->request_count; ++ return 0; +} + -+static inline u32 ovpn_opcode_compose(u8 opcode, u8 key_id, u32 peer_id) -+{ -+ const u8 op = (opcode << OVPN_OPCODE_SHIFT) | (key_id & OVPN_KEY_ID_MASK); -+ -+ return (op << 24) | (peer_id & OVPN_PEER_ID_MASK); ++/* ++ * ma_pivots() - Get a pointer to the maple node pivots. ++ * @node - the maple node ++ * @type - the node type ++ * ++ * Return: A pointer to the maple node pivots ++ */ ++static inline unsigned long *ma_pivots(struct maple_node *node, ++ enum maple_type type) ++{ ++ switch (type) { ++ case maple_arange_64: ++ return node->ma64.pivot; ++ case maple_range_64: ++ case maple_leaf_64: ++ return node->mr64.pivot; ++ case maple_dense: ++ return NULL; ++ } ++ return NULL; +} + -+#endif /* _NET_OVPN_DCO_OVPNPROTO_H_ */ -diff --git a/drivers/net/ovpn-dco/rcu.h b/drivers/net/ovpn-dco/rcu.h -new file mode 100644 -index 000000000000..02a50f49ba2e ---- /dev/null -+++ b/drivers/net/ovpn-dco/rcu.h -@@ -0,0 +1,21 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator ++/* ++ * ma_gaps() - Get a pointer to the maple node gaps. ++ * @node - the maple node ++ * @type - the node type + * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ * Return: A pointer to the maple node gaps ++ */ ++static inline unsigned long *ma_gaps(struct maple_node *node, ++ enum maple_type type) ++{ ++ switch (type) { ++ case maple_arange_64: ++ return node->ma64.gap; ++ case maple_range_64: ++ case maple_leaf_64: ++ case maple_dense: ++ return NULL; ++ } ++ return NULL; ++} ++ ++/* ++ * mte_pivot() - Get the pivot at @piv of the maple encoded node. ++ * @mn: The maple encoded node. ++ * @piv: The pivot. + * -+ * Author: James Yonan -+ * Antonio Quartulli ++ * Return: the pivot at @piv of @mn. + */ ++static inline unsigned long mte_pivot(const struct maple_enode *mn, ++ unsigned char piv) ++{ ++ struct maple_node *node = mte_to_node(mn); + -+#ifndef _NET_OVPN_DCO_OVPNRCU_H_ -+#define _NET_OVPN_DCO_OVPNRCU_H_ ++ if (piv >= mt_pivots[piv]) { ++ WARN_ON(1); ++ return 0; ++ } ++ switch (mte_node_type(mn)) { ++ case maple_arange_64: ++ return node->ma64.pivot[piv]; ++ case maple_range_64: ++ case maple_leaf_64: ++ return node->mr64.pivot[piv]; ++ case maple_dense: ++ return 0; ++ } ++ return 0; ++} + -+static inline void ovpn_rcu_lockdep_assert_held(void) ++/* ++ * mas_safe_pivot() - get the pivot at @piv or mas->max. ++ * @mas: The maple state ++ * @pivots: The pointer to the maple node pivots ++ * @piv: The pivot to fetch ++ * @type: The maple node type ++ * ++ * Return: The pivot at @piv within the limit of the @pivots array, @mas->max ++ * otherwise. ++ */ ++static inline unsigned long ++mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, ++ unsigned char piv, enum maple_type type) +{ -+#ifdef CONFIG_PROVE_RCU -+ RCU_LOCKDEP_WARN(!rcu_read_lock_held(), -+ "ovpn-dco RCU read lock not held"); -+#endif ++ if (piv >= mt_pivots[type]) ++ return mas->max; ++ ++ return pivots[piv]; +} + -+#endif /* _NET_OVPN_DCO_OVPNRCU_H_ */ -diff --git a/drivers/net/ovpn-dco/skb.h b/drivers/net/ovpn-dco/skb.h -new file mode 100644 -index 000000000000..d38dc2da01df ---- /dev/null -+++ b/drivers/net/ovpn-dco/skb.h -@@ -0,0 +1,54 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++/* ++ * mas_safe_min() - Return the minimum for a given offset. ++ * @mas: The maple state ++ * @pivots: The pointer to the maple node pivots ++ * @offset: The offset into the pivot array + * -+ * Author: Antonio Quartulli -+ * James Yonan ++ * Return: The minimum range value that is contained in @offset. + */ ++static inline unsigned long ++mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset) ++{ ++ if (likely(offset)) ++ return pivots[offset - 1] + 1; + -+#ifndef _NET_OVPN_DCO_SKB_H_ -+#define _NET_OVPN_DCO_SKB_H_ ++ return mas->min; ++} + -+#include -+#include -+#include -+#include -+#include -+#include ++/* ++ * mas_logical_pivot() - Get the logical pivot of a given offset. ++ * @mas: The maple state ++ * @pivots: The pointer to the maple node pivots ++ * @offset: The offset into the pivot array ++ * @type: The maple node type ++ * ++ * When there is no value at a pivot (beyond the end of the data), then the ++ * pivot is actually @mas->max. ++ * ++ * Return: the logical pivot of a given @offset. ++ */ ++static inline unsigned long ++mas_logical_pivot(struct ma_state *mas, unsigned long *pivots, ++ unsigned char offset, enum maple_type type) ++{ ++ unsigned long lpiv = mas_safe_pivot(mas, pivots, offset, type); + -+#define OVPN_SKB_CB(skb) ((struct ovpn_skb_cb *)&((skb)->cb)) ++ if (likely(lpiv)) ++ return lpiv; + -+struct ovpn_skb_cb { -+ /* original recv packet size for stats accounting */ -+ unsigned int rx_stats_size; ++ if (likely(offset)) ++ return mas->max; + -+ union { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } local; -+ sa_family_t sa_fam; -+}; ++ return lpiv; ++} + -+/* Return IP protocol version from skb header. -+ * Return 0 if protocol is not IPv4/IPv6 or cannot be read. ++/* ++ * mte_set_pivot() - Set a pivot to a value in an encoded maple node. ++ * @mn: The encoded maple node ++ * @piv: The pivot offset ++ * @val: The value of the pivot + */ -+static inline __be16 ovpn_ip_check_protocol(struct sk_buff *skb) ++static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, ++ unsigned long val) +{ -+ __be16 proto = 0; -+ -+ /* skb could be non-linear, -+ * make sure IP header is in non-fragmented part -+ */ -+ if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) -+ return 0; ++ struct maple_node *node = mte_to_node(mn); ++ enum maple_type type = mte_node_type(mn); + -+ if (ip_hdr(skb)->version == 4) -+ proto = htons(ETH_P_IP); -+ else if (ip_hdr(skb)->version == 6) -+ proto = htons(ETH_P_IPV6); ++ BUG_ON(piv >= mt_pivots[type]); ++ switch (type) { ++ default: ++ case maple_range_64: ++ case maple_leaf_64: ++ node->mr64.pivot[piv] = val; ++ break; ++ case maple_arange_64: ++ node->ma64.pivot[piv] = val; ++ break; ++ case maple_dense: ++ break; ++ } + -+ return proto; +} + -+#endif /* _NET_OVPN_DCO_SKB_H_ */ -diff --git a/drivers/net/ovpn-dco/sock.c b/drivers/net/ovpn-dco/sock.c -new file mode 100644 -index 000000000000..e92a4a9b952e ---- /dev/null -+++ b/drivers/net/ovpn-dco/sock.c -@@ -0,0 +1,134 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++/* ++ * ma_slots() - Get a pointer to the maple node slots. ++ * @mn: The maple node ++ * @mt: The maple node type + * -+ * Author: James Yonan -+ * Antonio Quartulli ++ * Return: A pointer to the maple node slots + */ ++static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) ++{ ++ switch (mt) { ++ default: ++ case maple_arange_64: ++ return mn->ma64.slot; ++ case maple_range_64: ++ case maple_leaf_64: ++ return mn->mr64.slot; ++ case maple_dense: ++ return mn->slot; ++ } ++} + -+#include "main.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "sock.h" -+#include "rcu.h" -+#include "tcp.h" -+#include "udp.h" ++static inline bool mt_locked(const struct maple_tree *mt) ++{ ++ return mt_external_lock(mt) ? mt_lock_is_held(mt) : ++ lockdep_is_held(&mt->ma_lock); ++} + -+#include -+#include ++static inline void *mt_slot(const struct maple_tree *mt, ++ void __rcu **slots, unsigned char offset) ++{ ++ return rcu_dereference_check(slots[offset], mt_locked(mt)); ++} + -+/* Finalize release of socket, called after RCU grace period */ -+static void ovpn_socket_detach(struct socket *sock) ++/* ++ * mas_slot_locked() - Get the slot value when holding the maple tree lock. ++ * @mas: The maple state ++ * @slots: The pointer to the slots ++ * @offset: The offset into the slots array to fetch ++ * ++ * Return: The entry stored in @slots at the @offset. ++ */ ++static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots, ++ unsigned char offset) +{ -+ if (!sock) -+ return; ++ return rcu_dereference_protected(slots[offset], mt_locked(mas->tree)); ++} + -+ if (sock->sk->sk_protocol == IPPROTO_UDP) -+ ovpn_udp_socket_detach(sock); -+ else if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ovpn_tcp_socket_detach(sock); ++/* ++ * mas_slot() - Get the slot value when not holding the maple tree lock. ++ * @mas: The maple state ++ * @slots: The pointer to the slots ++ * @offset: The offset into the slots array to fetch ++ * ++ * Return: The entry stored in @slots at the @offset ++ */ ++static inline void *mas_slot(struct ma_state *mas, void __rcu **slots, ++ unsigned char offset) ++{ ++ return mt_slot(mas->tree, slots, offset); ++} + -+ sockfd_put(sock); ++/* ++ * mas_root() - Get the maple tree root. ++ * @mas: The maple state. ++ * ++ * Return: The pointer to the root of the tree ++ */ ++static inline void *mas_root(struct ma_state *mas) ++{ ++ return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree)); +} + -+void ovpn_socket_release_kref(struct kref *kref) ++static inline void *mt_root_locked(struct maple_tree *mt) +{ -+ struct ovpn_socket *sock = container_of(kref, struct ovpn_socket, refcount); -+ -+ ovpn_socket_detach(sock->sock); -+ kfree_rcu(sock, rcu); ++ return rcu_dereference_protected(mt->ma_root, mt_locked(mt)); +} + -+static bool ovpn_socket_hold(struct ovpn_socket *sock) ++/* ++ * mas_root_locked() - Get the maple tree root when holding the maple tree lock. ++ * @mas: The maple state. ++ * ++ * Return: The pointer to the root of the tree ++ */ ++static inline void *mas_root_locked(struct ma_state *mas) +{ -+ return kref_get_unless_zero(&sock->refcount); ++ return mt_root_locked(mas->tree); +} + -+static struct ovpn_socket *ovpn_socket_get(struct socket *sock) ++static inline struct maple_metadata *ma_meta(struct maple_node *mn, ++ enum maple_type mt) +{ -+ struct ovpn_socket *ovpn_sock; -+ -+ rcu_read_lock(); -+ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); -+ if (!ovpn_socket_hold(ovpn_sock)) { -+ pr_warn("%s: found ovpn_socket with ref = 0\n", __func__); -+ ovpn_sock = NULL; ++ switch (mt) { ++ case maple_arange_64: ++ return &mn->ma64.meta; ++ default: ++ return &mn->mr64.meta; + } -+ rcu_read_unlock(); -+ -+ return ovpn_sock; +} + -+/* Finalize release of socket, called after RCU grace period */ -+static int ovpn_socket_attach(struct socket *sock, struct ovpn_peer *peer) ++/* ++ * ma_set_meta() - Set the metadata information of a node. ++ * @mn: The maple node ++ * @mt: The maple node type ++ * @offset: The offset of the highest sub-gap in this node. ++ * @end: The end of the data in this node. ++ */ ++static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt, ++ unsigned char offset, unsigned char end) +{ -+ int ret = -EOPNOTSUPP; ++ struct maple_metadata *meta = ma_meta(mn, mt); + -+ if (!sock || !peer) -+ return -EINVAL; ++ meta->gap = offset; ++ meta->end = end; ++} + -+ if (sock->sk->sk_protocol == IPPROTO_UDP) -+ ret = ovpn_udp_socket_attach(sock, peer->ovpn); -+ else if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ret = ovpn_tcp_socket_attach(sock, peer); ++/* ++ * ma_meta_end() - Get the data end of a node from the metadata ++ * @mn: The maple node ++ * @mt: The maple node type ++ */ ++static inline unsigned char ma_meta_end(struct maple_node *mn, ++ enum maple_type mt) ++{ ++ struct maple_metadata *meta = ma_meta(mn, mt); + -+ return ret; ++ return meta->end; +} + -+struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk) ++/* ++ * ma_meta_gap() - Get the largest gap location of a node from the metadata ++ * @mn: The maple node ++ * @mt: The maple node type ++ */ ++static inline unsigned char ma_meta_gap(struct maple_node *mn, ++ enum maple_type mt) +{ -+ struct ovpn_socket *ovpn_sock; -+ -+ ovpn_rcu_lockdep_assert_held(); ++ BUG_ON(mt != maple_arange_64); + -+ if (unlikely(READ_ONCE(udp_sk(sk)->encap_type) != UDP_ENCAP_OVPNINUDP)) -+ return NULL; ++ return mn->ma64.meta.gap; ++} + -+ ovpn_sock = rcu_dereference_sk_user_data(sk); -+ if (unlikely(!ovpn_sock)) -+ return NULL; ++/* ++ * ma_set_meta_gap() - Set the largest gap location in a nodes metadata ++ * @mn: The maple node ++ * @mn: The maple node type ++ * @offset: The location of the largest gap. ++ */ ++static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, ++ unsigned char offset) ++{ + -+ /* make sure that sk matches our stored transport socket */ -+ if (unlikely(!ovpn_sock->sock || sk != ovpn_sock->sock->sk)) -+ return NULL; ++ struct maple_metadata *meta = ma_meta(mn, mt); + -+ return ovpn_sock->ovpn; ++ meta->gap = offset; +} + -+struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ++/* ++ * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. ++ * @mat - the ma_topiary, a linked list of dead nodes. ++ * @dead_enode - the node to be marked as dead and added to the tail of the list ++ * ++ * Add the @dead_enode to the linked list in @mat. ++ */ ++static inline void mat_add(struct ma_topiary *mat, ++ struct maple_enode *dead_enode) +{ -+ struct ovpn_socket *ovpn_sock; -+ int ret; -+ -+ ret = ovpn_socket_attach(sock, peer); -+ if (ret < 0 && ret != -EALREADY) -+ return ERR_PTR(ret); -+ -+ /* if this socket is already owned by this interface, just increase the refcounter */ -+ if (ret == -EALREADY) { -+ /* caller is expected to increase the sock refcounter before passing it to this -+ * function. For this reason we drop it if not needed, like when this socket is -+ * already owned. -+ */ -+ ovpn_sock = ovpn_socket_get(sock); -+ sockfd_put(sock); -+ return ovpn_sock; ++ mte_set_node_dead(dead_enode); ++ mte_to_mat(dead_enode)->next = NULL; ++ if (!mat->tail) { ++ mat->tail = mat->head = dead_enode; ++ return; + } + -+ ovpn_sock = kzalloc(sizeof(*ovpn_sock), GFP_KERNEL); -+ if (!ovpn_sock) -+ return ERR_PTR(-ENOMEM); -+ -+ ovpn_sock->ovpn = peer->ovpn; -+ ovpn_sock->sock = sock; -+ kref_init(&ovpn_sock->refcount); ++ mte_to_mat(mat->tail)->next = dead_enode; ++ mat->tail = dead_enode; ++} + -+ /* TCP sockets are per-peer, therefore they are linked to their unique peer */ -+ if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ovpn_sock->peer = peer; ++static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); ++static inline void mas_free(struct ma_state *mas, struct maple_enode *used); + -+ rcu_assign_sk_user_data(sock->sk, ovpn_sock); ++/* ++ * mas_mat_free() - Free all nodes in a dead list. ++ * @mas - the maple state ++ * @mat - the ma_topiary linked list of dead nodes to free. ++ * ++ * Free walk a dead list. ++ */ ++static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat) ++{ ++ struct maple_enode *next; + -+ return ovpn_sock; ++ while (mat->head) { ++ next = mte_to_mat(mat->head)->next; ++ mas_free(mas, mat->head); ++ mat->head = next; ++ } +} -diff --git a/drivers/net/ovpn-dco/sock.h b/drivers/net/ovpn-dco/sock.h -new file mode 100644 -index 000000000000..9e79c1b5fe04 ---- /dev/null -+++ b/drivers/net/ovpn-dco/sock.h -@@ -0,0 +1,54 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ ++/* ++ * mas_mat_destroy() - Free all nodes and subtrees in a dead list. ++ * @mas - the maple state ++ * @mat - the ma_topiary linked list of dead nodes to free. + * -+ * Author: James Yonan -+ * Antonio Quartulli ++ * Destroy walk a dead list. + */ ++static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat) ++{ ++ struct maple_enode *next; + -+#ifndef _NET_OVPN_DCO_SOCK_H_ -+#define _NET_OVPN_DCO_SOCK_H_ ++ while (mat->head) { ++ next = mte_to_mat(mat->head)->next; ++ mte_destroy_walk(mat->head, mat->mtree); ++ mat->head = next; ++ } ++} ++/* ++ * mas_descend() - Descend into the slot stored in the ma_state. ++ * @mas - the maple state. ++ * ++ * Note: Not RCU safe, only use in write side or debug code. ++ */ ++static inline void mas_descend(struct ma_state *mas) ++{ ++ enum maple_type type; ++ unsigned long *pivots; ++ struct maple_node *node; ++ void __rcu **slots; + -+#include -+#include -+#include ++ node = mas_mn(mas); ++ type = mte_node_type(mas->node); ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); + -+#include "peer.h" ++ if (mas->offset) ++ mas->min = pivots[mas->offset - 1] + 1; ++ mas->max = mas_safe_pivot(mas, pivots, mas->offset, type); ++ mas->node = mas_slot(mas, slots, mas->offset); ++} + -+struct ovpn_struct; ++/* ++ * mte_set_gap() - Set a maple node gap. ++ * @mn: The encoded maple node ++ * @gap: The offset of the gap to set ++ * @val: The gap value ++ */ ++static inline void mte_set_gap(const struct maple_enode *mn, ++ unsigned char gap, unsigned long val) ++{ ++ switch (mte_node_type(mn)) { ++ default: ++ break; ++ case maple_arange_64: ++ mte_to_node(mn)->ma64.gap[gap] = val; ++ break; ++ } ++} + -+/** -+ * struct ovpn_socket - a kernel socket referenced in the ovpn-dco code ++/* ++ * mas_ascend() - Walk up a level of the tree. ++ * @mas: The maple state ++ * ++ * Sets the @mas->max and @mas->min to the correct values when walking up. This ++ * may cause several levels of walking up to find the correct min and max. ++ * May find a dead node which will cause a premature return. ++ * Return: 1 on dead node, 0 otherwise + */ -+struct ovpn_socket { -+ union { -+ /** @ovpn: the VPN session object owning this socket (UDP only) */ -+ struct ovpn_struct *ovpn; ++static int mas_ascend(struct ma_state *mas) ++{ ++ struct maple_enode *p_enode; /* parent enode. */ ++ struct maple_enode *a_enode; /* ancestor enode. */ ++ struct maple_node *a_node; /* ancestor node. */ ++ struct maple_node *p_node; /* parent node. */ ++ unsigned char a_slot; ++ enum maple_type a_type; ++ unsigned long min, max; ++ unsigned long *pivots; ++ unsigned char offset; ++ bool set_max = false, set_min = false; ++ ++ a_node = mas_mn(mas); ++ if (ma_is_root(a_node)) { ++ mas->offset = 0; ++ return 0; ++ } + -+ /** @peer: the unique peer transmitting over this socket (TCP only) */ -+ struct ovpn_peer *peer; -+ }; ++ p_node = mte_parent(mas->node); ++ if (unlikely(a_node == p_node)) ++ return 1; ++ a_type = mas_parent_enum(mas, mas->node); ++ offset = mte_parent_slot(mas->node); ++ a_enode = mt_mk_node(p_node, a_type); + -+ /** @sock: the kernel socket */ -+ struct socket *sock; ++ /* Check to make sure all parent information is still accurate */ ++ if (p_node != mte_parent(mas->node)) ++ return 1; + -+ /** @refcount: amount of contexts currently referencing this object */ -+ struct kref refcount; ++ mas->node = a_enode; ++ mas->offset = offset; + -+ /** @rcu: member used to schedule RCU destructor callback */ -+ struct rcu_head rcu; -+}; ++ if (mte_is_root(a_enode)) { ++ mas->max = ULONG_MAX; ++ mas->min = 0; ++ return 0; ++ } + -+struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk); ++ min = 0; ++ max = ULONG_MAX; ++ do { ++ p_enode = a_enode; ++ a_type = mas_parent_enum(mas, p_enode); ++ a_node = mte_parent(p_enode); ++ a_slot = mte_parent_slot(p_enode); ++ pivots = ma_pivots(a_node, a_type); ++ a_enode = mt_mk_node(a_node, a_type); ++ ++ if (!set_min && a_slot) { ++ set_min = true; ++ min = pivots[a_slot - 1] + 1; ++ } + -+void ovpn_socket_release_kref(struct kref *kref); ++ if (!set_max && a_slot < mt_pivots[a_type]) { ++ set_max = true; ++ max = pivots[a_slot]; ++ } + -+static inline void ovpn_socket_put(struct ovpn_socket *sock) -+{ -+ kref_put(&sock->refcount, ovpn_socket_release_kref); -+} ++ if (unlikely(ma_dead_node(a_node))) ++ return 1; + -+struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer); ++ if (unlikely(ma_is_root(a_node))) ++ break; + -+#endif /* _NET_OVPN_DCO_SOCK_H_ */ -diff --git a/drivers/net/ovpn-dco/stats.c b/drivers/net/ovpn-dco/stats.c -new file mode 100644 -index 000000000000..ee000b2a2177 ---- /dev/null -+++ b/drivers/net/ovpn-dco/stats.c -@@ -0,0 +1,20 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ } while (!set_min || !set_max); ++ ++ mas->max = max; ++ mas->min = min; ++ return 0; ++} ++ ++/* ++ * mas_pop_node() - Get a previously allocated maple node from the maple state. ++ * @mas: The maple state + * -+ * Author: James Yonan -+ * Antonio Quartulli ++ * Return: A pointer to a maple node. + */ ++static inline struct maple_node *mas_pop_node(struct ma_state *mas) ++{ ++ struct maple_alloc *ret, *node = mas->alloc; ++ unsigned long total = mas_allocated(mas); + -+#include "main.h" -+#include "stats.h" ++ /* nothing or a request pending. */ ++ if (unlikely(!total)) ++ return NULL; + -+void ovpn_peer_stats_init(struct ovpn_peer_stats *ps) -+{ -+ atomic64_set(&ps->rx.bytes, 0); -+ atomic_set(&ps->rx.packets, 0); ++ if (total == 1) { ++ /* single allocation in this ma_state */ ++ mas->alloc = NULL; ++ ret = node; ++ goto single_node; ++ } + -+ atomic64_set(&ps->tx.bytes, 0); -+ atomic_set(&ps->tx.packets, 0); ++ if (!node->node_count) { ++ /* Single allocation in this node. */ ++ mas->alloc = node->slot[0]; ++ node->slot[0] = NULL; ++ mas->alloc->total = node->total - 1; ++ ret = node; ++ goto new_head; ++ } ++ ++ node->total--; ++ ret = node->slot[node->node_count]; ++ node->slot[node->node_count--] = NULL; ++ ++single_node: ++new_head: ++ ret->total = 0; ++ ret->node_count = 0; ++ if (ret->request_count) { ++ mas_set_alloc_req(mas, ret->request_count + 1); ++ ret->request_count = 0; ++ } ++ return (struct maple_node *)ret; +} -diff --git a/drivers/net/ovpn-dco/stats.h b/drivers/net/ovpn-dco/stats.h -new file mode 100644 -index 000000000000..3aa6bdc049c6 ---- /dev/null -+++ b/drivers/net/ovpn-dco/stats.h -@@ -0,0 +1,67 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. ++ ++/* ++ * mas_push_node() - Push a node back on the maple state allocation. ++ * @mas: The maple state ++ * @used: The used maple node + * -+ * Author: James Yonan -+ * Antonio Quartulli -+ * Lev Stipakov ++ * Stores the maple node back into @mas->alloc for reuse. Updates allocated and ++ * requested node count as necessary. + */ ++static inline void mas_push_node(struct ma_state *mas, struct maple_node *used) ++{ ++ struct maple_alloc *reuse = (struct maple_alloc *)used; ++ struct maple_alloc *head = mas->alloc; ++ unsigned long count; ++ unsigned int requested = mas_alloc_req(mas); + -+#ifndef _NET_OVPN_DCO_OVPNSTATS_H_ -+#define _NET_OVPN_DCO_OVPNSTATS_H_ -+ -+#include -+#include ++ memset(reuse, 0, sizeof(*reuse)); ++ count = mas_allocated(mas); + -+struct ovpn_struct; ++ if (count && (head->node_count < MAPLE_ALLOC_SLOTS - 1)) { ++ if (head->slot[0]) ++ head->node_count++; ++ head->slot[head->node_count] = reuse; ++ head->total++; ++ goto done; ++ } + -+/* per-peer stats, measured on transport layer */ ++ reuse->total = 1; ++ if ((head) && !((unsigned long)head & 0x1)) { ++ head->request_count = 0; ++ reuse->slot[0] = head; ++ reuse->total += head->total; ++ } + -+/* one stat */ -+struct ovpn_peer_stat { -+ atomic64_t bytes; -+ atomic_t packets; -+}; ++ mas->alloc = reuse; ++done: ++ if (requested > 1) ++ mas_set_alloc_req(mas, requested - 1); ++} + -+/* rx and tx stats, enabled by notify_per != 0 or period != 0 */ -+struct ovpn_peer_stats { -+ struct ovpn_peer_stat rx; -+ struct ovpn_peer_stat tx; -+}; ++/* ++ * mas_alloc_nodes() - Allocate nodes into a maple state ++ * @mas: The maple state ++ * @gfp: The GFP Flags ++ */ ++static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) ++{ ++ struct maple_alloc *node; ++ struct maple_alloc **nodep = &mas->alloc; ++ unsigned long allocated = mas_allocated(mas); ++ unsigned long success = allocated; ++ unsigned int requested = mas_alloc_req(mas); ++ unsigned int count; ++ void **slots = NULL; ++ unsigned int max_req = 0; ++ ++ if (!requested) ++ return; + -+/* struct for OVPN_ERR_STATS */ ++ mas_set_alloc_req(mas, 0); ++ if (mas->mas_flags & MA_STATE_PREALLOC) { ++ if (allocated) ++ return; ++ WARN_ON(!allocated); ++ } + -+struct ovpn_err_stat { -+ unsigned int category; -+ int errcode; -+ u64 count; -+}; ++ if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS - 1) { ++ node = (struct maple_alloc *)mt_alloc_one(gfp); ++ if (!node) ++ goto nomem_one; + -+struct ovpn_err_stats { -+ /* total stats, returned by kovpn */ -+ unsigned int total_stats; -+ /* number of stats dimensioned below */ -+ unsigned int n_stats; -+ struct ovpn_err_stat stats[]; -+}; ++ if (allocated) ++ node->slot[0] = mas->alloc; + -+void ovpn_peer_stats_init(struct ovpn_peer_stats *ps); ++ success++; ++ mas->alloc = node; ++ requested--; ++ } + -+static inline void ovpn_peer_stats_increment(struct ovpn_peer_stat *stat, const unsigned int n) -+{ -+ atomic64_add(n, &stat->bytes); -+ atomic_inc(&stat->packets); -+} ++ node = mas->alloc; ++ while (requested) { ++ max_req = MAPLE_ALLOC_SLOTS; ++ if (node->slot[0]) { ++ unsigned int offset = node->node_count + 1; + -+static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats, const unsigned int n) -+{ -+ ovpn_peer_stats_increment(&stats->rx, n); -+} ++ slots = (void **)&node->slot[offset]; ++ max_req -= offset; ++ } else { ++ slots = (void **)&node->slot; ++ } + -+static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats, const unsigned int n) -+{ -+ ovpn_peer_stats_increment(&stats->tx, n); -+} ++ max_req = min(requested, max_req); ++ count = mt_alloc_bulk(gfp, max_req, slots); ++ if (!count) ++ goto nomem_bulk; + -+#endif /* _NET_OVPN_DCO_OVPNSTATS_H_ */ -diff --git a/drivers/net/ovpn-dco/tcp.c b/drivers/net/ovpn-dco/tcp.c -new file mode 100644 -index 000000000000..7e6690fee6e7 ---- /dev/null -+++ b/drivers/net/ovpn-dco/tcp.c -@@ -0,0 +1,326 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ ++ node->node_count += count; ++ /* zero indexed. */ ++ if (slots == (void **)&node->slot) ++ node->node_count--; + -+#include "main.h" -+#include "ovpnstruct.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "skb.h" -+#include "tcp.h" ++ success += count; ++ nodep = &node->slot[0]; ++ node = *nodep; ++ requested -= count; ++ } ++ mas->alloc->total = success; ++ return; + -+#include -+#include -+#include ++nomem_bulk: ++ /* Clean up potential freed allocations on bulk failure */ ++ memset(slots, 0, max_req * sizeof(unsigned long)); ++nomem_one: ++ mas_set_alloc_req(mas, requested); ++ if (mas->alloc && !(((unsigned long)mas->alloc & 0x1))) ++ mas->alloc->total = success; ++ mas_set_err(mas, -ENOMEM); ++ return; + -+static void ovpn_tcp_state_change(struct sock *sk) -+{ +} + -+static void ovpn_tcp_data_ready(struct sock *sk) ++/* ++ * mas_free() - Free an encoded maple node ++ * @mas: The maple state ++ * @used: The encoded maple node to free. ++ * ++ * Uses rcu free if necessary, pushes @used back on the maple state allocations ++ * otherwise. ++ */ ++static inline void mas_free(struct ma_state *mas, struct maple_enode *used) +{ -+ struct ovpn_socket *sock; -+ -+ rcu_read_lock(); -+ sock = rcu_dereference_sk_user_data(sk); -+ rcu_read_unlock(); -+ -+ if (!sock || !sock->peer) -+ return; ++ struct maple_node *tmp = mte_to_node(used); + -+ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.rx_work); ++ if (mt_in_rcu(mas->tree)) ++ ma_free_rcu(tmp); ++ else ++ mas_push_node(mas, tmp); +} + -+static void ovpn_tcp_write_space(struct sock *sk) ++/* ++ * mas_node_count() - Check if enough nodes are allocated and request more if ++ * there is not enough nodes. ++ * @mas: The maple state ++ * @count: The number of nodes needed ++ * @gfp: the gfp flags ++ */ ++static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp) +{ -+ struct ovpn_socket *sock; -+ -+ rcu_read_lock(); -+ sock = rcu_dereference_sk_user_data(sk); -+ rcu_read_unlock(); ++ unsigned long allocated = mas_allocated(mas); + -+ if (!sock || !sock->peer) -+ return; -+ -+ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.tx_work); ++ if (allocated < count) { ++ mas_set_alloc_req(mas, count - allocated); ++ mas_alloc_nodes(mas, gfp); ++ } +} + -+static void ovpn_destroy_skb(void *skb) ++/* ++ * mas_node_count() - Check if enough nodes are allocated and request more if ++ * there is not enough nodes. ++ * @mas: The maple state ++ * @count: The number of nodes needed ++ * ++ * Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags. ++ */ ++static void mas_node_count(struct ma_state *mas, int count) +{ -+ consume_skb(skb); ++ return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN); +} + -+void ovpn_tcp_socket_detach(struct socket *sock) ++/* ++ * mas_start() - Sets up maple state for operations. ++ * @mas: The maple state. ++ * ++ * If mas->node == MAS_START, then set the min, max, depth, and offset to ++ * defaults. ++ * ++ * Return: ++ * - If mas->node is an error or not MAS_START, return NULL. ++ * - If it's an empty tree: NULL & mas->node == MAS_NONE ++ * - If it's a single entry: The entry & mas->node == MAS_ROOT ++ * - If it's a tree: NULL & mas->node == safe root node. ++ */ ++static inline struct maple_enode *mas_start(struct ma_state *mas) +{ -+ struct ovpn_socket *ovpn_sock; -+ struct ovpn_peer *peer; ++ if (likely(mas_is_start(mas))) { ++ struct maple_enode *root; + -+ if (!sock) -+ return; ++ mas->node = MAS_NONE; ++ mas->min = 0; ++ mas->max = ULONG_MAX; ++ mas->depth = 0; ++ mas->offset = 0; + -+ rcu_read_lock(); -+ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); ++ root = mas_root(mas); ++ /* Tree with nodes */ ++ if (likely(xa_is_node(root))) { ++ mas->node = mte_safe_root(root); ++ return NULL; ++ } + -+ if (!ovpn_sock->peer) -+ return; ++ /* empty tree */ ++ if (unlikely(!root)) { ++ mas->offset = MAPLE_NODE_SLOTS; ++ return NULL; ++ } + -+ peer = ovpn_sock->peer; ++ /* Single entry tree */ ++ mas->node = MAS_ROOT; ++ mas->offset = MAPLE_NODE_SLOTS; + -+ /* restore CBs that were saved in ovpn_sock_set_tcp_cb() */ -+ write_lock_bh(&sock->sk->sk_callback_lock); -+ sock->sk->sk_state_change = peer->tcp.sk_cb.sk_state_change; -+ sock->sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; -+ sock->sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; -+ rcu_assign_sk_user_data(sock->sk, NULL); -+ write_unlock_bh(&sock->sk->sk_callback_lock); ++ /* Single entry tree. */ ++ if (mas->index > 0) ++ return NULL; + -+ /* cancel any ongoing work. Done after removing the CBs so that these workers cannot be -+ * re-armed -+ */ -+ cancel_work_sync(&peer->tcp.tx_work); -+ cancel_work_sync(&peer->tcp.rx_work); ++ return root; ++ } + -+ ptr_ring_cleanup(&peer->tcp.tx_ring, ovpn_destroy_skb); ++ return NULL; +} + -+/* Try to send one skb (or part of it) over the TCP stream. -+ * -+ * Return 0 on success or a negative error code otherwise. ++/* ++ * ma_data_end() - Find the end of the data in a node. ++ * @node: The maple node ++ * @type: The maple node type ++ * @pivots: The array of pivots in the node ++ * @max: The maximum value in the node + * -+ * Note that the skb is modified by putting away the data being sent, therefore -+ * the caller should check if skb->len is zero to understand if the full skb was -+ * sent or not. ++ * Uses metadata to find the end of the data when possible. ++ * Return: The zero indexed last slot with data (may be null). + */ -+static int ovpn_tcp_send_one(struct ovpn_peer *peer, struct sk_buff *skb) ++static inline unsigned char ma_data_end(struct maple_node *node, ++ enum maple_type type, ++ unsigned long *pivots, ++ unsigned long max) +{ -+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; -+ struct kvec iv = { 0 }; -+ int ret; -+ -+ if (skb_linearize(skb) < 0) { -+ net_err_ratelimited("%s: can't linearize packet\n", __func__); -+ return -ENOMEM; -+ } -+ -+ /* initialize iv structure now as skb_linearize() may have changed skb->data */ -+ iv.iov_base = skb->data; -+ iv.iov_len = skb->len; ++ unsigned char offset; + -+ ret = kernel_sendmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len); -+ if (ret > 0) { -+ __skb_pull(skb, ret); ++ if (type == maple_arange_64) ++ return ma_meta_end(node, type); + -+ /* since we update per-cpu stats in process context, -+ * we need to disable softirqs -+ */ -+ local_bh_disable(); -+ dev_sw_netstats_tx_add(peer->ovpn->dev, 1, ret); -+ local_bh_enable(); ++ offset = mt_pivots[type] - 1; ++ if (likely(!pivots[offset])) ++ return ma_meta_end(node, type); + -+ return 0; -+ } ++ if (likely(pivots[offset] == max)) ++ return offset; + -+ return ret; ++ return mt_pivots[type]; +} + -+/* Process packets in TCP TX queue */ -+static void ovpn_tcp_tx_work(struct work_struct *work) ++/* ++ * mas_data_end() - Find the end of the data (slot). ++ * @mas: the maple state ++ * ++ * This method is optimized to check the metadata of a node if the node type ++ * supports data end metadata. ++ * ++ * Return: The zero indexed last slot with data (may be null). ++ */ ++static inline unsigned char mas_data_end(struct ma_state *mas) +{ -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ int ret; -+ -+ peer = container_of(work, struct ovpn_peer, tcp.tx_work); -+ while ((skb = __ptr_ring_peek(&peer->tcp.tx_ring))) { -+ ret = ovpn_tcp_send_one(peer, skb); -+ if (ret < 0 && ret != -EAGAIN) { -+ net_warn_ratelimited("%s: cannot send TCP packet to peer %u: %d\n", __func__, -+ peer->id, ret); -+ /* in case of TCP error stop sending loop and delete peer */ -+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_ERROR); -+ break; -+ } else if (!skb->len) { -+ /* skb was entirely consumed and can now be removed from the ring */ -+ __ptr_ring_discard_one(&peer->tcp.tx_ring); -+ consume_skb(skb); -+ } ++ enum maple_type type; ++ struct maple_node *node; ++ unsigned char offset; ++ unsigned long *pivots; + -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+} ++ type = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ if (type == maple_arange_64) ++ return ma_meta_end(node, type); + -+static int ovpn_tcp_rx_one(struct ovpn_peer *peer) -+{ -+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; -+ struct ovpn_skb_cb *cb; -+ int status, ret; ++ pivots = ma_pivots(node, type); ++ offset = mt_pivots[type] - 1; ++ if (likely(!pivots[offset])) ++ return ma_meta_end(node, type); + -+ /* no skb allocated means that we have to read (or finish reading) the 2 bytes prefix -+ * containing the actual packet size. -+ */ -+ if (!peer->tcp.skb) { -+ struct kvec iv = { -+ .iov_base = peer->tcp.raw_len + peer->tcp.offset, -+ .iov_len = sizeof(u16) - peer->tcp.offset, -+ }; ++ if (likely(pivots[offset] == mas->max)) ++ return offset; + -+ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); -+ if (ret <= 0) -+ return ret; ++ return mt_pivots[type]; ++} + -+ peer->tcp.offset += ret; -+ /* the entire packet size was read, prepare skb for reading data */ -+ if (peer->tcp.offset == sizeof(u16)) { -+ u16 len = ntohs(*(__be16 *)peer->tcp.raw_len); -+ /* invalid packet length: this is a fatal TCP error */ -+ if (!len) { -+ netdev_err(peer->ovpn->dev, "%s: received invalid packet length\n", -+ __func__); -+ return -EINVAL; ++/* ++ * mas_leaf_max_gap() - Returns the largest gap in a leaf node ++ * @mas - the maple state ++ * ++ * Return: The maximum gap in the leaf. ++ */ ++static unsigned long mas_leaf_max_gap(struct ma_state *mas) ++{ ++ enum maple_type mt; ++ unsigned long pstart, gap, max_gap; ++ struct maple_node *mn; ++ unsigned long *pivots; ++ void __rcu **slots; ++ unsigned char i; ++ unsigned char max_piv; ++ ++ mt = mte_node_type(mas->node); ++ mn = mas_mn(mas); ++ slots = ma_slots(mn, mt); ++ max_gap = 0; ++ if (unlikely(ma_is_dense(mt))) { ++ gap = 0; ++ for (i = 0; i < mt_slots[mt]; i++) { ++ if (slots[i]) { ++ if (gap > max_gap) ++ max_gap = gap; ++ gap = 0; ++ } else { ++ gap++; + } -+ -+ peer->tcp.skb = netdev_alloc_skb_ip_align(peer->ovpn->dev, len); -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = len; + } ++ if (gap > max_gap) ++ max_gap = gap; ++ return max_gap; ++ } ++ ++ /* ++ * Check the first implied pivot optimizes the loop below and slot 1 may ++ * be skipped if there is a gap in slot 0. ++ */ ++ pivots = ma_pivots(mn, mt); ++ if (likely(!slots[0])) { ++ max_gap = pivots[0] - mas->min + 1; ++ i = 2; + } else { -+ struct kvec iv = { -+ .iov_base = peer->tcp.skb->data + peer->tcp.offset, -+ .iov_len = peer->tcp.data_len - peer->tcp.offset, -+ }; ++ i = 1; ++ } + -+ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); -+ if (ret <= 0) -+ return ret; ++ /* reduce max_piv as the special case is checked before the loop */ ++ max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1; ++ /* ++ * Check end implied pivot which can only be a gap on the right most ++ * node. ++ */ ++ if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) { ++ gap = ULONG_MAX - pivots[max_piv]; ++ if (gap > max_gap) ++ max_gap = gap; ++ } + -+ peer->tcp.offset += ret; -+ /* full packet received, send it up for processing */ -+ if (peer->tcp.offset == peer->tcp.data_len) { -+ /* update the skb data structure with the amount of data written by -+ * kernel_recvmsg() -+ */ -+ skb_put(peer->tcp.skb, peer->tcp.data_len); ++ for (; i <= max_piv; i++) { ++ /* data == no gap. */ ++ if (likely(slots[i])) ++ continue; + -+ /* do not perform IP caching for TCP connections */ -+ cb = OVPN_SKB_CB(peer->tcp.skb); -+ cb->sa_fam = AF_UNSPEC; ++ pstart = pivots[i - 1]; ++ gap = pivots[i] - pstart; ++ if (gap > max_gap) ++ max_gap = gap; + -+ /* hold reference to peer as requird by ovpn_recv() */ -+ ovpn_peer_hold(peer); -+ status = ovpn_recv(peer->ovpn, peer, peer->tcp.skb); -+ /* skb not consumed - free it now */ -+ if (unlikely(status < 0)) -+ kfree_skb(peer->tcp.skb); ++ /* There cannot be two gaps in a row. */ ++ i++; ++ } ++ return max_gap; ++} + -+ peer->tcp.skb = NULL; -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = 0; ++/* ++ * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) ++ * @node: The maple node ++ * @gaps: The pointer to the gaps ++ * @mt: The maple node type ++ * @*off: Pointer to store the offset location of the gap. ++ * ++ * Uses the metadata data end to scan backwards across set gaps. ++ * ++ * Return: The maximum gap value ++ */ ++static inline unsigned long ++ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, ++ unsigned char *off) ++{ ++ unsigned char offset, i; ++ unsigned long max_gap = 0; ++ ++ i = offset = ma_meta_end(node, mt); ++ do { ++ if (gaps[i] > max_gap) { ++ max_gap = gaps[i]; ++ offset = i; + } -+ } ++ } while (i--); + -+ return ret; ++ *off = offset; ++ return max_gap; +} + -+static void ovpn_tcp_rx_work(struct work_struct *work) ++/* ++ * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. ++ * @mas: The maple state. ++ * ++ * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap. ++ * ++ * Return: The gap value. ++ */ ++static inline unsigned long mas_max_gap(struct ma_state *mas) +{ -+ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, tcp.rx_work); -+ int ret; ++ unsigned long *gaps; ++ unsigned char offset; ++ enum maple_type mt; ++ struct maple_node *node; + -+ while (true) { -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); ++ mt = mte_node_type(mas->node); ++ if (ma_is_leaf(mt)) ++ return mas_leaf_max_gap(mas); + -+ ret = ovpn_tcp_rx_one(peer); -+ if (ret <= 0) -+ break; -+ } ++ node = mas_mn(mas); ++ offset = ma_meta_gap(node, mt); ++ if (offset == MAPLE_ARANGE64_META_MAX) ++ return 0; + -+ if (ret < 0 && ret != -EAGAIN) -+ netdev_err(peer->ovpn->dev, "%s: TCP socket error: %d\n", __func__, ret); ++ gaps = ma_gaps(node, mt); ++ return gaps[offset]; +} + -+/* Put packet into TCP TX queue and schedule a consumer */ -+void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ int ret; -+ -+ ret = ptr_ring_produce_bh(&peer->tcp.tx_ring, skb); -+ if (ret < 0) { -+ kfree_skb_list(skb); -+ return; -+ } ++/* ++ * mas_parent_gap() - Set the parent gap and any gaps above, as needed ++ * @mas: The maple state ++ * @offset: The gap offset in the parent to set ++ * @new: The new gap value. ++ * ++ * Set the parent gap then continue to set the gap upwards, using the metadata ++ * of the parent to see if it is necessary to check the node above. ++ */ ++static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, ++ unsigned long new) ++{ ++ unsigned long meta_gap = 0; ++ struct maple_node *pnode; ++ struct maple_enode *penode; ++ unsigned long *pgaps; ++ unsigned char meta_offset; ++ enum maple_type pmt; ++ ++ pnode = mte_parent(mas->node); ++ pmt = mas_parent_enum(mas, mas->node); ++ penode = mt_mk_node(pnode, pmt); ++ pgaps = ma_gaps(pnode, pmt); ++ ++ascend: ++ meta_offset = ma_meta_gap(pnode, pmt); ++ if (meta_offset == MAPLE_ARANGE64_META_MAX) ++ meta_gap = 0; ++ else ++ meta_gap = pgaps[meta_offset]; + -+ queue_work(peer->ovpn->events_wq, &peer->tcp.tx_work); -+} ++ pgaps[offset] = new; + -+/* Set TCP encapsulation callbacks */ -+int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer) -+{ -+ void *old_data; -+ int ret; ++ if (meta_gap == new) ++ return; + -+ INIT_WORK(&peer->tcp.tx_work, ovpn_tcp_tx_work); -+ INIT_WORK(&peer->tcp.rx_work, ovpn_tcp_rx_work); ++ if (offset != meta_offset) { ++ if (meta_gap > new) ++ return; + -+ ret = ptr_ring_init(&peer->tcp.tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(peer->ovpn->dev, "cannot allocate TCP TX ring\n"); -+ return ret; ++ ma_set_meta_gap(pnode, pmt, offset); ++ } else if (new < meta_gap) { ++ meta_offset = 15; ++ new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); ++ ma_set_meta_gap(pnode, pmt, meta_offset); + } + -+ peer->tcp.skb = NULL; -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = 0; ++ if (ma_is_root(pnode)) ++ return; + -+ write_lock_bh(&sock->sk->sk_callback_lock); ++ /* Go to the parent node. */ ++ pnode = mte_parent(penode); ++ pmt = mas_parent_enum(mas, penode); ++ pgaps = ma_gaps(pnode, pmt); ++ offset = mte_parent_slot(penode); ++ penode = mt_mk_node(pnode, pmt); ++ goto ascend; ++} + -+ /* make sure no pre-existing encapsulation handler exists */ -+ rcu_read_lock(); -+ old_data = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ if (old_data) { -+ netdev_err(peer->ovpn->dev, "provided socket already taken by other user\n"); -+ ret = -EBUSY; -+ goto err; -+ } ++/* ++ * mas_update_gap() - Update a nodes gaps and propagate up if necessary. ++ * @mas - the maple state. ++ */ ++static inline void mas_update_gap(struct ma_state *mas) ++{ ++ unsigned char pslot; ++ unsigned long p_gap; ++ unsigned long max_gap; + -+ /* sanity check */ -+ if (sock->sk->sk_protocol != IPPROTO_TCP) { -+ netdev_err(peer->ovpn->dev, "expected TCP socket\n"); -+ ret = -EINVAL; -+ goto err; -+ } ++ if (!mt_is_alloc(mas->tree)) ++ return; + -+ /* only a fully connected socket are expected. Connection should be handled in userspace */ -+ if (sock->sk->sk_state != TCP_ESTABLISHED) { -+ netdev_err(peer->ovpn->dev, "unexpected state for TCP socket: %d\n", -+ sock->sk->sk_state); -+ ret = -EINVAL; -+ goto err; -+ } ++ if (mte_is_root(mas->node)) ++ return; + -+ /* save current CBs so that they can be restored upon socket release */ -+ peer->tcp.sk_cb.sk_state_change = sock->sk->sk_state_change; -+ peer->tcp.sk_cb.sk_data_ready = sock->sk->sk_data_ready; -+ peer->tcp.sk_cb.sk_write_space = sock->sk->sk_write_space; ++ max_gap = mas_max_gap(mas); + -+ /* assign our static CBs */ -+ sock->sk->sk_state_change = ovpn_tcp_state_change; -+ sock->sk->sk_data_ready = ovpn_tcp_data_ready; -+ sock->sk->sk_write_space = ovpn_tcp_write_space; ++ pslot = mte_parent_slot(mas->node); ++ p_gap = ma_gaps(mte_parent(mas->node), ++ mas_parent_enum(mas, mas->node))[pslot]; + -+ write_unlock_bh(&sock->sk->sk_callback_lock); ++ if (p_gap != max_gap) ++ mas_parent_gap(mas, pslot, max_gap); ++} + -+ return 0; -+err: -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ ptr_ring_cleanup(&peer->tcp.tx_ring, NULL); ++/* ++ * mas_adopt_children() - Set the parent pointer of all nodes in @parent to ++ * @parent with the slot encoded. ++ * @mas - the maple state (for the tree) ++ * @parent - the maple encoded node containing the children. ++ */ ++static inline void mas_adopt_children(struct ma_state *mas, ++ struct maple_enode *parent) ++{ ++ enum maple_type type = mte_node_type(parent); ++ struct maple_node *node = mas_mn(mas); ++ void __rcu **slots = ma_slots(node, type); ++ unsigned long *pivots = ma_pivots(node, type); ++ struct maple_enode *child; ++ unsigned char offset; + -+ return ret; ++ offset = ma_data_end(node, type, pivots, mas->max); ++ do { ++ child = mas_slot_locked(mas, slots, offset); ++ mte_set_parent(child, parent, offset); ++ } while (offset--); +} -diff --git a/drivers/net/ovpn-dco/tcp.h b/drivers/net/ovpn-dco/tcp.h -new file mode 100644 -index 000000000000..d243a8e1c34e ---- /dev/null -+++ b/drivers/net/ovpn-dco/tcp.h -@@ -0,0 +1,38 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli ++ ++/* ++ * mas_replace() - Replace a maple node in the tree with mas->node. Uses the ++ * parent encoding to locate the maple node in the tree. ++ * @mas - the ma_state to use for operations. ++ * @advanced - boolean to adopt the child nodes and free the old node (false) or ++ * leave the node (true) and handle the adoption and free elsewhere. + */ ++static inline void mas_replace(struct ma_state *mas, bool advanced) ++ __must_hold(mas->tree->lock) ++{ ++ struct maple_node *mn = mas_mn(mas); ++ struct maple_enode *old_enode; ++ unsigned char offset = 0; ++ void __rcu **slots = NULL; + -+#ifndef _NET_OVPN_DCO_TCP_H_ -+#define _NET_OVPN_DCO_TCP_H_ ++ if (ma_is_root(mn)) { ++ old_enode = mas_root_locked(mas); ++ } else { ++ offset = mte_parent_slot(mas->node); ++ slots = ma_slots(mte_parent(mas->node), ++ mas_parent_enum(mas, mas->node)); ++ old_enode = mas_slot_locked(mas, slots, offset); ++ } + -+#include "peer.h" ++ if (!advanced && !mte_is_leaf(mas->node)) ++ mas_adopt_children(mas, mas->node); + -+#include -+#include -+#include -+#include ++ if (mte_is_root(mas->node)) { ++ mn->parent = ma_parent_ptr( ++ ((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); ++ mas_set_height(mas); ++ } else { ++ rcu_assign_pointer(slots[offset], mas->node); ++ } + -+void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb); ++ if (!advanced) ++ mas_free(mas, old_enode); ++} + -+int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer); -+void ovpn_tcp_socket_detach(struct socket *sock); ++/* ++ * mas_new_child() - Find the new child of a node. ++ * @mas: the maple state ++ * @child: the maple state to store the child. ++ */ ++static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) ++ __must_hold(mas->tree->lock) ++{ ++ enum maple_type mt; ++ unsigned char offset; ++ unsigned char end; ++ unsigned long *pivots; ++ struct maple_enode *entry; ++ struct maple_node *node; ++ void __rcu **slots; ++ ++ mt = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ end = ma_data_end(node, mt, pivots, mas->max); ++ for (offset = mas->offset; offset <= end; offset++) { ++ entry = mas_slot_locked(mas, slots, offset); ++ if (mte_parent(entry) == node) { ++ *child = *mas; ++ mas->offset = offset + 1; ++ child->offset = offset; ++ mas_descend(child); ++ child->offset = 0; ++ return true; ++ } ++ } ++ return false; ++} + -+/* Prepare skb and enqueue it for sending to peer. -+ * -+ * Preparation consist in prepending the skb payload with its size. -+ * Required by the OpenVPN protocol in order to extract packets from -+ * the TCP stream on the receiver side. ++/* ++ * mab_shift_right() - Shift the data in mab right. Note, does not clean out the ++ * old data or set b_node->b_end. ++ * @b_node: the maple_big_node ++ * @shift: the shift count + */ -+static inline void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sk_buff *skb) ++static inline void mab_shift_right(struct maple_big_node *b_node, ++ unsigned char shift) +{ -+ u16 len = skb->len; ++ unsigned long size = b_node->b_end * sizeof(unsigned long); + -+ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); -+ ovpn_queue_tcp_skb(peer, skb); ++ memmove(b_node->pivot + shift, b_node->pivot, size); ++ memmove(b_node->slot + shift, b_node->slot, size); ++ if (b_node->type == maple_arange_64) ++ memmove(b_node->gap + shift, b_node->gap, size); +} + -+#endif /* _NET_OVPN_DCO_TCP_H_ */ -diff --git a/drivers/net/ovpn-dco/udp.c b/drivers/net/ovpn-dco/udp.c -new file mode 100644 -index 000000000000..afa236d1f15c ---- /dev/null -+++ b/drivers/net/ovpn-dco/udp.c -@@ -0,0 +1,343 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. ++/* ++ * mab_middle_node() - Check if a middle node is needed (unlikely) ++ * @b_node: the maple_big_node that contains the data. ++ * @size: the amount of data in the b_node ++ * @split: the potential split location ++ * @slot_count: the size that can be stored in a single node being considered. + * -+ * Author: Antonio Quartulli ++ * Return: true if a middle node is required. + */ ++static inline bool mab_middle_node(struct maple_big_node *b_node, int split, ++ unsigned char slot_count) ++{ ++ unsigned char size = b_node->b_end; + -+#include "main.h" -+#include "bind.h" -+#include "ovpn.h" -+#include "ovpnstruct.h" -+#include "peer.h" -+#include "proto.h" -+#include "skb.h" -+#include "udp.h" ++ if (size >= 2 * slot_count) ++ return true; + -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++ if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) ++ return true; + -+/** -+ * ovpn_udp_encap_recv() - Start processing a received UDP packet. -+ * If the first byte of the payload is DATA_V2, the packet is further processed, -+ * otherwise it is forwarded to the UDP stack for delivery to user space. ++ return false; ++} ++ ++/* ++ * mab_no_null_split() - ensure the split doesn't fall on a NULL ++ * @b_node: the maple_big_node with the data ++ * @split: the suggested split location ++ * @slot_count: the number of slots in the node being considered. + * -+ * @sk: the socket the packet was received on -+ * @skb: the sk_buff containing the actual packet ++ * Return: the split location. ++ */ ++static inline int mab_no_null_split(struct maple_big_node *b_node, ++ unsigned char split, unsigned char slot_count) ++{ ++ if (!b_node->slot[split]) { ++ /* ++ * If the split is less than the max slot && the right side will ++ * still be sufficient, then increment the split on NULL. ++ */ ++ if ((split < slot_count - 1) && ++ (b_node->b_end - split) > (mt_min_slots[b_node->type])) ++ split++; ++ else ++ split--; ++ } ++ return split; ++} ++ ++/* ++ * mab_calc_split() - Calculate the split location and if there needs to be two ++ * splits. ++ * @bn: The maple_big_node with the data ++ * @mid_split: The second split, if required. 0 otherwise. + * -+ * Return codes: -+ * 0 : we consumed or dropped packet -+ * >0 : skb should be passed up to userspace as UDP (packet not consumed) -+ * <0 : skb should be resubmitted as proto -N (packet not consumed) ++ * Return: The first split location. The middle split is set in @mid_split. + */ -+static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) ++static inline int mab_calc_split(struct ma_state *mas, ++ struct maple_big_node *bn, unsigned char *mid_split, unsigned long min) +{ -+ struct ovpn_peer *peer = NULL; -+ struct ovpn_struct *ovpn; -+ u32 peer_id; -+ u8 opcode; -+ int ret; ++ unsigned char b_end = bn->b_end; ++ int split = b_end / 2; /* Assume equal split. */ ++ unsigned char slot_min, slot_count = mt_slots[bn->type]; + -+ ovpn = ovpn_from_udp_sock(sk); -+ if (unlikely(!ovpn)) { -+ net_err_ratelimited("%s: cannot obtain ovpn object from UDP socket\n", __func__); -+ goto drop; ++ /* ++ * To support gap tracking, all NULL entries are kept together and a node cannot ++ * end on a NULL entry, with the exception of the left-most leaf. The ++ * limitation means that the split of a node must be checked for this condition ++ * and be able to put more data in one direction or the other. ++ */ ++ if (unlikely((mas->mas_flags & MA_STATE_BULK))) { ++ *mid_split = 0; ++ split = b_end - mt_min_slots[bn->type]; ++ ++ if (!ma_is_leaf(bn->type)) ++ return split; ++ ++ mas->mas_flags |= MA_STATE_REBALANCE; ++ if (!bn->slot[split]) ++ split--; ++ return split; + } + -+ /* Make sure the first 4 bytes of the skb data buffer after the UDP header are accessible. -+ * They are required to fetch the OP code, the key ID and the peer ID. ++ /* ++ * Although extremely rare, it is possible to enter what is known as the 3-way ++ * split scenario. The 3-way split comes about by means of a store of a range ++ * that overwrites the end and beginning of two full nodes. The result is a set ++ * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can ++ * also be located in different parent nodes which are also full. This can ++ * carry upwards all the way to the root in the worst case. + */ -+ if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) + 4))) { -+ net_dbg_ratelimited("%s: packet too small\n", __func__); -+ goto drop; -+ } ++ if (unlikely(mab_middle_node(bn, split, slot_count))) { ++ split = b_end / 3; ++ *mid_split = split * 2; ++ } else { ++ slot_min = mt_min_slots[bn->type]; + -+ opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr)); -+ if (likely(opcode == OVPN_DATA_V2)) { -+ peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr)); -+ /* some OpenVPN server implementations send data packets with the peer-id set to -+ * undef. In this case we skip the peer lookup by peer-id and we try with the -+ * transport address ++ *mid_split = 0; ++ /* ++ * Avoid having a range less than the slot count unless it ++ * causes one node to be deficient. ++ * NOTE: mt_min_slots is 1 based, b_end and split are zero. + */ -+ if (peer_id != OVPN_PEER_ID_UNDEF) { -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) { -+ net_err_ratelimited("%s: received data from unknown peer (id: %d)\n", -+ __func__, peer_id); -+ goto drop; -+ } -+ -+ /* check if this peer changed it's IP address and update state */ -+ ovpn_peer_float(peer, skb); -+ } ++ while (((bn->pivot[split] - min) < slot_count - 1) && ++ (split < slot_count - 1) && (b_end - split > slot_min)) ++ split++; + } + -+ if (!peer) { -+ /* might be a control packet or a data packet with undef peer-id */ -+ peer = ovpn_peer_lookup_transp_addr(ovpn, skb); -+ if (unlikely(!peer)) { -+ if (opcode != OVPN_DATA_V2) { -+ netdev_dbg(ovpn->dev, -+ "%s: control packet from unknown peer, sending to userspace", -+ __func__); -+ return 1; -+ } ++ /* Avoid ending a node on a NULL entry */ ++ split = mab_no_null_split(bn, split, slot_count); ++ if (!(*mid_split)) ++ return split; + -+ netdev_dbg(ovpn->dev, -+ "%s: received data with undef peer-id from unknown source\n", -+ __func__); -+ goto drop; -+ } -+ } ++ *mid_split = mab_no_null_split(bn, *mid_split, slot_count); + -+ /* pop off outer UDP header */ -+ __skb_pull(skb, sizeof(struct udphdr)); ++ return split; ++} + -+ ret = ovpn_recv(ovpn, peer, skb); -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: cannot handle incoming packet: %d\n", __func__, ret); -+ goto drop; ++/* ++ * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node ++ * and set @b_node->b_end to the next free slot. ++ * @mas: The maple state ++ * @mas_start: The starting slot to copy ++ * @mas_end: The end slot to copy (inclusively) ++ * @b_node: The maple_big_node to place the data ++ * @mab_start: The starting location in maple_big_node to store the data. ++ */ ++static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, ++ unsigned char mas_end, struct maple_big_node *b_node, ++ unsigned char mab_start) ++{ ++ enum maple_type mt; ++ struct maple_node *node; ++ void __rcu **slots; ++ unsigned long *pivots, *gaps; ++ int i = mas_start, j = mab_start; ++ unsigned char piv_end; ++ ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ pivots = ma_pivots(node, mt); ++ if (!i) { ++ b_node->pivot[j] = pivots[i++]; ++ if (unlikely(i > mas_end)) ++ goto complete; ++ j++; ++ } ++ ++ piv_end = min(mas_end, mt_pivots[mt]); ++ for (; i < piv_end; i++, j++) { ++ b_node->pivot[j] = pivots[i]; ++ if (unlikely(!b_node->pivot[j])) ++ break; ++ ++ if (unlikely(mas->max == b_node->pivot[j])) ++ goto complete; + } + -+ /* should this be a non DATA_V2 packet, ret will be >0 and this will instruct the UDP -+ * stack to continue processing this packet as usual (i.e. deliver to user space) -+ */ -+ return ret; ++ if (likely(i <= mas_end)) ++ b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); + -+drop: -+ if (peer) -+ ovpn_peer_put(peer); -+ kfree_skb(skb); -+ return 0; ++complete: ++ b_node->b_end = ++j; ++ j -= mab_start; ++ slots = ma_slots(node, mt); ++ memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); ++ if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) { ++ gaps = ma_gaps(node, mt); ++ memcpy(b_node->gap + mab_start, gaps + mas_start, ++ sizeof(unsigned long) * j); ++ } +} + -+static int ovpn_udp4_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) ++/* ++ * mas_leaf_set_meta() - Set the metadata of a leaf if possible. ++ * @mas: The maple state ++ * @node: The maple node ++ * @pivots: pointer to the maple node pivots ++ * @mt: The maple type ++ * @end: The assumed end ++ * ++ * Note, end may be incremented within this function but not modified at the ++ * source. This is fine since the metadata is the last thing to be stored in a ++ * node during a write. ++ */ ++static inline void mas_leaf_set_meta(struct ma_state *mas, ++ struct maple_node *node, unsigned long *pivots, ++ enum maple_type mt, unsigned char end) +{ -+ struct rtable *rt; -+ struct flowi4 fl = { -+ .saddr = bind->local.ipv4.s_addr, -+ .daddr = bind->sa.in4.sin_addr.s_addr, -+ .fl4_sport = inet_sk(sk)->inet_sport, -+ .fl4_dport = bind->sa.in4.sin_port, -+ .flowi4_proto = sk->sk_protocol, -+ .flowi4_mark = sk->sk_mark, -+ }; -+ int ret; ++ /* There is no room for metadata already */ ++ if (mt_pivots[mt] <= end) ++ return; + -+ local_bh_disable(); -+ rt = dst_cache_get_ip4(cache, &fl.saddr); -+ if (rt) -+ goto transmit; ++ if (pivots[end] && pivots[end] < mas->max) ++ end++; + -+ if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { -+ /* we may end up here when the cached address is not usable anymore. -+ * In this case we reset address/cache and perform a new look up -+ */ -+ fl.saddr = 0; -+ bind->local.ipv4.s_addr = 0; -+ dst_cache_reset(cache); -+ } ++ if (end < mt_slots[mt] - 1) ++ ma_set_meta(node, mt, 0, end); ++} + -+ rt = ip_route_output_flow(sock_net(sk), &fl, sk); -+ if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { -+ fl.saddr = 0; -+ bind->local.ipv4.s_addr = 0; -+ dst_cache_reset(cache); ++/* ++ * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. ++ * @b_node: the maple_big_node that has the data ++ * @mab_start: the start location in @b_node. ++ * @mab_end: The end location in @b_node (inclusively) ++ * @mas: The maple state with the maple encoded node. ++ */ ++static inline void mab_mas_cp(struct maple_big_node *b_node, ++ unsigned char mab_start, unsigned char mab_end, ++ struct ma_state *mas, bool new_max) ++{ ++ int i, j = 0; ++ enum maple_type mt = mte_node_type(mas->node); ++ struct maple_node *node = mte_to_node(mas->node); ++ void __rcu **slots = ma_slots(node, mt); ++ unsigned long *pivots = ma_pivots(node, mt); ++ unsigned long *gaps = NULL; ++ unsigned char end; + -+ rt = ip_route_output_flow(sock_net(sk), &fl, sk); -+ } ++ if (mab_end - mab_start > mt_pivots[mt]) ++ mab_end--; + -+ if (IS_ERR(rt)) { -+ ret = PTR_ERR(rt); -+ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, -+ &bind->sa.in4, ret); -+ goto err; -+ } -+ dst_cache_set_ip4(cache, &rt->dst, fl.saddr); ++ if (!pivots[mt_pivots[mt] - 1]) ++ slots[mt_pivots[mt]] = NULL; + -+transmit: -+ udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, -+ ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, -+ fl.fl4_dport, false, sk->sk_no_check_tx); -+ ret = 0; -+err: -+ local_bh_enable(); -+ return ret; -+} ++ i = mab_start; ++ do { ++ pivots[j++] = b_node->pivot[i++]; ++ } while (i <= mab_end && likely(b_node->pivot[i])); + -+#if IS_ENABLED(CONFIG_IPV6) -+static int ovpn_udp6_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ struct dst_entry *dst; -+ int ret; ++ memcpy(slots, b_node->slot + mab_start, ++ sizeof(void *) * (i - mab_start)); + -+ struct flowi6 fl = { -+ .saddr = bind->local.ipv6, -+ .daddr = bind->sa.in6.sin6_addr, -+ .fl6_sport = inet_sk(sk)->inet_sport, -+ .fl6_dport = bind->sa.in6.sin6_port, -+ .flowi6_proto = sk->sk_protocol, -+ .flowi6_mark = sk->sk_mark, -+ .flowi6_oif = bind->sa.in6.sin6_scope_id, -+ }; ++ if (new_max) ++ mas->max = b_node->pivot[i - 1]; + -+ local_bh_disable(); -+ dst = dst_cache_get_ip6(cache, &fl.saddr); -+ if (dst) -+ goto transmit; ++ end = j - 1; ++ if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { ++ unsigned long max_gap = 0; ++ unsigned char offset = 15; + -+ if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { -+ /* we may end up here when the cached address is not usable anymore. -+ * In this case we reset address/cache and perform a new look up -+ */ -+ fl.saddr = in6addr_any; -+ bind->local.ipv6 = in6addr_any; -+ dst_cache_reset(cache); -+ } ++ gaps = ma_gaps(node, mt); ++ do { ++ gaps[--j] = b_node->gap[--i]; ++ if (gaps[j] > max_gap) { ++ offset = j; ++ max_gap = gaps[j]; ++ } ++ } while (j); + -+ dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); -+ if (IS_ERR(dst)) { -+ ret = PTR_ERR(dst); -+ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, -+ &bind->sa.in6, ret); -+ goto err; ++ ma_set_meta(node, mt, offset, end); ++ } else { ++ mas_leaf_set_meta(mas, node, pivots, mt, end); + } -+ dst_cache_set_ip6(cache, dst, &fl.saddr); -+ -+transmit: -+ udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0, -+ ip6_dst_hoplimit(dst), 0, fl.fl6_sport, -+ fl.fl6_dport, udp_get_no_check6_tx(sk)); -+ ret = 0; -+err: -+ local_bh_enable(); -+ return ret; +} -+#endif + -+/* Transmit skb utilizing kernel-provided UDP tunneling framework. ++/* ++ * mas_descend_adopt() - Descend through a sub-tree and adopt children. ++ * @mas: the maple state with the maple encoded node of the sub-tree. + * -+ * rcu_read_lock should be held on entry. -+ * On return, the skb is consumed. ++ * Descend through a sub-tree and adopt children who do not have the correct ++ * parents set. Follow the parents which have the correct parents as they are ++ * the new entries which need to be followed to find other incorrectly set ++ * parents. + */ -+static int ovpn_udp_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) ++static inline void mas_descend_adopt(struct ma_state *mas) +{ -+ int ret; ++ struct ma_state list[3], next[3]; ++ int i, n; + -+ ovpn_rcu_lockdep_assert_held(); -+ -+ /* set sk to null if skb is already orphaned */ -+ if (!skb->destructor) -+ skb->sk = NULL; ++ /* ++ * At each level there may be up to 3 correct parent pointers which indicates ++ * the new nodes which need to be walked to find any new nodes at a lower level. ++ */ + -+ switch (bind->sa.in4.sin_family) { -+ case AF_INET: -+ ret = ovpn_udp4_output(ovpn, bind, cache, sk, skb); -+ break; -+#if IS_ENABLED(CONFIG_IPV6) -+ case AF_INET6: -+ ret = ovpn_udp6_output(ovpn, bind, cache, sk, skb); -+ break; -+#endif -+ default: -+ ret = -EAFNOSUPPORT; -+ break; ++ for (i = 0; i < 3; i++) { ++ list[i] = *mas; ++ list[i].offset = 0; ++ next[i].offset = 0; + } ++ next[0] = *mas; + -+ return ret; ++ while (!mte_is_leaf(list[0].node)) { ++ n = 0; ++ for (i = 0; i < 3; i++) { ++ if (mas_is_none(&list[i])) ++ continue; ++ ++ if (i && list[i-1].node == list[i].node) ++ continue; ++ ++ while ((n < 3) && (mas_new_child(&list[i], &next[n]))) ++ n++; ++ ++ mas_adopt_children(&list[i], list[i].node); ++ } ++ ++ while (n < 3) ++ next[n++].node = MAS_NONE; ++ ++ /* descend by setting the list to the children */ ++ for (i = 0; i < 3; i++) ++ list[i] = next[i]; ++ } +} + -+void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, -+ struct sk_buff *skb) ++/* ++ * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. ++ * @mas: The maple state ++ * @end: The maple node end ++ * @mt: The maple node type ++ */ ++static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, ++ enum maple_type mt) +{ -+ struct ovpn_bind *bind; -+ struct socket *sock; -+ int ret = -1; ++ if (!(mas->mas_flags & MA_STATE_BULK)) ++ return; + -+ skb->dev = ovpn->dev; -+ /* no checksum performed at this layer */ -+ skb->ip_summed = CHECKSUM_NONE; ++ if (mte_is_root(mas->node)) ++ return; + -+ /* get socket info */ -+ sock = peer->sock->sock; -+ if (unlikely(!sock)) { -+ net_dbg_ratelimited("%s: no sock for remote peer\n", __func__); -+ goto out; ++ if (end > mt_min_slots[mt]) { ++ mas->mas_flags &= ~MA_STATE_REBALANCE; ++ return; + } ++} + -+ rcu_read_lock(); -+ /* get binding */ -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) { -+ net_dbg_ratelimited("%s: no bind for remote peer\n", __func__); -+ goto out_unlock; ++/* ++ * mas_store_b_node() - Store an @entry into the b_node while also copying the ++ * data from a maple encoded node. ++ * @wr_mas: the maple write state ++ * @b_node: the maple_big_node to fill with data ++ * @offset_end: the offset to end copying ++ * ++ * Return: The actual end of the data stored in @b_node ++ */ ++static inline void mas_store_b_node(struct ma_wr_state *wr_mas, ++ struct maple_big_node *b_node, unsigned char offset_end) ++{ ++ unsigned char slot; ++ unsigned char b_end; ++ /* Possible underflow of piv will wrap back to 0 before use. */ ++ unsigned long piv; ++ struct ma_state *mas = wr_mas->mas; ++ ++ b_node->type = wr_mas->type; ++ b_end = 0; ++ slot = mas->offset; ++ if (slot) { ++ /* Copy start data up to insert. */ ++ mas_mab_cp(mas, 0, slot - 1, b_node, 0); ++ b_end = b_node->b_end; ++ piv = b_node->pivot[b_end - 1]; ++ } else ++ piv = mas->min - 1; ++ ++ if (piv + 1 < mas->index) { ++ /* Handle range starting after old range */ ++ b_node->slot[b_end] = wr_mas->content; ++ if (!wr_mas->content) ++ b_node->gap[b_end] = mas->index - 1 - piv; ++ b_node->pivot[b_end++] = mas->index - 1; + } + -+ /* crypto layer -> transport (UDP) */ -+ ret = ovpn_udp_output(ovpn, bind, &peer->dst_cache, sock->sk, skb); ++ /* Store the new entry. */ ++ mas->offset = b_end; ++ b_node->slot[b_end] = wr_mas->entry; ++ b_node->pivot[b_end] = mas->last; + -+out_unlock: -+ rcu_read_unlock(); -+out: -+ if (ret < 0) -+ kfree_skb(skb); ++ /* Appended. */ ++ if (mas->last >= mas->max) ++ goto b_end; ++ ++ /* Handle new range ending before old range ends */ ++ piv = mas_logical_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); ++ if (piv > mas->last) { ++ if (piv == ULONG_MAX) ++ mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type); ++ ++ if (offset_end != slot) ++ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, ++ offset_end); ++ ++ b_node->slot[++b_end] = wr_mas->content; ++ if (!wr_mas->content) ++ b_node->gap[b_end] = piv - mas->last + 1; ++ b_node->pivot[b_end] = piv; ++ } ++ ++ slot = offset_end + 1; ++ if (slot > wr_mas->node_end) ++ goto b_end; ++ ++ /* Copy end data to the end of the node. */ ++ mas_mab_cp(mas, slot, wr_mas->node_end + 1, b_node, ++b_end); ++ b_node->b_end--; ++ return; ++ ++b_end: ++ b_node->b_end = b_end; +} + -+/* Set UDP encapsulation callbacks */ -+int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn) ++/* ++ * mas_prev_sibling() - Find the previous node with the same parent. ++ * @mas: the maple state ++ * ++ * Return: True if there is a previous sibling, false otherwise. ++ */ ++static inline bool mas_prev_sibling(struct ma_state *mas) +{ -+ struct udp_tunnel_sock_cfg cfg = { -+ .sk_user_data = ovpn, -+ .encap_type = UDP_ENCAP_OVPNINUDP, -+ .encap_rcv = ovpn_udp_encap_recv, -+ }; -+ struct ovpn_socket *old_data; ++ unsigned int p_slot = mte_parent_slot(mas->node); + -+ /* sanity check */ -+ if (sock->sk->sk_protocol != IPPROTO_UDP) { -+ netdev_err(ovpn->dev, "%s: expected UDP socket\n", __func__); -+ return -EINVAL; -+ } ++ if (mte_is_root(mas->node)) ++ return false; + -+ /* make sure no pre-existing encapsulation handler exists */ -+ rcu_read_lock(); -+ old_data = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ if (old_data) { -+ if (old_data->ovpn == ovpn) { -+ netdev_dbg(ovpn->dev, -+ "%s: provided socket already owned by this interface\n", -+ __func__); -+ return -EALREADY; -+ } ++ if (!p_slot) ++ return false; + -+ netdev_err(ovpn->dev, "%s: provided socket already taken by other user\n", -+ __func__); -+ return -EBUSY; -+ } ++ mas_ascend(mas); ++ mas->offset = p_slot - 1; ++ mas_descend(mas); ++ return true; ++} + -+ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); ++/* ++ * mas_next_sibling() - Find the next node with the same parent. ++ * @mas: the maple state ++ * ++ * Return: true if there is a next sibling, false otherwise. ++ */ ++static inline bool mas_next_sibling(struct ma_state *mas) ++{ ++ MA_STATE(parent, mas->tree, mas->index, mas->last); + -+ return 0; ++ if (mte_is_root(mas->node)) ++ return false; ++ ++ parent = *mas; ++ mas_ascend(&parent); ++ parent.offset = mte_parent_slot(mas->node) + 1; ++ if (parent.offset > mas_data_end(&parent)) ++ return false; ++ ++ *mas = parent; ++ mas_descend(mas); ++ return true; +} + -+/* Detach socket from encapsulation handler and/or other callbacks */ -+void ovpn_udp_socket_detach(struct socket *sock) ++/* ++ * mte_node_or_node() - Return the encoded node or MAS_NONE. ++ * @enode: The encoded maple node. ++ * ++ * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state. ++ * ++ * Return: @enode or MAS_NONE ++ */ ++static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) +{ -+ struct udp_tunnel_sock_cfg cfg = { }; ++ if (enode) ++ return enode; + -+ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); ++ return ma_enode_ptr(MAS_NONE); +} -diff --git a/drivers/net/ovpn-dco/udp.h b/drivers/net/ovpn-dco/udp.h -new file mode 100644 -index 000000000000..be94fb74669b ---- /dev/null -+++ b/drivers/net/ovpn-dco/udp.h -@@ -0,0 +1,25 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. ++ ++/* ++ * mas_wr_node_walk() - Find the correct offset for the index in the @mas. ++ * @wr_mas: The maple write state + * -+ * Author: Antonio Quartulli ++ * Uses mas_slot_locked() and does not need to worry about dead nodes. + */ ++static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ unsigned char count; ++ unsigned char offset; ++ unsigned long index, min, max; + -+#ifndef _NET_OVPN_DCO_UDP_H_ -+#define _NET_OVPN_DCO_UDP_H_ ++ if (unlikely(ma_is_dense(wr_mas->type))) { ++ wr_mas->r_max = wr_mas->r_min = mas->index; ++ mas->offset = mas->index = mas->min; ++ return; ++ } + -+#include "peer.h" -+#include "ovpnstruct.h" ++ wr_mas->node = mas_mn(wr_mas->mas); ++ wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); ++ count = wr_mas->node_end = ma_data_end(wr_mas->node, wr_mas->type, ++ wr_mas->pivots, mas->max); ++ offset = mas->offset; ++ min = mas_safe_min(mas, wr_mas->pivots, offset); ++ if (unlikely(offset == count)) ++ goto max; + -+#include -+#include -+#include -+#include ++ max = wr_mas->pivots[offset]; ++ index = mas->index; ++ if (unlikely(index <= max)) ++ goto done; + -+int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn); -+void ovpn_udp_socket_detach(struct socket *sock); -+void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, -+ struct sk_buff *skb); ++ if (unlikely(!max && offset)) ++ goto max; ++ ++ min = max + 1; ++ while (++offset < count) { ++ max = wr_mas->pivots[offset]; ++ if (index <= max) ++ goto done; ++ else if (unlikely(!max)) ++ break; ++ ++ min = max + 1; ++ } ++ ++max: ++ max = mas->max; ++done: ++ wr_mas->r_max = max; ++ wr_mas->r_min = min; ++ wr_mas->offset_end = mas->offset = offset; ++} + -+#endif /* _NET_OVPN_DCO_UDP_H_ */ -diff --git a/include/net/netlink.h b/include/net/netlink.h -index 7a2a9d3144ba..335f44871529 100644 ---- a/include/net/netlink.h -+++ b/include/net/netlink.h -@@ -441,6 +441,7 @@ struct nla_policy { - .max = _len \ - } - #define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len) -+#define NLA_POLICY_MAX_LEN(_len) NLA_POLICY_MAX(NLA_BINARY, _len) - - /** - * struct nl_info - netlink source information -diff --git a/include/uapi/linux/ovpn_dco.h b/include/uapi/linux/ovpn_dco.h -new file mode 100644 -index 000000000000..6afee8b3fedd ---- /dev/null -+++ b/include/uapi/linux/ovpn_dco.h -@@ -0,0 +1,265 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* -+ * OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli ++ * mas_topiary_range() - Add a range of slots to the topiary. ++ * @mas: The maple state ++ * @destroy: The topiary to add the slots (usually destroy) ++ * @start: The starting slot inclusively ++ * @end: The end slot inclusively + */ ++static inline void mas_topiary_range(struct ma_state *mas, ++ struct ma_topiary *destroy, unsigned char start, unsigned char end) ++{ ++ void __rcu **slots; ++ unsigned char offset; + -+#ifndef _UAPI_LINUX_OVPN_DCO_H_ -+#define _UAPI_LINUX_OVPN_DCO_H_ ++ MT_BUG_ON(mas->tree, mte_is_leaf(mas->node)); ++ slots = ma_slots(mas_mn(mas), mte_node_type(mas->node)); ++ for (offset = start; offset <= end; offset++) { ++ struct maple_enode *enode = mas_slot_locked(mas, slots, offset); + -+#define OVPN_NL_NAME "ovpn-dco" ++ if (mte_dead_node(enode)) ++ continue; + -+#define OVPN_NL_MULTICAST_GROUP_PEERS "peers" ++ mat_add(destroy, enode); ++ } ++} + -+/** -+ * enum ovpn_nl_commands - supported netlink commands ++/* ++ * mast_topiary() - Add the portions of the tree to the removal list; either to ++ * be freed or discarded (destroy walk). ++ * @mast: The maple_subtree_state. + */ -+enum ovpn_nl_commands { -+ /** -+ * @OVPN_CMD_UNSPEC: unspecified command to catch errors -+ */ -+ OVPN_CMD_UNSPEC = 0, ++static inline void mast_topiary(struct maple_subtree_state *mast) ++{ ++ MA_WR_STATE(wr_mas, mast->orig_l, NULL); ++ unsigned char r_start, r_end; ++ unsigned char l_start, l_end; ++ void __rcu **l_slots, **r_slots; + -+ /** -+ * @OVPN_CMD_NEW_PEER: Configure peer with its crypto keys -+ */ -+ OVPN_CMD_NEW_PEER, ++ wr_mas.type = mte_node_type(mast->orig_l->node); ++ mast->orig_l->index = mast->orig_l->last; ++ mas_wr_node_walk(&wr_mas); ++ l_start = mast->orig_l->offset + 1; ++ l_end = mas_data_end(mast->orig_l); ++ r_start = 0; ++ r_end = mast->orig_r->offset; + -+ /** -+ * @OVPN_CMD_SET_PEER: Tweak parameters for an existing peer -+ */ -+ OVPN_CMD_SET_PEER, ++ if (r_end) ++ r_end--; + -+ /** -+ * @OVPN_CMD_DEL_PEER: Remove peer from internal table -+ */ -+ OVPN_CMD_DEL_PEER, ++ l_slots = ma_slots(mas_mn(mast->orig_l), ++ mte_node_type(mast->orig_l->node)); + -+ OVPN_CMD_NEW_KEY, ++ r_slots = ma_slots(mas_mn(mast->orig_r), ++ mte_node_type(mast->orig_r->node)); + -+ OVPN_CMD_SWAP_KEYS, ++ if ((l_start < l_end) && ++ mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_start))) { ++ l_start++; ++ } + -+ OVPN_CMD_DEL_KEY, ++ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_end))) { ++ if (r_end) ++ r_end--; ++ } + -+ /** -+ * @OVPN_CMD_REGISTER_PACKET: Register for specific packet types to be -+ * forwarded to userspace -+ */ -+ OVPN_CMD_REGISTER_PACKET, ++ if ((l_start > r_end) && (mast->orig_l->node == mast->orig_r->node)) ++ return; + -+ /** -+ * @OVPN_CMD_PACKET: Send a packet from userspace to kernelspace. Also -+ * used to send to userspace packets for which a process had registered -+ * with OVPN_CMD_REGISTER_PACKET -+ */ -+ OVPN_CMD_PACKET, ++ /* At the node where left and right sides meet, add the parts between */ ++ if (mast->orig_l->node == mast->orig_r->node) { ++ return mas_topiary_range(mast->orig_l, mast->destroy, ++ l_start, r_end); ++ } + -+ /** -+ * @OVPN_CMD_GET_PEER: Retrieve the status of a peer or all peers -+ */ -+ OVPN_CMD_GET_PEER, -+}; ++ /* mast->orig_r is different and consumed. */ ++ if (mte_is_leaf(mast->orig_r->node)) ++ return; + -+enum ovpn_cipher_alg { -+ /** -+ * @OVPN_CIPHER_ALG_NONE: No encryption - reserved for debugging only -+ */ -+ OVPN_CIPHER_ALG_NONE = 0, -+ /** -+ * @OVPN_CIPHER_ALG_AES_GCM: AES-GCM AEAD cipher with any allowed key size -+ */ -+ OVPN_CIPHER_ALG_AES_GCM, -+ /** -+ * @OVPN_CIPHER_ALG_CHACHA20_POLY1305: ChaCha20Poly1305 AEAD cipher -+ */ -+ OVPN_CIPHER_ALG_CHACHA20_POLY1305, -+}; ++ if (mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_end))) ++ l_end--; + -+enum ovpn_del_peer_reason { -+ __OVPN_DEL_PEER_REASON_FIRST, -+ OVPN_DEL_PEER_REASON_TEARDOWN = __OVPN_DEL_PEER_REASON_FIRST, -+ OVPN_DEL_PEER_REASON_USERSPACE, -+ OVPN_DEL_PEER_REASON_EXPIRED, -+ OVPN_DEL_PEER_REASON_TRANSPORT_ERROR, -+ __OVPN_DEL_PEER_REASON_AFTER_LAST -+}; + -+enum ovpn_key_slot { -+ __OVPN_KEY_SLOT_FIRST, -+ OVPN_KEY_SLOT_PRIMARY = __OVPN_KEY_SLOT_FIRST, -+ OVPN_KEY_SLOT_SECONDARY, -+ __OVPN_KEY_SLOT_AFTER_LAST, -+}; ++ if (l_start <= l_end) ++ mas_topiary_range(mast->orig_l, mast->destroy, l_start, l_end); + -+enum ovpn_netlink_attrs { -+ OVPN_ATTR_UNSPEC = 0, -+ OVPN_ATTR_IFINDEX, -+ OVPN_ATTR_NEW_PEER, -+ OVPN_ATTR_SET_PEER, -+ OVPN_ATTR_DEL_PEER, -+ OVPN_ATTR_NEW_KEY, -+ OVPN_ATTR_SWAP_KEYS, -+ OVPN_ATTR_DEL_KEY, -+ OVPN_ATTR_PACKET, -+ OVPN_ATTR_GET_PEER, ++ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_start))) ++ r_start++; + -+ __OVPN_ATTR_AFTER_LAST, -+ OVPN_ATTR_MAX = __OVPN_ATTR_AFTER_LAST - 1, -+}; ++ if (r_start <= r_end) ++ mas_topiary_range(mast->orig_r, mast->destroy, 0, r_end); ++} + -+enum ovpn_netlink_key_dir_attrs { -+ OVPN_KEY_DIR_ATTR_UNSPEC = 0, -+ OVPN_KEY_DIR_ATTR_CIPHER_KEY, -+ OVPN_KEY_DIR_ATTR_NONCE_TAIL, ++/* ++ * mast_rebalance_next() - Rebalance against the next node ++ * @mast: The maple subtree state ++ * @old_r: The encoded maple node to the right (next node). ++ */ ++static inline void mast_rebalance_next(struct maple_subtree_state *mast) ++{ ++ unsigned char b_end = mast->bn->b_end; + -+ __OVPN_KEY_DIR_ATTR_AFTER_LAST, -+ OVPN_KEY_DIR_ATTR_MAX = __OVPN_KEY_DIR_ATTR_AFTER_LAST - 1, -+}; ++ mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), ++ mast->bn, b_end); ++ mast->orig_r->last = mast->orig_r->max; ++} + -+enum ovpn_netlink_new_key_attrs { -+ OVPN_NEW_KEY_ATTR_UNSPEC = 0, -+ OVPN_NEW_KEY_ATTR_PEER_ID, -+ OVPN_NEW_KEY_ATTR_KEY_SLOT, -+ OVPN_NEW_KEY_ATTR_KEY_ID, -+ OVPN_NEW_KEY_ATTR_CIPHER_ALG, -+ OVPN_NEW_KEY_ATTR_ENCRYPT_KEY, -+ OVPN_NEW_KEY_ATTR_DECRYPT_KEY, ++/* ++ * mast_rebalance_prev() - Rebalance against the previous node ++ * @mast: The maple subtree state ++ * @old_l: The encoded maple node to the left (previous node) ++ */ ++static inline void mast_rebalance_prev(struct maple_subtree_state *mast) ++{ ++ unsigned char end = mas_data_end(mast->orig_l) + 1; ++ unsigned char b_end = mast->bn->b_end; + -+ __OVPN_NEW_KEY_ATTR_AFTER_LAST, -+ OVPN_NEW_KEY_ATTR_MAX = __OVPN_NEW_KEY_ATTR_AFTER_LAST - 1, -+}; ++ mab_shift_right(mast->bn, end); ++ mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); ++ mast->l->min = mast->orig_l->min; ++ mast->orig_l->index = mast->orig_l->min; ++ mast->bn->b_end = end + b_end; ++ mast->l->offset += end; ++} + -+enum ovpn_netlink_del_key_attrs { -+ OVPN_DEL_KEY_ATTR_UNSPEC = 0, -+ OVPN_DEL_KEY_ATTR_PEER_ID, -+ OVPN_DEL_KEY_ATTR_KEY_SLOT, ++/* ++ * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring ++ * the node to the right. Checking the nodes to the right then the left at each ++ * level upwards until root is reached. Free and destroy as needed. ++ * Data is copied into the @mast->bn. ++ * @mast: The maple_subtree_state. ++ */ ++static inline ++bool mast_spanning_rebalance(struct maple_subtree_state *mast) ++{ ++ struct ma_state r_tmp = *mast->orig_r; ++ struct ma_state l_tmp = *mast->orig_l; ++ struct maple_enode *ancestor = NULL; ++ unsigned char start, end; ++ unsigned char depth = 0; + -+ __OVPN_DEL_KEY_ATTR_AFTER_LAST, -+ OVPN_DEL_KEY_ATTR_MAX = __OVPN_DEL_KEY_ATTR_AFTER_LAST - 1, -+}; ++ r_tmp = *mast->orig_r; ++ l_tmp = *mast->orig_l; ++ do { ++ mas_ascend(mast->orig_r); ++ mas_ascend(mast->orig_l); ++ depth++; ++ if (!ancestor && ++ (mast->orig_r->node == mast->orig_l->node)) { ++ ancestor = mast->orig_r->node; ++ end = mast->orig_r->offset - 1; ++ start = mast->orig_l->offset + 1; ++ } + -+enum ovpn_netlink_swap_keys_attrs { -+ OVPN_SWAP_KEYS_ATTR_UNSPEC = 0, -+ OVPN_SWAP_KEYS_ATTR_PEER_ID, ++ if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { ++ if (!ancestor) { ++ ancestor = mast->orig_r->node; ++ start = 0; ++ } + -+ __OVPN_SWAP_KEYS_ATTR_AFTER_LAST, -+ OVPN_SWAP_KEYS_ATTR_MAX = __OVPN_SWAP_KEYS_ATTR_AFTER_LAST - 1, ++ mast->orig_r->offset++; ++ do { ++ mas_descend(mast->orig_r); ++ mast->orig_r->offset = 0; ++ depth--; ++ } while (depth); + -+}; ++ mast_rebalance_next(mast); ++ do { ++ unsigned char l_off = 0; ++ struct maple_enode *child = r_tmp.node; + -+enum ovpn_netlink_new_peer_attrs { -+ OVPN_NEW_PEER_ATTR_UNSPEC = 0, -+ OVPN_NEW_PEER_ATTR_PEER_ID, -+ OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE, -+ OVPN_NEW_PEER_ATTR_SOCKET, -+ OVPN_NEW_PEER_ATTR_IPV4, -+ OVPN_NEW_PEER_ATTR_IPV6, -+ OVPN_NEW_PEER_ATTR_LOCAL_IP, ++ mas_ascend(&r_tmp); ++ if (ancestor == r_tmp.node) ++ l_off = start; + -+ __OVPN_NEW_PEER_ATTR_AFTER_LAST, -+ OVPN_NEW_PEER_ATTR_MAX = __OVPN_NEW_PEER_ATTR_AFTER_LAST - 1, -+}; ++ if (r_tmp.offset) ++ r_tmp.offset--; + -+enum ovpn_netlink_set_peer_attrs { -+ OVPN_SET_PEER_ATTR_UNSPEC = 0, -+ OVPN_SET_PEER_ATTR_PEER_ID, -+ OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT, ++ if (l_off < r_tmp.offset) ++ mas_topiary_range(&r_tmp, mast->destroy, ++ l_off, r_tmp.offset); + -+ __OVPN_SET_PEER_ATTR_AFTER_LAST, -+ OVPN_SET_PEER_ATTR_MAX = __OVPN_SET_PEER_ATTR_AFTER_LAST - 1, -+}; ++ if (l_tmp.node != child) ++ mat_add(mast->free, child); + -+enum ovpn_netlink_del_peer_attrs { -+ OVPN_DEL_PEER_ATTR_UNSPEC = 0, -+ OVPN_DEL_PEER_ATTR_REASON, -+ OVPN_DEL_PEER_ATTR_PEER_ID, ++ } while (r_tmp.node != ancestor); + -+ __OVPN_DEL_PEER_ATTR_AFTER_LAST, -+ OVPN_DEL_PEER_ATTR_MAX = __OVPN_DEL_PEER_ATTR_AFTER_LAST - 1, -+}; ++ *mast->orig_l = l_tmp; ++ return true; + -+enum ovpn_netlink_get_peer_attrs { -+ OVPN_GET_PEER_ATTR_UNSPEC = 0, -+ OVPN_GET_PEER_ATTR_PEER_ID, ++ } else if (mast->orig_l->offset != 0) { ++ if (!ancestor) { ++ ancestor = mast->orig_l->node; ++ end = mas_data_end(mast->orig_l); ++ } + -+ __OVPN_GET_PEER_ATTR_AFTER_LAST, -+ OVPN_GET_PEER_ATTR_MAX = __OVPN_GET_PEER_ATTR_AFTER_LAST - 1, -+}; ++ mast->orig_l->offset--; ++ do { ++ mas_descend(mast->orig_l); ++ mast->orig_l->offset = ++ mas_data_end(mast->orig_l); ++ depth--; ++ } while (depth); ++ ++ mast_rebalance_prev(mast); ++ do { ++ unsigned char r_off; ++ struct maple_enode *child = l_tmp.node; ++ ++ mas_ascend(&l_tmp); ++ if (ancestor == l_tmp.node) ++ r_off = end; ++ else ++ r_off = mas_data_end(&l_tmp); + -+enum ovpn_netlink_get_peer_response_attrs { -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC = 0, -+ OVPN_GET_PEER_RESP_ATTR_PEER_ID, -+ OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ OVPN_GET_PEER_RESP_ATTR_IPV4, -+ OVPN_GET_PEER_RESP_ATTR_IPV6, -+ OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, -+ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, -+ OVPN_GET_PEER_RESP_ATTR_RX_BYTES, -+ OVPN_GET_PEER_RESP_ATTR_TX_BYTES, -+ OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, -+ OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, ++ if (l_tmp.offset < r_off) ++ l_tmp.offset++; + -+ __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST, -+ OVPN_GET_PEER_RESP_ATTR_MAX = __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST - 1, -+}; ++ if (l_tmp.offset < r_off) ++ mas_topiary_range(&l_tmp, mast->destroy, ++ l_tmp.offset, r_off); + -+enum ovpn_netlink_peer_stats_attrs { -+ OVPN_PEER_STATS_ATTR_UNSPEC = 0, -+ OVPN_PEER_STATS_BYTES, -+ OVPN_PEER_STATS_PACKETS, ++ if (r_tmp.node != child) ++ mat_add(mast->free, child); + -+ __OVPN_PEER_STATS_ATTR_AFTER_LAST, -+ OVPN_PEER_STATS_ATTR_MAX = __OVPN_PEER_STATS_ATTR_AFTER_LAST - 1, -+}; ++ } while (l_tmp.node != ancestor); + -+enum ovpn_netlink_peer_attrs { -+ OVPN_PEER_ATTR_UNSPEC = 0, -+ OVPN_PEER_ATTR_PEER_ID, -+ OVPN_PEER_ATTR_SOCKADDR_REMOTE, -+ OVPN_PEER_ATTR_IPV4, -+ OVPN_PEER_ATTR_IPV6, -+ OVPN_PEER_ATTR_LOCAL_IP, -+ OVPN_PEER_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_PEER_ATTR_KEEPALIVE_TIMEOUT, -+ OVPN_PEER_ATTR_ENCRYPT_KEY, -+ OVPN_PEER_ATTR_DECRYPT_KEY, -+ OVPN_PEER_ATTR_RX_STATS, -+ OVPN_PEER_ATTR_TX_STATS, ++ *mast->orig_r = r_tmp; ++ return true; ++ } ++ } while (!mte_is_root(mast->orig_r->node)); + -+ __OVPN_PEER_ATTR_AFTER_LAST, -+ OVPN_PEER_ATTR_MAX = __OVPN_PEER_ATTR_AFTER_LAST - 1, -+}; ++ *mast->orig_r = r_tmp; ++ *mast->orig_l = l_tmp; ++ return false; ++} + -+enum ovpn_netlink_packet_attrs { -+ OVPN_PACKET_ATTR_UNSPEC = 0, -+ OVPN_PACKET_ATTR_PACKET, -+ OVPN_PACKET_ATTR_PEER_ID, ++/* ++ * mast_ascend_free() - Add current original maple state nodes to the free list ++ * and ascend. ++ * @mast: the maple subtree state. ++ * ++ * Ascend the original left and right sides and add the previous nodes to the ++ * free list. Set the slots to point to the correct location in the new nodes. ++ */ ++static inline void ++mast_ascend_free(struct maple_subtree_state *mast) ++{ ++ MA_WR_STATE(wr_mas, mast->orig_r, NULL); ++ struct maple_enode *left = mast->orig_l->node; ++ struct maple_enode *right = mast->orig_r->node; + -+ __OVPN_PACKET_ATTR_AFTER_LAST, -+ OVPN_PACKET_ATTR_MAX = __OVPN_PACKET_ATTR_AFTER_LAST - 1, -+}; ++ mas_ascend(mast->orig_l); ++ mas_ascend(mast->orig_r); ++ mat_add(mast->free, left); + -+enum ovpn_ifla_attrs { -+ IFLA_OVPN_UNSPEC = 0, -+ IFLA_OVPN_MODE, ++ if (left != right) ++ mat_add(mast->free, right); + -+ __IFLA_OVPN_AFTER_LAST, -+ IFLA_OVPN_MAX = __IFLA_OVPN_AFTER_LAST - 1, -+}; ++ mast->orig_r->offset = 0; ++ mast->orig_r->index = mast->r->max; ++ /* last should be larger than or equal to index */ ++ if (mast->orig_r->last < mast->orig_r->index) ++ mast->orig_r->last = mast->orig_r->index; ++ /* ++ * The node may not contain the value so set slot to ensure all ++ * of the nodes contents are freed or destroyed. ++ */ ++ wr_mas.type = mte_node_type(mast->orig_r->node); ++ mas_wr_node_walk(&wr_mas); ++ /* Set up the left side of things */ ++ mast->orig_l->offset = 0; ++ mast->orig_l->index = mast->l->min; ++ wr_mas.mas = mast->orig_l; ++ wr_mas.type = mte_node_type(mast->orig_l->node); ++ mas_wr_node_walk(&wr_mas); + -+enum ovpn_mode { -+ __OVPN_MODE_FIRST = 0, -+ OVPN_MODE_P2P = __OVPN_MODE_FIRST, -+ OVPN_MODE_MP, ++ mast->bn->type = wr_mas.type; ++} + -+ __OVPN_MODE_AFTER_LAST, -+}; ++/* ++ * mas_new_ma_node() - Create and return a new maple node. Helper function. ++ * @mas: the maple state with the allocations. ++ * @b_node: the maple_big_node with the type encoding. ++ * ++ * Use the node type from the maple_big_node to allocate a new node from the ++ * ma_state. This function exists mainly for code readability. ++ * ++ * Return: A new maple encoded node ++ */ ++static inline struct maple_enode ++*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) ++{ ++ return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); ++} + -+#endif /* _UAPI_LINUX_OVPN_DCO_H_ */ -diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h -index 4828794efcf8..8008c762e6b8 100644 ---- a/include/uapi/linux/udp.h -+++ b/include/uapi/linux/udp.h -@@ -43,5 +43,6 @@ struct udphdr { - #define UDP_ENCAP_GTP1U 5 /* 3GPP TS 29.060 */ - #define UDP_ENCAP_RXRPC 6 - #define TCP_ENCAP_ESPINTCP 7 /* Yikes, this is really xfrm encap types. */ -+#define UDP_ENCAP_OVPNINUDP 8 /* OpenVPN over UDP connection */ - - #endif /* _UAPI_LINUX_UDP_H */ --- -2.37.3 - -From 92568143bea4e1fe29fa24b6c871ce02fb2b00d2 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 26 Sep 2022 00:17:26 +0200 -Subject: [PATCH 05/13] lru - -Signed-off-by: Peter Jung ---- - Documentation/admin-guide/mm/index.rst | 1 + - Documentation/admin-guide/mm/multigen_lru.rst | 162 + - Documentation/mm/index.rst | 1 + - Documentation/mm/multigen_lru.rst | 159 + - arch/Kconfig | 8 + - arch/arm64/include/asm/pgtable.h | 15 +- - arch/x86/Kconfig | 1 + - arch/x86/include/asm/pgtable.h | 9 +- - arch/x86/mm/pgtable.c | 5 +- - fs/exec.c | 2 + - fs/fuse/dev.c | 3 +- - include/linux/cgroup.h | 15 +- - include/linux/memcontrol.h | 36 + - include/linux/mm.h | 5 + - include/linux/mm_inline.h | 231 +- - include/linux/mm_types.h | 76 + - include/linux/mmzone.h | 216 ++ - include/linux/nodemask.h | 1 + - include/linux/page-flags-layout.h | 16 +- - include/linux/page-flags.h | 4 +- - include/linux/pgtable.h | 17 +- - include/linux/sched.h | 4 + - include/linux/swap.h | 4 + - kernel/bounds.c | 7 + - kernel/cgroup/cgroup-internal.h | 1 - - kernel/exit.c | 1 + - kernel/fork.c | 9 + - kernel/sched/core.c | 1 + - mm/Kconfig | 26 + - mm/huge_memory.c | 3 +- - mm/internal.h | 1 + - mm/memcontrol.c | 28 + - mm/memory.c | 39 +- - mm/mm_init.c | 6 +- - mm/mmzone.c | 2 + - mm/rmap.c | 6 + - mm/swap.c | 54 +- - mm/vmscan.c | 3253 +++++++++++++++-- - mm/workingset.c | 110 +- - 39 files changed, 4252 insertions(+), 286 deletions(-) - create mode 100644 Documentation/admin-guide/mm/multigen_lru.rst - create mode 100644 Documentation/mm/multigen_lru.rst - -diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst -index 1bd11118dfb1..d1064e0ba34a 100644 ---- a/Documentation/admin-guide/mm/index.rst -+++ b/Documentation/admin-guide/mm/index.rst -@@ -32,6 +32,7 @@ the Linux memory management. - idle_page_tracking - ksm - memory-hotplug -+ multigen_lru - nommu-mmap - numa_memory_policy - numaperf -diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst -new file mode 100644 -index 000000000000..33e068830497 ---- /dev/null -+++ b/Documentation/admin-guide/mm/multigen_lru.rst -@@ -0,0 +1,162 @@ -+.. SPDX-License-Identifier: GPL-2.0 ++/* ++ * mas_mab_to_node() - Set up right and middle nodes ++ * ++ * @mas: the maple state that contains the allocations. ++ * @b_node: the node which contains the data. ++ * @left: The pointer which will have the left node ++ * @right: The pointer which may have the right node ++ * @middle: the pointer which may have the middle node (rare) ++ * @mid_split: the split location for the middle node ++ * ++ * Return: the split of left. ++ */ ++static inline unsigned char mas_mab_to_node(struct ma_state *mas, ++ struct maple_big_node *b_node, struct maple_enode **left, ++ struct maple_enode **right, struct maple_enode **middle, ++ unsigned char *mid_split, unsigned long min) ++{ ++ unsigned char split = 0; ++ unsigned char slot_count = mt_slots[b_node->type]; + -+============= -+Multi-Gen LRU -+============= -+The multi-gen LRU is an alternative LRU implementation that optimizes -+page reclaim and improves performance under memory pressure. Page -+reclaim decides the kernel's caching policy and ability to overcommit -+memory. It directly impacts the kswapd CPU usage and RAM efficiency. ++ *left = mas_new_ma_node(mas, b_node); ++ *right = NULL; ++ *middle = NULL; ++ *mid_split = 0; + -+Quick start -+=========== -+Build the kernel with the following configurations. ++ if (b_node->b_end < slot_count) { ++ split = b_node->b_end; ++ } else { ++ split = mab_calc_split(mas, b_node, mid_split, min); ++ *right = mas_new_ma_node(mas, b_node); ++ } + -+* ``CONFIG_LRU_GEN=y`` -+* ``CONFIG_LRU_GEN_ENABLED=y`` ++ if (*mid_split) ++ *middle = mas_new_ma_node(mas, b_node); + -+All set! ++ return split; + -+Runtime options -+=============== -+``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the -+following subsections. ++} + -+Kill switch -+----------- -+``enabled`` accepts different values to enable or disable the -+following components. Its default value depends on -+``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled -+unless some of them have unforeseen side effects. Writing to -+``enabled`` has no effect when a component is not supported by the -+hardware, and valid values will be accepted even when the main switch -+is off. ++/* ++ * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end ++ * pointer. ++ * @b_node - the big node to add the entry ++ * @mas - the maple state to get the pivot (mas->max) ++ * @entry - the entry to add, if NULL nothing happens. ++ */ ++static inline void mab_set_b_end(struct maple_big_node *b_node, ++ struct ma_state *mas, ++ void *entry) ++{ ++ if (!entry) ++ return; + -+====== =============================================================== -+Values Components -+====== =============================================================== -+0x0001 The main switch for the multi-gen LRU. -+0x0002 Clearing the accessed bit in leaf page table entries in large -+ batches, when MMU sets it (e.g., on x86). This behavior can -+ theoretically worsen lock contention (mmap_lock). If it is -+ disabled, the multi-gen LRU will suffer a minor performance -+ degradation for workloads that contiguously map hot pages, -+ whose accessed bits can be otherwise cleared by fewer larger -+ batches. -+0x0004 Clearing the accessed bit in non-leaf page table entries as -+ well, when MMU sets it (e.g., on x86). This behavior was not -+ verified on x86 varieties other than Intel and AMD. If it is -+ disabled, the multi-gen LRU will suffer a negligible -+ performance degradation. -+[yYnN] Apply to all the components above. -+====== =============================================================== ++ b_node->slot[b_node->b_end] = entry; ++ if (mt_is_alloc(mas->tree)) ++ b_node->gap[b_node->b_end] = mas_max_gap(mas); ++ b_node->pivot[b_node->b_end++] = mas->max; ++} + -+E.g., -+:: ++/* ++ * mas_set_split_parent() - combine_then_separate helper function. Sets the parent ++ * of @mas->node to either @left or @right, depending on @slot and @split ++ * ++ * @mas - the maple state with the node that needs a parent ++ * @left - possible parent 1 ++ * @right - possible parent 2 ++ * @slot - the slot the mas->node was placed ++ * @split - the split location between @left and @right ++ */ ++static inline void mas_set_split_parent(struct ma_state *mas, ++ struct maple_enode *left, ++ struct maple_enode *right, ++ unsigned char *slot, unsigned char split) ++{ ++ if (mas_is_none(mas)) ++ return; + -+ echo y >/sys/kernel/mm/lru_gen/enabled -+ cat /sys/kernel/mm/lru_gen/enabled -+ 0x0007 -+ echo 5 >/sys/kernel/mm/lru_gen/enabled -+ cat /sys/kernel/mm/lru_gen/enabled -+ 0x0005 ++ if ((*slot) <= split) ++ mte_set_parent(mas->node, left, *slot); ++ else if (right) ++ mte_set_parent(mas->node, right, (*slot) - split - 1); + -+Thrashing prevention -+-------------------- -+Personal computers are more sensitive to thrashing because it can -+cause janks (lags when rendering UI) and negatively impact user -+experience. The multi-gen LRU offers thrashing prevention to the -+majority of laptop and desktop users who do not have ``oomd``. ++ (*slot)++; ++} + -+Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of -+``N`` milliseconds from getting evicted. The OOM killer is triggered -+if this working set cannot be kept in memory. In other words, this -+option works as an adjustable pressure relief valve, and when open, it -+terminates applications that are hopefully not being used. ++/* ++ * mte_mid_split_check() - Check if the next node passes the mid-split ++ * @**l: Pointer to left encoded maple node. ++ * @**m: Pointer to middle encoded maple node. ++ * @**r: Pointer to right encoded maple node. ++ * @slot: The offset ++ * @*split: The split location. ++ * @mid_split: The middle split. ++ */ ++static inline void mte_mid_split_check(struct maple_enode **l, ++ struct maple_enode **r, ++ struct maple_enode *right, ++ unsigned char slot, ++ unsigned char *split, ++ unsigned char mid_split) ++{ ++ if (*r == right) ++ return; + -+Based on the average human detectable lag (~100ms), ``N=1000`` usually -+eliminates intolerable janks due to thrashing. Larger values like -+``N=3000`` make janks less noticeable at the risk of premature OOM -+kills. ++ if (slot < mid_split) ++ return; ++ ++ *l = *r; ++ *r = right; ++ *split = mid_split; ++} ++ ++/* ++ * mast_set_split_parents() - Helper function to set three nodes parents. Slot ++ * is taken from @mast->l. ++ * @mast - the maple subtree state ++ * @left - the left node ++ * @right - the right node ++ * @split - the split location. ++ */ ++static inline void mast_set_split_parents(struct maple_subtree_state *mast, ++ struct maple_enode *left, ++ struct maple_enode *middle, ++ struct maple_enode *right, ++ unsigned char split, ++ unsigned char mid_split) ++{ ++ unsigned char slot; ++ struct maple_enode *l = left; ++ struct maple_enode *r = right; ++ ++ if (mas_is_none(mast->l)) ++ return; ++ ++ if (middle) ++ r = middle; ++ ++ slot = mast->l->offset; ++ ++ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); ++ mas_set_split_parent(mast->l, l, r, &slot, split); ++ ++ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); ++ mas_set_split_parent(mast->m, l, r, &slot, split); ++ ++ mte_mid_split_check(&l, &r, right, slot, &split, mid_split); ++ mas_set_split_parent(mast->r, l, r, &slot, split); ++} ++ ++/* ++ * mas_wmb_replace() - Write memory barrier and replace ++ * @mas: The maple state ++ * @free: the maple topiary list of nodes to free ++ * @destroy: The maple topiary list of nodes to destroy (walk and free) ++ * ++ * Updates gap as necessary. ++ */ ++static inline void mas_wmb_replace(struct ma_state *mas, ++ struct ma_topiary *free, ++ struct ma_topiary *destroy) ++{ ++ /* All nodes must see old data as dead prior to replacing that data */ ++ smp_wmb(); /* Needed for RCU */ + -+The default value ``0`` means disabled. ++ /* Insert the new data in the tree */ ++ mas_replace(mas, true); + -+Experimental features -+===================== -+``/sys/kernel/debug/lru_gen`` accepts commands described in the -+following subsections. Multiple command lines are supported, so does -+concatenation with delimiters ``,`` and ``;``. ++ if (!mte_is_leaf(mas->node)) ++ mas_descend_adopt(mas); + -+``/sys/kernel/debug/lru_gen_full`` provides additional stats for -+debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from -+evicted generations in this file. ++ mas_mat_free(mas, free); + -+Working set estimation -+---------------------- -+Working set estimation measures how much memory an application needs -+in a given time interval, and it is usually done with little impact on -+the performance of the application. E.g., data centers want to -+optimize job scheduling (bin packing) to improve memory utilizations. -+When a new job comes in, the job scheduler needs to find out whether -+each server it manages can allocate a certain amount of memory for -+this new job before it can pick a candidate. To do so, the job -+scheduler needs to estimate the working sets of the existing jobs. ++ if (destroy) ++ mas_mat_destroy(mas, destroy); + -+When it is read, ``lru_gen`` returns a histogram of numbers of pages -+accessed over different time intervals for each memcg and node. -+``MAX_NR_GENS`` decides the number of bins for each histogram. The -+histograms are noncumulative. -+:: ++ if (mte_is_leaf(mas->node)) ++ return; + -+ memcg memcg_id memcg_path -+ node node_id -+ min_gen_nr age_in_ms nr_anon_pages nr_file_pages -+ ... -+ max_gen_nr age_in_ms nr_anon_pages nr_file_pages ++ mas_update_gap(mas); ++} + -+Each bin contains an estimated number of pages that have been accessed -+within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages -+and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of -+the former is the largest and that of the latter is the smallest. ++/* ++ * mast_new_root() - Set a new tree root during subtree creation ++ * @mast: The maple subtree state ++ * @mas: The maple state ++ */ ++static inline void mast_new_root(struct maple_subtree_state *mast, ++ struct ma_state *mas) ++{ ++ mas_mn(mast->l)->parent = ++ ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ if (!mte_dead_node(mast->orig_l->node) && ++ !mte_is_root(mast->orig_l->node)) { ++ do { ++ mast_ascend_free(mast); ++ mast_topiary(mast); ++ } while (!mte_is_root(mast->orig_l->node)); ++ } ++ if ((mast->orig_l->node != mas->node) && ++ (mast->l->depth > mas_mt_height(mas))) { ++ mat_add(mast->free, mas->node); ++ } ++} + -+Users can write the following command to ``lru_gen`` to create a new -+generation ``max_gen_nr+1``: ++/* ++ * mast_cp_to_nodes() - Copy data out to nodes. ++ * @mast: The maple subtree state ++ * @left: The left encoded maple node ++ * @middle: The middle encoded maple node ++ * @right: The right encoded maple node ++ * @split: The location to split between left and (middle ? middle : right) ++ * @mid_split: The location to split between middle and right. ++ */ ++static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, ++ struct maple_enode *left, struct maple_enode *middle, ++ struct maple_enode *right, unsigned char split, unsigned char mid_split) ++{ ++ bool new_lmax = true; + -+ ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` ++ mast->l->node = mte_node_or_none(left); ++ mast->m->node = mte_node_or_none(middle); ++ mast->r->node = mte_node_or_none(right); + -+``can_swap`` defaults to the swap setting and, if it is set to ``1``, -+it forces the scan of anon pages when swap is off, and vice versa. -+``force_scan`` defaults to ``1`` and, if it is set to ``0``, it -+employs heuristics to reduce the overhead, which is likely to reduce -+the coverage as well. ++ mast->l->min = mast->orig_l->min; ++ if (split == mast->bn->b_end) { ++ mast->l->max = mast->orig_r->max; ++ new_lmax = false; ++ } + -+A typical use case is that a job scheduler runs this command at a -+certain time interval to create new generations, and it ranks the -+servers it manages based on the sizes of their cold pages defined by -+this time interval. ++ mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); + -+Proactive reclaim -+----------------- -+Proactive reclaim induces page reclaim when there is no memory -+pressure. It usually targets cold pages only. E.g., when a new job -+comes in, the job scheduler wants to proactively reclaim cold pages on -+the server it selected, to improve the chance of successfully landing -+this new job. ++ if (middle) { ++ mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); ++ mast->m->min = mast->bn->pivot[split] + 1; ++ split = mid_split; ++ } + -+Users can write the following command to ``lru_gen`` to evict -+generations less than or equal to ``min_gen_nr``. ++ mast->r->max = mast->orig_r->max; ++ if (right) { ++ mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); ++ mast->r->min = mast->bn->pivot[split] + 1; ++ } ++} + -+ ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` ++/* ++ * mast_combine_cp_left - Copy in the original left side of the tree into the ++ * combined data set in the maple subtree state big node. ++ * @mast: The maple subtree state ++ */ ++static inline void mast_combine_cp_left(struct maple_subtree_state *mast) ++{ ++ unsigned char l_slot = mast->orig_l->offset; + -+``min_gen_nr`` should be less than ``max_gen_nr-1``, since -+``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to -+the active list) and therefore cannot be evicted. ``swappiness`` -+overrides the default value in ``/proc/sys/vm/swappiness``. -+``nr_to_reclaim`` limits the number of pages to evict. ++ if (!l_slot) ++ return; + -+A typical use case is that a job scheduler runs this command before it -+tries to land a new job on a server. If it fails to materialize enough -+cold pages because of the overestimation, it retries on the next -+server according to the ranking result obtained from the working set -+estimation step. This less forceful approach limits the impacts on the -+existing jobs. -diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst -index 575ccd40e30c..4aa12b8be278 100644 ---- a/Documentation/mm/index.rst -+++ b/Documentation/mm/index.rst -@@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. - ksm - memory-model - mmu_notifier -+ multigen_lru - numa - overcommit-accounting - page_migration -diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst -new file mode 100644 -index 000000000000..d7062c6a8946 ---- /dev/null -+++ b/Documentation/mm/multigen_lru.rst -@@ -0,0 +1,159 @@ -+.. SPDX-License-Identifier: GPL-2.0 ++ mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); ++} + -+============= -+Multi-Gen LRU -+============= -+The multi-gen LRU is an alternative LRU implementation that optimizes -+page reclaim and improves performance under memory pressure. Page -+reclaim decides the kernel's caching policy and ability to overcommit -+memory. It directly impacts the kswapd CPU usage and RAM efficiency. ++/* ++ * mast_combine_cp_right: Copy in the original right side of the tree into the ++ * combined data set in the maple subtree state big node. ++ * @mast: The maple subtree state ++ */ ++static inline void mast_combine_cp_right(struct maple_subtree_state *mast) ++{ ++ if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) ++ return; + -+Design overview -+=============== -+Objectives -+---------- -+The design objectives are: ++ mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, ++ mt_slot_count(mast->orig_r->node), mast->bn, ++ mast->bn->b_end); ++ mast->orig_r->last = mast->orig_r->max; ++} + -+* Good representation of access recency -+* Try to profit from spatial locality -+* Fast paths to make obvious choices -+* Simple self-correcting heuristics ++/* ++ * mast_sufficient: Check if the maple subtree state has enough data in the big ++ * node to create at least one sufficient node ++ * @mast: the maple subtree state ++ */ ++static inline bool mast_sufficient(struct maple_subtree_state *mast) ++{ ++ if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) ++ return true; + -+The representation of access recency is at the core of all LRU -+implementations. In the multi-gen LRU, each generation represents a -+group of pages with similar access recency. Generations establish a -+(time-based) common frame of reference and therefore help make better -+choices, e.g., between different memcgs on a computer or different -+computers in a data center (for job scheduling). ++ return false; ++} + -+Exploiting spatial locality improves efficiency when gathering the -+accessed bit. A rmap walk targets a single page and does not try to -+profit from discovering a young PTE. A page table walk can sweep all -+the young PTEs in an address space, but the address space can be too -+sparse to make a profit. The key is to optimize both methods and use -+them in combination. ++/* ++ * mast_overflow: Check if there is too much data in the subtree state for a ++ * single node. ++ * @mast: The maple subtree state ++ */ ++static inline bool mast_overflow(struct maple_subtree_state *mast) ++{ ++ if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node)) ++ return true; + -+Fast paths reduce code complexity and runtime overhead. Unmapped pages -+do not require TLB flushes; clean pages do not require writeback. -+These facts are only helpful when other conditions, e.g., access -+recency, are similar. With generations as a common frame of reference, -+additional factors stand out. But obvious choices might not be good -+choices; thus self-correction is necessary. ++ return false; ++} + -+The benefits of simple self-correcting heuristics are self-evident. -+Again, with generations as a common frame of reference, this becomes -+attainable. Specifically, pages in the same generation can be -+categorized based on additional factors, and a feedback loop can -+statistically compare the refault percentages across those categories -+and infer which of them are better choices. ++static inline void *mtree_range_walk(struct ma_state *mas) ++{ ++ unsigned long *pivots; ++ unsigned char offset; ++ struct maple_node *node; ++ struct maple_enode *next, *last; ++ enum maple_type type; ++ void __rcu **slots; ++ unsigned char end; ++ unsigned long max, min; ++ unsigned long prev_max, prev_min; + -+Assumptions -+----------- -+The protection of hot pages and the selection of cold pages are based -+on page access channels and patterns. There are two access channels: ++ last = next = mas->node; ++ prev_min = min = mas->min; ++ max = mas->max; ++ do { ++ offset = 0; ++ last = next; ++ node = mte_to_node(next); ++ type = mte_node_type(next); ++ pivots = ma_pivots(node, type); ++ end = ma_data_end(node, type, pivots, max); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ ++ if (pivots[offset] >= mas->index) { ++ prev_max = max; ++ prev_min = min; ++ max = pivots[offset]; ++ goto next; ++ } + -+* Accesses through page tables -+* Accesses through file descriptors ++ do { ++ offset++; ++ } while ((offset < end) && (pivots[offset] < mas->index)); + -+The protection of the former channel is by design stronger because: ++ prev_min = min; ++ min = pivots[offset - 1] + 1; ++ prev_max = max; ++ if (likely(offset < end && pivots[offset])) ++ max = pivots[offset]; + -+1. The uncertainty in determining the access patterns of the former -+ channel is higher due to the approximation of the accessed bit. -+2. The cost of evicting the former channel is higher due to the TLB -+ flushes required and the likelihood of encountering the dirty bit. -+3. The penalty of underprotecting the former channel is higher because -+ applications usually do not prepare themselves for major page -+ faults like they do for blocked I/O. E.g., GUI applications -+ commonly use dedicated I/O threads to avoid blocking rendering -+ threads. ++next: ++ slots = ma_slots(node, type); ++ next = mt_slot(mas->tree, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ } while (!ma_is_leaf(type)); ++ ++ mas->offset = offset; ++ mas->index = min; ++ mas->last = max; ++ mas->min = prev_min; ++ mas->max = prev_max; ++ mas->node = last; ++ return (void *) next; ++ ++dead_node: ++ mas_reset(mas); ++ return NULL; ++} + -+There are also two access patterns: ++/* ++ * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. ++ * @mas: The starting maple state ++ * @mast: The maple_subtree_state, keeps track of 4 maple states. ++ * @count: The estimated count of iterations needed. ++ * ++ * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root ++ * is hit. First @b_node is split into two entries which are inserted into the ++ * next iteration of the loop. @b_node is returned populated with the final ++ * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the ++ * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last ++ * to account of what has been copied into the new sub-tree. The update of ++ * orig_l_mas->last is used in mas_consume to find the slots that will need to ++ * be either freed or destroyed. orig_l_mas->depth keeps track of the height of ++ * the new sub-tree in case the sub-tree becomes the full tree. ++ * ++ * Return: the number of elements in b_node during the last loop. ++ */ ++static int mas_spanning_rebalance(struct ma_state *mas, ++ struct maple_subtree_state *mast, unsigned char count) ++{ ++ unsigned char split, mid_split; ++ unsigned char slot = 0; ++ struct maple_enode *left = NULL, *middle = NULL, *right = NULL; + -+* Accesses exhibiting temporal locality -+* Accesses not exhibiting temporal locality ++ MA_STATE(l_mas, mas->tree, mas->index, mas->index); ++ MA_STATE(r_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(m_mas, mas->tree, mas->index, mas->index); ++ MA_TOPIARY(free, mas->tree); ++ MA_TOPIARY(destroy, mas->tree); + -+For the reasons listed above, the former channel is assumed to follow -+the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is -+present, and the latter channel is assumed to follow the latter -+pattern unless outlying refaults have been observed. ++ /* ++ * The tree needs to be rebalanced and leaves need to be kept at the same level. ++ * Rebalancing is done by use of the ``struct maple_topiary``. ++ */ ++ mast->l = &l_mas; ++ mast->m = &m_mas; ++ mast->r = &r_mas; ++ mast->free = &free; ++ mast->destroy = &destroy; ++ l_mas.node = r_mas.node = m_mas.node = MAS_NONE; ++ if (!(mast->orig_l->min && mast->orig_r->max == ULONG_MAX) && ++ unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) ++ mast_spanning_rebalance(mast); ++ ++ mast->orig_l->depth = 0; + -+Workflow overview -+================= -+Evictable pages are divided into multiple generations for each -+``lruvec``. The youngest generation number is stored in -+``lrugen->max_seq`` for both anon and file types as they are aged on -+an equal footing. The oldest generation numbers are stored in -+``lrugen->min_seq[]`` separately for anon and file types as clean file -+pages can be evicted regardless of swap constraints. These three -+variables are monotonically increasing. ++ /* ++ * Each level of the tree is examined and balanced, pushing data to the left or ++ * right, or rebalancing against left or right nodes is employed to avoid ++ * rippling up the tree to limit the amount of churn. Once a new sub-section of ++ * the tree is created, there may be a mix of new and old nodes. The old nodes ++ * will have the incorrect parent pointers and currently be in two trees: the ++ * original tree and the partially new tree. To remedy the parent pointers in ++ * the old tree, the new data is swapped into the active tree and a walk down ++ * the tree is performed and the parent pointers are updated. ++ * See mas_descend_adopt() for more information.. ++ */ ++ while (count--) { ++ mast->bn->b_end--; ++ mast->bn->type = mte_node_type(mast->orig_l->node); ++ split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle, ++ &mid_split, mast->orig_l->min); ++ mast_set_split_parents(mast, left, middle, right, split, ++ mid_split); ++ mast_cp_to_nodes(mast, left, middle, right, split, mid_split); + -+Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` -+bits in order to fit into the gen counter in ``folio->flags``. Each -+truncated generation number is an index to ``lrugen->lists[]``. The -+sliding window technique is used to track at least ``MIN_NR_GENS`` and -+at most ``MAX_NR_GENS`` generations. The gen counter stores a value -+within ``[1, MAX_NR_GENS]`` while a page is on one of -+``lrugen->lists[]``; otherwise it stores zero. ++ /* ++ * Copy data from next level in the tree to mast->bn from next ++ * iteration ++ */ ++ memset(mast->bn, 0, sizeof(struct maple_big_node)); ++ mast->bn->type = mte_node_type(left); ++ mast->orig_l->depth++; ++ ++ /* Root already stored in l->node. */ ++ if (mas_is_root_limits(mast->l)) ++ goto new_root; ++ ++ mast_ascend_free(mast); ++ mast_combine_cp_left(mast); ++ l_mas.offset = mast->bn->b_end; ++ mab_set_b_end(mast->bn, &l_mas, left); ++ mab_set_b_end(mast->bn, &m_mas, middle); ++ mab_set_b_end(mast->bn, &r_mas, right); ++ ++ /* Copy anything necessary out of the right node. */ ++ mast_combine_cp_right(mast); ++ mast_topiary(mast); ++ mast->orig_l->last = mast->orig_l->max; ++ ++ if (mast_sufficient(mast)) ++ continue; + -+Each generation is divided into multiple tiers. A page accessed ``N`` -+times through file descriptors is in tier ``order_base_2(N)``. Unlike -+generations, tiers do not have dedicated ``lrugen->lists[]``. In -+contrast to moving across generations, which requires the LRU lock, -+moving across tiers only involves atomic operations on -+``folio->flags`` and therefore has a negligible cost. A feedback loop -+modeled after the PID controller monitors refaults over all the tiers -+from anon and file types and decides which tiers from which types to -+evict or protect. ++ if (mast_overflow(mast)) ++ continue; + -+There are two conceptually independent procedures: the aging and the -+eviction. They form a closed-loop system, i.e., the page reclaim. ++ /* May be a new root stored in mast->bn */ ++ if (mas_is_root_limits(mast->orig_l)) ++ break; + -+Aging -+----- -+The aging produces young generations. Given an ``lruvec``, it -+increments ``max_seq`` when ``max_seq-min_seq+1`` approaches -+``MIN_NR_GENS``. The aging promotes hot pages to the youngest -+generation when it finds them accessed through page tables; the -+demotion of cold pages happens consequently when it increments -+``max_seq``. The aging uses page table walks and rmap walks to find -+young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` -+and calls ``walk_page_range()`` with each ``mm_struct`` on this list -+to scan PTEs, and after each iteration, it increments ``max_seq``. For -+the latter, when the eviction walks the rmap and finds a young PTE, -+the aging scans the adjacent PTEs. For both, on finding a young PTE, -+the aging clears the accessed bit and updates the gen counter of the -+page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. ++ mast_spanning_rebalance(mast); + -+Eviction -+-------- -+The eviction consumes old generations. Given an ``lruvec``, it -+increments ``min_seq`` when ``lrugen->lists[]`` indexed by -+``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to -+evict from, it first compares ``min_seq[]`` to select the older type. -+If both types are equally old, it selects the one whose first tier has -+a lower refault percentage. The first tier contains single-use -+unmapped clean pages, which are the best bet. The eviction sorts a -+page according to its gen counter if the aging has found this page -+accessed through page tables and updated its gen counter. It also -+moves a page to the next generation, i.e., ``min_seq+1``, if this page -+was accessed multiple times through file descriptors and the feedback -+loop has detected outlying refaults from the tier this page is in. To -+this end, the feedback loop uses the first tier as the baseline, for -+the reason stated earlier. ++ /* rebalancing from other nodes may require another loop. */ ++ if (!count) ++ count++; ++ } + -+Summary -+------- -+The multi-gen LRU can be disassembled into the following parts: ++ l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), ++ mte_node_type(mast->orig_l->node)); ++ mast->orig_l->depth++; ++ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); ++ mte_set_parent(left, l_mas.node, slot); ++ if (middle) ++ mte_set_parent(middle, l_mas.node, ++slot); + -+* Generations -+* Rmap walks -+* Page table walks -+* Bloom filters -+* PID controller ++ if (right) ++ mte_set_parent(right, l_mas.node, ++slot); + -+The aging and the eviction form a producer-consumer model; -+specifically, the latter drives the former by the sliding window over -+generations. Within the aging, rmap walks drive page table walks by -+inserting hot densely populated page tables to the Bloom filters. -+Within the eviction, the PID controller uses refaults as the feedback -+to select types to evict and tiers to protect. -diff --git a/arch/Kconfig b/arch/Kconfig -index 8b311e400ec1..bf19a84fffa2 100644 ---- a/arch/Kconfig -+++ b/arch/Kconfig -@@ -1418,6 +1418,14 @@ config DYNAMIC_SIGFRAME - config HAVE_ARCH_NODE_DEV_GROUP - bool - -+config ARCH_HAS_NONLEAF_PMD_YOUNG -+ bool -+ help -+ Architectures that select this option are capable of setting the -+ accessed bit in non-leaf PMD entries when using them as part of linear -+ address translations. Page table walkers that clear the accessed bit -+ may use this capability to reduce their search space. ++ if (mas_is_root_limits(mast->l)) { ++new_root: ++ mast_new_root(mast, mas); ++ } else { ++ mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; ++ } + - source "kernel/gcov/Kconfig" - - source "scripts/gcc-plugins/Kconfig" -diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h -index b5df82aa99e6..71a1af42f0e8 100644 ---- a/arch/arm64/include/asm/pgtable.h -+++ b/arch/arm64/include/asm/pgtable.h -@@ -1082,24 +1082,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, - * page after fork() + CoW for pfn mappings. We don't always have a - * hardware-managed access flag on arm64. - */ --static inline bool arch_faults_on_old_pte(void) --{ -- /* The register read below requires a stable CPU to make any sense */ -- cant_migrate(); -- -- return !cpu_has_hw_af(); --} --#define arch_faults_on_old_pte arch_faults_on_old_pte -+#define arch_has_hw_pte_young cpu_has_hw_af - - /* - * Experimentally, it's cheap to set the access flag in hardware and we - * benefit from prefaulting mappings as 'old' to start with. - */ --static inline bool arch_wants_old_prefaulted_pte(void) --{ -- return !arch_faults_on_old_pte(); --} --#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte -+#define arch_wants_old_prefaulted_pte cpu_has_hw_af - - static inline bool pud_sect_supported(void) - { -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index f9920f1341c8..674d694a665e 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -85,6 +85,7 @@ config X86 - select ARCH_HAS_PMEM_API if X86_64 - select ARCH_HAS_PTE_DEVMAP if X86_64 - select ARCH_HAS_PTE_SPECIAL -+ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 - select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 - select ARCH_HAS_COPY_MC if X86_64 - select ARCH_HAS_SET_MEMORY -diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h -index 44e2d6f1dbaa..5059799bebe3 100644 ---- a/arch/x86/include/asm/pgtable.h -+++ b/arch/x86/include/asm/pgtable.h -@@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) - - static inline int pmd_bad(pmd_t pmd) - { -- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; -+ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != -+ (_KERNPG_TABLE & ~_PAGE_ACCESSED); - } - - static inline unsigned long pages_to_mb(unsigned long npg) -@@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) - return boot_cpu_has_bug(X86_BUG_L1TF); - } - --#define arch_faults_on_old_pte arch_faults_on_old_pte --static inline bool arch_faults_on_old_pte(void) -+#define arch_has_hw_pte_young arch_has_hw_pte_young -+static inline bool arch_has_hw_pte_young(void) - { -- return false; -+ return true; - } - - #ifdef CONFIG_PAGE_TABLE_CHECK -diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c -index a932d7712d85..8525f2876fb4 100644 ---- a/arch/x86/mm/pgtable.c -+++ b/arch/x86/mm/pgtable.c -@@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, - return ret; - } - --#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) - int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pmd_t *pmdp) - { -@@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, - - return ret; - } -+#endif ++ if (!mte_dead_node(mast->orig_l->node)) ++ mat_add(&free, mast->orig_l->node); + -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE - int pudp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long addr, pud_t *pudp) - { -diff --git a/fs/exec.c b/fs/exec.c -index d046dbb9cbd0..c67b12f0f577 100644 ---- a/fs/exec.c -+++ b/fs/exec.c -@@ -1011,6 +1011,7 @@ static int exec_mmap(struct mm_struct *mm) - active_mm = tsk->active_mm; - tsk->active_mm = mm; - tsk->mm = mm; -+ lru_gen_add_mm(mm); - /* - * This prevents preemption while active_mm is being loaded and - * it and mm are being updated, which could cause problems for -@@ -1026,6 +1027,7 @@ static int exec_mmap(struct mm_struct *mm) - tsk->mm->vmacache_seqnum = 0; - vmacache_flush(tsk); - task_unlock(tsk); -+ lru_gen_use_mm(mm); - if (old_mm) { - mmap_read_unlock(old_mm); - BUG_ON(active_mm != old_mm); -diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c -index 51897427a534..b4a6e0a1b945 100644 ---- a/fs/fuse/dev.c -+++ b/fs/fuse/dev.c -@@ -776,7 +776,8 @@ static int fuse_check_page(struct page *page) - 1 << PG_active | - 1 << PG_workingset | - 1 << PG_reclaim | -- 1 << PG_waiters))) { -+ 1 << PG_waiters | -+ LRU_GEN_MASK | LRU_REFS_MASK))) { - dump_page(page, "fuse: trying to steal weird page"); - return 1; - } -diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h -index ac5d0515680e..9179463c3c9f 100644 ---- a/include/linux/cgroup.h -+++ b/include/linux/cgroup.h -@@ -432,6 +432,18 @@ static inline void cgroup_put(struct cgroup *cgrp) - css_put(&cgrp->self); - } - -+extern struct mutex cgroup_mutex; ++ mas->depth = mast->orig_l->depth; ++ *mast->orig_l = l_mas; ++ mte_set_node_dead(mas->node); + -+static inline void cgroup_lock(void) -+{ -+ mutex_lock(&cgroup_mutex); ++ /* Set up mas for insertion. */ ++ mast->orig_l->depth = mas->depth; ++ mast->orig_l->alloc = mas->alloc; ++ *mas = *mast->orig_l; ++ mas_wmb_replace(mas, &free, &destroy); ++ mtree_range_walk(mas); ++ return mast->bn->b_end; +} + -+static inline void cgroup_unlock(void) ++/* ++ * mas_rebalance() - Rebalance a given node. ++ * @mas: The maple state ++ * @b_node: The big maple node. ++ * ++ * Rebalance two nodes into a single node or two new nodes that are sufficient. ++ * Continue upwards until tree is sufficient. ++ * ++ * Return: the number of elements in b_node during the last loop. ++ */ ++static inline int mas_rebalance(struct ma_state *mas, ++ struct maple_big_node *b_node) +{ -+ mutex_unlock(&cgroup_mutex); -+} ++ char empty_count = mas_mt_height(mas); ++ struct maple_subtree_state mast; ++ unsigned char shift, b_end = ++b_node->b_end; + - /** - * task_css_set_check - obtain a task's css_set with extra access conditions - * @task: the task to obtain css_set for -@@ -446,7 +458,6 @@ static inline void cgroup_put(struct cgroup *cgrp) - * as locks used during the cgroup_subsys::attach() methods. - */ - #ifdef CONFIG_PROVE_RCU --extern struct mutex cgroup_mutex; - extern spinlock_t css_set_lock; - #define task_css_set_check(task, __c) \ - rcu_dereference_check((task)->cgroups, \ -@@ -708,6 +719,8 @@ struct cgroup; - static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } - static inline void css_get(struct cgroup_subsys_state *css) {} - static inline void css_put(struct cgroup_subsys_state *css) {} -+static inline void cgroup_lock(void) {} -+static inline void cgroup_unlock(void) {} - static inline int cgroup_attach_task_all(struct task_struct *from, - struct task_struct *t) { return 0; } - static inline int cgroupstats_build(struct cgroupstats *stats, -diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h -index 6257867fbf95..207cfd3b42e5 100644 ---- a/include/linux/memcontrol.h -+++ b/include/linux/memcontrol.h -@@ -350,6 +350,11 @@ struct mem_cgroup { - struct deferred_split deferred_split_queue; - #endif - -+#ifdef CONFIG_LRU_GEN -+ /* per-memcg mm_struct list */ -+ struct lru_gen_mm_list mm_list; -+#endif ++ MA_STATE(l_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(r_mas, mas->tree, mas->index, mas->last); + - struct mem_cgroup_per_node *nodeinfo[]; - }; - -@@ -444,6 +449,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - * - * For a kmem folio a caller should hold an rcu read lock to protect memcg - * associated with a kmem folio from being released. -@@ -505,6 +511,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - * - * For a kmem page a caller should hold an rcu read lock to protect memcg - * associated with a kmem page from being released. -@@ -959,6 +966,23 @@ void unlock_page_memcg(struct page *page); - - void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); - -+/* try to stablize folio_memcg() for all the pages in a memcg */ -+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) -+{ -+ rcu_read_lock(); ++ trace_ma_op(__func__, mas); ++ ++ /* ++ * Rebalancing occurs if a node is insufficient. Data is rebalanced ++ * against the node to the right if it exists, otherwise the node to the ++ * left of this node is rebalanced against this node. If rebalancing ++ * causes just one node to be produced instead of two, then the parent ++ * is also examined and rebalanced if it is insufficient. Every level ++ * tries to combine the data in the same way. If one node contains the ++ * entire range of the tree, then that node is used as a new root node. ++ */ ++ mas_node_count(mas, 1 + empty_count * 3); ++ if (mas_is_err(mas)) ++ return 0; ++ ++ mast.orig_l = &l_mas; ++ mast.orig_r = &r_mas; ++ mast.bn = b_node; ++ mast.bn->type = mte_node_type(mas->node); ++ ++ l_mas = r_mas = *mas; + -+ if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) -+ return true; ++ if (mas_next_sibling(&r_mas)) { ++ mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); ++ r_mas.last = r_mas.index = r_mas.max; ++ } else { ++ mas_prev_sibling(&l_mas); ++ shift = mas_data_end(&l_mas) + 1; ++ mab_shift_right(b_node, shift); ++ mas->offset += shift; ++ mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); ++ b_node->b_end = shift + b_end; ++ l_mas.index = l_mas.last = l_mas.min; ++ } + -+ rcu_read_unlock(); -+ return false; ++ return mas_spanning_rebalance(mas, &mast, empty_count); +} + -+static inline void mem_cgroup_unlock_pages(void) -+{ -+ rcu_read_unlock(); ++/* ++ * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple ++ * state. ++ * @mas: The maple state ++ * @end: The end of the left-most node. ++ * ++ * During a mass-insert event (such as forking), it may be necessary to ++ * rebalance the left-most node when it is not sufficient. ++ */ ++static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end) ++{ ++ enum maple_type mt = mte_node_type(mas->node); ++ struct maple_node reuse, *newnode, *parent, *new_left, *left, *node; ++ struct maple_enode *eparent; ++ unsigned char offset, tmp, split = mt_slots[mt] / 2; ++ void __rcu **l_slots, **slots; ++ unsigned long *l_pivs, *pivs, gap; ++ bool in_rcu = mt_in_rcu(mas->tree); ++ ++ MA_STATE(l_mas, mas->tree, mas->index, mas->last); ++ ++ l_mas = *mas; ++ mas_prev_sibling(&l_mas); ++ ++ /* set up node. */ ++ if (in_rcu) { ++ /* Allocate for both left and right as well as parent. */ ++ mas_node_count(mas, 3); ++ if (mas_is_err(mas)) ++ return; ++ ++ newnode = mas_pop_node(mas); ++ } else { ++ newnode = &reuse; ++ } ++ ++ node = mas_mn(mas); ++ newnode->parent = node->parent; ++ slots = ma_slots(newnode, mt); ++ pivs = ma_pivots(newnode, mt); ++ left = mas_mn(&l_mas); ++ l_slots = ma_slots(left, mt); ++ l_pivs = ma_pivots(left, mt); ++ if (!l_slots[split]) ++ split++; ++ tmp = mas_data_end(&l_mas) - split; ++ ++ memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp); ++ memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp); ++ pivs[tmp] = l_mas.max; ++ memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end); ++ memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end); ++ ++ l_mas.max = l_pivs[split]; ++ mas->min = l_mas.max + 1; ++ eparent = mt_mk_node(mte_parent(l_mas.node), ++ mas_parent_enum(&l_mas, l_mas.node)); ++ tmp += end; ++ if (!in_rcu) { ++ unsigned char max_p = mt_pivots[mt]; ++ unsigned char max_s = mt_slots[mt]; ++ ++ if (tmp < max_p) ++ memset(pivs + tmp, 0, ++ sizeof(unsigned long *) * (max_p - tmp)); ++ ++ if (tmp < mt_slots[mt]) ++ memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp)); ++ ++ memcpy(node, newnode, sizeof(struct maple_node)); ++ ma_set_meta(node, mt, 0, tmp - 1); ++ mte_set_pivot(eparent, mte_parent_slot(l_mas.node), ++ l_pivs[split]); ++ ++ /* Remove data from l_pivs. */ ++ tmp = split + 1; ++ memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); ++ memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); ++ ma_set_meta(left, mt, 0, split); ++ ++ goto done; ++ } ++ ++ /* RCU requires replacing both l_mas, mas, and parent. */ ++ mas->node = mt_mk_node(newnode, mt); ++ ma_set_meta(newnode, mt, 0, tmp); ++ ++ new_left = mas_pop_node(mas); ++ new_left->parent = left->parent; ++ mt = mte_node_type(l_mas.node); ++ slots = ma_slots(new_left, mt); ++ pivs = ma_pivots(new_left, mt); ++ memcpy(slots, l_slots, sizeof(void *) * split); ++ memcpy(pivs, l_pivs, sizeof(unsigned long) * split); ++ ma_set_meta(new_left, mt, 0, split); ++ l_mas.node = mt_mk_node(new_left, mt); ++ ++ /* replace parent. */ ++ offset = mte_parent_slot(mas->node); ++ mt = mas_parent_enum(&l_mas, l_mas.node); ++ parent = mas_pop_node(mas); ++ slots = ma_slots(parent, mt); ++ pivs = ma_pivots(parent, mt); ++ memcpy(parent, mte_to_node(eparent), sizeof(struct maple_node)); ++ rcu_assign_pointer(slots[offset], mas->node); ++ rcu_assign_pointer(slots[offset - 1], l_mas.node); ++ pivs[offset - 1] = l_mas.max; ++ eparent = mt_mk_node(parent, mt); ++done: ++ gap = mas_leaf_max_gap(mas); ++ mte_set_gap(eparent, mte_parent_slot(mas->node), gap); ++ gap = mas_leaf_max_gap(&l_mas); ++ mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap); ++ mas_ascend(mas); ++ ++ if (in_rcu) ++ mas_replace(mas, false); ++ ++ mas_update_gap(mas); +} + - /* idx can be of type enum memcg_stat_item or node_stat_item */ - static inline void mod_memcg_state(struct mem_cgroup *memcg, - int idx, int val) -@@ -1433,6 +1457,18 @@ static inline void folio_memcg_unlock(struct folio *folio) - { - } - -+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) ++/* ++ * mas_split_final_node() - Split the final node in a subtree operation. ++ * @mast: the maple subtree state ++ * @mas: The maple state ++ * @height: The height of the tree in case it's a new root. ++ */ ++static inline bool mas_split_final_node(struct maple_subtree_state *mast, ++ struct ma_state *mas, int height) +{ -+ /* to match folio_memcg_rcu() */ -+ rcu_read_lock(); ++ struct maple_enode *ancestor; ++ ++ if (mte_is_root(mas->node)) { ++ if (mt_is_alloc(mas->tree)) ++ mast->bn->type = maple_arange_64; ++ else ++ mast->bn->type = maple_range_64; ++ mas->depth = height; ++ } ++ /* ++ * Only a single node is used here, could be root. ++ * The Big_node data should just fit in a single node. ++ */ ++ ancestor = mas_new_ma_node(mas, mast->bn); ++ mte_set_parent(mast->l->node, ancestor, mast->l->offset); ++ mte_set_parent(mast->r->node, ancestor, mast->r->offset); ++ mte_to_node(ancestor)->parent = mas_mn(mas)->parent; ++ ++ mast->l->node = ancestor; ++ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); ++ mas->offset = mast->bn->b_end - 1; + return true; +} + -+static inline void mem_cgroup_unlock_pages(void) ++/* ++ * mast_fill_bnode() - Copy data into the big node in the subtree state ++ * @mast: The maple subtree state ++ * @mas: the maple state ++ * @skip: The number of entries to skip for new nodes insertion. ++ */ ++static inline void mast_fill_bnode(struct maple_subtree_state *mast, ++ struct ma_state *mas, ++ unsigned char skip) +{ -+ rcu_read_unlock(); ++ bool cp = true; ++ struct maple_enode *old = mas->node; ++ unsigned char split; ++ ++ memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap)); ++ memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot)); ++ memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot)); ++ mast->bn->b_end = 0; ++ ++ if (mte_is_root(mas->node)) { ++ cp = false; ++ } else { ++ mas_ascend(mas); ++ mat_add(mast->free, old); ++ mas->offset = mte_parent_slot(mas->node); ++ } ++ ++ if (cp && mast->l->offset) ++ mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); ++ ++ split = mast->bn->b_end; ++ mab_set_b_end(mast->bn, mast->l, mast->l->node); ++ mast->r->offset = mast->bn->b_end; ++ mab_set_b_end(mast->bn, mast->r, mast->r->node); ++ if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) ++ cp = false; ++ ++ if (cp) ++ mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, ++ mast->bn, mast->bn->b_end); ++ ++ mast->bn->b_end--; ++ mast->bn->type = mte_node_type(mas->node); +} + - static inline void mem_cgroup_handle_over_high(void) - { - } -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 21f8b27bd9fd..88976a521ef5 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -1465,6 +1465,11 @@ static inline unsigned long folio_pfn(struct folio *folio) - return page_to_pfn(&folio->page); - } - -+static inline struct folio *pfn_folio(unsigned long pfn) ++/* ++ * mast_split_data() - Split the data in the subtree state big node into regular ++ * nodes. ++ * @mast: The maple subtree state ++ * @mas: The maple state ++ * @split: The location to split the big node ++ */ ++static inline void mast_split_data(struct maple_subtree_state *mast, ++ struct ma_state *mas, unsigned char split) ++{ ++ unsigned char p_slot; ++ ++ mab_mas_cp(mast->bn, 0, split, mast->l, true); ++ mte_set_pivot(mast->r->node, 0, mast->r->max); ++ mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); ++ mast->l->offset = mte_parent_slot(mas->node); ++ mast->l->max = mast->bn->pivot[split]; ++ mast->r->min = mast->l->max + 1; ++ if (mte_is_leaf(mas->node)) ++ return; ++ ++ p_slot = mast->orig_l->offset; ++ mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, ++ &p_slot, split); ++ mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, ++ &p_slot, split); ++} ++ ++/* ++ * mas_push_data() - Instead of splitting a node, it is beneficial to push the ++ * data to the right or left node if there is room. ++ * @mas: The maple state ++ * @height: The current height of the maple state ++ * @mast: The maple subtree state ++ * @left: Push left or not. ++ * ++ * Keeping the height of the tree low means faster lookups. ++ * ++ * Return: True if pushed, false otherwise. ++ */ ++static inline bool mas_push_data(struct ma_state *mas, int height, ++ struct maple_subtree_state *mast, bool left) +{ -+ return page_folio(pfn_to_page(pfn)); ++ unsigned char slot_total = mast->bn->b_end; ++ unsigned char end, space, split; ++ ++ MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); ++ tmp_mas = *mas; ++ tmp_mas.depth = mast->l->depth; ++ ++ if (left && !mas_prev_sibling(&tmp_mas)) ++ return false; ++ else if (!left && !mas_next_sibling(&tmp_mas)) ++ return false; ++ ++ end = mas_data_end(&tmp_mas); ++ slot_total += end; ++ space = 2 * mt_slot_count(mas->node) - 2; ++ /* -2 instead of -1 to ensure there isn't a triple split */ ++ if (ma_is_leaf(mast->bn->type)) ++ space--; ++ ++ if (mas->max == ULONG_MAX) ++ space--; ++ ++ if (slot_total >= space) ++ return false; ++ ++ /* Get the data; Fill mast->bn */ ++ mast->bn->b_end++; ++ if (left) { ++ mab_shift_right(mast->bn, end + 1); ++ mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); ++ mast->bn->b_end = slot_total + 1; ++ } else { ++ mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); ++ } ++ ++ /* Configure mast for splitting of mast->bn */ ++ split = mt_slots[mast->bn->type] - 2; ++ if (left) { ++ /* Switch mas to prev node */ ++ mat_add(mast->free, mas->node); ++ *mas = tmp_mas; ++ /* Start using mast->l for the left side. */ ++ tmp_mas.node = mast->l->node; ++ *mast->l = tmp_mas; ++ } else { ++ mat_add(mast->free, tmp_mas.node); ++ tmp_mas.node = mast->r->node; ++ *mast->r = tmp_mas; ++ split = slot_total - split; ++ } ++ split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); ++ /* Update parent slot for split calculation. */ ++ if (left) ++ mast->orig_l->offset += end + 1; ++ ++ mast_split_data(mast, mas, split); ++ mast_fill_bnode(mast, mas, 2); ++ mas_split_final_node(mast, mas, height + 1); ++ return true; +} + - static inline atomic_t *folio_pincount_ptr(struct folio *folio) - { - return &folio_page(folio, 1)->compound_pincount; -diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h -index 7b25b53c474a..4949eda9a9a2 100644 ---- a/include/linux/mm_inline.h -+++ b/include/linux/mm_inline.h -@@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) - return folio_is_file_lru(page_folio(page)); - } - --static __always_inline void update_lru_size(struct lruvec *lruvec, -+static __always_inline void __update_lru_size(struct lruvec *lruvec, - enum lru_list lru, enum zone_type zid, - long nr_pages) - { - struct pglist_data *pgdat = lruvec_pgdat(lruvec); - -+ lockdep_assert_held(&lruvec->lru_lock); -+ WARN_ON_ONCE(nr_pages != (int)nr_pages); ++/* ++ * mas_split() - Split data that is too big for one node into two. ++ * @mas: The maple state ++ * @b_node: The maple big node ++ * Return: 1 on success, 0 on failure. ++ */ ++static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) ++{ + - __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); - __mod_zone_page_state(&pgdat->node_zones[zid], - NR_ZONE_LRU_BASE + lru, nr_pages); ++ struct maple_subtree_state mast; ++ int height = 0; ++ unsigned char mid_split, split = 0; ++ ++ /* ++ * Splitting is handled differently from any other B-tree; the Maple ++ * Tree splits upwards. Splitting up means that the split operation ++ * occurs when the walk of the tree hits the leaves and not on the way ++ * down. The reason for splitting up is that it is impossible to know ++ * how much space will be needed until the leaf is (or leaves are) ++ * reached. Since overwriting data is allowed and a range could ++ * overwrite more than one range or result in changing one entry into 3 ++ * entries, it is impossible to know if a split is required until the ++ * data is examined. ++ * ++ * Splitting is a balancing act between keeping allocations to a minimum ++ * and avoiding a 'jitter' event where a tree is expanded to make room ++ * for an entry followed by a contraction when the entry is removed. To ++ * accomplish the balance, there are empty slots remaining in both left ++ * and right nodes after a split. ++ */ ++ MA_STATE(l_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(r_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); ++ MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); ++ MA_TOPIARY(mat, mas->tree); ++ ++ trace_ma_op(__func__, mas); ++ mas->depth = mas_mt_height(mas); ++ /* Allocation failures will happen early. */ ++ mas_node_count(mas, 1 + mas->depth * 2); ++ if (mas_is_err(mas)) ++ return 0; ++ ++ mast.l = &l_mas; ++ mast.r = &r_mas; ++ mast.orig_l = &prev_l_mas; ++ mast.orig_r = &prev_r_mas; ++ mast.free = &mat; ++ mast.bn = b_node; ++ ++ while (height++ <= mas->depth) { ++ if (mt_slots[b_node->type] > b_node->b_end) { ++ mas_split_final_node(&mast, mas, height); ++ break; ++ } ++ ++ l_mas = r_mas = *mas; ++ l_mas.node = mas_new_ma_node(mas, b_node); ++ r_mas.node = mas_new_ma_node(mas, b_node); ++ /* ++ * Another way that 'jitter' is avoided is to terminate a split up early if the ++ * left or right node has space to spare. This is referred to as "pushing left" ++ * or "pushing right" and is similar to the B* tree, except the nodes left or ++ * right can rarely be reused due to RCU, but the ripple upwards is halted which ++ * is a significant savings. ++ */ ++ /* Try to push left. */ ++ if (mas_push_data(mas, height, &mast, true)) ++ break; ++ ++ /* Try to push right. */ ++ if (mas_push_data(mas, height, &mast, false)) ++ break; ++ ++ split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min); ++ mast_split_data(&mast, mas, split); ++ /* ++ * Usually correct, mab_mas_cp in the above call overwrites ++ * r->max. ++ */ ++ mast.r->max = mas->max; ++ mast_fill_bnode(&mast, mas, 1); ++ prev_l_mas = *mast.l; ++ prev_r_mas = *mast.r; ++ } ++ ++ /* Set the original node as dead */ ++ mat_add(mast.free, mas->node); ++ mas->node = l_mas.node; ++ mas_wmb_replace(mas, mast.free, NULL); ++ mtree_range_walk(mas); ++ return 1; +} + -+static __always_inline void update_lru_size(struct lruvec *lruvec, -+ enum lru_list lru, enum zone_type zid, -+ long nr_pages) ++/* ++ * mas_reuse_node() - Reuse the node to store the data. ++ * @wr_mas: The maple write state ++ * @bn: The maple big node ++ * @end: The end of the data. ++ * ++ * Will always return false in RCU mode. ++ * ++ * Return: True if node was reused, false otherwise. ++ */ ++static inline bool mas_reuse_node(struct ma_wr_state *wr_mas, ++ struct maple_big_node *bn, unsigned char end) +{ -+ __update_lru_size(lruvec, lru, zid, nr_pages); - #ifdef CONFIG_MEMCG - mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); - #endif -@@ -94,11 +104,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) - return lru; - } - -+#ifdef CONFIG_LRU_GEN ++ /* Need to be rcu safe. */ ++ if (mt_in_rcu(wr_mas->mas->tree)) ++ return false; + -+#ifdef CONFIG_LRU_GEN_ENABLED -+static inline bool lru_gen_enabled(void) ++ if (end > bn->b_end) { ++ int clear = mt_slots[wr_mas->type] - bn->b_end; ++ ++ memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--); ++ memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear); ++ } ++ mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false); ++ return true; ++} ++ ++/* ++ * mas_commit_b_node() - Commit the big node into the tree. ++ * @wr_mas: The maple write state ++ * @b_node: The maple big node ++ * @end: The end of the data. ++ */ ++static inline int mas_commit_b_node(struct ma_wr_state *wr_mas, ++ struct maple_big_node *b_node, unsigned char end) +{ -+ DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); ++ struct maple_node *node; ++ unsigned char b_end = b_node->b_end; ++ enum maple_type b_type = b_node->type; + -+ return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); ++ if ((b_end < mt_min_slots[b_type]) && ++ (!mte_is_root(wr_mas->mas->node)) && ++ (mas_mt_height(wr_mas->mas) > 1)) ++ return mas_rebalance(wr_mas->mas, b_node); ++ ++ if (b_end >= mt_slots[b_type]) ++ return mas_split(wr_mas->mas, b_node); ++ ++ if (mas_reuse_node(wr_mas, b_node, end)) ++ goto reuse_node; ++ ++ mas_node_count(wr_mas->mas, 1); ++ if (mas_is_err(wr_mas->mas)) ++ return 0; ++ ++ node = mas_pop_node(wr_mas->mas); ++ node->parent = mas_mn(wr_mas->mas)->parent; ++ wr_mas->mas->node = mt_mk_node(node, b_type); ++ mab_mas_cp(b_node, 0, b_end, wr_mas->mas, true); ++ ++ mas_replace(wr_mas->mas, false); ++reuse_node: ++ mas_update_gap(wr_mas->mas); ++ return 1; +} -+#else -+static inline bool lru_gen_enabled(void) ++ ++/* ++ * mas_root_expand() - Expand a root to a node ++ * @mas: The maple state ++ * @entry: The entry to store into the tree ++ */ ++static inline int mas_root_expand(struct ma_state *mas, void *entry) +{ -+ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); ++ void *contents = mas_root_locked(mas); ++ enum maple_type type = maple_leaf_64; ++ struct maple_node *node; ++ void __rcu **slots; ++ unsigned long *pivots; ++ int slot = 0; + -+ return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); ++ mas_node_count(mas, 1); ++ if (unlikely(mas_is_err(mas))) ++ return 0; ++ ++ node = mas_pop_node(mas); ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ node->parent = ma_parent_ptr( ++ ((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ mas->node = mt_mk_node(node, type); ++ ++ if (mas->index) { ++ if (contents) { ++ rcu_assign_pointer(slots[slot], contents); ++ if (likely(mas->index > 1)) ++ slot++; ++ } ++ pivots[slot++] = mas->index - 1; ++ } ++ ++ rcu_assign_pointer(slots[slot], entry); ++ mas->offset = slot; ++ pivots[slot] = mas->last; ++ if (mas->last != ULONG_MAX) ++ slot++; ++ mas->depth = 1; ++ mas_set_height(mas); ++ ++ /* swap the new root into the tree */ ++ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); ++ ma_set_meta(node, maple_leaf_64, 0, slot); ++ return slot; +} -+#endif + -+static inline bool lru_gen_in_fault(void) ++static inline void mas_store_root(struct ma_state *mas, void *entry) +{ -+ return current->in_lru_fault; ++ if (likely((mas->last != 0) || (mas->index != 0))) ++ mas_root_expand(mas, entry); ++ else if (((unsigned long) (entry) & 3) == 2) ++ mas_root_expand(mas, entry); ++ else { ++ rcu_assign_pointer(mas->tree->ma_root, entry); ++ mas->node = MAS_START; ++ } +} + -+static inline int lru_gen_from_seq(unsigned long seq) ++/* ++ * mas_is_span_wr() - Check if the write needs to be treated as a write that ++ * spans the node. ++ * @mas: The maple state ++ * @piv: The pivot value being written ++ * @type: The maple node type ++ * @entry: The data to write ++ * ++ * Spanning writes are writes that start in one node and end in another OR if ++ * the write of a %NULL will cause the node to end with a %NULL. ++ * ++ * Return: True if this is a spanning write, false otherwise. ++ */ ++static bool mas_is_span_wr(struct ma_wr_state *wr_mas) +{ -+ return seq % MAX_NR_GENS; ++ unsigned long max; ++ unsigned long last = wr_mas->mas->last; ++ unsigned long piv = wr_mas->r_max; ++ enum maple_type type = wr_mas->type; ++ void *entry = wr_mas->entry; ++ ++ /* Contained in this pivot */ ++ if (piv > last) ++ return false; ++ ++ max = wr_mas->mas->max; ++ if (unlikely(ma_is_leaf(type))) { ++ /* Fits in the node, but may span slots. */ ++ if (last < max) ++ return false; ++ ++ /* Writes to the end of the node but not null. */ ++ if ((last == max) && entry) ++ return false; ++ ++ /* ++ * Writing ULONG_MAX is not a spanning write regardless of the ++ * value being written as long as the range fits in the node. ++ */ ++ if ((last == ULONG_MAX) && (last == max)) ++ return false; ++ } else if (piv == last) { ++ if (entry) ++ return false; ++ ++ /* Detect spanning store wr walk */ ++ if (last == ULONG_MAX) ++ return false; ++ } ++ ++ trace_ma_write(__func__, wr_mas->mas, piv, entry); ++ ++ return true; +} + -+static inline int lru_hist_from_seq(unsigned long seq) ++static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas) +{ -+ return seq % NR_HIST_GENS; ++ wr_mas->mas->depth++; ++ wr_mas->type = mte_node_type(wr_mas->mas->node); ++ mas_wr_node_walk(wr_mas); ++ wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); +} + -+static inline int lru_tier_from_refs(int refs) ++static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas) +{ -+ VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); ++ wr_mas->mas->max = wr_mas->r_max; ++ wr_mas->mas->min = wr_mas->r_min; ++ wr_mas->mas->node = wr_mas->content; ++ wr_mas->mas->offset = 0; ++} ++/* ++ * mas_wr_walk() - Walk the tree for a write. ++ * @wr_mas: The maple write state ++ * ++ * Uses mas_slot_locked() and does not need to worry about dead nodes. ++ * ++ * Return: True if it's contained in a node, false on spanning write. ++ */ ++static bool mas_wr_walk(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; + -+ /* see the comment in folio_lru_refs() */ -+ return order_base_2(refs + 1); ++ while (true) { ++ mas_wr_walk_descend(wr_mas); ++ if (unlikely(mas_is_span_wr(wr_mas))) ++ return false; ++ ++ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, ++ mas->offset); ++ if (ma_is_leaf(wr_mas->type)) ++ return true; ++ ++ mas_wr_walk_traverse(wr_mas); ++ } ++ ++ return true; +} + -+static inline int folio_lru_refs(struct folio *folio) ++static bool mas_wr_walk_index(struct ma_wr_state *wr_mas) +{ -+ unsigned long flags = READ_ONCE(folio->flags); -+ bool workingset = flags & BIT(PG_workingset); ++ struct ma_state *mas = wr_mas->mas; + -+ /* -+ * Return the number of accesses beyond PG_referenced, i.e., N-1 if the -+ * total number of accesses is N>1, since N=0,1 both map to the first -+ * tier. lru_tier_from_refs() will account for this off-by-one. Also see -+ * the comment on MAX_NR_TIERS. -+ */ -+ return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; ++ while (true) { ++ mas_wr_walk_descend(wr_mas); ++ wr_mas->content = mas_slot_locked(mas, wr_mas->slots, ++ mas->offset); ++ if (ma_is_leaf(wr_mas->type)) ++ return true; ++ mas_wr_walk_traverse(wr_mas); ++ ++ } ++ return true; +} ++/* ++ * mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs. ++ * @l_wr_mas: The left maple write state ++ * @r_wr_mas: The right maple write state ++ */ ++static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, ++ struct ma_wr_state *r_wr_mas) ++{ ++ struct ma_state *r_mas = r_wr_mas->mas; ++ struct ma_state *l_mas = l_wr_mas->mas; ++ unsigned char l_slot; ++ ++ l_slot = l_mas->offset; ++ if (!l_wr_mas->content) ++ l_mas->index = l_wr_mas->r_min; ++ ++ if ((l_mas->index == l_wr_mas->r_min) && ++ (l_slot && ++ !mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) { ++ if (l_slot > 1) ++ l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1; ++ else ++ l_mas->index = l_mas->min; + -+static inline int folio_lru_gen(struct folio *folio) -+{ -+ unsigned long flags = READ_ONCE(folio->flags); ++ l_mas->offset = l_slot - 1; ++ } + -+ return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ if (!r_wr_mas->content) { ++ if (r_mas->last < r_wr_mas->r_max) ++ r_mas->last = r_wr_mas->r_max; ++ r_mas->offset++; ++ } else if ((r_mas->last == r_wr_mas->r_max) && ++ (r_mas->last < r_mas->max) && ++ !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { ++ r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, ++ r_wr_mas->type, r_mas->offset + 1); ++ r_mas->offset++; ++ } +} + -+static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) ++static inline void *mas_state_walk(struct ma_state *mas) +{ -+ unsigned long max_seq = lruvec->lrugen.max_seq; ++ void *entry; + -+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); ++ entry = mas_start(mas); ++ if (mas_is_none(mas)) ++ return NULL; + -+ /* see the comment on MIN_NR_GENS */ -+ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); ++ if (mas_is_ptr(mas)) ++ return entry; ++ ++ return mtree_range_walk(mas); +} + -+static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, -+ int old_gen, int new_gen) -+{ -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++/* ++ * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up ++ * to date. ++ * ++ * @mas: The maple state. ++ * ++ * Note: Leaves mas in undesirable state. ++ * Return: The entry for @mas->index or %NULL on dead node. ++ */ ++static inline void *mtree_lookup_walk(struct ma_state *mas) ++{ ++ unsigned long *pivots; ++ unsigned char offset; ++ struct maple_node *node; ++ struct maple_enode *next; ++ enum maple_type type; ++ void __rcu **slots; ++ unsigned char end; ++ unsigned long max; ++ ++ next = mas->node; ++ max = ULONG_MAX; ++ do { ++ offset = 0; ++ node = mte_to_node(next); ++ type = mte_node_type(next); ++ pivots = ma_pivots(node, type); ++ end = ma_data_end(node, type, pivots, max); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ ++ if (pivots[offset] >= mas->index) ++ goto next; + -+ VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); ++ do { ++ offset++; ++ } while ((offset < end) && (pivots[offset] < mas->index)); + -+ if (old_gen >= 0) -+ WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], -+ lrugen->nr_pages[old_gen][type][zone] - delta); -+ if (new_gen >= 0) -+ WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], -+ lrugen->nr_pages[new_gen][type][zone] + delta); ++ if (likely(offset > end)) ++ max = pivots[offset]; + -+ /* addition */ -+ if (old_gen < 0) { -+ if (lru_gen_is_active(lruvec, new_gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, delta); -+ return; -+ } ++next: ++ slots = ma_slots(node, type); ++ next = mt_slot(mas->tree, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ goto dead_node; ++ } while (!ma_is_leaf(type)); + -+ /* deletion */ -+ if (new_gen < 0) { -+ if (lru_gen_is_active(lruvec, old_gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, -delta); -+ return; -+ } ++ return (void *) next; + -+ /* promotion */ -+ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { -+ __update_lru_size(lruvec, lru, zone, -delta); -+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); ++dead_node: ++ mas_reset(mas); ++ return NULL; ++} ++ ++/* ++ * mas_new_root() - Create a new root node that only contains the entry passed ++ * in. ++ * @mas: The maple state ++ * @entry: The entry to store. ++ * ++ * Only valid when the index == 0 and the last == ULONG_MAX ++ * ++ * Return 0 on error, 1 on success. ++ */ ++static inline int mas_new_root(struct ma_state *mas, void *entry) ++{ ++ struct maple_enode *root = mas_root_locked(mas); ++ enum maple_type type = maple_leaf_64; ++ struct maple_node *node; ++ void __rcu **slots; ++ unsigned long *pivots; ++ ++ if (!entry && !mas->index && mas->last == ULONG_MAX) { ++ mas->depth = 0; ++ mas_set_height(mas); ++ rcu_assign_pointer(mas->tree->ma_root, entry); ++ mas->node = MAS_START; ++ goto done; + } + -+ /* demotion requires isolation, e.g., lru_deactivate_fn() */ -+ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); -+} ++ mas_node_count(mas, 1); ++ if (mas_is_err(mas)) ++ return 0; + -+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++ node = mas_pop_node(mas); ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ node->parent = ma_parent_ptr( ++ ((unsigned long)mas->tree | MA_ROOT_PARENT)); ++ mas->node = mt_mk_node(node, type); ++ rcu_assign_pointer(slots[0], entry); ++ pivots[0] = mas->last; ++ mas->depth = 1; ++ mas_set_height(mas); ++ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); ++ ++done: ++ if (xa_is_node(root)) ++ mte_destroy_walk(root, mas->tree); ++ ++ return 1; ++} ++/* ++ * mas_wr_spanning_store() - Create a subtree with the store operation completed ++ * and new nodes where necessary, then place the sub-tree in the actual tree. ++ * Note that mas is expected to point to the node which caused the store to ++ * span. ++ * @wr_mas: The maple write state ++ * ++ * Return: 0 on error, positive on success. ++ */ ++static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) +{ -+ unsigned long seq; -+ unsigned long flags; -+ int gen = folio_lru_gen(folio); -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct maple_subtree_state mast; ++ struct maple_big_node b_node; ++ struct ma_state *mas; ++ unsigned char height; + -+ VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); ++ /* Left and Right side of spanning store */ ++ MA_STATE(l_mas, NULL, 0, 0); ++ MA_STATE(r_mas, NULL, 0, 0); ++ ++ MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); ++ MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); + -+ if (folio_test_unevictable(folio) || !lrugen->enabled) -+ return false; + /* -+ * There are three common cases for this page: -+ * 1. If it's hot, e.g., freshly faulted in or previously hot and -+ * migrated, add it to the youngest generation. -+ * 2. If it's cold but can't be evicted immediately, i.e., an anon page -+ * not in swapcache or a dirty page pending writeback, add it to the -+ * second oldest generation. -+ * 3. Everything else (clean, cold) is added to the oldest generation. ++ * A store operation that spans multiple nodes is called a spanning ++ * store and is handled early in the store call stack by the function ++ * mas_is_span_wr(). When a spanning store is identified, the maple ++ * state is duplicated. The first maple state walks the left tree path ++ * to ``index``, the duplicate walks the right tree path to ``last``. ++ * The data in the two nodes are combined into a single node, two nodes, ++ * or possibly three nodes (see the 3-way split above). A ``NULL`` ++ * written to the last entry of a node is considered a spanning store as ++ * a rebalance is required for the operation to complete and an overflow ++ * of data may happen. + */ -+ if (folio_test_active(folio)) -+ seq = lrugen->max_seq; -+ else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || -+ (folio_test_reclaim(folio) && -+ (folio_test_dirty(folio) || folio_test_writeback(folio)))) -+ seq = lrugen->min_seq[type] + 1; -+ else -+ seq = lrugen->min_seq[type]; ++ mas = wr_mas->mas; ++ trace_ma_op(__func__, mas); + -+ gen = lru_gen_from_seq(seq); -+ flags = (gen + 1UL) << LRU_GEN_PGOFF; -+ /* see the comment on MIN_NR_GENS about PG_active */ -+ set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); ++ if (unlikely(!mas->index && mas->last == ULONG_MAX)) ++ return mas_new_root(mas, wr_mas->entry); ++ /* ++ * Node rebalancing may occur due to this store, so there may be three new ++ * entries per level plus a new root. ++ */ ++ height = mas_mt_height(mas); ++ mas_node_count(mas, 1 + height * 3); ++ if (mas_is_err(mas)) ++ return 0; + -+ lru_gen_update_size(lruvec, folio, -1, gen); -+ /* for folio_rotate_reclaimable() */ -+ if (reclaiming) -+ list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); ++ /* ++ * Set up right side. Need to get to the next offset after the spanning ++ * store to ensure it's not NULL and to combine both the next node and ++ * the node with the start together. ++ */ ++ r_mas = *mas; ++ /* Avoid overflow, walk to next slot in the tree. */ ++ if (r_mas.last + 1) ++ r_mas.last++; ++ ++ r_mas.index = r_mas.last; ++ mas_wr_walk_index(&r_wr_mas); ++ r_mas.last = r_mas.index = mas->last; ++ ++ /* Set up left side. */ ++ l_mas = *mas; ++ mas_wr_walk_index(&l_wr_mas); ++ ++ if (!wr_mas->entry) { ++ mas_extend_spanning_null(&l_wr_mas, &r_wr_mas); ++ mas->offset = l_mas.offset; ++ mas->index = l_mas.index; ++ mas->last = l_mas.last = r_mas.last; ++ } ++ ++ /* expanding NULLs may make this cover the entire range */ ++ if (!l_mas.index && r_mas.last == ULONG_MAX) { ++ mas_set_range(mas, 0, ULONG_MAX); ++ return mas_new_root(mas, wr_mas->entry); ++ } ++ ++ memset(&b_node, 0, sizeof(struct maple_big_node)); ++ /* Copy l_mas and store the value in b_node. */ ++ mas_store_b_node(&l_wr_mas, &b_node, l_wr_mas.node_end); ++ /* Copy r_mas into b_node. */ ++ if (r_mas.offset <= r_wr_mas.node_end) ++ mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end, ++ &b_node, b_node.b_end + 1); + else -+ list_add(&folio->lru, &lrugen->lists[gen][type][zone]); ++ b_node.b_end++; + -+ return true; ++ /* Stop spanning searches by searching for just index. */ ++ l_mas.index = l_mas.last = mas->index; ++ ++ mast.bn = &b_node; ++ mast.orig_l = &l_mas; ++ mast.orig_r = &r_mas; ++ /* Combine l_mas and r_mas and split them up evenly again. */ ++ return mas_spanning_rebalance(mas, &mast, height + 1); +} + -+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ unsigned long flags; -+ int gen = folio_lru_gen(folio); ++/* ++ * mas_wr_node_store() - Attempt to store the value in a node ++ * @wr_mas: The maple write state ++ * ++ * Attempts to reuse the node, but may allocate. ++ * ++ * Return: True if stored, false otherwise ++ */ ++static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ void __rcu **dst_slots; ++ unsigned long *dst_pivots; ++ unsigned char dst_offset; ++ unsigned char new_end = wr_mas->node_end; ++ unsigned char offset; ++ unsigned char node_slots = mt_slots[wr_mas->type]; ++ struct maple_node reuse, *newnode; ++ unsigned char copy_size, max_piv = mt_pivots[wr_mas->type]; ++ bool in_rcu = mt_in_rcu(mas->tree); ++ ++ offset = mas->offset; ++ if (mas->last == wr_mas->r_max) { ++ /* runs right to the end of the node */ ++ if (mas->last == mas->max) ++ new_end = offset; ++ /* don't copy this offset */ ++ wr_mas->offset_end++; ++ } else if (mas->last < wr_mas->r_max) { ++ /* new range ends in this range */ ++ if (unlikely(wr_mas->r_max == ULONG_MAX)) ++ mas_bulk_rebalance(mas, wr_mas->node_end, wr_mas->type); ++ ++ new_end++; ++ } else { ++ if (wr_mas->end_piv == mas->last) ++ wr_mas->offset_end++; + -+ if (gen < 0) ++ new_end -= wr_mas->offset_end - offset - 1; ++ } ++ ++ /* new range starts within a range */ ++ if (wr_mas->r_min < mas->index) ++ new_end++; ++ ++ /* Not enough room */ ++ if (new_end >= node_slots) + return false; + -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); ++ /* Not enough data. */ ++ if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) && ++ !(mas->mas_flags & MA_STATE_BULK)) ++ return false; + -+ /* for folio_migrate_flags() */ -+ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; -+ flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); -+ gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ /* set up node. */ ++ if (in_rcu) { ++ mas_node_count(mas, 1); ++ if (mas_is_err(mas)) ++ return false; + -+ lru_gen_update_size(lruvec, folio, gen, -1); -+ list_del(&folio->lru); ++ newnode = mas_pop_node(mas); ++ } else { ++ memset(&reuse, 0, sizeof(struct maple_node)); ++ newnode = &reuse; ++ } + -+ return true; -+} ++ newnode->parent = mas_mn(mas)->parent; ++ dst_pivots = ma_pivots(newnode, wr_mas->type); ++ dst_slots = ma_slots(newnode, wr_mas->type); ++ /* Copy from start to insert point */ ++ memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * (offset + 1)); ++ memcpy(dst_slots, wr_mas->slots, sizeof(void *) * (offset + 1)); ++ dst_offset = offset; + -+#else /* !CONFIG_LRU_GEN */ ++ /* Handle insert of new range starting after old range */ ++ if (wr_mas->r_min < mas->index) { ++ mas->offset++; ++ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->content); ++ dst_pivots[dst_offset++] = mas->index - 1; ++ } + -+static inline bool lru_gen_enabled(void) -+{ -+ return false; -+} ++ /* Store the new entry and range end. */ ++ if (dst_offset < max_piv) ++ dst_pivots[dst_offset] = mas->last; ++ mas->offset = dst_offset; ++ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->entry); + -+static inline bool lru_gen_in_fault(void) -+{ -+ return false; -+} ++ /* ++ * this range wrote to the end of the node or it overwrote the rest of ++ * the data ++ */ ++ if (wr_mas->offset_end > wr_mas->node_end || mas->last >= mas->max) { ++ new_end = dst_offset; ++ goto done; ++ } + -+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -+{ -+ return false; ++ dst_offset++; ++ /* Copy to the end of node if necessary. */ ++ copy_size = wr_mas->node_end - wr_mas->offset_end + 1; ++ memcpy(dst_slots + dst_offset, wr_mas->slots + wr_mas->offset_end, ++ sizeof(void *) * copy_size); ++ if (dst_offset < max_piv) { ++ if (copy_size > max_piv - dst_offset) ++ copy_size = max_piv - dst_offset; ++ ++ memcpy(dst_pivots + dst_offset, ++ wr_mas->pivots + wr_mas->offset_end, ++ sizeof(unsigned long) * copy_size); ++ } ++ ++ if ((wr_mas->node_end == node_slots - 1) && (new_end < node_slots - 1)) ++ dst_pivots[new_end] = mas->max; ++ ++done: ++ mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end); ++ if (in_rcu) { ++ mas->node = mt_mk_node(newnode, wr_mas->type); ++ mas_replace(mas, false); ++ } else { ++ memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); ++ } ++ trace_ma_write(__func__, mas, 0, wr_mas->entry); ++ mas_update_gap(mas); ++ return true; +} + -+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++/* ++ * mas_wr_slot_store: Attempt to store a value in a slot. ++ * @wr_mas: the maple write state ++ * ++ * Return: True if stored, false otherwise ++ */ ++static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) +{ -+ return false; -+} ++ struct ma_state *mas = wr_mas->mas; ++ unsigned long lmax; /* Logical max. */ ++ unsigned char offset = mas->offset; + -+#endif /* CONFIG_LRU_GEN */ ++ if ((wr_mas->r_max > mas->last) && ((wr_mas->r_min != mas->index) || ++ (offset != wr_mas->node_end))) ++ return false; + - static __always_inline - void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_add_folio(lruvec, folio, false)) -+ return; ++ if (offset == wr_mas->node_end - 1) ++ lmax = mas->max; ++ else ++ lmax = wr_mas->pivots[offset + 1]; + - update_lru_size(lruvec, lru, folio_zonenum(folio), - folio_nr_pages(folio)); - if (lru != LRU_UNEVICTABLE) -@@ -116,6 +339,9 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_add_folio(lruvec, folio, true)) -+ return; ++ /* going to overwrite too many slots. */ ++ if (lmax < mas->last) ++ return false; + - update_lru_size(lruvec, lru, folio_zonenum(folio), - folio_nr_pages(folio)); - /* This is not expected to be used on LRU_UNEVICTABLE */ -@@ -133,6 +359,9 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) - { - enum lru_list lru = folio_lru_list(folio); - -+ if (lru_gen_del_folio(lruvec, folio, false)) -+ return; ++ if (wr_mas->r_min == mas->index) { ++ /* overwriting two or more ranges with one. */ ++ if (lmax == mas->last) ++ return false; + - if (lru != LRU_UNEVICTABLE) - list_del(&folio->lru); - update_lru_size(lruvec, lru, folio_zonenum(folio), -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 8ac6d7e53b17..15afe5e0c4c2 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -677,6 +677,22 @@ struct mm_struct { - */ - unsigned long ksm_rmap_items; - #endif -+#ifdef CONFIG_LRU_GEN -+ struct { -+ /* this mm_struct is on lru_gen_mm_list */ -+ struct list_head list; -+ /* -+ * Set when switching to this mm_struct, as a hint of -+ * whether it has been used since the last time per-node -+ * page table walkers cleared the corresponding bits. -+ */ -+ unsigned long bitmap; -+#ifdef CONFIG_MEMCG -+ /* points to the memcg of "owner" above */ -+ struct mem_cgroup *memcg; -+#endif -+ } lru_gen; -+#endif /* CONFIG_LRU_GEN */ - } __randomize_layout; - - /* -@@ -703,6 +719,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) - return (struct cpumask *)&mm->cpu_bitmap; - } - -+#ifdef CONFIG_LRU_GEN ++ /* Overwriting all of offset and a portion of offset + 1. */ ++ rcu_assign_pointer(wr_mas->slots[offset], wr_mas->entry); ++ wr_mas->pivots[offset] = mas->last; ++ goto done; ++ } + -+struct lru_gen_mm_list { -+ /* mm_struct list for page table walkers */ -+ struct list_head fifo; -+ /* protects the list above */ -+ spinlock_t lock; -+}; ++ /* Doesn't end on the next range end. */ ++ if (lmax != mas->last) ++ return false; + -+void lru_gen_add_mm(struct mm_struct *mm); -+void lru_gen_del_mm(struct mm_struct *mm); -+#ifdef CONFIG_MEMCG -+void lru_gen_migrate_mm(struct mm_struct *mm); -+#endif ++ /* Overwriting a portion of offset and all of offset + 1 */ ++ if ((offset + 1 < mt_pivots[wr_mas->type]) && ++ (wr_mas->entry || wr_mas->pivots[offset + 1])) ++ wr_mas->pivots[offset + 1] = mas->last; + -+static inline void lru_gen_init_mm(struct mm_struct *mm) -+{ -+ INIT_LIST_HEAD(&mm->lru_gen.list); -+ mm->lru_gen.bitmap = 0; -+#ifdef CONFIG_MEMCG -+ mm->lru_gen.memcg = NULL; -+#endif -+} ++ rcu_assign_pointer(wr_mas->slots[offset + 1], wr_mas->entry); ++ wr_mas->pivots[offset] = mas->index - 1; ++ mas->offset++; /* Keep mas accurate. */ + -+static inline void lru_gen_use_mm(struct mm_struct *mm) -+{ -+ /* -+ * When the bitmap is set, page reclaim knows this mm_struct has been -+ * used since the last time it cleared the bitmap. So it might be worth -+ * walking the page tables of this mm_struct to clear the accessed bit. -+ */ -+ WRITE_ONCE(mm->lru_gen.bitmap, -1); ++done: ++ trace_ma_write(__func__, mas, 0, wr_mas->entry); ++ mas_update_gap(mas); ++ return true; +} + -+#else /* !CONFIG_LRU_GEN */ -+ -+static inline void lru_gen_add_mm(struct mm_struct *mm) ++static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) +{ ++ while ((wr_mas->mas->last > wr_mas->end_piv) && ++ (wr_mas->offset_end < wr_mas->node_end)) ++ wr_mas->end_piv = wr_mas->pivots[++wr_mas->offset_end]; ++ ++ if (wr_mas->mas->last > wr_mas->end_piv) ++ wr_mas->end_piv = wr_mas->mas->max; +} + -+static inline void lru_gen_del_mm(struct mm_struct *mm) ++static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) +{ ++ struct ma_state *mas = wr_mas->mas; ++ ++ if (mas->last < wr_mas->end_piv && !wr_mas->slots[wr_mas->offset_end]) ++ mas->last = wr_mas->end_piv; ++ ++ /* Check next slot(s) if we are overwriting the end */ ++ if ((mas->last == wr_mas->end_piv) && ++ (wr_mas->node_end != wr_mas->offset_end) && ++ !wr_mas->slots[wr_mas->offset_end + 1]) { ++ wr_mas->offset_end++; ++ if (wr_mas->offset_end == wr_mas->node_end) ++ mas->last = mas->max; ++ else ++ mas->last = wr_mas->pivots[wr_mas->offset_end]; ++ wr_mas->end_piv = mas->last; ++ } ++ ++ if (!wr_mas->content) { ++ /* If this one is null, the next and prev are not */ ++ mas->index = wr_mas->r_min; ++ } else { ++ /* Check prev slot if we are overwriting the start */ ++ if (mas->index == wr_mas->r_min && mas->offset && ++ !wr_mas->slots[mas->offset - 1]) { ++ mas->offset--; ++ wr_mas->r_min = mas->index = ++ mas_safe_min(mas, wr_mas->pivots, mas->offset); ++ wr_mas->r_max = wr_mas->pivots[mas->offset]; ++ } ++ } +} + -+#ifdef CONFIG_MEMCG -+static inline void lru_gen_migrate_mm(struct mm_struct *mm) ++static inline bool mas_wr_append(struct ma_wr_state *wr_mas) +{ ++ unsigned char end = wr_mas->node_end; ++ unsigned char new_end = end + 1; ++ struct ma_state *mas = wr_mas->mas; ++ unsigned char node_pivots = mt_pivots[wr_mas->type]; ++ ++ if ((mas->index != wr_mas->r_min) && (mas->last == wr_mas->r_max)) { ++ if (new_end < node_pivots) ++ wr_mas->pivots[new_end] = wr_mas->pivots[end]; ++ ++ if (new_end < node_pivots) ++ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); ++ ++ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->entry); ++ mas->offset = new_end; ++ wr_mas->pivots[end] = mas->index - 1; ++ ++ return true; ++ } ++ ++ if ((mas->index == wr_mas->r_min) && (mas->last < wr_mas->r_max)) { ++ if (new_end < node_pivots) ++ wr_mas->pivots[new_end] = wr_mas->pivots[end]; ++ ++ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->content); ++ if (new_end < node_pivots) ++ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); ++ ++ wr_mas->pivots[end] = mas->last; ++ rcu_assign_pointer(wr_mas->slots[end], wr_mas->entry); ++ return true; ++ } ++ ++ return false; +} -+#endif + -+static inline void lru_gen_init_mm(struct mm_struct *mm) ++/* ++ * mas_wr_bnode() - Slow path for a modification. ++ * @wr_mas: The write maple state ++ * ++ * This is where split, rebalance end up. ++ */ ++static void mas_wr_bnode(struct ma_wr_state *wr_mas) +{ ++ struct maple_big_node b_node; ++ ++ trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry); ++ memset(&b_node, 0, sizeof(struct maple_big_node)); ++ mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); ++ mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end); +} + -+static inline void lru_gen_use_mm(struct mm_struct *mm) ++static inline void mas_wr_modify(struct ma_wr_state *wr_mas) +{ -+} ++ unsigned char node_slots; ++ unsigned char node_size; ++ struct ma_state *mas = wr_mas->mas; + -+#endif /* CONFIG_LRU_GEN */ ++ /* Direct replacement */ ++ if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) { ++ rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry); ++ if (!!wr_mas->entry ^ !!wr_mas->content) ++ mas_update_gap(mas); ++ return; ++ } + - struct mmu_gather; - extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); - extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); -diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index e24b40c52468..1543001feba9 100644 ---- a/include/linux/mmzone.h -+++ b/include/linux/mmzone.h -@@ -306,6 +306,8 @@ static inline bool is_active_lru(enum lru_list lru) - return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); - } - -+#define WORKINGSET_ANON 0 -+#define WORKINGSET_FILE 1 - #define ANON_AND_FILE 2 - - enum lruvec_flags { -@@ -314,6 +316,207 @@ enum lruvec_flags { - */ - }; - -+#endif /* !__GENERATING_BOUNDS_H */ ++ /* Attempt to append */ ++ node_slots = mt_slots[wr_mas->type]; ++ node_size = wr_mas->node_end - wr_mas->offset_end + mas->offset + 2; ++ if (mas->max == ULONG_MAX) ++ node_size++; ++ ++ /* slot and node store will not fit, go to the slow path */ ++ if (unlikely(node_size >= node_slots)) ++ goto slow_path; ++ ++ if (wr_mas->entry && (wr_mas->node_end < node_slots - 1) && ++ (mas->offset == wr_mas->node_end) && mas_wr_append(wr_mas)) { ++ if (!wr_mas->content || !wr_mas->entry) ++ mas_update_gap(mas); ++ return; ++ } ++ ++ if ((wr_mas->offset_end - mas->offset <= 1) && mas_wr_slot_store(wr_mas)) ++ return; ++ else if (mas_wr_node_store(wr_mas)) ++ return; ++ ++ if (mas_is_err(mas)) ++ return; ++ ++slow_path: ++ mas_wr_bnode(wr_mas); ++} + +/* -+ * Evictable pages are divided into multiple generations. The youngest and the -+ * oldest generation numbers, max_seq and min_seq, are monotonically increasing. -+ * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An -+ * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the -+ * corresponding generation. The gen counter in folio->flags stores gen+1 while -+ * a page is on one of lrugen->lists[]. Otherwise it stores 0. -+ * -+ * A page is added to the youngest generation on faulting. The aging needs to -+ * check the accessed bit at least twice before handing this page over to the -+ * eviction. The first check takes care of the accessed bit set on the initial -+ * fault; the second check makes sure this page hasn't been used since then. -+ * This process, AKA second chance, requires a minimum of two generations, -+ * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive -+ * LRU, e.g., /proc/vmstat, these two generations are considered active; the -+ * rest of generations, if they exist, are considered inactive. See -+ * lru_gen_is_active(). ++ * mas_wr_store_entry() - Internal call to store a value ++ * @mas: The maple state ++ * @entry: The entry to store. + * -+ * PG_active is always cleared while a page is on one of lrugen->lists[] so that -+ * the aging needs not to worry about it. And it's set again when a page -+ * considered active is isolated for non-reclaiming purposes, e.g., migration. -+ * See lru_gen_add_folio() and lru_gen_del_folio(). ++ * Return: The contents that was stored at the index. ++ */ ++static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas) ++{ ++ struct ma_state *mas = wr_mas->mas; ++ ++ wr_mas->content = mas_start(mas); ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_store_root(mas, wr_mas->entry); ++ return wr_mas->content; ++ } ++ ++ if (unlikely(!mas_wr_walk(wr_mas))) { ++ mas_wr_spanning_store(wr_mas); ++ return wr_mas->content; ++ } ++ ++ /* At this point, we are at the leaf node that needs to be altered. */ ++ wr_mas->end_piv = wr_mas->r_max; ++ mas_wr_end_piv(wr_mas); ++ ++ if (!wr_mas->entry) ++ mas_wr_extend_null(wr_mas); ++ ++ /* New root for a single pointer */ ++ if (unlikely(!mas->index && mas->last == ULONG_MAX)) { ++ mas_new_root(mas, wr_mas->entry); ++ return wr_mas->content; ++ } ++ ++ mas_wr_modify(wr_mas); ++ return wr_mas->content; ++} ++ ++/** ++ * mas_insert() - Internal call to insert a value ++ * @mas: The maple state ++ * @entry: The entry to store + * -+ * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the -+ * number of categories of the active/inactive LRU when keeping track of -+ * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits -+ * in folio->flags. ++ * Return: %NULL or the contents that already exists at the requested index ++ * otherwise. The maple state needs to be checked for error conditions. + */ -+#define MIN_NR_GENS 2U -+#define MAX_NR_GENS 4U ++static inline void *mas_insert(struct ma_state *mas, void *entry) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ /* ++ * Inserting a new range inserts either 0, 1, or 2 pivots within the ++ * tree. If the insert fits exactly into an existing gap with a value ++ * of NULL, then the slot only needs to be written with the new value. ++ * If the range being inserted is adjacent to another range, then only a ++ * single pivot needs to be inserted (as well as writing the entry). If ++ * the new range is within a gap but does not touch any other ranges, ++ * then two pivots need to be inserted: the start - 1, and the end. As ++ * usual, the entry must be written. Most operations require a new node ++ * to be allocated and replace an existing node to ensure RCU safety, ++ * when in RCU mode. The exception to requiring a newly allocated node ++ * is when inserting at the end of a node (appending). When done ++ * carefully, appending can reuse the node in place. ++ */ ++ wr_mas.content = mas_start(mas); ++ if (wr_mas.content) ++ goto exists; ++ ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_store_root(mas, entry); ++ return NULL; ++ } ++ ++ /* spanning writes always overwrite something */ ++ if (!mas_wr_walk(&wr_mas)) ++ goto exists; ++ ++ /* At this point, we are at the leaf node that needs to be altered. */ ++ wr_mas.offset_end = mas->offset; ++ wr_mas.end_piv = wr_mas.r_max; ++ ++ if (wr_mas.content || (mas->last > wr_mas.r_max)) ++ goto exists; ++ ++ if (!entry) ++ return NULL; ++ ++ mas_wr_modify(&wr_mas); ++ return wr_mas.content; ++ ++exists: ++ mas_set_err(mas, -EEXIST); ++ return wr_mas.content; ++ ++} + +/* -+ * Each generation is divided into multiple tiers. A page accessed N times -+ * through file descriptors is in tier order_base_2(N). A page in the first tier -+ * (N=0,1) is marked by PG_referenced unless it was faulted in through page -+ * tables or read ahead. A page in any other tier (N>1) is marked by -+ * PG_referenced and PG_workingset. This implies a minimum of two tiers is -+ * supported without using additional bits in folio->flags. ++ * mas_prev_node() - Find the prev non-null entry at the same level in the ++ * tree. The prev value will be mas->node[mas->offset] or MAS_NONE. ++ * @mas: The maple state ++ * @min: The lower limit to search + * -+ * In contrast to moving across generations which requires the LRU lock, moving -+ * across tiers only involves atomic operations on folio->flags and therefore -+ * has a negligible cost in the buffered access path. In the eviction path, -+ * comparisons of refaulted/(evicted+protected) from the first tier and the -+ * rest infer whether pages accessed multiple times through file descriptors -+ * are statistically hot and thus worth protecting. ++ * The prev node value will be mas->node[mas->offset] or MAS_NONE. ++ * Return: 1 if the node is dead, 0 otherwise. ++ */ ++static inline int mas_prev_node(struct ma_state *mas, unsigned long min) ++{ ++ enum maple_type mt; ++ int offset, level; ++ void __rcu **slots; ++ struct maple_node *node; ++ struct maple_enode *enode; ++ unsigned long *pivots; ++ ++ if (mas_is_none(mas)) ++ return 0; ++ ++ level = 0; ++ do { ++ node = mas_mn(mas); ++ if (ma_is_root(node)) ++ goto no_entry; ++ ++ /* Walk up. */ ++ if (unlikely(mas_ascend(mas))) ++ return 1; ++ offset = mas->offset; ++ level++; ++ } while (!offset); ++ ++ offset--; ++ mt = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ mas->max = pivots[offset]; ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ if (mas->max < min) ++ goto no_entry_min; ++ ++ while (level > 1) { ++ level--; ++ enode = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = enode; ++ mt = mte_node_type(mas->node); ++ node = mas_mn(mas); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ offset = ma_data_end(node, mt, pivots, mas->max); ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++ ++ if (offset < mt_pivots[mt]) ++ mas->max = pivots[offset]; ++ ++ if (mas->max < min) ++ goto no_entry; ++ } ++ ++ mas->node = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->offset = mas_data_end(mas); ++ if (unlikely(mte_dead_node(mas->node))) ++ return 1; ++ ++ return 0; ++ ++no_entry_min: ++ mas->offset = offset; ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++no_entry: ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = MAS_NONE; ++ return 0; ++} ++ ++/* ++ * mas_next_node() - Get the next node at the same level in the tree. ++ * @mas: The maple state ++ * @max: The maximum pivot value to check. + * -+ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the -+ * number of categories of the active/inactive LRU when keeping track of -+ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in -+ * folio->flags. ++ * The next value will be mas->node[mas->offset] or MAS_NONE. ++ * Return: 1 on dead node, 0 otherwise. + */ -+#define MAX_NR_TIERS 4U ++static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, ++ unsigned long max) ++{ ++ unsigned long min, pivot; ++ unsigned long *pivots; ++ struct maple_enode *enode; ++ int level = 0; ++ unsigned char offset; ++ enum maple_type mt; ++ void __rcu **slots; + -+#ifndef __GENERATING_BOUNDS_H ++ if (mas->max >= max) ++ goto no_entry; + -+struct lruvec; -+struct page_vma_mapped_walk; ++ level = 0; ++ do { ++ if (ma_is_root(node)) ++ goto no_entry; + -+#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) -+#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) ++ min = mas->max + 1; ++ if (min > max) ++ goto no_entry; + -+#ifdef CONFIG_LRU_GEN ++ if (unlikely(mas_ascend(mas))) ++ return 1; + -+enum { -+ LRU_GEN_ANON, -+ LRU_GEN_FILE, -+}; ++ offset = mas->offset; ++ level++; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ pivots = ma_pivots(node, mt); ++ } while (unlikely(offset == ma_data_end(node, mt, pivots, mas->max))); ++ ++ slots = ma_slots(node, mt); ++ pivot = mas_safe_pivot(mas, pivots, ++offset, mt); ++ while (unlikely(level > 1)) { ++ /* Descend, if necessary */ ++ enode = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; + -+enum { -+ LRU_GEN_CORE, -+ LRU_GEN_MM_WALK, -+ LRU_GEN_NONLEAF_YOUNG, -+ NR_LRU_GEN_CAPS -+}; ++ mas->node = enode; ++ level--; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ slots = ma_slots(node, mt); ++ pivots = ma_pivots(node, mt); ++ offset = 0; ++ pivot = pivots[0]; ++ } + -+#define MIN_LRU_BATCH BITS_PER_LONG -+#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) ++ enode = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(node))) ++ return 1; + -+/* whether to keep historical stats from evicted generations */ -+#ifdef CONFIG_LRU_GEN_STATS -+#define NR_HIST_GENS MAX_NR_GENS -+#else -+#define NR_HIST_GENS 1U -+#endif ++ mas->node = enode; ++ mas->min = min; ++ mas->max = pivot; ++ return 0; ++ ++no_entry: ++ if (unlikely(ma_dead_node(node))) ++ return 1; ++ ++ mas->node = MAS_NONE; ++ return 0; ++} + +/* -+ * The youngest generation number is stored in max_seq for both anon and file -+ * types as they are aged on an equal footing. The oldest generation numbers are -+ * stored in min_seq[] separately for anon and file types as clean file pages -+ * can be evicted regardless of swap constraints. ++ * mas_next_nentry() - Get the next node entry ++ * @mas: The maple state ++ * @max: The maximum value to check ++ * @*range_start: Pointer to store the start of the range. + * -+ * Normally anon and file min_seq are in sync. But if swapping is constrained, -+ * e.g., out of swap space, file min_seq is allowed to advance and leave anon -+ * min_seq behind. ++ * Sets @mas->offset to the offset of the next node entry, @mas->last to the ++ * pivot of the entry. + * -+ * The number of pages in each generation is eventually consistent and therefore -+ * can be transiently negative when reset_batch_size() is pending. ++ * Return: The next entry, %NULL otherwise + */ -+struct lru_gen_struct { -+ /* the aging increments the youngest generation number */ -+ unsigned long max_seq; -+ /* the eviction increments the oldest generation numbers */ -+ unsigned long min_seq[ANON_AND_FILE]; -+ /* the birth time of each generation in jiffies */ -+ unsigned long timestamps[MAX_NR_GENS]; -+ /* the multi-gen LRU lists, lazily sorted on eviction */ -+ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* the multi-gen LRU sizes, eventually consistent */ -+ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* the exponential moving average of refaulted */ -+ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; -+ /* the exponential moving average of evicted+protected */ -+ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; -+ /* the first tier doesn't need protection, hence the minus one */ -+ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; -+ /* can be modified without holding the LRU lock */ -+ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; -+ atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; -+ /* whether the multi-gen LRU is enabled */ -+ bool enabled; -+}; ++static inline void *mas_next_nentry(struct ma_state *mas, ++ struct maple_node *node, unsigned long max, enum maple_type type) ++{ ++ unsigned char count; ++ unsigned long pivot; ++ unsigned long *pivots; ++ void __rcu **slots; ++ void *entry; + -+enum { -+ MM_LEAF_TOTAL, /* total leaf entries */ -+ MM_LEAF_OLD, /* old leaf entries */ -+ MM_LEAF_YOUNG, /* young leaf entries */ -+ MM_NONLEAF_TOTAL, /* total non-leaf entries */ -+ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ -+ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ -+ NR_MM_STATS -+}; ++ if (mas->last == mas->max) { ++ mas->index = mas->max; ++ return NULL; ++ } + -+/* double-buffering Bloom filters */ -+#define NR_BLOOM_FILTERS 2 ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ mas->index = mas_safe_min(mas, pivots, mas->offset); ++ if (ma_dead_node(node)) ++ return NULL; + -+struct lru_gen_mm_state { -+ /* set to max_seq after each iteration */ -+ unsigned long seq; -+ /* where the current iteration continues (inclusive) */ -+ struct list_head *head; -+ /* where the last iteration ended (exclusive) */ -+ struct list_head *tail; -+ /* to wait for the last page table walker to finish */ -+ struct wait_queue_head wait; -+ /* Bloom filters flip after each iteration */ -+ unsigned long *filters[NR_BLOOM_FILTERS]; -+ /* the mm stats for debugging */ -+ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; -+ /* the number of concurrent page table walkers */ -+ int nr_walkers; -+}; ++ if (mas->index > max) ++ return NULL; + -+struct lru_gen_mm_walk { -+ /* the lruvec under reclaim */ -+ struct lruvec *lruvec; -+ /* unstable max_seq from lru_gen_struct */ -+ unsigned long max_seq; -+ /* the next address within an mm to scan */ -+ unsigned long next_addr; -+ /* to batch promoted pages */ -+ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; -+ /* to batch the mm stats */ -+ int mm_stats[NR_MM_STATS]; -+ /* total batched items */ -+ int batched; -+ bool can_swap; -+ bool force_scan; -+}; ++ count = ma_data_end(node, type, pivots, mas->max); ++ if (mas->offset > count) ++ return NULL; + -+void lru_gen_init_lruvec(struct lruvec *lruvec); -+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); ++ while (mas->offset < count) { ++ pivot = pivots[mas->offset]; ++ entry = mas_slot(mas, slots, mas->offset); ++ if (ma_dead_node(node)) ++ return NULL; + -+#ifdef CONFIG_MEMCG -+void lru_gen_init_memcg(struct mem_cgroup *memcg); -+void lru_gen_exit_memcg(struct mem_cgroup *memcg); -+#endif ++ if (entry) ++ goto found; + -+#else /* !CONFIG_LRU_GEN */ ++ if (pivot >= max) ++ return NULL; + -+static inline void lru_gen_init_lruvec(struct lruvec *lruvec) -+{ ++ mas->index = pivot + 1; ++ mas->offset++; ++ } ++ ++ if (mas->index > mas->max) { ++ mas->index = mas->last; ++ return NULL; ++ } ++ ++ pivot = mas_safe_pivot(mas, pivots, mas->offset, type); ++ entry = mas_slot(mas, slots, mas->offset); ++ if (ma_dead_node(node)) ++ return NULL; ++ ++ if (!pivot) ++ return NULL; ++ ++ if (!entry) ++ return NULL; ++ ++found: ++ mas->last = pivot; ++ return entry; +} + -+static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) ++static inline void mas_rewalk(struct ma_state *mas, unsigned long index) +{ ++ ++retry: ++ mas_set(mas, index); ++ mas_state_walk(mas); ++ if (mas_is_start(mas)) ++ goto retry; ++ ++ return; ++ +} + -+#ifdef CONFIG_MEMCG -+static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) -+{ ++/* ++ * mas_next_entry() - Internal function to get the next entry. ++ * @mas: The maple state ++ * @limit: The maximum range start. ++ * ++ * Set the @mas->node to the next entry and the range_start to ++ * the beginning value for the entry. Does not check beyond @limit. ++ * Sets @mas->index and @mas->last to the limit if it is hit. ++ * Restarts on dead nodes. ++ * ++ * Return: the next entry or %NULL. ++ */ ++static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) ++{ ++ void *entry = NULL; ++ struct maple_enode *prev_node; ++ struct maple_node *node; ++ unsigned char offset; ++ unsigned long last; ++ enum maple_type mt; ++ ++ last = mas->last; ++retry: ++ offset = mas->offset; ++ prev_node = mas->node; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ mas->offset++; ++ if (unlikely(mas->offset >= mt_slots[mt])) { ++ mas->offset = mt_slots[mt] - 1; ++ goto next_node; ++ } ++ ++ while (!mas_is_none(mas)) { ++ entry = mas_next_nentry(mas, node, limit, mt); ++ if (unlikely(ma_dead_node(node))) { ++ mas_rewalk(mas, last); ++ goto retry; ++ } ++ ++ if (likely(entry)) ++ return entry; ++ ++ if (unlikely((mas->index > limit))) ++ break; ++ ++next_node: ++ prev_node = mas->node; ++ offset = mas->offset; ++ if (unlikely(mas_next_node(mas, node, limit))) { ++ mas_rewalk(mas, last); ++ goto retry; ++ } ++ mas->offset = 0; ++ node = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ } ++ ++ mas->index = mas->last = limit; ++ mas->offset = offset; ++ mas->node = prev_node; ++ return NULL; +} + -+static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) -+{ ++/* ++ * mas_prev_nentry() - Get the previous node entry. ++ * @mas: The maple state. ++ * @limit: The lower limit to check for a value. ++ * ++ * Return: the entry, %NULL otherwise. ++ */ ++static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit, ++ unsigned long index) ++{ ++ unsigned long pivot, min; ++ unsigned char offset; ++ struct maple_node *mn; ++ enum maple_type mt; ++ unsigned long *pivots; ++ void __rcu **slots; ++ void *entry; ++ ++retry: ++ if (!mas->offset) ++ return NULL; ++ ++ mn = mas_mn(mas); ++ mt = mte_node_type(mas->node); ++ offset = mas->offset - 1; ++ if (offset >= mt_slots[mt]) ++ offset = mt_slots[mt] - 1; ++ ++ slots = ma_slots(mn, mt); ++ pivots = ma_pivots(mn, mt); ++ if (offset == mt_pivots[mt]) ++ pivot = mas->max; ++ else ++ pivot = pivots[offset]; ++ ++ if (unlikely(ma_dead_node(mn))) { ++ mas_rewalk(mas, index); ++ goto retry; ++ } ++ ++ while (offset && ((!mas_slot(mas, slots, offset) && pivot >= limit) || ++ !pivot)) ++ pivot = pivots[--offset]; ++ ++ min = mas_safe_min(mas, pivots, offset); ++ entry = mas_slot(mas, slots, offset); ++ if (unlikely(ma_dead_node(mn))) { ++ mas_rewalk(mas, index); ++ goto retry; ++ } ++ ++ if (likely(entry)) { ++ mas->offset = offset; ++ mas->last = pivot; ++ mas->index = min; ++ } ++ return entry; +} -+#endif + -+#endif /* CONFIG_LRU_GEN */ ++static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min) ++{ ++ void *entry; + - struct lruvec { - struct list_head lists[NR_LRU_LISTS]; - /* per lruvec lru_lock for memcg */ -@@ -331,6 +534,12 @@ struct lruvec { - unsigned long refaults[ANON_AND_FILE]; - /* Various lruvec state flags (enum lruvec_flags) */ - unsigned long flags; -+#ifdef CONFIG_LRU_GEN -+ /* evictable pages divided into generations */ -+ struct lru_gen_struct lrugen; -+ /* to concurrently iterate lru_gen_mm_list */ -+ struct lru_gen_mm_state mm_state; -+#endif - #ifdef CONFIG_MEMCG - struct pglist_data *pgdat; - #endif -@@ -746,6 +955,8 @@ static inline bool zone_is_empty(struct zone *zone) - #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) - #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) - #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) -+#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) -+#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) - - /* - * Define the bit shifts to access each section. For non-existent -@@ -1007,6 +1218,11 @@ typedef struct pglist_data { - - unsigned long flags; - -+#ifdef CONFIG_LRU_GEN -+ /* kswap mm walk data */ -+ struct lru_gen_mm_walk mm_walk; -+#endif ++retry: ++ while (likely(!mas_is_none(mas))) { ++ entry = mas_prev_nentry(mas, min, mas->index); ++ if (unlikely(mas->last < min)) ++ goto not_found; + - ZONE_PADDING(_pad2_) - - /* Per-node vmstats */ -diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h -index 4b71a96190a8..3a0eec9f2faa 100644 ---- a/include/linux/nodemask.h -+++ b/include/linux/nodemask.h -@@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) - #define first_online_node 0 - #define first_memory_node 0 - #define next_online_node(nid) (MAX_NUMNODES) -+#define next_memory_node(nid) (MAX_NUMNODES) - #define nr_node_ids 1U - #define nr_online_nodes 1U - -diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h -index ef1e3e736e14..7d79818dc065 100644 ---- a/include/linux/page-flags-layout.h -+++ b/include/linux/page-flags-layout.h -@@ -55,7 +55,8 @@ - #define SECTIONS_WIDTH 0 - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ -+ <= BITS_PER_LONG - NR_PAGEFLAGS - #define NODES_WIDTH NODES_SHIFT - #elif defined(CONFIG_SPARSEMEM_VMEMMAP) - #error "Vmemmap: No space for nodes field in page flags" -@@ -89,8 +90,8 @@ - #define LAST_CPUPID_SHIFT 0 - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ -- <= BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ -+ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS - #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT - #else - #define LAST_CPUPID_WIDTH 0 -@@ -100,10 +101,15 @@ - #define LAST_CPUPID_NOT_IN_PAGE_FLAGS - #endif - --#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ -- > BITS_PER_LONG - NR_PAGEFLAGS -+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ -+ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS - #error "Not enough bits in page flags" - #endif - -+/* see the comment on MAX_NR_TIERS */ -+#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ -+ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ -+ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) ++ if (likely(entry)) ++ return entry; ++ ++ if (unlikely(mas_prev_node(mas, min))) { ++ mas_rewalk(mas, mas->index); ++ goto retry; ++ } ++ ++ mas->offset++; ++ } ++ ++ mas->offset--; ++not_found: ++ mas->index = mas->last = min; ++ return NULL; ++} + - #endif - #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ -diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h -index 465ff35a8c00..0b0ae5084e60 100644 ---- a/include/linux/page-flags.h -+++ b/include/linux/page-flags.h -@@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) - 1UL << PG_private | 1UL << PG_private_2 | \ - 1UL << PG_writeback | 1UL << PG_reserved | \ - 1UL << PG_slab | 1UL << PG_active | \ -- 1UL << PG_unevictable | __PG_MLOCKED) -+ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) - - /* - * Flags checked when a page is prepped for return by the page allocator. -@@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) - * alloc-free cycle to prevent from reusing the page. - */ - #define PAGE_FLAGS_CHECK_AT_PREP \ -- (PAGEFLAGS_MASK & ~__PG_HWPOISON) -+ ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) - - #define PAGE_FLAGS_PRIVATE \ - (1UL << PG_private | 1UL << PG_private_2) -diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h -index 014ee8f0fbaa..d9095251bffd 100644 ---- a/include/linux/pgtable.h -+++ b/include/linux/pgtable.h -@@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - #endif - - #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG --#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) - static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) -@@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - BUILD_BUG(); - return 0; - } --#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -+#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ - #endif - - #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -@@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, - #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - #endif - -+#ifndef arch_has_hw_pte_young +/* -+ * Return whether the accessed bit is supported on the local CPU. ++ * mas_rev_awalk() - Internal function. Reverse allocation walk. Find the ++ * highest gap address of a given size in a given node and descend. ++ * @mas: The maple state ++ * @size: The needed size. ++ * ++ * Return: True if found in a leaf, false otherwise. + * -+ * This stub assumes accessing through an old PTE triggers a page fault. -+ * Architectures that automatically set the access bit should overwrite it. + */ -+static inline bool arch_has_hw_pte_young(void) ++static bool mas_rev_awalk(struct ma_state *mas, unsigned long size) +{ ++ enum maple_type type = mte_node_type(mas->node); ++ struct maple_node *node = mas_mn(mas); ++ unsigned long *pivots, *gaps; ++ void __rcu **slots; ++ unsigned long gap = 0; ++ unsigned long max, min, index; ++ unsigned char offset; ++ ++ if (unlikely(mas_is_err(mas))) ++ return true; ++ ++ if (ma_is_dense(type)) { ++ /* dense nodes. */ ++ mas->offset = (unsigned char)(mas->index - mas->min); ++ return true; ++ } ++ ++ pivots = ma_pivots(node, type); ++ slots = ma_slots(node, type); ++ gaps = ma_gaps(node, type); ++ offset = mas->offset; ++ min = mas_safe_min(mas, pivots, offset); ++ /* Skip out of bounds. */ ++ while (mas->last < min) ++ min = mas_safe_min(mas, pivots, --offset); ++ ++ max = mas_safe_pivot(mas, pivots, offset, type); ++ index = mas->index; ++ while (index <= max) { ++ gap = 0; ++ if (gaps) ++ gap = gaps[offset]; ++ else if (!mas_slot(mas, slots, offset)) ++ gap = max - min + 1; ++ ++ if (gap) { ++ if ((size <= gap) && (size <= mas->last - min + 1)) ++ break; ++ ++ if (!gaps) { ++ /* Skip the next slot, it cannot be a gap. */ ++ if (offset < 2) ++ goto ascend; ++ ++ offset -= 2; ++ max = pivots[offset]; ++ min = mas_safe_min(mas, pivots, offset); ++ continue; ++ } ++ } ++ ++ if (!offset) ++ goto ascend; ++ ++ offset--; ++ max = min - 1; ++ min = mas_safe_min(mas, pivots, offset); ++ } ++ ++ if (unlikely(index > max)) { ++ mas_set_err(mas, -EBUSY); ++ return false; ++ } ++ ++ if (unlikely(ma_is_leaf(type))) { ++ mas->offset = offset; ++ mas->min = min; ++ mas->max = min + gap - 1; ++ return true; ++ } ++ ++ /* descend, only happens under lock. */ ++ mas->node = mas_slot(mas, slots, offset); ++ mas->min = min; ++ mas->max = max; ++ mas->offset = mas_data_end(mas); ++ return false; ++ ++ascend: ++ if (mte_is_root(mas->node)) ++ mas_set_err(mas, -EBUSY); ++ + return false; +} -+#endif + - #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR - static inline pte_t ptep_get_and_clear(struct mm_struct *mm, - unsigned long address, -diff --git a/include/linux/sched.h b/include/linux/sched.h -index 367b30ed77cb..070b15ba2e1e 100644 ---- a/include/linux/sched.h -+++ b/include/linux/sched.h -@@ -914,6 +914,10 @@ struct task_struct { - #ifdef CONFIG_MEMCG - unsigned in_user_fault:1; - #endif -+#ifdef CONFIG_LRU_GEN -+ /* whether the LRU algorithm may apply to this access */ -+ unsigned in_lru_fault:1; -+#endif - #ifdef CONFIG_COMPAT_BRK - unsigned brk_randomized:1; - #endif -diff --git a/include/linux/swap.h b/include/linux/swap.h -index 43150b9bbc5c..6308150b234a 100644 ---- a/include/linux/swap.h -+++ b/include/linux/swap.h -@@ -162,6 +162,10 @@ union swap_header { - */ - struct reclaim_state { - unsigned long reclaimed_slab; -+#ifdef CONFIG_LRU_GEN -+ /* per-thread mm walk data */ -+ struct lru_gen_mm_walk *mm_walk; -+#endif - }; - - #ifdef __KERNEL__ -diff --git a/kernel/bounds.c b/kernel/bounds.c -index 9795d75b09b2..b529182e8b04 100644 ---- a/kernel/bounds.c -+++ b/kernel/bounds.c -@@ -22,6 +22,13 @@ int main(void) - DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); - #endif - DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); -+#ifdef CONFIG_LRU_GEN -+ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); -+ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); -+#else -+ DEFINE(LRU_GEN_WIDTH, 0); -+ DEFINE(__LRU_REFS_WIDTH, 0); -+#endif - /* End of constants */ - - return 0; -diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h -index 36b740cb3d59..63dc3e82be4f 100644 ---- a/kernel/cgroup/cgroup-internal.h -+++ b/kernel/cgroup/cgroup-internal.h -@@ -164,7 +164,6 @@ struct cgroup_mgctx { - #define DEFINE_CGROUP_MGCTX(name) \ - struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) - --extern struct mutex cgroup_mutex; - extern spinlock_t css_set_lock; - extern struct cgroup_subsys *cgroup_subsys[]; - extern struct list_head cgroup_roots; -diff --git a/kernel/exit.c b/kernel/exit.c -index 84021b24f79e..98a33bd7c25c 100644 ---- a/kernel/exit.c -+++ b/kernel/exit.c -@@ -466,6 +466,7 @@ void mm_update_next_owner(struct mm_struct *mm) - goto retry; - } - WRITE_ONCE(mm->owner, c); -+ lru_gen_migrate_mm(mm); - task_unlock(c); - put_task_struct(c); - } -diff --git a/kernel/fork.c b/kernel/fork.c -index 704fe6bc9cb4..b15aaae04403 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -1156,6 +1156,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - goto fail_nocontext; - - mm->user_ns = get_user_ns(user_ns); -+ lru_gen_init_mm(mm); - return mm; - - fail_nocontext: -@@ -1198,6 +1199,7 @@ static inline void __mmput(struct mm_struct *mm) - } - if (mm->binfmt) - module_put(mm->binfmt->module); -+ lru_gen_del_mm(mm); - mmdrop(mm); - } - -@@ -2700,6 +2702,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) - get_task_struct(p); - } - -+ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { -+ /* lock the task to synchronize with memcg migration */ -+ task_lock(p); -+ lru_gen_add_mm(p->mm); -+ task_unlock(p); ++static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) ++{ ++ enum maple_type type = mte_node_type(mas->node); ++ unsigned long pivot, min, gap = 0; ++ unsigned char count, offset; ++ unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type); ++ void __rcu **slots = ma_slots(mas_mn(mas), type); ++ bool found = false; ++ ++ if (ma_is_dense(type)) { ++ mas->offset = (unsigned char)(mas->index - mas->min); ++ return true; + } + - wake_up_new_task(p); - - /* forking complete and child started to run, tell ptracer */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index f2534e712a89..265944f14948 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -5178,6 +5178,7 @@ context_switch(struct rq *rq, struct task_struct *prev, - * finish_task_switch()'s mmdrop(). - */ - switch_mm_irqs_off(prev->active_mm, next->mm, next); -+ lru_gen_use_mm(next->mm); - - if (!prev->mm) { // from kernel - /* will mmdrop() in finish_task_switch(). */ -diff --git a/mm/Kconfig b/mm/Kconfig -index 0331f1461f81..96cd3ae25c6f 100644 ---- a/mm/Kconfig -+++ b/mm/Kconfig -@@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP - purposes. It is required to enable userfaultfd write protection on - file-backed memory types like shmem and hugetlbfs. - -+# multi-gen LRU { -+config LRU_GEN -+ bool "Multi-Gen LRU" -+ depends on MMU -+ # make sure folio->flags has enough spare bits -+ depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP -+ help -+ A high performance LRU implementation to overcommit memory. See -+ Documentation/admin-guide/mm/multigen_lru.rst for details. ++ gaps = ma_gaps(mte_to_node(mas->node), type); ++ offset = mas->offset; ++ count = mt_slots[type]; ++ min = mas_safe_min(mas, pivots, offset); ++ for (; offset < count; offset++) { ++ pivot = mas_safe_pivot(mas, pivots, offset, type); ++ if (offset && !pivot) ++ break; + -+config LRU_GEN_ENABLED -+ bool "Enable by default" -+ depends on LRU_GEN -+ help -+ This option enables the multi-gen LRU by default. ++ /* Not within lower bounds */ ++ if (mas->index > pivot) ++ goto next_slot; + -+config LRU_GEN_STATS -+ bool "Full stats for debugging" -+ depends on LRU_GEN -+ help -+ Do not enable this option unless you plan to look at historical stats -+ from evicted generations for debugging purpose. ++ if (gaps) ++ gap = gaps[offset]; ++ else if (!mas_slot(mas, slots, offset)) ++ gap = min(pivot, mas->last) - max(mas->index, min) + 1; ++ else ++ goto next_slot; + -+ This option has a per-memcg and per-node memory overhead. -+# } ++ if (gap >= size) { ++ if (ma_is_leaf(type)) { ++ found = true; ++ goto done; ++ } ++ if (mas->index <= pivot) { ++ mas->node = mas_slot(mas, slots, offset); ++ mas->min = min; ++ mas->max = pivot; ++ offset = 0; ++ type = mte_node_type(mas->node); ++ count = mt_slots[type]; ++ break; ++ } ++ } ++next_slot: ++ min = pivot + 1; ++ if (mas->last <= pivot) { ++ mas_set_err(mas, -EBUSY); ++ return true; ++ } ++ } + - source "mm/damon/Kconfig" - - endmenu -diff --git a/mm/huge_memory.c b/mm/huge_memory.c -index e9414ee57c5b..1eee94e39d0c 100644 ---- a/mm/huge_memory.c -+++ b/mm/huge_memory.c -@@ -2438,7 +2438,8 @@ static void __split_huge_page_tail(struct page *head, int tail, - #ifdef CONFIG_64BIT - (1L << PG_arch_2) | - #endif -- (1L << PG_dirty))); -+ (1L << PG_dirty) | -+ LRU_GEN_MASK | LRU_REFS_MASK)); - - /* ->mapping in first tail page is compound_mapcount */ - VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, -diff --git a/mm/internal.h b/mm/internal.h -index 785409805ed7..a1fddea6b34f 100644 ---- a/mm/internal.h -+++ b/mm/internal.h -@@ -83,6 +83,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); - void folio_rotate_reclaimable(struct folio *folio); - bool __folio_end_writeback(struct folio *folio); - void deactivate_file_folio(struct folio *folio); -+void folio_activate(struct folio *folio); - - void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, - unsigned long floor, unsigned long ceiling); -diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index 7eadbafc006b..48588e89867a 100644 ---- a/mm/memcontrol.c -+++ b/mm/memcontrol.c -@@ -2789,6 +2789,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) - * - LRU isolation - * - lock_page_memcg() - * - exclusive reference -+ * - mem_cgroup_trylock_pages() - */ - folio->memcg_data = (unsigned long)memcg; - } -@@ -5170,6 +5171,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) - - static void mem_cgroup_free(struct mem_cgroup *memcg) - { -+ lru_gen_exit_memcg(memcg); - memcg_wb_domain_exit(memcg); - __mem_cgroup_free(memcg); - } -@@ -5228,6 +5230,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) - memcg->deferred_split_queue.split_queue_len = 0; - #endif - idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); -+ lru_gen_init_memcg(memcg); - return memcg; - fail: - mem_cgroup_id_remove(memcg); -@@ -6196,6 +6199,30 @@ static void mem_cgroup_move_task(void) - } - #endif - -+#ifdef CONFIG_LRU_GEN -+static void mem_cgroup_attach(struct cgroup_taskset *tset) ++ if (mte_is_root(mas->node)) ++ found = true; ++done: ++ mas->offset = offset; ++ return found; ++} ++ ++/** ++ * mas_walk() - Search for @mas->index in the tree. ++ * @mas: The maple state. ++ * ++ * mas->index and mas->last will be set to the range if there is a value. If ++ * mas->node is MAS_NONE, reset to MAS_START. ++ * ++ * Return: the entry at the location or %NULL. ++ */ ++void *mas_walk(struct ma_state *mas) +{ -+ struct task_struct *task; -+ struct cgroup_subsys_state *css; ++ void *entry; + -+ /* find the first leader if there is any */ -+ cgroup_taskset_for_each_leader(task, css, tset) -+ break; ++retry: ++ entry = mas_state_walk(mas); ++ if (mas_is_start(mas)) ++ goto retry; + -+ if (!task) -+ return; ++ if (mas_is_ptr(mas)) { ++ if (!mas->index) { ++ mas->last = 0; ++ } else { ++ mas->index = 1; ++ mas->last = ULONG_MAX; ++ } ++ return entry; ++ } + -+ task_lock(task); -+ if (task->mm && READ_ONCE(task->mm->owner) == task) -+ lru_gen_migrate_mm(task->mm); -+ task_unlock(task); ++ if (mas_is_none(mas)) { ++ mas->index = 0; ++ mas->last = ULONG_MAX; ++ } ++ ++ return entry; +} -+#else -+static void mem_cgroup_attach(struct cgroup_taskset *tset) ++ ++static inline bool mas_rewind_node(struct ma_state *mas) +{ ++ unsigned char slot; ++ ++ do { ++ if (mte_is_root(mas->node)) { ++ slot = mas->offset; ++ if (!slot) ++ return false; ++ } else { ++ mas_ascend(mas); ++ slot = mas->offset; ++ } ++ } while (!slot); ++ ++ mas->offset = --slot; ++ return true; +} -+#endif /* CONFIG_LRU_GEN */ + - static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) - { - if (value == PAGE_COUNTER_MAX) -@@ -6601,6 +6628,7 @@ struct cgroup_subsys memory_cgrp_subsys = { - .css_reset = mem_cgroup_css_reset, - .css_rstat_flush = mem_cgroup_css_rstat_flush, - .can_attach = mem_cgroup_can_attach, -+ .attach = mem_cgroup_attach, - .cancel_attach = mem_cgroup_cancel_attach, - .post_attach = mem_cgroup_move_task, - .dfl_cftypes = memory_files, -diff --git a/mm/memory.c b/mm/memory.c -index 4ba73f5aa8bb..47daee6b421f 100644 ---- a/mm/memory.c -+++ b/mm/memory.c -@@ -125,18 +125,6 @@ int randomize_va_space __read_mostly = - 2; - #endif - --#ifndef arch_faults_on_old_pte --static inline bool arch_faults_on_old_pte(void) --{ -- /* -- * Those arches which don't have hw access flag feature need to -- * implement their own helper. By default, "true" means pagefault -- * will be hit on old pte. -- */ -- return true; --} --#endif -- - #ifndef arch_wants_old_prefaulted_pte - static inline bool arch_wants_old_prefaulted_pte(void) - { -@@ -2870,7 +2858,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, - * On architectures with software "accessed" bits, we would - * take a double page fault, so mark it accessed here. - */ -- if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { -+ if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { - pte_t entry; - - vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); -@@ -5114,6 +5102,27 @@ static inline void mm_account_fault(struct pt_regs *regs, - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); - } - -+#ifdef CONFIG_LRU_GEN -+static void lru_gen_enter_fault(struct vm_area_struct *vma) ++/* ++ * mas_skip_node() - Internal function. Skip over a node. ++ * @mas: The maple state. ++ * ++ * Return: true if there is another node, false otherwise. ++ */ ++static inline bool mas_skip_node(struct ma_state *mas) +{ -+ /* the LRU algorithm doesn't apply to sequential or random reads */ -+ current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); ++ unsigned char slot, slot_count; ++ unsigned long *pivots; ++ enum maple_type mt; ++ ++ mt = mte_node_type(mas->node); ++ slot_count = mt_slots[mt] - 1; ++ do { ++ if (mte_is_root(mas->node)) { ++ slot = mas->offset; ++ if (slot > slot_count) { ++ mas_set_err(mas, -EBUSY); ++ return false; ++ } ++ } else { ++ mas_ascend(mas); ++ slot = mas->offset; ++ mt = mte_node_type(mas->node); ++ slot_count = mt_slots[mt] - 1; ++ } ++ } while (slot > slot_count); ++ ++ mas->offset = ++slot; ++ pivots = ma_pivots(mas_mn(mas), mt); ++ if (slot > 0) ++ mas->min = pivots[slot - 1] + 1; ++ ++ if (slot <= slot_count) ++ mas->max = pivots[slot]; ++ ++ return true; +} + -+static void lru_gen_exit_fault(void) ++/* ++ * mas_awalk() - Allocation walk. Search from low address to high, for a gap of ++ * @size ++ * @mas: The maple state ++ * @size: The size of the gap required ++ * ++ * Search between @mas->index and @mas->last for a gap of @size. ++ */ ++static inline void mas_awalk(struct ma_state *mas, unsigned long size) +{ -+ current->in_lru_fault = false; ++ struct maple_enode *last = NULL; ++ ++ /* ++ * There are 4 options: ++ * go to child (descend) ++ * go back to parent (ascend) ++ * no gap found. (return, slot == MAPLE_NODE_SLOTS) ++ * found the gap. (return, slot != MAPLE_NODE_SLOTS) ++ */ ++ while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) { ++ if (last == mas->node) ++ mas_skip_node(mas); ++ else ++ last = mas->node; ++ } +} -+#else -+static void lru_gen_enter_fault(struct vm_area_struct *vma) ++ ++/* ++ * mas_fill_gap() - Fill a located gap with @entry. ++ * @mas: The maple state ++ * @entry: The value to store ++ * @slot: The offset into the node to store the @entry ++ * @size: The size of the entry ++ * @index: The start location ++ */ ++static inline void mas_fill_gap(struct ma_state *mas, void *entry, ++ unsigned char slot, unsigned long size, unsigned long *index) ++{ ++ MA_WR_STATE(wr_mas, mas, entry); ++ unsigned char pslot = mte_parent_slot(mas->node); ++ struct maple_enode *mn = mas->node; ++ unsigned long *pivots; ++ enum maple_type ptype; ++ /* ++ * mas->index is the start address for the search ++ * which may no longer be needed. ++ * mas->last is the end address for the search ++ */ ++ ++ *index = mas->index; ++ mas->last = mas->index + size - 1; ++ ++ /* ++ * It is possible that using mas->max and mas->min to correctly ++ * calculate the index and last will cause an issue in the gap ++ * calculation, so fix the ma_state here ++ */ ++ mas_ascend(mas); ++ ptype = mte_node_type(mas->node); ++ pivots = ma_pivots(mas_mn(mas), ptype); ++ mas->max = mas_safe_pivot(mas, pivots, pslot, ptype); ++ mas->min = mas_safe_min(mas, pivots, pslot); ++ mas->node = mn; ++ mas->offset = slot; ++ mas_wr_store_entry(&wr_mas); ++} ++ ++/* ++ * mas_sparse_area() - Internal function. Return upper or lower limit when ++ * searching for a gap in an empty tree. ++ * @mas: The maple state ++ * @min: the minimum range ++ * @max: The maximum range ++ * @size: The size of the gap ++ * @fwd: Searching forward or back ++ */ ++static inline void mas_sparse_area(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size, bool fwd) +{ ++ unsigned long start = 0; ++ ++ if (!unlikely(mas_is_none(mas))) ++ start++; ++ /* mas_is_ptr */ ++ ++ if (start < min) ++ start = min; ++ ++ if (fwd) { ++ mas->index = start; ++ mas->last = start + size - 1; ++ return; ++ } ++ ++ mas->index = max; +} + -+static void lru_gen_exit_fault(void) ++/* ++ * mas_empty_area() - Get the lowest address within the range that is ++ * sufficient for the size requested. ++ * @mas: The maple state ++ * @min: The lowest value of the range ++ * @max: The highest value of the range ++ * @size: The size needed ++ */ ++int mas_empty_area(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size) ++{ ++ unsigned char offset; ++ unsigned long *pivots; ++ enum maple_type mt; ++ ++ if (mas_is_start(mas)) ++ mas_start(mas); ++ else if (mas->offset >= 2) ++ mas->offset -= 2; ++ else if (!mas_skip_node(mas)) ++ return -EBUSY; ++ ++ /* Empty set */ ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_sparse_area(mas, min, max, size, true); ++ return 0; ++ } ++ ++ /* The start of the window can only be within these values */ ++ mas->index = min; ++ mas->last = max; ++ mas_awalk(mas, size); ++ ++ if (unlikely(mas_is_err(mas))) ++ return xa_err(mas->node); ++ ++ offset = mas->offset; ++ if (unlikely(offset == MAPLE_NODE_SLOTS)) ++ return -EBUSY; ++ ++ mt = mte_node_type(mas->node); ++ pivots = ma_pivots(mas_mn(mas), mt); ++ if (offset) ++ mas->min = pivots[offset - 1] + 1; ++ ++ if (offset < mt_pivots[mt]) ++ mas->max = pivots[offset]; ++ ++ if (mas->index < mas->min) ++ mas->index = mas->min; ++ ++ mas->last = mas->index + size - 1; ++ return 0; ++} ++ ++/* ++ * mas_empty_area_rev() - Get the highest address within the range that is ++ * sufficient for the size requested. ++ * @mas: The maple state ++ * @min: The lowest value of the range ++ * @max: The highest value of the range ++ * @size: The size needed ++ */ ++int mas_empty_area_rev(struct ma_state *mas, unsigned long min, ++ unsigned long max, unsigned long size) +{ ++ struct maple_enode *last = mas->node; ++ ++ if (mas_is_start(mas)) { ++ mas_start(mas); ++ mas->offset = mas_data_end(mas); ++ } else if (mas->offset >= 2) { ++ mas->offset -= 2; ++ } else if (!mas_rewind_node(mas)) { ++ return -EBUSY; ++ } ++ ++ /* Empty set. */ ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_sparse_area(mas, min, max, size, false); ++ return 0; ++ } ++ ++ /* The start of the window can only be within these values. */ ++ mas->index = min; ++ mas->last = max; ++ ++ while (!mas_rev_awalk(mas, size)) { ++ if (last == mas->node) { ++ if (!mas_rewind_node(mas)) ++ return -EBUSY; ++ } else { ++ last = mas->node; ++ } ++ } ++ ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); ++ ++ if (unlikely(mas->offset == MAPLE_NODE_SLOTS)) ++ return -EBUSY; ++ ++ /* ++ * mas_rev_awalk() has set mas->min and mas->max to the gap values. If ++ * the maximum is outside the window we are searching, then use the last ++ * location in the search. ++ * mas->max and mas->min is the range of the gap. ++ * mas->index and mas->last are currently set to the search range. ++ */ ++ ++ /* Trim the upper limit to the max. */ ++ if (mas->max <= mas->last) ++ mas->last = mas->max; ++ ++ mas->index = mas->last - size + 1; ++ return 0; +} -+#endif /* CONFIG_LRU_GEN */ + - /* - * By the time we get here, we already hold the mm semaphore - * -@@ -5145,11 +5154,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, - if (flags & FAULT_FLAG_USER) - mem_cgroup_enter_user_fault(); - -+ lru_gen_enter_fault(vma); ++static inline int mas_alloc(struct ma_state *mas, void *entry, ++ unsigned long size, unsigned long *index) ++{ ++ unsigned long min; + - if (unlikely(is_vm_hugetlb_page(vma))) - ret = hugetlb_fault(vma->vm_mm, vma, address, flags); - else - ret = __handle_mm_fault(vma, address, flags); - -+ lru_gen_exit_fault(); ++ mas_start(mas); ++ if (mas_is_none(mas) || mas_is_ptr(mas)) { ++ mas_root_expand(mas, entry); ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); + - if (flags & FAULT_FLAG_USER) { - mem_cgroup_exit_user_fault(); - /* -diff --git a/mm/mm_init.c b/mm/mm_init.c -index 9ddaf0e1b0ab..0d7b2bd2454a 100644 ---- a/mm/mm_init.c -+++ b/mm/mm_init.c -@@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) - - shift = 8 * sizeof(unsigned long); - width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH -- - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; -+ - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; - mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", -- "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", -+ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", - SECTIONS_WIDTH, - NODES_WIDTH, - ZONES_WIDTH, - LAST_CPUPID_WIDTH, - KASAN_TAG_WIDTH, -+ LRU_GEN_WIDTH, -+ LRU_REFS_WIDTH, - NR_PAGEFLAGS); - mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", - "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", -diff --git a/mm/mmzone.c b/mm/mmzone.c -index 0ae7571e35ab..68e1511be12d 100644 ---- a/mm/mmzone.c -+++ b/mm/mmzone.c -@@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) - * Poison its list head, so that any operations on it would crash. - */ - list_del(&lruvec->lists[LRU_UNEVICTABLE]); ++ if (!mas->index) ++ return mte_pivot(mas->node, 0); ++ return mte_pivot(mas->node, 1); ++ } + -+ lru_gen_init_lruvec(lruvec); - } - - #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) -diff --git a/mm/rmap.c b/mm/rmap.c -index 93d5a6f793d2..9e0ce48bca08 100644 ---- a/mm/rmap.c -+++ b/mm/rmap.c -@@ -833,6 +833,12 @@ static bool folio_referenced_one(struct folio *folio, - } - - if (pvmw.pte) { -+ if (lru_gen_enabled() && pte_young(*pvmw.pte) && -+ !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { -+ lru_gen_look_around(&pvmw); -+ referenced++; -+ } ++ /* Must be walking a tree. */ ++ mas_awalk(mas, size); ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); + - if (ptep_clear_flush_young_notify(vma, address, - pvmw.pte)) { - /* -diff --git a/mm/swap.c b/mm/swap.c -index 186b4e5dcecf..027943b341a0 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu) - folio_batch_move_lru(fbatch, folio_activate_fn); - } - --static void folio_activate(struct folio *folio) -+void folio_activate(struct folio *folio) - { - if (folio_test_lru(folio) && !folio_test_active(folio) && - !folio_test_unevictable(folio)) { -@@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu) - { - } - --static void folio_activate(struct folio *folio) -+void folio_activate(struct folio *folio) - { - struct lruvec *lruvec; - -@@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio) - local_unlock(&cpu_fbatches.lock); - } - -+#ifdef CONFIG_LRU_GEN -+static void folio_inc_refs(struct folio *folio) ++ if (mas->offset == MAPLE_NODE_SLOTS) ++ goto no_gap; ++ ++ /* ++ * At this point, mas->node points to the right node and we have an ++ * offset that has a sufficient gap. ++ */ ++ min = mas->min; ++ if (mas->offset) ++ min = mte_pivot(mas->node, mas->offset - 1) + 1; ++ ++ if (mas->index < min) ++ mas->index = min; ++ ++ mas_fill_gap(mas, entry, mas->offset, size, index); ++ return 0; ++ ++no_gap: ++ return -EBUSY; ++} ++ ++static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min, ++ unsigned long max, void *entry, ++ unsigned long size, unsigned long *index) +{ -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); ++ int ret = 0; ++ ++ ret = mas_empty_area_rev(mas, min, max, size); ++ if (ret) ++ return ret; + -+ if (folio_test_unevictable(folio)) -+ return; ++ if (mas_is_err(mas)) ++ return xa_err(mas->node); + -+ if (!folio_test_referenced(folio)) { -+ folio_set_referenced(folio); -+ return; -+ } ++ if (mas->offset == MAPLE_NODE_SLOTS) ++ goto no_gap; + -+ if (!folio_test_workingset(folio)) { -+ folio_set_workingset(folio); -+ return; -+ } ++ mas_fill_gap(mas, entry, mas->offset, size, index); ++ return 0; + -+ /* see the comment on MAX_NR_TIERS */ -+ do { -+ new_flags = old_flags & LRU_REFS_MASK; -+ if (new_flags == LRU_REFS_MASK) ++no_gap: ++ return -EBUSY; ++} ++ ++/* ++ * mas_dead_leaves() - Mark all leaves of a node as dead. ++ * @mas: The maple state ++ * @slots: Pointer to the slot array ++ * ++ * Must hold the write lock. ++ * ++ * Return: The number of leaves marked as dead. ++ */ ++static inline ++unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots) ++{ ++ struct maple_node *node; ++ enum maple_type type; ++ void *entry; ++ int offset; ++ ++ for (offset = 0; offset < mt_slot_count(mas->node); offset++) { ++ entry = mas_slot_locked(mas, slots, offset); ++ type = mte_node_type(entry); ++ node = mte_to_node(entry); ++ /* Use both node and type to catch LE & BE metadata */ ++ if (!node || !type) + break; + -+ new_flags += BIT(LRU_REFS_PGOFF); -+ new_flags |= old_flags & ~LRU_REFS_MASK; -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); ++ mte_set_node_dead(entry); ++ smp_wmb(); /* Needed for RCU */ ++ node->type = type; ++ rcu_assign_pointer(slots[offset], node); ++ } ++ ++ return offset; +} -+#else -+static void folio_inc_refs(struct folio *folio) ++ ++static void __rcu **mas_dead_walk(struct ma_state *mas, unsigned char offset) +{ -+} -+#endif /* CONFIG_LRU_GEN */ ++ struct maple_node *node, *next; ++ void __rcu **slots = NULL; + - /* - * Mark a page as having seen activity. - * -@@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio) - */ - void folio_mark_accessed(struct folio *folio) - { -+ if (lru_gen_enabled()) { -+ folio_inc_refs(folio); -+ return; -+ } ++ next = mas_mn(mas); ++ do { ++ mas->node = ma_enode_ptr(next); ++ node = mas_mn(mas); ++ slots = ma_slots(node, node->type); ++ next = mas_slot_locked(mas, slots, offset); ++ offset = 0; ++ } while (!ma_is_leaf(next->type)); + - if (!folio_test_referenced(folio)) { - folio_set_referenced(folio); - } else if (folio_test_unevictable(folio)) { -@@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio) - folio_test_unevictable(folio), folio); - VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); - -+ /* see the comment in lru_gen_add_folio() */ -+ if (lru_gen_enabled() && !folio_test_unevictable(folio) && -+ lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) -+ folio_set_active(folio); ++ return slots; ++} + - folio_get(folio); - local_lock(&cpu_fbatches.lock); - fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); -@@ -575,7 +619,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) - - static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) - { -- if (folio_test_active(folio) && !folio_test_unevictable(folio)) { -+ if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { - long nr_pages = folio_nr_pages(folio); - - lruvec_del_folio(lruvec, folio); -@@ -688,8 +732,8 @@ void deactivate_page(struct page *page) - { - struct folio *folio = page_folio(page); - -- if (folio_test_lru(folio) && folio_test_active(folio) && -- !folio_test_unevictable(folio)) { -+ if (folio_test_lru(folio) && !folio_test_unevictable(folio) && -+ (folio_test_active(folio) || lru_gen_enabled())) { - struct folio_batch *fbatch; - - folio_get(folio); -diff --git a/mm/vmscan.c b/mm/vmscan.c -index 0fc65ace3a4e..0ae7458b1e55 100644 ---- a/mm/vmscan.c -+++ b/mm/vmscan.c -@@ -49,6 +49,10 @@ - #include - #include - #include -+#include -+#include -+#include -+#include - - #include - #include -@@ -129,6 +133,12 @@ struct scan_control { - /* Always discard instead of demoting to lower tier memory */ - unsigned int no_demotion:1; - -+#ifdef CONFIG_LRU_GEN -+ /* help kswapd make better choices among multiple memcgs */ -+ unsigned int memcgs_need_aging:1; -+ unsigned long last_reclaimed; -+#endif ++static void mt_free_walk(struct rcu_head *head) ++{ ++ void __rcu **slots; ++ struct maple_node *node, *start; ++ struct maple_tree mt; ++ unsigned char offset; ++ enum maple_type type; ++ MA_STATE(mas, &mt, 0, 0); + - /* Allocation order */ - s8 order; - -@@ -1338,9 +1348,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, - - if (folio_test_swapcache(folio)) { - swp_entry_t swap = folio_swap_entry(folio); -- mem_cgroup_swapout(folio, swap); ++ node = container_of(head, struct maple_node, rcu); + -+ /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ - if (reclaimed && !mapping_exiting(mapping)) - shadow = workingset_eviction(folio, target_memcg); -+ mem_cgroup_swapout(folio, swap); - __delete_from_swap_cache(folio, swap, shadow); - xa_unlock_irq(&mapping->i_pages); - put_swap_page(&folio->page, swap); -@@ -1637,6 +1649,11 @@ static unsigned int shrink_page_list(struct list_head *page_list, - if (!sc->may_unmap && folio_mapped(folio)) - goto keep_locked; - -+ /* folio_update_gen() tried to promote this page? */ -+ if (lru_gen_enabled() && !ignore_references && -+ folio_mapped(folio) && folio_test_referenced(folio)) -+ goto keep_locked; ++ if (ma_is_leaf(node->type)) ++ goto free_leaf; + - /* - * The number of dirty pages determines if a node is marked - * reclaim_congested. kswapd will stall and start writing -@@ -2732,6 +2749,112 @@ enum scan_balance { - SCAN_FILE, - }; - -+static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) -+{ -+ unsigned long file; -+ struct lruvec *target_lruvec; ++ mt_init_flags(&mt, node->ma_flags); ++ mas_lock(&mas); ++ start = node; ++ mas.node = mt_mk_node(node, node->type); ++ slots = mas_dead_walk(&mas, 0); ++ node = mas_mn(&mas); ++ do { ++ mt_free_bulk(node->slot_len, slots); ++ offset = node->parent_slot + 1; ++ mas.node = node->piv_parent; ++ if (mas_mn(&mas) == node) ++ goto start_slots_free; + -+ if (lru_gen_enabled()) -+ return; ++ type = mte_node_type(mas.node); ++ slots = ma_slots(mte_to_node(mas.node), type); ++ if ((offset < mt_slots[type]) && (slots[offset])) ++ slots = mas_dead_walk(&mas, offset); + -+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); ++ node = mas_mn(&mas); ++ } while ((node != start) || (node->slot_len < offset)); + -+ /* -+ * Flush the memory cgroup stats, so that we read accurate per-memcg -+ * lruvec stats for heuristics. -+ */ -+ mem_cgroup_flush_stats(); ++ slots = ma_slots(node, node->type); ++ mt_free_bulk(node->slot_len, slots); + -+ /* -+ * Determine the scan balance between anon and file LRUs. -+ */ -+ spin_lock_irq(&target_lruvec->lru_lock); -+ sc->anon_cost = target_lruvec->anon_cost; -+ sc->file_cost = target_lruvec->file_cost; -+ spin_unlock_irq(&target_lruvec->lru_lock); ++start_slots_free: ++ mas_unlock(&mas); ++free_leaf: ++ mt_free_rcu(&node->rcu); ++} + -+ /* -+ * Target desirable inactive:active list ratios for the anon -+ * and file LRU lists. -+ */ -+ if (!sc->force_deactivate) { -+ unsigned long refaults; ++static inline void __rcu **mas_destroy_descend(struct ma_state *mas, ++ struct maple_enode *prev, unsigned char offset) ++{ ++ struct maple_node *node; ++ struct maple_enode *next = mas->node; ++ void __rcu **slots = NULL; + -+ /* -+ * When refaults are being observed, it means a new -+ * workingset is being established. Deactivate to get -+ * rid of any stale active pages quickly. -+ */ -+ refaults = lruvec_page_state(target_lruvec, -+ WORKINGSET_ACTIVATE_ANON); -+ if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || -+ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) -+ sc->may_deactivate |= DEACTIVATE_ANON; -+ else -+ sc->may_deactivate &= ~DEACTIVATE_ANON; ++ do { ++ mas->node = next; ++ node = mas_mn(mas); ++ slots = ma_slots(node, mte_node_type(mas->node)); ++ next = mas_slot_locked(mas, slots, 0); ++ if ((mte_dead_node(next))) ++ next = mas_slot_locked(mas, slots, 1); + -+ refaults = lruvec_page_state(target_lruvec, -+ WORKINGSET_ACTIVATE_FILE); -+ if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || -+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) -+ sc->may_deactivate |= DEACTIVATE_FILE; -+ else -+ sc->may_deactivate &= ~DEACTIVATE_FILE; -+ } else -+ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; ++ mte_set_node_dead(mas->node); ++ node->type = mte_node_type(mas->node); ++ node->piv_parent = prev; ++ node->parent_slot = offset; ++ offset = 0; ++ prev = mas->node; ++ } while (!mte_is_leaf(next)); + -+ /* -+ * If we have plenty of inactive file pages that aren't -+ * thrashing, try to reclaim those first before touching -+ * anonymous pages. -+ */ -+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); -+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) -+ sc->cache_trim_mode = 1; -+ else -+ sc->cache_trim_mode = 0; ++ return slots; ++} + -+ /* -+ * Prevent the reclaimer from falling into the cache trap: as -+ * cache pages start out inactive, every cache fault will tip -+ * the scan balance towards the file LRU. And as the file LRU -+ * shrinks, so does the window for rotation from references. -+ * This means we have a runaway feedback loop where a tiny -+ * thrashing file LRU becomes infinitely more attractive than -+ * anon pages. Try to detect this based on file LRU size. -+ */ -+ if (!cgroup_reclaim(sc)) { -+ unsigned long total_high_wmark = 0; -+ unsigned long free, anon; -+ int z; ++static void mt_destroy_walk(struct maple_enode *enode, unsigned char ma_flags, ++ bool free) ++{ ++ void __rcu **slots; ++ struct maple_node *node = mte_to_node(enode); ++ struct maple_enode *start; ++ struct maple_tree mt; + -+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); -+ file = node_page_state(pgdat, NR_ACTIVE_FILE) + -+ node_page_state(pgdat, NR_INACTIVE_FILE); ++ MA_STATE(mas, &mt, 0, 0); + -+ for (z = 0; z < MAX_NR_ZONES; z++) { -+ struct zone *zone = &pgdat->node_zones[z]; ++ if (mte_is_leaf(enode)) ++ goto free_leaf; + -+ if (!managed_zone(zone)) -+ continue; ++ mt_init_flags(&mt, ma_flags); ++ mas_lock(&mas); + -+ total_high_wmark += high_wmark_pages(zone); ++ mas.node = start = enode; ++ slots = mas_destroy_descend(&mas, start, 0); ++ node = mas_mn(&mas); ++ do { ++ enum maple_type type; ++ unsigned char offset; ++ struct maple_enode *parent, *tmp; ++ ++ node->slot_len = mas_dead_leaves(&mas, slots); ++ if (free) ++ mt_free_bulk(node->slot_len, slots); ++ offset = node->parent_slot + 1; ++ mas.node = node->piv_parent; ++ if (mas_mn(&mas) == node) ++ goto start_slots_free; ++ ++ type = mte_node_type(mas.node); ++ slots = ma_slots(mte_to_node(mas.node), type); ++ if (offset >= mt_slots[type]) ++ goto next; ++ ++ tmp = mas_slot_locked(&mas, slots, offset); ++ if (mte_node_type(tmp) && mte_to_node(tmp)) { ++ parent = mas.node; ++ mas.node = tmp; ++ slots = mas_destroy_descend(&mas, parent, offset); + } ++next: ++ node = mas_mn(&mas); ++ } while (start != mas.node); + -+ /* -+ * Consider anon: if that's low too, this isn't a -+ * runaway file reclaim problem, but rather just -+ * extreme pressure. Reclaim as per usual then. -+ */ -+ anon = node_page_state(pgdat, NR_INACTIVE_ANON); ++ node = mas_mn(&mas); ++ node->slot_len = mas_dead_leaves(&mas, slots); ++ if (free) ++ mt_free_bulk(node->slot_len, slots); + -+ sc->file_is_tiny = -+ file + free <= total_high_wmark && -+ !(sc->may_deactivate & DEACTIVATE_ANON) && -+ anon >> sc->priority; ++start_slots_free: ++ mas_unlock(&mas); ++ ++free_leaf: ++ if (free) ++ mt_free_rcu(&node->rcu); ++} ++ ++/* ++ * mte_destroy_walk() - Free a tree or sub-tree. ++ * @enode - the encoded maple node (maple_enode) to start ++ * @mn - the tree to free - needed for node types. ++ * ++ * Must hold the write lock. ++ */ ++static inline void mte_destroy_walk(struct maple_enode *enode, ++ struct maple_tree *mt) ++{ ++ struct maple_node *node = mte_to_node(enode); ++ ++ if (mt_in_rcu(mt)) { ++ mt_destroy_walk(enode, mt->ma_flags, false); ++ call_rcu(&node->rcu, mt_free_walk); ++ } else { ++ mt_destroy_walk(enode, mt->ma_flags, true); + } +} + - /* - * Determine how aggressively the anon and file LRU lists should be - * scanned. -@@ -2951,159 +3074,2912 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, - return can_demote(pgdat->node_id, sc); - } - --static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) --{ -- unsigned long nr[NR_LRU_LISTS]; -- unsigned long targets[NR_LRU_LISTS]; -- unsigned long nr_to_scan; -- enum lru_list lru; -- unsigned long nr_reclaimed = 0; -- unsigned long nr_to_reclaim = sc->nr_to_reclaim; -- struct blk_plug plug; -- bool scan_adjusted; -+#ifdef CONFIG_LRU_GEN - -- get_scan_count(lruvec, sc, nr); -+#ifdef CONFIG_LRU_GEN_ENABLED -+DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); -+#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) -+#else -+DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); -+#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) -+#endif - -- /* Record the original scan target for proportional adjustments later */ -- memcpy(targets, nr, sizeof(nr)); -+/****************************************************************************** -+ * shorthand helpers -+ ******************************************************************************/ - -- /* -- * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal -- * event that can occur when there is little memory pressure e.g. -- * multiple streaming readers/writers. Hence, we do not abort scanning -- * when the requested number of pages are reclaimed when scanning at -- * DEF_PRIORITY on the assumption that the fact we are direct -- * reclaiming implies that kswapd is not keeping up and it is best to -- * do a batch of work at once. For memcg reclaim one check is made to -- * abort proportional reclaim if either the file or anon lru has already -- * dropped to zero at the first pass. -- */ -- scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && -- sc->priority == DEF_PRIORITY); -+#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) - -- blk_start_plug(&plug); -- while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || -- nr[LRU_INACTIVE_FILE]) { -- unsigned long nr_anon, nr_file, percentage; -- unsigned long nr_scanned; -+#define DEFINE_MAX_SEQ(lruvec) \ -+ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) - -- for_each_evictable_lru(lru) { -- if (nr[lru]) { -- nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); -- nr[lru] -= nr_to_scan; -+#define DEFINE_MIN_SEQ(lruvec) \ -+ unsigned long min_seq[ANON_AND_FILE] = { \ -+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ -+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ ++static void mas_wr_store_setup(struct ma_wr_state *wr_mas) ++{ ++ if (!mas_is_start(wr_mas->mas)) { ++ if (mas_is_none(wr_mas->mas)) { ++ mas_reset(wr_mas->mas); ++ } else { ++ wr_mas->r_max = wr_mas->mas->max; ++ wr_mas->type = mte_node_type(wr_mas->mas->node); ++ if (mas_is_span_wr(wr_mas)) ++ mas_reset(wr_mas->mas); ++ } + } - -- nr_reclaimed += shrink_list(lru, nr_to_scan, -- lruvec, sc); -- } -- } -+#define for_each_gen_type_zone(gen, type, zone) \ -+ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ -+ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ -+ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) - -- cond_resched(); -+static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) ++ ++} ++ ++/* Interface */ ++ ++/** ++ * mas_store() - Store an @entry. ++ * @mas: The maple state. ++ * @entry: The entry to store. ++ * ++ * The @mas->index and @mas->last is used to set the range for the @entry. ++ * Note: The @mas should have pre-allocated entries to ensure there is memory to ++ * store the entry. Please see mas_expected_entries()/mas_destroy() for more details. ++ * ++ * Return: the first entry between mas->index and mas->last or %NULL. ++ */ ++void *mas_store(struct ma_state *mas, void *entry) +{ -+ struct pglist_data *pgdat = NODE_DATA(nid); - -- if (nr_reclaimed < nr_to_reclaim || scan_adjusted) -- continue; -+#ifdef CONFIG_MEMCG -+ if (memcg) { -+ struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; - -- /* -- * For kswapd and memcg, reclaim at least the number of pages -- * requested. Ensure that the anon and file LRUs are scanned -- * proportionally what was requested by get_scan_count(). We -- * stop reclaiming one LRU and reduce the amount scanning -- * proportional to the original scan target. -- */ -- nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; -- nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; -+ /* for hotadd_new_pgdat() */ -+ if (!lruvec->pgdat) -+ lruvec->pgdat = pgdat; - -- /* -- * It's just vindictive to attack the larger once the smaller -- * has gone to zero. And given the way we stop scanning the -- * smaller below, this makes sure that we only make one nudge -- * towards proportionality once we've got nr_to_reclaim. -- */ -- if (!nr_file || !nr_anon) -- break; -+ return lruvec; ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ trace_ma_write(__func__, mas, 0, entry); ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++ if (mas->index > mas->last) ++ pr_err("Error %lu > %lu %p\n", mas->index, mas->last, entry); ++ MT_BUG_ON(mas->tree, mas->index > mas->last); ++ if (mas->index > mas->last) { ++ mas_set_err(mas, -EINVAL); ++ return NULL; + } ++ +#endif -+ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); - -- if (nr_file > nr_anon) { -- unsigned long scan_target = targets[LRU_INACTIVE_ANON] + -- targets[LRU_ACTIVE_ANON] + 1; -- lru = LRU_BASE; -- percentage = nr_anon * 100 / scan_target; -- } else { -- unsigned long scan_target = targets[LRU_INACTIVE_FILE] + -- targets[LRU_ACTIVE_FILE] + 1; -- lru = LRU_FILE; -- percentage = nr_file * 100 / scan_target; -- } -+ return pgdat ? &pgdat->__lruvec : NULL; ++ ++ /* ++ * Storing is the same operation as insert with the added caveat that it ++ * can overwrite entries. Although this seems simple enough, one may ++ * want to examine what happens if a single store operation was to ++ * overwrite multiple entries within a self-balancing B-Tree. ++ */ ++ mas_wr_store_setup(&wr_mas); ++ mas_wr_store_entry(&wr_mas); ++ return wr_mas.content; +} - -- /* Stop scanning the smaller of the LRU */ -- nr[lru] = 0; -- nr[lru + LRU_ACTIVE] = 0; -+static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) ++ ++/** ++ * mas_store_gfp() - Store a value into the tree. ++ * @mas: The maple state ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations if necessary. ++ * ++ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not ++ * be allocated. ++ */ ++int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) +{ -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(lruvec); - -- /* -- * Recalculate the other LRU scan count based on its original -- * scan target and the percentage scanning already complete -- */ -- lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; -- nr_scanned = targets[lru] - nr[lru]; -- nr[lru] = targets[lru] * (100 - percentage) / 100; -- nr[lru] -= min(nr[lru], nr_scanned); -+ if (!can_demote(pgdat->node_id, sc) && -+ mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) -+ return 0; - -- lru += LRU_ACTIVE; -- nr_scanned = targets[lru] - nr[lru]; -- nr[lru] = targets[lru] * (100 - percentage) / 100; -- nr[lru] -= min(nr[lru], nr_scanned); -+ return mem_cgroup_swappiness(memcg); ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ mas_wr_store_setup(&wr_mas); ++ trace_ma_write(__func__, mas, 0, entry); ++retry: ++ mas_wr_store_entry(&wr_mas); ++ if (unlikely(mas_nomem(mas, gfp))) ++ goto retry; ++ ++ if (unlikely(mas_is_err(mas))) ++ return xa_err(mas->node); ++ ++ return 0; +} - -- scan_adjusted = true; -- } -- blk_finish_plug(&plug); -- sc->nr_reclaimed += nr_reclaimed; -+static int get_nr_gens(struct lruvec *lruvec, int type) ++ ++/** ++ * mas_store_prealloc() - Store a value into the tree using memory ++ * preallocated in the maple state. ++ * @mas: The maple state ++ * @entry: The entry to store. ++ */ ++void mas_store_prealloc(struct ma_state *mas, void *entry) +{ -+ return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; ++ MA_WR_STATE(wr_mas, mas, entry); ++ ++ mas_wr_store_setup(&wr_mas); ++ trace_ma_write(__func__, mas, 0, entry); ++ mas_wr_store_entry(&wr_mas); ++ BUG_ON(mas_is_err(mas)); ++ mas_destroy(mas); +} - -- /* -- * Even if we did not try to evict anon pages at all, we want to -- * rebalance the anon lru active/inactive ratio. -- */ -- if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && -- inactive_is_low(lruvec, LRU_INACTIVE_ANON)) -- shrink_active_list(SWAP_CLUSTER_MAX, lruvec, -- sc, LRU_ACTIVE_ANON); -+static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) ++ ++/** ++ * mas_preallocate() - Preallocate enough nodes for a store operation ++ * @mas: The maple state ++ * @entry: The entry that will be stored ++ * @gfp: The GFP_FLAGS to use for allocations. ++ * ++ * Return: 0 on success, -ENOMEM if memory could not be allocated. ++ */ ++int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) +{ -+ /* see the comment on lru_gen_struct */ -+ return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && -+ get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && -+ get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; - } - --/* Use reclaim/compaction for costly allocs or under memory pressure */ --static bool in_reclaim_compaction(struct scan_control *sc) -+/****************************************************************************** -+ * mm_struct list -+ ******************************************************************************/ ++ int ret; + -+static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) - { -- if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && -- (sc->order > PAGE_ALLOC_COSTLY_ORDER || -- sc->priority < DEF_PRIORITY - 2)) -- return true; -+ static struct lru_gen_mm_list mm_list = { -+ .fifo = LIST_HEAD_INIT(mm_list.fifo), -+ .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), -+ }; - -- return false; -+#ifdef CONFIG_MEMCG -+ if (memcg) -+ return &memcg->mm_list; -+#endif -+ VM_WARN_ON_ONCE(!mem_cgroup_disabled()); ++ mas_node_count_gfp(mas, 1 + mas_mt_height(mas) * 3, gfp); ++ mas->mas_flags |= MA_STATE_PREALLOC; ++ if (likely(!mas_is_err(mas))) ++ return 0; + -+ return &mm_list; - } - --/* -- * Reclaim/compaction is used for high-order allocation requests. It reclaims -- * order-0 pages before compacting the zone. should_continue_reclaim() returns -- * true if more pages should be reclaimed such that when the page allocator -- * calls try_to_compact_pages() that it will have enough free pages to succeed. -- * It will give up earlier than that if there is difficulty reclaiming pages. -- */ --static inline bool should_continue_reclaim(struct pglist_data *pgdat, -- unsigned long nr_reclaimed, -- struct scan_control *sc) -+void lru_gen_add_mm(struct mm_struct *mm) - { -- unsigned long pages_for_compaction; -- unsigned long inactive_lru_pages; -- int z; -+ int nid; -+ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); - -- /* If not in reclaim/compaction mode, stop */ -- if (!in_reclaim_compaction(sc)) -- return false; -+ VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); -+#ifdef CONFIG_MEMCG -+ VM_WARN_ON_ONCE(mm->lru_gen.memcg); -+ mm->lru_gen.memcg = memcg; -+#endif -+ spin_lock(&mm_list->lock); - -- /* -- * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX -- * number of pages that were scanned. This will return to the caller -- * with the risk reclaim/compaction and the resulting allocation attempt -- * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL -- * allocations through requiring that the full LRU list has been scanned -- * first, by assuming that zero delta of sc->nr_scanned means full LRU -- * scan, but that approximation was wrong, and there were corner cases -+ for_each_node_state(nid, N_MEMORY) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ mas_set_alloc_req(mas, 0); ++ ret = xa_err(mas->node); ++ mas_reset(mas); ++ mas_destroy(mas); ++ mas_reset(mas); ++ return ret; ++} ++ ++/* ++ * mas_destroy() - destroy a maple state. ++ * @mas: The maple state ++ * ++ * Upon completion, check the left-most node and rebalance against the node to ++ * the right if necessary. Frees any allocated nodes associated with this maple ++ * state. ++ */ ++void mas_destroy(struct ma_state *mas) ++{ ++ struct maple_alloc *node; ++ ++ /* ++ * When using mas_for_each() to insert an expected number of elements, ++ * it is possible that the number inserted is less than the expected ++ * number. To fix an invalid final node, a check is performed here to ++ * rebalance the previous node with the final node. ++ */ ++ if (mas->mas_flags & MA_STATE_REBALANCE) { ++ unsigned char end; ++ ++ if (mas_is_start(mas)) ++ mas_start(mas); + -+ if (!lruvec) -+ continue; ++ mtree_range_walk(mas); ++ end = mas_data_end(mas) + 1; ++ if (end < mt_min_slot_count(mas->node) - 1) ++ mas_destroy_rebalance(mas, end); + -+ /* the first addition since the last iteration */ -+ if (lruvec->mm_state.tail == &mm_list->fifo) -+ lruvec->mm_state.tail = &mm->lru_gen.list; ++ mas->mas_flags &= ~MA_STATE_REBALANCE; + } ++ mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC); + -+ list_add_tail(&mm->lru_gen.list, &mm_list->fifo); -+ -+ spin_unlock(&mm_list->lock); ++ while (mas->alloc && !((unsigned long)mas->alloc & 0x1)) { ++ node = mas->alloc; ++ mas->alloc = node->slot[0]; ++ if (node->node_count > 0) ++ mt_free_bulk(node->node_count, ++ (void __rcu **)&node->slot[1]); ++ kmem_cache_free(maple_node_cache, node); ++ } ++ mas->alloc = NULL; +} + -+void lru_gen_del_mm(struct mm_struct *mm) ++/* ++ * mas_expected_entries() - Set the expected number of entries that will be inserted. ++ * @mas: The maple state ++ * @nr_entries: The number of expected entries. ++ * ++ * This will attempt to pre-allocate enough nodes to store the expected number ++ * of entries. The allocations will occur using the bulk allocator interface ++ * for speed. Please call mas_destroy() on the @mas after inserting the entries ++ * to ensure any unused nodes are freed. ++ * ++ * Return: 0 on success, -ENOMEM if memory could not be allocated. ++ */ ++int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) +{ -+ int nid; -+ struct lru_gen_mm_list *mm_list; -+ struct mem_cgroup *memcg = NULL; ++ int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2; ++ struct maple_enode *enode = mas->node; ++ int nr_nodes; ++ int ret; + -+ if (list_empty(&mm->lru_gen.list)) -+ return; ++ /* ++ * Sometimes it is necessary to duplicate a tree to a new tree, such as ++ * forking a process and duplicating the VMAs from one tree to a new ++ * tree. When such a situation arises, it is known that the new tree is ++ * not going to be used until the entire tree is populated. For ++ * performance reasons, it is best to use a bulk load with RCU disabled. ++ * This allows for optimistic splitting that favours the left and reuse ++ * of nodes during the operation. ++ */ + -+#ifdef CONFIG_MEMCG -+ memcg = mm->lru_gen.memcg; -+#endif -+ mm_list = get_mm_list(memcg); ++ /* Optimize splitting for bulk insert in-order */ ++ mas->mas_flags |= MA_STATE_BULK; + -+ spin_lock(&mm_list->lock); ++ /* ++ * Avoid overflow, assume a gap between each entry and a trailing null. ++ * If this is wrong, it just means allocation can happen during ++ * insertion of entries. ++ */ ++ nr_nodes = max(nr_entries, nr_entries * 2 + 1); ++ if (!mt_is_alloc(mas->tree)) ++ nonleaf_cap = MAPLE_RANGE64_SLOTS - 2; + -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ /* Leaves; reduce slots to keep space for expansion */ ++ nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2); ++ /* Internal nodes */ ++ nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap); ++ /* Add working room for split (2 nodes) + new parents */ ++ mas_node_count(mas, nr_nodes + 3); + -+ if (!lruvec) -+ continue; ++ /* Detect if allocations run out */ ++ mas->mas_flags |= MA_STATE_PREALLOC; + -+ /* where the last iteration ended (exclusive) */ -+ if (lruvec->mm_state.tail == &mm->lru_gen.list) -+ lruvec->mm_state.tail = lruvec->mm_state.tail->next; ++ if (!mas_is_err(mas)) ++ return 0; + -+ /* where the current iteration continues (inclusive) */ -+ if (lruvec->mm_state.head != &mm->lru_gen.list) -+ continue; ++ ret = xa_err(mas->node); ++ mas->node = enode; ++ mas_destroy(mas); ++ return ret; + -+ lruvec->mm_state.head = lruvec->mm_state.head->next; -+ /* the deletion ends the current iteration */ -+ if (lruvec->mm_state.head == &mm_list->fifo) -+ WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); ++} ++ ++/** ++ * mas_next() - Get the next entry. ++ * @mas: The maple state ++ * @max: The maximum index to check. ++ * ++ * Returns the next entry after @mas->index. ++ * Must hold rcu_read_lock or the write lock. ++ * Can return the zero entry. ++ * ++ * Return: The next entry or %NULL ++ */ ++void *mas_next(struct ma_state *mas, unsigned long max) ++{ ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; ++ ++ if (mas_is_start(mas)) ++ mas_walk(mas); /* Retries on dead nodes handled by mas_walk */ ++ ++ if (mas_is_ptr(mas)) { ++ if (!mas->index) { ++ mas->index = 1; ++ mas->last = ULONG_MAX; ++ } ++ return NULL; + } + -+ list_del_init(&mm->lru_gen.list); ++ if (mas->last == ULONG_MAX) ++ return NULL; + -+ spin_unlock(&mm_list->lock); ++ /* Retries on dead nodes handled by mas_next_entry */ ++ return mas_next_entry(mas, max); ++} ++EXPORT_SYMBOL_GPL(mas_next); + -+#ifdef CONFIG_MEMCG -+ mem_cgroup_put(mm->lru_gen.memcg); -+ mm->lru_gen.memcg = NULL; -+#endif ++/** ++ * mt_next() - get the next value in the maple tree ++ * @mt: The maple tree ++ * @index: The start index ++ * @max: The maximum index to check ++ * ++ * Return: The entry at @index or higher, or %NULL if nothing is found. ++ */ ++void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max) ++{ ++ void *entry = NULL; ++ MA_STATE(mas, mt, index, index); ++ ++ rcu_read_lock(); ++ entry = mas_next(&mas, max); ++ rcu_read_unlock(); ++ return entry; +} ++EXPORT_SYMBOL_GPL(mt_next); + -+#ifdef CONFIG_MEMCG -+void lru_gen_migrate_mm(struct mm_struct *mm) ++/** ++ * mas_prev() - Get the previous entry ++ * @mas: The maple state ++ * @min: The minimum value to check. ++ * ++ * Must hold rcu_read_lock or the write lock. ++ * Will reset mas to MAS_START if the node is MAS_NONE. Will stop on not ++ * searchable nodes. ++ * ++ * Return: the previous value or %NULL. ++ */ ++void *mas_prev(struct ma_state *mas, unsigned long min) +{ -+ struct mem_cgroup *memcg; -+ struct task_struct *task = rcu_dereference_protected(mm->owner, true); ++ if (!mas->index) { ++ /* Nothing comes before 0 */ ++ mas->last = 0; ++ return NULL; ++ } + -+ VM_WARN_ON_ONCE(task->mm != mm); -+ lockdep_assert_held(&task->alloc_lock); ++ if (unlikely(mas_is_ptr(mas))) ++ return NULL; + -+ /* for mm_update_next_owner() */ -+ if (mem_cgroup_disabled()) -+ return; ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; + -+ rcu_read_lock(); -+ memcg = mem_cgroup_from_task(task); -+ rcu_read_unlock(); -+ if (memcg == mm->lru_gen.memcg) -+ return; ++ if (mas_is_start(mas)) { ++ mas_walk(mas); ++ if (!mas->index) ++ return NULL; ++ } + -+ VM_WARN_ON_ONCE(!mm->lru_gen.memcg); -+ VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); ++ if (mas_is_ptr(mas)) { ++ if (!mas->index) { ++ mas->last = 0; ++ return NULL; ++ } + -+ lru_gen_del_mm(mm); -+ lru_gen_add_mm(mm); ++ mas->index = mas->last = 0; ++ return mas_root_locked(mas); ++ } ++ return mas_prev_entry(mas, min); +} -+#endif ++EXPORT_SYMBOL_GPL(mas_prev); + -+/* -+ * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when -+ * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of -+ * bits in a bitmap, k is the number of hash functions and n is the number of -+ * inserted items. -+ * -+ * Page table walkers use one of the two filters to reduce their search space. -+ * To get rid of non-leaf entries that no longer have enough leaf entries, the -+ * aging uses the double-buffering technique to flip to the other filter each -+ * time it produces a new generation. For non-leaf entries that have enough -+ * leaf entries, the aging carries them over to the next generation in -+ * walk_pmd_range(); the eviction also report them when walking the rmap -+ * in lru_gen_look_around(). ++/** ++ * mt_prev() - get the previous value in the maple tree ++ * @mt: The maple tree ++ * @index: The start index ++ * @min: The minimum index to check + * -+ * For future optimizations: -+ * 1. It's not necessary to keep both filters all the time. The spare one can be -+ * freed after the RCU grace period and reallocated if needed again. -+ * 2. And when reallocating, it's worth scaling its size according to the number -+ * of inserted entries in the other filter, to reduce the memory overhead on -+ * small systems and false positives on large systems. -+ * 3. Jenkins' hash function is an alternative to Knuth's. ++ * Return: The entry at @index or lower, or %NULL if nothing is found. + */ -+#define BLOOM_FILTER_SHIFT 15 ++void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min) ++{ ++ void *entry = NULL; ++ MA_STATE(mas, mt, index, index); + -+static inline int filter_gen_from_seq(unsigned long seq) ++ rcu_read_lock(); ++ entry = mas_prev(&mas, min); ++ rcu_read_unlock(); ++ return entry; ++} ++EXPORT_SYMBOL_GPL(mt_prev); ++ ++/** ++ * mas_pause() - Pause a mas_find/mas_for_each to drop the lock. ++ * @mas: The maple state to pause ++ * ++ * Some users need to pause a walk and drop the lock they're holding in ++ * order to yield to a higher priority thread or carry out an operation ++ * on an entry. Those users should call this function before they drop ++ * the lock. It resets the @mas to be suitable for the next iteration ++ * of the loop after the user has reacquired the lock. If most entries ++ * found during a walk require you to call mas_pause(), the mt_for_each() ++ * iterator may be more appropriate. ++ * ++ */ ++void mas_pause(struct ma_state *mas) +{ -+ return seq % NR_BLOOM_FILTERS; ++ mas->node = MAS_PAUSE; +} ++EXPORT_SYMBOL_GPL(mas_pause); + -+static void get_item_key(void *item, int *key) ++/** ++ * mas_find() - On the first call, find the entry at or after mas->index up to ++ * %max. Otherwise, find the entry after mas->index. ++ * @mas: The maple state ++ * @max: The maximum value to check. ++ * ++ * Must hold rcu_read_lock or the write lock. ++ * If an entry exists, last and index are updated accordingly. ++ * May set @mas->node to MAS_NONE. ++ * ++ * Return: The entry or %NULL. ++ */ ++void *mas_find(struct ma_state *mas, unsigned long max) +{ -+ u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); ++ if (unlikely(mas_is_paused(mas))) { ++ if (unlikely(mas->last == ULONG_MAX)) { ++ mas->node = MAS_NONE; ++ return NULL; ++ } ++ mas->node = MAS_START; ++ mas->index = ++mas->last; ++ } + -+ BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); ++ if (unlikely(mas_is_start(mas))) { ++ /* First run or continue */ ++ void *entry; + -+ key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); -+ key[1] = hash >> BLOOM_FILTER_SHIFT; ++ if (mas->index > max) ++ return NULL; ++ ++ entry = mas_walk(mas); ++ if (entry) ++ return entry; ++ } ++ ++ if (unlikely(!mas_searchable(mas))) ++ return NULL; ++ ++ /* Retries on dead nodes handled by mas_next_entry */ ++ return mas_next_entry(mas, max); +} + -+static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) ++/** ++ * mas_find_rev: On the first call, find the first non-null entry at or below ++ * mas->index down to %min. Otherwise find the first non-null entry below ++ * mas->index down to %min. ++ * @mas: The maple state ++ * @min: The minimum value to check. ++ * ++ * Must hold rcu_read_lock or the write lock. ++ * If an entry exists, last and index are updated accordingly. ++ * May set @mas->node to MAS_NONE. ++ * ++ * Return: The entry or %NULL. ++ */ ++void *mas_find_rev(struct ma_state *mas, unsigned long min) +{ -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); ++ if (unlikely(mas_is_paused(mas))) { ++ if (unlikely(mas->last == ULONG_MAX)) { ++ mas->node = MAS_NONE; ++ return NULL; ++ } ++ mas->node = MAS_START; ++ mas->last = --mas->index; ++ } + -+ filter = lruvec->mm_state.filters[gen]; -+ if (filter) { -+ bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); -+ return; ++ if (unlikely(mas_is_start(mas))) { ++ /* First run or continue */ ++ void *entry; ++ ++ if (mas->index < min) ++ return NULL; ++ ++ entry = mas_walk(mas); ++ if (entry) ++ return entry; + } + -+ filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), -+ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); -+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter); ++ if (unlikely(!mas_searchable(mas))) ++ return NULL; ++ ++ if (mas->index < min) ++ return NULL; ++ ++ /* Retries on dead nodes handled by mas_next_entry */ ++ return mas_prev_entry(mas, min); +} ++EXPORT_SYMBOL_GPL(mas_find); + -+static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) ++/** ++ * mas_erase() - Find the range in which index resides and erase the entire ++ * range. ++ * @mas: The maple state ++ * ++ * Must hold the write lock. ++ * Searches for @mas->index, sets @mas->index and @mas->last to the range and ++ * erases that range. ++ * ++ * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. ++ */ ++void *mas_erase(struct ma_state *mas) +{ -+ int key[2]; -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); ++ void *entry; ++ MA_WR_STATE(wr_mas, mas, NULL); + -+ filter = READ_ONCE(lruvec->mm_state.filters[gen]); -+ if (!filter) -+ return; ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; + -+ get_item_key(item, key); ++ /* Retry unnecessary when holding the write lock. */ ++ entry = mas_state_walk(mas); ++ if (!entry) ++ return NULL; + -+ if (!test_bit(key[0], filter)) -+ set_bit(key[0], filter); -+ if (!test_bit(key[1], filter)) -+ set_bit(key[1], filter); ++write_retry: ++ /* Must reset to ensure spanning writes of last slot are detected */ ++ mas_reset(mas); ++ mas_wr_store_setup(&wr_mas); ++ mas_wr_store_entry(&wr_mas); ++ if (mas_nomem(mas, GFP_KERNEL)) ++ goto write_retry; ++ ++ return entry; +} ++EXPORT_SYMBOL_GPL(mas_erase); + -+static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) ++/** ++ * mas_nomem() - Check if there was an error allocating and do the allocation ++ * if necessary If there are allocations, then free them. ++ * @mas: The maple state ++ * @gfp: The GFP_FLAGS to use for allocations ++ * Return: true on allocation, false otherwise. ++ */ ++bool mas_nomem(struct ma_state *mas, gfp_t gfp) ++ __must_hold(mas->tree->lock) +{ -+ int key[2]; -+ unsigned long *filter; -+ int gen = filter_gen_from_seq(seq); ++ if (likely(mas->node != MA_ERROR(-ENOMEM))) { ++ mas_destroy(mas); ++ return false; ++ } + -+ filter = READ_ONCE(lruvec->mm_state.filters[gen]); -+ if (!filter) -+ return true; ++ if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { ++ mtree_unlock(mas->tree); ++ mas_alloc_nodes(mas, gfp); ++ mtree_lock(mas->tree); ++ } else { ++ mas_alloc_nodes(mas, gfp); ++ } + -+ get_item_key(item, key); ++ if (!mas_allocated(mas)) ++ return false; + -+ return test_bit(key[0], filter) && test_bit(key[1], filter); ++ mas->node = MAS_START; ++ return true; +} + -+static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) ++void __init maple_tree_init(void) +{ -+ int i; -+ int hist; ++ maple_node_cache = kmem_cache_create("maple_node", ++ sizeof(struct maple_node), sizeof(struct maple_node), ++ SLAB_PANIC, NULL); ++} + -+ lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); ++/** ++ * mtree_load() - Load a value stored in a maple tree ++ * @mt: The maple tree ++ * @index: The index to load ++ * ++ * Return: the entry or %NULL ++ */ ++void *mtree_load(struct maple_tree *mt, unsigned long index) ++{ ++ MA_STATE(mas, mt, index, index); ++ void *entry; + -+ if (walk) { -+ hist = lru_hist_from_seq(walk->max_seq); ++ trace_ma_read(__func__, &mas); ++ rcu_read_lock(); ++retry: ++ entry = mas_start(&mas); ++ if (unlikely(mas_is_none(&mas))) ++ goto unlock; + -+ for (i = 0; i < NR_MM_STATS; i++) { -+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], -+ lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); -+ walk->mm_stats[i] = 0; -+ } ++ if (unlikely(mas_is_ptr(&mas))) { ++ if (index) ++ entry = NULL; ++ ++ goto unlock; + } + -+ if (NR_HIST_GENS > 1 && last) { -+ hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); ++ entry = mtree_lookup_walk(&mas); ++ if (!entry && unlikely(mas_is_start(&mas))) ++ goto retry; ++unlock: ++ rcu_read_unlock(); ++ if (xa_is_zero(entry)) ++ return NULL; + -+ for (i = 0; i < NR_MM_STATS; i++) -+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); -+ } ++ return entry; +} ++EXPORT_SYMBOL(mtree_load); + -+static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) ++/** ++ * mtree_store_range() - Store an entry at a given range. ++ * @mt: The maple tree ++ * @index: The start of the range ++ * @last: The end of the range ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations ++ * ++ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not ++ * be allocated. ++ */ ++int mtree_store_range(struct maple_tree *mt, unsigned long index, ++ unsigned long last, void *entry, gfp_t gfp) +{ -+ int type; -+ unsigned long size = 0; -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); ++ MA_STATE(mas, mt, index, last); ++ MA_WR_STATE(wr_mas, &mas, entry); + -+ if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) -+ return true; ++ trace_ma_write(__func__, &mas, 0, entry); ++ if (WARN_ON_ONCE(xa_is_advanced(entry))) ++ return -EINVAL; + -+ clear_bit(key, &mm->lru_gen.bitmap); ++ if (index > last) ++ return -EINVAL; + -+ for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { -+ size += type ? get_mm_counter(mm, MM_FILEPAGES) : -+ get_mm_counter(mm, MM_ANONPAGES) + -+ get_mm_counter(mm, MM_SHMEMPAGES); -+ } ++ mtree_lock(mt); ++retry: ++ mas_wr_store_entry(&wr_mas); ++ if (mas_nomem(&mas, gfp)) ++ goto retry; + -+ if (size < MIN_LRU_BATCH) -+ return true; ++ mtree_unlock(mt); ++ if (mas_is_err(&mas)) ++ return xa_err(mas.node); + -+ return !mmget_not_zero(mm); ++ return 0; +} ++EXPORT_SYMBOL(mtree_store_range); + -+static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, -+ struct mm_struct **iter) ++/** ++ * mtree_store() - Store an entry at a given index. ++ * @mt: The maple tree ++ * @index: The index to store the value ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations ++ * ++ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not ++ * be allocated. ++ */ ++int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, ++ gfp_t gfp) +{ -+ bool first = false; -+ bool last = true; -+ struct mm_struct *mm = NULL; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); -+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; ++ return mtree_store_range(mt, index, index, entry, gfp); ++} ++EXPORT_SYMBOL(mtree_store); + -+ /* -+ * There are four interesting cases for this page table walker: -+ * 1. It tries to start a new iteration of mm_list with a stale max_seq; -+ * there is nothing left to do. -+ * 2. It's the first of the current generation, and it needs to reset -+ * the Bloom filter for the next generation. -+ * 3. It reaches the end of mm_list, and it needs to increment -+ * mm_state->seq; the iteration is done. -+ * 4. It's the last of the current generation, and it needs to reset the -+ * mm stats counters for the next generation. -+ */ -+ spin_lock(&mm_list->lock); ++/** ++ * mtree_insert_range() - Insert an entry at a give range if there is no value. ++ * @mt: The maple tree ++ * @first: The start of the range ++ * @last: The end of the range ++ * @entry: The entry to store ++ * @gfp: The GFP_FLAGS to use for allocations. ++ * ++ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid ++ * request, -ENOMEM if memory could not be allocated. ++ */ ++int mtree_insert_range(struct maple_tree *mt, unsigned long first, ++ unsigned long last, void *entry, gfp_t gfp) ++{ ++ MA_STATE(ms, mt, first, last); + -+ VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); -+ VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); -+ VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); ++ if (WARN_ON_ONCE(xa_is_advanced(entry))) ++ return -EINVAL; + -+ if (walk->max_seq <= mm_state->seq) { -+ if (!*iter) -+ last = false; -+ goto done; -+ } ++ if (first > last) ++ return -EINVAL; + -+ if (!mm_state->nr_walkers) { -+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); ++ mtree_lock(mt); ++retry: ++ mas_insert(&ms, entry); ++ if (mas_nomem(&ms, gfp)) ++ goto retry; + -+ mm_state->head = mm_list->fifo.next; -+ first = true; -+ } ++ mtree_unlock(mt); ++ if (mas_is_err(&ms)) ++ return xa_err(ms.node); + -+ while (!mm && mm_state->head != &mm_list->fifo) { -+ mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); ++ return 0; ++} ++EXPORT_SYMBOL(mtree_insert_range); + -+ mm_state->head = mm_state->head->next; ++/** ++ * mtree_insert() - Insert an entry at a give index if there is no value. ++ * @mt: The maple tree ++ * @index : The index to store the value ++ * @entry: The entry to store ++ * @gfp: The FGP_FLAGS to use for allocations. ++ * ++ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid ++ * request, -ENOMEM if memory could not be allocated. ++ */ ++int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, ++ gfp_t gfp) ++{ ++ return mtree_insert_range(mt, index, index, entry, gfp); ++} ++EXPORT_SYMBOL(mtree_insert); + -+ /* force scan for those added after the last iteration */ -+ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { -+ mm_state->tail = mm_state->head; -+ walk->force_scan = true; -+ } ++int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp) ++{ ++ int ret = 0; + -+ if (should_skip_mm(mm, walk)) -+ mm = NULL; -+ } ++ MA_STATE(mas, mt, min, max - size); ++ if (!mt_is_alloc(mt)) ++ return -EINVAL; + -+ if (mm_state->head == &mm_list->fifo) -+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); -+done: -+ if (*iter && !mm) -+ mm_state->nr_walkers--; -+ if (!*iter && mm) -+ mm_state->nr_walkers++; ++ if (WARN_ON_ONCE(mt_is_reserved(entry))) ++ return -EINVAL; + -+ if (mm_state->nr_walkers) -+ last = false; ++ if (min > max) ++ return -EINVAL; + -+ if (*iter || last) -+ reset_mm_stats(lruvec, walk, last); ++ if (max < size) ++ return -EINVAL; + -+ spin_unlock(&mm_list->lock); ++ if (!size) ++ return -EINVAL; + -+ if (mm && first) -+ reset_bloom_filter(lruvec, walk->max_seq + 1); ++ mtree_lock(mt); ++retry: ++ mas.offset = 0; ++ mas.index = min; ++ mas.last = max - size; ++ ret = mas_alloc(&mas, entry, size, startp); ++ if (mas_nomem(&mas, gfp)) ++ goto retry; + -+ if (*iter) -+ mmput_async(*iter); ++ mtree_unlock(mt); ++ return ret; ++} ++EXPORT_SYMBOL(mtree_alloc_range); + -+ *iter = mm; ++int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, ++ void *entry, unsigned long size, unsigned long min, ++ unsigned long max, gfp_t gfp) ++{ ++ int ret = 0; + -+ return last; ++ MA_STATE(mas, mt, min, max - size); ++ if (!mt_is_alloc(mt)) ++ return -EINVAL; ++ ++ if (WARN_ON_ONCE(mt_is_reserved(entry))) ++ return -EINVAL; ++ ++ if (min >= max) ++ return -EINVAL; ++ ++ if (max < size - 1) ++ return -EINVAL; ++ ++ if (!size) ++ return -EINVAL; ++ ++ mtree_lock(mt); ++retry: ++ ret = mas_rev_alloc(&mas, min, max, entry, size, startp); ++ if (mas_nomem(&mas, gfp)) ++ goto retry; ++ ++ mtree_unlock(mt); ++ return ret; +} ++EXPORT_SYMBOL(mtree_alloc_rrange); + -+static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) ++/** ++ * mtree_erase() - Find an index and erase the entire range. ++ * @mt: The maple tree ++ * @index: The index to erase ++ * ++ * Erasing is the same as a walk to an entry then a store of a NULL to that ++ * ENTIRE range. In fact, it is implemented as such using the advanced API. ++ * ++ * Return: The entry stored at the @index or %NULL ++ */ ++void *mtree_erase(struct maple_tree *mt, unsigned long index) +{ -+ bool success = false; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg); -+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state; ++ void *entry = NULL; + -+ spin_lock(&mm_list->lock); ++ MA_STATE(mas, mt, index, index); ++ trace_ma_op(__func__, &mas); + -+ VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); ++ mtree_lock(mt); ++ entry = mas_erase(&mas); ++ mtree_unlock(mt); + -+ if (max_seq > mm_state->seq && !mm_state->nr_walkers) { -+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); ++ return entry; ++} ++EXPORT_SYMBOL(mtree_erase); + -+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1); -+ reset_mm_stats(lruvec, NULL, true); -+ success = true; -+ } ++/** ++ * __mt_destroy() - Walk and free all nodes of a locked maple tree. ++ * @mt: The maple tree ++ * ++ * Note: Does not handle locking. ++ */ ++void __mt_destroy(struct maple_tree *mt) ++{ ++ void *root = mt_root_locked(mt); + -+ spin_unlock(&mm_list->lock); ++ rcu_assign_pointer(mt->ma_root, NULL); ++ if (xa_is_node(root)) ++ mte_destroy_walk(root, mt); + -+ return success; ++ mt->ma_flags = 0; +} ++EXPORT_SYMBOL_GPL(__mt_destroy); + -+/****************************************************************************** -+ * refault feedback loop -+ ******************************************************************************/ ++/** ++ * mtree_destroy() - Destroy a maple tree ++ * @mt: The maple tree ++ * ++ * Frees all resources used by the tree. Handles locking. ++ */ ++void mtree_destroy(struct maple_tree *mt) ++{ ++ mtree_lock(mt); ++ __mt_destroy(mt); ++ mtree_unlock(mt); ++} ++EXPORT_SYMBOL(mtree_destroy); + -+/* -+ * A feedback loop based on Proportional-Integral-Derivative (PID) controller. ++/** ++ * mt_find() - Search from the start up until an entry is found. ++ * @mt: The maple tree ++ * @index: Pointer which contains the start location of the search ++ * @max: The maximum value to check + * -+ * The P term is refaulted/(evicted+protected) from a tier in the generation -+ * currently being evicted; the I term is the exponential moving average of the -+ * P term over the generations previously evicted, using the smoothing factor -+ * 1/2; the D term isn't supported. ++ * Handles locking. @index will be incremented to one beyond the range. + * -+ * The setpoint (SP) is always the first tier of one type; the process variable -+ * (PV) is either any tier of the other type or any other tier of the same -+ * type. ++ * Return: The entry at or after the @index or %NULL ++ */ ++void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) ++{ ++ MA_STATE(mas, mt, *index, *index); ++ void *entry; ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++ unsigned long copy = *index; ++#endif ++ ++ trace_ma_read(__func__, &mas); ++ ++ if ((*index) > max) ++ return NULL; ++ ++ rcu_read_lock(); ++retry: ++ entry = mas_state_walk(&mas); ++ if (mas_is_start(&mas)) ++ goto retry; ++ ++ if (unlikely(xa_is_zero(entry))) ++ entry = NULL; ++ ++ if (entry) ++ goto unlock; ++ ++ while (mas_searchable(&mas) && (mas.index < max)) { ++ entry = mas_next_entry(&mas, max); ++ if (likely(entry && !xa_is_zero(entry))) ++ break; ++ } ++ ++ if (unlikely(xa_is_zero(entry))) ++ entry = NULL; ++unlock: ++ rcu_read_unlock(); ++ if (likely(entry)) { ++ *index = mas.last + 1; ++#ifdef CONFIG_DEBUG_MAPLE_TREE ++ if ((*index) && (*index) <= copy) ++ pr_err("index not increased! %lx <= %lx\n", ++ *index, copy); ++ MT_BUG_ON(mt, (*index) && ((*index) <= copy)); ++#endif ++ } ++ ++ return entry; ++} ++EXPORT_SYMBOL(mt_find); ++ ++/** ++ * mt_find_after() - Search from the start up until an entry is found. ++ * @mt: The maple tree ++ * @index: Pointer which contains the start location of the search ++ * @max: The maximum value to check + * -+ * The error is the difference between the SP and the PV; the correction is to -+ * turn off protection when SP>PV or turn on protection when SPlrugen; -+ int hist = lru_hist_from_seq(lrugen->min_seq[type]); ++ kmem_cache_set_non_kernel(maple_node_cache, val); ++} + -+ pos->refaulted = lrugen->avg_refaulted[type][tier] + -+ atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ pos->total = lrugen->avg_total[type][tier] + -+ atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ pos->total += lrugen->protected[hist][type][tier - 1]; -+ pos->gain = gain; ++extern unsigned long kmem_cache_get_alloc(struct kmem_cache *); ++unsigned long mt_get_alloc_size(void) ++{ ++ return kmem_cache_get_alloc(maple_node_cache); +} + -+static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) ++extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *); ++void mt_zero_nr_tallocated(void) +{ -+ int hist, tier; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; -+ unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; ++ kmem_cache_zero_nr_tallocated(maple_node_cache); ++} + -+ lockdep_assert_held(&lruvec->lru_lock); ++extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *); ++unsigned int mt_nr_tallocated(void) ++{ ++ return kmem_cache_nr_tallocated(maple_node_cache); ++} + -+ if (!carryover && !clear) -+ return; ++extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *); ++unsigned int mt_nr_allocated(void) ++{ ++ return kmem_cache_nr_allocated(maple_node_cache); ++} + -+ hist = lru_hist_from_seq(seq); ++/* ++ * mas_dead_node() - Check if the maple state is pointing to a dead node. ++ * @mas: The maple state ++ * @index: The index to restore in @mas. ++ * ++ * Used in test code. ++ * Return: 1 if @mas has been reset to MAS_START, 0 otherwise. ++ */ ++static inline int mas_dead_node(struct ma_state *mas, unsigned long index) ++{ ++ if (unlikely(!mas_searchable(mas) || mas_is_start(mas))) ++ return 0; + -+ for (tier = 0; tier < MAX_NR_TIERS; tier++) { -+ if (carryover) { -+ unsigned long sum; ++ if (likely(!mte_dead_node(mas->node))) ++ return 0; + -+ sum = lrugen->avg_refaulted[type][tier] + -+ atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); ++ mas_rewalk(mas, index); ++ return 1; ++} ++#endif /* not defined __KERNEL__ */ + -+ sum = lrugen->avg_total[type][tier] + -+ atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ sum += lrugen->protected[hist][type][tier - 1]; -+ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); -+ } ++/* ++ * mas_get_slot() - Get the entry in the maple state node stored at @offset. ++ * @mas: The maple state ++ * @offset: The offset into the slot array to fetch. ++ * ++ * Return: The entry stored at @offset. ++ */ ++static inline struct maple_enode *mas_get_slot(struct ma_state *mas, ++ unsigned char offset) ++{ ++ return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)), ++ offset); ++} + -+ if (clear) { -+ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); -+ atomic_long_set(&lrugen->evicted[hist][type][tier], 0); -+ if (tier) -+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); -+ } ++ ++/* ++ * mas_first_entry() - Go the first leaf and find the first entry. ++ * @mas: the maple state. ++ * @limit: the maximum index to check. ++ * @*r_start: Pointer to set to the range start. ++ * ++ * Sets mas->offset to the offset of the entry, r_start to the range minimum. ++ * ++ * Return: The first entry or MAS_NONE. ++ */ ++static inline void *mas_first_entry(struct ma_state *mas, struct maple_node *mn, ++ unsigned long limit, enum maple_type mt) ++ ++{ ++ unsigned long max; ++ unsigned long *pivots; ++ void __rcu **slots; ++ void *entry = NULL; ++ ++ mas->index = mas->min; ++ if (mas->index > limit) ++ goto none; ++ ++ max = mas->max; ++ mas->offset = 0; ++ while (likely(!ma_is_leaf(mt))) { ++ MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); ++ slots = ma_slots(mn, mt); ++ pivots = ma_pivots(mn, mt); ++ max = pivots[0]; ++ entry = mas_slot(mas, slots, 0); ++ if (unlikely(ma_dead_node(mn))) ++ return NULL; ++ mas->node = entry; ++ mn = mas_mn(mas); ++ mt = mte_node_type(mas->node); + } ++ MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); ++ ++ mas->max = max; ++ slots = ma_slots(mn, mt); ++ entry = mas_slot(mas, slots, 0); ++ if (unlikely(ma_dead_node(mn))) ++ return NULL; ++ ++ /* Slot 0 or 1 must be set */ ++ if (mas->index > limit) ++ goto none; ++ ++ if (likely(entry)) ++ return entry; ++ ++ pivots = ma_pivots(mn, mt); ++ mas->index = pivots[0] + 1; ++ mas->offset = 1; ++ entry = mas_slot(mas, slots, 1); ++ if (unlikely(ma_dead_node(mn))) ++ return NULL; ++ ++ if (mas->index > limit) ++ goto none; ++ ++ if (likely(entry)) ++ return entry; ++ ++none: ++ if (likely(!ma_dead_node(mn))) ++ mas->node = MAS_NONE; ++ return NULL; +} + -+static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ++/* Depth first search, post-order */ ++static void mas_dfs_postorder(struct ma_state *mas, unsigned long max) +{ -+ /* -+ * Return true if the PV has a limited number of refaults or a lower -+ * refaulted/total than the SP. -+ */ -+ return pv->refaulted < MIN_LRU_BATCH || -+ pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= -+ (sp->refaulted + 1) * pv->total * pv->gain; -+} + -+/****************************************************************************** -+ * the aging -+ ******************************************************************************/ ++ struct maple_enode *p = MAS_NONE, *mn = mas->node; ++ unsigned long p_min, p_max; + -+/* promote pages accessed through page tables */ -+static int folio_update_gen(struct folio *folio, int gen) -+{ -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); ++ mas_next_node(mas, mas_mn(mas), max); ++ if (!mas_is_none(mas)) ++ return; + -+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(!rcu_read_lock_held()); ++ if (mte_is_root(mn)) ++ return; + -+ do { -+ /* lru_gen_del_folio() has isolated this page? */ -+ if (!(old_flags & LRU_GEN_MASK)) { -+ /* for shrink_page_list() */ -+ new_flags = old_flags | BIT(PG_referenced); -+ continue; -+ } ++ mas->node = mn; ++ mas_ascend(mas); ++ while (mas->node != MAS_NONE) { ++ p = mas->node; ++ p_min = mas->min; ++ p_max = mas->max; ++ mas_prev_node(mas, 0); ++ } + -+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); -+ new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); ++ if (p == MAS_NONE) ++ return; + -+ return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; ++ mas->node = p; ++ mas->max = p_max; ++ mas->min = p_min; +} + -+/* protect pages accessed multiple times through file descriptors */ -+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) ++/* Tree validations */ ++static void mt_dump_node(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth); ++static void mt_dump_range(unsigned long min, unsigned long max, ++ unsigned int depth) +{ -+ int type = folio_is_file_lru(folio); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); -+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags); ++ static const char spaces[] = " "; + -+ VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); ++ if (min == max) ++ pr_info("%.*s%lu: ", depth * 2, spaces, min); ++ else ++ pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max); ++} + -+ do { -+ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; -+ /* folio_update_gen() has promoted this page? */ -+ if (new_gen >= 0 && new_gen != old_gen) -+ return new_gen; ++static void mt_dump_entry(void *entry, unsigned long min, unsigned long max, ++ unsigned int depth) ++{ ++ mt_dump_range(min, max, depth); + -+ new_gen = (old_gen + 1) % MAX_NR_GENS; ++ if (xa_is_value(entry)) ++ pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry), ++ xa_to_value(entry), entry); ++ else if (xa_is_zero(entry)) ++ pr_cont("zero (%ld)\n", xa_to_internal(entry)); ++ else if (mt_is_reserved(entry)) ++ pr_cont("UNKNOWN ENTRY (%p)\n", entry); ++ else ++ pr_cont("%p\n", entry); ++} + -+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); -+ new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; -+ /* for folio_end_writeback() */ -+ if (reclaiming) -+ new_flags |= BIT(PG_reclaim); -+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); ++static void mt_dump_range64(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth) ++{ ++ struct maple_range_64 *node = &mte_to_node(entry)->mr64; ++ bool leaf = mte_is_leaf(entry); ++ unsigned long first = min; ++ int i; + -+ lru_gen_update_size(lruvec, folio, old_gen, new_gen); ++ pr_cont(" contents: "); ++ for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) ++ pr_cont("%p %lu ", node->slot[i], node->pivot[i]); ++ pr_cont("%p\n", node->slot[i]); ++ for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) { ++ unsigned long last = max; + -+ return new_gen; ++ if (i < (MAPLE_RANGE64_SLOTS - 1)) ++ last = node->pivot[i]; ++ else if (!node->slot[i] && max != mt_max[mte_node_type(entry)]) ++ break; ++ if (last == 0 && i > 0) ++ break; ++ if (leaf) ++ mt_dump_entry(mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ else if (node->slot[i]) ++ mt_dump_node(mt, mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ ++ if (last == max) ++ break; ++ if (last > max) { ++ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", ++ node, last, max, i); ++ break; ++ } ++ first = last + 1; ++ } +} + -+static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, -+ int old_gen, int new_gen) ++static void mt_dump_arange64(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth) +{ -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); ++ struct maple_arange_64 *node = &mte_to_node(entry)->ma64; ++ bool leaf = mte_is_leaf(entry); ++ unsigned long first = min; ++ int i; + -+ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); -+ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); ++ pr_cont(" contents: "); ++ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) ++ pr_cont("%lu ", node->gap[i]); ++ pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap); ++ for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) ++ pr_cont("%p %lu ", node->slot[i], node->pivot[i]); ++ pr_cont("%p\n", node->slot[i]); ++ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) { ++ unsigned long last = max; ++ ++ if (i < (MAPLE_ARANGE64_SLOTS - 1)) ++ last = node->pivot[i]; ++ else if (!node->slot[i]) ++ break; ++ if (last == 0 && i > 0) ++ break; ++ if (leaf) ++ mt_dump_entry(mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ else if (node->slot[i]) ++ mt_dump_node(mt, mt_slot(mt, node->slot, i), ++ first, last, depth + 1); ++ ++ if (last == max) ++ break; ++ if (last > max) { ++ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", ++ node, last, max, i); ++ break; ++ } ++ first = last + 1; ++ } ++} + -+ walk->batched++; ++static void mt_dump_node(const struct maple_tree *mt, void *entry, ++ unsigned long min, unsigned long max, unsigned int depth) ++{ ++ struct maple_node *node = mte_to_node(entry); ++ unsigned int type = mte_node_type(entry); ++ unsigned int i; + -+ walk->nr_pages[old_gen][type][zone] -= delta; -+ walk->nr_pages[new_gen][type][zone] += delta; ++ mt_dump_range(min, max, depth); ++ ++ pr_cont("node %p depth %d type %d parent %p", node, depth, type, ++ node ? node->parent : NULL); ++ switch (type) { ++ case maple_dense: ++ pr_cont("\n"); ++ for (i = 0; i < MAPLE_NODE_SLOTS; i++) { ++ if (min + i > max) ++ pr_cont("OUT OF RANGE: "); ++ mt_dump_entry(mt_slot(mt, node->slot, i), ++ min + i, min + i, depth); ++ } ++ break; ++ case maple_leaf_64: ++ case maple_range_64: ++ mt_dump_range64(mt, entry, min, max, depth); ++ break; ++ case maple_arange_64: ++ mt_dump_arange64(mt, entry, min, max, depth); ++ break; ++ ++ default: ++ pr_cont(" UNKNOWN TYPE\n"); ++ } +} + -+static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) ++void mt_dump(const struct maple_tree *mt) +{ -+ int gen, type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt)); + -+ walk->batched = 0; ++ pr_info("maple_tree(%p) flags %X, height %u root %p\n", ++ mt, mt->ma_flags, mt_height(mt), entry); ++ if (!xa_is_node(entry)) ++ mt_dump_entry(entry, 0, 0, 0); ++ else if (entry) ++ mt_dump_node(mt, entry, 0, mt_max[mte_node_type(entry)], 0); ++} + -+ for_each_gen_type_zone(gen, type, zone) { -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ int delta = walk->nr_pages[gen][type][zone]; ++/* ++ * Calculate the maximum gap in a node and check if that's what is reported in ++ * the parent (unless root). ++ */ ++static void mas_validate_gaps(struct ma_state *mas) ++{ ++ struct maple_enode *mte = mas->node; ++ struct maple_node *p_mn; ++ unsigned long gap = 0, max_gap = 0; ++ unsigned long p_end, p_start = mas->min; ++ unsigned char p_slot; ++ unsigned long *gaps = NULL; ++ unsigned long *pivots = ma_pivots(mte_to_node(mte), mte_node_type(mte)); ++ int i; + -+ if (!delta) -+ continue; ++ if (ma_is_dense(mte_node_type(mte))) { ++ for (i = 0; i < mt_slot_count(mte); i++) { ++ if (mas_get_slot(mas, i)) { ++ if (gap > max_gap) ++ max_gap = gap; ++ gap = 0; ++ continue; ++ } ++ gap++; ++ } ++ goto counted; ++ } + -+ walk->nr_pages[gen][type][zone] = 0; -+ WRITE_ONCE(lrugen->nr_pages[gen][type][zone], -+ lrugen->nr_pages[gen][type][zone] + delta); ++ gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte)); ++ for (i = 0; i < mt_slot_count(mte); i++) { ++ p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte)); + -+ if (lru_gen_is_active(lruvec, gen)) -+ lru += LRU_ACTIVE; -+ __update_lru_size(lruvec, lru, zone, delta); ++ if (!gaps) { ++ if (mas_get_slot(mas, i)) { ++ gap = 0; ++ goto not_empty; ++ } ++ ++ gap += p_end - p_start + 1; ++ } else { ++ void *entry = mas_get_slot(mas, i); ++ ++ gap = gaps[i]; ++ if (!entry) { ++ if (gap != p_end - p_start + 1) { ++ pr_err("%p[%u] -> %p %lu != %lu - %lu + 1\n", ++ mas_mn(mas), i, ++ mas_get_slot(mas, i), gap, ++ p_end, p_start); ++ mt_dump(mas->tree); ++ ++ MT_BUG_ON(mas->tree, ++ gap != p_end - p_start + 1); ++ } ++ } else { ++ if (gap > p_end - p_start + 1) { ++ pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", ++ mas_mn(mas), i, gap, p_end, p_start, ++ p_end - p_start + 1); ++ MT_BUG_ON(mas->tree, ++ gap > p_end - p_start + 1); ++ } ++ } ++ } ++ ++ if (gap > max_gap) ++ max_gap = gap; ++not_empty: ++ p_start = p_end + 1; ++ if (p_end >= mas->max) ++ break; ++ } ++ ++counted: ++ if (mte_is_root(mte)) ++ return; ++ ++ p_slot = mte_parent_slot(mas->node); ++ p_mn = mte_parent(mte); ++ MT_BUG_ON(mas->tree, max_gap > mas->max); ++ if (ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap) { ++ pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap); ++ mt_dump(mas->tree); + } ++ ++ MT_BUG_ON(mas->tree, ++ ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap); +} + -+static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) ++static void mas_validate_parent_slot(struct ma_state *mas) +{ -+ struct address_space *mapping; -+ struct vm_area_struct *vma = args->vma; -+ struct lru_gen_mm_walk *walk = args->private; -+ -+ if (!vma_is_accessible(vma)) -+ return true; ++ struct maple_node *parent; ++ struct maple_enode *node; ++ enum maple_type p_type = mas_parent_enum(mas, mas->node); ++ unsigned char p_slot = mte_parent_slot(mas->node); ++ void __rcu **slots; ++ int i; + -+ if (is_vm_hugetlb_page(vma)) -+ return true; ++ if (mte_is_root(mas->node)) ++ return; + -+ if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) -+ return true; ++ parent = mte_parent(mas->node); ++ slots = ma_slots(parent, p_type); ++ MT_BUG_ON(mas->tree, mas_mn(mas) == parent); ++ ++ /* Check prev/next parent slot for duplicate node entry */ ++ ++ for (i = 0; i < mt_slots[p_type]; i++) { ++ node = mas_slot(mas, slots, i); ++ if (i == p_slot) { ++ if (node != mas->node) ++ pr_err("parent %p[%u] does not have %p\n", ++ parent, i, mas_mn(mas)); ++ MT_BUG_ON(mas->tree, node != mas->node); ++ } else if (node == mas->node) { ++ pr_err("Invalid child %p at parent %p[%u] p_slot %u\n", ++ mas_mn(mas), parent, i, p_slot); ++ MT_BUG_ON(mas->tree, node == mas->node); ++ } ++ } ++} + -+ if (vma == get_gate_vma(vma->vm_mm)) -+ return true; ++static void mas_validate_child_slot(struct ma_state *mas) ++{ ++ enum maple_type type = mte_node_type(mas->node); ++ void __rcu **slots = ma_slots(mte_to_node(mas->node), type); ++ unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type); ++ struct maple_enode *child; ++ unsigned char i; + -+ if (vma_is_anonymous(vma)) -+ return !walk->can_swap; ++ if (mte_is_leaf(mas->node)) ++ return; + -+ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) -+ return true; ++ for (i = 0; i < mt_slots[type]; i++) { ++ child = mas_slot(mas, slots, i); ++ if (!pivots[i] || pivots[i] == mas->max) ++ break; + -+ mapping = vma->vm_file->f_mapping; -+ if (mapping_unevictable(mapping)) -+ return true; ++ if (!child) ++ break; + -+ if (shmem_mapping(mapping)) -+ return !walk->can_swap; ++ if (mte_parent_slot(child) != i) { ++ pr_err("Slot error at %p[%u]: child %p has pslot %u\n", ++ mas_mn(mas), i, mte_to_node(child), ++ mte_parent_slot(child)); ++ MT_BUG_ON(mas->tree, 1); ++ } + -+ /* to exclude special mappings like dax, etc. */ -+ return !mapping->a_ops->read_folio; ++ if (mte_parent(child) != mte_to_node(mas->node)) { ++ pr_err("child %p has parent %p not %p\n", ++ mte_to_node(child), mte_parent(child), ++ mte_to_node(mas->node)); ++ MT_BUG_ON(mas->tree, 1); ++ } ++ } +} + +/* -+ * Some userspace memory allocators map many single-page VMAs. Instead of -+ * returning back to the PGD table for each of such VMAs, finish an entire PMD -+ * table to reduce zigzags and improve cache performance. ++ * Validate all pivots are within mas->min and mas->max. + */ -+static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, -+ unsigned long *vm_start, unsigned long *vm_end) ++static void mas_validate_limits(struct ma_state *mas) +{ -+ unsigned long start = round_up(*vm_end, size); -+ unsigned long end = (start | ~mask) + 1; ++ int i; ++ unsigned long prev_piv = 0; ++ enum maple_type type = mte_node_type(mas->node); ++ void __rcu **slots = ma_slots(mte_to_node(mas->node), type); ++ unsigned long *pivots = ma_pivots(mas_mn(mas), type); + -+ VM_WARN_ON_ONCE(mask & size); -+ VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); ++ /* all limits are fine here. */ ++ if (mte_is_root(mas->node)) ++ return; + -+ while (args->vma) { -+ if (start >= args->vma->vm_end) { -+ args->vma = args->vma->vm_next; -+ continue; -+ } ++ for (i = 0; i < mt_slots[type]; i++) { ++ unsigned long piv; + -+ if (end && end <= args->vma->vm_start) -+ return false; ++ piv = mas_safe_pivot(mas, pivots, i, type); + -+ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { -+ args->vma = args->vma->vm_next; -+ continue; -+ } ++ if (!piv && (i != 0)) ++ break; + -+ *vm_start = max(start, args->vma->vm_start); -+ *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; ++ if (!mte_is_leaf(mas->node)) { ++ void *entry = mas_slot(mas, slots, i); + -+ return true; -+ } ++ if (!entry) ++ pr_err("%p[%u] cannot be null\n", ++ mas_mn(mas), i); + -+ return false; -+} ++ MT_BUG_ON(mas->tree, !entry); ++ } + -+static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) -+{ -+ unsigned long pfn = pte_pfn(pte); ++ if (prev_piv > piv) { ++ pr_err("%p[%u] piv %lu < prev_piv %lu\n", ++ mas_mn(mas), i, piv, prev_piv); ++ MT_BUG_ON(mas->tree, piv < prev_piv); ++ } + -+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); ++ if (piv < mas->min) { ++ pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i, ++ piv, mas->min); ++ MT_BUG_ON(mas->tree, piv < mas->min); ++ } ++ if (piv > mas->max) { ++ pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i, ++ piv, mas->max); ++ MT_BUG_ON(mas->tree, piv > mas->max); ++ } ++ prev_piv = piv; ++ if (piv == mas->max) ++ break; ++ } ++ for (i += 1; i < mt_slots[type]; i++) { ++ void *entry = mas_slot(mas, slots, i); + -+ if (!pte_present(pte) || is_zero_pfn(pfn)) -+ return -1; ++ if (entry && (i != mt_slots[type] - 1)) { ++ pr_err("%p[%u] should not have entry %p\n", mas_mn(mas), ++ i, entry); ++ MT_BUG_ON(mas->tree, entry != NULL); ++ } + -+ if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) -+ return -1; ++ if (i < mt_pivots[type]) { ++ unsigned long piv = pivots[i]; + -+ if (WARN_ON_ONCE(!pfn_valid(pfn))) -+ return -1; ++ if (!piv) ++ continue; + -+ return pfn; ++ pr_err("%p[%u] should not have piv %lu\n", ++ mas_mn(mas), i, piv); ++ MT_BUG_ON(mas->tree, i < mt_pivots[type] - 1); ++ } ++ } +} + -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -+static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) ++static void mt_validate_nulls(struct maple_tree *mt) +{ -+ unsigned long pfn = pmd_pfn(pmd); -+ -+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); ++ void *entry, *last = (void *)1; ++ unsigned char offset = 0; ++ void __rcu **slots; ++ MA_STATE(mas, mt, 0, 0); + -+ if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) -+ return -1; ++ mas_start(&mas); ++ if (mas_is_none(&mas) || (mas.node == MAS_ROOT)) ++ return; + -+ if (WARN_ON_ONCE(pmd_devmap(pmd))) -+ return -1; ++ while (!mte_is_leaf(mas.node)) ++ mas_descend(&mas); + -+ if (WARN_ON_ONCE(!pfn_valid(pfn))) -+ return -1; ++ slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); ++ do { ++ entry = mas_slot(&mas, slots, offset); ++ if (!last && !entry) { ++ pr_err("Sequential nulls end at %p[%u]\n", ++ mas_mn(&mas), offset); ++ } ++ MT_BUG_ON(mt, !last && !entry); ++ last = entry; ++ if (offset == mas_data_end(&mas)) { ++ mas_next_node(&mas, mas_mn(&mas), ULONG_MAX); ++ if (mas_is_none(&mas)) ++ return; ++ offset = 0; ++ slots = ma_slots(mte_to_node(mas.node), ++ mte_node_type(mas.node)); ++ } else { ++ offset++; ++ } + -+ return pfn; ++ } while (!mas_is_none(&mas)); +} -+#endif + -+static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, -+ struct pglist_data *pgdat, bool can_swap) ++/* ++ * validate a maple tree by checking: ++ * 1. The limits (pivots are within mas->min to mas->max) ++ * 2. The gap is correctly set in the parents ++ */ ++void mt_validate(struct maple_tree *mt) +{ -+ struct folio *folio; -+ -+ /* try to avoid unnecessary memory loads */ -+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) -+ return NULL; ++ unsigned char end; + -+ folio = pfn_folio(pfn); -+ if (folio_nid(folio) != pgdat->node_id) -+ return NULL; ++ MA_STATE(mas, mt, 0, 0); ++ rcu_read_lock(); ++ mas_start(&mas); ++ if (!mas_searchable(&mas)) ++ goto done; + -+ if (folio_memcg_rcu(folio) != memcg) -+ return NULL; ++ mas_first_entry(&mas, mas_mn(&mas), ULONG_MAX, mte_node_type(mas.node)); ++ while (!mas_is_none(&mas)) { ++ MT_BUG_ON(mas.tree, mte_dead_node(mas.node)); ++ if (!mte_is_root(mas.node)) { ++ end = mas_data_end(&mas); ++ if ((end < mt_min_slot_count(mas.node)) && ++ (mas.max != ULONG_MAX)) { ++ pr_err("Invalid size %u of %p\n", end, ++ mas_mn(&mas)); ++ MT_BUG_ON(mas.tree, 1); ++ } + -+ /* file VMAs can contain anon pages from COW */ -+ if (!folio_is_file_lru(folio) && !can_swap) -+ return NULL; ++ } ++ mas_validate_parent_slot(&mas); ++ mas_validate_child_slot(&mas); ++ mas_validate_limits(&mas); ++ if (mt_is_alloc(mt)) ++ mas_validate_gaps(&mas); ++ mas_dfs_postorder(&mas, ULONG_MAX); ++ } ++ mt_validate_nulls(mt); ++done: ++ rcu_read_unlock(); + -+ return folio; +} + -+static bool suitable_to_scan(int total, int young) -+{ -+ int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); ++#endif /* CONFIG_DEBUG_MAPLE_TREE */ +diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c +new file mode 100644 +index 000000000000..4f69e009a015 +--- /dev/null ++++ b/lib/test_maple_tree.c +@@ -0,0 +1,38307 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++/* ++ * test_maple_tree.c: Test the maple tree API ++ * Copyright (c) 2018 Liam R. Howlett ++ * Author: Liam R. Howlett ++ */ + -+ /* suitable if the average number of young PTEs per cacheline is >=1 */ -+ return young * n >= total; ++#include ++#include ++#include ++#include ++ ++#define MTREE_ALLOC_MAX 0x2000000000000Ul ++#define CONFIG_DEBUG_MAPLE_TREE ++#define CONFIG_MAPLE_SEARCH ++/* #define BENCH_SLOT_STORE */ ++/* #define BENCH_NODE_STORE */ ++/* #define BENCH_AWALK */ ++/* #define BENCH_WALK */ ++/* #define BENCH_MT_FOR_EACH */ ++/* #define BENCH_FORK */ ++static ++int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp) ++{ ++ return mtree_insert(mt, index, xa_mk_value(index & LONG_MAX), gfp); +} + -+static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, -+ struct mm_walk *args) ++static void mtree_erase_index(struct maple_tree *mt, unsigned long index) +{ -+ int i; -+ pte_t *pte; -+ spinlock_t *ptl; -+ unsigned long addr; -+ int total = 0; -+ int young = 0; -+ struct lru_gen_mm_walk *walk = args->private; -+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); ++ MT_BUG_ON(mt, mtree_erase(mt, index) != xa_mk_value(index & LONG_MAX)); ++ MT_BUG_ON(mt, mtree_load(mt, index) != NULL); ++} + -+ VM_WARN_ON_ONCE(pmd_leaf(*pmd)); ++static int mtree_test_insert(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ return mtree_insert(mt, index, ptr, GFP_KERNEL); ++} + -+ ptl = pte_lockptr(args->mm, pmd); -+ if (!spin_trylock(ptl)) -+ return false; ++static int mtree_test_store_range(struct maple_tree *mt, unsigned long start, ++ unsigned long end, void *ptr) ++{ ++ return mtree_store_range(mt, start, end, ptr, GFP_KERNEL); ++} + -+ arch_enter_lazy_mmu_mode(); ++static int mtree_test_store(struct maple_tree *mt, unsigned long start, ++ void *ptr) ++{ ++ return mtree_test_store_range(mt, start, start, ptr); ++} + -+ pte = pte_offset_map(pmd, start & PMD_MASK); -+restart: -+ for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { -+ unsigned long pfn; -+ struct folio *folio; ++static int mtree_test_insert_range(struct maple_tree *mt, unsigned long start, ++ unsigned long end, void *ptr) ++{ ++ return mtree_insert_range(mt, start, end, ptr, GFP_KERNEL); ++} + -+ total++; -+ walk->mm_stats[MM_LEAF_TOTAL]++; ++static void *mtree_test_load(struct maple_tree *mt, unsigned long index) ++{ ++ return mtree_load(mt, index); ++} + -+ pfn = get_pte_pfn(pte[i], args->vma, addr); -+ if (pfn == -1) -+ continue; ++static void *mtree_test_erase(struct maple_tree *mt, unsigned long index) ++{ ++ return mtree_erase(mt, index); ++} + -+ if (!pte_young(pte[i])) { -+ walk->mm_stats[MM_LEAF_OLD]++; -+ continue; -+ } ++static noinline void check_mtree_alloc_range(struct maple_tree *mt, ++ unsigned long start, unsigned long end, unsigned long size, ++ unsigned long expected, int eret, void *ptr) ++{ + -+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); -+ if (!folio) -+ continue; ++ unsigned long result = expected + 1; ++ int ret; + -+ if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) -+ VM_WARN_ON_ONCE(true); ++ ret = mtree_alloc_range(mt, &result, ptr, size, start, end, ++ GFP_KERNEL); ++ MT_BUG_ON(mt, ret != eret); ++ if (ret) ++ return; + -+ young++; -+ walk->mm_stats[MM_LEAF_YOUNG]++; ++ MT_BUG_ON(mt, result != expected); ++} + -+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); ++static noinline void check_mtree_alloc_rrange(struct maple_tree *mt, ++ unsigned long start, unsigned long end, unsigned long size, ++ unsigned long expected, int eret, void *ptr) ++{ + -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen >= 0 && old_gen != new_gen) -+ update_batch_size(walk, folio, old_gen, new_gen); -+ } ++ unsigned long result = expected + 1; ++ int ret; + -+ if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) -+ goto restart; ++ ret = mtree_alloc_rrange(mt, &result, ptr, size, start, end - 1, ++ GFP_KERNEL); ++ MT_BUG_ON(mt, ret != eret); ++ if (ret) ++ return; + -+ pte_unmap(pte); ++ MT_BUG_ON(mt, result != expected); ++} + -+ arch_leave_lazy_mmu_mode(); -+ spin_unlock(ptl); ++static noinline void check_load(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ void *ret = mtree_test_load(mt, index); + -+ return suitable_to_scan(total, young); ++ if (ret != ptr) ++ pr_err("Load %lu returned %p expect %p\n", index, ret, ptr); ++ MT_BUG_ON(mt, ret != ptr); +} + -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, -+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) ++static noinline void check_store_range(struct maple_tree *mt, ++ unsigned long start, unsigned long end, void *ptr, int expected) +{ -+ int i; -+ pmd_t *pmd; -+ spinlock_t *ptl; -+ struct lru_gen_mm_walk *walk = args->private; -+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); -+ -+ VM_WARN_ON_ONCE(pud_leaf(*pud)); ++ int ret = -EINVAL; ++ unsigned long i; + -+ /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ -+ if (*start == -1) { -+ *start = next; -+ return; -+ } ++ ret = mtree_test_store_range(mt, start, end, ptr); ++ MT_BUG_ON(mt, ret != expected); + -+ i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); -+ if (i && i <= MIN_LRU_BATCH) { -+ __set_bit(i - 1, bitmap); ++ if (ret) + return; -+ } + -+ pmd = pmd_offset(pud, *start); ++ for (i = start; i <= end; i++) ++ check_load(mt, i, ptr); ++} + -+ ptl = pmd_lockptr(args->mm, pmd); -+ if (!spin_trylock(ptl)) -+ goto done; ++static noinline void check_insert_range(struct maple_tree *mt, ++ unsigned long start, unsigned long end, void *ptr, int expected) ++{ ++ int ret = -EINVAL; ++ unsigned long i; + -+ arch_enter_lazy_mmu_mode(); ++ ret = mtree_test_insert_range(mt, start, end, ptr); ++ MT_BUG_ON(mt, ret != expected); + -+ do { -+ unsigned long pfn; -+ struct folio *folio; -+ unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; ++ if (ret) ++ return; + -+ pfn = get_pmd_pfn(pmd[i], vma, addr); -+ if (pfn == -1) -+ goto next; ++ for (i = start; i <= end; i++) ++ check_load(mt, i, ptr); ++} + -+ if (!pmd_trans_huge(pmd[i])) { -+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && -+ get_cap(LRU_GEN_NONLEAF_YOUNG)) -+ pmdp_test_and_clear_young(vma, addr, pmd + i); -+ goto next; -+ } ++static noinline void check_insert(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ int ret = -EINVAL; + -+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); -+ if (!folio) -+ goto next; ++ ret = mtree_test_insert(mt, index, ptr); ++ MT_BUG_ON(mt, ret != 0); ++} + -+ if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) -+ goto next; ++static noinline void check_erase(struct maple_tree *mt, unsigned long index, ++ void *ptr) ++{ ++ MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr); ++} + -+ walk->mm_stats[MM_LEAF_YOUNG]++; ++static noinline void check_dup_insert(struct maple_tree *mt, ++ unsigned long index, void *ptr) ++{ ++ int ret = -EINVAL; + -+ if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); ++ ret = mtree_test_insert(mt, index, ptr); ++ MT_BUG_ON(mt, ret != -EEXIST); ++} + -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen >= 0 && old_gen != new_gen) -+ update_batch_size(walk, folio, old_gen, new_gen); -+next: -+ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; -+ } while (i <= MIN_LRU_BATCH); + -+ arch_leave_lazy_mmu_mode(); -+ spin_unlock(ptl); -+done: -+ *start = -1; -+ bitmap_zero(bitmap, MIN_LRU_BATCH); -+} -+#else -+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, -+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start) ++static noinline ++void check_index_load(struct maple_tree *mt, unsigned long index) +{ ++ return check_load(mt, index, xa_mk_value(index & LONG_MAX)); +} -+#endif + -+static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, -+ struct mm_walk *args) ++static noinline void check_nomem(struct maple_tree *mt) +{ -+ int i; -+ pmd_t *pmd; -+ unsigned long next; -+ unsigned long addr; -+ struct vm_area_struct *vma; -+ unsigned long pos = -1; -+ struct lru_gen_mm_walk *walk = args->private; -+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; ++ MA_STATE(ms, mt, 1, 1); + -+ VM_WARN_ON_ONCE(pud_leaf(*pud)); ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ /* Ensure no bypassing of allocation failures */ ++ mt_set_non_kernel(0); ++ ++ /* Storing something at 1 requires memory allocation */ ++ MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM); ++ /* Storing something at 0 does not */ ++ MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0); + + /* -+ * Finish an entire PMD in two passes: the first only reaches to PTE -+ * tables to avoid taking the PMD lock; the second, if necessary, takes -+ * the PMD lock to clear the accessed bit in PMD entries. ++ * Simulate two threads racing; the first one fails to allocate ++ * memory to insert an entry at 1, then the second one succeeds ++ * in allocating memory to insert an entry at 2. The first one ++ * then needs to free the node it allocated. LeakSanitizer will ++ * notice this, as will the 'nr_allocated' debugging aid in the ++ * userspace test suite. + */ -+ pmd = pmd_offset(pud, start & PUD_MASK); -+restart: -+ /* walk_pte_range() may call get_next_vma() */ -+ vma = args->vma; -+ for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { -+ pmd_t val = pmd_read_atomic(pmd + i); ++ mtree_lock(mt); ++ mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */ ++ MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM)); ++ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ ++ MT_BUG_ON(mt, ms.node != MAS_START); ++ mtree_unlock(mt); ++ MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0); ++ mtree_lock(mt); ++ mas_store(&ms, &ms); /* insert 1 -> &ms */ ++ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++} ++ ++static inline int not_empty(struct maple_node *node) ++{ ++ int i; + -+ /* for pmd_read_atomic() */ -+ barrier(); ++ if (node->parent) ++ return 1; + -+ next = pmd_addr_end(addr, end); ++ for (i = 0; i < ARRAY_SIZE(node->slot); i++) ++ if (node->slot[i]) ++ return 1; + -+ if (!pmd_present(val) || is_huge_zero_pmd(val)) { -+ walk->mm_stats[MM_LEAF_TOTAL]++; -+ continue; ++ return 0; ++} ++ ++static noinline void check_new_node(struct maple_tree *mt) ++{ ++ ++ struct maple_node *mn, *mn2, *mn3; ++ struct maple_alloc *smn; ++ struct maple_node *nodes[100]; ++ int i, j, total; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ /* Try allocating 3 nodes */ ++ mtree_lock(mt); ++ /* request 3 nodes to be allocated. */ ++ mas_node_count(&mas, 3); ++ /* Allocation request of 3. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); ++ /* Allocate failed. */ ++ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, mas.alloc == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); ++ mas_push_node(&mas, mn); ++ mas_nomem(&mas, GFP_KERNEL); /* free */ ++ mtree_unlock(mt); ++ ++ ++ /* Try allocating 1 node, then 2 more */ ++ mtree_lock(mt); ++ /* Set allocation request to 1. */ ++ mas_set_alloc_req(&mas, 1); ++ /* Check Allocation request of 1. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); ++ mas_set_err(&mas, -ENOMEM); ++ /* Validate allocation request. */ ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ /* Eat the requested node. */ ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, mn->slot[0] != NULL); ++ MT_BUG_ON(mt, mn->slot[1] != NULL); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ ma_free_rcu(mn); ++ mas.node = MAS_START; ++ mas_nomem(&mas, GFP_KERNEL); ++ /* Allocate 3 nodes, will fail. */ ++ mas_node_count(&mas, 3); ++ /* Drop the lock and allocate 3 nodes. */ ++ mas_nomem(&mas, GFP_KERNEL); ++ /* Ensure 3 are allocated. */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ /* Allocation request of 0. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); ++ ++ MT_BUG_ON(mt, mas.alloc == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); ++ /* Ensure we counted 3. */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ /* Free. */ ++ mas_nomem(&mas, GFP_KERNEL); ++ ++ /* Set allocation request to 1. */ ++ mas_set_alloc_req(&mas, 1); ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); ++ mas_set_err(&mas, -ENOMEM); ++ /* Validate allocation request. */ ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 1); ++ /* Check the node is only one node. */ ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, mn->slot[0] != NULL); ++ MT_BUG_ON(mt, mn->slot[1] != NULL); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 1); ++ MT_BUG_ON(mt, mas.alloc->node_count); ++ ++ mas_set_alloc_req(&mas, 2); /* request 2 more. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 3); ++ MT_BUG_ON(mt, mas.alloc == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); ++ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); ++ for (i = 2; i >= 0; i--) { ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); ++ MT_BUG_ON(mt, !mn); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ ++ total = 64; ++ mas_set_alloc_req(&mas, total); /* request 2 more. */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != total); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ for (i = total; i > 0; i--) { ++ unsigned int e = 0; /* expected node_count */ ++ ++ if (i >= 35) ++ e = i - 35; ++ else if (i >= 5) ++ e = i - 5; ++ else if (i >= 2) ++ e = i - 2; ++ MT_BUG_ON(mt, mas.alloc->node_count != e); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); ++ MT_BUG_ON(mt, !mn); ++ ma_free_rcu(mn); ++ } ++ ++ total = 100; ++ for (i = 1; i < total; i++) { ++ mas_set_alloc_req(&mas, i); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ for (j = i; j > 0; j--) { ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); ++ MT_BUG_ON(mt, !mn); ++ MT_BUG_ON(mt, not_empty(mn)); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != j); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ mas_set_alloc_req(&mas, i); ++ mas_set_err(&mas, -ENOMEM); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ for (j = 0; j <= i/2; j++) { ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); ++ nodes[j] = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); + } + -+#ifdef CONFIG_TRANSPARENT_HUGEPAGE -+ if (pmd_trans_huge(val)) { -+ unsigned long pfn = pmd_pfn(val); -+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); ++ while (j) { ++ j--; ++ mas_push_node(&mas, nodes[j]); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); ++ for (j = 0; j <= i/2; j++) { ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); ++ } ++ MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); ++ ++ } ++ ++ /* Set allocation request. */ ++ total = 500; ++ mas_node_count(&mas, total); ++ /* Drop the lock and allocate the nodes. */ ++ mas_nomem(&mas, GFP_KERNEL); ++ MT_BUG_ON(mt, !mas.alloc); ++ i = 1; ++ smn = mas.alloc; ++ while (i < total) { ++ for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) { ++ i++; ++ MT_BUG_ON(mt, !smn->slot[j]); ++ if (i == total) ++ break; ++ } ++ smn = smn->slot[0]; /* next. */ ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != total); ++ mas_nomem(&mas, GFP_KERNEL); /* Free. */ ++ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ for (i = 1; i < 128; i++) { ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ ++ for (j = i; j > 0; j--) { /*Free the requests */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ } ++ ++ for (i = 1; i < MAPLE_NODE_MASK + 1; i++) { ++ MA_STATE(mas2, mt, 0, 0); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ ++ for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ mas_push_node(&mas2, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas2) != j); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ MT_BUG_ON(mt, mas_allocated(&mas2) != i); ++ ++ for (j = i; j > 0; j--) { /*Free the requests */ ++ MT_BUG_ON(mt, mas_allocated(&mas2) != j); ++ mn = mas_pop_node(&mas2); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas2) != 0); ++ } ++ ++ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ ++ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); ++ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ MT_BUG_ON(mt, mn == NULL); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2); ++ ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); ++ ++ /* Check the limit of pop/push/pop */ ++ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ ++ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); ++ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); ++ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); ++ MT_BUG_ON(mt, mas_alloc_req(&mas)); ++ MT_BUG_ON(mt, mas.alloc->node_count); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); ++ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas.alloc->node_count); ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) { ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, not_empty(mn)); ++ ma_free_rcu(mn); ++ } ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ ++ for (i = 3; i < MAPLE_NODE_MASK * 3; i++) { ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ mas_push_node(&mas, mn); /* put it back */ ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ mn2 = mas_pop_node(&mas); /* get the next node. */ ++ mas_push_node(&mas, mn); /* put them back */ ++ mas_push_node(&mas, mn2); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ mn2 = mas_pop_node(&mas); /* get the next node. */ ++ mn3 = mas_pop_node(&mas); /* get the next node. */ ++ mas_push_node(&mas, mn); /* put them back */ ++ mas_push_node(&mas, mn2); ++ mas_push_node(&mas, mn3); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, i); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mn = mas_pop_node(&mas); /* get the next node. */ ++ ma_free_rcu(mn); ++ mas_destroy(&mas); ++ } ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, 5); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != 5); ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, 10); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mas.node = MAS_START; ++ MT_BUG_ON(mt, mas_allocated(&mas) != 10); ++ mas_destroy(&mas); ++ ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); ++ mas.node = MA_ERROR(-ENOMEM); ++ mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ ++ mas_nomem(&mas, GFP_KERNEL); /* Fill request */ ++ mas.node = MAS_START; ++ MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); ++ mas_destroy(&mas); ++ ++ mtree_unlock(mt); ++} ++ ++static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, ++ bool verbose) ++{ ++ unsigned long i = max, j; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ ++ mt_zero_nr_tallocated(); ++ while (i) { ++ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); ++ for (j = i; j <= max; j++) ++ check_index_load(mt, j); ++ ++ check_load(mt, i - 1, NULL); ++ mt_set_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mt_clear_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ i--; ++ } ++ check_load(mt, max + 1, NULL); ++ ++ if (verbose) { ++ rcu_barrier(); ++ mt_dump(mt); ++ pr_info(" %s test of 0-%lu %luK in %d active (%d total)\n", ++ __func__, max, mt_get_alloc_size()/1024, mt_nr_allocated(), ++ mt_nr_tallocated()); ++ } ++} ++ ++static noinline void check_seq(struct maple_tree *mt, unsigned long max, ++ bool verbose) ++{ ++ unsigned long i, j; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ ++ mt_zero_nr_tallocated(); ++ for (i = 0; i <= max; i++) { ++ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); ++ for (j = 0; j <= i; j++) ++ check_index_load(mt, j); ++ ++ if (i) ++ MT_BUG_ON(mt, !mt_height(mt)); ++ check_load(mt, i + 1, NULL); ++ } ++ if (verbose) { ++ rcu_barrier(); ++ mt_dump(mt); ++ pr_info(" seq test of 0-%lu %luK in %d active (%d total)\n", ++ max, mt_get_alloc_size()/1024, mt_nr_allocated(), ++ mt_nr_tallocated()); ++ } ++} ++ ++static noinline void check_lb_not_empty(struct maple_tree *mt) ++{ ++ unsigned long i, j; ++ unsigned long huge = 4000UL * 1000 * 1000; + -+ walk->mm_stats[MM_LEAF_TOTAL]++; + -+ if (!pmd_young(val)) { -+ walk->mm_stats[MM_LEAF_OLD]++; -+ continue; -+ } ++ i = huge; ++ while (i > 4096) { ++ check_insert(mt, i, (void *) i); ++ for (j = huge; j >= i; j /= 2) { ++ check_load(mt, j-1, NULL); ++ check_load(mt, j, (void *) j); ++ check_load(mt, j+1, NULL); ++ } ++ i /= 2; ++ } ++ mtree_destroy(mt); ++} + -+ /* try to avoid unnecessary memory loads */ -+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) -+ continue; ++static noinline void check_lower_bound_split(struct maple_tree *mt) ++{ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_lb_not_empty(mt); ++} + -+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); -+ continue; -+ } -+#endif -+ walk->mm_stats[MM_NONLEAF_TOTAL]++; ++static noinline void check_upper_bound_split(struct maple_tree *mt) ++{ ++ unsigned long i, j; ++ unsigned long huge = 4000UL * 1000 * 1000; + -+#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG -+ if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { -+ if (!pmd_young(val)) -+ continue; ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ ++ i = 4096; ++ while (i < huge) { ++ check_insert(mt, i, (void *) i); ++ for (j = i; j >= huge; j *= 2) { ++ check_load(mt, j-1, NULL); ++ check_load(mt, j, (void *) j); ++ check_load(mt, j+1, NULL); ++ } ++ i *= 2; ++ } ++ mtree_destroy(mt); ++} + -+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); -+ } -+#endif -+ if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) -+ continue; ++static noinline void check_mid_split(struct maple_tree *mt) ++{ ++ unsigned long huge = 8000UL * 1000 * 1000; + -+ walk->mm_stats[MM_NONLEAF_FOUND]++; ++ check_insert(mt, huge, (void *) huge); ++ check_insert(mt, 0, xa_mk_value(0)); ++ check_lb_not_empty(mt); ++} + -+ if (!walk_pte_range(&val, addr, next, args)) -+ continue; ++static noinline void check_rev_find(struct maple_tree *mt) ++{ ++ int i, nr_entries = 200; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); + -+ walk->mm_stats[MM_NONLEAF_ADDED]++; ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); + -+ /* carry over to the next generation */ -+ update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); -+ } ++ mas_set(&mas, 1000); ++ val = mas_find_rev(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(100)); ++ val = mas_find_rev(&mas, 1000); ++ MT_BUG_ON(mt, val != NULL); + -+ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); ++ mas_set(&mas, 999); ++ val = mas_find_rev(&mas, 997); ++ MT_BUG_ON(mt, val != NULL); + -+ if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) -+ goto restart; ++ mas_set(&mas, 1000); ++ val = mas_find_rev(&mas, 900); ++ MT_BUG_ON(mt, val != xa_mk_value(100)); ++ val = mas_find_rev(&mas, 900); ++ MT_BUG_ON(mt, val != xa_mk_value(99)); ++ ++ mas_set(&mas, 20); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(2)); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(1)); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(0)); ++ val = mas_find_rev(&mas, 0); ++ MT_BUG_ON(mt, val != NULL); +} + -+static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, -+ struct mm_walk *args) ++static noinline void check_find(struct maple_tree *mt) +{ -+ int i; -+ pud_t *pud; -+ unsigned long addr; -+ unsigned long next; -+ struct lru_gen_mm_walk *walk = args->private; ++ unsigned long val = 0; ++ unsigned long count = 20; ++ unsigned long max; ++ unsigned long last = 0, index = 0; ++ void *entry, *entry2; + -+ VM_WARN_ON_ONCE(p4d_leaf(*p4d)); ++ MA_STATE(mas, mt, 0, 0); + -+ pud = pud_offset(p4d, start & P4D_MASK); -+restart: -+ for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { -+ pud_t val = READ_ONCE(pud[i]); ++ /* Insert 0. */ ++ MT_BUG_ON(mt, mtree_insert_index(mt, val++, GFP_KERNEL)); + -+ next = pud_addr_end(addr, end); ++ for (int i = 0; i <= count; i++) { ++ if (val != 64) ++ MT_BUG_ON(mt, mtree_insert_index(mt, val, GFP_KERNEL)); ++ else ++ MT_BUG_ON(mt, mtree_insert(mt, val, ++ XA_ZERO_ENTRY, GFP_KERNEL)); + -+ if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) -+ continue; ++ val <<= 2; ++ } + -+ walk_pmd_range(&val, addr, next, args); ++ val = 0; ++ mas_set(&mas, val); ++ mas_lock(&mas); ++ while ((entry = mas_find(&mas, 268435456)) != NULL) { ++ if (val != 64) ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ else ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); + -+ /* a racy check to curtail the waiting time */ -+ if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) -+ return 1; ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ } ++ mas_unlock(&mas); + -+ if (need_resched() || walk->batched >= MAX_LRU_BATCH) { -+ end = (addr | ~PUD_MASK) + 1; -+ goto done; -+ } ++ val = 0; ++ mas_set(&mas, val); ++ mas_lock(&mas); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (val != 64) ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ else ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ } ++ mas_unlock(&mas); ++ ++ /* Test mas_pause */ ++ val = 0; ++ mas_set(&mas, val); ++ mas_lock(&mas); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (val != 64) ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ else ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ ++ mas_pause(&mas); ++ mas_unlock(&mas); ++ mas_lock(&mas); ++ } ++ mas_unlock(&mas); ++ ++ val = 0; ++ max = 300; /* A value big enough to include XA_ZERO_ENTRY at 64. */ ++ mt_for_each(mt, entry, index, max) { ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ val <<= 2; ++ if (val == 64) /* Skip zero entry. */ ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ } ++ ++ val = 0; ++ max = 0; ++ index = 0; ++ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); ++ mt_for_each(mt, entry, index, ULONG_MAX) { ++ if (val == 4398046511104) ++ MT_BUG_ON(mt, entry != ++ xa_mk_value(ULONG_MAX & LONG_MAX)); ++ else ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ val <<= 2; ++ if (val == 64) /* Skip zero entry. */ ++ val <<= 2; ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ max++; ++ MT_BUG_ON(mt, max > 25); ++ } ++ mtree_erase_index(mt, ULONG_MAX); ++ ++ mas_reset(&mas); ++ index = 17; ++ entry = mt_find(mt, &index, 512); ++ MT_BUG_ON(mt, xa_mk_value(256) != entry); ++ ++ mas_reset(&mas); ++ index = 17; ++ entry = mt_find(mt, &index, 20); ++ MT_BUG_ON(mt, entry != NULL); ++ ++ ++ /* Range check.. */ ++ /* Insert ULONG_MAX */ ++ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); ++ ++ val = 0; ++ mas_set(&mas, 0); ++ mas_lock(&mas); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (val == 64) ++ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); ++ else if (val == 4398046511104) ++ MT_BUG_ON(mt, entry != xa_mk_value(ULONG_MAX & LONG_MAX)); ++ else ++ MT_BUG_ON(mt, xa_mk_value(val) != entry); ++ val <<= 2; ++ ++ /* For zero check. */ ++ if (!val) ++ val = 1; ++ mas_pause(&mas); ++ mas_unlock(&mas); ++ mas_lock(&mas); + } ++ mas_unlock(&mas); + -+ if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) -+ goto restart; ++ mas_set(&mas, 1048576); ++ mas_lock(&mas); ++ entry = mas_find(&mas, 1048576); ++ mas_unlock(&mas); ++ MT_BUG_ON(mas.tree, entry == NULL); + -+ end = round_up(end, P4D_SIZE); -+done: -+ if (!end || !args->vma) -+ return 1; ++ /* ++ * Find last value. ++ * 1. get the expected value, leveraging the existence of an end entry ++ * 2. delete end entry ++ * 3. find the last value but searching for ULONG_MAX and then using ++ * prev ++ */ ++ /* First, get the expected result. */ ++ mas_lock(&mas); ++ mas_reset(&mas); ++ mas.index = ULONG_MAX; /* start at max.. */ ++ entry = mas_find(&mas, ULONG_MAX); ++ entry = mas_prev(&mas, 0); ++ index = mas.index; ++ last = mas.last; + -+ walk->next_addr = max(end, args->vma->vm_start); ++ /* Erase the last entry. */ ++ mas_reset(&mas); ++ mas.index = ULONG_MAX; ++ mas.last = ULONG_MAX; ++ mas_erase(&mas); + -+ return -EAGAIN; -+} ++ /* Get the previous value from MAS_START */ ++ mas_reset(&mas); ++ entry2 = mas_prev(&mas, 0); + -+static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) -+{ -+ static const struct mm_walk_ops mm_walk_ops = { -+ .test_walk = should_skip_vma, -+ .p4d_entry = walk_pud_range, -+ }; ++ /* Check results. */ ++ MT_BUG_ON(mt, entry != entry2); ++ MT_BUG_ON(mt, index != mas.index); ++ MT_BUG_ON(mt, last != mas.last); + -+ int err; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); + -+ walk->next_addr = FIRST_USER_ADDRESS; ++ mas.node = MAS_NONE; ++ mas.index = ULONG_MAX; ++ mas.last = ULONG_MAX; ++ entry2 = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, entry != entry2); + -+ do { -+ err = -EBUSY; ++ mas_set(&mas, 0); ++ MT_BUG_ON(mt, mas_prev(&mas, 0) != NULL); + -+ /* folio_update_gen() requires stable folio_memcg() */ -+ if (!mem_cgroup_trylock_pages(memcg)) -+ break; ++ mas_unlock(&mas); ++ mtree_destroy(mt); ++} + -+ /* the caller might be holding the lock for write */ -+ if (mmap_read_trylock(mm)) { -+ err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); ++static noinline void check_find_2(struct maple_tree *mt) ++{ ++ unsigned long i, j; ++ void *entry; + -+ mmap_read_unlock(mm); ++ MA_STATE(mas, mt, 0, 0); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) ++ MT_BUG_ON(mt, true); ++ rcu_read_unlock(); ++ ++ for (i = 0; i < 256; i++) { ++ mtree_insert_index(mt, i, GFP_KERNEL); ++ j = 0; ++ mas_set(&mas, 0); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ MT_BUG_ON(mt, entry != xa_mk_value(j)); ++ j++; + } ++ rcu_read_unlock(); ++ MT_BUG_ON(mt, j != i + 1); ++ } + -+ mem_cgroup_unlock_pages(); ++ for (i = 0; i < 256; i++) { ++ mtree_erase_index(mt, i); ++ j = i + 1; ++ mas_set(&mas, 0); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (xa_is_zero(entry)) ++ continue; + -+ if (walk->batched) { -+ spin_lock_irq(&lruvec->lru_lock); -+ reset_batch_size(lruvec, walk); -+ spin_unlock_irq(&lruvec->lru_lock); ++ MT_BUG_ON(mt, entry != xa_mk_value(j)); ++ j++; + } ++ rcu_read_unlock(); ++ MT_BUG_ON(mt, j != 256); ++ } + -+ cond_resched(); -+ } while (err == -EAGAIN); ++ /*MT_BUG_ON(mt, !mtree_empty(mt)); */ +} + -+static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) -+{ -+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; ++#define erase_ptr(i) entry[i%2] ++#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2]) ++#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2]) ++#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2]) + -+ if (pgdat && current_is_kswapd()) { -+ VM_WARN_ON_ONCE(walk); ++static noinline void check_erase_testset(struct maple_tree *mt) ++{ ++ unsigned long set[] = { 5015, 5014, 5017, 25, 1000, ++ 1001, 1002, 1003, 1005, 0, ++ 6003, 6002, 6008, 6012, 6015, ++ 7003, 7002, 7008, 7012, 7015, ++ 8003, 8002, 8008, 8012, 8015, ++ 9003, 9002, 9008, 9012, 9015, ++ 10003, 10002, 10008, 10012, 10015, ++ 11003, 11002, 11008, 11012, 11015, ++ 12003, 12002, 12008, 12012, 12015, ++ 13003, 13002, 13008, 13012, 13015, ++ 14003, 14002, 14008, 14012, 14015, ++ 15003, 15002, 15008, 15012, 15015, ++ }; + -+ walk = &pgdat->mm_walk; -+ } else if (!pgdat && !walk) { -+ VM_WARN_ON_ONCE(current_is_kswapd()); + -+ walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); -+ } ++ void *ptr = &set; ++ void *entry[2] = { ptr, mt }; ++ void *root_node; + -+ current->reclaim_state->mm_walk = walk; + -+ return walk; -+} ++ rcu_register_thread(); ++ mt_set_in_rcu(mt); ++ for (int i = 0; i < 4; i++) ++ erase_check_insert(mt, i); ++ for (int i = 0; i < 4; i++) ++ erase_check_load(mt, i); + -+static void clear_mm_walk(void) -+{ -+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 1); ++ erase_check_load(mt, 0); ++ check_load(mt, set[1], NULL); ++ for (int i = 2; i < 4; i++) ++ erase_check_load(mt, i); + -+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); -+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); + -+ current->reclaim_state->mm_walk = NULL; ++ erase_check_erase(mt, 2); ++ erase_check_load(mt, 0); ++ check_load(mt, set[1], NULL); ++ check_load(mt, set[2], NULL); + -+ if (!current_is_kswapd()) -+ kfree(walk); -+} ++ erase_check_insert(mt, 1); ++ erase_check_insert(mt, 2); + -+static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) -+{ -+ int zone; -+ int remaining = MAX_LRU_BATCH; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ for (int i = 0; i < 4; i++) ++ erase_check_load(mt, i); + -+ if (type == LRU_GEN_ANON && !can_swap) -+ goto done; ++ /* Check erase and load without an allocation. */ ++ erase_check_load(mt, 3); ++ erase_check_erase(mt, 1); ++ erase_check_load(mt, 0); ++ check_load(mt, set[1], NULL); ++ for (int i = 2; i < 4; i++) ++ erase_check_load(mt, i); + -+ /* prevent cold/hot inversion if force_scan is true */ -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ struct list_head *head = &lrugen->lists[old_gen][type][zone]; ++ /* ++ * Set the newly erased node. This will produce a different allocated ++ * node to avoid busy slots. ++ */ ++ root_node = mt->ma_root; ++ erase_check_insert(mt, 1); ++ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ erase_check_load(mt, 2); ++ check_load(mt, 5018, NULL); ++ erase_check_load(mt, 3); ++ ++ erase_check_erase(mt, 2); /* erase 5017 to check append */ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ check_load(mt, set[2], NULL); ++ check_load(mt, 5018, NULL); ++ ++ erase_check_load(mt, 3); ++ ++ root_node = mt->ma_root; ++ erase_check_insert(mt, 2); ++ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ erase_check_load(mt, 2); ++ check_load(mt, 5018, NULL); ++ erase_check_load(mt, 3); ++ ++ mt_set_non_kernel(1); ++ erase_check_erase(mt, 2); /* erase 5017 to check append */ ++ erase_check_load(mt, 0); ++ check_load(mt, 5016, NULL); ++ check_load(mt, set[2], NULL); ++ erase_check_erase(mt, 0); /* erase 5015 to check append */ ++ check_load(mt, set[0], NULL); ++ check_load(mt, 5016, NULL); ++ erase_check_insert(mt, 4); /* 1000 < Should not split. */ ++ check_load(mt, set[0], NULL); ++ check_load(mt, 5016, NULL); ++ erase_check_load(mt, 1); ++ check_load(mt, 5013, NULL); ++ check_load(mt, set[2], NULL); ++ check_load(mt, 5018, NULL); ++ erase_check_load(mt, 4); ++ check_load(mt, 999, NULL); ++ check_load(mt, 1001, NULL); ++ erase_check_load(mt, 4); ++ if (mt_in_rcu(mt)) ++ MT_BUG_ON(mt, root_node == mt->ma_root); ++ else ++ MT_BUG_ON(mt, root_node != mt->ma_root); + -+ while (!list_empty(head)) { -+ struct folio *folio = lru_to_folio(head); ++ /* Should not have split. */ ++ MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root)); + -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + -+ new_gen = folio_inc_gen(lruvec, folio, false); -+ list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); ++ /* Coalesce testing */ ++ erase_check_insert(mt, 0); ++ erase_check_insert(mt, 2); + -+ if (!--remaining) -+ return false; -+ } ++ for (int i = 5; i < 25; i++) { ++ erase_check_insert(mt, i); ++ for (int j = i; j >= 0; j--) ++ erase_check_load(mt, j); + } -+done: -+ reset_ctrl_pos(lruvec, type, true); -+ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); + -+ return true; -+} ++ erase_check_erase(mt, 14); /*6015 */ ++ for (int i = 0; i < 25; i++) { ++ if (i == 14) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ erase_check_erase(mt, 16); /*7002 */ ++ for (int i = 0; i < 25; i++) { ++ if (i == 16 || i == 14) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } + -+static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) -+{ -+ int gen, type, zone; -+ bool success = false; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ DEFINE_MIN_SEQ(lruvec); + -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); ++ mt_set_non_kernel(1); ++ erase_check_erase(mt, 13); /*6012 */ ++ for (int i = 0; i < 25; i++) { ++ if (i == 16 || i == 14 || i == 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } + -+ /* find the oldest populated generation */ -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { -+ gen = lru_gen_from_seq(min_seq[type]); ++ erase_check_erase(mt, 15); /*7003 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 16 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } + -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ if (!list_empty(&lrugen->lists[gen][type][zone])) -+ goto next; -+ } ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 17 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } + -+ min_seq[type]++; -+ } -+next: -+ ; ++ erase_check_erase(mt, 18); /*7012 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 18 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); + } + -+ /* see the comment on lru_gen_struct */ -+ if (can_swap) { -+ min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); -+ min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 19); /*7015 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 19 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); + } + -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ if (min_seq[type] == lrugen->min_seq[type]) -+ continue; ++ erase_check_erase(mt, 20); /*8003 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 20 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } + -+ reset_ctrl_pos(lruvec, type, true); -+ WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); -+ success = true; ++ erase_check_erase(mt, 21); /*8002 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 21 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); + } + -+ return success; -+} ++ mt_set_non_kernel(2); ++ erase_check_erase(mt, 22); /*8008 */ ++ for (int i = 0; i < 25; i++) { ++ if (i <= 22 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } ++ for (int i = 23; i < 25; i++) ++ erase_check_erase(mt, i); + -+static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) -+{ -+ int prev, next; -+ int type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ for (int i = 0; i < 25; i++) { ++ if (i <= 25 && i >= 13) ++ check_load(mt, set[i], NULL); ++ else ++ erase_check_load(mt, i); ++ } + -+ spin_lock_irq(&lruvec->lru_lock); ++ /* Shrinking tree test. */ + -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); ++ for (int i = 13; i < ARRAY_SIZE(set); i++) ++ erase_check_insert(mt, i); + -+ for (type = ANON_AND_FILE - 1; type >= 0; type--) { -+ if (get_nr_gens(lruvec, type) != MAX_NR_GENS) ++ mt_set_non_kernel(99); ++ for (int i = 18; i < ARRAY_SIZE(set); i++) { ++ erase_check_erase(mt, i); ++ for (int j = 0; j < ARRAY_SIZE(set); j++) { ++ if (j < 18 || j > i) ++ erase_check_load(mt, j); ++ else ++ check_load(mt, set[j], NULL); ++ } ++ } ++ mt_set_non_kernel(35); ++ for (int i = 0; i < 18; i++) { ++ erase_check_erase(mt, i); ++ for (int j = 0; j < ARRAY_SIZE(set); j++) { ++ if (j < 18 && j > i) ++ erase_check_load(mt, j); ++ else ++ check_load(mt, set[j], NULL); ++ } ++ } ++ erase_check_insert(mt, 8); ++ erase_check_insert(mt, 9); ++ erase_check_erase(mt, 8); ++ rcu_unregister_thread(); ++} ++ ++#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \ ++ a[(i)], a[(i + 1)], ptr) ++#define STORE 1 ++#define SNULL 2 ++#define ERASE 3 ++#define ec_type_str(x) \ ++ (((x) == STORE) ? \ ++ "STORE" : \ ++ (((x) == SNULL) ? \ ++ "SNULL" : "ERASE") \ ++ ) ++#define check_erase2_debug 0 ++void *mas_next(struct ma_state *mas, unsigned long max); ++ ++/* Calculate the overwritten entries. */ ++int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end, ++ void *s_entry, unsigned long s_min, ++ void *e_entry, unsigned long e_max, ++ unsigned long *set, int i, bool null_entry) ++{ ++ int count = 0, span = 0; ++ unsigned long retry = 0; ++ void *entry; ++ struct ma_state tmp; ++ ++ ++ /* count slots */ ++ memcpy(&tmp, mas_start, sizeof(tmp)); ++ entry = mas_next(&tmp, mas_end->last); ++ while (entry) { ++ BUG_ON(retry > 50); /* stop infinite retry on testing. */ ++ if (xa_is_zero(s_entry)) { ++ retry++; + continue; -+ -+ VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); -+ -+ while (!inc_min_seq(lruvec, type, can_swap)) { -+ spin_unlock_irq(&lruvec->lru_lock); -+ cond_resched(); -+ spin_lock_irq(&lruvec->lru_lock); ++ } ++ count++; ++ span++; ++ entry = mas_next(&tmp, mas_end->last); ++ } ++ ++ if (null_entry) { ++ /* Check splitting end. */ ++ if (e_entry && (e_max > mas_end->last)) ++ count--; ++ ++ /* check overwrite of entire start */ ++ if (s_entry && (s_min == mas_start->index)) ++ count++; ++ } else { /* !null_entry (store) */ ++ bool esplit = e_max > mas_end->last; ++ bool ssplit = s_min != mas_start->index; ++ ++ if (s_entry && e_entry) { ++ if (esplit && ssplit) ++ count--; ++ else if (ssplit) ++ count--; ++ else if (esplit) { ++ if (span) ++ count--; ++ } ++ } else if (s_entry && !e_entry) { ++ if (ssplit) ++ count--; ++ } else if (!s_entry && e_entry) { ++ if (esplit) ++ count--; ++ count--; ++ } else { ++ count--; + } + } ++ return count; ++} + -+ /* -+ * Update the active/inactive LRU sizes for compatibility. Both sides of -+ * the current max_seq need to be covered, since max_seq+1 can overlap -+ * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do -+ * overlap, cold/hot inversion happens. -+ */ -+ prev = lru_gen_from_seq(lrugen->max_seq - 1); -+ next = lru_gen_from_seq(lrugen->max_seq + 1); ++/* ++ * mas_node_walk() - Walk a maple node to offset of the index. ++ * @mas: The maple state ++ * @type: The maple node type ++ * @*range_min: Pointer to store the minimum range of the offset ++ * @*range_max: Pointer to store the maximum range of the offset ++ * ++ * The offset will be stored in the maple state. ++ * ++ */ ++static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, ++ enum maple_type type, unsigned long *range_min, ++ unsigned long *range_max) + -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) { -+ enum lru_list lru = type * LRU_INACTIVE_FILE; -+ long delta = lrugen->nr_pages[prev][type][zone] - -+ lrugen->nr_pages[next][type][zone]; ++{ ++ unsigned long *pivots; ++ unsigned char count; ++ unsigned long prev, max; ++ unsigned char offset; ++ unsigned long index; + -+ if (!delta) -+ continue; ++ if (unlikely(ma_is_dense(type))) { ++ (*range_max) = (*range_min) = mas->index; ++ if (unlikely(ma_dead_node(node))) ++ return; + -+ __update_lru_size(lruvec, lru, zone, delta); -+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); -+ } ++ mas->offset = mas->index = mas->min; ++ return; + } + -+ for (type = 0; type < ANON_AND_FILE; type++) -+ reset_ctrl_pos(lruvec, type, false); ++ pivots = ma_pivots(node, type); ++ max = pivots[0]; ++ if (unlikely(ma_dead_node(node))) ++ return; + -+ WRITE_ONCE(lrugen->timestamps[next], jiffies); -+ /* make sure preceding modifications appear */ -+ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); ++ offset = 0; ++ prev = mas->min; ++ index = mas->index; ++ if (unlikely(index <= max)) ++ goto offset_zero; ++ ++ count = mt_pivots[type]; ++ while (++offset < count) { ++ prev = max; ++ max = pivots[offset]; ++ if (unlikely(ma_dead_node(node))) ++ return; + -+ spin_unlock_irq(&lruvec->lru_lock); ++ if (index <= max) ++ goto offset_found; ++ else if (unlikely(!max)) ++ goto mas_max; ++ } ++ ++ prev = max; ++mas_max: ++ max = mas->max; ++offset_found: ++ prev++; ++offset_zero: ++ mas->offset = offset; ++ if (ma_is_leaf(type)) { ++ *range_max = max; ++ *range_min = prev; ++ } else { ++ mas->max = max; ++ mas->min = prev; ++ } +} + -+static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, -+ struct scan_control *sc, bool can_swap, bool force_scan) ++/* ++ * mas_descend_walk(): Locates a value and sets the mas->node and slot ++ * accordingly. range_min and range_max are set to the range which the entry is ++ * valid. ++ * @mas: The maple state ++ * @*range_min: A pointer to store the minimum of the range ++ * @*range_max: A pointer to store the maximum of the range ++ * ++ * Check mas->node is still valid on return of any value. ++ * ++ * Return: true if pointing to a valid node and offset. False otherwise. ++ */ ++static inline bool mas_descend_walk(struct ma_state *mas, ++ unsigned long *range_min, unsigned long *range_max) +{ -+ bool success; -+ struct lru_gen_mm_walk *walk; -+ struct mm_struct *mm = NULL; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct maple_enode *next; ++ struct maple_node *node; ++ enum maple_type type; + -+ VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); ++ next = mas->node; ++ while (true) { ++ node = mte_to_node(next); ++ type = mte_node_type(next); ++ mas_node_walk(mas, node, type, range_min, range_max); ++ next = mas_slot(mas, ma_slots(node, type), mas->offset); ++ if (unlikely(ma_dead_node(node))) ++ return false; + -+ /* see the comment in iterate_mm_list() */ -+ if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { -+ success = false; -+ goto done; ++ if (unlikely(ma_is_leaf(type))) ++ return true; ++ ++ /* Descend. */ ++ mas->node = next; + } ++ return false; ++} + -+ /* -+ * If the hardware doesn't automatically set the accessed bit, fallback -+ * to lru_gen_look_around(), which only clears the accessed bit in a -+ * handful of PTEs. Spreading the work out over a period of time usually -+ * is less efficient, but it avoids bursty page faults. -+ */ -+ if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { -+ success = iterate_mm_list_nowalk(lruvec, max_seq); -+ goto done; ++/* ++ * mas_tree_walk() - Walk to @mas->index and set the range values. ++ * @mas: The maple state. ++ * @*range_min: The minimum range to be set. ++ * @*range_max: The maximum range to be set. ++ * ++ * Ranges are only valid if there is a valid entry at @mas->index. ++ * ++ * Return: True if a value exists, false otherwise. ++ */ ++static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, ++ unsigned long *range_max) ++{ ++ bool ret; ++ ++retry: ++ ret = false; ++ mas_start(mas); ++ if (mas_is_none(mas)) ++ goto not_found; ++ ++ if (mas_is_ptr(mas)) { ++ *range_min = *range_max = 0; ++ if (!mas->index) ++ return true; ++ ++ goto not_found; + } + -+ walk = set_mm_walk(NULL); -+ if (!walk) { -+ success = iterate_mm_list_nowalk(lruvec, max_seq); -+ goto done; ++ ret = mas_descend_walk(mas, range_min, range_max); ++ if (unlikely(mte_dead_node(mas->node))) { ++ mas->node = MAS_START; ++ goto retry; + } + -+ walk->lruvec = lruvec; -+ walk->max_seq = max_seq; -+ walk->can_swap = can_swap; -+ walk->force_scan = force_scan; ++ return ret; + -+ do { -+ success = iterate_mm_list(lruvec, walk, &mm); -+ if (mm) -+ walk_mm(lruvec, mm, walk); ++not_found: ++ mas->offset = MAPLE_NODE_SLOTS; ++ return false; ++} + -+ cond_resched(); -+ } while (mm); -+done: -+ if (!success) { -+ if (sc->priority <= DEF_PRIORITY - 2) -+ wait_event_killable(lruvec->mm_state.wait, -+ max_seq < READ_ONCE(lrugen->max_seq)); ++static inline void *mas_range_load(struct ma_state *mas, ++ unsigned long *range_min, unsigned long *range_max) + -+ return max_seq < READ_ONCE(lrugen->max_seq); -+ } ++{ ++ void *entry = NULL; ++ unsigned long index = mas->index; + -+ VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); ++ if (mas_is_none(mas) || mas_is_paused(mas)) ++ mas->node = MAS_START; ++retry: ++ if (mas_tree_walk(mas, range_min, range_max)) ++ if (unlikely(mas->node == MAS_ROOT)) ++ return mas_root(mas); + -+ inc_max_seq(lruvec, can_swap, force_scan); -+ /* either this sees any waiters or they will see updated max_seq */ -+ if (wq_has_sleeper(&lruvec->mm_state.wait)) -+ wake_up_all(&lruvec->mm_state.wait); ++ if (likely(mas->offset != MAPLE_NODE_SLOTS)) ++ entry = mas_get_slot(mas, mas->offset); + -+ wakeup_flusher_threads(WB_REASON_VMSCAN); ++ if (mas_dead_node(mas, index)) ++ goto retry; + -+ return true; ++ return entry; +} -+ -+static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, -+ struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) ++static noinline void check_erase2_testset(struct maple_tree *mt, ++ unsigned long *set, unsigned long size) +{ -+ int gen, type, zone; -+ unsigned long old = 0; -+ unsigned long young = 0; -+ unsigned long total = 0; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ int entry_count = 0; ++ int check = 0; ++ void *foo; ++ unsigned long addr = 0; ++ void *s_entry = NULL, *e_entry = NULL; + -+ for (type = !can_swap; type < ANON_AND_FILE; type++) { -+ unsigned long seq; ++ MA_STATE(mas, mt, 0, 0); + -+ for (seq = min_seq[type]; seq <= max_seq; seq++) { -+ unsigned long size = 0; ++ for (int i = 0; i < size; i += 3) { ++ unsigned long s_min, s_max; ++ unsigned long e_min, e_max; ++ void *value = NULL; + -+ gen = lru_gen_from_seq(seq); ++ MA_STATE(mas_start, mt, set[i+1], set[i+1]); ++ MA_STATE(mas_end, mt, set[i+2], set[i+2]); ++ mt_set_non_kernel(127); ++#if check_erase2_debug ++ pr_err("%s: %d %s %lu - %lu\n", __func__, i, ++ ec_type_str(set[i]), ++ set[i+1], set[i+2]); ++#endif ++ s_entry = mas_range_load(&mas_start, &s_min, &s_max); ++ e_entry = mas_range_load(&mas_end, &e_min, &e_max); ++ ++ switch (set[i]) { ++ case SNULL: ++ if ((s_min == set[i+1]) && (s_max == set[i+2])) { ++ if (s_entry) ++ entry_count--; ++ } else if ((s_min != set[i+1]) && (s_max != set[i+2])) { ++ entry_count++; ++ } else if ((mas_start.node != mas_end.node) || ++ (mas_start.offset != mas_end.offset)) { ++ entry_count -= ++ mas_ce2_over_count(&mas_start, &mas_end, ++ s_entry, s_min, ++ e_entry, e_max, set, i, ++ true); ++ } + -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) -+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + -+ total += size; -+ if (seq == max_seq) -+ young += size; -+ else if (seq + MIN_NR_GENS == max_seq) -+ old += size; ++ erase_check_store_range(mt, set, i + 1, value); ++ break; ++ case STORE: ++ value = xa_mk_value(set[i + 1]); ++ if (mas_start.offset > mt_slot_count(mas_start.node)) { ++ entry_count++; /* appending an entry. */ ++ } else if ((s_min == e_min) && (s_max == e_max)) { ++ if (!entry_count) ++ entry_count++; ++ ++ else if (s_entry) { ++ if (e_max > mas_end.last) ++ entry_count++; ++ ++ if (s_min < mas_start.index) ++ entry_count++; ++ ++ } else { ++ entry_count++; ++ } ++ } else { ++ entry_count -= ++ mas_ce2_over_count(&mas_start, &mas_end, ++ s_entry, s_min, ++ e_entry, e_max, set, i, ++ false); ++ } ++ ++ erase_check_store_range(mt, set, i + 1, value); ++ break; ++ case ERASE: ++ if (!s_entry) ++ break; ++ check_erase(mt, set[i+1], xa_mk_value(set[i+1])); ++ entry_count--; ++ break; + } -+ } ++ mt_validate(mt); ++ if (entry_count) ++ MT_BUG_ON(mt, !mt_height(mt)); ++#if check_erase2_debug > 1 ++ mt_dump(mt); ++#endif ++#if check_erase2_debug ++ pr_err("Done\n"); ++#endif + -+ /* try to scrape all its memory if this memcg was deleted */ -+ *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; ++ check = 0; ++ addr = 0; ++ mt_for_each(mt, foo, addr, ULONG_MAX) { ++ check++; ++#if check_erase2_debug > 2 ++ pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check); ++#endif ++ if (check > entry_count) ++ break; ++ } + -+ /* -+ * The aging tries to be lazy to reduce the overhead, while the eviction -+ * stalls when the number of generations reaches MIN_NR_GENS. Hence, the -+ * ideal number of generations is MIN_NR_GENS+1. -+ */ -+ if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) -+ return true; -+ if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) -+ return false; ++#if check_erase2_debug > 2 ++ pr_err("mt_for_each %d and count %d\n", check, entry_count); ++#endif + -+ /* -+ * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) -+ * of the total number of pages for each generation. A reasonable range -+ * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The -+ * aging cares about the upper bound of hot pages, while the eviction -+ * cares about the lower bound of cold pages. -+ */ -+ if (young * MIN_NR_GENS > total) -+ return true; -+ if (old * (MIN_NR_GENS + 2) < total) -+ return true; ++ MT_BUG_ON(mt, check != entry_count); + -+ return false; ++ check = 0; ++ addr = 0; ++ mas_reset(&mas); ++ mas.index = 0; ++ rcu_read_lock(); ++ mas_for_each(&mas, foo, ULONG_MAX) { ++ if (xa_is_zero(foo)) { ++ if (addr == mas.index) { ++ mt_dump(mas.tree); ++ pr_err("retry failed %lu - %lu\n", ++ mas.index, mas.last); ++ MT_BUG_ON(mt, 1); ++ } ++ addr = mas.index; ++ continue; ++ } ++#if check_erase2_debug > 2 ++ pr_err("mas: %lu -> %p\n", mas.index, foo); ++#endif ++ check++; ++ if (check > entry_count) ++ break; ++ } ++ rcu_read_unlock(); ++#if check_erase2_debug > 2 ++ pr_err("mas_for_each %d and count %d\n", check, entry_count); ++ mt_validate(mt); ++#endif ++ ++ MT_BUG_ON(mt, check != entry_count); ++ ++ MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); ++ } +} + -+static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) ++ ++/* These tests were pulled from kvm tests. */ ++static noinline void check_erase2_sets(struct maple_tree *mt) +{ -+ bool need_aging; -+ unsigned long nr_to_scan; -+ int swappiness = get_swappiness(lruvec, sc); -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); ++ void *entry; ++ unsigned long start = 0; ++ unsigned long set[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140721266458624, 140737488351231, ++ERASE, 140721266458624, 140737488351231, ++STORE, 140721266458624, 140721266462719, ++STORE, 94735788949504, 94735789121535, ++ERASE, 94735788949504, 94735789121535, ++STORE, 94735788949504, 94735788965887, ++STORE, 94735788965888, 94735789121535, ++ERASE, 94735788965888, 94735789121535, ++STORE, 94735788965888, 94735789068287, ++STORE, 94735789068288, 94735789109247, ++STORE, 94735789109248, 94735789121535, ++STORE, 140253902692352, 140253902864383, ++ERASE, 140253902692352, 140253902864383, ++STORE, 140253902692352, 140253902696447, ++STORE, 140253902696448, 140253902864383, ++ }; ++ unsigned long set2[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735933583360, 140737488351231, ++ERASE, 140735933583360, 140737488351231, ++STORE, 140735933583360, 140735933587455, ++STORE, 94811003260928, 94811003432959, ++ERASE, 94811003260928, 94811003432959, ++STORE, 94811003260928, 94811003277311, ++STORE, 94811003277312, 94811003432959, ++ERASE, 94811003277312, 94811003432959, ++STORE, 94811003277312, 94811003379711, ++STORE, 94811003379712, 94811003420671, ++STORE, 94811003420672, 94811003432959, ++STORE, 140277094653952, 140277094825983, ++ERASE, 140277094653952, 140277094825983, ++STORE, 140277094653952, 140277094658047, ++STORE, 140277094658048, 140277094825983, ++ERASE, 140277094658048, 140277094825983, ++STORE, 140277094658048, 140277094780927, ++STORE, 140277094780928, 140277094813695, ++STORE, 140277094813696, 140277094821887, ++STORE, 140277094821888, 140277094825983, ++STORE, 140735933906944, 140735933911039, ++ }; ++ unsigned long set3[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735790264320, 140737488351231, ++ERASE, 140735790264320, 140737488351231, ++STORE, 140735790264320, 140735790268415, ++STORE, 94016597282816, 94016597454847, ++ERASE, 94016597282816, 94016597454847, ++STORE, 94016597282816, 94016597299199, ++STORE, 94016597299200, 94016597454847, ++ERASE, 94016597299200, 94016597454847, ++STORE, 94016597299200, 94016597401599, ++STORE, 94016597401600, 94016597442559, ++STORE, 94016597442560, 94016597454847, ++STORE, 140496959283200, 140496959455231, ++ERASE, 140496959283200, 140496959455231, ++STORE, 140496959283200, 140496959287295, ++STORE, 140496959287296, 140496959455231, ++ERASE, 140496959287296, 140496959455231, ++STORE, 140496959287296, 140496959410175, ++STORE, 140496959410176, 140496959442943, ++STORE, 140496959442944, 140496959451135, ++STORE, 140496959451136, 140496959455231, ++STORE, 140735791718400, 140735791722495, ++STORE, 140735791706112, 140735791718399, ++STORE, 47135835713536, 47135835721727, ++STORE, 47135835721728, 47135835729919, ++STORE, 47135835729920, 47135835893759, ++ERASE, 47135835729920, 47135835893759, ++STORE, 47135835729920, 47135835742207, ++STORE, 47135835742208, 47135835893759, ++STORE, 47135835840512, 47135835893759, ++STORE, 47135835742208, 47135835840511, ++ERASE, 47135835742208, 47135835840511, ++STORE, 47135835742208, 47135835840511, ++STORE, 47135835885568, 47135835893759, ++STORE, 47135835840512, 47135835885567, ++ERASE, 47135835840512, 47135835885567, ++STORE, 47135835840512, 47135835893759, ++ERASE, 47135835840512, 47135835893759, ++STORE, 47135835840512, 47135835885567, ++STORE, 47135835885568, 47135835893759, ++ }; + -+ VM_WARN_ON_ONCE(sc->memcg_low_reclaim); ++ unsigned long set4[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140728251703296, 140737488351231, ++ERASE, 140728251703296, 140737488351231, ++STORE, 140728251703296, 140728251707391, ++STORE, 94668429205504, 94668429377535, ++ERASE, 94668429205504, 94668429377535, ++STORE, 94668429205504, 94668429221887, ++STORE, 94668429221888, 94668429377535, ++ERASE, 94668429221888, 94668429377535, ++STORE, 94668429221888, 94668429324287, ++STORE, 94668429324288, 94668429365247, ++STORE, 94668429365248, 94668429377535, ++STORE, 47646523273216, 47646523445247, ++ERASE, 47646523273216, 47646523445247, ++STORE, 47646523273216, 47646523277311, ++STORE, 47646523277312, 47646523445247, ++ERASE, 47646523277312, 47646523445247, ++STORE, 47646523277312, 47646523400191, ++ }; + -+ mem_cgroup_calculate_protection(NULL, memcg); ++ unsigned long set5[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140726874062848, 140737488351231, ++ERASE, 140726874062848, 140737488351231, ++STORE, 140726874062848, 140726874066943, ++STORE, 94248892870656, 94248893042687, ++ERASE, 94248892870656, 94248893042687, ++STORE, 94248892870656, 94248892887039, ++STORE, 94248892887040, 94248893042687, ++ERASE, 94248892887040, 94248893042687, ++STORE, 94248892887040, 94248892989439, ++STORE, 94248892989440, 94248893030399, ++STORE, 94248893030400, 94248893042687, ++STORE, 47884786266112, 47884786438143, ++ERASE, 47884786266112, 47884786438143, ++STORE, 47884786266112, 47884786270207, ++STORE, 47884786270208, 47884786438143, ++ERASE, 47884786270208, 47884786438143, ++STORE, 47884786270208, 47884786393087, ++STORE, 47884786393088, 47884786425855, ++STORE, 47884786425856, 47884786434047, ++STORE, 47884786434048, 47884786438143, ++STORE, 140726874513408, 140726874517503, ++STORE, 140726874501120, 140726874513407, ++STORE, 47884786438144, 47884786446335, ++STORE, 47884786446336, 47884786454527, ++STORE, 47884786454528, 47884786618367, ++ERASE, 47884786454528, 47884786618367, ++STORE, 47884786454528, 47884786466815, ++STORE, 47884786466816, 47884786618367, ++STORE, 47884786565120, 47884786618367, ++STORE, 47884786466816, 47884786565119, ++ERASE, 47884786466816, 47884786565119, ++STORE, 47884786466816, 47884786565119, ++STORE, 47884786610176, 47884786618367, ++STORE, 47884786565120, 47884786610175, ++ERASE, 47884786565120, 47884786610175, ++STORE, 47884786565120, 47884786618367, ++ERASE, 47884786565120, 47884786618367, ++STORE, 47884786565120, 47884786610175, ++STORE, 47884786610176, 47884786618367, ++ERASE, 47884786610176, 47884786618367, ++STORE, 47884786610176, 47884786618367, ++STORE, 47884786618368, 47884789669887, ++STORE, 47884787163136, 47884789669887, ++STORE, 47884786618368, 47884787163135, ++ERASE, 47884787163136, 47884789669887, ++STORE, 47884787163136, 47884789448703, ++STORE, 47884789448704, 47884789669887, ++STORE, 47884788858880, 47884789448703, ++STORE, 47884787163136, 47884788858879, ++ERASE, 47884787163136, 47884788858879, ++STORE, 47884787163136, 47884788858879, ++STORE, 47884789444608, 47884789448703, ++STORE, 47884788858880, 47884789444607, ++ERASE, 47884788858880, 47884789444607, ++STORE, 47884788858880, 47884789444607, ++STORE, 47884789653504, 47884789669887, ++STORE, 47884789448704, 47884789653503, ++ERASE, 47884789448704, 47884789653503, ++STORE, 47884789448704, 47884789653503, ++ERASE, 47884789653504, 47884789669887, ++STORE, 47884789653504, 47884789669887, ++STORE, 47884789669888, 47884791508991, ++STORE, 47884789809152, 47884791508991, ++STORE, 47884789669888, 47884789809151, ++ERASE, 47884789809152, 47884791508991, ++STORE, 47884789809152, 47884791468031, ++STORE, 47884791468032, 47884791508991, ++STORE, 47884791152640, 47884791468031, ++STORE, 47884789809152, 47884791152639, ++ERASE, 47884789809152, 47884791152639, ++STORE, 47884789809152, 47884791152639, ++STORE, 47884791463936, 47884791468031, ++STORE, 47884791152640, 47884791463935, ++ERASE, 47884791152640, 47884791463935, ++STORE, 47884791152640, 47884791463935, ++STORE, 47884791492608, 47884791508991, ++STORE, 47884791468032, 47884791492607, ++ERASE, 47884791468032, 47884791492607, ++STORE, 47884791468032, 47884791492607, ++ERASE, 47884791492608, 47884791508991, ++STORE, 47884791492608, 47884791508991, ++STORE, 47884791508992, 47884791644159, ++ERASE, 47884791508992, 47884791644159, ++STORE, 47884791508992, 47884791533567, ++STORE, 47884791533568, 47884791644159, ++STORE, 47884791595008, 47884791644159, ++STORE, 47884791533568, 47884791595007, ++ERASE, 47884791533568, 47884791595007, ++STORE, 47884791533568, 47884791595007, ++STORE, 47884791619584, 47884791644159, ++STORE, 47884791595008, 47884791619583, ++ERASE, 47884791595008, 47884791619583, ++STORE, 47884791595008, 47884791644159, ++ERASE, 47884791595008, 47884791644159, ++STORE, 47884791595008, 47884791619583, ++STORE, 47884791619584, 47884791644159, ++STORE, 47884791627776, 47884791644159, ++STORE, 47884791619584, 47884791627775, ++ERASE, 47884791619584, 47884791627775, ++STORE, 47884791619584, 47884791627775, ++ERASE, 47884791627776, 47884791644159, ++STORE, 47884791627776, 47884791644159, ++STORE, 47884791644160, 47884791664639, ++ERASE, 47884791644160, 47884791664639, ++STORE, 47884791644160, 47884791648255, ++STORE, 47884791648256, 47884791664639, ++STORE, 47884791652352, 47884791664639, ++STORE, 47884791648256, 47884791652351, ++ERASE, 47884791648256, 47884791652351, ++STORE, 47884791648256, 47884791652351, ++STORE, 47884791656448, 47884791664639, ++STORE, 47884791652352, 47884791656447, ++ERASE, 47884791652352, 47884791656447, ++STORE, 47884791652352, 47884791664639, ++ERASE, 47884791652352, 47884791664639, ++STORE, 47884791652352, 47884791656447, ++STORE, 47884791656448, 47884791664639, ++ERASE, 47884791656448, 47884791664639, ++STORE, 47884791656448, 47884791664639, ++STORE, 47884791664640, 47884791672831, ++ERASE, 47884791468032, 47884791492607, ++STORE, 47884791468032, 47884791484415, ++STORE, 47884791484416, 47884791492607, ++ERASE, 47884791656448, 47884791664639, ++STORE, 47884791656448, 47884791660543, ++STORE, 47884791660544, 47884791664639, ++ERASE, 47884791619584, 47884791627775, ++STORE, 47884791619584, 47884791623679, ++STORE, 47884791623680, 47884791627775, ++ }; + -+ if (mem_cgroup_below_min(memcg)) -+ return false; ++ unsigned long set6[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722999021568, 140737488351231, ++ERASE, 140722999021568, 140737488351231, ++STORE, 140722999021568, 140722999025663, ++STORE, 94901500268544, 94901500440575, ++ERASE, 94901500268544, 94901500440575, ++STORE, 94901500268544, 94901500284927, ++STORE, 94901500284928, 94901500440575, ++ERASE, 94901500284928, 94901500440575, ++STORE, 94901500284928, 94901500387327, ++STORE, 94901500387328, 94901500428287, ++STORE, 94901500428288, 94901500440575, ++STORE, 47430426660864, 47430426832895, ++ERASE, 47430426660864, 47430426832895, ++STORE, 47430426660864, 47430426664959, ++STORE, 47430426664960, 47430426832895, ++ERASE, 47430426664960, 47430426832895, ++STORE, 47430426664960, 47430426787839, ++STORE, 47430426787840, 47430426820607, ++STORE, 47430426820608, 47430426828799, ++STORE, 47430426828800, 47430426832895, ++STORE, 140722999115776, 140722999119871, ++STORE, 140722999103488, 140722999115775, ++STORE, 47430426832896, 47430426841087, ++STORE, 47430426841088, 47430426849279, ++STORE, 47430426849280, 47430427013119, ++ERASE, 47430426849280, 47430427013119, ++STORE, 47430426849280, 47430426861567, ++STORE, 47430426861568, 47430427013119, ++STORE, 47430426959872, 47430427013119, ++STORE, 47430426861568, 47430426959871, ++ERASE, 47430426861568, 47430426959871, ++STORE, 47430426861568, 47430426959871, ++STORE, 47430427004928, 47430427013119, ++STORE, 47430426959872, 47430427004927, ++ERASE, 47430426959872, 47430427004927, ++STORE, 47430426959872, 47430427013119, ++ERASE, 47430426959872, 47430427013119, ++STORE, 47430426959872, 47430427004927, ++STORE, 47430427004928, 47430427013119, ++ERASE, 47430427004928, 47430427013119, ++STORE, 47430427004928, 47430427013119, ++STORE, 47430427013120, 47430430064639, ++STORE, 47430427557888, 47430430064639, ++STORE, 47430427013120, 47430427557887, ++ERASE, 47430427557888, 47430430064639, ++STORE, 47430427557888, 47430429843455, ++STORE, 47430429843456, 47430430064639, ++STORE, 47430429253632, 47430429843455, ++STORE, 47430427557888, 47430429253631, ++ERASE, 47430427557888, 47430429253631, ++STORE, 47430427557888, 47430429253631, ++STORE, 47430429839360, 47430429843455, ++STORE, 47430429253632, 47430429839359, ++ERASE, 47430429253632, 47430429839359, ++STORE, 47430429253632, 47430429839359, ++STORE, 47430430048256, 47430430064639, ++STORE, 47430429843456, 47430430048255, ++ERASE, 47430429843456, 47430430048255, ++STORE, 47430429843456, 47430430048255, ++ERASE, 47430430048256, 47430430064639, ++STORE, 47430430048256, 47430430064639, ++STORE, 47430430064640, 47430431903743, ++STORE, 47430430203904, 47430431903743, ++STORE, 47430430064640, 47430430203903, ++ERASE, 47430430203904, 47430431903743, ++STORE, 47430430203904, 47430431862783, ++STORE, 47430431862784, 47430431903743, ++STORE, 47430431547392, 47430431862783, ++STORE, 47430430203904, 47430431547391, ++ERASE, 47430430203904, 47430431547391, ++STORE, 47430430203904, 47430431547391, ++STORE, 47430431858688, 47430431862783, ++STORE, 47430431547392, 47430431858687, ++ERASE, 47430431547392, 47430431858687, ++STORE, 47430431547392, 47430431858687, ++STORE, 47430431887360, 47430431903743, ++STORE, 47430431862784, 47430431887359, ++ERASE, 47430431862784, 47430431887359, ++STORE, 47430431862784, 47430431887359, ++ERASE, 47430431887360, 47430431903743, ++STORE, 47430431887360, 47430431903743, ++STORE, 47430431903744, 47430432038911, ++ERASE, 47430431903744, 47430432038911, ++STORE, 47430431903744, 47430431928319, ++STORE, 47430431928320, 47430432038911, ++STORE, 47430431989760, 47430432038911, ++STORE, 47430431928320, 47430431989759, ++ERASE, 47430431928320, 47430431989759, ++STORE, 47430431928320, 47430431989759, ++STORE, 47430432014336, 47430432038911, ++STORE, 47430431989760, 47430432014335, ++ERASE, 47430431989760, 47430432014335, ++STORE, 47430431989760, 47430432038911, ++ERASE, 47430431989760, 47430432038911, ++STORE, 47430431989760, 47430432014335, ++STORE, 47430432014336, 47430432038911, ++STORE, 47430432022528, 47430432038911, ++STORE, 47430432014336, 47430432022527, ++ERASE, 47430432014336, 47430432022527, ++STORE, 47430432014336, 47430432022527, ++ERASE, 47430432022528, 47430432038911, ++STORE, 47430432022528, 47430432038911, ++STORE, 47430432038912, 47430432059391, ++ERASE, 47430432038912, 47430432059391, ++STORE, 47430432038912, 47430432043007, ++STORE, 47430432043008, 47430432059391, ++STORE, 47430432047104, 47430432059391, ++STORE, 47430432043008, 47430432047103, ++ERASE, 47430432043008, 47430432047103, ++STORE, 47430432043008, 47430432047103, ++STORE, 47430432051200, 47430432059391, ++STORE, 47430432047104, 47430432051199, ++ERASE, 47430432047104, 47430432051199, ++STORE, 47430432047104, 47430432059391, ++ERASE, 47430432047104, 47430432059391, ++STORE, 47430432047104, 47430432051199, ++STORE, 47430432051200, 47430432059391, ++ERASE, 47430432051200, 47430432059391, ++STORE, 47430432051200, 47430432059391, ++STORE, 47430432059392, 47430432067583, ++ERASE, 47430431862784, 47430431887359, ++STORE, 47430431862784, 47430431879167, ++STORE, 47430431879168, 47430431887359, ++ERASE, 47430432051200, 47430432059391, ++STORE, 47430432051200, 47430432055295, ++STORE, 47430432055296, 47430432059391, ++ERASE, 47430432014336, 47430432022527, ++STORE, 47430432014336, 47430432018431, ++STORE, 47430432018432, 47430432022527, ++ }; ++ unsigned long set7[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140729808330752, 140737488351231, ++ERASE, 140729808330752, 140737488351231, ++STORE, 140729808330752, 140729808334847, ++STORE, 94629632020480, 94629632192511, ++ERASE, 94629632020480, 94629632192511, ++STORE, 94629632020480, 94629632036863, ++STORE, 94629632036864, 94629632192511, ++ERASE, 94629632036864, 94629632192511, ++STORE, 94629632036864, 94629632139263, ++STORE, 94629632139264, 94629632180223, ++STORE, 94629632180224, 94629632192511, ++STORE, 47439981776896, 47439981948927, ++ERASE, 47439981776896, 47439981948927, ++STORE, 47439981776896, 47439981780991, ++STORE, 47439981780992, 47439981948927, ++ERASE, 47439981780992, 47439981948927, ++STORE, 47439981780992, 47439981903871, ++STORE, 47439981903872, 47439981936639, ++STORE, 47439981936640, 47439981944831, ++STORE, 47439981944832, 47439981948927, ++STORE, 140729808474112, 140729808478207, ++STORE, 140729808461824, 140729808474111, ++STORE, 47439981948928, 47439981957119, ++STORE, 47439981957120, 47439981965311, ++STORE, 47439981965312, 47439982129151, ++ERASE, 47439981965312, 47439982129151, ++STORE, 47439981965312, 47439981977599, ++STORE, 47439981977600, 47439982129151, ++STORE, 47439982075904, 47439982129151, ++STORE, 47439981977600, 47439982075903, ++ERASE, 47439981977600, 47439982075903, ++STORE, 47439981977600, 47439982075903, ++STORE, 47439982120960, 47439982129151, ++STORE, 47439982075904, 47439982120959, ++ERASE, 47439982075904, 47439982120959, ++STORE, 47439982075904, 47439982129151, ++ERASE, 47439982075904, 47439982129151, ++STORE, 47439982075904, 47439982120959, ++STORE, 47439982120960, 47439982129151, ++ERASE, 47439982120960, 47439982129151, ++STORE, 47439982120960, 47439982129151, ++STORE, 47439982129152, 47439985180671, ++STORE, 47439982673920, 47439985180671, ++STORE, 47439982129152, 47439982673919, ++ERASE, 47439982673920, 47439985180671, ++STORE, 47439982673920, 47439984959487, ++STORE, 47439984959488, 47439985180671, ++STORE, 47439984369664, 47439984959487, ++STORE, 47439982673920, 47439984369663, ++ERASE, 47439982673920, 47439984369663, ++STORE, 47439982673920, 47439984369663, ++STORE, 47439984955392, 47439984959487, ++STORE, 47439984369664, 47439984955391, ++ERASE, 47439984369664, 47439984955391, ++STORE, 47439984369664, 47439984955391, ++STORE, 47439985164288, 47439985180671, ++STORE, 47439984959488, 47439985164287, ++ERASE, 47439984959488, 47439985164287, ++STORE, 47439984959488, 47439985164287, ++ERASE, 47439985164288, 47439985180671, ++STORE, 47439985164288, 47439985180671, ++STORE, 47439985180672, 47439987019775, ++STORE, 47439985319936, 47439987019775, ++STORE, 47439985180672, 47439985319935, ++ERASE, 47439985319936, 47439987019775, ++STORE, 47439985319936, 47439986978815, ++STORE, 47439986978816, 47439987019775, ++STORE, 47439986663424, 47439986978815, ++STORE, 47439985319936, 47439986663423, ++ERASE, 47439985319936, 47439986663423, ++STORE, 47439985319936, 47439986663423, ++STORE, 47439986974720, 47439986978815, ++STORE, 47439986663424, 47439986974719, ++ERASE, 47439986663424, 47439986974719, ++STORE, 47439986663424, 47439986974719, ++STORE, 47439987003392, 47439987019775, ++STORE, 47439986978816, 47439987003391, ++ERASE, 47439986978816, 47439987003391, ++STORE, 47439986978816, 47439987003391, ++ERASE, 47439987003392, 47439987019775, ++STORE, 47439987003392, 47439987019775, ++STORE, 47439987019776, 47439987154943, ++ERASE, 47439987019776, 47439987154943, ++STORE, 47439987019776, 47439987044351, ++STORE, 47439987044352, 47439987154943, ++STORE, 47439987105792, 47439987154943, ++STORE, 47439987044352, 47439987105791, ++ERASE, 47439987044352, 47439987105791, ++STORE, 47439987044352, 47439987105791, ++STORE, 47439987130368, 47439987154943, ++STORE, 47439987105792, 47439987130367, ++ERASE, 47439987105792, 47439987130367, ++STORE, 47439987105792, 47439987154943, ++ERASE, 47439987105792, 47439987154943, ++STORE, 47439987105792, 47439987130367, ++STORE, 47439987130368, 47439987154943, ++STORE, 47439987138560, 47439987154943, ++STORE, 47439987130368, 47439987138559, ++ERASE, 47439987130368, 47439987138559, ++STORE, 47439987130368, 47439987138559, ++ERASE, 47439987138560, 47439987154943, ++STORE, 47439987138560, 47439987154943, ++STORE, 47439987154944, 47439987175423, ++ERASE, 47439987154944, 47439987175423, ++STORE, 47439987154944, 47439987159039, ++STORE, 47439987159040, 47439987175423, ++STORE, 47439987163136, 47439987175423, ++STORE, 47439987159040, 47439987163135, ++ERASE, 47439987159040, 47439987163135, ++STORE, 47439987159040, 47439987163135, ++STORE, 47439987167232, 47439987175423, ++STORE, 47439987163136, 47439987167231, ++ERASE, 47439987163136, 47439987167231, ++STORE, 47439987163136, 47439987175423, ++ERASE, 47439987163136, 47439987175423, ++STORE, 47439987163136, 47439987167231, ++STORE, 47439987167232, 47439987175423, ++ERASE, 47439987167232, 47439987175423, ++STORE, 47439987167232, 47439987175423, ++STORE, 47439987175424, 47439987183615, ++ERASE, 47439986978816, 47439987003391, ++STORE, 47439986978816, 47439986995199, ++STORE, 47439986995200, 47439987003391, ++ERASE, 47439987167232, 47439987175423, ++STORE, 47439987167232, 47439987171327, ++STORE, 47439987171328, 47439987175423, ++ERASE, 47439987130368, 47439987138559, ++STORE, 47439987130368, 47439987134463, ++STORE, 47439987134464, 47439987138559, ++ }; ++ unsigned long set8[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722482974720, 140737488351231, ++ERASE, 140722482974720, 140737488351231, ++STORE, 140722482974720, 140722482978815, ++STORE, 94121505034240, 94121505206271, ++ERASE, 94121505034240, 94121505206271, ++STORE, 94121505034240, 94121505050623, ++STORE, 94121505050624, 94121505206271, ++ERASE, 94121505050624, 94121505206271, ++STORE, 94121505050624, 94121505153023, ++STORE, 94121505153024, 94121505193983, ++STORE, 94121505193984, 94121505206271, ++STORE, 47708483284992, 47708483457023, ++ERASE, 47708483284992, 47708483457023, ++STORE, 47708483284992, 47708483289087, ++STORE, 47708483289088, 47708483457023, ++ERASE, 47708483289088, 47708483457023, ++STORE, 47708483289088, 47708483411967, ++STORE, 47708483411968, 47708483444735, ++STORE, 47708483444736, 47708483452927, ++STORE, 47708483452928, 47708483457023, ++STORE, 140722483142656, 140722483146751, ++STORE, 140722483130368, 140722483142655, ++STORE, 47708483457024, 47708483465215, ++STORE, 47708483465216, 47708483473407, ++STORE, 47708483473408, 47708483637247, ++ERASE, 47708483473408, 47708483637247, ++STORE, 47708483473408, 47708483485695, ++STORE, 47708483485696, 47708483637247, ++STORE, 47708483584000, 47708483637247, ++STORE, 47708483485696, 47708483583999, ++ERASE, 47708483485696, 47708483583999, ++STORE, 47708483485696, 47708483583999, ++STORE, 47708483629056, 47708483637247, ++STORE, 47708483584000, 47708483629055, ++ERASE, 47708483584000, 47708483629055, ++STORE, 47708483584000, 47708483637247, ++ERASE, 47708483584000, 47708483637247, ++STORE, 47708483584000, 47708483629055, ++STORE, 47708483629056, 47708483637247, ++ERASE, 47708483629056, 47708483637247, ++STORE, 47708483629056, 47708483637247, ++STORE, 47708483637248, 47708486688767, ++STORE, 47708484182016, 47708486688767, ++STORE, 47708483637248, 47708484182015, ++ERASE, 47708484182016, 47708486688767, ++STORE, 47708484182016, 47708486467583, ++STORE, 47708486467584, 47708486688767, ++STORE, 47708485877760, 47708486467583, ++STORE, 47708484182016, 47708485877759, ++ERASE, 47708484182016, 47708485877759, ++STORE, 47708484182016, 47708485877759, ++STORE, 47708486463488, 47708486467583, ++STORE, 47708485877760, 47708486463487, ++ERASE, 47708485877760, 47708486463487, ++STORE, 47708485877760, 47708486463487, ++STORE, 47708486672384, 47708486688767, ++STORE, 47708486467584, 47708486672383, ++ERASE, 47708486467584, 47708486672383, ++STORE, 47708486467584, 47708486672383, ++ERASE, 47708486672384, 47708486688767, ++STORE, 47708486672384, 47708486688767, ++STORE, 47708486688768, 47708488527871, ++STORE, 47708486828032, 47708488527871, ++STORE, 47708486688768, 47708486828031, ++ERASE, 47708486828032, 47708488527871, ++STORE, 47708486828032, 47708488486911, ++STORE, 47708488486912, 47708488527871, ++STORE, 47708488171520, 47708488486911, ++STORE, 47708486828032, 47708488171519, ++ERASE, 47708486828032, 47708488171519, ++STORE, 47708486828032, 47708488171519, ++STORE, 47708488482816, 47708488486911, ++STORE, 47708488171520, 47708488482815, ++ERASE, 47708488171520, 47708488482815, ++STORE, 47708488171520, 47708488482815, ++STORE, 47708488511488, 47708488527871, ++STORE, 47708488486912, 47708488511487, ++ERASE, 47708488486912, 47708488511487, ++STORE, 47708488486912, 47708488511487, ++ERASE, 47708488511488, 47708488527871, ++STORE, 47708488511488, 47708488527871, ++STORE, 47708488527872, 47708488663039, ++ERASE, 47708488527872, 47708488663039, ++STORE, 47708488527872, 47708488552447, ++STORE, 47708488552448, 47708488663039, ++STORE, 47708488613888, 47708488663039, ++STORE, 47708488552448, 47708488613887, ++ERASE, 47708488552448, 47708488613887, ++STORE, 47708488552448, 47708488613887, ++STORE, 47708488638464, 47708488663039, ++STORE, 47708488613888, 47708488638463, ++ERASE, 47708488613888, 47708488638463, ++STORE, 47708488613888, 47708488663039, ++ERASE, 47708488613888, 47708488663039, ++STORE, 47708488613888, 47708488638463, ++STORE, 47708488638464, 47708488663039, ++STORE, 47708488646656, 47708488663039, ++STORE, 47708488638464, 47708488646655, ++ERASE, 47708488638464, 47708488646655, ++STORE, 47708488638464, 47708488646655, ++ERASE, 47708488646656, 47708488663039, ++STORE, 47708488646656, 47708488663039, ++STORE, 47708488663040, 47708488683519, ++ERASE, 47708488663040, 47708488683519, ++STORE, 47708488663040, 47708488667135, ++STORE, 47708488667136, 47708488683519, ++STORE, 47708488671232, 47708488683519, ++STORE, 47708488667136, 47708488671231, ++ERASE, 47708488667136, 47708488671231, ++STORE, 47708488667136, 47708488671231, ++STORE, 47708488675328, 47708488683519, ++STORE, 47708488671232, 47708488675327, ++ERASE, 47708488671232, 47708488675327, ++STORE, 47708488671232, 47708488683519, ++ERASE, 47708488671232, 47708488683519, ++STORE, 47708488671232, 47708488675327, ++STORE, 47708488675328, 47708488683519, ++ERASE, 47708488675328, 47708488683519, ++STORE, 47708488675328, 47708488683519, ++STORE, 47708488683520, 47708488691711, ++ERASE, 47708488486912, 47708488511487, ++STORE, 47708488486912, 47708488503295, ++STORE, 47708488503296, 47708488511487, ++ERASE, 47708488675328, 47708488683519, ++STORE, 47708488675328, 47708488679423, ++STORE, 47708488679424, 47708488683519, ++ERASE, 47708488638464, 47708488646655, ++STORE, 47708488638464, 47708488642559, ++STORE, 47708488642560, 47708488646655, ++ }; + -+ need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); ++ unsigned long set9[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140736427839488, 140737488351231, ++ERASE, 140736427839488, 140736427839488, ++STORE, 140736427839488, 140736427843583, ++STORE, 94071213395968, 94071213567999, ++ERASE, 94071213395968, 94071213395968, ++STORE, 94071213395968, 94071213412351, ++STORE, 94071213412352, 94071213567999, ++ERASE, 94071213412352, 94071213412352, ++STORE, 94071213412352, 94071213514751, ++STORE, 94071213514752, 94071213555711, ++STORE, 94071213555712, 94071213567999, ++STORE, 139968410644480, 139968410816511, ++ERASE, 139968410644480, 139968410644480, ++STORE, 139968410644480, 139968410648575, ++STORE, 139968410648576, 139968410816511, ++ERASE, 139968410648576, 139968410648576, ++STORE, 139968410648576, 139968410771455, ++STORE, 139968410771456, 139968410804223, ++STORE, 139968410804224, 139968410812415, ++STORE, 139968410812416, 139968410816511, ++STORE, 140736429277184, 140736429281279, ++STORE, 140736429264896, 140736429277183, ++STORE, 47664384352256, 47664384360447, ++STORE, 47664384360448, 47664384368639, ++STORE, 47664384368640, 47664384532479, ++ERASE, 47664384368640, 47664384368640, ++STORE, 47664384368640, 47664384380927, ++STORE, 47664384380928, 47664384532479, ++STORE, 47664384479232, 47664384532479, ++STORE, 47664384380928, 47664384479231, ++ERASE, 47664384380928, 47664384380928, ++STORE, 47664384380928, 47664384479231, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384479232, 47664384524287, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384532479, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384524287, ++STORE, 47664384524288, 47664384532479, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384532480, 47664387583999, ++STORE, 47664385077248, 47664387583999, ++STORE, 47664384532480, 47664385077247, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664387362815, ++STORE, 47664387362816, 47664387583999, ++STORE, 47664386772992, 47664387362815, ++STORE, 47664385077248, 47664386772991, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664386772991, ++STORE, 47664387358720, 47664387362815, ++STORE, 47664386772992, 47664387358719, ++ERASE, 47664386772992, 47664386772992, ++STORE, 47664386772992, 47664387358719, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387567616, 47664387567616, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387584000, 47664389423103, ++STORE, 47664387723264, 47664389423103, ++STORE, 47664387584000, 47664387723263, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389382143, ++STORE, 47664389382144, 47664389423103, ++STORE, 47664389066752, 47664389382143, ++STORE, 47664387723264, 47664389066751, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389066751, ++STORE, 47664389378048, 47664389382143, ++STORE, 47664389066752, 47664389378047, ++ERASE, 47664389066752, 47664389066752, ++STORE, 47664389066752, 47664389378047, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389406720, 47664389406720, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389423104, 47664389558271, ++ERASE, 47664389423104, 47664389423104, ++STORE, 47664389423104, 47664389447679, ++STORE, 47664389447680, 47664389558271, ++STORE, 47664389509120, 47664389558271, ++STORE, 47664389447680, 47664389509119, ++ERASE, 47664389447680, 47664389447680, ++STORE, 47664389447680, 47664389509119, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389509120, 47664389533695, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389558271, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389533695, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389541888, 47664389541888, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389558272, 47664389578751, ++ERASE, 47664389558272, 47664389558272, ++STORE, 47664389558272, 47664389562367, ++STORE, 47664389562368, 47664389578751, ++STORE, 47664389566464, 47664389578751, ++STORE, 47664389562368, 47664389566463, ++ERASE, 47664389562368, 47664389562368, ++STORE, 47664389562368, 47664389566463, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389566464, 47664389570559, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389578751, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389570559, ++STORE, 47664389570560, 47664389578751, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389578752, 47664389586943, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389398527, ++STORE, 47664389398528, 47664389406719, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389574655, ++STORE, 47664389574656, 47664389578751, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389537791, ++STORE, 47664389537792, 47664389541887, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387559423, ++STORE, 47664387559424, 47664387567615, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384528383, ++STORE, 47664384528384, 47664384532479, ++ERASE, 94071213555712, 94071213555712, ++STORE, 94071213555712, 94071213563903, ++STORE, 94071213563904, 94071213567999, ++ERASE, 139968410804224, 139968410804224, ++STORE, 139968410804224, 139968410808319, ++STORE, 139968410808320, 139968410812415, ++ERASE, 47664384352256, 47664384352256, ++STORE, 94071244402688, 94071244537855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728271503360, 140737488351231, ++ERASE, 140728271503360, 140728271503360, ++STORE, 140728271503360, 140728271507455, ++STORE, 94410361982976, 94410362155007, ++ERASE, 94410361982976, 94410361982976, ++STORE, 94410361982976, 94410361999359, ++STORE, 94410361999360, 94410362155007, ++ERASE, 94410361999360, 94410361999360, ++STORE, 94410361999360, 94410362101759, ++STORE, 94410362101760, 94410362142719, ++STORE, 94410362142720, 94410362155007, ++STORE, 140351953997824, 140351954169855, ++ERASE, 140351953997824, 140351953997824, ++STORE, 140351953997824, 140351954001919, ++STORE, 140351954001920, 140351954169855, ++ERASE, 140351954001920, 140351954001920, ++STORE, 140351954001920, 140351954124799, ++STORE, 140351954124800, 140351954157567, ++STORE, 140351954157568, 140351954165759, ++STORE, 140351954165760, 140351954169855, ++STORE, 140728272429056, 140728272433151, ++STORE, 140728272416768, 140728272429055, ++STORE, 47280840998912, 47280841007103, ++STORE, 47280841007104, 47280841015295, ++STORE, 47280841015296, 47280841179135, ++ERASE, 47280841015296, 47280841015296, ++STORE, 47280841015296, 47280841027583, ++STORE, 47280841027584, 47280841179135, ++STORE, 47280841125888, 47280841179135, ++STORE, 47280841027584, 47280841125887, ++ERASE, 47280841027584, 47280841027584, ++STORE, 47280841027584, 47280841125887, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841125888, 47280841170943, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841179135, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841170943, ++STORE, 47280841170944, 47280841179135, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841179136, 47280844230655, ++STORE, 47280841723904, 47280844230655, ++STORE, 47280841179136, 47280841723903, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280844009471, ++STORE, 47280844009472, 47280844230655, ++STORE, 47280843419648, 47280844009471, ++STORE, 47280841723904, 47280843419647, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280843419647, ++STORE, 47280844005376, 47280844009471, ++STORE, 47280843419648, 47280844005375, ++ERASE, 47280843419648, 47280843419648, ++STORE, 47280843419648, 47280844005375, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844214272, 47280844214272, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844230656, 47280846069759, ++STORE, 47280844369920, 47280846069759, ++STORE, 47280844230656, 47280844369919, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280846028799, ++STORE, 47280846028800, 47280846069759, ++STORE, 47280845713408, 47280846028799, ++STORE, 47280844369920, 47280845713407, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280845713407, ++STORE, 47280846024704, 47280846028799, ++STORE, 47280845713408, 47280846024703, ++ERASE, 47280845713408, 47280845713408, ++STORE, 47280845713408, 47280846024703, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846053376, 47280846053376, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846069760, 47280846204927, ++ERASE, 47280846069760, 47280846069760, ++STORE, 47280846069760, 47280846094335, ++STORE, 47280846094336, 47280846204927, ++STORE, 47280846155776, 47280846204927, ++STORE, 47280846094336, 47280846155775, ++ERASE, 47280846094336, 47280846094336, ++STORE, 47280846094336, 47280846155775, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846155776, 47280846180351, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846204927, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846180351, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846188544, 47280846188544, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846204928, 47280846225407, ++ERASE, 47280846204928, 47280846204928, ++STORE, 47280846204928, 47280846209023, ++STORE, 47280846209024, 47280846225407, ++STORE, 47280846213120, 47280846225407, ++STORE, 47280846209024, 47280846213119, ++ERASE, 47280846209024, 47280846209024, ++STORE, 47280846209024, 47280846213119, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846213120, 47280846217215, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846225407, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846217215, ++STORE, 47280846217216, 47280846225407, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846225408, 47280846233599, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846045183, ++STORE, 47280846045184, 47280846053375, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846221311, ++STORE, 47280846221312, 47280846225407, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846184447, ++STORE, 47280846184448, 47280846188543, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844206079, ++STORE, 47280844206080, 47280844214271, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841175039, ++STORE, 47280841175040, 47280841179135, ++ERASE, 94410362142720, 94410362142720, ++STORE, 94410362142720, 94410362150911, ++STORE, 94410362150912, 94410362155007, ++ERASE, 140351954157568, 140351954157568, ++STORE, 140351954157568, 140351954161663, ++STORE, 140351954161664, 140351954165759, ++ERASE, 47280840998912, 47280840998912, ++STORE, 94410379456512, 94410379591679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732946362368, 140737488351231, ++ERASE, 140732946362368, 140732946362368, ++STORE, 140732946362368, 140732946366463, ++STORE, 94352937934848, 94352938106879, ++ERASE, 94352937934848, 94352937934848, ++STORE, 94352937934848, 94352937951231, ++STORE, 94352937951232, 94352938106879, ++ERASE, 94352937951232, 94352937951232, ++STORE, 94352937951232, 94352938053631, ++STORE, 94352938053632, 94352938094591, ++STORE, 94352938094592, 94352938106879, ++STORE, 140595518742528, 140595518914559, ++ERASE, 140595518742528, 140595518742528, ++STORE, 140595518742528, 140595518746623, ++STORE, 140595518746624, 140595518914559, ++ERASE, 140595518746624, 140595518746624, ++STORE, 140595518746624, 140595518869503, ++STORE, 140595518869504, 140595518902271, ++STORE, 140595518902272, 140595518910463, ++STORE, 140595518910464, 140595518914559, ++STORE, 140732947468288, 140732947472383, ++STORE, 140732947456000, 140732947468287, ++STORE, 47037276254208, 47037276262399, ++STORE, 47037276262400, 47037276270591, ++STORE, 47037276270592, 47037276434431, ++ERASE, 47037276270592, 47037276270592, ++STORE, 47037276270592, 47037276282879, ++STORE, 47037276282880, 47037276434431, ++STORE, 47037276381184, 47037276434431, ++STORE, 47037276282880, 47037276381183, ++ERASE, 47037276282880, 47037276282880, ++STORE, 47037276282880, 47037276381183, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276381184, 47037276426239, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276434431, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276426239, ++STORE, 47037276426240, 47037276434431, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276434432, 47037279485951, ++STORE, 47037276979200, 47037279485951, ++STORE, 47037276434432, 47037276979199, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037279264767, ++STORE, 47037279264768, 47037279485951, ++STORE, 47037278674944, 47037279264767, ++STORE, 47037276979200, 47037278674943, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037278674943, ++STORE, 47037279260672, 47037279264767, ++STORE, 47037278674944, 47037279260671, ++ERASE, 47037278674944, 47037278674944, ++STORE, 47037278674944, 47037279260671, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279469568, 47037279469568, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279485952, 47037281325055, ++STORE, 47037279625216, 47037281325055, ++STORE, 47037279485952, 47037279625215, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037281284095, ++STORE, 47037281284096, 47037281325055, ++STORE, 47037280968704, 47037281284095, ++STORE, 47037279625216, 47037280968703, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037280968703, ++STORE, 47037281280000, 47037281284095, ++STORE, 47037280968704, 47037281279999, ++ERASE, 47037280968704, 47037280968704, ++STORE, 47037280968704, 47037281279999, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281308672, 47037281308672, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281325056, 47037281460223, ++ERASE, 47037281325056, 47037281325056, ++STORE, 47037281325056, 47037281349631, ++STORE, 47037281349632, 47037281460223, ++STORE, 47037281411072, 47037281460223, ++STORE, 47037281349632, 47037281411071, ++ERASE, 47037281349632, 47037281349632, ++STORE, 47037281349632, 47037281411071, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281411072, 47037281435647, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281460223, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281435647, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281443840, 47037281443840, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281460224, 47037281480703, ++ERASE, 47037281460224, 47037281460224, ++STORE, 47037281460224, 47037281464319, ++STORE, 47037281464320, 47037281480703, ++STORE, 47037281468416, 47037281480703, ++STORE, 47037281464320, 47037281468415, ++ERASE, 47037281464320, 47037281464320, ++STORE, 47037281464320, 47037281468415, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281468416, 47037281472511, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281480703, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281472511, ++STORE, 47037281472512, 47037281480703, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281480704, 47037281488895, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281300479, ++STORE, 47037281300480, 47037281308671, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281476607, ++STORE, 47037281476608, 47037281480703, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281439743, ++STORE, 47037281439744, 47037281443839, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279461375, ++STORE, 47037279461376, 47037279469567, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276430335, ++STORE, 47037276430336, 47037276434431, ++ERASE, 94352938094592, 94352938094592, ++STORE, 94352938094592, 94352938102783, ++STORE, 94352938102784, 94352938106879, ++ERASE, 140595518902272, 140595518902272, ++STORE, 140595518902272, 140595518906367, ++STORE, 140595518906368, 140595518910463, ++ERASE, 47037276254208, 47037276254208, ++STORE, 94352938438656, 94352938573823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733506027520, 140737488351231, ++ERASE, 140733506027520, 140733506027520, ++STORE, 140733506027520, 140733506031615, ++STORE, 94150123073536, 94150123245567, ++ERASE, 94150123073536, 94150123073536, ++STORE, 94150123073536, 94150123089919, ++STORE, 94150123089920, 94150123245567, ++ERASE, 94150123089920, 94150123089920, ++STORE, 94150123089920, 94150123192319, ++STORE, 94150123192320, 94150123233279, ++STORE, 94150123233280, 94150123245567, ++STORE, 140081290375168, 140081290547199, ++ERASE, 140081290375168, 140081290375168, ++STORE, 140081290375168, 140081290379263, ++STORE, 140081290379264, 140081290547199, ++ERASE, 140081290379264, 140081290379264, ++STORE, 140081290379264, 140081290502143, ++STORE, 140081290502144, 140081290534911, ++STORE, 140081290534912, 140081290543103, ++STORE, 140081290543104, 140081290547199, ++STORE, 140733506707456, 140733506711551, ++STORE, 140733506695168, 140733506707455, ++STORE, 47551504621568, 47551504629759, ++STORE, 47551504629760, 47551504637951, ++STORE, 47551504637952, 47551504801791, ++ERASE, 47551504637952, 47551504637952, ++STORE, 47551504637952, 47551504650239, ++STORE, 47551504650240, 47551504801791, ++STORE, 47551504748544, 47551504801791, ++STORE, 47551504650240, 47551504748543, ++ERASE, 47551504650240, 47551504650240, ++STORE, 47551504650240, 47551504748543, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504748544, 47551504793599, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504801791, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504793599, ++STORE, 47551504793600, 47551504801791, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504801792, 47551507853311, ++STORE, 47551505346560, 47551507853311, ++STORE, 47551504801792, 47551505346559, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507632127, ++STORE, 47551507632128, 47551507853311, ++STORE, 47551507042304, 47551507632127, ++STORE, 47551505346560, 47551507042303, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507042303, ++STORE, 47551507628032, 47551507632127, ++STORE, 47551507042304, 47551507628031, ++ERASE, 47551507042304, 47551507042304, ++STORE, 47551507042304, 47551507628031, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507836928, 47551507836928, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507853312, 47551509692415, ++STORE, 47551507992576, 47551509692415, ++STORE, 47551507853312, 47551507992575, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509651455, ++STORE, 47551509651456, 47551509692415, ++STORE, 47551509336064, 47551509651455, ++STORE, 47551507992576, 47551509336063, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509336063, ++STORE, 47551509647360, 47551509651455, ++STORE, 47551509336064, 47551509647359, ++ERASE, 47551509336064, 47551509336064, ++STORE, 47551509336064, 47551509647359, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509676032, 47551509676032, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509692416, 47551509827583, ++ERASE, 47551509692416, 47551509692416, ++STORE, 47551509692416, 47551509716991, ++STORE, 47551509716992, 47551509827583, ++STORE, 47551509778432, 47551509827583, ++STORE, 47551509716992, 47551509778431, ++ERASE, 47551509716992, 47551509716992, ++STORE, 47551509716992, 47551509778431, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509778432, 47551509803007, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509827583, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509803007, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509811200, 47551509811200, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509827584, 47551509848063, ++ERASE, 47551509827584, 47551509827584, ++STORE, 47551509827584, 47551509831679, ++STORE, 47551509831680, 47551509848063, ++STORE, 47551509835776, 47551509848063, ++STORE, 47551509831680, 47551509835775, ++ERASE, 47551509831680, 47551509831680, ++STORE, 47551509831680, 47551509835775, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509835776, 47551509839871, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509848063, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509839871, ++STORE, 47551509839872, 47551509848063, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509848064, 47551509856255, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509667839, ++STORE, 47551509667840, 47551509676031, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509843967, ++STORE, 47551509843968, 47551509848063, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509807103, ++STORE, 47551509807104, 47551509811199, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507828735, ++STORE, 47551507828736, 47551507836927, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504797695, ++STORE, 47551504797696, 47551504801791, ++ERASE, 94150123233280, 94150123233280, ++STORE, 94150123233280, 94150123241471, ++STORE, 94150123241472, 94150123245567, ++ERASE, 140081290534912, 140081290534912, ++STORE, 140081290534912, 140081290539007, ++STORE, 140081290539008, 140081290543103, ++ERASE, 47551504621568, 47551504621568, ++STORE, 94150148112384, 94150148247551, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734389334016, 140737488351231, ++ERASE, 140734389334016, 140734389334016, ++STORE, 140734389334016, 140734389338111, ++STORE, 94844636606464, 94844636778495, ++ERASE, 94844636606464, 94844636606464, ++STORE, 94844636606464, 94844636622847, ++STORE, 94844636622848, 94844636778495, ++ERASE, 94844636622848, 94844636622848, ++STORE, 94844636622848, 94844636725247, ++STORE, 94844636725248, 94844636766207, ++STORE, 94844636766208, 94844636778495, ++STORE, 139922765217792, 139922765389823, ++ERASE, 139922765217792, 139922765217792, ++STORE, 139922765217792, 139922765221887, ++STORE, 139922765221888, 139922765389823, ++ERASE, 139922765221888, 139922765221888, ++STORE, 139922765221888, 139922765344767, ++STORE, 139922765344768, 139922765377535, ++STORE, 139922765377536, 139922765385727, ++STORE, 139922765385728, 139922765389823, ++STORE, 140734389678080, 140734389682175, ++STORE, 140734389665792, 140734389678079, ++STORE, 47710029778944, 47710029787135, ++STORE, 47710029787136, 47710029795327, ++STORE, 47710029795328, 47710029959167, ++ERASE, 47710029795328, 47710029795328, ++STORE, 47710029795328, 47710029807615, ++STORE, 47710029807616, 47710029959167, ++STORE, 47710029905920, 47710029959167, ++STORE, 47710029807616, 47710029905919, ++ERASE, 47710029807616, 47710029807616, ++STORE, 47710029807616, 47710029905919, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029905920, 47710029950975, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029959167, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029950975, ++STORE, 47710029950976, 47710029959167, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029959168, 47710033010687, ++STORE, 47710030503936, 47710033010687, ++STORE, 47710029959168, 47710030503935, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032789503, ++STORE, 47710032789504, 47710033010687, ++STORE, 47710032199680, 47710032789503, ++STORE, 47710030503936, 47710032199679, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032199679, ++STORE, 47710032785408, 47710032789503, ++STORE, 47710032199680, 47710032785407, ++ERASE, 47710032199680, 47710032199680, ++STORE, 47710032199680, 47710032785407, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032994304, 47710032994304, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710033010688, 47710034849791, ++STORE, 47710033149952, 47710034849791, ++STORE, 47710033010688, 47710033149951, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034808831, ++STORE, 47710034808832, 47710034849791, ++STORE, 47710034493440, 47710034808831, ++STORE, 47710033149952, 47710034493439, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034493439, ++STORE, 47710034804736, 47710034808831, ++STORE, 47710034493440, 47710034804735, ++ERASE, 47710034493440, 47710034493440, ++STORE, 47710034493440, 47710034804735, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034833408, 47710034833408, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034849792, 47710034984959, ++ERASE, 47710034849792, 47710034849792, ++STORE, 47710034849792, 47710034874367, ++STORE, 47710034874368, 47710034984959, ++STORE, 47710034935808, 47710034984959, ++STORE, 47710034874368, 47710034935807, ++ERASE, 47710034874368, 47710034874368, ++STORE, 47710034874368, 47710034935807, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034935808, 47710034960383, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034984959, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034960383, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034968576, 47710034968576, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034984960, 47710035005439, ++ERASE, 47710034984960, 47710034984960, ++STORE, 47710034984960, 47710034989055, ++STORE, 47710034989056, 47710035005439, ++STORE, 47710034993152, 47710035005439, ++STORE, 47710034989056, 47710034993151, ++ERASE, 47710034989056, 47710034989056, ++STORE, 47710034989056, 47710034993151, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710034993152, 47710034997247, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710035005439, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710034997247, ++STORE, 47710034997248, 47710035005439, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710035005440, 47710035013631, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034825215, ++STORE, 47710034825216, 47710034833407, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035001343, ++STORE, 47710035001344, 47710035005439, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034964479, ++STORE, 47710034964480, 47710034968575, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032986111, ++STORE, 47710032986112, 47710032994303, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029955071, ++STORE, 47710029955072, 47710029959167, ++ERASE, 94844636766208, 94844636766208, ++STORE, 94844636766208, 94844636774399, ++STORE, 94844636774400, 94844636778495, ++ERASE, 139922765377536, 139922765377536, ++STORE, 139922765377536, 139922765381631, ++STORE, 139922765381632, 139922765385727, ++ERASE, 47710029778944, 47710029778944, ++STORE, 94844641775616, 94844641910783, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732213886976, 140737488351231, ++ERASE, 140732213886976, 140732213886976, ++STORE, 140732213886976, 140732213891071, ++STORE, 94240508887040, 94240509059071, ++ERASE, 94240508887040, 94240508887040, ++STORE, 94240508887040, 94240508903423, ++STORE, 94240508903424, 94240509059071, ++ERASE, 94240508903424, 94240508903424, ++STORE, 94240508903424, 94240509005823, ++STORE, 94240509005824, 94240509046783, ++STORE, 94240509046784, 94240509059071, ++STORE, 140275106516992, 140275106689023, ++ERASE, 140275106516992, 140275106516992, ++STORE, 140275106516992, 140275106521087, ++STORE, 140275106521088, 140275106689023, ++ERASE, 140275106521088, 140275106521088, ++STORE, 140275106521088, 140275106643967, ++STORE, 140275106643968, 140275106676735, ++STORE, 140275106676736, 140275106684927, ++STORE, 140275106684928, 140275106689023, ++STORE, 140732213977088, 140732213981183, ++STORE, 140732213964800, 140732213977087, ++STORE, 47357688479744, 47357688487935, ++STORE, 47357688487936, 47357688496127, ++STORE, 47357688496128, 47357688659967, ++ERASE, 47357688496128, 47357688496128, ++STORE, 47357688496128, 47357688508415, ++STORE, 47357688508416, 47357688659967, ++STORE, 47357688606720, 47357688659967, ++STORE, 47357688508416, 47357688606719, ++ERASE, 47357688508416, 47357688508416, ++STORE, 47357688508416, 47357688606719, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688606720, 47357688651775, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688659967, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688651775, ++STORE, 47357688651776, 47357688659967, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688659968, 47357691711487, ++STORE, 47357689204736, 47357691711487, ++STORE, 47357688659968, 47357689204735, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357691490303, ++STORE, 47357691490304, 47357691711487, ++STORE, 47357690900480, 47357691490303, ++STORE, 47357689204736, 47357690900479, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357690900479, ++STORE, 47357691486208, 47357691490303, ++STORE, 47357690900480, 47357691486207, ++ERASE, 47357690900480, 47357690900480, ++STORE, 47357690900480, 47357691486207, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691695104, 47357691695104, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691711488, 47357693550591, ++STORE, 47357691850752, 47357693550591, ++STORE, 47357691711488, 47357691850751, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693509631, ++STORE, 47357693509632, 47357693550591, ++STORE, 47357693194240, 47357693509631, ++STORE, 47357691850752, 47357693194239, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693194239, ++STORE, 47357693505536, 47357693509631, ++STORE, 47357693194240, 47357693505535, ++ERASE, 47357693194240, 47357693194240, ++STORE, 47357693194240, 47357693505535, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693534208, 47357693534208, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693550592, 47357693685759, ++ERASE, 47357693550592, 47357693550592, ++STORE, 47357693550592, 47357693575167, ++STORE, 47357693575168, 47357693685759, ++STORE, 47357693636608, 47357693685759, ++STORE, 47357693575168, 47357693636607, ++ERASE, 47357693575168, 47357693575168, ++STORE, 47357693575168, 47357693636607, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693636608, 47357693661183, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693685759, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693661183, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693669376, 47357693669376, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693685760, 47357693706239, ++ERASE, 47357693685760, 47357693685760, ++STORE, 47357693685760, 47357693689855, ++STORE, 47357693689856, 47357693706239, ++STORE, 47357693693952, 47357693706239, ++STORE, 47357693689856, 47357693693951, ++ERASE, 47357693689856, 47357693689856, ++STORE, 47357693689856, 47357693693951, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693693952, 47357693698047, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693706239, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693698047, ++STORE, 47357693698048, 47357693706239, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693706240, 47357693714431, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693526015, ++STORE, 47357693526016, 47357693534207, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693702143, ++STORE, 47357693702144, 47357693706239, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693665279, ++STORE, 47357693665280, 47357693669375, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691686911, ++STORE, 47357691686912, 47357691695103, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688655871, ++STORE, 47357688655872, 47357688659967, ++ERASE, 94240509046784, 94240509046784, ++STORE, 94240509046784, 94240509054975, ++STORE, 94240509054976, 94240509059071, ++ERASE, 140275106676736, 140275106676736, ++STORE, 140275106676736, 140275106680831, ++STORE, 140275106680832, 140275106684927, ++ERASE, 47357688479744, 47357688479744, ++STORE, 94240518361088, 94240518496255, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732688277504, 140737488351231, ++ERASE, 140732688277504, 140732688277504, ++STORE, 140732688277504, 140732688281599, ++STORE, 94629171351552, 94629172064255, ++ERASE, 94629171351552, 94629171351552, ++STORE, 94629171351552, 94629171400703, ++STORE, 94629171400704, 94629172064255, ++ERASE, 94629171400704, 94629171400704, ++STORE, 94629171400704, 94629171945471, ++STORE, 94629171945472, 94629172043775, ++STORE, 94629172043776, 94629172064255, ++STORE, 139770707644416, 139770707816447, ++ERASE, 139770707644416, 139770707644416, ++STORE, 139770707644416, 139770707648511, ++STORE, 139770707648512, 139770707816447, ++ERASE, 139770707648512, 139770707648512, ++STORE, 139770707648512, 139770707771391, ++STORE, 139770707771392, 139770707804159, ++STORE, 139770707804160, 139770707812351, ++STORE, 139770707812352, 139770707816447, ++STORE, 140732689121280, 140732689125375, ++STORE, 140732689108992, 140732689121279, ++STORE, 47862087352320, 47862087360511, ++STORE, 47862087360512, 47862087368703, ++STORE, 47862087368704, 47862087475199, ++STORE, 47862087385088, 47862087475199, ++STORE, 47862087368704, 47862087385087, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087458815, ++STORE, 47862087458816, 47862087475199, ++STORE, 47862087438336, 47862087458815, ++STORE, 47862087385088, 47862087438335, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087438335, ++STORE, 47862087454720, 47862087458815, ++STORE, 47862087438336, 47862087454719, ++ERASE, 47862087438336, 47862087438336, ++STORE, 47862087438336, 47862087454719, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087467008, 47862087467008, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087475200, 47862089314303, ++STORE, 47862087614464, 47862089314303, ++STORE, 47862087475200, 47862087614463, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862089273343, ++STORE, 47862089273344, 47862089314303, ++STORE, 47862088957952, 47862089273343, ++STORE, 47862087614464, 47862088957951, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862088957951, ++STORE, 47862089269248, 47862089273343, ++STORE, 47862088957952, 47862089269247, ++ERASE, 47862088957952, 47862088957952, ++STORE, 47862088957952, 47862089269247, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089297920, 47862089297920, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089297920, 47862089326591, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089289727, ++STORE, 47862089289728, 47862089297919, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087462911, ++STORE, 47862087462912, 47862087467007, ++ERASE, 94629172043776, 94629172043776, ++STORE, 94629172043776, 94629172060159, ++STORE, 94629172060160, 94629172064255, ++ERASE, 139770707804160, 139770707804160, ++STORE, 139770707804160, 139770707808255, ++STORE, 139770707808256, 139770707812351, ++ERASE, 47862087352320, 47862087352320, ++STORE, 94629197533184, 94629197668351, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727540711424, 140737488351231, ++ERASE, 140727540711424, 140727540711424, ++STORE, 140727540711424, 140727540715519, ++STORE, 94299865313280, 94299866025983, ++ERASE, 94299865313280, 94299865313280, ++STORE, 94299865313280, 94299865362431, ++STORE, 94299865362432, 94299866025983, ++ERASE, 94299865362432, 94299865362432, ++STORE, 94299865362432, 94299865907199, ++STORE, 94299865907200, 94299866005503, ++STORE, 94299866005504, 94299866025983, ++STORE, 140680268763136, 140680268935167, ++ERASE, 140680268763136, 140680268763136, ++STORE, 140680268763136, 140680268767231, ++STORE, 140680268767232, 140680268935167, ++ERASE, 140680268767232, 140680268767232, ++STORE, 140680268767232, 140680268890111, ++STORE, 140680268890112, 140680268922879, ++STORE, 140680268922880, 140680268931071, ++STORE, 140680268931072, 140680268935167, ++STORE, 140727541424128, 140727541428223, ++STORE, 140727541411840, 140727541424127, ++STORE, 46952526233600, 46952526241791, ++STORE, 46952526241792, 46952526249983, ++STORE, 46952526249984, 46952526356479, ++STORE, 46952526266368, 46952526356479, ++STORE, 46952526249984, 46952526266367, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526340095, ++STORE, 46952526340096, 46952526356479, ++STORE, 46952526319616, 46952526340095, ++STORE, 46952526266368, 46952526319615, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526319615, ++STORE, 46952526336000, 46952526340095, ++STORE, 46952526319616, 46952526335999, ++ERASE, 46952526319616, 46952526319616, ++STORE, 46952526319616, 46952526335999, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526348288, 46952526348288, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526356480, 46952528195583, ++STORE, 46952526495744, 46952528195583, ++STORE, 46952526356480, 46952526495743, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952528154623, ++STORE, 46952528154624, 46952528195583, ++STORE, 46952527839232, 46952528154623, ++STORE, 46952526495744, 46952527839231, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952527839231, ++STORE, 46952528150528, 46952528154623, ++STORE, 46952527839232, 46952528150527, ++ERASE, 46952527839232, 46952527839232, ++STORE, 46952527839232, 46952528150527, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528179200, 46952528179200, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528179200, 46952528207871, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528171007, ++STORE, 46952528171008, 46952528179199, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526344191, ++STORE, 46952526344192, 46952526348287, ++ERASE, 94299866005504, 94299866005504, ++STORE, 94299866005504, 94299866021887, ++STORE, 94299866021888, 94299866025983, ++ERASE, 140680268922880, 140680268922880, ++STORE, 140680268922880, 140680268926975, ++STORE, 140680268926976, 140680268931071, ++ERASE, 46952526233600, 46952526233600, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722874793984, 140737488351231, ++ERASE, 140722874793984, 140722874793984, ++STORE, 140722874793984, 140722874798079, ++STORE, 94448916213760, 94448916926463, ++ERASE, 94448916213760, 94448916213760, ++STORE, 94448916213760, 94448916262911, ++STORE, 94448916262912, 94448916926463, ++ERASE, 94448916262912, 94448916262912, ++STORE, 94448916262912, 94448916807679, ++STORE, 94448916807680, 94448916905983, ++STORE, 94448916905984, 94448916926463, ++STORE, 140389117046784, 140389117218815, ++ERASE, 140389117046784, 140389117046784, ++STORE, 140389117046784, 140389117050879, ++STORE, 140389117050880, 140389117218815, ++ERASE, 140389117050880, 140389117050880, ++STORE, 140389117050880, 140389117173759, ++STORE, 140389117173760, 140389117206527, ++STORE, 140389117206528, 140389117214719, ++STORE, 140389117214720, 140389117218815, ++STORE, 140722875297792, 140722875301887, ++STORE, 140722875285504, 140722875297791, ++STORE, 47243677949952, 47243677958143, ++STORE, 47243677958144, 47243677966335, ++STORE, 47243677966336, 47243678072831, ++STORE, 47243677982720, 47243678072831, ++STORE, 47243677966336, 47243677982719, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678056447, ++STORE, 47243678056448, 47243678072831, ++STORE, 47243678035968, 47243678056447, ++STORE, 47243677982720, 47243678035967, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678035967, ++STORE, 47243678052352, 47243678056447, ++STORE, 47243678035968, 47243678052351, ++ERASE, 47243678035968, 47243678035968, ++STORE, 47243678035968, 47243678052351, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678064640, 47243678064640, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678072832, 47243679911935, ++STORE, 47243678212096, 47243679911935, ++STORE, 47243678072832, 47243678212095, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679870975, ++STORE, 47243679870976, 47243679911935, ++STORE, 47243679555584, 47243679870975, ++STORE, 47243678212096, 47243679555583, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679555583, ++STORE, 47243679866880, 47243679870975, ++STORE, 47243679555584, 47243679866879, ++ERASE, 47243679555584, 47243679555584, ++STORE, 47243679555584, 47243679866879, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679895552, 47243679895552, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679895552, 47243679924223, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679887359, ++STORE, 47243679887360, 47243679895551, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678060543, ++STORE, 47243678060544, 47243678064639, ++ERASE, 94448916905984, 94448916905984, ++STORE, 94448916905984, 94448916922367, ++STORE, 94448916922368, 94448916926463, ++ERASE, 140389117206528, 140389117206528, ++STORE, 140389117206528, 140389117210623, ++STORE, 140389117210624, 140389117214719, ++ERASE, 47243677949952, 47243677949952, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733068505088, 140737488351231, ++ERASE, 140733068505088, 140733068505088, ++STORE, 140733068505088, 140733068509183, ++STORE, 94207145750528, 94207146463231, ++ERASE, 94207145750528, 94207145750528, ++STORE, 94207145750528, 94207145799679, ++STORE, 94207145799680, 94207146463231, ++ERASE, 94207145799680, 94207145799680, ++STORE, 94207145799680, 94207146344447, ++STORE, 94207146344448, 94207146442751, ++STORE, 94207146442752, 94207146463231, ++STORE, 140684504911872, 140684505083903, ++ERASE, 140684504911872, 140684504911872, ++STORE, 140684504911872, 140684504915967, ++STORE, 140684504915968, 140684505083903, ++ERASE, 140684504915968, 140684504915968, ++STORE, 140684504915968, 140684505038847, ++STORE, 140684505038848, 140684505071615, ++STORE, 140684505071616, 140684505079807, ++STORE, 140684505079808, 140684505083903, ++STORE, 140733068607488, 140733068611583, ++STORE, 140733068595200, 140733068607487, ++STORE, 46948290084864, 46948290093055, ++STORE, 46948290093056, 46948290101247, ++STORE, 46948290101248, 46948290207743, ++STORE, 46948290117632, 46948290207743, ++STORE, 46948290101248, 46948290117631, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290191359, ++STORE, 46948290191360, 46948290207743, ++STORE, 46948290170880, 46948290191359, ++STORE, 46948290117632, 46948290170879, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290170879, ++STORE, 46948290187264, 46948290191359, ++STORE, 46948290170880, 46948290187263, ++ERASE, 46948290170880, 46948290170880, ++STORE, 46948290170880, 46948290187263, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290199552, 46948290199552, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290207744, 46948292046847, ++STORE, 46948290347008, 46948292046847, ++STORE, 46948290207744, 46948290347007, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948292005887, ++STORE, 46948292005888, 46948292046847, ++STORE, 46948291690496, 46948292005887, ++STORE, 46948290347008, 46948291690495, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948291690495, ++STORE, 46948292001792, 46948292005887, ++STORE, 46948291690496, 46948292001791, ++ERASE, 46948291690496, 46948291690496, ++STORE, 46948291690496, 46948292001791, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292030464, 46948292030464, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292030464, 46948292059135, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292022271, ++STORE, 46948292022272, 46948292030463, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290195455, ++STORE, 46948290195456, 46948290199551, ++ERASE, 94207146442752, 94207146442752, ++STORE, 94207146442752, 94207146459135, ++STORE, 94207146459136, 94207146463231, ++ERASE, 140684505071616, 140684505071616, ++STORE, 140684505071616, 140684505075711, ++STORE, 140684505075712, 140684505079807, ++ERASE, 46948290084864, 46948290084864, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726367158272, 140737488351231, ++ERASE, 140726367158272, 140726367158272, ++STORE, 140726367158272, 140726367162367, ++STORE, 94436124106752, 94436124819455, ++ERASE, 94436124106752, 94436124106752, ++STORE, 94436124106752, 94436124155903, ++STORE, 94436124155904, 94436124819455, ++ERASE, 94436124155904, 94436124155904, ++STORE, 94436124155904, 94436124700671, ++STORE, 94436124700672, 94436124798975, ++STORE, 94436124798976, 94436124819455, ++STORE, 140049025044480, 140049025216511, ++ERASE, 140049025044480, 140049025044480, ++STORE, 140049025044480, 140049025048575, ++STORE, 140049025048576, 140049025216511, ++ERASE, 140049025048576, 140049025048576, ++STORE, 140049025048576, 140049025171455, ++STORE, 140049025171456, 140049025204223, ++STORE, 140049025204224, 140049025212415, ++STORE, 140049025212416, 140049025216511, ++STORE, 140726367256576, 140726367260671, ++STORE, 140726367244288, 140726367256575, ++STORE, 47583769952256, 47583769960447, ++STORE, 47583769960448, 47583769968639, ++STORE, 47583769968640, 47583770075135, ++STORE, 47583769985024, 47583770075135, ++STORE, 47583769968640, 47583769985023, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770058751, ++STORE, 47583770058752, 47583770075135, ++STORE, 47583770038272, 47583770058751, ++STORE, 47583769985024, 47583770038271, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770038271, ++STORE, 47583770054656, 47583770058751, ++STORE, 47583770038272, 47583770054655, ++ERASE, 47583770038272, 47583770038272, ++STORE, 47583770038272, 47583770054655, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770066944, 47583770066944, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770075136, 47583771914239, ++STORE, 47583770214400, 47583771914239, ++STORE, 47583770075136, 47583770214399, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771873279, ++STORE, 47583771873280, 47583771914239, ++STORE, 47583771557888, 47583771873279, ++STORE, 47583770214400, 47583771557887, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771557887, ++STORE, 47583771869184, 47583771873279, ++STORE, 47583771557888, 47583771869183, ++ERASE, 47583771557888, 47583771557888, ++STORE, 47583771557888, 47583771869183, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771897856, 47583771897856, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771897856, 47583771926527, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771889663, ++STORE, 47583771889664, 47583771897855, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770062847, ++STORE, 47583770062848, 47583770066943, ++ERASE, 94436124798976, 94436124798976, ++STORE, 94436124798976, 94436124815359, ++STORE, 94436124815360, 94436124819455, ++ERASE, 140049025204224, 140049025204224, ++STORE, 140049025204224, 140049025208319, ++STORE, 140049025208320, 140049025212415, ++ERASE, 47583769952256, 47583769952256, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727116099584, 140737488351231, ++ERASE, 140727116099584, 140727116099584, ++STORE, 140727116099584, 140727116103679, ++STORE, 94166319734784, 94166320447487, ++ERASE, 94166319734784, 94166319734784, ++STORE, 94166319734784, 94166319783935, ++STORE, 94166319783936, 94166320447487, ++ERASE, 94166319783936, 94166319783936, ++STORE, 94166319783936, 94166320328703, ++STORE, 94166320328704, 94166320427007, ++STORE, 94166320427008, 94166320447487, ++STORE, 139976559542272, 139976559714303, ++ERASE, 139976559542272, 139976559542272, ++STORE, 139976559542272, 139976559546367, ++STORE, 139976559546368, 139976559714303, ++ERASE, 139976559546368, 139976559546368, ++STORE, 139976559546368, 139976559669247, ++STORE, 139976559669248, 139976559702015, ++STORE, 139976559702016, 139976559710207, ++STORE, 139976559710208, 139976559714303, ++STORE, 140727116222464, 140727116226559, ++STORE, 140727116210176, 140727116222463, ++STORE, 47656235454464, 47656235462655, ++STORE, 47656235462656, 47656235470847, ++STORE, 47656235470848, 47656235577343, ++STORE, 47656235487232, 47656235577343, ++STORE, 47656235470848, 47656235487231, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235560959, ++STORE, 47656235560960, 47656235577343, ++STORE, 47656235540480, 47656235560959, ++STORE, 47656235487232, 47656235540479, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235540479, ++STORE, 47656235556864, 47656235560959, ++STORE, 47656235540480, 47656235556863, ++ERASE, 47656235540480, 47656235540480, ++STORE, 47656235540480, 47656235556863, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235569152, 47656235569152, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235577344, 47656237416447, ++STORE, 47656235716608, 47656237416447, ++STORE, 47656235577344, 47656235716607, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237375487, ++STORE, 47656237375488, 47656237416447, ++STORE, 47656237060096, 47656237375487, ++STORE, 47656235716608, 47656237060095, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237060095, ++STORE, 47656237371392, 47656237375487, ++STORE, 47656237060096, 47656237371391, ++ERASE, 47656237060096, 47656237060096, ++STORE, 47656237060096, 47656237371391, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237400064, 47656237400064, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237400064, 47656237428735, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237391871, ++STORE, 47656237391872, 47656237400063, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235565055, ++STORE, 47656235565056, 47656235569151, ++ERASE, 94166320427008, 94166320427008, ++STORE, 94166320427008, 94166320443391, ++STORE, 94166320443392, 94166320447487, ++ERASE, 139976559702016, 139976559702016, ++STORE, 139976559702016, 139976559706111, ++STORE, 139976559706112, 139976559710207, ++ERASE, 47656235454464, 47656235454464, ++STORE, 94166332153856, 94166332289023, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726412816384, 140737488351231, ++ERASE, 140726412816384, 140726412816384, ++STORE, 140726412816384, 140726412820479, ++STORE, 94094884507648, 94094885220351, ++ERASE, 94094884507648, 94094884507648, ++STORE, 94094884507648, 94094884556799, ++STORE, 94094884556800, 94094885220351, ++ERASE, 94094884556800, 94094884556800, ++STORE, 94094884556800, 94094885101567, ++STORE, 94094885101568, 94094885199871, ++STORE, 94094885199872, 94094885220351, ++STORE, 139773773938688, 139773774110719, ++ERASE, 139773773938688, 139773773938688, ++STORE, 139773773938688, 139773773942783, ++STORE, 139773773942784, 139773774110719, ++ERASE, 139773773942784, 139773773942784, ++STORE, 139773773942784, 139773774065663, ++STORE, 139773774065664, 139773774098431, ++STORE, 139773774098432, 139773774106623, ++STORE, 139773774106624, 139773774110719, ++STORE, 140726412963840, 140726412967935, ++STORE, 140726412951552, 140726412963839, ++STORE, 47859021058048, 47859021066239, ++STORE, 47859021066240, 47859021074431, ++STORE, 47859021074432, 47859021180927, ++STORE, 47859021090816, 47859021180927, ++STORE, 47859021074432, 47859021090815, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021164543, ++STORE, 47859021164544, 47859021180927, ++STORE, 47859021144064, 47859021164543, ++STORE, 47859021090816, 47859021144063, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021144063, ++STORE, 47859021160448, 47859021164543, ++STORE, 47859021144064, 47859021160447, ++ERASE, 47859021144064, 47859021144064, ++STORE, 47859021144064, 47859021160447, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021172736, 47859021172736, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021180928, 47859023020031, ++STORE, 47859021320192, 47859023020031, ++STORE, 47859021180928, 47859021320191, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022979071, ++STORE, 47859022979072, 47859023020031, ++STORE, 47859022663680, 47859022979071, ++STORE, 47859021320192, 47859022663679, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022663679, ++STORE, 47859022974976, 47859022979071, ++STORE, 47859022663680, 47859022974975, ++ERASE, 47859022663680, 47859022663680, ++STORE, 47859022663680, 47859022974975, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859023003648, 47859023003648, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859023003648, 47859023032319, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859022995455, ++STORE, 47859022995456, 47859023003647, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021168639, ++STORE, 47859021168640, 47859021172735, ++ERASE, 94094885199872, 94094885199872, ++STORE, 94094885199872, 94094885216255, ++STORE, 94094885216256, 94094885220351, ++ERASE, 139773774098432, 139773774098432, ++STORE, 139773774098432, 139773774102527, ++STORE, 139773774102528, 139773774106623, ++ERASE, 47859021058048, 47859021058048, ++STORE, 94094901108736, 94094901243903, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736567963648, 140737488351231, ++ERASE, 140736567963648, 140736567963648, ++STORE, 140736567963648, 140736567967743, ++STORE, 94924425748480, 94924426461183, ++ERASE, 94924425748480, 94924425748480, ++STORE, 94924425748480, 94924425797631, ++STORE, 94924425797632, 94924426461183, ++ERASE, 94924425797632, 94924425797632, ++STORE, 94924425797632, 94924426342399, ++STORE, 94924426342400, 94924426440703, ++STORE, 94924426440704, 94924426461183, ++STORE, 140042126319616, 140042126491647, ++ERASE, 140042126319616, 140042126319616, ++STORE, 140042126319616, 140042126323711, ++STORE, 140042126323712, 140042126491647, ++ERASE, 140042126323712, 140042126323712, ++STORE, 140042126323712, 140042126446591, ++STORE, 140042126446592, 140042126479359, ++STORE, 140042126479360, 140042126487551, ++STORE, 140042126487552, 140042126491647, ++STORE, 140736568672256, 140736568676351, ++STORE, 140736568659968, 140736568672255, ++STORE, 47590668677120, 47590668685311, ++STORE, 47590668685312, 47590668693503, ++STORE, 47590668693504, 47590668799999, ++STORE, 47590668709888, 47590668799999, ++STORE, 47590668693504, 47590668709887, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668783615, ++STORE, 47590668783616, 47590668799999, ++STORE, 47590668763136, 47590668783615, ++STORE, 47590668709888, 47590668763135, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668763135, ++STORE, 47590668779520, 47590668783615, ++STORE, 47590668763136, 47590668779519, ++ERASE, 47590668763136, 47590668763136, ++STORE, 47590668763136, 47590668779519, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668791808, 47590668791808, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668800000, 47590670639103, ++STORE, 47590668939264, 47590670639103, ++STORE, 47590668800000, 47590668939263, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670598143, ++STORE, 47590670598144, 47590670639103, ++STORE, 47590670282752, 47590670598143, ++STORE, 47590668939264, 47590670282751, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670282751, ++STORE, 47590670594048, 47590670598143, ++STORE, 47590670282752, 47590670594047, ++ERASE, 47590670282752, 47590670282752, ++STORE, 47590670282752, 47590670594047, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670622720, 47590670622720, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670622720, 47590670651391, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670614527, ++STORE, 47590670614528, 47590670622719, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668787711, ++STORE, 47590668787712, 47590668791807, ++ERASE, 94924426440704, 94924426440704, ++STORE, 94924426440704, 94924426457087, ++STORE, 94924426457088, 94924426461183, ++ERASE, 140042126479360, 140042126479360, ++STORE, 140042126479360, 140042126483455, ++STORE, 140042126483456, 140042126487551, ++ERASE, 47590668677120, 47590668677120, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733281439744, 140737488351231, ++ERASE, 140733281439744, 140733281439744, ++STORE, 140733281439744, 140733281443839, ++STORE, 94490667069440, 94490667782143, ++ERASE, 94490667069440, 94490667069440, ++STORE, 94490667069440, 94490667118591, ++STORE, 94490667118592, 94490667782143, ++ERASE, 94490667118592, 94490667118592, ++STORE, 94490667118592, 94490667663359, ++STORE, 94490667663360, 94490667761663, ++STORE, 94490667761664, 94490667782143, ++STORE, 139878215118848, 139878215290879, ++ERASE, 139878215118848, 139878215118848, ++STORE, 139878215118848, 139878215122943, ++STORE, 139878215122944, 139878215290879, ++ERASE, 139878215122944, 139878215122944, ++STORE, 139878215122944, 139878215245823, ++STORE, 139878215245824, 139878215278591, ++STORE, 139878215278592, 139878215286783, ++STORE, 139878215286784, 139878215290879, ++STORE, 140733281464320, 140733281468415, ++STORE, 140733281452032, 140733281464319, ++STORE, 47754579877888, 47754579886079, ++STORE, 47754579886080, 47754579894271, ++STORE, 47754579894272, 47754580000767, ++STORE, 47754579910656, 47754580000767, ++STORE, 47754579894272, 47754579910655, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579984383, ++STORE, 47754579984384, 47754580000767, ++STORE, 47754579963904, 47754579984383, ++STORE, 47754579910656, 47754579963903, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579963903, ++STORE, 47754579980288, 47754579984383, ++STORE, 47754579963904, 47754579980287, ++ERASE, 47754579963904, 47754579963904, ++STORE, 47754579963904, 47754579980287, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579992576, 47754579992576, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754580000768, 47754581839871, ++STORE, 47754580140032, 47754581839871, ++STORE, 47754580000768, 47754580140031, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581798911, ++STORE, 47754581798912, 47754581839871, ++STORE, 47754581483520, 47754581798911, ++STORE, 47754580140032, 47754581483519, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581483519, ++STORE, 47754581794816, 47754581798911, ++STORE, 47754581483520, 47754581794815, ++ERASE, 47754581483520, 47754581483520, ++STORE, 47754581483520, 47754581794815, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581823488, 47754581823488, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581823488, 47754581852159, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581815295, ++STORE, 47754581815296, 47754581823487, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579988479, ++STORE, 47754579988480, 47754579992575, ++ERASE, 94490667761664, 94490667761664, ++STORE, 94490667761664, 94490667778047, ++STORE, 94490667778048, 94490667782143, ++ERASE, 139878215278592, 139878215278592, ++STORE, 139878215278592, 139878215282687, ++STORE, 139878215282688, 139878215286783, ++ERASE, 47754579877888, 47754579877888, ++STORE, 94490669649920, 94490669785087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140735382188032, 140737488351231, ++ERASE, 140735382188032, 140735382188032, ++STORE, 140735382188032, 140735382192127, ++STORE, 94150181302272, 94150182014975, ++ERASE, 94150181302272, 94150181302272, ++STORE, 94150181302272, 94150181351423, ++STORE, 94150181351424, 94150182014975, ++ERASE, 94150181351424, 94150181351424, ++STORE, 94150181351424, 94150181896191, ++STORE, 94150181896192, 94150181994495, ++STORE, 94150181994496, 94150182014975, ++STORE, 139679752458240, 139679752630271, ++ERASE, 139679752458240, 139679752458240, ++STORE, 139679752458240, 139679752462335, ++STORE, 139679752462336, 139679752630271, ++ERASE, 139679752462336, 139679752462336, ++STORE, 139679752462336, 139679752585215, ++STORE, 139679752585216, 139679752617983, ++STORE, 139679752617984, 139679752626175, ++STORE, 139679752626176, 139679752630271, ++STORE, 140735382536192, 140735382540287, ++STORE, 140735382523904, 140735382536191, ++STORE, 47953042538496, 47953042546687, ++STORE, 47953042546688, 47953042554879, ++STORE, 47953042554880, 47953042661375, ++STORE, 47953042571264, 47953042661375, ++STORE, 47953042554880, 47953042571263, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042644991, ++STORE, 47953042644992, 47953042661375, ++STORE, 47953042624512, 47953042644991, ++STORE, 47953042571264, 47953042624511, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042624511, ++STORE, 47953042640896, 47953042644991, ++STORE, 47953042624512, 47953042640895, ++ERASE, 47953042624512, 47953042624512, ++STORE, 47953042624512, 47953042640895, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042653184, 47953042653184, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042661376, 47953044500479, ++STORE, 47953042800640, 47953044500479, ++STORE, 47953042661376, 47953042800639, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044459519, ++STORE, 47953044459520, 47953044500479, ++STORE, 47953044144128, 47953044459519, ++STORE, 47953042800640, 47953044144127, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044144127, ++STORE, 47953044455424, 47953044459519, ++STORE, 47953044144128, 47953044455423, ++ERASE, 47953044144128, 47953044144128, ++STORE, 47953044144128, 47953044455423, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044484096, 47953044484096, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044484096, 47953044512767, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044475903, ++STORE, 47953044475904, 47953044484095, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042649087, ++STORE, 47953042649088, 47953042653183, ++ERASE, 94150181994496, 94150181994496, ++STORE, 94150181994496, 94150182010879, ++STORE, 94150182010880, 94150182014975, ++ERASE, 139679752617984, 139679752617984, ++STORE, 139679752617984, 139679752622079, ++STORE, 139679752622080, 139679752626175, ++ERASE, 47953042538496, 47953042538496, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737044123648, 140737488351231, ++ERASE, 140737044123648, 140737044123648, ++STORE, 140737044123648, 140737044127743, ++STORE, 94425324294144, 94425325006847, ++ERASE, 94425324294144, 94425324294144, ++STORE, 94425324294144, 94425324343295, ++STORE, 94425324343296, 94425325006847, ++ERASE, 94425324343296, 94425324343296, ++STORE, 94425324343296, 94425324888063, ++STORE, 94425324888064, 94425324986367, ++STORE, 94425324986368, 94425325006847, ++STORE, 140382015016960, 140382015188991, ++ERASE, 140382015016960, 140382015016960, ++STORE, 140382015016960, 140382015021055, ++STORE, 140382015021056, 140382015188991, ++ERASE, 140382015021056, 140382015021056, ++STORE, 140382015021056, 140382015143935, ++STORE, 140382015143936, 140382015176703, ++STORE, 140382015176704, 140382015184895, ++STORE, 140382015184896, 140382015188991, ++STORE, 140737045585920, 140737045590015, ++STORE, 140737045573632, 140737045585919, ++STORE, 47250779979776, 47250779987967, ++STORE, 47250779987968, 47250779996159, ++STORE, 47250779996160, 47250780102655, ++STORE, 47250780012544, 47250780102655, ++STORE, 47250779996160, 47250780012543, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780086271, ++STORE, 47250780086272, 47250780102655, ++STORE, 47250780065792, 47250780086271, ++STORE, 47250780012544, 47250780065791, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780065791, ++STORE, 47250780082176, 47250780086271, ++STORE, 47250780065792, 47250780082175, ++ERASE, 47250780065792, 47250780065792, ++STORE, 47250780065792, 47250780082175, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780094464, 47250780094464, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780102656, 47250781941759, ++STORE, 47250780241920, 47250781941759, ++STORE, 47250780102656, 47250780241919, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781900799, ++STORE, 47250781900800, 47250781941759, ++STORE, 47250781585408, 47250781900799, ++STORE, 47250780241920, 47250781585407, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781585407, ++STORE, 47250781896704, 47250781900799, ++STORE, 47250781585408, 47250781896703, ++ERASE, 47250781585408, 47250781585408, ++STORE, 47250781585408, 47250781896703, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781925376, 47250781925376, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781925376, 47250781954047, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781917183, ++STORE, 47250781917184, 47250781925375, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780090367, ++STORE, 47250780090368, 47250780094463, ++ERASE, 94425324986368, 94425324986368, ++STORE, 94425324986368, 94425325002751, ++STORE, 94425325002752, 94425325006847, ++ERASE, 140382015176704, 140382015176704, ++STORE, 140382015176704, 140382015180799, ++STORE, 140382015180800, 140382015184895, ++ERASE, 47250779979776, 47250779979776, ++STORE, 94425351438336, 94425351573503, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736801144832, 140737488351231, ++ERASE, 140736801144832, 140736801144832, ++STORE, 140736801144832, 140736801148927, ++STORE, 94629429358592, 94629430071295, ++ERASE, 94629429358592, 94629429358592, ++STORE, 94629429358592, 94629429407743, ++STORE, 94629429407744, 94629430071295, ++ERASE, 94629429407744, 94629429407744, ++STORE, 94629429407744, 94629429952511, ++STORE, 94629429952512, 94629430050815, ++STORE, 94629430050816, 94629430071295, ++STORE, 139801685483520, 139801685655551, ++ERASE, 139801685483520, 139801685483520, ++STORE, 139801685483520, 139801685487615, ++STORE, 139801685487616, 139801685655551, ++ERASE, 139801685487616, 139801685487616, ++STORE, 139801685487616, 139801685610495, ++STORE, 139801685610496, 139801685643263, ++STORE, 139801685643264, 139801685651455, ++STORE, 139801685651456, 139801685655551, ++STORE, 140736801198080, 140736801202175, ++STORE, 140736801185792, 140736801198079, ++STORE, 47831109513216, 47831109521407, ++STORE, 47831109521408, 47831109529599, ++STORE, 47831109529600, 47831109636095, ++STORE, 47831109545984, 47831109636095, ++STORE, 47831109529600, 47831109545983, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109619711, ++STORE, 47831109619712, 47831109636095, ++STORE, 47831109599232, 47831109619711, ++STORE, 47831109545984, 47831109599231, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109599231, ++STORE, 47831109615616, 47831109619711, ++STORE, 47831109599232, 47831109615615, ++ERASE, 47831109599232, 47831109599232, ++STORE, 47831109599232, 47831109615615, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109627904, 47831109627904, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109636096, 47831111475199, ++STORE, 47831109775360, 47831111475199, ++STORE, 47831109636096, 47831109775359, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111434239, ++STORE, 47831111434240, 47831111475199, ++STORE, 47831111118848, 47831111434239, ++STORE, 47831109775360, 47831111118847, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111118847, ++STORE, 47831111430144, 47831111434239, ++STORE, 47831111118848, 47831111430143, ++ERASE, 47831111118848, 47831111118848, ++STORE, 47831111118848, 47831111430143, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111458816, 47831111458816, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111458816, 47831111487487, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111450623, ++STORE, 47831111450624, 47831111458815, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109623807, ++STORE, 47831109623808, 47831109627903, ++ERASE, 94629430050816, 94629430050816, ++STORE, 94629430050816, 94629430067199, ++STORE, 94629430067200, 94629430071295, ++ERASE, 139801685643264, 139801685643264, ++STORE, 139801685643264, 139801685647359, ++STORE, 139801685647360, 139801685651455, ++ERASE, 47831109513216, 47831109513216, ++STORE, 140737488347136, 140737488351231, ++STORE, 140729419612160, 140737488351231, ++ERASE, 140729419612160, 140729419612160, ++STORE, 140729419612160, 140729419616255, ++STORE, 94443354148864, 94443354861567, ++ERASE, 94443354148864, 94443354148864, ++STORE, 94443354148864, 94443354198015, ++STORE, 94443354198016, 94443354861567, ++ERASE, 94443354198016, 94443354198016, ++STORE, 94443354198016, 94443354742783, ++STORE, 94443354742784, 94443354841087, ++STORE, 94443354841088, 94443354861567, ++STORE, 139741700038656, 139741700210687, ++ERASE, 139741700038656, 139741700038656, ++STORE, 139741700038656, 139741700042751, ++STORE, 139741700042752, 139741700210687, ++ERASE, 139741700042752, 139741700042752, ++STORE, 139741700042752, 139741700165631, ++STORE, 139741700165632, 139741700198399, ++STORE, 139741700198400, 139741700206591, ++STORE, 139741700206592, 139741700210687, ++STORE, 140729420574720, 140729420578815, ++STORE, 140729420562432, 140729420574719, ++STORE, 47891094958080, 47891094966271, ++STORE, 47891094966272, 47891094974463, ++STORE, 47891094974464, 47891095080959, ++STORE, 47891094990848, 47891095080959, ++STORE, 47891094974464, 47891094990847, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095064575, ++STORE, 47891095064576, 47891095080959, ++STORE, 47891095044096, 47891095064575, ++STORE, 47891094990848, 47891095044095, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095044095, ++STORE, 47891095060480, 47891095064575, ++STORE, 47891095044096, 47891095060479, ++ERASE, 47891095044096, 47891095044096, ++STORE, 47891095044096, 47891095060479, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095072768, 47891095072768, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095080960, 47891096920063, ++STORE, 47891095220224, 47891096920063, ++STORE, 47891095080960, 47891095220223, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096879103, ++STORE, 47891096879104, 47891096920063, ++STORE, 47891096563712, 47891096879103, ++STORE, 47891095220224, 47891096563711, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096563711, ++STORE, 47891096875008, 47891096879103, ++STORE, 47891096563712, 47891096875007, ++ERASE, 47891096563712, 47891096563712, ++STORE, 47891096563712, 47891096875007, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096903680, 47891096903680, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096903680, 47891096932351, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096895487, ++STORE, 47891096895488, 47891096903679, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095068671, ++STORE, 47891095068672, 47891095072767, ++ERASE, 94443354841088, 94443354841088, ++STORE, 94443354841088, 94443354857471, ++STORE, 94443354857472, 94443354861567, ++ERASE, 139741700198400, 139741700198400, ++STORE, 139741700198400, 139741700202495, ++STORE, 139741700202496, 139741700206591, ++ERASE, 47891094958080, 47891094958080, ++STORE, 94443360825344, 94443360960511, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722961661952, 140737488351231, ++ERASE, 140722961661952, 140722961661952, ++STORE, 140722961661952, 140722961666047, ++STORE, 94878388944896, 94878389657599, ++ERASE, 94878388944896, 94878388944896, ++STORE, 94878388944896, 94878388994047, ++STORE, 94878388994048, 94878389657599, ++ERASE, 94878388994048, 94878388994048, ++STORE, 94878388994048, 94878389538815, ++STORE, 94878389538816, 94878389637119, ++STORE, 94878389637120, 94878389657599, ++STORE, 140210690056192, 140210690228223, ++ERASE, 140210690056192, 140210690056192, ++STORE, 140210690056192, 140210690060287, ++STORE, 140210690060288, 140210690228223, ++ERASE, 140210690060288, 140210690060288, ++STORE, 140210690060288, 140210690183167, ++STORE, 140210690183168, 140210690215935, ++STORE, 140210690215936, 140210690224127, ++STORE, 140210690224128, 140210690228223, ++STORE, 140722963148800, 140722963152895, ++STORE, 140722963136512, 140722963148799, ++STORE, 47422104940544, 47422104948735, ++STORE, 47422104948736, 47422104956927, ++STORE, 47422104956928, 47422105063423, ++STORE, 47422104973312, 47422105063423, ++STORE, 47422104956928, 47422104973311, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105047039, ++STORE, 47422105047040, 47422105063423, ++STORE, 47422105026560, 47422105047039, ++STORE, 47422104973312, 47422105026559, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105026559, ++STORE, 47422105042944, 47422105047039, ++STORE, 47422105026560, 47422105042943, ++ERASE, 47422105026560, 47422105026560, ++STORE, 47422105026560, 47422105042943, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105055232, 47422105055232, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105063424, 47422106902527, ++STORE, 47422105202688, 47422106902527, ++STORE, 47422105063424, 47422105202687, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106861567, ++STORE, 47422106861568, 47422106902527, ++STORE, 47422106546176, 47422106861567, ++STORE, 47422105202688, 47422106546175, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106546175, ++STORE, 47422106857472, 47422106861567, ++STORE, 47422106546176, 47422106857471, ++ERASE, 47422106546176, 47422106546176, ++STORE, 47422106546176, 47422106857471, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106886144, 47422106886144, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106886144, 47422106914815, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106877951, ++STORE, 47422106877952, 47422106886143, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105051135, ++STORE, 47422105051136, 47422105055231, ++ERASE, 94878389637120, 94878389637120, ++STORE, 94878389637120, 94878389653503, ++STORE, 94878389653504, 94878389657599, ++ERASE, 140210690215936, 140210690215936, ++STORE, 140210690215936, 140210690220031, ++STORE, 140210690220032, 140210690224127, ++ERASE, 47422104940544, 47422104940544, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727690309632, 140737488351231, ++ERASE, 140727690309632, 140727690309632, ++STORE, 140727690309632, 140727690313727, ++STORE, 94121892208640, 94121892921343, ++ERASE, 94121892208640, 94121892208640, ++STORE, 94121892208640, 94121892257791, ++STORE, 94121892257792, 94121892921343, ++ERASE, 94121892257792, 94121892257792, ++STORE, 94121892257792, 94121892802559, ++STORE, 94121892802560, 94121892900863, ++STORE, 94121892900864, 94121892921343, ++STORE, 140662438326272, 140662438498303, ++ERASE, 140662438326272, 140662438326272, ++STORE, 140662438326272, 140662438330367, ++STORE, 140662438330368, 140662438498303, ++ERASE, 140662438330368, 140662438330368, ++STORE, 140662438330368, 140662438453247, ++STORE, 140662438453248, 140662438486015, ++STORE, 140662438486016, 140662438494207, ++STORE, 140662438494208, 140662438498303, ++STORE, 140727690379264, 140727690383359, ++STORE, 140727690366976, 140727690379263, ++STORE, 46970356670464, 46970356678655, ++STORE, 46970356678656, 46970356686847, ++STORE, 46970356686848, 46970356793343, ++STORE, 46970356703232, 46970356793343, ++STORE, 46970356686848, 46970356703231, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356776959, ++STORE, 46970356776960, 46970356793343, ++STORE, 46970356756480, 46970356776959, ++STORE, 46970356703232, 46970356756479, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356756479, ++STORE, 46970356772864, 46970356776959, ++STORE, 46970356756480, 46970356772863, ++ERASE, 46970356756480, 46970356756480, ++STORE, 46970356756480, 46970356772863, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356785152, 46970356785152, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356793344, 46970358632447, ++STORE, 46970356932608, 46970358632447, ++STORE, 46970356793344, 46970356932607, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358591487, ++STORE, 46970358591488, 46970358632447, ++STORE, 46970358276096, 46970358591487, ++STORE, 46970356932608, 46970358276095, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358276095, ++STORE, 46970358587392, 46970358591487, ++STORE, 46970358276096, 46970358587391, ++ERASE, 46970358276096, 46970358276096, ++STORE, 46970358276096, 46970358587391, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358616064, 46970358616064, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358616064, 46970358644735, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358607871, ++STORE, 46970358607872, 46970358616063, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356781055, ++STORE, 46970356781056, 46970356785151, ++ERASE, 94121892900864, 94121892900864, ++STORE, 94121892900864, 94121892917247, ++STORE, 94121892917248, 94121892921343, ++ERASE, 140662438486016, 140662438486016, ++STORE, 140662438486016, 140662438490111, ++STORE, 140662438490112, 140662438494207, ++ERASE, 46970356670464, 46970356670464, ++STORE, 94121898610688, 94121898745855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737189351424, 140737488351231, ++ERASE, 140737189351424, 140737189351424, ++STORE, 140737189351424, 140737189355519, ++STORE, 93847948832768, 93847949545471, ++ERASE, 93847948832768, 93847948832768, ++STORE, 93847948832768, 93847948881919, ++STORE, 93847948881920, 93847949545471, ++ERASE, 93847948881920, 93847948881920, ++STORE, 93847948881920, 93847949426687, ++STORE, 93847949426688, 93847949524991, ++STORE, 93847949524992, 93847949545471, ++STORE, 139698989985792, 139698990157823, ++ERASE, 139698989985792, 139698989985792, ++STORE, 139698989985792, 139698989989887, ++STORE, 139698989989888, 139698990157823, ++ERASE, 139698989989888, 139698989989888, ++STORE, 139698989989888, 139698990112767, ++STORE, 139698990112768, 139698990145535, ++STORE, 139698990145536, 139698990153727, ++STORE, 139698990153728, 139698990157823, ++STORE, 140737189744640, 140737189748735, ++STORE, 140737189732352, 140737189744639, ++STORE, 47933805010944, 47933805019135, ++STORE, 47933805019136, 47933805027327, ++STORE, 47933805027328, 47933805133823, ++STORE, 47933805043712, 47933805133823, ++STORE, 47933805027328, 47933805043711, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805117439, ++STORE, 47933805117440, 47933805133823, ++STORE, 47933805096960, 47933805117439, ++STORE, 47933805043712, 47933805096959, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805096959, ++STORE, 47933805113344, 47933805117439, ++STORE, 47933805096960, 47933805113343, ++ERASE, 47933805096960, 47933805096960, ++STORE, 47933805096960, 47933805113343, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805125632, 47933805125632, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805133824, 47933806972927, ++STORE, 47933805273088, 47933806972927, ++STORE, 47933805133824, 47933805273087, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806931967, ++STORE, 47933806931968, 47933806972927, ++STORE, 47933806616576, 47933806931967, ++STORE, 47933805273088, 47933806616575, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806616575, ++STORE, 47933806927872, 47933806931967, ++STORE, 47933806616576, 47933806927871, ++ERASE, 47933806616576, 47933806616576, ++STORE, 47933806616576, 47933806927871, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806956544, 47933806956544, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806956544, 47933806985215, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806948351, ++STORE, 47933806948352, 47933806956543, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805121535, ++STORE, 47933805121536, 47933805125631, ++ERASE, 93847949524992, 93847949524992, ++STORE, 93847949524992, 93847949541375, ++STORE, 93847949541376, 93847949545471, ++ERASE, 139698990145536, 139698990145536, ++STORE, 139698990145536, 139698990149631, ++STORE, 139698990149632, 139698990153727, ++ERASE, 47933805010944, 47933805010944, ++STORE, 140737488347136, 140737488351231, ++STORE, 140725553991680, 140737488351231, ++ERASE, 140725553991680, 140725553991680, ++STORE, 140725553991680, 140725553995775, ++STORE, 93980056248320, 93980056961023, ++ERASE, 93980056248320, 93980056248320, ++STORE, 93980056248320, 93980056297471, ++STORE, 93980056297472, 93980056961023, ++ERASE, 93980056297472, 93980056297472, ++STORE, 93980056297472, 93980056842239, ++STORE, 93980056842240, 93980056940543, ++STORE, 93980056940544, 93980056961023, ++STORE, 140146588971008, 140146589143039, ++ERASE, 140146588971008, 140146588971008, ++STORE, 140146588971008, 140146588975103, ++STORE, 140146588975104, 140146589143039, ++ERASE, 140146588975104, 140146588975104, ++STORE, 140146588975104, 140146589097983, ++STORE, 140146589097984, 140146589130751, ++STORE, 140146589130752, 140146589138943, ++STORE, 140146589138944, 140146589143039, ++STORE, 140725554860032, 140725554864127, ++STORE, 140725554847744, 140725554860031, ++STORE, 47486206025728, 47486206033919, ++STORE, 47486206033920, 47486206042111, ++STORE, 47486206042112, 47486206148607, ++STORE, 47486206058496, 47486206148607, ++STORE, 47486206042112, 47486206058495, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206132223, ++STORE, 47486206132224, 47486206148607, ++STORE, 47486206111744, 47486206132223, ++STORE, 47486206058496, 47486206111743, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206111743, ++STORE, 47486206128128, 47486206132223, ++STORE, 47486206111744, 47486206128127, ++ERASE, 47486206111744, 47486206111744, ++STORE, 47486206111744, 47486206128127, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206140416, 47486206140416, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206148608, 47486207987711, ++STORE, 47486206287872, 47486207987711, ++STORE, 47486206148608, 47486206287871, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207946751, ++STORE, 47486207946752, 47486207987711, ++STORE, 47486207631360, 47486207946751, ++STORE, 47486206287872, 47486207631359, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207631359, ++STORE, 47486207942656, 47486207946751, ++STORE, 47486207631360, 47486207942655, ++ERASE, 47486207631360, 47486207631360, ++STORE, 47486207631360, 47486207942655, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207971328, 47486207971328, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207971328, 47486207999999, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207963135, ++STORE, 47486207963136, 47486207971327, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206136319, ++STORE, 47486206136320, 47486206140415, ++ERASE, 93980056940544, 93980056940544, ++STORE, 93980056940544, 93980056956927, ++STORE, 93980056956928, 93980056961023, ++ERASE, 140146589130752, 140146589130752, ++STORE, 140146589130752, 140146589134847, ++STORE, 140146589134848, 140146589138943, ++ERASE, 47486206025728, 47486206025728, ++STORE, 93980070006784, 93980070141951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727334776832, 140737488351231, ++ERASE, 140727334776832, 140727334776832, ++STORE, 140727334776832, 140727334780927, ++STORE, 94049747247104, 94049747959807, ++ERASE, 94049747247104, 94049747247104, ++STORE, 94049747247104, 94049747296255, ++STORE, 94049747296256, 94049747959807, ++ERASE, 94049747296256, 94049747296256, ++STORE, 94049747296256, 94049747841023, ++STORE, 94049747841024, 94049747939327, ++STORE, 94049747939328, 94049747959807, ++STORE, 140227307216896, 140227307388927, ++ERASE, 140227307216896, 140227307216896, ++STORE, 140227307216896, 140227307220991, ++STORE, 140227307220992, 140227307388927, ++ERASE, 140227307220992, 140227307220992, ++STORE, 140227307220992, 140227307343871, ++STORE, 140227307343872, 140227307376639, ++STORE, 140227307376640, 140227307384831, ++STORE, 140227307384832, 140227307388927, ++STORE, 140727335337984, 140727335342079, ++STORE, 140727335325696, 140727335337983, ++STORE, 47405487779840, 47405487788031, ++STORE, 47405487788032, 47405487796223, ++STORE, 47405487796224, 47405487902719, ++STORE, 47405487812608, 47405487902719, ++STORE, 47405487796224, 47405487812607, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487886335, ++STORE, 47405487886336, 47405487902719, ++STORE, 47405487865856, 47405487886335, ++STORE, 47405487812608, 47405487865855, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487865855, ++STORE, 47405487882240, 47405487886335, ++STORE, 47405487865856, 47405487882239, ++ERASE, 47405487865856, 47405487865856, ++STORE, 47405487865856, 47405487882239, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487894528, 47405487894528, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487902720, 47405489741823, ++STORE, 47405488041984, 47405489741823, ++STORE, 47405487902720, 47405488041983, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489700863, ++STORE, 47405489700864, 47405489741823, ++STORE, 47405489385472, 47405489700863, ++STORE, 47405488041984, 47405489385471, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489385471, ++STORE, 47405489696768, 47405489700863, ++STORE, 47405489385472, 47405489696767, ++ERASE, 47405489385472, 47405489385472, ++STORE, 47405489385472, 47405489696767, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489725440, 47405489725440, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489725440, 47405489754111, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489717247, ++STORE, 47405489717248, 47405489725439, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487890431, ++STORE, 47405487890432, 47405487894527, ++ERASE, 94049747939328, 94049747939328, ++STORE, 94049747939328, 94049747955711, ++STORE, 94049747955712, 94049747959807, ++ERASE, 140227307376640, 140227307376640, ++STORE, 140227307376640, 140227307380735, ++STORE, 140227307380736, 140227307384831, ++ERASE, 47405487779840, 47405487779840, ++STORE, 94049758810112, 94049758945279, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727079718912, 140737488351231, ++ERASE, 140727079718912, 140727079718912, ++STORE, 140727079718912, 140727079723007, ++STORE, 94250996527104, 94250997239807, ++ERASE, 94250996527104, 94250996527104, ++STORE, 94250996527104, 94250996576255, ++STORE, 94250996576256, 94250997239807, ++ERASE, 94250996576256, 94250996576256, ++STORE, 94250996576256, 94250997121023, ++STORE, 94250997121024, 94250997219327, ++STORE, 94250997219328, 94250997239807, ++STORE, 140060022587392, 140060022759423, ++ERASE, 140060022587392, 140060022587392, ++STORE, 140060022587392, 140060022591487, ++STORE, 140060022591488, 140060022759423, ++ERASE, 140060022591488, 140060022591488, ++STORE, 140060022591488, 140060022714367, ++STORE, 140060022714368, 140060022747135, ++STORE, 140060022747136, 140060022755327, ++STORE, 140060022755328, 140060022759423, ++STORE, 140727079788544, 140727079792639, ++STORE, 140727079776256, 140727079788543, ++/* this next one caused issues when lowering the efficiency */ ++STORE, 47572772409344, 47572772417535, ++STORE, 47572772417536, 47572772425727, ++STORE, 47572772425728, 47572772532223, ++STORE, 47572772442112, 47572772532223, ++STORE, 47572772425728, 47572772442111, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772515839, ++STORE, 47572772515840, 47572772532223, ++STORE, 47572772495360, 47572772515839, ++STORE, 47572772442112, 47572772495359, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772495359, ++STORE, 47572772511744, 47572772515839, ++STORE, 47572772495360, 47572772511743, ++ERASE, 47572772495360, 47572772495360, ++STORE, 47572772495360, 47572772511743, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772524032, 47572772524032, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772532224, 47572774371327, ++STORE, 47572772671488, 47572774371327, ++STORE, 47572772532224, 47572772671487, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774330367, ++STORE, 47572774330368, 47572774371327, ++STORE, 47572774014976, 47572774330367, ++STORE, 47572772671488, 47572774014975, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774014975, ++STORE, 47572774326272, 47572774330367, ++STORE, 47572774014976, 47572774326271, ++ERASE, 47572774014976, 47572774014976, ++STORE, 47572774014976, 47572774326271, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774354944, 47572774354944, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774354944, 47572774383615, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774346751, ++STORE, 47572774346752, 47572774354943, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772519935, ++STORE, 47572772519936, 47572772524031, ++ERASE, 94250997219328, 94250997219328, ++STORE, 94250997219328, 94250997235711, ++STORE, 94250997235712, 94250997239807, ++ERASE, 140060022747136, 140060022747136, ++STORE, 140060022747136, 140060022751231, ++STORE, 140060022751232, 140060022755327, ++ERASE, 47572772409344, 47572772409344, ++STORE, 94251018305536, 94251018440703, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730012389376, 140737488351231, ++ERASE, 140730012389376, 140730012389376, ++STORE, 140730012389376, 140730012393471, ++STORE, 94382607675392, 94382607695871, ++ERASE, 94382607675392, 94382607675392, ++STORE, 94382607675392, 94382607679487, ++STORE, 94382607679488, 94382607695871, ++ERASE, 94382607679488, 94382607679488, ++STORE, 94382607679488, 94382607683583, ++STORE, 94382607683584, 94382607687679, ++STORE, 94382607687680, 94382607695871, ++STORE, 140252451454976, 140252451627007, ++ERASE, 140252451454976, 140252451454976, ++STORE, 140252451454976, 140252451459071, ++STORE, 140252451459072, 140252451627007, ++ERASE, 140252451459072, 140252451459072, ++STORE, 140252451459072, 140252451581951, ++STORE, 140252451581952, 140252451614719, ++STORE, 140252451614720, 140252451622911, ++STORE, 140252451622912, 140252451627007, ++STORE, 140730013548544, 140730013552639, ++STORE, 140730013536256, 140730013548543, ++STORE, 47380343541760, 47380343549951, ++STORE, 47380343549952, 47380343558143, ++STORE, 47380343558144, 47380345397247, ++STORE, 47380343697408, 47380345397247, ++STORE, 47380343558144, 47380343697407, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345356287, ++STORE, 47380345356288, 47380345397247, ++STORE, 47380345040896, 47380345356287, ++STORE, 47380343697408, 47380345040895, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345040895, ++STORE, 47380345352192, 47380345356287, ++STORE, 47380345040896, 47380345352191, ++ERASE, 47380345040896, 47380345040896, ++STORE, 47380345040896, 47380345352191, ++STORE, 47380345380864, 47380345397247, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345380864, 47380345380864, ++STORE, 47380345380864, 47380345397247, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345372671, ++STORE, 47380345372672, 47380345380863, ++ERASE, 94382607687680, 94382607687680, ++STORE, 94382607687680, 94382607691775, ++STORE, 94382607691776, 94382607695871, ++ERASE, 140252451614720, 140252451614720, ++STORE, 140252451614720, 140252451618815, ++STORE, 140252451618816, 140252451622911, ++ERASE, 47380343541760, 47380343541760, ++STORE, 94382626803712, 94382626938879, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730900271104, 140737488351231, ++ERASE, 140730900271104, 140730900271104, ++STORE, 140730900271104, 140730900275199, ++STORE, 93855478120448, 93855478337535, ++ERASE, 93855478120448, 93855478120448, ++STORE, 93855478120448, 93855478198271, ++STORE, 93855478198272, 93855478337535, ++ERASE, 93855478198272, 93855478198272, ++STORE, 93855478198272, 93855478243327, ++STORE, 93855478243328, 93855478288383, ++STORE, 93855478288384, 93855478337535, ++STORE, 140092686573568, 140092686745599, ++ERASE, 140092686573568, 140092686573568, ++STORE, 140092686573568, 140092686577663, ++STORE, 140092686577664, 140092686745599, ++ERASE, 140092686577664, 140092686577664, ++STORE, 140092686577664, 140092686700543, ++STORE, 140092686700544, 140092686733311, ++STORE, 140092686733312, 140092686741503, ++STORE, 140092686741504, 140092686745599, ++STORE, 140730900537344, 140730900541439, ++STORE, 140730900525056, 140730900537343, ++STORE, 47540108423168, 47540108431359, ++STORE, 47540108431360, 47540108439551, ++STORE, 47540108439552, 47540110278655, ++STORE, 47540108578816, 47540110278655, ++STORE, 47540108439552, 47540108578815, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540110237695, ++STORE, 47540110237696, 47540110278655, ++STORE, 47540109922304, 47540110237695, ++STORE, 47540108578816, 47540109922303, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540109922303, ++STORE, 47540110233600, 47540110237695, ++STORE, 47540109922304, 47540110233599, ++ERASE, 47540109922304, 47540109922304, ++STORE, 47540109922304, 47540110233599, ++STORE, 47540110262272, 47540110278655, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110262272, 47540110262272, ++STORE, 47540110262272, 47540110278655, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110254079, ++STORE, 47540110254080, 47540110262271, ++ERASE, 93855478288384, 93855478288384, ++STORE, 93855478288384, 93855478333439, ++STORE, 93855478333440, 93855478337535, ++ERASE, 140092686733312, 140092686733312, ++STORE, 140092686733312, 140092686737407, ++STORE, 140092686737408, 140092686741503, ++ERASE, 47540108423168, 47540108423168, ++STORE, 93855492222976, 93855492358143, ++STORE, 93855492222976, 93855492493311, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733498146816, 140737488351231, ++ERASE, 140733498146816, 140733498146816, ++STORE, 140733498146816, 140733498150911, ++STORE, 94170739654656, 94170740367359, ++ERASE, 94170739654656, 94170739654656, ++STORE, 94170739654656, 94170739703807, ++STORE, 94170739703808, 94170740367359, ++ERASE, 94170739703808, 94170739703808, ++STORE, 94170739703808, 94170740248575, ++STORE, 94170740248576, 94170740346879, ++STORE, 94170740346880, 94170740367359, ++STORE, 140024788877312, 140024789049343, ++ERASE, 140024788877312, 140024788877312, ++STORE, 140024788877312, 140024788881407, ++STORE, 140024788881408, 140024789049343, ++ERASE, 140024788881408, 140024788881408, ++STORE, 140024788881408, 140024789004287, ++STORE, 140024789004288, 140024789037055, ++STORE, 140024789037056, 140024789045247, ++STORE, 140024789045248, 140024789049343, ++STORE, 140733499023360, 140733499027455, ++STORE, 140733499011072, 140733499023359, ++STORE, 47608006119424, 47608006127615, ++STORE, 47608006127616, 47608006135807, ++STORE, 47608006135808, 47608006242303, ++STORE, 47608006152192, 47608006242303, ++STORE, 47608006135808, 47608006152191, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006225919, ++STORE, 47608006225920, 47608006242303, ++STORE, 47608006205440, 47608006225919, ++STORE, 47608006152192, 47608006205439, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006205439, ++STORE, 47608006221824, 47608006225919, ++STORE, 47608006205440, 47608006221823, ++ERASE, 47608006205440, 47608006205440, ++STORE, 47608006205440, 47608006221823, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006234112, 47608006234112, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006242304, 47608008081407, ++STORE, 47608006381568, 47608008081407, ++STORE, 47608006242304, 47608006381567, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608008040447, ++STORE, 47608008040448, 47608008081407, ++STORE, 47608007725056, 47608008040447, ++STORE, 47608006381568, 47608007725055, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608007725055, ++STORE, 47608008036352, 47608008040447, ++STORE, 47608007725056, 47608008036351, ++ERASE, 47608007725056, 47608007725056, ++STORE, 47608007725056, 47608008036351, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008065024, 47608008065024, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008065024, 47608008093695, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008056831, ++STORE, 47608008056832, 47608008065023, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006230015, ++STORE, 47608006230016, 47608006234111, ++ERASE, 94170740346880, 94170740346880, ++STORE, 94170740346880, 94170740363263, ++STORE, 94170740363264, 94170740367359, ++ERASE, 140024789037056, 140024789037056, ++STORE, 140024789037056, 140024789041151, ++STORE, 140024789041152, 140024789045247, ++ERASE, 47608006119424, 47608006119424, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730264326144, 140737488351231, ++ERASE, 140730264326144, 140730264326144, ++STORE, 140730264326144, 140730264330239, ++STORE, 94653216407552, 94653217120255, ++ERASE, 94653216407552, 94653216407552, ++STORE, 94653216407552, 94653216456703, ++STORE, 94653216456704, 94653217120255, ++ERASE, 94653216456704, 94653216456704, ++STORE, 94653216456704, 94653217001471, ++STORE, 94653217001472, 94653217099775, ++STORE, 94653217099776, 94653217120255, ++STORE, 140103617011712, 140103617183743, ++ERASE, 140103617011712, 140103617011712, ++STORE, 140103617011712, 140103617015807, ++STORE, 140103617015808, 140103617183743, ++ERASE, 140103617015808, 140103617015808, ++STORE, 140103617015808, 140103617138687, ++STORE, 140103617138688, 140103617171455, ++STORE, 140103617171456, 140103617179647, ++STORE, 140103617179648, 140103617183743, ++STORE, 140730265427968, 140730265432063, ++STORE, 140730265415680, 140730265427967, ++STORE, 47529177985024, 47529177993215, ++STORE, 47529177993216, 47529178001407, ++STORE, 47529178001408, 47529178107903, ++STORE, 47529178017792, 47529178107903, ++STORE, 47529178001408, 47529178017791, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178091519, ++STORE, 47529178091520, 47529178107903, ++STORE, 47529178071040, 47529178091519, ++STORE, 47529178017792, 47529178071039, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178071039, ++STORE, 47529178087424, 47529178091519, ++STORE, 47529178071040, 47529178087423, ++ERASE, 47529178071040, 47529178071040, ++STORE, 47529178071040, 47529178087423, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178099712, 47529178099712, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178107904, 47529179947007, ++STORE, 47529178247168, 47529179947007, ++STORE, 47529178107904, 47529178247167, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179906047, ++STORE, 47529179906048, 47529179947007, ++STORE, 47529179590656, 47529179906047, ++STORE, 47529178247168, 47529179590655, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179590655, ++STORE, 47529179901952, 47529179906047, ++STORE, 47529179590656, 47529179901951, ++ERASE, 47529179590656, 47529179590656, ++STORE, 47529179590656, 47529179901951, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179930624, 47529179930624, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179930624, 47529179959295, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179922431, ++STORE, 47529179922432, 47529179930623, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178095615, ++STORE, 47529178095616, 47529178099711, ++ERASE, 94653217099776, 94653217099776, ++STORE, 94653217099776, 94653217116159, ++STORE, 94653217116160, 94653217120255, ++ERASE, 140103617171456, 140103617171456, ++STORE, 140103617171456, 140103617175551, ++STORE, 140103617175552, 140103617179647, ++ERASE, 47529177985024, 47529177985024, ++STORE, 94653241135104, 94653241270271, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736284549120, 140737488351231, ++ERASE, 140736284549120, 140736284549120, ++STORE, 140736284549120, 140736284553215, ++STORE, 93963663822848, 93963664506879, ++ERASE, 93963663822848, 93963663822848, ++STORE, 93963663822848, 93963663884287, ++STORE, 93963663884288, 93963664506879, ++ERASE, 93963663884288, 93963663884288, ++STORE, 93963663884288, 93963664240639, ++STORE, 93963664240640, 93963664379903, ++STORE, 93963664379904, 93963664506879, ++STORE, 140450188439552, 140450188611583, ++ERASE, 140450188439552, 140450188439552, ++STORE, 140450188439552, 140450188443647, ++STORE, 140450188443648, 140450188611583, ++ERASE, 140450188443648, 140450188443648, ++STORE, 140450188443648, 140450188566527, ++STORE, 140450188566528, 140450188599295, ++STORE, 140450188599296, 140450188607487, ++STORE, 140450188607488, 140450188611583, ++STORE, 140736284577792, 140736284581887, ++STORE, 140736284565504, 140736284577791, ++STORE, 47182606557184, 47182606565375, ++STORE, 47182606565376, 47182606573567, ++STORE, 47182606573568, 47182608412671, ++STORE, 47182606712832, 47182608412671, ++STORE, 47182606573568, 47182606712831, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608371711, ++STORE, 47182608371712, 47182608412671, ++STORE, 47182608056320, 47182608371711, ++STORE, 47182606712832, 47182608056319, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608056319, ++STORE, 47182608367616, 47182608371711, ++STORE, 47182608056320, 47182608367615, ++ERASE, 47182608056320, 47182608056320, ++STORE, 47182608056320, 47182608367615, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608396288, 47182608396288, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608412672, 47182608523263, ++STORE, 47182608429056, 47182608523263, ++STORE, 47182608412672, 47182608429055, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608515071, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608490496, 47182608515071, ++STORE, 47182608429056, 47182608490495, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608490495, ++STORE, 47182608510976, 47182608515071, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608490496, 47182608490496, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608523264, 47182608568319, ++ERASE, 47182608523264, 47182608523264, ++STORE, 47182608523264, 47182608531455, ++STORE, 47182608531456, 47182608568319, ++STORE, 47182608551936, 47182608568319, ++STORE, 47182608531456, 47182608551935, ++ERASE, 47182608531456, 47182608531456, ++STORE, 47182608531456, 47182608551935, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608551936, 47182608560127, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608568319, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608560127, ++STORE, 47182608560128, 47182608568319, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608568320, 47182608916479, ++STORE, 47182608609280, 47182608916479, ++STORE, 47182608568320, 47182608609279, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608891903, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608822272, 47182608891903, ++STORE, 47182608609280, 47182608822271, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608822271, ++STORE, 47182608887808, 47182608891903, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608822272, 47182608822272, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608916480, 47182611177471, ++STORE, 47182609068032, 47182611177471, ++STORE, 47182608916480, 47182609068031, ++ERASE, 47182609068032, 47182609068032, ++STORE, 47182609068032, 47182611161087, ++STORE, 47182611161088, 47182611177471, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611169280, 47182611169280, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611177472, 47182611312639, ++ERASE, 47182611177472, 47182611177472, ++STORE, 47182611177472, 47182611202047, ++STORE, 47182611202048, 47182611312639, ++STORE, 47182611263488, 47182611312639, ++STORE, 47182611202048, 47182611263487, ++ERASE, 47182611202048, 47182611202048, ++STORE, 47182611202048, 47182611263487, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611263488, 47182611288063, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611312639, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611288063, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611296256, 47182611296256, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611296256, 47182611320831, ++STORE, 47182611320832, 47182611484671, ++ERASE, 47182611320832, 47182611320832, ++STORE, 47182611320832, 47182611333119, ++STORE, 47182611333120, 47182611484671, ++STORE, 47182611431424, 47182611484671, ++STORE, 47182611333120, 47182611431423, ++ERASE, 47182611333120, 47182611333120, ++STORE, 47182611333120, 47182611431423, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611431424, 47182611476479, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611484671, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611476479, ++STORE, 47182611476480, 47182611484671, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611484672, 47182612082687, ++STORE, 47182611603456, 47182612082687, ++STORE, 47182611484672, 47182611603455, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182612029439, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182611918848, 47182612029439, ++STORE, 47182611603456, 47182611918847, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182611918847, ++STORE, 47182612025344, 47182612029439, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182611918848, 47182611918848, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182612082688, 47182615134207, ++STORE, 47182612627456, 47182615134207, ++STORE, 47182612082688, 47182612627455, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614913023, ++STORE, 47182614913024, 47182615134207, ++STORE, 47182614323200, 47182614913023, ++STORE, 47182612627456, 47182614323199, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614323199, ++STORE, 47182614908928, 47182614913023, ++STORE, 47182614323200, 47182614908927, ++ERASE, 47182614323200, 47182614323200, ++STORE, 47182614323200, 47182614908927, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182615117824, 47182615117824, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182615134208, 47182615166975, ++ERASE, 47182615134208, 47182615134208, ++STORE, 47182615134208, 47182615142399, ++STORE, 47182615142400, 47182615166975, ++STORE, 47182615154688, 47182615166975, ++STORE, 47182615142400, 47182615154687, ++ERASE, 47182615142400, 47182615142400, ++STORE, 47182615142400, 47182615154687, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615154688, 47182615158783, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615166975, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615158783, ++STORE, 47182615158784, 47182615166975, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615166976, 47182615203839, ++ERASE, 47182615166976, 47182615166976, ++STORE, 47182615166976, 47182615175167, ++STORE, 47182615175168, 47182615203839, ++STORE, 47182615191552, 47182615203839, ++STORE, 47182615175168, 47182615191551, ++ERASE, 47182615175168, 47182615175168, ++STORE, 47182615175168, 47182615191551, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615191552, 47182615195647, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615203839, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615195647, ++STORE, 47182615195648, 47182615203839, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615203840, 47182615678975, ++ERASE, 47182615203840, 47182615203840, ++STORE, 47182615203840, 47182615212031, ++STORE, 47182615212032, 47182615678975, ++STORE, 47182615547904, 47182615678975, ++STORE, 47182615212032, 47182615547903, ++ERASE, 47182615212032, 47182615212032, ++STORE, 47182615212032, 47182615547903, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615547904, 47182615670783, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615678975, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615670783, ++STORE, 47182615670784, 47182615678975, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615678976, 47182615687167, ++STORE, 47182615687168, 47182615707647, ++ERASE, 47182615687168, 47182615687168, ++STORE, 47182615687168, 47182615691263, ++STORE, 47182615691264, 47182615707647, ++STORE, 47182615695360, 47182615707647, ++STORE, 47182615691264, 47182615695359, ++ERASE, 47182615691264, 47182615691264, ++STORE, 47182615691264, 47182615695359, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615695360, 47182615699455, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615707647, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615699455, ++STORE, 47182615699456, 47182615707647, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615707648, 47182615715839, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608388095, ++STORE, 47182608388096, 47182608396287, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615703551, ++STORE, 47182615703552, 47182615707647, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611292159, ++STORE, 47182611292160, 47182611296255, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615674879, ++STORE, 47182615674880, 47182615678975, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615199743, ++STORE, 47182615199744, 47182615203839, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615162879, ++STORE, 47182615162880, 47182615166975, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615109631, ++STORE, 47182615109632, 47182615117823, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612066303, ++STORE, 47182612066304, 47182612082687, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611480575, ++STORE, 47182611480576, 47182611484671, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611165183, ++STORE, 47182611165184, 47182611169279, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608912383, ++STORE, 47182608912384, 47182608916479, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608564223, ++STORE, 47182608564224, 47182608568319, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608519167, ++STORE, 47182608519168, 47182608523263, ++ERASE, 93963664379904, 93963664379904, ++STORE, 93963664379904, 93963664502783, ++STORE, 93963664502784, 93963664506879, ++ERASE, 140450188599296, 140450188599296, ++STORE, 140450188599296, 140450188603391, ++STORE, 140450188603392, 140450188607487, ++ERASE, 47182606557184, 47182606557184, ++STORE, 93963694723072, 93963694858239, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730313261056, 140737488351231, ++ERASE, 140730313261056, 140730313261056, ++STORE, 140730313261056, 140730313265151, ++STORE, 94386579017728, 94386579697663, ++ERASE, 94386579017728, 94386579017728, ++STORE, 94386579017728, 94386579083263, ++STORE, 94386579083264, 94386579697663, ++ERASE, 94386579083264, 94386579083264, ++STORE, 94386579083264, 94386579431423, ++STORE, 94386579431424, 94386579570687, ++STORE, 94386579570688, 94386579697663, ++STORE, 140124810838016, 140124811010047, ++ERASE, 140124810838016, 140124810838016, ++STORE, 140124810838016, 140124810842111, ++STORE, 140124810842112, 140124811010047, ++ERASE, 140124810842112, 140124810842112, ++STORE, 140124810842112, 140124810964991, ++STORE, 140124810964992, 140124810997759, ++STORE, 140124810997760, 140124811005951, ++STORE, 140124811005952, 140124811010047, ++STORE, 140730313601024, 140730313605119, ++STORE, 140730313588736, 140730313601023, ++STORE, 47507984158720, 47507984166911, ++STORE, 47507984166912, 47507984175103, ++STORE, 47507984175104, 47507986014207, ++STORE, 47507984314368, 47507986014207, ++STORE, 47507984175104, 47507984314367, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985973247, ++STORE, 47507985973248, 47507986014207, ++STORE, 47507985657856, 47507985973247, ++STORE, 47507984314368, 47507985657855, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985657855, ++STORE, 47507985969152, 47507985973247, ++STORE, 47507985657856, 47507985969151, ++ERASE, 47507985657856, 47507985657856, ++STORE, 47507985657856, 47507985969151, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985997824, 47507985997824, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507986014208, 47507986124799, ++STORE, 47507986030592, 47507986124799, ++STORE, 47507986014208, 47507986030591, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986116607, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986092032, 47507986116607, ++STORE, 47507986030592, 47507986092031, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986092031, ++STORE, 47507986112512, 47507986116607, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986092032, 47507986092032, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986124800, 47507986169855, ++ERASE, 47507986124800, 47507986124800, ++STORE, 47507986124800, 47507986132991, ++STORE, 47507986132992, 47507986169855, ++STORE, 47507986153472, 47507986169855, ++STORE, 47507986132992, 47507986153471, ++ERASE, 47507986132992, 47507986132992, ++STORE, 47507986132992, 47507986153471, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986153472, 47507986161663, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986169855, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986161663, ++STORE, 47507986161664, 47507986169855, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986169856, 47507986518015, ++STORE, 47507986210816, 47507986518015, ++STORE, 47507986169856, 47507986210815, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986493439, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986423808, 47507986493439, ++STORE, 47507986210816, 47507986423807, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986423807, ++STORE, 47507986489344, 47507986493439, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986423808, 47507986423808, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986518016, 47507988779007, ++STORE, 47507986669568, 47507988779007, ++STORE, 47507986518016, 47507986669567, ++ERASE, 47507986669568, 47507986669568, ++STORE, 47507986669568, 47507988762623, ++STORE, 47507988762624, 47507988779007, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988770816, 47507988770816, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988779008, 47507988914175, ++ERASE, 47507988779008, 47507988779008, ++STORE, 47507988779008, 47507988803583, ++STORE, 47507988803584, 47507988914175, ++STORE, 47507988865024, 47507988914175, ++STORE, 47507988803584, 47507988865023, ++ERASE, 47507988803584, 47507988803584, ++STORE, 47507988803584, 47507988865023, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988865024, 47507988889599, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988914175, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988889599, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988897792, 47507988897792, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988897792, 47507988922367, ++STORE, 47507988922368, 47507989086207, ++ERASE, 47507988922368, 47507988922368, ++STORE, 47507988922368, 47507988934655, ++STORE, 47507988934656, 47507989086207, ++STORE, 47507989032960, 47507989086207, ++STORE, 47507988934656, 47507989032959, ++ERASE, 47507988934656, 47507988934656, ++STORE, 47507988934656, 47507989032959, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989032960, 47507989078015, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989086207, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989078015, ++STORE, 47507989078016, 47507989086207, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989086208, 47507989684223, ++STORE, 47507989204992, 47507989684223, ++STORE, 47507989086208, 47507989204991, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989630975, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989520384, 47507989630975, ++STORE, 47507989204992, 47507989520383, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989520383, ++STORE, 47507989626880, 47507989630975, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989520384, 47507989520384, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989684224, 47507992735743, ++STORE, 47507990228992, 47507992735743, ++STORE, 47507989684224, 47507990228991, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507992514559, ++STORE, 47507992514560, 47507992735743, ++STORE, 47507991924736, 47507992514559, ++STORE, 47507990228992, 47507991924735, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507991924735, ++STORE, 47507992510464, 47507992514559, ++STORE, 47507991924736, 47507992510463, ++ERASE, 47507991924736, 47507991924736, ++STORE, 47507991924736, 47507992510463, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992719360, 47507992719360, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992735744, 47507992768511, ++ERASE, 47507992735744, 47507992735744, ++STORE, 47507992735744, 47507992743935, ++STORE, 47507992743936, 47507992768511, ++STORE, 47507992756224, 47507992768511, ++STORE, 47507992743936, 47507992756223, ++ERASE, 47507992743936, 47507992743936, ++STORE, 47507992743936, 47507992756223, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992756224, 47507992760319, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992768511, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992760319, ++STORE, 47507992760320, 47507992768511, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992768512, 47507992805375, ++ERASE, 47507992768512, 47507992768512, ++STORE, 47507992768512, 47507992776703, ++STORE, 47507992776704, 47507992805375, ++STORE, 47507992793088, 47507992805375, ++STORE, 47507992776704, 47507992793087, ++ERASE, 47507992776704, 47507992776704, ++STORE, 47507992776704, 47507992793087, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992793088, 47507992797183, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992805375, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992797183, ++STORE, 47507992797184, 47507992805375, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992805376, 47507993280511, ++ERASE, 47507992805376, 47507992805376, ++STORE, 47507992805376, 47507992813567, ++STORE, 47507992813568, 47507993280511, ++STORE, 47507993149440, 47507993280511, ++STORE, 47507992813568, 47507993149439, ++ERASE, 47507992813568, 47507992813568, ++STORE, 47507992813568, 47507993149439, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993149440, 47507993272319, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993280511, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993272319, ++STORE, 47507993272320, 47507993280511, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993280512, 47507993288703, ++STORE, 47507993288704, 47507993309183, ++ERASE, 47507993288704, 47507993288704, ++STORE, 47507993288704, 47507993292799, ++STORE, 47507993292800, 47507993309183, ++STORE, 47507993296896, 47507993309183, ++STORE, 47507993292800, 47507993296895, ++ERASE, 47507993292800, 47507993292800, ++STORE, 47507993292800, 47507993296895, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993296896, 47507993300991, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993309183, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993300991, ++STORE, 47507993300992, 47507993309183, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993309184, 47507993317375, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985989631, ++STORE, 47507985989632, 47507985997823, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993305087, ++STORE, 47507993305088, 47507993309183, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988893695, ++STORE, 47507988893696, 47507988897791, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993276415, ++STORE, 47507993276416, 47507993280511, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992801279, ++STORE, 47507992801280, 47507992805375, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992764415, ++STORE, 47507992764416, 47507992768511, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992711167, ++STORE, 47507992711168, 47507992719359, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989667839, ++STORE, 47507989667840, 47507989684223, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989082111, ++STORE, 47507989082112, 47507989086207, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988766719, ++STORE, 47507988766720, 47507988770815, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986513919, ++STORE, 47507986513920, 47507986518015, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986165759, ++STORE, 47507986165760, 47507986169855, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986120703, ++STORE, 47507986120704, 47507986124799, ++ERASE, 94386579570688, 94386579570688, ++STORE, 94386579570688, 94386579693567, ++STORE, 94386579693568, 94386579697663, ++ERASE, 140124810997760, 140124810997760, ++STORE, 140124810997760, 140124811001855, ++STORE, 140124811001856, 140124811005951, ++ERASE, 47507984158720, 47507984158720, ++STORE, 94386583982080, 94386584117247, ++STORE, 94386583982080, 94386584256511, ++ERASE, 94386583982080, 94386583982080, ++STORE, 94386583982080, 94386584223743, ++STORE, 94386584223744, 94386584256511, ++ERASE, 94386584223744, 94386584223744, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733763395584, 140737488351231, ++ERASE, 140733763395584, 140733763395584, ++STORE, 140733763395584, 140733763399679, ++STORE, 94011546472448, 94011547152383, ++ERASE, 94011546472448, 94011546472448, ++STORE, 94011546472448, 94011546537983, ++STORE, 94011546537984, 94011547152383, ++ERASE, 94011546537984, 94011546537984, ++STORE, 94011546537984, 94011546886143, ++STORE, 94011546886144, 94011547025407, ++STORE, 94011547025408, 94011547152383, ++STORE, 139757597949952, 139757598121983, ++ERASE, 139757597949952, 139757597949952, ++STORE, 139757597949952, 139757597954047, ++STORE, 139757597954048, 139757598121983, ++ERASE, 139757597954048, 139757597954048, ++STORE, 139757597954048, 139757598076927, ++STORE, 139757598076928, 139757598109695, ++STORE, 139757598109696, 139757598117887, ++STORE, 139757598117888, 139757598121983, ++STORE, 140733763596288, 140733763600383, ++STORE, 140733763584000, 140733763596287, ++STORE, 47875197046784, 47875197054975, ++STORE, 47875197054976, 47875197063167, ++STORE, 47875197063168, 47875198902271, ++STORE, 47875197202432, 47875198902271, ++STORE, 47875197063168, 47875197202431, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198861311, ++STORE, 47875198861312, 47875198902271, ++STORE, 47875198545920, 47875198861311, ++STORE, 47875197202432, 47875198545919, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198545919, ++STORE, 47875198857216, 47875198861311, ++STORE, 47875198545920, 47875198857215, ++ERASE, 47875198545920, 47875198545920, ++STORE, 47875198545920, 47875198857215, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198885888, 47875198885888, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198902272, 47875199012863, ++STORE, 47875198918656, 47875199012863, ++STORE, 47875198902272, 47875198918655, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875199004671, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875198980096, 47875199004671, ++STORE, 47875198918656, 47875198980095, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875198980095, ++STORE, 47875199000576, 47875199004671, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875198980096, 47875198980096, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875199012864, 47875199057919, ++ERASE, 47875199012864, 47875199012864, ++STORE, 47875199012864, 47875199021055, ++STORE, 47875199021056, 47875199057919, ++STORE, 47875199041536, 47875199057919, ++STORE, 47875199021056, 47875199041535, ++ERASE, 47875199021056, 47875199021056, ++STORE, 47875199021056, 47875199041535, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199041536, 47875199049727, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199057919, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199049727, ++STORE, 47875199049728, 47875199057919, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199057920, 47875199406079, ++STORE, 47875199098880, 47875199406079, ++STORE, 47875199057920, 47875199098879, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199381503, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199311872, 47875199381503, ++STORE, 47875199098880, 47875199311871, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199311871, ++STORE, 47875199377408, 47875199381503, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199311872, 47875199311872, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199406080, 47875201667071, ++STORE, 47875199557632, 47875201667071, ++STORE, 47875199406080, 47875199557631, ++ERASE, 47875199557632, 47875199557632, ++STORE, 47875199557632, 47875201650687, ++STORE, 47875201650688, 47875201667071, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201658880, 47875201658880, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201667072, 47875201802239, ++ERASE, 47875201667072, 47875201667072, ++STORE, 47875201667072, 47875201691647, ++STORE, 47875201691648, 47875201802239, ++STORE, 47875201753088, 47875201802239, ++STORE, 47875201691648, 47875201753087, ++ERASE, 47875201691648, 47875201691648, ++STORE, 47875201691648, 47875201753087, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201753088, 47875201777663, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201802239, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201777663, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201785856, 47875201785856, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201785856, 47875201810431, ++STORE, 47875201810432, 47875201974271, ++ERASE, 47875201810432, 47875201810432, ++STORE, 47875201810432, 47875201822719, ++STORE, 47875201822720, 47875201974271, ++STORE, 47875201921024, 47875201974271, ++STORE, 47875201822720, 47875201921023, ++ERASE, 47875201822720, 47875201822720, ++STORE, 47875201822720, 47875201921023, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201921024, 47875201966079, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201974271, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201966079, ++STORE, 47875201966080, 47875201974271, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201974272, 47875202572287, ++STORE, 47875202093056, 47875202572287, ++STORE, 47875201974272, 47875202093055, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202519039, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202408448, 47875202519039, ++STORE, 47875202093056, 47875202408447, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202408447, ++STORE, 47875202514944, 47875202519039, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202408448, 47875202408448, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202572288, 47875205623807, ++STORE, 47875203117056, 47875205623807, ++STORE, 47875202572288, 47875203117055, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875205402623, ++STORE, 47875205402624, 47875205623807, ++STORE, 47875204812800, 47875205402623, ++STORE, 47875203117056, 47875204812799, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875204812799, ++STORE, 47875205398528, 47875205402623, ++STORE, 47875204812800, 47875205398527, ++ERASE, 47875204812800, 47875204812800, ++STORE, 47875204812800, 47875205398527, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205607424, 47875205607424, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205623808, 47875205656575, ++ERASE, 47875205623808, 47875205623808, ++STORE, 47875205623808, 47875205631999, ++STORE, 47875205632000, 47875205656575, ++STORE, 47875205644288, 47875205656575, ++STORE, 47875205632000, 47875205644287, ++ERASE, 47875205632000, 47875205632000, ++STORE, 47875205632000, 47875205644287, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205644288, 47875205648383, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205656575, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205648383, ++STORE, 47875205648384, 47875205656575, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205656576, 47875205693439, ++ERASE, 47875205656576, 47875205656576, ++STORE, 47875205656576, 47875205664767, ++STORE, 47875205664768, 47875205693439, ++STORE, 47875205681152, 47875205693439, ++STORE, 47875205664768, 47875205681151, ++ERASE, 47875205664768, 47875205664768, ++STORE, 47875205664768, 47875205681151, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205681152, 47875205685247, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205693439, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205685247, ++STORE, 47875205685248, 47875205693439, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205693440, 47875206168575, ++ERASE, 47875205693440, 47875205693440, ++STORE, 47875205693440, 47875205701631, ++STORE, 47875205701632, 47875206168575, ++STORE, 47875206037504, 47875206168575, ++STORE, 47875205701632, 47875206037503, ++ERASE, 47875205701632, 47875205701632, ++STORE, 47875205701632, 47875206037503, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206037504, 47875206160383, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206168575, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206160383, ++STORE, 47875206160384, 47875206168575, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206168576, 47875206176767, ++STORE, 47875206176768, 47875206197247, ++ERASE, 47875206176768, 47875206176768, ++STORE, 47875206176768, 47875206180863, ++STORE, 47875206180864, 47875206197247, ++STORE, 47875206184960, 47875206197247, ++STORE, 47875206180864, 47875206184959, ++ERASE, 47875206180864, 47875206180864, ++STORE, 47875206180864, 47875206184959, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206184960, 47875206189055, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206197247, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206189055, ++STORE, 47875206189056, 47875206197247, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206197248, 47875206205439, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198877695, ++STORE, 47875198877696, 47875198885887, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206193151, ++STORE, 47875206193152, 47875206197247, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201781759, ++STORE, 47875201781760, 47875201785855, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206164479, ++STORE, 47875206164480, 47875206168575, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205689343, ++STORE, 47875205689344, 47875205693439, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205652479, ++STORE, 47875205652480, 47875205656575, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205599231, ++STORE, 47875205599232, 47875205607423, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202555903, ++STORE, 47875202555904, 47875202572287, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201970175, ++STORE, 47875201970176, 47875201974271, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201654783, ++STORE, 47875201654784, 47875201658879, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199401983, ++STORE, 47875199401984, 47875199406079, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199053823, ++STORE, 47875199053824, 47875199057919, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199008767, ++STORE, 47875199008768, 47875199012863, ++ERASE, 94011547025408, 94011547025408, ++STORE, 94011547025408, 94011547148287, ++STORE, 94011547148288, 94011547152383, ++ERASE, 139757598109696, 139757598109696, ++STORE, 139757598109696, 139757598113791, ++STORE, 139757598113792, 139757598117887, ++ERASE, 47875197046784, 47875197046784, ++STORE, 94011557584896, 94011557720063, ++STORE, 94011557584896, 94011557855231, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557851135, ++STORE, 94011557851136, 94011557855231, ++ERASE, 94011557851136, 94011557851136, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557847039, ++STORE, 94011557847040, 94011557851135, ++ERASE, 94011557847040, 94011557847040, ++STORE, 94011557584896, 94011557982207, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557978111, ++STORE, 94011557978112, 94011557982207, ++ERASE, 94011557978112, 94011557978112, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557974015, ++STORE, 94011557974016, 94011557978111, ++ERASE, 94011557974016, 94011557974016, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734130360320, 140737488351231, ++ERASE, 140734130360320, 140734130360320, ++STORE, 140734130360320, 140734130364415, ++STORE, 94641232105472, 94641232785407, ++ERASE, 94641232105472, 94641232105472, ++STORE, 94641232105472, 94641232171007, ++STORE, 94641232171008, 94641232785407, ++ERASE, 94641232171008, 94641232171008, ++STORE, 94641232171008, 94641232519167, ++STORE, 94641232519168, 94641232658431, ++STORE, 94641232658432, 94641232785407, ++STORE, 139726599516160, 139726599688191, ++ERASE, 139726599516160, 139726599516160, ++STORE, 139726599516160, 139726599520255, ++STORE, 139726599520256, 139726599688191, ++ERASE, 139726599520256, 139726599520256, ++STORE, 139726599520256, 139726599643135, ++STORE, 139726599643136, 139726599675903, ++STORE, 139726599675904, 139726599684095, ++STORE, 139726599684096, 139726599688191, ++STORE, 140734130446336, 140734130450431, ++STORE, 140734130434048, 140734130446335, ++STORE, 47906195480576, 47906195488767, ++STORE, 47906195488768, 47906195496959, ++STORE, 47906195496960, 47906197336063, ++STORE, 47906195636224, 47906197336063, ++STORE, 47906195496960, 47906195636223, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906197295103, ++STORE, 47906197295104, 47906197336063, ++STORE, 47906196979712, 47906197295103, ++STORE, 47906195636224, 47906196979711, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906196979711, ++STORE, 47906197291008, 47906197295103, ++STORE, 47906196979712, 47906197291007, ++ERASE, 47906196979712, 47906196979712, ++STORE, 47906196979712, 47906197291007, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197319680, 47906197319680, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197336064, 47906197446655, ++STORE, 47906197352448, 47906197446655, ++STORE, 47906197336064, 47906197352447, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197438463, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197413888, 47906197438463, ++STORE, 47906197352448, 47906197413887, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197413887, ++STORE, 47906197434368, 47906197438463, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197413888, 47906197413888, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197446656, 47906197491711, ++ERASE, 47906197446656, 47906197446656, ++STORE, 47906197446656, 47906197454847, ++STORE, 47906197454848, 47906197491711, ++STORE, 47906197475328, 47906197491711, ++STORE, 47906197454848, 47906197475327, ++ERASE, 47906197454848, 47906197454848, ++STORE, 47906197454848, 47906197475327, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197475328, 47906197483519, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197491711, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197483519, ++STORE, 47906197483520, 47906197491711, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197491712, 47906197839871, ++STORE, 47906197532672, 47906197839871, ++STORE, 47906197491712, 47906197532671, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197815295, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197745664, 47906197815295, ++STORE, 47906197532672, 47906197745663, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197745663, ++STORE, 47906197811200, 47906197815295, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197745664, 47906197745664, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197839872, 47906200100863, ++STORE, 47906197991424, 47906200100863, ++STORE, 47906197839872, 47906197991423, ++ERASE, 47906197991424, 47906197991424, ++STORE, 47906197991424, 47906200084479, ++STORE, 47906200084480, 47906200100863, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200092672, 47906200092672, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200100864, 47906200236031, ++ERASE, 47906200100864, 47906200100864, ++STORE, 47906200100864, 47906200125439, ++STORE, 47906200125440, 47906200236031, ++STORE, 47906200186880, 47906200236031, ++STORE, 47906200125440, 47906200186879, ++ERASE, 47906200125440, 47906200125440, ++STORE, 47906200125440, 47906200186879, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200186880, 47906200211455, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200236031, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200211455, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200219648, 47906200219648, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200219648, 47906200244223, ++STORE, 47906200244224, 47906200408063, ++ERASE, 47906200244224, 47906200244224, ++STORE, 47906200244224, 47906200256511, ++STORE, 47906200256512, 47906200408063, ++STORE, 47906200354816, 47906200408063, ++STORE, 47906200256512, 47906200354815, ++ERASE, 47906200256512, 47906200256512, ++STORE, 47906200256512, 47906200354815, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200354816, 47906200399871, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200408063, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200399871, ++STORE, 47906200399872, 47906200408063, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200408064, 47906201006079, ++STORE, 47906200526848, 47906201006079, ++STORE, 47906200408064, 47906200526847, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200952831, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906200842240, 47906200952831, ++STORE, 47906200526848, 47906200842239, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200842239, ++STORE, 47906200948736, 47906200952831, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200842240, 47906200842240, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906201006080, 47906204057599, ++STORE, 47906201550848, 47906204057599, ++STORE, 47906201006080, 47906201550847, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203836415, ++STORE, 47906203836416, 47906204057599, ++STORE, 47906203246592, 47906203836415, ++STORE, 47906201550848, 47906203246591, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203246591, ++STORE, 47906203832320, 47906203836415, ++STORE, 47906203246592, 47906203832319, ++ERASE, 47906203246592, 47906203246592, ++STORE, 47906203246592, 47906203832319, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906204041216, 47906204041216, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906204057600, 47906204090367, ++ERASE, 47906204057600, 47906204057600, ++STORE, 47906204057600, 47906204065791, ++STORE, 47906204065792, 47906204090367, ++STORE, 47906204078080, 47906204090367, ++STORE, 47906204065792, 47906204078079, ++ERASE, 47906204065792, 47906204065792, ++STORE, 47906204065792, 47906204078079, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204078080, 47906204082175, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204090367, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204082175, ++STORE, 47906204082176, 47906204090367, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204090368, 47906204127231, ++ERASE, 47906204090368, 47906204090368, ++STORE, 47906204090368, 47906204098559, ++STORE, 47906204098560, 47906204127231, ++STORE, 47906204114944, 47906204127231, ++STORE, 47906204098560, 47906204114943, ++ERASE, 47906204098560, 47906204098560, ++STORE, 47906204098560, 47906204114943, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204114944, 47906204119039, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204127231, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204119039, ++STORE, 47906204119040, 47906204127231, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204127232, 47906204602367, ++ERASE, 47906204127232, 47906204127232, ++STORE, 47906204127232, 47906204135423, ++STORE, 47906204135424, 47906204602367, ++STORE, 47906204471296, 47906204602367, ++STORE, 47906204135424, 47906204471295, ++ERASE, 47906204135424, 47906204135424, ++STORE, 47906204135424, 47906204471295, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204471296, 47906204594175, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204602367, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204594175, ++STORE, 47906204594176, 47906204602367, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204602368, 47906204610559, ++STORE, 47906204610560, 47906204631039, ++ERASE, 47906204610560, 47906204610560, ++STORE, 47906204610560, 47906204614655, ++STORE, 47906204614656, 47906204631039, ++STORE, 47906204618752, 47906204631039, ++STORE, 47906204614656, 47906204618751, ++ERASE, 47906204614656, 47906204614656, ++STORE, 47906204614656, 47906204618751, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204618752, 47906204622847, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204631039, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204622847, ++STORE, 47906204622848, 47906204631039, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204631040, 47906204639231, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197311487, ++STORE, 47906197311488, 47906197319679, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204626943, ++STORE, 47906204626944, 47906204631039, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200215551, ++STORE, 47906200215552, 47906200219647, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204598271, ++STORE, 47906204598272, 47906204602367, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204123135, ++STORE, 47906204123136, 47906204127231, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204086271, ++STORE, 47906204086272, 47906204090367, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204033023, ++STORE, 47906204033024, 47906204041215, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906200989695, ++STORE, 47906200989696, 47906201006079, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200403967, ++STORE, 47906200403968, 47906200408063, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200088575, ++STORE, 47906200088576, 47906200092671, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197835775, ++STORE, 47906197835776, 47906197839871, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197487615, ++STORE, 47906197487616, 47906197491711, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197442559, ++STORE, 47906197442560, 47906197446655, ++ERASE, 94641232658432, 94641232658432, ++STORE, 94641232658432, 94641232781311, ++STORE, 94641232781312, 94641232785407, ++ERASE, 139726599675904, 139726599675904, ++STORE, 139726599675904, 139726599679999, ++STORE, 139726599680000, 139726599684095, ++ERASE, 47906195480576, 47906195480576, ++STORE, 94641242615808, 94641242750975, ++ }; + -+ if (min_ttl) { -+ int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); -+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); ++ unsigned long set10[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140736427839488, 140737488351231, ++ERASE, 140736427839488, 140736427839488, ++STORE, 140736427839488, 140736427843583, ++STORE, 94071213395968, 94071213567999, ++ERASE, 94071213395968, 94071213395968, ++STORE, 94071213395968, 94071213412351, ++STORE, 94071213412352, 94071213567999, ++ERASE, 94071213412352, 94071213412352, ++STORE, 94071213412352, 94071213514751, ++STORE, 94071213514752, 94071213555711, ++STORE, 94071213555712, 94071213567999, ++STORE, 139968410644480, 139968410816511, ++ERASE, 139968410644480, 139968410644480, ++STORE, 139968410644480, 139968410648575, ++STORE, 139968410648576, 139968410816511, ++ERASE, 139968410648576, 139968410648576, ++STORE, 139968410648576, 139968410771455, ++STORE, 139968410771456, 139968410804223, ++STORE, 139968410804224, 139968410812415, ++STORE, 139968410812416, 139968410816511, ++STORE, 140736429277184, 140736429281279, ++STORE, 140736429264896, 140736429277183, ++STORE, 47664384352256, 47664384360447, ++STORE, 47664384360448, 47664384368639, ++STORE, 47664384368640, 47664384532479, ++ERASE, 47664384368640, 47664384368640, ++STORE, 47664384368640, 47664384380927, ++STORE, 47664384380928, 47664384532479, ++STORE, 47664384479232, 47664384532479, ++STORE, 47664384380928, 47664384479231, ++ERASE, 47664384380928, 47664384380928, ++STORE, 47664384380928, 47664384479231, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384479232, 47664384524287, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384532479, ++ERASE, 47664384479232, 47664384479232, ++STORE, 47664384479232, 47664384524287, ++STORE, 47664384524288, 47664384532479, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384532479, ++STORE, 47664384532480, 47664387583999, ++STORE, 47664385077248, 47664387583999, ++STORE, 47664384532480, 47664385077247, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664387362815, ++STORE, 47664387362816, 47664387583999, ++STORE, 47664386772992, 47664387362815, ++STORE, 47664385077248, 47664386772991, ++ERASE, 47664385077248, 47664385077248, ++STORE, 47664385077248, 47664386772991, ++STORE, 47664387358720, 47664387362815, ++STORE, 47664386772992, 47664387358719, ++ERASE, 47664386772992, 47664386772992, ++STORE, 47664386772992, 47664387358719, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387567615, ++ERASE, 47664387567616, 47664387567616, ++STORE, 47664387567616, 47664387583999, ++STORE, 47664387584000, 47664389423103, ++STORE, 47664387723264, 47664389423103, ++STORE, 47664387584000, 47664387723263, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389382143, ++STORE, 47664389382144, 47664389423103, ++STORE, 47664389066752, 47664389382143, ++STORE, 47664387723264, 47664389066751, ++ERASE, 47664387723264, 47664387723264, ++STORE, 47664387723264, 47664389066751, ++STORE, 47664389378048, 47664389382143, ++STORE, 47664389066752, 47664389378047, ++ERASE, 47664389066752, 47664389066752, ++STORE, 47664389066752, 47664389378047, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389406719, ++ERASE, 47664389406720, 47664389406720, ++STORE, 47664389406720, 47664389423103, ++STORE, 47664389423104, 47664389558271, ++ERASE, 47664389423104, 47664389423104, ++STORE, 47664389423104, 47664389447679, ++STORE, 47664389447680, 47664389558271, ++STORE, 47664389509120, 47664389558271, ++STORE, 47664389447680, 47664389509119, ++ERASE, 47664389447680, 47664389447680, ++STORE, 47664389447680, 47664389509119, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389509120, 47664389533695, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389558271, ++ERASE, 47664389509120, 47664389509120, ++STORE, 47664389509120, 47664389533695, ++STORE, 47664389533696, 47664389558271, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389541887, ++ERASE, 47664389541888, 47664389541888, ++STORE, 47664389541888, 47664389558271, ++STORE, 47664389558272, 47664389578751, ++ERASE, 47664389558272, 47664389558272, ++STORE, 47664389558272, 47664389562367, ++STORE, 47664389562368, 47664389578751, ++STORE, 47664389566464, 47664389578751, ++STORE, 47664389562368, 47664389566463, ++ERASE, 47664389562368, 47664389562368, ++STORE, 47664389562368, 47664389566463, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389566464, 47664389570559, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389578751, ++ERASE, 47664389566464, 47664389566464, ++STORE, 47664389566464, 47664389570559, ++STORE, 47664389570560, 47664389578751, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389578751, ++STORE, 47664389578752, 47664389586943, ++ERASE, 47664389382144, 47664389382144, ++STORE, 47664389382144, 47664389398527, ++STORE, 47664389398528, 47664389406719, ++ERASE, 47664389570560, 47664389570560, ++STORE, 47664389570560, 47664389574655, ++STORE, 47664389574656, 47664389578751, ++ERASE, 47664389533696, 47664389533696, ++STORE, 47664389533696, 47664389537791, ++STORE, 47664389537792, 47664389541887, ++ERASE, 47664387362816, 47664387362816, ++STORE, 47664387362816, 47664387559423, ++STORE, 47664387559424, 47664387567615, ++ERASE, 47664384524288, 47664384524288, ++STORE, 47664384524288, 47664384528383, ++STORE, 47664384528384, 47664384532479, ++ERASE, 94071213555712, 94071213555712, ++STORE, 94071213555712, 94071213563903, ++STORE, 94071213563904, 94071213567999, ++ERASE, 139968410804224, 139968410804224, ++STORE, 139968410804224, 139968410808319, ++STORE, 139968410808320, 139968410812415, ++ERASE, 47664384352256, 47664384352256, ++STORE, 94071244402688, 94071244537855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728271503360, 140737488351231, ++ERASE, 140728271503360, 140728271503360, ++STORE, 140728271503360, 140728271507455, ++STORE, 94410361982976, 94410362155007, ++ERASE, 94410361982976, 94410361982976, ++STORE, 94410361982976, 94410361999359, ++STORE, 94410361999360, 94410362155007, ++ERASE, 94410361999360, 94410361999360, ++STORE, 94410361999360, 94410362101759, ++STORE, 94410362101760, 94410362142719, ++STORE, 94410362142720, 94410362155007, ++STORE, 140351953997824, 140351954169855, ++ERASE, 140351953997824, 140351953997824, ++STORE, 140351953997824, 140351954001919, ++STORE, 140351954001920, 140351954169855, ++ERASE, 140351954001920, 140351954001920, ++STORE, 140351954001920, 140351954124799, ++STORE, 140351954124800, 140351954157567, ++STORE, 140351954157568, 140351954165759, ++STORE, 140351954165760, 140351954169855, ++STORE, 140728272429056, 140728272433151, ++STORE, 140728272416768, 140728272429055, ++STORE, 47280840998912, 47280841007103, ++STORE, 47280841007104, 47280841015295, ++STORE, 47280841015296, 47280841179135, ++ERASE, 47280841015296, 47280841015296, ++STORE, 47280841015296, 47280841027583, ++STORE, 47280841027584, 47280841179135, ++STORE, 47280841125888, 47280841179135, ++STORE, 47280841027584, 47280841125887, ++ERASE, 47280841027584, 47280841027584, ++STORE, 47280841027584, 47280841125887, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841125888, 47280841170943, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841179135, ++ERASE, 47280841125888, 47280841125888, ++STORE, 47280841125888, 47280841170943, ++STORE, 47280841170944, 47280841179135, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841179135, ++STORE, 47280841179136, 47280844230655, ++STORE, 47280841723904, 47280844230655, ++STORE, 47280841179136, 47280841723903, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280844009471, ++STORE, 47280844009472, 47280844230655, ++STORE, 47280843419648, 47280844009471, ++STORE, 47280841723904, 47280843419647, ++ERASE, 47280841723904, 47280841723904, ++STORE, 47280841723904, 47280843419647, ++STORE, 47280844005376, 47280844009471, ++STORE, 47280843419648, 47280844005375, ++ERASE, 47280843419648, 47280843419648, ++STORE, 47280843419648, 47280844005375, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844214271, ++ERASE, 47280844214272, 47280844214272, ++STORE, 47280844214272, 47280844230655, ++STORE, 47280844230656, 47280846069759, ++STORE, 47280844369920, 47280846069759, ++STORE, 47280844230656, 47280844369919, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280846028799, ++STORE, 47280846028800, 47280846069759, ++STORE, 47280845713408, 47280846028799, ++STORE, 47280844369920, 47280845713407, ++ERASE, 47280844369920, 47280844369920, ++STORE, 47280844369920, 47280845713407, ++STORE, 47280846024704, 47280846028799, ++STORE, 47280845713408, 47280846024703, ++ERASE, 47280845713408, 47280845713408, ++STORE, 47280845713408, 47280846024703, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846053375, ++ERASE, 47280846053376, 47280846053376, ++STORE, 47280846053376, 47280846069759, ++STORE, 47280846069760, 47280846204927, ++ERASE, 47280846069760, 47280846069760, ++STORE, 47280846069760, 47280846094335, ++STORE, 47280846094336, 47280846204927, ++STORE, 47280846155776, 47280846204927, ++STORE, 47280846094336, 47280846155775, ++ERASE, 47280846094336, 47280846094336, ++STORE, 47280846094336, 47280846155775, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846155776, 47280846180351, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846204927, ++ERASE, 47280846155776, 47280846155776, ++STORE, 47280846155776, 47280846180351, ++STORE, 47280846180352, 47280846204927, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846188543, ++ERASE, 47280846188544, 47280846188544, ++STORE, 47280846188544, 47280846204927, ++STORE, 47280846204928, 47280846225407, ++ERASE, 47280846204928, 47280846204928, ++STORE, 47280846204928, 47280846209023, ++STORE, 47280846209024, 47280846225407, ++STORE, 47280846213120, 47280846225407, ++STORE, 47280846209024, 47280846213119, ++ERASE, 47280846209024, 47280846209024, ++STORE, 47280846209024, 47280846213119, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846213120, 47280846217215, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846225407, ++ERASE, 47280846213120, 47280846213120, ++STORE, 47280846213120, 47280846217215, ++STORE, 47280846217216, 47280846225407, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846225407, ++STORE, 47280846225408, 47280846233599, ++ERASE, 47280846028800, 47280846028800, ++STORE, 47280846028800, 47280846045183, ++STORE, 47280846045184, 47280846053375, ++ERASE, 47280846217216, 47280846217216, ++STORE, 47280846217216, 47280846221311, ++STORE, 47280846221312, 47280846225407, ++ERASE, 47280846180352, 47280846180352, ++STORE, 47280846180352, 47280846184447, ++STORE, 47280846184448, 47280846188543, ++ERASE, 47280844009472, 47280844009472, ++STORE, 47280844009472, 47280844206079, ++STORE, 47280844206080, 47280844214271, ++ERASE, 47280841170944, 47280841170944, ++STORE, 47280841170944, 47280841175039, ++STORE, 47280841175040, 47280841179135, ++ERASE, 94410362142720, 94410362142720, ++STORE, 94410362142720, 94410362150911, ++STORE, 94410362150912, 94410362155007, ++ERASE, 140351954157568, 140351954157568, ++STORE, 140351954157568, 140351954161663, ++STORE, 140351954161664, 140351954165759, ++ERASE, 47280840998912, 47280840998912, ++STORE, 94410379456512, 94410379591679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732946362368, 140737488351231, ++ERASE, 140732946362368, 140732946362368, ++STORE, 140732946362368, 140732946366463, ++STORE, 94352937934848, 94352938106879, ++ERASE, 94352937934848, 94352937934848, ++STORE, 94352937934848, 94352937951231, ++STORE, 94352937951232, 94352938106879, ++ERASE, 94352937951232, 94352937951232, ++STORE, 94352937951232, 94352938053631, ++STORE, 94352938053632, 94352938094591, ++STORE, 94352938094592, 94352938106879, ++STORE, 140595518742528, 140595518914559, ++ERASE, 140595518742528, 140595518742528, ++STORE, 140595518742528, 140595518746623, ++STORE, 140595518746624, 140595518914559, ++ERASE, 140595518746624, 140595518746624, ++STORE, 140595518746624, 140595518869503, ++STORE, 140595518869504, 140595518902271, ++STORE, 140595518902272, 140595518910463, ++STORE, 140595518910464, 140595518914559, ++STORE, 140732947468288, 140732947472383, ++STORE, 140732947456000, 140732947468287, ++STORE, 47037276254208, 47037276262399, ++STORE, 47037276262400, 47037276270591, ++STORE, 47037276270592, 47037276434431, ++ERASE, 47037276270592, 47037276270592, ++STORE, 47037276270592, 47037276282879, ++STORE, 47037276282880, 47037276434431, ++STORE, 47037276381184, 47037276434431, ++STORE, 47037276282880, 47037276381183, ++ERASE, 47037276282880, 47037276282880, ++STORE, 47037276282880, 47037276381183, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276381184, 47037276426239, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276434431, ++ERASE, 47037276381184, 47037276381184, ++STORE, 47037276381184, 47037276426239, ++STORE, 47037276426240, 47037276434431, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276434431, ++STORE, 47037276434432, 47037279485951, ++STORE, 47037276979200, 47037279485951, ++STORE, 47037276434432, 47037276979199, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037279264767, ++STORE, 47037279264768, 47037279485951, ++STORE, 47037278674944, 47037279264767, ++STORE, 47037276979200, 47037278674943, ++ERASE, 47037276979200, 47037276979200, ++STORE, 47037276979200, 47037278674943, ++STORE, 47037279260672, 47037279264767, ++STORE, 47037278674944, 47037279260671, ++ERASE, 47037278674944, 47037278674944, ++STORE, 47037278674944, 47037279260671, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279469567, ++ERASE, 47037279469568, 47037279469568, ++STORE, 47037279469568, 47037279485951, ++STORE, 47037279485952, 47037281325055, ++STORE, 47037279625216, 47037281325055, ++STORE, 47037279485952, 47037279625215, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037281284095, ++STORE, 47037281284096, 47037281325055, ++STORE, 47037280968704, 47037281284095, ++STORE, 47037279625216, 47037280968703, ++ERASE, 47037279625216, 47037279625216, ++STORE, 47037279625216, 47037280968703, ++STORE, 47037281280000, 47037281284095, ++STORE, 47037280968704, 47037281279999, ++ERASE, 47037280968704, 47037280968704, ++STORE, 47037280968704, 47037281279999, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281308671, ++ERASE, 47037281308672, 47037281308672, ++STORE, 47037281308672, 47037281325055, ++STORE, 47037281325056, 47037281460223, ++ERASE, 47037281325056, 47037281325056, ++STORE, 47037281325056, 47037281349631, ++STORE, 47037281349632, 47037281460223, ++STORE, 47037281411072, 47037281460223, ++STORE, 47037281349632, 47037281411071, ++ERASE, 47037281349632, 47037281349632, ++STORE, 47037281349632, 47037281411071, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281411072, 47037281435647, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281460223, ++ERASE, 47037281411072, 47037281411072, ++STORE, 47037281411072, 47037281435647, ++STORE, 47037281435648, 47037281460223, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281443839, ++ERASE, 47037281443840, 47037281443840, ++STORE, 47037281443840, 47037281460223, ++STORE, 47037281460224, 47037281480703, ++ERASE, 47037281460224, 47037281460224, ++STORE, 47037281460224, 47037281464319, ++STORE, 47037281464320, 47037281480703, ++STORE, 47037281468416, 47037281480703, ++STORE, 47037281464320, 47037281468415, ++ERASE, 47037281464320, 47037281464320, ++STORE, 47037281464320, 47037281468415, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281468416, 47037281472511, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281480703, ++ERASE, 47037281468416, 47037281468416, ++STORE, 47037281468416, 47037281472511, ++STORE, 47037281472512, 47037281480703, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281480703, ++STORE, 47037281480704, 47037281488895, ++ERASE, 47037281284096, 47037281284096, ++STORE, 47037281284096, 47037281300479, ++STORE, 47037281300480, 47037281308671, ++ERASE, 47037281472512, 47037281472512, ++STORE, 47037281472512, 47037281476607, ++STORE, 47037281476608, 47037281480703, ++ERASE, 47037281435648, 47037281435648, ++STORE, 47037281435648, 47037281439743, ++STORE, 47037281439744, 47037281443839, ++ERASE, 47037279264768, 47037279264768, ++STORE, 47037279264768, 47037279461375, ++STORE, 47037279461376, 47037279469567, ++ERASE, 47037276426240, 47037276426240, ++STORE, 47037276426240, 47037276430335, ++STORE, 47037276430336, 47037276434431, ++ERASE, 94352938094592, 94352938094592, ++STORE, 94352938094592, 94352938102783, ++STORE, 94352938102784, 94352938106879, ++ERASE, 140595518902272, 140595518902272, ++STORE, 140595518902272, 140595518906367, ++STORE, 140595518906368, 140595518910463, ++ERASE, 47037276254208, 47037276254208, ++STORE, 94352938438656, 94352938573823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733506027520, 140737488351231, ++ERASE, 140733506027520, 140733506027520, ++STORE, 140733506027520, 140733506031615, ++STORE, 94150123073536, 94150123245567, ++ERASE, 94150123073536, 94150123073536, ++STORE, 94150123073536, 94150123089919, ++STORE, 94150123089920, 94150123245567, ++ERASE, 94150123089920, 94150123089920, ++STORE, 94150123089920, 94150123192319, ++STORE, 94150123192320, 94150123233279, ++STORE, 94150123233280, 94150123245567, ++STORE, 140081290375168, 140081290547199, ++ERASE, 140081290375168, 140081290375168, ++STORE, 140081290375168, 140081290379263, ++STORE, 140081290379264, 140081290547199, ++ERASE, 140081290379264, 140081290379264, ++STORE, 140081290379264, 140081290502143, ++STORE, 140081290502144, 140081290534911, ++STORE, 140081290534912, 140081290543103, ++STORE, 140081290543104, 140081290547199, ++STORE, 140733506707456, 140733506711551, ++STORE, 140733506695168, 140733506707455, ++STORE, 47551504621568, 47551504629759, ++STORE, 47551504629760, 47551504637951, ++STORE, 47551504637952, 47551504801791, ++ERASE, 47551504637952, 47551504637952, ++STORE, 47551504637952, 47551504650239, ++STORE, 47551504650240, 47551504801791, ++STORE, 47551504748544, 47551504801791, ++STORE, 47551504650240, 47551504748543, ++ERASE, 47551504650240, 47551504650240, ++STORE, 47551504650240, 47551504748543, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504748544, 47551504793599, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504801791, ++ERASE, 47551504748544, 47551504748544, ++STORE, 47551504748544, 47551504793599, ++STORE, 47551504793600, 47551504801791, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504801791, ++STORE, 47551504801792, 47551507853311, ++STORE, 47551505346560, 47551507853311, ++STORE, 47551504801792, 47551505346559, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507632127, ++STORE, 47551507632128, 47551507853311, ++STORE, 47551507042304, 47551507632127, ++STORE, 47551505346560, 47551507042303, ++ERASE, 47551505346560, 47551505346560, ++STORE, 47551505346560, 47551507042303, ++STORE, 47551507628032, 47551507632127, ++STORE, 47551507042304, 47551507628031, ++ERASE, 47551507042304, 47551507042304, ++STORE, 47551507042304, 47551507628031, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507836927, ++ERASE, 47551507836928, 47551507836928, ++STORE, 47551507836928, 47551507853311, ++STORE, 47551507853312, 47551509692415, ++STORE, 47551507992576, 47551509692415, ++STORE, 47551507853312, 47551507992575, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509651455, ++STORE, 47551509651456, 47551509692415, ++STORE, 47551509336064, 47551509651455, ++STORE, 47551507992576, 47551509336063, ++ERASE, 47551507992576, 47551507992576, ++STORE, 47551507992576, 47551509336063, ++STORE, 47551509647360, 47551509651455, ++STORE, 47551509336064, 47551509647359, ++ERASE, 47551509336064, 47551509336064, ++STORE, 47551509336064, 47551509647359, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509676031, ++ERASE, 47551509676032, 47551509676032, ++STORE, 47551509676032, 47551509692415, ++STORE, 47551509692416, 47551509827583, ++ERASE, 47551509692416, 47551509692416, ++STORE, 47551509692416, 47551509716991, ++STORE, 47551509716992, 47551509827583, ++STORE, 47551509778432, 47551509827583, ++STORE, 47551509716992, 47551509778431, ++ERASE, 47551509716992, 47551509716992, ++STORE, 47551509716992, 47551509778431, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509778432, 47551509803007, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509827583, ++ERASE, 47551509778432, 47551509778432, ++STORE, 47551509778432, 47551509803007, ++STORE, 47551509803008, 47551509827583, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509811199, ++ERASE, 47551509811200, 47551509811200, ++STORE, 47551509811200, 47551509827583, ++STORE, 47551509827584, 47551509848063, ++ERASE, 47551509827584, 47551509827584, ++STORE, 47551509827584, 47551509831679, ++STORE, 47551509831680, 47551509848063, ++STORE, 47551509835776, 47551509848063, ++STORE, 47551509831680, 47551509835775, ++ERASE, 47551509831680, 47551509831680, ++STORE, 47551509831680, 47551509835775, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509835776, 47551509839871, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509848063, ++ERASE, 47551509835776, 47551509835776, ++STORE, 47551509835776, 47551509839871, ++STORE, 47551509839872, 47551509848063, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509848063, ++STORE, 47551509848064, 47551509856255, ++ERASE, 47551509651456, 47551509651456, ++STORE, 47551509651456, 47551509667839, ++STORE, 47551509667840, 47551509676031, ++ERASE, 47551509839872, 47551509839872, ++STORE, 47551509839872, 47551509843967, ++STORE, 47551509843968, 47551509848063, ++ERASE, 47551509803008, 47551509803008, ++STORE, 47551509803008, 47551509807103, ++STORE, 47551509807104, 47551509811199, ++ERASE, 47551507632128, 47551507632128, ++STORE, 47551507632128, 47551507828735, ++STORE, 47551507828736, 47551507836927, ++ERASE, 47551504793600, 47551504793600, ++STORE, 47551504793600, 47551504797695, ++STORE, 47551504797696, 47551504801791, ++ERASE, 94150123233280, 94150123233280, ++STORE, 94150123233280, 94150123241471, ++STORE, 94150123241472, 94150123245567, ++ERASE, 140081290534912, 140081290534912, ++STORE, 140081290534912, 140081290539007, ++STORE, 140081290539008, 140081290543103, ++ERASE, 47551504621568, 47551504621568, ++STORE, 94150148112384, 94150148247551, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734389334016, 140737488351231, ++ERASE, 140734389334016, 140734389334016, ++STORE, 140734389334016, 140734389338111, ++STORE, 94844636606464, 94844636778495, ++ERASE, 94844636606464, 94844636606464, ++STORE, 94844636606464, 94844636622847, ++STORE, 94844636622848, 94844636778495, ++ERASE, 94844636622848, 94844636622848, ++STORE, 94844636622848, 94844636725247, ++STORE, 94844636725248, 94844636766207, ++STORE, 94844636766208, 94844636778495, ++STORE, 139922765217792, 139922765389823, ++ERASE, 139922765217792, 139922765217792, ++STORE, 139922765217792, 139922765221887, ++STORE, 139922765221888, 139922765389823, ++ERASE, 139922765221888, 139922765221888, ++STORE, 139922765221888, 139922765344767, ++STORE, 139922765344768, 139922765377535, ++STORE, 139922765377536, 139922765385727, ++STORE, 139922765385728, 139922765389823, ++STORE, 140734389678080, 140734389682175, ++STORE, 140734389665792, 140734389678079, ++STORE, 47710029778944, 47710029787135, ++STORE, 47710029787136, 47710029795327, ++STORE, 47710029795328, 47710029959167, ++ERASE, 47710029795328, 47710029795328, ++STORE, 47710029795328, 47710029807615, ++STORE, 47710029807616, 47710029959167, ++STORE, 47710029905920, 47710029959167, ++STORE, 47710029807616, 47710029905919, ++ERASE, 47710029807616, 47710029807616, ++STORE, 47710029807616, 47710029905919, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029905920, 47710029950975, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029959167, ++ERASE, 47710029905920, 47710029905920, ++STORE, 47710029905920, 47710029950975, ++STORE, 47710029950976, 47710029959167, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029959167, ++STORE, 47710029959168, 47710033010687, ++STORE, 47710030503936, 47710033010687, ++STORE, 47710029959168, 47710030503935, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032789503, ++STORE, 47710032789504, 47710033010687, ++STORE, 47710032199680, 47710032789503, ++STORE, 47710030503936, 47710032199679, ++ERASE, 47710030503936, 47710030503936, ++STORE, 47710030503936, 47710032199679, ++STORE, 47710032785408, 47710032789503, ++STORE, 47710032199680, 47710032785407, ++ERASE, 47710032199680, 47710032199680, ++STORE, 47710032199680, 47710032785407, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032994303, ++ERASE, 47710032994304, 47710032994304, ++STORE, 47710032994304, 47710033010687, ++STORE, 47710033010688, 47710034849791, ++STORE, 47710033149952, 47710034849791, ++STORE, 47710033010688, 47710033149951, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034808831, ++STORE, 47710034808832, 47710034849791, ++STORE, 47710034493440, 47710034808831, ++STORE, 47710033149952, 47710034493439, ++ERASE, 47710033149952, 47710033149952, ++STORE, 47710033149952, 47710034493439, ++STORE, 47710034804736, 47710034808831, ++STORE, 47710034493440, 47710034804735, ++ERASE, 47710034493440, 47710034493440, ++STORE, 47710034493440, 47710034804735, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034833407, ++ERASE, 47710034833408, 47710034833408, ++STORE, 47710034833408, 47710034849791, ++STORE, 47710034849792, 47710034984959, ++ERASE, 47710034849792, 47710034849792, ++STORE, 47710034849792, 47710034874367, ++STORE, 47710034874368, 47710034984959, ++STORE, 47710034935808, 47710034984959, ++STORE, 47710034874368, 47710034935807, ++ERASE, 47710034874368, 47710034874368, ++STORE, 47710034874368, 47710034935807, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034935808, 47710034960383, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034984959, ++ERASE, 47710034935808, 47710034935808, ++STORE, 47710034935808, 47710034960383, ++STORE, 47710034960384, 47710034984959, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034968575, ++ERASE, 47710034968576, 47710034968576, ++STORE, 47710034968576, 47710034984959, ++STORE, 47710034984960, 47710035005439, ++ERASE, 47710034984960, 47710034984960, ++STORE, 47710034984960, 47710034989055, ++STORE, 47710034989056, 47710035005439, ++STORE, 47710034993152, 47710035005439, ++STORE, 47710034989056, 47710034993151, ++ERASE, 47710034989056, 47710034989056, ++STORE, 47710034989056, 47710034993151, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710034993152, 47710034997247, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710035005439, ++ERASE, 47710034993152, 47710034993152, ++STORE, 47710034993152, 47710034997247, ++STORE, 47710034997248, 47710035005439, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035005439, ++STORE, 47710035005440, 47710035013631, ++ERASE, 47710034808832, 47710034808832, ++STORE, 47710034808832, 47710034825215, ++STORE, 47710034825216, 47710034833407, ++ERASE, 47710034997248, 47710034997248, ++STORE, 47710034997248, 47710035001343, ++STORE, 47710035001344, 47710035005439, ++ERASE, 47710034960384, 47710034960384, ++STORE, 47710034960384, 47710034964479, ++STORE, 47710034964480, 47710034968575, ++ERASE, 47710032789504, 47710032789504, ++STORE, 47710032789504, 47710032986111, ++STORE, 47710032986112, 47710032994303, ++ERASE, 47710029950976, 47710029950976, ++STORE, 47710029950976, 47710029955071, ++STORE, 47710029955072, 47710029959167, ++ERASE, 94844636766208, 94844636766208, ++STORE, 94844636766208, 94844636774399, ++STORE, 94844636774400, 94844636778495, ++ERASE, 139922765377536, 139922765377536, ++STORE, 139922765377536, 139922765381631, ++STORE, 139922765381632, 139922765385727, ++ERASE, 47710029778944, 47710029778944, ++STORE, 94844641775616, 94844641910783, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732213886976, 140737488351231, ++ERASE, 140732213886976, 140732213886976, ++STORE, 140732213886976, 140732213891071, ++STORE, 94240508887040, 94240509059071, ++ERASE, 94240508887040, 94240508887040, ++STORE, 94240508887040, 94240508903423, ++STORE, 94240508903424, 94240509059071, ++ERASE, 94240508903424, 94240508903424, ++STORE, 94240508903424, 94240509005823, ++STORE, 94240509005824, 94240509046783, ++STORE, 94240509046784, 94240509059071, ++STORE, 140275106516992, 140275106689023, ++ERASE, 140275106516992, 140275106516992, ++STORE, 140275106516992, 140275106521087, ++STORE, 140275106521088, 140275106689023, ++ERASE, 140275106521088, 140275106521088, ++STORE, 140275106521088, 140275106643967, ++STORE, 140275106643968, 140275106676735, ++STORE, 140275106676736, 140275106684927, ++STORE, 140275106684928, 140275106689023, ++STORE, 140732213977088, 140732213981183, ++STORE, 140732213964800, 140732213977087, ++STORE, 47357688479744, 47357688487935, ++STORE, 47357688487936, 47357688496127, ++STORE, 47357688496128, 47357688659967, ++ERASE, 47357688496128, 47357688496128, ++STORE, 47357688496128, 47357688508415, ++STORE, 47357688508416, 47357688659967, ++STORE, 47357688606720, 47357688659967, ++STORE, 47357688508416, 47357688606719, ++ERASE, 47357688508416, 47357688508416, ++STORE, 47357688508416, 47357688606719, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688606720, 47357688651775, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688659967, ++ERASE, 47357688606720, 47357688606720, ++STORE, 47357688606720, 47357688651775, ++STORE, 47357688651776, 47357688659967, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688659967, ++STORE, 47357688659968, 47357691711487, ++STORE, 47357689204736, 47357691711487, ++STORE, 47357688659968, 47357689204735, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357691490303, ++STORE, 47357691490304, 47357691711487, ++STORE, 47357690900480, 47357691490303, ++STORE, 47357689204736, 47357690900479, ++ERASE, 47357689204736, 47357689204736, ++STORE, 47357689204736, 47357690900479, ++STORE, 47357691486208, 47357691490303, ++STORE, 47357690900480, 47357691486207, ++ERASE, 47357690900480, 47357690900480, ++STORE, 47357690900480, 47357691486207, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691695103, ++ERASE, 47357691695104, 47357691695104, ++STORE, 47357691695104, 47357691711487, ++STORE, 47357691711488, 47357693550591, ++STORE, 47357691850752, 47357693550591, ++STORE, 47357691711488, 47357691850751, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693509631, ++STORE, 47357693509632, 47357693550591, ++STORE, 47357693194240, 47357693509631, ++STORE, 47357691850752, 47357693194239, ++ERASE, 47357691850752, 47357691850752, ++STORE, 47357691850752, 47357693194239, ++STORE, 47357693505536, 47357693509631, ++STORE, 47357693194240, 47357693505535, ++ERASE, 47357693194240, 47357693194240, ++STORE, 47357693194240, 47357693505535, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693534207, ++ERASE, 47357693534208, 47357693534208, ++STORE, 47357693534208, 47357693550591, ++STORE, 47357693550592, 47357693685759, ++ERASE, 47357693550592, 47357693550592, ++STORE, 47357693550592, 47357693575167, ++STORE, 47357693575168, 47357693685759, ++STORE, 47357693636608, 47357693685759, ++STORE, 47357693575168, 47357693636607, ++ERASE, 47357693575168, 47357693575168, ++STORE, 47357693575168, 47357693636607, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693636608, 47357693661183, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693685759, ++ERASE, 47357693636608, 47357693636608, ++STORE, 47357693636608, 47357693661183, ++STORE, 47357693661184, 47357693685759, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693669375, ++ERASE, 47357693669376, 47357693669376, ++STORE, 47357693669376, 47357693685759, ++STORE, 47357693685760, 47357693706239, ++ERASE, 47357693685760, 47357693685760, ++STORE, 47357693685760, 47357693689855, ++STORE, 47357693689856, 47357693706239, ++STORE, 47357693693952, 47357693706239, ++STORE, 47357693689856, 47357693693951, ++ERASE, 47357693689856, 47357693689856, ++STORE, 47357693689856, 47357693693951, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693693952, 47357693698047, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693706239, ++ERASE, 47357693693952, 47357693693952, ++STORE, 47357693693952, 47357693698047, ++STORE, 47357693698048, 47357693706239, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693706239, ++STORE, 47357693706240, 47357693714431, ++ERASE, 47357693509632, 47357693509632, ++STORE, 47357693509632, 47357693526015, ++STORE, 47357693526016, 47357693534207, ++ERASE, 47357693698048, 47357693698048, ++STORE, 47357693698048, 47357693702143, ++STORE, 47357693702144, 47357693706239, ++ERASE, 47357693661184, 47357693661184, ++STORE, 47357693661184, 47357693665279, ++STORE, 47357693665280, 47357693669375, ++ERASE, 47357691490304, 47357691490304, ++STORE, 47357691490304, 47357691686911, ++STORE, 47357691686912, 47357691695103, ++ERASE, 47357688651776, 47357688651776, ++STORE, 47357688651776, 47357688655871, ++STORE, 47357688655872, 47357688659967, ++ERASE, 94240509046784, 94240509046784, ++STORE, 94240509046784, 94240509054975, ++STORE, 94240509054976, 94240509059071, ++ERASE, 140275106676736, 140275106676736, ++STORE, 140275106676736, 140275106680831, ++STORE, 140275106680832, 140275106684927, ++ERASE, 47357688479744, 47357688479744, ++STORE, 94240518361088, 94240518496255, ++STORE, 140737488347136, 140737488351231, ++STORE, 140732688277504, 140737488351231, ++ERASE, 140732688277504, 140732688277504, ++STORE, 140732688277504, 140732688281599, ++STORE, 94629171351552, 94629172064255, ++ERASE, 94629171351552, 94629171351552, ++STORE, 94629171351552, 94629171400703, ++STORE, 94629171400704, 94629172064255, ++ERASE, 94629171400704, 94629171400704, ++STORE, 94629171400704, 94629171945471, ++STORE, 94629171945472, 94629172043775, ++STORE, 94629172043776, 94629172064255, ++STORE, 139770707644416, 139770707816447, ++ERASE, 139770707644416, 139770707644416, ++STORE, 139770707644416, 139770707648511, ++STORE, 139770707648512, 139770707816447, ++ERASE, 139770707648512, 139770707648512, ++STORE, 139770707648512, 139770707771391, ++STORE, 139770707771392, 139770707804159, ++STORE, 139770707804160, 139770707812351, ++STORE, 139770707812352, 139770707816447, ++STORE, 140732689121280, 140732689125375, ++STORE, 140732689108992, 140732689121279, ++STORE, 47862087352320, 47862087360511, ++STORE, 47862087360512, 47862087368703, ++STORE, 47862087368704, 47862087475199, ++STORE, 47862087385088, 47862087475199, ++STORE, 47862087368704, 47862087385087, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087458815, ++STORE, 47862087458816, 47862087475199, ++STORE, 47862087438336, 47862087458815, ++STORE, 47862087385088, 47862087438335, ++ERASE, 47862087385088, 47862087385088, ++STORE, 47862087385088, 47862087438335, ++STORE, 47862087454720, 47862087458815, ++STORE, 47862087438336, 47862087454719, ++ERASE, 47862087438336, 47862087438336, ++STORE, 47862087438336, 47862087454719, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087467007, ++ERASE, 47862087467008, 47862087467008, ++STORE, 47862087467008, 47862087475199, ++STORE, 47862087475200, 47862089314303, ++STORE, 47862087614464, 47862089314303, ++STORE, 47862087475200, 47862087614463, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862089273343, ++STORE, 47862089273344, 47862089314303, ++STORE, 47862088957952, 47862089273343, ++STORE, 47862087614464, 47862088957951, ++ERASE, 47862087614464, 47862087614464, ++STORE, 47862087614464, 47862088957951, ++STORE, 47862089269248, 47862089273343, ++STORE, 47862088957952, 47862089269247, ++ERASE, 47862088957952, 47862088957952, ++STORE, 47862088957952, 47862089269247, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089297919, ++ERASE, 47862089297920, 47862089297920, ++STORE, 47862089297920, 47862089314303, ++STORE, 47862089297920, 47862089326591, ++ERASE, 47862089273344, 47862089273344, ++STORE, 47862089273344, 47862089289727, ++STORE, 47862089289728, 47862089297919, ++ERASE, 47862087458816, 47862087458816, ++STORE, 47862087458816, 47862087462911, ++STORE, 47862087462912, 47862087467007, ++ERASE, 94629172043776, 94629172043776, ++STORE, 94629172043776, 94629172060159, ++STORE, 94629172060160, 94629172064255, ++ERASE, 139770707804160, 139770707804160, ++STORE, 139770707804160, 139770707808255, ++STORE, 139770707808256, 139770707812351, ++ERASE, 47862087352320, 47862087352320, ++STORE, 94629197533184, 94629197668351, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727540711424, 140737488351231, ++ERASE, 140727540711424, 140727540711424, ++STORE, 140727540711424, 140727540715519, ++STORE, 94299865313280, 94299866025983, ++ERASE, 94299865313280, 94299865313280, ++STORE, 94299865313280, 94299865362431, ++STORE, 94299865362432, 94299866025983, ++ERASE, 94299865362432, 94299865362432, ++STORE, 94299865362432, 94299865907199, ++STORE, 94299865907200, 94299866005503, ++STORE, 94299866005504, 94299866025983, ++STORE, 140680268763136, 140680268935167, ++ERASE, 140680268763136, 140680268763136, ++STORE, 140680268763136, 140680268767231, ++STORE, 140680268767232, 140680268935167, ++ERASE, 140680268767232, 140680268767232, ++STORE, 140680268767232, 140680268890111, ++STORE, 140680268890112, 140680268922879, ++STORE, 140680268922880, 140680268931071, ++STORE, 140680268931072, 140680268935167, ++STORE, 140727541424128, 140727541428223, ++STORE, 140727541411840, 140727541424127, ++STORE, 46952526233600, 46952526241791, ++STORE, 46952526241792, 46952526249983, ++STORE, 46952526249984, 46952526356479, ++STORE, 46952526266368, 46952526356479, ++STORE, 46952526249984, 46952526266367, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526340095, ++STORE, 46952526340096, 46952526356479, ++STORE, 46952526319616, 46952526340095, ++STORE, 46952526266368, 46952526319615, ++ERASE, 46952526266368, 46952526266368, ++STORE, 46952526266368, 46952526319615, ++STORE, 46952526336000, 46952526340095, ++STORE, 46952526319616, 46952526335999, ++ERASE, 46952526319616, 46952526319616, ++STORE, 46952526319616, 46952526335999, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526348287, ++ERASE, 46952526348288, 46952526348288, ++STORE, 46952526348288, 46952526356479, ++STORE, 46952526356480, 46952528195583, ++STORE, 46952526495744, 46952528195583, ++STORE, 46952526356480, 46952526495743, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952528154623, ++STORE, 46952528154624, 46952528195583, ++STORE, 46952527839232, 46952528154623, ++STORE, 46952526495744, 46952527839231, ++ERASE, 46952526495744, 46952526495744, ++STORE, 46952526495744, 46952527839231, ++STORE, 46952528150528, 46952528154623, ++STORE, 46952527839232, 46952528150527, ++ERASE, 46952527839232, 46952527839232, ++STORE, 46952527839232, 46952528150527, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528179199, ++ERASE, 46952528179200, 46952528179200, ++STORE, 46952528179200, 46952528195583, ++STORE, 46952528179200, 46952528207871, ++ERASE, 46952528154624, 46952528154624, ++STORE, 46952528154624, 46952528171007, ++STORE, 46952528171008, 46952528179199, ++ERASE, 46952526340096, 46952526340096, ++STORE, 46952526340096, 46952526344191, ++STORE, 46952526344192, 46952526348287, ++ERASE, 94299866005504, 94299866005504, ++STORE, 94299866005504, 94299866021887, ++STORE, 94299866021888, 94299866025983, ++ERASE, 140680268922880, 140680268922880, ++STORE, 140680268922880, 140680268926975, ++STORE, 140680268926976, 140680268931071, ++ERASE, 46952526233600, 46952526233600, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722874793984, 140737488351231, ++ERASE, 140722874793984, 140722874793984, ++STORE, 140722874793984, 140722874798079, ++STORE, 94448916213760, 94448916926463, ++ERASE, 94448916213760, 94448916213760, ++STORE, 94448916213760, 94448916262911, ++STORE, 94448916262912, 94448916926463, ++ERASE, 94448916262912, 94448916262912, ++STORE, 94448916262912, 94448916807679, ++STORE, 94448916807680, 94448916905983, ++STORE, 94448916905984, 94448916926463, ++STORE, 140389117046784, 140389117218815, ++ERASE, 140389117046784, 140389117046784, ++STORE, 140389117046784, 140389117050879, ++STORE, 140389117050880, 140389117218815, ++ERASE, 140389117050880, 140389117050880, ++STORE, 140389117050880, 140389117173759, ++STORE, 140389117173760, 140389117206527, ++STORE, 140389117206528, 140389117214719, ++STORE, 140389117214720, 140389117218815, ++STORE, 140722875297792, 140722875301887, ++STORE, 140722875285504, 140722875297791, ++STORE, 47243677949952, 47243677958143, ++STORE, 47243677958144, 47243677966335, ++STORE, 47243677966336, 47243678072831, ++STORE, 47243677982720, 47243678072831, ++STORE, 47243677966336, 47243677982719, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678056447, ++STORE, 47243678056448, 47243678072831, ++STORE, 47243678035968, 47243678056447, ++STORE, 47243677982720, 47243678035967, ++ERASE, 47243677982720, 47243677982720, ++STORE, 47243677982720, 47243678035967, ++STORE, 47243678052352, 47243678056447, ++STORE, 47243678035968, 47243678052351, ++ERASE, 47243678035968, 47243678035968, ++STORE, 47243678035968, 47243678052351, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678064639, ++ERASE, 47243678064640, 47243678064640, ++STORE, 47243678064640, 47243678072831, ++STORE, 47243678072832, 47243679911935, ++STORE, 47243678212096, 47243679911935, ++STORE, 47243678072832, 47243678212095, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679870975, ++STORE, 47243679870976, 47243679911935, ++STORE, 47243679555584, 47243679870975, ++STORE, 47243678212096, 47243679555583, ++ERASE, 47243678212096, 47243678212096, ++STORE, 47243678212096, 47243679555583, ++STORE, 47243679866880, 47243679870975, ++STORE, 47243679555584, 47243679866879, ++ERASE, 47243679555584, 47243679555584, ++STORE, 47243679555584, 47243679866879, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679895551, ++ERASE, 47243679895552, 47243679895552, ++STORE, 47243679895552, 47243679911935, ++STORE, 47243679895552, 47243679924223, ++ERASE, 47243679870976, 47243679870976, ++STORE, 47243679870976, 47243679887359, ++STORE, 47243679887360, 47243679895551, ++ERASE, 47243678056448, 47243678056448, ++STORE, 47243678056448, 47243678060543, ++STORE, 47243678060544, 47243678064639, ++ERASE, 94448916905984, 94448916905984, ++STORE, 94448916905984, 94448916922367, ++STORE, 94448916922368, 94448916926463, ++ERASE, 140389117206528, 140389117206528, ++STORE, 140389117206528, 140389117210623, ++STORE, 140389117210624, 140389117214719, ++ERASE, 47243677949952, 47243677949952, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733068505088, 140737488351231, ++ERASE, 140733068505088, 140733068505088, ++STORE, 140733068505088, 140733068509183, ++STORE, 94207145750528, 94207146463231, ++ERASE, 94207145750528, 94207145750528, ++STORE, 94207145750528, 94207145799679, ++STORE, 94207145799680, 94207146463231, ++ERASE, 94207145799680, 94207145799680, ++STORE, 94207145799680, 94207146344447, ++STORE, 94207146344448, 94207146442751, ++STORE, 94207146442752, 94207146463231, ++STORE, 140684504911872, 140684505083903, ++ERASE, 140684504911872, 140684504911872, ++STORE, 140684504911872, 140684504915967, ++STORE, 140684504915968, 140684505083903, ++ERASE, 140684504915968, 140684504915968, ++STORE, 140684504915968, 140684505038847, ++STORE, 140684505038848, 140684505071615, ++STORE, 140684505071616, 140684505079807, ++STORE, 140684505079808, 140684505083903, ++STORE, 140733068607488, 140733068611583, ++STORE, 140733068595200, 140733068607487, ++STORE, 46948290084864, 46948290093055, ++STORE, 46948290093056, 46948290101247, ++STORE, 46948290101248, 46948290207743, ++STORE, 46948290117632, 46948290207743, ++STORE, 46948290101248, 46948290117631, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290191359, ++STORE, 46948290191360, 46948290207743, ++STORE, 46948290170880, 46948290191359, ++STORE, 46948290117632, 46948290170879, ++ERASE, 46948290117632, 46948290117632, ++STORE, 46948290117632, 46948290170879, ++STORE, 46948290187264, 46948290191359, ++STORE, 46948290170880, 46948290187263, ++ERASE, 46948290170880, 46948290170880, ++STORE, 46948290170880, 46948290187263, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290199551, ++ERASE, 46948290199552, 46948290199552, ++STORE, 46948290199552, 46948290207743, ++STORE, 46948290207744, 46948292046847, ++STORE, 46948290347008, 46948292046847, ++STORE, 46948290207744, 46948290347007, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948292005887, ++STORE, 46948292005888, 46948292046847, ++STORE, 46948291690496, 46948292005887, ++STORE, 46948290347008, 46948291690495, ++ERASE, 46948290347008, 46948290347008, ++STORE, 46948290347008, 46948291690495, ++STORE, 46948292001792, 46948292005887, ++STORE, 46948291690496, 46948292001791, ++ERASE, 46948291690496, 46948291690496, ++STORE, 46948291690496, 46948292001791, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292030463, ++ERASE, 46948292030464, 46948292030464, ++STORE, 46948292030464, 46948292046847, ++STORE, 46948292030464, 46948292059135, ++ERASE, 46948292005888, 46948292005888, ++STORE, 46948292005888, 46948292022271, ++STORE, 46948292022272, 46948292030463, ++ERASE, 46948290191360, 46948290191360, ++STORE, 46948290191360, 46948290195455, ++STORE, 46948290195456, 46948290199551, ++ERASE, 94207146442752, 94207146442752, ++STORE, 94207146442752, 94207146459135, ++STORE, 94207146459136, 94207146463231, ++ERASE, 140684505071616, 140684505071616, ++STORE, 140684505071616, 140684505075711, ++STORE, 140684505075712, 140684505079807, ++ERASE, 46948290084864, 46948290084864, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726367158272, 140737488351231, ++ERASE, 140726367158272, 140726367158272, ++STORE, 140726367158272, 140726367162367, ++STORE, 94436124106752, 94436124819455, ++ERASE, 94436124106752, 94436124106752, ++STORE, 94436124106752, 94436124155903, ++STORE, 94436124155904, 94436124819455, ++ERASE, 94436124155904, 94436124155904, ++STORE, 94436124155904, 94436124700671, ++STORE, 94436124700672, 94436124798975, ++STORE, 94436124798976, 94436124819455, ++STORE, 140049025044480, 140049025216511, ++ERASE, 140049025044480, 140049025044480, ++STORE, 140049025044480, 140049025048575, ++STORE, 140049025048576, 140049025216511, ++ERASE, 140049025048576, 140049025048576, ++STORE, 140049025048576, 140049025171455, ++STORE, 140049025171456, 140049025204223, ++STORE, 140049025204224, 140049025212415, ++STORE, 140049025212416, 140049025216511, ++STORE, 140726367256576, 140726367260671, ++STORE, 140726367244288, 140726367256575, ++STORE, 47583769952256, 47583769960447, ++STORE, 47583769960448, 47583769968639, ++STORE, 47583769968640, 47583770075135, ++STORE, 47583769985024, 47583770075135, ++STORE, 47583769968640, 47583769985023, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770058751, ++STORE, 47583770058752, 47583770075135, ++STORE, 47583770038272, 47583770058751, ++STORE, 47583769985024, 47583770038271, ++ERASE, 47583769985024, 47583769985024, ++STORE, 47583769985024, 47583770038271, ++STORE, 47583770054656, 47583770058751, ++STORE, 47583770038272, 47583770054655, ++ERASE, 47583770038272, 47583770038272, ++STORE, 47583770038272, 47583770054655, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770066943, ++ERASE, 47583770066944, 47583770066944, ++STORE, 47583770066944, 47583770075135, ++STORE, 47583770075136, 47583771914239, ++STORE, 47583770214400, 47583771914239, ++STORE, 47583770075136, 47583770214399, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771873279, ++STORE, 47583771873280, 47583771914239, ++STORE, 47583771557888, 47583771873279, ++STORE, 47583770214400, 47583771557887, ++ERASE, 47583770214400, 47583770214400, ++STORE, 47583770214400, 47583771557887, ++STORE, 47583771869184, 47583771873279, ++STORE, 47583771557888, 47583771869183, ++ERASE, 47583771557888, 47583771557888, ++STORE, 47583771557888, 47583771869183, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771897855, ++ERASE, 47583771897856, 47583771897856, ++STORE, 47583771897856, 47583771914239, ++STORE, 47583771897856, 47583771926527, ++ERASE, 47583771873280, 47583771873280, ++STORE, 47583771873280, 47583771889663, ++STORE, 47583771889664, 47583771897855, ++ERASE, 47583770058752, 47583770058752, ++STORE, 47583770058752, 47583770062847, ++STORE, 47583770062848, 47583770066943, ++ERASE, 94436124798976, 94436124798976, ++STORE, 94436124798976, 94436124815359, ++STORE, 94436124815360, 94436124819455, ++ERASE, 140049025204224, 140049025204224, ++STORE, 140049025204224, 140049025208319, ++STORE, 140049025208320, 140049025212415, ++ERASE, 47583769952256, 47583769952256, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727116099584, 140737488351231, ++ERASE, 140727116099584, 140727116099584, ++STORE, 140727116099584, 140727116103679, ++STORE, 94166319734784, 94166320447487, ++ERASE, 94166319734784, 94166319734784, ++STORE, 94166319734784, 94166319783935, ++STORE, 94166319783936, 94166320447487, ++ERASE, 94166319783936, 94166319783936, ++STORE, 94166319783936, 94166320328703, ++STORE, 94166320328704, 94166320427007, ++STORE, 94166320427008, 94166320447487, ++STORE, 139976559542272, 139976559714303, ++ERASE, 139976559542272, 139976559542272, ++STORE, 139976559542272, 139976559546367, ++STORE, 139976559546368, 139976559714303, ++ERASE, 139976559546368, 139976559546368, ++STORE, 139976559546368, 139976559669247, ++STORE, 139976559669248, 139976559702015, ++STORE, 139976559702016, 139976559710207, ++STORE, 139976559710208, 139976559714303, ++STORE, 140727116222464, 140727116226559, ++STORE, 140727116210176, 140727116222463, ++STORE, 47656235454464, 47656235462655, ++STORE, 47656235462656, 47656235470847, ++STORE, 47656235470848, 47656235577343, ++STORE, 47656235487232, 47656235577343, ++STORE, 47656235470848, 47656235487231, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235560959, ++STORE, 47656235560960, 47656235577343, ++STORE, 47656235540480, 47656235560959, ++STORE, 47656235487232, 47656235540479, ++ERASE, 47656235487232, 47656235487232, ++STORE, 47656235487232, 47656235540479, ++STORE, 47656235556864, 47656235560959, ++STORE, 47656235540480, 47656235556863, ++ERASE, 47656235540480, 47656235540480, ++STORE, 47656235540480, 47656235556863, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235569151, ++ERASE, 47656235569152, 47656235569152, ++STORE, 47656235569152, 47656235577343, ++STORE, 47656235577344, 47656237416447, ++STORE, 47656235716608, 47656237416447, ++STORE, 47656235577344, 47656235716607, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237375487, ++STORE, 47656237375488, 47656237416447, ++STORE, 47656237060096, 47656237375487, ++STORE, 47656235716608, 47656237060095, ++ERASE, 47656235716608, 47656235716608, ++STORE, 47656235716608, 47656237060095, ++STORE, 47656237371392, 47656237375487, ++STORE, 47656237060096, 47656237371391, ++ERASE, 47656237060096, 47656237060096, ++STORE, 47656237060096, 47656237371391, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237400063, ++ERASE, 47656237400064, 47656237400064, ++STORE, 47656237400064, 47656237416447, ++STORE, 47656237400064, 47656237428735, ++ERASE, 47656237375488, 47656237375488, ++STORE, 47656237375488, 47656237391871, ++STORE, 47656237391872, 47656237400063, ++ERASE, 47656235560960, 47656235560960, ++STORE, 47656235560960, 47656235565055, ++STORE, 47656235565056, 47656235569151, ++ERASE, 94166320427008, 94166320427008, ++STORE, 94166320427008, 94166320443391, ++STORE, 94166320443392, 94166320447487, ++ERASE, 139976559702016, 139976559702016, ++STORE, 139976559702016, 139976559706111, ++STORE, 139976559706112, 139976559710207, ++ERASE, 47656235454464, 47656235454464, ++STORE, 94166332153856, 94166332289023, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726412816384, 140737488351231, ++ERASE, 140726412816384, 140726412816384, ++STORE, 140726412816384, 140726412820479, ++STORE, 94094884507648, 94094885220351, ++ERASE, 94094884507648, 94094884507648, ++STORE, 94094884507648, 94094884556799, ++STORE, 94094884556800, 94094885220351, ++ERASE, 94094884556800, 94094884556800, ++STORE, 94094884556800, 94094885101567, ++STORE, 94094885101568, 94094885199871, ++STORE, 94094885199872, 94094885220351, ++STORE, 139773773938688, 139773774110719, ++ERASE, 139773773938688, 139773773938688, ++STORE, 139773773938688, 139773773942783, ++STORE, 139773773942784, 139773774110719, ++ERASE, 139773773942784, 139773773942784, ++STORE, 139773773942784, 139773774065663, ++STORE, 139773774065664, 139773774098431, ++STORE, 139773774098432, 139773774106623, ++STORE, 139773774106624, 139773774110719, ++STORE, 140726412963840, 140726412967935, ++STORE, 140726412951552, 140726412963839, ++STORE, 47859021058048, 47859021066239, ++STORE, 47859021066240, 47859021074431, ++STORE, 47859021074432, 47859021180927, ++STORE, 47859021090816, 47859021180927, ++STORE, 47859021074432, 47859021090815, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021164543, ++STORE, 47859021164544, 47859021180927, ++STORE, 47859021144064, 47859021164543, ++STORE, 47859021090816, 47859021144063, ++ERASE, 47859021090816, 47859021090816, ++STORE, 47859021090816, 47859021144063, ++STORE, 47859021160448, 47859021164543, ++STORE, 47859021144064, 47859021160447, ++ERASE, 47859021144064, 47859021144064, ++STORE, 47859021144064, 47859021160447, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021172735, ++ERASE, 47859021172736, 47859021172736, ++STORE, 47859021172736, 47859021180927, ++STORE, 47859021180928, 47859023020031, ++STORE, 47859021320192, 47859023020031, ++STORE, 47859021180928, 47859021320191, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022979071, ++STORE, 47859022979072, 47859023020031, ++STORE, 47859022663680, 47859022979071, ++STORE, 47859021320192, 47859022663679, ++ERASE, 47859021320192, 47859021320192, ++STORE, 47859021320192, 47859022663679, ++STORE, 47859022974976, 47859022979071, ++STORE, 47859022663680, 47859022974975, ++ERASE, 47859022663680, 47859022663680, ++STORE, 47859022663680, 47859022974975, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859023003647, ++ERASE, 47859023003648, 47859023003648, ++STORE, 47859023003648, 47859023020031, ++STORE, 47859023003648, 47859023032319, ++ERASE, 47859022979072, 47859022979072, ++STORE, 47859022979072, 47859022995455, ++STORE, 47859022995456, 47859023003647, ++ERASE, 47859021164544, 47859021164544, ++STORE, 47859021164544, 47859021168639, ++STORE, 47859021168640, 47859021172735, ++ERASE, 94094885199872, 94094885199872, ++STORE, 94094885199872, 94094885216255, ++STORE, 94094885216256, 94094885220351, ++ERASE, 139773774098432, 139773774098432, ++STORE, 139773774098432, 139773774102527, ++STORE, 139773774102528, 139773774106623, ++ERASE, 47859021058048, 47859021058048, ++STORE, 94094901108736, 94094901243903, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736567963648, 140737488351231, ++ERASE, 140736567963648, 140736567963648, ++STORE, 140736567963648, 140736567967743, ++STORE, 94924425748480, 94924426461183, ++ERASE, 94924425748480, 94924425748480, ++STORE, 94924425748480, 94924425797631, ++STORE, 94924425797632, 94924426461183, ++ERASE, 94924425797632, 94924425797632, ++STORE, 94924425797632, 94924426342399, ++STORE, 94924426342400, 94924426440703, ++STORE, 94924426440704, 94924426461183, ++STORE, 140042126319616, 140042126491647, ++ERASE, 140042126319616, 140042126319616, ++STORE, 140042126319616, 140042126323711, ++STORE, 140042126323712, 140042126491647, ++ERASE, 140042126323712, 140042126323712, ++STORE, 140042126323712, 140042126446591, ++STORE, 140042126446592, 140042126479359, ++STORE, 140042126479360, 140042126487551, ++STORE, 140042126487552, 140042126491647, ++STORE, 140736568672256, 140736568676351, ++STORE, 140736568659968, 140736568672255, ++STORE, 47590668677120, 47590668685311, ++STORE, 47590668685312, 47590668693503, ++STORE, 47590668693504, 47590668799999, ++STORE, 47590668709888, 47590668799999, ++STORE, 47590668693504, 47590668709887, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668783615, ++STORE, 47590668783616, 47590668799999, ++STORE, 47590668763136, 47590668783615, ++STORE, 47590668709888, 47590668763135, ++ERASE, 47590668709888, 47590668709888, ++STORE, 47590668709888, 47590668763135, ++STORE, 47590668779520, 47590668783615, ++STORE, 47590668763136, 47590668779519, ++ERASE, 47590668763136, 47590668763136, ++STORE, 47590668763136, 47590668779519, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668791807, ++ERASE, 47590668791808, 47590668791808, ++STORE, 47590668791808, 47590668799999, ++STORE, 47590668800000, 47590670639103, ++STORE, 47590668939264, 47590670639103, ++STORE, 47590668800000, 47590668939263, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670598143, ++STORE, 47590670598144, 47590670639103, ++STORE, 47590670282752, 47590670598143, ++STORE, 47590668939264, 47590670282751, ++ERASE, 47590668939264, 47590668939264, ++STORE, 47590668939264, 47590670282751, ++STORE, 47590670594048, 47590670598143, ++STORE, 47590670282752, 47590670594047, ++ERASE, 47590670282752, 47590670282752, ++STORE, 47590670282752, 47590670594047, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670622719, ++ERASE, 47590670622720, 47590670622720, ++STORE, 47590670622720, 47590670639103, ++STORE, 47590670622720, 47590670651391, ++ERASE, 47590670598144, 47590670598144, ++STORE, 47590670598144, 47590670614527, ++STORE, 47590670614528, 47590670622719, ++ERASE, 47590668783616, 47590668783616, ++STORE, 47590668783616, 47590668787711, ++STORE, 47590668787712, 47590668791807, ++ERASE, 94924426440704, 94924426440704, ++STORE, 94924426440704, 94924426457087, ++STORE, 94924426457088, 94924426461183, ++ERASE, 140042126479360, 140042126479360, ++STORE, 140042126479360, 140042126483455, ++STORE, 140042126483456, 140042126487551, ++ERASE, 47590668677120, 47590668677120, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733281439744, 140737488351231, ++ERASE, 140733281439744, 140733281439744, ++STORE, 140733281439744, 140733281443839, ++STORE, 94490667069440, 94490667782143, ++ERASE, 94490667069440, 94490667069440, ++STORE, 94490667069440, 94490667118591, ++STORE, 94490667118592, 94490667782143, ++ERASE, 94490667118592, 94490667118592, ++STORE, 94490667118592, 94490667663359, ++STORE, 94490667663360, 94490667761663, ++STORE, 94490667761664, 94490667782143, ++STORE, 139878215118848, 139878215290879, ++ERASE, 139878215118848, 139878215118848, ++STORE, 139878215118848, 139878215122943, ++STORE, 139878215122944, 139878215290879, ++ERASE, 139878215122944, 139878215122944, ++STORE, 139878215122944, 139878215245823, ++STORE, 139878215245824, 139878215278591, ++STORE, 139878215278592, 139878215286783, ++STORE, 139878215286784, 139878215290879, ++STORE, 140733281464320, 140733281468415, ++STORE, 140733281452032, 140733281464319, ++STORE, 47754579877888, 47754579886079, ++STORE, 47754579886080, 47754579894271, ++STORE, 47754579894272, 47754580000767, ++STORE, 47754579910656, 47754580000767, ++STORE, 47754579894272, 47754579910655, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579984383, ++STORE, 47754579984384, 47754580000767, ++STORE, 47754579963904, 47754579984383, ++STORE, 47754579910656, 47754579963903, ++ERASE, 47754579910656, 47754579910656, ++STORE, 47754579910656, 47754579963903, ++STORE, 47754579980288, 47754579984383, ++STORE, 47754579963904, 47754579980287, ++ERASE, 47754579963904, 47754579963904, ++STORE, 47754579963904, 47754579980287, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579992575, ++ERASE, 47754579992576, 47754579992576, ++STORE, 47754579992576, 47754580000767, ++STORE, 47754580000768, 47754581839871, ++STORE, 47754580140032, 47754581839871, ++STORE, 47754580000768, 47754580140031, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581798911, ++STORE, 47754581798912, 47754581839871, ++STORE, 47754581483520, 47754581798911, ++STORE, 47754580140032, 47754581483519, ++ERASE, 47754580140032, 47754580140032, ++STORE, 47754580140032, 47754581483519, ++STORE, 47754581794816, 47754581798911, ++STORE, 47754581483520, 47754581794815, ++ERASE, 47754581483520, 47754581483520, ++STORE, 47754581483520, 47754581794815, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581823487, ++ERASE, 47754581823488, 47754581823488, ++STORE, 47754581823488, 47754581839871, ++STORE, 47754581823488, 47754581852159, ++ERASE, 47754581798912, 47754581798912, ++STORE, 47754581798912, 47754581815295, ++STORE, 47754581815296, 47754581823487, ++ERASE, 47754579984384, 47754579984384, ++STORE, 47754579984384, 47754579988479, ++STORE, 47754579988480, 47754579992575, ++ERASE, 94490667761664, 94490667761664, ++STORE, 94490667761664, 94490667778047, ++STORE, 94490667778048, 94490667782143, ++ERASE, 139878215278592, 139878215278592, ++STORE, 139878215278592, 139878215282687, ++STORE, 139878215282688, 139878215286783, ++ERASE, 47754579877888, 47754579877888, ++STORE, 94490669649920, 94490669785087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140735382188032, 140737488351231, ++ERASE, 140735382188032, 140735382188032, ++STORE, 140735382188032, 140735382192127, ++STORE, 94150181302272, 94150182014975, ++ERASE, 94150181302272, 94150181302272, ++STORE, 94150181302272, 94150181351423, ++STORE, 94150181351424, 94150182014975, ++ERASE, 94150181351424, 94150181351424, ++STORE, 94150181351424, 94150181896191, ++STORE, 94150181896192, 94150181994495, ++STORE, 94150181994496, 94150182014975, ++STORE, 139679752458240, 139679752630271, ++ERASE, 139679752458240, 139679752458240, ++STORE, 139679752458240, 139679752462335, ++STORE, 139679752462336, 139679752630271, ++ERASE, 139679752462336, 139679752462336, ++STORE, 139679752462336, 139679752585215, ++STORE, 139679752585216, 139679752617983, ++STORE, 139679752617984, 139679752626175, ++STORE, 139679752626176, 139679752630271, ++STORE, 140735382536192, 140735382540287, ++STORE, 140735382523904, 140735382536191, ++STORE, 47953042538496, 47953042546687, ++STORE, 47953042546688, 47953042554879, ++STORE, 47953042554880, 47953042661375, ++STORE, 47953042571264, 47953042661375, ++STORE, 47953042554880, 47953042571263, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042644991, ++STORE, 47953042644992, 47953042661375, ++STORE, 47953042624512, 47953042644991, ++STORE, 47953042571264, 47953042624511, ++ERASE, 47953042571264, 47953042571264, ++STORE, 47953042571264, 47953042624511, ++STORE, 47953042640896, 47953042644991, ++STORE, 47953042624512, 47953042640895, ++ERASE, 47953042624512, 47953042624512, ++STORE, 47953042624512, 47953042640895, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042653183, ++ERASE, 47953042653184, 47953042653184, ++STORE, 47953042653184, 47953042661375, ++STORE, 47953042661376, 47953044500479, ++STORE, 47953042800640, 47953044500479, ++STORE, 47953042661376, 47953042800639, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044459519, ++STORE, 47953044459520, 47953044500479, ++STORE, 47953044144128, 47953044459519, ++STORE, 47953042800640, 47953044144127, ++ERASE, 47953042800640, 47953042800640, ++STORE, 47953042800640, 47953044144127, ++STORE, 47953044455424, 47953044459519, ++STORE, 47953044144128, 47953044455423, ++ERASE, 47953044144128, 47953044144128, ++STORE, 47953044144128, 47953044455423, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044484095, ++ERASE, 47953044484096, 47953044484096, ++STORE, 47953044484096, 47953044500479, ++STORE, 47953044484096, 47953044512767, ++ERASE, 47953044459520, 47953044459520, ++STORE, 47953044459520, 47953044475903, ++STORE, 47953044475904, 47953044484095, ++ERASE, 47953042644992, 47953042644992, ++STORE, 47953042644992, 47953042649087, ++STORE, 47953042649088, 47953042653183, ++ERASE, 94150181994496, 94150181994496, ++STORE, 94150181994496, 94150182010879, ++STORE, 94150182010880, 94150182014975, ++ERASE, 139679752617984, 139679752617984, ++STORE, 139679752617984, 139679752622079, ++STORE, 139679752622080, 139679752626175, ++ERASE, 47953042538496, 47953042538496, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737044123648, 140737488351231, ++ERASE, 140737044123648, 140737044123648, ++STORE, 140737044123648, 140737044127743, ++STORE, 94425324294144, 94425325006847, ++ERASE, 94425324294144, 94425324294144, ++STORE, 94425324294144, 94425324343295, ++STORE, 94425324343296, 94425325006847, ++ERASE, 94425324343296, 94425324343296, ++STORE, 94425324343296, 94425324888063, ++STORE, 94425324888064, 94425324986367, ++STORE, 94425324986368, 94425325006847, ++STORE, 140382015016960, 140382015188991, ++ERASE, 140382015016960, 140382015016960, ++STORE, 140382015016960, 140382015021055, ++STORE, 140382015021056, 140382015188991, ++ERASE, 140382015021056, 140382015021056, ++STORE, 140382015021056, 140382015143935, ++STORE, 140382015143936, 140382015176703, ++STORE, 140382015176704, 140382015184895, ++STORE, 140382015184896, 140382015188991, ++STORE, 140737045585920, 140737045590015, ++STORE, 140737045573632, 140737045585919, ++STORE, 47250779979776, 47250779987967, ++STORE, 47250779987968, 47250779996159, ++STORE, 47250779996160, 47250780102655, ++STORE, 47250780012544, 47250780102655, ++STORE, 47250779996160, 47250780012543, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780086271, ++STORE, 47250780086272, 47250780102655, ++STORE, 47250780065792, 47250780086271, ++STORE, 47250780012544, 47250780065791, ++ERASE, 47250780012544, 47250780012544, ++STORE, 47250780012544, 47250780065791, ++STORE, 47250780082176, 47250780086271, ++STORE, 47250780065792, 47250780082175, ++ERASE, 47250780065792, 47250780065792, ++STORE, 47250780065792, 47250780082175, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780094463, ++ERASE, 47250780094464, 47250780094464, ++STORE, 47250780094464, 47250780102655, ++STORE, 47250780102656, 47250781941759, ++STORE, 47250780241920, 47250781941759, ++STORE, 47250780102656, 47250780241919, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781900799, ++STORE, 47250781900800, 47250781941759, ++STORE, 47250781585408, 47250781900799, ++STORE, 47250780241920, 47250781585407, ++ERASE, 47250780241920, 47250780241920, ++STORE, 47250780241920, 47250781585407, ++STORE, 47250781896704, 47250781900799, ++STORE, 47250781585408, 47250781896703, ++ERASE, 47250781585408, 47250781585408, ++STORE, 47250781585408, 47250781896703, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781925375, ++ERASE, 47250781925376, 47250781925376, ++STORE, 47250781925376, 47250781941759, ++STORE, 47250781925376, 47250781954047, ++ERASE, 47250781900800, 47250781900800, ++STORE, 47250781900800, 47250781917183, ++STORE, 47250781917184, 47250781925375, ++ERASE, 47250780086272, 47250780086272, ++STORE, 47250780086272, 47250780090367, ++STORE, 47250780090368, 47250780094463, ++ERASE, 94425324986368, 94425324986368, ++STORE, 94425324986368, 94425325002751, ++STORE, 94425325002752, 94425325006847, ++ERASE, 140382015176704, 140382015176704, ++STORE, 140382015176704, 140382015180799, ++STORE, 140382015180800, 140382015184895, ++ERASE, 47250779979776, 47250779979776, ++STORE, 94425351438336, 94425351573503, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736801144832, 140737488351231, ++ERASE, 140736801144832, 140736801144832, ++STORE, 140736801144832, 140736801148927, ++STORE, 94629429358592, 94629430071295, ++ERASE, 94629429358592, 94629429358592, ++STORE, 94629429358592, 94629429407743, ++STORE, 94629429407744, 94629430071295, ++ERASE, 94629429407744, 94629429407744, ++STORE, 94629429407744, 94629429952511, ++STORE, 94629429952512, 94629430050815, ++STORE, 94629430050816, 94629430071295, ++STORE, 139801685483520, 139801685655551, ++ERASE, 139801685483520, 139801685483520, ++STORE, 139801685483520, 139801685487615, ++STORE, 139801685487616, 139801685655551, ++ERASE, 139801685487616, 139801685487616, ++STORE, 139801685487616, 139801685610495, ++STORE, 139801685610496, 139801685643263, ++STORE, 139801685643264, 139801685651455, ++STORE, 139801685651456, 139801685655551, ++STORE, 140736801198080, 140736801202175, ++STORE, 140736801185792, 140736801198079, ++STORE, 47831109513216, 47831109521407, ++STORE, 47831109521408, 47831109529599, ++STORE, 47831109529600, 47831109636095, ++STORE, 47831109545984, 47831109636095, ++STORE, 47831109529600, 47831109545983, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109619711, ++STORE, 47831109619712, 47831109636095, ++STORE, 47831109599232, 47831109619711, ++STORE, 47831109545984, 47831109599231, ++ERASE, 47831109545984, 47831109545984, ++STORE, 47831109545984, 47831109599231, ++STORE, 47831109615616, 47831109619711, ++STORE, 47831109599232, 47831109615615, ++ERASE, 47831109599232, 47831109599232, ++STORE, 47831109599232, 47831109615615, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109627903, ++ERASE, 47831109627904, 47831109627904, ++STORE, 47831109627904, 47831109636095, ++STORE, 47831109636096, 47831111475199, ++STORE, 47831109775360, 47831111475199, ++STORE, 47831109636096, 47831109775359, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111434239, ++STORE, 47831111434240, 47831111475199, ++STORE, 47831111118848, 47831111434239, ++STORE, 47831109775360, 47831111118847, ++ERASE, 47831109775360, 47831109775360, ++STORE, 47831109775360, 47831111118847, ++STORE, 47831111430144, 47831111434239, ++STORE, 47831111118848, 47831111430143, ++ERASE, 47831111118848, 47831111118848, ++STORE, 47831111118848, 47831111430143, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111458815, ++ERASE, 47831111458816, 47831111458816, ++STORE, 47831111458816, 47831111475199, ++STORE, 47831111458816, 47831111487487, ++ERASE, 47831111434240, 47831111434240, ++STORE, 47831111434240, 47831111450623, ++STORE, 47831111450624, 47831111458815, ++ERASE, 47831109619712, 47831109619712, ++STORE, 47831109619712, 47831109623807, ++STORE, 47831109623808, 47831109627903, ++ERASE, 94629430050816, 94629430050816, ++STORE, 94629430050816, 94629430067199, ++STORE, 94629430067200, 94629430071295, ++ERASE, 139801685643264, 139801685643264, ++STORE, 139801685643264, 139801685647359, ++STORE, 139801685647360, 139801685651455, ++ERASE, 47831109513216, 47831109513216, ++STORE, 140737488347136, 140737488351231, ++STORE, 140729419612160, 140737488351231, ++ERASE, 140729419612160, 140729419612160, ++STORE, 140729419612160, 140729419616255, ++STORE, 94443354148864, 94443354861567, ++ERASE, 94443354148864, 94443354148864, ++STORE, 94443354148864, 94443354198015, ++STORE, 94443354198016, 94443354861567, ++ERASE, 94443354198016, 94443354198016, ++STORE, 94443354198016, 94443354742783, ++STORE, 94443354742784, 94443354841087, ++STORE, 94443354841088, 94443354861567, ++STORE, 139741700038656, 139741700210687, ++ERASE, 139741700038656, 139741700038656, ++STORE, 139741700038656, 139741700042751, ++STORE, 139741700042752, 139741700210687, ++ERASE, 139741700042752, 139741700042752, ++STORE, 139741700042752, 139741700165631, ++STORE, 139741700165632, 139741700198399, ++STORE, 139741700198400, 139741700206591, ++STORE, 139741700206592, 139741700210687, ++STORE, 140729420574720, 140729420578815, ++STORE, 140729420562432, 140729420574719, ++STORE, 47891094958080, 47891094966271, ++STORE, 47891094966272, 47891094974463, ++STORE, 47891094974464, 47891095080959, ++STORE, 47891094990848, 47891095080959, ++STORE, 47891094974464, 47891094990847, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095064575, ++STORE, 47891095064576, 47891095080959, ++STORE, 47891095044096, 47891095064575, ++STORE, 47891094990848, 47891095044095, ++ERASE, 47891094990848, 47891094990848, ++STORE, 47891094990848, 47891095044095, ++STORE, 47891095060480, 47891095064575, ++STORE, 47891095044096, 47891095060479, ++ERASE, 47891095044096, 47891095044096, ++STORE, 47891095044096, 47891095060479, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095072767, ++ERASE, 47891095072768, 47891095072768, ++STORE, 47891095072768, 47891095080959, ++STORE, 47891095080960, 47891096920063, ++STORE, 47891095220224, 47891096920063, ++STORE, 47891095080960, 47891095220223, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096879103, ++STORE, 47891096879104, 47891096920063, ++STORE, 47891096563712, 47891096879103, ++STORE, 47891095220224, 47891096563711, ++ERASE, 47891095220224, 47891095220224, ++STORE, 47891095220224, 47891096563711, ++STORE, 47891096875008, 47891096879103, ++STORE, 47891096563712, 47891096875007, ++ERASE, 47891096563712, 47891096563712, ++STORE, 47891096563712, 47891096875007, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096903679, ++ERASE, 47891096903680, 47891096903680, ++STORE, 47891096903680, 47891096920063, ++STORE, 47891096903680, 47891096932351, ++ERASE, 47891096879104, 47891096879104, ++STORE, 47891096879104, 47891096895487, ++STORE, 47891096895488, 47891096903679, ++ERASE, 47891095064576, 47891095064576, ++STORE, 47891095064576, 47891095068671, ++STORE, 47891095068672, 47891095072767, ++ERASE, 94443354841088, 94443354841088, ++STORE, 94443354841088, 94443354857471, ++STORE, 94443354857472, 94443354861567, ++ERASE, 139741700198400, 139741700198400, ++STORE, 139741700198400, 139741700202495, ++STORE, 139741700202496, 139741700206591, ++ERASE, 47891094958080, 47891094958080, ++STORE, 94443360825344, 94443360960511, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722961661952, 140737488351231, ++ERASE, 140722961661952, 140722961661952, ++STORE, 140722961661952, 140722961666047, ++STORE, 94878388944896, 94878389657599, ++ERASE, 94878388944896, 94878388944896, ++STORE, 94878388944896, 94878388994047, ++STORE, 94878388994048, 94878389657599, ++ERASE, 94878388994048, 94878388994048, ++STORE, 94878388994048, 94878389538815, ++STORE, 94878389538816, 94878389637119, ++STORE, 94878389637120, 94878389657599, ++STORE, 140210690056192, 140210690228223, ++ERASE, 140210690056192, 140210690056192, ++STORE, 140210690056192, 140210690060287, ++STORE, 140210690060288, 140210690228223, ++ERASE, 140210690060288, 140210690060288, ++STORE, 140210690060288, 140210690183167, ++STORE, 140210690183168, 140210690215935, ++STORE, 140210690215936, 140210690224127, ++STORE, 140210690224128, 140210690228223, ++STORE, 140722963148800, 140722963152895, ++STORE, 140722963136512, 140722963148799, ++STORE, 47422104940544, 47422104948735, ++STORE, 47422104948736, 47422104956927, ++STORE, 47422104956928, 47422105063423, ++STORE, 47422104973312, 47422105063423, ++STORE, 47422104956928, 47422104973311, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105047039, ++STORE, 47422105047040, 47422105063423, ++STORE, 47422105026560, 47422105047039, ++STORE, 47422104973312, 47422105026559, ++ERASE, 47422104973312, 47422104973312, ++STORE, 47422104973312, 47422105026559, ++STORE, 47422105042944, 47422105047039, ++STORE, 47422105026560, 47422105042943, ++ERASE, 47422105026560, 47422105026560, ++STORE, 47422105026560, 47422105042943, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105055231, ++ERASE, 47422105055232, 47422105055232, ++STORE, 47422105055232, 47422105063423, ++STORE, 47422105063424, 47422106902527, ++STORE, 47422105202688, 47422106902527, ++STORE, 47422105063424, 47422105202687, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106861567, ++STORE, 47422106861568, 47422106902527, ++STORE, 47422106546176, 47422106861567, ++STORE, 47422105202688, 47422106546175, ++ERASE, 47422105202688, 47422105202688, ++STORE, 47422105202688, 47422106546175, ++STORE, 47422106857472, 47422106861567, ++STORE, 47422106546176, 47422106857471, ++ERASE, 47422106546176, 47422106546176, ++STORE, 47422106546176, 47422106857471, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106886143, ++ERASE, 47422106886144, 47422106886144, ++STORE, 47422106886144, 47422106902527, ++STORE, 47422106886144, 47422106914815, ++ERASE, 47422106861568, 47422106861568, ++STORE, 47422106861568, 47422106877951, ++STORE, 47422106877952, 47422106886143, ++ERASE, 47422105047040, 47422105047040, ++STORE, 47422105047040, 47422105051135, ++STORE, 47422105051136, 47422105055231, ++ERASE, 94878389637120, 94878389637120, ++STORE, 94878389637120, 94878389653503, ++STORE, 94878389653504, 94878389657599, ++ERASE, 140210690215936, 140210690215936, ++STORE, 140210690215936, 140210690220031, ++STORE, 140210690220032, 140210690224127, ++ERASE, 47422104940544, 47422104940544, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727690309632, 140737488351231, ++ERASE, 140727690309632, 140727690309632, ++STORE, 140727690309632, 140727690313727, ++STORE, 94121892208640, 94121892921343, ++ERASE, 94121892208640, 94121892208640, ++STORE, 94121892208640, 94121892257791, ++STORE, 94121892257792, 94121892921343, ++ERASE, 94121892257792, 94121892257792, ++STORE, 94121892257792, 94121892802559, ++STORE, 94121892802560, 94121892900863, ++STORE, 94121892900864, 94121892921343, ++STORE, 140662438326272, 140662438498303, ++ERASE, 140662438326272, 140662438326272, ++STORE, 140662438326272, 140662438330367, ++STORE, 140662438330368, 140662438498303, ++ERASE, 140662438330368, 140662438330368, ++STORE, 140662438330368, 140662438453247, ++STORE, 140662438453248, 140662438486015, ++STORE, 140662438486016, 140662438494207, ++STORE, 140662438494208, 140662438498303, ++STORE, 140727690379264, 140727690383359, ++STORE, 140727690366976, 140727690379263, ++STORE, 46970356670464, 46970356678655, ++STORE, 46970356678656, 46970356686847, ++STORE, 46970356686848, 46970356793343, ++STORE, 46970356703232, 46970356793343, ++STORE, 46970356686848, 46970356703231, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356776959, ++STORE, 46970356776960, 46970356793343, ++STORE, 46970356756480, 46970356776959, ++STORE, 46970356703232, 46970356756479, ++ERASE, 46970356703232, 46970356703232, ++STORE, 46970356703232, 46970356756479, ++STORE, 46970356772864, 46970356776959, ++STORE, 46970356756480, 46970356772863, ++ERASE, 46970356756480, 46970356756480, ++STORE, 46970356756480, 46970356772863, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356785151, ++ERASE, 46970356785152, 46970356785152, ++STORE, 46970356785152, 46970356793343, ++STORE, 46970356793344, 46970358632447, ++STORE, 46970356932608, 46970358632447, ++STORE, 46970356793344, 46970356932607, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358591487, ++STORE, 46970358591488, 46970358632447, ++STORE, 46970358276096, 46970358591487, ++STORE, 46970356932608, 46970358276095, ++ERASE, 46970356932608, 46970356932608, ++STORE, 46970356932608, 46970358276095, ++STORE, 46970358587392, 46970358591487, ++STORE, 46970358276096, 46970358587391, ++ERASE, 46970358276096, 46970358276096, ++STORE, 46970358276096, 46970358587391, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358616063, ++ERASE, 46970358616064, 46970358616064, ++STORE, 46970358616064, 46970358632447, ++STORE, 46970358616064, 46970358644735, ++ERASE, 46970358591488, 46970358591488, ++STORE, 46970358591488, 46970358607871, ++STORE, 46970358607872, 46970358616063, ++ERASE, 46970356776960, 46970356776960, ++STORE, 46970356776960, 46970356781055, ++STORE, 46970356781056, 46970356785151, ++ERASE, 94121892900864, 94121892900864, ++STORE, 94121892900864, 94121892917247, ++STORE, 94121892917248, 94121892921343, ++ERASE, 140662438486016, 140662438486016, ++STORE, 140662438486016, 140662438490111, ++STORE, 140662438490112, 140662438494207, ++ERASE, 46970356670464, 46970356670464, ++STORE, 94121898610688, 94121898745855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737189351424, 140737488351231, ++ERASE, 140737189351424, 140737189351424, ++STORE, 140737189351424, 140737189355519, ++STORE, 93847948832768, 93847949545471, ++ERASE, 93847948832768, 93847948832768, ++STORE, 93847948832768, 93847948881919, ++STORE, 93847948881920, 93847949545471, ++ERASE, 93847948881920, 93847948881920, ++STORE, 93847948881920, 93847949426687, ++STORE, 93847949426688, 93847949524991, ++STORE, 93847949524992, 93847949545471, ++STORE, 139698989985792, 139698990157823, ++ERASE, 139698989985792, 139698989985792, ++STORE, 139698989985792, 139698989989887, ++STORE, 139698989989888, 139698990157823, ++ERASE, 139698989989888, 139698989989888, ++STORE, 139698989989888, 139698990112767, ++STORE, 139698990112768, 139698990145535, ++STORE, 139698990145536, 139698990153727, ++STORE, 139698990153728, 139698990157823, ++STORE, 140737189744640, 140737189748735, ++STORE, 140737189732352, 140737189744639, ++STORE, 47933805010944, 47933805019135, ++STORE, 47933805019136, 47933805027327, ++STORE, 47933805027328, 47933805133823, ++STORE, 47933805043712, 47933805133823, ++STORE, 47933805027328, 47933805043711, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805117439, ++STORE, 47933805117440, 47933805133823, ++STORE, 47933805096960, 47933805117439, ++STORE, 47933805043712, 47933805096959, ++ERASE, 47933805043712, 47933805043712, ++STORE, 47933805043712, 47933805096959, ++STORE, 47933805113344, 47933805117439, ++STORE, 47933805096960, 47933805113343, ++ERASE, 47933805096960, 47933805096960, ++STORE, 47933805096960, 47933805113343, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805125631, ++ERASE, 47933805125632, 47933805125632, ++STORE, 47933805125632, 47933805133823, ++STORE, 47933805133824, 47933806972927, ++STORE, 47933805273088, 47933806972927, ++STORE, 47933805133824, 47933805273087, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806931967, ++STORE, 47933806931968, 47933806972927, ++STORE, 47933806616576, 47933806931967, ++STORE, 47933805273088, 47933806616575, ++ERASE, 47933805273088, 47933805273088, ++STORE, 47933805273088, 47933806616575, ++STORE, 47933806927872, 47933806931967, ++STORE, 47933806616576, 47933806927871, ++ERASE, 47933806616576, 47933806616576, ++STORE, 47933806616576, 47933806927871, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806956543, ++ERASE, 47933806956544, 47933806956544, ++STORE, 47933806956544, 47933806972927, ++STORE, 47933806956544, 47933806985215, ++ERASE, 47933806931968, 47933806931968, ++STORE, 47933806931968, 47933806948351, ++STORE, 47933806948352, 47933806956543, ++ERASE, 47933805117440, 47933805117440, ++STORE, 47933805117440, 47933805121535, ++STORE, 47933805121536, 47933805125631, ++ERASE, 93847949524992, 93847949524992, ++STORE, 93847949524992, 93847949541375, ++STORE, 93847949541376, 93847949545471, ++ERASE, 139698990145536, 139698990145536, ++STORE, 139698990145536, 139698990149631, ++STORE, 139698990149632, 139698990153727, ++ERASE, 47933805010944, 47933805010944, ++STORE, 140737488347136, 140737488351231, ++STORE, 140725553991680, 140737488351231, ++ERASE, 140725553991680, 140725553991680, ++STORE, 140725553991680, 140725553995775, ++STORE, 93980056248320, 93980056961023, ++ERASE, 93980056248320, 93980056248320, ++STORE, 93980056248320, 93980056297471, ++STORE, 93980056297472, 93980056961023, ++ERASE, 93980056297472, 93980056297472, ++STORE, 93980056297472, 93980056842239, ++STORE, 93980056842240, 93980056940543, ++STORE, 93980056940544, 93980056961023, ++STORE, 140146588971008, 140146589143039, ++ERASE, 140146588971008, 140146588971008, ++STORE, 140146588971008, 140146588975103, ++STORE, 140146588975104, 140146589143039, ++ERASE, 140146588975104, 140146588975104, ++STORE, 140146588975104, 140146589097983, ++STORE, 140146589097984, 140146589130751, ++STORE, 140146589130752, 140146589138943, ++STORE, 140146589138944, 140146589143039, ++STORE, 140725554860032, 140725554864127, ++STORE, 140725554847744, 140725554860031, ++STORE, 47486206025728, 47486206033919, ++STORE, 47486206033920, 47486206042111, ++STORE, 47486206042112, 47486206148607, ++STORE, 47486206058496, 47486206148607, ++STORE, 47486206042112, 47486206058495, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206132223, ++STORE, 47486206132224, 47486206148607, ++STORE, 47486206111744, 47486206132223, ++STORE, 47486206058496, 47486206111743, ++ERASE, 47486206058496, 47486206058496, ++STORE, 47486206058496, 47486206111743, ++STORE, 47486206128128, 47486206132223, ++STORE, 47486206111744, 47486206128127, ++ERASE, 47486206111744, 47486206111744, ++STORE, 47486206111744, 47486206128127, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206140415, ++ERASE, 47486206140416, 47486206140416, ++STORE, 47486206140416, 47486206148607, ++STORE, 47486206148608, 47486207987711, ++STORE, 47486206287872, 47486207987711, ++STORE, 47486206148608, 47486206287871, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207946751, ++STORE, 47486207946752, 47486207987711, ++STORE, 47486207631360, 47486207946751, ++STORE, 47486206287872, 47486207631359, ++ERASE, 47486206287872, 47486206287872, ++STORE, 47486206287872, 47486207631359, ++STORE, 47486207942656, 47486207946751, ++STORE, 47486207631360, 47486207942655, ++ERASE, 47486207631360, 47486207631360, ++STORE, 47486207631360, 47486207942655, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207971327, ++ERASE, 47486207971328, 47486207971328, ++STORE, 47486207971328, 47486207987711, ++STORE, 47486207971328, 47486207999999, ++ERASE, 47486207946752, 47486207946752, ++STORE, 47486207946752, 47486207963135, ++STORE, 47486207963136, 47486207971327, ++ERASE, 47486206132224, 47486206132224, ++STORE, 47486206132224, 47486206136319, ++STORE, 47486206136320, 47486206140415, ++ERASE, 93980056940544, 93980056940544, ++STORE, 93980056940544, 93980056956927, ++STORE, 93980056956928, 93980056961023, ++ERASE, 140146589130752, 140146589130752, ++STORE, 140146589130752, 140146589134847, ++STORE, 140146589134848, 140146589138943, ++ERASE, 47486206025728, 47486206025728, ++STORE, 93980070006784, 93980070141951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727334776832, 140737488351231, ++ERASE, 140727334776832, 140727334776832, ++STORE, 140727334776832, 140727334780927, ++STORE, 94049747247104, 94049747959807, ++ERASE, 94049747247104, 94049747247104, ++STORE, 94049747247104, 94049747296255, ++STORE, 94049747296256, 94049747959807, ++ERASE, 94049747296256, 94049747296256, ++STORE, 94049747296256, 94049747841023, ++STORE, 94049747841024, 94049747939327, ++STORE, 94049747939328, 94049747959807, ++STORE, 140227307216896, 140227307388927, ++ERASE, 140227307216896, 140227307216896, ++STORE, 140227307216896, 140227307220991, ++STORE, 140227307220992, 140227307388927, ++ERASE, 140227307220992, 140227307220992, ++STORE, 140227307220992, 140227307343871, ++STORE, 140227307343872, 140227307376639, ++STORE, 140227307376640, 140227307384831, ++STORE, 140227307384832, 140227307388927, ++STORE, 140727335337984, 140727335342079, ++STORE, 140727335325696, 140727335337983, ++STORE, 47405487779840, 47405487788031, ++STORE, 47405487788032, 47405487796223, ++STORE, 47405487796224, 47405487902719, ++STORE, 47405487812608, 47405487902719, ++STORE, 47405487796224, 47405487812607, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487886335, ++STORE, 47405487886336, 47405487902719, ++STORE, 47405487865856, 47405487886335, ++STORE, 47405487812608, 47405487865855, ++ERASE, 47405487812608, 47405487812608, ++STORE, 47405487812608, 47405487865855, ++STORE, 47405487882240, 47405487886335, ++STORE, 47405487865856, 47405487882239, ++ERASE, 47405487865856, 47405487865856, ++STORE, 47405487865856, 47405487882239, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487894527, ++ERASE, 47405487894528, 47405487894528, ++STORE, 47405487894528, 47405487902719, ++STORE, 47405487902720, 47405489741823, ++STORE, 47405488041984, 47405489741823, ++STORE, 47405487902720, 47405488041983, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489700863, ++STORE, 47405489700864, 47405489741823, ++STORE, 47405489385472, 47405489700863, ++STORE, 47405488041984, 47405489385471, ++ERASE, 47405488041984, 47405488041984, ++STORE, 47405488041984, 47405489385471, ++STORE, 47405489696768, 47405489700863, ++STORE, 47405489385472, 47405489696767, ++ERASE, 47405489385472, 47405489385472, ++STORE, 47405489385472, 47405489696767, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489725439, ++ERASE, 47405489725440, 47405489725440, ++STORE, 47405489725440, 47405489741823, ++STORE, 47405489725440, 47405489754111, ++ERASE, 47405489700864, 47405489700864, ++STORE, 47405489700864, 47405489717247, ++STORE, 47405489717248, 47405489725439, ++ERASE, 47405487886336, 47405487886336, ++STORE, 47405487886336, 47405487890431, ++STORE, 47405487890432, 47405487894527, ++ERASE, 94049747939328, 94049747939328, ++STORE, 94049747939328, 94049747955711, ++STORE, 94049747955712, 94049747959807, ++ERASE, 140227307376640, 140227307376640, ++STORE, 140227307376640, 140227307380735, ++STORE, 140227307380736, 140227307384831, ++ERASE, 47405487779840, 47405487779840, ++STORE, 94049758810112, 94049758945279, ++STORE, 140737488347136, 140737488351231, ++STORE, 140727079718912, 140737488351231, ++ERASE, 140727079718912, 140727079718912, ++STORE, 140727079718912, 140727079723007, ++STORE, 94250996527104, 94250997239807, ++ERASE, 94250996527104, 94250996527104, ++STORE, 94250996527104, 94250996576255, ++STORE, 94250996576256, 94250997239807, ++ERASE, 94250996576256, 94250996576256, ++STORE, 94250996576256, 94250997121023, ++STORE, 94250997121024, 94250997219327, ++STORE, 94250997219328, 94250997239807, ++STORE, 140060022587392, 140060022759423, ++ERASE, 140060022587392, 140060022587392, ++STORE, 140060022587392, 140060022591487, ++STORE, 140060022591488, 140060022759423, ++ERASE, 140060022591488, 140060022591488, ++STORE, 140060022591488, 140060022714367, ++STORE, 140060022714368, 140060022747135, ++STORE, 140060022747136, 140060022755327, ++STORE, 140060022755328, 140060022759423, ++STORE, 140727079788544, 140727079792639, ++STORE, 140727079776256, 140727079788543, ++STORE, 47572772409344, 47572772417535, ++STORE, 47572772417536, 47572772425727, ++STORE, 47572772425728, 47572772532223, ++STORE, 47572772442112, 47572772532223, ++STORE, 47572772425728, 47572772442111, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772515839, ++STORE, 47572772515840, 47572772532223, ++STORE, 47572772495360, 47572772515839, ++STORE, 47572772442112, 47572772495359, ++ERASE, 47572772442112, 47572772442112, ++STORE, 47572772442112, 47572772495359, ++STORE, 47572772511744, 47572772515839, ++STORE, 47572772495360, 47572772511743, ++ERASE, 47572772495360, 47572772495360, ++STORE, 47572772495360, 47572772511743, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772524031, ++ERASE, 47572772524032, 47572772524032, ++STORE, 47572772524032, 47572772532223, ++STORE, 47572772532224, 47572774371327, ++STORE, 47572772671488, 47572774371327, ++STORE, 47572772532224, 47572772671487, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774330367, ++STORE, 47572774330368, 47572774371327, ++STORE, 47572774014976, 47572774330367, ++STORE, 47572772671488, 47572774014975, ++ERASE, 47572772671488, 47572772671488, ++STORE, 47572772671488, 47572774014975, ++STORE, 47572774326272, 47572774330367, ++STORE, 47572774014976, 47572774326271, ++ERASE, 47572774014976, 47572774014976, ++STORE, 47572774014976, 47572774326271, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774354943, ++ERASE, 47572774354944, 47572774354944, ++STORE, 47572774354944, 47572774371327, ++STORE, 47572774354944, 47572774383615, ++ERASE, 47572774330368, 47572774330368, ++STORE, 47572774330368, 47572774346751, ++STORE, 47572774346752, 47572774354943, ++ERASE, 47572772515840, 47572772515840, ++STORE, 47572772515840, 47572772519935, ++STORE, 47572772519936, 47572772524031, ++ERASE, 94250997219328, 94250997219328, ++STORE, 94250997219328, 94250997235711, ++STORE, 94250997235712, 94250997239807, ++ERASE, 140060022747136, 140060022747136, ++STORE, 140060022747136, 140060022751231, ++STORE, 140060022751232, 140060022755327, ++ERASE, 47572772409344, 47572772409344, ++STORE, 94251018305536, 94251018440703, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730012389376, 140737488351231, ++ERASE, 140730012389376, 140730012389376, ++STORE, 140730012389376, 140730012393471, ++STORE, 94382607675392, 94382607695871, ++ERASE, 94382607675392, 94382607675392, ++STORE, 94382607675392, 94382607679487, ++STORE, 94382607679488, 94382607695871, ++ERASE, 94382607679488, 94382607679488, ++STORE, 94382607679488, 94382607683583, ++STORE, 94382607683584, 94382607687679, ++STORE, 94382607687680, 94382607695871, ++STORE, 140252451454976, 140252451627007, ++ERASE, 140252451454976, 140252451454976, ++STORE, 140252451454976, 140252451459071, ++STORE, 140252451459072, 140252451627007, ++ERASE, 140252451459072, 140252451459072, ++STORE, 140252451459072, 140252451581951, ++STORE, 140252451581952, 140252451614719, ++STORE, 140252451614720, 140252451622911, ++STORE, 140252451622912, 140252451627007, ++STORE, 140730013548544, 140730013552639, ++STORE, 140730013536256, 140730013548543, ++STORE, 47380343541760, 47380343549951, ++STORE, 47380343549952, 47380343558143, ++STORE, 47380343558144, 47380345397247, ++STORE, 47380343697408, 47380345397247, ++STORE, 47380343558144, 47380343697407, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345356287, ++STORE, 47380345356288, 47380345397247, ++STORE, 47380345040896, 47380345356287, ++STORE, 47380343697408, 47380345040895, ++ERASE, 47380343697408, 47380343697408, ++STORE, 47380343697408, 47380345040895, ++STORE, 47380345352192, 47380345356287, ++STORE, 47380345040896, 47380345352191, ++ERASE, 47380345040896, 47380345040896, ++STORE, 47380345040896, 47380345352191, ++STORE, 47380345380864, 47380345397247, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345380863, ++ERASE, 47380345380864, 47380345380864, ++STORE, 47380345380864, 47380345397247, ++ERASE, 47380345356288, 47380345356288, ++STORE, 47380345356288, 47380345372671, ++STORE, 47380345372672, 47380345380863, ++ERASE, 94382607687680, 94382607687680, ++STORE, 94382607687680, 94382607691775, ++STORE, 94382607691776, 94382607695871, ++ERASE, 140252451614720, 140252451614720, ++STORE, 140252451614720, 140252451618815, ++STORE, 140252451618816, 140252451622911, ++ERASE, 47380343541760, 47380343541760, ++STORE, 94382626803712, 94382626938879, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730900271104, 140737488351231, ++ERASE, 140730900271104, 140730900271104, ++STORE, 140730900271104, 140730900275199, ++STORE, 93855478120448, 93855478337535, ++ERASE, 93855478120448, 93855478120448, ++STORE, 93855478120448, 93855478198271, ++STORE, 93855478198272, 93855478337535, ++ERASE, 93855478198272, 93855478198272, ++STORE, 93855478198272, 93855478243327, ++STORE, 93855478243328, 93855478288383, ++STORE, 93855478288384, 93855478337535, ++STORE, 140092686573568, 140092686745599, ++ERASE, 140092686573568, 140092686573568, ++STORE, 140092686573568, 140092686577663, ++STORE, 140092686577664, 140092686745599, ++ERASE, 140092686577664, 140092686577664, ++STORE, 140092686577664, 140092686700543, ++STORE, 140092686700544, 140092686733311, ++STORE, 140092686733312, 140092686741503, ++STORE, 140092686741504, 140092686745599, ++STORE, 140730900537344, 140730900541439, ++STORE, 140730900525056, 140730900537343, ++STORE, 47540108423168, 47540108431359, ++STORE, 47540108431360, 47540108439551, ++STORE, 47540108439552, 47540110278655, ++STORE, 47540108578816, 47540110278655, ++STORE, 47540108439552, 47540108578815, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540110237695, ++STORE, 47540110237696, 47540110278655, ++STORE, 47540109922304, 47540110237695, ++STORE, 47540108578816, 47540109922303, ++ERASE, 47540108578816, 47540108578816, ++STORE, 47540108578816, 47540109922303, ++STORE, 47540110233600, 47540110237695, ++STORE, 47540109922304, 47540110233599, ++ERASE, 47540109922304, 47540109922304, ++STORE, 47540109922304, 47540110233599, ++STORE, 47540110262272, 47540110278655, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110262271, ++ERASE, 47540110262272, 47540110262272, ++STORE, 47540110262272, 47540110278655, ++ERASE, 47540110237696, 47540110237696, ++STORE, 47540110237696, 47540110254079, ++STORE, 47540110254080, 47540110262271, ++ERASE, 93855478288384, 93855478288384, ++STORE, 93855478288384, 93855478333439, ++STORE, 93855478333440, 93855478337535, ++ERASE, 140092686733312, 140092686733312, ++STORE, 140092686733312, 140092686737407, ++STORE, 140092686737408, 140092686741503, ++ERASE, 47540108423168, 47540108423168, ++STORE, 93855492222976, 93855492358143, ++STORE, 93855492222976, 93855492493311, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733498146816, 140737488351231, ++ERASE, 140733498146816, 140733498146816, ++STORE, 140733498146816, 140733498150911, ++STORE, 94170739654656, 94170740367359, ++ERASE, 94170739654656, 94170739654656, ++STORE, 94170739654656, 94170739703807, ++STORE, 94170739703808, 94170740367359, ++ERASE, 94170739703808, 94170739703808, ++STORE, 94170739703808, 94170740248575, ++STORE, 94170740248576, 94170740346879, ++STORE, 94170740346880, 94170740367359, ++STORE, 140024788877312, 140024789049343, ++ERASE, 140024788877312, 140024788877312, ++STORE, 140024788877312, 140024788881407, ++STORE, 140024788881408, 140024789049343, ++ERASE, 140024788881408, 140024788881408, ++STORE, 140024788881408, 140024789004287, ++STORE, 140024789004288, 140024789037055, ++STORE, 140024789037056, 140024789045247, ++STORE, 140024789045248, 140024789049343, ++STORE, 140733499023360, 140733499027455, ++STORE, 140733499011072, 140733499023359, ++STORE, 47608006119424, 47608006127615, ++STORE, 47608006127616, 47608006135807, ++STORE, 47608006135808, 47608006242303, ++STORE, 47608006152192, 47608006242303, ++STORE, 47608006135808, 47608006152191, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006225919, ++STORE, 47608006225920, 47608006242303, ++STORE, 47608006205440, 47608006225919, ++STORE, 47608006152192, 47608006205439, ++ERASE, 47608006152192, 47608006152192, ++STORE, 47608006152192, 47608006205439, ++STORE, 47608006221824, 47608006225919, ++STORE, 47608006205440, 47608006221823, ++ERASE, 47608006205440, 47608006205440, ++STORE, 47608006205440, 47608006221823, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006234111, ++ERASE, 47608006234112, 47608006234112, ++STORE, 47608006234112, 47608006242303, ++STORE, 47608006242304, 47608008081407, ++STORE, 47608006381568, 47608008081407, ++STORE, 47608006242304, 47608006381567, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608008040447, ++STORE, 47608008040448, 47608008081407, ++STORE, 47608007725056, 47608008040447, ++STORE, 47608006381568, 47608007725055, ++ERASE, 47608006381568, 47608006381568, ++STORE, 47608006381568, 47608007725055, ++STORE, 47608008036352, 47608008040447, ++STORE, 47608007725056, 47608008036351, ++ERASE, 47608007725056, 47608007725056, ++STORE, 47608007725056, 47608008036351, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008065023, ++ERASE, 47608008065024, 47608008065024, ++STORE, 47608008065024, 47608008081407, ++STORE, 47608008065024, 47608008093695, ++ERASE, 47608008040448, 47608008040448, ++STORE, 47608008040448, 47608008056831, ++STORE, 47608008056832, 47608008065023, ++ERASE, 47608006225920, 47608006225920, ++STORE, 47608006225920, 47608006230015, ++STORE, 47608006230016, 47608006234111, ++ERASE, 94170740346880, 94170740346880, ++STORE, 94170740346880, 94170740363263, ++STORE, 94170740363264, 94170740367359, ++ERASE, 140024789037056, 140024789037056, ++STORE, 140024789037056, 140024789041151, ++STORE, 140024789041152, 140024789045247, ++ERASE, 47608006119424, 47608006119424, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730264326144, 140737488351231, ++ERASE, 140730264326144, 140730264326144, ++STORE, 140730264326144, 140730264330239, ++STORE, 94653216407552, 94653217120255, ++ERASE, 94653216407552, 94653216407552, ++STORE, 94653216407552, 94653216456703, ++STORE, 94653216456704, 94653217120255, ++ERASE, 94653216456704, 94653216456704, ++STORE, 94653216456704, 94653217001471, ++STORE, 94653217001472, 94653217099775, ++STORE, 94653217099776, 94653217120255, ++STORE, 140103617011712, 140103617183743, ++ERASE, 140103617011712, 140103617011712, ++STORE, 140103617011712, 140103617015807, ++STORE, 140103617015808, 140103617183743, ++ERASE, 140103617015808, 140103617015808, ++STORE, 140103617015808, 140103617138687, ++STORE, 140103617138688, 140103617171455, ++STORE, 140103617171456, 140103617179647, ++STORE, 140103617179648, 140103617183743, ++STORE, 140730265427968, 140730265432063, ++STORE, 140730265415680, 140730265427967, ++STORE, 47529177985024, 47529177993215, ++STORE, 47529177993216, 47529178001407, ++STORE, 47529178001408, 47529178107903, ++STORE, 47529178017792, 47529178107903, ++STORE, 47529178001408, 47529178017791, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178091519, ++STORE, 47529178091520, 47529178107903, ++STORE, 47529178071040, 47529178091519, ++STORE, 47529178017792, 47529178071039, ++ERASE, 47529178017792, 47529178017792, ++STORE, 47529178017792, 47529178071039, ++STORE, 47529178087424, 47529178091519, ++STORE, 47529178071040, 47529178087423, ++ERASE, 47529178071040, 47529178071040, ++STORE, 47529178071040, 47529178087423, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178099711, ++ERASE, 47529178099712, 47529178099712, ++STORE, 47529178099712, 47529178107903, ++STORE, 47529178107904, 47529179947007, ++STORE, 47529178247168, 47529179947007, ++STORE, 47529178107904, 47529178247167, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179906047, ++STORE, 47529179906048, 47529179947007, ++STORE, 47529179590656, 47529179906047, ++STORE, 47529178247168, 47529179590655, ++ERASE, 47529178247168, 47529178247168, ++STORE, 47529178247168, 47529179590655, ++STORE, 47529179901952, 47529179906047, ++STORE, 47529179590656, 47529179901951, ++ERASE, 47529179590656, 47529179590656, ++STORE, 47529179590656, 47529179901951, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179930623, ++ERASE, 47529179930624, 47529179930624, ++STORE, 47529179930624, 47529179947007, ++STORE, 47529179930624, 47529179959295, ++ERASE, 47529179906048, 47529179906048, ++STORE, 47529179906048, 47529179922431, ++STORE, 47529179922432, 47529179930623, ++ERASE, 47529178091520, 47529178091520, ++STORE, 47529178091520, 47529178095615, ++STORE, 47529178095616, 47529178099711, ++ERASE, 94653217099776, 94653217099776, ++STORE, 94653217099776, 94653217116159, ++STORE, 94653217116160, 94653217120255, ++ERASE, 140103617171456, 140103617171456, ++STORE, 140103617171456, 140103617175551, ++STORE, 140103617175552, 140103617179647, ++ERASE, 47529177985024, 47529177985024, ++STORE, 94653241135104, 94653241270271, ++STORE, 140737488347136, 140737488351231, ++STORE, 140736284549120, 140737488351231, ++ERASE, 140736284549120, 140736284549120, ++STORE, 140736284549120, 140736284553215, ++STORE, 93963663822848, 93963664506879, ++ERASE, 93963663822848, 93963663822848, ++STORE, 93963663822848, 93963663884287, ++STORE, 93963663884288, 93963664506879, ++ERASE, 93963663884288, 93963663884288, ++STORE, 93963663884288, 93963664240639, ++STORE, 93963664240640, 93963664379903, ++STORE, 93963664379904, 93963664506879, ++STORE, 140450188439552, 140450188611583, ++ERASE, 140450188439552, 140450188439552, ++STORE, 140450188439552, 140450188443647, ++STORE, 140450188443648, 140450188611583, ++ERASE, 140450188443648, 140450188443648, ++STORE, 140450188443648, 140450188566527, ++STORE, 140450188566528, 140450188599295, ++STORE, 140450188599296, 140450188607487, ++STORE, 140450188607488, 140450188611583, ++STORE, 140736284577792, 140736284581887, ++STORE, 140736284565504, 140736284577791, ++STORE, 47182606557184, 47182606565375, ++STORE, 47182606565376, 47182606573567, ++STORE, 47182606573568, 47182608412671, ++STORE, 47182606712832, 47182608412671, ++STORE, 47182606573568, 47182606712831, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608371711, ++STORE, 47182608371712, 47182608412671, ++STORE, 47182608056320, 47182608371711, ++STORE, 47182606712832, 47182608056319, ++ERASE, 47182606712832, 47182606712832, ++STORE, 47182606712832, 47182608056319, ++STORE, 47182608367616, 47182608371711, ++STORE, 47182608056320, 47182608367615, ++ERASE, 47182608056320, 47182608056320, ++STORE, 47182608056320, 47182608367615, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608396287, ++ERASE, 47182608396288, 47182608396288, ++STORE, 47182608396288, 47182608412671, ++STORE, 47182608412672, 47182608523263, ++STORE, 47182608429056, 47182608523263, ++STORE, 47182608412672, 47182608429055, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608515071, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608490496, 47182608515071, ++STORE, 47182608429056, 47182608490495, ++ERASE, 47182608429056, 47182608429056, ++STORE, 47182608429056, 47182608490495, ++STORE, 47182608510976, 47182608515071, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608490496, 47182608490496, ++STORE, 47182608490496, 47182608510975, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608523263, ++STORE, 47182608523264, 47182608568319, ++ERASE, 47182608523264, 47182608523264, ++STORE, 47182608523264, 47182608531455, ++STORE, 47182608531456, 47182608568319, ++STORE, 47182608551936, 47182608568319, ++STORE, 47182608531456, 47182608551935, ++ERASE, 47182608531456, 47182608531456, ++STORE, 47182608531456, 47182608551935, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608551936, 47182608560127, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608568319, ++ERASE, 47182608551936, 47182608551936, ++STORE, 47182608551936, 47182608560127, ++STORE, 47182608560128, 47182608568319, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608568319, ++STORE, 47182608568320, 47182608916479, ++STORE, 47182608609280, 47182608916479, ++STORE, 47182608568320, 47182608609279, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608891903, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608822272, 47182608891903, ++STORE, 47182608609280, 47182608822271, ++ERASE, 47182608609280, 47182608609280, ++STORE, 47182608609280, 47182608822271, ++STORE, 47182608887808, 47182608891903, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608822272, 47182608822272, ++STORE, 47182608822272, 47182608887807, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608916479, ++STORE, 47182608916480, 47182611177471, ++STORE, 47182609068032, 47182611177471, ++STORE, 47182608916480, 47182609068031, ++ERASE, 47182609068032, 47182609068032, ++STORE, 47182609068032, 47182611161087, ++STORE, 47182611161088, 47182611177471, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611169279, ++ERASE, 47182611169280, 47182611169280, ++STORE, 47182611169280, 47182611177471, ++STORE, 47182611177472, 47182611312639, ++ERASE, 47182611177472, 47182611177472, ++STORE, 47182611177472, 47182611202047, ++STORE, 47182611202048, 47182611312639, ++STORE, 47182611263488, 47182611312639, ++STORE, 47182611202048, 47182611263487, ++ERASE, 47182611202048, 47182611202048, ++STORE, 47182611202048, 47182611263487, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611263488, 47182611288063, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611312639, ++ERASE, 47182611263488, 47182611263488, ++STORE, 47182611263488, 47182611288063, ++STORE, 47182611288064, 47182611312639, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611296255, ++ERASE, 47182611296256, 47182611296256, ++STORE, 47182611296256, 47182611312639, ++STORE, 47182611296256, 47182611320831, ++STORE, 47182611320832, 47182611484671, ++ERASE, 47182611320832, 47182611320832, ++STORE, 47182611320832, 47182611333119, ++STORE, 47182611333120, 47182611484671, ++STORE, 47182611431424, 47182611484671, ++STORE, 47182611333120, 47182611431423, ++ERASE, 47182611333120, 47182611333120, ++STORE, 47182611333120, 47182611431423, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611431424, 47182611476479, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611484671, ++ERASE, 47182611431424, 47182611431424, ++STORE, 47182611431424, 47182611476479, ++STORE, 47182611476480, 47182611484671, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611484671, ++STORE, 47182611484672, 47182612082687, ++STORE, 47182611603456, 47182612082687, ++STORE, 47182611484672, 47182611603455, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182612029439, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182611918848, 47182612029439, ++STORE, 47182611603456, 47182611918847, ++ERASE, 47182611603456, 47182611603456, ++STORE, 47182611603456, 47182611918847, ++STORE, 47182612025344, 47182612029439, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182611918848, 47182611918848, ++STORE, 47182611918848, 47182612025343, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612082687, ++STORE, 47182612082688, 47182615134207, ++STORE, 47182612627456, 47182615134207, ++STORE, 47182612082688, 47182612627455, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614913023, ++STORE, 47182614913024, 47182615134207, ++STORE, 47182614323200, 47182614913023, ++STORE, 47182612627456, 47182614323199, ++ERASE, 47182612627456, 47182612627456, ++STORE, 47182612627456, 47182614323199, ++STORE, 47182614908928, 47182614913023, ++STORE, 47182614323200, 47182614908927, ++ERASE, 47182614323200, 47182614323200, ++STORE, 47182614323200, 47182614908927, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615117823, ++ERASE, 47182615117824, 47182615117824, ++STORE, 47182615117824, 47182615134207, ++STORE, 47182615134208, 47182615166975, ++ERASE, 47182615134208, 47182615134208, ++STORE, 47182615134208, 47182615142399, ++STORE, 47182615142400, 47182615166975, ++STORE, 47182615154688, 47182615166975, ++STORE, 47182615142400, 47182615154687, ++ERASE, 47182615142400, 47182615142400, ++STORE, 47182615142400, 47182615154687, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615154688, 47182615158783, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615166975, ++ERASE, 47182615154688, 47182615154688, ++STORE, 47182615154688, 47182615158783, ++STORE, 47182615158784, 47182615166975, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615166975, ++STORE, 47182615166976, 47182615203839, ++ERASE, 47182615166976, 47182615166976, ++STORE, 47182615166976, 47182615175167, ++STORE, 47182615175168, 47182615203839, ++STORE, 47182615191552, 47182615203839, ++STORE, 47182615175168, 47182615191551, ++ERASE, 47182615175168, 47182615175168, ++STORE, 47182615175168, 47182615191551, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615191552, 47182615195647, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615203839, ++ERASE, 47182615191552, 47182615191552, ++STORE, 47182615191552, 47182615195647, ++STORE, 47182615195648, 47182615203839, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615203839, ++STORE, 47182615203840, 47182615678975, ++ERASE, 47182615203840, 47182615203840, ++STORE, 47182615203840, 47182615212031, ++STORE, 47182615212032, 47182615678975, ++STORE, 47182615547904, 47182615678975, ++STORE, 47182615212032, 47182615547903, ++ERASE, 47182615212032, 47182615212032, ++STORE, 47182615212032, 47182615547903, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615547904, 47182615670783, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615678975, ++ERASE, 47182615547904, 47182615547904, ++STORE, 47182615547904, 47182615670783, ++STORE, 47182615670784, 47182615678975, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615678975, ++STORE, 47182615678976, 47182615687167, ++STORE, 47182615687168, 47182615707647, ++ERASE, 47182615687168, 47182615687168, ++STORE, 47182615687168, 47182615691263, ++STORE, 47182615691264, 47182615707647, ++STORE, 47182615695360, 47182615707647, ++STORE, 47182615691264, 47182615695359, ++ERASE, 47182615691264, 47182615691264, ++STORE, 47182615691264, 47182615695359, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615695360, 47182615699455, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615707647, ++ERASE, 47182615695360, 47182615695360, ++STORE, 47182615695360, 47182615699455, ++STORE, 47182615699456, 47182615707647, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615707647, ++STORE, 47182615707648, 47182615715839, ++ERASE, 47182608371712, 47182608371712, ++STORE, 47182608371712, 47182608388095, ++STORE, 47182608388096, 47182608396287, ++ERASE, 47182615699456, 47182615699456, ++STORE, 47182615699456, 47182615703551, ++STORE, 47182615703552, 47182615707647, ++ERASE, 47182611288064, 47182611288064, ++STORE, 47182611288064, 47182611292159, ++STORE, 47182611292160, 47182611296255, ++ERASE, 47182615670784, 47182615670784, ++STORE, 47182615670784, 47182615674879, ++STORE, 47182615674880, 47182615678975, ++ERASE, 47182615195648, 47182615195648, ++STORE, 47182615195648, 47182615199743, ++STORE, 47182615199744, 47182615203839, ++ERASE, 47182615158784, 47182615158784, ++STORE, 47182615158784, 47182615162879, ++STORE, 47182615162880, 47182615166975, ++ERASE, 47182614913024, 47182614913024, ++STORE, 47182614913024, 47182615109631, ++STORE, 47182615109632, 47182615117823, ++ERASE, 47182612029440, 47182612029440, ++STORE, 47182612029440, 47182612066303, ++STORE, 47182612066304, 47182612082687, ++ERASE, 47182611476480, 47182611476480, ++STORE, 47182611476480, 47182611480575, ++STORE, 47182611480576, 47182611484671, ++ERASE, 47182611161088, 47182611161088, ++STORE, 47182611161088, 47182611165183, ++STORE, 47182611165184, 47182611169279, ++ERASE, 47182608891904, 47182608891904, ++STORE, 47182608891904, 47182608912383, ++STORE, 47182608912384, 47182608916479, ++ERASE, 47182608560128, 47182608560128, ++STORE, 47182608560128, 47182608564223, ++STORE, 47182608564224, 47182608568319, ++ERASE, 47182608515072, 47182608515072, ++STORE, 47182608515072, 47182608519167, ++STORE, 47182608519168, 47182608523263, ++ERASE, 93963664379904, 93963664379904, ++STORE, 93963664379904, 93963664502783, ++STORE, 93963664502784, 93963664506879, ++ERASE, 140450188599296, 140450188599296, ++STORE, 140450188599296, 140450188603391, ++STORE, 140450188603392, 140450188607487, ++ERASE, 47182606557184, 47182606557184, ++STORE, 93963694723072, 93963694858239, ++STORE, 140737488347136, 140737488351231, ++STORE, 140730313261056, 140737488351231, ++ERASE, 140730313261056, 140730313261056, ++STORE, 140730313261056, 140730313265151, ++STORE, 94386579017728, 94386579697663, ++ERASE, 94386579017728, 94386579017728, ++STORE, 94386579017728, 94386579083263, ++STORE, 94386579083264, 94386579697663, ++ERASE, 94386579083264, 94386579083264, ++STORE, 94386579083264, 94386579431423, ++STORE, 94386579431424, 94386579570687, ++STORE, 94386579570688, 94386579697663, ++STORE, 140124810838016, 140124811010047, ++ERASE, 140124810838016, 140124810838016, ++STORE, 140124810838016, 140124810842111, ++STORE, 140124810842112, 140124811010047, ++ERASE, 140124810842112, 140124810842112, ++STORE, 140124810842112, 140124810964991, ++STORE, 140124810964992, 140124810997759, ++STORE, 140124810997760, 140124811005951, ++STORE, 140124811005952, 140124811010047, ++STORE, 140730313601024, 140730313605119, ++STORE, 140730313588736, 140730313601023, ++STORE, 47507984158720, 47507984166911, ++STORE, 47507984166912, 47507984175103, ++STORE, 47507984175104, 47507986014207, ++STORE, 47507984314368, 47507986014207, ++STORE, 47507984175104, 47507984314367, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985973247, ++STORE, 47507985973248, 47507986014207, ++STORE, 47507985657856, 47507985973247, ++STORE, 47507984314368, 47507985657855, ++ERASE, 47507984314368, 47507984314368, ++STORE, 47507984314368, 47507985657855, ++STORE, 47507985969152, 47507985973247, ++STORE, 47507985657856, 47507985969151, ++ERASE, 47507985657856, 47507985657856, ++STORE, 47507985657856, 47507985969151, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985997823, ++ERASE, 47507985997824, 47507985997824, ++STORE, 47507985997824, 47507986014207, ++STORE, 47507986014208, 47507986124799, ++STORE, 47507986030592, 47507986124799, ++STORE, 47507986014208, 47507986030591, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986116607, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986092032, 47507986116607, ++STORE, 47507986030592, 47507986092031, ++ERASE, 47507986030592, 47507986030592, ++STORE, 47507986030592, 47507986092031, ++STORE, 47507986112512, 47507986116607, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986092032, 47507986092032, ++STORE, 47507986092032, 47507986112511, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986124799, ++STORE, 47507986124800, 47507986169855, ++ERASE, 47507986124800, 47507986124800, ++STORE, 47507986124800, 47507986132991, ++STORE, 47507986132992, 47507986169855, ++STORE, 47507986153472, 47507986169855, ++STORE, 47507986132992, 47507986153471, ++ERASE, 47507986132992, 47507986132992, ++STORE, 47507986132992, 47507986153471, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986153472, 47507986161663, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986169855, ++ERASE, 47507986153472, 47507986153472, ++STORE, 47507986153472, 47507986161663, ++STORE, 47507986161664, 47507986169855, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986169855, ++STORE, 47507986169856, 47507986518015, ++STORE, 47507986210816, 47507986518015, ++STORE, 47507986169856, 47507986210815, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986493439, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986423808, 47507986493439, ++STORE, 47507986210816, 47507986423807, ++ERASE, 47507986210816, 47507986210816, ++STORE, 47507986210816, 47507986423807, ++STORE, 47507986489344, 47507986493439, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986423808, 47507986423808, ++STORE, 47507986423808, 47507986489343, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986518015, ++STORE, 47507986518016, 47507988779007, ++STORE, 47507986669568, 47507988779007, ++STORE, 47507986518016, 47507986669567, ++ERASE, 47507986669568, 47507986669568, ++STORE, 47507986669568, 47507988762623, ++STORE, 47507988762624, 47507988779007, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988770815, ++ERASE, 47507988770816, 47507988770816, ++STORE, 47507988770816, 47507988779007, ++STORE, 47507988779008, 47507988914175, ++ERASE, 47507988779008, 47507988779008, ++STORE, 47507988779008, 47507988803583, ++STORE, 47507988803584, 47507988914175, ++STORE, 47507988865024, 47507988914175, ++STORE, 47507988803584, 47507988865023, ++ERASE, 47507988803584, 47507988803584, ++STORE, 47507988803584, 47507988865023, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988865024, 47507988889599, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988914175, ++ERASE, 47507988865024, 47507988865024, ++STORE, 47507988865024, 47507988889599, ++STORE, 47507988889600, 47507988914175, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988897791, ++ERASE, 47507988897792, 47507988897792, ++STORE, 47507988897792, 47507988914175, ++STORE, 47507988897792, 47507988922367, ++STORE, 47507988922368, 47507989086207, ++ERASE, 47507988922368, 47507988922368, ++STORE, 47507988922368, 47507988934655, ++STORE, 47507988934656, 47507989086207, ++STORE, 47507989032960, 47507989086207, ++STORE, 47507988934656, 47507989032959, ++ERASE, 47507988934656, 47507988934656, ++STORE, 47507988934656, 47507989032959, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989032960, 47507989078015, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989086207, ++ERASE, 47507989032960, 47507989032960, ++STORE, 47507989032960, 47507989078015, ++STORE, 47507989078016, 47507989086207, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989086207, ++STORE, 47507989086208, 47507989684223, ++STORE, 47507989204992, 47507989684223, ++STORE, 47507989086208, 47507989204991, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989630975, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989520384, 47507989630975, ++STORE, 47507989204992, 47507989520383, ++ERASE, 47507989204992, 47507989204992, ++STORE, 47507989204992, 47507989520383, ++STORE, 47507989626880, 47507989630975, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989520384, 47507989520384, ++STORE, 47507989520384, 47507989626879, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989684223, ++STORE, 47507989684224, 47507992735743, ++STORE, 47507990228992, 47507992735743, ++STORE, 47507989684224, 47507990228991, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507992514559, ++STORE, 47507992514560, 47507992735743, ++STORE, 47507991924736, 47507992514559, ++STORE, 47507990228992, 47507991924735, ++ERASE, 47507990228992, 47507990228992, ++STORE, 47507990228992, 47507991924735, ++STORE, 47507992510464, 47507992514559, ++STORE, 47507991924736, 47507992510463, ++ERASE, 47507991924736, 47507991924736, ++STORE, 47507991924736, 47507992510463, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992719359, ++ERASE, 47507992719360, 47507992719360, ++STORE, 47507992719360, 47507992735743, ++STORE, 47507992735744, 47507992768511, ++ERASE, 47507992735744, 47507992735744, ++STORE, 47507992735744, 47507992743935, ++STORE, 47507992743936, 47507992768511, ++STORE, 47507992756224, 47507992768511, ++STORE, 47507992743936, 47507992756223, ++ERASE, 47507992743936, 47507992743936, ++STORE, 47507992743936, 47507992756223, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992756224, 47507992760319, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992768511, ++ERASE, 47507992756224, 47507992756224, ++STORE, 47507992756224, 47507992760319, ++STORE, 47507992760320, 47507992768511, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992768511, ++STORE, 47507992768512, 47507992805375, ++ERASE, 47507992768512, 47507992768512, ++STORE, 47507992768512, 47507992776703, ++STORE, 47507992776704, 47507992805375, ++STORE, 47507992793088, 47507992805375, ++STORE, 47507992776704, 47507992793087, ++ERASE, 47507992776704, 47507992776704, ++STORE, 47507992776704, 47507992793087, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992793088, 47507992797183, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992805375, ++ERASE, 47507992793088, 47507992793088, ++STORE, 47507992793088, 47507992797183, ++STORE, 47507992797184, 47507992805375, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992805375, ++STORE, 47507992805376, 47507993280511, ++ERASE, 47507992805376, 47507992805376, ++STORE, 47507992805376, 47507992813567, ++STORE, 47507992813568, 47507993280511, ++STORE, 47507993149440, 47507993280511, ++STORE, 47507992813568, 47507993149439, ++ERASE, 47507992813568, 47507992813568, ++STORE, 47507992813568, 47507993149439, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993149440, 47507993272319, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993280511, ++ERASE, 47507993149440, 47507993149440, ++STORE, 47507993149440, 47507993272319, ++STORE, 47507993272320, 47507993280511, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993280511, ++STORE, 47507993280512, 47507993288703, ++STORE, 47507993288704, 47507993309183, ++ERASE, 47507993288704, 47507993288704, ++STORE, 47507993288704, 47507993292799, ++STORE, 47507993292800, 47507993309183, ++STORE, 47507993296896, 47507993309183, ++STORE, 47507993292800, 47507993296895, ++ERASE, 47507993292800, 47507993292800, ++STORE, 47507993292800, 47507993296895, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993296896, 47507993300991, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993309183, ++ERASE, 47507993296896, 47507993296896, ++STORE, 47507993296896, 47507993300991, ++STORE, 47507993300992, 47507993309183, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993309183, ++STORE, 47507993309184, 47507993317375, ++ERASE, 47507985973248, 47507985973248, ++STORE, 47507985973248, 47507985989631, ++STORE, 47507985989632, 47507985997823, ++ERASE, 47507993300992, 47507993300992, ++STORE, 47507993300992, 47507993305087, ++STORE, 47507993305088, 47507993309183, ++ERASE, 47507988889600, 47507988889600, ++STORE, 47507988889600, 47507988893695, ++STORE, 47507988893696, 47507988897791, ++ERASE, 47507993272320, 47507993272320, ++STORE, 47507993272320, 47507993276415, ++STORE, 47507993276416, 47507993280511, ++ERASE, 47507992797184, 47507992797184, ++STORE, 47507992797184, 47507992801279, ++STORE, 47507992801280, 47507992805375, ++ERASE, 47507992760320, 47507992760320, ++STORE, 47507992760320, 47507992764415, ++STORE, 47507992764416, 47507992768511, ++ERASE, 47507992514560, 47507992514560, ++STORE, 47507992514560, 47507992711167, ++STORE, 47507992711168, 47507992719359, ++ERASE, 47507989630976, 47507989630976, ++STORE, 47507989630976, 47507989667839, ++STORE, 47507989667840, 47507989684223, ++ERASE, 47507989078016, 47507989078016, ++STORE, 47507989078016, 47507989082111, ++STORE, 47507989082112, 47507989086207, ++ERASE, 47507988762624, 47507988762624, ++STORE, 47507988762624, 47507988766719, ++STORE, 47507988766720, 47507988770815, ++ERASE, 47507986493440, 47507986493440, ++STORE, 47507986493440, 47507986513919, ++STORE, 47507986513920, 47507986518015, ++ERASE, 47507986161664, 47507986161664, ++STORE, 47507986161664, 47507986165759, ++STORE, 47507986165760, 47507986169855, ++ERASE, 47507986116608, 47507986116608, ++STORE, 47507986116608, 47507986120703, ++STORE, 47507986120704, 47507986124799, ++ERASE, 94386579570688, 94386579570688, ++STORE, 94386579570688, 94386579693567, ++STORE, 94386579693568, 94386579697663, ++ERASE, 140124810997760, 140124810997760, ++STORE, 140124810997760, 140124811001855, ++STORE, 140124811001856, 140124811005951, ++ERASE, 47507984158720, 47507984158720, ++STORE, 94386583982080, 94386584117247, ++STORE, 94386583982080, 94386584256511, ++ERASE, 94386583982080, 94386583982080, ++STORE, 94386583982080, 94386584223743, ++STORE, 94386584223744, 94386584256511, ++ERASE, 94386584223744, 94386584223744, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733763395584, 140737488351231, ++ERASE, 140733763395584, 140733763395584, ++STORE, 140733763395584, 140733763399679, ++STORE, 94011546472448, 94011547152383, ++ERASE, 94011546472448, 94011546472448, ++STORE, 94011546472448, 94011546537983, ++STORE, 94011546537984, 94011547152383, ++ERASE, 94011546537984, 94011546537984, ++STORE, 94011546537984, 94011546886143, ++STORE, 94011546886144, 94011547025407, ++STORE, 94011547025408, 94011547152383, ++STORE, 139757597949952, 139757598121983, ++ERASE, 139757597949952, 139757597949952, ++STORE, 139757597949952, 139757597954047, ++STORE, 139757597954048, 139757598121983, ++ERASE, 139757597954048, 139757597954048, ++STORE, 139757597954048, 139757598076927, ++STORE, 139757598076928, 139757598109695, ++STORE, 139757598109696, 139757598117887, ++STORE, 139757598117888, 139757598121983, ++STORE, 140733763596288, 140733763600383, ++STORE, 140733763584000, 140733763596287, ++STORE, 47875197046784, 47875197054975, ++STORE, 47875197054976, 47875197063167, ++STORE, 47875197063168, 47875198902271, ++STORE, 47875197202432, 47875198902271, ++STORE, 47875197063168, 47875197202431, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198861311, ++STORE, 47875198861312, 47875198902271, ++STORE, 47875198545920, 47875198861311, ++STORE, 47875197202432, 47875198545919, ++ERASE, 47875197202432, 47875197202432, ++STORE, 47875197202432, 47875198545919, ++STORE, 47875198857216, 47875198861311, ++STORE, 47875198545920, 47875198857215, ++ERASE, 47875198545920, 47875198545920, ++STORE, 47875198545920, 47875198857215, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198885887, ++ERASE, 47875198885888, 47875198885888, ++STORE, 47875198885888, 47875198902271, ++STORE, 47875198902272, 47875199012863, ++STORE, 47875198918656, 47875199012863, ++STORE, 47875198902272, 47875198918655, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875199004671, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875198980096, 47875199004671, ++STORE, 47875198918656, 47875198980095, ++ERASE, 47875198918656, 47875198918656, ++STORE, 47875198918656, 47875198980095, ++STORE, 47875199000576, 47875199004671, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875198980096, 47875198980096, ++STORE, 47875198980096, 47875199000575, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199012863, ++STORE, 47875199012864, 47875199057919, ++ERASE, 47875199012864, 47875199012864, ++STORE, 47875199012864, 47875199021055, ++STORE, 47875199021056, 47875199057919, ++STORE, 47875199041536, 47875199057919, ++STORE, 47875199021056, 47875199041535, ++ERASE, 47875199021056, 47875199021056, ++STORE, 47875199021056, 47875199041535, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199041536, 47875199049727, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199057919, ++ERASE, 47875199041536, 47875199041536, ++STORE, 47875199041536, 47875199049727, ++STORE, 47875199049728, 47875199057919, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199057919, ++STORE, 47875199057920, 47875199406079, ++STORE, 47875199098880, 47875199406079, ++STORE, 47875199057920, 47875199098879, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199381503, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199311872, 47875199381503, ++STORE, 47875199098880, 47875199311871, ++ERASE, 47875199098880, 47875199098880, ++STORE, 47875199098880, 47875199311871, ++STORE, 47875199377408, 47875199381503, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199311872, 47875199311872, ++STORE, 47875199311872, 47875199377407, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199406079, ++STORE, 47875199406080, 47875201667071, ++STORE, 47875199557632, 47875201667071, ++STORE, 47875199406080, 47875199557631, ++ERASE, 47875199557632, 47875199557632, ++STORE, 47875199557632, 47875201650687, ++STORE, 47875201650688, 47875201667071, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201658879, ++ERASE, 47875201658880, 47875201658880, ++STORE, 47875201658880, 47875201667071, ++STORE, 47875201667072, 47875201802239, ++ERASE, 47875201667072, 47875201667072, ++STORE, 47875201667072, 47875201691647, ++STORE, 47875201691648, 47875201802239, ++STORE, 47875201753088, 47875201802239, ++STORE, 47875201691648, 47875201753087, ++ERASE, 47875201691648, 47875201691648, ++STORE, 47875201691648, 47875201753087, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201753088, 47875201777663, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201802239, ++ERASE, 47875201753088, 47875201753088, ++STORE, 47875201753088, 47875201777663, ++STORE, 47875201777664, 47875201802239, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201785855, ++ERASE, 47875201785856, 47875201785856, ++STORE, 47875201785856, 47875201802239, ++STORE, 47875201785856, 47875201810431, ++STORE, 47875201810432, 47875201974271, ++ERASE, 47875201810432, 47875201810432, ++STORE, 47875201810432, 47875201822719, ++STORE, 47875201822720, 47875201974271, ++STORE, 47875201921024, 47875201974271, ++STORE, 47875201822720, 47875201921023, ++ERASE, 47875201822720, 47875201822720, ++STORE, 47875201822720, 47875201921023, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201921024, 47875201966079, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201974271, ++ERASE, 47875201921024, 47875201921024, ++STORE, 47875201921024, 47875201966079, ++STORE, 47875201966080, 47875201974271, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201974271, ++STORE, 47875201974272, 47875202572287, ++STORE, 47875202093056, 47875202572287, ++STORE, 47875201974272, 47875202093055, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202519039, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202408448, 47875202519039, ++STORE, 47875202093056, 47875202408447, ++ERASE, 47875202093056, 47875202093056, ++STORE, 47875202093056, 47875202408447, ++STORE, 47875202514944, 47875202519039, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202408448, 47875202408448, ++STORE, 47875202408448, 47875202514943, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202572287, ++STORE, 47875202572288, 47875205623807, ++STORE, 47875203117056, 47875205623807, ++STORE, 47875202572288, 47875203117055, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875205402623, ++STORE, 47875205402624, 47875205623807, ++STORE, 47875204812800, 47875205402623, ++STORE, 47875203117056, 47875204812799, ++ERASE, 47875203117056, 47875203117056, ++STORE, 47875203117056, 47875204812799, ++STORE, 47875205398528, 47875205402623, ++STORE, 47875204812800, 47875205398527, ++ERASE, 47875204812800, 47875204812800, ++STORE, 47875204812800, 47875205398527, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205607423, ++ERASE, 47875205607424, 47875205607424, ++STORE, 47875205607424, 47875205623807, ++STORE, 47875205623808, 47875205656575, ++ERASE, 47875205623808, 47875205623808, ++STORE, 47875205623808, 47875205631999, ++STORE, 47875205632000, 47875205656575, ++STORE, 47875205644288, 47875205656575, ++STORE, 47875205632000, 47875205644287, ++ERASE, 47875205632000, 47875205632000, ++STORE, 47875205632000, 47875205644287, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205644288, 47875205648383, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205656575, ++ERASE, 47875205644288, 47875205644288, ++STORE, 47875205644288, 47875205648383, ++STORE, 47875205648384, 47875205656575, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205656575, ++STORE, 47875205656576, 47875205693439, ++ERASE, 47875205656576, 47875205656576, ++STORE, 47875205656576, 47875205664767, ++STORE, 47875205664768, 47875205693439, ++STORE, 47875205681152, 47875205693439, ++STORE, 47875205664768, 47875205681151, ++ERASE, 47875205664768, 47875205664768, ++STORE, 47875205664768, 47875205681151, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205681152, 47875205685247, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205693439, ++ERASE, 47875205681152, 47875205681152, ++STORE, 47875205681152, 47875205685247, ++STORE, 47875205685248, 47875205693439, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205693439, ++STORE, 47875205693440, 47875206168575, ++ERASE, 47875205693440, 47875205693440, ++STORE, 47875205693440, 47875205701631, ++STORE, 47875205701632, 47875206168575, ++STORE, 47875206037504, 47875206168575, ++STORE, 47875205701632, 47875206037503, ++ERASE, 47875205701632, 47875205701632, ++STORE, 47875205701632, 47875206037503, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206037504, 47875206160383, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206168575, ++ERASE, 47875206037504, 47875206037504, ++STORE, 47875206037504, 47875206160383, ++STORE, 47875206160384, 47875206168575, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206168575, ++STORE, 47875206168576, 47875206176767, ++STORE, 47875206176768, 47875206197247, ++ERASE, 47875206176768, 47875206176768, ++STORE, 47875206176768, 47875206180863, ++STORE, 47875206180864, 47875206197247, ++STORE, 47875206184960, 47875206197247, ++STORE, 47875206180864, 47875206184959, ++ERASE, 47875206180864, 47875206180864, ++STORE, 47875206180864, 47875206184959, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206184960, 47875206189055, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206197247, ++ERASE, 47875206184960, 47875206184960, ++STORE, 47875206184960, 47875206189055, ++STORE, 47875206189056, 47875206197247, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206197247, ++STORE, 47875206197248, 47875206205439, ++ERASE, 47875198861312, 47875198861312, ++STORE, 47875198861312, 47875198877695, ++STORE, 47875198877696, 47875198885887, ++ERASE, 47875206189056, 47875206189056, ++STORE, 47875206189056, 47875206193151, ++STORE, 47875206193152, 47875206197247, ++ERASE, 47875201777664, 47875201777664, ++STORE, 47875201777664, 47875201781759, ++STORE, 47875201781760, 47875201785855, ++ERASE, 47875206160384, 47875206160384, ++STORE, 47875206160384, 47875206164479, ++STORE, 47875206164480, 47875206168575, ++ERASE, 47875205685248, 47875205685248, ++STORE, 47875205685248, 47875205689343, ++STORE, 47875205689344, 47875205693439, ++ERASE, 47875205648384, 47875205648384, ++STORE, 47875205648384, 47875205652479, ++STORE, 47875205652480, 47875205656575, ++ERASE, 47875205402624, 47875205402624, ++STORE, 47875205402624, 47875205599231, ++STORE, 47875205599232, 47875205607423, ++ERASE, 47875202519040, 47875202519040, ++STORE, 47875202519040, 47875202555903, ++STORE, 47875202555904, 47875202572287, ++ERASE, 47875201966080, 47875201966080, ++STORE, 47875201966080, 47875201970175, ++STORE, 47875201970176, 47875201974271, ++ERASE, 47875201650688, 47875201650688, ++STORE, 47875201650688, 47875201654783, ++STORE, 47875201654784, 47875201658879, ++ERASE, 47875199381504, 47875199381504, ++STORE, 47875199381504, 47875199401983, ++STORE, 47875199401984, 47875199406079, ++ERASE, 47875199049728, 47875199049728, ++STORE, 47875199049728, 47875199053823, ++STORE, 47875199053824, 47875199057919, ++ERASE, 47875199004672, 47875199004672, ++STORE, 47875199004672, 47875199008767, ++STORE, 47875199008768, 47875199012863, ++ERASE, 94011547025408, 94011547025408, ++STORE, 94011547025408, 94011547148287, ++STORE, 94011547148288, 94011547152383, ++ERASE, 139757598109696, 139757598109696, ++STORE, 139757598109696, 139757598113791, ++STORE, 139757598113792, 139757598117887, ++ERASE, 47875197046784, 47875197046784, ++STORE, 94011557584896, 94011557720063, ++STORE, 94011557584896, 94011557855231, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557851135, ++STORE, 94011557851136, 94011557855231, ++ERASE, 94011557851136, 94011557851136, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557847039, ++STORE, 94011557847040, 94011557851135, ++ERASE, 94011557847040, 94011557847040, ++STORE, 94011557584896, 94011557982207, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557978111, ++STORE, 94011557978112, 94011557982207, ++ERASE, 94011557978112, 94011557978112, ++ERASE, 94011557584896, 94011557584896, ++STORE, 94011557584896, 94011557974015, ++STORE, 94011557974016, 94011557978111, ++ERASE, 94011557974016, 94011557974016, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734130360320, 140737488351231, ++ERASE, 140734130360320, 140734130360320, ++STORE, 140734130360320, 140734130364415, ++STORE, 94641232105472, 94641232785407, ++ERASE, 94641232105472, 94641232105472, ++STORE, 94641232105472, 94641232171007, ++STORE, 94641232171008, 94641232785407, ++ERASE, 94641232171008, 94641232171008, ++STORE, 94641232171008, 94641232519167, ++STORE, 94641232519168, 94641232658431, ++STORE, 94641232658432, 94641232785407, ++STORE, 139726599516160, 139726599688191, ++ERASE, 139726599516160, 139726599516160, ++STORE, 139726599516160, 139726599520255, ++STORE, 139726599520256, 139726599688191, ++ERASE, 139726599520256, 139726599520256, ++STORE, 139726599520256, 139726599643135, ++STORE, 139726599643136, 139726599675903, ++STORE, 139726599675904, 139726599684095, ++STORE, 139726599684096, 139726599688191, ++STORE, 140734130446336, 140734130450431, ++STORE, 140734130434048, 140734130446335, ++STORE, 47906195480576, 47906195488767, ++STORE, 47906195488768, 47906195496959, ++STORE, 47906195496960, 47906197336063, ++STORE, 47906195636224, 47906197336063, ++STORE, 47906195496960, 47906195636223, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906197295103, ++STORE, 47906197295104, 47906197336063, ++STORE, 47906196979712, 47906197295103, ++STORE, 47906195636224, 47906196979711, ++ERASE, 47906195636224, 47906195636224, ++STORE, 47906195636224, 47906196979711, ++STORE, 47906197291008, 47906197295103, ++STORE, 47906196979712, 47906197291007, ++ERASE, 47906196979712, 47906196979712, ++STORE, 47906196979712, 47906197291007, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197319679, ++ERASE, 47906197319680, 47906197319680, ++STORE, 47906197319680, 47906197336063, ++STORE, 47906197336064, 47906197446655, ++STORE, 47906197352448, 47906197446655, ++STORE, 47906197336064, 47906197352447, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197438463, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197413888, 47906197438463, ++STORE, 47906197352448, 47906197413887, ++ERASE, 47906197352448, 47906197352448, ++STORE, 47906197352448, 47906197413887, ++STORE, 47906197434368, 47906197438463, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197413888, 47906197413888, ++STORE, 47906197413888, 47906197434367, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197446655, ++STORE, 47906197446656, 47906197491711, ++ERASE, 47906197446656, 47906197446656, ++STORE, 47906197446656, 47906197454847, ++STORE, 47906197454848, 47906197491711, ++STORE, 47906197475328, 47906197491711, ++STORE, 47906197454848, 47906197475327, ++ERASE, 47906197454848, 47906197454848, ++STORE, 47906197454848, 47906197475327, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197475328, 47906197483519, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197491711, ++ERASE, 47906197475328, 47906197475328, ++STORE, 47906197475328, 47906197483519, ++STORE, 47906197483520, 47906197491711, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197491711, ++STORE, 47906197491712, 47906197839871, ++STORE, 47906197532672, 47906197839871, ++STORE, 47906197491712, 47906197532671, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197815295, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197745664, 47906197815295, ++STORE, 47906197532672, 47906197745663, ++ERASE, 47906197532672, 47906197532672, ++STORE, 47906197532672, 47906197745663, ++STORE, 47906197811200, 47906197815295, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197745664, 47906197745664, ++STORE, 47906197745664, 47906197811199, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197839871, ++STORE, 47906197839872, 47906200100863, ++STORE, 47906197991424, 47906200100863, ++STORE, 47906197839872, 47906197991423, ++ERASE, 47906197991424, 47906197991424, ++STORE, 47906197991424, 47906200084479, ++STORE, 47906200084480, 47906200100863, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200092671, ++ERASE, 47906200092672, 47906200092672, ++STORE, 47906200092672, 47906200100863, ++STORE, 47906200100864, 47906200236031, ++ERASE, 47906200100864, 47906200100864, ++STORE, 47906200100864, 47906200125439, ++STORE, 47906200125440, 47906200236031, ++STORE, 47906200186880, 47906200236031, ++STORE, 47906200125440, 47906200186879, ++ERASE, 47906200125440, 47906200125440, ++STORE, 47906200125440, 47906200186879, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200186880, 47906200211455, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200236031, ++ERASE, 47906200186880, 47906200186880, ++STORE, 47906200186880, 47906200211455, ++STORE, 47906200211456, 47906200236031, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200219647, ++ERASE, 47906200219648, 47906200219648, ++STORE, 47906200219648, 47906200236031, ++STORE, 47906200219648, 47906200244223, ++STORE, 47906200244224, 47906200408063, ++ERASE, 47906200244224, 47906200244224, ++STORE, 47906200244224, 47906200256511, ++STORE, 47906200256512, 47906200408063, ++STORE, 47906200354816, 47906200408063, ++STORE, 47906200256512, 47906200354815, ++ERASE, 47906200256512, 47906200256512, ++STORE, 47906200256512, 47906200354815, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200354816, 47906200399871, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200408063, ++ERASE, 47906200354816, 47906200354816, ++STORE, 47906200354816, 47906200399871, ++STORE, 47906200399872, 47906200408063, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200408063, ++STORE, 47906200408064, 47906201006079, ++STORE, 47906200526848, 47906201006079, ++STORE, 47906200408064, 47906200526847, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200952831, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906200842240, 47906200952831, ++STORE, 47906200526848, 47906200842239, ++ERASE, 47906200526848, 47906200526848, ++STORE, 47906200526848, 47906200842239, ++STORE, 47906200948736, 47906200952831, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200842240, 47906200842240, ++STORE, 47906200842240, 47906200948735, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906201006079, ++STORE, 47906201006080, 47906204057599, ++STORE, 47906201550848, 47906204057599, ++STORE, 47906201006080, 47906201550847, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203836415, ++STORE, 47906203836416, 47906204057599, ++STORE, 47906203246592, 47906203836415, ++STORE, 47906201550848, 47906203246591, ++ERASE, 47906201550848, 47906201550848, ++STORE, 47906201550848, 47906203246591, ++STORE, 47906203832320, 47906203836415, ++STORE, 47906203246592, 47906203832319, ++ERASE, 47906203246592, 47906203246592, ++STORE, 47906203246592, 47906203832319, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204041215, ++ERASE, 47906204041216, 47906204041216, ++STORE, 47906204041216, 47906204057599, ++STORE, 47906204057600, 47906204090367, ++ERASE, 47906204057600, 47906204057600, ++STORE, 47906204057600, 47906204065791, ++STORE, 47906204065792, 47906204090367, ++STORE, 47906204078080, 47906204090367, ++STORE, 47906204065792, 47906204078079, ++ERASE, 47906204065792, 47906204065792, ++STORE, 47906204065792, 47906204078079, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204078080, 47906204082175, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204090367, ++ERASE, 47906204078080, 47906204078080, ++STORE, 47906204078080, 47906204082175, ++STORE, 47906204082176, 47906204090367, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204090367, ++STORE, 47906204090368, 47906204127231, ++ERASE, 47906204090368, 47906204090368, ++STORE, 47906204090368, 47906204098559, ++STORE, 47906204098560, 47906204127231, ++STORE, 47906204114944, 47906204127231, ++STORE, 47906204098560, 47906204114943, ++ERASE, 47906204098560, 47906204098560, ++STORE, 47906204098560, 47906204114943, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204114944, 47906204119039, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204127231, ++ERASE, 47906204114944, 47906204114944, ++STORE, 47906204114944, 47906204119039, ++STORE, 47906204119040, 47906204127231, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204127231, ++STORE, 47906204127232, 47906204602367, ++ERASE, 47906204127232, 47906204127232, ++STORE, 47906204127232, 47906204135423, ++STORE, 47906204135424, 47906204602367, ++STORE, 47906204471296, 47906204602367, ++STORE, 47906204135424, 47906204471295, ++ERASE, 47906204135424, 47906204135424, ++STORE, 47906204135424, 47906204471295, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204471296, 47906204594175, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204602367, ++ERASE, 47906204471296, 47906204471296, ++STORE, 47906204471296, 47906204594175, ++STORE, 47906204594176, 47906204602367, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204602367, ++STORE, 47906204602368, 47906204610559, ++STORE, 47906204610560, 47906204631039, ++ERASE, 47906204610560, 47906204610560, ++STORE, 47906204610560, 47906204614655, ++STORE, 47906204614656, 47906204631039, ++STORE, 47906204618752, 47906204631039, ++STORE, 47906204614656, 47906204618751, ++ERASE, 47906204614656, 47906204614656, ++STORE, 47906204614656, 47906204618751, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204618752, 47906204622847, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204631039, ++ERASE, 47906204618752, 47906204618752, ++STORE, 47906204618752, 47906204622847, ++STORE, 47906204622848, 47906204631039, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204631039, ++STORE, 47906204631040, 47906204639231, ++ERASE, 47906197295104, 47906197295104, ++STORE, 47906197295104, 47906197311487, ++STORE, 47906197311488, 47906197319679, ++ERASE, 47906204622848, 47906204622848, ++STORE, 47906204622848, 47906204626943, ++STORE, 47906204626944, 47906204631039, ++ERASE, 47906200211456, 47906200211456, ++STORE, 47906200211456, 47906200215551, ++STORE, 47906200215552, 47906200219647, ++ERASE, 47906204594176, 47906204594176, ++STORE, 47906204594176, 47906204598271, ++STORE, 47906204598272, 47906204602367, ++ERASE, 47906204119040, 47906204119040, ++STORE, 47906204119040, 47906204123135, ++STORE, 47906204123136, 47906204127231, ++ERASE, 47906204082176, 47906204082176, ++STORE, 47906204082176, 47906204086271, ++STORE, 47906204086272, 47906204090367, ++ERASE, 47906203836416, 47906203836416, ++STORE, 47906203836416, 47906204033023, ++STORE, 47906204033024, 47906204041215, ++ERASE, 47906200952832, 47906200952832, ++STORE, 47906200952832, 47906200989695, ++STORE, 47906200989696, 47906201006079, ++ERASE, 47906200399872, 47906200399872, ++STORE, 47906200399872, 47906200403967, ++STORE, 47906200403968, 47906200408063, ++ERASE, 47906200084480, 47906200084480, ++STORE, 47906200084480, 47906200088575, ++STORE, 47906200088576, 47906200092671, ++ERASE, 47906197815296, 47906197815296, ++STORE, 47906197815296, 47906197835775, ++STORE, 47906197835776, 47906197839871, ++ERASE, 47906197483520, 47906197483520, ++STORE, 47906197483520, 47906197487615, ++STORE, 47906197487616, 47906197491711, ++ERASE, 47906197438464, 47906197438464, ++STORE, 47906197438464, 47906197442559, ++STORE, 47906197442560, 47906197446655, ++ERASE, 94641232658432, 94641232658432, ++STORE, 94641232658432, 94641232781311, ++STORE, 94641232781312, 94641232785407, ++ERASE, 139726599675904, 139726599675904, ++STORE, 139726599675904, 139726599679999, ++STORE, 139726599680000, 139726599684095, ++ERASE, 47906195480576, 47906195480576, ++STORE, 94641242615808, 94641242750975, ++ }; ++ unsigned long set11[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140732658499584, 140737488351231, ++ERASE, 140732658499584, 140732658499584, ++STORE, 140732658499584, 140732658503679, ++STORE, 94029856579584, 94029856751615, ++ERASE, 94029856579584, 94029856579584, ++STORE, 94029856579584, 94029856595967, ++STORE, 94029856595968, 94029856751615, ++ERASE, 94029856595968, 94029856595968, ++STORE, 94029856595968, 94029856698367, ++STORE, 94029856698368, 94029856739327, ++STORE, 94029856739328, 94029856751615, ++STORE, 140014592573440, 140014592745471, ++ERASE, 140014592573440, 140014592573440, ++STORE, 140014592573440, 140014592577535, ++STORE, 140014592577536, 140014592745471, ++ERASE, 140014592577536, 140014592577536, ++STORE, 140014592577536, 140014592700415, ++STORE, 140014592700416, 140014592733183, ++STORE, 140014592733184, 140014592741375, ++STORE, 140014592741376, 140014592745471, ++STORE, 140732658565120, 140732658569215, ++STORE, 140732658552832, 140732658565119, ++ }; + -+ if (time_is_after_jiffies(birth + min_ttl)) -+ return false; ++ unsigned long set12[] = { /* contains 12 values. */ ++STORE, 140737488347136, 140737488351231, ++STORE, 140732658499584, 140737488351231, ++ERASE, 140732658499584, 140732658499584, ++STORE, 140732658499584, 140732658503679, ++STORE, 94029856579584, 94029856751615, ++ERASE, 94029856579584, 94029856579584, ++STORE, 94029856579584, 94029856595967, ++STORE, 94029856595968, 94029856751615, ++ERASE, 94029856595968, 94029856595968, ++STORE, 94029856595968, 94029856698367, ++STORE, 94029856698368, 94029856739327, ++STORE, 94029856739328, 94029856751615, ++STORE, 140014592573440, 140014592745471, ++ERASE, 140014592573440, 140014592573440, ++STORE, 140014592573440, 140014592577535, ++STORE, 140014592577536, 140014592745471, ++ERASE, 140014592577536, 140014592577536, ++STORE, 140014592577536, 140014592700415, ++STORE, 140014592700416, 140014592733183, ++STORE, 140014592733184, 140014592741375, ++STORE, 140014592741376, 140014592745471, ++STORE, 140732658565120, 140732658569215, ++STORE, 140732658552832, 140732658565119, ++STORE, 140014592741375, 140014592741375, /* contrived */ ++STORE, 140014592733184, 140014592741376, /* creates first entry retry. */ ++ }; ++ unsigned long set13[] = { ++STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */ ++STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */ ++STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */ ++STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */ ++STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */ ++STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */ ++STORE, 140373518663680, 140373518684159,/*: deleted (257) */ ++STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */ ++STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */ ++STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */ ++STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */ ++ }; ++ unsigned long set14[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140731667996672, 140737488351231, ++SNULL, 140731668000767, 140737488351231, ++STORE, 140731667996672, 140731668000767, ++STORE, 140731667865600, 140731668000767, ++STORE, 94077521272832, 94077521313791, ++SNULL, 94077521301503, 94077521313791, ++STORE, 94077521272832, 94077521301503, ++STORE, 94077521301504, 94077521313791, ++ERASE, 94077521301504, 94077521313791, ++STORE, 94077521305600, 94077521313791, ++STORE, 139826134630400, 139826136883199, ++SNULL, 139826134773759, 139826136883199, ++STORE, 139826134630400, 139826134773759, ++STORE, 139826134773760, 139826136883199, ++ERASE, 139826134773760, 139826136883199, ++STORE, 139826136870912, 139826136879103, ++STORE, 139826136879104, 139826136883199, ++STORE, 140731668013056, 140731668017151, ++STORE, 140731668000768, 140731668013055, ++STORE, 139826136862720, 139826136870911, ++STORE, 139826132406272, 139826134630399, ++SNULL, 139826134056959, 139826134630399, ++STORE, 139826132406272, 139826134056959, ++STORE, 139826134056960, 139826134630399, ++SNULL, 139826134056960, 139826134626303, ++STORE, 139826134626304, 139826134630399, ++STORE, 139826134056960, 139826134626303, ++ERASE, 139826134056960, 139826134626303, ++STORE, 139826134056960, 139826134626303, ++ERASE, 139826134626304, 139826134630399, ++STORE, 139826134626304, 139826134630399, ++STORE, 139826136842240, 139826136862719, ++STORE, 139826130022400, 139826132406271, ++SNULL, 139826130022400, 139826130288639, ++STORE, 139826130288640, 139826132406271, ++STORE, 139826130022400, 139826130288639, ++SNULL, 139826132381695, 139826132406271, ++STORE, 139826130288640, 139826132381695, ++STORE, 139826132381696, 139826132406271, ++SNULL, 139826132381696, 139826132402175, ++STORE, 139826132402176, 139826132406271, ++STORE, 139826132381696, 139826132402175, ++ERASE, 139826132381696, 139826132402175, ++STORE, 139826132381696, 139826132402175, ++ERASE, 139826132402176, 139826132406271, ++STORE, 139826132402176, 139826132406271, ++STORE, 139826127806464, 139826130022399, ++SNULL, 139826127806464, 139826127904767, ++STORE, 139826127904768, 139826130022399, ++STORE, 139826127806464, 139826127904767, ++SNULL, 139826129997823, 139826130022399, ++STORE, 139826127904768, 139826129997823, ++STORE, 139826129997824, 139826130022399, ++SNULL, 139826129997824, 139826130006015, ++STORE, 139826130006016, 139826130022399, ++STORE, 139826129997824, 139826130006015, ++ERASE, 139826129997824, 139826130006015, ++STORE, 139826129997824, 139826130006015, ++ERASE, 139826130006016, 139826130022399, ++STORE, 139826130006016, 139826130022399, ++STORE, 139826124009472, 139826127806463, ++SNULL, 139826124009472, 139826125668351, ++STORE, 139826125668352, 139826127806463, ++STORE, 139826124009472, 139826125668351, ++SNULL, 139826127765503, 139826127806463, ++STORE, 139826125668352, 139826127765503, ++STORE, 139826127765504, 139826127806463, ++SNULL, 139826127765504, 139826127790079, ++STORE, 139826127790080, 139826127806463, ++STORE, 139826127765504, 139826127790079, ++ERASE, 139826127765504, 139826127790079, ++STORE, 139826127765504, 139826127790079, ++ERASE, 139826127790080, 139826127806463, ++STORE, 139826127790080, 139826127806463, ++STORE, 139826121748480, 139826124009471, ++SNULL, 139826121748480, 139826121900031, ++STORE, 139826121900032, 139826124009471, ++STORE, 139826121748480, 139826121900031, ++SNULL, 139826123993087, 139826124009471, ++STORE, 139826121900032, 139826123993087, ++STORE, 139826123993088, 139826124009471, ++SNULL, 139826123993088, 139826124001279, ++STORE, 139826124001280, 139826124009471, ++STORE, 139826123993088, 139826124001279, ++ERASE, 139826123993088, 139826124001279, ++STORE, 139826123993088, 139826124001279, ++ERASE, 139826124001280, 139826124009471, ++STORE, 139826124001280, 139826124009471, ++STORE, 139826119626752, 139826121748479, ++SNULL, 139826119626752, 139826119643135, ++STORE, 139826119643136, 139826121748479, ++STORE, 139826119626752, 139826119643135, ++SNULL, 139826121740287, 139826121748479, ++STORE, 139826119643136, 139826121740287, ++STORE, 139826121740288, 139826121748479, ++ERASE, 139826121740288, 139826121748479, ++STORE, 139826121740288, 139826121748479, ++STORE, 139826136834048, 139826136842239, ++STORE, 139826117496832, 139826119626751, ++SNULL, 139826117496832, 139826117525503, ++STORE, 139826117525504, 139826119626751, ++STORE, 139826117496832, 139826117525503, ++SNULL, 139826119618559, 139826119626751, ++STORE, 139826117525504, 139826119618559, ++STORE, 139826119618560, 139826119626751, ++ERASE, 139826119618560, 139826119626751, ++STORE, 139826119618560, 139826119626751, ++STORE, 139826115244032, 139826117496831, ++SNULL, 139826115244032, 139826115395583, ++STORE, 139826115395584, 139826117496831, ++STORE, 139826115244032, 139826115395583, ++SNULL, 139826117488639, 139826117496831, ++STORE, 139826115395584, 139826117488639, ++STORE, 139826117488640, 139826117496831, ++ERASE, 139826117488640, 139826117496831, ++STORE, 139826117488640, 139826117496831, ++STORE, 139826113073152, 139826115244031, ++SNULL, 139826113073152, 139826113142783, ++STORE, 139826113142784, 139826115244031, ++STORE, 139826113073152, 139826113142783, ++SNULL, 139826115235839, 139826115244031, ++STORE, 139826113142784, 139826115235839, ++STORE, 139826115235840, 139826115244031, ++ERASE, 139826115235840, 139826115244031, ++STORE, 139826115235840, 139826115244031, ++STORE, 139826109861888, 139826113073151, ++SNULL, 139826109861888, 139826110939135, ++STORE, 139826110939136, 139826113073151, ++STORE, 139826109861888, 139826110939135, ++SNULL, 139826113036287, 139826113073151, ++STORE, 139826110939136, 139826113036287, ++STORE, 139826113036288, 139826113073151, ++ERASE, 139826113036288, 139826113073151, ++STORE, 139826113036288, 139826113073151, ++STORE, 139826107727872, 139826109861887, ++SNULL, 139826107727872, 139826107756543, ++STORE, 139826107756544, 139826109861887, ++STORE, 139826107727872, 139826107756543, ++SNULL, 139826109853695, 139826109861887, ++STORE, 139826107756544, 139826109853695, ++STORE, 139826109853696, 139826109861887, ++ERASE, 139826109853696, 139826109861887, ++STORE, 139826109853696, 139826109861887, ++STORE, 139826105417728, 139826107727871, ++SNULL, 139826105417728, 139826105622527, ++STORE, 139826105622528, 139826107727871, ++STORE, 139826105417728, 139826105622527, ++SNULL, 139826107719679, 139826107727871, ++STORE, 139826105622528, 139826107719679, ++STORE, 139826107719680, 139826107727871, ++ERASE, 139826107719680, 139826107727871, ++STORE, 139826107719680, 139826107727871, ++STORE, 139826136825856, 139826136842239, ++STORE, 139826103033856, 139826105417727, ++SNULL, 139826103033856, 139826103226367, ++STORE, 139826103226368, 139826105417727, ++STORE, 139826103033856, 139826103226367, ++SNULL, 139826105319423, 139826105417727, ++STORE, 139826103226368, 139826105319423, ++STORE, 139826105319424, 139826105417727, ++ERASE, 139826105319424, 139826105417727, ++STORE, 139826105319424, 139826105417727, ++STORE, 139826100916224, 139826103033855, ++SNULL, 139826100916224, 139826100932607, ++STORE, 139826100932608, 139826103033855, ++STORE, 139826100916224, 139826100932607, ++SNULL, 139826103025663, 139826103033855, ++STORE, 139826100932608, 139826103025663, ++STORE, 139826103025664, 139826103033855, ++ERASE, 139826103025664, 139826103033855, ++STORE, 139826103025664, 139826103033855, ++STORE, 139826098348032, 139826100916223, ++SNULL, 139826098348032, 139826098814975, ++STORE, 139826098814976, 139826100916223, ++STORE, 139826098348032, 139826098814975, ++SNULL, 139826100908031, 139826100916223, ++STORE, 139826098814976, 139826100908031, ++STORE, 139826100908032, 139826100916223, ++ERASE, 139826100908032, 139826100916223, ++STORE, 139826100908032, 139826100916223, ++STORE, 139826096234496, 139826098348031, ++SNULL, 139826096234496, 139826096246783, ++STORE, 139826096246784, 139826098348031, ++STORE, 139826096234496, 139826096246783, ++SNULL, 139826098339839, 139826098348031, ++STORE, 139826096246784, 139826098339839, ++STORE, 139826098339840, 139826098348031, ++ERASE, 139826098339840, 139826098348031, ++STORE, 139826098339840, 139826098348031, ++STORE, 139826094055424, 139826096234495, ++SNULL, 139826094055424, 139826094133247, ++STORE, 139826094133248, 139826096234495, ++STORE, 139826094055424, 139826094133247, ++SNULL, 139826096226303, 139826096234495, ++STORE, 139826094133248, 139826096226303, ++STORE, 139826096226304, 139826096234495, ++ERASE, 139826096226304, 139826096234495, ++STORE, 139826096226304, 139826096234495, ++STORE, 139826136817664, 139826136842239, ++STORE, 139826091937792, 139826094055423, ++SNULL, 139826091937792, 139826091954175, ++STORE, 139826091954176, 139826094055423, ++STORE, 139826091937792, 139826091954175, ++SNULL, 139826094047231, 139826094055423, ++STORE, 139826091954176, 139826094047231, ++STORE, 139826094047232, 139826094055423, ++ERASE, 139826094047232, 139826094055423, ++STORE, 139826094047232, 139826094055423, ++STORE, 139826136809472, 139826136842239, ++SNULL, 139826127781887, 139826127790079, ++STORE, 139826127765504, 139826127781887, ++STORE, 139826127781888, 139826127790079, ++SNULL, 139826094051327, 139826094055423, ++STORE, 139826094047232, 139826094051327, ++STORE, 139826094051328, 139826094055423, ++SNULL, 139826096230399, 139826096234495, ++STORE, 139826096226304, 139826096230399, ++STORE, 139826096230400, 139826096234495, ++SNULL, 139826098343935, 139826098348031, ++STORE, 139826098339840, 139826098343935, ++STORE, 139826098343936, 139826098348031, ++SNULL, 139826130001919, 139826130006015, ++STORE, 139826129997824, 139826130001919, ++STORE, 139826130001920, 139826130006015, ++SNULL, 139826100912127, 139826100916223, ++STORE, 139826100908032, 139826100912127, ++STORE, 139826100912128, 139826100916223, ++SNULL, 139826103029759, 139826103033855, ++STORE, 139826103025664, 139826103029759, ++STORE, 139826103029760, 139826103033855, ++SNULL, 139826105413631, 139826105417727, ++STORE, 139826105319424, 139826105413631, ++STORE, 139826105413632, 139826105417727, ++SNULL, 139826107723775, 139826107727871, ++STORE, 139826107719680, 139826107723775, ++STORE, 139826107723776, 139826107727871, ++SNULL, 139826109857791, 139826109861887, ++STORE, 139826109853696, 139826109857791, ++STORE, 139826109857792, 139826109861887, ++SNULL, 139826113044479, 139826113073151, ++STORE, 139826113036288, 139826113044479, ++STORE, 139826113044480, 139826113073151, ++SNULL, 139826115239935, 139826115244031, ++STORE, 139826115235840, 139826115239935, ++STORE, 139826115239936, 139826115244031, ++SNULL, 139826117492735, 139826117496831, ++STORE, 139826117488640, 139826117492735, ++STORE, 139826117492736, 139826117496831, ++SNULL, 139826119622655, 139826119626751, ++STORE, 139826119618560, 139826119622655, ++STORE, 139826119622656, 139826119626751, ++SNULL, 139826121744383, 139826121748479, ++STORE, 139826121740288, 139826121744383, ++STORE, 139826121744384, 139826121748479, ++SNULL, 139826123997183, 139826124001279, ++STORE, 139826123993088, 139826123997183, ++STORE, 139826123997184, 139826124001279, ++SNULL, 139826132398079, 139826132402175, ++STORE, 139826132381696, 139826132398079, ++STORE, 139826132398080, 139826132402175, ++SNULL, 139826134622207, 139826134626303, ++STORE, 139826134056960, 139826134622207, ++STORE, 139826134622208, 139826134626303, ++SNULL, 94077521309695, 94077521313791, ++STORE, 94077521305600, 94077521309695, ++STORE, 94077521309696, 94077521313791, ++SNULL, 139826136875007, 139826136879103, ++STORE, 139826136870912, 139826136875007, ++STORE, 139826136875008, 139826136879103, ++ERASE, 139826136842240, 139826136862719, ++STORE, 94077554049024, 94077554184191, ++STORE, 139826136543232, 139826136842239, ++STORE, 139826136276992, 139826136842239, ++STORE, 139826136010752, 139826136842239, ++STORE, 139826135744512, 139826136842239, ++SNULL, 139826136543231, 139826136842239, ++STORE, 139826135744512, 139826136543231, ++STORE, 139826136543232, 139826136842239, ++SNULL, 139826136543232, 139826136809471, ++STORE, 139826136809472, 139826136842239, ++STORE, 139826136543232, 139826136809471, ++ }; ++ unsigned long set15[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722061451264, 140737488351231, ++SNULL, 140722061455359, 140737488351231, ++STORE, 140722061451264, 140722061455359, ++STORE, 140722061320192, 140722061455359, ++STORE, 94728600248320, 94728600289279, ++SNULL, 94728600276991, 94728600289279, ++STORE, 94728600248320, 94728600276991, ++STORE, 94728600276992, 94728600289279, ++ERASE, 94728600276992, 94728600289279, ++STORE, 94728600281088, 94728600289279, ++STORE, 139906806779904, 139906809032703, ++SNULL, 139906806923263, 139906809032703, ++STORE, 139906806779904, 139906806923263, ++STORE, 139906806923264, 139906809032703, ++ERASE, 139906806923264, 139906809032703, ++STORE, 139906809020416, 139906809028607, ++STORE, 139906809028608, 139906809032703, ++STORE, 140722061692928, 140722061697023, ++STORE, 140722061680640, 140722061692927, ++STORE, 139906809012224, 139906809020415, ++STORE, 139906804555776, 139906806779903, ++SNULL, 139906806206463, 139906806779903, ++STORE, 139906804555776, 139906806206463, ++STORE, 139906806206464, 139906806779903, ++SNULL, 139906806206464, 139906806775807, ++STORE, 139906806775808, 139906806779903, ++STORE, 139906806206464, 139906806775807, ++ERASE, 139906806206464, 139906806775807, ++STORE, 139906806206464, 139906806775807, ++ERASE, 139906806775808, 139906806779903, ++STORE, 139906806775808, 139906806779903, ++STORE, 139906808991744, 139906809012223, ++STORE, 139906802171904, 139906804555775, ++SNULL, 139906802171904, 139906802438143, ++STORE, 139906802438144, 139906804555775, ++STORE, 139906802171904, 139906802438143, ++SNULL, 139906804531199, 139906804555775, ++STORE, 139906802438144, 139906804531199, ++STORE, 139906804531200, 139906804555775, ++SNULL, 139906804531200, 139906804551679, ++STORE, 139906804551680, 139906804555775, ++STORE, 139906804531200, 139906804551679, ++ERASE, 139906804531200, 139906804551679, ++STORE, 139906804531200, 139906804551679, ++ERASE, 139906804551680, 139906804555775, ++STORE, 139906804551680, 139906804555775, ++STORE, 139906799955968, 139906802171903, ++SNULL, 139906799955968, 139906800054271, ++STORE, 139906800054272, 139906802171903, ++STORE, 139906799955968, 139906800054271, ++SNULL, 139906802147327, 139906802171903, ++STORE, 139906800054272, 139906802147327, ++STORE, 139906802147328, 139906802171903, ++SNULL, 139906802147328, 139906802155519, ++STORE, 139906802155520, 139906802171903, ++STORE, 139906802147328, 139906802155519, ++ERASE, 139906802147328, 139906802155519, ++STORE, 139906802147328, 139906802155519, ++ERASE, 139906802155520, 139906802171903, ++STORE, 139906802155520, 139906802171903, ++STORE, 139906796158976, 139906799955967, ++SNULL, 139906796158976, 139906797817855, ++STORE, 139906797817856, 139906799955967, ++STORE, 139906796158976, 139906797817855, ++SNULL, 139906799915007, 139906799955967, ++STORE, 139906797817856, 139906799915007, ++STORE, 139906799915008, 139906799955967, ++SNULL, 139906799915008, 139906799939583, ++STORE, 139906799939584, 139906799955967, ++STORE, 139906799915008, 139906799939583, ++ERASE, 139906799915008, 139906799939583, ++STORE, 139906799915008, 139906799939583, ++ERASE, 139906799939584, 139906799955967, ++STORE, 139906799939584, 139906799955967, ++STORE, 139906793897984, 139906796158975, ++SNULL, 139906793897984, 139906794049535, ++STORE, 139906794049536, 139906796158975, ++STORE, 139906793897984, 139906794049535, ++SNULL, 139906796142591, 139906796158975, ++STORE, 139906794049536, 139906796142591, ++STORE, 139906796142592, 139906796158975, ++SNULL, 139906796142592, 139906796150783, ++STORE, 139906796150784, 139906796158975, ++STORE, 139906796142592, 139906796150783, ++ERASE, 139906796142592, 139906796150783, ++STORE, 139906796142592, 139906796150783, ++ERASE, 139906796150784, 139906796158975, ++STORE, 139906796150784, 139906796158975, ++STORE, 139906791776256, 139906793897983, ++SNULL, 139906791776256, 139906791792639, ++STORE, 139906791792640, 139906793897983, ++STORE, 139906791776256, 139906791792639, ++SNULL, 139906793889791, 139906793897983, ++STORE, 139906791792640, 139906793889791, ++STORE, 139906793889792, 139906793897983, ++ERASE, 139906793889792, 139906793897983, ++STORE, 139906793889792, 139906793897983, ++STORE, 139906808983552, 139906808991743, ++STORE, 139906789646336, 139906791776255, ++SNULL, 139906789646336, 139906789675007, ++STORE, 139906789675008, 139906791776255, ++STORE, 139906789646336, 139906789675007, ++SNULL, 139906791768063, 139906791776255, ++STORE, 139906789675008, 139906791768063, ++STORE, 139906791768064, 139906791776255, ++ERASE, 139906791768064, 139906791776255, ++STORE, 139906791768064, 139906791776255, ++STORE, 139906787393536, 139906789646335, ++SNULL, 139906787393536, 139906787545087, ++STORE, 139906787545088, 139906789646335, ++STORE, 139906787393536, 139906787545087, ++SNULL, 139906789638143, 139906789646335, ++STORE, 139906787545088, 139906789638143, ++STORE, 139906789638144, 139906789646335, ++ERASE, 139906789638144, 139906789646335, ++STORE, 139906789638144, 139906789646335, ++STORE, 139906785222656, 139906787393535, ++SNULL, 139906785222656, 139906785292287, ++STORE, 139906785292288, 139906787393535, ++STORE, 139906785222656, 139906785292287, ++SNULL, 139906787385343, 139906787393535, ++STORE, 139906785292288, 139906787385343, ++STORE, 139906787385344, 139906787393535, ++ERASE, 139906787385344, 139906787393535, ++STORE, 139906787385344, 139906787393535, ++STORE, 139906782011392, 139906785222655, ++SNULL, 139906782011392, 139906783088639, ++STORE, 139906783088640, 139906785222655, ++STORE, 139906782011392, 139906783088639, ++SNULL, 139906785185791, 139906785222655, ++STORE, 139906783088640, 139906785185791, ++STORE, 139906785185792, 139906785222655, ++ERASE, 139906785185792, 139906785222655, ++STORE, 139906785185792, 139906785222655, ++STORE, 139906779877376, 139906782011391, ++SNULL, 139906779877376, 139906779906047, ++STORE, 139906779906048, 139906782011391, ++STORE, 139906779877376, 139906779906047, ++SNULL, 139906782003199, 139906782011391, ++STORE, 139906779906048, 139906782003199, ++STORE, 139906782003200, 139906782011391, ++ERASE, 139906782003200, 139906782011391, ++STORE, 139906782003200, 139906782011391, ++STORE, 139906777567232, 139906779877375, ++SNULL, 139906777567232, 139906777772031, ++STORE, 139906777772032, 139906779877375, ++STORE, 139906777567232, 139906777772031, ++SNULL, 139906779869183, 139906779877375, ++STORE, 139906777772032, 139906779869183, ++STORE, 139906779869184, 139906779877375, ++ERASE, 139906779869184, 139906779877375, ++STORE, 139906779869184, 139906779877375, ++STORE, 139906808975360, 139906808991743, ++STORE, 139906775183360, 139906777567231, ++SNULL, 139906775183360, 139906775375871, ++STORE, 139906775375872, 139906777567231, ++STORE, 139906775183360, 139906775375871, ++SNULL, 139906777468927, 139906777567231, ++STORE, 139906775375872, 139906777468927, ++STORE, 139906777468928, 139906777567231, ++ERASE, 139906777468928, 139906777567231, ++STORE, 139906777468928, 139906777567231, ++STORE, 139906773065728, 139906775183359, ++SNULL, 139906773065728, 139906773082111, ++STORE, 139906773082112, 139906775183359, ++STORE, 139906773065728, 139906773082111, ++SNULL, 139906775175167, 139906775183359, ++STORE, 139906773082112, 139906775175167, ++STORE, 139906775175168, 139906775183359, ++ERASE, 139906775175168, 139906775183359, ++STORE, 139906775175168, 139906775183359, ++STORE, 139906770497536, 139906773065727, ++SNULL, 139906770497536, 139906770964479, ++STORE, 139906770964480, 139906773065727, ++STORE, 139906770497536, 139906770964479, ++SNULL, 139906773057535, 139906773065727, ++STORE, 139906770964480, 139906773057535, ++STORE, 139906773057536, 139906773065727, ++ERASE, 139906773057536, 139906773065727, ++STORE, 139906773057536, 139906773065727, ++STORE, 139906768384000, 139906770497535, ++SNULL, 139906768384000, 139906768396287, ++STORE, 139906768396288, 139906770497535, ++STORE, 139906768384000, 139906768396287, ++SNULL, 139906770489343, 139906770497535, ++STORE, 139906768396288, 139906770489343, ++STORE, 139906770489344, 139906770497535, ++ERASE, 139906770489344, 139906770497535, ++STORE, 139906770489344, 139906770497535, ++STORE, 139906766204928, 139906768383999, ++SNULL, 139906766204928, 139906766282751, ++STORE, 139906766282752, 139906768383999, ++STORE, 139906766204928, 139906766282751, ++SNULL, 139906768375807, 139906768383999, ++STORE, 139906766282752, 139906768375807, ++STORE, 139906768375808, 139906768383999, ++ERASE, 139906768375808, 139906768383999, ++STORE, 139906768375808, 139906768383999, ++STORE, 139906808967168, 139906808991743, ++STORE, 139906764087296, 139906766204927, ++SNULL, 139906764087296, 139906764103679, ++STORE, 139906764103680, 139906766204927, ++STORE, 139906764087296, 139906764103679, ++SNULL, 139906766196735, 139906766204927, ++STORE, 139906764103680, 139906766196735, ++STORE, 139906766196736, 139906766204927, ++ERASE, 139906766196736, 139906766204927, ++STORE, 139906766196736, 139906766204927, ++STORE, 139906808958976, 139906808991743, ++SNULL, 139906799931391, 139906799939583, ++STORE, 139906799915008, 139906799931391, ++STORE, 139906799931392, 139906799939583, ++SNULL, 139906766200831, 139906766204927, ++STORE, 139906766196736, 139906766200831, ++STORE, 139906766200832, 139906766204927, ++SNULL, 139906768379903, 139906768383999, ++STORE, 139906768375808, 139906768379903, ++STORE, 139906768379904, 139906768383999, ++SNULL, 139906770493439, 139906770497535, ++STORE, 139906770489344, 139906770493439, ++STORE, 139906770493440, 139906770497535, ++SNULL, 139906802151423, 139906802155519, ++STORE, 139906802147328, 139906802151423, ++STORE, 139906802151424, 139906802155519, ++SNULL, 139906773061631, 139906773065727, ++STORE, 139906773057536, 139906773061631, ++STORE, 139906773061632, 139906773065727, ++SNULL, 139906775179263, 139906775183359, ++STORE, 139906775175168, 139906775179263, ++STORE, 139906775179264, 139906775183359, ++SNULL, 139906777563135, 139906777567231, ++STORE, 139906777468928, 139906777563135, ++STORE, 139906777563136, 139906777567231, ++SNULL, 139906779873279, 139906779877375, ++STORE, 139906779869184, 139906779873279, ++STORE, 139906779873280, 139906779877375, ++SNULL, 139906782007295, 139906782011391, ++STORE, 139906782003200, 139906782007295, ++STORE, 139906782007296, 139906782011391, ++SNULL, 139906785193983, 139906785222655, ++STORE, 139906785185792, 139906785193983, ++STORE, 139906785193984, 139906785222655, ++SNULL, 139906787389439, 139906787393535, ++STORE, 139906787385344, 139906787389439, ++STORE, 139906787389440, 139906787393535, ++SNULL, 139906789642239, 139906789646335, ++STORE, 139906789638144, 139906789642239, ++STORE, 139906789642240, 139906789646335, ++SNULL, 139906791772159, 139906791776255, ++STORE, 139906791768064, 139906791772159, ++STORE, 139906791772160, 139906791776255, ++SNULL, 139906793893887, 139906793897983, ++STORE, 139906793889792, 139906793893887, ++STORE, 139906793893888, 139906793897983, ++SNULL, 139906796146687, 139906796150783, ++STORE, 139906796142592, 139906796146687, ++STORE, 139906796146688, 139906796150783, ++SNULL, 139906804547583, 139906804551679, ++STORE, 139906804531200, 139906804547583, ++STORE, 139906804547584, 139906804551679, ++SNULL, 139906806771711, 139906806775807, ++STORE, 139906806206464, 139906806771711, ++STORE, 139906806771712, 139906806775807, ++SNULL, 94728600285183, 94728600289279, ++STORE, 94728600281088, 94728600285183, ++STORE, 94728600285184, 94728600289279, ++SNULL, 139906809024511, 139906809028607, ++STORE, 139906809020416, 139906809024511, ++STORE, 139906809024512, 139906809028607, ++ERASE, 139906808991744, 139906809012223, ++STORE, 94728620138496, 94728620273663, ++STORE, 139906808692736, 139906808991743, ++STORE, 139906808426496, 139906808991743, ++STORE, 139906808160256, 139906808991743, ++STORE, 139906807894016, 139906808991743, ++SNULL, 139906808692735, 139906808991743, ++STORE, 139906807894016, 139906808692735, ++STORE, 139906808692736, 139906808991743, ++SNULL, 139906808692736, 139906808958975, ++STORE, 139906808958976, 139906808991743, ++STORE, 139906808692736, 139906808958975, ++ }; + -+ /* the size is likely too small to be helpful */ -+ if (!nr_to_scan && sc->priority != DEF_PRIORITY) -+ return false; -+ } ++ unsigned long set16[] = { ++STORE, 94174808662016, 94174809321471, ++STORE, 94174811414528, 94174811426815, ++STORE, 94174811426816, 94174811430911, ++STORE, 94174811430912, 94174811443199, ++STORE, 94174841700352, 94174841835519, ++STORE, 140173257838592, 140173259497471, ++STORE, 140173259497472, 140173261594623, ++STORE, 140173261594624, 140173261611007, ++STORE, 140173261611008, 140173261619199, ++STORE, 140173261619200, 140173261635583, ++STORE, 140173261635584, 140173261778943, ++STORE, 140173263863808, 140173263871999, ++STORE, 140173263876096, 140173263880191, ++STORE, 140173263880192, 140173263884287, ++STORE, 140173263884288, 140173263888383, ++STORE, 140729801007104, 140729801142271, ++STORE, 140729801617408, 140729801629695, ++STORE, 140729801629696, 140729801633791, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728166858752, 140737488351231, ++SNULL, 140728166862847, 140737488351231, ++STORE, 140728166858752, 140728166862847, ++STORE, 140728166727680, 140728166862847, ++STORE, 93912949866496, 93912950337535, ++SNULL, 93912950288383, 93912950337535, ++STORE, 93912949866496, 93912950288383, ++STORE, 93912950288384, 93912950337535, ++ERASE, 93912950288384, 93912950337535, ++STORE, 93912950292480, 93912950337535, ++STORE, 139921863385088, 139921865637887, ++SNULL, 139921863528447, 139921865637887, ++STORE, 139921863385088, 139921863528447, ++STORE, 139921863528448, 139921865637887, ++ERASE, 139921863528448, 139921865637887, ++STORE, 139921865625600, 139921865633791, ++STORE, 139921865633792, 139921865637887, ++STORE, 140728167899136, 140728167903231, ++STORE, 140728167886848, 140728167899135, ++STORE, 139921865601024, 139921865625599, ++STORE, 139921865592832, 139921865601023, ++STORE, 139921861251072, 139921863385087, ++SNULL, 139921861251072, 139921861279743, ++STORE, 139921861279744, 139921863385087, ++STORE, 139921861251072, 139921861279743, ++SNULL, 139921863376895, 139921863385087, ++STORE, 139921861279744, 139921863376895, ++STORE, 139921863376896, 139921863385087, ++ERASE, 139921863376896, 139921863385087, ++STORE, 139921863376896, 139921863385087, ++STORE, 139921858867200, 139921861251071, ++SNULL, 139921858867200, 139921859133439, ++STORE, 139921859133440, 139921861251071, ++STORE, 139921858867200, 139921859133439, ++SNULL, 139921861226495, 139921861251071, ++STORE, 139921859133440, 139921861226495, ++STORE, 139921861226496, 139921861251071, ++SNULL, 139921861226496, 139921861246975, ++STORE, 139921861246976, 139921861251071, ++STORE, 139921861226496, 139921861246975, ++ERASE, 139921861226496, 139921861246975, ++STORE, 139921861226496, 139921861246975, ++ERASE, 139921861246976, 139921861251071, ++STORE, 139921861246976, 139921861251071, ++STORE, 139921856675840, 139921858867199, ++SNULL, 139921856675840, 139921856765951, ++STORE, 139921856765952, 139921858867199, ++STORE, 139921856675840, 139921856765951, ++SNULL, 139921858859007, 139921858867199, ++STORE, 139921856765952, 139921858859007, ++STORE, 139921858859008, 139921858867199, ++ERASE, 139921858859008, 139921858867199, ++STORE, 139921858859008, 139921858867199, ++STORE, 139921854414848, 139921856675839, ++SNULL, 139921854414848, 139921854566399, ++STORE, 139921854566400, 139921856675839, ++STORE, 139921854414848, 139921854566399, ++SNULL, 139921856659455, 139921856675839, ++STORE, 139921854566400, 139921856659455, ++STORE, 139921856659456, 139921856675839, ++SNULL, 139921856659456, 139921856667647, ++STORE, 139921856667648, 139921856675839, ++STORE, 139921856659456, 139921856667647, ++ERASE, 139921856659456, 139921856667647, ++STORE, 139921856659456, 139921856667647, ++ERASE, 139921856667648, 139921856675839, ++STORE, 139921856667648, 139921856675839, ++STORE, 139921852284928, 139921854414847, ++SNULL, 139921852284928, 139921852313599, ++STORE, 139921852313600, 139921854414847, ++STORE, 139921852284928, 139921852313599, ++SNULL, 139921854406655, 139921854414847, ++STORE, 139921852313600, 139921854406655, ++STORE, 139921854406656, 139921854414847, ++ERASE, 139921854406656, 139921854414847, ++STORE, 139921854406656, 139921854414847, ++STORE, 139921850068992, 139921852284927, ++SNULL, 139921850068992, 139921850167295, ++STORE, 139921850167296, 139921852284927, ++STORE, 139921850068992, 139921850167295, ++SNULL, 139921852260351, 139921852284927, ++STORE, 139921850167296, 139921852260351, ++STORE, 139921852260352, 139921852284927, ++SNULL, 139921852260352, 139921852268543, ++STORE, 139921852268544, 139921852284927, ++STORE, 139921852260352, 139921852268543, ++ERASE, 139921852260352, 139921852268543, ++STORE, 139921852260352, 139921852268543, ++ERASE, 139921852268544, 139921852284927, ++STORE, 139921852268544, 139921852284927, ++STORE, 139921865584640, 139921865601023, ++STORE, 139921846272000, 139921850068991, ++SNULL, 139921846272000, 139921847930879, ++STORE, 139921847930880, 139921850068991, ++STORE, 139921846272000, 139921847930879, ++SNULL, 139921850028031, 139921850068991, ++STORE, 139921847930880, 139921850028031, ++STORE, 139921850028032, 139921850068991, ++SNULL, 139921850028032, 139921850052607, ++STORE, 139921850052608, 139921850068991, ++STORE, 139921850028032, 139921850052607, ++ERASE, 139921850028032, 139921850052607, ++STORE, 139921850028032, 139921850052607, ++ERASE, 139921850052608, 139921850068991, ++STORE, 139921850052608, 139921850068991, ++STORE, 139921844154368, 139921846271999, ++SNULL, 139921844154368, 139921844170751, ++STORE, 139921844170752, 139921846271999, ++STORE, 139921844154368, 139921844170751, ++SNULL, 139921846263807, 139921846271999, ++STORE, 139921844170752, 139921846263807, ++STORE, 139921846263808, 139921846271999, ++ERASE, 139921846263808, 139921846271999, ++STORE, 139921846263808, 139921846271999, ++STORE, 139921842036736, 139921844154367, ++SNULL, 139921842036736, 139921842053119, ++STORE, 139921842053120, 139921844154367, ++STORE, 139921842036736, 139921842053119, ++SNULL, 139921844146175, 139921844154367, ++STORE, 139921842053120, 139921844146175, ++STORE, 139921844146176, 139921844154367, ++ERASE, 139921844146176, 139921844154367, ++STORE, 139921844146176, 139921844154367, ++STORE, 139921839468544, 139921842036735, ++SNULL, 139921839468544, 139921839935487, ++STORE, 139921839935488, 139921842036735, ++STORE, 139921839468544, 139921839935487, ++SNULL, 139921842028543, 139921842036735, ++STORE, 139921839935488, 139921842028543, ++STORE, 139921842028544, 139921842036735, ++ERASE, 139921842028544, 139921842036735, ++STORE, 139921842028544, 139921842036735, ++STORE, 139921837355008, 139921839468543, ++SNULL, 139921837355008, 139921837367295, ++STORE, 139921837367296, 139921839468543, ++STORE, 139921837355008, 139921837367295, ++SNULL, 139921839460351, 139921839468543, ++STORE, 139921837367296, 139921839460351, ++STORE, 139921839460352, 139921839468543, ++ERASE, 139921839460352, 139921839468543, ++STORE, 139921839460352, 139921839468543, ++STORE, 139921865576448, 139921865601023, ++STORE, 139921865564160, 139921865601023, ++SNULL, 139921850044415, 139921850052607, ++STORE, 139921850028032, 139921850044415, ++STORE, 139921850044416, 139921850052607, ++SNULL, 139921839464447, 139921839468543, ++STORE, 139921839460352, 139921839464447, ++STORE, 139921839464448, 139921839468543, ++SNULL, 139921852264447, 139921852268543, ++STORE, 139921852260352, 139921852264447, ++STORE, 139921852264448, 139921852268543, ++SNULL, 139921842032639, 139921842036735, ++STORE, 139921842028544, 139921842032639, ++STORE, 139921842032640, 139921842036735, ++SNULL, 139921844150271, 139921844154367, ++STORE, 139921844146176, 139921844150271, ++STORE, 139921844150272, 139921844154367, ++SNULL, 139921846267903, 139921846271999, ++STORE, 139921846263808, 139921846267903, ++STORE, 139921846267904, 139921846271999, ++SNULL, 139921854410751, 139921854414847, ++STORE, 139921854406656, 139921854410751, ++STORE, 139921854410752, 139921854414847, ++SNULL, 139921856663551, 139921856667647, ++STORE, 139921856659456, 139921856663551, ++STORE, 139921856663552, 139921856667647, ++SNULL, 139921858863103, 139921858867199, ++STORE, 139921858859008, 139921858863103, ++STORE, 139921858863104, 139921858867199, ++SNULL, 139921861242879, 139921861246975, ++STORE, 139921861226496, 139921861242879, ++STORE, 139921861242880, 139921861246975, ++SNULL, 139921863380991, 139921863385087, ++STORE, 139921863376896, 139921863380991, ++STORE, 139921863380992, 139921863385087, ++SNULL, 93912950333439, 93912950337535, ++STORE, 93912950292480, 93912950333439, ++STORE, 93912950333440, 93912950337535, ++SNULL, 139921865629695, 139921865633791, ++STORE, 139921865625600, 139921865629695, ++STORE, 139921865629696, 139921865633791, ++ERASE, 139921865601024, 139921865625599, ++STORE, 93912968110080, 93912968245247, ++STORE, 139921828913152, 139921837355007, ++STORE, 139921865621504, 139921865625599, ++STORE, 139921865617408, 139921865621503, ++STORE, 139921865613312, 139921865617407, ++STORE, 139921865547776, 139921865564159, ++ }; + -+ if (need_aging) -+ try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); ++ unsigned long set17[] = { ++STORE, 94397057224704, 94397057646591, ++STORE, 94397057650688, 94397057691647, ++STORE, 94397057691648, 94397057695743, ++STORE, 94397075271680, 94397075406847, ++STORE, 139953169051648, 139953169063935, ++STORE, 139953169063936, 139953171156991, ++STORE, 139953171156992, 139953171161087, ++STORE, 139953171161088, 139953171165183, ++STORE, 139953171165184, 139953171632127, ++STORE, 139953171632128, 139953173725183, ++STORE, 139953173725184, 139953173729279, ++STORE, 139953173729280, 139953173733375, ++STORE, 139953173733376, 139953173749759, ++STORE, 139953173749760, 139953175842815, ++STORE, 139953175842816, 139953175846911, ++STORE, 139953175846912, 139953175851007, ++STORE, 139953175851008, 139953175867391, ++STORE, 139953175867392, 139953177960447, ++STORE, 139953177960448, 139953177964543, ++STORE, 139953177964544, 139953177968639, ++STORE, 139953177968640, 139953179627519, ++STORE, 139953179627520, 139953181724671, ++STORE, 139953181724672, 139953181741055, ++STORE, 139953181741056, 139953181749247, ++STORE, 139953181749248, 139953181765631, ++STORE, 139953181765632, 139953181863935, ++STORE, 139953181863936, 139953183956991, ++STORE, 139953183956992, 139953183961087, ++STORE, 139953183961088, 139953183965183, ++STORE, 139953183965184, 139953183981567, ++STORE, 139953183981568, 139953184010239, ++STORE, 139953184010240, 139953186103295, ++STORE, 139953186103296, 139953186107391, ++STORE, 139953186107392, 139953186111487, ++STORE, 139953186111488, 139953186263039, ++STORE, 139953186263040, 139953188356095, ++STORE, 139953188356096, 139953188360191, ++STORE, 139953188360192, 139953188364287, ++STORE, 139953188364288, 139953188372479, ++STORE, 139953188372480, 139953188462591, ++STORE, 139953188462592, 139953190555647, ++STORE, 139953190555648, 139953190559743, ++STORE, 139953190559744, 139953190563839, ++STORE, 139953190563840, 139953190830079, ++STORE, 139953190830080, 139953192923135, ++STORE, 139953192923136, 139953192939519, ++STORE, 139953192939520, 139953192943615, ++STORE, 139953192943616, 139953192947711, ++STORE, 139953192947712, 139953192976383, ++STORE, 139953192976384, 139953195073535, ++STORE, 139953195073536, 139953195077631, ++STORE, 139953195077632, 139953195081727, ++STORE, 139953195081728, 139953195225087, ++STORE, 139953197281280, 139953197318143, ++STORE, 139953197322240, 139953197326335, ++STORE, 139953197326336, 139953197330431, ++STORE, 139953197330432, 139953197334527, ++STORE, 140720477511680, 140720477646847, ++STORE, 140720478302208, 140720478314495, ++STORE, 140720478314496, 140720478318591, ++ }; ++ unsigned long set18[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140724953673728, 140737488351231, ++SNULL, 140724953677823, 140737488351231, ++STORE, 140724953673728, 140724953677823, ++STORE, 140724953542656, 140724953677823, ++STORE, 94675199266816, 94675199311871, ++SNULL, 94675199303679, 94675199311871, ++STORE, 94675199266816, 94675199303679, ++STORE, 94675199303680, 94675199311871, ++ERASE, 94675199303680, 94675199311871, ++STORE, 94675199303680, 94675199311871, ++STORE, 140222970605568, 140222972858367, ++SNULL, 140222970748927, 140222972858367, ++STORE, 140222970605568, 140222970748927, ++STORE, 140222970748928, 140222972858367, ++ERASE, 140222970748928, 140222972858367, ++STORE, 140222972846080, 140222972854271, ++STORE, 140222972854272, 140222972858367, ++STORE, 140724954365952, 140724954370047, ++STORE, 140724954353664, 140724954365951, ++STORE, 140222972841984, 140222972846079, ++STORE, 140222972833792, 140222972841983, ++STORE, 140222968475648, 140222970605567, ++SNULL, 140222968475648, 140222968504319, ++STORE, 140222968504320, 140222970605567, ++STORE, 140222968475648, 140222968504319, ++SNULL, 140222970597375, 140222970605567, ++STORE, 140222968504320, 140222970597375, ++STORE, 140222970597376, 140222970605567, ++ERASE, 140222970597376, 140222970605567, ++STORE, 140222970597376, 140222970605567, ++ }; ++ unsigned long set19[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140725182459904, 140737488351231, ++SNULL, 140725182463999, 140737488351231, ++STORE, 140725182459904, 140725182463999, ++STORE, 140725182328832, 140725182463999, ++STORE, 94730166636544, 94730166763519, ++SNULL, 94730166747135, 94730166763519, ++STORE, 94730166636544, 94730166747135, ++STORE, 94730166747136, 94730166763519, ++ERASE, 94730166747136, 94730166763519, ++STORE, 94730166751232, 94730166763519, ++STORE, 140656834555904, 140656836808703, ++SNULL, 140656834699263, 140656836808703, ++STORE, 140656834555904, 140656834699263, ++STORE, 140656834699264, 140656836808703, ++ERASE, 140656834699264, 140656836808703, ++STORE, 140656836796416, 140656836804607, ++STORE, 140656836804608, 140656836808703, ++STORE, 140725183389696, 140725183393791, ++STORE, 140725183377408, 140725183389695, ++STORE, 140656836788224, 140656836796415, ++STORE, 140656832331776, 140656834555903, ++SNULL, 140656833982463, 140656834555903, ++STORE, 140656832331776, 140656833982463, ++STORE, 140656833982464, 140656834555903, ++SNULL, 140656833982464, 140656834551807, ++STORE, 140656834551808, 140656834555903, ++STORE, 140656833982464, 140656834551807, ++ERASE, 140656833982464, 140656834551807, ++STORE, 140656833982464, 140656834551807, ++ERASE, 140656834551808, 140656834555903, ++STORE, 140656834551808, 140656834555903, ++STORE, 140656836763648, 140656836788223, ++STORE, 140656830070784, 140656832331775, ++SNULL, 140656830070784, 140656830222335, ++STORE, 140656830222336, 140656832331775, ++STORE, 140656830070784, 140656830222335, ++SNULL, 140656832315391, 140656832331775, ++STORE, 140656830222336, 140656832315391, ++STORE, 140656832315392, 140656832331775, ++SNULL, 140656832315392, 140656832323583, ++STORE, 140656832323584, 140656832331775, ++STORE, 140656832315392, 140656832323583, ++ERASE, 140656832315392, 140656832323583, ++STORE, 140656832315392, 140656832323583, ++ERASE, 140656832323584, 140656832331775, ++STORE, 140656832323584, 140656832331775, ++STORE, 140656827940864, 140656830070783, ++SNULL, 140656827940864, 140656827969535, ++STORE, 140656827969536, 140656830070783, ++STORE, 140656827940864, 140656827969535, ++SNULL, 140656830062591, 140656830070783, ++STORE, 140656827969536, 140656830062591, ++STORE, 140656830062592, 140656830070783, ++ERASE, 140656830062592, 140656830070783, ++STORE, 140656830062592, 140656830070783, ++STORE, 140656825724928, 140656827940863, ++SNULL, 140656825724928, 140656825823231, ++STORE, 140656825823232, 140656827940863, ++STORE, 140656825724928, 140656825823231, ++SNULL, 140656827916287, 140656827940863, ++STORE, 140656825823232, 140656827916287, ++STORE, 140656827916288, 140656827940863, ++SNULL, 140656827916288, 140656827924479, ++STORE, 140656827924480, 140656827940863, ++STORE, 140656827916288, 140656827924479, ++ERASE, 140656827916288, 140656827924479, ++STORE, 140656827916288, 140656827924479, ++ERASE, 140656827924480, 140656827940863, ++STORE, 140656827924480, 140656827940863, ++STORE, 140656821927936, 140656825724927, ++SNULL, 140656821927936, 140656823586815, ++STORE, 140656823586816, 140656825724927, ++STORE, 140656821927936, 140656823586815, ++SNULL, 140656825683967, 140656825724927, ++STORE, 140656823586816, 140656825683967, ++STORE, 140656825683968, 140656825724927, ++SNULL, 140656825683968, 140656825708543, ++STORE, 140656825708544, 140656825724927, ++STORE, 140656825683968, 140656825708543, ++ERASE, 140656825683968, 140656825708543, ++STORE, 140656825683968, 140656825708543, ++ERASE, 140656825708544, 140656825724927, ++STORE, 140656825708544, 140656825724927, ++STORE, 140656819806208, 140656821927935, ++SNULL, 140656819806208, 140656819822591, ++STORE, 140656819822592, 140656821927935, ++STORE, 140656819806208, 140656819822591, ++SNULL, 140656821919743, 140656821927935, ++STORE, 140656819822592, 140656821919743, ++STORE, 140656821919744, 140656821927935, ++ERASE, 140656821919744, 140656821927935, ++STORE, 140656821919744, 140656821927935, ++STORE, 140656836755456, 140656836763647, ++STORE, 140656817553408, 140656819806207, ++SNULL, 140656817553408, 140656817704959, ++STORE, 140656817704960, 140656819806207, ++STORE, 140656817553408, 140656817704959, ++SNULL, 140656819798015, 140656819806207, ++STORE, 140656817704960, 140656819798015, ++STORE, 140656819798016, 140656819806207, ++ERASE, 140656819798016, 140656819806207, ++STORE, 140656819798016, 140656819806207, ++STORE, 140656815382528, 140656817553407, ++SNULL, 140656815382528, 140656815452159, ++STORE, 140656815452160, 140656817553407, ++STORE, 140656815382528, 140656815452159, ++SNULL, 140656817545215, 140656817553407, ++STORE, 140656815452160, 140656817545215, ++STORE, 140656817545216, 140656817553407, ++ERASE, 140656817545216, 140656817553407, ++STORE, 140656817545216, 140656817553407, ++STORE, 140656812171264, 140656815382527, ++SNULL, 140656812171264, 140656813248511, ++STORE, 140656813248512, 140656815382527, ++STORE, 140656812171264, 140656813248511, ++SNULL, 140656815345663, 140656815382527, ++STORE, 140656813248512, 140656815345663, ++STORE, 140656815345664, 140656815382527, ++ERASE, 140656815345664, 140656815382527, ++STORE, 140656815345664, 140656815382527, ++STORE, 140656810037248, 140656812171263, ++SNULL, 140656810037248, 140656810065919, ++STORE, 140656810065920, 140656812171263, ++STORE, 140656810037248, 140656810065919, ++SNULL, 140656812163071, 140656812171263, ++STORE, 140656810065920, 140656812163071, ++STORE, 140656812163072, 140656812171263, ++ERASE, 140656812163072, 140656812171263, ++STORE, 140656812163072, 140656812171263, ++STORE, 140656807727104, 140656810037247, ++SNULL, 140656807727104, 140656807931903, ++STORE, 140656807931904, 140656810037247, ++STORE, 140656807727104, 140656807931903, ++SNULL, 140656810029055, 140656810037247, ++STORE, 140656807931904, 140656810029055, ++STORE, 140656810029056, 140656810037247, ++ERASE, 140656810029056, 140656810037247, ++STORE, 140656810029056, 140656810037247, ++STORE, 140656805343232, 140656807727103, ++SNULL, 140656805343232, 140656805535743, ++STORE, 140656805535744, 140656807727103, ++STORE, 140656805343232, 140656805535743, ++SNULL, 140656807628799, 140656807727103, ++STORE, 140656805535744, 140656807628799, ++STORE, 140656807628800, 140656807727103, ++ERASE, 140656807628800, 140656807727103, ++STORE, 140656807628800, 140656807727103, ++STORE, 140656836747264, 140656836763647, ++STORE, 140656802775040, 140656805343231, ++SNULL, 140656802775040, 140656803241983, ++STORE, 140656803241984, 140656805343231, ++STORE, 140656802775040, 140656803241983, ++SNULL, 140656805335039, 140656805343231, ++STORE, 140656803241984, 140656805335039, ++STORE, 140656805335040, 140656805343231, ++ERASE, 140656805335040, 140656805343231, ++STORE, 140656805335040, 140656805343231, ++STORE, 140656800661504, 140656802775039, ++SNULL, 140656800661504, 140656800673791, ++STORE, 140656800673792, 140656802775039, ++STORE, 140656800661504, 140656800673791, ++SNULL, 140656802766847, 140656802775039, ++STORE, 140656800673792, 140656802766847, ++STORE, 140656802766848, 140656802775039, ++ERASE, 140656802766848, 140656802775039, ++STORE, 140656802766848, 140656802775039, ++STORE, 140656798482432, 140656800661503, ++SNULL, 140656798482432, 140656798560255, ++STORE, 140656798560256, 140656800661503, ++STORE, 140656798482432, 140656798560255, ++SNULL, 140656800653311, 140656800661503, ++STORE, 140656798560256, 140656800653311, ++STORE, 140656800653312, 140656800661503, ++ERASE, 140656800653312, 140656800661503, ++STORE, 140656800653312, 140656800661503, ++STORE, 140656796364800, 140656798482431, ++SNULL, 140656796364800, 140656796381183, ++STORE, 140656796381184, 140656798482431, ++STORE, 140656796364800, 140656796381183, ++SNULL, 140656798474239, 140656798482431, ++STORE, 140656796381184, 140656798474239, ++STORE, 140656798474240, 140656798482431, ++ERASE, 140656798474240, 140656798482431, ++STORE, 140656798474240, 140656798482431, ++STORE, 140656836739072, 140656836763647, ++STORE, 140656836726784, 140656836763647, ++SNULL, 140656825700351, 140656825708543, ++STORE, 140656825683968, 140656825700351, ++STORE, 140656825700352, 140656825708543, ++SNULL, 140656798478335, 140656798482431, ++STORE, 140656798474240, 140656798478335, ++STORE, 140656798478336, 140656798482431, ++SNULL, 140656800657407, 140656800661503, ++STORE, 140656800653312, 140656800657407, ++STORE, 140656800657408, 140656800661503, ++SNULL, 140656802770943, 140656802775039, ++STORE, 140656802766848, 140656802770943, ++STORE, 140656802770944, 140656802775039, ++SNULL, 140656827920383, 140656827924479, ++STORE, 140656827916288, 140656827920383, ++STORE, 140656827920384, 140656827924479, ++SNULL, 140656805339135, 140656805343231, ++STORE, 140656805335040, 140656805339135, ++STORE, 140656805339136, 140656805343231, ++SNULL, 140656807723007, 140656807727103, ++STORE, 140656807628800, 140656807723007, ++STORE, 140656807723008, 140656807727103, ++SNULL, 140656810033151, 140656810037247, ++STORE, 140656810029056, 140656810033151, ++STORE, 140656810033152, 140656810037247, ++SNULL, 140656812167167, 140656812171263, ++STORE, 140656812163072, 140656812167167, ++STORE, 140656812167168, 140656812171263, ++SNULL, 140656815353855, 140656815382527, ++STORE, 140656815345664, 140656815353855, ++STORE, 140656815353856, 140656815382527, ++SNULL, 140656817549311, 140656817553407, ++STORE, 140656817545216, 140656817549311, ++STORE, 140656817549312, 140656817553407, ++SNULL, 140656819802111, 140656819806207, ++STORE, 140656819798016, 140656819802111, ++STORE, 140656819802112, 140656819806207, ++SNULL, 140656821923839, 140656821927935, ++STORE, 140656821919744, 140656821923839, ++STORE, 140656821923840, 140656821927935, ++SNULL, 140656830066687, 140656830070783, ++STORE, 140656830062592, 140656830066687, ++STORE, 140656830066688, 140656830070783, ++SNULL, 140656832319487, 140656832323583, ++STORE, 140656832315392, 140656832319487, ++STORE, 140656832319488, 140656832323583, ++SNULL, 140656834547711, 140656834551807, ++STORE, 140656833982464, 140656834547711, ++STORE, 140656834547712, 140656834551807, ++SNULL, 94730166759423, 94730166763519, ++STORE, 94730166751232, 94730166759423, ++STORE, 94730166759424, 94730166763519, ++SNULL, 140656836800511, 140656836804607, ++STORE, 140656836796416, 140656836800511, ++STORE, 140656836800512, 140656836804607, ++ERASE, 140656836763648, 140656836788223, ++STORE, 94730171318272, 94730171453439, ++STORE, 140656836784128, 140656836788223, ++STORE, 140656836780032, 140656836784127, ++STORE, 140656791920640, 140656796364799, ++STORE, 140656836775936, 140656836780031, ++STORE, 140656787476480, 140656791920639, ++STORE, 140656779083776, 140656787476479, ++SNULL, 140656779087871, 140656787476479, ++STORE, 140656779083776, 140656779087871, ++STORE, 140656779087872, 140656787476479, ++STORE, 140656836771840, 140656836775935, ++STORE, 140656774639616, 140656779083775, ++STORE, 140656766246912, 140656774639615, ++SNULL, 140656766251007, 140656774639615, ++STORE, 140656766246912, 140656766251007, ++STORE, 140656766251008, 140656774639615, ++ERASE, 140656791920640, 140656796364799, ++ERASE, 140656836780032, 140656836784127, ++ERASE, 140656787476480, 140656791920639, ++ERASE, 140656836775936, 140656836780031, ++STORE, 140656836780032, 140656836784127, ++STORE, 140656791920640, 140656796364799, ++STORE, 140656836775936, 140656836780031, ++STORE, 140656787476480, 140656791920639, ++ERASE, 140656774639616, 140656779083775, ++ }; ++ unsigned long set20[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735952392192, 140737488351231, ++SNULL, 140735952396287, 140737488351231, ++STORE, 140735952392192, 140735952396287, ++STORE, 140735952261120, 140735952396287, ++STORE, 94849008947200, 94849009414143, ++SNULL, 94849009364991, 94849009414143, ++STORE, 94849008947200, 94849009364991, ++STORE, 94849009364992, 94849009414143, ++ERASE, 94849009364992, 94849009414143, ++STORE, 94849009364992, 94849009414143, ++STORE, 140590397943808, 140590400196607, ++SNULL, 140590398087167, 140590400196607, ++STORE, 140590397943808, 140590398087167, ++STORE, 140590398087168, 140590400196607, ++ERASE, 140590398087168, 140590400196607, ++STORE, 140590400184320, 140590400192511, ++STORE, 140590400192512, 140590400196607, ++STORE, 140735952850944, 140735952855039, ++STORE, 140735952838656, 140735952850943, ++STORE, 140590400180224, 140590400184319, ++STORE, 140590400172032, 140590400180223, ++STORE, 140590395809792, 140590397943807, ++SNULL, 140590395809792, 140590395838463, ++STORE, 140590395838464, 140590397943807, ++STORE, 140590395809792, 140590395838463, ++SNULL, 140590397935615, 140590397943807, ++STORE, 140590395838464, 140590397935615, ++STORE, 140590397935616, 140590397943807, ++ERASE, 140590397935616, 140590397943807, ++STORE, 140590397935616, 140590397943807, ++STORE, 140590393425920, 140590395809791, ++SNULL, 140590393425920, 140590393692159, ++STORE, 140590393692160, 140590395809791, ++STORE, 140590393425920, 140590393692159, ++SNULL, 140590395785215, 140590395809791, ++STORE, 140590393692160, 140590395785215, ++STORE, 140590395785216, 140590395809791, ++SNULL, 140590395785216, 140590395805695, ++STORE, 140590395805696, 140590395809791, ++STORE, 140590395785216, 140590395805695, ++ERASE, 140590395785216, 140590395805695, ++STORE, 140590395785216, 140590395805695, ++ERASE, 140590395805696, 140590395809791, ++STORE, 140590395805696, 140590395809791, ++STORE, 140590391234560, 140590393425919, ++SNULL, 140590391234560, 140590391324671, ++STORE, 140590391324672, 140590393425919, ++STORE, 140590391234560, 140590391324671, ++SNULL, 140590393417727, 140590393425919, ++STORE, 140590391324672, 140590393417727, ++STORE, 140590393417728, 140590393425919, ++ERASE, 140590393417728, 140590393425919, ++STORE, 140590393417728, 140590393425919, ++STORE, 140590388973568, 140590391234559, ++SNULL, 140590388973568, 140590389125119, ++STORE, 140590389125120, 140590391234559, ++STORE, 140590388973568, 140590389125119, ++SNULL, 140590391218175, 140590391234559, ++STORE, 140590389125120, 140590391218175, ++STORE, 140590391218176, 140590391234559, ++SNULL, 140590391218176, 140590391226367, ++STORE, 140590391226368, 140590391234559, ++STORE, 140590391218176, 140590391226367, ++ERASE, 140590391218176, 140590391226367, ++STORE, 140590391218176, 140590391226367, ++ERASE, 140590391226368, 140590391234559, ++STORE, 140590391226368, 140590391234559, ++STORE, 140590386843648, 140590388973567, ++SNULL, 140590386843648, 140590386872319, ++STORE, 140590386872320, 140590388973567, ++STORE, 140590386843648, 140590386872319, ++SNULL, 140590388965375, 140590388973567, ++STORE, 140590386872320, 140590388965375, ++STORE, 140590388965376, 140590388973567, ++ERASE, 140590388965376, 140590388973567, ++STORE, 140590388965376, 140590388973567, ++STORE, 140590384627712, 140590386843647, ++SNULL, 140590384627712, 140590384726015, ++STORE, 140590384726016, 140590386843647, ++STORE, 140590384627712, 140590384726015, ++SNULL, 140590386819071, 140590386843647, ++STORE, 140590384726016, 140590386819071, ++STORE, 140590386819072, 140590386843647, ++SNULL, 140590386819072, 140590386827263, ++STORE, 140590386827264, 140590386843647, ++STORE, 140590386819072, 140590386827263, ++ERASE, 140590386819072, 140590386827263, ++STORE, 140590386819072, 140590386827263, ++ERASE, 140590386827264, 140590386843647, ++STORE, 140590386827264, 140590386843647, ++STORE, 140590400163840, 140590400180223, ++STORE, 140590380830720, 140590384627711, ++SNULL, 140590380830720, 140590382489599, ++STORE, 140590382489600, 140590384627711, ++STORE, 140590380830720, 140590382489599, ++SNULL, 140590384586751, 140590384627711, ++STORE, 140590382489600, 140590384586751, ++STORE, 140590384586752, 140590384627711, ++SNULL, 140590384586752, 140590384611327, ++STORE, 140590384611328, 140590384627711, ++STORE, 140590384586752, 140590384611327, ++ERASE, 140590384586752, 140590384611327, ++STORE, 140590384586752, 140590384611327, ++ERASE, 140590384611328, 140590384627711, ++STORE, 140590384611328, 140590384627711, ++STORE, 140590378713088, 140590380830719, ++SNULL, 140590378713088, 140590378729471, ++STORE, 140590378729472, 140590380830719, ++STORE, 140590378713088, 140590378729471, ++SNULL, 140590380822527, 140590380830719, ++STORE, 140590378729472, 140590380822527, ++STORE, 140590380822528, 140590380830719, ++ERASE, 140590380822528, 140590380830719, ++STORE, 140590380822528, 140590380830719, ++STORE, 140590376595456, 140590378713087, ++SNULL, 140590376595456, 140590376611839, ++STORE, 140590376611840, 140590378713087, ++STORE, 140590376595456, 140590376611839, ++SNULL, 140590378704895, 140590378713087, ++STORE, 140590376611840, 140590378704895, ++STORE, 140590378704896, 140590378713087, ++ERASE, 140590378704896, 140590378713087, ++STORE, 140590378704896, 140590378713087, ++STORE, 140590374027264, 140590376595455, ++SNULL, 140590374027264, 140590374494207, ++STORE, 140590374494208, 140590376595455, ++STORE, 140590374027264, 140590374494207, ++SNULL, 140590376587263, 140590376595455, ++STORE, 140590374494208, 140590376587263, ++STORE, 140590376587264, 140590376595455, ++ERASE, 140590376587264, 140590376595455, ++STORE, 140590376587264, 140590376595455, ++STORE, 140590371913728, 140590374027263, ++SNULL, 140590371913728, 140590371926015, ++STORE, 140590371926016, 140590374027263, ++STORE, 140590371913728, 140590371926015, ++SNULL, 140590374019071, 140590374027263, ++STORE, 140590371926016, 140590374019071, ++STORE, 140590374019072, 140590374027263, ++ERASE, 140590374019072, 140590374027263, ++STORE, 140590374019072, 140590374027263, ++STORE, 140590400155648, 140590400180223, ++STORE, 140590400143360, 140590400180223, ++SNULL, 140590384603135, 140590384611327, ++STORE, 140590384586752, 140590384603135, ++STORE, 140590384603136, 140590384611327, ++SNULL, 140590374023167, 140590374027263, ++STORE, 140590374019072, 140590374023167, ++STORE, 140590374023168, 140590374027263, ++SNULL, 140590386823167, 140590386827263, ++STORE, 140590386819072, 140590386823167, ++STORE, 140590386823168, 140590386827263, ++SNULL, 140590376591359, 140590376595455, ++ }; ++ unsigned long set21[] = { ++STORE, 93874710941696, 93874711363583, ++STORE, 93874711367680, 93874711408639, ++STORE, 93874711408640, 93874711412735, ++STORE, 93874720989184, 93874721124351, ++STORE, 140708365086720, 140708365099007, ++STORE, 140708365099008, 140708367192063, ++STORE, 140708367192064, 140708367196159, ++STORE, 140708367196160, 140708367200255, ++STORE, 140708367200256, 140708367667199, ++STORE, 140708367667200, 140708369760255, ++STORE, 140708369760256, 140708369764351, ++STORE, 140708369764352, 140708369768447, ++STORE, 140708369768448, 140708369784831, ++STORE, 140708369784832, 140708371877887, ++STORE, 140708371877888, 140708371881983, ++STORE, 140708371881984, 140708371886079, ++STORE, 140708371886080, 140708371902463, ++STORE, 140708371902464, 140708373995519, ++STORE, 140708373995520, 140708373999615, ++STORE, 140708373999616, 140708374003711, ++STORE, 140708374003712, 140708375662591, ++STORE, 140708375662592, 140708377759743, ++STORE, 140708377759744, 140708377776127, ++STORE, 140708377776128, 140708377784319, ++STORE, 140708377784320, 140708377800703, ++STORE, 140708377800704, 140708377899007, ++STORE, 140708377899008, 140708379992063, ++STORE, 140708379992064, 140708379996159, ++STORE, 140708379996160, 140708380000255, ++STORE, 140708380000256, 140708380016639, ++STORE, 140708380016640, 140708380045311, ++STORE, 140708380045312, 140708382138367, ++STORE, 140708382138368, 140708382142463, ++STORE, 140708382142464, 140708382146559, ++STORE, 140708382146560, 140708382298111, ++STORE, 140708382298112, 140708384391167, ++STORE, 140708384391168, 140708384395263, ++STORE, 140708384395264, 140708384399359, ++STORE, 140708384399360, 140708384407551, ++STORE, 140708384407552, 140708384497663, ++STORE, 140708384497664, 140708386590719, ++STORE, 140708386590720, 140708386594815, ++STORE, 140708386594816, 140708386598911, ++STORE, 140708386598912, 140708386865151, ++STORE, 140708386865152, 140708388958207, ++STORE, 140708388958208, 140708388974591, ++STORE, 140708388974592, 140708388978687, ++STORE, 140708388978688, 140708388982783, ++STORE, 140708388982784, 140708389011455, ++STORE, 140708389011456, 140708391108607, ++STORE, 140708391108608, 140708391112703, ++STORE, 140708391112704, 140708391116799, ++STORE, 140708391116800, 140708391260159, ++STORE, 140708393291776, 140708393308159, ++STORE, 140708393308160, 140708393312255, ++STORE, 140708393312256, 140708393316351, ++STORE, 140708393316352, 140708393353215, ++STORE, 140708393353216, 140708393357311, ++STORE, 140708393357312, 140708393361407, ++STORE, 140708393361408, 140708393365503, ++STORE, 140708393365504, 140708393369599, ++STORE, 140730557042688, 140730557177855, ++STORE, 140730557235200, 140730557247487, ++STORE, 140730557247488, 140730557251583, ++ERASE, 140708393353216, 140708393357311, ++ERASE, 140708393312256, 140708393316351, ++ERASE, 140708393308160, 140708393312255, ++ERASE, 140708393291776, 140708393308159, ++ }; ++ unsigned long set22[] = { ++STORE, 93951397134336, 93951397183487, ++STORE, 93951397183488, 93951397728255, ++STORE, 93951397728256, 93951397826559, ++STORE, 93951397826560, 93951397842943, ++STORE, 93951397842944, 93951397847039, ++STORE, 93951425974272, 93951426109439, ++STORE, 140685152665600, 140685152677887, ++STORE, 140685152677888, 140685152829439, ++STORE, 140685152829440, 140685154181119, ++STORE, 140685154181120, 140685154484223, ++STORE, 140685154484224, 140685154496511, ++STORE, 140685154496512, 140685154508799, ++STORE, 140685154508800, 140685154525183, ++STORE, 140685154525184, 140685154541567, ++STORE, 140685154541568, 140685154590719, ++STORE, 140685154590720, 140685154603007, ++STORE, 140685154603008, 140685154607103, ++STORE, 140685154607104, 140685154611199, ++STORE, 140685154611200, 140685154615295, ++STORE, 140685154615296, 140685154631679, ++STORE, 140685154639872, 140685154643967, ++STORE, 140685154643968, 140685154766847, ++STORE, 140685154766848, 140685154799615, ++STORE, 140685154803712, 140685154807807, ++STORE, 140685154807808, 140685154811903, ++STORE, 140685154811904, 140685154815999, ++STORE, 140722188902400, 140722189037567, ++STORE, 140722189512704, 140722189524991, ++STORE, 140722189524992, 140722189529087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733429354496, 140737488351231, ++SNULL, 140733429358591, 140737488351231, ++STORE, 140733429354496, 140733429358591, ++STORE, 140733429223424, 140733429358591, ++STORE, 94526683537408, 94526683660287, ++SNULL, 94526683553791, 94526683660287, ++STORE, 94526683537408, 94526683553791, ++STORE, 94526683553792, 94526683660287, ++ERASE, 94526683553792, 94526683660287, ++STORE, 94526683553792, 94526683623423, ++STORE, 94526683623424, 94526683647999, ++STORE, 94526683652096, 94526683660287, ++STORE, 140551363747840, 140551363923967, ++SNULL, 140551363751935, 140551363923967, ++STORE, 140551363747840, 140551363751935, ++STORE, 140551363751936, 140551363923967, ++ERASE, 140551363751936, 140551363923967, ++STORE, 140551363751936, 140551363874815, ++STORE, 140551363874816, 140551363907583, ++STORE, 140551363911680, 140551363919871, ++STORE, 140551363919872, 140551363923967, ++STORE, 140733429690368, 140733429694463, ++STORE, 140733429678080, 140733429690367, ++STORE, 140551363739648, 140551363747839, ++STORE, 140551363731456, 140551363739647, ++STORE, 140551363379200, 140551363731455, ++SNULL, 140551363379200, 140551363420159, ++STORE, 140551363420160, 140551363731455, ++STORE, 140551363379200, 140551363420159, ++SNULL, 140551363706879, 140551363731455, ++STORE, 140551363420160, 140551363706879, ++STORE, 140551363706880, 140551363731455, ++SNULL, 140551363420160, 140551363637247, ++STORE, 140551363637248, 140551363706879, ++STORE, 140551363420160, 140551363637247, ++ERASE, 140551363420160, 140551363637247, ++STORE, 140551363420160, 140551363637247, ++SNULL, 140551363637248, 140551363702783, ++STORE, 140551363702784, 140551363706879, ++STORE, 140551363637248, 140551363702783, ++ERASE, 140551363637248, 140551363702783, ++STORE, 140551363637248, 140551363702783, ++ERASE, 140551363706880, 140551363731455, ++STORE, 140551363706880, 140551363731455, ++STORE, 140551361531904, 140551363379199, ++SNULL, 140551361683455, 140551363379199, ++STORE, 140551361531904, 140551361683455, ++STORE, 140551361683456, 140551363379199, ++SNULL, 140551361683456, 140551363035135, ++STORE, 140551363035136, 140551363379199, ++STORE, 140551361683456, 140551363035135, ++ERASE, 140551361683456, 140551363035135, ++STORE, 140551361683456, 140551363035135, ++SNULL, 140551363035136, 140551363338239, ++STORE, 140551363338240, 140551363379199, ++STORE, 140551363035136, 140551363338239, ++ERASE, 140551363035136, 140551363338239, ++STORE, 140551363035136, 140551363379199, ++SNULL, 140551363338239, 140551363379199, ++STORE, 140551363035136, 140551363338239, ++STORE, 140551363338240, 140551363379199, ++SNULL, 140551363338240, 140551363362815, ++STORE, 140551363362816, 140551363379199, ++STORE, 140551363338240, 140551363362815, ++ERASE, 140551363338240, 140551363362815, ++STORE, 140551363338240, 140551363362815, ++ERASE, 140551363362816, 140551363379199, ++STORE, 140551363362816, 140551363379199, ++STORE, 140551361519616, 140551361531903, ++SNULL, 140551363350527, 140551363362815, ++STORE, 140551363338240, 140551363350527, ++STORE, 140551363350528, 140551363362815, ++SNULL, 140551363727359, 140551363731455, ++STORE, 140551363706880, 140551363727359, ++STORE, 140551363727360, 140551363731455, ++SNULL, 94526683656191, 94526683660287, ++STORE, 94526683652096, 94526683656191, ++STORE, 94526683656192, 94526683660287, ++SNULL, 140551363915775, 140551363919871, ++STORE, 140551363911680, 140551363915775, ++STORE, 140551363915776, 140551363919871, ++ERASE, 140551363739648, 140551363747839, ++STORE, 94526715490304, 94526715625471, ++STORE, 140551361253376, 140551361531903, ++STORE, 140551360987136, 140551361531903, ++STORE, 140551360720896, 140551361531903, ++STORE, 140551360454656, 140551361531903, ++SNULL, 140551361253375, 140551361531903, ++STORE, 140551360454656, 140551361253375, ++STORE, 140551361253376, 140551361531903, ++SNULL, 140551361253376, 140551361519615, ++STORE, 140551361519616, 140551361531903, ++STORE, 140551361253376, 140551361519615, ++ERASE, 140551361253376, 140551361519615, ++ }; + -+ return true; -+} ++ unsigned long set23[] = { ++STORE, 94014447943680, 94014448156671, ++STORE, 94014450253824, 94014450257919, ++STORE, 94014450257920, 94014450266111, ++STORE, 94014450266112, 94014450278399, ++STORE, 94014464225280, 94014464630783, ++STORE, 139761764306944, 139761765965823, ++STORE, 139761765965824, 139761768062975, ++STORE, 139761768062976, 139761768079359, ++STORE, 139761768079360, 139761768087551, ++STORE, 139761768087552, 139761768103935, ++STORE, 139761768103936, 139761768116223, ++STORE, 139761768116224, 139761770209279, ++STORE, 139761770209280, 139761770213375, ++STORE, 139761770213376, 139761770217471, ++STORE, 139761770217472, 139761770360831, ++STORE, 139761770729472, 139761772412927, ++STORE, 139761772412928, 139761772429311, ++STORE, 139761772457984, 139761772462079, ++STORE, 139761772462080, 139761772466175, ++STORE, 139761772466176, 139761772470271, ++STORE, 140724336517120, 140724336652287, ++STORE, 140724336955392, 140724336967679, ++STORE, 140724336967680, 140724336971775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140721840295936, 140737488351231, ++SNULL, 140721840300031, 140737488351231, ++STORE, 140721840295936, 140721840300031, ++STORE, 140721840164864, 140721840300031, ++STORE, 93937913667584, 93937915830271, ++SNULL, 93937913729023, 93937915830271, ++STORE, 93937913667584, 93937913729023, ++STORE, 93937913729024, 93937915830271, ++ERASE, 93937913729024, 93937915830271, ++STORE, 93937915822080, 93937915830271, ++STORE, 140598835335168, 140598837587967, ++SNULL, 140598835478527, 140598837587967, ++STORE, 140598835335168, 140598835478527, ++STORE, 140598835478528, 140598837587967, ++ERASE, 140598835478528, 140598837587967, ++STORE, 140598837575680, 140598837583871, ++STORE, 140598837583872, 140598837587967, ++STORE, 140721841086464, 140721841090559, ++STORE, 140721841074176, 140721841086463, ++STORE, 140598837547008, 140598837575679, ++STORE, 140598837538816, 140598837547007, ++STORE, 140598831538176, 140598835335167, ++SNULL, 140598831538176, 140598833197055, ++STORE, 140598833197056, 140598835335167, ++STORE, 140598831538176, 140598833197055, ++SNULL, 140598835294207, 140598835335167, ++STORE, 140598833197056, 140598835294207, ++STORE, 140598835294208, 140598835335167, ++SNULL, 140598835294208, 140598835318783, ++STORE, 140598835318784, 140598835335167, ++STORE, 140598835294208, 140598835318783, ++ERASE, 140598835294208, 140598835318783, ++STORE, 140598835294208, 140598835318783, ++ERASE, 140598835318784, 140598835335167, ++STORE, 140598835318784, 140598835335167, ++SNULL, 140598835310591, 140598835318783, ++STORE, 140598835294208, 140598835310591, ++STORE, 140598835310592, 140598835318783, ++SNULL, 93937915826175, 93937915830271, ++STORE, 93937915822080, 93937915826175, ++STORE, 93937915826176, 93937915830271, ++SNULL, 140598837579775, 140598837583871, ++STORE, 140598837575680, 140598837579775, ++STORE, 140598837579776, 140598837583871, ++ERASE, 140598837547008, 140598837575679, ++STORE, 93937929179136, 93937929314303, ++STORE, 140598835855360, 140598837538815, ++STORE, 140737488347136, 140737488351231, ++STORE, 140728187723776, 140737488351231, ++SNULL, 140728187727871, 140737488351231, ++STORE, 140728187723776, 140728187727871, ++STORE, 140728187592704, 140728187727871, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140583951437824, 140583953690623, ++SNULL, 140583951581183, 140583953690623, ++STORE, 140583951437824, 140583951581183, ++STORE, 140583951581184, 140583953690623, ++ERASE, 140583951581184, 140583953690623, ++STORE, 140583953678336, 140583953686527, ++STORE, 140583953686528, 140583953690623, ++STORE, 140728189116416, 140728189120511, ++STORE, 140728189104128, 140728189116415, ++STORE, 140583953649664, 140583953678335, ++STORE, 140583953641472, 140583953649663, ++STORE, 140583948275712, 140583951437823, ++SNULL, 140583948275712, 140583949336575, ++STORE, 140583949336576, 140583951437823, ++STORE, 140583948275712, 140583949336575, ++SNULL, 140583951429631, 140583951437823, ++STORE, 140583949336576, 140583951429631, ++STORE, 140583951429632, 140583951437823, ++ERASE, 140583951429632, 140583951437823, ++STORE, 140583951429632, 140583951437823, ++STORE, 140583944478720, 140583948275711, ++SNULL, 140583944478720, 140583946137599, ++STORE, 140583946137600, 140583948275711, ++STORE, 140583944478720, 140583946137599, ++SNULL, 140583948234751, 140583948275711, ++STORE, 140583946137600, 140583948234751, ++STORE, 140583948234752, 140583948275711, ++SNULL, 140583948234752, 140583948259327, ++STORE, 140583948259328, 140583948275711, ++STORE, 140583948234752, 140583948259327, ++ERASE, 140583948234752, 140583948259327, ++STORE, 140583948234752, 140583948259327, ++ERASE, 140583948259328, 140583948275711, ++STORE, 140583948259328, 140583948275711, ++STORE, 140583953629184, 140583953649663, ++SNULL, 140583948251135, 140583948259327, ++STORE, 140583948234752, 140583948251135, ++STORE, 140583948251136, 140583948259327, ++SNULL, 140583951433727, 140583951437823, ++STORE, 140583951429632, 140583951433727, ++STORE, 140583951433728, 140583951437823, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140583953682431, 140583953686527, ++STORE, 140583953678336, 140583953682431, ++STORE, 140583953682432, 140583953686527, ++ERASE, 140583953649664, 140583953678335, ++STORE, 17821696, 17956863, ++STORE, 17821696, 18104319, ++STORE, 140583951945728, 140583953629183, ++STORE, 94014447943680, 94014448156671, ++STORE, 94014450253824, 94014450257919, ++STORE, 94014450257920, 94014450266111, ++STORE, 94014450266112, 94014450278399, ++STORE, 94014464225280, 94014465196031, ++STORE, 139761764306944, 139761765965823, ++STORE, 139761765965824, 139761768062975, ++STORE, 139761768062976, 139761768079359, ++STORE, 139761768079360, 139761768087551, ++STORE, 139761768087552, 139761768103935, ++STORE, 139761768103936, 139761768116223, ++STORE, 139761768116224, 139761770209279, ++STORE, 139761770209280, 139761770213375, ++STORE, 139761770213376, 139761770217471, ++STORE, 139761770217472, 139761770360831, ++STORE, 139761770729472, 139761772412927, ++STORE, 139761772412928, 139761772429311, ++STORE, 139761772457984, 139761772462079, ++STORE, 139761772462080, 139761772466175, ++STORE, 139761772466176, 139761772470271, ++STORE, 140724336517120, 140724336652287, ++STORE, 140724336955392, 140724336967679, ++STORE, 140724336967680, 140724336971775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726063296512, 140737488351231, ++SNULL, 140726063300607, 140737488351231, ++STORE, 140726063296512, 140726063300607, ++STORE, 140726063165440, 140726063300607, ++STORE, 94016795934720, 94016798158847, ++SNULL, 94016796045311, 94016798158847, ++STORE, 94016795934720, 94016796045311, ++STORE, 94016796045312, 94016798158847, ++ERASE, 94016796045312, 94016798158847, ++STORE, 94016798138368, 94016798150655, ++STORE, 94016798150656, 94016798158847, ++STORE, 139975915966464, 139975918219263, ++SNULL, 139975916109823, 139975918219263, ++STORE, 139975915966464, 139975916109823, ++STORE, 139975916109824, 139975918219263, ++ERASE, 139975916109824, 139975918219263, ++STORE, 139975918206976, 139975918215167, ++STORE, 139975918215168, 139975918219263, ++STORE, 140726064541696, 140726064545791, ++STORE, 140726064529408, 140726064541695, ++STORE, 139975918178304, 139975918206975, ++STORE, 139975918170112, 139975918178303, ++STORE, 139975912169472, 139975915966463, ++SNULL, 139975912169472, 139975913828351, ++STORE, 139975913828352, 139975915966463, ++STORE, 139975912169472, 139975913828351, ++SNULL, 139975915925503, 139975915966463, ++STORE, 139975913828352, 139975915925503, ++STORE, 139975915925504, 139975915966463, ++SNULL, 139975915925504, 139975915950079, ++STORE, 139975915950080, 139975915966463, ++STORE, 139975915925504, 139975915950079, ++ERASE, 139975915925504, 139975915950079, ++STORE, 139975915925504, 139975915950079, ++ERASE, 139975915950080, 139975915966463, ++STORE, 139975915950080, 139975915966463, ++SNULL, 139975915941887, 139975915950079, ++STORE, 139975915925504, 139975915941887, ++STORE, 139975915941888, 139975915950079, ++SNULL, 94016798146559, 94016798150655, ++STORE, 94016798138368, 94016798146559, ++STORE, 94016798146560, 94016798150655, ++SNULL, 139975918211071, 139975918215167, ++STORE, 139975918206976, 139975918211071, ++STORE, 139975918211072, 139975918215167, ++ERASE, 139975918178304, 139975918206975, ++STORE, 94016804925440, 94016805060607, ++STORE, 94596177661952, 94596177772543, ++STORE, 94596179865600, 94596179873791, ++STORE, 94596179873792, 94596179877887, ++STORE, 94596179877888, 94596179886079, ++STORE, 94596211597312, 94596211863551, ++STORE, 140127351840768, 140127353499647, ++STORE, 140127353499648, 140127355596799, ++STORE, 140127355596800, 140127355613183, ++STORE, 140127355613184, 140127355621375, ++STORE, 140127355621376, 140127355637759, ++STORE, 140127355637760, 140127355781119, ++STORE, 140127357841408, 140127357849599, ++STORE, 140127357878272, 140127357882367, ++STORE, 140127357882368, 140127357886463, ++STORE, 140127357886464, 140127357890559, ++STORE, 140726167252992, 140726167392255, ++STORE, 140726167838720, 140726167851007, ++STORE, 140726167851008, 140726167855103, ++STORE, 140737488347136, 140737488351231, ++STORE, 140731874017280, 140737488351231, ++SNULL, 140731874021375, 140737488351231, ++STORE, 140731874017280, 140731874021375, ++STORE, 140731873886208, 140731874021375, ++STORE, 94178682265600, 94178684489727, ++SNULL, 94178682376191, 94178684489727, ++STORE, 94178682265600, 94178682376191, ++STORE, 94178682376192, 94178684489727, ++ERASE, 94178682376192, 94178684489727, ++STORE, 94178684469248, 94178684481535, ++STORE, 94178684481536, 94178684489727, ++STORE, 140460853403648, 140460855656447, ++SNULL, 140460853547007, 140460855656447, ++STORE, 140460853403648, 140460853547007, ++STORE, 140460853547008, 140460855656447, ++ERASE, 140460853547008, 140460855656447, ++STORE, 140460855644160, 140460855652351, ++STORE, 140460855652352, 140460855656447, ++STORE, 140731874103296, 140731874107391, ++STORE, 140731874091008, 140731874103295, ++STORE, 140460855615488, 140460855644159, ++STORE, 140460855607296, 140460855615487, ++STORE, 140460849606656, 140460853403647, ++SNULL, 140460849606656, 140460851265535, ++STORE, 140460851265536, 140460853403647, ++STORE, 140460849606656, 140460851265535, ++SNULL, 140460853362687, 140460853403647, ++STORE, 140460851265536, 140460853362687, ++STORE, 140460853362688, 140460853403647, ++SNULL, 140460853362688, 140460853387263, ++STORE, 140460853387264, 140460853403647, ++STORE, 140460853362688, 140460853387263, ++ERASE, 140460853362688, 140460853387263, ++STORE, 140460853362688, 140460853387263, ++ERASE, 140460853387264, 140460853403647, ++STORE, 140460853387264, 140460853403647, ++SNULL, 140460853379071, 140460853387263, ++STORE, 140460853362688, 140460853379071, ++STORE, 140460853379072, 140460853387263, ++SNULL, 94178684477439, 94178684481535, ++STORE, 94178684469248, 94178684477439, ++STORE, 94178684477440, 94178684481535, ++SNULL, 140460855648255, 140460855652351, ++STORE, 140460855644160, 140460855648255, ++STORE, 140460855648256, 140460855652351, ++ERASE, 140460855615488, 140460855644159, ++STORE, 94178692063232, 94178692198399, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733096603648, 140737488351231, ++SNULL, 140733096611839, 140737488351231, ++STORE, 140733096603648, 140733096611839, ++STORE, 140733096472576, 140733096611839, ++STORE, 94796716122112, 94796718325759, ++SNULL, 94796716224511, 94796718325759, ++STORE, 94796716122112, 94796716224511, ++STORE, 94796716224512, 94796718325759, ++ERASE, 94796716224512, 94796718325759, ++STORE, 94796718317568, 94796718325759, ++STORE, 139667892793344, 139667895046143, ++SNULL, 139667892936703, 139667895046143, ++STORE, 139667892793344, 139667892936703, ++STORE, 139667892936704, 139667895046143, ++ERASE, 139667892936704, 139667895046143, ++STORE, 139667895033856, 139667895042047, ++STORE, 139667895042048, 139667895046143, ++STORE, 140733096857600, 140733096861695, ++STORE, 140733096845312, 140733096857599, ++STORE, 139667895005184, 139667895033855, ++STORE, 139667894996992, 139667895005183, ++STORE, 139667890532352, 139667892793343, ++SNULL, 139667890532352, 139667890683903, ++STORE, 139667890683904, 139667892793343, ++STORE, 139667890532352, 139667890683903, ++SNULL, 139667892776959, 139667892793343, ++STORE, 139667890683904, 139667892776959, ++STORE, 139667892776960, 139667892793343, ++SNULL, 139667892776960, 139667892785151, ++STORE, 139667892785152, 139667892793343, ++STORE, 139667892776960, 139667892785151, ++ERASE, 139667892776960, 139667892785151, ++STORE, 139667892776960, 139667892785151, ++ERASE, 139667892785152, 139667892793343, ++STORE, 139667892785152, 139667892793343, ++STORE, 139667886735360, 139667890532351, ++SNULL, 139667886735360, 139667888394239, ++STORE, 139667888394240, 139667890532351, ++STORE, 139667886735360, 139667888394239, ++SNULL, 139667890491391, 139667890532351, ++STORE, 139667888394240, 139667890491391, ++STORE, 139667890491392, 139667890532351, ++SNULL, 139667890491392, 139667890515967, ++STORE, 139667890515968, 139667890532351, ++STORE, 139667890491392, 139667890515967, ++ERASE, 139667890491392, 139667890515967, ++STORE, 139667890491392, 139667890515967, ++ERASE, 139667890515968, 139667890532351, ++STORE, 139667890515968, 139667890532351, ++STORE, 139667884167168, 139667886735359, ++SNULL, 139667884167168, 139667884634111, ++STORE, 139667884634112, 139667886735359, ++STORE, 139667884167168, 139667884634111, ++SNULL, 139667886727167, 139667886735359, ++STORE, 139667884634112, 139667886727167, ++STORE, 139667886727168, 139667886735359, ++ERASE, 139667886727168, 139667886735359, ++STORE, 139667886727168, 139667886735359, ++STORE, 139667882053632, 139667884167167, ++SNULL, 139667882053632, 139667882065919, ++STORE, 139667882065920, 139667884167167, ++STORE, 139667882053632, 139667882065919, ++SNULL, 139667884158975, 139667884167167, ++STORE, 139667882065920, 139667884158975, ++STORE, 139667884158976, 139667884167167, ++ERASE, 139667884158976, 139667884167167, ++STORE, 139667884158976, 139667884167167, ++STORE, 139667879837696, 139667882053631, ++SNULL, 139667879837696, 139667879935999, ++STORE, 139667879936000, 139667882053631, ++STORE, 139667879837696, 139667879935999, ++SNULL, 139667882029055, 139667882053631, ++STORE, 139667879936000, 139667882029055, ++STORE, 139667882029056, 139667882053631, ++SNULL, 139667882029056, 139667882037247, ++STORE, 139667882037248, 139667882053631, ++STORE, 139667882029056, 139667882037247, ++ERASE, 139667882029056, 139667882037247, ++STORE, 139667882029056, 139667882037247, ++ERASE, 139667882037248, 139667882053631, ++STORE, 139667882037248, 139667882053631, ++STORE, 139667894988800, 139667895005183, ++SNULL, 139667890507775, 139667890515967, ++STORE, 139667890491392, 139667890507775, ++STORE, 139667890507776, 139667890515967, ++SNULL, 139667882033151, 139667882037247, ++STORE, 139667882029056, 139667882033151, ++STORE, 139667882033152, 139667882037247, ++SNULL, 139667884163071, 139667884167167, ++STORE, 139667884158976, 139667884163071, ++STORE, 139667884163072, 139667884167167, ++SNULL, 139667886731263, 139667886735359, ++STORE, 139667886727168, 139667886731263, ++STORE, 139667886731264, 139667886735359, ++SNULL, 139667892781055, 139667892785151, ++STORE, 139667892776960, 139667892781055, ++STORE, 139667892781056, 139667892785151, ++SNULL, 94796718321663, 94796718325759, ++STORE, 94796718317568, 94796718321663, ++STORE, 94796718321664, 94796718325759, ++SNULL, 139667895037951, 139667895042047, ++STORE, 139667895033856, 139667895037951, ++STORE, 139667895037952, 139667895042047, ++ERASE, 139667895005184, 139667895033855, ++STORE, 94796726063104, 94796726198271, ++STORE, 139667893305344, 139667894988799, ++STORE, 139667895005184, 139667895033855, ++STORE, 94796726063104, 94796726333439, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722489507840, 140737488351231, ++SNULL, 140722489516031, 140737488351231, ++STORE, 140722489507840, 140722489516031, ++STORE, 140722489376768, 140722489516031, ++STORE, 93980993265664, 93980995489791, ++SNULL, 93980993376255, 93980995489791, ++STORE, 93980993265664, 93980993376255, ++STORE, 93980993376256, 93980995489791, ++ERASE, 93980993376256, 93980995489791, ++STORE, 93980995469312, 93980995481599, ++STORE, 93980995481600, 93980995489791, ++STORE, 140261313593344, 140261315846143, ++SNULL, 140261313736703, 140261315846143, ++STORE, 140261313593344, 140261313736703, ++STORE, 140261313736704, 140261315846143, ++ERASE, 140261313736704, 140261315846143, ++STORE, 140261315833856, 140261315842047, ++STORE, 140261315842048, 140261315846143, ++STORE, 140722489675776, 140722489679871, ++STORE, 140722489663488, 140722489675775, ++STORE, 140261315805184, 140261315833855, ++STORE, 140261315796992, 140261315805183, ++STORE, 140261309796352, 140261313593343, ++SNULL, 140261309796352, 140261311455231, ++STORE, 140261311455232, 140261313593343, ++STORE, 140261309796352, 140261311455231, ++SNULL, 140261313552383, 140261313593343, ++STORE, 140261311455232, 140261313552383, ++STORE, 140261313552384, 140261313593343, ++SNULL, 140261313552384, 140261313576959, ++STORE, 140261313576960, 140261313593343, ++STORE, 140261313552384, 140261313576959, ++ERASE, 140261313552384, 140261313576959, ++STORE, 140261313552384, 140261313576959, ++ERASE, 140261313576960, 140261313593343, ++STORE, 140261313576960, 140261313593343, ++SNULL, 140261313568767, 140261313576959, ++STORE, 140261313552384, 140261313568767, ++STORE, 140261313568768, 140261313576959, ++SNULL, 93980995477503, 93980995481599, ++STORE, 93980995469312, 93980995477503, ++STORE, 93980995477504, 93980995481599, ++SNULL, 140261315837951, 140261315842047, ++STORE, 140261315833856, 140261315837951, ++STORE, 140261315837952, 140261315842047, ++ERASE, 140261315805184, 140261315833855, ++STORE, 93980997443584, 93980997578751, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140734059450368, 140737488351231, ++SNULL, 140734059462655, 140737488351231, ++STORE, 140734059450368, 140734059462655, ++STORE, 140734059319296, 140734059462655, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140307554983936, 140307557236735, ++SNULL, 140307555127295, 140307557236735, ++STORE, 140307554983936, 140307555127295, ++STORE, 140307555127296, 140307557236735, ++ERASE, 140307555127296, 140307557236735, ++STORE, 140307557224448, 140307557232639, ++STORE, 140307557232640, 140307557236735, ++STORE, 140734059483136, 140734059487231, ++STORE, 140734059470848, 140734059483135, ++STORE, 140307557195776, 140307557224447, ++STORE, 140307557187584, 140307557195775, ++STORE, 140307551821824, 140307554983935, ++SNULL, 140307551821824, 140307552882687, ++STORE, 140307552882688, 140307554983935, ++STORE, 140307551821824, 140307552882687, ++SNULL, 140307554975743, 140307554983935, ++STORE, 140307552882688, 140307554975743, ++STORE, 140307554975744, 140307554983935, ++ERASE, 140307554975744, 140307554983935, ++STORE, 140307554975744, 140307554983935, ++STORE, 140307548024832, 140307551821823, ++SNULL, 140307548024832, 140307549683711, ++STORE, 140307549683712, 140307551821823, ++STORE, 140307548024832, 140307549683711, ++SNULL, 140307551780863, 140307551821823, ++STORE, 140307549683712, 140307551780863, ++STORE, 140307551780864, 140307551821823, ++SNULL, 140307551780864, 140307551805439, ++STORE, 140307551805440, 140307551821823, ++STORE, 140307551780864, 140307551805439, ++ERASE, 140307551780864, 140307551805439, ++STORE, 140307551780864, 140307551805439, ++ERASE, 140307551805440, 140307551821823, ++STORE, 140307551805440, 140307551821823, ++STORE, 140307557175296, 140307557195775, ++SNULL, 140307551797247, 140307551805439, ++STORE, 140307551780864, 140307551797247, ++STORE, 140307551797248, 140307551805439, ++SNULL, 140307554979839, 140307554983935, ++STORE, 140307554975744, 140307554979839, ++STORE, 140307554979840, 140307554983935, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140307557228543, 140307557232639, ++STORE, 140307557224448, 140307557228543, ++STORE, 140307557228544, 140307557232639, ++ERASE, 140307557195776, 140307557224447, ++STORE, 39698432, 39833599, ++STORE, 39698432, 39981055, ++STORE, 94306485321728, 94306485432319, ++STORE, 94306487525376, 94306487533567, ++STORE, 94306487533568, 94306487537663, ++STORE, 94306487537664, 94306487545855, ++STORE, 94306488868864, 94306489004031, ++STORE, 140497673998336, 140497675657215, ++STORE, 140497675657216, 140497677754367, ++STORE, 140497677754368, 140497677770751, ++STORE, 140497677770752, 140497677778943, ++STORE, 140497677778944, 140497677795327, ++STORE, 140497677795328, 140497677938687, ++STORE, 140497679998976, 140497680007167, ++STORE, 140497680035840, 140497680039935, ++STORE, 140497680039936, 140497680044031, ++STORE, 140497680044032, 140497680048127, ++STORE, 140732780462080, 140732780601343, ++STORE, 140732782239744, 140732782252031, ++STORE, 140732782252032, 140732782256127, ++STORE, 94236915900416, 94236916011007, ++STORE, 94236918104064, 94236918112255, ++STORE, 94236918112256, 94236918116351, ++STORE, 94236918116352, 94236918124543, ++STORE, 94236939489280, 94236939624447, ++STORE, 140046091743232, 140046093402111, ++STORE, 140046093402112, 140046095499263, ++STORE, 140046095499264, 140046095515647, ++STORE, 140046095515648, 140046095523839, ++STORE, 140046095523840, 140046095540223, ++STORE, 140046095540224, 140046095683583, ++STORE, 140046097743872, 140046097752063, ++STORE, 140046097780736, 140046097784831, ++STORE, 140046097784832, 140046097788927, ++STORE, 140046097788928, 140046097793023, ++STORE, 140726694449152, 140726694588415, ++STORE, 140726695313408, 140726695325695, ++STORE, 140726695325696, 140726695329791, ++STORE, 94894582779904, 94894582992895, ++STORE, 94894585090048, 94894585094143, ++STORE, 94894585094144, 94894585102335, ++STORE, 94894585102336, 94894585114623, ++STORE, 94894592868352, 94894594293759, ++STORE, 139733563842560, 139733565501439, ++STORE, 139733565501440, 139733567598591, ++STORE, 139733567598592, 139733567614975, ++STORE, 139733567614976, 139733567623167, ++STORE, 139733567623168, 139733567639551, ++STORE, 139733567639552, 139733567651839, ++STORE, 139733567651840, 139733569744895, ++STORE, 139733569744896, 139733569748991, ++STORE, 139733569748992, 139733569753087, ++STORE, 139733569753088, 139733569896447, ++STORE, 139733570265088, 139733571948543, ++STORE, 139733571948544, 139733571964927, ++STORE, 139733571993600, 139733571997695, ++STORE, 139733571997696, 139733572001791, ++STORE, 139733572001792, 139733572005887, ++STORE, 140726369255424, 140726369394687, ++STORE, 140726370402304, 140726370414591, ++STORE, 140726370414592, 140726370418687, ++STORE, 94899236483072, 94899236696063, ++STORE, 94899238793216, 94899238797311, ++STORE, 94899238797312, 94899238805503, ++STORE, 94899238805504, 94899238817791, ++STORE, 94899263045632, 94899263979519, ++STORE, 140040959893504, 140040961552383, ++STORE, 140040961552384, 140040963649535, ++STORE, 140040963649536, 140040963665919, ++STORE, 140040963665920, 140040963674111, ++STORE, 140040963674112, 140040963690495, ++STORE, 140040963690496, 140040963702783, ++STORE, 140040963702784, 140040965795839, ++STORE, 140040965795840, 140040965799935, ++STORE, 140040965799936, 140040965804031, ++STORE, 140040965804032, 140040965947391, ++STORE, 140040966316032, 140040967999487, ++STORE, 140040967999488, 140040968015871, ++STORE, 140040968044544, 140040968048639, ++STORE, 140040968048640, 140040968052735, ++STORE, 140040968052736, 140040968056831, ++STORE, 140729921359872, 140729921499135, ++STORE, 140729921613824, 140729921626111, ++STORE, 140729921626112, 140729921630207, ++STORE, 94818265190400, 94818265403391, ++STORE, 94818267500544, 94818267504639, ++STORE, 94818267504640, 94818267512831, ++STORE, 94818267512832, 94818267525119, ++STORE, 94818283372544, 94818285858815, ++STORE, 139818425675776, 139818427334655, ++STORE, 139818427334656, 139818429431807, ++STORE, 139818429431808, 139818429448191, ++STORE, 139818429448192, 139818429456383, ++STORE, 139818429456384, 139818429472767, ++STORE, 139818429472768, 139818429485055, ++STORE, 139818429485056, 139818431578111, ++STORE, 139818431578112, 139818431582207, ++STORE, 139818431582208, 139818431586303, ++STORE, 139818431586304, 139818431729663, ++STORE, 139818432098304, 139818433781759, ++STORE, 139818433781760, 139818433798143, ++STORE, 139818433826816, 139818433830911, ++STORE, 139818433830912, 139818433835007, ++STORE, 139818433835008, 139818433839103, ++STORE, 140726170509312, 140726170648575, ++STORE, 140726171824128, 140726171836415, ++STORE, 140726171836416, 140726171840511, ++STORE, 94611513188352, 94611513401343, ++STORE, 94611515498496, 94611515502591, ++STORE, 94611515502592, 94611515510783, ++STORE, 94611515510784, 94611515523071, ++STORE, 94611516502016, 94611516907519, ++STORE, 140596246388736, 140596248047615, ++STORE, 140596248047616, 140596250144767, ++STORE, 140596250144768, 140596250161151, ++STORE, 140596250161152, 140596250169343, ++STORE, 140596250169344, 140596250185727, ++STORE, 140596250185728, 140596250198015, ++STORE, 140596250198016, 140596252291071, ++STORE, 140596252291072, 140596252295167, ++STORE, 140596252295168, 140596252299263, ++STORE, 140596252299264, 140596252442623, ++STORE, 140596252811264, 140596254494719, ++STORE, 140596254494720, 140596254511103, ++STORE, 140596254539776, 140596254543871, ++STORE, 140596254543872, 140596254547967, ++STORE, 140596254547968, 140596254552063, ++STORE, 140731551338496, 140731551477759, ++STORE, 140731551780864, 140731551793151, ++STORE, 140731551793152, 140731551797247, ++STORE, 94313835851776, 94313836064767, ++STORE, 94313838161920, 94313838166015, ++STORE, 94313838166016, 94313838174207, ++STORE, 94313838174208, 94313838186495, ++STORE, 94313858416640, 94313861906431, ++STORE, 140693503918080, 140693505576959, ++STORE, 140693505576960, 140693507674111, ++STORE, 140693507674112, 140693507690495, ++STORE, 140693507690496, 140693507698687, ++STORE, 140693507698688, 140693507715071, ++STORE, 140693507715072, 140693507727359, ++STORE, 140693507727360, 140693509820415, ++STORE, 140693509820416, 140693509824511, ++STORE, 140693509824512, 140693509828607, ++STORE, 140693509828608, 140693509971967, ++STORE, 140693510340608, 140693512024063, ++STORE, 140693512024064, 140693512040447, ++STORE, 140693512069120, 140693512073215, ++STORE, 140693512073216, 140693512077311, ++STORE, 140693512077312, 140693512081407, ++STORE, 140721116065792, 140721116205055, ++STORE, 140721117831168, 140721117843455, ++STORE, 140721117843456, 140721117847551, ++STORE, 94843650150400, 94843650363391, ++STORE, 94843652460544, 94843652464639, ++STORE, 94843652464640, 94843652472831, ++STORE, 94843652472832, 94843652485119, ++STORE, 94843685388288, 94843686281215, ++STORE, 140484193681408, 140484195340287, ++STORE, 140484195340288, 140484197437439, ++STORE, 140484197437440, 140484197453823, ++STORE, 140484197453824, 140484197462015, ++STORE, 140484197462016, 140484197478399, ++STORE, 140484197478400, 140484197490687, ++STORE, 140484197490688, 140484199583743, ++STORE, 140484199583744, 140484199587839, ++STORE, 140484199587840, 140484199591935, ++STORE, 140484199591936, 140484199735295, ++STORE, 140484200103936, 140484201787391, ++STORE, 140484201787392, 140484201803775, ++STORE, 140484201832448, 140484201836543, ++STORE, 140484201836544, 140484201840639, ++STORE, 140484201840640, 140484201844735, ++STORE, 140726294315008, 140726294454271, ++STORE, 140726295646208, 140726295658495, ++STORE, 140726295658496, 140726295662591, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720422371328, 140737488351231, ++SNULL, 140720422379519, 140737488351231, ++STORE, 140720422371328, 140720422379519, ++STORE, 140720422240256, 140720422379519, ++STORE, 94417967845376, 94417970180095, ++SNULL, 94417968058367, 94417970180095, ++STORE, 94417967845376, 94417968058367, ++STORE, 94417968058368, 94417970180095, ++ERASE, 94417968058368, 94417970180095, ++STORE, 94417970155520, 94417970167807, ++STORE, 94417970167808, 94417970180095, ++STORE, 140252450045952, 140252452298751, ++SNULL, 140252450189311, 140252452298751, ++STORE, 140252450045952, 140252450189311, ++STORE, 140252450189312, 140252452298751, ++ERASE, 140252450189312, 140252452298751, ++STORE, 140252452286464, 140252452294655, ++STORE, 140252452294656, 140252452298751, ++STORE, 140720422416384, 140720422420479, ++STORE, 140720422404096, 140720422416383, ++STORE, 140252452257792, 140252452286463, ++STORE, 140252452249600, 140252452257791, ++STORE, 140252447932416, 140252450045951, ++SNULL, 140252447932416, 140252447944703, ++STORE, 140252447944704, 140252450045951, ++STORE, 140252447932416, 140252447944703, ++SNULL, 140252450037759, 140252450045951, ++STORE, 140252447944704, 140252450037759, ++STORE, 140252450037760, 140252450045951, ++ERASE, 140252450037760, 140252450045951, ++STORE, 140252450037760, 140252450045951, ++STORE, 140252444135424, 140252447932415, ++SNULL, 140252444135424, 140252445794303, ++STORE, 140252445794304, 140252447932415, ++STORE, 140252444135424, 140252445794303, ++SNULL, 140252447891455, 140252447932415, ++STORE, 140252445794304, 140252447891455, ++STORE, 140252447891456, 140252447932415, ++SNULL, 140252447891456, 140252447916031, ++STORE, 140252447916032, 140252447932415, ++STORE, 140252447891456, 140252447916031, ++ERASE, 140252447891456, 140252447916031, ++STORE, 140252447891456, 140252447916031, ++ERASE, 140252447916032, 140252447932415, ++STORE, 140252447916032, 140252447932415, ++STORE, 140252452241408, 140252452257791, ++SNULL, 140252447907839, 140252447916031, ++STORE, 140252447891456, 140252447907839, ++STORE, 140252447907840, 140252447916031, ++SNULL, 140252450041855, 140252450045951, ++STORE, 140252450037760, 140252450041855, ++STORE, 140252450041856, 140252450045951, ++SNULL, 94417970159615, 94417970167807, ++STORE, 94417970155520, 94417970159615, ++STORE, 94417970159616, 94417970167807, ++SNULL, 140252452290559, 140252452294655, ++STORE, 140252452286464, 140252452290559, ++STORE, 140252452290560, 140252452294655, ++ERASE, 140252452257792, 140252452286463, ++STORE, 94417996333056, 94417996468223, ++STORE, 140252450557952, 140252452241407, ++STORE, 94417996333056, 94417996603391, ++STORE, 94417996333056, 94417996738559, ++STORE, 94417996333056, 94417996910591, ++SNULL, 94417996881919, 94417996910591, ++STORE, 94417996333056, 94417996881919, ++STORE, 94417996881920, 94417996910591, ++ERASE, 94417996881920, 94417996910591, ++STORE, 94417996333056, 94417997017087, ++STORE, 94417996333056, 94417997152255, ++SNULL, 94417997135871, 94417997152255, ++STORE, 94417996333056, 94417997135871, ++STORE, 94417997135872, 94417997152255, ++ERASE, 94417997135872, 94417997152255, ++STORE, 94417996333056, 94417997291519, ++SNULL, 94417997271039, 94417997291519, ++STORE, 94417996333056, 94417997271039, ++STORE, 94417997271040, 94417997291519, ++ERASE, 94417997271040, 94417997291519, ++STORE, 94417996333056, 94417997406207, ++SNULL, 94417997381631, 94417997406207, ++STORE, 94417996333056, 94417997381631, ++STORE, 94417997381632, 94417997406207, ++ERASE, 94417997381632, 94417997406207, ++STORE, 94417996333056, 94417997516799, ++SNULL, 94417997488127, 94417997516799, ++STORE, 94417996333056, 94417997488127, ++STORE, 94417997488128, 94417997516799, ++ERASE, 94417997488128, 94417997516799, ++STORE, 94417996333056, 94417997643775, ++SNULL, 94417997631487, 94417997643775, ++STORE, 94417996333056, 94417997631487, ++STORE, 94417997631488, 94417997643775, ++ERASE, 94417997631488, 94417997643775, ++SNULL, 94417997590527, 94417997631487, ++STORE, 94417996333056, 94417997590527, ++STORE, 94417997590528, 94417997631487, ++ERASE, 94417997590528, 94417997631487, ++STORE, 94417996333056, 94417997733887, ++STORE, 94417996333056, 94417997869055, ++STORE, 94417996333056, 94417998004223, ++SNULL, 94417998000127, 94417998004223, ++STORE, 94417996333056, 94417998000127, ++STORE, 94417998000128, 94417998004223, ++ERASE, 94417998000128, 94417998004223, ++STORE, 94049170993152, 94049171206143, ++STORE, 94049173303296, 94049173307391, ++STORE, 94049173307392, 94049173315583, ++STORE, 94049173315584, 94049173327871, ++STORE, 94049176236032, 94049183645695, ++STORE, 139807795544064, 139807797202943, ++STORE, 139807797202944, 139807799300095, ++STORE, 139807799300096, 139807799316479, ++STORE, 139807799316480, 139807799324671, ++STORE, 139807799324672, 139807799341055, ++STORE, 139807799341056, 139807799353343, ++STORE, 139807799353344, 139807801446399, ++STORE, 139807801446400, 139807801450495, ++STORE, 139807801450496, 139807801454591, ++STORE, 139807801454592, 139807801597951, ++STORE, 139807801966592, 139807803650047, ++STORE, 139807803650048, 139807803666431, ++STORE, 139807803695104, 139807803699199, ++STORE, 139807803699200, 139807803703295, ++STORE, 139807803703296, 139807803707391, ++STORE, 140727555538944, 140727555678207, ++STORE, 140727555940352, 140727555952639, ++STORE, 140727555952640, 140727555956735, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722483441664, 140737488351231, ++SNULL, 140722483449855, 140737488351231, ++STORE, 140722483441664, 140722483449855, ++STORE, 140722483310592, 140722483449855, ++STORE, 94416704921600, 94416707145727, ++SNULL, 94416705032191, 94416707145727, ++STORE, 94416704921600, 94416705032191, ++STORE, 94416705032192, 94416707145727, ++ERASE, 94416705032192, 94416707145727, ++STORE, 94416707125248, 94416707137535, ++STORE, 94416707137536, 94416707145727, ++STORE, 140555439296512, 140555441549311, ++SNULL, 140555439439871, 140555441549311, ++STORE, 140555439296512, 140555439439871, ++STORE, 140555439439872, 140555441549311, ++ERASE, 140555439439872, 140555441549311, ++STORE, 140555441537024, 140555441545215, ++STORE, 140555441545216, 140555441549311, ++STORE, 140722484781056, 140722484785151, ++STORE, 140722484768768, 140722484781055, ++STORE, 140555441508352, 140555441537023, ++STORE, 140555441500160, 140555441508351, ++STORE, 140555435499520, 140555439296511, ++SNULL, 140555435499520, 140555437158399, ++STORE, 140555437158400, 140555439296511, ++STORE, 140555435499520, 140555437158399, ++SNULL, 140555439255551, 140555439296511, ++STORE, 140555437158400, 140555439255551, ++STORE, 140555439255552, 140555439296511, ++SNULL, 140555439255552, 140555439280127, ++STORE, 140555439280128, 140555439296511, ++STORE, 140555439255552, 140555439280127, ++ERASE, 140555439255552, 140555439280127, ++STORE, 140555439255552, 140555439280127, ++ERASE, 140555439280128, 140555439296511, ++STORE, 140555439280128, 140555439296511, ++SNULL, 140555439271935, 140555439280127, ++STORE, 140555439255552, 140555439271935, ++STORE, 140555439271936, 140555439280127, ++SNULL, 94416707133439, 94416707137535, ++STORE, 94416707125248, 94416707133439, ++STORE, 94416707133440, 94416707137535, ++SNULL, 140555441541119, 140555441545215, ++STORE, 140555441537024, 140555441541119, ++STORE, 140555441541120, 140555441545215, ++ERASE, 140555441508352, 140555441537023, ++STORE, 94416724672512, 94416724807679, ++STORE, 94686636953600, 94686637166591, ++STORE, 94686639263744, 94686639267839, ++STORE, 94686639267840, 94686639276031, ++STORE, 94686639276032, 94686639288319, ++STORE, 94686662193152, 94686663163903, ++STORE, 140312944431104, 140312946089983, ++STORE, 140312946089984, 140312948187135, ++STORE, 140312948187136, 140312948203519, ++STORE, 140312948203520, 140312948211711, ++STORE, 140312948211712, 140312948228095, ++STORE, 140312948228096, 140312948240383, ++STORE, 140312948240384, 140312950333439, ++STORE, 140312950333440, 140312950337535, ++STORE, 140312950337536, 140312950341631, ++STORE, 140312950341632, 140312950484991, ++STORE, 140312950853632, 140312952537087, ++STORE, 140312952537088, 140312952553471, ++STORE, 140312952582144, 140312952586239, ++STORE, 140312952586240, 140312952590335, ++STORE, 140312952590336, 140312952594431, ++STORE, 140730598920192, 140730599059455, ++STORE, 140730599108608, 140730599120895, ++STORE, 140730599120896, 140730599124991, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726234079232, 140737488351231, ++SNULL, 140726234087423, 140737488351231, ++STORE, 140726234079232, 140726234087423, ++STORE, 140726233948160, 140726234087423, ++STORE, 94589467578368, 94589469802495, ++SNULL, 94589467688959, 94589469802495, ++STORE, 94589467578368, 94589467688959, ++STORE, 94589467688960, 94589469802495, ++ERASE, 94589467688960, 94589469802495, ++STORE, 94589469782016, 94589469794303, ++STORE, 94589469794304, 94589469802495, ++STORE, 140587082842112, 140587085094911, ++SNULL, 140587082985471, 140587085094911, ++STORE, 140587082842112, 140587082985471, ++STORE, 140587082985472, 140587085094911, ++ERASE, 140587082985472, 140587085094911, ++STORE, 140587085082624, 140587085090815, ++STORE, 140587085090816, 140587085094911, ++STORE, 140726234103808, 140726234107903, ++STORE, 140726234091520, 140726234103807, ++STORE, 140587085053952, 140587085082623, ++STORE, 140587085045760, 140587085053951, ++STORE, 140587079045120, 140587082842111, ++SNULL, 140587079045120, 140587080703999, ++STORE, 140587080704000, 140587082842111, ++STORE, 140587079045120, 140587080703999, ++SNULL, 140587082801151, 140587082842111, ++STORE, 140587080704000, 140587082801151, ++STORE, 140587082801152, 140587082842111, ++SNULL, 140587082801152, 140587082825727, ++STORE, 140587082825728, 140587082842111, ++STORE, 140587082801152, 140587082825727, ++ERASE, 140587082801152, 140587082825727, ++STORE, 140587082801152, 140587082825727, ++ERASE, 140587082825728, 140587082842111, ++STORE, 140587082825728, 140587082842111, ++SNULL, 140587082817535, 140587082825727, ++STORE, 140587082801152, 140587082817535, ++STORE, 140587082817536, 140587082825727, ++SNULL, 94589469790207, 94589469794303, ++STORE, 94589469782016, 94589469790207, ++STORE, 94589469790208, 94589469794303, ++SNULL, 140587085086719, 140587085090815, ++STORE, 140587085082624, 140587085086719, ++STORE, 140587085086720, 140587085090815, ++ERASE, 140587085053952, 140587085082623, ++STORE, 94589477507072, 94589477642239, ++STORE, 94225448325120, 94225448538111, ++STORE, 94225450635264, 94225450639359, ++STORE, 94225450639360, 94225450647551, ++STORE, 94225450647552, 94225450659839, ++STORE, 94225470246912, 94225473548287, ++STORE, 140199245496320, 140199247155199, ++STORE, 140199247155200, 140199249252351, ++STORE, 140199249252352, 140199249268735, ++STORE, 140199249268736, 140199249276927, ++STORE, 140199249276928, 140199249293311, ++STORE, 140199249293312, 140199249305599, ++STORE, 140199249305600, 140199251398655, ++STORE, 140199251398656, 140199251402751, ++STORE, 140199251402752, 140199251406847, ++STORE, 140199251406848, 140199251550207, ++STORE, 140199251918848, 140199253602303, ++STORE, 140199253602304, 140199253618687, ++STORE, 140199253647360, 140199253651455, ++STORE, 140199253651456, 140199253655551, ++STORE, 140199253655552, 140199253659647, ++STORE, 140726264414208, 140726264553471, ++STORE, 140726265843712, 140726265855999, ++STORE, 140726265856000, 140726265860095, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733508358144, 140737488351231, ++SNULL, 140733508366335, 140737488351231, ++STORE, 140733508358144, 140733508366335, ++STORE, 140733508227072, 140733508366335, ++STORE, 94766263947264, 94766266171391, ++SNULL, 94766264057855, 94766266171391, ++STORE, 94766263947264, 94766264057855, ++STORE, 94766264057856, 94766266171391, ++ERASE, 94766264057856, 94766266171391, ++STORE, 94766266150912, 94766266163199, ++STORE, 94766266163200, 94766266171391, ++STORE, 140693985132544, 140693987385343, ++SNULL, 140693985275903, 140693987385343, ++STORE, 140693985132544, 140693985275903, ++STORE, 140693985275904, 140693987385343, ++ERASE, 140693985275904, 140693987385343, ++STORE, 140693987373056, 140693987381247, ++STORE, 140693987381248, 140693987385343, ++STORE, 140733509939200, 140733509943295, ++STORE, 140733509926912, 140733509939199, ++STORE, 140693987344384, 140693987373055, ++STORE, 140693987336192, 140693987344383, ++STORE, 140693981335552, 140693985132543, ++SNULL, 140693981335552, 140693982994431, ++STORE, 140693982994432, 140693985132543, ++STORE, 140693981335552, 140693982994431, ++SNULL, 140693985091583, 140693985132543, ++STORE, 140693982994432, 140693985091583, ++STORE, 140693985091584, 140693985132543, ++SNULL, 140693985091584, 140693985116159, ++STORE, 140693985116160, 140693985132543, ++STORE, 140693985091584, 140693985116159, ++ERASE, 140693985091584, 140693985116159, ++STORE, 140693985091584, 140693985116159, ++ERASE, 140693985116160, 140693985132543, ++STORE, 140693985116160, 140693985132543, ++SNULL, 140693985107967, 140693985116159, ++STORE, 140693985091584, 140693985107967, ++STORE, 140693985107968, 140693985116159, ++SNULL, 94766266159103, 94766266163199, ++STORE, 94766266150912, 94766266159103, ++STORE, 94766266159104, 94766266163199, ++SNULL, 140693987377151, 140693987381247, ++STORE, 140693987373056, 140693987377151, ++STORE, 140693987377152, 140693987381247, ++ERASE, 140693987344384, 140693987373055, ++STORE, 94766282035200, 94766282170367, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724769353728, 140737488351231, ++SNULL, 140724769361919, 140737488351231, ++STORE, 140724769353728, 140724769361919, ++STORE, 140724769222656, 140724769361919, ++STORE, 94710460526592, 94710462750719, ++SNULL, 94710460637183, 94710462750719, ++STORE, 94710460526592, 94710460637183, ++STORE, 94710460637184, 94710462750719, ++ERASE, 94710460637184, 94710462750719, ++STORE, 94710462730240, 94710462742527, ++STORE, 94710462742528, 94710462750719, ++STORE, 140469764395008, 140469766647807, ++SNULL, 140469764538367, 140469766647807, ++STORE, 140469764395008, 140469764538367, ++STORE, 140469764538368, 140469766647807, ++ERASE, 140469764538368, 140469766647807, ++STORE, 140469766635520, 140469766643711, ++STORE, 140469766643712, 140469766647807, ++STORE, 140724770877440, 140724770881535, ++STORE, 140724770865152, 140724770877439, ++STORE, 140469766606848, 140469766635519, ++STORE, 140469766598656, 140469766606847, ++STORE, 140469760598016, 140469764395007, ++SNULL, 140469760598016, 140469762256895, ++STORE, 140469762256896, 140469764395007, ++STORE, 140469760598016, 140469762256895, ++SNULL, 140469764354047, 140469764395007, ++STORE, 140469762256896, 140469764354047, ++STORE, 140469764354048, 140469764395007, ++SNULL, 140469764354048, 140469764378623, ++STORE, 140469764378624, 140469764395007, ++STORE, 140469764354048, 140469764378623, ++ERASE, 140469764354048, 140469764378623, ++STORE, 140469764354048, 140469764378623, ++ERASE, 140469764378624, 140469764395007, ++STORE, 140469764378624, 140469764395007, ++SNULL, 140469764370431, 140469764378623, ++STORE, 140469764354048, 140469764370431, ++STORE, 140469764370432, 140469764378623, ++SNULL, 94710462738431, 94710462742527, ++STORE, 94710462730240, 94710462738431, ++STORE, 94710462738432, 94710462742527, ++SNULL, 140469766639615, 140469766643711, ++STORE, 140469766635520, 140469766639615, ++STORE, 140469766639616, 140469766643711, ++ERASE, 140469766606848, 140469766635519, ++STORE, 94710485581824, 94710485716991, ++STORE, 94105755795456, 94105756008447, ++STORE, 94105758105600, 94105758109695, ++STORE, 94105758109696, 94105758117887, ++STORE, 94105758117888, 94105758130175, ++STORE, 94105788981248, 94105794871295, ++STORE, 140641190031360, 140641191690239, ++STORE, 140641191690240, 140641193787391, ++STORE, 140641193787392, 140641193803775, ++STORE, 140641193803776, 140641193811967, ++STORE, 140641193811968, 140641193828351, ++STORE, 140641193828352, 140641193840639, ++STORE, 140641193840640, 140641195933695, ++STORE, 140641195933696, 140641195937791, ++STORE, 140641195937792, 140641195941887, ++STORE, 140641195941888, 140641196085247, ++STORE, 140641196453888, 140641198137343, ++STORE, 140641198137344, 140641198153727, ++STORE, 140641198182400, 140641198186495, ++STORE, 140641198186496, 140641198190591, ++STORE, 140641198190592, 140641198194687, ++STORE, 140731980034048, 140731980173311, ++STORE, 140731981078528, 140731981090815, ++STORE, 140731981090816, 140731981094911, ++STORE, 93828086431744, 93828086644735, ++STORE, 93828088741888, 93828088745983, ++STORE, 93828088745984, 93828088754175, ++STORE, 93828088754176, 93828088766463, ++STORE, 93828094193664, 93828096831487, ++STORE, 139844717334528, 139844718993407, ++STORE, 139844718993408, 139844721090559, ++STORE, 139844721090560, 139844721106943, ++STORE, 139844721106944, 139844721115135, ++STORE, 139844721115136, 139844721131519, ++STORE, 139844721131520, 139844721143807, ++STORE, 139844721143808, 139844723236863, ++STORE, 139844723236864, 139844723240959, ++STORE, 139844723240960, 139844723245055, ++STORE, 139844723245056, 139844723388415, ++STORE, 139844723757056, 139844725440511, ++STORE, 139844725440512, 139844725456895, ++STORE, 139844725485568, 139844725489663, ++STORE, 139844725489664, 139844725493759, ++STORE, 139844725493760, 139844725497855, ++STORE, 140729996185600, 140729996324863, ++STORE, 140729996828672, 140729996840959, ++STORE, 140729996840960, 140729996845055, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722494771200, 140737488351231, ++SNULL, 140722494775295, 140737488351231, ++STORE, 140722494771200, 140722494775295, ++STORE, 140722494640128, 140722494775295, ++STORE, 94324011311104, 94324013535231, ++SNULL, 94324011421695, 94324013535231, ++STORE, 94324011311104, 94324011421695, ++STORE, 94324011421696, 94324013535231, ++ERASE, 94324011421696, 94324013535231, ++STORE, 94324013514752, 94324013527039, ++STORE, 94324013527040, 94324013535231, ++STORE, 140151462309888, 140151464562687, ++SNULL, 140151462453247, 140151464562687, ++STORE, 140151462309888, 140151462453247, ++STORE, 140151462453248, 140151464562687, ++ERASE, 140151462453248, 140151464562687, ++STORE, 140151464550400, 140151464558591, ++STORE, 140151464558592, 140151464562687, ++STORE, 140722495467520, 140722495471615, ++STORE, 140722495455232, 140722495467519, ++STORE, 140151464521728, 140151464550399, ++STORE, 140151464513536, 140151464521727, ++STORE, 140151458512896, 140151462309887, ++SNULL, 140151458512896, 140151460171775, ++STORE, 140151460171776, 140151462309887, ++STORE, 140151458512896, 140151460171775, ++SNULL, 140151462268927, 140151462309887, ++STORE, 140151460171776, 140151462268927, ++STORE, 140151462268928, 140151462309887, ++SNULL, 140151462268928, 140151462293503, ++STORE, 140151462293504, 140151462309887, ++STORE, 140151462268928, 140151462293503, ++ERASE, 140151462268928, 140151462293503, ++STORE, 140151462268928, 140151462293503, ++ERASE, 140151462293504, 140151462309887, ++STORE, 140151462293504, 140151462309887, ++SNULL, 140151462285311, 140151462293503, ++STORE, 140151462268928, 140151462285311, ++STORE, 140151462285312, 140151462293503, ++SNULL, 94324013522943, 94324013527039, ++STORE, 94324013514752, 94324013522943, ++STORE, 94324013522944, 94324013527039, ++SNULL, 140151464554495, 140151464558591, ++STORE, 140151464550400, 140151464554495, ++STORE, 140151464554496, 140151464558591, ++ERASE, 140151464521728, 140151464550399, ++STORE, 94324024778752, 94324024913919, ++STORE, 94899262967808, 94899263180799, ++STORE, 94899265277952, 94899265282047, ++STORE, 94899265282048, 94899265290239, ++STORE, 94899265290240, 94899265302527, ++STORE, 94899295469568, 94899298689023, ++STORE, 140434388418560, 140434390077439, ++STORE, 140434390077440, 140434392174591, ++STORE, 140434392174592, 140434392190975, ++STORE, 140434392190976, 140434392199167, ++STORE, 140434392199168, 140434392215551, ++STORE, 140434392215552, 140434392227839, ++STORE, 140434392227840, 140434394320895, ++STORE, 140434394320896, 140434394324991, ++STORE, 140434394324992, 140434394329087, ++STORE, 140434394329088, 140434394472447, ++STORE, 140434394841088, 140434396524543, ++STORE, 140434396524544, 140434396540927, ++STORE, 140434396569600, 140434396573695, ++STORE, 140434396573696, 140434396577791, ++STORE, 140434396577792, 140434396581887, ++STORE, 140720618135552, 140720618274815, ++STORE, 140720618418176, 140720618430463, ++STORE, 140720618430464, 140720618434559, ++STORE, 94425529798656, 94425530011647, ++STORE, 94425532108800, 94425532112895, ++STORE, 94425532112896, 94425532121087, ++STORE, 94425532121088, 94425532133375, ++STORE, 94425557753856, 94425566576639, ++STORE, 140600528470016, 140600530128895, ++STORE, 140600530128896, 140600532226047, ++STORE, 140600532226048, 140600532242431, ++STORE, 140600532242432, 140600532250623, ++STORE, 140600532250624, 140600532267007, ++STORE, 140600532267008, 140600532279295, ++STORE, 140600532279296, 140600534372351, ++STORE, 140600534372352, 140600534376447, ++STORE, 140600534376448, 140600534380543, ++STORE, 140600534380544, 140600534523903, ++STORE, 140600534892544, 140600536575999, ++STORE, 140600536576000, 140600536592383, ++STORE, 140600536621056, 140600536625151, ++STORE, 140600536625152, 140600536629247, ++STORE, 140600536629248, 140600536633343, ++STORE, 140721857785856, 140721857925119, ++STORE, 140721858068480, 140721858080767, ++STORE, 140721858080768, 140721858084863, ++STORE, 94425529798656, 94425530011647, ++STORE, 94425532108800, 94425532112895, ++STORE, 94425532112896, 94425532121087, ++STORE, 94425532121088, 94425532133375, ++STORE, 94425557753856, 94425568772095, ++STORE, 140600528470016, 140600530128895, ++STORE, 140600530128896, 140600532226047, ++STORE, 140600532226048, 140600532242431, ++STORE, 140600532242432, 140600532250623, ++STORE, 140600532250624, 140600532267007, ++STORE, 140600532267008, 140600532279295, ++STORE, 140600532279296, 140600534372351, ++STORE, 140600534372352, 140600534376447, ++STORE, 140600534376448, 140600534380543, ++STORE, 140600534380544, 140600534523903, ++STORE, 140600534892544, 140600536575999, ++STORE, 140600536576000, 140600536592383, ++STORE, 140600536621056, 140600536625151, ++STORE, 140600536625152, 140600536629247, ++STORE, 140600536629248, 140600536633343, ++STORE, 140721857785856, 140721857925119, ++STORE, 140721858068480, 140721858080767, ++STORE, 140721858080768, 140721858084863, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735611645952, 140737488351231, ++SNULL, 140735611654143, 140737488351231, ++STORE, 140735611645952, 140735611654143, ++STORE, 140735611514880, 140735611654143, ++STORE, 94592137641984, 94592139866111, ++SNULL, 94592137752575, 94592139866111, ++STORE, 94592137641984, 94592137752575, ++STORE, 94592137752576, 94592139866111, ++ERASE, 94592137752576, 94592139866111, ++STORE, 94592139845632, 94592139857919, ++STORE, 94592139857920, 94592139866111, ++STORE, 140350425030656, 140350427283455, ++SNULL, 140350425174015, 140350427283455, ++STORE, 140350425030656, 140350425174015, ++STORE, 140350425174016, 140350427283455, ++ERASE, 140350425174016, 140350427283455, ++STORE, 140350427271168, 140350427279359, ++STORE, 140350427279360, 140350427283455, ++STORE, 140735612043264, 140735612047359, ++STORE, 140735612030976, 140735612043263, ++STORE, 140350427242496, 140350427271167, ++STORE, 140350427234304, 140350427242495, ++STORE, 140350421233664, 140350425030655, ++SNULL, 140350421233664, 140350422892543, ++STORE, 140350422892544, 140350425030655, ++STORE, 140350421233664, 140350422892543, ++SNULL, 140350424989695, 140350425030655, ++STORE, 140350422892544, 140350424989695, ++STORE, 140350424989696, 140350425030655, ++SNULL, 140350424989696, 140350425014271, ++STORE, 140350425014272, 140350425030655, ++STORE, 140350424989696, 140350425014271, ++ERASE, 140350424989696, 140350425014271, ++STORE, 140350424989696, 140350425014271, ++ERASE, 140350425014272, 140350425030655, ++STORE, 140350425014272, 140350425030655, ++SNULL, 140350425006079, 140350425014271, ++STORE, 140350424989696, 140350425006079, ++STORE, 140350425006080, 140350425014271, ++SNULL, 94592139853823, 94592139857919, ++STORE, 94592139845632, 94592139853823, ++STORE, 94592139853824, 94592139857919, ++SNULL, 140350427275263, 140350427279359, ++STORE, 140350427271168, 140350427275263, ++STORE, 140350427275264, 140350427279359, ++ERASE, 140350427242496, 140350427271167, ++STORE, 94592164823040, 94592164958207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723500535808, 140737488351231, ++SNULL, 140723500543999, 140737488351231, ++STORE, 140723500535808, 140723500543999, ++STORE, 140723500404736, 140723500543999, ++STORE, 94458379010048, 94458381234175, ++SNULL, 94458379120639, 94458381234175, ++STORE, 94458379010048, 94458379120639, ++STORE, 94458379120640, 94458381234175, ++ERASE, 94458379120640, 94458381234175, ++STORE, 94458381213696, 94458381225983, ++STORE, 94458381225984, 94458381234175, ++STORE, 139771674230784, 139771676483583, ++SNULL, 139771674374143, 139771676483583, ++STORE, 139771674230784, 139771674374143, ++STORE, 139771674374144, 139771676483583, ++ERASE, 139771674374144, 139771676483583, ++STORE, 139771676471296, 139771676479487, ++STORE, 139771676479488, 139771676483583, ++STORE, 140723500769280, 140723500773375, ++STORE, 140723500756992, 140723500769279, ++STORE, 139771676442624, 139771676471295, ++STORE, 139771676434432, 139771676442623, ++STORE, 139771670433792, 139771674230783, ++SNULL, 139771670433792, 139771672092671, ++STORE, 139771672092672, 139771674230783, ++STORE, 139771670433792, 139771672092671, ++SNULL, 139771674189823, 139771674230783, ++STORE, 139771672092672, 139771674189823, ++STORE, 139771674189824, 139771674230783, ++SNULL, 139771674189824, 139771674214399, ++STORE, 139771674214400, 139771674230783, ++STORE, 139771674189824, 139771674214399, ++ERASE, 139771674189824, 139771674214399, ++STORE, 139771674189824, 139771674214399, ++ERASE, 139771674214400, 139771674230783, ++STORE, 139771674214400, 139771674230783, ++SNULL, 139771674206207, 139771674214399, ++STORE, 139771674189824, 139771674206207, ++STORE, 139771674206208, 139771674214399, ++SNULL, 94458381221887, 94458381225983, ++STORE, 94458381213696, 94458381221887, ++STORE, 94458381221888, 94458381225983, ++SNULL, 139771676475391, 139771676479487, ++STORE, 139771676471296, 139771676475391, ++STORE, 139771676475392, 139771676479487, ++ERASE, 139771676442624, 139771676471295, ++STORE, 94458401873920, 94458402009087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731316264960, 140737488351231, ++SNULL, 140731316273151, 140737488351231, ++STORE, 140731316264960, 140731316273151, ++STORE, 140731316133888, 140731316273151, ++STORE, 94437830881280, 94437833215999, ++SNULL, 94437831094271, 94437833215999, ++STORE, 94437830881280, 94437831094271, ++STORE, 94437831094272, 94437833215999, ++ERASE, 94437831094272, 94437833215999, ++STORE, 94437833191424, 94437833203711, ++STORE, 94437833203712, 94437833215999, ++STORE, 140265986031616, 140265988284415, ++SNULL, 140265986174975, 140265988284415, ++STORE, 140265986031616, 140265986174975, ++STORE, 140265986174976, 140265988284415, ++ERASE, 140265986174976, 140265988284415, ++STORE, 140265988272128, 140265988280319, ++STORE, 140265988280320, 140265988284415, ++STORE, 140731316318208, 140731316322303, ++STORE, 140731316305920, 140731316318207, ++STORE, 140265988243456, 140265988272127, ++STORE, 140265988235264, 140265988243455, ++STORE, 140265983918080, 140265986031615, ++SNULL, 140265983918080, 140265983930367, ++STORE, 140265983930368, 140265986031615, ++STORE, 140265983918080, 140265983930367, ++SNULL, 140265986023423, 140265986031615, ++STORE, 140265983930368, 140265986023423, ++STORE, 140265986023424, 140265986031615, ++ERASE, 140265986023424, 140265986031615, ++STORE, 140265986023424, 140265986031615, ++STORE, 140265980121088, 140265983918079, ++SNULL, 140265980121088, 140265981779967, ++STORE, 140265981779968, 140265983918079, ++STORE, 140265980121088, 140265981779967, ++SNULL, 140265983877119, 140265983918079, ++STORE, 140265981779968, 140265983877119, ++STORE, 140265983877120, 140265983918079, ++SNULL, 140265983877120, 140265983901695, ++STORE, 140265983901696, 140265983918079, ++STORE, 140265983877120, 140265983901695, ++ERASE, 140265983877120, 140265983901695, ++STORE, 140265983877120, 140265983901695, ++ERASE, 140265983901696, 140265983918079, ++STORE, 140265983901696, 140265983918079, ++STORE, 140265988227072, 140265988243455, ++SNULL, 140265983893503, 140265983901695, ++STORE, 140265983877120, 140265983893503, ++STORE, 140265983893504, 140265983901695, ++SNULL, 140265986027519, 140265986031615, ++STORE, 140265986023424, 140265986027519, ++STORE, 140265986027520, 140265986031615, ++SNULL, 94437833195519, 94437833203711, ++STORE, 94437833191424, 94437833195519, ++STORE, 94437833195520, 94437833203711, ++SNULL, 140265988276223, 140265988280319, ++STORE, 140265988272128, 140265988276223, ++STORE, 140265988276224, 140265988280319, ++ERASE, 140265988243456, 140265988272127, ++STORE, 94437847638016, 94437847773183, ++STORE, 140265986543616, 140265988227071, ++STORE, 94437847638016, 94437847908351, ++STORE, 94437847638016, 94437848043519, ++STORE, 94437847638016, 94437848190975, ++SNULL, 94437848178687, 94437848190975, ++STORE, 94437847638016, 94437848178687, ++STORE, 94437848178688, 94437848190975, ++ERASE, 94437848178688, 94437848190975, ++STORE, 94437847638016, 94437848330239, ++STORE, 94437847638016, 94437848465407, ++SNULL, 94437848444927, 94437848465407, ++STORE, 94437847638016, 94437848444927, ++STORE, 94437848444928, 94437848465407, ++ERASE, 94437848444928, 94437848465407, ++STORE, 94437847638016, 94437848584191, ++STORE, 94437847638016, 94437848719359, ++SNULL, 94437848678399, 94437848719359, ++STORE, 94437847638016, 94437848678399, ++STORE, 94437848678400, 94437848719359, ++ERASE, 94437848678400, 94437848719359, ++STORE, 94437847638016, 94437848842239, ++SNULL, 94437848825855, 94437848842239, ++STORE, 94437847638016, 94437848825855, ++STORE, 94437848825856, 94437848842239, ++ERASE, 94437848825856, 94437848842239, ++STORE, 94437847638016, 94437848961023, ++STORE, 94437847638016, 94437849096191, ++STORE, 94661814710272, 94661814923263, ++STORE, 94661817020416, 94661817024511, ++STORE, 94661817024512, 94661817032703, ++STORE, 94661817032704, 94661817044991, ++STORE, 94661840424960, 94661841240063, ++STORE, 140582259814400, 140582261473279, ++STORE, 140582261473280, 140582263570431, ++STORE, 140582263570432, 140582263586815, ++STORE, 140582263586816, 140582263595007, ++STORE, 140582263595008, 140582263611391, ++STORE, 140582263611392, 140582263623679, ++STORE, 140582263623680, 140582265716735, ++STORE, 140582265716736, 140582265720831, ++STORE, 140582265720832, 140582265724927, ++STORE, 140582265724928, 140582265868287, ++STORE, 140582266236928, 140582267920383, ++STORE, 140582267920384, 140582267936767, ++STORE, 140582267965440, 140582267969535, ++STORE, 140582267969536, 140582267973631, ++STORE, 140582267973632, 140582267977727, ++STORE, 140735472508928, 140735472648191, ++STORE, 140735472672768, 140735472685055, ++STORE, 140735472685056, 140735472689151, ++STORE, 94440069140480, 94440069353471, ++STORE, 94440071450624, 94440071454719, ++STORE, 94440071454720, 94440071462911, ++STORE, 94440071462912, 94440071475199, ++STORE, 94440072122368, 94440079048703, ++STORE, 140112218095616, 140112219754495, ++STORE, 140112219754496, 140112221851647, ++STORE, 140112221851648, 140112221868031, ++STORE, 140112221868032, 140112221876223, ++STORE, 140112221876224, 140112221892607, ++STORE, 140112221892608, 140112221904895, ++STORE, 140112221904896, 140112223997951, ++STORE, 140112223997952, 140112224002047, ++STORE, 140112224002048, 140112224006143, ++STORE, 140112224006144, 140112224149503, ++STORE, 140112224518144, 140112226201599, ++STORE, 140112226201600, 140112226217983, ++STORE, 140112226246656, 140112226250751, ++STORE, 140112226250752, 140112226254847, ++STORE, 140112226254848, 140112226258943, ++STORE, 140737460969472, 140737461108735, ++STORE, 140737462083584, 140737462095871, ++STORE, 140737462095872, 140737462099967, ++STORE, 94257654345728, 94257654390783, ++STORE, 94257656483840, 94257656487935, ++STORE, 94257656487936, 94257656492031, ++STORE, 94257656492032, 94257656496127, ++STORE, 94257665859584, 94257665994751, ++STORE, 140507070345216, 140507070386175, ++STORE, 140507070386176, 140507072483327, ++STORE, 140507072483328, 140507072487423, ++STORE, 140507072487424, 140507072491519, ++STORE, 140507072491520, 140507072516095, ++STORE, 140507072516096, 140507072561151, ++STORE, 140507072561152, 140507074654207, ++STORE, 140507074654208, 140507074658303, ++STORE, 140507074658304, 140507074662399, ++STORE, 140507074662400, 140507074744319, ++STORE, 140507074744320, 140507076841471, ++STORE, 140507076841472, 140507076845567, ++STORE, 140507076845568, 140507076849663, ++STORE, 140507076849664, 140507076857855, ++STORE, 140507076857856, 140507076886527, ++STORE, 140507076886528, 140507078979583, ++STORE, 140507078979584, 140507078983679, ++STORE, 140507078983680, 140507078987775, ++STORE, 140507078987776, 140507079086079, ++STORE, 140507079086080, 140507081179135, ++STORE, 140507081179136, 140507081183231, ++STORE, 140507081183232, 140507081187327, ++STORE, 140507081187328, 140507081203711, ++STORE, 140507081203712, 140507081220095, ++STORE, 140507081220096, 140507083317247, ++STORE, 140507083317248, 140507083321343, ++STORE, 140507083321344, 140507083325439, ++STORE, 140507083325440, 140507083792383, ++STORE, 140507083792384, 140507085885439, ++STORE, 140507085885440, 140507085889535, ++STORE, 140507085889536, 140507085893631, ++STORE, 140507085893632, 140507085905919, ++STORE, 140507085905920, 140507087998975, ++STORE, 140507087998976, 140507088003071, ++STORE, 140507088003072, 140507088007167, ++STORE, 140507088007168, 140507088125951, ++STORE, 140507088125952, 140507090219007, ++STORE, 140507090219008, 140507090223103, ++STORE, 140507090223104, 140507090227199, ++STORE, 140507090227200, 140507090268159, ++STORE, 140507090268160, 140507091927039, ++STORE, 140507091927040, 140507094024191, ++STORE, 140507094024192, 140507094040575, ++STORE, 140507094040576, 140507094048767, ++STORE, 140507094048768, 140507094065151, ++STORE, 140507094065152, 140507094216703, ++STORE, 140507094216704, 140507096309759, ++STORE, 140507096309760, 140507096313855, ++STORE, 140507096313856, 140507096317951, ++STORE, 140507096317952, 140507096326143, ++STORE, 140507096326144, 140507096379391, ++STORE, 140507096379392, 140507098472447, ++STORE, 140507098472448, 140507098476543, ++STORE, 140507098476544, 140507098480639, ++STORE, 140507098480640, 140507098623999, ++STORE, 140507098980352, 140507100663807, ++STORE, 140507100663808, 140507100692479, ++STORE, 140507100721152, 140507100725247, ++STORE, 140507100725248, 140507100729343, ++STORE, 140507100729344, 140507100733439, ++STORE, 140728152780800, 140728152915967, ++STORE, 140728153698304, 140728153710591, ++STORE, 140728153710592, 140728153714687, ++STORE, 140507068137472, 140507070345215, ++SNULL, 140507068137472, 140507068190719, ++STORE, 140507068190720, 140507070345215, ++STORE, 140507068137472, 140507068190719, ++SNULL, 140507070287871, 140507070345215, ++STORE, 140507068190720, 140507070287871, ++STORE, 140507070287872, 140507070345215, ++SNULL, 140507070287872, 140507070296063, ++STORE, 140507070296064, 140507070345215, ++STORE, 140507070287872, 140507070296063, ++ERASE, 140507070287872, 140507070296063, ++STORE, 140507070287872, 140507070296063, ++ERASE, 140507070296064, 140507070345215, ++STORE, 140507070296064, 140507070345215, ++STORE, 140507100692480, 140507100721151, ++STORE, 140507065810944, 140507068137471, ++SNULL, 140507065810944, 140507065843711, ++STORE, 140507065843712, 140507068137471, ++STORE, 140507065810944, 140507065843711, ++SNULL, 140507067940863, 140507068137471, ++STORE, 140507065843712, 140507067940863, ++STORE, 140507067940864, 140507068137471, ++SNULL, 140507067940864, 140507067949055, ++STORE, 140507067949056, 140507068137471, ++STORE, 140507067940864, 140507067949055, ++ERASE, 140507067940864, 140507067949055, ++STORE, 140507067940864, 140507067949055, ++ERASE, 140507067949056, 140507068137471, ++STORE, 140507067949056, 140507068137471, ++SNULL, 140507067944959, 140507067949055, ++STORE, 140507067940864, 140507067944959, ++STORE, 140507067944960, 140507067949055, ++SNULL, 140507070291967, 140507070296063, ++STORE, 140507070287872, 140507070291967, ++STORE, 140507070291968, 140507070296063, ++ERASE, 140507100692480, 140507100721151, ++STORE, 140507063705600, 140507065810943, ++SNULL, 140507063705600, 140507063709695, ++STORE, 140507063709696, 140507065810943, ++STORE, 140507063705600, 140507063709695, ++SNULL, 140507065802751, 140507065810943, ++STORE, 140507063709696, 140507065802751, ++STORE, 140507065802752, 140507065810943, ++ERASE, 140507065802752, 140507065810943, ++STORE, 140507065802752, 140507065810943, ++SNULL, 140507065806847, 140507065810943, ++STORE, 140507065802752, 140507065806847, ++STORE, 140507065806848, 140507065810943, ++STORE, 140507061600256, 140507063705599, ++SNULL, 140507061600256, 140507061604351, ++STORE, 140507061604352, 140507063705599, ++STORE, 140507061600256, 140507061604351, ++SNULL, 140507063697407, 140507063705599, ++STORE, 140507061604352, 140507063697407, ++STORE, 140507063697408, 140507063705599, ++ERASE, 140507063697408, 140507063705599, ++STORE, 140507063697408, 140507063705599, ++SNULL, 140507063701503, 140507063705599, ++STORE, 140507063697408, 140507063701503, ++STORE, 140507063701504, 140507063705599, ++STORE, 140507059490816, 140507061600255, ++SNULL, 140507059490816, 140507059499007, ++STORE, 140507059499008, 140507061600255, ++STORE, 140507059490816, 140507059499007, ++SNULL, 140507061592063, 140507061600255, ++STORE, 140507059499008, 140507061592063, ++STORE, 140507061592064, 140507061600255, ++ERASE, 140507061592064, 140507061600255, ++STORE, 140507061592064, 140507061600255, ++SNULL, 140507061596159, 140507061600255, ++STORE, 140507061592064, 140507061596159, ++STORE, 140507061596160, 140507061600255, ++STORE, 140507057377280, 140507059490815, ++SNULL, 140507057377280, 140507057389567, ++STORE, 140507057389568, 140507059490815, ++STORE, 140507057377280, 140507057389567, ++SNULL, 140507059482623, 140507059490815, ++STORE, 140507057389568, 140507059482623, ++STORE, 140507059482624, 140507059490815, ++ERASE, 140507059482624, 140507059490815, ++STORE, 140507059482624, 140507059490815, ++SNULL, 140507059486719, 140507059490815, ++STORE, 140507059482624, 140507059486719, ++STORE, 140507059486720, 140507059490815, ++STORE, 140507055255552, 140507057377279, ++SNULL, 140507055255552, 140507055276031, ++STORE, 140507055276032, 140507057377279, ++STORE, 140507055255552, 140507055276031, ++SNULL, 140507057369087, 140507057377279, ++STORE, 140507055276032, 140507057369087, ++STORE, 140507057369088, 140507057377279, ++ERASE, 140507057369088, 140507057377279, ++STORE, 140507057369088, 140507057377279, ++SNULL, 140507057373183, 140507057377279, ++STORE, 140507057369088, 140507057373183, ++STORE, 140507057373184, 140507057377279, ++STORE, 140507098693632, 140507098980351, ++SNULL, 140507098959871, 140507098980351, ++STORE, 140507098693632, 140507098959871, ++STORE, 140507098959872, 140507098980351, ++SNULL, 140507098959872, 140507098976255, ++STORE, 140507098976256, 140507098980351, ++STORE, 140507098959872, 140507098976255, ++ERASE, 140507098959872, 140507098976255, ++STORE, 140507098959872, 140507098976255, ++ERASE, 140507098976256, 140507098980351, ++STORE, 140507098976256, 140507098980351, ++STORE, 140507100692480, 140507100721151, ++STORE, 140507053125632, 140507055255551, ++SNULL, 140507053125632, 140507053154303, ++STORE, 140507053154304, 140507055255551, ++STORE, 140507053125632, 140507053154303, ++SNULL, 140507055247359, 140507055255551, ++STORE, 140507053154304, 140507055247359, ++STORE, 140507055247360, 140507055255551, ++ERASE, 140507055247360, 140507055255551, ++STORE, 140507055247360, 140507055255551, ++STORE, 140507051012096, 140507053125631, ++SNULL, 140507051012096, 140507051024383, ++STORE, 140507051024384, 140507053125631, ++STORE, 140507051012096, 140507051024383, ++SNULL, 140507053117439, 140507053125631, ++STORE, 140507051024384, 140507053117439, ++STORE, 140507053117440, 140507053125631, ++ERASE, 140507053117440, 140507053125631, ++STORE, 140507053117440, 140507053125631, ++SNULL, 140507053121535, 140507053125631, ++STORE, 140507053117440, 140507053121535, ++STORE, 140507053121536, 140507053125631, ++SNULL, 140507055251455, 140507055255551, ++STORE, 140507055247360, 140507055251455, ++STORE, 140507055251456, 140507055255551, ++SNULL, 140507098972159, 140507098976255, ++STORE, 140507098959872, 140507098972159, ++STORE, 140507098972160, 140507098976255, ++ERASE, 140507100692480, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100717056, 140507100721151, ++ERASE, 140507100717056, 140507100721151, ++STORE, 140507100692480, 140507100721151, ++ERASE, 140507068137472, 140507068190719, ++ERASE, 140507068190720, 140507070287871, ++ERASE, 140507070287872, 140507070291967, ++ERASE, 140507070291968, 140507070296063, ++ERASE, 140507070296064, 140507070345215, ++ERASE, 140507065810944, 140507065843711, ++ERASE, 140507065843712, 140507067940863, ++ERASE, 140507067940864, 140507067944959, ++ERASE, 140507067944960, 140507067949055, ++ERASE, 140507067949056, 140507068137471, ++ERASE, 140507063705600, 140507063709695, ++ERASE, 140507063709696, 140507065802751, ++ERASE, 140507065802752, 140507065806847, ++ERASE, 140507065806848, 140507065810943, ++ERASE, 140507061600256, 140507061604351, ++ERASE, 140507061604352, 140507063697407, ++ERASE, 140507063697408, 140507063701503, ++ERASE, 140507063701504, 140507063705599, ++ERASE, 140507059490816, 140507059499007, ++ERASE, 140507059499008, 140507061592063, ++ERASE, 140507061592064, 140507061596159, ++ERASE, 140507061596160, 140507061600255, ++ERASE, 140507057377280, 140507057389567, ++ERASE, 140507057389568, 140507059482623, ++ERASE, 140507059482624, 140507059486719, ++ERASE, 140507059486720, 140507059490815, ++ERASE, 140507055255552, 140507055276031, ++ERASE, 140507055276032, 140507057369087, ++ERASE, 140507057369088, 140507057373183, ++ERASE, 140507057373184, 140507057377279, ++ERASE, 140507098693632, 140507098959871, ++ERASE, 140507098959872, 140507098972159, ++ERASE, 140507098972160, 140507098976255, ++ERASE, 140507098976256, 140507098980351, ++ERASE, 140507051012096, 140507051024383, ++ERASE, 140507051024384, 140507053117439, ++ERASE, 140507053117440, 140507053121535, ++ERASE, 140507053121536, 140507053125631, ++STORE, 94036448296960, 94036448509951, ++STORE, 94036450607104, 94036450611199, ++STORE, 94036450611200, 94036450619391, ++STORE, 94036450619392, 94036450631679, ++STORE, 94036482445312, 94036502376447, ++STORE, 140469487013888, 140469488672767, ++STORE, 140469488672768, 140469490769919, ++STORE, 140469490769920, 140469490786303, ++STORE, 140469490786304, 140469490794495, ++STORE, 140469490794496, 140469490810879, ++STORE, 140469490810880, 140469490823167, ++STORE, 140469490823168, 140469492916223, ++STORE, 140469492916224, 140469492920319, ++STORE, 140469492920320, 140469492924415, ++STORE, 140469492924416, 140469493067775, ++STORE, 140469493436416, 140469495119871, ++STORE, 140469495119872, 140469495136255, ++STORE, 140469495164928, 140469495169023, ++STORE, 140469495169024, 140469495173119, ++STORE, 140469495173120, 140469495177215, ++STORE, 140732281446400, 140732281585663, ++STORE, 140732282736640, 140732282748927, ++STORE, 140732282748928, 140732282753023, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723411931136, 140737488351231, ++SNULL, 140723411939327, 140737488351231, ++STORE, 140723411931136, 140723411939327, ++STORE, 140723411800064, 140723411939327, ++STORE, 93993768685568, 93993770909695, ++SNULL, 93993768796159, 93993770909695, ++STORE, 93993768685568, 93993768796159, ++STORE, 93993768796160, 93993770909695, ++ERASE, 93993768796160, 93993770909695, ++STORE, 93993770889216, 93993770901503, ++STORE, 93993770901504, 93993770909695, ++STORE, 140508681740288, 140508683993087, ++SNULL, 140508681883647, 140508683993087, ++STORE, 140508681740288, 140508681883647, ++STORE, 140508681883648, 140508683993087, ++ERASE, 140508681883648, 140508683993087, ++STORE, 140508683980800, 140508683988991, ++STORE, 140508683988992, 140508683993087, ++STORE, 140723412070400, 140723412074495, ++STORE, 140723412058112, 140723412070399, ++STORE, 140508683952128, 140508683980799, ++STORE, 140508683943936, 140508683952127, ++STORE, 140508677943296, 140508681740287, ++SNULL, 140508677943296, 140508679602175, ++STORE, 140508679602176, 140508681740287, ++STORE, 140508677943296, 140508679602175, ++SNULL, 140508681699327, 140508681740287, ++STORE, 140508679602176, 140508681699327, ++STORE, 140508681699328, 140508681740287, ++SNULL, 140508681699328, 140508681723903, ++STORE, 140508681723904, 140508681740287, ++STORE, 140508681699328, 140508681723903, ++ERASE, 140508681699328, 140508681723903, ++STORE, 140508681699328, 140508681723903, ++ERASE, 140508681723904, 140508681740287, ++STORE, 140508681723904, 140508681740287, ++SNULL, 140508681715711, 140508681723903, ++STORE, 140508681699328, 140508681715711, ++STORE, 140508681715712, 140508681723903, ++SNULL, 93993770897407, 93993770901503, ++STORE, 93993770889216, 93993770897407, ++STORE, 93993770897408, 93993770901503, ++SNULL, 140508683984895, 140508683988991, ++STORE, 140508683980800, 140508683984895, ++STORE, 140508683984896, 140508683988991, ++ERASE, 140508683952128, 140508683980799, ++STORE, 93993791582208, 93993791717375, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140734685458432, 140737488351231, ++SNULL, 140734685466623, 140737488351231, ++STORE, 140734685458432, 140734685466623, ++STORE, 140734685327360, 140734685466623, ++STORE, 93832321548288, 93832323772415, ++SNULL, 93832321658879, 93832323772415, ++STORE, 93832321548288, 93832321658879, ++STORE, 93832321658880, 93832323772415, ++ERASE, 93832321658880, 93832323772415, ++STORE, 93832323751936, 93832323764223, ++STORE, 93832323764224, 93832323772415, ++STORE, 140650945118208, 140650947371007, ++SNULL, 140650945261567, 140650947371007, ++STORE, 140650945118208, 140650945261567, ++STORE, 140650945261568, 140650947371007, ++ERASE, 140650945261568, 140650947371007, ++STORE, 140650947358720, 140650947366911, ++STORE, 140650947366912, 140650947371007, ++STORE, 140734686081024, 140734686085119, ++STORE, 140734686068736, 140734686081023, ++STORE, 140650947330048, 140650947358719, ++STORE, 140650947321856, 140650947330047, ++STORE, 140650941321216, 140650945118207, ++SNULL, 140650941321216, 140650942980095, ++STORE, 140650942980096, 140650945118207, ++STORE, 140650941321216, 140650942980095, ++SNULL, 140650945077247, 140650945118207, ++STORE, 140650942980096, 140650945077247, ++STORE, 140650945077248, 140650945118207, ++SNULL, 140650945077248, 140650945101823, ++STORE, 140650945101824, 140650945118207, ++STORE, 140650945077248, 140650945101823, ++ERASE, 140650945077248, 140650945101823, ++STORE, 140650945077248, 140650945101823, ++ERASE, 140650945101824, 140650945118207, ++STORE, 140650945101824, 140650945118207, ++SNULL, 140650945093631, 140650945101823, ++STORE, 140650945077248, 140650945093631, ++STORE, 140650945093632, 140650945101823, ++SNULL, 93832323760127, 93832323764223, ++STORE, 93832323751936, 93832323760127, ++STORE, 93832323760128, 93832323764223, ++SNULL, 140650947362815, 140650947366911, ++STORE, 140650947358720, 140650947362815, ++STORE, 140650947362816, 140650947366911, ++ERASE, 140650947330048, 140650947358719, ++STORE, 93832331890688, 93832332025855, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728333520896, 140737488351231, ++SNULL, 140728333529087, 140737488351231, ++STORE, 140728333520896, 140728333529087, ++STORE, 140728333389824, 140728333529087, ++STORE, 94872734732288, 94872736956415, ++SNULL, 94872734842879, 94872736956415, ++STORE, 94872734732288, 94872734842879, ++STORE, 94872734842880, 94872736956415, ++ERASE, 94872734842880, 94872736956415, ++STORE, 94872736935936, 94872736948223, ++STORE, 94872736948224, 94872736956415, ++STORE, 139755193257984, 139755195510783, ++SNULL, 139755193401343, 139755195510783, ++STORE, 139755193257984, 139755193401343, ++STORE, 139755193401344, 139755195510783, ++ERASE, 139755193401344, 139755195510783, ++STORE, 139755195498496, 139755195506687, ++STORE, 139755195506688, 139755195510783, ++STORE, 140728333926400, 140728333930495, ++STORE, 140728333914112, 140728333926399, ++STORE, 139755195469824, 139755195498495, ++STORE, 139755195461632, 139755195469823, ++STORE, 139755189460992, 139755193257983, ++SNULL, 139755189460992, 139755191119871, ++STORE, 139755191119872, 139755193257983, ++STORE, 139755189460992, 139755191119871, ++SNULL, 139755193217023, 139755193257983, ++STORE, 139755191119872, 139755193217023, ++STORE, 139755193217024, 139755193257983, ++SNULL, 139755193217024, 139755193241599, ++STORE, 139755193241600, 139755193257983, ++STORE, 139755193217024, 139755193241599, ++ERASE, 139755193217024, 139755193241599, ++STORE, 139755193217024, 139755193241599, ++ERASE, 139755193241600, 139755193257983, ++STORE, 139755193241600, 139755193257983, ++SNULL, 139755193233407, 139755193241599, ++STORE, 139755193217024, 139755193233407, ++STORE, 139755193233408, 139755193241599, ++SNULL, 94872736944127, 94872736948223, ++STORE, 94872736935936, 94872736944127, ++STORE, 94872736944128, 94872736948223, ++SNULL, 139755195502591, 139755195506687, ++STORE, 139755195498496, 139755195502591, ++STORE, 139755195502592, 139755195506687, ++ERASE, 139755195469824, 139755195498495, ++STORE, 94872749744128, 94872749879295, ++STORE, 94720243642368, 94720243855359, ++STORE, 94720245952512, 94720245956607, ++STORE, 94720245956608, 94720245964799, ++STORE, 94720245964800, 94720245977087, ++STORE, 94720277745664, 94720278151167, ++STORE, 140453174497280, 140453176156159, ++STORE, 140453176156160, 140453178253311, ++STORE, 140453178253312, 140453178269695, ++STORE, 140453178269696, 140453178277887, ++STORE, 140453178277888, 140453178294271, ++STORE, 140453178294272, 140453178306559, ++STORE, 140453178306560, 140453180399615, ++STORE, 140453180399616, 140453180403711, ++STORE, 140453180403712, 140453180407807, ++STORE, 140453180407808, 140453180551167, ++STORE, 140453180919808, 140453182603263, ++STORE, 140453182603264, 140453182619647, ++STORE, 140453182648320, 140453182652415, ++STORE, 140453182652416, 140453182656511, ++STORE, 140453182656512, 140453182660607, ++STORE, 140733223923712, 140733224062975, ++STORE, 140733224808448, 140733224820735, ++STORE, 140733224820736, 140733224824831, ++STORE, 94321091141632, 94321091354623, ++STORE, 94321093451776, 94321093455871, ++STORE, 94321093455872, 94321093464063, ++STORE, 94321093464064, 94321093476351, ++STORE, 94321115873280, 94321117229055, ++STORE, 139695978840064, 139695980498943, ++STORE, 139695980498944, 139695982596095, ++STORE, 139695982596096, 139695982612479, ++STORE, 139695982612480, 139695982620671, ++STORE, 139695982620672, 139695982637055, ++STORE, 139695982637056, 139695982649343, ++STORE, 139695982649344, 139695984742399, ++STORE, 139695984742400, 139695984746495, ++STORE, 139695984746496, 139695984750591, ++STORE, 139695984750592, 139695984893951, ++STORE, 139695985262592, 139695986946047, ++STORE, 139695986946048, 139695986962431, ++STORE, 139695986991104, 139695986995199, ++STORE, 139695986995200, 139695986999295, ++STORE, 139695986999296, 139695987003391, ++STORE, 140734650564608, 140734650703871, ++STORE, 140734650785792, 140734650798079, ++STORE, 140734650798080, 140734650802175, ++STORE, 94523438456832, 94523438669823, ++STORE, 94523440766976, 94523440771071, ++STORE, 94523440771072, 94523440779263, ++STORE, 94523440779264, 94523440791551, ++STORE, 94523464544256, 94523465842687, ++STORE, 140453231493120, 140453233151999, ++STORE, 140453233152000, 140453235249151, ++STORE, 140453235249152, 140453235265535, ++STORE, 140453235265536, 140453235273727, ++STORE, 140453235273728, 140453235290111, ++STORE, 140453235290112, 140453235302399, ++STORE, 140453235302400, 140453237395455, ++STORE, 140453237395456, 140453237399551, ++STORE, 140453237399552, 140453237403647, ++STORE, 140453237403648, 140453237547007, ++STORE, 140453237915648, 140453239599103, ++STORE, 140453239599104, 140453239615487, ++STORE, 140453239644160, 140453239648255, ++STORE, 140453239648256, 140453239652351, ++STORE, 140453239652352, 140453239656447, ++STORE, 140734679445504, 140734679584767, ++STORE, 140734680018944, 140734680031231, ++STORE, 140734680031232, 140734680035327, ++STORE, 94614776987648, 94614777200639, ++STORE, 94614779297792, 94614779301887, ++STORE, 94614779301888, 94614779310079, ++STORE, 94614779310080, 94614779322367, ++STORE, 94614798467072, 94614800699391, ++STORE, 139677037182976, 139677038841855, ++STORE, 139677038841856, 139677040939007, ++STORE, 139677040939008, 139677040955391, ++STORE, 139677040955392, 139677040963583, ++STORE, 139677040963584, 139677040979967, ++STORE, 139677040979968, 139677040992255, ++STORE, 139677040992256, 139677043085311, ++STORE, 139677043085312, 139677043089407, ++STORE, 139677043089408, 139677043093503, ++STORE, 139677043093504, 139677043236863, ++STORE, 139677043605504, 139677045288959, ++STORE, 139677045288960, 139677045305343, ++STORE, 139677045334016, 139677045338111, ++STORE, 139677045338112, 139677045342207, ++STORE, 139677045342208, 139677045346303, ++STORE, 140721604411392, 140721604550655, ++STORE, 140721606135808, 140721606148095, ++STORE, 140721606148096, 140721606152191, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140729280544768, 140737488351231, ++SNULL, 140729280552959, 140737488351231, ++STORE, 140729280544768, 140729280552959, ++STORE, 140729280413696, 140729280552959, ++STORE, 94863939334144, 94863941558271, ++SNULL, 94863939444735, 94863941558271, ++STORE, 94863939334144, 94863939444735, ++STORE, 94863939444736, 94863941558271, ++ERASE, 94863939444736, 94863941558271, ++STORE, 94863941537792, 94863941550079, ++STORE, 94863941550080, 94863941558271, ++STORE, 139691047276544, 139691049529343, ++SNULL, 139691047419903, 139691049529343, ++STORE, 139691047276544, 139691047419903, ++STORE, 139691047419904, 139691049529343, ++ERASE, 139691047419904, 139691049529343, ++STORE, 139691049517056, 139691049525247, ++STORE, 139691049525248, 139691049529343, ++STORE, 140729281679360, 140729281683455, ++STORE, 140729281667072, 140729281679359, ++STORE, 139691049488384, 139691049517055, ++STORE, 139691049480192, 139691049488383, ++STORE, 139691043479552, 139691047276543, ++SNULL, 139691043479552, 139691045138431, ++STORE, 139691045138432, 139691047276543, ++STORE, 139691043479552, 139691045138431, ++SNULL, 139691047235583, 139691047276543, ++STORE, 139691045138432, 139691047235583, ++STORE, 139691047235584, 139691047276543, ++SNULL, 139691047235584, 139691047260159, ++STORE, 139691047260160, 139691047276543, ++STORE, 139691047235584, 139691047260159, ++ERASE, 139691047235584, 139691047260159, ++STORE, 139691047235584, 139691047260159, ++ERASE, 139691047260160, 139691047276543, ++STORE, 139691047260160, 139691047276543, ++SNULL, 139691047251967, 139691047260159, ++STORE, 139691047235584, 139691047251967, ++STORE, 139691047251968, 139691047260159, ++SNULL, 94863941545983, 94863941550079, ++STORE, 94863941537792, 94863941545983, ++STORE, 94863941545984, 94863941550079, ++SNULL, 139691049521151, 139691049525247, ++STORE, 139691049517056, 139691049521151, ++STORE, 139691049521152, 139691049525247, ++ERASE, 139691049488384, 139691049517055, ++STORE, 94863951294464, 94863951429631, ++STORE, 93998209294336, 93998209507327, ++STORE, 93998211604480, 93998211608575, ++STORE, 93998211608576, 93998211616767, ++STORE, 93998211616768, 93998211629055, ++STORE, 93998227210240, 93998227615743, ++STORE, 140243029913600, 140243031572479, ++STORE, 140243031572480, 140243033669631, ++STORE, 140243033669632, 140243033686015, ++STORE, 140243033686016, 140243033694207, ++STORE, 140243033694208, 140243033710591, ++STORE, 140243033710592, 140243033722879, ++STORE, 140243033722880, 140243035815935, ++STORE, 140243035815936, 140243035820031, ++STORE, 140243035820032, 140243035824127, ++STORE, 140243035824128, 140243035967487, ++STORE, 140243036336128, 140243038019583, ++STORE, 140243038019584, 140243038035967, ++STORE, 140243038064640, 140243038068735, ++STORE, 140243038068736, 140243038072831, ++STORE, 140243038072832, 140243038076927, ++STORE, 140734976479232, 140734976618495, ++STORE, 140734977978368, 140734977990655, ++STORE, 140734977990656, 140734977994751, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722742775808, 140737488351231, ++SNULL, 140722742783999, 140737488351231, ++STORE, 140722742775808, 140722742783999, ++STORE, 140722742644736, 140722742783999, ++STORE, 93857673662464, 93857675997183, ++SNULL, 93857673875455, 93857675997183, ++STORE, 93857673662464, 93857673875455, ++STORE, 93857673875456, 93857675997183, ++ERASE, 93857673875456, 93857675997183, ++STORE, 93857675972608, 93857675984895, ++STORE, 93857675984896, 93857675997183, ++STORE, 140629677498368, 140629679751167, ++SNULL, 140629677641727, 140629679751167, ++STORE, 140629677498368, 140629677641727, ++STORE, 140629677641728, 140629679751167, ++ERASE, 140629677641728, 140629679751167, ++STORE, 140629679738880, 140629679747071, ++STORE, 140629679747072, 140629679751167, ++STORE, 140722743222272, 140722743226367, ++STORE, 140722743209984, 140722743222271, ++STORE, 140629679710208, 140629679738879, ++STORE, 140629679702016, 140629679710207, ++STORE, 140629675384832, 140629677498367, ++SNULL, 140629675384832, 140629675397119, ++STORE, 140629675397120, 140629677498367, ++STORE, 140629675384832, 140629675397119, ++SNULL, 140629677490175, 140629677498367, ++STORE, 140629675397120, 140629677490175, ++STORE, 140629677490176, 140629677498367, ++ERASE, 140629677490176, 140629677498367, ++STORE, 140629677490176, 140629677498367, ++STORE, 140629671587840, 140629675384831, ++SNULL, 140629671587840, 140629673246719, ++STORE, 140629673246720, 140629675384831, ++STORE, 140629671587840, 140629673246719, ++SNULL, 140629675343871, 140629675384831, ++STORE, 140629673246720, 140629675343871, ++STORE, 140629675343872, 140629675384831, ++SNULL, 140629675343872, 140629675368447, ++STORE, 140629675368448, 140629675384831, ++STORE, 140629675343872, 140629675368447, ++ERASE, 140629675343872, 140629675368447, ++STORE, 140629675343872, 140629675368447, ++ERASE, 140629675368448, 140629675384831, ++STORE, 140629675368448, 140629675384831, ++STORE, 140629679693824, 140629679710207, ++SNULL, 140629675360255, 140629675368447, ++STORE, 140629675343872, 140629675360255, ++STORE, 140629675360256, 140629675368447, ++SNULL, 140629677494271, 140629677498367, ++STORE, 140629677490176, 140629677494271, ++STORE, 140629677494272, 140629677498367, ++SNULL, 93857675976703, 93857675984895, ++STORE, 93857675972608, 93857675976703, ++STORE, 93857675976704, 93857675984895, ++SNULL, 140629679742975, 140629679747071, ++STORE, 140629679738880, 140629679742975, ++STORE, 140629679742976, 140629679747071, ++ERASE, 140629679710208, 140629679738879, ++STORE, 93857705832448, 93857705967615, ++STORE, 140629678010368, 140629679693823, ++STORE, 93857705832448, 93857706102783, ++STORE, 93857705832448, 93857706237951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735922421760, 140737488351231, ++SNULL, 140735922429951, 140737488351231, ++STORE, 140735922421760, 140735922429951, ++STORE, 140735922290688, 140735922429951, ++STORE, 94651136139264, 94651138363391, ++SNULL, 94651136249855, 94651138363391, ++STORE, 94651136139264, 94651136249855, ++STORE, 94651136249856, 94651138363391, ++ERASE, 94651136249856, 94651138363391, ++STORE, 94651138342912, 94651138355199, ++STORE, 94651138355200, 94651138363391, ++STORE, 140325788266496, 140325790519295, ++SNULL, 140325788409855, 140325790519295, ++STORE, 140325788266496, 140325788409855, ++STORE, 140325788409856, 140325790519295, ++ERASE, 140325788409856, 140325790519295, ++STORE, 140325790507008, 140325790515199, ++STORE, 140325790515200, 140325790519295, ++STORE, 140735923572736, 140735923576831, ++STORE, 140735923560448, 140735923572735, ++STORE, 140325790478336, 140325790507007, ++STORE, 140325790470144, 140325790478335, ++STORE, 140325784469504, 140325788266495, ++SNULL, 140325784469504, 140325786128383, ++STORE, 140325786128384, 140325788266495, ++STORE, 140325784469504, 140325786128383, ++SNULL, 140325788225535, 140325788266495, ++STORE, 140325786128384, 140325788225535, ++STORE, 140325788225536, 140325788266495, ++SNULL, 140325788225536, 140325788250111, ++STORE, 140325788250112, 140325788266495, ++STORE, 140325788225536, 140325788250111, ++ERASE, 140325788225536, 140325788250111, ++STORE, 140325788225536, 140325788250111, ++ERASE, 140325788250112, 140325788266495, ++STORE, 140325788250112, 140325788266495, ++SNULL, 140325788241919, 140325788250111, ++STORE, 140325788225536, 140325788241919, ++STORE, 140325788241920, 140325788250111, ++SNULL, 94651138351103, 94651138355199, ++STORE, 94651138342912, 94651138351103, ++STORE, 94651138351104, 94651138355199, ++SNULL, 140325790511103, 140325790515199, ++STORE, 140325790507008, 140325790511103, ++STORE, 140325790511104, 140325790515199, ++ERASE, 140325790478336, 140325790507007, ++STORE, 94651146297344, 94651146432511, ++STORE, 94212330168320, 94212330381311, ++STORE, 94212332478464, 94212332482559, ++STORE, 94212332482560, 94212332490751, ++STORE, 94212332490752, 94212332503039, ++STORE, 94212348891136, 94212349825023, ++STORE, 140611630604288, 140611632263167, ++STORE, 140611632263168, 140611634360319, ++STORE, 140611634360320, 140611634376703, ++STORE, 140611634376704, 140611634384895, ++STORE, 140611634384896, 140611634401279, ++STORE, 140611634401280, 140611634413567, ++STORE, 140611634413568, 140611636506623, ++STORE, 140611636506624, 140611636510719, ++STORE, 140611636510720, 140611636514815, ++STORE, 140611636514816, 140611636658175, ++STORE, 140611637026816, 140611638710271, ++STORE, 140611638710272, 140611638726655, ++STORE, 140611638755328, 140611638759423, ++STORE, 140611638759424, 140611638763519, ++STORE, 140611638763520, 140611638767615, ++STORE, 140726974533632, 140726974672895, ++STORE, 140726974943232, 140726974955519, ++STORE, 140726974955520, 140726974959615, ++STORE, 94572463521792, 94572463734783, ++STORE, 94572465831936, 94572465836031, ++STORE, 94572465836032, 94572465844223, ++STORE, 94572465844224, 94572465856511, ++STORE, 94572491534336, 94572492865535, ++STORE, 140644351492096, 140644353150975, ++STORE, 140644353150976, 140644355248127, ++STORE, 140644355248128, 140644355264511, ++STORE, 140644355264512, 140644355272703, ++STORE, 140644355272704, 140644355289087, ++STORE, 140644355289088, 140644355301375, ++STORE, 140644355301376, 140644357394431, ++STORE, 140644357394432, 140644357398527, ++STORE, 140644357398528, 140644357402623, ++STORE, 140644357402624, 140644357545983, ++STORE, 140644357914624, 140644359598079, ++STORE, 140644359598080, 140644359614463, ++STORE, 140644359643136, 140644359647231, ++STORE, 140644359647232, 140644359651327, ++STORE, 140644359651328, 140644359655423, ++STORE, 140727841824768, 140727841964031, ++STORE, 140727843188736, 140727843201023, ++STORE, 140727843201024, 140727843205119, ++STORE, 94144315457536, 94144315670527, ++STORE, 94144317767680, 94144317771775, ++STORE, 94144317771776, 94144317779967, ++STORE, 94144317779968, 94144317792255, ++STORE, 94144318369792, 94144320815103, ++STORE, 140316717645824, 140316719304703, ++STORE, 140316719304704, 140316721401855, ++STORE, 140316721401856, 140316721418239, ++STORE, 140316721418240, 140316721426431, ++STORE, 140316721426432, 140316721442815, ++STORE, 140316721442816, 140316721455103, ++STORE, 140316721455104, 140316723548159, ++STORE, 140316723548160, 140316723552255, ++STORE, 140316723552256, 140316723556351, ++STORE, 140316723556352, 140316723699711, ++STORE, 140316724068352, 140316725751807, ++STORE, 140316725751808, 140316725768191, ++STORE, 140316725796864, 140316725800959, ++STORE, 140316725800960, 140316725805055, ++STORE, 140316725805056, 140316725809151, ++STORE, 140725744283648, 140725744422911, ++STORE, 140725745852416, 140725745864703, ++STORE, 140725745864704, 140725745868799, ++STORE, 94646858846208, 94646859059199, ++STORE, 94646861156352, 94646861160447, ++STORE, 94646861160448, 94646861168639, ++STORE, 94646861168640, 94646861180927, ++STORE, 94646879805440, 94646881894399, ++STORE, 140435449745408, 140435451404287, ++STORE, 140435451404288, 140435453501439, ++STORE, 140435453501440, 140435453517823, ++STORE, 140435453517824, 140435453526015, ++STORE, 140435453526016, 140435453542399, ++STORE, 140435453542400, 140435453554687, ++STORE, 140435453554688, 140435455647743, ++STORE, 140435455647744, 140435455651839, ++STORE, 140435455651840, 140435455655935, ++STORE, 140435455655936, 140435455799295, ++STORE, 140435456167936, 140435457851391, ++STORE, 140435457851392, 140435457867775, ++STORE, 140435457896448, 140435457900543, ++STORE, 140435457900544, 140435457904639, ++STORE, 140435457904640, 140435457908735, ++STORE, 140721033818112, 140721033957375, ++STORE, 140721034018816, 140721034031103, ++STORE, 140721034031104, 140721034035199, ++STORE, 94872903438336, 94872903651327, ++STORE, 94872905748480, 94872905752575, ++STORE, 94872905752576, 94872905760767, ++STORE, 94872905760768, 94872905773055, ++STORE, 94872931246080, 94872931651583, ++STORE, 139771607810048, 139771609468927, ++STORE, 139771609468928, 139771611566079, ++STORE, 139771611566080, 139771611582463, ++STORE, 139771611582464, 139771611590655, ++STORE, 139771611590656, 139771611607039, ++STORE, 139771611607040, 139771611619327, ++STORE, 139771611619328, 139771613712383, ++STORE, 139771613712384, 139771613716479, ++STORE, 139771613716480, 139771613720575, ++STORE, 139771613720576, 139771613863935, ++STORE, 139771614232576, 139771615916031, ++STORE, 139771615916032, 139771615932415, ++STORE, 139771615961088, 139771615965183, ++STORE, 139771615965184, 139771615969279, ++STORE, 139771615969280, 139771615973375, ++STORE, 140725402931200, 140725403070463, ++STORE, 140725403852800, 140725403865087, ++STORE, 140725403865088, 140725403869183, ++STORE, 94740737736704, 94740737949695, ++STORE, 94740740046848, 94740740050943, ++STORE, 94740740050944, 94740740059135, ++STORE, 94740740059136, 94740740071423, ++STORE, 94740743249920, 94740744724479, ++STORE, 140640287010816, 140640288669695, ++STORE, 140640288669696, 140640290766847, ++STORE, 140640290766848, 140640290783231, ++STORE, 140640290783232, 140640290791423, ++STORE, 140640290791424, 140640290807807, ++STORE, 140640290807808, 140640290820095, ++STORE, 140640290820096, 140640292913151, ++STORE, 140640292913152, 140640292917247, ++STORE, 140640292917248, 140640292921343, ++STORE, 140640292921344, 140640293064703, ++STORE, 140640293433344, 140640295116799, ++STORE, 140640295116800, 140640295133183, ++STORE, 140640295161856, 140640295165951, ++STORE, 140640295165952, 140640295170047, ++STORE, 140640295170048, 140640295174143, ++STORE, 140725133303808, 140725133443071, ++STORE, 140725133684736, 140725133697023, ++STORE, 140725133697024, 140725133701119, ++STORE, 140737488347136, 140737488351231, ++STORE, 140722826371072, 140737488351231, ++SNULL, 140722826375167, 140737488351231, ++STORE, 140722826371072, 140722826375167, ++STORE, 140722826240000, 140722826375167, ++STORE, 94113818611712, 94113820835839, ++SNULL, 94113818722303, 94113820835839, ++STORE, 94113818611712, 94113818722303, ++STORE, 94113818722304, 94113820835839, ++ERASE, 94113818722304, 94113820835839, ++STORE, 94113820815360, 94113820827647, ++STORE, 94113820827648, 94113820835839, ++STORE, 139628194508800, 139628196761599, ++SNULL, 139628194652159, 139628196761599, ++STORE, 139628194508800, 139628194652159, ++STORE, 139628194652160, 139628196761599, ++ERASE, 139628194652160, 139628196761599, ++STORE, 139628196749312, 139628196757503, ++STORE, 139628196757504, 139628196761599, ++STORE, 140722826727424, 140722826731519, ++STORE, 140722826715136, 140722826727423, ++STORE, 139628196720640, 139628196749311, ++STORE, 139628196712448, 139628196720639, ++STORE, 139628190711808, 139628194508799, ++SNULL, 139628190711808, 139628192370687, ++STORE, 139628192370688, 139628194508799, ++STORE, 139628190711808, 139628192370687, ++SNULL, 139628194467839, 139628194508799, ++STORE, 139628192370688, 139628194467839, ++STORE, 139628194467840, 139628194508799, ++SNULL, 139628194467840, 139628194492415, ++STORE, 139628194492416, 139628194508799, ++STORE, 139628194467840, 139628194492415, ++ERASE, 139628194467840, 139628194492415, ++STORE, 139628194467840, 139628194492415, ++ERASE, 139628194492416, 139628194508799, ++STORE, 139628194492416, 139628194508799, ++SNULL, 139628194484223, 139628194492415, ++STORE, 139628194467840, 139628194484223, ++STORE, 139628194484224, 139628194492415, ++SNULL, 94113820823551, 94113820827647, ++STORE, 94113820815360, 94113820823551, ++STORE, 94113820823552, 94113820827647, ++SNULL, 139628196753407, 139628196757503, ++STORE, 139628196749312, 139628196753407, ++STORE, 139628196753408, 139628196757503, ++ERASE, 139628196720640, 139628196749311, ++STORE, 94113830850560, 94113830985727, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731865833472, 140737488351231, ++SNULL, 140731865841663, 140737488351231, ++STORE, 140731865833472, 140731865841663, ++STORE, 140731865702400, 140731865841663, ++STORE, 94763339386880, 94763341611007, ++SNULL, 94763339497471, 94763341611007, ++STORE, 94763339386880, 94763339497471, ++STORE, 94763339497472, 94763341611007, ++ERASE, 94763339497472, 94763341611007, ++STORE, 94763341590528, 94763341602815, ++STORE, 94763341602816, 94763341611007, ++STORE, 139778398486528, 139778400739327, ++SNULL, 139778398629887, 139778400739327, ++STORE, 139778398486528, 139778398629887, ++STORE, 139778398629888, 139778400739327, ++ERASE, 139778398629888, 139778400739327, ++STORE, 139778400727040, 139778400735231, ++STORE, 139778400735232, 139778400739327, ++STORE, 140731865858048, 140731865862143, ++STORE, 140731865845760, 140731865858047, ++STORE, 139778400698368, 139778400727039, ++STORE, 139778400690176, 139778400698367, ++STORE, 139778394689536, 139778398486527, ++SNULL, 139778394689536, 139778396348415, ++STORE, 139778396348416, 139778398486527, ++STORE, 139778394689536, 139778396348415, ++SNULL, 139778398445567, 139778398486527, ++STORE, 139778396348416, 139778398445567, ++STORE, 139778398445568, 139778398486527, ++SNULL, 139778398445568, 139778398470143, ++STORE, 139778398470144, 139778398486527, ++STORE, 139778398445568, 139778398470143, ++ERASE, 139778398445568, 139778398470143, ++STORE, 139778398445568, 139778398470143, ++ERASE, 139778398470144, 139778398486527, ++STORE, 139778398470144, 139778398486527, ++SNULL, 139778398461951, 139778398470143, ++STORE, 139778398445568, 139778398461951, ++STORE, 139778398461952, 139778398470143, ++SNULL, 94763341598719, 94763341602815, ++STORE, 94763341590528, 94763341598719, ++STORE, 94763341598720, 94763341602815, ++SNULL, 139778400731135, 139778400735231, ++STORE, 139778400727040, 139778400731135, ++STORE, 139778400731136, 139778400735231, ++ERASE, 139778400698368, 139778400727039, ++STORE, 94763362197504, 94763362332671, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140732053192704, 140737488351231, ++SNULL, 140732053204991, 140737488351231, ++STORE, 140732053192704, 140732053204991, ++STORE, 140732053061632, 140732053204991, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 140176018599936, 140176020852735, ++SNULL, 140176018743295, 140176020852735, ++STORE, 140176018599936, 140176018743295, ++STORE, 140176018743296, 140176020852735, ++ERASE, 140176018743296, 140176020852735, ++STORE, 140176020840448, 140176020848639, ++STORE, 140176020848640, 140176020852735, ++STORE, 140732053381120, 140732053385215, ++STORE, 140732053368832, 140732053381119, ++STORE, 140176020811776, 140176020840447, ++STORE, 140176020803584, 140176020811775, ++STORE, 140176014766080, 140176018599935, ++SNULL, 140176014766080, 140176016474111, ++STORE, 140176016474112, 140176018599935, ++STORE, 140176014766080, 140176016474111, ++SNULL, 140176018567167, 140176018599935, ++STORE, 140176016474112, 140176018567167, ++STORE, 140176018567168, 140176018599935, ++ERASE, 140176018567168, 140176018599935, ++STORE, 140176018567168, 140176018599935, ++STORE, 140176012570624, 140176014766079, ++SNULL, 140176012570624, 140176012664831, ++STORE, 140176012664832, 140176014766079, ++STORE, 140176012570624, 140176012664831, ++SNULL, 140176014757887, 140176014766079, ++STORE, 140176012664832, 140176014757887, ++STORE, 140176014757888, 140176014766079, ++ERASE, 140176014757888, 140176014766079, ++STORE, 140176014757888, 140176014766079, ++STORE, 140176010051584, 140176012570623, ++SNULL, 140176010051584, 140176010465279, ++STORE, 140176010465280, 140176012570623, ++STORE, 140176010051584, 140176010465279, ++SNULL, 140176012558335, 140176012570623, ++STORE, 140176010465280, 140176012558335, ++STORE, 140176012558336, 140176012570623, ++ERASE, 140176012558336, 140176012570623, ++STORE, 140176012558336, 140176012570623, ++STORE, 140176007417856, 140176010051583, ++SNULL, 140176007417856, 140176007946239, ++STORE, 140176007946240, 140176010051583, ++STORE, 140176007417856, 140176007946239, ++SNULL, 140176010043391, 140176010051583, ++STORE, 140176007946240, 140176010043391, ++STORE, 140176010043392, 140176010051583, ++ERASE, 140176010043392, 140176010051583, ++STORE, 140176010043392, 140176010051583, ++STORE, 140176005304320, 140176007417855, ++SNULL, 140176005304320, 140176005316607, ++STORE, 140176005316608, 140176007417855, ++STORE, 140176005304320, 140176005316607, ++SNULL, 140176007409663, 140176007417855, ++STORE, 140176005316608, 140176007409663, ++STORE, 140176007409664, 140176007417855, ++ERASE, 140176007409664, 140176007417855, ++STORE, 140176007409664, 140176007417855, ++STORE, 140176003100672, 140176005304319, ++SNULL, 140176003100672, 140176003203071, ++STORE, 140176003203072, 140176005304319, ++STORE, 140176003100672, 140176003203071, ++SNULL, 140176005296127, 140176005304319, ++STORE, 140176003203072, 140176005296127, ++STORE, 140176005296128, 140176005304319, ++ERASE, 140176005296128, 140176005304319, ++STORE, 140176005296128, 140176005304319, ++STORE, 140176020795392, 140176020811775, ++STORE, 140175999938560, 140176003100671, ++SNULL, 140175999938560, 140176000999423, ++STORE, 140176000999424, 140176003100671, ++STORE, 140175999938560, 140176000999423, ++SNULL, 140176003092479, 140176003100671, ++STORE, 140176000999424, 140176003092479, ++STORE, 140176003092480, 140176003100671, ++ERASE, 140176003092480, 140176003100671, ++STORE, 140176003092480, 140176003100671, ++STORE, 140175996141568, 140175999938559, ++SNULL, 140175996141568, 140175997800447, ++STORE, 140175997800448, 140175999938559, ++STORE, 140175996141568, 140175997800447, ++SNULL, 140175999897599, 140175999938559, ++STORE, 140175997800448, 140175999897599, ++STORE, 140175999897600, 140175999938559, ++SNULL, 140175999897600, 140175999922175, ++STORE, 140175999922176, 140175999938559, ++STORE, 140175999897600, 140175999922175, ++ERASE, 140175999897600, 140175999922175, ++STORE, 140175999897600, 140175999922175, ++ERASE, 140175999922176, 140175999938559, ++STORE, 140175999922176, 140175999938559, ++STORE, 140176020783104, 140176020811775, ++SNULL, 140175999913983, 140175999922175, ++STORE, 140175999897600, 140175999913983, ++STORE, 140175999913984, 140175999922175, ++SNULL, 140176003096575, 140176003100671, ++STORE, 140176003092480, 140176003096575, ++STORE, 140176003096576, 140176003100671, ++SNULL, 140176005300223, 140176005304319, ++STORE, 140176005296128, 140176005300223, ++STORE, 140176005300224, 140176005304319, ++SNULL, 140176007413759, 140176007417855, ++STORE, 140176007409664, 140176007413759, ++STORE, 140176007413760, 140176007417855, ++SNULL, 140176010047487, 140176010051583, ++STORE, 140176010043392, 140176010047487, ++STORE, 140176010047488, 140176010051583, ++SNULL, 140176012566527, 140176012570623, ++STORE, 140176012558336, 140176012566527, ++STORE, 140176012566528, 140176012570623, ++SNULL, 140176014761983, 140176014766079, ++STORE, 140176014757888, 140176014761983, ++STORE, 140176014761984, 140176014766079, ++SNULL, 140176018571263, 140176018599935, ++STORE, 140176018567168, 140176018571263, ++STORE, 140176018571264, 140176018599935, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 140176020844543, 140176020848639, ++STORE, 140176020840448, 140176020844543, ++STORE, 140176020844544, 140176020848639, ++ERASE, 140176020811776, 140176020840447, ++STORE, 53080064, 53215231, ++STORE, 140176019099648, 140176020783103, ++STORE, 140176020836352, 140176020840447, ++STORE, 140176018964480, 140176019099647, ++STORE, 53080064, 53358591, ++STORE, 140175994044416, 140175996141567, ++STORE, 140176020828160, 140176020840447, ++STORE, 140176020819968, 140176020840447, ++STORE, 140176020783104, 140176020819967, ++STORE, 140176018948096, 140176019099647, ++STORE, 53080064, 53493759, ++STORE, 53080064, 53649407, ++STORE, 140176018939904, 140176019099647, ++STORE, 140176018931712, 140176019099647, ++STORE, 53080064, 53784575, ++STORE, 53080064, 53919743, ++STORE, 140176018915328, 140176019099647, ++STORE, 140176018907136, 140176019099647, ++STORE, 53080064, 54059007, ++STORE, 140175993769984, 140175996141567, ++STORE, 140176018747392, 140176019099647, ++STORE, 53080064, 54198271, ++SNULL, 54190079, 54198271, ++STORE, 53080064, 54190079, ++STORE, 54190080, 54198271, ++ERASE, 54190080, 54198271, ++SNULL, 54181887, 54190079, ++STORE, 53080064, 54181887, ++STORE, 54181888, 54190079, ++ERASE, 54181888, 54190079, ++SNULL, 54173695, 54181887, ++STORE, 53080064, 54173695, ++STORE, 54173696, 54181887, ++ERASE, 54173696, 54181887, ++SNULL, 54165503, 54173695, ++STORE, 53080064, 54165503, ++STORE, 54165504, 54173695, ++ERASE, 54165504, 54173695, ++STORE, 140175993753600, 140175996141567, ++STORE, 140175993688064, 140175996141567, ++STORE, 140175993655296, 140175996141567, ++STORE, 140175991558144, 140175996141567, ++STORE, 140175991492608, 140175996141567, ++STORE, 53080064, 54312959, ++STORE, 140175991361536, 140175996141567, ++STORE, 140175991099392, 140175996141567, ++STORE, 140175991091200, 140175996141567, ++STORE, 140175991074816, 140175996141567, ++STORE, 140175991066624, 140175996141567, ++STORE, 140175991058432, 140175996141567, ++STORE, 53080064, 54448127, ++SNULL, 54439935, 54448127, ++STORE, 53080064, 54439935, ++STORE, 54439936, 54448127, ++ERASE, 54439936, 54448127, ++SNULL, 54431743, 54439935, ++STORE, 53080064, 54431743, ++STORE, 54431744, 54439935, ++ERASE, 54431744, 54439935, ++SNULL, 54419455, 54431743, ++STORE, 53080064, 54419455, ++STORE, 54419456, 54431743, ++ERASE, 54419456, 54431743, ++SNULL, 54403071, 54419455, ++STORE, 53080064, 54403071, ++STORE, 54403072, 54419455, ++ERASE, 54403072, 54419455, ++STORE, 140175991042048, 140175996141567, ++STORE, 53080064, 54538239, ++SNULL, 54534143, 54538239, ++STORE, 53080064, 54534143, ++STORE, 54534144, 54538239, ++ERASE, 54534144, 54538239, ++SNULL, 54530047, 54534143, ++STORE, 53080064, 54530047, ++STORE, 54530048, 54534143, ++ERASE, 54530048, 54534143, ++SNULL, 54525951, 54530047, ++STORE, 53080064, 54525951, ++STORE, 54525952, 54530047, ++ERASE, 54525952, 54530047, ++SNULL, 54521855, 54525951, ++STORE, 53080064, 54521855, ++STORE, 54521856, 54525951, ++ERASE, 54521856, 54525951, ++SNULL, 54517759, 54521855, ++STORE, 53080064, 54517759, ++STORE, 54517760, 54521855, ++ERASE, 54517760, 54521855, ++SNULL, 54513663, 54517759, ++STORE, 53080064, 54513663, ++STORE, 54513664, 54517759, ++ERASE, 54513664, 54517759, ++SNULL, 54509567, 54513663, ++STORE, 53080064, 54509567, ++STORE, 54509568, 54513663, ++ERASE, 54509568, 54513663, ++STORE, 140175991025664, 140175996141567, ++STORE, 140175990992896, 140175996141567, ++STORE, 53080064, 54644735, ++SNULL, 54628351, 54644735, ++STORE, 53080064, 54628351, ++STORE, 54628352, 54644735, ++ERASE, 54628352, 54644735, ++SNULL, 54616063, 54628351, ++STORE, 53080064, 54616063, ++STORE, 54616064, 54628351, ++ERASE, 54616064, 54628351, ++STORE, 140175988895744, 140175996141567, ++STORE, 53080064, 54767615, ++STORE, 140175988879360, 140175996141567, ++STORE, 140175988617216, 140175996141567, ++STORE, 140175988609024, 140175996141567, ++STORE, 140175988600832, 140175996141567, ++STORE, 53080064, 54906879, ++SNULL, 54898687, 54906879, ++STORE, 53080064, 54898687, ++STORE, 54898688, 54906879, ++ERASE, 54898688, 54906879, ++SNULL, 54853631, 54898687, ++STORE, 53080064, 54853631, ++STORE, 54853632, 54898687, ++ERASE, 54853632, 54898687, ++STORE, 140175986503680, 140175996141567, ++STORE, 53080064, 54996991, ++STORE, 140175986495488, 140175996141567, ++STORE, 140175986487296, 140175996141567, ++STORE, 140175985438720, 140175996141567, ++STORE, 53080064, 55136255, ++STORE, 140175985405952, 140175996141567, ++STORE, 140175985139712, 140175996141567, ++SNULL, 140176018964479, 140176019099647, ++STORE, 140176018747392, 140176018964479, ++STORE, 140176018964480, 140176019099647, ++ERASE, 140176018964480, 140176019099647, ++STORE, 140175983042560, 140175996141567, ++STORE, 140175982518272, 140175996141567, ++STORE, 140175980421120, 140175996141567, ++STORE, 53080064, 55287807, ++STORE, 53080064, 55427071, ++STORE, 140176019091456, 140176019099647, ++STORE, 140176019083264, 140176019099647, ++STORE, 140176019075072, 140176019099647, ++STORE, 140176019066880, 140176019099647, ++STORE, 140176019058688, 140176019099647, ++STORE, 140175980158976, 140175996141567, ++STORE, 140176019050496, 140176019099647, ++STORE, 140176019042304, 140176019099647, ++STORE, 140176019034112, 140176019099647, ++STORE, 140176019025920, 140176019099647, ++STORE, 140176019017728, 140176019099647, ++STORE, 140176019009536, 140176019099647, ++STORE, 140176019001344, 140176019099647, ++STORE, 140176018993152, 140176019099647, ++STORE, 140176018984960, 140176019099647, ++STORE, 140176018976768, 140176019099647, ++STORE, 140176018968576, 140176019099647, ++STORE, 140175978061824, 140175996141567, ++STORE, 53080064, 55603199, ++STORE, 140175978029056, 140175996141567, ++STORE, 140175977996288, 140175996141567, ++STORE, 53080064, 55738367, ++STORE, 53080064, 55881727, ++STORE, 140175977963520, 140175996141567, ++STORE, 140175977930752, 140175996141567, ++STORE, 53080064, 56041471, ++STORE, 140175977897984, 140175996141567, ++STORE, 140175977865216, 140175996141567, ++SNULL, 55881727, 56041471, ++STORE, 53080064, 55881727, ++STORE, 55881728, 56041471, ++ERASE, 55881728, 56041471, ++SNULL, 55721983, 55881727, ++STORE, 53080064, 55721983, ++STORE, 55721984, 55881727, ++ERASE, 55721984, 55881727, ++SNULL, 55570431, 55721983, ++STORE, 53080064, 55570431, ++STORE, 55570432, 55721983, ++ERASE, 55570432, 55721983, ++STORE, 140175977857024, 140175996141567, ++STORE, 140175975759872, 140175996141567, ++STORE, 53080064, 55754751, ++STORE, 53080064, 55943167, ++STORE, 140175975751680, 140175996141567, ++STORE, 140175975743488, 140175996141567, ++STORE, 140175975735296, 140175996141567, ++STORE, 140175975727104, 140175996141567, ++STORE, 140175975718912, 140175996141567, ++STORE, 140175975710720, 140175996141567, ++STORE, 140175975702528, 140175996141567, ++STORE, 140175975694336, 140175996141567, ++STORE, 140175975686144, 140175996141567, ++STORE, 140175975677952, 140175996141567, ++STORE, 140175975669760, 140175996141567, ++STORE, 140175974621184, 140175996141567, ++STORE, 140175974612992, 140175996141567, ++STORE, 53080064, 56139775, ++STORE, 140175972515840, 140175996141567, ++STORE, 53080064, 56401919, ++STORE, 140175970418688, 140175996141567, ++STORE, 140175970410496, 140175996141567, ++STORE, 140175970402304, 140175996141567, ++STORE, 140175970394112, 140175996141567, ++STORE, 53080064, 56569855, ++STORE, 140175969865728, 140175996141567, ++SNULL, 140175985139711, 140175996141567, ++STORE, 140175969865728, 140175985139711, ++STORE, 140175985139712, 140175996141567, ++SNULL, 140175985139712, 140175985405951, ++STORE, 140175985405952, 140175996141567, ++STORE, 140175985139712, 140175985405951, ++ERASE, 140175985139712, 140175985405951, ++STORE, 140175965671424, 140175985139711, ++STORE, 140175985397760, 140175996141567, ++STORE, 140175985389568, 140175996141567, ++STORE, 140175985381376, 140175996141567, ++STORE, 140175985373184, 140175996141567, ++STORE, 140175985364992, 140175996141567, ++STORE, 140175985356800, 140175996141567, ++STORE, 140175985348608, 140175996141567, ++STORE, 140175985340416, 140175996141567, ++STORE, 140175985332224, 140175996141567, ++STORE, 140175985324032, 140175996141567, ++STORE, 140175985315840, 140175996141567, ++STORE, 140175985307648, 140175996141567, ++STORE, 140175985299456, 140175996141567, ++STORE, 140175985291264, 140175996141567, ++STORE, 140175985283072, 140175996141567, ++STORE, 140175985274880, 140175996141567, ++STORE, 140175963574272, 140175985139711, ++STORE, 140175985266688, 140175996141567, ++STORE, 140175961477120, 140175985139711, ++STORE, 53080064, 56831999, ++STORE, 140175959379968, 140175985139711, ++STORE, 140175985258496, 140175996141567, ++STORE, 140175957282816, 140175985139711, ++STORE, 140175985250304, 140175996141567, ++STORE, 140175985242112, 140175996141567, ++STORE, 140175985233920, 140175996141567, ++STORE, 140175985225728, 140175996141567, ++STORE, 140175985217536, 140175996141567, ++STORE, 140175957151744, 140175985139711, ++STORE, 140175956627456, 140175985139711, ++SNULL, 140175980158975, 140175985139711, ++STORE, 140175956627456, 140175980158975, ++STORE, 140175980158976, 140175985139711, ++SNULL, 140175980158976, 140175980421119, ++STORE, 140175980421120, 140175985139711, ++STORE, 140175980158976, 140175980421119, ++ERASE, 140175980158976, 140175980421119, ++STORE, 140175954530304, 140175980158975, ++STORE, 140175985209344, 140175996141567, ++STORE, 53080064, 57094143, ++STORE, 140175952433152, 140175980158975, ++STORE, 140175985192960, 140175996141567, ++STORE, 140175985184768, 140175996141567, ++STORE, 140175985176576, 140175996141567, ++STORE, 140175985168384, 140175996141567, ++STORE, 140175985160192, 140175996141567, ++STORE, 140175985152000, 140175996141567, ++STORE, 140175985143808, 140175996141567, ++STORE, 140175980412928, 140175985139711, ++STORE, 140175980404736, 140175985139711, ++STORE, 140175980396544, 140175985139711, ++STORE, 140175980388352, 140175985139711, ++STORE, 140175980380160, 140175985139711, ++STORE, 140175980371968, 140175985139711, ++STORE, 140175980363776, 140175985139711, ++STORE, 140175980355584, 140175985139711, ++STORE, 140175980347392, 140175985139711, ++STORE, 140175980339200, 140175985139711, ++STORE, 53080064, 57356287, ++SNULL, 140176018747392, 140176018907135, ++STORE, 140176018907136, 140176018964479, ++STORE, 140176018747392, 140176018907135, ++ERASE, 140176018747392, 140176018907135, ++STORE, 140175952146432, 140175980158975, ++STORE, 140175950049280, 140175980158975, ++SNULL, 140175952146431, 140175980158975, ++STORE, 140175950049280, 140175952146431, ++STORE, 140175952146432, 140175980158975, ++SNULL, 140175952146432, 140175952433151, ++STORE, 140175952433152, 140175980158975, ++STORE, 140175952146432, 140175952433151, ++ERASE, 140175952146432, 140175952433151, ++STORE, 140176018898944, 140176018964479, ++STORE, 53080064, 57749503, ++STORE, 140175949520896, 140175952146431, ++STORE, 140175947423744, 140175952146431, ++SNULL, 140175993769983, 140175996141567, ++STORE, 140175985143808, 140175993769983, ++STORE, 140175993769984, 140175996141567, ++SNULL, 140175993769984, 140175994044415, ++STORE, 140175994044416, 140175996141567, ++STORE, 140175993769984, 140175994044415, ++ERASE, 140175993769984, 140175994044415, ++STORE, 140176018890752, 140176018964479, ++STORE, 140176018882560, 140176018964479, ++STORE, 140176018874368, 140176018964479, ++STORE, 140176018866176, 140176018964479, ++STORE, 140176018849792, 140176018964479, ++STORE, 140176018841600, 140176018964479, ++STORE, 140176018825216, 140176018964479, ++STORE, 140176018817024, 140176018964479, ++STORE, 140176018800640, 140176018964479, ++STORE, 140176018792448, 140176018964479, ++STORE, 140176018759680, 140176018964479, ++STORE, 140176018751488, 140176018964479, ++STORE, 140175994028032, 140175996141567, ++STORE, 140176018743296, 140176018964479, ++STORE, 140175994011648, 140175996141567, ++STORE, 140175994003456, 140175996141567, ++STORE, 140175993987072, 140175996141567, ++STORE, 140175993978880, 140175996141567, ++STORE, 140175993946112, 140175996141567, ++STORE, 140175993937920, 140175996141567, ++STORE, 140175993921536, 140175996141567, ++STORE, 140175993913344, 140175996141567, ++STORE, 140175993896960, 140175996141567, ++STORE, 140175993888768, 140175996141567, ++STORE, 140175993872384, 140175996141567, ++STORE, 140175993864192, 140175996141567, ++STORE, 140175993831424, 140175996141567, ++STORE, 140175993823232, 140175996141567, ++STORE, 140175993806848, 140175996141567, ++STORE, 140175993798656, 140175996141567, ++STORE, 140175993782272, 140175996141567, ++STORE, 140175993774080, 140175996141567, ++STORE, 140175980322816, 140175985139711, ++STORE, 140175980314624, 140175985139711, ++STORE, 140175980281856, 140175985139711, ++STORE, 140175980273664, 140175985139711, ++STORE, 140175980257280, 140175985139711, ++STORE, 140175945326592, 140175952146431, ++STORE, 140175980249088, 140175985139711, ++STORE, 140175980232704, 140175985139711, ++STORE, 140175980224512, 140175985139711, ++STORE, 140175980208128, 140175985139711, ++STORE, 140175980199936, 140175985139711, ++STORE, 140175980167168, 140175985139711, ++STORE, 140175952433152, 140175985139711, ++STORE, 140175952416768, 140175985139711, ++STORE, 140175952408576, 140175985139711, ++STORE, 140175952392192, 140175985139711, ++STORE, 140175952384000, 140175985139711, ++STORE, 140175952367616, 140175985139711, ++STORE, 140175943229440, 140175952146431, ++STORE, 140175952359424, 140175985139711, ++STORE, 140175952326656, 140175985139711, ++STORE, 140175952318464, 140175985139711, ++STORE, 140175952302080, 140175985139711, ++STORE, 140175952293888, 140175985139711, ++STORE, 140175952277504, 140175985139711, ++STORE, 140175952269312, 140175985139711, ++STORE, 140175952252928, 140175985139711, ++STORE, 140175952244736, 140175985139711, ++STORE, 140175952211968, 140175985139711, ++STORE, 140175952203776, 140175985139711, ++STORE, 140175952187392, 140175985139711, ++STORE, 140175952179200, 140175985139711, ++STORE, 140175952162816, 140175985139711, ++STORE, 140175952154624, 140175985139711, ++STORE, 140175943213056, 140175952146431, ++STORE, 140175943213056, 140175985139711, ++STORE, 140175943180288, 140175985139711, ++STORE, 140175943172096, 140175985139711, ++STORE, 140175943155712, 140175985139711, ++STORE, 140175943147520, 140175985139711, ++STORE, 140175943131136, 140175985139711, ++STORE, 140175943122944, 140175985139711, ++STORE, 140175943106560, 140175985139711, ++STORE, 140175943098368, 140175985139711, ++STORE, 140175943065600, 140175985139711, ++STORE, 140175943057408, 140175985139711, ++STORE, 140175943041024, 140175985139711, ++STORE, 140175943032832, 140175985139711, ++STORE, 140175943016448, 140175985139711, ++STORE, 140175943008256, 140175985139711, ++STORE, 140175942991872, 140175985139711, ++STORE, 140175942983680, 140175985139711, ++STORE, 140175942950912, 140175985139711, ++STORE, 140175942942720, 140175985139711, ++STORE, 140175942926336, 140175985139711, ++STORE, 140175942918144, 140175985139711, ++STORE, 140175942901760, 140175985139711, ++STORE, 140175942893568, 140175985139711, ++STORE, 140175942877184, 140175985139711, ++STORE, 140175942868992, 140175985139711, ++STORE, 140175942836224, 140175985139711, ++STORE, 140175942828032, 140175985139711, ++STORE, 140175942811648, 140175985139711, ++STORE, 140175942803456, 140175985139711, ++STORE, 140175942787072, 140175985139711, ++STORE, 140175942778880, 140175985139711, ++STORE, 140175942762496, 140175985139711, ++STORE, 140175942754304, 140175985139711, ++STORE, 140175942721536, 140175985139711, ++STORE, 140175942713344, 140175985139711, ++STORE, 140175942696960, 140175985139711, ++STORE, 140175942688768, 140175985139711, ++STORE, 140175942672384, 140175985139711, ++STORE, 140175942664192, 140175985139711, ++STORE, 140175942647808, 140175985139711, ++STORE, 140175942639616, 140175985139711, ++STORE, 140175942606848, 140175985139711, ++STORE, 140175942598656, 140175985139711, ++STORE, 140175942582272, 140175985139711, ++STORE, 140175942574080, 140175985139711, ++STORE, 140175942557696, 140175985139711, ++STORE, 140175942549504, 140175985139711, ++STORE, 140175942533120, 140175985139711, ++STORE, 140175942524928, 140175985139711, ++STORE, 140175942492160, 140175985139711, ++STORE, 140175942483968, 140175985139711, ++STORE, 140175942467584, 140175985139711, ++STORE, 140175942459392, 140175985139711, ++STORE, 140175942443008, 140175985139711, ++STORE, 140175942434816, 140175985139711, ++STORE, 140175942418432, 140175985139711, ++STORE, 140175942410240, 140175985139711, ++STORE, 140175942377472, 140175985139711, ++STORE, 140175942369280, 140175985139711, ++STORE, 140175942352896, 140175985139711, ++STORE, 140175942344704, 140175985139711, ++STORE, 140175942328320, 140175985139711, ++STORE, 140175942320128, 140175985139711, ++STORE, 140175942303744, 140175985139711, ++STORE, 140175942295552, 140175985139711, ++STORE, 140175942262784, 140175985139711, ++STORE, 140175942254592, 140175985139711, ++STORE, 140175942238208, 140175985139711, ++STORE, 140175942230016, 140175985139711, ++STORE, 140175942213632, 140175985139711, ++STORE, 140175942205440, 140175985139711, ++STORE, 140175942189056, 140175985139711, ++STORE, 140175942180864, 140175985139711, ++STORE, 140175942148096, 140175985139711, ++STORE, 140175942139904, 140175985139711, ++STORE, 140175942123520, 140175985139711, ++STORE, 140175942115328, 140175985139711, ++STORE, 140175942098944, 140175985139711, ++STORE, 140175942090752, 140175985139711, ++STORE, 140175942074368, 140175985139711, ++STORE, 140175942066176, 140175985139711, ++STORE, 140175942033408, 140175985139711, ++STORE, 140175942025216, 140175985139711, ++STORE, 140175942008832, 140175985139711, ++STORE, 140175942000640, 140175985139711, ++STORE, 140175941984256, 140175985139711, ++STORE, 140175941976064, 140175985139711, ++STORE, 140175941959680, 140175985139711, ++STORE, 140175939862528, 140175985139711, ++STORE, 140175939854336, 140175985139711, ++STORE, 140175939821568, 140175985139711, ++STORE, 140175939813376, 140175985139711, ++STORE, 140175939796992, 140175985139711, ++STORE, 140175939788800, 140175985139711, ++STORE, 140175939772416, 140175985139711, ++STORE, 140175939764224, 140175985139711, ++STORE, 140175939747840, 140175985139711, ++STORE, 140175939739648, 140175985139711, ++STORE, 140175939706880, 140175985139711, ++STORE, 140175939698688, 140175985139711, ++STORE, 140175939682304, 140175985139711, ++STORE, 140175939674112, 140175985139711, ++STORE, 140175939657728, 140175985139711, ++STORE, 140175939649536, 140175985139711, ++STORE, 140175939633152, 140175985139711, ++STORE, 140175939624960, 140175985139711, ++STORE, 140175939592192, 140175985139711, ++STORE, 140175939584000, 140175985139711, ++STORE, 140175939567616, 140175985139711, ++STORE, 140175939559424, 140175985139711, ++STORE, 140175939543040, 140175985139711, ++STORE, 140175939534848, 140175985139711, ++STORE, 140175939518464, 140175985139711, ++STORE, 140175939510272, 140175985139711, ++STORE, 140175939477504, 140175985139711, ++STORE, 140175939469312, 140175985139711, ++STORE, 140175939452928, 140175985139711, ++STORE, 140175939444736, 140175985139711, ++STORE, 140175939428352, 140175985139711, ++STORE, 140175939420160, 140175985139711, ++STORE, 140175939403776, 140175985139711, ++STORE, 140175939395584, 140175985139711, ++STORE, 140175939362816, 140175985139711, ++STORE, 140175939354624, 140175985139711, ++STORE, 140175939338240, 140175985139711, ++STORE, 140175939330048, 140175985139711, ++STORE, 140175939313664, 140175985139711, ++STORE, 140175939305472, 140175985139711, ++STORE, 140175939289088, 140175985139711, ++STORE, 140175939280896, 140175985139711, ++STORE, 140175939248128, 140175985139711, ++STORE, 140175939239936, 140175985139711, ++STORE, 140175939223552, 140175985139711, ++STORE, 140175939215360, 140175985139711, ++STORE, 140175939198976, 140175985139711, ++STORE, 140175939190784, 140175985139711, ++STORE, 140175939174400, 140175985139711, ++STORE, 140175939166208, 140175985139711, ++STORE, 140175939133440, 140175985139711, ++STORE, 140175939125248, 140175985139711, ++STORE, 140175939108864, 140175985139711, ++STORE, 140175939100672, 140175985139711, ++STORE, 140175939084288, 140175985139711, ++STORE, 140175939076096, 140175985139711, ++STORE, 140175939059712, 140175985139711, ++STORE, 140175939051520, 140175985139711, ++STORE, 140175939018752, 140175985139711, ++STORE, 140175939010560, 140175985139711, ++STORE, 140175938994176, 140175985139711, ++STORE, 140175938985984, 140175985139711, ++STORE, 140175938969600, 140175985139711, ++STORE, 140175938961408, 140175985139711, ++STORE, 140175938945024, 140175985139711, ++STORE, 140175938936832, 140175985139711, ++STORE, 140175938904064, 140175985139711, ++STORE, 140175938895872, 140175985139711, ++STORE, 140175938879488, 140175985139711, ++STORE, 140175938871296, 140175985139711, ++STORE, 140175938854912, 140175985139711, ++STORE, 140175938846720, 140175985139711, ++STORE, 140175938830336, 140175985139711, ++STORE, 140175938822144, 140175985139711, ++STORE, 140175938789376, 140175985139711, ++STORE, 140175938781184, 140175985139711, ++STORE, 140175938764800, 140175985139711, ++STORE, 140175938756608, 140175985139711, ++STORE, 140175938740224, 140175985139711, ++STORE, 140175938732032, 140175985139711, ++STORE, 140175938715648, 140175985139711, ++STORE, 140175938707456, 140175985139711, ++STORE, 140175938674688, 140175985139711, ++STORE, 140175938666496, 140175985139711, ++STORE, 140175938650112, 140175985139711, ++STORE, 140175938641920, 140175985139711, ++STORE, 140175938625536, 140175985139711, ++STORE, 140175938617344, 140175985139711, ++STORE, 140175938600960, 140175985139711, ++STORE, 140175938592768, 140175985139711, ++STORE, 140175938560000, 140175985139711, ++STORE, 140175938551808, 140175985139711, ++STORE, 140175938535424, 140175985139711, ++STORE, 140175938527232, 140175985139711, ++STORE, 140175938510848, 140175985139711, ++STORE, 140175938502656, 140175985139711, ++STORE, 140175938486272, 140175985139711, ++STORE, 140175938478080, 140175985139711, ++STORE, 140175938445312, 140175985139711, ++STORE, 140175938437120, 140175985139711, ++STORE, 140175938420736, 140175985139711, ++STORE, 140175938412544, 140175985139711, ++STORE, 140175938396160, 140175985139711, ++STORE, 140175938387968, 140175985139711, ++STORE, 140175938371584, 140175985139711, ++STORE, 140175938363392, 140175985139711, ++STORE, 140175938330624, 140175985139711, ++STORE, 140175938322432, 140175985139711, ++STORE, 140175938306048, 140175985139711, ++STORE, 140175938297856, 140175985139711, ++STORE, 140175938281472, 140175985139711, ++STORE, 140175938273280, 140175985139711, ++STORE, 140175938256896, 140175985139711, ++STORE, 140175938248704, 140175985139711, ++STORE, 140175938215936, 140175985139711, ++STORE, 140175938207744, 140175985139711, ++STORE, 140175938191360, 140175985139711, ++STORE, 140175938183168, 140175985139711, ++STORE, 140175938166784, 140175985139711, ++STORE, 140175938158592, 140175985139711, ++STORE, 140175938142208, 140175985139711, ++STORE, 140175936045056, 140175985139711, ++STORE, 140175936036864, 140175985139711, ++STORE, 140175936004096, 140175985139711, ++STORE, 140175935995904, 140175985139711, ++STORE, 140175935979520, 140175985139711, ++STORE, 140175935971328, 140175985139711, ++STORE, 140175935954944, 140175985139711, ++STORE, 140175935946752, 140175985139711, ++STORE, 140175935930368, 140175985139711, ++STORE, 140175935922176, 140175985139711, ++STORE, 140175935889408, 140175985139711, ++STORE, 140175935881216, 140175985139711, ++STORE, 140175935864832, 140175985139711, ++STORE, 140175935856640, 140175985139711, ++STORE, 140175935840256, 140175985139711, ++STORE, 140175935832064, 140175985139711, ++STORE, 140175935815680, 140175985139711, ++STORE, 140175935807488, 140175985139711, ++STORE, 140175935774720, 140175985139711, ++STORE, 140175935766528, 140175985139711, ++STORE, 140175935750144, 140175985139711, ++STORE, 140175935741952, 140175985139711, ++STORE, 140175935725568, 140175985139711, ++STORE, 140175935717376, 140175985139711, ++STORE, 140175935700992, 140175985139711, ++STORE, 140175935692800, 140175985139711, ++STORE, 140175935660032, 140175985139711, ++STORE, 140175935651840, 140175985139711, ++STORE, 140175935635456, 140175985139711, ++STORE, 140175935627264, 140175985139711, ++STORE, 140175935610880, 140175985139711, ++STORE, 140175935602688, 140175985139711, ++STORE, 140175935586304, 140175985139711, ++STORE, 140175935578112, 140175985139711, ++STORE, 140175935545344, 140175985139711, ++STORE, 140175935537152, 140175985139711, ++STORE, 140175935520768, 140175985139711, ++STORE, 140175935512576, 140175985139711, ++STORE, 140175935496192, 140175985139711, ++STORE, 140175935488000, 140175985139711, ++STORE, 140175935471616, 140175985139711, ++STORE, 140175935463424, 140175985139711, ++STORE, 140175935430656, 140175985139711, ++STORE, 140175935422464, 140175985139711, ++STORE, 140175935406080, 140175985139711, ++STORE, 140175935397888, 140175985139711, ++STORE, 140175935381504, 140175985139711, ++STORE, 140175935373312, 140175985139711, ++STORE, 140175935356928, 140175985139711, ++STORE, 140175935348736, 140175985139711, ++STORE, 140175935315968, 140175985139711, ++STORE, 140175935307776, 140175985139711, ++STORE, 140175935291392, 140175985139711, ++STORE, 140175935283200, 140175985139711, ++STORE, 140175935266816, 140175985139711, ++STORE, 140175935258624, 140175985139711, ++STORE, 140175935242240, 140175985139711, ++STORE, 140175935234048, 140175985139711, ++STORE, 140175935201280, 140175985139711, ++STORE, 140175935193088, 140175985139711, ++STORE, 140175935176704, 140175985139711, ++STORE, 140175935168512, 140175985139711, ++STORE, 140175935152128, 140175985139711, ++STORE, 140175935143936, 140175985139711, ++STORE, 140175935127552, 140175985139711, ++STORE, 140175935119360, 140175985139711, ++STORE, 140175935086592, 140175985139711, ++STORE, 140175935078400, 140175985139711, ++STORE, 140175935062016, 140175985139711, ++STORE, 140175935053824, 140175985139711, ++STORE, 140175935037440, 140175985139711, ++STORE, 140175935029248, 140175985139711, ++STORE, 140175935012864, 140175985139711, ++STORE, 140175935004672, 140175985139711, ++STORE, 140175934971904, 140175985139711, ++STORE, 140175934963712, 140175985139711, ++STORE, 140175934947328, 140175985139711, ++STORE, 140175934939136, 140175985139711, ++STORE, 140175934922752, 140175985139711, ++STORE, 140175934914560, 140175985139711, ++STORE, 140175934898176, 140175985139711, ++STORE, 140175934889984, 140175985139711, ++STORE, 140175934857216, 140175985139711, ++STORE, 140175934849024, 140175985139711, ++STORE, 140175934832640, 140175985139711, ++STORE, 140175934824448, 140175985139711, ++STORE, 140175934808064, 140175985139711, ++STORE, 140175934799872, 140175985139711, ++STORE, 140175934783488, 140175985139711, ++STORE, 140175934775296, 140175985139711, ++STORE, 140175934742528, 140175985139711, ++STORE, 140175934734336, 140175985139711, ++STORE, 140175934717952, 140175985139711, ++STORE, 140175934709760, 140175985139711, ++STORE, 140175934693376, 140175985139711, ++STORE, 140175934685184, 140175985139711, ++STORE, 140175934668800, 140175985139711, ++STORE, 140175934660608, 140175985139711, ++STORE, 140175934627840, 140175985139711, ++STORE, 140175934619648, 140175985139711, ++STORE, 140175934603264, 140175985139711, ++STORE, 140175934595072, 140175985139711, ++STORE, 140175934578688, 140175985139711, ++STORE, 140175934570496, 140175985139711, ++STORE, 140175934554112, 140175985139711, ++STORE, 140175934545920, 140175985139711, ++STORE, 140175934513152, 140175985139711, ++STORE, 140175934504960, 140175985139711, ++STORE, 140175934488576, 140175985139711, ++STORE, 140175934480384, 140175985139711, ++STORE, 140175934464000, 140175985139711, ++STORE, 140175934455808, 140175985139711, ++STORE, 140175934439424, 140175985139711, ++STORE, 140175934431232, 140175985139711, ++STORE, 140175934398464, 140175985139711, ++STORE, 140175934390272, 140175985139711, ++STORE, 140175934373888, 140175985139711, ++STORE, 140175934365696, 140175985139711, ++STORE, 140175934349312, 140175985139711, ++STORE, 140175934341120, 140175985139711, ++STORE, 140175934324736, 140175985139711, ++STORE, 140175932227584, 140175985139711, ++STORE, 140175932219392, 140175985139711, ++STORE, 140175932186624, 140175985139711, ++STORE, 140175932178432, 140175985139711, ++STORE, 140175932162048, 140175985139711, ++STORE, 140175932153856, 140175985139711, ++STORE, 140175932137472, 140175985139711, ++STORE, 53080064, 57884671, ++STORE, 140175932129280, 140175985139711, ++STORE, 140175932112896, 140175985139711, ++STORE, 140175932104704, 140175985139711, ++STORE, 140175932071936, 140175985139711, ++STORE, 140175932063744, 140175985139711, ++STORE, 140175932047360, 140175985139711, ++STORE, 140175932039168, 140175985139711, ++STORE, 140175932022784, 140175985139711, ++STORE, 140175932014592, 140175985139711, ++STORE, 140175931998208, 140175985139711, ++STORE, 140175931990016, 140175985139711, ++STORE, 140175931957248, 140175985139711, ++STORE, 140175931949056, 140175985139711, ++STORE, 140175931932672, 140175985139711, ++STORE, 140175931924480, 140175985139711, ++STORE, 140175931908096, 140175985139711, ++STORE, 140175931899904, 140175985139711, ++STORE, 140175931883520, 140175985139711, ++STORE, 140175931875328, 140175985139711, ++STORE, 140175931842560, 140175985139711, ++STORE, 140175931834368, 140175985139711, ++STORE, 140175931817984, 140175985139711, ++STORE, 140175931809792, 140175985139711, ++STORE, 140175931793408, 140175985139711, ++STORE, 140175931785216, 140175985139711, ++STORE, 140175931768832, 140175985139711, ++STORE, 140175931760640, 140175985139711, ++STORE, 140175931727872, 140175985139711, ++STORE, 140175931719680, 140175985139711, ++STORE, 140175931703296, 140175985139711, ++STORE, 140175931695104, 140175985139711, ++STORE, 140175931678720, 140175985139711, ++STORE, 140175931670528, 140175985139711, ++STORE, 140175931654144, 140175985139711, ++STORE, 140175931645952, 140175985139711, ++STORE, 140175931613184, 140175985139711, ++STORE, 140175931604992, 140175985139711, ++STORE, 140175931588608, 140175985139711, ++STORE, 140175931580416, 140175985139711, ++STORE, 140175931564032, 140175985139711, ++STORE, 140175931555840, 140175985139711, ++STORE, 140175931539456, 140175985139711, ++STORE, 140175931531264, 140175985139711, ++STORE, 140175931498496, 140175985139711, ++STORE, 140175931490304, 140175985139711, ++STORE, 140175931473920, 140175985139711, ++STORE, 140175931465728, 140175985139711, ++STORE, 140175931449344, 140175985139711, ++STORE, 140175931441152, 140175985139711, ++STORE, 140175931424768, 140175985139711, ++STORE, 140175931416576, 140175985139711, ++STORE, 140175931383808, 140175985139711, ++STORE, 140175931375616, 140175985139711, ++STORE, 140175931359232, 140175985139711, ++STORE, 140175931351040, 140175985139711, ++STORE, 140175931334656, 140175985139711, ++STORE, 140175931326464, 140175985139711, ++STORE, 140175931310080, 140175985139711, ++STORE, 140175931301888, 140175985139711, ++STORE, 140175931269120, 140175985139711, ++STORE, 140175931260928, 140175985139711, ++STORE, 140175931244544, 140175985139711, ++STORE, 140175931236352, 140175985139711, ++STORE, 140175931219968, 140175985139711, ++STORE, 140175931211776, 140175985139711, ++STORE, 140175931195392, 140175985139711, ++STORE, 140175931187200, 140175985139711, ++STORE, 140175931154432, 140175985139711, ++STORE, 140175931146240, 140175985139711, ++STORE, 140175931129856, 140175985139711, ++STORE, 140175931121664, 140175985139711, ++STORE, 140175931105280, 140175985139711, ++STORE, 140175931097088, 140175985139711, ++STORE, 140175931080704, 140175985139711, ++STORE, 140175931072512, 140175985139711, ++STORE, 140175931039744, 140175985139711, ++STORE, 140175931031552, 140175985139711, ++STORE, 140175931015168, 140175985139711, ++STORE, 140175931006976, 140175985139711, ++STORE, 140175930990592, 140175985139711, ++STORE, 140175930982400, 140175985139711, ++STORE, 140175930966016, 140175985139711, ++STORE, 140175930957824, 140175985139711, ++STORE, 140175930925056, 140175985139711, ++STORE, 140175930916864, 140175985139711, ++STORE, 140175930900480, 140175985139711, ++STORE, 140175930892288, 140175985139711, ++STORE, 140175930875904, 140175985139711, ++STORE, 140175930867712, 140175985139711, ++STORE, 140175930851328, 140175985139711, ++STORE, 140175930843136, 140175985139711, ++STORE, 140175930810368, 140175985139711, ++STORE, 140175930802176, 140175985139711, ++STORE, 140175930785792, 140175985139711, ++STORE, 140175930777600, 140175985139711, ++STORE, 140175930761216, 140175985139711, ++STORE, 140175930753024, 140175985139711, ++STORE, 140175930736640, 140175985139711, ++STORE, 140175930728448, 140175985139711, ++STORE, 140175930695680, 140175985139711, ++STORE, 140175930687488, 140175985139711, ++STORE, 140175930671104, 140175985139711, ++STORE, 140175930662912, 140175985139711, ++STORE, 140175930646528, 140175985139711, ++STORE, 140175930638336, 140175985139711, ++STORE, 140175930621952, 140175985139711, ++STORE, 140175930613760, 140175985139711, ++STORE, 140175930580992, 140175985139711, ++STORE, 140175930572800, 140175985139711, ++STORE, 140175930556416, 140175985139711, ++STORE, 140175930548224, 140175985139711, ++STORE, 140175930531840, 140175985139711, ++STORE, 140175930523648, 140175985139711, ++STORE, 140175930507264, 140175985139711, ++STORE, 140175928410112, 140175985139711, ++STORE, 140175928401920, 140175985139711, ++STORE, 140175928369152, 140175985139711, ++STORE, 140175928360960, 140175985139711, ++STORE, 140175928344576, 140175985139711, ++STORE, 140175928336384, 140175985139711, ++STORE, 140175928320000, 140175985139711, ++STORE, 140175928311808, 140175985139711, ++STORE, 140175928295424, 140175985139711, ++STORE, 140175927242752, 140175985139711, ++SNULL, 140175956627455, 140175985139711, ++STORE, 140175927242752, 140175956627455, ++STORE, 140175956627456, 140175985139711, ++ }; ++ unsigned long set24[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140735281639424, 140737488351231, ++SNULL, 140735281643519, 140737488351231, ++STORE, 140735281639424, 140735281643519, ++STORE, 140735281508352, 140735281643519, ++STORE, 94717834911744, 94717834928127, ++SNULL, 94717834915839, 94717834928127, ++STORE, 94717834911744, 94717834915839, ++STORE, 94717834915840, 94717834928127, ++ERASE, 94717834915840, 94717834928127, ++STORE, 94717834919936, 94717834928127, ++STORE, 140428246065152, 140428248317951, ++SNULL, 140428246208511, 140428248317951, ++STORE, 140428246065152, 140428246208511, ++STORE, 140428246208512, 140428248317951, ++ERASE, 140428246208512, 140428248317951, ++STORE, 140428248305664, 140428248313855, ++STORE, 140428248313856, 140428248317951, ++STORE, 140735281811456, 140735281815551, ++STORE, 140735281799168, 140735281811455, ++STORE, 140428248297472, 140428248305663, ++STORE, 140428243841024, 140428246065151, ++SNULL, 140428245491711, 140428246065151, ++STORE, 140428243841024, 140428245491711, ++STORE, 140428245491712, 140428246065151, ++SNULL, 140428245491712, 140428246061055, ++STORE, 140428246061056, 140428246065151, ++STORE, 140428245491712, 140428246061055, ++ERASE, 140428245491712, 140428246061055, ++STORE, 140428245491712, 140428246061055, ++ERASE, 140428246061056, 140428246065151, ++STORE, 140428246061056, 140428246065151, ++STORE, 140428248268800, 140428248297471, ++STORE, 140428241625088, 140428243841023, ++SNULL, 140428241625088, 140428241723391, ++STORE, 140428241723392, 140428243841023, ++STORE, 140428241625088, 140428241723391, ++SNULL, 140428243816447, 140428243841023, ++STORE, 140428241723392, 140428243816447, ++STORE, 140428243816448, 140428243841023, ++SNULL, 140428243816448, 140428243824639, ++STORE, 140428243824640, 140428243841023, ++STORE, 140428243816448, 140428243824639, ++ERASE, 140428243816448, 140428243824639, ++STORE, 140428243816448, 140428243824639, ++ERASE, 140428243824640, 140428243841023, ++STORE, 140428243824640, 140428243841023, ++STORE, 140428237828096, 140428241625087, ++SNULL, 140428237828096, 140428239486975, ++STORE, 140428239486976, 140428241625087, ++STORE, 140428237828096, 140428239486975, ++SNULL, 140428241584127, 140428241625087, ++STORE, 140428239486976, 140428241584127, ++STORE, 140428241584128, 140428241625087, ++SNULL, 140428241584128, 140428241608703, ++STORE, 140428241608704, 140428241625087, ++STORE, 140428241584128, 140428241608703, ++ERASE, 140428241584128, 140428241608703, ++STORE, 140428241584128, 140428241608703, ++ERASE, 140428241608704, 140428241625087, ++STORE, 140428241608704, 140428241625087, ++STORE, 140428235567104, 140428237828095, ++SNULL, 140428235567104, 140428235718655, ++STORE, 140428235718656, 140428237828095, ++STORE, 140428235567104, 140428235718655, ++SNULL, 140428237811711, 140428237828095, ++STORE, 140428235718656, 140428237811711, ++STORE, 140428237811712, 140428237828095, ++SNULL, 140428237811712, 140428237819903, ++STORE, 140428237819904, 140428237828095, ++STORE, 140428237811712, 140428237819903, ++ERASE, 140428237811712, 140428237819903, ++STORE, 140428237811712, 140428237819903, ++ERASE, 140428237819904, 140428237828095, ++STORE, 140428237819904, 140428237828095, ++STORE, 140428233445376, 140428235567103, ++SNULL, 140428233445376, 140428233461759, ++STORE, 140428233461760, 140428235567103, ++STORE, 140428233445376, 140428233461759, ++SNULL, 140428235558911, 140428235567103, ++STORE, 140428233461760, 140428235558911, ++STORE, 140428235558912, 140428235567103, ++ERASE, 140428235558912, 140428235567103, ++STORE, 140428235558912, 140428235567103, ++STORE, 140428231315456, 140428233445375, ++SNULL, 140428231315456, 140428231344127, ++STORE, 140428231344128, 140428233445375, ++STORE, 140428231315456, 140428231344127, ++SNULL, 140428233437183, 140428233445375, ++STORE, 140428231344128, 140428233437183, ++STORE, 140428233437184, 140428233445375, ++ERASE, 140428233437184, 140428233445375, ++STORE, 140428233437184, 140428233445375, ++STORE, 140428248260608, 140428248268799, ++STORE, 140428229062656, 140428231315455, ++SNULL, 140428229062656, 140428229214207, ++STORE, 140428229214208, 140428231315455, ++STORE, 140428229062656, 140428229214207, ++SNULL, 140428231307263, 140428231315455, ++STORE, 140428229214208, 140428231307263, ++STORE, 140428231307264, 140428231315455, ++ERASE, 140428231307264, 140428231315455, ++STORE, 140428231307264, 140428231315455, ++STORE, 140428226891776, 140428229062655, ++SNULL, 140428226891776, 140428226961407, ++STORE, 140428226961408, 140428229062655, ++STORE, 140428226891776, 140428226961407, ++SNULL, 140428229054463, 140428229062655, ++STORE, 140428226961408, 140428229054463, ++STORE, 140428229054464, 140428229062655, ++ERASE, 140428229054464, 140428229062655, ++STORE, 140428229054464, 140428229062655, ++STORE, 140428223680512, 140428226891775, ++SNULL, 140428223680512, 140428224757759, ++STORE, 140428224757760, 140428226891775, ++STORE, 140428223680512, 140428224757759, ++SNULL, 140428226854911, 140428226891775, ++STORE, 140428224757760, 140428226854911, ++STORE, 140428226854912, 140428226891775, ++ERASE, 140428226854912, 140428226891775, ++STORE, 140428226854912, 140428226891775, ++STORE, 140428221546496, 140428223680511, ++SNULL, 140428221546496, 140428221575167, ++STORE, 140428221575168, 140428223680511, ++STORE, 140428221546496, 140428221575167, ++SNULL, 140428223672319, 140428223680511, ++STORE, 140428221575168, 140428223672319, ++STORE, 140428223672320, 140428223680511, ++ERASE, 140428223672320, 140428223680511, ++STORE, 140428223672320, 140428223680511, ++STORE, 140428219236352, 140428221546495, ++SNULL, 140428219236352, 140428219441151, ++STORE, 140428219441152, 140428221546495, ++STORE, 140428219236352, 140428219441151, ++SNULL, 140428221538303, 140428221546495, ++STORE, 140428219441152, 140428221538303, ++STORE, 140428221538304, 140428221546495, ++ERASE, 140428221538304, 140428221546495, ++STORE, 140428221538304, 140428221546495, ++STORE, 140428216852480, 140428219236351, ++SNULL, 140428216852480, 140428217044991, ++STORE, 140428217044992, 140428219236351, ++STORE, 140428216852480, 140428217044991, ++SNULL, 140428219138047, 140428219236351, ++STORE, 140428217044992, 140428219138047, ++STORE, 140428219138048, 140428219236351, ++ERASE, 140428219138048, 140428219236351, ++STORE, 140428219138048, 140428219236351, ++STORE, 140428248252416, 140428248268799, ++STORE, 140428214284288, 140428216852479, ++SNULL, 140428214284288, 140428214751231, ++STORE, 140428214751232, 140428216852479, ++STORE, 140428214284288, 140428214751231, ++SNULL, 140428216844287, 140428216852479, ++STORE, 140428214751232, 140428216844287, ++STORE, 140428216844288, 140428216852479, ++ERASE, 140428216844288, 140428216852479, ++STORE, 140428216844288, 140428216852479, ++STORE, 140428212170752, 140428214284287, ++SNULL, 140428212170752, 140428212183039, ++STORE, 140428212183040, 140428214284287, ++STORE, 140428212170752, 140428212183039, ++SNULL, 140428214276095, 140428214284287, ++STORE, 140428212183040, 140428214276095, ++STORE, 140428214276096, 140428214284287, ++ERASE, 140428214276096, 140428214284287, ++STORE, 140428214276096, 140428214284287, ++STORE, 140428209991680, 140428212170751, ++SNULL, 140428209991680, 140428210069503, ++STORE, 140428210069504, 140428212170751, ++STORE, 140428209991680, 140428210069503, ++SNULL, 140428212162559, 140428212170751, ++STORE, 140428210069504, 140428212162559, ++STORE, 140428212162560, 140428212170751, ++ERASE, 140428212162560, 140428212170751, ++STORE, 140428212162560, 140428212170751, ++STORE, 140428207874048, 140428209991679, ++SNULL, 140428207874048, 140428207890431, ++STORE, 140428207890432, 140428209991679, ++STORE, 140428207874048, 140428207890431, ++SNULL, 140428209983487, 140428209991679, ++STORE, 140428207890432, 140428209983487, ++STORE, 140428209983488, 140428209991679, ++ERASE, 140428209983488, 140428209991679, ++STORE, 140428209983488, 140428209991679, ++STORE, 140428248244224, 140428248268799, ++STORE, 140428248231936, 140428248268799, ++SNULL, 140428241600511, 140428241608703, ++STORE, 140428241584128, 140428241600511, ++STORE, 140428241600512, 140428241608703, ++SNULL, 140428209987583, 140428209991679, ++STORE, 140428209983488, 140428209987583, ++STORE, 140428209987584, 140428209991679, ++SNULL, 140428212166655, 140428212170751, ++STORE, 140428212162560, 140428212166655, ++STORE, 140428212166656, 140428212170751, ++SNULL, 140428214280191, 140428214284287, ++STORE, 140428214276096, 140428214280191, ++STORE, 140428214280192, 140428214284287, ++SNULL, 140428243820543, 140428243824639, ++STORE, 140428243816448, 140428243820543, ++STORE, 140428243820544, 140428243824639, ++SNULL, 140428216848383, 140428216852479, ++STORE, 140428216844288, 140428216848383, ++STORE, 140428216848384, 140428216852479, ++SNULL, 140428219232255, 140428219236351, ++STORE, 140428219138048, 140428219232255, ++STORE, 140428219232256, 140428219236351, ++SNULL, 140428221542399, 140428221546495, ++STORE, 140428221538304, 140428221542399, ++STORE, 140428221542400, 140428221546495, ++SNULL, 140428223676415, 140428223680511, ++STORE, 140428223672320, 140428223676415, ++STORE, 140428223676416, 140428223680511, ++SNULL, 140428226863103, 140428226891775, ++STORE, 140428226854912, 140428226863103, ++STORE, 140428226863104, 140428226891775, ++SNULL, 140428229058559, 140428229062655, ++STORE, 140428229054464, 140428229058559, ++STORE, 140428229058560, 140428229062655, ++SNULL, 140428231311359, 140428231315455, ++STORE, 140428231307264, 140428231311359, ++STORE, 140428231311360, 140428231315455, ++SNULL, 140428233441279, 140428233445375, ++STORE, 140428233437184, 140428233441279, ++STORE, 140428233441280, 140428233445375, ++SNULL, 140428235563007, 140428235567103, ++STORE, 140428235558912, 140428235563007, ++STORE, 140428235563008, 140428235567103, ++SNULL, 140428237815807, 140428237819903, ++STORE, 140428237811712, 140428237815807, ++STORE, 140428237815808, 140428237819903, ++SNULL, 140428246056959, 140428246061055, ++STORE, 140428245491712, 140428246056959, ++STORE, 140428246056960, 140428246061055, ++SNULL, 94717834924031, 94717834928127, ++STORE, 94717834919936, 94717834924031, ++STORE, 94717834924032, 94717834928127, ++SNULL, 140428248309759, 140428248313855, ++STORE, 140428248305664, 140428248309759, ++STORE, 140428248309760, 140428248313855, ++ERASE, 140428248268800, 140428248297471, ++STORE, 94717843058688, 94717843193855, ++STORE, 94749677137920, 94749677559807, ++STORE, 94749677563904, 94749677604863, ++STORE, 94749677604864, 94749677608959, ++STORE, 94749710970880, 94749711241215, ++STORE, 140490884894720, 140490884935679, ++STORE, 140490884935680, 140490887032831, ++STORE, 140490887032832, 140490887036927, ++STORE, 140490887036928, 140490887041023, ++STORE, 140490887041024, 140490887065599, ++STORE, 140490887065600, 140490887110655, ++STORE, 140490887110656, 140490889203711, ++STORE, 140490889203712, 140490889207807, ++STORE, 140490889207808, 140490889211903, ++STORE, 140490889211904, 140490889293823, ++STORE, 140490889293824, 140490891390975, ++STORE, 140490891390976, 140490891395071, ++STORE, 140490891395072, 140490891399167, ++STORE, 140490891399168, 140490891407359, ++STORE, 140490891407360, 140490891436031, ++STORE, 140490891436032, 140490893529087, ++STORE, 140490893529088, 140490893533183, ++STORE, 140490893533184, 140490893537279, ++STORE, 140490893537280, 140490901979135, ++STORE, 140490901979136, 140490901991423, ++STORE, 140490901991424, 140490904084479, ++STORE, 140490904084480, 140490904088575, ++STORE, 140490904088576, 140490904092671, ++STORE, 140490904092672, 140490904559615, ++STORE, 140490904559616, 140490906652671, ++STORE, 140490906652672, 140490906656767, ++STORE, 140490906656768, 140490906660863, ++STORE, 140490906660864, 140490906677247, ++STORE, 140490906677248, 140490908770303, ++STORE, 140490908770304, 140490908774399, ++STORE, 140490908774400, 140490908778495, ++STORE, 140490908778496, 140490908794879, ++STORE, 140490908794880, 140490910887935, ++STORE, 140490910887936, 140490910892031, ++STORE, 140490910892032, 140490910896127, ++STORE, 140490910896128, 140490912555007, ++STORE, 140490912555008, 140490914652159, ++STORE, 140490914652160, 140490914668543, ++STORE, 140490914668544, 140490914676735, ++STORE, 140490914676736, 140490914693119, ++STORE, 140490914693120, 140490914791423, ++STORE, 140490914791424, 140490916884479, ++STORE, 140490916884480, 140490916888575, ++STORE, 140490916888576, 140490916892671, ++STORE, 140490916892672, 140490916909055, ++STORE, 140490916909056, 140490916937727, ++STORE, 140490916937728, 140490919030783, ++STORE, 140490919030784, 140490919034879, ++STORE, 140490919034880, 140490919038975, ++STORE, 140490919038976, 140490919190527, ++STORE, 140490919190528, 140490921283583, ++STORE, 140490921283584, 140490921287679, ++STORE, 140490921287680, 140490921291775, ++STORE, 140490921291776, 140490921299967, ++STORE, 140490921299968, 140490921390079, ++STORE, 140490921390080, 140490923483135, ++STORE, 140490923483136, 140490923487231, ++STORE, 140490923487232, 140490923491327, ++STORE, 140490923491328, 140490923757567, ++STORE, 140490923757568, 140490925850623, ++STORE, 140490925850624, 140490925867007, ++STORE, 140490925867008, 140490925871103, ++STORE, 140490925871104, 140490925875199, ++STORE, 140490925875200, 140490925903871, ++STORE, 140490925903872, 140490928001023, ++STORE, 140490928001024, 140490928005119, ++STORE, 140490928005120, 140490928009215, ++STORE, 140490928009216, 140490928152575, ++STORE, 140490930184192, 140490930221055, ++STORE, 140490930221056, 140490930237439, ++STORE, 140490930237440, 140490930241535, ++STORE, 140490930241536, 140490930245631, ++STORE, 140490930245632, 140490930249727, ++STORE, 140490930249728, 140490930253823, ++STORE, 140490930253824, 140490930257919, ++STORE, 140490930257920, 140490930262015, ++STORE, 140724611694592, 140724611829759, ++STORE, 140724612427776, 140724612440063, ++STORE, 140724612440064, 140724612444159, ++STORE, 94103163662336, 94103163772927, ++STORE, 94103165865984, 94103165874175, ++STORE, 94103165874176, 94103165878271, ++STORE, 94103165878272, 94103165886463, ++STORE, 94103182548992, 94103182684159, ++STORE, 140092694708224, 140092696367103, ++STORE, 140092696367104, 140092698464255, ++STORE, 140092698464256, 140092698480639, ++STORE, 140092698480640, 140092698488831, ++STORE, 140092698488832, 140092698505215, ++STORE, 140092698505216, 140092698648575, ++STORE, 140092700708864, 140092700717055, ++STORE, 140092700745728, 140092700749823, ++STORE, 140092700749824, 140092700753919, ++STORE, 140092700753920, 140092700758015, ++STORE, 140736800911360, 140736801046527, ++STORE, 140736802308096, 140736802320383, ++STORE, 140736802320384, 140736802324479, ++STORE, 93948802064384, 93948802174975, ++STORE, 93948804268032, 93948804276223, ++STORE, 93948804276224, 93948804280319, ++STORE, 93948804280320, 93948804288511, ++STORE, 93948806266880, 93948806402047, ++STORE, 140222999113728, 140223000772607, ++STORE, 140223000772608, 140223002869759, ++STORE, 140223002869760, 140223002886143, ++STORE, 140223002886144, 140223002894335, ++STORE, 140223002894336, 140223002910719, ++STORE, 140223002910720, 140223003054079, ++STORE, 140223005114368, 140223005122559, ++STORE, 140223005151232, 140223005155327, ++STORE, 140223005155328, 140223005159423, ++STORE, 140223005159424, 140223005163519, ++STORE, 140720877506560, 140720877641727, ++STORE, 140720878231552, 140720878243839, ++STORE, 140720878243840, 140720878247935, ++STORE, 140737488347136, 140737488351231, ++STORE, 140733232087040, 140737488351231, ++SNULL, 140733232091135, 140737488351231, ++STORE, 140733232087040, 140733232091135, ++STORE, 140733231955968, 140733232091135, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140161681321984, 140161683574783, ++SNULL, 140161681465343, 140161683574783, ++STORE, 140161681321984, 140161681465343, ++STORE, 140161681465344, 140161683574783, ++ERASE, 140161681465344, 140161683574783, ++STORE, 140161683562496, 140161683570687, ++STORE, 140161683570688, 140161683574783, ++STORE, 140733232214016, 140733232218111, ++STORE, 140733232201728, 140733232214015, ++STORE, 140161683533824, 140161683562495, ++STORE, 140161683525632, 140161683533823, ++STORE, 140161678159872, 140161681321983, ++SNULL, 140161678159872, 140161679220735, ++STORE, 140161679220736, 140161681321983, ++STORE, 140161678159872, 140161679220735, ++SNULL, 140161681313791, 140161681321983, ++STORE, 140161679220736, 140161681313791, ++STORE, 140161681313792, 140161681321983, ++ERASE, 140161681313792, 140161681321983, ++STORE, 140161681313792, 140161681321983, ++STORE, 140161674362880, 140161678159871, ++SNULL, 140161674362880, 140161676021759, ++STORE, 140161676021760, 140161678159871, ++STORE, 140161674362880, 140161676021759, ++SNULL, 140161678118911, 140161678159871, ++STORE, 140161676021760, 140161678118911, ++STORE, 140161678118912, 140161678159871, ++SNULL, 140161678118912, 140161678143487, ++STORE, 140161678143488, 140161678159871, ++STORE, 140161678118912, 140161678143487, ++ERASE, 140161678118912, 140161678143487, ++STORE, 140161678118912, 140161678143487, ++ERASE, 140161678143488, 140161678159871, ++STORE, 140161678143488, 140161678159871, ++STORE, 140161683513344, 140161683533823, ++SNULL, 140161678135295, 140161678143487, ++STORE, 140161678118912, 140161678135295, ++STORE, 140161678135296, 140161678143487, ++SNULL, 140161681317887, 140161681321983, ++STORE, 140161681313792, 140161681317887, ++STORE, 140161681317888, 140161681321983, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140161683566591, 140161683570687, ++STORE, 140161683562496, 140161683566591, ++STORE, 140161683566592, 140161683570687, ++ERASE, 140161683533824, 140161683562495, ++STORE, 25477120, 25612287, ++STORE, 25477120, 25759743, ++STORE, 140161681829888, 140161683513343, ++STORE, 25477120, 25915391, ++STORE, 25477120, 26054655, ++SNULL, 25800703, 26054655, ++STORE, 25477120, 25800703, ++STORE, 25800704, 26054655, ++ERASE, 25800704, 26054655, ++STORE, 140737488347136, 140737488351231, ++STORE, 140723218452480, 140737488351231, ++SNULL, 140723218456575, 140737488351231, ++STORE, 140723218452480, 140723218456575, ++STORE, 140723218321408, 140723218456575, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 140398872264704, 140398874517503, ++SNULL, 140398872408063, 140398874517503, ++STORE, 140398872264704, 140398872408063, ++STORE, 140398872408064, 140398874517503, ++ERASE, 140398872408064, 140398874517503, ++STORE, 140398874505216, 140398874513407, ++STORE, 140398874513408, 140398874517503, ++STORE, 140723219247104, 140723219251199, ++STORE, 140723219234816, 140723219247103, ++STORE, 140398874476544, 140398874505215, ++STORE, 140398874468352, 140398874476543, ++STORE, 140398868430848, 140398872264703, ++SNULL, 140398868430848, 140398870138879, ++STORE, 140398870138880, 140398872264703, ++STORE, 140398868430848, 140398870138879, ++SNULL, 140398872231935, 140398872264703, ++STORE, 140398870138880, 140398872231935, ++STORE, 140398872231936, 140398872264703, ++ERASE, 140398872231936, 140398872264703, ++STORE, 140398872231936, 140398872264703, ++STORE, 140398866235392, 140398868430847, ++SNULL, 140398866235392, 140398866329599, ++STORE, 140398866329600, 140398868430847, ++STORE, 140398866235392, 140398866329599, ++SNULL, 140398868422655, 140398868430847, ++STORE, 140398866329600, 140398868422655, ++STORE, 140398868422656, 140398868430847, ++ERASE, 140398868422656, 140398868430847, ++STORE, 140398868422656, 140398868430847, ++STORE, 140398863716352, 140398866235391, ++SNULL, 140398863716352, 140398864130047, ++STORE, 140398864130048, 140398866235391, ++STORE, 140398863716352, 140398864130047, ++SNULL, 140398866223103, 140398866235391, ++STORE, 140398864130048, 140398866223103, ++STORE, 140398866223104, 140398866235391, ++ERASE, 140398866223104, 140398866235391, ++STORE, 140398866223104, 140398866235391, ++STORE, 140398861082624, 140398863716351, ++SNULL, 140398861082624, 140398861611007, ++STORE, 140398861611008, 140398863716351, ++STORE, 140398861082624, 140398861611007, ++SNULL, 140398863708159, 140398863716351, ++STORE, 140398861611008, 140398863708159, ++STORE, 140398863708160, 140398863716351, ++ERASE, 140398863708160, 140398863716351, ++STORE, 140398863708160, 140398863716351, ++STORE, 140398858969088, 140398861082623, ++SNULL, 140398858969088, 140398858981375, ++STORE, 140398858981376, 140398861082623, ++STORE, 140398858969088, 140398858981375, ++SNULL, 140398861074431, 140398861082623, ++STORE, 140398858981376, 140398861074431, ++STORE, 140398861074432, 140398861082623, ++ERASE, 140398861074432, 140398861082623, ++STORE, 140398861074432, 140398861082623, ++STORE, 140398856765440, 140398858969087, ++SNULL, 140398856765440, 140398856867839, ++STORE, 140398856867840, 140398858969087, ++STORE, 140398856765440, 140398856867839, ++SNULL, 140398858960895, 140398858969087, ++STORE, 140398856867840, 140398858960895, ++STORE, 140398858960896, 140398858969087, ++ERASE, 140398858960896, 140398858969087, ++STORE, 140398858960896, 140398858969087, ++STORE, 140398874460160, 140398874476543, ++STORE, 140398853603328, 140398856765439, ++SNULL, 140398853603328, 140398854664191, ++STORE, 140398854664192, 140398856765439, ++STORE, 140398853603328, 140398854664191, ++SNULL, 140398856757247, 140398856765439, ++STORE, 140398854664192, 140398856757247, ++STORE, 140398856757248, 140398856765439, ++ERASE, 140398856757248, 140398856765439, ++STORE, 140398856757248, 140398856765439, ++STORE, 140398849806336, 140398853603327, ++SNULL, 140398849806336, 140398851465215, ++STORE, 140398851465216, 140398853603327, ++STORE, 140398849806336, 140398851465215, ++SNULL, 140398853562367, 140398853603327, ++STORE, 140398851465216, 140398853562367, ++STORE, 140398853562368, 140398853603327, ++SNULL, 140398853562368, 140398853586943, ++STORE, 140398853586944, 140398853603327, ++STORE, 140398853562368, 140398853586943, ++ERASE, 140398853562368, 140398853586943, ++STORE, 140398853562368, 140398853586943, ++ERASE, 140398853586944, 140398853603327, ++STORE, 140398853586944, 140398853603327, ++STORE, 140398874447872, 140398874476543, ++SNULL, 140398853578751, 140398853586943, ++STORE, 140398853562368, 140398853578751, ++STORE, 140398853578752, 140398853586943, ++SNULL, 140398856761343, 140398856765439, ++STORE, 140398856757248, 140398856761343, ++STORE, 140398856761344, 140398856765439, ++SNULL, 140398858964991, 140398858969087, ++STORE, 140398858960896, 140398858964991, ++STORE, 140398858964992, 140398858969087, ++SNULL, 140398861078527, 140398861082623, ++STORE, 140398861074432, 140398861078527, ++STORE, 140398861078528, 140398861082623, ++SNULL, 140398863712255, 140398863716351, ++STORE, 140398863708160, 140398863712255, ++STORE, 140398863712256, 140398863716351, ++SNULL, 140398866231295, 140398866235391, ++STORE, 140398866223104, 140398866231295, ++STORE, 140398866231296, 140398866235391, ++SNULL, 140398868426751, 140398868430847, ++STORE, 140398868422656, 140398868426751, ++STORE, 140398868426752, 140398868430847, ++SNULL, 140398872236031, 140398872264703, ++STORE, 140398872231936, 140398872236031, ++STORE, 140398872236032, 140398872264703, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 140398874509311, 140398874513407, ++STORE, 140398874505216, 140398874509311, ++STORE, 140398874509312, 140398874513407, ++ERASE, 140398874476544, 140398874505215, ++STORE, 43278336, 43413503, ++STORE, 140398872764416, 140398874447871, ++STORE, 140398874501120, 140398874505215, ++STORE, 140398872629248, 140398872764415, ++STORE, 43278336, 43556863, ++STORE, 140398847709184, 140398849806335, ++STORE, 140398874492928, 140398874505215, ++STORE, 140398874484736, 140398874505215, ++STORE, 140398874447872, 140398874484735, ++STORE, 140398872612864, 140398872764415, ++STORE, 43278336, 43692031, ++STORE, 43278336, 43880447, ++STORE, 140398872604672, 140398872764415, ++STORE, 140398872596480, 140398872764415, ++STORE, 43278336, 44044287, ++STORE, 140398872580096, 140398872764415, ++STORE, 140737488347136, 140737488351231, ++STORE, 140734403092480, 140737488351231, ++SNULL, 140734403096575, 140737488351231, ++STORE, 140734403092480, 140734403096575, ++STORE, 140734402961408, 140734403096575, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140240662380544, 140240664633343, ++SNULL, 140240662523903, 140240664633343, ++STORE, 140240662380544, 140240662523903, ++STORE, 140240662523904, 140240664633343, ++ERASE, 140240662523904, 140240664633343, ++STORE, 140240664621056, 140240664629247, ++STORE, 140240664629248, 140240664633343, ++STORE, 140734403145728, 140734403149823, ++STORE, 140734403133440, 140734403145727, ++STORE, 140240664592384, 140240664621055, ++STORE, 140240664584192, 140240664592383, ++STORE, 140240659218432, 140240662380543, ++SNULL, 140240659218432, 140240660279295, ++STORE, 140240660279296, 140240662380543, ++STORE, 140240659218432, 140240660279295, ++SNULL, 140240662372351, 140240662380543, ++STORE, 140240660279296, 140240662372351, ++STORE, 140240662372352, 140240662380543, ++ERASE, 140240662372352, 140240662380543, ++STORE, 140240662372352, 140240662380543, ++STORE, 140240655421440, 140240659218431, ++SNULL, 140240655421440, 140240657080319, ++STORE, 140240657080320, 140240659218431, ++STORE, 140240655421440, 140240657080319, ++SNULL, 140240659177471, 140240659218431, ++STORE, 140240657080320, 140240659177471, ++STORE, 140240659177472, 140240659218431, ++SNULL, 140240659177472, 140240659202047, ++STORE, 140240659202048, 140240659218431, ++STORE, 140240659177472, 140240659202047, ++ERASE, 140240659177472, 140240659202047, ++STORE, 140240659177472, 140240659202047, ++ERASE, 140240659202048, 140240659218431, ++STORE, 140240659202048, 140240659218431, ++STORE, 140240664571904, 140240664592383, ++SNULL, 140240659193855, 140240659202047, ++STORE, 140240659177472, 140240659193855, ++STORE, 140240659193856, 140240659202047, ++SNULL, 140240662376447, 140240662380543, ++STORE, 140240662372352, 140240662376447, ++STORE, 140240662376448, 140240662380543, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140240664625151, 140240664629247, ++STORE, 140240664621056, 140240664625151, ++STORE, 140240664625152, 140240664629247, ++ERASE, 140240664592384, 140240664621055, ++STORE, 30646272, 30781439, ++STORE, 30646272, 30928895, ++STORE, 140240662888448, 140240664571903, ++STORE, 94256659468288, 94256659578879, ++STORE, 94256661671936, 94256661680127, ++STORE, 94256661680128, 94256661684223, ++STORE, 94256661684224, 94256661692415, ++STORE, 94256687980544, 94256688115711, ++STORE, 139801712504832, 139801714163711, ++STORE, 139801714163712, 139801716260863, ++STORE, 139801716260864, 139801716277247, ++STORE, 139801716277248, 139801716285439, ++STORE, 139801716285440, 139801716301823, ++STORE, 139801716301824, 139801716445183, ++STORE, 139801718505472, 139801718513663, ++STORE, 139801718542336, 139801718546431, ++STORE, 139801718546432, 139801718550527, ++STORE, 139801718550528, 139801718554623, ++STORE, 140721575538688, 140721575673855, ++STORE, 140721577013248, 140721577025535, ++STORE, 140721577025536, 140721577029631, ++STORE, 140737488347136, 140737488351231, ++STORE, 140729259393024, 140737488351231, ++SNULL, 140729259397119, 140737488351231, ++STORE, 140729259393024, 140729259397119, ++STORE, 140729259261952, 140729259397119, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 139682376638464, 139682378891263, ++SNULL, 139682376781823, 139682378891263, ++STORE, 139682376638464, 139682376781823, ++STORE, 139682376781824, 139682378891263, ++ERASE, 139682376781824, 139682378891263, ++STORE, 139682378878976, 139682378887167, ++STORE, 139682378887168, 139682378891263, ++STORE, 140729260462080, 140729260466175, ++STORE, 140729260449792, 140729260462079, ++STORE, 139682378850304, 139682378878975, ++STORE, 139682378842112, 139682378850303, ++STORE, 139682373476352, 139682376638463, ++SNULL, 139682373476352, 139682374537215, ++STORE, 139682374537216, 139682376638463, ++STORE, 139682373476352, 139682374537215, ++SNULL, 139682376630271, 139682376638463, ++STORE, 139682374537216, 139682376630271, ++STORE, 139682376630272, 139682376638463, ++ERASE, 139682376630272, 139682376638463, ++STORE, 139682376630272, 139682376638463, ++STORE, 139682369679360, 139682373476351, ++SNULL, 139682369679360, 139682371338239, ++STORE, 139682371338240, 139682373476351, ++STORE, 139682369679360, 139682371338239, ++SNULL, 139682373435391, 139682373476351, ++STORE, 139682371338240, 139682373435391, ++STORE, 139682373435392, 139682373476351, ++SNULL, 139682373435392, 139682373459967, ++STORE, 139682373459968, 139682373476351, ++STORE, 139682373435392, 139682373459967, ++ERASE, 139682373435392, 139682373459967, ++STORE, 139682373435392, 139682373459967, ++ERASE, 139682373459968, 139682373476351, ++STORE, 139682373459968, 139682373476351, ++STORE, 139682378829824, 139682378850303, ++SNULL, 139682373451775, 139682373459967, ++STORE, 139682373435392, 139682373451775, ++STORE, 139682373451776, 139682373459967, ++SNULL, 139682376634367, 139682376638463, ++STORE, 139682376630272, 139682376634367, ++STORE, 139682376634368, 139682376638463, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 139682378883071, 139682378887167, ++STORE, 139682378878976, 139682378883071, ++STORE, 139682378883072, 139682378887167, ++ERASE, 139682378850304, 139682378878975, ++STORE, 10022912, 10158079, ++STORE, 10022912, 10305535, ++STORE, 139682377146368, 139682378829823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140731831926784, 140737488351231, ++SNULL, 140731831930879, 140737488351231, ++STORE, 140731831926784, 140731831930879, ++STORE, 140731831795712, 140731831930879, ++STORE, 94615305261056, 94615307485183, ++SNULL, 94615305371647, 94615307485183, ++STORE, 94615305261056, 94615305371647, ++STORE, 94615305371648, 94615307485183, ++ERASE, 94615305371648, 94615307485183, ++STORE, 94615307464704, 94615307476991, ++STORE, 94615307476992, 94615307485183, ++STORE, 140163912994816, 140163915247615, ++SNULL, 140163913138175, 140163915247615, ++STORE, 140163912994816, 140163913138175, ++STORE, 140163913138176, 140163915247615, ++ERASE, 140163913138176, 140163915247615, ++STORE, 140163915235328, 140163915243519, ++STORE, 140163915243520, 140163915247615, ++STORE, 140731832217600, 140731832221695, ++STORE, 140731832205312, 140731832217599, ++STORE, 140163915206656, 140163915235327, ++STORE, 140163915198464, 140163915206655, ++STORE, 140163909197824, 140163912994815, ++SNULL, 140163909197824, 140163910856703, ++STORE, 140163910856704, 140163912994815, ++STORE, 140163909197824, 140163910856703, ++SNULL, 140163912953855, 140163912994815, ++STORE, 140163910856704, 140163912953855, ++STORE, 140163912953856, 140163912994815, ++SNULL, 140163912953856, 140163912978431, ++STORE, 140163912978432, 140163912994815, ++STORE, 140163912953856, 140163912978431, ++ERASE, 140163912953856, 140163912978431, ++STORE, 140163912953856, 140163912978431, ++ERASE, 140163912978432, 140163912994815, ++STORE, 140163912978432, 140163912994815, ++SNULL, 140163912970239, 140163912978431, ++STORE, 140163912953856, 140163912970239, ++STORE, 140163912970240, 140163912978431, ++SNULL, 94615307472895, 94615307476991, ++STORE, 94615307464704, 94615307472895, ++STORE, 94615307472896, 94615307476991, ++SNULL, 140163915239423, 140163915243519, ++STORE, 140163915235328, 140163915239423, ++STORE, 140163915239424, 140163915243519, ++ERASE, 140163915206656, 140163915235327, ++STORE, 94615330672640, 94615330807807, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725254479872, 140737488351231, ++SNULL, 140725254488063, 140737488351231, ++STORE, 140725254479872, 140725254488063, ++STORE, 140725254348800, 140725254488063, ++STORE, 94572781277184, 94572785741823, ++SNULL, 94572783312895, 94572785741823, ++STORE, 94572781277184, 94572783312895, ++STORE, 94572783312896, 94572785741823, ++ERASE, 94572783312896, 94572785741823, ++STORE, 94572785405952, 94572785455103, ++STORE, 94572785455104, 94572785741823, ++STORE, 139636001341440, 139636003594239, ++SNULL, 139636001484799, 139636003594239, ++STORE, 139636001341440, 139636001484799, ++STORE, 139636001484800, 139636003594239, ++ERASE, 139636001484800, 139636003594239, ++STORE, 139636003581952, 139636003590143, ++STORE, 139636003590144, 139636003594239, ++STORE, 140725255557120, 140725255561215, ++STORE, 140725255544832, 140725255557119, ++STORE, 139636003553280, 139636003581951, ++STORE, 139636003545088, 139636003553279, ++STORE, 139635998773248, 139636001341439, ++SNULL, 139635998773248, 139635999240191, ++STORE, 139635999240192, 139636001341439, ++STORE, 139635998773248, 139635999240191, ++SNULL, 139636001333247, 139636001341439, ++STORE, 139635999240192, 139636001333247, ++STORE, 139636001333248, 139636001341439, ++ERASE, 139636001333248, 139636001341439, ++STORE, 139636001333248, 139636001341439, ++STORE, 139635996569600, 139635998773247, ++SNULL, 139635996569600, 139635996671999, ++STORE, 139635996672000, 139635998773247, ++STORE, 139635996569600, 139635996671999, ++SNULL, 139635998765055, 139635998773247, ++STORE, 139635996672000, 139635998765055, ++STORE, 139635998765056, 139635998773247, ++ERASE, 139635998765056, 139635998773247, ++STORE, 139635998765056, 139635998773247, ++STORE, 139635994353664, 139635996569599, ++SNULL, 139635994353664, 139635994451967, ++STORE, 139635994451968, 139635996569599, ++STORE, 139635994353664, 139635994451967, ++SNULL, 139635996545023, 139635996569599, ++STORE, 139635994451968, 139635996545023, ++STORE, 139635996545024, 139635996569599, ++SNULL, 139635996545024, 139635996553215, ++STORE, 139635996553216, 139635996569599, ++STORE, 139635996545024, 139635996553215, ++ERASE, 139635996545024, 139635996553215, ++STORE, 139635996545024, 139635996553215, ++ERASE, 139635996553216, 139635996569599, ++STORE, 139635996553216, 139635996569599, ++STORE, 139635992223744, 139635994353663, ++SNULL, 139635992223744, 139635992252415, ++STORE, 139635992252416, 139635994353663, ++STORE, 139635992223744, 139635992252415, ++SNULL, 139635994345471, 139635994353663, ++STORE, 139635992252416, 139635994345471, ++STORE, 139635994345472, 139635994353663, ++ERASE, 139635994345472, 139635994353663, ++STORE, 139635994345472, 139635994353663, ++STORE, 139635988426752, 139635992223743, ++SNULL, 139635988426752, 139635990085631, ++STORE, 139635990085632, 139635992223743, ++STORE, 139635988426752, 139635990085631, ++SNULL, 139635992182783, 139635992223743, ++STORE, 139635990085632, 139635992182783, ++STORE, 139635992182784, 139635992223743, ++SNULL, 139635992182784, 139635992207359, ++STORE, 139635992207360, 139635992223743, ++STORE, 139635992182784, 139635992207359, ++ERASE, 139635992182784, 139635992207359, ++STORE, 139635992182784, 139635992207359, ++ERASE, 139635992207360, 139635992223743, ++STORE, 139635992207360, 139635992223743, ++STORE, 139636003536896, 139636003553279, ++SNULL, 139635992199167, 139635992207359, ++STORE, 139635992182784, 139635992199167, ++STORE, 139635992199168, 139635992207359, ++SNULL, 139635996549119, 139635996553215, ++STORE, 139635996545024, 139635996549119, ++STORE, 139635996549120, 139635996553215, ++SNULL, 139635994349567, 139635994353663, ++STORE, 139635994345472, 139635994349567, ++STORE, 139635994349568, 139635994353663, ++SNULL, 139635998769151, 139635998773247, ++STORE, 139635998765056, 139635998769151, ++STORE, 139635998769152, 139635998773247, ++SNULL, 139636001337343, 139636001341439, ++STORE, 139636001333248, 139636001337343, ++STORE, 139636001337344, 139636001341439, ++SNULL, 94572785418239, 94572785455103, ++STORE, 94572785405952, 94572785418239, ++STORE, 94572785418240, 94572785455103, ++SNULL, 139636003586047, 139636003590143, ++STORE, 139636003581952, 139636003586047, ++STORE, 139636003586048, 139636003590143, ++ERASE, 139636003553280, 139636003581951, ++STORE, 94572798435328, 94572798570495, ++STORE, 139636001853440, 139636003536895, ++STORE, 139635981426688, 139635988426751, ++STORE, 139635980615680, 139635981426687, ++STORE, 94572798435328, 94572798705663, ++STORE, 94572798435328, 94572798840831, ++STORE, 94572798435328, 94572798975999, ++STORE, 94572798435328, 94572799111167, ++STORE, 94572798435328, 94572799246335, ++STORE, 94572798435328, 94572799381503, ++STORE, 94572798435328, 94572799516671, ++STORE, 94572798435328, 94572799651839, ++STORE, 94572798435328, 94572799787007, ++STORE, 94572798435328, 94572799922175, ++STORE, 94572798435328, 94572800057343, ++STORE, 94572798435328, 94572800192511, ++STORE, 94572798435328, 94572800327679, ++STORE, 94572798435328, 94572800462847, ++STORE, 94572798435328, 94572800598015, ++STORE, 94572798435328, 94572800733183, ++STORE, 94572798435328, 94572800868351, ++STORE, 94572798435328, 94572801003519, ++STORE, 94572798435328, 94572801138687, ++STORE, 94572798435328, 94572801273855, ++STORE, 94572798435328, 94572801409023, ++STORE, 94572798435328, 94572801544191, ++STORE, 94572798435328, 94572801679359, ++STORE, 94572798435328, 94572801814527, ++STORE, 94572798435328, 94572801949695, ++STORE, 94572798435328, 94572802084863, ++STORE, 94572798435328, 94572802220031, ++STORE, 94572798435328, 94572802355199, ++STORE, 94572798435328, 94572802490367, ++STORE, 94572798435328, 94572802625535, ++STORE, 94572798435328, 94572802760703, ++STORE, 94572798435328, 94572802895871, ++STORE, 94572798435328, 94572803031039, ++STORE, 94572798435328, 94572803166207, ++STORE, 94572798435328, 94572803301375, ++STORE, 94572798435328, 94572803436543, ++STORE, 94572798435328, 94572803571711, ++STORE, 94572798435328, 94572803706879, ++STORE, 94572798435328, 94572803842047, ++STORE, 94572798435328, 94572803977215, ++STORE, 94572798435328, 94572804112383, ++STORE, 94572798435328, 94572804247551, ++STORE, 94572798435328, 94572804382719, ++STORE, 94572798435328, 94572804517887, ++STORE, 94572798435328, 94572804653055, ++STORE, 94572798435328, 94572804788223, ++STORE, 94572798435328, 94572804923391, ++STORE, 94572798435328, 94572805058559, ++STORE, 94572798435328, 94572805193727, ++STORE, 94572798435328, 94572805328895, ++STORE, 94572798435328, 94572805464063, ++STORE, 94572798435328, 94572805599231, ++STORE, 94572798435328, 94572805734399, ++STORE, 94572798435328, 94572805869567, ++STORE, 94572798435328, 94572806004735, ++STORE, 94572798435328, 94572806139903, ++STORE, 94572798435328, 94572806275071, ++STORE, 94572798435328, 94572806410239, ++STORE, 94572798435328, 94572806545407, ++STORE, 94572798435328, 94572806680575, ++STORE, 94572798435328, 94572806815743, ++STORE, 94572798435328, 94572806950911, ++STORE, 94572798435328, 94572807086079, ++STORE, 94572798435328, 94572807221247, ++STORE, 94572798435328, 94572807356415, ++STORE, 94572798435328, 94572807491583, ++STORE, 94572798435328, 94572807626751, ++STORE, 94572798435328, 94572807761919, ++STORE, 94572798435328, 94572807897087, ++STORE, 94572798435328, 94572808032255, ++STORE, 94572798435328, 94572808167423, ++STORE, 94572798435328, 94572808302591, ++STORE, 94572798435328, 94572808437759, ++STORE, 94572798435328, 94572808572927, ++ERASE, 139635981426688, 139635988426751, ++STORE, 139635985088512, 139635988426751, ++STORE, 139635778273280, 139635980615679, ++STORE, 139635567632384, 139635778273279, ++STORE, 94572798435328, 94572808716287, ++STORE, 139635984564224, 139635985088511, ++STORE, 139635559239680, 139635567632383, ++SNULL, 139635559243775, 139635567632383, ++STORE, 139635559239680, 139635559243775, ++STORE, 139635559243776, 139635567632383, ++STORE, 139635550846976, 139635559239679, ++SNULL, 139635550851071, 139635559239679, ++STORE, 139635550846976, 139635550851071, ++STORE, 139635550851072, 139635559239679, ++STORE, 139635542454272, 139635550846975, ++STORE, 139635408236544, 139635542454271, ++SNULL, 139635408236544, 139635426590719, ++STORE, 139635426590720, 139635542454271, ++STORE, 139635408236544, 139635426590719, ++ERASE, 139635408236544, 139635426590719, ++STORE, 139635292372992, 139635542454271, ++SNULL, 139635359481855, 139635542454271, ++STORE, 139635292372992, 139635359481855, ++STORE, 139635359481856, 139635542454271, ++SNULL, 139635359481856, 139635426590719, ++STORE, 139635426590720, 139635542454271, ++STORE, 139635359481856, 139635426590719, ++ERASE, 139635359481856, 139635426590719, ++SNULL, 139635542458367, 139635550846975, ++STORE, 139635542454272, 139635542458367, ++STORE, 139635542458368, 139635550846975, ++STORE, 139635418198016, 139635426590719, ++SNULL, 139635493699583, 139635542454271, ++STORE, 139635426590720, 139635493699583, ++STORE, 139635493699584, 139635542454271, ++ERASE, 139635493699584, 139635542454271, ++SNULL, 139635426725887, 139635493699583, ++STORE, 139635426590720, 139635426725887, ++STORE, 139635426725888, 139635493699583, ++SNULL, 139635292508159, 139635359481855, ++STORE, 139635292372992, 139635292508159, ++STORE, 139635292508160, 139635359481855, ++SNULL, 139635418202111, 139635426590719, ++STORE, 139635418198016, 139635418202111, ++STORE, 139635418202112, 139635426590719, ++STORE, 139635225264128, 139635292372991, ++STORE, 139635534061568, 139635542454271, ++SNULL, 139635534065663, 139635542454271, ++STORE, 139635534061568, 139635534065663, ++STORE, 139635534065664, 139635542454271, ++STORE, 139635525668864, 139635534061567, ++SNULL, 139635525672959, 139635534061567, ++STORE, 139635525668864, 139635525672959, ++STORE, 139635525672960, 139635534061567, ++SNULL, 139635225399295, 139635292372991, ++STORE, 139635225264128, 139635225399295, ++STORE, 139635225399296, 139635292372991, ++STORE, 139635091046400, 139635225264127, ++SNULL, 139635158155263, 139635225264127, ++STORE, 139635091046400, 139635158155263, ++STORE, 139635158155264, 139635225264127, ++ERASE, 139635158155264, 139635225264127, ++STORE, 139634956828672, 139635158155263, ++STORE, 139635517276160, 139635525668863, ++SNULL, 139635517280255, 139635525668863, ++STORE, 139635517276160, 139635517280255, ++STORE, 139635517280256, 139635525668863, ++SNULL, 139634956828672, 139635091046399, ++STORE, 139635091046400, 139635158155263, ++STORE, 139634956828672, 139635091046399, ++SNULL, 139635091181567, 139635158155263, ++STORE, 139635091046400, 139635091181567, ++STORE, 139635091181568, 139635158155263, ++SNULL, 139635023937535, 139635091046399, ++STORE, 139634956828672, 139635023937535, ++STORE, 139635023937536, 139635091046399, ++ERASE, 139635023937536, 139635091046399, ++STORE, 139634956828672, 139635091046399, ++SNULL, 139634956828672, 139635023937535, ++STORE, 139635023937536, 139635091046399, ++STORE, 139634956828672, 139635023937535, ++SNULL, 139635024072703, 139635091046399, ++STORE, 139635023937536, 139635024072703, ++STORE, 139635024072704, 139635091046399, ++STORE, 139635508883456, 139635517276159, ++SNULL, 139635508887551, 139635517276159, ++STORE, 139635508883456, 139635508887551, ++STORE, 139635508887552, 139635517276159, ++STORE, 139634822610944, 139635023937535, ++SNULL, 139634822610944, 139634956828671, ++STORE, 139634956828672, 139635023937535, ++STORE, 139634822610944, 139634956828671, ++SNULL, 139634956963839, 139635023937535, ++STORE, 139634956828672, 139634956963839, ++STORE, 139634956963840, 139635023937535, ++STORE, 139635500490752, 139635508883455, ++SNULL, 139634889719807, 139634956828671, ++STORE, 139634822610944, 139634889719807, ++STORE, 139634889719808, 139634956828671, ++ERASE, 139634889719808, 139634956828671, ++SNULL, 139635500494847, 139635508883455, ++STORE, 139635500490752, 139635500494847, ++STORE, 139635500494848, 139635508883455, ++SNULL, 139634822746111, 139634889719807, ++STORE, 139634822610944, 139634822746111, ++STORE, 139634822746112, 139634889719807, ++STORE, 139635409805312, 139635418198015, ++STORE, 139634822746112, 139634956828671, ++SNULL, 139634822746112, 139634889719807, ++STORE, 139634889719808, 139634956828671, ++STORE, 139634822746112, 139634889719807, ++SNULL, 139634889854975, 139634956828671, ++STORE, 139634889719808, 139634889854975, ++STORE, 139634889854976, 139634956828671, ++SNULL, 139635409809407, 139635418198015, ++STORE, 139635409805312, 139635409809407, ++STORE, 139635409809408, 139635418198015, ++STORE, 139635401412608, 139635409805311, ++STORE, 139634688393216, 139634822610943, ++SNULL, 139634755502079, 139634822610943, ++STORE, 139634688393216, 139634755502079, ++STORE, 139634755502080, 139634822610943, ++ERASE, 139634755502080, 139634822610943, ++SNULL, 139635401416703, 139635409805311, ++STORE, 139635401412608, 139635401416703, ++STORE, 139635401416704, 139635409805311, ++STORE, 139634554175488, 139634755502079, ++SNULL, 139634554175488, 139634688393215, ++STORE, 139634688393216, 139634755502079, ++STORE, 139634554175488, 139634688393215, ++SNULL, 139634688528383, 139634755502079, ++STORE, 139634688393216, 139634688528383, ++STORE, 139634688528384, 139634755502079, ++STORE, 139635393019904, 139635401412607, ++SNULL, 139634621284351, 139634688393215, ++STORE, 139634554175488, 139634621284351, ++STORE, 139634621284352, 139634688393215, ++ERASE, 139634621284352, 139634688393215, ++SNULL, 139634554310655, 139634621284351, ++STORE, 139634554175488, 139634554310655, ++STORE, 139634554310656, 139634621284351, ++STORE, 139634554310656, 139634688393215, ++SNULL, 139635393023999, 139635401412607, ++STORE, 139635393019904, 139635393023999, ++STORE, 139635393024000, 139635401412607, ++SNULL, 139634554310656, 139634621284351, ++STORE, 139634621284352, 139634688393215, ++STORE, 139634554310656, 139634621284351, ++SNULL, 139634621419519, 139634688393215, ++STORE, 139634621284352, 139634621419519, ++STORE, 139634621419520, 139634688393215, ++STORE, 139635384627200, 139635393019903, ++SNULL, 139635384631295, 139635393019903, ++STORE, 139635384627200, 139635384631295, ++STORE, 139635384631296, 139635393019903, ++STORE, 139635376234496, 139635384627199, ++SNULL, 139635376238591, 139635384627199, ++STORE, 139635376234496, 139635376238591, ++STORE, 139635376238592, 139635384627199, ++STORE, 139635367841792, 139635376234495, ++SNULL, 139635367845887, 139635376234495, ++STORE, 139635367841792, 139635367845887, ++STORE, 139635367845888, 139635376234495, ++STORE, 139634419957760, 139634554175487, ++SNULL, 139634487066623, 139634554175487, ++STORE, 139634419957760, 139634487066623, ++STORE, 139634487066624, 139634554175487, ++ERASE, 139634487066624, 139634554175487, ++STORE, 139635216871424, 139635225264127, ++SNULL, 139635216875519, 139635225264127, ++STORE, 139635216871424, 139635216875519, ++STORE, 139635216875520, 139635225264127, ++SNULL, 139634420092927, 139634487066623, ++STORE, 139634419957760, 139634420092927, ++STORE, 139634420092928, 139634487066623, ++STORE, 139635208478720, 139635216871423, ++SNULL, 139635208482815, 139635216871423, ++STORE, 139635208478720, 139635208482815, ++STORE, 139635208482816, 139635216871423, ++STORE, 139635200086016, 139635208478719, ++SNULL, 139635200090111, 139635208478719, ++STORE, 139635200086016, 139635200090111, ++STORE, 139635200090112, 139635208478719, ++STORE, 139635191693312, 139635200086015, ++SNULL, 139635191697407, 139635200086015, ++STORE, 139635191693312, 139635191697407, ++STORE, 139635191697408, 139635200086015, ++STORE, 139635183300608, 139635191693311, ++SNULL, 139635183304703, 139635191693311, ++STORE, 139635183300608, 139635183304703, ++STORE, 139635183304704, 139635191693311, ++STORE, 139634420092928, 139634554175487, ++SNULL, 139634420092928, 139634487066623, ++STORE, 139634487066624, 139634554175487, ++STORE, 139634420092928, 139634487066623, ++SNULL, 139634487201791, 139634554175487, ++STORE, 139634487066624, 139634487201791, ++STORE, 139634487201792, 139634554175487, ++ERASE, 139635559239680, 139635559243775, ++ERASE, 139635559243776, 139635567632383, ++ERASE, 139635550846976, 139635550851071, ++ERASE, 139635550851072, 139635559239679, ++ERASE, 139635542454272, 139635542458367, ++ERASE, 139635542458368, 139635550846975, ++ERASE, 139635418198016, 139635418202111, ++ERASE, 139635418202112, 139635426590719, ++ERASE, 139635534061568, 139635534065663, ++ERASE, 139635534065664, 139635542454271, ++ERASE, 139635525668864, 139635525672959, ++ERASE, 139635525672960, 139635534061567, ++ERASE, 139635517276160, 139635517280255, ++ERASE, 139635517280256, 139635525668863, ++ERASE, 139635508883456, 139635508887551, ++ERASE, 139635508887552, 139635517276159, ++ERASE, 139635500490752, 139635500494847, ++ERASE, 139635500494848, 139635508883455, ++ERASE, 139635409805312, 139635409809407, ++ERASE, 139635409809408, 139635418198015, ++ERASE, 139635401412608, 139635401416703, ++ERASE, 139635401416704, 139635409805311, ++ERASE, 139635393019904, 139635393023999, ++ERASE, 139635393024000, 139635401412607, ++ERASE, 139635384627200, 139635384631295, ++ERASE, 139635384631296, 139635393019903, ++ }; ++ unsigned long set25[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722547441664, 140737488351231, ++SNULL, 140722547449855, 140737488351231, ++STORE, 140722547441664, 140722547449855, ++STORE, 140722547310592, 140722547449855, ++STORE, 94827521732608, 94827523956735, ++SNULL, 94827521843199, 94827523956735, ++STORE, 94827521732608, 94827521843199, ++STORE, 94827521843200, 94827523956735, ++ERASE, 94827521843200, 94827523956735, ++STORE, 94827523936256, 94827523948543, ++STORE, 94827523948544, 94827523956735, ++STORE, 139816136847360, 139816139100159, ++SNULL, 139816136990719, 139816139100159, ++STORE, 139816136847360, 139816136990719, ++STORE, 139816136990720, 139816139100159, ++ERASE, 139816136990720, 139816139100159, ++STORE, 139816139087872, 139816139096063, ++STORE, 139816139096064, 139816139100159, ++STORE, 140722548142080, 140722548146175, ++STORE, 140722548129792, 140722548142079, ++STORE, 139816139059200, 139816139087871, ++STORE, 139816139051008, 139816139059199, ++STORE, 139816133050368, 139816136847359, ++SNULL, 139816133050368, 139816134709247, ++STORE, 139816134709248, 139816136847359, ++STORE, 139816133050368, 139816134709247, ++SNULL, 139816136806399, 139816136847359, ++STORE, 139816134709248, 139816136806399, ++STORE, 139816136806400, 139816136847359, ++SNULL, 139816136806400, 139816136830975, ++STORE, 139816136830976, 139816136847359, ++STORE, 139816136806400, 139816136830975, ++ERASE, 139816136806400, 139816136830975, ++STORE, 139816136806400, 139816136830975, ++ERASE, 139816136830976, 139816136847359, ++STORE, 139816136830976, 139816136847359, ++SNULL, 139816136822783, 139816136830975, ++STORE, 139816136806400, 139816136822783, ++STORE, 139816136822784, 139816136830975, ++SNULL, 94827523944447, 94827523948543, ++STORE, 94827523936256, 94827523944447, ++STORE, 94827523944448, 94827523948543, ++SNULL, 139816139091967, 139816139096063, ++STORE, 139816139087872, 139816139091967, ++STORE, 139816139091968, 139816139096063, ++ERASE, 139816139059200, 139816139087871, ++STORE, 94827534970880, 94827535106047, ++STORE, 94114394132480, 94114394345471, ++STORE, 94114396442624, 94114396446719, ++STORE, 94114396446720, 94114396454911, ++STORE, 94114396454912, 94114396467199, ++STORE, 94114421575680, 94114427715583, ++STORE, 139934313955328, 139934315614207, ++STORE, 139934315614208, 139934317711359, ++STORE, 139934317711360, 139934317727743, ++STORE, 139934317727744, 139934317735935, ++STORE, 139934317735936, 139934317752319, ++STORE, 139934317752320, 139934317764607, ++STORE, 139934317764608, 139934319857663, ++STORE, 139934319857664, 139934319861759, ++STORE, 139934319861760, 139934319865855, ++STORE, 139934319865856, 139934320009215, ++STORE, 139934320377856, 139934322061311, ++STORE, 139934322061312, 139934322077695, ++STORE, 139934322106368, 139934322110463, ++STORE, 139934322110464, 139934322114559, ++STORE, 139934322114560, 139934322118655, ++STORE, 140731200376832, 140731200516095, ++STORE, 140731200929792, 140731200942079, ++STORE, 140731200942080, 140731200946175, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140734133174272, 140737488351231, ++SNULL, 140734133182463, 140737488351231, ++STORE, 140734133174272, 140734133182463, ++STORE, 140734133043200, 140734133182463, ++STORE, 94412675600384, 94412677824511, ++SNULL, 94412675710975, 94412677824511, ++STORE, 94412675600384, 94412675710975, ++STORE, 94412675710976, 94412677824511, ++ERASE, 94412675710976, 94412677824511, ++STORE, 94412677804032, 94412677816319, ++STORE, 94412677816320, 94412677824511, ++STORE, 140320087945216, 140320090198015, ++SNULL, 140320088088575, 140320090198015, ++STORE, 140320087945216, 140320088088575, ++STORE, 140320088088576, 140320090198015, ++ERASE, 140320088088576, 140320090198015, ++STORE, 140320090185728, 140320090193919, ++STORE, 140320090193920, 140320090198015, ++STORE, 140734134591488, 140734134595583, ++STORE, 140734134579200, 140734134591487, ++STORE, 140320090157056, 140320090185727, ++STORE, 140320090148864, 140320090157055, ++STORE, 140320084148224, 140320087945215, ++SNULL, 140320084148224, 140320085807103, ++STORE, 140320085807104, 140320087945215, ++STORE, 140320084148224, 140320085807103, ++SNULL, 140320087904255, 140320087945215, ++STORE, 140320085807104, 140320087904255, ++STORE, 140320087904256, 140320087945215, ++SNULL, 140320087904256, 140320087928831, ++STORE, 140320087928832, 140320087945215, ++STORE, 140320087904256, 140320087928831, ++ERASE, 140320087904256, 140320087928831, ++STORE, 140320087904256, 140320087928831, ++ERASE, 140320087928832, 140320087945215, ++STORE, 140320087928832, 140320087945215, ++SNULL, 140320087920639, 140320087928831, ++STORE, 140320087904256, 140320087920639, ++STORE, 140320087920640, 140320087928831, ++SNULL, 94412677812223, 94412677816319, ++STORE, 94412677804032, 94412677812223, ++STORE, 94412677812224, 94412677816319, ++SNULL, 140320090189823, 140320090193919, ++STORE, 140320090185728, 140320090189823, ++STORE, 140320090189824, 140320090193919, ++ERASE, 140320090157056, 140320090185727, ++STORE, 94412684546048, 94412684681215, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723005485056, 140737488351231, ++SNULL, 140723005493247, 140737488351231, ++STORE, 140723005485056, 140723005493247, ++STORE, 140723005353984, 140723005493247, ++STORE, 94387431936000, 94387434160127, ++SNULL, 94387432046591, 94387434160127, ++STORE, 94387431936000, 94387432046591, ++STORE, 94387432046592, 94387434160127, ++ERASE, 94387432046592, 94387434160127, ++STORE, 94387434139648, 94387434151935, ++STORE, 94387434151936, 94387434160127, ++STORE, 140151675392000, 140151677644799, ++SNULL, 140151675535359, 140151677644799, ++STORE, 140151675392000, 140151675535359, ++STORE, 140151675535360, 140151677644799, ++ERASE, 140151675535360, 140151677644799, ++STORE, 140151677632512, 140151677640703, ++STORE, 140151677640704, 140151677644799, ++STORE, 140723005784064, 140723005788159, ++STORE, 140723005771776, 140723005784063, ++STORE, 140151677603840, 140151677632511, ++STORE, 140151677595648, 140151677603839, ++STORE, 140151671595008, 140151675391999, ++SNULL, 140151671595008, 140151673253887, ++STORE, 140151673253888, 140151675391999, ++STORE, 140151671595008, 140151673253887, ++SNULL, 140151675351039, 140151675391999, ++STORE, 140151673253888, 140151675351039, ++STORE, 140151675351040, 140151675391999, ++SNULL, 140151675351040, 140151675375615, ++STORE, 140151675375616, 140151675391999, ++STORE, 140151675351040, 140151675375615, ++ERASE, 140151675351040, 140151675375615, ++STORE, 140151675351040, 140151675375615, ++ERASE, 140151675375616, 140151675391999, ++STORE, 140151675375616, 140151675391999, ++SNULL, 140151675367423, 140151675375615, ++STORE, 140151675351040, 140151675367423, ++STORE, 140151675367424, 140151675375615, ++SNULL, 94387434147839, 94387434151935, ++STORE, 94387434139648, 94387434147839, ++STORE, 94387434147840, 94387434151935, ++SNULL, 140151677636607, 140151677640703, ++STORE, 140151677632512, 140151677636607, ++STORE, 140151677636608, 140151677640703, ++ERASE, 140151677603840, 140151677632511, ++STORE, 94387458818048, 94387458953215, ++STORE, 94909010997248, 94909011210239, ++STORE, 94909013307392, 94909013311487, ++STORE, 94909013311488, 94909013319679, ++STORE, 94909013319680, 94909013331967, ++STORE, 94909014827008, 94909023371263, ++STORE, 140712411975680, 140712413634559, ++STORE, 140712413634560, 140712415731711, ++STORE, 140712415731712, 140712415748095, ++STORE, 140712415748096, 140712415756287, ++STORE, 140712415756288, 140712415772671, ++STORE, 140712415772672, 140712415784959, ++STORE, 140712415784960, 140712417878015, ++STORE, 140712417878016, 140712417882111, ++STORE, 140712417882112, 140712417886207, ++STORE, 140712417886208, 140712418029567, ++STORE, 140712418398208, 140712420081663, ++STORE, 140712420081664, 140712420098047, ++STORE, 140712420126720, 140712420130815, ++STORE, 140712420130816, 140712420134911, ++STORE, 140712420134912, 140712420139007, ++STORE, 140729293111296, 140729293250559, ++STORE, 140729293307904, 140729293320191, ++STORE, 140729293320192, 140729293324287, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720541691904, 140737488351231, ++SNULL, 140720541700095, 140737488351231, ++STORE, 140720541691904, 140720541700095, ++STORE, 140720541560832, 140720541700095, ++STORE, 94203603419136, 94203605643263, ++SNULL, 94203603529727, 94203605643263, ++STORE, 94203603419136, 94203603529727, ++STORE, 94203603529728, 94203605643263, ++ERASE, 94203603529728, 94203605643263, ++STORE, 94203605622784, 94203605635071, ++STORE, 94203605635072, 94203605643263, ++STORE, 139847623081984, 139847625334783, ++SNULL, 139847623225343, 139847625334783, ++STORE, 139847623081984, 139847623225343, ++STORE, 139847623225344, 139847625334783, ++ERASE, 139847623225344, 139847625334783, ++STORE, 139847625322496, 139847625330687, ++STORE, 139847625330688, 139847625334783, ++STORE, 140720542547968, 140720542552063, ++STORE, 140720542535680, 140720542547967, ++STORE, 139847625293824, 139847625322495, ++STORE, 139847625285632, 139847625293823, ++STORE, 139847619284992, 139847623081983, ++SNULL, 139847619284992, 139847620943871, ++STORE, 139847620943872, 139847623081983, ++STORE, 139847619284992, 139847620943871, ++SNULL, 139847623041023, 139847623081983, ++STORE, 139847620943872, 139847623041023, ++STORE, 139847623041024, 139847623081983, ++SNULL, 139847623041024, 139847623065599, ++STORE, 139847623065600, 139847623081983, ++STORE, 139847623041024, 139847623065599, ++ERASE, 139847623041024, 139847623065599, ++STORE, 139847623041024, 139847623065599, ++ERASE, 139847623065600, 139847623081983, ++STORE, 139847623065600, 139847623081983, ++SNULL, 139847623057407, 139847623065599, ++STORE, 139847623041024, 139847623057407, ++STORE, 139847623057408, 139847623065599, ++SNULL, 94203605630975, 94203605635071, ++STORE, 94203605622784, 94203605630975, ++STORE, 94203605630976, 94203605635071, ++SNULL, 139847625326591, 139847625330687, ++STORE, 139847625322496, 139847625326591, ++STORE, 139847625326592, 139847625330687, ++ERASE, 139847625293824, 139847625322495, ++STORE, 94203634880512, 94203635015679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721428738048, 140737488351231, ++SNULL, 140721428746239, 140737488351231, ++STORE, 140721428738048, 140721428746239, ++STORE, 140721428606976, 140721428746239, ++STORE, 93968808378368, 93968810602495, ++SNULL, 93968808488959, 93968810602495, ++STORE, 93968808378368, 93968808488959, ++STORE, 93968808488960, 93968810602495, ++ERASE, 93968808488960, 93968810602495, ++STORE, 93968810582016, 93968810594303, ++STORE, 93968810594304, 93968810602495, ++STORE, 140397757026304, 140397759279103, ++SNULL, 140397757169663, 140397759279103, ++STORE, 140397757026304, 140397757169663, ++STORE, 140397757169664, 140397759279103, ++ERASE, 140397757169664, 140397759279103, ++STORE, 140397759266816, 140397759275007, ++STORE, 140397759275008, 140397759279103, ++STORE, 140721430368256, 140721430372351, ++STORE, 140721430355968, 140721430368255, ++STORE, 140397759238144, 140397759266815, ++STORE, 140397759229952, 140397759238143, ++STORE, 140397753229312, 140397757026303, ++SNULL, 140397753229312, 140397754888191, ++STORE, 140397754888192, 140397757026303, ++STORE, 140397753229312, 140397754888191, ++SNULL, 140397756985343, 140397757026303, ++STORE, 140397754888192, 140397756985343, ++STORE, 140397756985344, 140397757026303, ++SNULL, 140397756985344, 140397757009919, ++STORE, 140397757009920, 140397757026303, ++STORE, 140397756985344, 140397757009919, ++ERASE, 140397756985344, 140397757009919, ++STORE, 140397756985344, 140397757009919, ++ERASE, 140397757009920, 140397757026303, ++STORE, 140397757009920, 140397757026303, ++SNULL, 140397757001727, 140397757009919, ++STORE, 140397756985344, 140397757001727, ++STORE, 140397757001728, 140397757009919, ++SNULL, 93968810590207, 93968810594303, ++STORE, 93968810582016, 93968810590207, ++STORE, 93968810590208, 93968810594303, ++SNULL, 140397759270911, 140397759275007, ++STORE, 140397759266816, 140397759270911, ++STORE, 140397759270912, 140397759275007, ++ERASE, 140397759238144, 140397759266815, ++STORE, 93968837025792, 93968837160959, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721751044096, 140737488351231, ++SNULL, 140721751052287, 140737488351231, ++STORE, 140721751044096, 140721751052287, ++STORE, 140721750913024, 140721751052287, ++STORE, 94426051657728, 94426053881855, ++SNULL, 94426051768319, 94426053881855, ++STORE, 94426051657728, 94426051768319, ++STORE, 94426051768320, 94426053881855, ++ERASE, 94426051768320, 94426053881855, ++STORE, 94426053861376, 94426053873663, ++STORE, 94426053873664, 94426053881855, ++STORE, 140228456181760, 140228458434559, ++SNULL, 140228456325119, 140228458434559, ++STORE, 140228456181760, 140228456325119, ++STORE, 140228456325120, 140228458434559, ++ERASE, 140228456325120, 140228458434559, ++STORE, 140228458422272, 140228458430463, ++STORE, 140228458430464, 140228458434559, ++STORE, 140721751117824, 140721751121919, ++STORE, 140721751105536, 140721751117823, ++STORE, 140228458393600, 140228458422271, ++STORE, 140228458385408, 140228458393599, ++STORE, 140228452384768, 140228456181759, ++SNULL, 140228452384768, 140228454043647, ++STORE, 140228454043648, 140228456181759, ++STORE, 140228452384768, 140228454043647, ++SNULL, 140228456140799, 140228456181759, ++STORE, 140228454043648, 140228456140799, ++STORE, 140228456140800, 140228456181759, ++SNULL, 140228456140800, 140228456165375, ++STORE, 140228456165376, 140228456181759, ++STORE, 140228456140800, 140228456165375, ++ERASE, 140228456140800, 140228456165375, ++STORE, 140228456140800, 140228456165375, ++ERASE, 140228456165376, 140228456181759, ++STORE, 140228456165376, 140228456181759, ++SNULL, 140228456157183, 140228456165375, ++STORE, 140228456140800, 140228456157183, ++STORE, 140228456157184, 140228456165375, ++SNULL, 94426053869567, 94426053873663, ++STORE, 94426053861376, 94426053869567, ++STORE, 94426053869568, 94426053873663, ++SNULL, 140228458426367, 140228458430463, ++STORE, 140228458422272, 140228458426367, ++STORE, 140228458426368, 140228458430463, ++ERASE, 140228458393600, 140228458422271, ++STORE, 94426073681920, 94426073817087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732727623680, 140737488351231, ++SNULL, 140732727631871, 140737488351231, ++STORE, 140732727623680, 140732727631871, ++STORE, 140732727492608, 140732727631871, ++STORE, 94537485996032, 94537488220159, ++SNULL, 94537486106623, 94537488220159, ++STORE, 94537485996032, 94537486106623, ++STORE, 94537486106624, 94537488220159, ++ERASE, 94537486106624, 94537488220159, ++STORE, 94537488199680, 94537488211967, ++STORE, 94537488211968, 94537488220159, ++STORE, 140446578036736, 140446580289535, ++SNULL, 140446578180095, 140446580289535, ++STORE, 140446578036736, 140446578180095, ++STORE, 140446578180096, 140446580289535, ++ERASE, 140446578180096, 140446580289535, ++STORE, 140446580277248, 140446580285439, ++STORE, 140446580285440, 140446580289535, ++STORE, 140732727758848, 140732727762943, ++STORE, 140732727746560, 140732727758847, ++STORE, 140446580248576, 140446580277247, ++STORE, 140446580240384, 140446580248575, ++STORE, 140446574239744, 140446578036735, ++SNULL, 140446574239744, 140446575898623, ++STORE, 140446575898624, 140446578036735, ++STORE, 140446574239744, 140446575898623, ++SNULL, 140446577995775, 140446578036735, ++STORE, 140446575898624, 140446577995775, ++STORE, 140446577995776, 140446578036735, ++SNULL, 140446577995776, 140446578020351, ++STORE, 140446578020352, 140446578036735, ++STORE, 140446577995776, 140446578020351, ++ERASE, 140446577995776, 140446578020351, ++STORE, 140446577995776, 140446578020351, ++ERASE, 140446578020352, 140446578036735, ++STORE, 140446578020352, 140446578036735, ++SNULL, 140446578012159, 140446578020351, ++STORE, 140446577995776, 140446578012159, ++STORE, 140446578012160, 140446578020351, ++SNULL, 94537488207871, 94537488211967, ++STORE, 94537488199680, 94537488207871, ++STORE, 94537488207872, 94537488211967, ++SNULL, 140446580281343, 140446580285439, ++STORE, 140446580277248, 140446580281343, ++STORE, 140446580281344, 140446580285439, ++ERASE, 140446580248576, 140446580277247, ++STORE, 94537489014784, 94537489149951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728766808064, 140737488351231, ++SNULL, 140728766816255, 140737488351231, ++STORE, 140728766808064, 140728766816255, ++STORE, 140728766676992, 140728766816255, ++STORE, 94418513866752, 94418516090879, ++SNULL, 94418513977343, 94418516090879, ++STORE, 94418513866752, 94418513977343, ++STORE, 94418513977344, 94418516090879, ++ERASE, 94418513977344, 94418516090879, ++STORE, 94418516070400, 94418516082687, ++STORE, 94418516082688, 94418516090879, ++STORE, 140556479520768, 140556481773567, ++SNULL, 140556479664127, 140556481773567, ++STORE, 140556479520768, 140556479664127, ++STORE, 140556479664128, 140556481773567, ++ERASE, 140556479664128, 140556481773567, ++STORE, 140556481761280, 140556481769471, ++STORE, 140556481769472, 140556481773567, ++STORE, 140728767148032, 140728767152127, ++STORE, 140728767135744, 140728767148031, ++STORE, 140556481732608, 140556481761279, ++STORE, 140556481724416, 140556481732607, ++STORE, 140556475723776, 140556479520767, ++SNULL, 140556475723776, 140556477382655, ++STORE, 140556477382656, 140556479520767, ++STORE, 140556475723776, 140556477382655, ++SNULL, 140556479479807, 140556479520767, ++STORE, 140556477382656, 140556479479807, ++STORE, 140556479479808, 140556479520767, ++SNULL, 140556479479808, 140556479504383, ++STORE, 140556479504384, 140556479520767, ++STORE, 140556479479808, 140556479504383, ++ERASE, 140556479479808, 140556479504383, ++STORE, 140556479479808, 140556479504383, ++ERASE, 140556479504384, 140556479520767, ++STORE, 140556479504384, 140556479520767, ++SNULL, 140556479496191, 140556479504383, ++STORE, 140556479479808, 140556479496191, ++STORE, 140556479496192, 140556479504383, ++SNULL, 94418516078591, 94418516082687, ++STORE, 94418516070400, 94418516078591, ++STORE, 94418516078592, 94418516082687, ++SNULL, 140556481765375, 140556481769471, ++STORE, 140556481761280, 140556481765375, ++STORE, 140556481765376, 140556481769471, ++ERASE, 140556481732608, 140556481761279, ++STORE, 94418541113344, 94418541248511, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723945873408, 140737488351231, ++SNULL, 140723945881599, 140737488351231, ++STORE, 140723945873408, 140723945881599, ++STORE, 140723945742336, 140723945881599, ++STORE, 94543169773568, 94543171997695, ++SNULL, 94543169884159, 94543171997695, ++STORE, 94543169773568, 94543169884159, ++STORE, 94543169884160, 94543171997695, ++ERASE, 94543169884160, 94543171997695, ++STORE, 94543171977216, 94543171989503, ++STORE, 94543171989504, 94543171997695, ++STORE, 139890420883456, 139890423136255, ++SNULL, 139890421026815, 139890423136255, ++STORE, 139890420883456, 139890421026815, ++STORE, 139890421026816, 139890423136255, ++ERASE, 139890421026816, 139890423136255, ++STORE, 139890423123968, 139890423132159, ++STORE, 139890423132160, 139890423136255, ++STORE, 140723946102784, 140723946106879, ++STORE, 140723946090496, 140723946102783, ++STORE, 139890423095296, 139890423123967, ++STORE, 139890423087104, 139890423095295, ++STORE, 139890417086464, 139890420883455, ++SNULL, 139890417086464, 139890418745343, ++STORE, 139890418745344, 139890420883455, ++STORE, 139890417086464, 139890418745343, ++SNULL, 139890420842495, 139890420883455, ++STORE, 139890418745344, 139890420842495, ++STORE, 139890420842496, 139890420883455, ++SNULL, 139890420842496, 139890420867071, ++STORE, 139890420867072, 139890420883455, ++STORE, 139890420842496, 139890420867071, ++ERASE, 139890420842496, 139890420867071, ++STORE, 139890420842496, 139890420867071, ++ERASE, 139890420867072, 139890420883455, ++STORE, 139890420867072, 139890420883455, ++SNULL, 139890420858879, 139890420867071, ++STORE, 139890420842496, 139890420858879, ++STORE, 139890420858880, 139890420867071, ++SNULL, 94543171985407, 94543171989503, ++STORE, 94543171977216, 94543171985407, ++STORE, 94543171985408, 94543171989503, ++SNULL, 139890423128063, 139890423132159, ++STORE, 139890423123968, 139890423128063, ++STORE, 139890423128064, 139890423132159, ++ERASE, 139890423095296, 139890423123967, ++STORE, 94543197097984, 94543197233151, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736205979648, 140737488351231, ++SNULL, 140736205987839, 140737488351231, ++STORE, 140736205979648, 140736205987839, ++STORE, 140736205848576, 140736205987839, ++STORE, 94913209913344, 94913212137471, ++SNULL, 94913210023935, 94913212137471, ++STORE, 94913209913344, 94913210023935, ++STORE, 94913210023936, 94913212137471, ++ERASE, 94913210023936, 94913212137471, ++STORE, 94913212116992, 94913212129279, ++STORE, 94913212129280, 94913212137471, ++STORE, 140006323052544, 140006325305343, ++SNULL, 140006323195903, 140006325305343, ++STORE, 140006323052544, 140006323195903, ++STORE, 140006323195904, 140006325305343, ++ERASE, 140006323195904, 140006325305343, ++STORE, 140006325293056, 140006325301247, ++STORE, 140006325301248, 140006325305343, ++STORE, 140736206716928, 140736206721023, ++STORE, 140736206704640, 140736206716927, ++STORE, 140006325264384, 140006325293055, ++STORE, 140006325256192, 140006325264383, ++STORE, 140006319255552, 140006323052543, ++SNULL, 140006319255552, 140006320914431, ++STORE, 140006320914432, 140006323052543, ++STORE, 140006319255552, 140006320914431, ++SNULL, 140006323011583, 140006323052543, ++STORE, 140006320914432, 140006323011583, ++STORE, 140006323011584, 140006323052543, ++SNULL, 140006323011584, 140006323036159, ++STORE, 140006323036160, 140006323052543, ++STORE, 140006323011584, 140006323036159, ++ERASE, 140006323011584, 140006323036159, ++STORE, 140006323011584, 140006323036159, ++ERASE, 140006323036160, 140006323052543, ++STORE, 140006323036160, 140006323052543, ++SNULL, 140006323027967, 140006323036159, ++STORE, 140006323011584, 140006323027967, ++STORE, 140006323027968, 140006323036159, ++SNULL, 94913212125183, 94913212129279, ++STORE, 94913212116992, 94913212125183, ++STORE, 94913212125184, 94913212129279, ++SNULL, 140006325297151, 140006325301247, ++STORE, 140006325293056, 140006325297151, ++STORE, 140006325297152, 140006325301247, ++ERASE, 140006325264384, 140006325293055, ++STORE, 94913239932928, 94913240068095, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726926897152, 140737488351231, ++SNULL, 140726926905343, 140737488351231, ++STORE, 140726926897152, 140726926905343, ++STORE, 140726926766080, 140726926905343, ++STORE, 94213246820352, 94213249044479, ++SNULL, 94213246930943, 94213249044479, ++STORE, 94213246820352, 94213246930943, ++STORE, 94213246930944, 94213249044479, ++ERASE, 94213246930944, 94213249044479, ++STORE, 94213249024000, 94213249036287, ++STORE, 94213249036288, 94213249044479, ++STORE, 140368830242816, 140368832495615, ++SNULL, 140368830386175, 140368832495615, ++STORE, 140368830242816, 140368830386175, ++STORE, 140368830386176, 140368832495615, ++ERASE, 140368830386176, 140368832495615, ++STORE, 140368832483328, 140368832491519, ++STORE, 140368832491520, 140368832495615, ++STORE, 140726926999552, 140726927003647, ++STORE, 140726926987264, 140726926999551, ++STORE, 140368832454656, 140368832483327, ++STORE, 140368832446464, 140368832454655, ++STORE, 140368826445824, 140368830242815, ++SNULL, 140368826445824, 140368828104703, ++STORE, 140368828104704, 140368830242815, ++STORE, 140368826445824, 140368828104703, ++SNULL, 140368830201855, 140368830242815, ++STORE, 140368828104704, 140368830201855, ++STORE, 140368830201856, 140368830242815, ++SNULL, 140368830201856, 140368830226431, ++STORE, 140368830226432, 140368830242815, ++STORE, 140368830201856, 140368830226431, ++ERASE, 140368830201856, 140368830226431, ++STORE, 140368830201856, 140368830226431, ++ERASE, 140368830226432, 140368830242815, ++STORE, 140368830226432, 140368830242815, ++SNULL, 140368830218239, 140368830226431, ++STORE, 140368830201856, 140368830218239, ++STORE, 140368830218240, 140368830226431, ++SNULL, 94213249032191, 94213249036287, ++STORE, 94213249024000, 94213249032191, ++STORE, 94213249032192, 94213249036287, ++SNULL, 140368832487423, 140368832491519, ++STORE, 140368832483328, 140368832487423, ++STORE, 140368832487424, 140368832491519, ++ERASE, 140368832454656, 140368832483327, ++STORE, 94213267435520, 94213267570687, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728954130432, 140737488351231, ++SNULL, 140728954138623, 140737488351231, ++STORE, 140728954130432, 140728954138623, ++STORE, 140728953999360, 140728954138623, ++STORE, 94672570966016, 94672573190143, ++SNULL, 94672571076607, 94672573190143, ++STORE, 94672570966016, 94672571076607, ++STORE, 94672571076608, 94672573190143, ++ERASE, 94672571076608, 94672573190143, ++STORE, 94672573169664, 94672573181951, ++STORE, 94672573181952, 94672573190143, ++STORE, 140201696735232, 140201698988031, ++SNULL, 140201696878591, 140201698988031, ++STORE, 140201696735232, 140201696878591, ++STORE, 140201696878592, 140201698988031, ++ERASE, 140201696878592, 140201698988031, ++STORE, 140201698975744, 140201698983935, ++STORE, 140201698983936, 140201698988031, ++STORE, 140728954163200, 140728954167295, ++STORE, 140728954150912, 140728954163199, ++STORE, 140201698947072, 140201698975743, ++STORE, 140201698938880, 140201698947071, ++STORE, 140201692938240, 140201696735231, ++SNULL, 140201692938240, 140201694597119, ++STORE, 140201694597120, 140201696735231, ++STORE, 140201692938240, 140201694597119, ++SNULL, 140201696694271, 140201696735231, ++STORE, 140201694597120, 140201696694271, ++STORE, 140201696694272, 140201696735231, ++SNULL, 140201696694272, 140201696718847, ++STORE, 140201696718848, 140201696735231, ++STORE, 140201696694272, 140201696718847, ++ERASE, 140201696694272, 140201696718847, ++STORE, 140201696694272, 140201696718847, ++ERASE, 140201696718848, 140201696735231, ++STORE, 140201696718848, 140201696735231, ++SNULL, 140201696710655, 140201696718847, ++STORE, 140201696694272, 140201696710655, ++STORE, 140201696710656, 140201696718847, ++SNULL, 94672573177855, 94672573181951, ++STORE, 94672573169664, 94672573177855, ++STORE, 94672573177856, 94672573181951, ++SNULL, 140201698979839, 140201698983935, ++STORE, 140201698975744, 140201698979839, ++STORE, 140201698979840, 140201698983935, ++ERASE, 140201698947072, 140201698975743, ++STORE, 94672595689472, 94672595824639, ++STORE, 94114394132480, 94114394345471, ++STORE, 94114396442624, 94114396446719, ++STORE, 94114396446720, 94114396454911, ++STORE, 94114396454912, 94114396467199, ++STORE, 94114421575680, 94114428256255, ++STORE, 139934313955328, 139934315614207, ++STORE, 139934315614208, 139934317711359, ++STORE, 139934317711360, 139934317727743, ++STORE, 139934317727744, 139934317735935, ++STORE, 139934317735936, 139934317752319, ++STORE, 139934317752320, 139934317764607, ++STORE, 139934317764608, 139934319857663, ++STORE, 139934319857664, 139934319861759, ++STORE, 139934319861760, 139934319865855, ++STORE, 139934319865856, 139934320009215, ++STORE, 139934320377856, 139934322061311, ++STORE, 139934322061312, 139934322077695, ++STORE, 139934322106368, 139934322110463, ++STORE, 139934322110464, 139934322114559, ++STORE, 139934322114560, 139934322118655, ++STORE, 140731200376832, 140731200516095, ++STORE, 140731200929792, 140731200942079, ++STORE, 140731200942080, 140731200946175, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721532362752, 140737488351231, ++SNULL, 140721532370943, 140737488351231, ++STORE, 140721532362752, 140721532370943, ++STORE, 140721532231680, 140721532370943, ++STORE, 94467222597632, 94467224821759, ++SNULL, 94467222708223, 94467224821759, ++STORE, 94467222597632, 94467222708223, ++STORE, 94467222708224, 94467224821759, ++ERASE, 94467222708224, 94467224821759, ++STORE, 94467224801280, 94467224813567, ++STORE, 94467224813568, 94467224821759, ++STORE, 140191433543680, 140191435796479, ++SNULL, 140191433687039, 140191435796479, ++STORE, 140191433543680, 140191433687039, ++STORE, 140191433687040, 140191435796479, ++ERASE, 140191433687040, 140191435796479, ++STORE, 140191435784192, 140191435792383, ++STORE, 140191435792384, 140191435796479, ++STORE, 140721533034496, 140721533038591, ++STORE, 140721533022208, 140721533034495, ++STORE, 140191435755520, 140191435784191, ++STORE, 140191435747328, 140191435755519, ++STORE, 140191429746688, 140191433543679, ++SNULL, 140191429746688, 140191431405567, ++STORE, 140191431405568, 140191433543679, ++STORE, 140191429746688, 140191431405567, ++SNULL, 140191433502719, 140191433543679, ++STORE, 140191431405568, 140191433502719, ++STORE, 140191433502720, 140191433543679, ++SNULL, 140191433502720, 140191433527295, ++STORE, 140191433527296, 140191433543679, ++STORE, 140191433502720, 140191433527295, ++ERASE, 140191433502720, 140191433527295, ++STORE, 140191433502720, 140191433527295, ++ERASE, 140191433527296, 140191433543679, ++STORE, 140191433527296, 140191433543679, ++SNULL, 140191433519103, 140191433527295, ++STORE, 140191433502720, 140191433519103, ++STORE, 140191433519104, 140191433527295, ++SNULL, 94467224809471, 94467224813567, ++STORE, 94467224801280, 94467224809471, ++STORE, 94467224809472, 94467224813567, ++SNULL, 140191435788287, 140191435792383, ++STORE, 140191435784192, 140191435788287, ++STORE, 140191435788288, 140191435792383, ++ERASE, 140191435755520, 140191435784191, ++STORE, 94467251847168, 94467251982335, ++STORE, 94367895400448, 94367895613439, ++STORE, 94367897710592, 94367897714687, ++STORE, 94367897714688, 94367897722879, ++STORE, 94367897722880, 94367897735167, ++STORE, 94367925264384, 94367926861823, ++STORE, 139801317548032, 139801319206911, ++STORE, 139801319206912, 139801321304063, ++STORE, 139801321304064, 139801321320447, ++STORE, 139801321320448, 139801321328639, ++STORE, 139801321328640, 139801321345023, ++STORE, 139801321345024, 139801321357311, ++STORE, 139801321357312, 139801323450367, ++STORE, 139801323450368, 139801323454463, ++STORE, 139801323454464, 139801323458559, ++STORE, 139801323458560, 139801323601919, ++STORE, 139801323970560, 139801325654015, ++STORE, 139801325654016, 139801325670399, ++STORE, 139801325699072, 139801325703167, ++STORE, 139801325703168, 139801325707263, ++STORE, 139801325707264, 139801325711359, ++STORE, 140724442861568, 140724443000831, ++STORE, 140724443611136, 140724443623423, ++STORE, 140724443623424, 140724443627519, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731353149440, 140737488351231, ++SNULL, 140731353157631, 140737488351231, ++STORE, 140731353149440, 140731353157631, ++STORE, 140731353018368, 140731353157631, ++STORE, 94310379503616, 94310381838335, ++SNULL, 94310379716607, 94310381838335, ++STORE, 94310379503616, 94310379716607, ++STORE, 94310379716608, 94310381838335, ++ERASE, 94310379716608, 94310381838335, ++STORE, 94310381813760, 94310381826047, ++STORE, 94310381826048, 94310381838335, ++STORE, 140515434659840, 140515436912639, ++SNULL, 140515434803199, 140515436912639, ++STORE, 140515434659840, 140515434803199, ++STORE, 140515434803200, 140515436912639, ++ERASE, 140515434803200, 140515436912639, ++STORE, 140515436900352, 140515436908543, ++STORE, 140515436908544, 140515436912639, ++STORE, 140731353886720, 140731353890815, ++STORE, 140731353874432, 140731353886719, ++STORE, 140515436871680, 140515436900351, ++STORE, 140515436863488, 140515436871679, ++STORE, 140515432546304, 140515434659839, ++SNULL, 140515432546304, 140515432558591, ++STORE, 140515432558592, 140515434659839, ++STORE, 140515432546304, 140515432558591, ++SNULL, 140515434651647, 140515434659839, ++STORE, 140515432558592, 140515434651647, ++STORE, 140515434651648, 140515434659839, ++ERASE, 140515434651648, 140515434659839, ++STORE, 140515434651648, 140515434659839, ++STORE, 140515428749312, 140515432546303, ++SNULL, 140515428749312, 140515430408191, ++STORE, 140515430408192, 140515432546303, ++STORE, 140515428749312, 140515430408191, ++SNULL, 140515432505343, 140515432546303, ++STORE, 140515430408192, 140515432505343, ++STORE, 140515432505344, 140515432546303, ++SNULL, 140515432505344, 140515432529919, ++STORE, 140515432529920, 140515432546303, ++STORE, 140515432505344, 140515432529919, ++ERASE, 140515432505344, 140515432529919, ++STORE, 140515432505344, 140515432529919, ++ERASE, 140515432529920, 140515432546303, ++STORE, 140515432529920, 140515432546303, ++STORE, 140515436855296, 140515436871679, ++SNULL, 140515432521727, 140515432529919, ++STORE, 140515432505344, 140515432521727, ++STORE, 140515432521728, 140515432529919, ++SNULL, 140515434655743, 140515434659839, ++STORE, 140515434651648, 140515434655743, ++STORE, 140515434655744, 140515434659839, ++SNULL, 94310381817855, 94310381826047, ++STORE, 94310381813760, 94310381817855, ++STORE, 94310381817856, 94310381826047, ++SNULL, 140515436904447, 140515436908543, ++STORE, 140515436900352, 140515436904447, ++STORE, 140515436904448, 140515436908543, ++ERASE, 140515436871680, 140515436900351, ++STORE, 94310395457536, 94310395592703, ++STORE, 140515435171840, 140515436855295, ++STORE, 94310395457536, 94310395727871, ++STORE, 94310395457536, 94310395863039, ++STORE, 94310395457536, 94310396047359, ++SNULL, 94310396022783, 94310396047359, ++STORE, 94310395457536, 94310396022783, ++STORE, 94310396022784, 94310396047359, ++ERASE, 94310396022784, 94310396047359, ++STORE, 94310395457536, 94310396157951, ++STORE, 94310395457536, 94310396293119, ++SNULL, 94310396276735, 94310396293119, ++STORE, 94310395457536, 94310396276735, ++STORE, 94310396276736, 94310396293119, ++ERASE, 94310396276736, 94310396293119, ++STORE, 94310395457536, 94310396411903, ++SNULL, 94310396383231, 94310396411903, ++STORE, 94310395457536, 94310396383231, ++STORE, 94310396383232, 94310396411903, ++ERASE, 94310396383232, 94310396411903, ++STORE, 94310395457536, 94310396522495, ++STORE, 94310395457536, 94310396674047, ++SNULL, 94310396657663, 94310396674047, ++STORE, 94310395457536, 94310396657663, ++STORE, 94310396657664, 94310396674047, ++ERASE, 94310396657664, 94310396674047, ++SNULL, 94310396624895, 94310396657663, ++STORE, 94310395457536, 94310396624895, ++STORE, 94310396624896, 94310396657663, ++ERASE, 94310396624896, 94310396657663, ++STORE, 94310395457536, 94310396776447, ++SNULL, 94310396764159, 94310396776447, ++STORE, 94310395457536, 94310396764159, ++STORE, 94310396764160, 94310396776447, ++ERASE, 94310396764160, 94310396776447, ++SNULL, 94310396739583, 94310396764159, ++STORE, 94310395457536, 94310396739583, ++STORE, 94310396739584, 94310396764159, ++ERASE, 94310396739584, 94310396764159, ++STORE, 94310395457536, 94310396882943, ++STORE, 94310395457536, 94310397018111, ++STORE, 94310395457536, 94310397161471, ++STORE, 94310395457536, 94310397300735, ++SNULL, 94310397292543, 94310397300735, ++STORE, 94310395457536, 94310397292543, ++STORE, 94310397292544, 94310397300735, ++ERASE, 94310397292544, 94310397300735, ++STORE, 94359222210560, 94359222423551, ++STORE, 94359224520704, 94359224524799, ++STORE, 94359224524800, 94359224532991, ++STORE, 94359224532992, 94359224545279, ++STORE, 94359238348800, 94359239385087, ++STORE, 140675699838976, 140675701497855, ++STORE, 140675701497856, 140675703595007, ++STORE, 140675703595008, 140675703611391, ++STORE, 140675703611392, 140675703619583, ++STORE, 140675703619584, 140675703635967, ++STORE, 140675703635968, 140675703648255, ++STORE, 140675703648256, 140675705741311, ++STORE, 140675705741312, 140675705745407, ++STORE, 140675705745408, 140675705749503, ++STORE, 140675705749504, 140675705892863, ++STORE, 140675706261504, 140675707944959, ++STORE, 140675707944960, 140675707961343, ++STORE, 140675707990016, 140675707994111, ++STORE, 140675707994112, 140675707998207, ++STORE, 140675707998208, 140675708002303, ++STORE, 140721324634112, 140721324773375, ++STORE, 140721324810240, 140721324822527, ++STORE, 140721324822528, 140721324826623, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724099678208, 140737488351231, ++SNULL, 140724099686399, 140737488351231, ++STORE, 140724099678208, 140724099686399, ++STORE, 140724099547136, 140724099686399, ++STORE, 94586638516224, 94586640850943, ++SNULL, 94586638729215, 94586640850943, ++STORE, 94586638516224, 94586638729215, ++STORE, 94586638729216, 94586640850943, ++ERASE, 94586638729216, 94586640850943, ++STORE, 94586640826368, 94586640838655, ++STORE, 94586640838656, 94586640850943, ++STORE, 140371033796608, 140371036049407, ++SNULL, 140371033939967, 140371036049407, ++STORE, 140371033796608, 140371033939967, ++STORE, 140371033939968, 140371036049407, ++ERASE, 140371033939968, 140371036049407, ++STORE, 140371036037120, 140371036045311, ++STORE, 140371036045312, 140371036049407, ++STORE, 140724100001792, 140724100005887, ++STORE, 140724099989504, 140724100001791, ++STORE, 140371036008448, 140371036037119, ++STORE, 140371036000256, 140371036008447, ++STORE, 140371031683072, 140371033796607, ++SNULL, 140371031683072, 140371031695359, ++STORE, 140371031695360, 140371033796607, ++STORE, 140371031683072, 140371031695359, ++SNULL, 140371033788415, 140371033796607, ++STORE, 140371031695360, 140371033788415, ++STORE, 140371033788416, 140371033796607, ++ERASE, 140371033788416, 140371033796607, ++STORE, 140371033788416, 140371033796607, ++STORE, 140371027886080, 140371031683071, ++SNULL, 140371027886080, 140371029544959, ++STORE, 140371029544960, 140371031683071, ++STORE, 140371027886080, 140371029544959, ++SNULL, 140371031642111, 140371031683071, ++STORE, 140371029544960, 140371031642111, ++STORE, 140371031642112, 140371031683071, ++SNULL, 140371031642112, 140371031666687, ++STORE, 140371031666688, 140371031683071, ++STORE, 140371031642112, 140371031666687, ++ERASE, 140371031642112, 140371031666687, ++STORE, 140371031642112, 140371031666687, ++ERASE, 140371031666688, 140371031683071, ++STORE, 140371031666688, 140371031683071, ++STORE, 140371035992064, 140371036008447, ++SNULL, 140371031658495, 140371031666687, ++STORE, 140371031642112, 140371031658495, ++STORE, 140371031658496, 140371031666687, ++SNULL, 140371033792511, 140371033796607, ++STORE, 140371033788416, 140371033792511, ++STORE, 140371033792512, 140371033796607, ++SNULL, 94586640830463, 94586640838655, ++STORE, 94586640826368, 94586640830463, ++STORE, 94586640830464, 94586640838655, ++SNULL, 140371036041215, 140371036045311, ++STORE, 140371036037120, 140371036041215, ++STORE, 140371036041216, 140371036045311, ++ERASE, 140371036008448, 140371036037119, ++STORE, 94586663849984, 94586663985151, ++STORE, 140371034308608, 140371035992063, ++STORE, 94586663849984, 94586664120319, ++STORE, 94586663849984, 94586664255487, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727532937216, 140737488351231, ++SNULL, 140727532945407, 140737488351231, ++STORE, 140727532937216, 140727532945407, ++STORE, 140727532806144, 140727532945407, ++STORE, 94849780191232, 94849782525951, ++SNULL, 94849780404223, 94849782525951, ++STORE, 94849780191232, 94849780404223, ++STORE, 94849780404224, 94849782525951, ++ERASE, 94849780404224, 94849782525951, ++STORE, 94849782501376, 94849782513663, ++STORE, 94849782513664, 94849782525951, ++STORE, 140382070218752, 140382072471551, ++SNULL, 140382070362111, 140382072471551, ++STORE, 140382070218752, 140382070362111, ++STORE, 140382070362112, 140382072471551, ++ERASE, 140382070362112, 140382072471551, ++STORE, 140382072459264, 140382072467455, ++STORE, 140382072467456, 140382072471551, ++STORE, 140727533092864, 140727533096959, ++STORE, 140727533080576, 140727533092863, ++STORE, 140382072430592, 140382072459263, ++STORE, 140382072422400, 140382072430591, ++STORE, 140382068105216, 140382070218751, ++SNULL, 140382068105216, 140382068117503, ++STORE, 140382068117504, 140382070218751, ++STORE, 140382068105216, 140382068117503, ++SNULL, 140382070210559, 140382070218751, ++STORE, 140382068117504, 140382070210559, ++STORE, 140382070210560, 140382070218751, ++ERASE, 140382070210560, 140382070218751, ++STORE, 140382070210560, 140382070218751, ++STORE, 140382064308224, 140382068105215, ++SNULL, 140382064308224, 140382065967103, ++STORE, 140382065967104, 140382068105215, ++STORE, 140382064308224, 140382065967103, ++SNULL, 140382068064255, 140382068105215, ++STORE, 140382065967104, 140382068064255, ++STORE, 140382068064256, 140382068105215, ++SNULL, 140382068064256, 140382068088831, ++STORE, 140382068088832, 140382068105215, ++STORE, 140382068064256, 140382068088831, ++ERASE, 140382068064256, 140382068088831, ++STORE, 140382068064256, 140382068088831, ++ERASE, 140382068088832, 140382068105215, ++STORE, 140382068088832, 140382068105215, ++STORE, 140382072414208, 140382072430591, ++SNULL, 140382068080639, 140382068088831, ++STORE, 140382068064256, 140382068080639, ++STORE, 140382068080640, 140382068088831, ++SNULL, 140382070214655, 140382070218751, ++STORE, 140382070210560, 140382070214655, ++STORE, 140382070214656, 140382070218751, ++SNULL, 94849782505471, 94849782513663, ++STORE, 94849782501376, 94849782505471, ++STORE, 94849782505472, 94849782513663, ++SNULL, 140382072463359, 140382072467455, ++STORE, 140382072459264, 140382072463359, ++STORE, 140382072463360, 140382072467455, ++ERASE, 140382072430592, 140382072459263, ++STORE, 94849782845440, 94849782980607, ++STORE, 140382070730752, 140382072414207, ++STORE, 94849782845440, 94849783115775, ++STORE, 94849782845440, 94849783250943, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722594377728, 140737488351231, ++SNULL, 140722594385919, 140737488351231, ++STORE, 140722594377728, 140722594385919, ++STORE, 140722594246656, 140722594385919, ++STORE, 94421466353664, 94421468577791, ++SNULL, 94421466464255, 94421468577791, ++STORE, 94421466353664, 94421466464255, ++STORE, 94421466464256, 94421468577791, ++ERASE, 94421466464256, 94421468577791, ++STORE, 94421468557312, 94421468569599, ++STORE, 94421468569600, 94421468577791, ++STORE, 140345458057216, 140345460310015, ++SNULL, 140345458200575, 140345460310015, ++STORE, 140345458057216, 140345458200575, ++STORE, 140345458200576, 140345460310015, ++ERASE, 140345458200576, 140345460310015, ++STORE, 140345460297728, 140345460305919, ++STORE, 140345460305920, 140345460310015, ++STORE, 140722595557376, 140722595561471, ++STORE, 140722595545088, 140722595557375, ++STORE, 140345460269056, 140345460297727, ++STORE, 140345460260864, 140345460269055, ++STORE, 140345454260224, 140345458057215, ++SNULL, 140345454260224, 140345455919103, ++STORE, 140345455919104, 140345458057215, ++STORE, 140345454260224, 140345455919103, ++SNULL, 140345458016255, 140345458057215, ++STORE, 140345455919104, 140345458016255, ++STORE, 140345458016256, 140345458057215, ++SNULL, 140345458016256, 140345458040831, ++STORE, 140345458040832, 140345458057215, ++STORE, 140345458016256, 140345458040831, ++ERASE, 140345458016256, 140345458040831, ++STORE, 140345458016256, 140345458040831, ++ERASE, 140345458040832, 140345458057215, ++STORE, 140345458040832, 140345458057215, ++SNULL, 140345458032639, 140345458040831, ++STORE, 140345458016256, 140345458032639, ++STORE, 140345458032640, 140345458040831, ++SNULL, 94421468565503, 94421468569599, ++STORE, 94421468557312, 94421468565503, ++STORE, 94421468565504, 94421468569599, ++SNULL, 140345460301823, 140345460305919, ++STORE, 140345460297728, 140345460301823, ++STORE, 140345460301824, 140345460305919, ++ERASE, 140345460269056, 140345460297727, ++STORE, 94421496004608, 94421496139775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726096302080, 140737488351231, ++SNULL, 140726096310271, 140737488351231, ++STORE, 140726096302080, 140726096310271, ++STORE, 140726096171008, 140726096310271, ++STORE, 94101992124416, 94101994459135, ++SNULL, 94101992337407, 94101994459135, ++STORE, 94101992124416, 94101992337407, ++STORE, 94101992337408, 94101994459135, ++ERASE, 94101992337408, 94101994459135, ++STORE, 94101994434560, 94101994446847, ++STORE, 94101994446848, 94101994459135, ++STORE, 140192085594112, 140192087846911, ++SNULL, 140192085737471, 140192087846911, ++STORE, 140192085594112, 140192085737471, ++STORE, 140192085737472, 140192087846911, ++ERASE, 140192085737472, 140192087846911, ++STORE, 140192087834624, 140192087842815, ++STORE, 140192087842816, 140192087846911, ++STORE, 140726096375808, 140726096379903, ++STORE, 140726096363520, 140726096375807, ++STORE, 140192087805952, 140192087834623, ++STORE, 140192087797760, 140192087805951, ++STORE, 140192083480576, 140192085594111, ++SNULL, 140192083480576, 140192083492863, ++STORE, 140192083492864, 140192085594111, ++STORE, 140192083480576, 140192083492863, ++SNULL, 140192085585919, 140192085594111, ++STORE, 140192083492864, 140192085585919, ++STORE, 140192085585920, 140192085594111, ++ERASE, 140192085585920, 140192085594111, ++STORE, 140192085585920, 140192085594111, ++STORE, 140192079683584, 140192083480575, ++SNULL, 140192079683584, 140192081342463, ++STORE, 140192081342464, 140192083480575, ++STORE, 140192079683584, 140192081342463, ++SNULL, 140192083439615, 140192083480575, ++STORE, 140192081342464, 140192083439615, ++STORE, 140192083439616, 140192083480575, ++SNULL, 140192083439616, 140192083464191, ++STORE, 140192083464192, 140192083480575, ++STORE, 140192083439616, 140192083464191, ++ERASE, 140192083439616, 140192083464191, ++STORE, 140192083439616, 140192083464191, ++ERASE, 140192083464192, 140192083480575, ++STORE, 140192083464192, 140192083480575, ++STORE, 140192087789568, 140192087805951, ++SNULL, 140192083455999, 140192083464191, ++STORE, 140192083439616, 140192083455999, ++STORE, 140192083456000, 140192083464191, ++SNULL, 140192085590015, 140192085594111, ++STORE, 140192085585920, 140192085590015, ++STORE, 140192085590016, 140192085594111, ++SNULL, 94101994438655, 94101994446847, ++STORE, 94101994434560, 94101994438655, ++STORE, 94101994438656, 94101994446847, ++SNULL, 140192087838719, 140192087842815, ++STORE, 140192087834624, 140192087838719, ++STORE, 140192087838720, 140192087842815, ++ERASE, 140192087805952, 140192087834623, ++STORE, 94102011887616, 94102012022783, ++STORE, 140192086106112, 140192087789567, ++STORE, 94102011887616, 94102012157951, ++STORE, 94102011887616, 94102012293119, ++STORE, 94102011887616, 94102012440575, ++SNULL, 94102012428287, 94102012440575, ++STORE, 94102011887616, 94102012428287, ++STORE, 94102012428288, 94102012440575, ++ERASE, 94102012428288, 94102012440575, ++STORE, 94102011887616, 94102012579839, ++STORE, 94102011887616, 94102012715007, ++SNULL, 94102012694527, 94102012715007, ++STORE, 94102011887616, 94102012694527, ++STORE, 94102012694528, 94102012715007, ++ERASE, 94102012694528, 94102012715007, ++STORE, 94102011887616, 94102012833791, ++STORE, 94102011887616, 94102012968959, ++SNULL, 94102012927999, 94102012968959, ++STORE, 94102011887616, 94102012927999, ++STORE, 94102012928000, 94102012968959, ++ERASE, 94102012928000, 94102012968959, ++STORE, 94102011887616, 94102013091839, ++SNULL, 94102013075455, 94102013091839, ++STORE, 94102011887616, 94102013075455, ++STORE, 94102013075456, 94102013091839, ++ERASE, 94102013075456, 94102013091839, ++STORE, 94102011887616, 94102013210623, ++STORE, 94102011887616, 94102013345791, ++STORE, 93968727965696, 93968728178687, ++STORE, 93968730275840, 93968730279935, ++STORE, 93968730279936, 93968730288127, ++STORE, 93968730288128, 93968730300415, ++STORE, 93968731140096, 93968732704767, ++STORE, 140588443168768, 140588444827647, ++STORE, 140588444827648, 140588446924799, ++STORE, 140588446924800, 140588446941183, ++STORE, 140588446941184, 140588446949375, ++STORE, 140588446949376, 140588446965759, ++STORE, 140588446965760, 140588446978047, ++STORE, 140588446978048, 140588449071103, ++STORE, 140588449071104, 140588449075199, ++STORE, 140588449075200, 140588449079295, ++STORE, 140588449079296, 140588449222655, ++STORE, 140588449591296, 140588451274751, ++STORE, 140588451274752, 140588451291135, ++STORE, 140588451319808, 140588451323903, ++STORE, 140588451323904, 140588451327999, ++STORE, 140588451328000, 140588451332095, ++STORE, 140733877239808, 140733877379071, ++STORE, 140733878702080, 140733878714367, ++STORE, 140733878714368, 140733878718463, ++STORE, 93968727965696, 93968728178687, ++STORE, 93968730275840, 93968730279935, ++STORE, 93968730279936, 93968730288127, ++STORE, 93968730288128, 93968730300415, ++STORE, 93968731140096, 93968732991487, ++STORE, 140588443168768, 140588444827647, ++STORE, 140588444827648, 140588446924799, ++STORE, 140588446924800, 140588446941183, ++STORE, 140588446941184, 140588446949375, ++STORE, 140588446949376, 140588446965759, ++STORE, 140588446965760, 140588446978047, ++STORE, 140588446978048, 140588449071103, ++STORE, 140588449071104, 140588449075199, ++STORE, 140588449075200, 140588449079295, ++STORE, 140588449079296, 140588449222655, ++STORE, 140588449591296, 140588451274751, ++STORE, 140588451274752, 140588451291135, ++STORE, 140588451319808, 140588451323903, ++STORE, 140588451323904, 140588451327999, ++STORE, 140588451328000, 140588451332095, ++STORE, 140733877239808, 140733877379071, ++STORE, 140733878702080, 140733878714367, ++STORE, 140733878714368, 140733878718463, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733054472192, 140737488351231, ++SNULL, 140733054480383, 140737488351231, ++STORE, 140733054472192, 140733054480383, ++STORE, 140733054341120, 140733054480383, ++STORE, 93992873623552, 93992875847679, ++SNULL, 93992873734143, 93992875847679, ++STORE, 93992873623552, 93992873734143, ++STORE, 93992873734144, 93992875847679, ++ERASE, 93992873734144, 93992875847679, ++STORE, 93992875827200, 93992875839487, ++STORE, 93992875839488, 93992875847679, ++STORE, 139790881488896, 139790883741695, ++SNULL, 139790881632255, 139790883741695, ++STORE, 139790881488896, 139790881632255, ++STORE, 139790881632256, 139790883741695, ++ERASE, 139790881632256, 139790883741695, ++STORE, 139790883729408, 139790883737599, ++STORE, 139790883737600, 139790883741695, ++STORE, 140733054754816, 140733054758911, ++STORE, 140733054742528, 140733054754815, ++STORE, 139790883700736, 139790883729407, ++STORE, 139790883692544, 139790883700735, ++STORE, 139790877691904, 139790881488895, ++SNULL, 139790877691904, 139790879350783, ++STORE, 139790879350784, 139790881488895, ++STORE, 139790877691904, 139790879350783, ++SNULL, 139790881447935, 139790881488895, ++STORE, 139790879350784, 139790881447935, ++STORE, 139790881447936, 139790881488895, ++SNULL, 139790881447936, 139790881472511, ++STORE, 139790881472512, 139790881488895, ++STORE, 139790881447936, 139790881472511, ++ERASE, 139790881447936, 139790881472511, ++STORE, 139790881447936, 139790881472511, ++ERASE, 139790881472512, 139790881488895, ++STORE, 139790881472512, 139790881488895, ++SNULL, 139790881464319, 139790881472511, ++STORE, 139790881447936, 139790881464319, ++STORE, 139790881464320, 139790881472511, ++SNULL, 93992875835391, 93992875839487, ++STORE, 93992875827200, 93992875835391, ++STORE, 93992875835392, 93992875839487, ++SNULL, 139790883733503, 139790883737599, ++STORE, 139790883729408, 139790883733503, ++STORE, 139790883733504, 139790883737599, ++ERASE, 139790883700736, 139790883729407, ++STORE, 93992877031424, 93992877166591, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728550887424, 140737488351231, ++SNULL, 140728550895615, 140737488351231, ++STORE, 140728550887424, 140728550895615, ++STORE, 140728550756352, 140728550895615, ++STORE, 94707634077696, 94707636301823, ++SNULL, 94707634188287, 94707636301823, ++STORE, 94707634077696, 94707634188287, ++STORE, 94707634188288, 94707636301823, ++ERASE, 94707634188288, 94707636301823, ++STORE, 94707636281344, 94707636293631, ++STORE, 94707636293632, 94707636301823, ++STORE, 140553545666560, 140553547919359, ++SNULL, 140553545809919, 140553547919359, ++STORE, 140553545666560, 140553545809919, ++STORE, 140553545809920, 140553547919359, ++ERASE, 140553545809920, 140553547919359, ++STORE, 140553547907072, 140553547915263, ++STORE, 140553547915264, 140553547919359, ++STORE, 140728552374272, 140728552378367, ++STORE, 140728552361984, 140728552374271, ++STORE, 140553547878400, 140553547907071, ++STORE, 140553547870208, 140553547878399, ++STORE, 140553541869568, 140553545666559, ++SNULL, 140553541869568, 140553543528447, ++STORE, 140553543528448, 140553545666559, ++STORE, 140553541869568, 140553543528447, ++SNULL, 140553545625599, 140553545666559, ++STORE, 140553543528448, 140553545625599, ++STORE, 140553545625600, 140553545666559, ++SNULL, 140553545625600, 140553545650175, ++STORE, 140553545650176, 140553545666559, ++STORE, 140553545625600, 140553545650175, ++ERASE, 140553545625600, 140553545650175, ++STORE, 140553545625600, 140553545650175, ++ERASE, 140553545650176, 140553545666559, ++STORE, 140553545650176, 140553545666559, ++SNULL, 140553545641983, 140553545650175, ++STORE, 140553545625600, 140553545641983, ++STORE, 140553545641984, 140553545650175, ++SNULL, 94707636289535, 94707636293631, ++STORE, 94707636281344, 94707636289535, ++STORE, 94707636289536, 94707636293631, ++SNULL, 140553547911167, 140553547915263, ++STORE, 140553547907072, 140553547911167, ++STORE, 140553547911168, 140553547915263, ++ERASE, 140553547878400, 140553547907071, ++STORE, 94707651411968, 94707651547135, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732168695808, 140737488351231, ++SNULL, 140732168703999, 140737488351231, ++STORE, 140732168695808, 140732168703999, ++STORE, 140732168564736, 140732168703999, ++STORE, 94454287859712, 94454290083839, ++SNULL, 94454287970303, 94454290083839, ++STORE, 94454287859712, 94454287970303, ++STORE, 94454287970304, 94454290083839, ++ERASE, 94454287970304, 94454290083839, ++STORE, 94454290063360, 94454290075647, ++STORE, 94454290075648, 94454290083839, ++STORE, 140564947107840, 140564949360639, ++SNULL, 140564947251199, 140564949360639, ++STORE, 140564947107840, 140564947251199, ++STORE, 140564947251200, 140564949360639, ++ERASE, 140564947251200, 140564949360639, ++STORE, 140564949348352, 140564949356543, ++STORE, 140564949356544, 140564949360639, ++STORE, 140732168843264, 140732168847359, ++STORE, 140732168830976, 140732168843263, ++STORE, 140564949319680, 140564949348351, ++STORE, 140564949311488, 140564949319679, ++STORE, 140564943310848, 140564947107839, ++SNULL, 140564943310848, 140564944969727, ++STORE, 140564944969728, 140564947107839, ++STORE, 140564943310848, 140564944969727, ++SNULL, 140564947066879, 140564947107839, ++STORE, 140564944969728, 140564947066879, ++STORE, 140564947066880, 140564947107839, ++SNULL, 140564947066880, 140564947091455, ++STORE, 140564947091456, 140564947107839, ++STORE, 140564947066880, 140564947091455, ++ERASE, 140564947066880, 140564947091455, ++STORE, 140564947066880, 140564947091455, ++ERASE, 140564947091456, 140564947107839, ++STORE, 140564947091456, 140564947107839, ++SNULL, 140564947083263, 140564947091455, ++STORE, 140564947066880, 140564947083263, ++STORE, 140564947083264, 140564947091455, ++SNULL, 94454290071551, 94454290075647, ++STORE, 94454290063360, 94454290071551, ++STORE, 94454290071552, 94454290075647, ++SNULL, 140564949352447, 140564949356543, ++STORE, 140564949348352, 140564949352447, ++STORE, 140564949352448, 140564949356543, ++ERASE, 140564949319680, 140564949348351, ++STORE, 94454316236800, 94454316371967, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735155617792, 140737488351231, ++SNULL, 140735155625983, 140737488351231, ++STORE, 140735155617792, 140735155625983, ++STORE, 140735155486720, 140735155625983, ++STORE, 93915969556480, 93915971780607, ++SNULL, 93915969667071, 93915971780607, ++STORE, 93915969556480, 93915969667071, ++STORE, 93915969667072, 93915971780607, ++ERASE, 93915969667072, 93915971780607, ++STORE, 93915971760128, 93915971772415, ++STORE, 93915971772416, 93915971780607, ++STORE, 140141164605440, 140141166858239, ++SNULL, 140141164748799, 140141166858239, ++STORE, 140141164605440, 140141164748799, ++STORE, 140141164748800, 140141166858239, ++ERASE, 140141164748800, 140141166858239, ++STORE, 140141166845952, 140141166854143, ++STORE, 140141166854144, 140141166858239, ++STORE, 140735155691520, 140735155695615, ++STORE, 140735155679232, 140735155691519, ++STORE, 140141166817280, 140141166845951, ++STORE, 140141166809088, 140141166817279, ++STORE, 140141160808448, 140141164605439, ++SNULL, 140141160808448, 140141162467327, ++STORE, 140141162467328, 140141164605439, ++STORE, 140141160808448, 140141162467327, ++SNULL, 140141164564479, 140141164605439, ++STORE, 140141162467328, 140141164564479, ++STORE, 140141164564480, 140141164605439, ++SNULL, 140141164564480, 140141164589055, ++STORE, 140141164589056, 140141164605439, ++STORE, 140141164564480, 140141164589055, ++ERASE, 140141164564480, 140141164589055, ++STORE, 140141164564480, 140141164589055, ++ERASE, 140141164589056, 140141164605439, ++STORE, 140141164589056, 140141164605439, ++SNULL, 140141164580863, 140141164589055, ++STORE, 140141164564480, 140141164580863, ++STORE, 140141164580864, 140141164589055, ++SNULL, 93915971768319, 93915971772415, ++STORE, 93915971760128, 93915971768319, ++STORE, 93915971768320, 93915971772415, ++SNULL, 140141166850047, 140141166854143, ++STORE, 140141166845952, 140141166850047, ++STORE, 140141166850048, 140141166854143, ++ERASE, 140141166817280, 140141166845951, ++STORE, 93916002775040, 93916002910207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728988409856, 140737488351231, ++SNULL, 140728988418047, 140737488351231, ++STORE, 140728988409856, 140728988418047, ++STORE, 140728988278784, 140728988418047, ++STORE, 94021634813952, 94021637038079, ++SNULL, 94021634924543, 94021637038079, ++STORE, 94021634813952, 94021634924543, ++STORE, 94021634924544, 94021637038079, ++ERASE, 94021634924544, 94021637038079, ++STORE, 94021637017600, 94021637029887, ++STORE, 94021637029888, 94021637038079, ++STORE, 140638014038016, 140638016290815, ++SNULL, 140638014181375, 140638016290815, ++STORE, 140638014038016, 140638014181375, ++STORE, 140638014181376, 140638016290815, ++ERASE, 140638014181376, 140638016290815, ++STORE, 140638016278528, 140638016286719, ++STORE, 140638016286720, 140638016290815, ++STORE, 140728988536832, 140728988540927, ++STORE, 140728988524544, 140728988536831, ++STORE, 140638016249856, 140638016278527, ++STORE, 140638016241664, 140638016249855, ++STORE, 140638010241024, 140638014038015, ++SNULL, 140638010241024, 140638011899903, ++STORE, 140638011899904, 140638014038015, ++STORE, 140638010241024, 140638011899903, ++SNULL, 140638013997055, 140638014038015, ++STORE, 140638011899904, 140638013997055, ++STORE, 140638013997056, 140638014038015, ++SNULL, 140638013997056, 140638014021631, ++STORE, 140638014021632, 140638014038015, ++STORE, 140638013997056, 140638014021631, ++ERASE, 140638013997056, 140638014021631, ++STORE, 140638013997056, 140638014021631, ++ERASE, 140638014021632, 140638014038015, ++STORE, 140638014021632, 140638014038015, ++SNULL, 140638014013439, 140638014021631, ++STORE, 140638013997056, 140638014013439, ++STORE, 140638014013440, 140638014021631, ++SNULL, 94021637025791, 94021637029887, ++STORE, 94021637017600, 94021637025791, ++STORE, 94021637025792, 94021637029887, ++SNULL, 140638016282623, 140638016286719, ++STORE, 140638016278528, 140638016282623, ++STORE, 140638016282624, 140638016286719, ++ERASE, 140638016249856, 140638016278527, ++STORE, 94021643124736, 94021643259903, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731219275776, 140737488351231, ++SNULL, 140731219283967, 140737488351231, ++STORE, 140731219275776, 140731219283967, ++STORE, 140731219144704, 140731219283967, ++STORE, 93888803647488, 93888805871615, ++SNULL, 93888803758079, 93888805871615, ++STORE, 93888803647488, 93888803758079, ++STORE, 93888803758080, 93888805871615, ++ERASE, 93888803758080, 93888805871615, ++STORE, 93888805851136, 93888805863423, ++STORE, 93888805863424, 93888805871615, ++STORE, 139630576934912, 139630579187711, ++SNULL, 139630577078271, 139630579187711, ++STORE, 139630576934912, 139630577078271, ++STORE, 139630577078272, 139630579187711, ++ERASE, 139630577078272, 139630579187711, ++STORE, 139630579175424, 139630579183615, ++STORE, 139630579183616, 139630579187711, ++STORE, 140731219718144, 140731219722239, ++STORE, 140731219705856, 140731219718143, ++STORE, 139630579146752, 139630579175423, ++STORE, 139630579138560, 139630579146751, ++STORE, 139630573137920, 139630576934911, ++SNULL, 139630573137920, 139630574796799, ++STORE, 139630574796800, 139630576934911, ++STORE, 139630573137920, 139630574796799, ++SNULL, 139630576893951, 139630576934911, ++STORE, 139630574796800, 139630576893951, ++STORE, 139630576893952, 139630576934911, ++SNULL, 139630576893952, 139630576918527, ++STORE, 139630576918528, 139630576934911, ++STORE, 139630576893952, 139630576918527, ++ERASE, 139630576893952, 139630576918527, ++STORE, 139630576893952, 139630576918527, ++ERASE, 139630576918528, 139630576934911, ++STORE, 139630576918528, 139630576934911, ++SNULL, 139630576910335, 139630576918527, ++STORE, 139630576893952, 139630576910335, ++STORE, 139630576910336, 139630576918527, ++SNULL, 93888805859327, 93888805863423, ++STORE, 93888805851136, 93888805859327, ++STORE, 93888805859328, 93888805863423, ++SNULL, 139630579179519, 139630579183615, ++STORE, 139630579175424, 139630579179519, ++STORE, 139630579179520, 139630579183615, ++ERASE, 139630579146752, 139630579175423, ++STORE, 93888822235136, 93888822370303, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733391151104, 140737488351231, ++SNULL, 140733391159295, 140737488351231, ++STORE, 140733391151104, 140733391159295, ++STORE, 140733391020032, 140733391159295, ++STORE, 94393875324928, 94393877549055, ++SNULL, 94393875435519, 94393877549055, ++STORE, 94393875324928, 94393875435519, ++STORE, 94393875435520, 94393877549055, ++ERASE, 94393875435520, 94393877549055, ++STORE, 94393877528576, 94393877540863, ++STORE, 94393877540864, 94393877549055, ++STORE, 140292111740928, 140292113993727, ++SNULL, 140292111884287, 140292113993727, ++STORE, 140292111740928, 140292111884287, ++STORE, 140292111884288, 140292113993727, ++ERASE, 140292111884288, 140292113993727, ++STORE, 140292113981440, 140292113989631, ++STORE, 140292113989632, 140292113993727, ++STORE, 140733391532032, 140733391536127, ++STORE, 140733391519744, 140733391532031, ++STORE, 140292113952768, 140292113981439, ++STORE, 140292113944576, 140292113952767, ++STORE, 140292107943936, 140292111740927, ++SNULL, 140292107943936, 140292109602815, ++STORE, 140292109602816, 140292111740927, ++STORE, 140292107943936, 140292109602815, ++SNULL, 140292111699967, 140292111740927, ++STORE, 140292109602816, 140292111699967, ++STORE, 140292111699968, 140292111740927, ++SNULL, 140292111699968, 140292111724543, ++STORE, 140292111724544, 140292111740927, ++STORE, 140292111699968, 140292111724543, ++ERASE, 140292111699968, 140292111724543, ++STORE, 140292111699968, 140292111724543, ++ERASE, 140292111724544, 140292111740927, ++STORE, 140292111724544, 140292111740927, ++SNULL, 140292111716351, 140292111724543, ++STORE, 140292111699968, 140292111716351, ++STORE, 140292111716352, 140292111724543, ++SNULL, 94393877536767, 94393877540863, ++STORE, 94393877528576, 94393877536767, ++STORE, 94393877536768, 94393877540863, ++SNULL, 140292113985535, 140292113989631, ++STORE, 140292113981440, 140292113985535, ++STORE, 140292113985536, 140292113989631, ++ERASE, 140292113952768, 140292113981439, ++STORE, 94393909342208, 94393909477375, ++STORE, 94458367512576, 94458367725567, ++STORE, 94458369822720, 94458369826815, ++STORE, 94458369826816, 94458369835007, ++STORE, 94458369835008, 94458369847295, ++STORE, 94458393292800, 94458399666175, ++STORE, 140619773841408, 140619775500287, ++STORE, 140619775500288, 140619777597439, ++STORE, 140619777597440, 140619777613823, ++STORE, 140619777613824, 140619777622015, ++STORE, 140619777622016, 140619777638399, ++STORE, 140619777638400, 140619777650687, ++STORE, 140619777650688, 140619779743743, ++STORE, 140619779743744, 140619779747839, ++STORE, 140619779747840, 140619779751935, ++STORE, 140619779751936, 140619779895295, ++STORE, 140619780263936, 140619781947391, ++STORE, 140619781947392, 140619781963775, ++STORE, 140619781992448, 140619781996543, ++STORE, 140619781996544, 140619782000639, ++STORE, 140619782000640, 140619782004735, ++STORE, 140725811675136, 140725811814399, ++STORE, 140725812813824, 140725812826111, ++STORE, 140725812826112, 140725812830207, ++STORE, 94458367512576, 94458367725567, ++STORE, 94458369822720, 94458369826815, ++STORE, 94458369826816, 94458369835007, ++STORE, 94458369835008, 94458369847295, ++STORE, 94458393292800, 94458400366591, ++STORE, 140619773841408, 140619775500287, ++STORE, 140619775500288, 140619777597439, ++STORE, 140619777597440, 140619777613823, ++STORE, 140619777613824, 140619777622015, ++STORE, 140619777622016, 140619777638399, ++STORE, 140619777638400, 140619777650687, ++STORE, 140619777650688, 140619779743743, ++STORE, 140619779743744, 140619779747839, ++STORE, 140619779747840, 140619779751935, ++STORE, 140619779751936, 140619779895295, ++STORE, 140619780263936, 140619781947391, ++STORE, 140619781947392, 140619781963775, ++STORE, 140619781992448, 140619781996543, ++STORE, 140619781996544, 140619782000639, ++STORE, 140619782000640, 140619782004735, ++STORE, 140725811675136, 140725811814399, ++STORE, 140725812813824, 140725812826111, ++STORE, 140725812826112, 140725812830207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728740679680, 140737488351231, ++SNULL, 140728740687871, 140737488351231, ++STORE, 140728740679680, 140728740687871, ++STORE, 140728740548608, 140728740687871, ++STORE, 94764075249664, 94764077473791, ++SNULL, 94764075360255, 94764077473791, ++STORE, 94764075249664, 94764075360255, ++STORE, 94764075360256, 94764077473791, ++ERASE, 94764075360256, 94764077473791, ++STORE, 94764077453312, 94764077465599, ++STORE, 94764077465600, 94764077473791, ++STORE, 139766406791168, 139766409043967, ++SNULL, 139766406934527, 139766409043967, ++STORE, 139766406791168, 139766406934527, ++STORE, 139766406934528, 139766409043967, ++ERASE, 139766406934528, 139766409043967, ++STORE, 139766409031680, 139766409039871, ++STORE, 139766409039872, 139766409043967, ++STORE, 140728740913152, 140728740917247, ++STORE, 140728740900864, 140728740913151, ++STORE, 139766409003008, 139766409031679, ++STORE, 139766408994816, 139766409003007, ++STORE, 139766402994176, 139766406791167, ++SNULL, 139766402994176, 139766404653055, ++STORE, 139766404653056, 139766406791167, ++STORE, 139766402994176, 139766404653055, ++SNULL, 139766406750207, 139766406791167, ++STORE, 139766404653056, 139766406750207, ++STORE, 139766406750208, 139766406791167, ++SNULL, 139766406750208, 139766406774783, ++STORE, 139766406774784, 139766406791167, ++STORE, 139766406750208, 139766406774783, ++ERASE, 139766406750208, 139766406774783, ++STORE, 139766406750208, 139766406774783, ++ERASE, 139766406774784, 139766406791167, ++STORE, 139766406774784, 139766406791167, ++SNULL, 139766406766591, 139766406774783, ++STORE, 139766406750208, 139766406766591, ++STORE, 139766406766592, 139766406774783, ++SNULL, 94764077461503, 94764077465599, ++STORE, 94764077453312, 94764077461503, ++STORE, 94764077461504, 94764077465599, ++SNULL, 139766409035775, 139766409039871, ++STORE, 139766409031680, 139766409035775, ++STORE, 139766409035776, 139766409039871, ++ERASE, 139766409003008, 139766409031679, ++STORE, 94764090458112, 94764090593279, ++STORE, 94758057480192, 94758057590783, ++STORE, 94758059683840, 94758059692031, ++STORE, 94758059692032, 94758059696127, ++STORE, 94758059696128, 94758059704319, ++STORE, 94758083215360, 94758083350527, ++STORE, 139951456772096, 139951458430975, ++STORE, 139951458430976, 139951460528127, ++STORE, 139951460528128, 139951460544511, ++STORE, 139951460544512, 139951460552703, ++STORE, 139951460552704, 139951460569087, ++STORE, 139951460569088, 139951460712447, ++STORE, 139951462772736, 139951462780927, ++STORE, 139951462809600, 139951462813695, ++STORE, 139951462813696, 139951462817791, ++STORE, 139951462817792, 139951462821887, ++STORE, 140734098313216, 140734098452479, ++STORE, 140734098911232, 140734098923519, ++STORE, 140734098923520, 140734098927615, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724904095744, 140737488351231, ++SNULL, 140724904103935, 140737488351231, ++STORE, 140724904095744, 140724904103935, ++STORE, 140724903964672, 140724904103935, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140408497864704, 140408500117503, ++SNULL, 140408498008063, 140408500117503, ++STORE, 140408497864704, 140408498008063, ++STORE, 140408498008064, 140408500117503, ++ERASE, 140408498008064, 140408500117503, ++STORE, 140408500105216, 140408500113407, ++STORE, 140408500113408, 140408500117503, ++STORE, 140724905369600, 140724905373695, ++STORE, 140724905357312, 140724905369599, ++STORE, 140408500076544, 140408500105215, ++STORE, 140408500068352, 140408500076543, ++STORE, 140408494702592, 140408497864703, ++SNULL, 140408494702592, 140408495763455, ++STORE, 140408495763456, 140408497864703, ++STORE, 140408494702592, 140408495763455, ++SNULL, 140408497856511, 140408497864703, ++STORE, 140408495763456, 140408497856511, ++STORE, 140408497856512, 140408497864703, ++ERASE, 140408497856512, 140408497864703, ++STORE, 140408497856512, 140408497864703, ++STORE, 140408490905600, 140408494702591, ++SNULL, 140408490905600, 140408492564479, ++STORE, 140408492564480, 140408494702591, ++STORE, 140408490905600, 140408492564479, ++SNULL, 140408494661631, 140408494702591, ++STORE, 140408492564480, 140408494661631, ++STORE, 140408494661632, 140408494702591, ++SNULL, 140408494661632, 140408494686207, ++STORE, 140408494686208, 140408494702591, ++STORE, 140408494661632, 140408494686207, ++ERASE, 140408494661632, 140408494686207, ++STORE, 140408494661632, 140408494686207, ++ERASE, 140408494686208, 140408494702591, ++STORE, 140408494686208, 140408494702591, ++STORE, 140408500056064, 140408500076543, ++SNULL, 140408494678015, 140408494686207, ++STORE, 140408494661632, 140408494678015, ++STORE, 140408494678016, 140408494686207, ++SNULL, 140408497860607, 140408497864703, ++STORE, 140408497856512, 140408497860607, ++STORE, 140408497860608, 140408497864703, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140408500109311, 140408500113407, ++STORE, 140408500105216, 140408500109311, ++STORE, 140408500109312, 140408500113407, ++ERASE, 140408500076544, 140408500105215, ++STORE, 25235456, 25370623, ++STORE, 25235456, 25518079, ++STORE, 140408498372608, 140408500056063, ++STORE, 94543937388544, 94543937499135, ++STORE, 94543939592192, 94543939600383, ++STORE, 94543939600384, 94543939604479, ++STORE, 94543939604480, 94543939612671, ++STORE, 94543941447680, 94543941582847, ++STORE, 140282621947904, 140282623606783, ++STORE, 140282623606784, 140282625703935, ++STORE, 140282625703936, 140282625720319, ++STORE, 140282625720320, 140282625728511, ++STORE, 140282625728512, 140282625744895, ++STORE, 140282625744896, 140282625888255, ++STORE, 140282627948544, 140282627956735, ++STORE, 140282627985408, 140282627989503, ++STORE, 140282627989504, 140282627993599, ++STORE, 140282627993600, 140282627997695, ++STORE, 140728295723008, 140728295862271, ++STORE, 140728296476672, 140728296488959, ++STORE, 140728296488960, 140728296493055, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431510691839, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431510691839, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140736944451584, 140737488351231, ++SNULL, 140736944463871, 140737488351231, ++STORE, 140736944451584, 140736944463871, ++STORE, 140736944320512, 140736944463871, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 139693609893888, 139693612146687, ++SNULL, 139693610037247, 139693612146687, ++STORE, 139693609893888, 139693610037247, ++STORE, 139693610037248, 139693612146687, ++ERASE, 139693610037248, 139693612146687, ++STORE, 139693612134400, 139693612142591, ++STORE, 139693612142592, 139693612146687, ++STORE, 140736945152000, 140736945156095, ++STORE, 140736945139712, 140736945151999, ++STORE, 139693612105728, 139693612134399, ++STORE, 139693612097536, 139693612105727, ++STORE, 139693606060032, 139693609893887, ++SNULL, 139693606060032, 139693607768063, ++STORE, 139693607768064, 139693609893887, ++STORE, 139693606060032, 139693607768063, ++SNULL, 139693609861119, 139693609893887, ++STORE, 139693607768064, 139693609861119, ++STORE, 139693609861120, 139693609893887, ++ERASE, 139693609861120, 139693609893887, ++STORE, 139693609861120, 139693609893887, ++STORE, 139693603864576, 139693606060031, ++SNULL, 139693603864576, 139693603958783, ++STORE, 139693603958784, 139693606060031, ++STORE, 139693603864576, 139693603958783, ++SNULL, 139693606051839, 139693606060031, ++STORE, 139693603958784, 139693606051839, ++STORE, 139693606051840, 139693606060031, ++ERASE, 139693606051840, 139693606060031, ++STORE, 139693606051840, 139693606060031, ++STORE, 139693601345536, 139693603864575, ++SNULL, 139693601345536, 139693601759231, ++STORE, 139693601759232, 139693603864575, ++STORE, 139693601345536, 139693601759231, ++SNULL, 139693603852287, 139693603864575, ++STORE, 139693601759232, 139693603852287, ++STORE, 139693603852288, 139693603864575, ++ERASE, 139693603852288, 139693603864575, ++STORE, 139693603852288, 139693603864575, ++STORE, 139693598711808, 139693601345535, ++SNULL, 139693598711808, 139693599240191, ++STORE, 139693599240192, 139693601345535, ++STORE, 139693598711808, 139693599240191, ++SNULL, 139693601337343, 139693601345535, ++STORE, 139693599240192, 139693601337343, ++STORE, 139693601337344, 139693601345535, ++ERASE, 139693601337344, 139693601345535, ++STORE, 139693601337344, 139693601345535, ++STORE, 139693596598272, 139693598711807, ++SNULL, 139693596598272, 139693596610559, ++STORE, 139693596610560, 139693598711807, ++STORE, 139693596598272, 139693596610559, ++SNULL, 139693598703615, 139693598711807, ++STORE, 139693596610560, 139693598703615, ++STORE, 139693598703616, 139693598711807, ++ERASE, 139693598703616, 139693598711807, ++STORE, 139693598703616, 139693598711807, ++STORE, 139693594394624, 139693596598271, ++SNULL, 139693594394624, 139693594497023, ++STORE, 139693594497024, 139693596598271, ++STORE, 139693594394624, 139693594497023, ++SNULL, 139693596590079, 139693596598271, ++STORE, 139693594497024, 139693596590079, ++STORE, 139693596590080, 139693596598271, ++ERASE, 139693596590080, 139693596598271, ++STORE, 139693596590080, 139693596598271, ++STORE, 139693612089344, 139693612105727, ++STORE, 139693591232512, 139693594394623, ++SNULL, 139693591232512, 139693592293375, ++STORE, 139693592293376, 139693594394623, ++STORE, 139693591232512, 139693592293375, ++SNULL, 139693594386431, 139693594394623, ++STORE, 139693592293376, 139693594386431, ++STORE, 139693594386432, 139693594394623, ++ERASE, 139693594386432, 139693594394623, ++STORE, 139693594386432, 139693594394623, ++STORE, 139693587435520, 139693591232511, ++SNULL, 139693587435520, 139693589094399, ++STORE, 139693589094400, 139693591232511, ++STORE, 139693587435520, 139693589094399, ++SNULL, 139693591191551, 139693591232511, ++STORE, 139693589094400, 139693591191551, ++STORE, 139693591191552, 139693591232511, ++SNULL, 139693591191552, 139693591216127, ++STORE, 139693591216128, 139693591232511, ++STORE, 139693591191552, 139693591216127, ++ERASE, 139693591191552, 139693591216127, ++STORE, 139693591191552, 139693591216127, ++ERASE, 139693591216128, 139693591232511, ++STORE, 139693591216128, 139693591232511, ++STORE, 139693612077056, 139693612105727, ++SNULL, 139693591207935, 139693591216127, ++STORE, 139693591191552, 139693591207935, ++STORE, 139693591207936, 139693591216127, ++SNULL, 139693594390527, 139693594394623, ++STORE, 139693594386432, 139693594390527, ++STORE, 139693594390528, 139693594394623, ++SNULL, 139693596594175, 139693596598271, ++STORE, 139693596590080, 139693596594175, ++STORE, 139693596594176, 139693596598271, ++SNULL, 139693598707711, 139693598711807, ++STORE, 139693598703616, 139693598707711, ++STORE, 139693598707712, 139693598711807, ++SNULL, 139693601341439, 139693601345535, ++STORE, 139693601337344, 139693601341439, ++STORE, 139693601341440, 139693601345535, ++SNULL, 139693603860479, 139693603864575, ++STORE, 139693603852288, 139693603860479, ++STORE, 139693603860480, 139693603864575, ++SNULL, 139693606055935, 139693606060031, ++STORE, 139693606051840, 139693606055935, ++STORE, 139693606055936, 139693606060031, ++SNULL, 139693609865215, 139693609893887, ++STORE, 139693609861120, 139693609865215, ++STORE, 139693609865216, 139693609893887, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 139693612138495, 139693612142591, ++STORE, 139693612134400, 139693612138495, ++STORE, 139693612138496, 139693612142591, ++ERASE, 139693612105728, 139693612134399, ++STORE, 39976960, 40112127, ++STORE, 139693610393600, 139693612077055, ++STORE, 139693612130304, 139693612134399, ++STORE, 139693610258432, 139693610393599, ++STORE, 39976960, 40255487, ++STORE, 139693585338368, 139693587435519, ++STORE, 139693612122112, 139693612134399, ++STORE, 139693612113920, 139693612134399, ++STORE, 139693612077056, 139693612113919, ++STORE, 139693610242048, 139693610393599, ++STORE, 39976960, 40390655, ++STORE, 39976960, 40546303, ++STORE, 139693610233856, 139693610393599, ++STORE, 139693610225664, 139693610393599, ++STORE, 39976960, 40714239, ++STORE, 139693610209280, 139693610393599, ++STORE, 39976960, 40861695, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431528759295, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140729993904128, 140737488351231, ++SNULL, 140729993912319, 140737488351231, ++STORE, 140729993904128, 140729993912319, ++STORE, 140729993773056, 140729993912319, ++STORE, 93926271991808, 93926274215935, ++SNULL, 93926272102399, 93926274215935, ++STORE, 93926271991808, 93926272102399, ++STORE, 93926272102400, 93926274215935, ++ERASE, 93926272102400, 93926274215935, ++STORE, 93926274195456, 93926274207743, ++STORE, 93926274207744, 93926274215935, ++STORE, 139962167296000, 139962169548799, ++SNULL, 139962167439359, 139962169548799, ++STORE, 139962167296000, 139962167439359, ++STORE, 139962167439360, 139962169548799, ++ERASE, 139962167439360, 139962169548799, ++STORE, 139962169536512, 139962169544703, ++STORE, 139962169544704, 139962169548799, ++STORE, 140729995096064, 140729995100159, ++STORE, 140729995083776, 140729995096063, ++STORE, 139962169507840, 139962169536511, ++STORE, 139962169499648, 139962169507839, ++STORE, 139962163499008, 139962167295999, ++SNULL, 139962163499008, 139962165157887, ++STORE, 139962165157888, 139962167295999, ++STORE, 139962163499008, 139962165157887, ++SNULL, 139962167255039, 139962167295999, ++STORE, 139962165157888, 139962167255039, ++STORE, 139962167255040, 139962167295999, ++SNULL, 139962167255040, 139962167279615, ++STORE, 139962167279616, 139962167295999, ++STORE, 139962167255040, 139962167279615, ++ERASE, 139962167255040, 139962167279615, ++STORE, 139962167255040, 139962167279615, ++ERASE, 139962167279616, 139962167295999, ++STORE, 139962167279616, 139962167295999, ++SNULL, 139962167271423, 139962167279615, ++STORE, 139962167255040, 139962167271423, ++STORE, 139962167271424, 139962167279615, ++SNULL, 93926274203647, 93926274207743, ++STORE, 93926274195456, 93926274203647, ++STORE, 93926274203648, 93926274207743, ++SNULL, 139962169540607, 139962169544703, ++STORE, 139962169536512, 139962169540607, ++STORE, 139962169540608, 139962169544703, ++ERASE, 139962169507840, 139962169536511, ++STORE, 93926291120128, 93926291255295, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724960579584, 140737488351231, ++SNULL, 140724960587775, 140737488351231, ++STORE, 140724960579584, 140724960587775, ++STORE, 140724960448512, 140724960587775, ++STORE, 94246489489408, 94246491713535, ++SNULL, 94246489599999, 94246491713535, ++STORE, 94246489489408, 94246489599999, ++STORE, 94246489600000, 94246491713535, ++ERASE, 94246489600000, 94246491713535, ++STORE, 94246491693056, 94246491705343, ++STORE, 94246491705344, 94246491713535, ++STORE, 140098174926848, 140098177179647, ++SNULL, 140098175070207, 140098177179647, ++STORE, 140098174926848, 140098175070207, ++STORE, 140098175070208, 140098177179647, ++ERASE, 140098175070208, 140098177179647, ++STORE, 140098177167360, 140098177175551, ++STORE, 140098177175552, 140098177179647, ++STORE, 140724961439744, 140724961443839, ++STORE, 140724961427456, 140724961439743, ++STORE, 140098177138688, 140098177167359, ++STORE, 140098177130496, 140098177138687, ++STORE, 140098171129856, 140098174926847, ++SNULL, 140098171129856, 140098172788735, ++STORE, 140098172788736, 140098174926847, ++STORE, 140098171129856, 140098172788735, ++SNULL, 140098174885887, 140098174926847, ++STORE, 140098172788736, 140098174885887, ++STORE, 140098174885888, 140098174926847, ++SNULL, 140098174885888, 140098174910463, ++STORE, 140098174910464, 140098174926847, ++STORE, 140098174885888, 140098174910463, ++ERASE, 140098174885888, 140098174910463, ++STORE, 140098174885888, 140098174910463, ++ERASE, 140098174910464, 140098174926847, ++STORE, 140098174910464, 140098174926847, ++SNULL, 140098174902271, 140098174910463, ++STORE, 140098174885888, 140098174902271, ++STORE, 140098174902272, 140098174910463, ++SNULL, 94246491701247, 94246491705343, ++STORE, 94246491693056, 94246491701247, ++STORE, 94246491701248, 94246491705343, ++SNULL, 140098177171455, 140098177175551, ++STORE, 140098177167360, 140098177171455, ++STORE, 140098177171456, 140098177175551, ++ERASE, 140098177138688, 140098177167359, ++STORE, 94246516998144, 94246517133311, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730522918912, 140737488351231, ++SNULL, 140730522927103, 140737488351231, ++STORE, 140730522918912, 140730522927103, ++STORE, 140730522787840, 140730522927103, ++STORE, 94196043120640, 94196045344767, ++SNULL, 94196043231231, 94196045344767, ++STORE, 94196043120640, 94196043231231, ++STORE, 94196043231232, 94196045344767, ++ERASE, 94196043231232, 94196045344767, ++STORE, 94196045324288, 94196045336575, ++STORE, 94196045336576, 94196045344767, ++STORE, 139815918940160, 139815921192959, ++SNULL, 139815919083519, 139815921192959, ++STORE, 139815918940160, 139815919083519, ++STORE, 139815919083520, 139815921192959, ++ERASE, 139815919083520, 139815921192959, ++STORE, 139815921180672, 139815921188863, ++STORE, 139815921188864, 139815921192959, ++STORE, 140730523344896, 140730523348991, ++STORE, 140730523332608, 140730523344895, ++STORE, 139815921152000, 139815921180671, ++STORE, 139815921143808, 139815921151999, ++STORE, 139815915143168, 139815918940159, ++SNULL, 139815915143168, 139815916802047, ++STORE, 139815916802048, 139815918940159, ++STORE, 139815915143168, 139815916802047, ++SNULL, 139815918899199, 139815918940159, ++STORE, 139815916802048, 139815918899199, ++STORE, 139815918899200, 139815918940159, ++SNULL, 139815918899200, 139815918923775, ++STORE, 139815918923776, 139815918940159, ++STORE, 139815918899200, 139815918923775, ++ERASE, 139815918899200, 139815918923775, ++STORE, 139815918899200, 139815918923775, ++ERASE, 139815918923776, 139815918940159, ++STORE, 139815918923776, 139815918940159, ++SNULL, 139815918915583, 139815918923775, ++STORE, 139815918899200, 139815918915583, ++STORE, 139815918915584, 139815918923775, ++SNULL, 94196045332479, 94196045336575, ++STORE, 94196045324288, 94196045332479, ++STORE, 94196045332480, 94196045336575, ++SNULL, 139815921184767, 139815921188863, ++STORE, 139815921180672, 139815921184767, ++STORE, 139815921184768, 139815921188863, ++ERASE, 139815921152000, 139815921180671, ++STORE, 94196076183552, 94196076318719, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722460393472, 140737488351231, ++SNULL, 140722460401663, 140737488351231, ++STORE, 140722460393472, 140722460401663, ++STORE, 140722460262400, 140722460401663, ++STORE, 94569810399232, 94569812623359, ++SNULL, 94569810509823, 94569812623359, ++STORE, 94569810399232, 94569810509823, ++STORE, 94569810509824, 94569812623359, ++ERASE, 94569810509824, 94569812623359, ++STORE, 94569812602880, 94569812615167, ++STORE, 94569812615168, 94569812623359, ++STORE, 139681565450240, 139681567703039, ++SNULL, 139681565593599, 139681567703039, ++STORE, 139681565450240, 139681565593599, ++STORE, 139681565593600, 139681567703039, ++ERASE, 139681565593600, 139681567703039, ++STORE, 139681567690752, 139681567698943, ++STORE, 139681567698944, 139681567703039, ++STORE, 140722460569600, 140722460573695, ++STORE, 140722460557312, 140722460569599, ++STORE, 139681567662080, 139681567690751, ++STORE, 139681567653888, 139681567662079, ++STORE, 139681561653248, 139681565450239, ++SNULL, 139681561653248, 139681563312127, ++STORE, 139681563312128, 139681565450239, ++STORE, 139681561653248, 139681563312127, ++SNULL, 139681565409279, 139681565450239, ++STORE, 139681563312128, 139681565409279, ++STORE, 139681565409280, 139681565450239, ++SNULL, 139681565409280, 139681565433855, ++STORE, 139681565433856, 139681565450239, ++STORE, 139681565409280, 139681565433855, ++ERASE, 139681565409280, 139681565433855, ++STORE, 139681565409280, 139681565433855, ++ERASE, 139681565433856, 139681565450239, ++STORE, 139681565433856, 139681565450239, ++SNULL, 139681565425663, 139681565433855, ++STORE, 139681565409280, 139681565425663, ++STORE, 139681565425664, 139681565433855, ++SNULL, 94569812611071, 94569812615167, ++STORE, 94569812602880, 94569812611071, ++STORE, 94569812611072, 94569812615167, ++SNULL, 139681567694847, 139681567698943, ++STORE, 139681567690752, 139681567694847, ++STORE, 139681567694848, 139681567698943, ++ERASE, 139681567662080, 139681567690751, ++STORE, 94569818066944, 94569818202111, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431534280703, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725452365824, 140737488351231, ++SNULL, 140725452374015, 140737488351231, ++STORE, 140725452365824, 140725452374015, ++STORE, 140725452234752, 140725452374015, ++STORE, 94395067465728, 94395069689855, ++SNULL, 94395067576319, 94395069689855, ++STORE, 94395067465728, 94395067576319, ++STORE, 94395067576320, 94395069689855, ++ERASE, 94395067576320, 94395069689855, ++STORE, 94395069669376, 94395069681663, ++STORE, 94395069681664, 94395069689855, ++STORE, 140269941211136, 140269943463935, ++SNULL, 140269941354495, 140269943463935, ++STORE, 140269941211136, 140269941354495, ++STORE, 140269941354496, 140269943463935, ++ERASE, 140269941354496, 140269943463935, ++STORE, 140269943451648, 140269943459839, ++STORE, 140269943459840, 140269943463935, ++STORE, 140725452558336, 140725452562431, ++STORE, 140725452546048, 140725452558335, ++STORE, 140269943422976, 140269943451647, ++STORE, 140269943414784, 140269943422975, ++STORE, 140269937414144, 140269941211135, ++SNULL, 140269937414144, 140269939073023, ++STORE, 140269939073024, 140269941211135, ++STORE, 140269937414144, 140269939073023, ++SNULL, 140269941170175, 140269941211135, ++STORE, 140269939073024, 140269941170175, ++STORE, 140269941170176, 140269941211135, ++SNULL, 140269941170176, 140269941194751, ++STORE, 140269941194752, 140269941211135, ++STORE, 140269941170176, 140269941194751, ++ERASE, 140269941170176, 140269941194751, ++STORE, 140269941170176, 140269941194751, ++ERASE, 140269941194752, 140269941211135, ++STORE, 140269941194752, 140269941211135, ++SNULL, 140269941186559, 140269941194751, ++STORE, 140269941170176, 140269941186559, ++STORE, 140269941186560, 140269941194751, ++SNULL, 94395069677567, 94395069681663, ++STORE, 94395069669376, 94395069677567, ++STORE, 94395069677568, 94395069681663, ++SNULL, 140269943455743, 140269943459839, ++STORE, 140269943451648, 140269943455743, ++STORE, 140269943455744, 140269943459839, ++ERASE, 140269943422976, 140269943451647, ++STORE, 94395101691904, 94395101827071, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733860118528, 140737488351231, ++SNULL, 140733860126719, 140737488351231, ++STORE, 140733860118528, 140733860126719, ++STORE, 140733859987456, 140733860126719, ++STORE, 94484752990208, 94484755214335, ++SNULL, 94484753100799, 94484755214335, ++STORE, 94484752990208, 94484753100799, ++STORE, 94484753100800, 94484755214335, ++ERASE, 94484753100800, 94484755214335, ++STORE, 94484755193856, 94484755206143, ++STORE, 94484755206144, 94484755214335, ++STORE, 139958922309632, 139958924562431, ++SNULL, 139958922452991, 139958924562431, ++STORE, 139958922309632, 139958922452991, ++STORE, 139958922452992, 139958924562431, ++ERASE, 139958922452992, 139958924562431, ++STORE, 139958924550144, 139958924558335, ++STORE, 139958924558336, 139958924562431, ++STORE, 140733860253696, 140733860257791, ++STORE, 140733860241408, 140733860253695, ++STORE, 139958924521472, 139958924550143, ++STORE, 139958924513280, 139958924521471, ++STORE, 139958918512640, 139958922309631, ++SNULL, 139958918512640, 139958920171519, ++STORE, 139958920171520, 139958922309631, ++STORE, 139958918512640, 139958920171519, ++SNULL, 139958922268671, 139958922309631, ++STORE, 139958920171520, 139958922268671, ++STORE, 139958922268672, 139958922309631, ++SNULL, 139958922268672, 139958922293247, ++STORE, 139958922293248, 139958922309631, ++STORE, 139958922268672, 139958922293247, ++ERASE, 139958922268672, 139958922293247, ++STORE, 139958922268672, 139958922293247, ++ERASE, 139958922293248, 139958922309631, ++STORE, 139958922293248, 139958922309631, ++SNULL, 139958922285055, 139958922293247, ++STORE, 139958922268672, 139958922285055, ++STORE, 139958922285056, 139958922293247, ++SNULL, 94484755202047, 94484755206143, ++STORE, 94484755193856, 94484755202047, ++STORE, 94484755202048, 94484755206143, ++SNULL, 139958924554239, 139958924558335, ++STORE, 139958924550144, 139958924554239, ++STORE, 139958924554240, 139958924558335, ++ERASE, 139958924521472, 139958924550143, ++STORE, 94484777615360, 94484777750527, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731051036672, 140737488351231, ++SNULL, 140731051044863, 140737488351231, ++STORE, 140731051036672, 140731051044863, ++STORE, 140731050905600, 140731051044863, ++STORE, 93945822998528, 93945825222655, ++SNULL, 93945823109119, 93945825222655, ++STORE, 93945822998528, 93945823109119, ++STORE, 93945823109120, 93945825222655, ++ERASE, 93945823109120, 93945825222655, ++STORE, 93945825202176, 93945825214463, ++STORE, 93945825214464, 93945825222655, ++STORE, 140153503997952, 140153506250751, ++SNULL, 140153504141311, 140153506250751, ++STORE, 140153503997952, 140153504141311, ++STORE, 140153504141312, 140153506250751, ++ERASE, 140153504141312, 140153506250751, ++STORE, 140153506238464, 140153506246655, ++STORE, 140153506246656, 140153506250751, ++STORE, 140731051331584, 140731051335679, ++STORE, 140731051319296, 140731051331583, ++STORE, 140153506209792, 140153506238463, ++STORE, 140153506201600, 140153506209791, ++STORE, 140153500200960, 140153503997951, ++SNULL, 140153500200960, 140153501859839, ++STORE, 140153501859840, 140153503997951, ++STORE, 140153500200960, 140153501859839, ++SNULL, 140153503956991, 140153503997951, ++STORE, 140153501859840, 140153503956991, ++STORE, 140153503956992, 140153503997951, ++SNULL, 140153503956992, 140153503981567, ++STORE, 140153503981568, 140153503997951, ++STORE, 140153503956992, 140153503981567, ++ERASE, 140153503956992, 140153503981567, ++STORE, 140153503956992, 140153503981567, ++ERASE, 140153503981568, 140153503997951, ++STORE, 140153503981568, 140153503997951, ++SNULL, 140153503973375, 140153503981567, ++STORE, 140153503956992, 140153503973375, ++STORE, 140153503973376, 140153503981567, ++SNULL, 93945825210367, 93945825214463, ++STORE, 93945825202176, 93945825210367, ++STORE, 93945825210368, 93945825214463, ++SNULL, 140153506242559, 140153506246655, ++STORE, 140153506238464, 140153506242559, ++STORE, 140153506242560, 140153506246655, ++ERASE, 140153506209792, 140153506238463, ++STORE, 93945854537728, 93945854672895, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431537885183, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736025325568, 140737488351231, ++SNULL, 140736025333759, 140737488351231, ++STORE, 140736025325568, 140736025333759, ++STORE, 140736025194496, 140736025333759, ++STORE, 94809095172096, 94809097396223, ++SNULL, 94809095282687, 94809097396223, ++STORE, 94809095172096, 94809095282687, ++STORE, 94809095282688, 94809097396223, ++ERASE, 94809095282688, 94809097396223, ++STORE, 94809097375744, 94809097388031, ++STORE, 94809097388032, 94809097396223, ++STORE, 140194992517120, 140194994769919, ++SNULL, 140194992660479, 140194994769919, ++STORE, 140194992517120, 140194992660479, ++STORE, 140194992660480, 140194994769919, ++ERASE, 140194992660480, 140194994769919, ++STORE, 140194994757632, 140194994765823, ++STORE, 140194994765824, 140194994769919, ++STORE, 140736026173440, 140736026177535, ++STORE, 140736026161152, 140736026173439, ++STORE, 140194994728960, 140194994757631, ++STORE, 140194994720768, 140194994728959, ++STORE, 140194988720128, 140194992517119, ++SNULL, 140194988720128, 140194990379007, ++STORE, 140194990379008, 140194992517119, ++STORE, 140194988720128, 140194990379007, ++SNULL, 140194992476159, 140194992517119, ++STORE, 140194990379008, 140194992476159, ++STORE, 140194992476160, 140194992517119, ++SNULL, 140194992476160, 140194992500735, ++STORE, 140194992500736, 140194992517119, ++STORE, 140194992476160, 140194992500735, ++ERASE, 140194992476160, 140194992500735, ++STORE, 140194992476160, 140194992500735, ++ERASE, 140194992500736, 140194992517119, ++STORE, 140194992500736, 140194992517119, ++SNULL, 140194992492543, 140194992500735, ++STORE, 140194992476160, 140194992492543, ++STORE, 140194992492544, 140194992500735, ++SNULL, 94809097383935, 94809097388031, ++STORE, 94809097375744, 94809097383935, ++STORE, 94809097383936, 94809097388031, ++SNULL, 140194994761727, 140194994765823, ++STORE, 140194994757632, 140194994761727, ++STORE, 140194994761728, 140194994765823, ++ERASE, 140194994728960, 140194994757631, ++STORE, 94809124286464, 94809124421631, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140726342660096, 140737488351231, ++SNULL, 140726342668287, 140737488351231, ++STORE, 140726342660096, 140726342668287, ++STORE, 140726342529024, 140726342668287, ++STORE, 94140331462656, 94140333686783, ++SNULL, 94140331573247, 94140333686783, ++STORE, 94140331462656, 94140331573247, ++STORE, 94140331573248, 94140333686783, ++ERASE, 94140331573248, 94140333686783, ++STORE, 94140333666304, 94140333678591, ++STORE, 94140333678592, 94140333686783, ++STORE, 140714077208576, 140714079461375, ++SNULL, 140714077351935, 140714079461375, ++STORE, 140714077208576, 140714077351935, ++STORE, 140714077351936, 140714079461375, ++ERASE, 140714077351936, 140714079461375, ++STORE, 140714079449088, 140714079457279, ++STORE, 140714079457280, 140714079461375, ++STORE, 140726343933952, 140726343938047, ++STORE, 140726343921664, 140726343933951, ++STORE, 140714079420416, 140714079449087, ++STORE, 140714079412224, 140714079420415, ++STORE, 140714073411584, 140714077208575, ++SNULL, 140714073411584, 140714075070463, ++STORE, 140714075070464, 140714077208575, ++STORE, 140714073411584, 140714075070463, ++SNULL, 140714077167615, 140714077208575, ++STORE, 140714075070464, 140714077167615, ++STORE, 140714077167616, 140714077208575, ++SNULL, 140714077167616, 140714077192191, ++STORE, 140714077192192, 140714077208575, ++STORE, 140714077167616, 140714077192191, ++ERASE, 140714077167616, 140714077192191, ++STORE, 140714077167616, 140714077192191, ++ERASE, 140714077192192, 140714077208575, ++STORE, 140714077192192, 140714077208575, ++SNULL, 140714077183999, 140714077192191, ++STORE, 140714077167616, 140714077183999, ++STORE, 140714077184000, 140714077192191, ++SNULL, 94140333674495, 94140333678591, ++STORE, 94140333666304, 94140333674495, ++STORE, 94140333674496, 94140333678591, ++SNULL, 140714079453183, 140714079457279, ++STORE, 140714079449088, 140714079453183, ++STORE, 140714079453184, 140714079457279, ++ERASE, 140714079420416, 140714079449087, ++STORE, 94140341432320, 94140341567487, ++STORE, 94431504838656, 94431505051647, ++STORE, 94431507148800, 94431507152895, ++STORE, 94431507152896, 94431507161087, ++STORE, 94431507161088, 94431507173375, ++STORE, 94431510286336, 94431539601407, ++STORE, 139818797948928, 139818799607807, ++STORE, 139818799607808, 139818801704959, ++STORE, 139818801704960, 139818801721343, ++STORE, 139818801721344, 139818801729535, ++STORE, 139818801729536, 139818801745919, ++STORE, 139818801745920, 139818801758207, ++STORE, 139818801758208, 139818803851263, ++STORE, 139818803851264, 139818803855359, ++STORE, 139818803855360, 139818803859455, ++STORE, 139818803859456, 139818804002815, ++STORE, 139818804371456, 139818806054911, ++STORE, 139818806054912, 139818806071295, ++STORE, 139818806099968, 139818806104063, ++STORE, 139818806104064, 139818806108159, ++STORE, 139818806108160, 139818806112255, ++STORE, 140731430457344, 140731430596607, ++STORE, 140731431227392, 140731431239679, ++STORE, 140731431239680, 140731431243775, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725843607552, 140737488351231, ++SNULL, 140725843615743, 140737488351231, ++STORE, 140725843607552, 140725843615743, ++STORE, 140725843476480, 140725843615743, ++STORE, 94889043505152, 94889045839871, ++SNULL, 94889043718143, 94889045839871, ++STORE, 94889043505152, 94889043718143, ++STORE, 94889043718144, 94889045839871, ++ERASE, 94889043718144, 94889045839871, ++STORE, 94889045815296, 94889045827583, ++STORE, 94889045827584, 94889045839871, ++STORE, 140250965946368, 140250968199167, ++SNULL, 140250966089727, 140250968199167, ++STORE, 140250965946368, 140250966089727, ++STORE, 140250966089728, 140250968199167, ++ERASE, 140250966089728, 140250968199167, ++STORE, 140250968186880, 140250968195071, ++STORE, 140250968195072, 140250968199167, ++STORE, 140725844500480, 140725844504575, ++STORE, 140725844488192, 140725844500479, ++STORE, 140250968158208, 140250968186879, ++STORE, 140250968150016, 140250968158207, ++STORE, 140250963832832, 140250965946367, ++SNULL, 140250963832832, 140250963845119, ++STORE, 140250963845120, 140250965946367, ++STORE, 140250963832832, 140250963845119, ++SNULL, 140250965938175, 140250965946367, ++STORE, 140250963845120, 140250965938175, ++STORE, 140250965938176, 140250965946367, ++ERASE, 140250965938176, 140250965946367, ++STORE, 140250965938176, 140250965946367, ++STORE, 140250960035840, 140250963832831, ++SNULL, 140250960035840, 140250961694719, ++STORE, 140250961694720, 140250963832831, ++STORE, 140250960035840, 140250961694719, ++SNULL, 140250963791871, 140250963832831, ++STORE, 140250961694720, 140250963791871, ++STORE, 140250963791872, 140250963832831, ++SNULL, 140250963791872, 140250963816447, ++STORE, 140250963816448, 140250963832831, ++STORE, 140250963791872, 140250963816447, ++ERASE, 140250963791872, 140250963816447, ++STORE, 140250963791872, 140250963816447, ++ERASE, 140250963816448, 140250963832831, ++STORE, 140250963816448, 140250963832831, ++STORE, 140250968141824, 140250968158207, ++SNULL, 140250963808255, 140250963816447, ++STORE, 140250963791872, 140250963808255, ++STORE, 140250963808256, 140250963816447, ++SNULL, 140250965942271, 140250965946367, ++STORE, 140250965938176, 140250965942271, ++STORE, 140250965942272, 140250965946367, ++SNULL, 94889045819391, 94889045827583, ++STORE, 94889045815296, 94889045819391, ++STORE, 94889045819392, 94889045827583, ++SNULL, 140250968190975, 140250968195071, ++STORE, 140250968186880, 140250968190975, ++STORE, 140250968190976, 140250968195071, ++ERASE, 140250968158208, 140250968186879, ++STORE, 94889052213248, 94889052348415, ++STORE, 140250966458368, 140250968141823, ++STORE, 94889052213248, 94889052483583, ++STORE, 94889052213248, 94889052618751, ++STORE, 94170851819520, 94170852032511, ++STORE, 94170854129664, 94170854133759, ++STORE, 94170854133760, 94170854141951, ++STORE, 94170854141952, 94170854154239, ++STORE, 94170866515968, 94170867740671, ++STORE, 140062030422016, 140062032080895, ++STORE, 140062032080896, 140062034178047, ++STORE, 140062034178048, 140062034194431, ++STORE, 140062034194432, 140062034202623, ++STORE, 140062034202624, 140062034219007, ++STORE, 140062034219008, 140062034231295, ++STORE, 140062034231296, 140062036324351, ++STORE, 140062036324352, 140062036328447, ++STORE, 140062036328448, 140062036332543, ++STORE, 140062036332544, 140062036475903, ++STORE, 140062036844544, 140062038527999, ++STORE, 140062038528000, 140062038544383, ++STORE, 140062038573056, 140062038577151, ++STORE, 140062038577152, 140062038581247, ++STORE, 140062038581248, 140062038585343, ++STORE, 140736210550784, 140736210690047, ++STORE, 140736210759680, 140736210771967, ++STORE, 140736210771968, 140736210776063, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724272365568, 140737488351231, ++SNULL, 140724272373759, 140737488351231, ++STORE, 140724272365568, 140724272373759, ++STORE, 140724272234496, 140724272373759, ++STORE, 94607711965184, 94607714189311, ++SNULL, 94607712075775, 94607714189311, ++STORE, 94607711965184, 94607712075775, ++STORE, 94607712075776, 94607714189311, ++ERASE, 94607712075776, 94607714189311, ++STORE, 94607714168832, 94607714181119, ++STORE, 94607714181120, 94607714189311, ++STORE, 140054949253120, 140054951505919, ++SNULL, 140054949396479, 140054951505919, ++STORE, 140054949253120, 140054949396479, ++STORE, 140054949396480, 140054951505919, ++ERASE, 140054949396480, 140054951505919, ++STORE, 140054951493632, 140054951501823, ++STORE, 140054951501824, 140054951505919, ++STORE, 140724272992256, 140724272996351, ++STORE, 140724272979968, 140724272992255, ++STORE, 140054951464960, 140054951493631, ++STORE, 140054951456768, 140054951464959, ++STORE, 140054945456128, 140054949253119, ++SNULL, 140054945456128, 140054947115007, ++STORE, 140054947115008, 140054949253119, ++STORE, 140054945456128, 140054947115007, ++SNULL, 140054949212159, 140054949253119, ++STORE, 140054947115008, 140054949212159, ++STORE, 140054949212160, 140054949253119, ++SNULL, 140054949212160, 140054949236735, ++STORE, 140054949236736, 140054949253119, ++STORE, 140054949212160, 140054949236735, ++ERASE, 140054949212160, 140054949236735, ++STORE, 140054949212160, 140054949236735, ++ERASE, 140054949236736, 140054949253119, ++STORE, 140054949236736, 140054949253119, ++SNULL, 140054949228543, 140054949236735, ++STORE, 140054949212160, 140054949228543, ++STORE, 140054949228544, 140054949236735, ++SNULL, 94607714177023, 94607714181119, ++STORE, 94607714168832, 94607714177023, ++STORE, 94607714177024, 94607714181119, ++SNULL, 140054951497727, 140054951501823, ++STORE, 140054951493632, 140054951497727, ++STORE, 140054951497728, 140054951501823, ++ERASE, 140054951464960, 140054951493631, ++STORE, 94607733374976, 94607733510143, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733586923520, 140737488351231, ++SNULL, 140733586931711, 140737488351231, ++STORE, 140733586923520, 140733586931711, ++STORE, 140733586792448, 140733586931711, ++STORE, 93901634904064, 93901637128191, ++SNULL, 93901635014655, 93901637128191, ++STORE, 93901634904064, 93901635014655, ++STORE, 93901635014656, 93901637128191, ++ERASE, 93901635014656, 93901637128191, ++STORE, 93901637107712, 93901637119999, ++STORE, 93901637120000, 93901637128191, ++STORE, 140086104784896, 140086107037695, ++SNULL, 140086104928255, 140086107037695, ++STORE, 140086104784896, 140086104928255, ++STORE, 140086104928256, 140086107037695, ++ERASE, 140086104928256, 140086107037695, ++STORE, 140086107025408, 140086107033599, ++STORE, 140086107033600, 140086107037695, ++STORE, 140733587263488, 140733587267583, ++STORE, 140733587251200, 140733587263487, ++STORE, 140086106996736, 140086107025407, ++STORE, 140086106988544, 140086106996735, ++STORE, 140086100987904, 140086104784895, ++SNULL, 140086100987904, 140086102646783, ++STORE, 140086102646784, 140086104784895, ++STORE, 140086100987904, 140086102646783, ++SNULL, 140086104743935, 140086104784895, ++STORE, 140086102646784, 140086104743935, ++STORE, 140086104743936, 140086104784895, ++SNULL, 140086104743936, 140086104768511, ++STORE, 140086104768512, 140086104784895, ++STORE, 140086104743936, 140086104768511, ++ERASE, 140086104743936, 140086104768511, ++STORE, 140086104743936, 140086104768511, ++ERASE, 140086104768512, 140086104784895, ++STORE, 140086104768512, 140086104784895, ++SNULL, 140086104760319, 140086104768511, ++STORE, 140086104743936, 140086104760319, ++STORE, 140086104760320, 140086104768511, ++SNULL, 93901637115903, 93901637119999, ++STORE, 93901637107712, 93901637115903, ++STORE, 93901637115904, 93901637119999, ++SNULL, 140086107029503, 140086107033599, ++STORE, 140086107025408, 140086107029503, ++STORE, 140086107029504, 140086107033599, ++ERASE, 140086106996736, 140086107025407, ++STORE, 93901662715904, 93901662851071, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723365613568, 140737488351231, ++SNULL, 140723365621759, 140737488351231, ++STORE, 140723365613568, 140723365621759, ++STORE, 140723365482496, 140723365621759, ++STORE, 94759193546752, 94759195770879, ++SNULL, 94759193657343, 94759195770879, ++STORE, 94759193546752, 94759193657343, ++STORE, 94759193657344, 94759195770879, ++ERASE, 94759193657344, 94759195770879, ++STORE, 94759195750400, 94759195762687, ++STORE, 94759195762688, 94759195770879, ++STORE, 140607636246528, 140607638499327, ++SNULL, 140607636389887, 140607638499327, ++STORE, 140607636246528, 140607636389887, ++STORE, 140607636389888, 140607638499327, ++ERASE, 140607636389888, 140607638499327, ++STORE, 140607638487040, 140607638495231, ++STORE, 140607638495232, 140607638499327, ++STORE, 140723365900288, 140723365904383, ++STORE, 140723365888000, 140723365900287, ++STORE, 140607638458368, 140607638487039, ++STORE, 140607638450176, 140607638458367, ++STORE, 140607632449536, 140607636246527, ++SNULL, 140607632449536, 140607634108415, ++STORE, 140607634108416, 140607636246527, ++STORE, 140607632449536, 140607634108415, ++SNULL, 140607636205567, 140607636246527, ++STORE, 140607634108416, 140607636205567, ++STORE, 140607636205568, 140607636246527, ++SNULL, 140607636205568, 140607636230143, ++STORE, 140607636230144, 140607636246527, ++STORE, 140607636205568, 140607636230143, ++ERASE, 140607636205568, 140607636230143, ++STORE, 140607636205568, 140607636230143, ++ERASE, 140607636230144, 140607636246527, ++STORE, 140607636230144, 140607636246527, ++SNULL, 140607636221951, 140607636230143, ++STORE, 140607636205568, 140607636221951, ++STORE, 140607636221952, 140607636230143, ++SNULL, 94759195758591, 94759195762687, ++STORE, 94759195750400, 94759195758591, ++STORE, 94759195758592, 94759195762687, ++SNULL, 140607638491135, 140607638495231, ++STORE, 140607638487040, 140607638491135, ++STORE, 140607638491136, 140607638495231, ++ERASE, 140607638458368, 140607638487039, ++STORE, 94759204995072, 94759205130239, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732503789568, 140737488351231, ++SNULL, 140732503797759, 140737488351231, ++STORE, 140732503789568, 140732503797759, ++STORE, 140732503658496, 140732503797759, ++STORE, 94077792956416, 94077795180543, ++SNULL, 94077793067007, 94077795180543, ++STORE, 94077792956416, 94077793067007, ++STORE, 94077793067008, 94077795180543, ++ERASE, 94077793067008, 94077795180543, ++STORE, 94077795160064, 94077795172351, ++STORE, 94077795172352, 94077795180543, ++STORE, 140359874252800, 140359876505599, ++SNULL, 140359874396159, 140359876505599, ++STORE, 140359874252800, 140359874396159, ++STORE, 140359874396160, 140359876505599, ++ERASE, 140359874396160, 140359876505599, ++STORE, 140359876493312, 140359876501503, ++STORE, 140359876501504, 140359876505599, ++STORE, 140732504465408, 140732504469503, ++STORE, 140732504453120, 140732504465407, ++STORE, 140359876464640, 140359876493311, ++STORE, 140359876456448, 140359876464639, ++STORE, 140359870455808, 140359874252799, ++SNULL, 140359870455808, 140359872114687, ++STORE, 140359872114688, 140359874252799, ++STORE, 140359870455808, 140359872114687, ++SNULL, 140359874211839, 140359874252799, ++STORE, 140359872114688, 140359874211839, ++STORE, 140359874211840, 140359874252799, ++SNULL, 140359874211840, 140359874236415, ++STORE, 140359874236416, 140359874252799, ++STORE, 140359874211840, 140359874236415, ++ERASE, 140359874211840, 140359874236415, ++STORE, 140359874211840, 140359874236415, ++ERASE, 140359874236416, 140359874252799, ++STORE, 140359874236416, 140359874252799, ++SNULL, 140359874228223, 140359874236415, ++STORE, 140359874211840, 140359874228223, ++STORE, 140359874228224, 140359874236415, ++SNULL, 94077795168255, 94077795172351, ++STORE, 94077795160064, 94077795168255, ++STORE, 94077795168256, 94077795172351, ++SNULL, 140359876497407, 140359876501503, ++STORE, 140359876493312, 140359876497407, ++STORE, 140359876497408, 140359876501503, ++ERASE, 140359876464640, 140359876493311, ++STORE, 94077808717824, 94077808852991, ++STORE, 94549486252032, 94549486465023, ++STORE, 94549488562176, 94549488566271, ++STORE, 94549488566272, 94549488574463, ++STORE, 94549488574464, 94549488586751, ++STORE, 94549503492096, 94549506121727, ++STORE, 140085800894464, 140085802553343, ++STORE, 140085802553344, 140085804650495, ++STORE, 140085804650496, 140085804666879, ++STORE, 140085804666880, 140085804675071, ++STORE, 140085804675072, 140085804691455, ++STORE, 140085804691456, 140085804703743, ++STORE, 140085804703744, 140085806796799, ++STORE, 140085806796800, 140085806800895, ++STORE, 140085806800896, 140085806804991, ++STORE, 140085806804992, 140085806948351, ++STORE, 140085807316992, 140085809000447, ++STORE, 140085809000448, 140085809016831, ++STORE, 140085809045504, 140085809049599, ++STORE, 140085809049600, 140085809053695, ++STORE, 140085809053696, 140085809057791, ++STORE, 140731810545664, 140731810684927, ++STORE, 140731810967552, 140731810979839, ++STORE, 140731810979840, 140731810983935, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724752330752, 140737488351231, ++SNULL, 140724752338943, 140737488351231, ++STORE, 140724752330752, 140724752338943, ++STORE, 140724752199680, 140724752338943, ++STORE, 94656357539840, 94656359874559, ++SNULL, 94656357752831, 94656359874559, ++STORE, 94656357539840, 94656357752831, ++STORE, 94656357752832, 94656359874559, ++ERASE, 94656357752832, 94656359874559, ++STORE, 94656359849984, 94656359862271, ++STORE, 94656359862272, 94656359874559, ++STORE, 139632585203712, 139632587456511, ++SNULL, 139632585347071, 139632587456511, ++STORE, 139632585203712, 139632585347071, ++STORE, 139632585347072, 139632587456511, ++ERASE, 139632585347072, 139632587456511, ++STORE, 139632587444224, 139632587452415, ++STORE, 139632587452416, 139632587456511, ++STORE, 139632587440128, 139632587444223, ++STORE, 139632587427840, 139632587440127, ++STORE, 139632587399168, 139632587427839, ++STORE, 139632587390976, 139632587399167, ++STORE, 139632583090176, 139632585203711, ++SNULL, 139632583090176, 139632583102463, ++STORE, 139632583102464, 139632585203711, ++STORE, 139632583090176, 139632583102463, ++SNULL, 139632585195519, 139632585203711, ++STORE, 139632583102464, 139632585195519, ++STORE, 139632585195520, 139632585203711, ++ERASE, 139632585195520, 139632585203711, ++STORE, 139632585195520, 139632585203711, ++STORE, 139632579293184, 139632583090175, ++SNULL, 139632579293184, 139632580952063, ++STORE, 139632580952064, 139632583090175, ++STORE, 139632579293184, 139632580952063, ++SNULL, 139632583049215, 139632583090175, ++STORE, 139632580952064, 139632583049215, ++STORE, 139632583049216, 139632583090175, ++SNULL, 139632583049216, 139632583073791, ++STORE, 139632583073792, 139632583090175, ++STORE, 139632583049216, 139632583073791, ++ERASE, 139632583049216, 139632583073791, ++STORE, 139632583049216, 139632583073791, ++ERASE, 139632583073792, 139632583090175, ++STORE, 139632583073792, 139632583090175, ++STORE, 139632587382784, 139632587399167, ++SNULL, 139632583065599, 139632583073791, ++STORE, 139632583049216, 139632583065599, ++STORE, 139632583065600, 139632583073791, ++SNULL, 139632585199615, 139632585203711, ++STORE, 139632585195520, 139632585199615, ++STORE, 139632585199616, 139632585203711, ++SNULL, 94656359854079, 94656359862271, ++STORE, 94656359849984, 94656359854079, ++STORE, 94656359854080, 94656359862271, ++SNULL, 139632587448319, 139632587452415, ++STORE, 139632587444224, 139632587448319, ++STORE, 139632587448320, 139632587452415, ++ERASE, 139632587399168, 139632587427839, ++STORE, 94656378912768, 94656379047935, ++STORE, 139632585699328, 139632587382783, ++STORE, 94656378912768, 94656379183103, ++STORE, 94656378912768, 94656379318271, ++STORE, 94656378912768, 94656379494399, ++SNULL, 94656379469823, 94656379494399, ++STORE, 94656378912768, 94656379469823, ++STORE, 94656379469824, 94656379494399, ++ERASE, 94656379469824, 94656379494399, ++STORE, 94656378912768, 94656379621375, ++STORE, 94656378912768, 94656379756543, ++STORE, 94656378912768, 94656379912191, ++STORE, 94656378912768, 94656380055551, ++STORE, 94656378912768, 94656380190719, ++STORE, 94656378912768, 94656380338175, ++SNULL, 94656380313599, 94656380338175, ++STORE, 94656378912768, 94656380313599, ++STORE, 94656380313600, 94656380338175, ++ERASE, 94656380313600, 94656380338175, ++STORE, 94656378912768, 94656380448767, ++SNULL, 94656380432383, 94656380448767, ++STORE, 94656378912768, 94656380432383, ++STORE, 94656380432384, 94656380448767, ++ERASE, 94656380432384, 94656380448767, ++STORE, 94656378912768, 94656380567551, ++STORE, 94656378912768, 94656380719103, ++STORE, 94656378912768, 94656380858367, ++STORE, 94656378912768, 94656380997631, ++STORE, 94656378912768, 94656381132799, ++SNULL, 94656381124607, 94656381132799, ++STORE, 94656378912768, 94656381124607, ++STORE, 94656381124608, 94656381132799, ++ERASE, 94656381124608, 94656381132799, ++STORE, 94656378912768, 94656381276159, ++STORE, 94656378912768, 94656381427711, ++STORE, 94604087611392, 94604087824383, ++STORE, 94604089921536, 94604089925631, ++STORE, 94604089925632, 94604089933823, ++STORE, 94604089933824, 94604089946111, ++STORE, 94604105125888, 94604106424319, ++STORE, 140454937694208, 140454939353087, ++STORE, 140454939353088, 140454941450239, ++STORE, 140454941450240, 140454941466623, ++STORE, 140454941466624, 140454941474815, ++STORE, 140454941474816, 140454941491199, ++STORE, 140454941491200, 140454941503487, ++STORE, 140454941503488, 140454943596543, ++STORE, 140454943596544, 140454943600639, ++STORE, 140454943600640, 140454943604735, ++STORE, 140454943604736, 140454943748095, ++STORE, 140454944116736, 140454945800191, ++STORE, 140454945800192, 140454945816575, ++STORE, 140454945845248, 140454945849343, ++STORE, 140454945849344, 140454945853439, ++STORE, 140454945853440, 140454945857535, ++STORE, 140728438214656, 140728438353919, ++STORE, 140728439095296, 140728439107583, ++STORE, 140728439107584, 140728439111679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727821099008, 140737488351231, ++SNULL, 140727821107199, 140737488351231, ++STORE, 140727821099008, 140727821107199, ++STORE, 140727820967936, 140727821107199, ++STORE, 94088457240576, 94088459575295, ++SNULL, 94088457453567, 94088459575295, ++STORE, 94088457240576, 94088457453567, ++STORE, 94088457453568, 94088459575295, ++ERASE, 94088457453568, 94088459575295, ++STORE, 94088459550720, 94088459563007, ++STORE, 94088459563008, 94088459575295, ++STORE, 140234378989568, 140234381242367, ++SNULL, 140234379132927, 140234381242367, ++STORE, 140234378989568, 140234379132927, ++STORE, 140234379132928, 140234381242367, ++ERASE, 140234379132928, 140234381242367, ++STORE, 140234381230080, 140234381238271, ++STORE, 140234381238272, 140234381242367, ++STORE, 140727822077952, 140727822082047, ++STORE, 140727822065664, 140727822077951, ++STORE, 140234381201408, 140234381230079, ++STORE, 140234381193216, 140234381201407, ++STORE, 140234376876032, 140234378989567, ++SNULL, 140234376876032, 140234376888319, ++STORE, 140234376888320, 140234378989567, ++STORE, 140234376876032, 140234376888319, ++SNULL, 140234378981375, 140234378989567, ++STORE, 140234376888320, 140234378981375, ++STORE, 140234378981376, 140234378989567, ++ERASE, 140234378981376, 140234378989567, ++STORE, 140234378981376, 140234378989567, ++STORE, 140234373079040, 140234376876031, ++SNULL, 140234373079040, 140234374737919, ++STORE, 140234374737920, 140234376876031, ++STORE, 140234373079040, 140234374737919, ++SNULL, 140234376835071, 140234376876031, ++STORE, 140234374737920, 140234376835071, ++STORE, 140234376835072, 140234376876031, ++SNULL, 140234376835072, 140234376859647, ++STORE, 140234376859648, 140234376876031, ++STORE, 140234376835072, 140234376859647, ++ERASE, 140234376835072, 140234376859647, ++STORE, 140234376835072, 140234376859647, ++ERASE, 140234376859648, 140234376876031, ++STORE, 140234376859648, 140234376876031, ++STORE, 140234381185024, 140234381201407, ++SNULL, 140234376851455, 140234376859647, ++STORE, 140234376835072, 140234376851455, ++STORE, 140234376851456, 140234376859647, ++SNULL, 140234378985471, 140234378989567, ++STORE, 140234378981376, 140234378985471, ++STORE, 140234378985472, 140234378989567, ++SNULL, 94088459554815, 94088459563007, ++STORE, 94088459550720, 94088459554815, ++STORE, 94088459554816, 94088459563007, ++SNULL, 140234381234175, 140234381238271, ++STORE, 140234381230080, 140234381234175, ++STORE, 140234381234176, 140234381238271, ++ERASE, 140234381201408, 140234381230079, ++STORE, 94088468852736, 94088468987903, ++STORE, 140234379501568, 140234381185023, ++STORE, 94088468852736, 94088469123071, ++STORE, 94088468852736, 94088469258239, ++STORE, 94110050402304, 94110050615295, ++STORE, 94110052712448, 94110052716543, ++STORE, 94110052716544, 94110052724735, ++STORE, 94110052724736, 94110052737023, ++STORE, 94110061875200, 94110062415871, ++STORE, 140139439357952, 140139441016831, ++STORE, 140139441016832, 140139443113983, ++STORE, 140139443113984, 140139443130367, ++STORE, 140139443130368, 140139443138559, ++STORE, 140139443138560, 140139443154943, ++STORE, 140139443154944, 140139443167231, ++STORE, 140139443167232, 140139445260287, ++STORE, 140139445260288, 140139445264383, ++STORE, 140139445264384, 140139445268479, ++STORE, 140139445268480, 140139445411839, ++STORE, 140139445780480, 140139447463935, ++STORE, 140139447463936, 140139447480319, ++STORE, 140139447508992, 140139447513087, ++STORE, 140139447513088, 140139447517183, ++STORE, 140139447517184, 140139447521279, ++STORE, 140731901427712, 140731901566975, ++STORE, 140731902259200, 140731902271487, ++STORE, 140731902271488, 140731902275583, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727282622464, 140737488351231, ++SNULL, 140727282630655, 140737488351231, ++STORE, 140727282622464, 140727282630655, ++STORE, 140727282491392, 140727282630655, ++STORE, 94266649866240, 94266652200959, ++SNULL, 94266650079231, 94266652200959, ++STORE, 94266649866240, 94266650079231, ++STORE, 94266650079232, 94266652200959, ++ERASE, 94266650079232, 94266652200959, ++STORE, 94266652176384, 94266652188671, ++STORE, 94266652188672, 94266652200959, ++STORE, 139888497991680, 139888500244479, ++SNULL, 139888498135039, 139888500244479, ++STORE, 139888497991680, 139888498135039, ++STORE, 139888498135040, 139888500244479, ++ERASE, 139888498135040, 139888500244479, ++STORE, 139888500232192, 139888500240383, ++STORE, 139888500240384, 139888500244479, ++STORE, 140727283113984, 140727283118079, ++STORE, 140727283101696, 140727283113983, ++STORE, 139888500203520, 139888500232191, ++STORE, 139888500195328, 139888500203519, ++STORE, 139888495878144, 139888497991679, ++SNULL, 139888495878144, 139888495890431, ++STORE, 139888495890432, 139888497991679, ++STORE, 139888495878144, 139888495890431, ++SNULL, 139888497983487, 139888497991679, ++STORE, 139888495890432, 139888497983487, ++STORE, 139888497983488, 139888497991679, ++ERASE, 139888497983488, 139888497991679, ++STORE, 139888497983488, 139888497991679, ++STORE, 139888492081152, 139888495878143, ++SNULL, 139888492081152, 139888493740031, ++STORE, 139888493740032, 139888495878143, ++STORE, 139888492081152, 139888493740031, ++SNULL, 139888495837183, 139888495878143, ++STORE, 139888493740032, 139888495837183, ++STORE, 139888495837184, 139888495878143, ++SNULL, 139888495837184, 139888495861759, ++STORE, 139888495861760, 139888495878143, ++STORE, 139888495837184, 139888495861759, ++ERASE, 139888495837184, 139888495861759, ++STORE, 139888495837184, 139888495861759, ++ERASE, 139888495861760, 139888495878143, ++STORE, 139888495861760, 139888495878143, ++STORE, 139888500187136, 139888500203519, ++SNULL, 139888495853567, 139888495861759, ++STORE, 139888495837184, 139888495853567, ++STORE, 139888495853568, 139888495861759, ++SNULL, 139888497987583, 139888497991679, ++STORE, 139888497983488, 139888497987583, ++STORE, 139888497987584, 139888497991679, ++SNULL, 94266652180479, 94266652188671, ++STORE, 94266652176384, 94266652180479, ++STORE, 94266652180480, 94266652188671, ++SNULL, 139888500236287, 139888500240383, ++STORE, 139888500232192, 139888500236287, ++STORE, 139888500236288, 139888500240383, ++ERASE, 139888500203520, 139888500232191, ++STORE, 94266678542336, 94266678677503, ++STORE, 139888498503680, 139888500187135, ++STORE, 94266678542336, 94266678812671, ++STORE, 94266678542336, 94266678947839, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722507702272, 140737488351231, ++SNULL, 140722507710463, 140737488351231, ++STORE, 140722507702272, 140722507710463, ++STORE, 140722507571200, 140722507710463, ++STORE, 94313981394944, 94313983729663, ++SNULL, 94313981607935, 94313983729663, ++STORE, 94313981394944, 94313981607935, ++STORE, 94313981607936, 94313983729663, ++ERASE, 94313981607936, 94313983729663, ++STORE, 94313983705088, 94313983717375, ++STORE, 94313983717376, 94313983729663, ++STORE, 140456286076928, 140456288329727, ++SNULL, 140456286220287, 140456288329727, ++STORE, 140456286076928, 140456286220287, ++STORE, 140456286220288, 140456288329727, ++ERASE, 140456286220288, 140456288329727, ++STORE, 140456288317440, 140456288325631, ++STORE, 140456288325632, 140456288329727, ++STORE, 140722507997184, 140722508001279, ++STORE, 140722507984896, 140722507997183, ++STORE, 140456288288768, 140456288317439, ++STORE, 140456288280576, 140456288288767, ++STORE, 140456283963392, 140456286076927, ++SNULL, 140456283963392, 140456283975679, ++STORE, 140456283975680, 140456286076927, ++STORE, 140456283963392, 140456283975679, ++SNULL, 140456286068735, 140456286076927, ++STORE, 140456283975680, 140456286068735, ++STORE, 140456286068736, 140456286076927, ++ERASE, 140456286068736, 140456286076927, ++STORE, 140456286068736, 140456286076927, ++STORE, 140456280166400, 140456283963391, ++SNULL, 140456280166400, 140456281825279, ++STORE, 140456281825280, 140456283963391, ++STORE, 140456280166400, 140456281825279, ++SNULL, 140456283922431, 140456283963391, ++STORE, 140456281825280, 140456283922431, ++STORE, 140456283922432, 140456283963391, ++SNULL, 140456283922432, 140456283947007, ++STORE, 140456283947008, 140456283963391, ++STORE, 140456283922432, 140456283947007, ++ERASE, 140456283922432, 140456283947007, ++STORE, 140456283922432, 140456283947007, ++ERASE, 140456283947008, 140456283963391, ++STORE, 140456283947008, 140456283963391, ++STORE, 140456288272384, 140456288288767, ++SNULL, 140456283938815, 140456283947007, ++STORE, 140456283922432, 140456283938815, ++STORE, 140456283938816, 140456283947007, ++SNULL, 140456286072831, 140456286076927, ++STORE, 140456286068736, 140456286072831, ++STORE, 140456286072832, 140456286076927, ++SNULL, 94313983709183, 94313983717375, ++STORE, 94313983705088, 94313983709183, ++STORE, 94313983709184, 94313983717375, ++SNULL, 140456288321535, 140456288325631, ++STORE, 140456288317440, 140456288321535, ++STORE, 140456288321536, 140456288325631, ++ERASE, 140456288288768, 140456288317439, ++STORE, 94314006716416, 94314006851583, ++STORE, 140456286588928, 140456288272383, ++STORE, 94314006716416, 94314006986751, ++STORE, 94314006716416, 94314007121919, ++STORE, 93948644454400, 93948644667391, ++STORE, 93948646764544, 93948646768639, ++STORE, 93948646768640, 93948646776831, ++STORE, 93948646776832, 93948646789119, ++STORE, 93948664999936, 93948667142143, ++STORE, 140187350659072, 140187352317951, ++STORE, 140187352317952, 140187354415103, ++STORE, 140187354415104, 140187354431487, ++STORE, 140187354431488, 140187354439679, ++STORE, 140187354439680, 140187354456063, ++STORE, 140187354456064, 140187354468351, ++STORE, 140187354468352, 140187356561407, ++STORE, 140187356561408, 140187356565503, ++STORE, 140187356565504, 140187356569599, ++STORE, 140187356569600, 140187356712959, ++STORE, 140187357081600, 140187358765055, ++STORE, 140187358765056, 140187358781439, ++STORE, 140187358810112, 140187358814207, ++STORE, 140187358814208, 140187358818303, ++STORE, 140187358818304, 140187358822399, ++STORE, 140730484518912, 140730484658175, ++STORE, 140730485690368, 140730485702655, ++STORE, 140730485702656, 140730485706751, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721211551744, 140737488351231, ++SNULL, 140721211559935, 140737488351231, ++STORE, 140721211551744, 140721211559935, ++STORE, 140721211420672, 140721211559935, ++STORE, 94105221423104, 94105223757823, ++SNULL, 94105221636095, 94105223757823, ++STORE, 94105221423104, 94105221636095, ++STORE, 94105221636096, 94105223757823, ++ERASE, 94105221636096, 94105223757823, ++STORE, 94105223733248, 94105223745535, ++STORE, 94105223745536, 94105223757823, ++STORE, 140474453676032, 140474455928831, ++SNULL, 140474453819391, 140474455928831, ++STORE, 140474453676032, 140474453819391, ++STORE, 140474453819392, 140474455928831, ++ERASE, 140474453819392, 140474455928831, ++STORE, 140474455916544, 140474455924735, ++STORE, 140474455924736, 140474455928831, ++STORE, 140721211703296, 140721211707391, ++STORE, 140721211691008, 140721211703295, ++STORE, 140474455887872, 140474455916543, ++STORE, 140474455879680, 140474455887871, ++STORE, 140474451562496, 140474453676031, ++SNULL, 140474451562496, 140474451574783, ++STORE, 140474451574784, 140474453676031, ++STORE, 140474451562496, 140474451574783, ++SNULL, 140474453667839, 140474453676031, ++STORE, 140474451574784, 140474453667839, ++STORE, 140474453667840, 140474453676031, ++ERASE, 140474453667840, 140474453676031, ++STORE, 140474453667840, 140474453676031, ++STORE, 140474447765504, 140474451562495, ++SNULL, 140474447765504, 140474449424383, ++STORE, 140474449424384, 140474451562495, ++STORE, 140474447765504, 140474449424383, ++SNULL, 140474451521535, 140474451562495, ++STORE, 140474449424384, 140474451521535, ++STORE, 140474451521536, 140474451562495, ++SNULL, 140474451521536, 140474451546111, ++STORE, 140474451546112, 140474451562495, ++STORE, 140474451521536, 140474451546111, ++ERASE, 140474451521536, 140474451546111, ++STORE, 140474451521536, 140474451546111, ++ERASE, 140474451546112, 140474451562495, ++STORE, 140474451546112, 140474451562495, ++STORE, 140474455871488, 140474455887871, ++SNULL, 140474451537919, 140474451546111, ++STORE, 140474451521536, 140474451537919, ++STORE, 140474451537920, 140474451546111, ++SNULL, 140474453671935, 140474453676031, ++STORE, 140474453667840, 140474453671935, ++STORE, 140474453671936, 140474453676031, ++SNULL, 94105223737343, 94105223745535, ++STORE, 94105223733248, 94105223737343, ++STORE, 94105223737344, 94105223745535, ++SNULL, 140474455920639, 140474455924735, ++STORE, 140474455916544, 140474455920639, ++STORE, 140474455920640, 140474455924735, ++ERASE, 140474455887872, 140474455916543, ++STORE, 94105238712320, 94105238847487, ++STORE, 140474454188032, 140474455871487, ++STORE, 94105238712320, 94105238982655, ++STORE, 94105238712320, 94105239117823, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732356354048, 140737488351231, ++SNULL, 140732356362239, 140737488351231, ++STORE, 140732356354048, 140732356362239, ++STORE, 140732356222976, 140732356362239, ++STORE, 94461165989888, 94461168324607, ++SNULL, 94461166202879, 94461168324607, ++STORE, 94461165989888, 94461166202879, ++STORE, 94461166202880, 94461168324607, ++ERASE, 94461166202880, 94461168324607, ++STORE, 94461168300032, 94461168312319, ++STORE, 94461168312320, 94461168324607, ++STORE, 140317255110656, 140317257363455, ++SNULL, 140317255254015, 140317257363455, ++STORE, 140317255110656, 140317255254015, ++STORE, 140317255254016, 140317257363455, ++ERASE, 140317255254016, 140317257363455, ++STORE, 140317257351168, 140317257359359, ++STORE, 140317257359360, 140317257363455, ++STORE, 140732356583424, 140732356587519, ++STORE, 140732356571136, 140732356583423, ++STORE, 140317257322496, 140317257351167, ++STORE, 140317257314304, 140317257322495, ++STORE, 140317252997120, 140317255110655, ++SNULL, 140317252997120, 140317253009407, ++STORE, 140317253009408, 140317255110655, ++STORE, 140317252997120, 140317253009407, ++SNULL, 140317255102463, 140317255110655, ++STORE, 140317253009408, 140317255102463, ++STORE, 140317255102464, 140317255110655, ++ERASE, 140317255102464, 140317255110655, ++STORE, 140317255102464, 140317255110655, ++STORE, 140317249200128, 140317252997119, ++SNULL, 140317249200128, 140317250859007, ++STORE, 140317250859008, 140317252997119, ++STORE, 140317249200128, 140317250859007, ++SNULL, 140317252956159, 140317252997119, ++STORE, 140317250859008, 140317252956159, ++STORE, 140317252956160, 140317252997119, ++SNULL, 140317252956160, 140317252980735, ++STORE, 140317252980736, 140317252997119, ++STORE, 140317252956160, 140317252980735, ++ERASE, 140317252956160, 140317252980735, ++STORE, 140317252956160, 140317252980735, ++ERASE, 140317252980736, 140317252997119, ++STORE, 140317252980736, 140317252997119, ++STORE, 140317257306112, 140317257322495, ++SNULL, 140317252972543, 140317252980735, ++STORE, 140317252956160, 140317252972543, ++STORE, 140317252972544, 140317252980735, ++SNULL, 140317255106559, 140317255110655, ++STORE, 140317255102464, 140317255106559, ++STORE, 140317255106560, 140317255110655, ++SNULL, 94461168304127, 94461168312319, ++STORE, 94461168300032, 94461168304127, ++STORE, 94461168304128, 94461168312319, ++SNULL, 140317257355263, 140317257359359, ++STORE, 140317257351168, 140317257355263, ++STORE, 140317257355264, 140317257359359, ++ERASE, 140317257322496, 140317257351167, ++STORE, 94461195268096, 94461195403263, ++STORE, 140317255622656, 140317257306111, ++STORE, 94461195268096, 94461195538431, ++STORE, 94461195268096, 94461195673599, ++STORE, 94110050402304, 94110050615295, ++STORE, 94110052712448, 94110052716543, ++STORE, 94110052716544, 94110052724735, ++STORE, 94110052724736, 94110052737023, ++STORE, 94110061875200, 94110062415871, ++STORE, 140139439357952, 140139441016831, ++STORE, 140139441016832, 140139443113983, ++STORE, 140139443113984, 140139443130367, ++STORE, 140139443130368, 140139443138559, ++STORE, 140139443138560, 140139443154943, ++STORE, 140139443154944, 140139443167231, ++STORE, 140139443167232, 140139445260287, ++STORE, 140139445260288, 140139445264383, ++STORE, 140139445264384, 140139445268479, ++STORE, 140139445268480, 140139445411839, ++STORE, 140139445780480, 140139447463935, ++STORE, 140139447463936, 140139447480319, ++STORE, 140139447508992, 140139447513087, ++STORE, 140139447513088, 140139447517183, ++STORE, 140139447517184, 140139447521279, ++STORE, 140731901427712, 140731901566975, ++STORE, 140731902259200, 140731902271487, ++STORE, 140731902271488, 140731902275583, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720941613056, 140737488351231, ++SNULL, 140720941621247, 140737488351231, ++STORE, 140720941613056, 140720941621247, ++STORE, 140720941481984, 140720941621247, ++STORE, 93902377721856, 93902379945983, ++SNULL, 93902377832447, 93902379945983, ++STORE, 93902377721856, 93902377832447, ++STORE, 93902377832448, 93902379945983, ++ERASE, 93902377832448, 93902379945983, ++STORE, 93902379925504, 93902379937791, ++STORE, 93902379937792, 93902379945983, ++STORE, 139836543635456, 139836545888255, ++SNULL, 139836543778815, 139836545888255, ++STORE, 139836543635456, 139836543778815, ++STORE, 139836543778816, 139836545888255, ++ERASE, 139836543778816, 139836545888255, ++STORE, 139836545875968, 139836545884159, ++STORE, 139836545884160, 139836545888255, ++STORE, 140720941711360, 140720941715455, ++STORE, 140720941699072, 140720941711359, ++STORE, 139836545847296, 139836545875967, ++STORE, 139836545839104, 139836545847295, ++STORE, 139836539838464, 139836543635455, ++SNULL, 139836539838464, 139836541497343, ++STORE, 139836541497344, 139836543635455, ++STORE, 139836539838464, 139836541497343, ++SNULL, 139836543594495, 139836543635455, ++STORE, 139836541497344, 139836543594495, ++STORE, 139836543594496, 139836543635455, ++SNULL, 139836543594496, 139836543619071, ++STORE, 139836543619072, 139836543635455, ++STORE, 139836543594496, 139836543619071, ++ERASE, 139836543594496, 139836543619071, ++STORE, 139836543594496, 139836543619071, ++ERASE, 139836543619072, 139836543635455, ++STORE, 139836543619072, 139836543635455, ++SNULL, 139836543610879, 139836543619071, ++STORE, 139836543594496, 139836543610879, ++STORE, 139836543610880, 139836543619071, ++SNULL, 93902379933695, 93902379937791, ++STORE, 93902379925504, 93902379933695, ++STORE, 93902379933696, 93902379937791, ++SNULL, 139836545880063, 139836545884159, ++STORE, 139836545875968, 139836545880063, ++STORE, 139836545880064, 139836545884159, ++ERASE, 139836545847296, 139836545875967, ++STORE, 93902396891136, 93902397026303, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736538206208, 140737488351231, ++SNULL, 140736538214399, 140737488351231, ++STORE, 140736538206208, 140736538214399, ++STORE, 140736538075136, 140736538214399, ++STORE, 94173471399936, 94173473734655, ++SNULL, 94173471612927, 94173473734655, ++STORE, 94173471399936, 94173471612927, ++STORE, 94173471612928, 94173473734655, ++ERASE, 94173471612928, 94173473734655, ++STORE, 94173473710080, 94173473722367, ++STORE, 94173473722368, 94173473734655, ++STORE, 140035513556992, 140035515809791, ++SNULL, 140035513700351, 140035515809791, ++STORE, 140035513556992, 140035513700351, ++STORE, 140035513700352, 140035515809791, ++ERASE, 140035513700352, 140035515809791, ++STORE, 140035515797504, 140035515805695, ++STORE, 140035515805696, 140035515809791, ++STORE, 140736538329088, 140736538333183, ++STORE, 140736538316800, 140736538329087, ++STORE, 140035515768832, 140035515797503, ++STORE, 140035515760640, 140035515768831, ++STORE, 140035511443456, 140035513556991, ++SNULL, 140035511443456, 140035511455743, ++STORE, 140035511455744, 140035513556991, ++STORE, 140035511443456, 140035511455743, ++SNULL, 140035513548799, 140035513556991, ++STORE, 140035511455744, 140035513548799, ++STORE, 140035513548800, 140035513556991, ++ERASE, 140035513548800, 140035513556991, ++STORE, 140035513548800, 140035513556991, ++STORE, 140035507646464, 140035511443455, ++SNULL, 140035507646464, 140035509305343, ++STORE, 140035509305344, 140035511443455, ++STORE, 140035507646464, 140035509305343, ++SNULL, 140035511402495, 140035511443455, ++STORE, 140035509305344, 140035511402495, ++STORE, 140035511402496, 140035511443455, ++SNULL, 140035511402496, 140035511427071, ++STORE, 140035511427072, 140035511443455, ++STORE, 140035511402496, 140035511427071, ++ERASE, 140035511402496, 140035511427071, ++STORE, 140035511402496, 140035511427071, ++ERASE, 140035511427072, 140035511443455, ++STORE, 140035511427072, 140035511443455, ++STORE, 140035515752448, 140035515768831, ++SNULL, 140035511418879, 140035511427071, ++STORE, 140035511402496, 140035511418879, ++STORE, 140035511418880, 140035511427071, ++SNULL, 140035513552895, 140035513556991, ++STORE, 140035513548800, 140035513552895, ++STORE, 140035513552896, 140035513556991, ++SNULL, 94173473714175, 94173473722367, ++STORE, 94173473710080, 94173473714175, ++STORE, 94173473714176, 94173473722367, ++SNULL, 140035515801599, 140035515805695, ++STORE, 140035515797504, 140035515801599, ++STORE, 140035515801600, 140035515805695, ++ERASE, 140035515768832, 140035515797503, ++STORE, 94173478645760, 94173478780927, ++STORE, 140035514068992, 140035515752447, ++STORE, 94173478645760, 94173478916095, ++STORE, 94173478645760, 94173479051263, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140724216176640, 140737488351231, ++SNULL, 140724216184831, 140737488351231, ++STORE, 140724216176640, 140724216184831, ++STORE, 140724216045568, 140724216184831, ++STORE, 94870930628608, 94870932963327, ++SNULL, 94870930841599, 94870932963327, ++STORE, 94870930628608, 94870930841599, ++STORE, 94870930841600, 94870932963327, ++ERASE, 94870930841600, 94870932963327, ++STORE, 94870932938752, 94870932951039, ++STORE, 94870932951040, 94870932963327, ++STORE, 140453683736576, 140453685989375, ++SNULL, 140453683879935, 140453685989375, ++STORE, 140453683736576, 140453683879935, ++STORE, 140453683879936, 140453685989375, ++ERASE, 140453683879936, 140453685989375, ++STORE, 140453685977088, 140453685985279, ++STORE, 140453685985280, 140453685989375, ++STORE, 140724216832000, 140724216836095, ++STORE, 140724216819712, 140724216831999, ++STORE, 140453685948416, 140453685977087, ++STORE, 140453685940224, 140453685948415, ++STORE, 140453681623040, 140453683736575, ++SNULL, 140453681623040, 140453681635327, ++STORE, 140453681635328, 140453683736575, ++STORE, 140453681623040, 140453681635327, ++SNULL, 140453683728383, 140453683736575, ++STORE, 140453681635328, 140453683728383, ++STORE, 140453683728384, 140453683736575, ++ERASE, 140453683728384, 140453683736575, ++STORE, 140453683728384, 140453683736575, ++STORE, 140453677826048, 140453681623039, ++SNULL, 140453677826048, 140453679484927, ++STORE, 140453679484928, 140453681623039, ++STORE, 140453677826048, 140453679484927, ++SNULL, 140453681582079, 140453681623039, ++STORE, 140453679484928, 140453681582079, ++STORE, 140453681582080, 140453681623039, ++SNULL, 140453681582080, 140453681606655, ++STORE, 140453681606656, 140453681623039, ++STORE, 140453681582080, 140453681606655, ++ERASE, 140453681582080, 140453681606655, ++STORE, 140453681582080, 140453681606655, ++ERASE, 140453681606656, 140453681623039, ++STORE, 140453681606656, 140453681623039, ++STORE, 140453685932032, 140453685948415, ++SNULL, 140453681598463, 140453681606655, ++STORE, 140453681582080, 140453681598463, ++STORE, 140453681598464, 140453681606655, ++SNULL, 140453683732479, 140453683736575, ++STORE, 140453683728384, 140453683732479, ++STORE, 140453683732480, 140453683736575, ++SNULL, 94870932942847, 94870932951039, ++STORE, 94870932938752, 94870932942847, ++STORE, 94870932942848, 94870932951039, ++SNULL, 140453685981183, 140453685985279, ++STORE, 140453685977088, 140453685981183, ++STORE, 140453685981184, 140453685985279, ++ERASE, 140453685948416, 140453685977087, ++STORE, 94870940565504, 94870940700671, ++STORE, 140453684248576, 140453685932031, ++STORE, 94870940565504, 94870940835839, ++STORE, 94870940565504, 94870940971007, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731275661312, 140737488351231, ++SNULL, 140731275669503, 140737488351231, ++STORE, 140731275661312, 140731275669503, ++STORE, 140731275530240, 140731275669503, ++STORE, 94642788548608, 94642790883327, ++SNULL, 94642788761599, 94642790883327, ++STORE, 94642788548608, 94642788761599, ++STORE, 94642788761600, 94642790883327, ++ERASE, 94642788761600, 94642790883327, ++STORE, 94642790858752, 94642790871039, ++STORE, 94642790871040, 94642790883327, ++STORE, 140228458749952, 140228461002751, ++SNULL, 140228458893311, 140228461002751, ++STORE, 140228458749952, 140228458893311, ++STORE, 140228458893312, 140228461002751, ++ERASE, 140228458893312, 140228461002751, ++STORE, 140228460990464, 140228460998655, ++STORE, 140228460998656, 140228461002751, ++STORE, 140731276349440, 140731276353535, ++STORE, 140731276337152, 140731276349439, ++STORE, 140228460961792, 140228460990463, ++STORE, 140228460953600, 140228460961791, ++STORE, 140228456636416, 140228458749951, ++SNULL, 140228456636416, 140228456648703, ++STORE, 140228456648704, 140228458749951, ++STORE, 140228456636416, 140228456648703, ++SNULL, 140228458741759, 140228458749951, ++STORE, 140228456648704, 140228458741759, ++STORE, 140228458741760, 140228458749951, ++ERASE, 140228458741760, 140228458749951, ++STORE, 140228458741760, 140228458749951, ++STORE, 140228452839424, 140228456636415, ++SNULL, 140228452839424, 140228454498303, ++STORE, 140228454498304, 140228456636415, ++STORE, 140228452839424, 140228454498303, ++SNULL, 140228456595455, 140228456636415, ++STORE, 140228454498304, 140228456595455, ++STORE, 140228456595456, 140228456636415, ++SNULL, 140228456595456, 140228456620031, ++STORE, 140228456620032, 140228456636415, ++STORE, 140228456595456, 140228456620031, ++ERASE, 140228456595456, 140228456620031, ++STORE, 140228456595456, 140228456620031, ++ERASE, 140228456620032, 140228456636415, ++STORE, 140228456620032, 140228456636415, ++STORE, 140228460945408, 140228460961791, ++SNULL, 140228456611839, 140228456620031, ++STORE, 140228456595456, 140228456611839, ++STORE, 140228456611840, 140228456620031, ++SNULL, 140228458745855, 140228458749951, ++STORE, 140228458741760, 140228458745855, ++STORE, 140228458745856, 140228458749951, ++SNULL, 94642790862847, 94642790871039, ++STORE, 94642790858752, 94642790862847, ++STORE, 94642790862848, 94642790871039, ++SNULL, 140228460994559, 140228460998655, ++STORE, 140228460990464, 140228460994559, ++STORE, 140228460994560, 140228460998655, ++ERASE, 140228460961792, 140228460990463, ++STORE, 94642801549312, 94642801684479, ++STORE, 140228459261952, 140228460945407, ++STORE, 94642801549312, 94642801819647, ++STORE, 94642801549312, 94642801954815, ++STORE, 94604087611392, 94604087824383, ++STORE, 94604089921536, 94604089925631, ++STORE, 94604089925632, 94604089933823, ++STORE, 94604089933824, 94604089946111, ++STORE, 94604105125888, 94604106424319, ++STORE, 140454937694208, 140454939353087, ++STORE, 140454939353088, 140454941450239, ++STORE, 140454941450240, 140454941466623, ++STORE, 140454941466624, 140454941474815, ++STORE, 140454941474816, 140454941491199, ++STORE, 140454941491200, 140454941503487, ++STORE, 140454941503488, 140454943596543, ++STORE, 140454943596544, 140454943600639, ++STORE, 140454943600640, 140454943604735, ++STORE, 140454943604736, 140454943748095, ++STORE, 140454944116736, 140454945800191, ++STORE, 140454945800192, 140454945816575, ++STORE, 140454945845248, 140454945849343, ++STORE, 140454945849344, 140454945853439, ++STORE, 140454945853440, 140454945857535, ++STORE, 140728438214656, 140728438353919, ++STORE, 140728439095296, 140728439107583, ++STORE, 140728439107584, 140728439111679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721843453952, 140737488351231, ++SNULL, 140721843462143, 140737488351231, ++STORE, 140721843453952, 140721843462143, ++STORE, 140721843322880, 140721843462143, ++STORE, 94465962455040, 94465964789759, ++SNULL, 94465962668031, 94465964789759, ++STORE, 94465962455040, 94465962668031, ++STORE, 94465962668032, 94465964789759, ++ERASE, 94465962668032, 94465964789759, ++STORE, 94465964765184, 94465964777471, ++STORE, 94465964777472, 94465964789759, ++STORE, 139913488314368, 139913490567167, ++SNULL, 139913488457727, 139913490567167, ++STORE, 139913488314368, 139913488457727, ++STORE, 139913488457728, 139913490567167, ++ERASE, 139913488457728, 139913490567167, ++STORE, 139913490554880, 139913490563071, ++STORE, 139913490563072, 139913490567167, ++STORE, 140721843503104, 140721843507199, ++STORE, 140721843490816, 140721843503103, ++STORE, 139913490526208, 139913490554879, ++STORE, 139913490518016, 139913490526207, ++STORE, 139913486200832, 139913488314367, ++SNULL, 139913486200832, 139913486213119, ++STORE, 139913486213120, 139913488314367, ++STORE, 139913486200832, 139913486213119, ++SNULL, 139913488306175, 139913488314367, ++STORE, 139913486213120, 139913488306175, ++STORE, 139913488306176, 139913488314367, ++ERASE, 139913488306176, 139913488314367, ++STORE, 139913488306176, 139913488314367, ++STORE, 139913482403840, 139913486200831, ++SNULL, 139913482403840, 139913484062719, ++STORE, 139913484062720, 139913486200831, ++STORE, 139913482403840, 139913484062719, ++SNULL, 139913486159871, 139913486200831, ++STORE, 139913484062720, 139913486159871, ++STORE, 139913486159872, 139913486200831, ++SNULL, 139913486159872, 139913486184447, ++STORE, 139913486184448, 139913486200831, ++STORE, 139913486159872, 139913486184447, ++ERASE, 139913486159872, 139913486184447, ++STORE, 139913486159872, 139913486184447, ++ERASE, 139913486184448, 139913486200831, ++STORE, 139913486184448, 139913486200831, ++STORE, 139913490509824, 139913490526207, ++SNULL, 139913486176255, 139913486184447, ++STORE, 139913486159872, 139913486176255, ++STORE, 139913486176256, 139913486184447, ++SNULL, 139913488310271, 139913488314367, ++STORE, 139913488306176, 139913488310271, ++STORE, 139913488310272, 139913488314367, ++SNULL, 94465964769279, 94465964777471, ++STORE, 94465964765184, 94465964769279, ++STORE, 94465964769280, 94465964777471, ++SNULL, 139913490558975, 139913490563071, ++STORE, 139913490554880, 139913490558975, ++STORE, 139913490558976, 139913490563071, ++ERASE, 139913490526208, 139913490554879, ++STORE, 94465970024448, 94465970159615, ++STORE, 139913488826368, 139913490509823, ++STORE, 94465970024448, 94465970294783, ++STORE, 94465970024448, 94465970429951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140720583307264, 140737488351231, ++SNULL, 140720583315455, 140737488351231, ++STORE, 140720583307264, 140720583315455, ++STORE, 140720583176192, 140720583315455, ++STORE, 94212322082816, 94212324417535, ++SNULL, 94212322295807, 94212324417535, ++STORE, 94212322082816, 94212322295807, ++STORE, 94212322295808, 94212324417535, ++ERASE, 94212322295808, 94212324417535, ++STORE, 94212324392960, 94212324405247, ++STORE, 94212324405248, 94212324417535, ++STORE, 139659688538112, 139659690790911, ++SNULL, 139659688681471, 139659690790911, ++STORE, 139659688538112, 139659688681471, ++STORE, 139659688681472, 139659690790911, ++ERASE, 139659688681472, 139659690790911, ++STORE, 139659690778624, 139659690786815, ++STORE, 139659690786816, 139659690790911, ++STORE, 140720584781824, 140720584785919, ++STORE, 140720584769536, 140720584781823, ++STORE, 139659690749952, 139659690778623, ++STORE, 139659690741760, 139659690749951, ++STORE, 139659686424576, 139659688538111, ++SNULL, 139659686424576, 139659686436863, ++STORE, 139659686436864, 139659688538111, ++STORE, 139659686424576, 139659686436863, ++SNULL, 139659688529919, 139659688538111, ++STORE, 139659686436864, 139659688529919, ++STORE, 139659688529920, 139659688538111, ++ERASE, 139659688529920, 139659688538111, ++STORE, 139659688529920, 139659688538111, ++STORE, 139659682627584, 139659686424575, ++SNULL, 139659682627584, 139659684286463, ++STORE, 139659684286464, 139659686424575, ++STORE, 139659682627584, 139659684286463, ++SNULL, 139659686383615, 139659686424575, ++STORE, 139659684286464, 139659686383615, ++STORE, 139659686383616, 139659686424575, ++SNULL, 139659686383616, 139659686408191, ++STORE, 139659686408192, 139659686424575, ++STORE, 139659686383616, 139659686408191, ++ERASE, 139659686383616, 139659686408191, ++STORE, 139659686383616, 139659686408191, ++ERASE, 139659686408192, 139659686424575, ++STORE, 139659686408192, 139659686424575, ++STORE, 139659690733568, 139659690749951, ++SNULL, 139659686399999, 139659686408191, ++STORE, 139659686383616, 139659686399999, ++STORE, 139659686400000, 139659686408191, ++SNULL, 139659688534015, 139659688538111, ++STORE, 139659688529920, 139659688534015, ++STORE, 139659688534016, 139659688538111, ++SNULL, 94212324397055, 94212324405247, ++STORE, 94212324392960, 94212324397055, ++STORE, 94212324397056, 94212324405247, ++SNULL, 139659690782719, 139659690786815, ++STORE, 139659690778624, 139659690782719, ++STORE, 139659690782720, 139659690786815, ++ERASE, 139659690749952, 139659690778623, ++STORE, 94212355014656, 94212355149823, ++STORE, 139659689050112, 139659690733567, ++STORE, 94212355014656, 94212355284991, ++STORE, 94212355014656, 94212355420159, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140727689830400, 140737488351231, ++SNULL, 140727689838591, 140737488351231, ++STORE, 140727689830400, 140727689838591, ++STORE, 140727689699328, 140727689838591, ++STORE, 94572390281216, 94572392615935, ++SNULL, 94572390494207, 94572392615935, ++STORE, 94572390281216, 94572390494207, ++STORE, 94572390494208, 94572392615935, ++ERASE, 94572390494208, 94572392615935, ++STORE, 94572392591360, 94572392603647, ++STORE, 94572392603648, 94572392615935, ++STORE, 140575923769344, 140575926022143, ++SNULL, 140575923912703, 140575926022143, ++STORE, 140575923769344, 140575923912703, ++STORE, 140575923912704, 140575926022143, ++ERASE, 140575923912704, 140575926022143, ++STORE, 140575926009856, 140575926018047, ++STORE, 140575926018048, 140575926022143, ++STORE, 140727689871360, 140727689875455, ++STORE, 140727689859072, 140727689871359, ++STORE, 140575925981184, 140575926009855, ++STORE, 140575925972992, 140575925981183, ++STORE, 140575921655808, 140575923769343, ++SNULL, 140575921655808, 140575921668095, ++STORE, 140575921668096, 140575923769343, ++STORE, 140575921655808, 140575921668095, ++SNULL, 140575923761151, 140575923769343, ++STORE, 140575921668096, 140575923761151, ++STORE, 140575923761152, 140575923769343, ++ERASE, 140575923761152, 140575923769343, ++STORE, 140575923761152, 140575923769343, ++STORE, 140575917858816, 140575921655807, ++SNULL, 140575917858816, 140575919517695, ++STORE, 140575919517696, 140575921655807, ++STORE, 140575917858816, 140575919517695, ++SNULL, 140575921614847, 140575921655807, ++STORE, 140575919517696, 140575921614847, ++STORE, 140575921614848, 140575921655807, ++SNULL, 140575921614848, 140575921639423, ++STORE, 140575921639424, 140575921655807, ++STORE, 140575921614848, 140575921639423, ++ERASE, 140575921614848, 140575921639423, ++STORE, 140575921614848, 140575921639423, ++ERASE, 140575921639424, 140575921655807, ++STORE, 140575921639424, 140575921655807, ++STORE, 140575925964800, 140575925981183, ++SNULL, 140575921631231, 140575921639423, ++STORE, 140575921614848, 140575921631231, ++STORE, 140575921631232, 140575921639423, ++SNULL, 140575923765247, 140575923769343, ++STORE, 140575923761152, 140575923765247, ++STORE, 140575923765248, 140575923769343, ++SNULL, 94572392595455, 94572392603647, ++STORE, 94572392591360, 94572392595455, ++STORE, 94572392595456, 94572392603647, ++SNULL, 140575926013951, 140575926018047, ++STORE, 140575926009856, 140575926013951, ++STORE, 140575926013952, 140575926018047, ++ERASE, 140575925981184, 140575926009855, ++STORE, 94572402278400, 94572402413567, ++STORE, 140575924281344, 140575925964799, ++STORE, 94572402278400, 94572402548735, ++STORE, 94572402278400, 94572402683903, ++STORE, 94572402278400, 94572402851839, ++SNULL, 94572402827263, 94572402851839, ++STORE, 94572402278400, 94572402827263, ++STORE, 94572402827264, 94572402851839, ++ERASE, 94572402827264, 94572402851839, ++STORE, 94572402278400, 94572402966527, ++STORE, 94572402278400, 94572403109887, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725520506880, 140737488351231, ++SNULL, 140725520515071, 140737488351231, ++STORE, 140725520506880, 140725520515071, ++STORE, 140725520375808, 140725520515071, ++STORE, 93829948788736, 93829951012863, ++SNULL, 93829948899327, 93829951012863, ++STORE, 93829948788736, 93829948899327, ++STORE, 93829948899328, 93829951012863, ++ERASE, 93829948899328, 93829951012863, ++STORE, 93829950992384, 93829951004671, ++STORE, 93829951004672, 93829951012863, ++STORE, 140133696794624, 140133699047423, ++SNULL, 140133696937983, 140133699047423, ++STORE, 140133696794624, 140133696937983, ++STORE, 140133696937984, 140133699047423, ++ERASE, 140133696937984, 140133699047423, ++STORE, 140133699035136, 140133699043327, ++STORE, 140133699043328, 140133699047423, ++STORE, 140725520875520, 140725520879615, ++STORE, 140725520863232, 140725520875519, ++STORE, 140133699006464, 140133699035135, ++STORE, 140133698998272, 140133699006463, ++STORE, 140133692997632, 140133696794623, ++SNULL, 140133692997632, 140133694656511, ++STORE, 140133694656512, 140133696794623, ++STORE, 140133692997632, 140133694656511, ++SNULL, 140133696753663, 140133696794623, ++STORE, 140133694656512, 140133696753663, ++STORE, 140133696753664, 140133696794623, ++SNULL, 140133696753664, 140133696778239, ++STORE, 140133696778240, 140133696794623, ++STORE, 140133696753664, 140133696778239, ++ERASE, 140133696753664, 140133696778239, ++STORE, 140133696753664, 140133696778239, ++ERASE, 140133696778240, 140133696794623, ++STORE, 140133696778240, 140133696794623, ++SNULL, 140133696770047, 140133696778239, ++STORE, 140133696753664, 140133696770047, ++STORE, 140133696770048, 140133696778239, ++SNULL, 93829951000575, 93829951004671, ++STORE, 93829950992384, 93829951000575, ++STORE, 93829951000576, 93829951004671, ++SNULL, 140133699039231, 140133699043327, ++STORE, 140133699035136, 140133699039231, ++STORE, 140133699039232, 140133699043327, ++ERASE, 140133699006464, 140133699035135, ++STORE, 93829978693632, 93829978828799, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140736118022144, 140737488351231, ++SNULL, 140736118030335, 140737488351231, ++STORE, 140736118022144, 140736118030335, ++STORE, 140736117891072, 140736118030335, ++STORE, 94467663982592, 94467666206719, ++SNULL, 94467664093183, 94467666206719, ++STORE, 94467663982592, 94467664093183, ++STORE, 94467664093184, 94467666206719, ++ERASE, 94467664093184, 94467666206719, ++STORE, 94467666186240, 94467666198527, ++STORE, 94467666198528, 94467666206719, ++STORE, 140525377327104, 140525379579903, ++SNULL, 140525377470463, 140525379579903, ++STORE, 140525377327104, 140525377470463, ++STORE, 140525377470464, 140525379579903, ++ERASE, 140525377470464, 140525379579903, ++STORE, 140525379567616, 140525379575807, ++STORE, 140525379575808, 140525379579903, ++STORE, 140736118771712, 140736118775807, ++STORE, 140736118759424, 140736118771711, ++STORE, 140525379538944, 140525379567615, ++STORE, 140525379530752, 140525379538943, ++STORE, 140525373530112, 140525377327103, ++SNULL, 140525373530112, 140525375188991, ++STORE, 140525375188992, 140525377327103, ++STORE, 140525373530112, 140525375188991, ++SNULL, 140525377286143, 140525377327103, ++STORE, 140525375188992, 140525377286143, ++STORE, 140525377286144, 140525377327103, ++SNULL, 140525377286144, 140525377310719, ++STORE, 140525377310720, 140525377327103, ++STORE, 140525377286144, 140525377310719, ++ERASE, 140525377286144, 140525377310719, ++STORE, 140525377286144, 140525377310719, ++ERASE, 140525377310720, 140525377327103, ++STORE, 140525377310720, 140525377327103, ++SNULL, 140525377302527, 140525377310719, ++STORE, 140525377286144, 140525377302527, ++STORE, 140525377302528, 140525377310719, ++SNULL, 94467666194431, 94467666198527, ++STORE, 94467666186240, 94467666194431, ++STORE, 94467666194432, 94467666198527, ++SNULL, 140525379571711, 140525379575807, ++STORE, 140525379567616, 140525379571711, ++STORE, 140525379571712, 140525379575807, ++ERASE, 140525379538944, 140525379567615, ++STORE, 94467693379584, 94467693514751, ++STORE, 94200172744704, 94200172957695, ++STORE, 94200175054848, 94200175058943, ++STORE, 94200175058944, 94200175067135, ++STORE, 94200175067136, 94200175079423, ++STORE, 94200196673536, 94200198905855, ++STORE, 140053867720704, 140053869379583, ++STORE, 140053869379584, 140053871476735, ++STORE, 140053871476736, 140053871493119, ++STORE, 140053871493120, 140053871501311, ++STORE, 140053871501312, 140053871517695, ++STORE, 140053871517696, 140053871529983, ++STORE, 140053871529984, 140053873623039, ++STORE, 140053873623040, 140053873627135, ++STORE, 140053873627136, 140053873631231, ++STORE, 140053873631232, 140053873774591, ++STORE, 140053874143232, 140053875826687, ++STORE, 140053875826688, 140053875843071, ++STORE, 140053875871744, 140053875875839, ++STORE, 140053875875840, 140053875879935, ++STORE, 140053875879936, 140053875884031, ++STORE, 140728538484736, 140728538623999, ++STORE, 140728538652672, 140728538664959, ++STORE, 140728538664960, 140728538669055, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732307775488, 140737488351231, ++SNULL, 140732307783679, 140737488351231, ++STORE, 140732307775488, 140732307783679, ++STORE, 140732307644416, 140732307783679, ++STORE, 93831417630720, 93831419965439, ++SNULL, 93831417843711, 93831419965439, ++STORE, 93831417630720, 93831417843711, ++STORE, 93831417843712, 93831419965439, ++ERASE, 93831417843712, 93831419965439, ++STORE, 93831419940864, 93831419953151, ++STORE, 93831419953152, 93831419965439, ++STORE, 140241062088704, 140241064341503, ++SNULL, 140241062232063, 140241064341503, ++STORE, 140241062088704, 140241062232063, ++STORE, 140241062232064, 140241064341503, ++ERASE, 140241062232064, 140241064341503, ++STORE, 140241064329216, 140241064337407, ++STORE, 140241064337408, 140241064341503, ++STORE, 140732308140032, 140732308144127, ++STORE, 140732308127744, 140732308140031, ++STORE, 140241064300544, 140241064329215, ++STORE, 140241064292352, 140241064300543, ++STORE, 140241059975168, 140241062088703, ++SNULL, 140241059975168, 140241059987455, ++STORE, 140241059987456, 140241062088703, ++STORE, 140241059975168, 140241059987455, ++SNULL, 140241062080511, 140241062088703, ++STORE, 140241059987456, 140241062080511, ++STORE, 140241062080512, 140241062088703, ++ERASE, 140241062080512, 140241062088703, ++STORE, 140241062080512, 140241062088703, ++STORE, 140241056178176, 140241059975167, ++SNULL, 140241056178176, 140241057837055, ++STORE, 140241057837056, 140241059975167, ++STORE, 140241056178176, 140241057837055, ++SNULL, 140241059934207, 140241059975167, ++STORE, 140241057837056, 140241059934207, ++STORE, 140241059934208, 140241059975167, ++SNULL, 140241059934208, 140241059958783, ++STORE, 140241059958784, 140241059975167, ++STORE, 140241059934208, 140241059958783, ++ERASE, 140241059934208, 140241059958783, ++STORE, 140241059934208, 140241059958783, ++ERASE, 140241059958784, 140241059975167, ++STORE, 140241059958784, 140241059975167, ++STORE, 140241064284160, 140241064300543, ++SNULL, 140241059950591, 140241059958783, ++STORE, 140241059934208, 140241059950591, ++STORE, 140241059950592, 140241059958783, ++SNULL, 140241062084607, 140241062088703, ++STORE, 140241062080512, 140241062084607, ++STORE, 140241062084608, 140241062088703, ++SNULL, 93831419944959, 93831419953151, ++STORE, 93831419940864, 93831419944959, ++STORE, 93831419944960, 93831419953151, ++SNULL, 140241064333311, 140241064337407, ++STORE, 140241064329216, 140241064333311, ++STORE, 140241064333312, 140241064337407, ++ERASE, 140241064300544, 140241064329215, ++STORE, 93831435284480, 93831435419647, ++STORE, 140241062600704, 140241064284159, ++STORE, 93831435284480, 93831435554815, ++STORE, 93831435284480, 93831435689983, ++STORE, 93831435284480, 93831435862015, ++SNULL, 93831435837439, 93831435862015, ++STORE, 93831435284480, 93831435837439, ++STORE, 93831435837440, 93831435862015, ++ERASE, 93831435837440, 93831435862015, ++STORE, 93831435284480, 93831435972607, ++STORE, 93831435284480, 93831436107775, ++SNULL, 93831436091391, 93831436107775, ++STORE, 93831435284480, 93831436091391, ++STORE, 93831436091392, 93831436107775, ++ERASE, 93831436091392, 93831436107775, ++STORE, 93831435284480, 93831436226559, ++STORE, 93831435284480, 93831436361727, ++STORE, 93831435284480, 93831436505087, ++STORE, 93831435284480, 93831436652543, ++STORE, 93831435284480, 93831436787711, ++STORE, 93831435284480, 93831436926975, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140728546775040, 140737488351231, ++SNULL, 140728546783231, 140737488351231, ++STORE, 140728546775040, 140728546783231, ++STORE, 140728546643968, 140728546783231, ++STORE, 94456178786304, 94456181010431, ++SNULL, 94456178896895, 94456181010431, ++STORE, 94456178786304, 94456178896895, ++STORE, 94456178896896, 94456181010431, ++ERASE, 94456178896896, 94456181010431, ++STORE, 94456180989952, 94456181002239, ++STORE, 94456181002240, 94456181010431, ++STORE, 140221893091328, 140221895344127, ++SNULL, 140221893234687, 140221895344127, ++STORE, 140221893091328, 140221893234687, ++STORE, 140221893234688, 140221895344127, ++ERASE, 140221893234688, 140221895344127, ++STORE, 140221895331840, 140221895340031, ++STORE, 140221895340032, 140221895344127, ++STORE, 140728547803136, 140728547807231, ++STORE, 140728547790848, 140728547803135, ++STORE, 140221895303168, 140221895331839, ++STORE, 140221895294976, 140221895303167, ++STORE, 140221889294336, 140221893091327, ++SNULL, 140221889294336, 140221890953215, ++STORE, 140221890953216, 140221893091327, ++STORE, 140221889294336, 140221890953215, ++SNULL, 140221893050367, 140221893091327, ++STORE, 140221890953216, 140221893050367, ++STORE, 140221893050368, 140221893091327, ++SNULL, 140221893050368, 140221893074943, ++STORE, 140221893074944, 140221893091327, ++STORE, 140221893050368, 140221893074943, ++ERASE, 140221893050368, 140221893074943, ++STORE, 140221893050368, 140221893074943, ++ERASE, 140221893074944, 140221893091327, ++STORE, 140221893074944, 140221893091327, ++SNULL, 140221893066751, 140221893074943, ++STORE, 140221893050368, 140221893066751, ++STORE, 140221893066752, 140221893074943, ++SNULL, 94456180998143, 94456181002239, ++STORE, 94456180989952, 94456180998143, ++STORE, 94456180998144, 94456181002239, ++SNULL, 140221895335935, 140221895340031, ++STORE, 140221895331840, 140221895335935, ++STORE, 140221895335936, 140221895340031, ++ERASE, 140221895303168, 140221895331839, ++STORE, 94456203730944, 94456203866111, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140734438637568, 140737488351231, ++SNULL, 140734438645759, 140737488351231, ++STORE, 140734438637568, 140734438645759, ++STORE, 140734438506496, 140734438645759, ++STORE, 94652233351168, 94652235575295, ++SNULL, 94652233461759, 94652235575295, ++STORE, 94652233351168, 94652233461759, ++STORE, 94652233461760, 94652235575295, ++ERASE, 94652233461760, 94652235575295, ++STORE, 94652235554816, 94652235567103, ++STORE, 94652235567104, 94652235575295, ++STORE, 140536493195264, 140536495448063, ++SNULL, 140536493338623, 140536495448063, ++STORE, 140536493195264, 140536493338623, ++STORE, 140536493338624, 140536495448063, ++ERASE, 140536493338624, 140536495448063, ++STORE, 140536495435776, 140536495443967, ++STORE, 140536495443968, 140536495448063, ++STORE, 140734439002112, 140734439006207, ++STORE, 140734438989824, 140734439002111, ++STORE, 140536495407104, 140536495435775, ++STORE, 140536495398912, 140536495407103, ++STORE, 140536489398272, 140536493195263, ++SNULL, 140536489398272, 140536491057151, ++STORE, 140536491057152, 140536493195263, ++STORE, 140536489398272, 140536491057151, ++SNULL, 140536493154303, 140536493195263, ++STORE, 140536491057152, 140536493154303, ++STORE, 140536493154304, 140536493195263, ++SNULL, 140536493154304, 140536493178879, ++STORE, 140536493178880, 140536493195263, ++STORE, 140536493154304, 140536493178879, ++ERASE, 140536493154304, 140536493178879, ++STORE, 140536493154304, 140536493178879, ++ERASE, 140536493178880, 140536493195263, ++STORE, 140536493178880, 140536493195263, ++SNULL, 140536493170687, 140536493178879, ++STORE, 140536493154304, 140536493170687, ++STORE, 140536493170688, 140536493178879, ++SNULL, 94652235563007, 94652235567103, ++STORE, 94652235554816, 94652235563007, ++STORE, 94652235563008, 94652235567103, ++SNULL, 140536495439871, 140536495443967, ++STORE, 140536495435776, 140536495439871, ++STORE, 140536495439872, 140536495443967, ++ERASE, 140536495407104, 140536495435775, ++STORE, 94652265619456, 94652265754623, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140721814200320, 140737488351231, ++SNULL, 140721814208511, 140737488351231, ++STORE, 140721814200320, 140721814208511, ++STORE, 140721814069248, 140721814208511, ++STORE, 94062800691200, 94062802915327, ++SNULL, 94062800801791, 94062802915327, ++STORE, 94062800691200, 94062800801791, ++STORE, 94062800801792, 94062802915327, ++ERASE, 94062800801792, 94062802915327, ++STORE, 94062802894848, 94062802907135, ++STORE, 94062802907136, 94062802915327, ++STORE, 139717739700224, 139717741953023, ++SNULL, 139717739843583, 139717741953023, ++STORE, 139717739700224, 139717739843583, ++STORE, 139717739843584, 139717741953023, ++ERASE, 139717739843584, 139717741953023, ++STORE, 139717741940736, 139717741948927, ++STORE, 139717741948928, 139717741953023, ++STORE, 140721814224896, 140721814228991, ++STORE, 140721814212608, 140721814224895, ++STORE, 139717741912064, 139717741940735, ++STORE, 139717741903872, 139717741912063, ++STORE, 139717735903232, 139717739700223, ++SNULL, 139717735903232, 139717737562111, ++STORE, 139717737562112, 139717739700223, ++STORE, 139717735903232, 139717737562111, ++SNULL, 139717739659263, 139717739700223, ++STORE, 139717737562112, 139717739659263, ++STORE, 139717739659264, 139717739700223, ++SNULL, 139717739659264, 139717739683839, ++STORE, 139717739683840, 139717739700223, ++STORE, 139717739659264, 139717739683839, ++ERASE, 139717739659264, 139717739683839, ++STORE, 139717739659264, 139717739683839, ++ERASE, 139717739683840, 139717739700223, ++STORE, 139717739683840, 139717739700223, ++SNULL, 139717739675647, 139717739683839, ++STORE, 139717739659264, 139717739675647, ++STORE, 139717739675648, 139717739683839, ++SNULL, 94062802903039, 94062802907135, ++STORE, 94062802894848, 94062802903039, ++STORE, 94062802903040, 94062802907135, ++SNULL, 139717741944831, 139717741948927, ++STORE, 139717741940736, 139717741944831, ++STORE, 139717741944832, 139717741948927, ++ERASE, 139717741912064, 139717741940735, ++STORE, 94062814060544, 94062814195711, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723945754624, 140737488351231, ++SNULL, 140723945762815, 140737488351231, ++STORE, 140723945754624, 140723945762815, ++STORE, 140723945623552, 140723945762815, ++STORE, 94886119305216, 94886121639935, ++SNULL, 94886119518207, 94886121639935, ++STORE, 94886119305216, 94886119518207, ++STORE, 94886119518208, 94886121639935, ++ERASE, 94886119518208, 94886121639935, ++STORE, 94886121615360, 94886121627647, ++STORE, 94886121627648, 94886121639935, ++STORE, 140152532131840, 140152534384639, ++SNULL, 140152532275199, 140152534384639, ++STORE, 140152532131840, 140152532275199, ++STORE, 140152532275200, 140152534384639, ++ERASE, 140152532275200, 140152534384639, ++STORE, 140152534372352, 140152534380543, ++STORE, 140152534380544, 140152534384639, ++STORE, 140723946213376, 140723946217471, ++STORE, 140723946201088, 140723946213375, ++STORE, 140152534343680, 140152534372351, ++STORE, 140152534335488, 140152534343679, ++STORE, 140152530018304, 140152532131839, ++SNULL, 140152530018304, 140152530030591, ++STORE, 140152530030592, 140152532131839, ++STORE, 140152530018304, 140152530030591, ++SNULL, 140152532123647, 140152532131839, ++STORE, 140152530030592, 140152532123647, ++STORE, 140152532123648, 140152532131839, ++ERASE, 140152532123648, 140152532131839, ++STORE, 140152532123648, 140152532131839, ++STORE, 140152526221312, 140152530018303, ++SNULL, 140152526221312, 140152527880191, ++STORE, 140152527880192, 140152530018303, ++STORE, 140152526221312, 140152527880191, ++SNULL, 140152529977343, 140152530018303, ++STORE, 140152527880192, 140152529977343, ++STORE, 140152529977344, 140152530018303, ++SNULL, 140152529977344, 140152530001919, ++STORE, 140152530001920, 140152530018303, ++STORE, 140152529977344, 140152530001919, ++ERASE, 140152529977344, 140152530001919, ++STORE, 140152529977344, 140152530001919, ++ERASE, 140152530001920, 140152530018303, ++STORE, 140152530001920, 140152530018303, ++STORE, 140152534327296, 140152534343679, ++SNULL, 140152529993727, 140152530001919, ++STORE, 140152529977344, 140152529993727, ++STORE, 140152529993728, 140152530001919, ++SNULL, 140152532127743, 140152532131839, ++STORE, 140152532123648, 140152532127743, ++STORE, 140152532127744, 140152532131839, ++SNULL, 94886121619455, 94886121627647, ++STORE, 94886121615360, 94886121619455, ++STORE, 94886121619456, 94886121627647, ++SNULL, 140152534376447, 140152534380543, ++STORE, 140152534372352, 140152534376447, ++STORE, 140152534376448, 140152534380543, ++ERASE, 140152534343680, 140152534372351, ++STORE, 94886129770496, 94886129905663, ++STORE, 140152532643840, 140152534327295, ++STORE, 94886129770496, 94886130040831, ++STORE, 94886129770496, 94886130175999, ++STORE, 94886129770496, 94886130348031, ++SNULL, 94886130323455, 94886130348031, ++STORE, 94886129770496, 94886130323455, ++STORE, 94886130323456, 94886130348031, ++ERASE, 94886130323456, 94886130348031, ++STORE, 94886129770496, 94886130458623, ++STORE, 94886129770496, 94886130606079, ++SNULL, 94886130573311, 94886130606079, ++STORE, 94886129770496, 94886130573311, ++STORE, 94886130573312, 94886130606079, ++ERASE, 94886130573312, 94886130606079, ++STORE, 94886129770496, 94886130724863, ++STORE, 94886129770496, 94886130876415, ++STORE, 94886129770496, 94886131023871, ++STORE, 94886129770496, 94886131175423, ++STORE, 94886129770496, 94886131318783, ++STORE, 94886129770496, 94886131453951, ++SNULL, 94886131449855, 94886131453951, ++STORE, 94886129770496, 94886131449855, ++STORE, 94886131449856, 94886131453951, ++ERASE, 94886131449856, 94886131453951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735450779648, 140737488351231, ++SNULL, 140735450787839, 140737488351231, ++STORE, 140735450779648, 140735450787839, ++STORE, 140735450648576, 140735450787839, ++STORE, 93947794079744, 93947796414463, ++SNULL, 93947794292735, 93947796414463, ++STORE, 93947794079744, 93947794292735, ++STORE, 93947794292736, 93947796414463, ++ERASE, 93947794292736, 93947796414463, ++STORE, 93947796389888, 93947796402175, ++STORE, 93947796402176, 93947796414463, ++STORE, 139841993433088, 139841995685887, ++SNULL, 139841993576447, 139841995685887, ++STORE, 139841993433088, 139841993576447, ++STORE, 139841993576448, 139841995685887, ++ERASE, 139841993576448, 139841995685887, ++STORE, 139841995673600, 139841995681791, ++STORE, 139841995681792, 139841995685887, ++STORE, 140735451308032, 140735451312127, ++STORE, 140735451295744, 140735451308031, ++STORE, 139841995644928, 139841995673599, ++STORE, 139841995636736, 139841995644927, ++STORE, 139841991319552, 139841993433087, ++SNULL, 139841991319552, 139841991331839, ++STORE, 139841991331840, 139841993433087, ++STORE, 139841991319552, 139841991331839, ++SNULL, 139841993424895, 139841993433087, ++STORE, 139841991331840, 139841993424895, ++STORE, 139841993424896, 139841993433087, ++ERASE, 139841993424896, 139841993433087, ++STORE, 139841993424896, 139841993433087, ++STORE, 139841987522560, 139841991319551, ++SNULL, 139841987522560, 139841989181439, ++STORE, 139841989181440, 139841991319551, ++STORE, 139841987522560, 139841989181439, ++SNULL, 139841991278591, 139841991319551, ++STORE, 139841989181440, 139841991278591, ++STORE, 139841991278592, 139841991319551, ++SNULL, 139841991278592, 139841991303167, ++STORE, 139841991303168, 139841991319551, ++STORE, 139841991278592, 139841991303167, ++ERASE, 139841991278592, 139841991303167, ++STORE, 139841991278592, 139841991303167, ++ERASE, 139841991303168, 139841991319551, ++STORE, 139841991303168, 139841991319551, ++STORE, 139841995628544, 139841995644927, ++SNULL, 139841991294975, 139841991303167, ++STORE, 139841991278592, 139841991294975, ++STORE, 139841991294976, 139841991303167, ++SNULL, 139841993428991, 139841993433087, ++STORE, 139841993424896, 139841993428991, ++STORE, 139841993428992, 139841993433087, ++SNULL, 93947796393983, 93947796402175, ++STORE, 93947796389888, 93947796393983, ++STORE, 93947796393984, 93947796402175, ++SNULL, 139841995677695, 139841995681791, ++STORE, 139841995673600, 139841995677695, ++STORE, 139841995677696, 139841995681791, ++ERASE, 139841995644928, 139841995673599, ++STORE, 93947829739520, 93947829874687, ++STORE, 139841993945088, 139841995628543, ++STORE, 93947829739520, 93947830009855, ++STORE, 93947829739520, 93947830145023, ++STORE, 94659351814144, 94659352027135, ++STORE, 94659354124288, 94659354128383, ++STORE, 94659354128384, 94659354136575, ++STORE, 94659354136576, 94659354148863, ++STORE, 94659383476224, 94659385057279, ++STORE, 139959054557184, 139959056216063, ++STORE, 139959056216064, 139959058313215, ++STORE, 139959058313216, 139959058329599, ++STORE, 139959058329600, 139959058337791, ++STORE, 139959058337792, 139959058354175, ++STORE, 139959058354176, 139959058366463, ++STORE, 139959058366464, 139959060459519, ++STORE, 139959060459520, 139959060463615, ++STORE, 139959060463616, 139959060467711, ++STORE, 139959060467712, 139959060611071, ++STORE, 139959060979712, 139959062663167, ++STORE, 139959062663168, 139959062679551, ++STORE, 139959062708224, 139959062712319, ++STORE, 139959062712320, 139959062716415, ++STORE, 139959062716416, 139959062720511, ++STORE, 140735532539904, 140735532679167, ++STORE, 140735532830720, 140735532843007, ++STORE, 140735532843008, 140735532847103, ++STORE, 93894361829376, 93894362042367, ++STORE, 93894364139520, 93894364143615, ++STORE, 93894364143616, 93894364151807, ++STORE, 93894364151808, 93894364164095, ++STORE, 93894396944384, 93894397624319, ++STORE, 140075612573696, 140075614232575, ++STORE, 140075614232576, 140075616329727, ++STORE, 140075616329728, 140075616346111, ++STORE, 140075616346112, 140075616354303, ++STORE, 140075616354304, 140075616370687, ++STORE, 140075616370688, 140075616382975, ++STORE, 140075616382976, 140075618476031, ++STORE, 140075618476032, 140075618480127, ++STORE, 140075618480128, 140075618484223, ++STORE, 140075618484224, 140075618627583, ++STORE, 140075618996224, 140075620679679, ++STORE, 140075620679680, 140075620696063, ++STORE, 140075620724736, 140075620728831, ++STORE, 140075620728832, 140075620732927, ++STORE, 140075620732928, 140075620737023, ++STORE, 140720830312448, 140720830451711, ++STORE, 140720830631936, 140720830644223, ++STORE, 140720830644224, 140720830648319, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735116226560, 140737488351231, ++SNULL, 140735116234751, 140737488351231, ++STORE, 140735116226560, 140735116234751, ++STORE, 140735116095488, 140735116234751, ++STORE, 94873398054912, 94873400279039, ++SNULL, 94873398165503, 94873400279039, ++STORE, 94873398054912, 94873398165503, ++STORE, 94873398165504, 94873400279039, ++ERASE, 94873398165504, 94873400279039, ++STORE, 94873400258560, 94873400270847, ++STORE, 94873400270848, 94873400279039, ++STORE, 140303828606976, 140303830859775, ++SNULL, 140303828750335, 140303830859775, ++STORE, 140303828606976, 140303828750335, ++STORE, 140303828750336, 140303830859775, ++ERASE, 140303828750336, 140303830859775, ++STORE, 140303830847488, 140303830855679, ++STORE, 140303830855680, 140303830859775, ++STORE, 140735116251136, 140735116255231, ++STORE, 140735116238848, 140735116251135, ++STORE, 140303830818816, 140303830847487, ++STORE, 140303830810624, 140303830818815, ++STORE, 140303824809984, 140303828606975, ++SNULL, 140303824809984, 140303826468863, ++STORE, 140303826468864, 140303828606975, ++STORE, 140303824809984, 140303826468863, ++SNULL, 140303828566015, 140303828606975, ++STORE, 140303826468864, 140303828566015, ++STORE, 140303828566016, 140303828606975, ++SNULL, 140303828566016, 140303828590591, ++STORE, 140303828590592, 140303828606975, ++STORE, 140303828566016, 140303828590591, ++ERASE, 140303828566016, 140303828590591, ++STORE, 140303828566016, 140303828590591, ++ERASE, 140303828590592, 140303828606975, ++STORE, 140303828590592, 140303828606975, ++SNULL, 140303828582399, 140303828590591, ++STORE, 140303828566016, 140303828582399, ++STORE, 140303828582400, 140303828590591, ++SNULL, 94873400266751, 94873400270847, ++STORE, 94873400258560, 94873400266751, ++STORE, 94873400266752, 94873400270847, ++SNULL, 140303830851583, 140303830855679, ++STORE, 140303830847488, 140303830851583, ++STORE, 140303830851584, 140303830855679, ++ERASE, 140303830818816, 140303830847487, ++STORE, 94873413713920, 94873413849087, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140732349956096, 140737488351231, ++SNULL, 140732349964287, 140737488351231, ++STORE, 140732349956096, 140732349964287, ++STORE, 140732349825024, 140732349964287, ++STORE, 94009652736000, 94009655070719, ++SNULL, 94009652948991, 94009655070719, ++STORE, 94009652736000, 94009652948991, ++STORE, 94009652948992, 94009655070719, ++ERASE, 94009652948992, 94009655070719, ++STORE, 94009655046144, 94009655058431, ++STORE, 94009655058432, 94009655070719, ++STORE, 140295688531968, 140295690784767, ++SNULL, 140295688675327, 140295690784767, ++STORE, 140295688531968, 140295688675327, ++STORE, 140295688675328, 140295690784767, ++ERASE, 140295688675328, 140295690784767, ++STORE, 140295690772480, 140295690780671, ++STORE, 140295690780672, 140295690784767, ++STORE, 140732350005248, 140732350009343, ++STORE, 140732349992960, 140732350005247, ++STORE, 140295690743808, 140295690772479, ++STORE, 140295690735616, 140295690743807, ++STORE, 140295686418432, 140295688531967, ++SNULL, 140295686418432, 140295686430719, ++STORE, 140295686430720, 140295688531967, ++STORE, 140295686418432, 140295686430719, ++SNULL, 140295688523775, 140295688531967, ++STORE, 140295686430720, 140295688523775, ++STORE, 140295688523776, 140295688531967, ++ERASE, 140295688523776, 140295688531967, ++STORE, 140295688523776, 140295688531967, ++STORE, 140295682621440, 140295686418431, ++SNULL, 140295682621440, 140295684280319, ++STORE, 140295684280320, 140295686418431, ++STORE, 140295682621440, 140295684280319, ++SNULL, 140295686377471, 140295686418431, ++STORE, 140295684280320, 140295686377471, ++STORE, 140295686377472, 140295686418431, ++SNULL, 140295686377472, 140295686402047, ++STORE, 140295686402048, 140295686418431, ++STORE, 140295686377472, 140295686402047, ++ERASE, 140295686377472, 140295686402047, ++STORE, 140295686377472, 140295686402047, ++ERASE, 140295686402048, 140295686418431, ++STORE, 140295686402048, 140295686418431, ++STORE, 140295690727424, 140295690743807, ++SNULL, 140295686393855, 140295686402047, ++STORE, 140295686377472, 140295686393855, ++STORE, 140295686393856, 140295686402047, ++SNULL, 140295688527871, 140295688531967, ++STORE, 140295688523776, 140295688527871, ++STORE, 140295688527872, 140295688531967, ++SNULL, 94009655050239, 94009655058431, ++STORE, 94009655046144, 94009655050239, ++STORE, 94009655050240, 94009655058431, ++SNULL, 140295690776575, 140295690780671, ++STORE, 140295690772480, 140295690776575, ++STORE, 140295690776576, 140295690780671, ++ERASE, 140295690743808, 140295690772479, ++STORE, 94009672114176, 94009672249343, ++STORE, 140295689043968, 140295690727423, ++STORE, 94009672114176, 94009672384511, ++STORE, 94009672114176, 94009672519679, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722376515584, 140737488351231, ++SNULL, 140722376523775, 140737488351231, ++STORE, 140722376515584, 140722376523775, ++STORE, 140722376384512, 140722376523775, ++STORE, 94089815773184, 94089818107903, ++SNULL, 94089815986175, 94089818107903, ++STORE, 94089815773184, 94089815986175, ++STORE, 94089815986176, 94089818107903, ++ERASE, 94089815986176, 94089818107903, ++STORE, 94089818083328, 94089818095615, ++STORE, 94089818095616, 94089818107903, ++STORE, 140265595711488, 140265597964287, ++SNULL, 140265595854847, 140265597964287, ++STORE, 140265595711488, 140265595854847, ++STORE, 140265595854848, 140265597964287, ++ERASE, 140265595854848, 140265597964287, ++STORE, 140265597952000, 140265597960191, ++STORE, 140265597960192, 140265597964287, ++STORE, 140722378297344, 140722378301439, ++STORE, 140722378285056, 140722378297343, ++STORE, 140265597923328, 140265597951999, ++STORE, 140265597915136, 140265597923327, ++STORE, 140265593597952, 140265595711487, ++SNULL, 140265593597952, 140265593610239, ++STORE, 140265593610240, 140265595711487, ++STORE, 140265593597952, 140265593610239, ++SNULL, 140265595703295, 140265595711487, ++STORE, 140265593610240, 140265595703295, ++STORE, 140265595703296, 140265595711487, ++ERASE, 140265595703296, 140265595711487, ++STORE, 140265595703296, 140265595711487, ++STORE, 140265589800960, 140265593597951, ++SNULL, 140265589800960, 140265591459839, ++STORE, 140265591459840, 140265593597951, ++STORE, 140265589800960, 140265591459839, ++SNULL, 140265593556991, 140265593597951, ++STORE, 140265591459840, 140265593556991, ++STORE, 140265593556992, 140265593597951, ++SNULL, 140265593556992, 140265593581567, ++STORE, 140265593581568, 140265593597951, ++STORE, 140265593556992, 140265593581567, ++ERASE, 140265593556992, 140265593581567, ++STORE, 140265593556992, 140265593581567, ++ERASE, 140265593581568, 140265593597951, ++STORE, 140265593581568, 140265593597951, ++STORE, 140265597906944, 140265597923327, ++SNULL, 140265593573375, 140265593581567, ++STORE, 140265593556992, 140265593573375, ++STORE, 140265593573376, 140265593581567, ++SNULL, 140265595707391, 140265595711487, ++STORE, 140265595703296, 140265595707391, ++STORE, 140265595707392, 140265595711487, ++SNULL, 94089818087423, 94089818095615, ++STORE, 94089818083328, 94089818087423, ++STORE, 94089818087424, 94089818095615, ++SNULL, 140265597956095, 140265597960191, ++STORE, 140265597952000, 140265597956095, ++STORE, 140265597956096, 140265597960191, ++ERASE, 140265597923328, 140265597951999, ++STORE, 94089837146112, 94089837281279, ++STORE, 140265596223488, 140265597906943, ++STORE, 94089837146112, 94089837416447, ++STORE, 94089837146112, 94089837551615, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735265218560, 140737488351231, ++SNULL, 140735265226751, 140737488351231, ++STORE, 140735265218560, 140735265226751, ++STORE, 140735265087488, 140735265226751, ++STORE, 94250422370304, 94250424705023, ++SNULL, 94250422583295, 94250424705023, ++STORE, 94250422370304, 94250422583295, ++STORE, 94250422583296, 94250424705023, ++ERASE, 94250422583296, 94250424705023, ++STORE, 94250424680448, 94250424692735, ++STORE, 94250424692736, 94250424705023, ++STORE, 140344442474496, 140344444727295, ++SNULL, 140344442617855, 140344444727295, ++STORE, 140344442474496, 140344442617855, ++STORE, 140344442617856, 140344444727295, ++ERASE, 140344442617856, 140344444727295, ++STORE, 140344444715008, 140344444723199, ++STORE, 140344444723200, 140344444727295, ++STORE, 140735265341440, 140735265345535, ++STORE, 140735265329152, 140735265341439, ++STORE, 140344444686336, 140344444715007, ++STORE, 140344444678144, 140344444686335, ++STORE, 140344440360960, 140344442474495, ++SNULL, 140344440360960, 140344440373247, ++STORE, 140344440373248, 140344442474495, ++STORE, 140344440360960, 140344440373247, ++SNULL, 140344442466303, 140344442474495, ++STORE, 140344440373248, 140344442466303, ++STORE, 140344442466304, 140344442474495, ++ERASE, 140344442466304, 140344442474495, ++STORE, 140344442466304, 140344442474495, ++STORE, 140344436563968, 140344440360959, ++SNULL, 140344436563968, 140344438222847, ++STORE, 140344438222848, 140344440360959, ++STORE, 140344436563968, 140344438222847, ++SNULL, 140344440319999, 140344440360959, ++STORE, 140344438222848, 140344440319999, ++STORE, 140344440320000, 140344440360959, ++SNULL, 140344440320000, 140344440344575, ++STORE, 140344440344576, 140344440360959, ++STORE, 140344440320000, 140344440344575, ++ERASE, 140344440320000, 140344440344575, ++STORE, 140344440320000, 140344440344575, ++ERASE, 140344440344576, 140344440360959, ++STORE, 140344440344576, 140344440360959, ++STORE, 140344444669952, 140344444686335, ++SNULL, 140344440336383, 140344440344575, ++STORE, 140344440320000, 140344440336383, ++STORE, 140344440336384, 140344440344575, ++SNULL, 140344442470399, 140344442474495, ++STORE, 140344442466304, 140344442470399, ++STORE, 140344442470400, 140344442474495, ++SNULL, 94250424684543, 94250424692735, ++STORE, 94250424680448, 94250424684543, ++STORE, 94250424684544, 94250424692735, ++SNULL, 140344444719103, 140344444723199, ++STORE, 140344444715008, 140344444719103, ++STORE, 140344444719104, 140344444723199, ++ERASE, 140344444686336, 140344444715007, ++STORE, 94250445512704, 94250445647871, ++STORE, 140344442986496, 140344444669951, ++STORE, 94250445512704, 94250445783039, ++STORE, 94250445512704, 94250445918207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725762719744, 140737488351231, ++SNULL, 140725762727935, 140737488351231, ++STORE, 140725762719744, 140725762727935, ++STORE, 140725762588672, 140725762727935, ++STORE, 94819009097728, 94819011432447, ++SNULL, 94819009310719, 94819011432447, ++STORE, 94819009097728, 94819009310719, ++STORE, 94819009310720, 94819011432447, ++ERASE, 94819009310720, 94819011432447, ++STORE, 94819011407872, 94819011420159, ++STORE, 94819011420160, 94819011432447, ++STORE, 139987985596416, 139987987849215, ++SNULL, 139987985739775, 139987987849215, ++STORE, 139987985596416, 139987985739775, ++STORE, 139987985739776, 139987987849215, ++ERASE, 139987985739776, 139987987849215, ++STORE, 139987987836928, 139987987845119, ++STORE, 139987987845120, 139987987849215, ++STORE, 140725763072000, 140725763076095, ++STORE, 140725763059712, 140725763071999, ++STORE, 139987987808256, 139987987836927, ++STORE, 139987987800064, 139987987808255, ++STORE, 139987983482880, 139987985596415, ++SNULL, 139987983482880, 139987983495167, ++STORE, 139987983495168, 139987985596415, ++STORE, 139987983482880, 139987983495167, ++SNULL, 139987985588223, 139987985596415, ++STORE, 139987983495168, 139987985588223, ++STORE, 139987985588224, 139987985596415, ++ERASE, 139987985588224, 139987985596415, ++STORE, 139987985588224, 139987985596415, ++STORE, 139987979685888, 139987983482879, ++SNULL, 139987979685888, 139987981344767, ++STORE, 139987981344768, 139987983482879, ++STORE, 139987979685888, 139987981344767, ++SNULL, 139987983441919, 139987983482879, ++STORE, 139987981344768, 139987983441919, ++STORE, 139987983441920, 139987983482879, ++SNULL, 139987983441920, 139987983466495, ++STORE, 139987983466496, 139987983482879, ++STORE, 139987983441920, 139987983466495, ++ERASE, 139987983441920, 139987983466495, ++STORE, 139987983441920, 139987983466495, ++ERASE, 139987983466496, 139987983482879, ++STORE, 139987983466496, 139987983482879, ++STORE, 139987987791872, 139987987808255, ++SNULL, 139987983458303, 139987983466495, ++STORE, 139987983441920, 139987983458303, ++STORE, 139987983458304, 139987983466495, ++SNULL, 139987985592319, 139987985596415, ++STORE, 139987985588224, 139987985592319, ++STORE, 139987985592320, 139987985596415, ++SNULL, 94819011411967, 94819011420159, ++STORE, 94819011407872, 94819011411967, ++STORE, 94819011411968, 94819011420159, ++SNULL, 139987987841023, 139987987845119, ++STORE, 139987987836928, 139987987841023, ++STORE, 139987987841024, 139987987845119, ++ERASE, 139987987808256, 139987987836927, ++STORE, 94819028176896, 94819028312063, ++STORE, 139987986108416, 139987987791871, ++STORE, 94819028176896, 94819028447231, ++STORE, 94819028176896, 94819028582399, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722475413504, 140737488351231, ++SNULL, 140722475421695, 140737488351231, ++STORE, 140722475413504, 140722475421695, ++STORE, 140722475282432, 140722475421695, ++STORE, 94620599119872, 94620601343999, ++SNULL, 94620599230463, 94620601343999, ++STORE, 94620599119872, 94620599230463, ++STORE, 94620599230464, 94620601343999, ++ERASE, 94620599230464, 94620601343999, ++STORE, 94620601323520, 94620601335807, ++STORE, 94620601335808, 94620601343999, ++STORE, 139891763060736, 139891765313535, ++SNULL, 139891763204095, 139891765313535, ++STORE, 139891763060736, 139891763204095, ++STORE, 139891763204096, 139891765313535, ++ERASE, 139891763204096, 139891765313535, ++STORE, 139891765301248, 139891765309439, ++STORE, 139891765309440, 139891765313535, ++STORE, 140722475700224, 140722475704319, ++STORE, 140722475687936, 140722475700223, ++STORE, 139891765272576, 139891765301247, ++STORE, 139891765264384, 139891765272575, ++STORE, 139891759263744, 139891763060735, ++SNULL, 139891759263744, 139891760922623, ++STORE, 139891760922624, 139891763060735, ++STORE, 139891759263744, 139891760922623, ++SNULL, 139891763019775, 139891763060735, ++STORE, 139891760922624, 139891763019775, ++STORE, 139891763019776, 139891763060735, ++SNULL, 139891763019776, 139891763044351, ++STORE, 139891763044352, 139891763060735, ++STORE, 139891763019776, 139891763044351, ++ERASE, 139891763019776, 139891763044351, ++STORE, 139891763019776, 139891763044351, ++ERASE, 139891763044352, 139891763060735, ++STORE, 139891763044352, 139891763060735, ++SNULL, 139891763036159, 139891763044351, ++STORE, 139891763019776, 139891763036159, ++STORE, 139891763036160, 139891763044351, ++SNULL, 94620601331711, 94620601335807, ++STORE, 94620601323520, 94620601331711, ++STORE, 94620601331712, 94620601335807, ++SNULL, 139891765305343, 139891765309439, ++STORE, 139891765301248, 139891765305343, ++STORE, 139891765305344, 139891765309439, ++ERASE, 139891765272576, 139891765301247, ++STORE, 94620610027520, 94620610162687, ++STORE, 94031976210432, 94031976423423, ++STORE, 94031978520576, 94031978524671, ++STORE, 94031978524672, 94031978532863, ++STORE, 94031978532864, 94031978545151, ++STORE, 94031990398976, 94031992565759, ++STORE, 140336240640000, 140336242298879, ++STORE, 140336242298880, 140336244396031, ++STORE, 140336244396032, 140336244412415, ++STORE, 140336244412416, 140336244420607, ++STORE, 140336244420608, 140336244436991, ++STORE, 140336244436992, 140336244449279, ++STORE, 140336244449280, 140336246542335, ++STORE, 140336246542336, 140336246546431, ++STORE, 140336246546432, 140336246550527, ++STORE, 140336246550528, 140336246693887, ++STORE, 140336247062528, 140336248745983, ++STORE, 140336248745984, 140336248762367, ++STORE, 140336248791040, 140336248795135, ++STORE, 140336248795136, 140336248799231, ++STORE, 140336248799232, 140336248803327, ++STORE, 140728500064256, 140728500203519, ++STORE, 140728501501952, 140728501514239, ++STORE, 140728501514240, 140728501518335, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730503987200, 140737488351231, ++SNULL, 140730503995391, 140737488351231, ++STORE, 140730503987200, 140730503995391, ++STORE, 140730503856128, 140730503995391, ++STORE, 93866544205824, 93866546429951, ++SNULL, 93866544316415, 93866546429951, ++STORE, 93866544205824, 93866544316415, ++STORE, 93866544316416, 93866546429951, ++ERASE, 93866544316416, 93866546429951, ++STORE, 93866546409472, 93866546421759, ++STORE, 93866546421760, 93866546429951, ++STORE, 140216311959552, 140216314212351, ++SNULL, 140216312102911, 140216314212351, ++STORE, 140216311959552, 140216312102911, ++STORE, 140216312102912, 140216314212351, ++ERASE, 140216312102912, 140216314212351, ++STORE, 140216314200064, 140216314208255, ++STORE, 140216314208256, 140216314212351, ++STORE, 140730504626176, 140730504630271, ++STORE, 140730504613888, 140730504626175, ++STORE, 140216314171392, 140216314200063, ++STORE, 140216314163200, 140216314171391, ++STORE, 140216308162560, 140216311959551, ++SNULL, 140216308162560, 140216309821439, ++STORE, 140216309821440, 140216311959551, ++STORE, 140216308162560, 140216309821439, ++SNULL, 140216311918591, 140216311959551, ++STORE, 140216309821440, 140216311918591, ++STORE, 140216311918592, 140216311959551, ++SNULL, 140216311918592, 140216311943167, ++STORE, 140216311943168, 140216311959551, ++STORE, 140216311918592, 140216311943167, ++ERASE, 140216311918592, 140216311943167, ++STORE, 140216311918592, 140216311943167, ++ERASE, 140216311943168, 140216311959551, ++STORE, 140216311943168, 140216311959551, ++SNULL, 140216311934975, 140216311943167, ++STORE, 140216311918592, 140216311934975, ++STORE, 140216311934976, 140216311943167, ++SNULL, 93866546417663, 93866546421759, ++STORE, 93866546409472, 93866546417663, ++STORE, 93866546417664, 93866546421759, ++SNULL, 140216314204159, 140216314208255, ++STORE, 140216314200064, 140216314204159, ++STORE, 140216314204160, 140216314208255, ++ERASE, 140216314171392, 140216314200063, ++STORE, 93866550386688, 93866550521855, ++STORE, 94074292674560, 94074292887551, ++STORE, 94074294984704, 94074294988799, ++STORE, 94074294988800, 94074294996991, ++STORE, 94074294996992, 94074295009279, ++STORE, 94074300219392, 94074301378559, ++STORE, 139781563256832, 139781564915711, ++STORE, 139781564915712, 139781567012863, ++STORE, 139781567012864, 139781567029247, ++STORE, 139781567029248, 139781567037439, ++STORE, 139781567037440, 139781567053823, ++STORE, 139781567053824, 139781567066111, ++STORE, 139781567066112, 139781569159167, ++STORE, 139781569159168, 139781569163263, ++STORE, 139781569163264, 139781569167359, ++STORE, 139781569167360, 139781569310719, ++STORE, 139781569679360, 139781571362815, ++STORE, 139781571362816, 139781571379199, ++STORE, 139781571407872, 139781571411967, ++STORE, 139781571411968, 139781571416063, ++STORE, 139781571416064, 139781571420159, ++STORE, 140723688488960, 140723688628223, ++STORE, 140723689005056, 140723689017343, ++STORE, 140723689017344, 140723689021439, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735189745664, 140737488351231, ++SNULL, 140735189753855, 140737488351231, ++STORE, 140735189745664, 140735189753855, ++STORE, 140735189614592, 140735189753855, ++STORE, 94172072177664, 94172074512383, ++SNULL, 94172072390655, 94172074512383, ++STORE, 94172072177664, 94172072390655, ++STORE, 94172072390656, 94172074512383, ++ERASE, 94172072390656, 94172074512383, ++STORE, 94172074487808, 94172074500095, ++STORE, 94172074500096, 94172074512383, ++STORE, 140687827263488, 140687829516287, ++SNULL, 140687827406847, 140687829516287, ++STORE, 140687827263488, 140687827406847, ++STORE, 140687827406848, 140687829516287, ++ERASE, 140687827406848, 140687829516287, ++STORE, 140687829504000, 140687829512191, ++STORE, 140687829512192, 140687829516287, ++STORE, 140735189766144, 140735189770239, ++STORE, 140735189753856, 140735189766143, ++STORE, 140687829475328, 140687829503999, ++STORE, 140687829467136, 140687829475327, ++STORE, 140687825149952, 140687827263487, ++SNULL, 140687825149952, 140687825162239, ++STORE, 140687825162240, 140687827263487, ++STORE, 140687825149952, 140687825162239, ++SNULL, 140687827255295, 140687827263487, ++STORE, 140687825162240, 140687827255295, ++STORE, 140687827255296, 140687827263487, ++ERASE, 140687827255296, 140687827263487, ++STORE, 140687827255296, 140687827263487, ++STORE, 140687821352960, 140687825149951, ++SNULL, 140687821352960, 140687823011839, ++STORE, 140687823011840, 140687825149951, ++STORE, 140687821352960, 140687823011839, ++SNULL, 140687825108991, 140687825149951, ++STORE, 140687823011840, 140687825108991, ++STORE, 140687825108992, 140687825149951, ++SNULL, 140687825108992, 140687825133567, ++STORE, 140687825133568, 140687825149951, ++STORE, 140687825108992, 140687825133567, ++ERASE, 140687825108992, 140687825133567, ++STORE, 140687825108992, 140687825133567, ++ERASE, 140687825133568, 140687825149951, ++STORE, 140687825133568, 140687825149951, ++STORE, 140687829458944, 140687829475327, ++SNULL, 140687825125375, 140687825133567, ++STORE, 140687825108992, 140687825125375, ++STORE, 140687825125376, 140687825133567, ++SNULL, 140687827259391, 140687827263487, ++STORE, 140687827255296, 140687827259391, ++STORE, 140687827259392, 140687827263487, ++SNULL, 94172074491903, 94172074500095, ++STORE, 94172074487808, 94172074491903, ++STORE, 94172074491904, 94172074500095, ++SNULL, 140687829508095, 140687829512191, ++STORE, 140687829504000, 140687829508095, ++STORE, 140687829508096, 140687829512191, ++ERASE, 140687829475328, 140687829503999, ++STORE, 94172092432384, 94172092567551, ++STORE, 140687827775488, 140687829458943, ++STORE, 94172092432384, 94172092702719, ++STORE, 94172092432384, 94172092837887, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737229504512, 140737488351231, ++SNULL, 140737229512703, 140737488351231, ++STORE, 140737229504512, 140737229512703, ++STORE, 140737229373440, 140737229512703, ++STORE, 94155246866432, 94155249090559, ++SNULL, 94155246977023, 94155249090559, ++STORE, 94155246866432, 94155246977023, ++STORE, 94155246977024, 94155249090559, ++ERASE, 94155246977024, 94155249090559, ++STORE, 94155249070080, 94155249082367, ++STORE, 94155249082368, 94155249090559, ++STORE, 140640993693696, 140640995946495, ++SNULL, 140640993837055, 140640995946495, ++STORE, 140640993693696, 140640993837055, ++STORE, 140640993837056, 140640995946495, ++ERASE, 140640993837056, 140640995946495, ++STORE, 140640995934208, 140640995942399, ++STORE, 140640995942400, 140640995946495, ++STORE, 140737230004224, 140737230008319, ++STORE, 140737229991936, 140737230004223, ++STORE, 140640995905536, 140640995934207, ++STORE, 140640995897344, 140640995905535, ++STORE, 140640989896704, 140640993693695, ++SNULL, 140640989896704, 140640991555583, ++STORE, 140640991555584, 140640993693695, ++STORE, 140640989896704, 140640991555583, ++SNULL, 140640993652735, 140640993693695, ++STORE, 140640991555584, 140640993652735, ++STORE, 140640993652736, 140640993693695, ++SNULL, 140640993652736, 140640993677311, ++STORE, 140640993677312, 140640993693695, ++STORE, 140640993652736, 140640993677311, ++ERASE, 140640993652736, 140640993677311, ++STORE, 140640993652736, 140640993677311, ++ERASE, 140640993677312, 140640993693695, ++STORE, 140640993677312, 140640993693695, ++SNULL, 140640993669119, 140640993677311, ++STORE, 140640993652736, 140640993669119, ++STORE, 140640993669120, 140640993677311, ++SNULL, 94155249078271, 94155249082367, ++STORE, 94155249070080, 94155249078271, ++STORE, 94155249078272, 94155249082367, ++SNULL, 140640995938303, 140640995942399, ++STORE, 140640995934208, 140640995938303, ++STORE, 140640995938304, 140640995942399, ++ERASE, 140640995905536, 140640995934207, ++STORE, 94155281035264, 94155281170431, ++STORE, 94088066453504, 94088066564095, ++STORE, 94088068657152, 94088068665343, ++STORE, 94088068665344, 94088068669439, ++STORE, 94088068669440, 94088068677631, ++STORE, 94088090214400, 94088090349567, ++STORE, 140503024627712, 140503026286591, ++STORE, 140503026286592, 140503028383743, ++STORE, 140503028383744, 140503028400127, ++STORE, 140503028400128, 140503028408319, ++STORE, 140503028408320, 140503028424703, ++STORE, 140503028424704, 140503028568063, ++STORE, 140503030628352, 140503030636543, ++STORE, 140503030665216, 140503030669311, ++STORE, 140503030669312, 140503030673407, ++STORE, 140503030673408, 140503030677503, ++STORE, 140730894725120, 140730894864383, ++STORE, 140730894880768, 140730894893055, ++STORE, 140730894893056, 140730894897151, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730434342912, 140737488351231, ++SNULL, 140730434351103, 140737488351231, ++STORE, 140730434342912, 140730434351103, ++STORE, 140730434211840, 140730434351103, ++STORE, 4194304, 5128191, ++STORE, 7221248, 7241727, ++STORE, 7241728, 7249919, ++STORE, 140109041938432, 140109044191231, ++SNULL, 140109042081791, 140109044191231, ++STORE, 140109041938432, 140109042081791, ++STORE, 140109042081792, 140109044191231, ++ERASE, 140109042081792, 140109044191231, ++STORE, 140109044178944, 140109044187135, ++STORE, 140109044187136, 140109044191231, ++STORE, 140730434850816, 140730434854911, ++STORE, 140730434838528, 140730434850815, ++STORE, 140109044150272, 140109044178943, ++STORE, 140109044142080, 140109044150271, ++STORE, 140109038776320, 140109041938431, ++SNULL, 140109038776320, 140109039837183, ++STORE, 140109039837184, 140109041938431, ++STORE, 140109038776320, 140109039837183, ++SNULL, 140109041930239, 140109041938431, ++STORE, 140109039837184, 140109041930239, ++STORE, 140109041930240, 140109041938431, ++ERASE, 140109041930240, 140109041938431, ++STORE, 140109041930240, 140109041938431, ++STORE, 140109034979328, 140109038776319, ++SNULL, 140109034979328, 140109036638207, ++STORE, 140109036638208, 140109038776319, ++STORE, 140109034979328, 140109036638207, ++SNULL, 140109038735359, 140109038776319, ++STORE, 140109036638208, 140109038735359, ++STORE, 140109038735360, 140109038776319, ++SNULL, 140109038735360, 140109038759935, ++STORE, 140109038759936, 140109038776319, ++STORE, 140109038735360, 140109038759935, ++ERASE, 140109038735360, 140109038759935, ++STORE, 140109038735360, 140109038759935, ++ERASE, 140109038759936, 140109038776319, ++STORE, 140109038759936, 140109038776319, ++STORE, 140109044129792, 140109044150271, ++SNULL, 140109038751743, 140109038759935, ++STORE, 140109038735360, 140109038751743, ++STORE, 140109038751744, 140109038759935, ++SNULL, 140109041934335, 140109041938431, ++STORE, 140109041930240, 140109041934335, ++STORE, 140109041934336, 140109041938431, ++SNULL, 7233535, 7241727, ++STORE, 7221248, 7233535, ++STORE, 7233536, 7241727, ++SNULL, 140109044183039, 140109044187135, ++STORE, 140109044178944, 140109044183039, ++STORE, 140109044183040, 140109044187135, ++ERASE, 140109044150272, 140109044178943, ++STORE, 20000768, 20135935, ++STORE, 20000768, 20283391, ++STORE, 140109042446336, 140109044129791, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140730853408768, 140737488351231, ++SNULL, 140730853416959, 140737488351231, ++STORE, 140730853408768, 140730853416959, ++STORE, 140730853277696, 140730853416959, ++STORE, 94865902977024, 94865905311743, ++SNULL, 94865903190015, 94865905311743, ++STORE, 94865902977024, 94865903190015, ++STORE, 94865903190016, 94865905311743, ++ERASE, 94865903190016, 94865905311743, ++STORE, 94865905287168, 94865905299455, ++STORE, 94865905299456, 94865905311743, ++STORE, 139768865738752, 139768867991551, ++SNULL, 139768865882111, 139768867991551, ++STORE, 139768865738752, 139768865882111, ++STORE, 139768865882112, 139768867991551, ++ERASE, 139768865882112, 139768867991551, ++STORE, 139768867979264, 139768867987455, ++STORE, 139768867987456, 139768867991551, ++STORE, 140730853957632, 140730853961727, ++STORE, 140730853945344, 140730853957631, ++STORE, 139768867950592, 139768867979263, ++STORE, 139768867942400, 139768867950591, ++STORE, 139768863625216, 139768865738751, ++SNULL, 139768863625216, 139768863637503, ++STORE, 139768863637504, 139768865738751, ++STORE, 139768863625216, 139768863637503, ++SNULL, 139768865730559, 139768865738751, ++STORE, 139768863637504, 139768865730559, ++STORE, 139768865730560, 139768865738751, ++ERASE, 139768865730560, 139768865738751, ++STORE, 139768865730560, 139768865738751, ++STORE, 139768859828224, 139768863625215, ++SNULL, 139768859828224, 139768861487103, ++STORE, 139768861487104, 139768863625215, ++STORE, 139768859828224, 139768861487103, ++SNULL, 139768863584255, 139768863625215, ++STORE, 139768861487104, 139768863584255, ++STORE, 139768863584256, 139768863625215, ++SNULL, 139768863584256, 139768863608831, ++STORE, 139768863608832, 139768863625215, ++STORE, 139768863584256, 139768863608831, ++ERASE, 139768863584256, 139768863608831, ++STORE, 139768863584256, 139768863608831, ++ERASE, 139768863608832, 139768863625215, ++STORE, 139768863608832, 139768863625215, ++STORE, 139768867934208, 139768867950591, ++SNULL, 139768863600639, 139768863608831, ++STORE, 139768863584256, 139768863600639, ++STORE, 139768863600640, 139768863608831, ++SNULL, 139768865734655, 139768865738751, ++STORE, 139768865730560, 139768865734655, ++STORE, 139768865734656, 139768865738751, ++SNULL, 94865905291263, 94865905299455, ++STORE, 94865905287168, 94865905291263, ++STORE, 94865905291264, 94865905299455, ++SNULL, 139768867983359, 139768867987455, ++STORE, 139768867979264, 139768867983359, ++STORE, 139768867983360, 139768867987455, ++ERASE, 139768867950592, 139768867979263, ++STORE, 94865923670016, 94865923805183, ++STORE, 139768866250752, 139768867934207, ++STORE, 94865923670016, 94865923940351, ++STORE, 94865923670016, 94865924075519, ++STORE, 94865923670016, 94865924222975, ++SNULL, 94865924210687, 94865924222975, ++STORE, 94865923670016, 94865924210687, ++STORE, 94865924210688, 94865924222975, ++ERASE, 94865924210688, 94865924222975, ++STORE, 94865923670016, 94865924349951, ++STORE, 94865923670016, 94865924493311, ++STORE, 94865923670016, 94865924640767, ++SNULL, 94865924603903, 94865924640767, ++STORE, 94865923670016, 94865924603903, ++STORE, 94865924603904, 94865924640767, ++ERASE, 94865924603904, 94865924640767, ++STORE, 94865923670016, 94865924747263, ++STORE, 94865923670016, 94865924898815, ++SNULL, 94865924874239, 94865924898815, ++STORE, 94865923670016, 94865924874239, ++STORE, 94865924874240, 94865924898815, ++ERASE, 94865924874240, 94865924898815, ++STORE, 94865923670016, 94865925025791, ++SNULL, 94865925013503, 94865925025791, ++STORE, 94865923670016, 94865925013503, ++STORE, 94865925013504, 94865925025791, ++ERASE, 94865925013504, 94865925025791, ++SNULL, 94865924988927, 94865925013503, ++STORE, 94865923670016, 94865924988927, ++STORE, 94865924988928, 94865925013503, ++ERASE, 94865924988928, 94865925013503, ++STORE, 94865923670016, 94865925152767, ++SNULL, 94865925136383, 94865925152767, ++STORE, 94865923670016, 94865925136383, ++STORE, 94865925136384, 94865925152767, ++ERASE, 94865925136384, 94865925152767, ++STORE, 94865923670016, 94865925292031, ++SNULL, 94865925279743, 94865925292031, ++STORE, 94865923670016, 94865925279743, ++STORE, 94865925279744, 94865925292031, ++ERASE, 94865925279744, 94865925292031, ++SNULL, 94865925255167, 94865925279743, ++STORE, 94865923670016, 94865925255167, ++STORE, 94865925255168, 94865925279743, ++ERASE, 94865925255168, 94865925279743, ++STORE, 94865923670016, 94865925406719, ++SNULL, 94865925394431, 94865925406719, ++STORE, 94865923670016, 94865925394431, ++STORE, 94865925394432, 94865925406719, ++ERASE, 94865925394432, 94865925406719, ++STORE, 94865923670016, 94865925545983, ++SNULL, 94865925533695, 94865925545983, ++STORE, 94865923670016, 94865925533695, ++STORE, 94865925533696, 94865925545983, ++ERASE, 94865925533696, 94865925545983, ++SNULL, 94865925492735, 94865925533695, ++STORE, 94865923670016, 94865925492735, ++STORE, 94865925492736, 94865925533695, ++ERASE, 94865925492736, 94865925533695, ++STORE, 94865923670016, 94865925627903, ++SNULL, 94865925599231, 94865925627903, ++STORE, 94865923670016, 94865925599231, ++STORE, 94865925599232, 94865925627903, ++ERASE, 94865925599232, 94865925627903, ++STORE, 94865923670016, 94865925738495, ++SNULL, 94865925726207, 94865925738495, ++STORE, 94865923670016, 94865925726207, ++STORE, 94865925726208, 94865925738495, ++ERASE, 94865925726208, 94865925738495, ++STORE, 94865923670016, 94865925877759, ++SNULL, 94865925865471, 94865925877759, ++STORE, 94865923670016, 94865925865471, ++STORE, 94865925865472, 94865925877759, ++ERASE, 94865925865472, 94865925877759, ++STORE, 94865923670016, 94865926021119, ++SNULL, 94865926008831, 94865926021119, ++STORE, 94865923670016, 94865926008831, ++STORE, 94865926008832, 94865926021119, ++ERASE, 94865926008832, 94865926021119, ++SNULL, 94865925971967, 94865926008831, ++STORE, 94865923670016, 94865925971967, ++STORE, 94865925971968, 94865926008831, ++ERASE, 94865925971968, 94865926008831, ++STORE, 94865923670016, 94865926115327, ++STORE, 94865923670016, 94865926254591, ++SNULL, 94865926246399, 94865926254591, ++STORE, 94865923670016, 94865926246399, ++STORE, 94865926246400, 94865926254591, ++ERASE, 94865926246400, 94865926254591, ++STORE, 94865923670016, 94865926385663, ++STORE, 94865923670016, 94865926537215, ++STORE, 94865923670016, 94865926672383, ++STORE, 94865923670016, 94865926815743, ++STORE, 94865923670016, 94865926955007, ++STORE, 94865923670016, 94865927094271, ++STORE, 94865923670016, 94865927233535, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140731148435456, 140737488351231, ++SNULL, 140731148443647, 140737488351231, ++STORE, 140731148435456, 140731148443647, ++STORE, 140731148304384, 140731148443647, ++STORE, 94090775400448, 94090777735167, ++SNULL, 94090775613439, 94090777735167, ++STORE, 94090775400448, 94090775613439, ++STORE, 94090775613440, 94090777735167, ++ERASE, 94090775613440, 94090777735167, ++STORE, 94090777710592, 94090777722879, ++STORE, 94090777722880, 94090777735167, ++STORE, 140301090283520, 140301092536319, ++SNULL, 140301090426879, 140301092536319, ++STORE, 140301090283520, 140301090426879, ++STORE, 140301090426880, 140301092536319, ++ERASE, 140301090426880, 140301092536319, ++STORE, 140301092524032, 140301092532223, ++STORE, 140301092532224, 140301092536319, ++STORE, 140731148570624, 140731148574719, ++STORE, 140731148558336, 140731148570623, ++STORE, 140301092495360, 140301092524031, ++STORE, 140301092487168, 140301092495359, ++STORE, 140301088169984, 140301090283519, ++SNULL, 140301088169984, 140301088182271, ++STORE, 140301088182272, 140301090283519, ++STORE, 140301088169984, 140301088182271, ++SNULL, 140301090275327, 140301090283519, ++STORE, 140301088182272, 140301090275327, ++STORE, 140301090275328, 140301090283519, ++ERASE, 140301090275328, 140301090283519, ++STORE, 140301090275328, 140301090283519, ++STORE, 140301084372992, 140301088169983, ++SNULL, 140301084372992, 140301086031871, ++STORE, 140301086031872, 140301088169983, ++STORE, 140301084372992, 140301086031871, ++SNULL, 140301088129023, 140301088169983, ++STORE, 140301086031872, 140301088129023, ++STORE, 140301088129024, 140301088169983, ++SNULL, 140301088129024, 140301088153599, ++STORE, 140301088153600, 140301088169983, ++STORE, 140301088129024, 140301088153599, ++ERASE, 140301088129024, 140301088153599, ++STORE, 140301088129024, 140301088153599, ++ERASE, 140301088153600, 140301088169983, ++STORE, 140301088153600, 140301088169983, ++STORE, 140301092478976, 140301092495359, ++SNULL, 140301088145407, 140301088153599, ++STORE, 140301088129024, 140301088145407, ++STORE, 140301088145408, 140301088153599, ++SNULL, 140301090279423, 140301090283519, ++STORE, 140301090275328, 140301090279423, ++STORE, 140301090279424, 140301090283519, ++SNULL, 94090777714687, 94090777722879, ++STORE, 94090777710592, 94090777714687, ++STORE, 94090777714688, 94090777722879, ++SNULL, 140301092528127, 140301092532223, ++STORE, 140301092524032, 140301092528127, ++STORE, 140301092528128, 140301092532223, ++ERASE, 140301092495360, 140301092524031, ++STORE, 94090794590208, 94090794725375, ++STORE, 140301090795520, 140301092478975, ++STORE, 94090794590208, 94090794860543, ++STORE, 94090794590208, 94090794995711, ++STORE, 94090794590208, 94090795163647, ++SNULL, 94090795139071, 94090795163647, ++STORE, 94090794590208, 94090795139071, ++STORE, 94090795139072, 94090795163647, ++ERASE, 94090795139072, 94090795163647, ++STORE, 94090794590208, 94090795278335, ++STORE, 94090794590208, 94090795425791, ++SNULL, 94090795388927, 94090795425791, ++STORE, 94090794590208, 94090795388927, ++STORE, 94090795388928, 94090795425791, ++ERASE, 94090795388928, 94090795425791, ++STORE, 94090794590208, 94090795528191, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733084430336, 140737488351231, ++SNULL, 140733084438527, 140737488351231, ++STORE, 140733084430336, 140733084438527, ++STORE, 140733084299264, 140733084438527, ++STORE, 94116169183232, 94116171517951, ++SNULL, 94116169396223, 94116171517951, ++STORE, 94116169183232, 94116169396223, ++STORE, 94116169396224, 94116171517951, ++ERASE, 94116169396224, 94116171517951, ++STORE, 94116171493376, 94116171505663, ++STORE, 94116171505664, 94116171517951, ++STORE, 139772214128640, 139772216381439, ++SNULL, 139772214271999, 139772216381439, ++STORE, 139772214128640, 139772214271999, ++STORE, 139772214272000, 139772216381439, ++ERASE, 139772214272000, 139772216381439, ++STORE, 139772216369152, 139772216377343, ++STORE, 139772216377344, 139772216381439, ++STORE, 140733085270016, 140733085274111, ++STORE, 140733085257728, 140733085270015, ++STORE, 139772216340480, 139772216369151, ++STORE, 139772216332288, 139772216340479, ++STORE, 139772212015104, 139772214128639, ++SNULL, 139772212015104, 139772212027391, ++STORE, 139772212027392, 139772214128639, ++STORE, 139772212015104, 139772212027391, ++SNULL, 139772214120447, 139772214128639, ++STORE, 139772212027392, 139772214120447, ++STORE, 139772214120448, 139772214128639, ++ERASE, 139772214120448, 139772214128639, ++STORE, 139772214120448, 139772214128639, ++STORE, 139772208218112, 139772212015103, ++SNULL, 139772208218112, 139772209876991, ++STORE, 139772209876992, 139772212015103, ++STORE, 139772208218112, 139772209876991, ++SNULL, 139772211974143, 139772212015103, ++STORE, 139772209876992, 139772211974143, ++STORE, 139772211974144, 139772212015103, ++SNULL, 139772211974144, 139772211998719, ++STORE, 139772211998720, 139772212015103, ++STORE, 139772211974144, 139772211998719, ++ERASE, 139772211974144, 139772211998719, ++STORE, 139772211974144, 139772211998719, ++ERASE, 139772211998720, 139772212015103, ++STORE, 139772211998720, 139772212015103, ++STORE, 139772216324096, 139772216340479, ++SNULL, 139772211990527, 139772211998719, ++STORE, 139772211974144, 139772211990527, ++STORE, 139772211990528, 139772211998719, ++SNULL, 139772214124543, 139772214128639, ++STORE, 139772214120448, 139772214124543, ++STORE, 139772214124544, 139772214128639, ++SNULL, 94116171497471, 94116171505663, ++STORE, 94116171493376, 94116171497471, ++STORE, 94116171497472, 94116171505663, ++SNULL, 139772216373247, 139772216377343, ++STORE, 139772216369152, 139772216373247, ++STORE, 139772216373248, 139772216377343, ++ERASE, 139772216340480, 139772216369151, ++STORE, 94116199383040, 94116199518207, ++STORE, 139772214640640, 139772216324095, ++STORE, 94116199383040, 94116199653375, ++STORE, 94116199383040, 94116199788543, ++STORE, 140737488347136, 140737488351231, ++STORE, 140726067826688, 140737488351231, ++SNULL, 140726067830783, 140737488351231, ++STORE, 140726067826688, 140726067830783, ++STORE, 140726067695616, 140726067830783, ++STORE, 94535150673920, 94535152898047, ++SNULL, 94535150784511, 94535152898047, ++STORE, 94535150673920, 94535150784511, ++STORE, 94535150784512, 94535152898047, ++ERASE, 94535150784512, 94535152898047, ++STORE, 94535152877568, 94535152889855, ++STORE, 94535152889856, 94535152898047, ++STORE, 140381257314304, 140381259567103, ++SNULL, 140381257457663, 140381259567103, ++STORE, 140381257314304, 140381257457663, ++STORE, 140381257457664, 140381259567103, ++ERASE, 140381257457664, 140381259567103, ++STORE, 140381259554816, 140381259563007, ++STORE, 140381259563008, 140381259567103, ++STORE, 140726068060160, 140726068064255, ++STORE, 140726068047872, 140726068060159, ++STORE, 140381259526144, 140381259554815, ++STORE, 140381259517952, 140381259526143, ++STORE, 140381253517312, 140381257314303, ++SNULL, 140381253517312, 140381255176191, ++STORE, 140381255176192, 140381257314303, ++STORE, 140381253517312, 140381255176191, ++SNULL, 140381257273343, 140381257314303, ++STORE, 140381255176192, 140381257273343, ++STORE, 140381257273344, 140381257314303, ++SNULL, 140381257273344, 140381257297919, ++STORE, 140381257297920, 140381257314303, ++STORE, 140381257273344, 140381257297919, ++ERASE, 140381257273344, 140381257297919, ++STORE, 140381257273344, 140381257297919, ++ERASE, 140381257297920, 140381257314303, ++STORE, 140381257297920, 140381257314303, ++SNULL, 140381257289727, 140381257297919, ++STORE, 140381257273344, 140381257289727, ++STORE, 140381257289728, 140381257297919, ++SNULL, 94535152885759, 94535152889855, ++STORE, 94535152877568, 94535152885759, ++STORE, 94535152885760, 94535152889855, ++SNULL, 140381259558911, 140381259563007, ++STORE, 140381259554816, 140381259558911, ++STORE, 140381259558912, 140381259563007, ++ERASE, 140381259526144, 140381259554815, ++STORE, 94535186296832, 94535186431999, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140729189425152, 140737488351231, ++SNULL, 140729189433343, 140737488351231, ++STORE, 140729189425152, 140729189433343, ++STORE, 140729189294080, 140729189433343, ++STORE, 94428200128512, 94428202352639, ++SNULL, 94428200239103, 94428202352639, ++STORE, 94428200128512, 94428200239103, ++STORE, 94428200239104, 94428202352639, ++ERASE, 94428200239104, 94428202352639, ++STORE, 94428202332160, 94428202344447, ++STORE, 94428202344448, 94428202352639, ++STORE, 139707216986112, 139707219238911, ++SNULL, 139707217129471, 139707219238911, ++STORE, 139707216986112, 139707217129471, ++STORE, 139707217129472, 139707219238911, ++ERASE, 139707217129472, 139707219238911, ++STORE, 139707219226624, 139707219234815, ++STORE, 139707219234816, 139707219238911, ++STORE, 140729189785600, 140729189789695, ++STORE, 140729189773312, 140729189785599, ++STORE, 139707219197952, 139707219226623, ++STORE, 139707219189760, 139707219197951, ++STORE, 139707213189120, 139707216986111, ++SNULL, 139707213189120, 139707214847999, ++STORE, 139707214848000, 139707216986111, ++STORE, 139707213189120, 139707214847999, ++SNULL, 139707216945151, 139707216986111, ++STORE, 139707214848000, 139707216945151, ++STORE, 139707216945152, 139707216986111, ++SNULL, 139707216945152, 139707216969727, ++STORE, 139707216969728, 139707216986111, ++STORE, 139707216945152, 139707216969727, ++ERASE, 139707216945152, 139707216969727, ++STORE, 139707216945152, 139707216969727, ++ERASE, 139707216969728, 139707216986111, ++STORE, 139707216969728, 139707216986111, ++SNULL, 139707216961535, 139707216969727, ++STORE, 139707216945152, 139707216961535, ++STORE, 139707216961536, 139707216969727, ++SNULL, 94428202340351, 94428202344447, ++STORE, 94428202332160, 94428202340351, ++STORE, 94428202340352, 94428202344447, ++SNULL, 139707219230719, 139707219234815, ++STORE, 139707219226624, 139707219230719, ++STORE, 139707219230720, 139707219234815, ++ERASE, 139707219197952, 139707219226623, ++STORE, 94428208599040, 94428208734207, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722000953344, 140737488351231, ++SNULL, 140722000961535, 140737488351231, ++STORE, 140722000953344, 140722000961535, ++STORE, 140722000822272, 140722000961535, ++STORE, 94636494757888, 94636496982015, ++SNULL, 94636494868479, 94636496982015, ++STORE, 94636494757888, 94636494868479, ++STORE, 94636494868480, 94636496982015, ++ERASE, 94636494868480, 94636496982015, ++STORE, 94636496961536, 94636496973823, ++STORE, 94636496973824, 94636496982015, ++STORE, 140142275100672, 140142277353471, ++SNULL, 140142275244031, 140142277353471, ++STORE, 140142275100672, 140142275244031, ++STORE, 140142275244032, 140142277353471, ++ERASE, 140142275244032, 140142277353471, ++STORE, 140142277341184, 140142277349375, ++STORE, 140142277349376, 140142277353471, ++STORE, 140722002747392, 140722002751487, ++STORE, 140722002735104, 140722002747391, ++STORE, 140142277312512, 140142277341183, ++STORE, 140142277304320, 140142277312511, ++STORE, 140142271303680, 140142275100671, ++SNULL, 140142271303680, 140142272962559, ++STORE, 140142272962560, 140142275100671, ++STORE, 140142271303680, 140142272962559, ++SNULL, 140142275059711, 140142275100671, ++STORE, 140142272962560, 140142275059711, ++STORE, 140142275059712, 140142275100671, ++SNULL, 140142275059712, 140142275084287, ++STORE, 140142275084288, 140142275100671, ++STORE, 140142275059712, 140142275084287, ++ERASE, 140142275059712, 140142275084287, ++STORE, 140142275059712, 140142275084287, ++ERASE, 140142275084288, 140142275100671, ++STORE, 140142275084288, 140142275100671, ++SNULL, 140142275076095, 140142275084287, ++STORE, 140142275059712, 140142275076095, ++STORE, 140142275076096, 140142275084287, ++SNULL, 94636496969727, 94636496973823, ++STORE, 94636496961536, 94636496969727, ++STORE, 94636496969728, 94636496973823, ++SNULL, 140142277345279, 140142277349375, ++STORE, 140142277341184, 140142277345279, ++STORE, 140142277345280, 140142277349375, ++ERASE, 140142277312512, 140142277341183, ++STORE, 94636516286464, 94636516421631, ++STORE, 94071103692800, 94071103905791, ++STORE, 94071106002944, 94071106007039, ++STORE, 94071106007040, 94071106015231, ++STORE, 94071106015232, 94071106027519, ++STORE, 94071138521088, 94071140368383, ++STORE, 140145668190208, 140145669849087, ++STORE, 140145669849088, 140145671946239, ++STORE, 140145671946240, 140145671962623, ++STORE, 140145671962624, 140145671970815, ++STORE, 140145671970816, 140145671987199, ++STORE, 140145671987200, 140145671999487, ++STORE, 140145671999488, 140145674092543, ++STORE, 140145674092544, 140145674096639, ++STORE, 140145674096640, 140145674100735, ++STORE, 140145674100736, 140145674244095, ++STORE, 140145674612736, 140145676296191, ++STORE, 140145676296192, 140145676312575, ++STORE, 140145676341248, 140145676345343, ++STORE, 140145676345344, 140145676349439, ++STORE, 140145676349440, 140145676353535, ++STORE, 140734927740928, 140734927880191, ++STORE, 140734928842752, 140734928855039, ++STORE, 140734928855040, 140734928859135, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722342535168, 140737488351231, ++SNULL, 140722342543359, 140737488351231, ++STORE, 140722342535168, 140722342543359, ++STORE, 140722342404096, 140722342543359, ++STORE, 94399699714048, 94399702048767, ++SNULL, 94399699927039, 94399702048767, ++STORE, 94399699714048, 94399699927039, ++STORE, 94399699927040, 94399702048767, ++ERASE, 94399699927040, 94399702048767, ++STORE, 94399702024192, 94399702036479, ++STORE, 94399702036480, 94399702048767, ++STORE, 139811024748544, 139811027001343, ++SNULL, 139811024891903, 139811027001343, ++STORE, 139811024748544, 139811024891903, ++STORE, 139811024891904, 139811027001343, ++ERASE, 139811024891904, 139811027001343, ++STORE, 139811026989056, 139811026997247, ++STORE, 139811026997248, 139811027001343, ++STORE, 140722342707200, 140722342711295, ++STORE, 140722342694912, 140722342707199, ++STORE, 139811026960384, 139811026989055, ++STORE, 139811026952192, 139811026960383, ++STORE, 139811022635008, 139811024748543, ++SNULL, 139811022635008, 139811022647295, ++STORE, 139811022647296, 139811024748543, ++STORE, 139811022635008, 139811022647295, ++SNULL, 139811024740351, 139811024748543, ++STORE, 139811022647296, 139811024740351, ++STORE, 139811024740352, 139811024748543, ++ERASE, 139811024740352, 139811024748543, ++STORE, 139811024740352, 139811024748543, ++STORE, 139811018838016, 139811022635007, ++SNULL, 139811018838016, 139811020496895, ++STORE, 139811020496896, 139811022635007, ++STORE, 139811018838016, 139811020496895, ++SNULL, 139811022594047, 139811022635007, ++STORE, 139811020496896, 139811022594047, ++STORE, 139811022594048, 139811022635007, ++SNULL, 139811022594048, 139811022618623, ++STORE, 139811022618624, 139811022635007, ++STORE, 139811022594048, 139811022618623, ++ERASE, 139811022594048, 139811022618623, ++STORE, 139811022594048, 139811022618623, ++ERASE, 139811022618624, 139811022635007, ++STORE, 139811022618624, 139811022635007, ++STORE, 139811026944000, 139811026960383, ++SNULL, 139811022610431, 139811022618623, ++STORE, 139811022594048, 139811022610431, ++STORE, 139811022610432, 139811022618623, ++SNULL, 139811024744447, 139811024748543, ++STORE, 139811024740352, 139811024744447, ++STORE, 139811024744448, 139811024748543, ++SNULL, 94399702028287, 94399702036479, ++STORE, 94399702024192, 94399702028287, ++STORE, 94399702028288, 94399702036479, ++SNULL, 139811026993151, 139811026997247, ++STORE, 139811026989056, 139811026993151, ++STORE, 139811026993152, 139811026997247, ++ERASE, 139811026960384, 139811026989055, ++STORE, 94399723880448, 94399724015615, ++STORE, 139811025260544, 139811026943999, ++STORE, 94399723880448, 94399724150783, ++STORE, 94399723880448, 94399724285951, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140735364939776, 140737488351231, ++SNULL, 140735364947967, 140737488351231, ++STORE, 140735364939776, 140735364947967, ++STORE, 140735364808704, 140735364947967, ++STORE, 94421528674304, 94421531009023, ++SNULL, 94421528887295, 94421531009023, ++STORE, 94421528674304, 94421528887295, ++STORE, 94421528887296, 94421531009023, ++ERASE, 94421528887296, 94421531009023, ++STORE, 94421530984448, 94421530996735, ++STORE, 94421530996736, 94421531009023, ++STORE, 140162004742144, 140162006994943, ++SNULL, 140162004885503, 140162006994943, ++STORE, 140162004742144, 140162004885503, ++STORE, 140162004885504, 140162006994943, ++ERASE, 140162004885504, 140162006994943, ++STORE, 140162006982656, 140162006990847, ++STORE, 140162006990848, 140162006994943, ++STORE, 140735365402624, 140735365406719, ++STORE, 140735365390336, 140735365402623, ++STORE, 140162006953984, 140162006982655, ++STORE, 140162006945792, 140162006953983, ++STORE, 140162002628608, 140162004742143, ++SNULL, 140162002628608, 140162002640895, ++STORE, 140162002640896, 140162004742143, ++STORE, 140162002628608, 140162002640895, ++SNULL, 140162004733951, 140162004742143, ++STORE, 140162002640896, 140162004733951, ++STORE, 140162004733952, 140162004742143, ++ERASE, 140162004733952, 140162004742143, ++STORE, 140162004733952, 140162004742143, ++STORE, 140161998831616, 140162002628607, ++SNULL, 140161998831616, 140162000490495, ++STORE, 140162000490496, 140162002628607, ++STORE, 140161998831616, 140162000490495, ++SNULL, 140162002587647, 140162002628607, ++STORE, 140162000490496, 140162002587647, ++STORE, 140162002587648, 140162002628607, ++SNULL, 140162002587648, 140162002612223, ++STORE, 140162002612224, 140162002628607, ++STORE, 140162002587648, 140162002612223, ++ERASE, 140162002587648, 140162002612223, ++STORE, 140162002587648, 140162002612223, ++ERASE, 140162002612224, 140162002628607, ++STORE, 140162002612224, 140162002628607, ++STORE, 140162006937600, 140162006953983, ++SNULL, 140162002604031, 140162002612223, ++STORE, 140162002587648, 140162002604031, ++STORE, 140162002604032, 140162002612223, ++SNULL, 140162004738047, 140162004742143, ++STORE, 140162004733952, 140162004738047, ++STORE, 140162004738048, 140162004742143, ++SNULL, 94421530988543, 94421530996735, ++STORE, 94421530984448, 94421530988543, ++STORE, 94421530988544, 94421530996735, ++SNULL, 140162006986751, 140162006990847, ++STORE, 140162006982656, 140162006986751, ++STORE, 140162006986752, 140162006990847, ++ERASE, 140162006953984, 140162006982655, ++STORE, 94421551697920, 94421551833087, ++STORE, 140162005254144, 140162006937599, ++STORE, 94421551697920, 94421551968255, ++STORE, 94421551697920, 94421552103423, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140733498486784, 140737488351231, ++SNULL, 140733498494975, 140737488351231, ++STORE, 140733498486784, 140733498494975, ++STORE, 140733498355712, 140733498494975, ++STORE, 94567985836032, 94567988170751, ++SNULL, 94567986049023, 94567988170751, ++STORE, 94567985836032, 94567986049023, ++STORE, 94567986049024, 94567988170751, ++ERASE, 94567986049024, 94567988170751, ++STORE, 94567988146176, 94567988158463, ++STORE, 94567988158464, 94567988170751, ++STORE, 139634278572032, 139634280824831, ++SNULL, 139634278715391, 139634280824831, ++STORE, 139634278572032, 139634278715391, ++STORE, 139634278715392, 139634280824831, ++ERASE, 139634278715392, 139634280824831, ++STORE, 139634280812544, 139634280820735, ++STORE, 139634280820736, 139634280824831, ++STORE, 140733498544128, 140733498548223, ++STORE, 140733498531840, 140733498544127, ++STORE, 139634280783872, 139634280812543, ++STORE, 139634280775680, 139634280783871, ++STORE, 139634276458496, 139634278572031, ++SNULL, 139634276458496, 139634276470783, ++STORE, 139634276470784, 139634278572031, ++STORE, 139634276458496, 139634276470783, ++SNULL, 139634278563839, 139634278572031, ++STORE, 139634276470784, 139634278563839, ++STORE, 139634278563840, 139634278572031, ++ERASE, 139634278563840, 139634278572031, ++STORE, 139634278563840, 139634278572031, ++STORE, 139634272661504, 139634276458495, ++SNULL, 139634272661504, 139634274320383, ++STORE, 139634274320384, 139634276458495, ++STORE, 139634272661504, 139634274320383, ++SNULL, 139634276417535, 139634276458495, ++STORE, 139634274320384, 139634276417535, ++STORE, 139634276417536, 139634276458495, ++SNULL, 139634276417536, 139634276442111, ++STORE, 139634276442112, 139634276458495, ++STORE, 139634276417536, 139634276442111, ++ERASE, 139634276417536, 139634276442111, ++STORE, 139634276417536, 139634276442111, ++ERASE, 139634276442112, 139634276458495, ++STORE, 139634276442112, 139634276458495, ++STORE, 139634280767488, 139634280783871, ++SNULL, 139634276433919, 139634276442111, ++STORE, 139634276417536, 139634276433919, ++STORE, 139634276433920, 139634276442111, ++SNULL, 139634278567935, 139634278572031, ++STORE, 139634278563840, 139634278567935, ++STORE, 139634278567936, 139634278572031, ++SNULL, 94567988150271, 94567988158463, ++STORE, 94567988146176, 94567988150271, ++STORE, 94567988150272, 94567988158463, ++SNULL, 139634280816639, 139634280820735, ++STORE, 139634280812544, 139634280816639, ++STORE, 139634280816640, 139634280820735, ++ERASE, 139634280783872, 139634280812543, ++STORE, 94567996379136, 94567996514303, ++STORE, 139634279084032, 139634280767487, ++STORE, 94567996379136, 94567996649471, ++STORE, 94567996379136, 94567996784639, ++STORE, 94567996379136, 94567996960767, ++SNULL, 94567996932095, 94567996960767, ++STORE, 94567996379136, 94567996932095, ++STORE, 94567996932096, 94567996960767, ++ERASE, 94567996932096, 94567996960767, ++STORE, 94567996379136, 94567997071359, ++STORE, 94567996379136, 94567997206527, ++SNULL, 94567997186047, 94567997206527, ++STORE, 94567996379136, 94567997186047, ++STORE, 94567997186048, 94567997206527, ++ERASE, 94567997186048, 94567997206527, ++STORE, 94567996379136, 94567997358079, ++STORE, 94567996379136, 94567997493247, ++SNULL, 94567997476863, 94567997493247, ++STORE, 94567996379136, 94567997476863, ++STORE, 94567997476864, 94567997493247, ++ERASE, 94567997476864, 94567997493247, ++STORE, 94567996379136, 94567997612031, ++STORE, 94567996379136, 94567997767679, ++SNULL, 94567997739007, 94567997767679, ++STORE, 94567996379136, 94567997739007, ++STORE, 94567997739008, 94567997767679, ++ERASE, 94567997739008, 94567997767679, ++SNULL, 94567997698047, 94567997739007, ++STORE, 94567996379136, 94567997698047, ++STORE, 94567997698048, 94567997739007, ++ERASE, 94567997698048, 94567997739007, ++STORE, 94567996379136, 94567997853695, ++STORE, 94567996379136, 94567997988863, ++STORE, 94567996379136, 94567998132223, ++STORE, 94567996379136, 94567998275583, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140723667759104, 140737488351231, ++SNULL, 140723667767295, 140737488351231, ++STORE, 140723667759104, 140723667767295, ++STORE, 140723667628032, 140723667767295, ++STORE, 94231598800896, 94231601135615, ++SNULL, 94231599013887, 94231601135615, ++STORE, 94231598800896, 94231599013887, ++STORE, 94231599013888, 94231601135615, ++ERASE, 94231599013888, 94231601135615, ++STORE, 94231601111040, 94231601123327, ++STORE, 94231601123328, 94231601135615, ++STORE, 140269472649216, 140269474902015, ++SNULL, 140269472792575, 140269474902015, ++STORE, 140269472649216, 140269472792575, ++STORE, 140269472792576, 140269474902015, ++ERASE, 140269472792576, 140269474902015, ++STORE, 140269474889728, 140269474897919, ++STORE, 140269474897920, 140269474902015, ++STORE, 140723667836928, 140723667841023, ++STORE, 140723667824640, 140723667836927, ++STORE, 140269474861056, 140269474889727, ++STORE, 140269474852864, 140269474861055, ++STORE, 140269470535680, 140269472649215, ++SNULL, 140269470535680, 140269470547967, ++STORE, 140269470547968, 140269472649215, ++STORE, 140269470535680, 140269470547967, ++SNULL, 140269472641023, 140269472649215, ++STORE, 140269470547968, 140269472641023, ++STORE, 140269472641024, 140269472649215, ++ERASE, 140269472641024, 140269472649215, ++STORE, 140269472641024, 140269472649215, ++STORE, 140269466738688, 140269470535679, ++SNULL, 140269466738688, 140269468397567, ++STORE, 140269468397568, 140269470535679, ++STORE, 140269466738688, 140269468397567, ++SNULL, 140269470494719, 140269470535679, ++STORE, 140269468397568, 140269470494719, ++STORE, 140269470494720, 140269470535679, ++SNULL, 140269470494720, 140269470519295, ++STORE, 140269470519296, 140269470535679, ++STORE, 140269470494720, 140269470519295, ++ERASE, 140269470494720, 140269470519295, ++STORE, 140269470494720, 140269470519295, ++ERASE, 140269470519296, 140269470535679, ++STORE, 140269470519296, 140269470535679, ++STORE, 140269474844672, 140269474861055, ++SNULL, 140269470511103, 140269470519295, ++STORE, 140269470494720, 140269470511103, ++STORE, 140269470511104, 140269470519295, ++SNULL, 140269472645119, 140269472649215, ++STORE, 140269472641024, 140269472645119, ++STORE, 140269472645120, 140269472649215, ++SNULL, 94231601115135, 94231601123327, ++STORE, 94231601111040, 94231601115135, ++STORE, 94231601115136, 94231601123327, ++SNULL, 140269474893823, 140269474897919, ++STORE, 140269474889728, 140269474893823, ++STORE, 140269474893824, 140269474897919, ++ERASE, 140269474861056, 140269474889727, ++STORE, 94231626592256, 94231626727423, ++STORE, 140269473161216, 140269474844671, ++STORE, 94231626592256, 94231626862591, ++STORE, 94231626592256, 94231626997759, ++STORE, 94327178862592, 94327179075583, ++STORE, 94327181172736, 94327181176831, ++STORE, 94327181176832, 94327181185023, ++STORE, 94327181185024, 94327181197311, ++STORE, 94327185715200, 94327186685951, ++STORE, 140172071755776, 140172073414655, ++STORE, 140172073414656, 140172075511807, ++STORE, 140172075511808, 140172075528191, ++STORE, 140172075528192, 140172075536383, ++STORE, 140172075536384, 140172075552767, ++STORE, 140172075552768, 140172075565055, ++STORE, 140172075565056, 140172077658111, ++STORE, 140172077658112, 140172077662207, ++STORE, 140172077662208, 140172077666303, ++STORE, 140172077666304, 140172077809663, ++STORE, 140172078178304, 140172079861759, ++STORE, 140172079861760, 140172079878143, ++STORE, 140172079878144, 140172079906815, ++STORE, 140172079906816, 140172079910911, ++STORE, 140172079910912, 140172079915007, ++STORE, 140172079915008, 140172079919103, ++STORE, 140720358359040, 140720358494207, ++STORE, 140720358498304, 140720358510591, ++STORE, 140720358510592, 140720358514687, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140722548621312, 140737488351231, ++SNULL, 140722548629503, 140737488351231, ++STORE, 140722548621312, 140722548629503, ++STORE, 140722548490240, 140722548629503, ++STORE, 93949289504768, 93949291728895, ++SNULL, 93949289615359, 93949291728895, ++STORE, 93949289504768, 93949289615359, ++STORE, 93949289615360, 93949291728895, ++ERASE, 93949289615360, 93949291728895, ++STORE, 93949291708416, 93949291720703, ++STORE, 93949291720704, 93949291728895, ++STORE, 140305861902336, 140305864155135, ++SNULL, 140305862045695, 140305864155135, ++STORE, 140305861902336, 140305862045695, ++STORE, 140305862045696, 140305864155135, ++ERASE, 140305862045696, 140305864155135, ++STORE, 140305864142848, 140305864151039, ++STORE, 140305864151040, 140305864155135, ++STORE, 140722549821440, 140722549825535, ++STORE, 140722549809152, 140722549821439, ++STORE, 140305864114176, 140305864142847, ++STORE, 140305864105984, 140305864114175, ++STORE, 140305858105344, 140305861902335, ++SNULL, 140305858105344, 140305859764223, ++STORE, 140305859764224, 140305861902335, ++STORE, 140305858105344, 140305859764223, ++SNULL, 140305861861375, 140305861902335, ++STORE, 140305859764224, 140305861861375, ++STORE, 140305861861376, 140305861902335, ++SNULL, 140305861861376, 140305861885951, ++STORE, 140305861885952, 140305861902335, ++STORE, 140305861861376, 140305861885951, ++ERASE, 140305861861376, 140305861885951, ++STORE, 140305861861376, 140305861885951, ++ERASE, 140305861885952, 140305861902335, ++STORE, 140305861885952, 140305861902335, ++SNULL, 140305861877759, 140305861885951, ++STORE, 140305861861376, 140305861877759, ++STORE, 140305861877760, 140305861885951, ++SNULL, 93949291716607, 93949291720703, ++STORE, 93949291708416, 93949291716607, ++STORE, 93949291716608, 93949291720703, ++SNULL, 140305864146943, 140305864151039, ++STORE, 140305864142848, 140305864146943, ++STORE, 140305864146944, 140305864151039, ++ERASE, 140305864114176, 140305864142847, ++STORE, 93949324136448, 93949324271615, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140725754908672, 140737488351231, ++SNULL, 140725754916863, 140737488351231, ++STORE, 140725754908672, 140725754916863, ++STORE, 140725754777600, 140725754916863, ++STORE, 94831184375808, 94831186599935, ++SNULL, 94831184486399, 94831186599935, ++STORE, 94831184375808, 94831184486399, ++STORE, 94831184486400, 94831186599935, ++ERASE, 94831184486400, 94831186599935, ++STORE, 94831186579456, 94831186591743, ++STORE, 94831186591744, 94831186599935, ++STORE, 140605482479616, 140605484732415, ++SNULL, 140605482622975, 140605484732415, ++STORE, 140605482479616, 140605482622975, ++STORE, 140605482622976, 140605484732415, ++ERASE, 140605482622976, 140605484732415, ++STORE, 140605484720128, 140605484728319, ++STORE, 140605484728320, 140605484732415, ++STORE, 140725755670528, 140725755674623, ++STORE, 140725755658240, 140725755670527, ++STORE, 140605484691456, 140605484720127, ++STORE, 140605484683264, 140605484691455, ++STORE, 140605478682624, 140605482479615, ++SNULL, 140605478682624, 140605480341503, ++STORE, 140605480341504, 140605482479615, ++STORE, 140605478682624, 140605480341503, ++SNULL, 140605482438655, 140605482479615, ++STORE, 140605480341504, 140605482438655, ++STORE, 140605482438656, 140605482479615, ++SNULL, 140605482438656, 140605482463231, ++STORE, 140605482463232, 140605482479615, ++STORE, 140605482438656, 140605482463231, ++ERASE, 140605482438656, 140605482463231, ++STORE, 140605482438656, 140605482463231, ++ERASE, 140605482463232, 140605482479615, ++STORE, 140605482463232, 140605482479615, ++SNULL, 140605482455039, 140605482463231, ++STORE, 140605482438656, 140605482455039, ++STORE, 140605482455040, 140605482463231, ++SNULL, 94831186587647, 94831186591743, ++STORE, 94831186579456, 94831186587647, ++STORE, 94831186587648, 94831186591743, ++SNULL, 140605484724223, 140605484728319, ++STORE, 140605484720128, 140605484724223, ++STORE, 140605484724224, 140605484728319, ++ERASE, 140605484691456, 140605484720127, ++STORE, 94831217156096, 94831217291263, ++STORE, 94327178862592, 94327179075583, ++STORE, 94327181172736, 94327181176831, ++STORE, 94327181176832, 94327181185023, ++STORE, 94327181185024, 94327181197311, ++STORE, 94327185715200, 94327186685951, ++STORE, 140172071755776, 140172073414655, ++STORE, 140172073414656, 140172075511807, ++STORE, 140172075511808, 140172075528191, ++STORE, 140172075528192, 140172075536383, ++STORE, 140172075536384, 140172075552767, ++STORE, 140172075552768, 140172075565055, ++STORE, 140172075565056, 140172077658111, ++STORE, 140172077658112, 140172077662207, ++STORE, 140172077662208, 140172077666303, ++STORE, 140172077666304, 140172077809663, ++STORE, 140172078178304, 140172079861759, ++STORE, 140172079861760, 140172079878143, ++STORE, 140172079878144, 140172079906815, ++STORE, 140172079906816, 140172079910911, ++STORE, 140172079910912, 140172079915007, ++STORE, 140172079915008, 140172079919103, ++STORE, 140720358359040, 140720358494207, ++STORE, 140720358498304, 140720358510591, ++STORE, 140720358510592, 140720358514687, ++STORE, 140737488347136, 140737488351231, ++STORE, 140737488343040, 140737488351231, ++STORE, 140737488338944, 140737488351231, ++STORE, 140734529933312, 140737488351231, ++SNULL, 140734529945599, 140737488351231, ++STORE, 140734529933312, 140734529945599, ++STORE, 140734529802240, 140734529945599, ++STORE, 4194304, 26279935, ++STORE, 28372992, 28454911, ++STORE, 28454912, 29806591, ++STORE, 140249744060416, 140249746313215, ++SNULL, 140249744203775, 140249746313215, ++STORE, 140249744060416, 140249744203775, ++STORE, 140249744203776, 140249746313215, ++ERASE, 140249744203776, 140249746313215, ++STORE, 140249746300928, 140249746309119, ++STORE, 140249746309120, 140249746313215, ++STORE, 140734530174976, 140734530179071, ++STORE, 140734530162688, 140734530174975, ++STORE, 140249746272256, 140249746300927, ++STORE, 140249746264064, 140249746272255, ++STORE, 140249740226560, 140249744060415, ++SNULL, 140249740226560, 140249741934591, ++STORE, 140249741934592, 140249744060415, ++STORE, 140249740226560, 140249741934591, ++SNULL, 140249744027647, 140249744060415, ++STORE, 140249741934592, 140249744027647, ++STORE, 140249744027648, 140249744060415, ++ERASE, 140249744027648, 140249744060415, ++STORE, 140249744027648, 140249744060415, ++STORE, 140249738031104, 140249740226559, ++SNULL, 140249738031104, 140249738125311, ++STORE, 140249738125312, 140249740226559, ++STORE, 140249738031104, 140249738125311, ++SNULL, 140249740218367, 140249740226559, ++STORE, 140249738125312, 140249740218367, ++STORE, 140249740218368, 140249740226559, ++ERASE, 140249740218368, 140249740226559, ++STORE, 140249740218368, 140249740226559, ++STORE, 140249735512064, 140249738031103, ++SNULL, 140249735512064, 140249735925759, ++STORE, 140249735925760, 140249738031103, ++STORE, 140249735512064, 140249735925759, ++SNULL, 140249738018815, 140249738031103, ++STORE, 140249735925760, 140249738018815, ++STORE, 140249738018816, 140249738031103, ++ERASE, 140249738018816, 140249738031103, ++STORE, 140249738018816, 140249738031103, ++STORE, 140249732878336, 140249735512063, ++SNULL, 140249732878336, 140249733406719, ++STORE, 140249733406720, 140249735512063, ++STORE, 140249732878336, 140249733406719, ++SNULL, 140249735503871, 140249735512063, ++STORE, 140249733406720, 140249735503871, ++STORE, 140249735503872, 140249735512063, ++ERASE, 140249735503872, 140249735512063, ++STORE, 140249735503872, 140249735512063, ++STORE, 140249730764800, 140249732878335, ++SNULL, 140249730764800, 140249730777087, ++STORE, 140249730777088, 140249732878335, ++STORE, 140249730764800, 140249730777087, ++SNULL, 140249732870143, 140249732878335, ++STORE, 140249730777088, 140249732870143, ++STORE, 140249732870144, 140249732878335, ++ERASE, 140249732870144, 140249732878335, ++STORE, 140249732870144, 140249732878335, ++STORE, 140249728561152, 140249730764799, ++SNULL, 140249728561152, 140249728663551, ++STORE, 140249728663552, 140249730764799, ++STORE, 140249728561152, 140249728663551, ++SNULL, 140249730756607, 140249730764799, ++STORE, 140249728663552, 140249730756607, ++STORE, 140249730756608, 140249730764799, ++ERASE, 140249730756608, 140249730764799, ++STORE, 140249730756608, 140249730764799, ++STORE, 140249746255872, 140249746272255, ++STORE, 140249725399040, 140249728561151, ++SNULL, 140249725399040, 140249726459903, ++STORE, 140249726459904, 140249728561151, ++STORE, 140249725399040, 140249726459903, ++SNULL, 140249728552959, 140249728561151, ++STORE, 140249726459904, 140249728552959, ++STORE, 140249728552960, 140249728561151, ++ERASE, 140249728552960, 140249728561151, ++STORE, 140249728552960, 140249728561151, ++STORE, 140249721602048, 140249725399039, ++SNULL, 140249721602048, 140249723260927, ++STORE, 140249723260928, 140249725399039, ++STORE, 140249721602048, 140249723260927, ++SNULL, 140249725358079, 140249725399039, ++STORE, 140249723260928, 140249725358079, ++STORE, 140249725358080, 140249725399039, ++SNULL, 140249725358080, 140249725382655, ++STORE, 140249725382656, 140249725399039, ++STORE, 140249725358080, 140249725382655, ++ERASE, 140249725358080, 140249725382655, ++STORE, 140249725358080, 140249725382655, ++ERASE, 140249725382656, 140249725399039, ++STORE, 140249725382656, 140249725399039, ++STORE, 140249746243584, 140249746272255, ++SNULL, 140249725374463, 140249725382655, ++STORE, 140249725358080, 140249725374463, ++STORE, 140249725374464, 140249725382655, ++SNULL, 140249728557055, 140249728561151, ++STORE, 140249728552960, 140249728557055, ++STORE, 140249728557056, 140249728561151, ++SNULL, 140249730760703, 140249730764799, ++STORE, 140249730756608, 140249730760703, ++STORE, 140249730760704, 140249730764799, ++SNULL, 140249732874239, 140249732878335, ++STORE, 140249732870144, 140249732874239, ++STORE, 140249732874240, 140249732878335, ++SNULL, 140249735507967, 140249735512063, ++STORE, 140249735503872, 140249735507967, ++STORE, 140249735507968, 140249735512063, ++SNULL, 140249738027007, 140249738031103, ++STORE, 140249738018816, 140249738027007, ++STORE, 140249738027008, 140249738031103, ++SNULL, 140249740222463, 140249740226559, ++STORE, 140249740218368, 140249740222463, ++STORE, 140249740222464, 140249740226559, ++SNULL, 140249744031743, 140249744060415, ++STORE, 140249744027648, 140249744031743, ++STORE, 140249744031744, 140249744060415, ++SNULL, 28405759, 28454911, ++STORE, 28372992, 28405759, ++STORE, 28405760, 28454911, ++SNULL, 140249746305023, 140249746309119, ++STORE, 140249746300928, 140249746305023, ++STORE, 140249746305024, 140249746309119, ++ERASE, 140249746272256, 140249746300927, ++STORE, 33853440, 33988607, ++STORE, 140249744560128, 140249746243583, ++STORE, 140249746296832, 140249746300927, ++STORE, 140249744424960, 140249744560127, ++STORE, 33853440, 34131967, ++STORE, 140249719504896, 140249721602047, ++STORE, 140249746288640, 140249746300927, ++STORE, 140249746280448, 140249746300927, ++STORE, 140249746243584, 140249746280447, ++STORE, 140249744408576, 140249744560127, ++STORE, 33853440, 34267135, ++STORE, 33853440, 34422783, ++STORE, 140249744400384, 140249744560127, ++STORE, 140249744392192, 140249744560127, ++STORE, 33853440, 34557951, ++STORE, 33853440, 34693119, ++STORE, 140249744375808, 140249744560127, ++STORE, 140249744367616, 140249744560127, ++STORE, 33853440, 34832383, ++STORE, 140249719230464, 140249721602047, ++STORE, 140249744207872, 140249744560127, ++STORE, 33853440, 34971647, ++SNULL, 34963455, 34971647, ++STORE, 33853440, 34963455, ++STORE, 34963456, 34971647, ++ERASE, 34963456, 34971647, ++SNULL, 34955263, 34963455, ++STORE, 33853440, 34955263, ++STORE, 34955264, 34963455, ++ERASE, 34955264, 34963455, ++SNULL, 34947071, 34955263, ++STORE, 33853440, 34947071, ++STORE, 34947072, 34955263, ++ERASE, 34947072, 34955263, ++SNULL, 34938879, 34947071, ++STORE, 33853440, 34938879, ++STORE, 34938880, 34947071, ++ERASE, 34938880, 34947071, ++STORE, 140249719214080, 140249721602047, ++STORE, 140249719148544, 140249721602047, ++STORE, 140249719115776, 140249721602047, ++STORE, 140249717018624, 140249721602047, ++STORE, 140249716953088, 140249721602047, ++STORE, 33853440, 35086335, ++STORE, 140249716822016, 140249721602047, ++STORE, 140249716559872, 140249721602047, ++STORE, 140249716551680, 140249721602047, ++STORE, 140249716535296, 140249721602047, ++STORE, 140249716527104, 140249721602047, ++STORE, 140249716518912, 140249721602047, ++STORE, 33853440, 35221503, ++SNULL, 35213311, 35221503, ++STORE, 33853440, 35213311, ++STORE, 35213312, 35221503, ++ERASE, 35213312, 35221503, ++SNULL, 35205119, 35213311, ++STORE, 33853440, 35205119, ++STORE, 35205120, 35213311, ++ERASE, 35205120, 35213311, ++SNULL, 35192831, 35205119, ++STORE, 33853440, 35192831, ++STORE, 35192832, 35205119, ++ERASE, 35192832, 35205119, ++SNULL, 35176447, 35192831, ++STORE, 33853440, 35176447, ++STORE, 35176448, 35192831, ++ERASE, 35176448, 35192831, ++STORE, 140249716502528, 140249721602047, ++STORE, 33853440, 35311615, ++SNULL, 35307519, 35311615, ++STORE, 33853440, 35307519, ++STORE, 35307520, 35311615, ++ERASE, 35307520, 35311615, ++SNULL, 35303423, 35307519, ++STORE, 33853440, 35303423, ++STORE, 35303424, 35307519, ++ERASE, 35303424, 35307519, ++SNULL, 35299327, 35303423, ++STORE, 33853440, 35299327, ++STORE, 35299328, 35303423, ++ERASE, 35299328, 35303423, ++SNULL, 35295231, 35299327, ++STORE, 33853440, 35295231, ++STORE, 35295232, 35299327, ++ERASE, 35295232, 35299327, ++SNULL, 35291135, 35295231, ++STORE, 33853440, 35291135, ++STORE, 35291136, 35295231, ++ERASE, 35291136, 35295231, ++SNULL, 35287039, 35291135, ++STORE, 33853440, 35287039, ++STORE, 35287040, 35291135, ++ERASE, 35287040, 35291135, ++SNULL, 35282943, 35287039, ++STORE, 33853440, 35282943, ++STORE, 35282944, 35287039, ++ERASE, 35282944, 35287039, ++STORE, 140249716486144, 140249721602047, ++STORE, 140249716453376, 140249721602047, ++STORE, 33853440, 35418111, ++SNULL, 35401727, 35418111, ++STORE, 33853440, 35401727, ++STORE, 35401728, 35418111, ++ERASE, 35401728, 35418111, ++SNULL, 35389439, 35401727, ++STORE, 33853440, 35389439, ++STORE, 35389440, 35401727, ++ERASE, 35389440, 35401727, ++STORE, 140249714356224, 140249721602047, ++STORE, 33853440, 35540991, ++STORE, 140249714339840, 140249721602047, ++STORE, 140249714077696, 140249721602047, ++STORE, 140249714069504, 140249721602047, ++STORE, 140249714061312, 140249721602047, ++STORE, 33853440, 35680255, ++SNULL, 35672063, 35680255, ++STORE, 33853440, 35672063, ++STORE, 35672064, 35680255, ++ERASE, 35672064, 35680255, ++SNULL, 35627007, 35672063, ++STORE, 33853440, 35627007, ++STORE, 35627008, 35672063, ++ERASE, 35627008, 35672063, ++STORE, 140249711964160, 140249721602047, ++STORE, 33853440, 35762175, ++SNULL, 35753983, 35762175, ++STORE, 33853440, 35753983, ++STORE, 35753984, 35762175, ++ERASE, 35753984, 35762175, ++SNULL, 35745791, 35753983, ++STORE, 33853440, 35745791, ++STORE, 35745792, 35753983, ++ERASE, 35745792, 35753983, ++STORE, 140249711955968, 140249721602047, ++STORE, 140249711947776, 140249721602047, ++STORE, 140249710899200, 140249721602047, ++STORE, 140249710866432, 140249721602047, ++STORE, 140249710600192, 140249721602047, ++SNULL, 140249744424959, 140249744560127, ++STORE, 140249744207872, 140249744424959, ++STORE, 140249744424960, 140249744560127, ++ERASE, 140249744424960, 140249744560127, ++STORE, 140249708503040, 140249721602047, ++STORE, 33853440, 35885055, ++STORE, 140249707978752, 140249721602047, ++STORE, 140249705881600, 140249721602047, ++STORE, 33853440, 36036607, ++STORE, 33853440, 36175871, ++STORE, 140249744551936, 140249744560127, ++STORE, 140249744543744, 140249744560127, ++STORE, 140249744535552, 140249744560127, ++STORE, 140249744527360, 140249744560127, ++STORE, 140249744519168, 140249744560127, ++STORE, 140249705619456, 140249721602047, ++STORE, 140249744510976, 140249744560127, ++STORE, 140249744502784, 140249744560127, ++STORE, 140249744494592, 140249744560127, ++STORE, 140249744486400, 140249744560127, ++STORE, 140249744478208, 140249744560127, ++STORE, 140249744470016, 140249744560127, ++STORE, 140249744461824, 140249744560127, ++STORE, 140249744453632, 140249744560127, ++STORE, 140249744445440, 140249744560127, ++STORE, 140249744437248, 140249744560127, ++STORE, 140249744429056, 140249744560127, ++STORE, 140249703522304, 140249721602047, ++STORE, 33853440, 36311039, ++STORE, 140249703489536, 140249721602047, ++STORE, 33853440, 36474879, ++STORE, 140249703456768, 140249721602047, ++STORE, 33853440, 36622335, ++STORE, 140249703424000, 140249721602047, ++STORE, 140249703391232, 140249721602047, ++STORE, 33853440, 36810751, ++STORE, 140249703358464, 140249721602047, ++STORE, 140249703325696, 140249721602047, ++SNULL, 36655103, 36810751, ++STORE, 33853440, 36655103, ++STORE, 36655104, 36810751, ++ERASE, 36655104, 36810751, ++SNULL, 36438015, 36655103, ++STORE, 33853440, 36438015, ++STORE, 36438016, 36655103, ++ERASE, 36438016, 36655103, ++STORE, 140249703317504, 140249721602047, ++STORE, 140249701220352, 140249721602047, ++STORE, 33853440, 36585471, ++STORE, 33853440, 36782079, ++STORE, 140249701212160, 140249721602047, ++STORE, 140249701203968, 140249721602047, ++STORE, 140249701195776, 140249721602047, ++STORE, 140249701187584, 140249721602047, ++STORE, 140249701179392, 140249721602047, ++STORE, 140249701171200, 140249721602047, ++STORE, 140249701163008, 140249721602047, ++STORE, 140249701154816, 140249721602047, ++STORE, 140249701146624, 140249721602047, ++STORE, 140249701138432, 140249721602047, ++STORE, 140249701130240, 140249721602047, ++STORE, 140249700081664, 140249721602047, ++STORE, 140249700073472, 140249721602047, ++STORE, 33853440, 36978687, ++STORE, 140249697976320, 140249721602047, ++STORE, 33853440, 37240831, ++STORE, 140249695879168, 140249721602047, ++STORE, 140249695870976, 140249721602047, ++STORE, 140249695862784, 140249721602047, ++STORE, 140249695854592, 140249721602047, ++STORE, 140249695326208, 140249721602047, ++SNULL, 140249710600191, 140249721602047, ++STORE, 140249695326208, 140249710600191, ++STORE, 140249710600192, 140249721602047, ++SNULL, 140249710600192, 140249710866431, ++STORE, 140249710866432, 140249721602047, ++STORE, 140249710600192, 140249710866431, ++ERASE, 140249710600192, 140249710866431, ++STORE, 140249691131904, 140249710600191, ++STORE, 33853440, 37474303, ++STORE, 140249710858240, 140249721602047, ++STORE, 140249710850048, 140249721602047, ++STORE, 140249710841856, 140249721602047, ++STORE, 140249710833664, 140249721602047, ++STORE, 140249710825472, 140249721602047, ++STORE, 140249710817280, 140249721602047, ++STORE, 140249710809088, 140249721602047, ++STORE, 140249710800896, 140249721602047, ++STORE, 140249710792704, 140249721602047, ++STORE, 140249710784512, 140249721602047, ++STORE, 140249710776320, 140249721602047, ++STORE, 140249710768128, 140249721602047, ++STORE, 140249710759936, 140249721602047, ++STORE, 140249710751744, 140249721602047, ++STORE, 140249710743552, 140249721602047, ++STORE, 140249710735360, 140249721602047, ++STORE, 140249689034752, 140249710600191, ++STORE, 140249710727168, 140249721602047, ++STORE, 140249686937600, 140249710600191, ++STORE, 33853440, 37867519, ++STORE, 140249684840448, 140249710600191, ++STORE, 140249710718976, 140249721602047, ++STORE, 140249682743296, 140249710600191, ++STORE, 140249710710784, 140249721602047, ++STORE, 140249710702592, 140249721602047, ++STORE, 140249710694400, 140249721602047, ++STORE, 140249710686208, 140249721602047, ++STORE, 140249710678016, 140249721602047, ++STORE, 140249682612224, 140249710600191, ++STORE, 140249682087936, 140249710600191, ++SNULL, 140249705619455, 140249710600191, ++STORE, 140249682087936, 140249705619455, ++STORE, 140249705619456, 140249710600191, ++SNULL, 140249705619456, 140249705881599, ++STORE, 140249705881600, 140249710600191, ++STORE, 140249705619456, 140249705881599, ++ERASE, 140249705619456, 140249705881599, ++STORE, 140249679990784, 140249705619455, ++STORE, 140249710669824, 140249721602047, ++STORE, 140249677893632, 140249705619455, ++STORE, 140249710653440, 140249721602047, ++STORE, 140249710645248, 140249721602047, ++STORE, 140249710637056, 140249721602047, ++STORE, 140249710628864, 140249721602047, ++STORE, 140249710620672, 140249721602047, ++STORE, 140249710612480, 140249721602047, ++STORE, 140249710604288, 140249721602047, ++STORE, 140249705873408, 140249710600191, ++STORE, 140249705865216, 140249710600191, ++STORE, 140249705857024, 140249710600191, ++STORE, 140249705848832, 140249710600191, ++STORE, 140249705840640, 140249710600191, ++STORE, 140249705832448, 140249710600191, ++STORE, 140249705824256, 140249710600191, ++STORE, 140249705816064, 140249710600191, ++STORE, 140249705807872, 140249710600191, ++STORE, 140249705799680, 140249710600191, ++STORE, 33853440, 38129663, ++SNULL, 140249744207872, 140249744367615, ++STORE, 140249744367616, 140249744424959, ++STORE, 140249744207872, 140249744367615, ++ERASE, 140249744207872, 140249744367615, ++STORE, 140249677606912, 140249705619455, ++STORE, 140249675509760, 140249705619455, ++SNULL, 140249677606911, 140249705619455, ++STORE, 140249675509760, 140249677606911, ++STORE, 140249677606912, 140249705619455, ++SNULL, 140249677606912, 140249677893631, ++STORE, 140249677893632, 140249705619455, ++STORE, 140249677606912, 140249677893631, ++ERASE, 140249677606912, 140249677893631, ++STORE, 140249744359424, 140249744424959, ++STORE, 33853440, 38391807, ++STORE, 140249674981376, 140249677606911, ++STORE, 140249672884224, 140249677606911, ++SNULL, 140249719230463, 140249721602047, ++STORE, 140249710604288, 140249719230463, ++STORE, 140249719230464, 140249721602047, ++SNULL, 140249719230464, 140249719504895, ++STORE, 140249719504896, 140249721602047, ++STORE, 140249719230464, 140249719504895, ++ERASE, 140249719230464, 140249719504895, ++STORE, 140249744351232, 140249744424959, ++STORE, 140249744343040, 140249744424959, ++STORE, 140249744334848, 140249744424959, ++STORE, 140249744326656, 140249744424959, ++STORE, 140249744310272, 140249744424959, ++STORE, 140249744302080, 140249744424959, ++STORE, 140249744285696, 140249744424959, ++STORE, 140249744277504, 140249744424959, ++STORE, 140249744261120, 140249744424959, ++STORE, 140249744252928, 140249744424959, ++STORE, 140249744220160, 140249744424959, ++STORE, 140249744211968, 140249744424959, ++STORE, 140249719488512, 140249721602047, ++STORE, 140249744203776, 140249744424959, ++STORE, 140249719472128, 140249721602047, ++STORE, 140249719463936, 140249721602047, ++STORE, 140249719447552, 140249721602047, ++STORE, 140249719439360, 140249721602047, ++STORE, 140249719406592, 140249721602047, ++STORE, 140249719398400, 140249721602047, ++STORE, 140249719382016, 140249721602047, ++STORE, 140249719373824, 140249721602047, ++STORE, 140249719357440, 140249721602047, ++STORE, 140249719349248, 140249721602047, ++STORE, 140249719332864, 140249721602047, ++STORE, 140249719324672, 140249721602047, ++STORE, 140249719291904, 140249721602047, ++STORE, 140249719283712, 140249721602047, ++STORE, 140249719267328, 140249721602047, ++STORE, 140249719259136, 140249721602047, ++STORE, 140249719242752, 140249721602047, ++STORE, 140249719234560, 140249721602047, ++STORE, 140249705783296, 140249710600191, ++STORE, 140249705775104, 140249710600191, ++STORE, 140249705742336, 140249710600191, ++STORE, 140249705734144, 140249710600191, ++STORE, 140249705717760, 140249710600191, ++STORE, 140249670787072, 140249677606911, ++STORE, 140249705709568, 140249710600191, ++STORE, 140249705693184, 140249710600191, ++STORE, 140249705684992, 140249710600191, ++STORE, 140249705668608, 140249710600191, ++STORE, 140249705660416, 140249710600191, ++STORE, 140249705627648, 140249710600191, ++STORE, 140249677893632, 140249710600191, ++STORE, 140249677877248, 140249710600191, ++STORE, 140249677869056, 140249710600191, ++STORE, 140249677852672, 140249710600191, ++STORE, 140249677844480, 140249710600191, ++STORE, 140249677828096, 140249710600191, ++STORE, 140249668689920, 140249677606911, ++STORE, 140249677819904, 140249710600191, ++STORE, 140249677787136, 140249710600191, ++STORE, 140249677778944, 140249710600191, ++STORE, 140249677762560, 140249710600191, ++STORE, 140249677754368, 140249710600191, ++STORE, 140249677737984, 140249710600191, ++STORE, 140249677729792, 140249710600191, ++STORE, 140249677713408, 140249710600191, ++STORE, 140249677705216, 140249710600191, ++STORE, 140249677672448, 140249710600191, ++STORE, 140249677664256, 140249710600191, ++STORE, 140249677647872, 140249710600191, ++STORE, 140249677639680, 140249710600191, ++STORE, 140249677623296, 140249710600191, ++STORE, 140249677615104, 140249710600191, ++STORE, 140249668673536, 140249677606911, ++STORE, 140249668673536, 140249710600191, ++STORE, 140249668640768, 140249710600191, ++STORE, 140249668632576, 140249710600191, ++STORE, 140249668616192, 140249710600191, ++STORE, 140249668608000, 140249710600191, ++STORE, 140249668591616, 140249710600191, ++STORE, 140249668583424, 140249710600191, ++STORE, 140249668567040, 140249710600191, ++STORE, 140249668558848, 140249710600191, ++STORE, 140249668526080, 140249710600191, ++STORE, 140249668517888, 140249710600191, ++STORE, 140249668501504, 140249710600191, ++STORE, 140249668493312, 140249710600191, ++STORE, 140249668476928, 140249710600191, ++STORE, 140249668468736, 140249710600191, ++STORE, 140249668452352, 140249710600191, ++STORE, 140249668444160, 140249710600191, ++STORE, 140249668411392, 140249710600191, ++STORE, 140249668403200, 140249710600191, ++STORE, 140249668386816, 140249710600191, ++STORE, 140249668378624, 140249710600191, ++STORE, 140249668362240, 140249710600191, ++STORE, 140249668354048, 140249710600191, ++STORE, 140249668337664, 140249710600191, ++STORE, 140249668329472, 140249710600191, ++STORE, 140249668296704, 140249710600191, ++STORE, 140249668288512, 140249710600191, ++STORE, 140249668272128, 140249710600191, ++STORE, 140249668263936, 140249710600191, ++STORE, 140249668247552, 140249710600191, ++STORE, 140249668239360, 140249710600191, ++STORE, 140249668222976, 140249710600191, ++STORE, 140249668214784, 140249710600191, ++STORE, 140249668182016, 140249710600191, ++STORE, 140249668173824, 140249710600191, ++STORE, 140249668157440, 140249710600191, ++STORE, 140249668149248, 140249710600191, ++STORE, 140249668132864, 140249710600191, ++STORE, 140249668124672, 140249710600191, ++STORE, 140249668108288, 140249710600191, ++STORE, 140249668100096, 140249710600191, ++STORE, 140249668067328, 140249710600191, ++STORE, 140249668059136, 140249710600191, ++STORE, 140249668042752, 140249710600191, ++STORE, 140249668034560, 140249710600191, ++STORE, 140249668018176, 140249710600191, ++STORE, 140249668009984, 140249710600191, ++STORE, 140249667993600, 140249710600191, ++STORE, 140249667985408, 140249710600191, ++STORE, 140249667952640, 140249710600191, ++STORE, 140249667944448, 140249710600191, ++STORE, 140249667928064, 140249710600191, ++STORE, 140249667919872, 140249710600191, ++STORE, 140249667903488, 140249710600191, ++STORE, 140249667895296, 140249710600191, ++STORE, 140249667878912, 140249710600191, ++STORE, 140249667870720, 140249710600191, ++STORE, 140249667837952, 140249710600191, ++STORE, 140249667829760, 140249710600191, ++STORE, 140249667813376, 140249710600191, ++STORE, 140249667805184, 140249710600191, ++STORE, 140249667788800, 140249710600191, ++STORE, 140249667780608, 140249710600191, ++STORE, 140249667764224, 140249710600191, ++STORE, 140249667756032, 140249710600191, ++STORE, 140249667723264, 140249710600191, ++STORE, 140249667715072, 140249710600191, ++STORE, 140249667698688, 140249710600191, ++STORE, 140249667690496, 140249710600191, ++STORE, 140249667674112, 140249710600191, ++STORE, 140249667665920, 140249710600191, ++STORE, 140249667649536, 140249710600191, ++STORE, 140249667641344, 140249710600191, ++STORE, 140249667608576, 140249710600191, ++STORE, 140249667600384, 140249710600191, ++STORE, 140249667584000, 140249710600191, ++STORE, 140249667575808, 140249710600191, ++STORE, 140249667559424, 140249710600191, ++STORE, 140249667551232, 140249710600191, ++STORE, 140249667534848, 140249710600191, ++STORE, 140249667526656, 140249710600191, ++STORE, 140249667493888, 140249710600191, ++STORE, 140249667485696, 140249710600191, ++STORE, 140249667469312, 140249710600191, ++STORE, 140249667461120, 140249710600191, ++STORE, 140249667444736, 140249710600191, ++STORE, 140249667436544, 140249710600191, ++STORE, 140249667420160, 140249710600191, ++STORE, 140249665323008, 140249710600191, ++STORE, 140249665314816, 140249710600191, ++STORE, 140249665282048, 140249710600191, ++STORE, 140249665273856, 140249710600191, ++STORE, 140249665257472, 140249710600191, ++STORE, 140249665249280, 140249710600191, ++STORE, 140249665232896, 140249710600191, ++STORE, 140249665224704, 140249710600191, ++STORE, 140249665208320, 140249710600191, ++STORE, 140249665200128, 140249710600191, ++STORE, 140249665167360, 140249710600191, ++STORE, 140249665159168, 140249710600191, ++STORE, 140249665142784, 140249710600191, ++STORE, 140249665134592, 140249710600191, ++STORE, 140249665118208, 140249710600191, ++STORE, 140249665110016, 140249710600191, ++STORE, 140249665093632, 140249710600191, ++STORE, 140249665085440, 140249710600191, ++STORE, 140249665052672, 140249710600191, ++STORE, 140249665044480, 140249710600191, ++STORE, 140249665028096, 140249710600191, ++STORE, 140249665019904, 140249710600191, ++STORE, 140249665003520, 140249710600191, ++STORE, 140249664995328, 140249710600191, ++STORE, 140249664978944, 140249710600191, ++STORE, 140249664970752, 140249710600191, ++STORE, 140249664937984, 140249710600191, ++STORE, 140249664929792, 140249710600191, ++STORE, 140249664913408, 140249710600191, ++STORE, 140249664905216, 140249710600191, ++STORE, 140249664888832, 140249710600191, ++STORE, 140249664880640, 140249710600191, ++STORE, 140249664864256, 140249710600191, ++STORE, 140249664856064, 140249710600191, ++STORE, 140249664823296, 140249710600191, ++STORE, 140249664815104, 140249710600191, ++STORE, 140249664798720, 140249710600191, ++STORE, 140249664790528, 140249710600191, ++STORE, 140249664774144, 140249710600191, ++STORE, 140249664765952, 140249710600191, ++STORE, 140249664749568, 140249710600191, ++STORE, 140249664741376, 140249710600191, ++STORE, 140249664708608, 140249710600191, ++STORE, 140249664700416, 140249710600191, ++STORE, 140249664684032, 140249710600191, ++STORE, 140249664675840, 140249710600191, ++STORE, 140249664659456, 140249710600191, ++STORE, 140249664651264, 140249710600191, ++STORE, 140249664634880, 140249710600191, ++STORE, 140249664626688, 140249710600191, ++STORE, 140249664593920, 140249710600191, ++STORE, 140249664585728, 140249710600191, ++STORE, 140249664569344, 140249710600191, ++STORE, 140249664561152, 140249710600191, ++STORE, 140249664544768, 140249710600191, ++STORE, 140249664536576, 140249710600191, ++STORE, 140249664520192, 140249710600191, ++STORE, 140249664512000, 140249710600191, ++STORE, 140249664479232, 140249710600191, ++STORE, 140249664471040, 140249710600191, ++STORE, 140249664454656, 140249710600191, ++STORE, 140249664446464, 140249710600191, ++STORE, 140249664430080, 140249710600191, ++STORE, 140249664421888, 140249710600191, ++STORE, 140249664405504, 140249710600191, ++STORE, 140249664397312, 140249710600191, ++STORE, 140249664364544, 140249710600191, ++STORE, 140249664356352, 140249710600191, ++STORE, 140249664339968, 140249710600191, ++STORE, 140249664331776, 140249710600191, ++STORE, 140249664315392, 140249710600191, ++STORE, 140249664307200, 140249710600191, ++STORE, 140249664290816, 140249710600191, ++STORE, 140249664282624, 140249710600191, ++STORE, 140249664249856, 140249710600191, ++STORE, 140249664241664, 140249710600191, ++STORE, 140249664225280, 140249710600191, ++STORE, 140249664217088, 140249710600191, ++STORE, 140249664200704, 140249710600191, ++STORE, 140249664192512, 140249710600191, ++STORE, 140249664176128, 140249710600191, ++STORE, 140249664167936, 140249710600191, ++STORE, 140249664135168, 140249710600191, ++STORE, 140249664126976, 140249710600191, ++STORE, 140249664110592, 140249710600191, ++STORE, 140249664102400, 140249710600191, ++STORE, 140249664086016, 140249710600191, ++STORE, 140249664077824, 140249710600191, ++STORE, 140249664061440, 140249710600191, ++STORE, 140249664053248, 140249710600191, ++STORE, 140249664020480, 140249710600191, ++STORE, 140249664012288, 140249710600191, ++STORE, 140249663995904, 140249710600191, ++STORE, 140249663987712, 140249710600191, ++STORE, 140249663971328, 140249710600191, ++STORE, 140249663963136, 140249710600191, ++STORE, 140249663946752, 140249710600191, ++STORE, 140249663938560, 140249710600191, ++STORE, 140249663905792, 140249710600191, ++STORE, 140249663897600, 140249710600191, ++STORE, 140249663881216, 140249710600191, ++STORE, 140249663873024, 140249710600191, ++STORE, 140249663856640, 140249710600191, ++STORE, 140249663848448, 140249710600191, ++STORE, 140249663832064, 140249710600191, ++STORE, 140249663823872, 140249710600191, ++STORE, 140249663791104, 140249710600191, ++STORE, 140249663782912, 140249710600191, ++STORE, 140249663766528, 140249710600191, ++STORE, 140249663758336, 140249710600191, ++STORE, 140249663741952, 140249710600191, ++STORE, 140249663733760, 140249710600191, ++STORE, 140249663717376, 140249710600191, ++STORE, 140249663709184, 140249710600191, ++STORE, 140249663676416, 140249710600191, ++STORE, 140249663668224, 140249710600191, ++STORE, 140249663651840, 140249710600191, ++STORE, 140249663643648, 140249710600191, ++STORE, 140249663627264, 140249710600191, ++STORE, 33853440, 38526975, ++STORE, 140249663619072, 140249710600191, ++STORE, 140249663602688, 140249710600191, ++STORE, 140249661505536, 140249710600191, ++STORE, 140249661497344, 140249710600191, ++STORE, 140249661464576, 140249710600191, ++STORE, 140249661456384, 140249710600191, ++STORE, 140249661440000, 140249710600191, ++STORE, 140249661431808, 140249710600191, ++STORE, 140249661415424, 140249710600191, ++STORE, 140249661407232, 140249710600191, ++STORE, 140249661390848, 140249710600191, ++STORE, 140249661382656, 140249710600191, ++STORE, 140249661349888, 140249710600191, ++STORE, 140249661341696, 140249710600191, ++STORE, 140249661325312, 140249710600191, ++STORE, 140249661317120, 140249710600191, ++STORE, 140249661300736, 140249710600191, ++STORE, 140249661292544, 140249710600191, ++STORE, 140249661276160, 140249710600191, ++STORE, 140249661267968, 140249710600191, ++STORE, 140249661235200, 140249710600191, ++STORE, 140249661227008, 140249710600191, ++STORE, 140249661210624, 140249710600191, ++STORE, 140249661202432, 140249710600191, ++STORE, 140249661186048, 140249710600191, ++STORE, 140249661177856, 140249710600191, ++STORE, 140249661161472, 140249710600191, ++STORE, 140249661153280, 140249710600191, ++STORE, 140249661120512, 140249710600191, ++STORE, 140249661112320, 140249710600191, ++STORE, 140249661095936, 140249710600191, ++STORE, 140249661087744, 140249710600191, ++STORE, 140249661071360, 140249710600191, ++STORE, 140249661063168, 140249710600191, ++STORE, 140249661046784, 140249710600191, ++STORE, 140249661038592, 140249710600191, ++STORE, 140249661005824, 140249710600191, ++STORE, 140249660997632, 140249710600191, ++STORE, 140249660981248, 140249710600191, ++STORE, 140249660973056, 140249710600191, ++STORE, 140249660956672, 140249710600191, ++STORE, 140249660948480, 140249710600191, ++STORE, 140249660932096, 140249710600191, ++STORE, 140249660923904, 140249710600191, ++STORE, 140249660891136, 140249710600191, ++STORE, 140249660882944, 140249710600191, ++STORE, 140249660866560, 140249710600191, ++STORE, 140249660858368, 140249710600191, ++STORE, 140249660841984, 140249710600191, ++STORE, 140249660833792, 140249710600191, ++STORE, 140249660817408, 140249710600191, ++STORE, 140249660809216, 140249710600191, ++STORE, 140249660776448, 140249710600191, ++STORE, 140249660768256, 140249710600191, ++STORE, 140249660751872, 140249710600191, ++STORE, 140249660743680, 140249710600191, ++STORE, 140249660727296, 140249710600191, ++STORE, 140249660719104, 140249710600191, ++STORE, 140249660702720, 140249710600191, ++STORE, 140249660694528, 140249710600191, ++STORE, 140249660661760, 140249710600191, ++STORE, 140249660653568, 140249710600191, ++STORE, 140249660637184, 140249710600191, ++STORE, 140249660628992, 140249710600191, ++STORE, 140249660612608, 140249710600191, ++STORE, 140249660604416, 140249710600191, ++STORE, 140249660588032, 140249710600191, ++STORE, 140249660579840, 140249710600191, ++STORE, 140249660547072, 140249710600191, ++STORE, 140249660538880, 140249710600191, ++STORE, 140249660522496, 140249710600191, ++STORE, 140249660514304, 140249710600191, ++STORE, 140249660497920, 140249710600191, ++STORE, 140249660489728, 140249710600191, ++STORE, 140249660473344, 140249710600191, ++STORE, 140249660465152, 140249710600191, ++STORE, 140249660432384, 140249710600191, ++STORE, 140249660424192, 140249710600191, ++STORE, 140249660407808, 140249710600191, ++STORE, 140249660399616, 140249710600191, ++STORE, 140249660383232, 140249710600191, ++STORE, 140249660375040, 140249710600191, ++STORE, 140249660358656, 140249710600191, ++STORE, 140249660350464, 140249710600191, ++STORE, 140249660317696, 140249710600191, ++STORE, 140249660309504, 140249710600191, ++STORE, 140249660293120, 140249710600191, ++STORE, 140249660284928, 140249710600191, ++STORE, 140249660268544, 140249710600191, ++STORE, 140249660260352, 140249710600191, ++STORE, 140249660243968, 140249710600191, ++STORE, 140249660235776, 140249710600191, ++STORE, 140249660203008, 140249710600191, ++STORE, 140249660194816, 140249710600191, ++STORE, 140249660178432, 140249710600191, ++STORE, 140249660170240, 140249710600191, ++STORE, 140249660153856, 140249710600191, ++STORE, 140249660145664, 140249710600191, ++STORE, 140249660129280, 140249710600191, ++STORE, 140249660121088, 140249710600191, ++STORE, 140249660088320, 140249710600191, ++STORE, 140249660080128, 140249710600191, ++STORE, 140249660063744, 140249710600191, ++STORE, 140249660055552, 140249710600191, ++STORE, 140249660039168, 140249710600191, ++STORE, 140249660030976, 140249710600191, ++STORE, 140249660014592, 140249710600191, ++STORE, 140249660006400, 140249710600191, ++STORE, 140249659973632, 140249710600191, ++STORE, 140249659965440, 140249710600191, ++STORE, 140249659949056, 140249710600191, ++STORE, 140249659940864, 140249710600191, ++STORE, 140249659924480, 140249710600191, ++STORE, 140249659916288, 140249710600191, ++STORE, 140249659899904, 140249710600191, ++STORE, 140249659891712, 140249710600191, ++STORE, 140249659858944, 140249710600191, ++STORE, 140249659850752, 140249710600191, ++STORE, 140249659834368, 140249710600191, ++STORE, 140249659826176, 140249710600191, ++STORE, 140249659809792, 140249710600191, ++STORE, 140249659801600, 140249710600191, ++STORE, 140249659785216, 140249710600191, ++STORE, 140249657688064, 140249710600191, ++STORE, 140249657679872, 140249710600191, ++STORE, 140249657647104, 140249710600191, ++STORE, 140249657638912, 140249710600191, ++STORE, 140249657622528, 140249710600191, ++STORE, 140249657614336, 140249710600191, ++STORE, 140249657597952, 140249710600191, ++STORE, 140249657589760, 140249710600191, ++STORE, 140249657573376, 140249710600191, ++STORE, 140249657565184, 140249710600191, ++STORE, 140249657532416, 140249710600191, ++STORE, 140249657524224, 140249710600191, ++STORE, 140249657507840, 140249710600191, ++STORE, 140249657499648, 140249710600191, ++STORE, 140249657483264, 140249710600191, ++STORE, 140249657475072, 140249710600191, ++STORE, 140249657458688, 140249710600191, ++STORE, 140249657450496, 140249710600191, ++STORE, 140249657417728, 140249710600191, ++STORE, 140249657409536, 140249710600191, ++STORE, 140249657393152, 140249710600191, ++STORE, 140249657384960, 140249710600191, ++STORE, 140249657368576, 140249710600191, ++STORE, 140249657360384, 140249710600191, ++STORE, 140249657344000, 140249710600191, ++STORE, 140249657335808, 140249710600191, ++STORE, 140249657303040, 140249710600191, ++STORE, 140249657294848, 140249710600191, ++STORE, 140249657278464, 140249710600191, ++STORE, 140249657270272, 140249710600191, ++STORE, 140249657253888, 140249710600191, ++STORE, 140249657245696, 140249710600191, ++STORE, 140249657229312, 140249710600191, ++STORE, 140249657221120, 140249710600191, ++STORE, 140249657188352, 140249710600191, ++STORE, 140249657180160, 140249710600191, ++STORE, 140249657163776, 140249710600191, ++STORE, 140249657155584, 140249710600191, ++STORE, 140249657139200, 140249710600191, ++STORE, 140249657131008, 140249710600191, ++STORE, 140249657114624, 140249710600191, ++STORE, 140249657106432, 140249710600191, ++STORE, 140249657073664, 140249710600191, ++STORE, 140249657065472, 140249710600191, ++STORE, 140249657049088, 140249710600191, ++STORE, 140249657040896, 140249710600191, ++STORE, 140249657024512, 140249710600191, ++STORE, 140249657016320, 140249710600191, ++STORE, 140249656999936, 140249710600191, ++STORE, 140249656991744, 140249710600191, ++STORE, 140249656958976, 140249710600191, ++STORE, 140249656950784, 140249710600191, ++STORE, 140249656934400, 140249710600191, ++STORE, 140249656926208, 140249710600191, ++STORE, 140249656909824, 140249710600191, ++STORE, 140249656901632, 140249710600191, ++STORE, 140249656885248, 140249710600191, ++STORE, 140249656877056, 140249710600191, ++STORE, 140249656844288, 140249710600191, ++STORE, 140249656836096, 140249710600191, ++STORE, 140249656819712, 140249710600191, ++STORE, 140249656811520, 140249710600191, ++STORE, 140249656795136, 140249710600191, ++STORE, 33853440, 38662143, ++STORE, 140249656786944, 140249710600191, ++STORE, 140249656770560, 140249710600191, ++STORE, 140249656762368, 140249710600191, ++STORE, 140249656729600, 140249710600191, ++STORE, 140249656721408, 140249710600191, ++STORE, 140249656705024, 140249710600191, ++STORE, 140249656696832, 140249710600191, ++STORE, 140249656680448, 140249710600191, ++STORE, 140249656672256, 140249710600191, ++STORE, 140249656655872, 140249710600191, ++STORE, 140249656647680, 140249710600191, ++STORE, 140249656614912, 140249710600191, ++STORE, 140249656606720, 140249710600191, ++STORE, 140249656590336, 140249710600191, ++STORE, 140249656582144, 140249710600191, ++STORE, 140249656565760, 140249710600191, ++STORE, 140249656557568, 140249710600191, ++STORE, 140249656541184, 140249710600191, ++STORE, 140249656532992, 140249710600191, ++STORE, 140249656500224, 140249710600191, ++STORE, 140249656492032, 140249710600191, ++STORE, 140249656475648, 140249710600191, ++STORE, 140249656467456, 140249710600191, ++STORE, 140249656451072, 140249710600191, ++STORE, 140249656442880, 140249710600191, ++STORE, 140249656426496, 140249710600191, ++STORE, 140249656418304, 140249710600191, ++STORE, 140249656385536, 140249710600191, ++STORE, 140249656377344, 140249710600191, ++STORE, 140249656360960, 140249710600191, ++STORE, 140249656352768, 140249710600191, ++STORE, 140249656336384, 140249710600191, ++STORE, 140249656328192, 140249710600191, ++STORE, 140249656311808, 140249710600191, ++STORE, 140249656303616, 140249710600191, ++STORE, 140249656270848, 140249710600191, ++STORE, 140249656262656, 140249710600191, ++STORE, 140249656246272, 140249710600191, ++STORE, 140249656238080, 140249710600191, ++STORE, 140249656221696, 140249710600191, ++STORE, 140249656213504, 140249710600191, ++STORE, 140249656197120, 140249710600191, ++STORE, 140249656188928, 140249710600191, ++STORE, 140249656156160, 140249710600191, ++STORE, 140249656147968, 140249710600191, ++STORE, 140249656131584, 140249710600191, ++STORE, 140249656123392, 140249710600191, ++STORE, 140249656107008, 140249710600191, ++STORE, 140249656098816, 140249710600191, ++STORE, 140249656082432, 140249710600191, ++STORE, 140249656074240, 140249710600191, ++STORE, 140249656041472, 140249710600191, ++STORE, 140249656033280, 140249710600191, ++STORE, 140249656016896, 140249710600191, ++STORE, 140249656008704, 140249710600191, ++STORE, 140249655992320, 140249710600191, ++STORE, 140249655984128, 140249710600191, ++STORE, 140249655967744, 140249710600191, ++STORE, 140249653870592, 140249710600191, ++STORE, 140249653862400, 140249710600191, ++STORE, 140249653829632, 140249710600191, ++STORE, 140249653821440, 140249710600191, ++STORE, 140249653805056, 140249710600191, ++STORE, 140249653796864, 140249710600191, ++STORE, 140249653780480, 140249710600191, ++STORE, 140249653772288, 140249710600191, ++STORE, 140249653755904, 140249710600191, ++STORE, 140249652703232, 140249710600191, ++SNULL, 140249682087935, 140249710600191, ++STORE, 140249652703232, 140249682087935, ++STORE, 140249682087936, 140249710600191, ++ }; + -+/* to protect the working set of the last N jiffies */ -+static unsigned long lru_gen_min_ttl __read_mostly; ++ unsigned long set26[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140729464770560, 140737488351231, ++SNULL, 140729464774655, 140737488351231, ++STORE, 140729464770560, 140729464774655, ++STORE, 140729464639488, 140729464774655, ++STORE, 4194304, 5066751, ++STORE, 7159808, 7172095, ++STORE, 7172096, 7180287, ++STORE, 140729465114624, 140729465118719, ++STORE, 140729465102336, 140729465114623, ++STORE, 30867456, 30875647, ++STORE, 30867456, 31010815, ++STORE, 140109040988160, 140109042671615, ++STORE, 140109040959488, 140109040988159, ++STORE, 140109040943104, 140109040959487, ++ERASE, 140109040943104, 140109040959487, ++STORE, 140109040840704, 140109040959487, ++ERASE, 140109040840704, 140109040959487, ++STORE, 140109040951296, 140109040959487, ++ERASE, 140109040951296, 140109040959487, ++STORE, 140109040955392, 140109040959487, ++ERASE, 140109040955392, 140109040959487, ++ }; ++ unsigned long set27[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140726128070656, 140737488351231, ++SNULL, 140726128074751, 140737488351231, ++STORE, 140726128070656, 140726128074751, ++STORE, 140726127939584, 140726128074751, ++STORE, 94478497189888, 94478499303423, ++SNULL, 94478497202175, 94478499303423, ++STORE, 94478497189888, 94478497202175, ++STORE, 94478497202176, 94478499303423, ++ERASE, 94478497202176, 94478499303423, ++STORE, 94478499295232, 94478499303423, ++STORE, 140415605723136, 140415607975935, ++SNULL, 140415605866495, 140415607975935, ++STORE, 140415605723136, 140415605866495, ++STORE, 140415605866496, 140415607975935, ++ERASE, 140415605866496, 140415607975935, ++STORE, 140415607963648, 140415607971839, ++STORE, 140415607971840, 140415607975935, ++STORE, 140726130024448, 140726130028543, ++STORE, 140726130012160, 140726130024447, ++STORE, 140415607934976, 140415607963647, ++STORE, 140415607926784, 140415607934975, ++STORE, 140415603245056, 140415605723135, ++SNULL, 140415603245056, 140415603613695, ++STORE, 140415603613696, 140415605723135, ++STORE, 140415603245056, 140415603613695, ++SNULL, 140415605710847, 140415605723135, ++STORE, 140415603613696, 140415605710847, ++STORE, 140415605710848, 140415605723135, ++ERASE, 140415605710848, 140415605723135, ++STORE, 140415605710848, 140415605723135, ++STORE, 140415599370240, 140415603245055, ++SNULL, 140415599370240, 140415601111039, ++STORE, 140415601111040, 140415603245055, ++STORE, 140415599370240, 140415601111039, ++SNULL, 140415603208191, 140415603245055, ++STORE, 140415601111040, 140415603208191, ++STORE, 140415603208192, 140415603245055, ++ERASE, 140415603208192, 140415603245055, ++STORE, 140415603208192, 140415603245055, ++STORE, 140415595692032, 140415599370239, ++SNULL, 140415595692032, 140415597207551, ++STORE, 140415597207552, 140415599370239, ++STORE, 140415595692032, 140415597207551, ++SNULL, 140415599304703, 140415599370239, ++STORE, 140415597207552, 140415599304703, ++STORE, 140415599304704, 140415599370239, ++SNULL, 140415599304704, 140415599353855, ++STORE, 140415599353856, 140415599370239, ++STORE, 140415599304704, 140415599353855, ++ERASE, 140415599304704, 140415599353855, ++STORE, 140415599304704, 140415599353855, ++ERASE, 140415599353856, 140415599370239, ++STORE, 140415599353856, 140415599370239, ++STORE, 140415593500672, 140415595692031, ++SNULL, 140415593500672, 140415593590783, ++STORE, 140415593590784, 140415595692031, ++STORE, 140415593500672, 140415593590783, ++SNULL, 140415595683839, 140415595692031, ++STORE, 140415593590784, 140415595683839, ++STORE, 140415595683840, 140415595692031, ++ERASE, 140415595683840, 140415595692031, ++STORE, 140415595683840, 140415595692031, ++STORE, 140415589703680, 140415593500671, ++SNULL, 140415589703680, 140415591362559, ++STORE, 140415591362560, 140415593500671, ++STORE, 140415589703680, 140415591362559, ++SNULL, 140415593459711, 140415593500671, ++STORE, 140415591362560, 140415593459711, ++STORE, 140415593459712, 140415593500671, ++SNULL, 140415593459712, 140415593484287, ++STORE, 140415593484288, 140415593500671, ++STORE, 140415593459712, 140415593484287, ++ERASE, 140415593459712, 140415593484287, ++STORE, 140415593459712, 140415593484287, ++ERASE, 140415593484288, 140415593500671, ++STORE, 140415593484288, 140415593500671, ++STORE, 140415587590144, 140415589703679, ++SNULL, 140415587590144, 140415587602431, ++STORE, 140415587602432, 140415589703679, ++STORE, 140415587590144, 140415587602431, ++SNULL, 140415589695487, 140415589703679, ++STORE, 140415587602432, 140415589695487, ++STORE, 140415589695488, 140415589703679, ++ERASE, 140415589695488, 140415589703679, ++STORE, 140415589695488, 140415589703679, ++STORE, 140415607918592, 140415607934975, ++STORE, 140415585398784, 140415587590143, ++SNULL, 140415585398784, 140415585480703, ++STORE, 140415585480704, 140415587590143, ++STORE, 140415585398784, 140415585480703, ++SNULL, 140415587573759, 140415587590143, ++STORE, 140415585480704, 140415587573759, ++STORE, 140415587573760, 140415587590143, ++SNULL, 140415587573760, 140415587581951, ++STORE, 140415587581952, 140415587590143, ++STORE, 140415587573760, 140415587581951, ++ERASE, 140415587573760, 140415587581951, ++STORE, 140415587573760, 140415587581951, ++ERASE, 140415587581952, 140415587590143, ++STORE, 140415587581952, 140415587590143, ++STORE, 140415583182848, 140415585398783, ++SNULL, 140415583182848, 140415583281151, ++STORE, 140415583281152, 140415585398783, ++STORE, 140415583182848, 140415583281151, ++SNULL, 140415585374207, 140415585398783, ++STORE, 140415583281152, 140415585374207, ++STORE, 140415585374208, 140415585398783, ++SNULL, 140415585374208, 140415585382399, ++STORE, 140415585382400, 140415585398783, ++STORE, 140415585374208, 140415585382399, ++ERASE, 140415585374208, 140415585382399, ++STORE, 140415585374208, 140415585382399, ++ERASE, 140415585382400, 140415585398783, ++STORE, 140415585382400, 140415585398783, ++STORE, 140415580979200, 140415583182847, ++SNULL, 140415580979200, 140415581081599, ++STORE, 140415581081600, 140415583182847, ++STORE, 140415580979200, 140415581081599, ++SNULL, 140415583174655, 140415583182847, ++STORE, 140415581081600, 140415583174655, ++STORE, 140415583174656, 140415583182847, ++ERASE, 140415583174656, 140415583182847, ++STORE, 140415583174656, 140415583182847, ++STORE, 140415578816512, 140415580979199, ++SNULL, 140415578816512, 140415578877951, ++STORE, 140415578877952, 140415580979199, ++STORE, 140415578816512, 140415578877951, ++SNULL, 140415580971007, 140415580979199, ++STORE, 140415578877952, 140415580971007, ++STORE, 140415580971008, 140415580979199, ++ERASE, 140415580971008, 140415580979199, ++STORE, 140415580971008, 140415580979199, ++STORE, 140415576563712, 140415578816511, ++SNULL, 140415576563712, 140415576715263, ++STORE, 140415576715264, 140415578816511, ++STORE, 140415576563712, 140415576715263, ++SNULL, 140415578808319, 140415578816511, ++STORE, 140415576715264, 140415578808319, ++STORE, 140415578808320, 140415578816511, ++ERASE, 140415578808320, 140415578816511, ++STORE, 140415578808320, 140415578816511, ++STORE, 140415574392832, 140415576563711, ++SNULL, 140415574392832, 140415574462463, ++STORE, 140415574462464, 140415576563711, ++STORE, 140415574392832, 140415574462463, ++SNULL, 140415576555519, 140415576563711, ++STORE, 140415574462464, 140415576555519, ++STORE, 140415576555520, 140415576563711, ++ERASE, 140415576555520, 140415576563711, ++STORE, 140415576555520, 140415576563711, ++STORE, 140415607910400, 140415607934975, ++STORE, 140415571230720, 140415574392831, ++SNULL, 140415571230720, 140415572291583, ++STORE, 140415572291584, 140415574392831, ++STORE, 140415571230720, 140415572291583, ++SNULL, 140415574384639, 140415574392831, ++STORE, 140415572291584, 140415574384639, ++STORE, 140415574384640, 140415574392831, ++ERASE, 140415574384640, 140415574392831, ++STORE, 140415574384640, 140415574392831, ++STORE, 140415607902208, 140415607934975, ++SNULL, 140415593476095, 140415593484287, ++STORE, 140415593459712, 140415593476095, ++STORE, 140415593476096, 140415593484287, ++SNULL, 140415574388735, 140415574392831, ++STORE, 140415574384640, 140415574388735, ++STORE, 140415574388736, 140415574392831, ++SNULL, 140415576559615, 140415576563711, ++STORE, 140415576555520, 140415576559615, ++STORE, 140415576559616, 140415576563711, ++SNULL, 140415589699583, 140415589703679, ++STORE, 140415589695488, 140415589699583, ++STORE, 140415589699584, 140415589703679, ++SNULL, 140415585378303, 140415585382399, ++STORE, 140415585374208, 140415585378303, ++STORE, 140415585378304, 140415585382399, ++SNULL, 140415578812415, 140415578816511, ++STORE, 140415578808320, 140415578812415, ++STORE, 140415578812416, 140415578816511, ++SNULL, 140415580975103, 140415580979199, ++STORE, 140415580971008, 140415580975103, ++STORE, 140415580975104, 140415580979199, ++SNULL, 140415583178751, 140415583182847, ++STORE, 140415583174656, 140415583178751, ++STORE, 140415583178752, 140415583182847, ++SNULL, 140415587577855, 140415587581951, ++STORE, 140415587573760, 140415587577855, ++STORE, 140415587577856, 140415587581951, ++SNULL, 140415595687935, 140415595692031, ++STORE, 140415595683840, 140415595687935, ++STORE, 140415595687936, 140415595692031, ++STORE, 140415607894016, 140415607934975, ++SNULL, 140415599345663, 140415599353855, ++STORE, 140415599304704, 140415599345663, ++STORE, 140415599345664, 140415599353855, ++SNULL, 140415603240959, 140415603245055, ++STORE, 140415603208192, 140415603240959, ++STORE, 140415603240960, 140415603245055, ++SNULL, 140415605719039, 140415605723135, ++STORE, 140415605710848, 140415605719039, ++STORE, 140415605719040, 140415605723135, ++SNULL, 94478499299327, 94478499303423, ++STORE, 94478499295232, 94478499299327, ++STORE, 94478499299328, 94478499303423, ++SNULL, 140415607967743, 140415607971839, ++STORE, 140415607963648, 140415607967743, ++STORE, 140415607967744, 140415607971839, ++ERASE, 140415607934976, 140415607963647, ++STORE, 94478511173632, 94478511378431, ++STORE, 140415606210560, 140415607894015, ++STORE, 140415607934976, 140415607963647, ++STORE, 94478511173632, 94478511513599, ++STORE, 94478511173632, 94478511648767, ++SNULL, 94478511615999, 94478511648767, ++STORE, 94478511173632, 94478511615999, ++STORE, 94478511616000, 94478511648767, ++ERASE, 94478511616000, 94478511648767, ++STORE, 94478511173632, 94478511751167, ++SNULL, 94478511747071, 94478511751167, ++STORE, 94478511173632, 94478511747071, ++STORE, 94478511747072, 94478511751167, ++ERASE, 94478511747072, 94478511751167, ++STORE, 94478511173632, 94478511882239, ++SNULL, 94478511878143, 94478511882239, ++STORE, 94478511173632, 94478511878143, ++STORE, 94478511878144, 94478511882239, ++ERASE, 94478511878144, 94478511882239, ++STORE, 94478511173632, 94478512013311, ++SNULL, 94478512009215, 94478512013311, ++STORE, 94478511173632, 94478512009215, ++STORE, 94478512009216, 94478512013311, ++ERASE, 94478512009216, 94478512013311, ++STORE, 94478511173632, 94478512144383, ++STORE, 94478511173632, 94478512279551, ++STORE, 140415606181888, 140415606210559, ++STORE, 140415569100800, 140415571230719, ++SNULL, 140415569100800, 140415569129471, ++STORE, 140415569129472, 140415571230719, ++STORE, 140415569100800, 140415569129471, ++SNULL, 140415571222527, 140415571230719, ++STORE, 140415569129472, 140415571222527, ++STORE, 140415571222528, 140415571230719, ++ERASE, 140415571222528, 140415571230719, ++STORE, 140415571222528, 140415571230719, ++STORE, 140415566905344, 140415569100799, ++SNULL, 140415566905344, 140415566987263, ++STORE, 140415566987264, 140415569100799, ++STORE, 140415566905344, 140415566987263, ++SNULL, 140415569084415, 140415569100799, ++STORE, 140415566987264, 140415569084415, ++STORE, 140415569084416, 140415569100799, ++SNULL, 140415569084416, 140415569092607, ++STORE, 140415569092608, 140415569100799, ++STORE, 140415569084416, 140415569092607, ++ERASE, 140415569084416, 140415569092607, ++STORE, 140415569084416, 140415569092607, ++ERASE, 140415569092608, 140415569100799, ++STORE, 140415569092608, 140415569100799, ++SNULL, 140415569088511, 140415569092607, ++STORE, 140415569084416, 140415569088511, ++STORE, 140415569088512, 140415569092607, ++SNULL, 140415571226623, 140415571230719, ++STORE, 140415571222528, 140415571226623, ++STORE, 140415571226624, 140415571230719, ++ERASE, 140415606181888, 140415606210559, ++STORE, 140415606181888, 140415606210559, ++STORE, 140415564759040, 140415566905343, ++SNULL, 140415564759040, 140415564804095, ++STORE, 140415564804096, 140415566905343, ++STORE, 140415564759040, 140415564804095, ++SNULL, 140415566897151, 140415566905343, ++STORE, 140415564804096, 140415566897151, ++STORE, 140415566897152, 140415566905343, ++ERASE, 140415566897152, 140415566905343, ++STORE, 140415566897152, 140415566905343, ++STORE, 140415562588160, 140415564759039, ++SNULL, 140415562588160, 140415562629119, ++STORE, 140415562629120, 140415564759039, ++STORE, 140415562588160, 140415562629119, ++SNULL, 140415564726271, 140415564759039, ++STORE, 140415562629120, 140415564726271, ++STORE, 140415564726272, 140415564759039, ++SNULL, 140415564726272, 140415564734463, ++STORE, 140415564734464, 140415564759039, ++STORE, 140415564726272, 140415564734463, ++ERASE, 140415564726272, 140415564734463, ++STORE, 140415564726272, 140415564734463, ++ERASE, 140415564734464, 140415564759039, ++STORE, 140415564734464, 140415564759039, ++SNULL, 140415564730367, 140415564734463, ++STORE, 140415564726272, 140415564730367, ++STORE, 140415564730368, 140415564734463, ++SNULL, 140415566901247, 140415566905343, ++STORE, 140415566897152, 140415566901247, ++STORE, 140415566901248, 140415566905343, ++ERASE, 140415606181888, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415605944320, 140415606210559, ++ERASE, 140415605944320, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 94478511173632, 94478512414719, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 140415606206464, 140415606210559, ++ERASE, 140415606206464, 140415606210559, ++STORE, 94478511173632, 94478512652287, ++STORE, 94478511173632, 94478512787455, ++STORE, 94478511173632, 94478512922623, ++STORE, 94478511173632, 94478513057791, ++STORE, 140415537422336, 140415562588159, ++STORE, 94478511173632, 94478513192959, ++STORE, 94478511173632, 94478513356799, ++STORE, 94478511173632, 94478513491967, ++STORE, 94478511173632, 94478513627135, ++STORE, 94478511173632, 94478513790975, ++STORE, 94478511173632, 94478513926143, ++STORE, 94478511173632, 94478514061311, ++STORE, 94478511173632, 94478514196479, ++STORE, 94478511173632, 94478514331647, ++STORE, 94478511173632, 94478514606079, ++STORE, 94478511173632, 94478514741247, ++STORE, 94478511173632, 94478514876415, ++STORE, 94478511173632, 94478515011583, ++STORE, 94478511173632, 94478515146751, ++STORE, 94478511173632, 94478515281919, ++STORE, 94478511173632, 94478515474431, ++STORE, 94478511173632, 94478515609599, ++STORE, 94478511173632, 94478515744767, ++STORE, 140415536922624, 140415562588159, ++STORE, 94478511173632, 94478515879935, ++STORE, 94478511173632, 94478516015103, ++STORE, 94478511173632, 94478516150271, ++STORE, 94478511173632, 94478516285439, ++STORE, 94478511173632, 94478516420607, ++STORE, 94478511173632, 94478516555775, ++STORE, 94478511173632, 94478516690943, ++STORE, 94478511173632, 94478516826111, ++STORE, 94478511173632, 94478516961279, ++STORE, 94478511173632, 94478517231615, ++STORE, 94478511173632, 94478517366783, ++STORE, 94478511173632, 94478517501951, ++STORE, 94478511173632, 94478517637119, ++STORE, 94478511173632, 94478517772287, ++STORE, 94478511173632, 94478517907455, ++STORE, 94478511173632, 94478518042623, ++STORE, 94478511173632, 94478518177791, ++STORE, 94478511173632, 94478518312959, ++STORE, 94478511173632, 94478518448127, ++STORE, 140415535910912, 140415562588159, ++SNULL, 140415536922623, 140415562588159, ++STORE, 140415535910912, 140415536922623, ++STORE, 140415536922624, 140415562588159, ++SNULL, 140415536922624, 140415537422335, ++STORE, 140415537422336, 140415562588159, ++STORE, 140415536922624, 140415537422335, ++ERASE, 140415536922624, 140415537422335, ++STORE, 94478511173632, 94478518583295, ++STORE, 94478511173632, 94478518718463, ++STORE, 94478511173632, 94478518853631, ++STORE, 94478511173632, 94478518988799, ++STORE, 94478511173632, 94478519123967, ++STORE, 94478511173632, 94478519259135, ++STORE, 140415509696512, 140415535910911, ++ERASE, 140415537422336, 140415562588159, ++STORE, 140415482433536, 140415509696511, ++ }; ++ unsigned long set28[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722475622400, 140737488351231, ++SNULL, 140722475626495, 140737488351231, ++STORE, 140722475622400, 140722475626495, ++STORE, 140722475491328, 140722475626495, ++STORE, 93865834291200, 93865836548095, ++SNULL, 93865834422271, 93865836548095, ++STORE, 93865834291200, 93865834422271, ++STORE, 93865834422272, 93865836548095, ++ERASE, 93865834422272, 93865836548095, ++STORE, 93865836519424, 93865836527615, ++STORE, 93865836527616, 93865836548095, ++STORE, 139918411104256, 139918413357055, ++SNULL, 139918411247615, 139918413357055, ++STORE, 139918411104256, 139918411247615, ++STORE, 139918411247616, 139918413357055, ++ERASE, 139918411247616, 139918413357055, ++STORE, 139918413344768, 139918413352959, ++STORE, 139918413352960, 139918413357055, ++STORE, 140722476642304, 140722476646399, ++STORE, 140722476630016, 140722476642303, ++STORE, 139918413316096, 139918413344767, ++STORE, 139918413307904, 139918413316095, ++STORE, 139918408888320, 139918411104255, ++SNULL, 139918408888320, 139918408986623, ++STORE, 139918408986624, 139918411104255, ++STORE, 139918408888320, 139918408986623, ++SNULL, 139918411079679, 139918411104255, ++STORE, 139918408986624, 139918411079679, ++STORE, 139918411079680, 139918411104255, ++SNULL, 139918411079680, 139918411087871, ++STORE, 139918411087872, 139918411104255, ++STORE, 139918411079680, 139918411087871, ++ERASE, 139918411079680, 139918411087871, ++STORE, 139918411079680, 139918411087871, ++ERASE, 139918411087872, 139918411104255, ++STORE, 139918411087872, 139918411104255, ++STORE, 139918405091328, 139918408888319, ++SNULL, 139918405091328, 139918406750207, ++STORE, 139918406750208, 139918408888319, ++STORE, 139918405091328, 139918406750207, ++SNULL, 139918408847359, 139918408888319, ++STORE, 139918406750208, 139918408847359, ++STORE, 139918408847360, 139918408888319, ++SNULL, 139918408847360, 139918408871935, ++STORE, 139918408871936, 139918408888319, ++STORE, 139918408847360, 139918408871935, ++ERASE, 139918408847360, 139918408871935, ++STORE, 139918408847360, 139918408871935, ++ERASE, 139918408871936, 139918408888319, ++STORE, 139918408871936, 139918408888319, ++STORE, 139918413299712, 139918413316095, ++SNULL, 139918408863743, 139918408871935, ++STORE, 139918408847360, 139918408863743, ++STORE, 139918408863744, 139918408871935, ++SNULL, 139918411083775, 139918411087871, ++STORE, 139918411079680, 139918411083775, ++STORE, 139918411083776, 139918411087871, ++SNULL, 93865836523519, 93865836527615, ++STORE, 93865836519424, 93865836523519, ++STORE, 93865836523520, 93865836527615, ++SNULL, 139918413348863, 139918413352959, ++STORE, 139918413344768, 139918413348863, ++STORE, 139918413348864, 139918413352959, ++ERASE, 139918413316096, 139918413344767, ++STORE, 93865848528896, 93865848664063, ++ }; ++ unsigned long set29[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734467944448, 140737488351231, ++SNULL, 140734467948543, 140737488351231, ++STORE, 140734467944448, 140734467948543, ++STORE, 140734467813376, 140734467948543, ++STORE, 94880407924736, 94880410177535, ++SNULL, 94880408055807, 94880410177535, ++STORE, 94880407924736, 94880408055807, ++STORE, 94880408055808, 94880410177535, ++ERASE, 94880408055808, 94880410177535, ++STORE, 94880410148864, 94880410157055, ++STORE, 94880410157056, 94880410177535, ++STORE, 140143367815168, 140143370067967, ++SNULL, 140143367958527, 140143370067967, ++STORE, 140143367815168, 140143367958527, ++STORE, 140143367958528, 140143370067967, ++ERASE, 140143367958528, 140143370067967, ++STORE, 140143370055680, 140143370063871, ++STORE, 140143370063872, 140143370067967, ++STORE, 140734468329472, 140734468333567, ++STORE, 140734468317184, 140734468329471, ++STORE, 140143370027008, 140143370055679, ++STORE, 140143370018816, 140143370027007, ++STORE, 140143365599232, 140143367815167, ++SNULL, 140143365599232, 140143365697535, ++STORE, 140143365697536, 140143367815167, ++STORE, 140143365599232, 140143365697535, ++SNULL, 140143367790591, 140143367815167, ++STORE, 140143365697536, 140143367790591, ++STORE, 140143367790592, 140143367815167, ++SNULL, 140143367790592, 140143367798783, ++STORE, 140143367798784, 140143367815167, ++STORE, 140143367790592, 140143367798783, ++ERASE, 140143367790592, 140143367798783, ++STORE, 140143367790592, 140143367798783, ++ERASE, 140143367798784, 140143367815167, ++STORE, 140143367798784, 140143367815167, ++STORE, 140143361802240, 140143365599231, ++SNULL, 140143361802240, 140143363461119, ++STORE, 140143363461120, 140143365599231, ++STORE, 140143361802240, 140143363461119, ++SNULL, 140143365558271, 140143365599231, ++STORE, 140143363461120, 140143365558271, ++STORE, 140143365558272, 140143365599231, ++SNULL, 140143365558272, 140143365582847, ++STORE, 140143365582848, 140143365599231, ++STORE, 140143365558272, 140143365582847, ++ERASE, 140143365558272, 140143365582847, ++STORE, 140143365558272, 140143365582847, ++ERASE, 140143365582848, 140143365599231, ++STORE, 140143365582848, 140143365599231, ++STORE, 140143370010624, 140143370027007, ++SNULL, 140143365574655, 140143365582847, ++STORE, 140143365558272, 140143365574655, ++STORE, 140143365574656, 140143365582847, ++SNULL, 140143367794687, 140143367798783, ++STORE, 140143367790592, 140143367794687, ++STORE, 140143367794688, 140143367798783, ++SNULL, 94880410152959, 94880410157055, ++STORE, 94880410148864, 94880410152959, ++STORE, 94880410152960, 94880410157055, ++SNULL, 140143370059775, 140143370063871, ++STORE, 140143370055680, 140143370059775, ++STORE, 140143370059776, 140143370063871, ++ERASE, 140143370027008, 140143370055679, ++STORE, 94880442400768, 94880442535935, ++STORE, 140143353409536, 140143361802239, ++SNULL, 140143353413631, 140143361802239, ++STORE, 140143353409536, 140143353413631, ++STORE, 140143353413632, 140143361802239, ++STORE, 140143345016832, 140143353409535, ++STORE, 140143210799104, 140143345016831, ++SNULL, 140143210799104, 140143239364607, ++STORE, 140143239364608, 140143345016831, ++STORE, 140143210799104, 140143239364607, ++ERASE, 140143210799104, 140143239364607, ++SNULL, 140143306473471, 140143345016831, ++STORE, 140143239364608, 140143306473471, ++STORE, 140143306473472, 140143345016831, ++ERASE, 140143306473472, 140143345016831, ++SNULL, 140143239499775, 140143306473471, ++STORE, 140143239364608, 140143239499775, ++STORE, 140143239499776, 140143306473471, ++SNULL, 140143345020927, 140143353409535, ++STORE, 140143345016832, 140143345020927, ++STORE, 140143345020928, 140143353409535, ++STORE, 140143336624128, 140143345016831, ++SNULL, 140143336628223, 140143345016831, ++STORE, 140143336624128, 140143336628223, ++STORE, 140143336628224, 140143345016831, ++STORE, 140143328231424, 140143336624127, ++SNULL, 140143328235519, 140143336624127, ++STORE, 140143328231424, 140143328235519, ++STORE, 140143328235520, 140143336624127, ++STORE, 140143319838720, 140143328231423, ++SNULL, 140143319842815, 140143328231423, ++STORE, 140143319838720, 140143319842815, ++STORE, 140143319842816, 140143328231423, ++STORE, 140143311446016, 140143319838719, ++STORE, 140143105146880, 140143239364607, ++STORE, 140143096754176, 140143105146879, ++STORE, 140143029645312, 140143096754175, ++ERASE, 140143029645312, 140143096754175, ++STORE, 140142962536448, 140143096754175, ++SNULL, 140142962536448, 140142970929151, ++STORE, 140142970929152, 140143096754175, ++STORE, 140142962536448, 140142970929151, ++ERASE, 140142962536448, 140142970929151, ++STORE, 140142962536448, 140142970929151, ++STORE, 140142828318720, 140142962536447, ++STORE, 140142819926016, 140142828318719, ++SNULL, 140142828318720, 140142836711423, ++STORE, 140142836711424, 140142962536447, ++STORE, 140142828318720, 140142836711423, ++ERASE, 140142828318720, 140142836711423, ++SNULL, 140143172255743, 140143239364607, ++STORE, 140143105146880, 140143172255743, ++STORE, 140143172255744, 140143239364607, ++ERASE, 140143172255744, 140143239364607, ++SNULL, 140143105282047, 140143172255743, ++STORE, 140143105146880, 140143105282047, ++STORE, 140143105282048, 140143172255743, ++SNULL, 140143038038015, 140143096754175, ++STORE, 140142970929152, 140143038038015, ++STORE, 140143038038016, 140143096754175, ++ERASE, 140143038038016, 140143096754175, ++SNULL, 140142971064319, 140143038038015, ++STORE, 140142970929152, 140142971064319, ++STORE, 140142971064320, 140143038038015, ++SNULL, 140142903820287, 140142962536447, ++STORE, 140142836711424, 140142903820287, ++STORE, 140142903820288, 140142962536447, ++ERASE, 140142903820288, 140142962536447, ++SNULL, 140142836846591, 140142903820287, ++STORE, 140142836711424, 140142836846591, ++STORE, 140142836846592, 140142903820287, ++STORE, 140142685708288, 140142819926015, ++SNULL, 140143311450111, 140143319838719, ++STORE, 140143311446016, 140143311450111, ++STORE, 140143311450112, 140143319838719, ++SNULL, 140142962540543, 140142970929151, ++STORE, 140142962536448, 140142962540543, ++STORE, 140142962540544, 140142970929151, ++SNULL, 140142685708288, 140142702493695, ++STORE, 140142702493696, 140142819926015, ++STORE, 140142685708288, 140142702493695, ++ERASE, 140142685708288, 140142702493695, ++SNULL, 140142769602559, 140142819926015, ++STORE, 140142702493696, 140142769602559, ++STORE, 140142769602560, 140142819926015, ++ERASE, 140142769602560, 140142819926015, ++SNULL, 140142702628863, 140142769602559, ++STORE, 140142702493696, 140142702628863, ++STORE, 140142702628864, 140142769602559, ++STORE, 140143230971904, 140143239364607, ++SNULL, 140143230975999, 140143239364607, ++STORE, 140143230971904, 140143230975999, ++STORE, 140143230976000, 140143239364607, ++SNULL, 140143096758271, 140143105146879, ++STORE, 140143096754176, 140143096758271, ++STORE, 140143096758272, 140143105146879, ++STORE, 140143222579200, 140143230971903, ++SNULL, 140143222583295, 140143230971903, ++STORE, 140143222579200, 140143222583295, ++STORE, 140143222583296, 140143230971903, ++STORE, 140143214186496, 140143222579199, ++SNULL, 140142819930111, 140142828318719, ++STORE, 140142819926016, 140142819930111, ++STORE, 140142819930112, 140142828318719, ++STORE, 140143205793792, 140143222579199, ++SNULL, 140143205793792, 140143214186495, ++STORE, 140143214186496, 140143222579199, ++STORE, 140143205793792, 140143214186495, ++SNULL, 140143214190591, 140143222579199, ++STORE, 140143214186496, 140143214190591, ++STORE, 140143214190592, 140143222579199, ++SNULL, 140143205797887, 140143214186495, ++STORE, 140143205793792, 140143205797887, ++STORE, 140143205797888, 140143214186495, ++STORE, 140143197401088, 140143205793791, ++SNULL, 140143197405183, 140143205793791, ++STORE, 140143197401088, 140143197405183, ++STORE, 140143197405184, 140143205793791, ++STORE, 140143189008384, 140143197401087, ++STORE, 140143180615680, 140143197401087, ++STORE, 140143088361472, 140143096754175, ++SNULL, 140143180619775, 140143197401087, ++STORE, 140143180615680, 140143180619775, ++STORE, 140143180619776, 140143197401087, ++SNULL, 140143180619776, 140143189008383, ++STORE, 140143189008384, 140143197401087, ++STORE, 140143180619776, 140143189008383, ++SNULL, 140143189012479, 140143197401087, ++STORE, 140143189008384, 140143189012479, ++STORE, 140143189012480, 140143197401087, ++SNULL, 140143088365567, 140143096754175, ++STORE, 140143088361472, 140143088365567, ++STORE, 140143088365568, 140143096754175, ++STORE, 140143079968768, 140143088361471, ++SNULL, 140143079972863, 140143088361471, ++STORE, 140143079968768, 140143079972863, ++STORE, 140143079972864, 140143088361471, ++STORE, 140143071576064, 140143079968767, ++SNULL, 140143071580159, 140143079968767, ++STORE, 140143071576064, 140143071580159, ++STORE, 140143071580160, 140143079968767, ++STORE, 140143063183360, 140143071576063, ++STORE, 140143054790656, 140143071576063, ++SNULL, 140143054794751, 140143071576063, ++STORE, 140143054790656, 140143054794751, ++STORE, 140143054794752, 140143071576063, ++SNULL, 140143054794752, 140143063183359, ++STORE, 140143063183360, 140143071576063, ++STORE, 140143054794752, 140143063183359, ++SNULL, 140143063187455, 140143071576063, ++STORE, 140143063183360, 140143063187455, ++STORE, 140143063187456, 140143071576063, ++STORE, 140143046397952, 140143054790655, ++STORE, 140142954143744, 140142962536447, ++STORE, 140142945751040, 140142962536447, ++STORE, 140142937358336, 140142962536447, ++STORE, 140142928965632, 140142962536447, ++STORE, 140142568275968, 140142702493695, ++SNULL, 140142635384831, 140142702493695, ++STORE, 140142568275968, 140142635384831, ++STORE, 140142635384832, 140142702493695, ++ERASE, 140142635384832, 140142702493695, ++STORE, 140142920572928, 140142962536447, ++STORE, 140142912180224, 140142962536447, ++STORE, 140142568275968, 140142702493695, ++SNULL, 140142568275968, 140142635384831, ++STORE, 140142635384832, 140142702493695, ++STORE, 140142568275968, 140142635384831, ++SNULL, 140142635519999, 140142702493695, ++STORE, 140142635384832, 140142635519999, ++STORE, 140142635520000, 140142702493695, ++STORE, 140142819930112, 140142836711423, ++STORE, 140142811533312, 140142819926015, ++STORE, 140142434058240, 140142635384831, ++SNULL, 140142501167103, 140142635384831, ++STORE, 140142434058240, 140142501167103, ++STORE, 140142501167104, 140142635384831, ++SNULL, 140142501167104, 140142568275967, ++STORE, 140142568275968, 140142635384831, ++STORE, 140142501167104, 140142568275967, ++ERASE, 140142501167104, 140142568275967, ++STORE, 140142299840512, 140142501167103, ++STORE, 140142803140608, 140142819926015, ++SNULL, 140142366949375, 140142501167103, ++STORE, 140142299840512, 140142366949375, ++STORE, 140142366949376, 140142501167103, ++SNULL, 140142366949376, 140142434058239, ++STORE, 140142434058240, 140142501167103, ++STORE, 140142366949376, 140142434058239, ++ERASE, 140142366949376, 140142434058239, ++STORE, 140142794747904, 140142819926015, ++STORE, 140142786355200, 140142819926015, ++STORE, 140142299840512, 140142501167103, ++STORE, 140142777962496, 140142819926015, ++STORE, 140142559883264, 140142568275967, ++STORE, 140142232731648, 140142501167103, ++STORE, 140142551490560, 140142568275967, ++SNULL, 140142777962496, 140142803140607, ++STORE, 140142803140608, 140142819926015, ++STORE, 140142777962496, 140142803140607, ++SNULL, 140142803144703, 140142819926015, ++STORE, 140142803140608, 140142803144703, ++STORE, 140142803144704, 140142819926015, ++STORE, 140142543097856, 140142568275967, ++STORE, 140142098513920, 140142501167103, ++SNULL, 140142165622783, 140142501167103, ++STORE, 140142098513920, 140142165622783, ++STORE, 140142165622784, 140142501167103, ++SNULL, 140142165622784, 140142232731647, ++STORE, 140142232731648, 140142501167103, ++STORE, 140142165622784, 140142232731647, ++ERASE, 140142165622784, 140142232731647, ++SNULL, 140142568411135, 140142635384831, ++STORE, 140142568275968, 140142568411135, ++STORE, 140142568411136, 140142635384831, ++STORE, 140141964296192, 140142165622783, ++SNULL, 140142912180224, 140142928965631, ++STORE, 140142928965632, 140142962536447, ++STORE, 140142912180224, 140142928965631, ++SNULL, 140142928969727, 140142962536447, ++STORE, 140142928965632, 140142928969727, ++STORE, 140142928969728, 140142962536447, ++STORE, 140141830078464, 140142165622783, ++SNULL, 140142912184319, 140142928965631, ++STORE, 140142912180224, 140142912184319, ++STORE, 140142912184320, 140142928965631, ++SNULL, 140142232731648, 140142434058239, ++STORE, 140142434058240, 140142501167103, ++STORE, 140142232731648, 140142434058239, ++SNULL, 140142434193407, 140142501167103, ++STORE, 140142434058240, 140142434193407, ++STORE, 140142434193408, 140142501167103, ++SNULL, 140142232731648, 140142299840511, ++STORE, 140142299840512, 140142434058239, ++STORE, 140142232731648, 140142299840511, ++SNULL, 140142299975679, 140142434058239, ++STORE, 140142299840512, 140142299975679, ++STORE, 140142299975680, 140142434058239, ++SNULL, 140142928969728, 140142954143743, ++STORE, 140142954143744, 140142962536447, ++STORE, 140142928969728, 140142954143743, ++SNULL, 140142954147839, 140142962536447, ++STORE, 140142954143744, 140142954147839, ++STORE, 140142954147840, 140142962536447, ++STORE, 140141830078464, 140142299840511, ++SNULL, 140142543097856, 140142559883263, ++STORE, 140142559883264, 140142568275967, ++STORE, 140142543097856, 140142559883263, ++SNULL, 140142559887359, 140142568275967, ++STORE, 140142559883264, 140142559887359, ++STORE, 140142559887360, 140142568275967, ++STORE, 140142534705152, 140142559883263, ++SNULL, 140142928969728, 140142945751039, ++STORE, 140142945751040, 140142954143743, ++STORE, 140142928969728, 140142945751039, ++SNULL, 140142945755135, 140142954143743, ++STORE, 140142945751040, 140142945755135, ++STORE, 140142945755136, 140142954143743, ++SNULL, 140142299975680, 140142366949375, ++STORE, 140142366949376, 140142434058239, ++STORE, 140142299975680, 140142366949375, ++SNULL, 140142367084543, 140142434058239, ++STORE, 140142366949376, 140142367084543, ++STORE, 140142367084544, 140142434058239, ++SNULL, 140142928969728, 140142937358335, ++STORE, 140142937358336, 140142945751039, ++STORE, 140142928969728, 140142937358335, ++SNULL, 140142937362431, 140142945751039, ++STORE, 140142937358336, 140142937362431, ++STORE, 140142937362432, 140142945751039, ++SNULL, 140141830078464, 140142232731647, ++STORE, 140142232731648, 140142299840511, ++STORE, 140141830078464, 140142232731647, ++SNULL, 140142232866815, 140142299840511, ++STORE, 140142232731648, 140142232866815, ++STORE, 140142232866816, 140142299840511, ++SNULL, 140142534705152, 140142543097855, ++STORE, 140142543097856, 140142559883263, ++STORE, 140142534705152, 140142543097855, ++SNULL, 140142543101951, 140142559883263, ++STORE, 140142543097856, 140142543101951, ++STORE, 140142543101952, 140142559883263, ++STORE, 140142526312448, 140142543097855, ++STORE, 140142517919744, 140142543097855, ++SNULL, 140141830078464, 140142098513919, ++STORE, 140142098513920, 140142232731647, ++STORE, 140141830078464, 140142098513919, ++SNULL, 140142098649087, 140142232731647, ++STORE, 140142098513920, 140142098649087, ++STORE, 140142098649088, 140142232731647, ++SNULL, 140142031405055, 140142098513919, ++STORE, 140141830078464, 140142031405055, ++STORE, 140142031405056, 140142098513919, ++ERASE, 140142031405056, 140142098513919, ++SNULL, 140141830078464, 140141964296191, ++STORE, 140141964296192, 140142031405055, ++STORE, 140141830078464, 140141964296191, ++SNULL, 140141964431359, 140142031405055, ++STORE, 140141964296192, 140141964431359, ++STORE, 140141964431360, 140142031405055, ++STORE, 140142509527040, 140142543097855, ++SNULL, 140141897187327, 140141964296191, ++STORE, 140141830078464, 140141897187327, ++STORE, 140141897187328, 140141964296191, ++ERASE, 140141897187328, 140141964296191, ++SNULL, 140141830213631, 140141897187327, ++STORE, 140141830078464, 140141830213631, ++STORE, 140141830213632, 140141897187327, ++SNULL, 140142803144704, 140142811533311, ++STORE, 140142811533312, 140142819926015, ++STORE, 140142803144704, 140142811533311, ++SNULL, 140142811537407, 140142819926015, ++STORE, 140142811533312, 140142811537407, ++STORE, 140142811537408, 140142819926015, ++SNULL, 140142098649088, 140142165622783, ++STORE, 140142165622784, 140142232731647, ++STORE, 140142098649088, 140142165622783, ++SNULL, 140142165757951, 140142232731647, ++STORE, 140142165622784, 140142165757951, ++STORE, 140142165757952, 140142232731647, ++STORE, 140142090121216, 140142098513919, ++SNULL, 140142777962496, 140142786355199, ++STORE, 140142786355200, 140142803140607, ++STORE, 140142777962496, 140142786355199, ++SNULL, 140142786359295, 140142803140607, ++STORE, 140142786355200, 140142786359295, ++STORE, 140142786359296, 140142803140607, ++SNULL, 140142509527040, 140142534705151, ++STORE, 140142534705152, 140142543097855, ++STORE, 140142509527040, 140142534705151, ++SNULL, 140142534709247, 140142543097855, ++STORE, 140142534705152, 140142534709247, ++STORE, 140142534709248, 140142543097855, ++STORE, 140142081728512, 140142098513919, ++SNULL, 140142786359296, 140142794747903, ++STORE, 140142794747904, 140142803140607, ++STORE, 140142786359296, 140142794747903, ++SNULL, 140142794751999, 140142803140607, ++STORE, 140142794747904, 140142794751999, ++STORE, 140142794752000, 140142803140607, ++STORE, 140142073335808, 140142098513919, ++SNULL, 140142073339903, 140142098513919, ++STORE, 140142073335808, 140142073339903, ++STORE, 140142073339904, 140142098513919, ++SNULL, 140142543101952, 140142551490559, ++STORE, 140142551490560, 140142559883263, ++STORE, 140142543101952, 140142551490559, ++SNULL, 140142551494655, 140142559883263, ++STORE, 140142551490560, 140142551494655, ++STORE, 140142551494656, 140142559883263, ++SNULL, 140142509527040, 140142517919743, ++STORE, 140142517919744, 140142534705151, ++STORE, 140142509527040, 140142517919743, ++SNULL, 140142517923839, 140142534705151, ++STORE, 140142517919744, 140142517923839, ++STORE, 140142517923840, 140142534705151, ++STORE, 140142064943104, 140142073335807, ++SNULL, 140142073339904, 140142090121215, ++STORE, 140142090121216, 140142098513919, ++STORE, 140142073339904, 140142090121215, ++SNULL, 140142090125311, 140142098513919, ++STORE, 140142090121216, 140142090125311, ++STORE, 140142090125312, 140142098513919, ++STORE, 140142056550400, 140142073335807, ++SNULL, 140142056554495, 140142073335807, ++STORE, 140142056550400, 140142056554495, ++STORE, 140142056554496, 140142073335807, ++STORE, 140142048157696, 140142056550399, ++SNULL, 140142509531135, 140142517919743, ++STORE, 140142509527040, 140142509531135, ++STORE, 140142509531136, 140142517919743, ++SNULL, 140142777966591, 140142786355199, ++STORE, 140142777962496, 140142777966591, ++STORE, 140142777966592, 140142786355199, ++SNULL, 140143046402047, 140143054790655, ++STORE, 140143046397952, 140143046402047, ++STORE, 140143046402048, 140143054790655, ++SNULL, 140142912184320, 140142920572927, ++STORE, 140142920572928, 140142928965631, ++STORE, 140142912184320, 140142920572927, ++SNULL, 140142920577023, 140142928965631, ++STORE, 140142920572928, 140142920577023, ++STORE, 140142920577024, 140142928965631, ++STORE, 140142039764992, 140142056550399, ++STORE, 140141955903488, 140141964296191, ++SNULL, 140142819930112, 140142828318719, ++STORE, 140142828318720, 140142836711423, ++STORE, 140142819930112, 140142828318719, ++SNULL, 140142828322815, 140142836711423, ++STORE, 140142828318720, 140142828322815, ++STORE, 140142828322816, 140142836711423, ++SNULL, 140142517923840, 140142526312447, ++STORE, 140142526312448, 140142534705151, ++STORE, 140142517923840, 140142526312447, ++SNULL, 140142526316543, 140142534705151, ++STORE, 140142526312448, 140142526316543, ++STORE, 140142526316544, 140142534705151, ++STORE, 140141947510784, 140141964296191, ++SNULL, 140142056554496, 140142064943103, ++STORE, 140142064943104, 140142073335807, ++STORE, 140142056554496, 140142064943103, ++SNULL, 140142064947199, 140142073335807, ++STORE, 140142064943104, 140142064947199, ++STORE, 140142064947200, 140142073335807, ++SNULL, 140142073339904, 140142081728511, ++STORE, 140142081728512, 140142090121215, ++STORE, 140142073339904, 140142081728511, ++SNULL, 140142081732607, 140142090121215, ++STORE, 140142081728512, 140142081732607, ++STORE, 140142081732608, 140142090121215, ++STORE, 140141939118080, 140141964296191, ++STORE, 140141930725376, 140141964296191, ++STORE, 140141922332672, 140141964296191, ++STORE, 140141913939968, 140141964296191, ++SNULL, 140141913939968, 140141922332671, ++STORE, 140141922332672, 140141964296191, ++STORE, 140141913939968, 140141922332671, ++SNULL, 140141922336767, 140141964296191, ++STORE, 140141922332672, 140141922336767, ++STORE, 140141922336768, 140141964296191, ++STORE, 140141905547264, 140141922332671, ++SNULL, 140141905551359, 140141922332671, ++STORE, 140141905547264, 140141905551359, ++STORE, 140141905551360, 140141922332671, ++STORE, 140141821685760, 140141830078463, ++STORE, 140141813293056, 140141830078463, ++STORE, 140141804900352, 140141830078463, ++STORE, 140141796507648, 140141830078463, ++SNULL, 140141796511743, 140141830078463, ++STORE, 140141796507648, 140141796511743, ++STORE, 140141796511744, 140141830078463, ++SNULL, 140141922336768, 140141955903487, ++STORE, 140141955903488, 140141964296191, ++STORE, 140141922336768, 140141955903487, ++SNULL, 140141955907583, 140141964296191, ++STORE, 140141955903488, 140141955907583, ++STORE, 140141955907584, 140141964296191, ++STORE, 140141788114944, 140141796507647, ++STORE, 140141779722240, 140141796507647, ++SNULL, 140141779722240, 140141788114943, ++STORE, 140141788114944, 140141796507647, ++STORE, 140141779722240, 140141788114943, ++SNULL, 140141788119039, 140141796507647, ++STORE, 140141788114944, 140141788119039, ++STORE, 140141788119040, 140141796507647, ++SNULL, 140141922336768, 140141947510783, ++STORE, 140141947510784, 140141955903487, ++STORE, 140141922336768, 140141947510783, ++SNULL, 140141947514879, 140141955903487, ++STORE, 140141947510784, 140141947514879, ++STORE, 140141947514880, 140141955903487, ++SNULL, 140142039764992, 140142048157695, ++STORE, 140142048157696, 140142056550399, ++STORE, 140142039764992, 140142048157695, ++SNULL, 140142048161791, 140142056550399, ++STORE, 140142048157696, 140142048161791, ++STORE, 140142048161792, 140142056550399, ++SNULL, 140142039769087, 140142048157695, ++STORE, 140142039764992, 140142039769087, ++STORE, 140142039769088, 140142048157695, ++SNULL, 140141796511744, 140141804900351, ++STORE, 140141804900352, 140141830078463, ++STORE, 140141796511744, 140141804900351, ++SNULL, 140141804904447, 140141830078463, ++STORE, 140141804900352, 140141804904447, ++STORE, 140141804904448, 140141830078463, ++STORE, 140141771329536, 140141788114943, ++STORE, 140141762936832, 140141788114943, ++STORE, 140141754544128, 140141788114943, ++SNULL, 140141804904448, 140141821685759, ++STORE, 140141821685760, 140141830078463, ++STORE, 140141804904448, 140141821685759, ++SNULL, 140141821689855, 140141830078463, ++STORE, 140141821685760, 140141821689855, ++STORE, 140141821689856, 140141830078463, ++SNULL, 140141922336768, 140141939118079, ++STORE, 140141939118080, 140141947510783, ++STORE, 140141922336768, 140141939118079, ++SNULL, 140141939122175, 140141947510783, ++STORE, 140141939118080, 140141939122175, ++STORE, 140141939122176, 140141947510783, ++SNULL, 140141905551360, 140141913939967, ++STORE, 140141913939968, 140141922332671, ++STORE, 140141905551360, 140141913939967, ++SNULL, 140141913944063, 140141922332671, ++STORE, 140141913939968, 140141913944063, ++STORE, 140141913944064, 140141922332671, ++STORE, 140141746151424, 140141788114943, ++STORE, 140141737758720, 140141788114943, ++SNULL, 140141804904448, 140141813293055, ++STORE, 140141813293056, 140141821685759, ++STORE, 140141804904448, 140141813293055, ++SNULL, 140141813297151, 140141821685759, ++STORE, 140141813293056, 140141813297151, ++STORE, 140141813297152, 140141821685759, ++STORE, 140141729366016, 140141788114943, ++STORE, 140141720973312, 140141788114943, ++STORE, 140141712580608, 140141788114943, ++SNULL, 140141712584703, 140141788114943, ++STORE, 140141712580608, 140141712584703, ++STORE, 140141712584704, 140141788114943, ++SNULL, 140141922336768, 140141930725375, ++STORE, 140141930725376, 140141939118079, ++STORE, 140141922336768, 140141930725375, ++SNULL, 140141930729471, 140141939118079, ++STORE, 140141930725376, 140141930729471, ++STORE, 140141930729472, 140141939118079, ++STORE, 140141704187904, 140141712580607, ++SNULL, 140141704191999, 140141712580607, ++STORE, 140141704187904, 140141704191999, ++STORE, 140141704192000, 140141712580607, ++STORE, 140141695795200, 140141704187903, ++STORE, 140141687402496, 140141704187903, ++SNULL, 140141712584704, 140141771329535, ++STORE, 140141771329536, 140141788114943, ++STORE, 140141712584704, 140141771329535, ++SNULL, 140141771333631, 140141788114943, ++STORE, 140141771329536, 140141771333631, ++STORE, 140141771333632, 140141788114943, ++SNULL, 140141771333632, 140141779722239, ++STORE, 140141779722240, 140141788114943, ++STORE, 140141771333632, 140141779722239, ++SNULL, 140141779726335, 140141788114943, ++STORE, 140141779722240, 140141779726335, ++STORE, 140141779726336, 140141788114943, ++STORE, 140141679009792, 140141704187903, ++SNULL, 140141679013887, 140141704187903, ++STORE, 140141679009792, 140141679013887, ++STORE, 140141679013888, 140141704187903, ++STORE, 140141670617088, 140141679009791, ++SNULL, 140141670621183, 140141679009791, ++STORE, 140141670617088, 140141670621183, ++STORE, 140141670621184, 140141679009791, ++STORE, 140141662224384, 140141670617087, ++SNULL, 140141712584704, 140141737758719, ++STORE, 140141737758720, 140141771329535, ++STORE, 140141712584704, 140141737758719, ++SNULL, 140141737762815, 140141771329535, ++STORE, 140141737758720, 140141737762815, ++STORE, 140141737762816, 140141771329535, ++SNULL, 140141712584704, 140141729366015, ++STORE, 140141729366016, 140141737758719, ++STORE, 140141712584704, 140141729366015, ++SNULL, 140141729370111, 140141737758719, ++STORE, 140141729366016, 140141729370111, ++STORE, 140141729370112, 140141737758719, ++SNULL, 140141737762816, 140141746151423, ++STORE, 140141746151424, 140141771329535, ++STORE, 140141737762816, 140141746151423, ++SNULL, 140141746155519, 140141771329535, ++STORE, 140141746151424, 140141746155519, ++STORE, 140141746155520, 140141771329535, ++STORE, 140141653831680, 140141670617087, ++SNULL, 140141746155520, 140141762936831, ++STORE, 140141762936832, 140141771329535, ++STORE, 140141746155520, 140141762936831, ++SNULL, 140141762940927, 140141771329535, ++STORE, 140141762936832, 140141762940927, ++STORE, 140141762940928, 140141771329535, ++STORE, 140141645438976, 140141670617087, ++SNULL, 140141645443071, 140141670617087, ++STORE, 140141645438976, 140141645443071, ++STORE, 140141645443072, 140141670617087, ++SNULL, 140141712584704, 140141720973311, ++STORE, 140141720973312, 140141729366015, ++STORE, 140141712584704, 140141720973311, ++SNULL, 140141720977407, 140141729366015, ++STORE, 140141720973312, 140141720977407, ++STORE, 140141720977408, 140141729366015, ++STORE, 140141637046272, 140141645438975, ++SNULL, 140141637050367, 140141645438975, ++STORE, 140141637046272, 140141637050367, ++STORE, 140141637050368, 140141645438975, ++STORE, 140141628653568, 140141637046271, ++SNULL, 140141628657663, 140141637046271, ++STORE, 140141628653568, 140141628657663, ++STORE, 140141628657664, 140141637046271, ++STORE, 140141620260864, 140141628653567, ++SNULL, 140141679013888, 140141687402495, ++STORE, 140141687402496, 140141704187903, ++STORE, 140141679013888, 140141687402495, ++SNULL, 140141687406591, 140141704187903, ++STORE, 140141687402496, 140141687406591, ++STORE, 140141687406592, 140141704187903, ++SNULL, 140141746155520, 140141754544127, ++STORE, 140141754544128, 140141762936831, ++STORE, 140141746155520, 140141754544127, ++SNULL, 140141754548223, 140141762936831, ++STORE, 140141754544128, 140141754548223, ++STORE, 140141754548224, 140141762936831, ++SNULL, 140141687406592, 140141695795199, ++STORE, 140141695795200, 140141704187903, ++STORE, 140141687406592, 140141695795199, ++SNULL, 140141695799295, 140141704187903, ++STORE, 140141695795200, 140141695799295, ++STORE, 140141695799296, 140141704187903, ++STORE, 140141611868160, 140141628653567, ++SNULL, 140141611872255, 140141628653567, ++STORE, 140141611868160, 140141611872255, ++STORE, 140141611872256, 140141628653567, ++SNULL, 140141645443072, 140141662224383, ++STORE, 140141662224384, 140141670617087, ++STORE, 140141645443072, 140141662224383, ++SNULL, 140141662228479, 140141670617087, ++STORE, 140141662224384, 140141662228479, ++STORE, 140141662228480, 140141670617087, ++STORE, 140141603475456, 140141611868159, ++SNULL, 140141603479551, 140141611868159, ++STORE, 140141603475456, 140141603479551, ++STORE, 140141603479552, 140141611868159, ++STORE, 140141595082752, 140141603475455, ++SNULL, 140141645443072, 140141653831679, ++STORE, 140141653831680, 140141662224383, ++STORE, 140141645443072, 140141653831679, ++SNULL, 140141653835775, 140141662224383, ++STORE, 140141653831680, 140141653835775, ++STORE, 140141653835776, 140141662224383, ++STORE, 140141586690048, 140141603475455, ++SNULL, 140141611872256, 140141620260863, ++STORE, 140141620260864, 140141628653567, ++STORE, 140141611872256, 140141620260863, ++SNULL, 140141620264959, 140141628653567, ++STORE, 140141620260864, 140141620264959, ++STORE, 140141620264960, 140141628653567, ++SNULL, 140141586690048, 140141595082751, ++STORE, 140141595082752, 140141603475455, ++STORE, 140141586690048, 140141595082751, ++SNULL, 140141595086847, 140141603475455, ++STORE, 140141595082752, 140141595086847, ++STORE, 140141595086848, 140141603475455, ++STORE, 140141578297344, 140141595082751, ++SNULL, 140141578301439, 140141595082751, ++STORE, 140141578297344, 140141578301439, ++STORE, 140141578301440, 140141595082751, ++SNULL, 140141578301440, 140141586690047, ++STORE, 140141586690048, 140141595082751, ++STORE, 140141578301440, 140141586690047, ++SNULL, 140141586694143, 140141595082751, ++STORE, 140141586690048, 140141586694143, ++STORE, 140141586694144, 140141595082751, ++STORE, 140143370027008, 140143370055679, ++STORE, 140143309254656, 140143311446015, ++SNULL, 140143309254656, 140143309344767, ++STORE, 140143309344768, 140143311446015, ++STORE, 140143309254656, 140143309344767, ++SNULL, 140143311437823, 140143311446015, ++STORE, 140143309344768, 140143311437823, ++STORE, 140143311437824, 140143311446015, ++ERASE, 140143311437824, 140143311446015, ++STORE, 140143311437824, 140143311446015, ++SNULL, 140143311441919, 140143311446015, ++STORE, 140143311437824, 140143311441919, ++STORE, 140143311441920, 140143311446015, ++ERASE, 140143370027008, 140143370055679, ++ERASE, 140142912180224, 140142912184319, ++ERASE, 140142912184320, 140142920572927, ++ERASE, 140142945751040, 140142945755135, ++ERASE, 140142945755136, 140142954143743, ++ERASE, 140142090121216, 140142090125311, ++ERASE, 140142090125312, 140142098513919, ++ERASE, 140142794747904, 140142794751999, ++ERASE, 140142794752000, 140142803140607, ++ERASE, 140141913939968, 140141913944063, ++ERASE, 140141913944064, 140141922332671, ++ERASE, 140141746151424, 140141746155519, ++ERASE, 140141746155520, 140141754544127, ++ERASE, 140142954143744, 140142954147839, ++ERASE, 140142954147840, 140142962536447, ++ERASE, 140142081728512, 140142081732607, ++ERASE, 140142081732608, 140142090121215, ++ERASE, 140141905547264, 140141905551359, ++ERASE, 140141905551360, 140141913939967, ++ERASE, 140141729366016, 140141729370111, ++ERASE, 140141729370112, 140141737758719, ++ERASE, 140142920572928, 140142920577023, ++ERASE, 140142920577024, 140142928965631, ++ERASE, 140142039764992, 140142039769087, ++ERASE, 140142039769088, 140142048157695, ++ERASE, 140141679009792, 140141679013887, ++ERASE, 140141679013888, 140141687402495, ++ERASE, 140142551490560, 140142551494655, ++ERASE, 140142551494656, 140142559883263, ++ERASE, 140141947510784, 140141947514879, ++ERASE, 140141947514880, 140141955903487, ++ERASE, 140141771329536, 140141771333631, ++ERASE, 140141771333632, 140141779722239, ++ERASE, 140142928965632, 140142928969727, ++ERASE, 140142928969728, 140142937358335, ++ERASE, 140142073335808, 140142073339903, ++ERASE, 140142073339904, 140142081728511, ++ERASE, 140142543097856, 140142543101951, ++ERASE, 140142543101952, 140142551490559, ++ERASE, 140141955903488, 140141955907583, ++ERASE, 140141955907584, 140141964296191, ++ERASE, 140141704187904, 140141704191999, ++ERASE, 140141704192000, 140141712580607, ++ERASE, 140142786355200, 140142786359295, ++ERASE, 140142786359296, 140142794747903, ++ERASE, 140142056550400, 140142056554495, ++ERASE, 140142056554496, 140142064943103, ++ERASE, 140142828318720, 140142828322815, ++ERASE, 140142828322816, 140142836711423, ++ERASE, 140141788114944, 140141788119039, ++ERASE, 140141788119040, 140141796507647, ++ERASE, 140141695795200, 140141695799295, ++ERASE, 140141695799296, 140141704187903, ++ERASE, 140141578297344, 140141578301439, ++ERASE, 140141578301440, 140141586690047, ++ERASE, 140141611868160, 140141611872255, ++ERASE, 140141611872256, 140141620260863, ++ERASE, 140142811533312, 140142811537407, ++ERASE, 140142811537408, 140142819926015, ++ERASE, 140142064943104, 140142064947199, ++ERASE, 140142064947200, 140142073335807, ++ERASE, 140141628653568, 140141628657663, ++ERASE, 140141628657664, 140141637046271, ++ERASE, 140143046397952, 140143046402047, ++ERASE, 140143046402048, 140143054790655, ++ERASE, 140141796507648, 140141796511743, ++ERASE, 140141796511744, 140141804900351, ++ERASE, 140142803140608, 140142803144703, ++ERASE, 140142803144704, 140142811533311, ++ERASE, 140142509527040, 140142509531135, ++ERASE, 140142509531136, 140142517919743, ++ERASE, 140141821685760, 140141821689855, ++ERASE, 140141821689856, 140141830078463, ++ERASE, 140142777962496, 140142777966591, ++ERASE, 140142777966592, 140142786355199, ++ERASE, 140141804900352, 140141804904447, ++ERASE, 140141804904448, 140141813293055, ++ERASE, 140141930725376, 140141930729471, ++ERASE, 140141930729472, 140141939118079, ++ERASE, 140142937358336, 140142937362431, ++ERASE, 140142937362432, 140142945751039, ++ERASE, 140142559883264, 140142559887359, ++ERASE, 140142559887360, 140142568275967, ++ERASE, 140142534705152, 140142534709247, ++ERASE, 140142534709248, 140142543097855, ++ERASE, 140142048157696, 140142048161791, ++ERASE, 140142048161792, 140142056550399, ++ERASE, 140141754544128, 140141754548223, ++ERASE, 140141754548224, 140141762936831, ++ERASE, 140141939118080, 140141939122175, ++ERASE, 140141939122176, 140141947510783, ++ERASE, 140141653831680, 140141653835775, ++ERASE, 140141653835776, 140141662224383, ++ERASE, 140141712580608, 140141712584703, ++ERASE, 140141712584704, 140141720973311, ++ERASE, 140141645438976, 140141645443071, ++ERASE, 140141645443072, 140141653831679, ++ERASE, 140141687402496, 140141687406591, ++ERASE, 140141687406592, 140141695795199, ++ERASE, 140141662224384, 140141662228479, ++ERASE, 140141662228480, 140141670617087, ++ERASE, 140141922332672, 140141922336767, ++ERASE, 140141922336768, 140141930725375, ++ERASE, 140141737758720, 140141737762815, ++ERASE, 140141737762816, 140141746151423, ++ERASE, 140141637046272, 140141637050367, ++ERASE, 140141637050368, 140141645438975, ++ERASE, 140142517919744, 140142517923839, ++ERASE, 140142517923840, 140142526312447, ++ERASE, 140143096754176, 140143096758271, ++ERASE, 140143096758272, 140143105146879, ++ERASE, 140141595082752, 140141595086847, ++ERASE, 140141595086848, 140141603475455, ++ERASE, 140141762936832, 140141762940927, ++ERASE, 140141762940928, 140141771329535, ++ERASE, 140143311446016, 140143311450111, ++ERASE, 140143311450112, 140143319838719, ++ERASE, 140142526312448, 140142526316543, ++ERASE, 140142526316544, 140142534705151, ++ERASE, 140142819926016, 140142819930111, ++ERASE, 140142819930112, 140142828318719, ++ERASE, 140143180615680, 140143180619775, ++ERASE, 140143180619776, 140143189008383, ++ERASE, 140142962536448, 140142962540543, ++ERASE, 140142962540544, 140142970929151, ++ERASE, 140143214186496, 140143214190591, ++ERASE, 140143214190592, 140143222579199, ++ERASE, 140143088361472, 140143088365567, ++ERASE, 140143088365568, 140143096754175, ++ERASE, 140141586690048, 140141586694143, ++ERASE, 140141586694144, 140141595082751, ++ERASE, 140143230971904, 140143230975999, ++ERASE, 140143230976000, 140143239364607, ++ERASE, 140141779722240, 140141779726335, ++ERASE, 140141779726336, 140141788114943, ++ERASE, 140141670617088, 140141670621183, ++ERASE, 140141670621184, 140141679009791, ++ERASE, 140141813293056, 140141813297151, ++ERASE, 140141813297152, 140141821685759, ++ERASE, 140143222579200, 140143222583295, ++ERASE, 140143222583296, 140143230971903, ++ERASE, 140143189008384, 140143189012479, ++ERASE, 140143189012480, 140143197401087, ++ERASE, 140143071576064, 140143071580159, ++ERASE, 140143071580160, 140143079968767, ++ERASE, 140141620260864, 140141620264959, ++ERASE, 140141620264960, 140141628653567, ++ERASE, 140141603475456, 140141603479551, ++ERASE, 140141603479552, 140141611868159, ++ERASE, 140141720973312, 140141720977407, ++ERASE, 140141720977408, 140141729366015, ++ERASE, 140143079968768, 140143079972863, ++ERASE, 140143079972864, 140143088361471, ++ERASE, 140143205793792, 140143205797887, ++ERASE, 140143205797888, 140143214186495, ++ }; ++ unsigned long set30[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140733436743680, 140737488351231, ++SNULL, 140733436747775, 140737488351231, ++STORE, 140733436743680, 140733436747775, ++STORE, 140733436612608, 140733436747775, ++STORE, 94630728904704, 94630731157503, ++SNULL, 94630729035775, 94630731157503, ++STORE, 94630728904704, 94630729035775, ++STORE, 94630729035776, 94630731157503, ++ERASE, 94630729035776, 94630731157503, ++STORE, 94630731128832, 94630731137023, ++STORE, 94630731137024, 94630731157503, ++STORE, 140165750841344, 140165753094143, ++SNULL, 140165750984703, 140165753094143, ++STORE, 140165750841344, 140165750984703, ++STORE, 140165750984704, 140165753094143, ++ERASE, 140165750984704, 140165753094143, ++STORE, 140165753081856, 140165753090047, ++STORE, 140165753090048, 140165753094143, ++STORE, 140733436887040, 140733436891135, ++STORE, 140733436874752, 140733436887039, ++STORE, 140165753053184, 140165753081855, ++STORE, 140165753044992, 140165753053183, ++STORE, 140165748625408, 140165750841343, ++SNULL, 140165748625408, 140165748723711, ++STORE, 140165748723712, 140165750841343, ++STORE, 140165748625408, 140165748723711, ++SNULL, 140165750816767, 140165750841343, ++STORE, 140165748723712, 140165750816767, ++STORE, 140165750816768, 140165750841343, ++SNULL, 140165750816768, 140165750824959, ++STORE, 140165750824960, 140165750841343, ++STORE, 140165750816768, 140165750824959, ++ERASE, 140165750816768, 140165750824959, ++STORE, 140165750816768, 140165750824959, ++ERASE, 140165750824960, 140165750841343, ++STORE, 140165750824960, 140165750841343, ++STORE, 140165744828416, 140165748625407, ++SNULL, 140165744828416, 140165746487295, ++STORE, 140165746487296, 140165748625407, ++STORE, 140165744828416, 140165746487295, ++SNULL, 140165748584447, 140165748625407, ++STORE, 140165746487296, 140165748584447, ++STORE, 140165748584448, 140165748625407, ++SNULL, 140165748584448, 140165748609023, ++STORE, 140165748609024, 140165748625407, ++STORE, 140165748584448, 140165748609023, ++ERASE, 140165748584448, 140165748609023, ++STORE, 140165748584448, 140165748609023, ++ERASE, 140165748609024, 140165748625407, ++STORE, 140165748609024, 140165748625407, ++STORE, 140165753036800, 140165753053183, ++SNULL, 140165748600831, 140165748609023, ++STORE, 140165748584448, 140165748600831, ++STORE, 140165748600832, 140165748609023, ++SNULL, 140165750820863, 140165750824959, ++STORE, 140165750816768, 140165750820863, ++STORE, 140165750820864, 140165750824959, ++SNULL, 94630731132927, 94630731137023, ++STORE, 94630731128832, 94630731132927, ++STORE, 94630731132928, 94630731137023, ++SNULL, 140165753085951, 140165753090047, ++STORE, 140165753081856, 140165753085951, ++STORE, 140165753085952, 140165753090047, ++ERASE, 140165753053184, 140165753081855, ++STORE, 94630743547904, 94630743683071, ++STORE, 140165736435712, 140165744828415, ++SNULL, 140165736439807, 140165744828415, ++STORE, 140165736435712, 140165736439807, ++STORE, 140165736439808, 140165744828415, ++STORE, 140165728043008, 140165736435711, ++STORE, 140165593825280, 140165728043007, ++SNULL, 140165593825280, 140165653725183, ++STORE, 140165653725184, 140165728043007, ++STORE, 140165593825280, 140165653725183, ++ERASE, 140165593825280, 140165653725183, ++SNULL, 140165720834047, 140165728043007, ++STORE, 140165653725184, 140165720834047, ++STORE, 140165720834048, 140165728043007, ++ERASE, 140165720834048, 140165728043007, ++SNULL, 140165653860351, 140165720834047, ++STORE, 140165653725184, 140165653860351, ++STORE, 140165653860352, 140165720834047, ++SNULL, 140165728047103, 140165736435711, ++STORE, 140165728043008, 140165728047103, ++STORE, 140165728047104, 140165736435711, ++STORE, 140165645332480, 140165653725183, ++SNULL, 140165645336575, 140165653725183, ++STORE, 140165645332480, 140165645336575, ++STORE, 140165645336576, 140165653725183, ++STORE, 140165636939776, 140165645332479, ++SNULL, 140165636943871, 140165645332479, ++STORE, 140165636939776, 140165636943871, ++STORE, 140165636943872, 140165645332479, ++STORE, 140165628547072, 140165636939775, ++SNULL, 140165628551167, 140165636939775, ++STORE, 140165628547072, 140165628551167, ++STORE, 140165628551168, 140165636939775, ++STORE, 140165620154368, 140165628547071, ++STORE, 140165611761664, 140165628547071, ++STORE, 140165603368960, 140165628547071, ++STORE, 140165469151232, 140165603368959, ++SNULL, 140165469151232, 140165519507455, ++STORE, 140165519507456, 140165603368959, ++STORE, 140165469151232, 140165519507455, ++ERASE, 140165469151232, 140165519507455, ++SNULL, 140165586616319, 140165603368959, ++STORE, 140165519507456, 140165586616319, ++STORE, 140165586616320, 140165603368959, ++ERASE, 140165586616320, 140165603368959, ++STORE, 140165594976256, 140165628547071, ++STORE, 140165385289728, 140165586616319, ++SNULL, 140165452398591, 140165586616319, ++STORE, 140165385289728, 140165452398591, ++STORE, 140165452398592, 140165586616319, ++SNULL, 140165452398592, 140165519507455, ++STORE, 140165519507456, 140165586616319, ++STORE, 140165452398592, 140165519507455, ++ERASE, 140165452398592, 140165519507455, ++STORE, 140165251072000, 140165452398591, ++SNULL, 140165318180863, 140165452398591, ++STORE, 140165251072000, 140165318180863, ++STORE, 140165318180864, 140165452398591, ++SNULL, 140165318180864, 140165385289727, ++STORE, 140165385289728, 140165452398591, ++STORE, 140165318180864, 140165385289727, ++ERASE, 140165318180864, 140165385289727, ++SNULL, 140165519642623, 140165586616319, ++STORE, 140165519507456, 140165519642623, ++STORE, 140165519642624, 140165586616319, ++SNULL, 140165594976256, 140165611761663, ++STORE, 140165611761664, 140165628547071, ++STORE, 140165594976256, 140165611761663, ++SNULL, 140165611765759, 140165628547071, ++STORE, 140165611761664, 140165611765759, ++STORE, 140165611765760, 140165628547071, ++STORE, 140165385289728, 140165519507455, ++SNULL, 140165385424895, 140165519507455, ++STORE, 140165385289728, 140165385424895, ++STORE, 140165385424896, 140165519507455, ++SNULL, 140165594976256, 140165603368959, ++STORE, 140165603368960, 140165611761663, ++STORE, 140165594976256, 140165603368959, ++SNULL, 140165603373055, 140165611761663, ++STORE, 140165603368960, 140165603373055, ++STORE, 140165603373056, 140165611761663, ++SNULL, 140165251207167, 140165318180863, ++STORE, 140165251072000, 140165251207167, ++STORE, 140165251207168, 140165318180863, ++STORE, 140165376897024, 140165385289727, ++SNULL, 140165376901119, 140165385289727, ++STORE, 140165376897024, 140165376901119, ++STORE, 140165376901120, 140165385289727, ++SNULL, 140165385424896, 140165452398591, ++STORE, 140165452398592, 140165519507455, ++STORE, 140165385424896, 140165452398591, ++SNULL, 140165452533759, 140165519507455, ++STORE, 140165452398592, 140165452533759, ++STORE, 140165452533760, 140165519507455, ++STORE, 140165368504320, 140165376897023, ++SNULL, 140165594980351, 140165603368959, ++STORE, 140165594976256, 140165594980351, ++STORE, 140165594980352, 140165603368959, ++SNULL, 140165368508415, 140165376897023, ++STORE, 140165368504320, 140165368508415, ++STORE, 140165368508416, 140165376897023, ++SNULL, 140165611765760, 140165620154367, ++STORE, 140165620154368, 140165628547071, ++STORE, 140165611765760, 140165620154367, ++SNULL, 140165620158463, 140165628547071, ++STORE, 140165620154368, 140165620158463, ++STORE, 140165620158464, 140165628547071, ++STORE, 140165360111616, 140165368504319, ++STORE, 140165351718912, 140165368504319, ++STORE, 140165343326208, 140165368504319, ++SNULL, 140165343326208, 140165351718911, ++STORE, 140165351718912, 140165368504319, ++STORE, 140165343326208, 140165351718911, ++SNULL, 140165351723007, 140165368504319, ++STORE, 140165351718912, 140165351723007, ++STORE, 140165351723008, 140165368504319, ++SNULL, 140165343330303, 140165351718911, ++STORE, 140165343326208, 140165343330303, ++STORE, 140165343330304, 140165351718911, ++SNULL, 140165351723008, 140165360111615, ++STORE, 140165360111616, 140165368504319, ++STORE, 140165351723008, 140165360111615, ++SNULL, 140165360115711, 140165368504319, ++STORE, 140165360111616, 140165360115711, ++STORE, 140165360115712, 140165368504319, ++STORE, 140165334933504, 140165343326207, ++SNULL, 140165334937599, 140165343326207, ++STORE, 140165334933504, 140165334937599, ++STORE, 140165334937600, 140165343326207, ++STORE, 140165326540800, 140165334933503, ++STORE, 140165242679296, 140165251071999, ++SNULL, 140165242683391, 140165251071999, ++STORE, 140165242679296, 140165242683391, ++STORE, 140165242683392, 140165251071999, ++STORE, 140165234286592, 140165242679295, ++STORE, 140165225893888, 140165242679295, ++SNULL, 140165225897983, 140165242679295, ++STORE, 140165225893888, 140165225897983, ++STORE, 140165225897984, 140165242679295, ++SNULL, 140165225897984, 140165234286591, ++STORE, 140165234286592, 140165242679295, ++STORE, 140165225897984, 140165234286591, ++SNULL, 140165234290687, 140165242679295, ++STORE, 140165234286592, 140165234290687, ++STORE, 140165234290688, 140165242679295, ++SNULL, 140165326544895, 140165334933503, ++STORE, 140165326540800, 140165326544895, ++STORE, 140165326544896, 140165334933503, ++STORE, 140165217501184, 140165225893887, ++STORE, 140165209108480, 140165225893887, ++SNULL, 140165209108480, 140165217501183, ++STORE, 140165217501184, 140165225893887, ++STORE, 140165209108480, 140165217501183, ++SNULL, 140165217505279, 140165225893887, ++STORE, 140165217501184, 140165217505279, ++STORE, 140165217505280, 140165225893887, ++SNULL, 140165209112575, 140165217501183, ++STORE, 140165209108480, 140165209112575, ++STORE, 140165209112576, 140165217501183, ++STORE, 140165200715776, 140165209108479, ++STORE, 140165066498048, 140165200715775, ++SNULL, 140165066498048, 140165116854271, ++STORE, 140165116854272, 140165200715775, ++STORE, 140165066498048, 140165116854271, ++ERASE, 140165066498048, 140165116854271, ++SNULL, 140165183963135, 140165200715775, ++STORE, 140165116854272, 140165183963135, ++STORE, 140165183963136, 140165200715775, ++ERASE, 140165183963136, 140165200715775, ++SNULL, 140165116989439, 140165183963135, ++STORE, 140165116854272, 140165116989439, ++STORE, 140165116989440, 140165183963135, ++STORE, 140165192323072, 140165209108479, ++STORE, 140165108461568, 140165116854271, ++STORE, 140164974243840, 140165108461567, ++STORE, 140164965851136, 140164974243839, ++SNULL, 140164974243840, 140164982636543, ++STORE, 140164982636544, 140165108461567, ++STORE, 140164974243840, 140164982636543, ++ERASE, 140164974243840, 140164982636543, ++STORE, 140164965851136, 140164982636543, ++STORE, 140164957458432, 140164982636543, ++STORE, 140164949065728, 140164982636543, ++STORE, 140164940673024, 140164982636543, ++STORE, 140164806455296, 140164940673023, ++STORE, 140164798062592, 140164806455295, ++STORE, 140164789669888, 140164806455295, ++STORE, 140164655452160, 140164789669887, ++STORE, 140164647059456, 140164655452159, ++STORE, 140164638666752, 140164655452159, ++SNULL, 140164655452160, 140164714201087, ++STORE, 140164714201088, 140164789669887, ++STORE, 140164655452160, 140164714201087, ++ERASE, 140164655452160, 140164714201087, ++STORE, 140164705808384, 140164714201087, ++STORE, 140164697415680, 140164714201087, ++STORE, 140164504449024, 140164638666751, ++SNULL, 140164504449024, 140164512874495, ++STORE, 140164512874496, 140164638666751, ++STORE, 140164504449024, 140164512874495, ++ERASE, 140164504449024, 140164512874495, ++STORE, 140164689022976, 140164714201087, ++STORE, 140164680630272, 140164714201087, ++SNULL, 140164680634367, 140164714201087, ++STORE, 140164680630272, 140164680634367, ++STORE, 140164680634368, 140164714201087, ++STORE, 140164378656768, 140164638666751, ++SNULL, 140165192323072, 140165200715775, ++STORE, 140165200715776, 140165209108479, ++STORE, 140165192323072, 140165200715775, ++SNULL, 140165200719871, 140165209108479, ++STORE, 140165200715776, 140165200719871, ++STORE, 140165200719872, 140165209108479, ++SNULL, 140165049745407, 140165108461567, ++STORE, 140164982636544, 140165049745407, ++STORE, 140165049745408, 140165108461567, ++ERASE, 140165049745408, 140165108461567, ++SNULL, 140164982771711, 140165049745407, ++STORE, 140164982636544, 140164982771711, ++STORE, 140164982771712, 140165049745407, ++STORE, 140164244439040, 140164638666751, ++SNULL, 140164311547903, 140164638666751, ++STORE, 140164244439040, 140164311547903, ++STORE, 140164311547904, 140164638666751, ++SNULL, 140164311547904, 140164378656767, ++STORE, 140164378656768, 140164638666751, ++STORE, 140164311547904, 140164378656767, ++ERASE, 140164311547904, 140164378656767, ++SNULL, 140164806455296, 140164848418815, ++STORE, 140164848418816, 140164940673023, ++STORE, 140164806455296, 140164848418815, ++ERASE, 140164806455296, 140164848418815, ++SNULL, 140164915527679, 140164940673023, ++STORE, 140164848418816, 140164915527679, ++STORE, 140164915527680, 140164940673023, ++ERASE, 140164915527680, 140164940673023, ++STORE, 140164110221312, 140164311547903, ++SNULL, 140164177330175, 140164311547903, ++STORE, 140164110221312, 140164177330175, ++STORE, 140164177330176, 140164311547903, ++SNULL, 140164177330176, 140164244439039, ++STORE, 140164244439040, 140164311547903, ++STORE, 140164177330176, 140164244439039, ++ERASE, 140164177330176, 140164244439039, ++SNULL, 140164781309951, 140164789669887, ++STORE, 140164714201088, 140164781309951, ++STORE, 140164781309952, 140164789669887, ++ERASE, 140164781309952, 140164789669887, ++STORE, 140163976003584, 140164177330175, ++SNULL, 140164043112447, 140164177330175, ++STORE, 140163976003584, 140164043112447, ++STORE, 140164043112448, 140164177330175, ++SNULL, 140164043112448, 140164110221311, ++STORE, 140164110221312, 140164177330175, ++STORE, 140164043112448, 140164110221311, ++ERASE, 140164043112448, 140164110221311, ++SNULL, 140164579983359, 140164638666751, ++STORE, 140164378656768, 140164579983359, ++STORE, 140164579983360, 140164638666751, ++ERASE, 140164579983360, 140164638666751, ++STORE, 140163841785856, 140164043112447, ++SNULL, 140163908894719, 140164043112447, ++STORE, 140163841785856, 140163908894719, ++STORE, 140163908894720, 140164043112447, ++SNULL, 140163908894720, 140163976003583, ++STORE, 140163976003584, 140164043112447, ++STORE, 140163908894720, 140163976003583, ++ERASE, 140163908894720, 140163976003583, ++SNULL, 140164940673024, 140164965851135, ++STORE, 140164965851136, 140164982636543, ++STORE, 140164940673024, 140164965851135, ++SNULL, 140164965855231, 140164982636543, ++STORE, 140164965851136, 140164965855231, ++STORE, 140164965855232, 140164982636543, ++SNULL, 140164965855232, 140164974243839, ++STORE, 140164974243840, 140164982636543, ++STORE, 140164965855232, 140164974243839, ++SNULL, 140164974247935, 140164982636543, ++STORE, 140164974243840, 140164974247935, ++STORE, 140164974247936, 140164982636543, ++SNULL, 140164445765631, 140164579983359, ++STORE, 140164378656768, 140164445765631, ++STORE, 140164445765632, 140164579983359, ++SNULL, 140164445765632, 140164512874495, ++STORE, 140164512874496, 140164579983359, ++STORE, 140164445765632, 140164512874495, ++ERASE, 140164445765632, 140164512874495, ++SNULL, 140164378791935, 140164445765631, ++STORE, 140164378656768, 140164378791935, ++STORE, 140164378791936, 140164445765631, ++SNULL, 140164789673983, 140164806455295, ++STORE, 140164789669888, 140164789673983, ++STORE, 140164789673984, 140164806455295, ++SNULL, 140164789673984, 140164798062591, ++STORE, 140164798062592, 140164806455295, ++STORE, 140164789673984, 140164798062591, ++SNULL, 140164798066687, 140164806455295, ++STORE, 140164798062592, 140164798066687, ++STORE, 140164798066688, 140164806455295, ++SNULL, 140164638670847, 140164655452159, ++STORE, 140164638666752, 140164638670847, ++STORE, 140164638670848, 140164655452159, ++STORE, 140165100068864, 140165116854271, ++STORE, 140165091676160, 140165116854271, ++STORE, 140165083283456, 140165116854271, ++SNULL, 140164244574207, 140164311547903, ++STORE, 140164244439040, 140164244574207, ++STORE, 140164244574208, 140164311547903, ++SNULL, 140164848553983, 140164915527679, ++STORE, 140164848418816, 140164848553983, ++STORE, 140164848553984, 140164915527679, ++SNULL, 140164110356479, 140164177330175, ++STORE, 140164110221312, 140164110356479, ++STORE, 140164110356480, 140164177330175, ++SNULL, 140164714336255, 140164781309951, ++STORE, 140164714201088, 140164714336255, ++STORE, 140164714336256, 140164781309951, ++SNULL, 140163976138751, 140164043112447, ++STORE, 140163976003584, 140163976138751, ++STORE, 140163976138752, 140164043112447, ++SNULL, 140164513009663, 140164579983359, ++STORE, 140164512874496, 140164513009663, ++STORE, 140164513009664, 140164579983359, ++SNULL, 140163841921023, 140163908894719, ++STORE, 140163841785856, 140163841921023, ++STORE, 140163841921024, 140163908894719, ++SNULL, 140165083283456, 140165100068863, ++STORE, 140165100068864, 140165116854271, ++STORE, 140165083283456, 140165100068863, ++SNULL, 140165100072959, 140165116854271, ++STORE, 140165100068864, 140165100072959, ++STORE, 140165100072960, 140165116854271, ++SNULL, 140165100072960, 140165108461567, ++STORE, 140165108461568, 140165116854271, ++STORE, 140165100072960, 140165108461567, ++SNULL, 140165108465663, 140165116854271, ++STORE, 140165108461568, 140165108465663, ++STORE, 140165108465664, 140165116854271, ++STORE, 140165074890752, 140165100068863, ++SNULL, 140165074894847, 140165100068863, ++STORE, 140165074890752, 140165074894847, ++STORE, 140165074894848, 140165100068863, ++STORE, 140165066498048, 140165074890751, ++STORE, 140165058105344, 140165074890751, ++STORE, 140164932280320, 140164965851135, ++SNULL, 140165192327167, 140165200715775, ++STORE, 140165192323072, 140165192327167, ++STORE, 140165192327168, 140165200715775, ++STORE, 140164923887616, 140164965851135, ++SNULL, 140164923891711, 140164965851135, ++STORE, 140164923887616, 140164923891711, ++STORE, 140164923891712, 140164965851135, ++SNULL, 140164680634368, 140164705808383, ++STORE, 140164705808384, 140164714201087, ++STORE, 140164680634368, 140164705808383, ++SNULL, 140164705812479, 140164714201087, ++STORE, 140164705808384, 140164705812479, ++STORE, 140164705812480, 140164714201087, ++SNULL, 140164680634368, 140164697415679, ++STORE, 140164697415680, 140164705808383, ++STORE, 140164680634368, 140164697415679, ++SNULL, 140164697419775, 140164705808383, ++STORE, 140164697415680, 140164697419775, ++STORE, 140164697419776, 140164705808383, ++STORE, 140164840026112, 140164848418815, ++STORE, 140164831633408, 140164848418815, ++STORE, 140164823240704, 140164848418815, ++SNULL, 140165074894848, 140165083283455, ++STORE, 140165083283456, 140165100068863, ++STORE, 140165074894848, 140165083283455, ++SNULL, 140165083287551, 140165100068863, ++STORE, 140165083283456, 140165083287551, ++STORE, 140165083287552, 140165100068863, ++SNULL, 140165083287552, 140165091676159, ++STORE, 140165091676160, 140165100068863, ++STORE, 140165083287552, 140165091676159, ++SNULL, 140165091680255, 140165100068863, ++STORE, 140165091676160, 140165091680255, ++STORE, 140165091680256, 140165100068863, ++SNULL, 140164638670848, 140164647059455, ++STORE, 140164647059456, 140164655452159, ++STORE, 140164638670848, 140164647059455, ++SNULL, 140164647063551, 140164655452159, ++STORE, 140164647059456, 140164647063551, ++STORE, 140164647063552, 140164655452159, ++SNULL, 140164923891712, 140164940673023, ++STORE, 140164940673024, 140164965851135, ++STORE, 140164923891712, 140164940673023, ++SNULL, 140164940677119, 140164965851135, ++STORE, 140164940673024, 140164940677119, ++STORE, 140164940677120, 140164965851135, ++SNULL, 140164940677120, 140164949065727, ++STORE, 140164949065728, 140164965851135, ++STORE, 140164940677120, 140164949065727, ++SNULL, 140164949069823, 140164965851135, ++STORE, 140164949065728, 140164949069823, ++STORE, 140164949069824, 140164965851135, ++SNULL, 140164949069824, 140164957458431, ++STORE, 140164957458432, 140164965851135, ++STORE, 140164949069824, 140164957458431, ++SNULL, 140164957462527, 140164965851135, ++STORE, 140164957458432, 140164957462527, ++STORE, 140164957462528, 140164965851135, ++SNULL, 140164680634368, 140164689022975, ++STORE, 140164689022976, 140164697415679, ++STORE, 140164680634368, 140164689022975, ++SNULL, 140164689027071, 140164697415679, ++STORE, 140164689022976, 140164689027071, ++STORE, 140164689027072, 140164697415679, ++STORE, 140164814848000, 140164848418815, ++SNULL, 140165058105344, 140165066498047, ++STORE, 140165066498048, 140165074890751, ++STORE, 140165058105344, 140165066498047, ++SNULL, 140165066502143, 140165074890751, ++STORE, 140165066498048, 140165066502143, ++STORE, 140165066502144, 140165074890751, ++SNULL, 140165058109439, 140165066498047, ++STORE, 140165058105344, 140165058109439, ++STORE, 140165058109440, 140165066498047, ++STORE, 140164798066688, 140164814847999, ++SNULL, 140164798066688, 140164806455295, ++STORE, 140164806455296, 140164814847999, ++STORE, 140164798066688, 140164806455295, ++SNULL, 140164806459391, 140164814847999, ++STORE, 140164806455296, 140164806459391, ++STORE, 140164806459392, 140164814847999, ++SNULL, 140164923891712, 140164932280319, ++STORE, 140164932280320, 140164940673023, ++STORE, 140164923891712, 140164932280319, ++SNULL, 140164932284415, 140164940673023, ++STORE, 140164932280320, 140164932284415, ++STORE, 140164932284416, 140164940673023, ++STORE, 140164672237568, 140164680630271, ++STORE, 140164663844864, 140164680630271, ++STORE, 140164647063552, 140164680630271, ++SNULL, 140164647063552, 140164655452159, ++STORE, 140164655452160, 140164680630271, ++STORE, 140164647063552, 140164655452159, ++SNULL, 140164655456255, 140164680630271, ++STORE, 140164655452160, 140164655456255, ++STORE, 140164655456256, 140164680630271, ++STORE, 140164630274048, 140164638666751, ++SNULL, 140164814852095, 140164848418815, ++STORE, 140164814848000, 140164814852095, ++STORE, 140164814852096, 140164848418815, ++SNULL, 140164814852096, 140164831633407, ++STORE, 140164831633408, 140164848418815, ++STORE, 140164814852096, 140164831633407, ++SNULL, 140164831637503, 140164848418815, ++STORE, 140164831633408, 140164831637503, ++STORE, 140164831637504, 140164848418815, ++STORE, 140164621881344, 140164638666751, ++SNULL, 140164831637504, 140164840026111, ++STORE, 140164840026112, 140164848418815, ++STORE, 140164831637504, 140164840026111, ++SNULL, 140164840030207, 140164848418815, ++STORE, 140164840026112, 140164840030207, ++STORE, 140164840030208, 140164848418815, ++STORE, 140164613488640, 140164638666751, ++SNULL, 140164613492735, 140164638666751, ++STORE, 140164613488640, 140164613492735, ++STORE, 140164613492736, 140164638666751, ++STORE, 140164605095936, 140164613488639, ++SNULL, 140164605100031, 140164613488639, ++STORE, 140164605095936, 140164605100031, ++STORE, 140164605100032, 140164613488639, ++STORE, 140164596703232, 140164605095935, ++STORE, 140164588310528, 140164605095935, ++SNULL, 140164588314623, 140164605095935, ++STORE, 140164588310528, 140164588314623, ++STORE, 140164588314624, 140164605095935, ++STORE, 140164504481792, 140164512874495, ++STORE, 140164496089088, 140164512874495, ++SNULL, 140164496089088, 140164504481791, ++STORE, 140164504481792, 140164512874495, ++STORE, 140164496089088, 140164504481791, ++SNULL, 140164504485887, 140164512874495, ++STORE, 140164504481792, 140164504485887, ++STORE, 140164504485888, 140164512874495, ++SNULL, 140164613492736, 140164630274047, ++STORE, 140164630274048, 140164638666751, ++STORE, 140164613492736, 140164630274047, ++SNULL, 140164630278143, 140164638666751, ++STORE, 140164630274048, 140164630278143, ++STORE, 140164630278144, 140164638666751, ++STORE, 140164487696384, 140164504481791, ++STORE, 140164479303680, 140164504481791, ++SNULL, 140164814852096, 140164823240703, ++STORE, 140164823240704, 140164831633407, ++STORE, 140164814852096, 140164823240703, ++SNULL, 140164823244799, 140164831633407, ++STORE, 140164823240704, 140164823244799, ++STORE, 140164823244800, 140164831633407, ++STORE, 140164470910976, 140164504481791, ++SNULL, 140164470910976, 140164496089087, ++STORE, 140164496089088, 140164504481791, ++STORE, 140164470910976, 140164496089087, ++SNULL, 140164496093183, 140164504481791, ++STORE, 140164496089088, 140164496093183, ++STORE, 140164496093184, 140164504481791, ++SNULL, 140164655456256, 140164672237567, ++STORE, 140164672237568, 140164680630271, ++STORE, 140164655456256, 140164672237567, ++SNULL, 140164672241663, 140164680630271, ++STORE, 140164672237568, 140164672241663, ++STORE, 140164672241664, 140164680630271, ++STORE, 140164462518272, 140164496089087, ++STORE, 140164454125568, 140164496089087, ++SNULL, 140164655456256, 140164663844863, ++STORE, 140164663844864, 140164672237567, ++STORE, 140164655456256, 140164663844863, ++SNULL, 140164663848959, 140164672237567, ++STORE, 140164663844864, 140164663848959, ++STORE, 140164663848960, 140164672237567, ++STORE, 140164370264064, 140164378656767, ++STORE, 140164361871360, 140164378656767, ++STORE, 140164353478656, 140164378656767, ++STORE, 140164345085952, 140164378656767, ++SNULL, 140164345085952, 140164353478655, ++STORE, 140164353478656, 140164378656767, ++STORE, 140164345085952, 140164353478655, ++SNULL, 140164353482751, 140164378656767, ++STORE, 140164353478656, 140164353482751, ++STORE, 140164353482752, 140164378656767, ++SNULL, 140164454125568, 140164487696383, ++STORE, 140164487696384, 140164496089087, ++STORE, 140164454125568, 140164487696383, ++SNULL, 140164487700479, 140164496089087, ++STORE, 140164487696384, 140164487700479, ++STORE, 140164487700480, 140164496089087, ++STORE, 140164336693248, 140164353478655, ++SNULL, 140164336697343, 140164353478655, ++STORE, 140164336693248, 140164336697343, ++STORE, 140164336697344, 140164353478655, ++STORE, 140164328300544, 140164336693247, ++SNULL, 140164454125568, 140164479303679, ++STORE, 140164479303680, 140164487696383, ++STORE, 140164454125568, 140164479303679, ++SNULL, 140164479307775, 140164487696383, ++STORE, 140164479303680, 140164479307775, ++STORE, 140164479307776, 140164487696383, ++STORE, 140164319907840, 140164336693247, ++STORE, 140164236046336, 140164244439039, ++SNULL, 140164588314624, 140164596703231, ++STORE, 140164596703232, 140164605095935, ++STORE, 140164588314624, 140164596703231, ++SNULL, 140164596707327, 140164605095935, ++STORE, 140164596703232, 140164596707327, ++STORE, 140164596707328, 140164605095935, ++SNULL, 140164454125568, 140164462518271, ++STORE, 140164462518272, 140164479303679, ++STORE, 140164454125568, 140164462518271, ++SNULL, 140164462522367, 140164479303679, ++STORE, 140164462518272, 140164462522367, ++STORE, 140164462522368, 140164479303679, ++STORE, 140164227653632, 140164244439039, ++SNULL, 140164227657727, 140164244439039, ++STORE, 140164227653632, 140164227657727, ++STORE, 140164227657728, 140164244439039, ++SNULL, 140164462522368, 140164470910975, ++STORE, 140164470910976, 140164479303679, ++STORE, 140164462522368, 140164470910975, ++SNULL, 140164470915071, 140164479303679, ++STORE, 140164470910976, 140164470915071, ++STORE, 140164470915072, 140164479303679, ++SNULL, 140164613492736, 140164621881343, ++STORE, 140164621881344, 140164630274047, ++STORE, 140164613492736, 140164621881343, ++SNULL, 140164621885439, 140164630274047, ++STORE, 140164621881344, 140164621885439, ++STORE, 140164621885440, 140164630274047, ++SNULL, 140164353482752, 140164370264063, ++STORE, 140164370264064, 140164378656767, ++STORE, 140164353482752, 140164370264063, ++SNULL, 140164370268159, 140164378656767, ++STORE, 140164370264064, 140164370268159, ++STORE, 140164370268160, 140164378656767, ++STORE, 140164219260928, 140164227653631, ++SNULL, 140164319911935, 140164336693247, ++STORE, 140164319907840, 140164319911935, ++STORE, 140164319911936, 140164336693247, ++SNULL, 140164336697344, 140164345085951, ++STORE, 140164345085952, 140164353478655, ++STORE, 140164336697344, 140164345085951, ++SNULL, 140164345090047, 140164353478655, ++STORE, 140164345085952, 140164345090047, ++STORE, 140164345090048, 140164353478655, ++SNULL, 140164319911936, 140164328300543, ++STORE, 140164328300544, 140164336693247, ++STORE, 140164319911936, 140164328300543, ++SNULL, 140164328304639, 140164336693247, ++STORE, 140164328300544, 140164328304639, ++STORE, 140164328304640, 140164336693247, ++SNULL, 140164454129663, 140164462518271, ++STORE, 140164454125568, 140164454129663, ++STORE, 140164454129664, 140164462518271, ++STORE, 140164210868224, 140164227653631, ++STORE, 140164202475520, 140164227653631, ++STORE, 140164194082816, 140164227653631, ++SNULL, 140164194086911, 140164227653631, ++STORE, 140164194082816, 140164194086911, ++STORE, 140164194086912, 140164227653631, ++SNULL, 140164353482752, 140164361871359, ++STORE, 140164361871360, 140164370264063, ++STORE, 140164353482752, 140164361871359, ++SNULL, 140164361875455, 140164370264063, ++STORE, 140164361871360, 140164361875455, ++STORE, 140164361875456, 140164370264063, ++SNULL, 140164227657728, 140164236046335, ++STORE, 140164236046336, 140164244439039, ++STORE, 140164227657728, 140164236046335, ++SNULL, 140164236050431, 140164244439039, ++STORE, 140164236046336, 140164236050431, ++STORE, 140164236050432, 140164244439039, ++STORE, 140164185690112, 140164194082815, ++SNULL, 140164194086912, 140164219260927, ++STORE, 140164219260928, 140164227653631, ++STORE, 140164194086912, 140164219260927, ++SNULL, 140164219265023, 140164227653631, ++STORE, 140164219260928, 140164219265023, ++STORE, 140164219265024, 140164227653631, ++STORE, 140164101828608, 140164110221311, ++STORE, 140164093435904, 140164110221311, ++STORE, 140164085043200, 140164110221311, ++SNULL, 140164085047295, 140164110221311, ++STORE, 140164085043200, 140164085047295, ++STORE, 140164085047296, 140164110221311, ++STORE, 140164076650496, 140164085043199, ++SNULL, 140164185694207, 140164194082815, ++STORE, 140164185690112, 140164185694207, ++STORE, 140164185694208, 140164194082815, ++SNULL, 140164085047296, 140164101828607, ++STORE, 140164101828608, 140164110221311, ++STORE, 140164085047296, 140164101828607, ++SNULL, 140164101832703, 140164110221311, ++STORE, 140164101828608, 140164101832703, ++STORE, 140164101832704, 140164110221311, ++SNULL, 140164085047296, 140164093435903, ++STORE, 140164093435904, 140164101828607, ++STORE, 140164085047296, 140164093435903, ++SNULL, 140164093439999, 140164101828607, ++STORE, 140164093435904, 140164093439999, ++STORE, 140164093440000, 140164101828607, ++SNULL, 140164194086912, 140164202475519, ++STORE, 140164202475520, 140164219260927, ++STORE, 140164194086912, 140164202475519, ++SNULL, 140164202479615, 140164219260927, ++STORE, 140164202475520, 140164202479615, ++STORE, 140164202479616, 140164219260927, ++SNULL, 140164202479616, 140164210868223, ++STORE, 140164210868224, 140164219260927, ++STORE, 140164202479616, 140164210868223, ++SNULL, 140164210872319, 140164219260927, ++STORE, 140164210868224, 140164210872319, ++STORE, 140164210872320, 140164219260927, ++SNULL, 140164076654591, 140164085043199, ++STORE, 140164076650496, 140164076654591, ++STORE, 140164076654592, 140164085043199, ++STORE, 140164068257792, 140164076650495, ++SNULL, 140164068261887, 140164076650495, ++STORE, 140164068257792, 140164068261887, ++STORE, 140164068261888, 140164076650495, ++STORE, 140165753053184, 140165753081855, ++STORE, 140165725851648, 140165728043007, ++SNULL, 140165725851648, 140165725941759, ++STORE, 140165725941760, 140165728043007, ++STORE, 140165725851648, 140165725941759, ++SNULL, 140165728034815, 140165728043007, ++STORE, 140165725941760, 140165728034815, ++STORE, 140165728034816, 140165728043007, ++ERASE, 140165728034816, 140165728043007, ++STORE, 140165728034816, 140165728043007, ++SNULL, 140165728038911, 140165728043007, ++STORE, 140165728034816, 140165728038911, ++STORE, 140165728038912, 140165728043007, ++ERASE, 140165753053184, 140165753081855, ++ERASE, 140164638666752, 140164638670847, ++ERASE, 140164638670848, 140164647059455, ++ERASE, 140165091676160, 140165091680255, ++ERASE, 140165091680256, 140165100068863, ++ERASE, 140164613488640, 140164613492735, ++ERASE, 140164613492736, 140164621881343, ++ERASE, 140164319907840, 140164319911935, ++ERASE, 140164319911936, 140164328300543, ++ERASE, 140165620154368, 140165620158463, ++ERASE, 140165620158464, 140165628547071, ++ERASE, 140164798062592, 140164798066687, ++ERASE, 140164798066688, 140164806455295, ++ERASE, 140164789669888, 140164789673983, ++ERASE, 140164789673984, 140164798062591, ++ERASE, 140164965851136, 140164965855231, ++ERASE, 140164965855232, 140164974243839, ++ERASE, 140165074890752, 140165074894847, ++ERASE, 140165074894848, 140165083283455, ++ERASE, 140164672237568, 140164672241663, ++ERASE, 140164672241664, 140164680630271, ++ERASE, 140164454125568, 140164454129663, ++ERASE, 140164454129664, 140164462518271, ++ERASE, 140165200715776, 140165200719871, ++ERASE, 140165200719872, 140165209108479, ++ERASE, 140164932280320, 140164932284415, ++ERASE, 140164932284416, 140164940673023, ++ERASE, 140164663844864, 140164663848959, ++ERASE, 140164663848960, 140164672237567, ++ERASE, 140164697415680, 140164697419775, ++ERASE, 140164697419776, 140164705808383, ++ERASE, 140164831633408, 140164831637503, ++ERASE, 140164831637504, 140164840026111, ++ERASE, 140165192323072, 140165192327167, ++ERASE, 140165192327168, 140165200715775, ++ERASE, 140165108461568, 140165108465663, ++ERASE, 140165108465664, 140165116854271, ++ERASE, 140164840026112, 140164840030207, ++ERASE, 140164840030208, 140164848418815, ++ERASE, 140164647059456, 140164647063551, ++ERASE, 140164647063552, 140164655452159, ++ERASE, 140165083283456, 140165083287551, ++ERASE, 140165083287552, 140165091676159, ++ERASE, 140164923887616, 140164923891711, ++ERASE, 140164923891712, 140164932280319, ++ERASE, 140164823240704, 140164823244799, ++ERASE, 140164823244800, 140164831633407, ++ERASE, 140164227653632, 140164227657727, ++ERASE, 140164227657728, 140164236046335, ++ERASE, 140164957458432, 140164957462527, ++ERASE, 140164957462528, 140164965851135, ++ERASE, 140164680630272, 140164680634367, ++ERASE, 140164680634368, 140164689022975, ++ERASE, 140164974243840, 140164974247935, ++ERASE, 140164974247936, 140164982636543, ++ERASE, 140165066498048, 140165066502143, ++ERASE, 140165066502144, 140165074890751, ++ERASE, 140164621881344, 140164621885439, ++ERASE, 140164621885440, 140164630274047, ++ERASE, 140164949065728, 140164949069823, ++ERASE, 140164949069824, 140164957458431, ++ERASE, 140164588310528, 140164588314623, ++ERASE, 140164588314624, 140164596703231, ++ERASE, 140164806455296, 140164806459391, ++ERASE, 140164806459392, 140164814847999, ++ERASE, 140164940673024, 140164940677119, ++ERASE, 140164940677120, 140164949065727, ++ERASE, 140164596703232, 140164596707327, ++ERASE, 140164596707328, 140164605095935, ++ERASE, 140164605095936, 140164605100031, ++ERASE, 140164605100032, 140164613488639, ++ERASE, 140164655452160, 140164655456255, ++ERASE, 140164655456256, 140164663844863, ++ERASE, 140164705808384, 140164705812479, ++ERASE, 140164705812480, 140164714201087, ++ERASE, 140164689022976, 140164689027071, ++ERASE, 140164689027072, 140164697415679, ++ERASE, 140164630274048, 140164630278143, ++ERASE, 140164630278144, 140164638666751, ++ERASE, 140164479303680, 140164479307775, ++ERASE, 140164479307776, 140164487696383, ++ERASE, 140164236046336, 140164236050431, ++ERASE, 140164236050432, 140164244439039, ++ERASE, 140164085043200, 140164085047295, ++ERASE, 140164085047296, 140164093435903, ++ERASE, 140164345085952, 140164345090047, ++ERASE, 140164345090048, 140164353478655, ++ERASE, 140164101828608, 140164101832703, ++ERASE, 140164101832704, 140164110221311, ++ERASE, 140164370264064, 140164370268159, ++ERASE, 140164370268160, 140164378656767, ++ERASE, 140164336693248, 140164336697343, ++ERASE, 140164336697344, 140164345085951, ++ERASE, 140164194082816, 140164194086911, ++ERASE, 140164194086912, 140164202475519, ++ERASE, 140164353478656, 140164353482751, ++ERASE, 140164353482752, 140164361871359, ++ERASE, 140164210868224, 140164210872319, ++ERASE, 140164210872320, 140164219260927, ++ERASE, 140164814848000, 140164814852095, ++ERASE, 140164814852096, 140164823240703, ++ERASE, 140164504481792, 140164504485887, ++ERASE, 140164504485888, 140164512874495, ++ERASE, 140165100068864, 140165100072959, ++ERASE, 140165100072960, 140165108461567, ++ERASE, 140164361871360, 140164361875455, ++ERASE, 140164361875456, 140164370264063, ++ERASE, 140164470910976, 140164470915071, ++ERASE, 140164470915072, 140164479303679, ++ERASE, 140164076650496, 140164076654591, ++ERASE, 140164076654592, 140164085043199, ++ERASE, 140164202475520, 140164202479615, ++ERASE, 140164202479616, 140164210868223, ++ERASE, 140164462518272, 140164462522367, ++ERASE, 140164462522368, 140164470910975, ++ERASE, 140165351718912, 140165351723007, ++ERASE, 140165351723008, 140165360111615, ++ERASE, 140164328300544, 140164328304639, ++ERASE, 140164328304640, 140164336693247, ++ERASE, 140164093435904, 140164093439999, ++ERASE, 140164093440000, 140164101828607, ++ERASE, 140165603368960, 140165603373055, ++ERASE, 140165603373056, 140165611761663, ++ERASE, 140165368504320, 140165368508415, ++ERASE, 140165368508416, 140165376897023, ++ERASE, 140165334933504, 140165334937599, ++ERASE, 140165334937600, 140165343326207, ++ERASE, 140165594976256, 140165594980351, ++ERASE, 140165594980352, 140165603368959, ++ERASE, 140164487696384, 140164487700479, ++ERASE, 140164487700480, 140164496089087, ++ERASE, 140164219260928, 140164219265023, ++ERASE, 140164219265024, 140164227653631, ++ERASE, 140164185690112, 140164185694207, ++ERASE, 140164185694208, 140164194082815, ++ERASE, 140164068257792, 140164068261887, ++ERASE, 140164068261888, 140164076650495, ++ERASE, 140165225893888, 140165225897983, ++ERASE, 140165225897984, 140165234286591, ++ERASE, 140165058105344, 140165058109439, ++ }; ++ unsigned long set31[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140730890784768, 140737488351231, ++SNULL, 140730890788863, 140737488351231, ++STORE, 140730890784768, 140730890788863, ++STORE, 140730890653696, 140730890788863, ++STORE, 94577123659776, 94577125912575, ++SNULL, 94577123790847, 94577125912575, ++STORE, 94577123659776, 94577123790847, ++STORE, 94577123790848, 94577125912575, ++ERASE, 94577123790848, 94577125912575, ++STORE, 94577125883904, 94577125892095, ++STORE, 94577125892096, 94577125912575, ++STORE, 140624060407808, 140624062660607, ++SNULL, 140624060551167, 140624062660607, ++STORE, 140624060407808, 140624060551167, ++STORE, 140624060551168, 140624062660607, ++ERASE, 140624060551168, 140624062660607, ++STORE, 140624062648320, 140624062656511, ++STORE, 140624062656512, 140624062660607, ++STORE, 140730892140544, 140730892144639, ++STORE, 140730892128256, 140730892140543, ++STORE, 140624062619648, 140624062648319, ++STORE, 140624062611456, 140624062619647, ++STORE, 140624058191872, 140624060407807, ++SNULL, 140624058191872, 140624058290175, ++STORE, 140624058290176, 140624060407807, ++STORE, 140624058191872, 140624058290175, ++SNULL, 140624060383231, 140624060407807, ++STORE, 140624058290176, 140624060383231, ++STORE, 140624060383232, 140624060407807, ++SNULL, 140624060383232, 140624060391423, ++STORE, 140624060391424, 140624060407807, ++STORE, 140624060383232, 140624060391423, ++ERASE, 140624060383232, 140624060391423, ++STORE, 140624060383232, 140624060391423, ++ERASE, 140624060391424, 140624060407807, ++STORE, 140624060391424, 140624060407807, ++STORE, 140624054394880, 140624058191871, ++SNULL, 140624054394880, 140624056053759, ++STORE, 140624056053760, 140624058191871, ++STORE, 140624054394880, 140624056053759, ++SNULL, 140624058150911, 140624058191871, ++STORE, 140624056053760, 140624058150911, ++STORE, 140624058150912, 140624058191871, ++SNULL, 140624058150912, 140624058175487, ++STORE, 140624058175488, 140624058191871, ++STORE, 140624058150912, 140624058175487, ++ERASE, 140624058150912, 140624058175487, ++STORE, 140624058150912, 140624058175487, ++ERASE, 140624058175488, 140624058191871, ++STORE, 140624058175488, 140624058191871, ++STORE, 140624062603264, 140624062619647, ++SNULL, 140624058167295, 140624058175487, ++STORE, 140624058150912, 140624058167295, ++STORE, 140624058167296, 140624058175487, ++SNULL, 140624060387327, 140624060391423, ++STORE, 140624060383232, 140624060387327, ++STORE, 140624060387328, 140624060391423, ++SNULL, 94577125887999, 94577125892095, ++STORE, 94577125883904, 94577125887999, ++STORE, 94577125888000, 94577125892095, ++SNULL, 140624062652415, 140624062656511, ++STORE, 140624062648320, 140624062652415, ++STORE, 140624062652416, 140624062656511, ++ERASE, 140624062619648, 140624062648319, ++STORE, 94577157709824, 94577157844991, ++STORE, 140624046002176, 140624054394879, ++SNULL, 140624046006271, 140624054394879, ++STORE, 140624046002176, 140624046006271, ++STORE, 140624046006272, 140624054394879, ++STORE, 140624037609472, 140624046002175, ++STORE, 140623903391744, 140624037609471, ++SNULL, 140623903391744, 140623940157439, ++STORE, 140623940157440, 140624037609471, ++STORE, 140623903391744, 140623940157439, ++ERASE, 140623903391744, 140623940157439, ++SNULL, 140624007266303, 140624037609471, ++STORE, 140623940157440, 140624007266303, ++STORE, 140624007266304, 140624037609471, ++ERASE, 140624007266304, 140624037609471, ++SNULL, 140623940292607, 140624007266303, ++STORE, 140623940157440, 140623940292607, ++STORE, 140623940292608, 140624007266303, ++SNULL, 140624037613567, 140624046002175, ++STORE, 140624037609472, 140624037613567, ++STORE, 140624037613568, 140624046002175, ++STORE, 140624029216768, 140624037609471, ++SNULL, 140624029220863, 140624037609471, ++STORE, 140624029216768, 140624029220863, ++STORE, 140624029220864, 140624037609471, ++STORE, 140624020824064, 140624029216767, ++SNULL, 140624020828159, 140624029216767, ++STORE, 140624020824064, 140624020828159, ++STORE, 140624020828160, 140624029216767, ++STORE, 140624012431360, 140624020824063, ++SNULL, 140624012435455, 140624020824063, ++STORE, 140624012431360, 140624012435455, ++STORE, 140624012435456, 140624020824063, ++STORE, 140623931764736, 140623940157439, ++STORE, 140623797547008, 140623931764735, ++SNULL, 140623797547008, 140623805939711, ++STORE, 140623805939712, 140623931764735, ++STORE, 140623797547008, 140623805939711, ++ERASE, 140623797547008, 140623805939711, ++SNULL, 140623873048575, 140623931764735, ++STORE, 140623805939712, 140623873048575, ++STORE, 140623873048576, 140623931764735, ++ERASE, 140623873048576, 140623931764735, ++STORE, 140623923372032, 140623940157439, ++STORE, 140623914979328, 140623940157439, ++STORE, 140623906586624, 140623940157439, ++STORE, 140623671721984, 140623873048575, ++SNULL, 140623738830847, 140623873048575, ++STORE, 140623671721984, 140623738830847, ++STORE, 140623738830848, 140623873048575, ++SNULL, 140623738830848, 140623805939711, ++STORE, 140623805939712, 140623873048575, ++STORE, 140623738830848, 140623805939711, ++ERASE, 140623738830848, 140623805939711, ++SNULL, 140623806074879, 140623873048575, ++STORE, 140623805939712, 140623806074879, ++STORE, 140623806074880, 140623873048575, ++SNULL, 140623906586624, 140623931764735, ++STORE, 140623931764736, 140623940157439, ++STORE, 140623906586624, 140623931764735, ++SNULL, 140623931768831, 140623940157439, ++STORE, 140623931764736, 140623931768831, ++STORE, 140623931768832, 140623940157439, ++STORE, 140623537504256, 140623738830847, ++SNULL, 140623537504256, 140623671721983, ++STORE, 140623671721984, 140623738830847, ++STORE, 140623537504256, 140623671721983, ++SNULL, 140623671857151, 140623738830847, ++STORE, 140623671721984, 140623671857151, ++STORE, 140623671857152, 140623738830847, ++SNULL, 140623604613119, 140623671721983, ++STORE, 140623537504256, 140623604613119, ++STORE, 140623604613120, 140623671721983, ++ERASE, 140623604613120, 140623671721983, ++SNULL, 140623537639423, 140623604613119, ++STORE, 140623537504256, 140623537639423, ++STORE, 140623537639424, 140623604613119, ++STORE, 140623537639424, 140623671721983, ++SNULL, 140623537639424, 140623604613119, ++STORE, 140623604613120, 140623671721983, ++STORE, 140623537639424, 140623604613119, ++SNULL, 140623604748287, 140623671721983, ++STORE, 140623604613120, 140623604748287, ++STORE, 140623604748288, 140623671721983, ++STORE, 140623898193920, 140623931764735, ++SNULL, 140623898193920, 140623923372031, ++STORE, 140623923372032, 140623931764735, ++STORE, 140623898193920, 140623923372031, ++SNULL, 140623923376127, 140623931764735, ++STORE, 140623923372032, 140623923376127, ++STORE, 140623923376128, 140623931764735, ++STORE, 140623889801216, 140623923372031, ++SNULL, 140623889801216, 140623898193919, ++STORE, 140623898193920, 140623923372031, ++STORE, 140623889801216, 140623898193919, ++SNULL, 140623898198015, 140623923372031, ++STORE, 140623898193920, 140623898198015, ++STORE, 140623898198016, 140623923372031, ++SNULL, 140623889805311, 140623898193919, ++STORE, 140623889801216, 140623889805311, ++STORE, 140623889805312, 140623898193919, ++SNULL, 140623898198016, 140623906586623, ++STORE, 140623906586624, 140623923372031, ++STORE, 140623898198016, 140623906586623, ++SNULL, 140623906590719, 140623923372031, ++STORE, 140623906586624, 140623906590719, ++STORE, 140623906590720, 140623923372031, ++STORE, 140623881408512, 140623889801215, ++SNULL, 140623906590720, 140623914979327, ++STORE, 140623914979328, 140623923372031, ++STORE, 140623906590720, 140623914979327, ++SNULL, 140623914983423, 140623923372031, ++STORE, 140623914979328, 140623914983423, ++STORE, 140623914983424, 140623923372031, ++SNULL, 140623881412607, 140623889801215, ++STORE, 140623881408512, 140623881412607, ++STORE, 140623881412608, 140623889801215, ++STORE, 140623797547008, 140623805939711, ++STORE, 140623789154304, 140623805939711, ++STORE, 140623780761600, 140623805939711, ++SNULL, 140623780761600, 140623789154303, ++STORE, 140623789154304, 140623805939711, ++STORE, 140623780761600, 140623789154303, ++SNULL, 140623789158399, 140623805939711, ++STORE, 140623789154304, 140623789158399, ++STORE, 140623789158400, 140623805939711, ++STORE, 140623772368896, 140623789154303, ++STORE, 140623763976192, 140623789154303, ++SNULL, 140623763976192, 140623780761599, ++STORE, 140623780761600, 140623789154303, ++STORE, 140623763976192, 140623780761599, ++SNULL, 140623780765695, 140623789154303, ++STORE, 140623780761600, 140623780765695, ++STORE, 140623780765696, 140623789154303, ++SNULL, 140623789158400, 140623797547007, ++STORE, 140623797547008, 140623805939711, ++STORE, 140623789158400, 140623797547007, ++SNULL, 140623797551103, 140623805939711, ++STORE, 140623797547008, 140623797551103, ++STORE, 140623797551104, 140623805939711, ++SNULL, 140623763976192, 140623772368895, ++STORE, 140623772368896, 140623780761599, ++STORE, 140623763976192, 140623772368895, ++SNULL, 140623772372991, 140623780761599, ++STORE, 140623772368896, 140623772372991, ++STORE, 140623772372992, 140623780761599, ++SNULL, 140623763980287, 140623772368895, ++STORE, 140623763976192, 140623763980287, ++STORE, 140623763980288, 140623772368895, ++STORE, 140623755583488, 140623763976191, ++STORE, 140623747190784, 140623763976191, ++SNULL, 140623747190784, 140623755583487, ++STORE, 140623755583488, 140623763976191, ++STORE, 140623747190784, 140623755583487, ++SNULL, 140623755587583, 140623763976191, ++STORE, 140623755583488, 140623755587583, ++STORE, 140623755587584, 140623763976191, ++STORE, 140623529111552, 140623537504255, ++SNULL, 140623747194879, 140623755583487, ++STORE, 140623747190784, 140623747194879, ++STORE, 140623747194880, 140623755583487, ++SNULL, 140623529115647, 140623537504255, ++STORE, 140623529111552, 140623529115647, ++STORE, 140623529115648, 140623537504255, ++STORE, 140623520718848, 140623529111551, ++SNULL, 140623520722943, 140623529111551, ++STORE, 140623520718848, 140623520722943, ++STORE, 140623520722944, 140623529111551, ++STORE, 140623512326144, 140623520718847, ++STORE, 140623503933440, 140623520718847, ++STORE, 140623495540736, 140623520718847, ++STORE, 140623361323008, 140623495540735, ++STORE, 140623227105280, 140623495540735, ++STORE, 140623218712576, 140623227105279, ++STORE, 140623084494848, 140623218712575, ++STORE, 140623076102144, 140623084494847, ++STORE, 140622941884416, 140623076102143, ++SNULL, 140622941884416, 140623000633343, ++STORE, 140623000633344, 140623076102143, ++STORE, 140622941884416, 140623000633343, ++ERASE, 140622941884416, 140623000633343, ++STORE, 140622992240640, 140623000633343, ++STORE, 140622983847936, 140623000633343, ++STORE, 140622849630208, 140622983847935, ++STORE, 140622841237504, 140622849630207, ++SNULL, 140622849630208, 140622866415615, ++STORE, 140622866415616, 140622983847935, ++STORE, 140622849630208, 140622866415615, ++ERASE, 140622849630208, 140622866415615, ++STORE, 140622858022912, 140622866415615, ++SNULL, 140622933524479, 140622983847935, ++STORE, 140622866415616, 140622933524479, ++STORE, 140622933524480, 140622983847935, ++ERASE, 140622933524480, 140622983847935, ++STORE, 140622975455232, 140623000633343, ++STORE, 140622707019776, 140622841237503, ++STORE, 140622967062528, 140623000633343, ++STORE, 140622572802048, 140622841237503, ++STORE, 140622958669824, 140623000633343, ++STORE, 140622438584320, 140622841237503, ++STORE, 140622950277120, 140623000633343, ++SNULL, 140622858027007, 140622866415615, ++STORE, 140622858022912, 140622858027007, ++STORE, 140622858027008, 140622866415615, ++STORE, 140622941884416, 140623000633343, ++STORE, 140622841237504, 140622858022911, ++SNULL, 140622841237504, 140622849630207, ++STORE, 140622849630208, 140622858022911, ++STORE, 140622841237504, 140622849630207, ++SNULL, 140622849634303, 140622858022911, ++STORE, 140622849630208, 140622849634303, ++STORE, 140622849634304, 140622858022911, ++STORE, 140622430191616, 140622438584319, ++SNULL, 140622430195711, 140622438584319, ++STORE, 140622430191616, 140622430195711, ++STORE, 140622430195712, 140622438584319, ++SNULL, 140623361323007, 140623495540735, ++STORE, 140623227105280, 140623361323007, ++STORE, 140623361323008, 140623495540735, ++SNULL, 140623361323008, 140623403286527, ++STORE, 140623403286528, 140623495540735, ++STORE, 140623361323008, 140623403286527, ++ERASE, 140623361323008, 140623403286527, ++SNULL, 140623470395391, 140623495540735, ++STORE, 140623403286528, 140623470395391, ++STORE, 140623470395392, 140623495540735, ++ERASE, 140623470395392, 140623495540735, ++SNULL, 140623227105280, 140623269068799, ++STORE, 140623269068800, 140623361323007, ++STORE, 140623227105280, 140623269068799, ++ERASE, 140623227105280, 140623269068799, ++SNULL, 140623084494848, 140623134851071, ++STORE, 140623134851072, 140623218712575, ++STORE, 140623084494848, 140623134851071, ++ERASE, 140623084494848, 140623134851071, ++SNULL, 140623201959935, 140623218712575, ++STORE, 140623134851072, 140623201959935, ++STORE, 140623201959936, 140623218712575, ++ERASE, 140623201959936, 140623218712575, ++SNULL, 140623067742207, 140623076102143, ++STORE, 140623000633344, 140623067742207, ++STORE, 140623067742208, 140623076102143, ++ERASE, 140623067742208, 140623076102143, ++STORE, 140622295973888, 140622430191615, ++SNULL, 140622295973888, 140622329544703, ++STORE, 140622329544704, 140622430191615, ++STORE, 140622295973888, 140622329544703, ++ERASE, 140622295973888, 140622329544703, ++SNULL, 140622866550783, 140622933524479, ++STORE, 140622866415616, 140622866550783, ++STORE, 140622866550784, 140622933524479, ++SNULL, 140622707019775, 140622841237503, ++STORE, 140622438584320, 140622707019775, ++STORE, 140622707019776, 140622841237503, ++SNULL, 140622707019776, 140622732197887, ++STORE, 140622732197888, 140622841237503, ++STORE, 140622707019776, 140622732197887, ++ERASE, 140622707019776, 140622732197887, ++SNULL, 140622799306751, 140622841237503, ++STORE, 140622732197888, 140622799306751, ++STORE, 140622799306752, 140622841237503, ++ERASE, 140622799306752, 140622841237503, ++SNULL, 140622572802047, 140622707019775, ++STORE, 140622438584320, 140622572802047, ++STORE, 140622572802048, 140622707019775, ++SNULL, 140622572802048, 140622597980159, ++STORE, 140622597980160, 140622707019775, ++STORE, 140622572802048, 140622597980159, ++ERASE, 140622572802048, 140622597980159, ++SNULL, 140622438584320, 140622463762431, ++STORE, 140622463762432, 140622572802047, ++STORE, 140622438584320, 140622463762431, ++ERASE, 140622438584320, 140622463762431, ++SNULL, 140622530871295, 140622572802047, ++STORE, 140622463762432, 140622530871295, ++STORE, 140622530871296, 140622572802047, ++ERASE, 140622530871296, 140622572802047, ++STORE, 140622195326976, 140622430191615, ++SNULL, 140622262435839, 140622430191615, ++STORE, 140622195326976, 140622262435839, ++STORE, 140622262435840, 140622430191615, ++SNULL, 140622262435840, 140622329544703, ++STORE, 140622329544704, 140622430191615, ++STORE, 140622262435840, 140622329544703, ++ERASE, 140622262435840, 140622329544703, ++SNULL, 140622841241599, 140622849630207, ++STORE, 140622841237504, 140622841241599, ++STORE, 140622841241600, 140622849630207, ++STORE, 140623487148032, 140623520718847, ++STORE, 140623478755328, 140623520718847, ++SNULL, 140622941884416, 140622983847935, ++STORE, 140622983847936, 140623000633343, ++STORE, 140622941884416, 140622983847935, ++SNULL, 140622983852031, 140623000633343, ++STORE, 140622983847936, 140622983852031, ++STORE, 140622983852032, 140623000633343, ++STORE, 140623394893824, 140623403286527, ++SNULL, 140623394897919, 140623403286527, ++STORE, 140623394893824, 140623394897919, ++STORE, 140623394897920, 140623403286527, ++SNULL, 140623403421695, 140623470395391, ++STORE, 140623403286528, 140623403421695, ++STORE, 140623403421696, 140623470395391, ++SNULL, 140623478755328, 140623503933439, ++STORE, 140623503933440, 140623520718847, ++STORE, 140623478755328, 140623503933439, ++SNULL, 140623503937535, 140623520718847, ++STORE, 140623503933440, 140623503937535, ++STORE, 140623503937536, 140623520718847, ++SNULL, 140623336177663, 140623361323007, ++STORE, 140623269068800, 140623336177663, ++STORE, 140623336177664, 140623361323007, ++ERASE, 140623336177664, 140623361323007, ++SNULL, 140623269203967, 140623336177663, ++STORE, 140623269068800, 140623269203967, ++STORE, 140623269203968, 140623336177663, ++SNULL, 140623134986239, 140623201959935, ++STORE, 140623134851072, 140623134986239, ++STORE, 140623134986240, 140623201959935, ++SNULL, 140623000768511, 140623067742207, ++STORE, 140623000633344, 140623000768511, ++STORE, 140623000768512, 140623067742207, ++SNULL, 140622396653567, 140622430191615, ++STORE, 140622329544704, 140622396653567, ++STORE, 140622396653568, 140622430191615, ++ERASE, 140622396653568, 140622430191615, ++SNULL, 140622732333055, 140622799306751, ++STORE, 140622732197888, 140622732333055, ++STORE, 140622732333056, 140622799306751, ++SNULL, 140622941884416, 140622975455231, ++STORE, 140622975455232, 140622983847935, ++STORE, 140622941884416, 140622975455231, ++SNULL, 140622975459327, 140622983847935, ++STORE, 140622975455232, 140622975459327, ++STORE, 140622975459328, 140622983847935, ++SNULL, 140622665089023, 140622707019775, ++STORE, 140622597980160, 140622665089023, ++STORE, 140622665089024, 140622707019775, ++ERASE, 140622665089024, 140622707019775, ++SNULL, 140622598115327, 140622665089023, ++STORE, 140622597980160, 140622598115327, ++STORE, 140622598115328, 140622665089023, ++SNULL, 140622463897599, 140622530871295, ++STORE, 140622463762432, 140622463897599, ++STORE, 140622463897600, 140622530871295, ++SNULL, 140622195462143, 140622262435839, ++STORE, 140622195326976, 140622195462143, ++STORE, 140622195462144, 140622262435839, ++STORE, 140623386501120, 140623394893823, ++SNULL, 140622941884416, 140622950277119, ++STORE, 140622950277120, 140622975455231, ++STORE, 140622941884416, 140622950277119, ++SNULL, 140622950281215, 140622975455231, ++STORE, 140622950277120, 140622950281215, ++STORE, 140622950281216, 140622975455231, ++SNULL, 140622941888511, 140622950277119, ++STORE, 140622941884416, 140622941888511, ++STORE, 140622941888512, 140622950277119, ++STORE, 140623378108416, 140623394893823, ++SNULL, 140623478755328, 140623495540735, ++STORE, 140623495540736, 140623503933439, ++STORE, 140623478755328, 140623495540735, ++SNULL, 140623495544831, 140623503933439, ++STORE, 140623495540736, 140623495544831, ++STORE, 140623495544832, 140623503933439, ++SNULL, 140623478755328, 140623487148031, ++STORE, 140623487148032, 140623495540735, ++STORE, 140623478755328, 140623487148031, ++SNULL, 140623487152127, 140623495540735, ++STORE, 140623487148032, 140623487152127, ++STORE, 140623487152128, 140623495540735, ++SNULL, 140623218716671, 140623227105279, ++STORE, 140623218712576, 140623218716671, ++STORE, 140623218716672, 140623227105279, ++SNULL, 140623076106239, 140623084494847, ++STORE, 140623076102144, 140623076106239, ++STORE, 140623076106240, 140623084494847, ++SNULL, 140622329679871, 140622396653567, ++STORE, 140622329544704, 140622329679871, ++STORE, 140622329679872, 140622396653567, ++SNULL, 140622950281216, 140622958669823, ++STORE, 140622958669824, 140622975455231, ++STORE, 140622950281216, 140622958669823, ++SNULL, 140622958673919, 140622975455231, ++STORE, 140622958669824, 140622958673919, ++STORE, 140622958673920, 140622975455231, ++SNULL, 140623503937536, 140623512326143, ++STORE, 140623512326144, 140623520718847, ++STORE, 140623503937536, 140623512326143, ++SNULL, 140623512330239, 140623520718847, ++STORE, 140623512326144, 140623512330239, ++STORE, 140623512330240, 140623520718847, ++SNULL, 140623378108416, 140623386501119, ++STORE, 140623386501120, 140623394893823, ++STORE, 140623378108416, 140623386501119, ++SNULL, 140623386505215, 140623394893823, ++STORE, 140623386501120, 140623386505215, ++STORE, 140623386505216, 140623394893823, ++STORE, 140623369715712, 140623386501119, ++STORE, 140623361323008, 140623386501119, ++STORE, 140623352930304, 140623386501119, ++SNULL, 140623352930304, 140623361323007, ++STORE, 140623361323008, 140623386501119, ++STORE, 140623352930304, 140623361323007, ++SNULL, 140623361327103, 140623386501119, ++STORE, 140623361323008, 140623361327103, ++STORE, 140623361327104, 140623386501119, ++SNULL, 140623478759423, 140623487148031, ++STORE, 140623478755328, 140623478759423, ++STORE, 140623478759424, 140623487148031, ++STORE, 140623344537600, 140623361323007, ++STORE, 140623260676096, 140623269068799, ++SNULL, 140622958673920, 140622967062527, ++STORE, 140622967062528, 140622975455231, ++STORE, 140622958673920, 140622967062527, ++SNULL, 140622967066623, 140622975455231, ++STORE, 140622967062528, 140622967066623, ++STORE, 140622967066624, 140622975455231, ++STORE, 140623252283392, 140623269068799, ++STORE, 140623243890688, 140623269068799, ++SNULL, 140622983852032, 140622992240639, ++STORE, 140622992240640, 140623000633343, ++STORE, 140622983852032, 140622992240639, ++SNULL, 140622992244735, 140623000633343, ++STORE, 140622992240640, 140622992244735, ++STORE, 140622992244736, 140623000633343, ++STORE, 140623235497984, 140623269068799, ++STORE, 140623218716672, 140623235497983, ++STORE, 140623210319872, 140623218712575, ++STORE, 140623126458368, 140623134851071, ++SNULL, 140623210323967, 140623218712575, ++STORE, 140623210319872, 140623210323967, ++STORE, 140623210323968, 140623218712575, ++SNULL, 140623218716672, 140623227105279, ++STORE, 140623227105280, 140623235497983, ++STORE, 140623218716672, 140623227105279, ++SNULL, 140623227109375, 140623235497983, ++STORE, 140623227105280, 140623227109375, ++STORE, 140623227109376, 140623235497983, ++STORE, 140623118065664, 140623134851071, ++STORE, 140623109672960, 140623134851071, ++SNULL, 140623109677055, 140623134851071, ++STORE, 140623109672960, 140623109677055, ++STORE, 140623109677056, 140623134851071, ++STORE, 140623101280256, 140623109672959, ++STORE, 140623092887552, 140623109672959, ++SNULL, 140623092887552, 140623101280255, ++STORE, 140623101280256, 140623109672959, ++STORE, 140623092887552, 140623101280255, ++SNULL, 140623101284351, 140623109672959, ++STORE, 140623101280256, 140623101284351, ++STORE, 140623101284352, 140623109672959, ++SNULL, 140623361327104, 140623378108415, ++STORE, 140623378108416, 140623386501119, ++STORE, 140623361327104, 140623378108415, ++SNULL, 140623378112511, 140623386501119, ++STORE, 140623378108416, 140623378112511, ++STORE, 140623378112512, 140623386501119, ++SNULL, 140623235497984, 140623243890687, ++STORE, 140623243890688, 140623269068799, ++STORE, 140623235497984, 140623243890687, ++SNULL, 140623243894783, 140623269068799, ++STORE, 140623243890688, 140623243894783, ++STORE, 140623243894784, 140623269068799, ++SNULL, 140623361327104, 140623369715711, ++STORE, 140623369715712, 140623378108415, ++STORE, 140623361327104, 140623369715711, ++SNULL, 140623369719807, 140623378108415, ++STORE, 140623369715712, 140623369719807, ++STORE, 140623369719808, 140623378108415, ++SNULL, 140623243894784, 140623252283391, ++STORE, 140623252283392, 140623269068799, ++STORE, 140623243894784, 140623252283391, ++SNULL, 140623252287487, 140623269068799, ++STORE, 140623252283392, 140623252287487, ++STORE, 140623252287488, 140623269068799, ++SNULL, 140623235502079, 140623243890687, ++STORE, 140623235497984, 140623235502079, ++STORE, 140623235502080, 140623243890687, ++SNULL, 140623344541695, 140623361323007, ++STORE, 140623344537600, 140623344541695, ++STORE, 140623344541696, 140623361323007, ++STORE, 140623076106240, 140623092887551, ++SNULL, 140623076106240, 140623084494847, ++STORE, 140623084494848, 140623092887551, ++STORE, 140623076106240, 140623084494847, ++SNULL, 140623084498943, 140623092887551, ++STORE, 140623084494848, 140623084498943, ++STORE, 140623084498944, 140623092887551, ++SNULL, 140623344541696, 140623352930303, ++STORE, 140623352930304, 140623361323007, ++STORE, 140623344541696, 140623352930303, ++SNULL, 140623352934399, 140623361323007, ++STORE, 140623352930304, 140623352934399, ++STORE, 140623352934400, 140623361323007, ++SNULL, 140623109677056, 140623118065663, ++STORE, 140623118065664, 140623134851071, ++STORE, 140623109677056, 140623118065663, ++SNULL, 140623118069759, 140623134851071, ++STORE, 140623118065664, 140623118069759, ++STORE, 140623118069760, 140623134851071, ++STORE, 140622832844800, 140622841237503, ++STORE, 140622824452096, 140622841237503, ++SNULL, 140622824452096, 140622832844799, ++STORE, 140622832844800, 140622841237503, ++STORE, 140622824452096, 140622832844799, ++SNULL, 140622832848895, 140622841237503, ++STORE, 140622832844800, 140622832848895, ++STORE, 140622832848896, 140622841237503, ++STORE, 140622816059392, 140622832844799, ++SNULL, 140623092891647, 140623101280255, ++STORE, 140623092887552, 140623092891647, ++STORE, 140623092891648, 140623101280255, ++SNULL, 140623118069760, 140623126458367, ++STORE, 140623126458368, 140623134851071, ++STORE, 140623118069760, 140623126458367, ++SNULL, 140623126462463, 140623134851071, ++STORE, 140623126458368, 140623126462463, ++STORE, 140623126462464, 140623134851071, ++SNULL, 140623252287488, 140623260676095, ++STORE, 140623260676096, 140623269068799, ++STORE, 140623252287488, 140623260676095, ++SNULL, 140623260680191, 140623269068799, ++STORE, 140623260676096, 140623260680191, ++STORE, 140623260680192, 140623269068799, ++STORE, 140622807666688, 140622832844799, ++STORE, 140622723805184, 140622732197887, ++STORE, 140622715412480, 140622732197887, ++STORE, 140622707019776, 140622732197887, ++SNULL, 140622707023871, 140622732197887, ++STORE, 140622707019776, 140622707023871, ++STORE, 140622707023872, 140622732197887, ++STORE, 140622698627072, 140622707019775, ++STORE, 140622690234368, 140622707019775, ++SNULL, 140622690238463, 140622707019775, ++STORE, 140622690234368, 140622690238463, ++STORE, 140622690238464, 140622707019775, ++SNULL, 140622807666688, 140622816059391, ++STORE, 140622816059392, 140622832844799, ++STORE, 140622807666688, 140622816059391, ++SNULL, 140622816063487, 140622832844799, ++STORE, 140622816059392, 140622816063487, ++STORE, 140622816063488, 140622832844799, ++STORE, 140622681841664, 140622690234367, ++STORE, 140622673448960, 140622690234367, ++SNULL, 140622673453055, 140622690234367, ++STORE, 140622673448960, 140622673453055, ++STORE, 140622673453056, 140622690234367, ++STORE, 140622589587456, 140622597980159, ++SNULL, 140622807670783, 140622816059391, ++STORE, 140622807666688, 140622807670783, ++STORE, 140622807670784, 140622816059391, ++STORE, 140622581194752, 140622597980159, ++SNULL, 140622581198847, 140622597980159, ++STORE, 140622581194752, 140622581198847, ++STORE, 140622581198848, 140622597980159, ++SNULL, 140622816063488, 140622824452095, ++STORE, 140622824452096, 140622832844799, ++STORE, 140622816063488, 140622824452095, ++SNULL, 140622824456191, 140622832844799, ++STORE, 140622824452096, 140622824456191, ++STORE, 140622824456192, 140622832844799, ++STORE, 140622572802048, 140622581194751, ++SNULL, 140622572806143, 140622581194751, ++STORE, 140622572802048, 140622572806143, ++STORE, 140622572806144, 140622581194751, ++STORE, 140622564409344, 140622572802047, ++STORE, 140622556016640, 140622572802047, ++SNULL, 140622556016640, 140622564409343, ++STORE, 140622564409344, 140622572802047, ++STORE, 140622556016640, 140622564409343, ++SNULL, 140622564413439, 140622572802047, ++STORE, 140622564409344, 140622564413439, ++STORE, 140622564413440, 140622572802047, ++SNULL, 140622690238464, 140622698627071, ++STORE, 140622698627072, 140622707019775, ++STORE, 140622690238464, 140622698627071, ++SNULL, 140622698631167, 140622707019775, ++STORE, 140622698627072, 140622698631167, ++STORE, 140622698631168, 140622707019775, ++SNULL, 140622707023872, 140622723805183, ++STORE, 140622723805184, 140622732197887, ++STORE, 140622707023872, 140622723805183, ++SNULL, 140622723809279, 140622732197887, ++STORE, 140622723805184, 140622723809279, ++STORE, 140622723809280, 140622732197887, ++SNULL, 140622707023872, 140622715412479, ++STORE, 140622715412480, 140622723805183, ++STORE, 140622707023872, 140622715412479, ++SNULL, 140622715416575, 140622723805183, ++STORE, 140622715412480, 140622715416575, ++STORE, 140622715416576, 140622723805183, ++STORE, 140622547623936, 140622564409343, ++SNULL, 140622547628031, 140622564409343, ++STORE, 140622547623936, 140622547628031, ++STORE, 140622547628032, 140622564409343, ++STORE, 140622539231232, 140622547623935, ++SNULL, 140622539235327, 140622547623935, ++STORE, 140622539231232, 140622539235327, ++STORE, 140622539235328, 140622547623935, ++SNULL, 140622581198848, 140622589587455, ++STORE, 140622589587456, 140622597980159, ++STORE, 140622581198848, 140622589587455, ++SNULL, 140622589591551, 140622597980159, ++STORE, 140622589587456, 140622589591551, ++STORE, 140622589591552, 140622597980159, ++STORE, 140622455369728, 140622463762431, ++SNULL, 140622455373823, 140622463762431, ++STORE, 140622455369728, 140622455373823, ++STORE, 140622455373824, 140622463762431, ++STORE, 140622446977024, 140622455369727, ++SNULL, 140622446981119, 140622455369727, ++STORE, 140622446977024, 140622446981119, ++STORE, 140622446981120, 140622455369727, ++SNULL, 140622547628032, 140622556016639, ++STORE, 140622556016640, 140622564409343, ++STORE, 140622547628032, 140622556016639, ++SNULL, 140622556020735, 140622564409343, ++STORE, 140622556016640, 140622556020735, ++STORE, 140622556020736, 140622564409343, ++STORE, 140622430195712, 140622446977023, ++STORE, 140622421798912, 140622430191615, ++SNULL, 140622430195712, 140622438584319, ++STORE, 140622438584320, 140622446977023, ++STORE, 140622430195712, 140622438584319, ++SNULL, 140622438588415, 140622446977023, ++STORE, 140622438584320, 140622438588415, ++STORE, 140622438588416, 140622446977023, ++STORE, 140622413406208, 140622430191615, ++STORE, 140622405013504, 140622430191615, ++SNULL, 140622405013504, 140622413406207, ++STORE, 140622413406208, 140622430191615, ++STORE, 140622405013504, 140622413406207, ++SNULL, 140622413410303, 140622430191615, ++STORE, 140622413406208, 140622413410303, ++STORE, 140622413410304, 140622430191615, ++SNULL, 140622673453056, 140622681841663, ++STORE, 140622681841664, 140622690234367, ++STORE, 140622673453056, 140622681841663, ++SNULL, 140622681845759, 140622690234367, ++STORE, 140622681841664, 140622681845759, ++STORE, 140622681845760, 140622690234367, ++STORE, 140622321152000, 140622329544703, ++SNULL, 140622413410304, 140622421798911, ++STORE, 140622421798912, 140622430191615, ++STORE, 140622413410304, 140622421798911, ++SNULL, 140622421803007, 140622430191615, ++STORE, 140622421798912, 140622421803007, ++STORE, 140622421803008, 140622430191615, ++STORE, 140622312759296, 140622329544703, ++SNULL, 140622312763391, 140622329544703, ++STORE, 140622312759296, 140622312763391, ++STORE, 140622312763392, 140622329544703, ++SNULL, 140622405017599, 140622413406207, ++STORE, 140622405013504, 140622405017599, ++STORE, 140622405017600, 140622413406207, ++STORE, 140622304366592, 140622312759295, ++SNULL, 140622304370687, 140622312759295, ++STORE, 140622304366592, 140622304370687, ++STORE, 140622304370688, 140622312759295, ++SNULL, 140622312763392, 140622321151999, ++STORE, 140622321152000, 140622329544703, ++STORE, 140622312763392, 140622321151999, ++SNULL, 140622321156095, 140622329544703, ++STORE, 140622321152000, 140622321156095, ++STORE, 140622321156096, 140622329544703, ++STORE, 140624062619648, 140624062648319, ++STORE, 140624010240000, 140624012431359, ++SNULL, 140624010240000, 140624010330111, ++STORE, 140624010330112, 140624012431359, ++STORE, 140624010240000, 140624010330111, ++SNULL, 140624012423167, 140624012431359, ++STORE, 140624010330112, 140624012423167, ++STORE, 140624012423168, 140624012431359, ++ERASE, 140624012423168, 140624012431359, ++STORE, 140624012423168, 140624012431359, ++SNULL, 140624012427263, 140624012431359, ++STORE, 140624012423168, 140624012427263, ++STORE, 140624012427264, 140624012431359, ++ERASE, 140624062619648, 140624062648319, ++ERASE, 140622849630208, 140622849634303, ++ERASE, 140622849634304, 140622858022911, ++ERASE, 140623394893824, 140623394897919, ++ERASE, 140623394897920, 140623403286527, ++ERASE, 140623361323008, 140623361327103, ++ERASE, 140623361327104, 140623369715711, ++ERASE, 140623084494848, 140623084498943, ++ERASE, 140623084498944, 140623092887551, ++ERASE, 140623931764736, 140623931768831, ++ERASE, 140623931768832, 140623940157439, ++ERASE, 140622841237504, 140622841241599, ++ERASE, 140622841241600, 140622849630207, ++ERASE, 140623487148032, 140623487152127, ++ERASE, 140623487152128, 140623495540735, ++ERASE, 140623109672960, 140623109677055, ++ERASE, 140623109677056, 140623118065663, ++ERASE, 140622983847936, 140622983852031, ++ERASE, 140622983852032, 140622992240639, ++ERASE, 140623352930304, 140623352934399, ++ERASE, 140623352934400, 140623361323007, ++ERASE, 140622564409344, 140622564413439, ++ERASE, 140622564413440, 140622572802047, ++ERASE, 140622430191616, 140622430195711, ++ERASE, 140622430195712, 140622438584319, ++ERASE, 140622958669824, 140622958673919, ++ERASE, 140622958673920, 140622967062527, ++ERASE, 140622992240640, 140622992244735, ++ERASE, 140622992244736, 140623000633343, ++ERASE, 140623227105280, 140623227109375, ++ERASE, 140623227109376, 140623235497983, ++ERASE, 140622321152000, 140622321156095, ++ERASE, 140622321156096, 140622329544703, ++ERASE, 140622858022912, 140622858027007, ++ERASE, 140622858027008, 140622866415615, ++ERASE, 140622975455232, 140622975459327, ++ERASE, 140622975459328, 140622983847935, ++ERASE, 140623378108416, 140623378112511, ++ERASE, 140623378112512, 140623386501119, ++ERASE, 140623495540736, 140623495544831, ++ERASE, 140623495544832, 140623503933439, ++ERASE, 140623118065664, 140623118069759, ++ERASE, 140623118069760, 140623126458367, ++ERASE, 140622572802048, 140622572806143, ++ERASE, 140622572806144, 140622581194751, ++ERASE, 140622421798912, 140622421803007, ++ERASE, 140622421803008, 140622430191615, ++ERASE, 140622967062528, 140622967066623, ++ERASE, 140622967066624, 140622975455231, ++ERASE, 140623252283392, 140623252287487, ++ERASE, 140623252287488, 140623260676095, ++ERASE, 140622673448960, 140622673453055, ++ERASE, 140622673453056, 140622681841663, ++ERASE, 140623076102144, 140623076106239, ++ERASE, 140623076106240, 140623084494847, ++ERASE, 140623101280256, 140623101284351, ++ERASE, 140623101284352, 140623109672959, ++ERASE, 140622715412480, 140622715416575, ++ERASE, 140622715416576, 140622723805183, ++ERASE, 140622405013504, 140622405017599, ++ERASE, 140622405017600, 140622413406207, ++ERASE, 140623478755328, 140623478759423, ++ERASE, 140623478759424, 140623487148031, ++ERASE, 140623906586624, 140623906590719, ++ERASE, 140623906590720, 140623914979327, ++ERASE, 140622950277120, 140622950281215, ++ERASE, 140622950281216, 140622958669823, ++ }; ++ unsigned long set32[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140731244212224, 140737488351231, ++SNULL, 140731244216319, 140737488351231, ++STORE, 140731244212224, 140731244216319, ++STORE, 140731244081152, 140731244216319, ++STORE, 94427773984768, 94427776237567, ++SNULL, 94427774115839, 94427776237567, ++STORE, 94427773984768, 94427774115839, ++STORE, 94427774115840, 94427776237567, ++ERASE, 94427774115840, 94427776237567, ++STORE, 94427776208896, 94427776217087, ++STORE, 94427776217088, 94427776237567, ++STORE, 140401464893440, 140401467146239, ++SNULL, 140401465036799, 140401467146239, ++STORE, 140401464893440, 140401465036799, ++STORE, 140401465036800, 140401467146239, ++ERASE, 140401465036800, 140401467146239, ++STORE, 140401467133952, 140401467142143, ++STORE, 140401467142144, 140401467146239, ++STORE, 140731244507136, 140731244511231, ++STORE, 140731244494848, 140731244507135, ++STORE, 140401467105280, 140401467133951, ++STORE, 140401467097088, 140401467105279, ++STORE, 140401462677504, 140401464893439, ++SNULL, 140401462677504, 140401462775807, ++STORE, 140401462775808, 140401464893439, ++STORE, 140401462677504, 140401462775807, ++SNULL, 140401464868863, 140401464893439, ++STORE, 140401462775808, 140401464868863, ++STORE, 140401464868864, 140401464893439, ++SNULL, 140401464868864, 140401464877055, ++STORE, 140401464877056, 140401464893439, ++STORE, 140401464868864, 140401464877055, ++ERASE, 140401464868864, 140401464877055, ++STORE, 140401464868864, 140401464877055, ++ERASE, 140401464877056, 140401464893439, ++STORE, 140401464877056, 140401464893439, ++STORE, 140401458880512, 140401462677503, ++SNULL, 140401458880512, 140401460539391, ++STORE, 140401460539392, 140401462677503, ++STORE, 140401458880512, 140401460539391, ++SNULL, 140401462636543, 140401462677503, ++STORE, 140401460539392, 140401462636543, ++STORE, 140401462636544, 140401462677503, ++SNULL, 140401462636544, 140401462661119, ++STORE, 140401462661120, 140401462677503, ++STORE, 140401462636544, 140401462661119, ++ERASE, 140401462636544, 140401462661119, ++STORE, 140401462636544, 140401462661119, ++ERASE, 140401462661120, 140401462677503, ++STORE, 140401462661120, 140401462677503, ++STORE, 140401467088896, 140401467105279, ++SNULL, 140401462652927, 140401462661119, ++STORE, 140401462636544, 140401462652927, ++STORE, 140401462652928, 140401462661119, ++SNULL, 140401464872959, 140401464877055, ++STORE, 140401464868864, 140401464872959, ++STORE, 140401464872960, 140401464877055, ++SNULL, 94427776212991, 94427776217087, ++STORE, 94427776208896, 94427776212991, ++STORE, 94427776212992, 94427776217087, ++SNULL, 140401467138047, 140401467142143, ++STORE, 140401467133952, 140401467138047, ++STORE, 140401467138048, 140401467142143, ++ERASE, 140401467105280, 140401467133951, ++STORE, 94427784683520, 94427784818687, ++STORE, 140401450487808, 140401458880511, ++SNULL, 140401450491903, 140401458880511, ++STORE, 140401450487808, 140401450491903, ++STORE, 140401450491904, 140401458880511, ++STORE, 140401442095104, 140401450487807, ++STORE, 140401307877376, 140401442095103, ++SNULL, 140401307877376, 140401340055551, ++STORE, 140401340055552, 140401442095103, ++STORE, 140401307877376, 140401340055551, ++ERASE, 140401307877376, 140401340055551, ++SNULL, 140401407164415, 140401442095103, ++STORE, 140401340055552, 140401407164415, ++STORE, 140401407164416, 140401442095103, ++ERASE, 140401407164416, 140401442095103, ++SNULL, 140401340190719, 140401407164415, ++STORE, 140401340055552, 140401340190719, ++STORE, 140401340190720, 140401407164415, ++SNULL, 140401442099199, 140401450487807, ++STORE, 140401442095104, 140401442099199, ++STORE, 140401442099200, 140401450487807, ++STORE, 140401433702400, 140401442095103, ++SNULL, 140401433706495, 140401442095103, ++STORE, 140401433702400, 140401433706495, ++STORE, 140401433706496, 140401442095103, ++STORE, 140401425309696, 140401433702399, ++SNULL, 140401425313791, 140401433702399, ++STORE, 140401425309696, 140401425313791, ++STORE, 140401425313792, 140401433702399, ++STORE, 140401416916992, 140401425309695, ++SNULL, 140401416921087, 140401425309695, ++STORE, 140401416916992, 140401416921087, ++STORE, 140401416921088, 140401425309695, ++STORE, 140401408524288, 140401416916991, ++STORE, 140401205837824, 140401340055551, ++SNULL, 140401272946687, 140401340055551, ++STORE, 140401205837824, 140401272946687, ++STORE, 140401272946688, 140401340055551, ++ERASE, 140401272946688, 140401340055551, ++SNULL, 140401205972991, 140401272946687, ++STORE, 140401205837824, 140401205972991, ++STORE, 140401205972992, 140401272946687, ++STORE, 140401331662848, 140401340055551, ++STORE, 140401323270144, 140401340055551, ++STORE, 140401138728960, 140401205837823, ++STORE, 140401314877440, 140401340055551, ++SNULL, 140401408528383, 140401416916991, ++STORE, 140401408524288, 140401408528383, ++STORE, 140401408528384, 140401416916991, ++SNULL, 140401138864127, 140401205837823, ++STORE, 140401138728960, 140401138864127, ++STORE, 140401138864128, 140401205837823, ++STORE, 140401004511232, 140401138728959, ++SNULL, 140401071620095, 140401138728959, ++STORE, 140401004511232, 140401071620095, ++STORE, 140401071620096, 140401138728959, ++ERASE, 140401071620096, 140401138728959, ++STORE, 140400870293504, 140401071620095, ++SNULL, 140400937402367, 140401071620095, ++STORE, 140400870293504, 140400937402367, ++STORE, 140400937402368, 140401071620095, ++SNULL, 140400937402368, 140401004511231, ++STORE, 140401004511232, 140401071620095, ++STORE, 140400937402368, 140401004511231, ++ERASE, 140400937402368, 140401004511231, ++STORE, 140401306484736, 140401340055551, ++SNULL, 140401306484736, 140401323270143, ++STORE, 140401323270144, 140401340055551, ++STORE, 140401306484736, 140401323270143, ++SNULL, 140401323274239, 140401340055551, ++STORE, 140401323270144, 140401323274239, ++STORE, 140401323274240, 140401340055551, ++SNULL, 140401004646399, 140401071620095, ++STORE, 140401004511232, 140401004646399, ++STORE, 140401004646400, 140401071620095, ++SNULL, 140400870428671, 140400937402367, ++STORE, 140400870293504, 140400870428671, ++STORE, 140400870428672, 140400937402367, ++SNULL, 140401306488831, 140401323270143, ++STORE, 140401306484736, 140401306488831, ++STORE, 140401306488832, 140401323270143, ++STORE, 140401298092032, 140401306484735, ++SNULL, 140401306488832, 140401314877439, ++STORE, 140401314877440, 140401323270143, ++STORE, 140401306488832, 140401314877439, ++SNULL, 140401314881535, 140401323270143, ++STORE, 140401314877440, 140401314881535, ++STORE, 140401314881536, 140401323270143, ++SNULL, 140401323274240, 140401331662847, ++STORE, 140401331662848, 140401340055551, ++STORE, 140401323274240, 140401331662847, ++SNULL, 140401331666943, 140401340055551, ++STORE, 140401331662848, 140401331666943, ++STORE, 140401331666944, 140401340055551, ++SNULL, 140401298096127, 140401306484735, ++STORE, 140401298092032, 140401298096127, ++STORE, 140401298096128, 140401306484735, ++STORE, 140401289699328, 140401298092031, ++STORE, 140401281306624, 140401298092031, ++STORE, 140401130336256, 140401138728959, ++SNULL, 140401281306624, 140401289699327, ++STORE, 140401289699328, 140401298092031, ++STORE, 140401281306624, 140401289699327, ++SNULL, 140401289703423, 140401298092031, ++STORE, 140401289699328, 140401289703423, ++STORE, 140401289703424, 140401298092031, ++STORE, 140401121943552, 140401138728959, ++STORE, 140401113550848, 140401138728959, ++SNULL, 140401281310719, 140401289699327, ++STORE, 140401281306624, 140401281310719, ++STORE, 140401281310720, 140401289699327, ++SNULL, 140401113550848, 140401121943551, ++STORE, 140401121943552, 140401138728959, ++STORE, 140401113550848, 140401121943551, ++SNULL, 140401121947647, 140401138728959, ++STORE, 140401121943552, 140401121947647, ++STORE, 140401121947648, 140401138728959, ++STORE, 140401105158144, 140401121943551, ++SNULL, 140401121947648, 140401130336255, ++STORE, 140401130336256, 140401138728959, ++STORE, 140401121947648, 140401130336255, ++SNULL, 140401130340351, 140401138728959, ++STORE, 140401130336256, 140401130340351, ++STORE, 140401130340352, 140401138728959, ++STORE, 140401096765440, 140401121943551, ++SNULL, 140401096765440, 140401113550847, ++STORE, 140401113550848, 140401121943551, ++STORE, 140401096765440, 140401113550847, ++SNULL, 140401113554943, 140401121943551, ++STORE, 140401113550848, 140401113554943, ++STORE, 140401113554944, 140401121943551, ++STORE, 140401088372736, 140401113550847, ++SNULL, 140401088372736, 140401096765439, ++STORE, 140401096765440, 140401113550847, ++STORE, 140401088372736, 140401096765439, ++SNULL, 140401096769535, 140401113550847, ++STORE, 140401096765440, 140401096769535, ++STORE, 140401096769536, 140401113550847, ++SNULL, 140401096769536, 140401105158143, ++STORE, 140401105158144, 140401113550847, ++STORE, 140401096769536, 140401105158143, ++SNULL, 140401105162239, 140401113550847, ++STORE, 140401105158144, 140401105162239, ++STORE, 140401105162240, 140401113550847, ++SNULL, 140401088376831, 140401096765439, ++STORE, 140401088372736, 140401088376831, ++STORE, 140401088376832, 140401096765439, ++STORE, 140401079980032, 140401088372735, ++STORE, 140400996118528, 140401004511231, ++SNULL, 140401079984127, 140401088372735, ++STORE, 140401079980032, 140401079984127, ++STORE, 140401079984128, 140401088372735, ++SNULL, 140400996122623, 140401004511231, ++STORE, 140400996118528, 140400996122623, ++STORE, 140400996122624, 140401004511231, ++STORE, 140400987725824, 140400996118527, ++STORE, 140400979333120, 140400996118527, ++STORE, 140400803184640, 140400870293503, ++SNULL, 140400803319807, 140400870293503, ++STORE, 140400803184640, 140400803319807, ++STORE, 140400803319808, 140400870293503, ++SNULL, 140400979333120, 140400987725823, ++STORE, 140400987725824, 140400996118527, ++STORE, 140400979333120, 140400987725823, ++SNULL, 140400987729919, 140400996118527, ++STORE, 140400987725824, 140400987729919, ++STORE, 140400987729920, 140400996118527, ++STORE, 140400970940416, 140400987725823, ++STORE, 140400962547712, 140400987725823, ++STORE, 140400668966912, 140400803184639, ++STORE, 140400954155008, 140400987725823, ++STORE, 140400945762304, 140400987725823, ++STORE, 140400660574208, 140400668966911, ++STORE, 140400593465344, 140400660574207, ++STORE, 140400585072640, 140400593465343, ++STORE, 140400450854912, 140400585072639, ++STORE, 140400442462208, 140400450854911, ++STORE, 140400434069504, 140400450854911, ++STORE, 140400299851776, 140400434069503, ++STORE, 140400291459072, 140400299851775, ++SNULL, 140400299851776, 140400333422591, ++STORE, 140400333422592, 140400434069503, ++STORE, 140400299851776, 140400333422591, ++ERASE, 140400299851776, 140400333422591, ++STORE, 140400325029888, 140400333422591, ++STORE, 140400157241344, 140400291459071, ++STORE, 140400316637184, 140400333422591, ++STORE, 140400308244480, 140400333422591, ++STORE, 140400023023616, 140400291459071, ++STORE, 140400291459072, 140400333422591, ++SNULL, 140400023023616, 140400064987135, ++STORE, 140400064987136, 140400291459071, ++STORE, 140400023023616, 140400064987135, ++ERASE, 140400023023616, 140400064987135, ++STORE, 140400056594432, 140400064987135, ++SNULL, 140400056598527, 140400064987135, ++STORE, 140400056594432, 140400056598527, ++STORE, 140400056598528, 140400064987135, ++STORE, 140399989485568, 140400056594431, ++SNULL, 140400291459072, 140400316637183, ++STORE, 140400316637184, 140400333422591, ++STORE, 140400291459072, 140400316637183, ++SNULL, 140400316641279, 140400333422591, ++STORE, 140400316637184, 140400316641279, ++STORE, 140400316641280, 140400333422591, ++STORE, 140399855267840, 140400056594431, ++SNULL, 140399855267840, 140399863660543, ++STORE, 140399863660544, 140400056594431, ++STORE, 140399855267840, 140399863660543, ++ERASE, 140399855267840, 140399863660543, ++SNULL, 140400736075775, 140400803184639, ++STORE, 140400668966912, 140400736075775, ++STORE, 140400736075776, 140400803184639, ++ERASE, 140400736075776, 140400803184639, ++SNULL, 140400669102079, 140400736075775, ++STORE, 140400668966912, 140400669102079, ++STORE, 140400669102080, 140400736075775, ++STORE, 140400669102080, 140400803184639, ++SNULL, 140400669102080, 140400736075775, ++STORE, 140400736075776, 140400803184639, ++STORE, 140400669102080, 140400736075775, ++SNULL, 140400736210943, 140400803184639, ++STORE, 140400736075776, 140400736210943, ++STORE, 140400736210944, 140400803184639, ++ERASE, 140400593465344, 140400660574207, ++SNULL, 140400450854912, 140400467640319, ++STORE, 140400467640320, 140400585072639, ++STORE, 140400450854912, 140400467640319, ++ERASE, 140400450854912, 140400467640319, ++STORE, 140399729442816, 140400056594431, ++SNULL, 140400400531455, 140400434069503, ++STORE, 140400333422592, 140400400531455, ++STORE, 140400400531456, 140400434069503, ++ERASE, 140400400531456, 140400434069503, ++SNULL, 140400333557759, 140400400531455, ++STORE, 140400333422592, 140400333557759, ++STORE, 140400333557760, 140400400531455, ++SNULL, 140400157241343, 140400291459071, ++STORE, 140400064987136, 140400157241343, ++STORE, 140400157241344, 140400291459071, ++SNULL, 140400157241344, 140400199204863, ++STORE, 140400199204864, 140400291459071, ++STORE, 140400157241344, 140400199204863, ++ERASE, 140400157241344, 140400199204863, ++SNULL, 140400266313727, 140400291459071, ++STORE, 140400199204864, 140400266313727, ++STORE, 140400266313728, 140400291459071, ++ERASE, 140400266313728, 140400291459071, ++SNULL, 140400132095999, 140400157241343, ++STORE, 140400064987136, 140400132095999, ++STORE, 140400132096000, 140400157241343, ++ERASE, 140400132096000, 140400157241343, ++SNULL, 140400065122303, 140400132095999, ++STORE, 140400064987136, 140400065122303, ++STORE, 140400065122304, 140400132095999, ++SNULL, 140400945762304, 140400954155007, ++STORE, 140400954155008, 140400987725823, ++STORE, 140400945762304, 140400954155007, ++SNULL, 140400954159103, 140400987725823, ++STORE, 140400954155008, 140400954159103, ++STORE, 140400954159104, 140400987725823, ++SNULL, 140400434069504, 140400442462207, ++STORE, 140400442462208, 140400450854911, ++STORE, 140400434069504, 140400442462207, ++SNULL, 140400442466303, 140400450854911, ++STORE, 140400442462208, 140400442466303, ++STORE, 140400442466304, 140400450854911, ++SNULL, 140400291463167, 140400316637183, ++STORE, 140400291459072, 140400291463167, ++STORE, 140400291463168, 140400316637183, ++STORE, 140400652181504, 140400668966911, ++STORE, 140400643788800, 140400668966911, ++SNULL, 140400291463168, 140400299851775, ++STORE, 140400299851776, 140400316637183, ++STORE, 140400291463168, 140400299851775, ++SNULL, 140400299855871, 140400316637183, ++STORE, 140400299851776, 140400299855871, ++STORE, 140400299855872, 140400316637183, ++STORE, 140400635396096, 140400668966911, ++SNULL, 140400635396096, 140400643788799, ++STORE, 140400643788800, 140400668966911, ++STORE, 140400635396096, 140400643788799, ++SNULL, 140400643792895, 140400668966911, ++STORE, 140400643788800, 140400643792895, ++STORE, 140400643792896, 140400668966911, ++SNULL, 140399989485567, 140400056594431, ++STORE, 140399729442816, 140399989485567, ++STORE, 140399989485568, 140400056594431, ++ERASE, 140399989485568, 140400056594431, ++SNULL, 140399930769407, 140399989485567, ++STORE, 140399729442816, 140399930769407, ++STORE, 140399930769408, 140399989485567, ++ERASE, 140399930769408, 140399989485567, ++SNULL, 140400945766399, 140400954155007, ++STORE, 140400945762304, 140400945766399, ++STORE, 140400945766400, 140400954155007, ++SNULL, 140400534749183, 140400585072639, ++STORE, 140400467640320, 140400534749183, ++STORE, 140400534749184, 140400585072639, ++ERASE, 140400534749184, 140400585072639, ++SNULL, 140399796551679, 140399930769407, ++STORE, 140399729442816, 140399796551679, ++STORE, 140399796551680, 140399930769407, ++SNULL, 140399796551680, 140399863660543, ++STORE, 140399863660544, 140399930769407, ++STORE, 140399796551680, 140399863660543, ++ERASE, 140399796551680, 140399863660543, ++SNULL, 140400199340031, 140400266313727, ++STORE, 140400199204864, 140400199340031, ++STORE, 140400199340032, 140400266313727, ++STORE, 140400627003392, 140400643788799, ++SNULL, 140400316641280, 140400325029887, ++STORE, 140400325029888, 140400333422591, ++STORE, 140400316641280, 140400325029887, ++SNULL, 140400325033983, 140400333422591, ++STORE, 140400325029888, 140400325033983, ++STORE, 140400325033984, 140400333422591, ++SNULL, 140400627003392, 140400635396095, ++STORE, 140400635396096, 140400643788799, ++STORE, 140400627003392, 140400635396095, ++SNULL, 140400635400191, 140400643788799, ++STORE, 140400635396096, 140400635400191, ++STORE, 140400635400192, 140400643788799, ++SNULL, 140400434073599, 140400442462207, ++STORE, 140400434069504, 140400434073599, ++STORE, 140400434073600, 140400442462207, ++STORE, 140400618610688, 140400635396095, ++STORE, 140400610217984, 140400635396095, ++SNULL, 140400954159104, 140400962547711, ++STORE, 140400962547712, 140400987725823, ++STORE, 140400954159104, 140400962547711, ++SNULL, 140400962551807, 140400987725823, ++STORE, 140400962547712, 140400962551807, ++STORE, 140400962551808, 140400987725823, ++SNULL, 140400299855872, 140400308244479, ++STORE, 140400308244480, 140400316637183, ++STORE, 140400299855872, 140400308244479, ++SNULL, 140400308248575, 140400316637183, ++STORE, 140400308244480, 140400308248575, ++STORE, 140400308248576, 140400316637183, ++STORE, 140400601825280, 140400635396095, ++SNULL, 140400601829375, 140400635396095, ++STORE, 140400601825280, 140400601829375, ++STORE, 140400601829376, 140400635396095, ++STORE, 140400576679936, 140400593465343, ++SNULL, 140400576684031, 140400593465343, ++STORE, 140400576679936, 140400576684031, ++STORE, 140400576684032, 140400593465343, ++SNULL, 140400643792896, 140400652181503, ++STORE, 140400652181504, 140400668966911, ++STORE, 140400643792896, 140400652181503, ++SNULL, 140400652185599, 140400668966911, ++STORE, 140400652181504, 140400652185599, ++STORE, 140400652185600, 140400668966911, ++STORE, 140399595225088, 140399796551679, ++SNULL, 140399662333951, 140399796551679, ++STORE, 140399595225088, 140399662333951, ++STORE, 140399662333952, 140399796551679, ++SNULL, 140399662333952, 140399729442815, ++STORE, 140399729442816, 140399796551679, ++STORE, 140399662333952, 140399729442815, ++ERASE, 140399662333952, 140399729442815, ++SNULL, 140399863795711, 140399930769407, ++STORE, 140399863660544, 140399863795711, ++STORE, 140399863795712, 140399930769407, ++STORE, 140400568287232, 140400576679935, ++SNULL, 140400568291327, 140400576679935, ++STORE, 140400568287232, 140400568291327, ++STORE, 140400568291328, 140400576679935, ++SNULL, 140400467775487, 140400534749183, ++STORE, 140400467640320, 140400467775487, ++STORE, 140400467775488, 140400534749183, ++SNULL, 140399729577983, 140399796551679, ++STORE, 140399729442816, 140399729577983, ++STORE, 140399729577984, 140399796551679, ++SNULL, 140400601829376, 140400627003391, ++STORE, 140400627003392, 140400635396095, ++STORE, 140400601829376, 140400627003391, ++SNULL, 140400627007487, 140400635396095, ++STORE, 140400627003392, 140400627007487, ++STORE, 140400627007488, 140400635396095, ++STORE, 140400559894528, 140400568287231, ++STORE, 140400551501824, 140400568287231, ++STORE, 140400543109120, 140400568287231, ++STORE, 140400459247616, 140400467640319, ++STORE, 140400442466304, 140400467640319, ++SNULL, 140399595360255, 140399662333951, ++STORE, 140399595225088, 140399595360255, ++STORE, 140399595360256, 140399662333951, ++SNULL, 140400962551808, 140400970940415, ++STORE, 140400970940416, 140400987725823, ++STORE, 140400962551808, 140400970940415, ++SNULL, 140400970944511, 140400987725823, ++STORE, 140400970940416, 140400970944511, ++STORE, 140400970944512, 140400987725823, ++SNULL, 140400652185600, 140400660574207, ++STORE, 140400660574208, 140400668966911, ++STORE, 140400652185600, 140400660574207, ++SNULL, 140400660578303, 140400668966911, ++STORE, 140400660574208, 140400660578303, ++STORE, 140400660578304, 140400668966911, ++SNULL, 140400576684032, 140400585072639, ++STORE, 140400585072640, 140400593465343, ++STORE, 140400576684032, 140400585072639, ++SNULL, 140400585076735, 140400593465343, ++STORE, 140400585072640, 140400585076735, ++STORE, 140400585076736, 140400593465343, ++STORE, 140400425676800, 140400434069503, ++STORE, 140400417284096, 140400434069503, ++STORE, 140400408891392, 140400434069503, ++SNULL, 140400408891392, 140400417284095, ++STORE, 140400417284096, 140400434069503, ++STORE, 140400408891392, 140400417284095, ++SNULL, 140400417288191, 140400434069503, ++STORE, 140400417284096, 140400417288191, ++STORE, 140400417288192, 140400434069503, ++STORE, 140400283066368, 140400291459071, ++SNULL, 140400601829376, 140400618610687, ++STORE, 140400618610688, 140400627003391, ++STORE, 140400601829376, 140400618610687, ++SNULL, 140400618614783, 140400627003391, ++STORE, 140400618610688, 140400618614783, ++STORE, 140400618614784, 140400627003391, ++SNULL, 140400601829376, 140400610217983, ++STORE, 140400610217984, 140400618610687, ++STORE, 140400601829376, 140400610217983, ++SNULL, 140400610222079, 140400618610687, ++STORE, 140400610217984, 140400610222079, ++STORE, 140400610222080, 140400618610687, ++STORE, 140400274673664, 140400291459071, ++STORE, 140400190812160, 140400199204863, ++STORE, 140400182419456, 140400199204863, ++SNULL, 140400442466304, 140400450854911, ++STORE, 140400450854912, 140400467640319, ++STORE, 140400442466304, 140400450854911, ++SNULL, 140400450859007, 140400467640319, ++STORE, 140400450854912, 140400450859007, ++STORE, 140400450859008, 140400467640319, ++SNULL, 140400543109120, 140400559894527, ++STORE, 140400559894528, 140400568287231, ++STORE, 140400543109120, 140400559894527, ++SNULL, 140400559898623, 140400568287231, ++STORE, 140400559894528, 140400559898623, ++STORE, 140400559898624, 140400568287231, ++SNULL, 140400450859008, 140400459247615, ++STORE, 140400459247616, 140400467640319, ++STORE, 140400450859008, 140400459247615, ++SNULL, 140400459251711, 140400467640319, ++STORE, 140400459247616, 140400459251711, ++STORE, 140400459251712, 140400467640319, ++SNULL, 140400543113215, 140400559894527, ++STORE, 140400543109120, 140400543113215, ++STORE, 140400543113216, 140400559894527, ++SNULL, 140400970944512, 140400979333119, ++STORE, 140400979333120, 140400987725823, ++STORE, 140400970944512, 140400979333119, ++SNULL, 140400979337215, 140400987725823, ++STORE, 140400979333120, 140400979337215, ++STORE, 140400979337216, 140400987725823, ++STORE, 140400174026752, 140400199204863, ++SNULL, 140400174030847, 140400199204863, ++STORE, 140400174026752, 140400174030847, ++STORE, 140400174030848, 140400199204863, ++SNULL, 140400274673664, 140400283066367, ++STORE, 140400283066368, 140400291459071, ++STORE, 140400274673664, 140400283066367, ++SNULL, 140400283070463, 140400291459071, ++STORE, 140400283066368, 140400283070463, ++STORE, 140400283070464, 140400291459071, ++STORE, 140400165634048, 140400174026751, ++SNULL, 140400165638143, 140400174026751, ++STORE, 140400165634048, 140400165638143, ++STORE, 140400165638144, 140400174026751, ++SNULL, 140400174030848, 140400182419455, ++STORE, 140400182419456, 140400199204863, ++STORE, 140400174030848, 140400182419455, ++SNULL, 140400182423551, 140400199204863, ++STORE, 140400182419456, 140400182423551, ++STORE, 140400182423552, 140400199204863, ++SNULL, 140400182423552, 140400190812159, ++STORE, 140400190812160, 140400199204863, ++STORE, 140400182423552, 140400190812159, ++SNULL, 140400190816255, 140400199204863, ++STORE, 140400190812160, 140400190816255, ++STORE, 140400190816256, 140400199204863, ++STORE, 140400157241344, 140400165634047, ++SNULL, 140400157245439, 140400165634047, ++STORE, 140400157241344, 140400157245439, ++STORE, 140400157245440, 140400165634047, ++SNULL, 140400408895487, 140400417284095, ++STORE, 140400408891392, 140400408895487, ++STORE, 140400408895488, 140400417284095, ++SNULL, 140400417288192, 140400425676799, ++STORE, 140400425676800, 140400434069503, ++STORE, 140400417288192, 140400425676799, ++SNULL, 140400425680895, 140400434069503, ++STORE, 140400425676800, 140400425680895, ++STORE, 140400425680896, 140400434069503, ++STORE, 140400148848640, 140400157241343, ++SNULL, 140400148852735, 140400157241343, ++STORE, 140400148848640, 140400148852735, ++STORE, 140400148852736, 140400157241343, ++SNULL, 140400543113216, 140400551501823, ++STORE, 140400551501824, 140400559894527, ++STORE, 140400543113216, 140400551501823, ++SNULL, 140400551505919, 140400559894527, ++STORE, 140400551501824, 140400551505919, ++STORE, 140400551505920, 140400559894527, ++STORE, 140400140455936, 140400148848639, ++STORE, 140400048201728, 140400056594431, ++SNULL, 140400140460031, 140400148848639, ++STORE, 140400140455936, 140400140460031, ++STORE, 140400140460032, 140400148848639, ++STORE, 140400039809024, 140400056594431, ++SNULL, 140400039813119, 140400056594431, ++STORE, 140400039809024, 140400039813119, ++STORE, 140400039813120, 140400056594431, ++STORE, 140400031416320, 140400039809023, ++STORE, 140400023023616, 140400039809023, ++SNULL, 140400274677759, 140400283066367, ++STORE, 140400274673664, 140400274677759, ++STORE, 140400274677760, 140400283066367, ++STORE, 140400014630912, 140400039809023, ++STORE, 140400006238208, 140400039809023, ++STORE, 140399997845504, 140400039809023, ++SNULL, 140399997849599, 140400039809023, ++STORE, 140399997845504, 140399997849599, ++STORE, 140399997849600, 140400039809023, ++STORE, 140399989452800, 140399997845503, ++SNULL, 140399989456895, 140399997845503, ++STORE, 140399989452800, 140399989456895, ++STORE, 140399989456896, 140399997845503, ++STORE, 140399981060096, 140399989452799, ++SNULL, 140399981064191, 140399989452799, ++STORE, 140399981060096, 140399981064191, ++STORE, 140399981064192, 140399989452799, ++STORE, 140399972667392, 140399981060095, ++STORE, 140399964274688, 140399981060095, ++SNULL, 140399964278783, 140399981060095, ++STORE, 140399964274688, 140399964278783, ++STORE, 140399964278784, 140399981060095, ++SNULL, 140400039813120, 140400048201727, ++STORE, 140400048201728, 140400056594431, ++STORE, 140400039813120, 140400048201727, ++SNULL, 140400048205823, 140400056594431, ++STORE, 140400048201728, 140400048205823, ++STORE, 140400048205824, 140400056594431, ++SNULL, 140399997849600, 140400031416319, ++STORE, 140400031416320, 140400039809023, ++STORE, 140399997849600, 140400031416319, ++SNULL, 140400031420415, 140400039809023, ++STORE, 140400031416320, 140400031420415, ++STORE, 140400031420416, 140400039809023, ++STORE, 140399955881984, 140399964274687, ++SNULL, 140399955886079, 140399964274687, ++STORE, 140399955881984, 140399955886079, ++STORE, 140399955886080, 140399964274687, ++STORE, 140399947489280, 140399955881983, ++STORE, 140399939096576, 140399955881983, ++STORE, 140399855267840, 140399863660543, ++SNULL, 140399939100671, 140399955881983, ++STORE, 140399939096576, 140399939100671, ++STORE, 140399939100672, 140399955881983, ++SNULL, 140399997849600, 140400014630911, ++STORE, 140400014630912, 140400031416319, ++STORE, 140399997849600, 140400014630911, ++SNULL, 140400014635007, 140400031416319, ++STORE, 140400014630912, 140400014635007, ++STORE, 140400014635008, 140400031416319, ++SNULL, 140400014635008, 140400023023615, ++STORE, 140400023023616, 140400031416319, ++STORE, 140400014635008, 140400023023615, ++SNULL, 140400023027711, 140400031416319, ++STORE, 140400023023616, 140400023027711, ++STORE, 140400023027712, 140400031416319, ++SNULL, 140399997849600, 140400006238207, ++STORE, 140400006238208, 140400014630911, ++STORE, 140399997849600, 140400006238207, ++SNULL, 140400006242303, 140400014630911, ++STORE, 140400006238208, 140400006242303, ++STORE, 140400006242304, 140400014630911, ++STORE, 140399846875136, 140399863660543, ++STORE, 140399838482432, 140399863660543, ++SNULL, 140399838486527, 140399863660543, ++STORE, 140399838482432, 140399838486527, ++STORE, 140399838486528, 140399863660543, ++SNULL, 140399939100672, 140399947489279, ++STORE, 140399947489280, 140399955881983, ++STORE, 140399939100672, 140399947489279, ++SNULL, 140399947493375, 140399955881983, ++STORE, 140399947489280, 140399947493375, ++STORE, 140399947493376, 140399955881983, ++SNULL, 140399964278784, 140399972667391, ++STORE, 140399972667392, 140399981060095, ++STORE, 140399964278784, 140399972667391, ++SNULL, 140399972671487, 140399981060095, ++STORE, 140399972667392, 140399972671487, ++STORE, 140399972671488, 140399981060095, ++SNULL, 140399838486528, 140399855267839, ++STORE, 140399855267840, 140399863660543, ++STORE, 140399838486528, 140399855267839, ++SNULL, 140399855271935, 140399863660543, ++STORE, 140399855267840, 140399855271935, ++STORE, 140399855271936, 140399863660543, ++STORE, 140399830089728, 140399838482431, ++SNULL, 140399830093823, 140399838482431, ++STORE, 140399830089728, 140399830093823, ++STORE, 140399830093824, 140399838482431, ++STORE, 140399821697024, 140399830089727, ++SNULL, 140399821701119, 140399830089727, ++STORE, 140399821697024, 140399821701119, ++STORE, 140399821701120, 140399830089727, ++SNULL, 140399838486528, 140399846875135, ++STORE, 140399846875136, 140399855267839, ++STORE, 140399838486528, 140399846875135, ++SNULL, 140399846879231, 140399855267839, ++STORE, 140399846875136, 140399846879231, ++STORE, 140399846879232, 140399855267839, ++STORE, 140399813304320, 140399821697023, ++STORE, 140399804911616, 140399821697023, ++SNULL, 140399804915711, 140399821697023, ++STORE, 140399804911616, 140399804915711, ++STORE, 140399804915712, 140399821697023, ++STORE, 140399721050112, 140399729442815, ++SNULL, 140399804915712, 140399813304319, ++STORE, 140399813304320, 140399821697023, ++STORE, 140399804915712, 140399813304319, ++SNULL, 140399813308415, 140399821697023, ++STORE, 140399813304320, 140399813308415, ++STORE, 140399813308416, 140399821697023, ++SNULL, 140399721054207, 140399729442815, ++STORE, 140399721050112, 140399721054207, ++STORE, 140399721054208, 140399729442815, ++STORE, 140401467105280, 140401467133951, ++STORE, 140401279115264, 140401281306623, ++SNULL, 140401279115264, 140401279205375, ++STORE, 140401279205376, 140401281306623, ++STORE, 140401279115264, 140401279205375, ++SNULL, 140401281298431, 140401281306623, ++STORE, 140401279205376, 140401281298431, ++STORE, 140401281298432, 140401281306623, ++ERASE, 140401281298432, 140401281306623, ++STORE, 140401281298432, 140401281306623, ++SNULL, 140401281302527, 140401281306623, ++STORE, 140401281298432, 140401281302527, ++STORE, 140401281302528, 140401281306623, ++ERASE, 140401467105280, 140401467133951, ++ERASE, 140400056594432, 140400056598527, ++ERASE, 140400056598528, 140400064987135, ++ERASE, 140400635396096, 140400635400191, ++ERASE, 140400635400192, 140400643788799, ++ERASE, 140400408891392, 140400408895487, ++ERASE, 140400408895488, 140400417284095, ++ERASE, 140400299851776, 140400299855871, ++ERASE, 140400299855872, 140400308244479, ++ERASE, 140400627003392, 140400627007487, ++ERASE, 140400627007488, 140400635396095, ++ERASE, 140400954155008, 140400954159103, ++ERASE, 140400954159104, 140400962547711, ++ERASE, 140400291459072, 140400291463167, ++ERASE, 140400291463168, 140400299851775, ++ERASE, 140400643788800, 140400643792895, ++ERASE, 140400643792896, 140400652181503, ++ERASE, 140400325029888, 140400325033983, ++ERASE, 140400325033984, 140400333422591, ++ERASE, 140400610217984, 140400610222079, ++ERASE, 140400610222080, 140400618610687, ++ERASE, 140400190812160, 140400190816255, ++ERASE, 140400190816256, 140400199204863, ++ERASE, 140399964274688, 140399964278783, ++ERASE, 140399964278784, 140399972667391, ++ERASE, 140400945762304, 140400945766399, ++ERASE, 140400945766400, 140400954155007, ++ERASE, 140400568287232, 140400568291327, ++ERASE, 140400568291328, 140400576679935, ++ERASE, 140399972667392, 140399972671487, ++ERASE, 140399972671488, 140399981060095, ++ERASE, 140400962547712, 140400962551807, ++ERASE, 140400962551808, 140400970940415, ++ERASE, 140400987725824, 140400987729919, ++ERASE, 140400987729920, 140400996118527, ++ERASE, 140400652181504, 140400652185599, ++ERASE, 140400652185600, 140400660574207, ++ERASE, 140400450854912, 140400450859007, ++ERASE, 140400450859008, 140400459247615, ++ERASE, 140400031416320, 140400031420415, ++ERASE, 140400031420416, 140400039809023, ++ERASE, 140400308244480, 140400308248575, ++ERASE, 140400308248576, 140400316637183, ++ERASE, 140400434069504, 140400434073599, ++ERASE, 140400434073600, 140400442462207, ++ERASE, 140400543109120, 140400543113215, ++ERASE, 140400543113216, 140400551501823, ++ERASE, 140400023023616, 140400023027711, ++ERASE, 140400023027712, 140400031416319, ++ERASE, 140399813304320, 140399813308415, ++ERASE, 140399813308416, 140399821697023, ++ERASE, 140400316637184, 140400316641279, ++ERASE, 140400316641280, 140400325029887, ++ERASE, 140400585072640, 140400585076735, ++ERASE, 140400585076736, 140400593465343, ++ERASE, 140400148848640, 140400148852735, ++ERASE, 140400148852736, 140400157241343, ++ERASE, 140399955881984, 140399955886079, ++ERASE, 140399955886080, 140399964274687, ++ERASE, 140399821697024, 140399821701119, ++ERASE, 140399821701120, 140399830089727, ++ERASE, 140400601825280, 140400601829375, ++ERASE, 140400601829376, 140400610217983, ++ERASE, 140400979333120, 140400979337215, ++ERASE, 140400979337216, 140400987725823, ++ERASE, 140399997845504, 140399997849599, ++ERASE, 140399997849600, 140400006238207, ++ERASE, 140400459247616, 140400459251711, ++ERASE, 140400459251712, 140400467640319, ++ERASE, 140400551501824, 140400551505919, ++ERASE, 140400551505920, 140400559894527, ++ERASE, 140399939096576, 140399939100671, ++ERASE, 140399939100672, 140399947489279, ++ERASE, 140400442462208, 140400442466303, ++ERASE, 140400442466304, 140400450854911, ++ERASE, 140400576679936, 140400576684031, ++ERASE, 140400576684032, 140400585072639, ++ERASE, 140400559894528, 140400559898623, ++ERASE, 140400559898624, 140400568287231, ++ERASE, 140400417284096, 140400417288191, ++ERASE, 140400417288192, 140400425676799, ++ERASE, 140400283066368, 140400283070463, ++ERASE, 140400283070464, 140400291459071, ++ }; ++ unsigned long set33[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734562918400, 140737488351231, ++SNULL, 140734562922495, 140737488351231, ++STORE, 140734562918400, 140734562922495, ++STORE, 140734562787328, 140734562922495, ++STORE, 94133878984704, 94133881237503, ++SNULL, 94133879115775, 94133881237503, ++STORE, 94133878984704, 94133879115775, ++STORE, 94133879115776, 94133881237503, ++ERASE, 94133879115776, 94133881237503, ++STORE, 94133881208832, 94133881217023, ++STORE, 94133881217024, 94133881237503, ++STORE, 140583654043648, 140583656296447, ++SNULL, 140583654187007, 140583656296447, ++STORE, 140583654043648, 140583654187007, ++STORE, 140583654187008, 140583656296447, ++ERASE, 140583654187008, 140583656296447, ++STORE, 140583656284160, 140583656292351, ++STORE, 140583656292352, 140583656296447, ++STORE, 140734564319232, 140734564323327, ++STORE, 140734564306944, 140734564319231, ++STORE, 140583656255488, 140583656284159, ++STORE, 140583656247296, 140583656255487, ++STORE, 140583651827712, 140583654043647, ++SNULL, 140583651827712, 140583651926015, ++STORE, 140583651926016, 140583654043647, ++STORE, 140583651827712, 140583651926015, ++SNULL, 140583654019071, 140583654043647, ++STORE, 140583651926016, 140583654019071, ++STORE, 140583654019072, 140583654043647, ++SNULL, 140583654019072, 140583654027263, ++STORE, 140583654027264, 140583654043647, ++STORE, 140583654019072, 140583654027263, ++ERASE, 140583654019072, 140583654027263, ++STORE, 140583654019072, 140583654027263, ++ERASE, 140583654027264, 140583654043647, ++STORE, 140583654027264, 140583654043647, ++STORE, 140583648030720, 140583651827711, ++SNULL, 140583648030720, 140583649689599, ++STORE, 140583649689600, 140583651827711, ++STORE, 140583648030720, 140583649689599, ++SNULL, 140583651786751, 140583651827711, ++STORE, 140583649689600, 140583651786751, ++STORE, 140583651786752, 140583651827711, ++SNULL, 140583651786752, 140583651811327, ++STORE, 140583651811328, 140583651827711, ++STORE, 140583651786752, 140583651811327, ++ERASE, 140583651786752, 140583651811327, ++STORE, 140583651786752, 140583651811327, ++ERASE, 140583651811328, 140583651827711, ++STORE, 140583651811328, 140583651827711, ++STORE, 140583656239104, 140583656255487, ++SNULL, 140583651803135, 140583651811327, ++STORE, 140583651786752, 140583651803135, ++STORE, 140583651803136, 140583651811327, ++SNULL, 140583654023167, 140583654027263, ++STORE, 140583654019072, 140583654023167, ++STORE, 140583654023168, 140583654027263, ++SNULL, 94133881212927, 94133881217023, ++STORE, 94133881208832, 94133881212927, ++STORE, 94133881212928, 94133881217023, ++SNULL, 140583656288255, 140583656292351, ++STORE, 140583656284160, 140583656288255, ++STORE, 140583656288256, 140583656292351, ++ERASE, 140583656255488, 140583656284159, ++STORE, 94133881733120, 94133881868287, ++STORE, 140583639638016, 140583648030719, ++SNULL, 140583639642111, 140583648030719, ++STORE, 140583639638016, 140583639642111, ++STORE, 140583639642112, 140583648030719, ++STORE, 140583631245312, 140583639638015, ++STORE, 140583497027584, 140583631245311, ++SNULL, 140583497027584, 140583540621311, ++STORE, 140583540621312, 140583631245311, ++STORE, 140583497027584, 140583540621311, ++ERASE, 140583497027584, 140583540621311, ++SNULL, 140583607730175, 140583631245311, ++STORE, 140583540621312, 140583607730175, ++STORE, 140583607730176, 140583631245311, ++ERASE, 140583607730176, 140583631245311, ++SNULL, 140583540756479, 140583607730175, ++STORE, 140583540621312, 140583540756479, ++STORE, 140583540756480, 140583607730175, ++SNULL, 140583631249407, 140583639638015, ++STORE, 140583631245312, 140583631249407, ++STORE, 140583631249408, 140583639638015, ++STORE, 140583622852608, 140583631245311, ++SNULL, 140583622856703, 140583631245311, ++STORE, 140583622852608, 140583622856703, ++STORE, 140583622856704, 140583631245311, ++STORE, 140583614459904, 140583622852607, ++SNULL, 140583614463999, 140583622852607, ++STORE, 140583614459904, 140583614463999, ++STORE, 140583614464000, 140583622852607, ++STORE, 140583532228608, 140583540621311, ++SNULL, 140583532232703, 140583540621311, ++STORE, 140583532228608, 140583532232703, ++STORE, 140583532232704, 140583540621311, ++STORE, 140583523835904, 140583532228607, ++STORE, 140583515443200, 140583532228607, ++STORE, 140583507050496, 140583532228607, ++STORE, 140583372832768, 140583507050495, ++STORE, 140583364440064, 140583372832767, ++STORE, 140583230222336, 140583364440063, ++STORE, 140583096004608, 140583364440063, ++SNULL, 140583230222335, 140583364440063, ++STORE, 140583096004608, 140583230222335, ++STORE, 140583230222336, 140583364440063, ++SNULL, 140583230222336, 140583272185855, ++STORE, 140583272185856, 140583364440063, ++STORE, 140583230222336, 140583272185855, ++ERASE, 140583230222336, 140583272185855, ++STORE, 140582961786880, 140583230222335, ++SNULL, 140583372832768, 140583406403583, ++STORE, 140583406403584, 140583507050495, ++STORE, 140583372832768, 140583406403583, ++ERASE, 140583372832768, 140583406403583, ++SNULL, 140583473512447, 140583507050495, ++STORE, 140583406403584, 140583473512447, ++STORE, 140583473512448, 140583507050495, ++ERASE, 140583473512448, 140583507050495, ++SNULL, 140583096004607, 140583230222335, ++STORE, 140582961786880, 140583096004607, ++STORE, 140583096004608, 140583230222335, ++SNULL, 140583096004608, 140583137968127, ++STORE, 140583137968128, 140583230222335, ++STORE, 140583096004608, 140583137968127, ++ERASE, 140583096004608, 140583137968127, ++SNULL, 140583339294719, 140583364440063, ++STORE, 140583272185856, 140583339294719, ++STORE, 140583339294720, 140583364440063, ++ERASE, 140583339294720, 140583364440063, ++SNULL, 140583272321023, 140583339294719, ++STORE, 140583272185856, 140583272321023, ++STORE, 140583272321024, 140583339294719, ++SNULL, 140582961786880, 140583003750399, ++STORE, 140583003750400, 140583096004607, ++STORE, 140582961786880, 140583003750399, ++ERASE, 140582961786880, 140583003750399, ++ }; + -+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -+{ -+ struct mem_cgroup *memcg; -+ bool success = false; -+ unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); ++ unsigned long set34[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140731327180800, 140737488351231, ++SNULL, 140731327184895, 140737488351231, ++STORE, 140731327180800, 140731327184895, ++STORE, 140731327049728, 140731327184895, ++STORE, 94632924487680, 94632926740479, ++SNULL, 94632924618751, 94632926740479, ++STORE, 94632924487680, 94632924618751, ++STORE, 94632924618752, 94632926740479, ++ERASE, 94632924618752, 94632926740479, ++STORE, 94632926711808, 94632926719999, ++STORE, 94632926720000, 94632926740479, ++STORE, 140012544888832, 140012547141631, ++SNULL, 140012545032191, 140012547141631, ++STORE, 140012544888832, 140012545032191, ++STORE, 140012545032192, 140012547141631, ++ERASE, 140012545032192, 140012547141631, ++STORE, 140012547129344, 140012547137535, ++STORE, 140012547137536, 140012547141631, ++STORE, 140731327725568, 140731327729663, ++STORE, 140731327713280, 140731327725567, ++STORE, 140012547100672, 140012547129343, ++STORE, 140012547092480, 140012547100671, ++STORE, 140012542672896, 140012544888831, ++SNULL, 140012542672896, 140012542771199, ++STORE, 140012542771200, 140012544888831, ++STORE, 140012542672896, 140012542771199, ++SNULL, 140012544864255, 140012544888831, ++STORE, 140012542771200, 140012544864255, ++STORE, 140012544864256, 140012544888831, ++SNULL, 140012544864256, 140012544872447, ++STORE, 140012544872448, 140012544888831, ++STORE, 140012544864256, 140012544872447, ++ERASE, 140012544864256, 140012544872447, ++STORE, 140012544864256, 140012544872447, ++ERASE, 140012544872448, 140012544888831, ++STORE, 140012544872448, 140012544888831, ++STORE, 140012538875904, 140012542672895, ++SNULL, 140012538875904, 140012540534783, ++STORE, 140012540534784, 140012542672895, ++STORE, 140012538875904, 140012540534783, ++SNULL, 140012542631935, 140012542672895, ++STORE, 140012540534784, 140012542631935, ++STORE, 140012542631936, 140012542672895, ++SNULL, 140012542631936, 140012542656511, ++STORE, 140012542656512, 140012542672895, ++STORE, 140012542631936, 140012542656511, ++ERASE, 140012542631936, 140012542656511, ++STORE, 140012542631936, 140012542656511, ++ERASE, 140012542656512, 140012542672895, ++STORE, 140012542656512, 140012542672895, ++STORE, 140012547084288, 140012547100671, ++SNULL, 140012542648319, 140012542656511, ++STORE, 140012542631936, 140012542648319, ++STORE, 140012542648320, 140012542656511, ++SNULL, 140012544868351, 140012544872447, ++STORE, 140012544864256, 140012544868351, ++STORE, 140012544868352, 140012544872447, ++SNULL, 94632926715903, 94632926719999, ++STORE, 94632926711808, 94632926715903, ++STORE, 94632926715904, 94632926719999, ++SNULL, 140012547133439, 140012547137535, ++STORE, 140012547129344, 140012547133439, ++STORE, 140012547133440, 140012547137535, ++ERASE, 140012547100672, 140012547129343, ++STORE, 94632939606016, 94632939741183, ++STORE, 140012530483200, 140012538875903, ++SNULL, 140012530487295, 140012538875903, ++STORE, 140012530483200, 140012530487295, ++STORE, 140012530487296, 140012538875903, ++STORE, 140012522090496, 140012530483199, ++STORE, 140012387872768, 140012522090495, ++SNULL, 140012387872768, 140012444188671, ++STORE, 140012444188672, 140012522090495, ++STORE, 140012387872768, 140012444188671, ++ERASE, 140012387872768, 140012444188671, ++SNULL, 140012511297535, 140012522090495, ++STORE, 140012444188672, 140012511297535, ++STORE, 140012511297536, 140012522090495, ++ERASE, 140012511297536, 140012522090495, ++SNULL, 140012444323839, 140012511297535, ++STORE, 140012444188672, 140012444323839, ++STORE, 140012444323840, 140012511297535, ++SNULL, 140012522094591, 140012530483199, ++STORE, 140012522090496, 140012522094591, ++STORE, 140012522094592, 140012530483199, ++STORE, 140012513697792, 140012522090495, ++SNULL, 140012513701887, 140012522090495, ++STORE, 140012513697792, 140012513701887, ++STORE, 140012513701888, 140012522090495, ++STORE, 140012435795968, 140012444188671, ++SNULL, 140012435800063, 140012444188671, ++STORE, 140012435795968, 140012435800063, ++STORE, 140012435800064, 140012444188671, ++STORE, 140012427403264, 140012435795967, ++SNULL, 140012427407359, 140012435795967, ++STORE, 140012427403264, 140012427407359, ++STORE, 140012427407360, 140012435795967, ++STORE, 140012419010560, 140012427403263, ++STORE, 140012410617856, 140012427403263, ++STORE, 140012276400128, 140012410617855, ++STORE, 140012268007424, 140012276400127, ++STORE, 140012133789696, 140012268007423, ++SNULL, 140012133789696, 140012175753215, ++STORE, 140012175753216, 140012268007423, ++STORE, 140012133789696, 140012175753215, ++ERASE, 140012133789696, 140012175753215, ++STORE, 140012041535488, 140012268007423, ++SNULL, 140012108644351, 140012268007423, ++STORE, 140012041535488, 140012108644351, ++STORE, 140012108644352, 140012268007423, ++SNULL, 140012108644352, 140012175753215, ++STORE, 140012175753216, 140012268007423, ++STORE, 140012108644352, 140012175753215, ++ERASE, 140012108644352, 140012175753215, ++SNULL, 140012276400128, 140012309970943, ++STORE, 140012309970944, 140012410617855, ++STORE, 140012276400128, 140012309970943, ++ERASE, 140012276400128, 140012309970943, ++STORE, 140012301578240, 140012309970943, ++STORE, 140012041535488, 140012268007423, ++SNULL, 140012242862079, 140012268007423, ++STORE, 140012041535488, 140012242862079, ++STORE, 140012242862080, 140012268007423, ++ERASE, 140012242862080, 140012268007423, ++SNULL, 140012041670655, 140012242862079, ++STORE, 140012041535488, 140012041670655, ++STORE, 140012041670656, 140012242862079, ++SNULL, 140012041670656, 140012108644351, ++STORE, 140012108644352, 140012242862079, ++STORE, 140012041670656, 140012108644351, ++SNULL, 140012108779519, 140012242862079, ++STORE, 140012108644352, 140012108779519, ++STORE, 140012108779520, 140012242862079, ++SNULL, 140012377079807, 140012410617855, ++STORE, 140012309970944, 140012377079807, ++STORE, 140012377079808, 140012410617855, ++ERASE, 140012377079808, 140012410617855, ++SNULL, 140012310106111, 140012377079807, ++STORE, 140012309970944, 140012310106111, ++STORE, 140012310106112, 140012377079807, ++SNULL, 140012410621951, 140012427403263, ++STORE, 140012410617856, 140012410621951, ++STORE, 140012410621952, 140012427403263, ++SNULL, 140012108779520, 140012175753215, ++STORE, 140012175753216, 140012242862079, ++STORE, 140012108779520, 140012175753215, ++SNULL, 140012175888383, 140012242862079, ++STORE, 140012175753216, 140012175888383, ++STORE, 140012175888384, 140012242862079, ++SNULL, 140012301582335, 140012309970943, ++STORE, 140012301578240, 140012301582335, ++STORE, 140012301582336, 140012309970943, ++SNULL, 140012410621952, 140012419010559, ++STORE, 140012419010560, 140012427403263, ++STORE, 140012410621952, 140012419010559, ++SNULL, 140012419014655, 140012427403263, ++STORE, 140012419010560, 140012419014655, ++STORE, 140012419014656, 140012427403263, ++SNULL, 140012268011519, 140012276400127, ++STORE, 140012268007424, 140012268011519, ++STORE, 140012268011520, 140012276400127, ++STORE, 140012402225152, 140012410617855, ++STORE, 140012393832448, 140012410617855, ++SNULL, 140012393832448, 140012402225151, ++STORE, 140012402225152, 140012410617855, ++STORE, 140012393832448, 140012402225151, ++SNULL, 140012402229247, 140012410617855, ++STORE, 140012402225152, 140012402229247, ++STORE, 140012402229248, 140012410617855, ++STORE, 140012385439744, 140012402225151, ++SNULL, 140012385439744, 140012393832447, ++STORE, 140012393832448, 140012402225151, ++STORE, 140012385439744, 140012393832447, ++SNULL, 140012393836543, 140012402225151, ++STORE, 140012393832448, 140012393836543, ++STORE, 140012393836544, 140012402225151, ++STORE, 140012293185536, 140012301578239, ++STORE, 140012284792832, 140012301578239, ++SNULL, 140012284792832, 140012293185535, ++STORE, 140012293185536, 140012301578239, ++STORE, 140012284792832, 140012293185535, ++SNULL, 140012293189631, 140012301578239, ++STORE, 140012293185536, 140012293189631, ++STORE, 140012293189632, 140012301578239, ++STORE, 140012268011520, 140012284792831, ++SNULL, 140012385443839, 140012393832447, ++STORE, 140012385439744, 140012385443839, ++STORE, 140012385443840, 140012393832447, ++STORE, 140012259614720, 140012268007423, ++SNULL, 140012259618815, 140012268007423, ++STORE, 140012259614720, 140012259618815, ++STORE, 140012259618816, 140012268007423, ++STORE, 140012251222016, 140012259614719, ++SNULL, 140012251226111, 140012259614719, ++STORE, 140012251222016, 140012251226111, ++STORE, 140012251226112, 140012259614719, ++SNULL, 140012284796927, 140012293185535, ++STORE, 140012284792832, 140012284796927, ++STORE, 140012284796928, 140012293185535, ++SNULL, 140012268011520, 140012276400127, ++STORE, 140012276400128, 140012284792831, ++STORE, 140012268011520, 140012276400127, ++SNULL, 140012276404223, 140012284792831, ++STORE, 140012276400128, 140012276404223, ++STORE, 140012276404224, 140012284792831, ++STORE, 140012033142784, 140012041535487, ++SNULL, 140012033146879, 140012041535487, ++STORE, 140012033142784, 140012033146879, ++STORE, 140012033146880, 140012041535487, ++STORE, 140012024750080, 140012033142783, ++STORE, 140012016357376, 140012033142783, ++SNULL, 140012016357376, 140012024750079, ++STORE, 140012024750080, 140012033142783, ++STORE, 140012016357376, 140012024750079, ++SNULL, 140012024754175, 140012033142783, ++STORE, 140012024750080, 140012024754175, ++STORE, 140012024754176, 140012033142783, ++SNULL, 140012016361471, 140012024750079, ++STORE, 140012016357376, 140012016361471, ++STORE, 140012016361472, 140012024750079, ++STORE, 140012007964672, 140012016357375, ++SNULL, 140012007968767, 140012016357375, ++STORE, 140012007964672, 140012007968767, ++STORE, 140012007968768, 140012016357375, ++STORE, 140011999571968, 140012007964671, ++STORE, 140011991179264, 140012007964671, ++STORE, 140011856961536, 140011991179263, ++STORE, 140011848568832, 140011856961535, ++STORE, 140011714351104, 140011848568831, ++SNULL, 140011714351104, 140011773100031, ++STORE, 140011773100032, 140011848568831, ++STORE, 140011714351104, 140011773100031, ++ERASE, 140011714351104, 140011773100031, ++STORE, 140011764707328, 140011773100031, ++STORE, 140011756314624, 140011773100031, ++STORE, 140011622096896, 140011756314623, ++STORE, 140011613704192, 140011622096895, ++STORE, 140011479486464, 140011613704191, ++STORE, 140011471093760, 140011479486463, ++SNULL, 140011479486464, 140011504664575, ++STORE, 140011504664576, 140011613704191, ++STORE, 140011479486464, 140011504664575, ++ERASE, 140011479486464, 140011504664575, ++STORE, 140011496271872, 140011504664575, ++STORE, 140011487879168, 140011504664575, ++STORE, 140011336876032, 140011471093759, ++SNULL, 140011336876032, 140011370446847, ++STORE, 140011370446848, 140011471093759, ++STORE, 140011336876032, 140011370446847, ++ERASE, 140011336876032, 140011370446847, ++STORE, 140011471093760, 140011487879167, ++STORE, 140011362054144, 140011370446847, ++SNULL, 140011362058239, 140011370446847, ++STORE, 140011362054144, 140011362058239, ++STORE, 140011362058240, 140011370446847, ++STORE, 140011353661440, 140011362054143, ++STORE, 140011345268736, 140011362054143, ++SNULL, 140011345272831, 140011362054143, ++STORE, 140011345268736, 140011345272831, ++STORE, 140011345272832, 140011362054143, ++STORE, 140011336876032, 140011345268735, ++STORE, 140011328483328, 140011345268735, ++SNULL, 140011328487423, 140011345268735, ++STORE, 140011328483328, 140011328487423, ++STORE, 140011328487424, 140011345268735, ++STORE, 140011320090624, 140011328483327, ++STORE, 140011185872896, 140011320090623, ++SNULL, 140011185872896, 140011236229119, ++STORE, 140011236229120, 140011320090623, ++STORE, 140011185872896, 140011236229119, ++ERASE, 140011185872896, 140011236229119, ++SNULL, 140011856961536, 140011907317759, ++STORE, 140011907317760, 140011991179263, ++STORE, 140011856961536, 140011907317759, ++ERASE, 140011856961536, 140011907317759, ++SNULL, 140011974426623, 140011991179263, ++STORE, 140011907317760, 140011974426623, ++STORE, 140011974426624, 140011991179263, ++ERASE, 140011974426624, 140011991179263, ++SNULL, 140011840208895, 140011848568831, ++STORE, 140011773100032, 140011840208895, ++STORE, 140011840208896, 140011848568831, ++ERASE, 140011840208896, 140011848568831, ++SNULL, 140011773235199, 140011840208895, ++STORE, 140011773100032, 140011773235199, ++STORE, 140011773235200, 140011840208895, ++STORE, 140011102011392, 140011320090623, ++SNULL, 140011169120255, 140011320090623, ++STORE, 140011102011392, 140011169120255, ++STORE, 140011169120256, 140011320090623, ++SNULL, 140011169120256, 140011236229119, ++STORE, 140011236229120, 140011320090623, ++STORE, 140011169120256, 140011236229119, ++ERASE, 140011169120256, 140011236229119, ++SNULL, 140011622096896, 140011638882303, ++STORE, 140011638882304, 140011756314623, ++STORE, 140011622096896, 140011638882303, ++ERASE, 140011622096896, 140011638882303, ++SNULL, 140011705991167, 140011756314623, ++STORE, 140011638882304, 140011705991167, ++STORE, 140011705991168, 140011756314623, ++ERASE, 140011705991168, 140011756314623, ++SNULL, 140011571773439, 140011613704191, ++STORE, 140011504664576, 140011571773439, ++STORE, 140011571773440, 140011613704191, ++ERASE, 140011571773440, 140011613704191, ++STORE, 140010967793664, 140011169120255, ++SNULL, 140011034902527, 140011169120255, ++STORE, 140010967793664, 140011034902527, ++STORE, 140011034902528, 140011169120255, ++SNULL, 140011034902528, 140011102011391, ++STORE, 140011102011392, 140011169120255, ++STORE, 140011034902528, 140011102011391, ++ERASE, 140011034902528, 140011102011391, ++STORE, 140010833575936, 140011034902527, ++SNULL, 140011437555711, 140011471093759, ++STORE, 140011370446848, 140011437555711, ++STORE, 140011437555712, 140011471093759, ++ERASE, 140011437555712, 140011471093759, ++SNULL, 140011370582015, 140011437555711, ++STORE, 140011370446848, 140011370582015, ++STORE, 140011370582016, 140011437555711, ++STORE, 140010699358208, 140011034902527, ++SNULL, 140011487883263, 140011504664575, ++STORE, 140011487879168, 140011487883263, ++STORE, 140011487883264, 140011504664575, ++SNULL, 140011345272832, 140011353661439, ++STORE, 140011353661440, 140011362054143, ++STORE, 140011345272832, 140011353661439, ++SNULL, 140011353665535, 140011362054143, ++STORE, 140011353661440, 140011353665535, ++STORE, 140011353665536, 140011362054143, ++SNULL, 140011328487424, 140011336876031, ++STORE, 140011336876032, 140011345268735, ++STORE, 140011328487424, 140011336876031, ++SNULL, 140011336880127, 140011345268735, ++STORE, 140011336876032, 140011336880127, ++STORE, 140011336880128, 140011345268735, ++SNULL, 140011303337983, 140011320090623, ++STORE, 140011236229120, 140011303337983, ++STORE, 140011303337984, 140011320090623, ++ERASE, 140011303337984, 140011320090623, ++SNULL, 140011907452927, 140011974426623, ++STORE, 140011907317760, 140011907452927, ++STORE, 140011907452928, 140011974426623, ++SNULL, 140011102146559, 140011169120255, ++STORE, 140011102011392, 140011102146559, ++STORE, 140011102146560, 140011169120255, ++SNULL, 140011639017471, 140011705991167, ++STORE, 140011638882304, 140011639017471, ++STORE, 140011639017472, 140011705991167, ++SNULL, 140011504799743, 140011571773439, ++STORE, 140011504664576, 140011504799743, ++STORE, 140011504799744, 140011571773439, ++SNULL, 140011613708287, 140011622096895, ++STORE, 140011613704192, 140011613708287, ++STORE, 140011613708288, 140011622096895, ++SNULL, 140010699358208, 140010967793663, ++STORE, 140010967793664, 140011034902527, ++STORE, 140010699358208, 140010967793663, ++SNULL, 140010967928831, 140011034902527, ++STORE, 140010967793664, 140010967928831, ++STORE, 140010967928832, 140011034902527, ++SNULL, 140010900684799, 140010967793663, ++STORE, 140010699358208, 140010900684799, ++STORE, 140010900684800, 140010967793663, ++ERASE, 140010900684800, 140010967793663, ++SNULL, 140010766467071, 140010900684799, ++STORE, 140010699358208, 140010766467071, ++STORE, 140010766467072, 140010900684799, ++SNULL, 140010766467072, 140010833575935, ++STORE, 140010833575936, 140010900684799, ++STORE, 140010766467072, 140010833575935, ++ERASE, 140010766467072, 140010833575935, ++SNULL, 140010699493375, 140010766467071, ++STORE, 140010699358208, 140010699493375, ++STORE, 140010699493376, 140010766467071, ++SNULL, 140011848572927, 140011856961535, ++STORE, 140011848568832, 140011848572927, ++STORE, 140011848572928, 140011856961535, ++STORE, 140011982786560, 140012007964671, ++STORE, 140011898925056, 140011907317759, ++SNULL, 140011898929151, 140011907317759, ++STORE, 140011898925056, 140011898929151, ++STORE, 140011898929152, 140011907317759, ++SNULL, 140011320094719, 140011328483327, ++STORE, 140011320090624, 140011320094719, ++STORE, 140011320094720, 140011328483327, ++STORE, 140011890532352, 140011898925055, ++STORE, 140011882139648, 140011898925055, ++SNULL, 140011882143743, 140011898925055, ++STORE, 140011882139648, 140011882143743, ++STORE, 140011882143744, 140011898925055, ++STORE, 140011873746944, 140011882139647, ++SNULL, 140011873751039, 140011882139647, ++STORE, 140011873746944, 140011873751039, ++STORE, 140011873751040, 140011882139647, ++SNULL, 140011236364287, 140011303337983, ++STORE, 140011236229120, 140011236364287, ++STORE, 140011236364288, 140011303337983, ++SNULL, 140011756318719, 140011773100031, ++STORE, 140011756314624, 140011756318719, ++STORE, 140011756318720, 140011773100031, ++SNULL, 140011756318720, 140011764707327, ++STORE, 140011764707328, 140011773100031, ++STORE, 140011756318720, 140011764707327, ++SNULL, 140011764711423, 140011773100031, ++STORE, 140011764707328, 140011764711423, ++STORE, 140011764711424, 140011773100031, ++SNULL, 140011471097855, 140011487879167, ++STORE, 140011471093760, 140011471097855, ++STORE, 140011471097856, 140011487879167, ++SNULL, 140010833711103, 140010900684799, ++STORE, 140010833575936, 140010833711103, ++STORE, 140010833711104, 140010900684799, ++SNULL, 140011982790655, 140012007964671, ++STORE, 140011982786560, 140011982790655, ++STORE, 140011982790656, 140012007964671, ++STORE, 140011865354240, 140011873746943, ++STORE, 140011848572928, 140011865354239, ++SNULL, 140011848572928, 140011856961535, ++STORE, 140011856961536, 140011865354239, ++STORE, 140011848572928, 140011856961535, ++SNULL, 140011856965631, 140011865354239, ++STORE, 140011856961536, 140011856965631, ++STORE, 140011856965632, 140011865354239, ++STORE, 140011747921920, 140011756314623, ++STORE, 140011739529216, 140011756314623, ++SNULL, 140011471097856, 140011479486463, ++STORE, 140011479486464, 140011487879167, ++STORE, 140011471097856, 140011479486463, ++SNULL, 140011479490559, 140011487879167, ++STORE, 140011479486464, 140011479490559, ++STORE, 140011479490560, 140011487879167, ++STORE, 140011731136512, 140011756314623, ++STORE, 140011722743808, 140011756314623, ++SNULL, 140011982790656, 140011999571967, ++STORE, 140011999571968, 140012007964671, ++STORE, 140011982790656, 140011999571967, ++SNULL, 140011999576063, 140012007964671, ++STORE, 140011999571968, 140011999576063, ++STORE, 140011999576064, 140012007964671, ++STORE, 140011714351104, 140011756314623, ++SNULL, 140011882143744, 140011890532351, ++STORE, 140011890532352, 140011898925055, ++STORE, 140011882143744, 140011890532351, ++SNULL, 140011890536447, 140011898925055, ++STORE, 140011890532352, 140011890536447, ++STORE, 140011890536448, 140011898925055, ++STORE, 140011630489600, 140011638882303, ++STORE, 140011613708288, 140011638882303, ++STORE, 140011605311488, 140011613704191, ++STORE, 140011596918784, 140011613704191, ++STORE, 140011588526080, 140011613704191, ++SNULL, 140011487883264, 140011496271871, ++STORE, 140011496271872, 140011504664575, ++STORE, 140011487883264, 140011496271871, ++SNULL, 140011496275967, 140011504664575, ++STORE, 140011496271872, 140011496275967, ++STORE, 140011496275968, 140011504664575, ++STORE, 140011580133376, 140011613704191, ++SNULL, 140011580137471, 140011613704191, ++STORE, 140011580133376, 140011580137471, ++STORE, 140011580137472, 140011613704191, ++SNULL, 140011982790656, 140011991179263, ++STORE, 140011991179264, 140011999571967, ++STORE, 140011982790656, 140011991179263, ++SNULL, 140011991183359, 140011999571967, ++STORE, 140011991179264, 140011991183359, ++STORE, 140011991183360, 140011999571967, ++SNULL, 140011865358335, 140011873746943, ++STORE, 140011865354240, 140011865358335, ++STORE, 140011865358336, 140011873746943, ++STORE, 140011462701056, 140011471093759, ++SNULL, 140011714351104, 140011739529215, ++STORE, 140011739529216, 140011756314623, ++STORE, 140011714351104, 140011739529215, ++SNULL, 140011739533311, 140011756314623, ++STORE, 140011739529216, 140011739533311, ++STORE, 140011739533312, 140011756314623, ++SNULL, 140011739533312, 140011747921919, ++STORE, 140011747921920, 140011756314623, ++STORE, 140011739533312, 140011747921919, ++SNULL, 140011747926015, 140011756314623, ++STORE, 140011747921920, 140011747926015, ++STORE, 140011747926016, 140011756314623, ++SNULL, 140011613708288, 140011630489599, ++STORE, 140011630489600, 140011638882303, ++STORE, 140011613708288, 140011630489599, ++SNULL, 140011630493695, 140011638882303, ++STORE, 140011630489600, 140011630493695, ++STORE, 140011630493696, 140011638882303, ++SNULL, 140011714351104, 140011722743807, ++STORE, 140011722743808, 140011739529215, ++STORE, 140011714351104, 140011722743807, ++SNULL, 140011722747903, 140011739529215, ++STORE, 140011722743808, 140011722747903, ++STORE, 140011722747904, 140011739529215, ++SNULL, 140011714355199, 140011722743807, ++STORE, 140011714351104, 140011714355199, ++STORE, 140011714355200, 140011722743807, ++SNULL, 140011722747904, 140011731136511, ++STORE, 140011731136512, 140011739529215, ++STORE, 140011722747904, 140011731136511, ++SNULL, 140011731140607, 140011739529215, ++STORE, 140011731136512, 140011731140607, ++STORE, 140011731140608, 140011739529215, ++STORE, 140011454308352, 140011471093759, ++STORE, 140011445915648, 140011471093759, ++SNULL, 140011580137472, 140011588526079, ++STORE, 140011588526080, 140011613704191, ++STORE, 140011580137472, 140011588526079, ++SNULL, 140011588530175, 140011613704191, ++STORE, 140011588526080, 140011588530175, ++STORE, 140011588530176, 140011613704191, ++SNULL, 140011445915648, 140011462701055, ++STORE, 140011462701056, 140011471093759, ++STORE, 140011445915648, 140011462701055, ++SNULL, 140011462705151, 140011471093759, ++STORE, 140011462701056, 140011462705151, ++STORE, 140011462705152, 140011471093759, ++SNULL, 140011588530176, 140011596918783, ++STORE, 140011596918784, 140011613704191, ++STORE, 140011588530176, 140011596918783, ++SNULL, 140011596922879, 140011613704191, ++STORE, 140011596918784, 140011596922879, ++STORE, 140011596922880, 140011613704191, ++SNULL, 140011596922880, 140011605311487, ++STORE, 140011605311488, 140011613704191, ++STORE, 140011596922880, 140011605311487, ++SNULL, 140011605315583, 140011613704191, ++STORE, 140011605311488, 140011605315583, ++STORE, 140011605315584, 140011613704191, ++SNULL, 140011613708288, 140011622096895, ++STORE, 140011622096896, 140011630489599, ++STORE, 140011613708288, 140011622096895, ++SNULL, 140011622100991, 140011630489599, ++STORE, 140011622096896, 140011622100991, ++STORE, 140011622100992, 140011630489599, ++STORE, 140011311697920, 140011320090623, ++STORE, 140011227836416, 140011236229119, ++STORE, 140011219443712, 140011236229119, ++SNULL, 140011219447807, 140011236229119, ++STORE, 140011219443712, 140011219447807, ++STORE, 140011219447808, 140011236229119, ++STORE, 140011211051008, 140011219443711, ++STORE, 140011202658304, 140011219443711, ++SNULL, 140011202662399, 140011219443711, ++STORE, 140011202658304, 140011202662399, ++STORE, 140011202662400, 140011219443711, ++STORE, 140011194265600, 140011202658303, ++STORE, 140011185872896, 140011202658303, ++STORE, 140011177480192, 140011202658303, ++STORE, 140011093618688, 140011102011391, ++SNULL, 140011445915648, 140011454308351, ++STORE, 140011454308352, 140011462701055, ++STORE, 140011445915648, 140011454308351, ++SNULL, 140011454312447, 140011462701055, ++STORE, 140011454308352, 140011454312447, ++STORE, 140011454312448, 140011462701055, ++STORE, 140011085225984, 140011102011391, ++SNULL, 140011085230079, 140011102011391, ++STORE, 140011085225984, 140011085230079, ++STORE, 140011085230080, 140011102011391, ++SNULL, 140011177484287, 140011202658303, ++STORE, 140011177480192, 140011177484287, ++STORE, 140011177484288, 140011202658303, ++SNULL, 140011445919743, 140011454308351, ++STORE, 140011445915648, 140011445919743, ++STORE, 140011445919744, 140011454308351, ++SNULL, 140011177484288, 140011185872895, ++STORE, 140011185872896, 140011202658303, ++STORE, 140011177484288, 140011185872895, ++SNULL, 140011185876991, 140011202658303, ++STORE, 140011185872896, 140011185876991, ++STORE, 140011185876992, 140011202658303, ++STORE, 140011076833280, 140011085225983, ++SNULL, 140011202662400, 140011211051007, ++STORE, 140011211051008, 140011219443711, ++STORE, 140011202662400, 140011211051007, ++SNULL, 140011211055103, 140011219443711, ++STORE, 140011211051008, 140011211055103, ++STORE, 140011211055104, 140011219443711, ++SNULL, 140011185876992, 140011194265599, ++STORE, 140011194265600, 140011202658303, ++STORE, 140011185876992, 140011194265599, ++SNULL, 140011194269695, 140011202658303, ++STORE, 140011194265600, 140011194269695, ++STORE, 140011194269696, 140011202658303, ++STORE, 140011068440576, 140011085225983, ++SNULL, 140011311702015, 140011320090623, ++STORE, 140011311697920, 140011311702015, ++STORE, 140011311702016, 140011320090623, ++STORE, 140011060047872, 140011085225983, ++SNULL, 140011060051967, 140011085225983, ++STORE, 140011060047872, 140011060051967, ++STORE, 140011060051968, 140011085225983, ++STORE, 140011051655168, 140011060047871, ++STORE, 140011043262464, 140011060047871, ++SNULL, 140011043266559, 140011060047871, ++STORE, 140011043262464, 140011043266559, ++STORE, 140011043266560, 140011060047871, ++SNULL, 140011219447808, 140011227836415, ++STORE, 140011227836416, 140011236229119, ++STORE, 140011219447808, 140011227836415, ++SNULL, 140011227840511, 140011236229119, ++STORE, 140011227836416, 140011227840511, ++STORE, 140011227840512, 140011236229119, ++SNULL, 140011085230080, 140011093618687, ++STORE, 140011093618688, 140011102011391, ++STORE, 140011085230080, 140011093618687, ++SNULL, 140011093622783, 140011102011391, ++STORE, 140011093618688, 140011093622783, ++STORE, 140011093622784, 140011102011391, ++STORE, 140010959400960, 140010967793663, ++STORE, 140010951008256, 140010967793663, ++SNULL, 140010951008256, 140010959400959, ++STORE, 140010959400960, 140010967793663, ++STORE, 140010951008256, 140010959400959, ++SNULL, 140010959405055, 140010967793663, ++STORE, 140010959400960, 140010959405055, ++STORE, 140010959405056, 140010967793663, ++STORE, 140010942615552, 140010959400959, ++STORE, 140010934222848, 140010959400959, ++SNULL, 140011060051968, 140011076833279, ++STORE, 140011076833280, 140011085225983, ++STORE, 140011060051968, 140011076833279, ++SNULL, 140011076837375, 140011085225983, ++STORE, 140011076833280, 140011076837375, ++STORE, 140011076837376, 140011085225983, ++SNULL, 140011043266560, 140011051655167, ++STORE, 140011051655168, 140011060047871, ++STORE, 140011043266560, 140011051655167, ++SNULL, 140011051659263, 140011060047871, ++STORE, 140011051655168, 140011051659263, ++STORE, 140011051659264, 140011060047871, ++STORE, 140010925830144, 140010959400959, ++SNULL, 140011060051968, 140011068440575, ++STORE, 140011068440576, 140011076833279, ++STORE, 140011060051968, 140011068440575, ++SNULL, 140011068444671, 140011076833279, ++STORE, 140011068440576, 140011068444671, ++STORE, 140011068444672, 140011076833279, ++STORE, 140010917437440, 140010959400959, ++STORE, 140010909044736, 140010959400959, ++STORE, 140010825183232, 140010833575935, ++SNULL, 140010909044736, 140010942615551, ++STORE, 140010942615552, 140010959400959, ++STORE, 140010909044736, 140010942615551, ++SNULL, 140010942619647, 140010959400959, ++STORE, 140010942615552, 140010942619647, ++STORE, 140010942619648, 140010959400959, ++SNULL, 140010909044736, 140010934222847, ++STORE, 140010934222848, 140010942615551, ++STORE, 140010909044736, 140010934222847, ++SNULL, 140010934226943, 140010942615551, ++STORE, 140010934222848, 140010934226943, ++STORE, 140010934226944, 140010942615551, ++SNULL, 140010909048831, 140010934222847, ++STORE, 140010909044736, 140010909048831, ++STORE, 140010909048832, 140010934222847, ++STORE, 140010816790528, 140010833575935, ++SNULL, 140010816794623, 140010833575935, ++STORE, 140010816790528, 140010816794623, ++STORE, 140010816794624, 140010833575935, ++STORE, 140010808397824, 140010816790527, ++SNULL, 140010942619648, 140010951008255, ++STORE, 140010951008256, 140010959400959, ++STORE, 140010942619648, 140010951008255, ++SNULL, 140010951012351, 140010959400959, ++STORE, 140010951008256, 140010951012351, ++STORE, 140010951012352, 140010959400959, ++STORE, 140010800005120, 140010816790527, ++SNULL, 140010800009215, 140010816790527, ++STORE, 140010800005120, 140010800009215, ++STORE, 140010800009216, 140010816790527, ++SNULL, 140010909048832, 140010925830143, ++STORE, 140010925830144, 140010934222847, ++STORE, 140010909048832, 140010925830143, ++SNULL, 140010925834239, 140010934222847, ++STORE, 140010925830144, 140010925834239, ++STORE, 140010925834240, 140010934222847, ++SNULL, 140010816794624, 140010825183231, ++STORE, 140010825183232, 140010833575935, ++STORE, 140010816794624, 140010825183231, ++SNULL, 140010825187327, 140010833575935, ++STORE, 140010825183232, 140010825187327, ++STORE, 140010825187328, 140010833575935, ++SNULL, 140010909048832, 140010917437439, ++STORE, 140010917437440, 140010925830143, ++STORE, 140010909048832, 140010917437439, ++SNULL, 140010917441535, 140010925830143, ++STORE, 140010917437440, 140010917441535, ++STORE, 140010917441536, 140010925830143, ++SNULL, 140010800009216, 140010808397823, ++STORE, 140010808397824, 140010816790527, ++STORE, 140010800009216, 140010808397823, ++SNULL, 140010808401919, 140010816790527, ++STORE, 140010808397824, 140010808401919, ++STORE, 140010808401920, 140010816790527, ++STORE, 140010791612416, 140010800005119, ++SNULL, 140010791616511, 140010800005119, ++STORE, 140010791612416, 140010791616511, ++STORE, 140010791616512, 140010800005119, ++STORE, 140012547100672, 140012547129343, ++STORE, 140012511506432, 140012513697791, ++SNULL, 140012511506432, 140012511596543, ++STORE, 140012511596544, 140012513697791, ++STORE, 140012511506432, 140012511596543, ++SNULL, 140012513689599, 140012513697791, ++STORE, 140012511596544, 140012513689599, ++STORE, 140012513689600, 140012513697791, ++ERASE, 140012513689600, 140012513697791, ++STORE, 140012513689600, 140012513697791, ++SNULL, 140012513693695, 140012513697791, ++STORE, 140012513689600, 140012513693695, ++STORE, 140012513693696, 140012513697791, ++ERASE, 140012547100672, 140012547129343, ++ERASE, 140011362054144, 140011362058239, ++ERASE, 140011362058240, 140011370446847, ++ERASE, 140011882139648, 140011882143743, ++ERASE, 140011882143744, 140011890532351, ++ERASE, 140011873746944, 140011873751039, ++ERASE, 140011873751040, 140011882139647, ++ERASE, 140011588526080, 140011588530175, ++ERASE, 140011588530176, 140011596918783, ++ERASE, 140011328483328, 140011328487423, ++ERASE, 140011328487424, 140011336876031, ++ERASE, 140011898925056, 140011898929151, ++ERASE, 140011898929152, 140011907317759, ++ERASE, 140011353661440, 140011353665535, ++ERASE, 140011353665536, 140011362054143, ++ERASE, 140011336876032, 140011336880127, ++ERASE, 140011336880128, 140011345268735, ++ERASE, 140011731136512, 140011731140607, ++ERASE, 140011731140608, 140011739529215, ++ERASE, 140011479486464, 140011479490559, ++ERASE, 140011479490560, 140011487879167, ++ERASE, 140011756314624, 140011756318719, ++ERASE, 140011756318720, 140011764707327, ++ERASE, 140011580133376, 140011580137471, ++ERASE, 140011580137472, 140011588526079, ++ERASE, 140011219443712, 140011219447807, ++ERASE, 140011219447808, 140011227836415, ++ERASE, 140011051655168, 140011051659263, ++ERASE, 140011051659264, 140011060047871, ++ERASE, 140011999571968, 140011999576063, ++ERASE, 140011999576064, 140012007964671, ++ERASE, 140011714351104, 140011714355199, ++ERASE, 140011714355200, 140011722743807, ++ERASE, 140011739529216, 140011739533311, ++ERASE, 140011739533312, 140011747921919, ++ERASE, 140011320090624, 140011320094719, ++ERASE, 140011320094720, 140011328483327, ++ERASE, 140011630489600, 140011630493695, ++ERASE, 140011630493696, 140011638882303, ++ERASE, 140011345268736, 140011345272831, ++ERASE, 140011345272832, 140011353661439, ++ERASE, 140011496271872, 140011496275967, ++ERASE, 140011496275968, 140011504664575, ++ERASE, 140011194265600, 140011194269695, ++ERASE, 140011194269696, 140011202658303, ++ERASE, 140011068440576, 140011068444671, ++ERASE, 140011068444672, 140011076833279, ++ERASE, 140010909044736, 140010909048831, ++ERASE, 140010909048832, 140010917437439, ++ERASE, 140011764707328, 140011764711423, ++ERASE, 140011764711424, 140011773100031, ++ERASE, 140011462701056, 140011462705151, ++ERASE, 140011462705152, 140011471093759, ++ERASE, 140011076833280, 140011076837375, ++ERASE, 140011076837376, 140011085225983, ++ERASE, 140011991179264, 140011991183359, ++ERASE, 140011991183360, 140011999571967, ++ERASE, 140011211051008, 140011211055103, ++ERASE, 140011211055104, 140011219443711, ++ERASE, 140010917437440, 140010917441535, ++ERASE, 140010917441536, 140010925830143, ++ERASE, 140011085225984, 140011085230079, ++ERASE, 140011085230080, 140011093618687, ++ERASE, 140011487879168, 140011487883263, ++ERASE, 140011487883264, 140011496271871, ++ERASE, 140011856961536, 140011856965631, ++ERASE, 140011856965632, 140011865354239, ++ERASE, 140011982786560, 140011982790655, ++ERASE, 140011982790656, 140011991179263, ++ERASE, 140011722743808, 140011722747903, ++ERASE, 140011722747904, 140011731136511, ++ERASE, 140011177480192, 140011177484287, ++ERASE, 140011177484288, 140011185872895, ++ERASE, 140011848568832, 140011848572927, ++ERASE, 140011848572928, 140011856961535, ++ERASE, 140011890532352, 140011890536447, ++ERASE, 140011890536448, 140011898925055, ++ERASE, 140011622096896, 140011622100991, ++ERASE, 140011622100992, 140011630489599, ++ERASE, 140011311697920, 140011311702015, ++ERASE, 140011311702016, 140011320090623, ++ERASE, 140011471093760, 140011471097855, ++ERASE, 140011471097856, 140011479486463, ++ERASE, 140011605311488, 140011605315583, ++ERASE, 140011605315584, 140011613704191, ++ERASE, 140010791612416, 140010791616511, ++ERASE, 140010791616512, 140010800005119, ++ERASE, 140010959400960, 140010959405055, ++ERASE, 140010959405056, 140010967793663, ++ERASE, 140011185872896, 140011185876991, ++ERASE, 140011185876992, 140011194265599, ++ERASE, 140011454308352, 140011454312447, ++ERASE, 140011454312448, 140011462701055, ++ERASE, 140011596918784, 140011596922879, ++ERASE, 140011596922880, 140011605311487, ++ERASE, 140011060047872, 140011060051967, ++ERASE, 140011060051968, 140011068440575, ++ERASE, 140010925830144, 140010925834239, ++ERASE, 140010925834240, 140010934222847, ++ERASE, 140011747921920, 140011747926015, ++ERASE, 140011747926016, 140011756314623, ++ERASE, 140011202658304, 140011202662399, ++ERASE, 140011202662400, 140011211051007, ++ERASE, 140010800005120, 140010800009215, ++ERASE, 140010800009216, 140010808397823, ++ERASE, 140011093618688, 140011093622783, ++ERASE, 140011093622784, 140011102011391, ++ERASE, 140010808397824, 140010808401919, ++ERASE, 140010808401920, 140010816790527, ++ERASE, 140012419010560, 140012419014655, ++ERASE, 140012419014656, 140012427403263, ++ERASE, 140010934222848, 140010934226943, ++ERASE, 140010934226944, 140010942615551, ++ERASE, 140010942615552, 140010942619647, ++ERASE, 140010942619648, 140010951008255, ++ERASE, 140011613704192, 140011613708287, ++ERASE, 140011613708288, 140011622096895, ++ERASE, 140011865354240, 140011865358335, ++ERASE, 140011865358336, 140011873746943, ++ERASE, 140012301578240, 140012301582335, ++ERASE, 140012301582336, 140012309970943, ++ERASE, 140012393832448, 140012393836543, ++ERASE, 140012393836544, 140012402225151, ++ERASE, 140012410617856, 140012410621951, ++ERASE, 140012410621952, 140012419010559, ++ERASE, 140012402225152, 140012402229247, ++ERASE, 140012402229248, 140012410617855, ++ERASE, 140012259614720, 140012259618815, ++ERASE, 140012259618816, 140012268007423, ++ERASE, 140012251222016, 140012251226111, ++ERASE, 140012251226112, 140012259614719, ++ERASE, 140012284792832, 140012284796927, ++ERASE, 140012284796928, 140012293185535, ++ERASE, 140011445915648, 140011445919743, ++ERASE, 140011445919744, 140011454308351, ++ERASE, 140010951008256, 140010951012351, ++ERASE, 140010951012352, 140010959400959, ++ERASE, 140011043262464, 140011043266559, ++ERASE, 140011043266560, 140011051655167, ++ERASE, 140010825183232, 140010825187327, ++ERASE, 140010825187328, 140010833575935, ++ERASE, 140012293185536, 140012293189631, ++ERASE, 140012293189632, 140012301578239, ++ERASE, 140012276400128, 140012276404223, ++ERASE, 140012276404224, 140012284792831, ++ERASE, 140012016357376, 140012016361471, ++ERASE, 140012016361472, 140012024750079, ++ERASE, 140012024750080, 140012024754175, ++ERASE, 140012024754176, 140012033142783, ++ERASE, 140011227836416, 140011227840511, ++ERASE, 140011227840512, 140011236229119, ++ERASE, 140010816790528, 140010816794623, ++ERASE, 140010816794624, 140010825183231, ++ERASE, 140012268007424, 140012268011519, ++ERASE, 140012268011520, 140012276400127, ++ERASE, 140012385439744, 140012385443839, ++ERASE, 140012385443840, 140012393832447, ++ERASE, 140012522090496, 140012522094591, ++ERASE, 140012522094592, 140012530483199, ++ERASE, 140012033142784, 140012033146879, ++ERASE, 140012033146880, 140012041535487, ++ }; ++ unsigned long set35[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140730536939520, 140737488351231, ++SNULL, 140730536943615, 140737488351231, ++STORE, 140730536939520, 140730536943615, ++STORE, 140730536808448, 140730536943615, ++STORE, 94245239877632, 94245242130431, ++SNULL, 94245240008703, 94245242130431, ++STORE, 94245239877632, 94245240008703, ++STORE, 94245240008704, 94245242130431, ++ERASE, 94245240008704, 94245242130431, ++STORE, 94245242101760, 94245242109951, ++STORE, 94245242109952, 94245242130431, ++STORE, 140475575263232, 140475577516031, ++SNULL, 140475575406591, 140475577516031, ++STORE, 140475575263232, 140475575406591, ++STORE, 140475575406592, 140475577516031, ++ERASE, 140475575406592, 140475577516031, ++STORE, 140475577503744, 140475577511935, ++STORE, 140475577511936, 140475577516031, ++STORE, 140730538164224, 140730538168319, ++STORE, 140730538151936, 140730538164223, ++STORE, 140475577475072, 140475577503743, ++STORE, 140475577466880, 140475577475071, ++STORE, 140475573047296, 140475575263231, ++SNULL, 140475573047296, 140475573145599, ++STORE, 140475573145600, 140475575263231, ++STORE, 140475573047296, 140475573145599, ++SNULL, 140475575238655, 140475575263231, ++STORE, 140475573145600, 140475575238655, ++STORE, 140475575238656, 140475575263231, ++SNULL, 140475575238656, 140475575246847, ++STORE, 140475575246848, 140475575263231, ++STORE, 140475575238656, 140475575246847, ++ERASE, 140475575238656, 140475575246847, ++STORE, 140475575238656, 140475575246847, ++ERASE, 140475575246848, 140475575263231, ++STORE, 140475575246848, 140475575263231, ++STORE, 140475569250304, 140475573047295, ++SNULL, 140475569250304, 140475570909183, ++STORE, 140475570909184, 140475573047295, ++STORE, 140475569250304, 140475570909183, ++SNULL, 140475573006335, 140475573047295, ++STORE, 140475570909184, 140475573006335, ++STORE, 140475573006336, 140475573047295, ++SNULL, 140475573006336, 140475573030911, ++STORE, 140475573030912, 140475573047295, ++STORE, 140475573006336, 140475573030911, ++ERASE, 140475573006336, 140475573030911, ++STORE, 140475573006336, 140475573030911, ++ERASE, 140475573030912, 140475573047295, ++STORE, 140475573030912, 140475573047295, ++STORE, 140475577458688, 140475577475071, ++SNULL, 140475573022719, 140475573030911, ++STORE, 140475573006336, 140475573022719, ++STORE, 140475573022720, 140475573030911, ++SNULL, 140475575242751, 140475575246847, ++STORE, 140475575238656, 140475575242751, ++STORE, 140475575242752, 140475575246847, ++SNULL, 94245242105855, 94245242109951, ++STORE, 94245242101760, 94245242105855, ++STORE, 94245242105856, 94245242109951, ++SNULL, 140475577507839, 140475577511935, ++STORE, 140475577503744, 140475577507839, ++STORE, 140475577507840, 140475577511935, ++ERASE, 140475577475072, 140475577503743, ++STORE, 94245271216128, 94245271351295, ++STORE, 140475560857600, 140475569250303, ++SNULL, 140475560861695, 140475569250303, ++STORE, 140475560857600, 140475560861695, ++STORE, 140475560861696, 140475569250303, ++STORE, 140475552464896, 140475560857599, ++STORE, 140475418247168, 140475552464895, ++SNULL, 140475418247168, 140475428241407, ++STORE, 140475428241408, 140475552464895, ++STORE, 140475418247168, 140475428241407, ++ERASE, 140475418247168, 140475428241407, ++SNULL, 140475495350271, 140475552464895, ++STORE, 140475428241408, 140475495350271, ++STORE, 140475495350272, 140475552464895, ++ERASE, 140475495350272, 140475552464895, ++SNULL, 140475428376575, 140475495350271, ++STORE, 140475428241408, 140475428376575, ++STORE, 140475428376576, 140475495350271, ++SNULL, 140475552468991, 140475560857599, ++STORE, 140475552464896, 140475552468991, ++STORE, 140475552468992, 140475560857599, ++STORE, 140475544072192, 140475552464895, ++SNULL, 140475544076287, 140475552464895, ++STORE, 140475544072192, 140475544076287, ++STORE, 140475544076288, 140475552464895, ++STORE, 140475535679488, 140475544072191, ++SNULL, 140475535683583, 140475544072191, ++STORE, 140475535679488, 140475535683583, ++STORE, 140475535683584, 140475544072191, ++STORE, 140475527286784, 140475535679487, ++SNULL, 140475527290879, 140475535679487, ++STORE, 140475527286784, 140475527290879, ++STORE, 140475527290880, 140475535679487, ++STORE, 140475518894080, 140475527286783, ++STORE, 140475510501376, 140475527286783, ++STORE, 140475502108672, 140475527286783, ++STORE, 140475419848704, 140475428241407, ++STORE, 140475285630976, 140475419848703, ++SNULL, 140475285630976, 140475294023679, ++STORE, 140475294023680, 140475419848703, ++STORE, 140475285630976, 140475294023679, ++ERASE, 140475285630976, 140475294023679, ++STORE, 140475159805952, 140475419848703, ++STORE, 140475025588224, 140475419848703, ++SNULL, 140475092697087, 140475419848703, ++STORE, 140475025588224, 140475092697087, ++STORE, 140475092697088, 140475419848703, ++SNULL, 140475092697088, 140475159805951, ++STORE, 140475159805952, 140475419848703, ++STORE, 140475092697088, 140475159805951, ++ERASE, 140475092697088, 140475159805951, ++STORE, 140474891370496, 140475092697087, ++SNULL, 140474958479359, 140475092697087, ++STORE, 140474891370496, 140474958479359, ++STORE, 140474958479360, 140475092697087, ++SNULL, 140474958479360, 140475025588223, ++STORE, 140475025588224, 140475092697087, ++STORE, 140474958479360, 140475025588223, ++ERASE, 140474958479360, 140475025588223, ++SNULL, 140475361132543, 140475419848703, ++STORE, 140475159805952, 140475361132543, ++STORE, 140475361132544, 140475419848703, ++ERASE, 140475361132544, 140475419848703, ++SNULL, 140475159805952, 140475294023679, ++STORE, 140475294023680, 140475361132543, ++STORE, 140475159805952, 140475294023679, ++SNULL, 140475294158847, 140475361132543, ++STORE, 140475294023680, 140475294158847, ++STORE, 140475294158848, 140475361132543, ++SNULL, 140475226914815, 140475294023679, ++STORE, 140475159805952, 140475226914815, ++STORE, 140475226914816, 140475294023679, ++ERASE, 140475226914816, 140475294023679, ++SNULL, 140475025723391, 140475092697087, ++STORE, 140475025588224, 140475025723391, ++STORE, 140475025723392, 140475092697087, ++SNULL, 140475159941119, 140475226914815, ++STORE, 140475159805952, 140475159941119, ++STORE, 140475159941120, 140475226914815, ++SNULL, 140474891505663, 140474958479359, ++STORE, 140474891370496, 140474891505663, ++STORE, 140474891505664, 140474958479359, ++SNULL, 140475502108672, 140475518894079, ++STORE, 140475518894080, 140475527286783, ++STORE, 140475502108672, 140475518894079, ++SNULL, 140475518898175, 140475527286783, ++STORE, 140475518894080, 140475518898175, ++STORE, 140475518898176, 140475527286783, ++STORE, 140475411456000, 140475428241407, ++SNULL, 140475502112767, 140475518894079, ++STORE, 140475502108672, 140475502112767, ++STORE, 140475502112768, 140475518894079, ++SNULL, 140475411460095, 140475428241407, ++STORE, 140475411456000, 140475411460095, ++STORE, 140475411460096, 140475428241407, ++SNULL, 140475411460096, 140475419848703, ++STORE, 140475419848704, 140475428241407, ++STORE, 140475411460096, 140475419848703, ++SNULL, 140475419852799, 140475428241407, ++STORE, 140475419848704, 140475419852799, ++STORE, 140475419852800, 140475428241407, ++STORE, 140475403063296, 140475411455999, ++SNULL, 140475502112768, 140475510501375, ++STORE, 140475510501376, 140475518894079, ++STORE, 140475502112768, 140475510501375, ++SNULL, 140475510505471, 140475518894079, ++STORE, 140475510501376, 140475510505471, ++STORE, 140475510505472, 140475518894079, ++SNULL, 140475403067391, 140475411455999, ++STORE, 140475403063296, 140475403067391, ++STORE, 140475403067392, 140475411455999, ++STORE, 140475394670592, 140475403063295, ++SNULL, 140475394674687, 140475403063295, ++STORE, 140475394670592, 140475394674687, ++STORE, 140475394674688, 140475403063295, ++STORE, 140475386277888, 140475394670591, ++STORE, 140475377885184, 140475394670591, ++STORE, 140475369492480, 140475394670591, ++SNULL, 140475369496575, 140475394670591, ++STORE, 140475369492480, 140475369496575, ++STORE, 140475369496576, 140475394670591, ++SNULL, 140475369496576, 140475377885183, ++STORE, 140475377885184, 140475394670591, ++STORE, 140475369496576, 140475377885183, ++SNULL, 140475377889279, 140475394670591, ++STORE, 140475377885184, 140475377889279, ++STORE, 140475377889280, 140475394670591, ++STORE, 140475285630976, 140475294023679, ++SNULL, 140475377889280, 140475386277887, ++STORE, 140475386277888, 140475394670591, ++STORE, 140475377889280, 140475386277887, ++SNULL, 140475386281983, 140475394670591, ++STORE, 140475386277888, 140475386281983, ++STORE, 140475386281984, 140475394670591, ++SNULL, 140475285635071, 140475294023679, ++STORE, 140475285630976, 140475285635071, ++STORE, 140475285635072, 140475294023679, ++STORE, 140475277238272, 140475285630975, ++STORE, 140475268845568, 140475285630975, ++SNULL, 140475268845568, 140475277238271, ++STORE, 140475277238272, 140475285630975, ++STORE, 140475268845568, 140475277238271, ++SNULL, 140475277242367, 140475285630975, ++STORE, 140475277238272, 140475277242367, ++STORE, 140475277242368, 140475285630975, ++STORE, 140475260452864, 140475277238271, ++SNULL, 140475260452864, 140475268845567, ++STORE, 140475268845568, 140475277238271, ++STORE, 140475260452864, 140475268845567, ++SNULL, 140475268849663, 140475277238271, ++STORE, 140475268845568, 140475268849663, ++STORE, 140475268849664, 140475277238271, ++SNULL, 140475260456959, 140475268845567, ++STORE, 140475260452864, 140475260456959, ++STORE, 140475260456960, 140475268845567, ++STORE, 140475252060160, 140475260452863, ++SNULL, 140475252064255, 140475260452863, ++STORE, 140475252060160, 140475252064255, ++STORE, 140475252064256, 140475260452863, ++STORE, 140475243667456, 140475252060159, ++SNULL, 140475243671551, 140475252060159, ++STORE, 140475243667456, 140475243671551, ++STORE, 140475243671552, 140475252060159, ++STORE, 140475235274752, 140475243667455, ++STORE, 140475151413248, 140475159805951, ++STORE, 140474891505664, 140475025588223, ++STORE, 140475143020544, 140475159805951, ++SNULL, 140474891505664, 140474958479359, ++STORE, 140474958479360, 140475025588223, ++STORE, 140474891505664, 140474958479359, ++SNULL, 140474958614527, 140475025588223, ++STORE, 140474958479360, 140474958614527, ++STORE, 140474958614528, 140475025588223, ++STORE, 140474824261632, 140474891370495, ++SNULL, 140474824396799, 140474891370495, ++STORE, 140474824261632, 140474824396799, ++STORE, 140474824396800, 140474891370495, ++STORE, 140475134627840, 140475159805951, ++STORE, 140474690043904, 140474824261631, ++STORE, 140475126235136, 140475159805951, ++STORE, 140475117842432, 140475159805951, ++STORE, 140474622935040, 140474824261631, ++STORE, 140475109449728, 140475159805951, ++STORE, 140474488717312, 140474824261631, ++STORE, 140475101057024, 140475159805951, ++STORE, 140474480324608, 140474488717311, ++STORE, 140474413215744, 140474480324607, ++STORE, 140474404823040, 140474413215743, ++ERASE, 140474413215744, 140474480324607, ++STORE, 140474471931904, 140474488717311, ++STORE, 140474270605312, 140474404823039, ++SNULL, 140475101057024, 140475126235135, ++STORE, 140475126235136, 140475159805951, ++STORE, 140475101057024, 140475126235135, ++SNULL, 140475126239231, 140475159805951, ++STORE, 140475126235136, 140475126239231, ++STORE, 140475126239232, 140475159805951, ++STORE, 140474463539200, 140474488717311, ++STORE, 140474455146496, 140474488717311, ++SNULL, 140474455150591, 140474488717311, ++STORE, 140474455146496, 140474455150591, ++STORE, 140474455150592, 140474488717311, ++STORE, 140474446753792, 140474455146495, ++SNULL, 140474446757887, 140474455146495, ++STORE, 140474446753792, 140474446757887, ++STORE, 140474446757888, 140474455146495, ++STORE, 140474438361088, 140474446753791, ++STORE, 140474429968384, 140474446753791, ++SNULL, 140474429972479, 140474446753791, ++STORE, 140474429968384, 140474429972479, ++STORE, 140474429972480, 140474446753791, ++SNULL, 140475235278847, 140475243667455, ++STORE, 140475235274752, 140475235278847, ++STORE, 140475235278848, 140475243667455, ++SNULL, 140474757152767, 140474824261631, ++STORE, 140474488717312, 140474757152767, ++STORE, 140474757152768, 140474824261631, ++ERASE, 140474757152768, 140474824261631, ++SNULL, 140474488717312, 140474690043903, ++STORE, 140474690043904, 140474757152767, ++STORE, 140474488717312, 140474690043903, ++SNULL, 140474690179071, 140474757152767, ++STORE, 140474690043904, 140474690179071, ++STORE, 140474690179072, 140474757152767, ++SNULL, 140474488717312, 140474622935039, ++STORE, 140474622935040, 140474690043903, ++STORE, 140474488717312, 140474622935039, ++SNULL, 140474623070207, 140474690043903, ++STORE, 140474622935040, 140474623070207, ++STORE, 140474623070208, 140474690043903, ++SNULL, 140475101057024, 140475117842431, ++STORE, 140475117842432, 140475126235135, ++STORE, 140475101057024, 140475117842431, ++SNULL, 140475117846527, 140475126235135, ++STORE, 140475117842432, 140475117846527, ++STORE, 140475117846528, 140475126235135, ++SNULL, 140474555826175, 140474622935039, ++STORE, 140474488717312, 140474555826175, ++STORE, 140474555826176, 140474622935039, ++ERASE, 140474555826176, 140474622935039, ++STORE, 140474136387584, 140474404823039, ++SNULL, 140474136387584, 140474153172991, ++STORE, 140474153172992, 140474404823039, ++STORE, 140474136387584, 140474153172991, ++ERASE, 140474136387584, 140474153172991, ++STORE, 140474018955264, 140474404823039, ++STORE, 140473884737536, 140474404823039, ++SNULL, 140474086064127, 140474404823039, ++STORE, 140473884737536, 140474086064127, ++STORE, 140474086064128, 140474404823039, ++SNULL, 140474086064128, 140474153172991, ++STORE, 140474153172992, 140474404823039, ++STORE, 140474086064128, 140474153172991, ++ERASE, 140474086064128, 140474153172991, ++STORE, 140473750519808, 140474086064127, ++SNULL, 140473817628671, 140474086064127, ++STORE, 140473750519808, 140473817628671, ++STORE, 140473817628672, 140474086064127, ++SNULL, 140473817628672, 140473884737535, ++STORE, 140473884737536, 140474086064127, ++STORE, 140473817628672, 140473884737535, ++ERASE, 140473817628672, 140473884737535, ++SNULL, 140475126239232, 140475151413247, ++STORE, 140475151413248, 140475159805951, ++STORE, 140475126239232, 140475151413247, ++SNULL, 140475151417343, 140475159805951, ++STORE, 140475151413248, 140475151417343, ++STORE, 140475151417344, 140475159805951, ++SNULL, 140474270605311, 140474404823039, ++STORE, 140474153172992, 140474270605311, ++STORE, 140474270605312, 140474404823039, ++SNULL, 140474270605312, 140474287390719, ++STORE, 140474287390720, 140474404823039, ++STORE, 140474270605312, 140474287390719, ++ERASE, 140474270605312, 140474287390719, ++SNULL, 140474429972480, 140474438361087, ++STORE, 140474438361088, 140474446753791, ++STORE, 140474429972480, 140474438361087, ++SNULL, 140474438365183, 140474446753791, ++STORE, 140474438361088, 140474438365183, ++STORE, 140474438365184, 140474446753791, ++STORE, 140474815868928, 140474824261631, ++SNULL, 140474815873023, 140474824261631, ++STORE, 140474815868928, 140474815873023, ++STORE, 140474815873024, 140474824261631, ++SNULL, 140474220281855, 140474270605311, ++STORE, 140474153172992, 140474220281855, ++STORE, 140474220281856, 140474270605311, ++ERASE, 140474220281856, 140474270605311, ++SNULL, 140474488852479, 140474555826175, ++STORE, 140474488717312, 140474488852479, ++STORE, 140474488852480, 140474555826175, ++SNULL, 140475101057024, 140475109449727, ++STORE, 140475109449728, 140475117842431, ++STORE, 140475101057024, 140475109449727, ++SNULL, 140475109453823, 140475117842431, ++STORE, 140475109449728, 140475109453823, ++STORE, 140475109453824, 140475117842431, ++SNULL, 140473951846399, 140474086064127, ++STORE, 140473884737536, 140473951846399, ++STORE, 140473951846400, 140474086064127, ++SNULL, 140473951846400, 140474018955263, ++STORE, 140474018955264, 140474086064127, ++STORE, 140473951846400, 140474018955263, ++ERASE, 140473951846400, 140474018955263, ++SNULL, 140473884872703, 140473951846399, ++STORE, 140473884737536, 140473884872703, ++STORE, 140473884872704, 140473951846399, ++SNULL, 140474019090431, 140474086064127, ++STORE, 140474018955264, 140474019090431, ++STORE, 140474019090432, 140474086064127, ++SNULL, 140473750654975, 140473817628671, ++STORE, 140473750519808, 140473750654975, ++STORE, 140473750654976, 140473817628671, ++SNULL, 140474455150592, 140474463539199, ++STORE, 140474463539200, 140474488717311, ++STORE, 140474455150592, 140474463539199, ++SNULL, 140474463543295, 140474488717311, ++STORE, 140474463539200, 140474463543295, ++STORE, 140474463543296, 140474488717311, ++STORE, 140474807476224, 140474815868927, ++SNULL, 140474463543296, 140474471931903, ++STORE, 140474471931904, 140474488717311, ++STORE, 140474463543296, 140474471931903, ++SNULL, 140474471935999, 140474488717311, ++STORE, 140474471931904, 140474471935999, ++STORE, 140474471936000, 140474488717311, ++STORE, 140474799083520, 140474815868927, ++STORE, 140474790690816, 140474815868927, ++SNULL, 140474790690816, 140474799083519, ++STORE, 140474799083520, 140474815868927, ++STORE, 140474790690816, 140474799083519, ++SNULL, 140474799087615, 140474815868927, ++STORE, 140474799083520, 140474799087615, ++STORE, 140474799087616, 140474815868927, ++SNULL, 140474354499583, 140474404823039, ++STORE, 140474287390720, 140474354499583, ++STORE, 140474354499584, 140474404823039, ++ERASE, 140474354499584, 140474404823039, ++SNULL, 140474287525887, 140474354499583, ++STORE, 140474287390720, 140474287525887, ++STORE, 140474287525888, 140474354499583, ++STORE, 140474782298112, 140474799083519, ++STORE, 140474773905408, 140474799083519, ++SNULL, 140474773909503, 140474799083519, ++STORE, 140474773905408, 140474773909503, ++STORE, 140474773909504, 140474799083519, ++SNULL, 140475126239232, 140475134627839, ++STORE, 140475134627840, 140475151413247, ++STORE, 140475126239232, 140475134627839, ++SNULL, 140475134631935, 140475151413247, ++STORE, 140475134627840, 140475134631935, ++STORE, 140475134631936, 140475151413247, ++STORE, 140474765512704, 140474773905407, ++STORE, 140474614542336, 140474622935039, ++SNULL, 140474153308159, 140474220281855, ++STORE, 140474153172992, 140474153308159, ++STORE, 140474153308160, 140474220281855, ++SNULL, 140474404827135, 140474413215743, ++STORE, 140474404823040, 140474404827135, ++STORE, 140474404827136, 140474413215743, ++STORE, 140474606149632, 140474622935039, ++SNULL, 140474606153727, 140474622935039, ++STORE, 140474606149632, 140474606153727, ++STORE, 140474606153728, 140474622935039, ++STORE, 140474597756928, 140474606149631, ++SNULL, 140474597761023, 140474606149631, ++STORE, 140474597756928, 140474597761023, ++STORE, 140474597761024, 140474606149631, ++SNULL, 140475134631936, 140475143020543, ++STORE, 140475143020544, 140475151413247, ++STORE, 140475134631936, 140475143020543, ++SNULL, 140475143024639, 140475151413247, ++STORE, 140475143020544, 140475143024639, ++STORE, 140475143024640, 140475151413247, ++STORE, 140474589364224, 140474597756927, ++SNULL, 140474606153728, 140474614542335, ++STORE, 140474614542336, 140474622935039, ++STORE, 140474606153728, 140474614542335, ++SNULL, 140474614546431, 140474622935039, ++STORE, 140474614542336, 140474614546431, ++STORE, 140474614546432, 140474622935039, ++SNULL, 140474765516799, 140474773905407, ++STORE, 140474765512704, 140474765516799, ++STORE, 140474765516800, 140474773905407, ++STORE, 140474580971520, 140474597756927, ++SNULL, 140474773909504, 140474782298111, ++STORE, 140474782298112, 140474799083519, ++STORE, 140474773909504, 140474782298111, ++SNULL, 140474782302207, 140474799083519, ++STORE, 140474782298112, 140474782302207, ++STORE, 140474782302208, 140474799083519, ++SNULL, 140474471936000, 140474480324607, ++STORE, 140474480324608, 140474488717311, ++STORE, 140474471936000, 140474480324607, ++SNULL, 140474480328703, 140474488717311, ++STORE, 140474480324608, 140474480328703, ++STORE, 140474480328704, 140474488717311, ++STORE, 140474572578816, 140474597756927, ++SNULL, 140474572582911, 140474597756927, ++STORE, 140474572578816, 140474572582911, ++STORE, 140474572582912, 140474597756927, ++SNULL, 140474782302208, 140474790690815, ++STORE, 140474790690816, 140474799083519, ++STORE, 140474782302208, 140474790690815, ++SNULL, 140474790694911, 140474799083519, ++STORE, 140474790690816, 140474790694911, ++STORE, 140474790694912, 140474799083519, ++STORE, 140474564186112, 140474572578815, ++STORE, 140474421575680, 140474429968383, ++STORE, 140474396430336, 140474404823039, ++SNULL, 140474396434431, 140474404823039, ++STORE, 140474396430336, 140474396434431, ++STORE, 140474396434432, 140474404823039, ++STORE, 140474388037632, 140474396430335, ++SNULL, 140474799087616, 140474807476223, ++STORE, 140474807476224, 140474815868927, ++STORE, 140474799087616, 140474807476223, ++SNULL, 140474807480319, 140474815868927, ++STORE, 140474807476224, 140474807480319, ++STORE, 140474807480320, 140474815868927, ++SNULL, 140475101061119, 140475109449727, ++STORE, 140475101057024, 140475101061119, ++STORE, 140475101061120, 140475109449727, ++STORE, 140474379644928, 140474396430335, ++SNULL, 140474572582912, 140474589364223, ++STORE, 140474589364224, 140474597756927, ++STORE, 140474572582912, 140474589364223, ++SNULL, 140474589368319, 140474597756927, ++STORE, 140474589364224, 140474589368319, ++STORE, 140474589368320, 140474597756927, ++STORE, 140474371252224, 140474396430335, ++STORE, 140474362859520, 140474396430335, ++STORE, 140474278998016, 140474287390719, ++STORE, 140474270605312, 140474287390719, ++STORE, 140474262212608, 140474287390719, ++SNULL, 140474262216703, 140474287390719, ++STORE, 140474262212608, 140474262216703, ++STORE, 140474262216704, 140474287390719, ++STORE, 140474253819904, 140474262212607, ++SNULL, 140474253823999, 140474262212607, ++STORE, 140474253819904, 140474253823999, ++STORE, 140474253824000, 140474262212607, ++SNULL, 140474362859520, 140474388037631, ++STORE, 140474388037632, 140474396430335, ++STORE, 140474362859520, 140474388037631, ++SNULL, 140474388041727, 140474396430335, ++STORE, 140474388037632, 140474388041727, ++STORE, 140474388041728, 140474396430335, ++SNULL, 140474362859520, 140474379644927, ++STORE, 140474379644928, 140474388037631, ++STORE, 140474362859520, 140474379644927, ++SNULL, 140474379649023, 140474388037631, ++STORE, 140474379644928, 140474379649023, ++STORE, 140474379649024, 140474388037631, ++STORE, 140474245427200, 140474253819903, ++STORE, 140474237034496, 140474253819903, ++STORE, 140474228641792, 140474253819903, ++STORE, 140474144780288, 140474153172991, ++SNULL, 140474228645887, 140474253819903, ++STORE, 140474228641792, 140474228645887, ++STORE, 140474228645888, 140474253819903, ++SNULL, 140474564190207, 140474572578815, ++STORE, 140474564186112, 140474564190207, ++STORE, 140474564190208, 140474572578815, ++STORE, 140474136387584, 140474153172991, ++SNULL, 140474362859520, 140474371252223, ++STORE, 140474371252224, 140474379644927, ++STORE, 140474362859520, 140474371252223, ++SNULL, 140474371256319, 140474379644927, ++STORE, 140474371252224, 140474371256319, ++STORE, 140474371256320, 140474379644927, ++STORE, 140474127994880, 140474153172991, ++STORE, 140474119602176, 140474153172991, ++SNULL, 140474421579775, 140474429968383, ++STORE, 140474421575680, 140474421579775, ++STORE, 140474421579776, 140474429968383, ++STORE, 140474111209472, 140474153172991, ++SNULL, 140474111213567, 140474153172991, ++STORE, 140474111209472, 140474111213567, ++STORE, 140474111213568, 140474153172991, ++SNULL, 140474262216704, 140474270605311, ++STORE, 140474270605312, 140474287390719, ++STORE, 140474262216704, 140474270605311, ++SNULL, 140474270609407, 140474287390719, ++STORE, 140474270605312, 140474270609407, ++STORE, 140474270609408, 140474287390719, ++STORE, 140474102816768, 140474111209471, ++SNULL, 140474102820863, 140474111209471, ++STORE, 140474102816768, 140474102820863, ++STORE, 140474102820864, 140474111209471, ++SNULL, 140474270609408, 140474278998015, ++STORE, 140474278998016, 140474287390719, ++STORE, 140474270609408, 140474278998015, ++SNULL, 140474279002111, 140474287390719, ++STORE, 140474278998016, 140474279002111, ++STORE, 140474279002112, 140474287390719, ++STORE, 140474094424064, 140474102816767, ++SNULL, 140474572582912, 140474580971519, ++STORE, 140474580971520, 140474589364223, ++STORE, 140474572582912, 140474580971519, ++SNULL, 140474580975615, 140474589364223, ++STORE, 140474580971520, 140474580975615, ++STORE, 140474580975616, 140474589364223, ++SNULL, 140474362863615, 140474371252223, ++STORE, 140474362859520, 140474362863615, ++STORE, 140474362863616, 140474371252223, ++STORE, 140474010562560, 140474018955263, ++SNULL, 140474228645888, 140474245427199, ++STORE, 140474245427200, 140474253819903, ++STORE, 140474228645888, 140474245427199, ++SNULL, 140474245431295, 140474253819903, ++STORE, 140474245427200, 140474245431295, ++STORE, 140474245431296, 140474253819903, ++SNULL, 140474111213568, 140474136387583, ++STORE, 140474136387584, 140474153172991, ++STORE, 140474111213568, 140474136387583, ++SNULL, 140474136391679, 140474153172991, ++STORE, 140474136387584, 140474136391679, ++STORE, 140474136391680, 140474153172991, ++STORE, 140474002169856, 140474018955263, ++STORE, 140473993777152, 140474018955263, ++SNULL, 140474111213568, 140474127994879, ++STORE, 140474127994880, 140474136387583, ++STORE, 140474111213568, 140474127994879, ++SNULL, 140474127998975, 140474136387583, ++STORE, 140474127994880, 140474127998975, ++STORE, 140474127998976, 140474136387583, ++SNULL, 140474228645888, 140474237034495, ++STORE, 140474237034496, 140474245427199, ++STORE, 140474228645888, 140474237034495, ++SNULL, 140474237038591, 140474245427199, ++STORE, 140474237034496, 140474237038591, ++STORE, 140474237038592, 140474245427199, ++SNULL, 140474136391680, 140474144780287, ++STORE, 140474144780288, 140474153172991, ++STORE, 140474136391680, 140474144780287, ++SNULL, 140474144784383, 140474153172991, ++STORE, 140474144780288, 140474144784383, ++STORE, 140474144784384, 140474153172991, ++STORE, 140473985384448, 140474018955263, ++STORE, 140473976991744, 140474018955263, ++STORE, 140473968599040, 140474018955263, ++SNULL, 140473968603135, 140474018955263, ++STORE, 140473968599040, 140473968603135, ++STORE, 140473968603136, 140474018955263, ++SNULL, 140474111213568, 140474119602175, ++STORE, 140474119602176, 140474127994879, ++STORE, 140474111213568, 140474119602175, ++SNULL, 140474119606271, 140474127994879, ++STORE, 140474119602176, 140474119606271, ++STORE, 140474119606272, 140474127994879, ++STORE, 140473960206336, 140473968599039, ++SNULL, 140474094428159, 140474102816767, ++STORE, 140474094424064, 140474094428159, ++STORE, 140474094428160, 140474102816767, ++STORE, 140473876344832, 140473884737535, ++STORE, 140473867952128, 140473884737535, ++STORE, 140473859559424, 140473884737535, ++SNULL, 140473859563519, 140473884737535, ++STORE, 140473859559424, 140473859563519, ++STORE, 140473859563520, 140473884737535, ++SNULL, 140473968603136, 140473993777151, ++STORE, 140473993777152, 140474018955263, ++STORE, 140473968603136, 140473993777151, ++SNULL, 140473993781247, 140474018955263, ++STORE, 140473993777152, 140473993781247, ++STORE, 140473993781248, 140474018955263, ++SNULL, 140473960210431, 140473968599039, ++STORE, 140473960206336, 140473960210431, ++STORE, 140473960210432, 140473968599039, ++SNULL, 140473993781248, 140474010562559, ++STORE, 140474010562560, 140474018955263, ++STORE, 140473993781248, 140474010562559, ++SNULL, 140474010566655, 140474018955263, ++STORE, 140474010562560, 140474010566655, ++STORE, 140474010566656, 140474018955263, ++SNULL, 140473968603136, 140473985384447, ++STORE, 140473985384448, 140473993777151, ++STORE, 140473968603136, 140473985384447, ++SNULL, 140473985388543, 140473993777151, ++STORE, 140473985384448, 140473985388543, ++STORE, 140473985388544, 140473993777151, ++SNULL, 140473993781248, 140474002169855, ++STORE, 140474002169856, 140474010562559, ++STORE, 140473993781248, 140474002169855, ++SNULL, 140474002173951, 140474010562559, ++STORE, 140474002169856, 140474002173951, ++STORE, 140474002173952, 140474010562559, ++STORE, 140473851166720, 140473859559423, ++SNULL, 140473851170815, 140473859559423, ++STORE, 140473851166720, 140473851170815, ++STORE, 140473851170816, 140473859559423, ++SNULL, 140473968603136, 140473976991743, ++STORE, 140473976991744, 140473985384447, ++STORE, 140473968603136, 140473976991743, ++SNULL, 140473976995839, 140473985384447, ++STORE, 140473976991744, 140473976995839, ++STORE, 140473976995840, 140473985384447, ++STORE, 140473842774016, 140473851166719, ++SNULL, 140473859563520, 140473867952127, ++STORE, 140473867952128, 140473884737535, ++STORE, 140473859563520, 140473867952127, ++SNULL, 140473867956223, 140473884737535, ++STORE, 140473867952128, 140473867956223, ++STORE, 140473867956224, 140473884737535, ++SNULL, 140473867956224, 140473876344831, ++STORE, 140473876344832, 140473884737535, ++STORE, 140473867956224, 140473876344831, ++SNULL, 140473876348927, 140473884737535, ++STORE, 140473876344832, 140473876348927, ++STORE, 140473876348928, 140473884737535, ++STORE, 140473834381312, 140473851166719, ++SNULL, 140473834385407, 140473851166719, ++STORE, 140473834381312, 140473834385407, ++STORE, 140473834385408, 140473851166719, ++SNULL, 140473834385408, 140473842774015, ++STORE, 140473842774016, 140473851166719, ++STORE, 140473834385408, 140473842774015, ++SNULL, 140473842778111, 140473851166719, ++STORE, 140473842774016, 140473842778111, ++STORE, 140473842778112, 140473851166719, ++STORE, 140473825988608, 140473834381311, ++SNULL, 140473825992703, 140473834381311, ++STORE, 140473825988608, 140473825992703, ++STORE, 140473825992704, 140473834381311, ++STORE, 140475577475072, 140475577503743, ++STORE, 140475499917312, 140475502108671, ++SNULL, 140475499917312, 140475500007423, ++STORE, 140475500007424, 140475502108671, ++STORE, 140475499917312, 140475500007423, ++SNULL, 140475502100479, 140475502108671, ++STORE, 140475500007424, 140475502100479, ++STORE, 140475502100480, 140475502108671, ++ERASE, 140475502100480, 140475502108671, ++STORE, 140475502100480, 140475502108671, ++SNULL, 140475502104575, 140475502108671, ++STORE, 140475502100480, 140475502104575, ++STORE, 140475502104576, 140475502108671, ++ERASE, 140475577475072, 140475577503743, ++ERASE, 140475235274752, 140475235278847, ++ERASE, 140475235278848, 140475243667455, ++ERASE, 140474815868928, 140474815873023, ++ERASE, 140474815873024, 140474824261631, ++ERASE, 140474606149632, 140474606153727, ++ERASE, 140474606153728, 140474614542335, ++ERASE, 140474270605312, 140474270609407, ++ERASE, 140474270609408, 140474278998015, ++ERASE, 140474438361088, 140474438365183, ++ERASE, 140474438365184, 140474446753791, ++ERASE, 140474597756928, 140474597761023, ++ERASE, 140474597761024, 140474606149631, ++ERASE, 140475126235136, 140475126239231, ++ERASE, 140475126239232, 140475134627839, ++ERASE, 140474463539200, 140474463543295, ++ERASE, 140474463543296, 140474471931903, ++ERASE, 140474388037632, 140474388041727, ++ERASE, 140474388041728, 140474396430335, ++ERASE, 140474404823040, 140474404827135, ++ERASE, 140474404827136, 140474413215743, ++ERASE, 140474278998016, 140474279002111, ++ERASE, 140474279002112, 140474287390719, ++ERASE, 140474094424064, 140474094428159, ++ERASE, 140474094428160, 140474102816767, ++ERASE, 140473867952128, 140473867956223, ++ERASE, 140473867956224, 140473876344831, ++ERASE, 140475151413248, 140475151417343, ++ERASE, 140475151417344, 140475159805951, ++ERASE, 140474455146496, 140474455150591, ++ERASE, 140474455150592, 140474463539199, ++ERASE, 140474807476224, 140474807480319, ++ERASE, 140474807480320, 140474815868927, ++ERASE, 140475117842432, 140475117846527, ++ERASE, 140475117846528, 140475126235135, ++ERASE, 140474446753792, 140474446757887, ++ERASE, 140474446757888, 140474455146495, ++ERASE, 140474429968384, 140474429972479, ++ERASE, 140474429972480, 140474438361087, ++ERASE, 140474782298112, 140474782302207, ++ERASE, 140474782302208, 140474790690815, ++ERASE, 140474136387584, 140474136391679, ++ERASE, 140474136391680, 140474144780287, ++ERASE, 140474002169856, 140474002173951, ++ERASE, 140474002173952, 140474010562559, ++ERASE, 140475134627840, 140475134631935, ++ERASE, 140475134631936, 140475143020543, ++ERASE, 140474471931904, 140474471935999, ++ERASE, 140474471936000, 140474480324607, ++ERASE, 140474396430336, 140474396434431, ++ERASE, 140474396434432, 140474404823039, ++ }; ++ unsigned long set36[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140723893125120, 140737488351231, ++SNULL, 140723893129215, 140737488351231, ++STORE, 140723893125120, 140723893129215, ++STORE, 140723892994048, 140723893129215, ++STORE, 94076829786112, 94076832038911, ++SNULL, 94076829917183, 94076832038911, ++STORE, 94076829786112, 94076829917183, ++STORE, 94076829917184, 94076832038911, ++ERASE, 94076829917184, 94076832038911, ++STORE, 94076832010240, 94076832018431, ++STORE, 94076832018432, 94076832038911, ++STORE, 140122444345344, 140122446598143, ++SNULL, 140122444488703, 140122446598143, ++STORE, 140122444345344, 140122444488703, ++STORE, 140122444488704, 140122446598143, ++ERASE, 140122444488704, 140122446598143, ++STORE, 140122446585856, 140122446594047, ++STORE, 140122446594048, 140122446598143, ++STORE, 140723893538816, 140723893542911, ++STORE, 140723893526528, 140723893538815, ++STORE, 140122446557184, 140122446585855, ++STORE, 140122446548992, 140122446557183, ++STORE, 140122442129408, 140122444345343, ++SNULL, 140122442129408, 140122442227711, ++STORE, 140122442227712, 140122444345343, ++STORE, 140122442129408, 140122442227711, ++SNULL, 140122444320767, 140122444345343, ++STORE, 140122442227712, 140122444320767, ++STORE, 140122444320768, 140122444345343, ++SNULL, 140122444320768, 140122444328959, ++STORE, 140122444328960, 140122444345343, ++STORE, 140122444320768, 140122444328959, ++ERASE, 140122444320768, 140122444328959, ++STORE, 140122444320768, 140122444328959, ++ERASE, 140122444328960, 140122444345343, ++STORE, 140122444328960, 140122444345343, ++STORE, 140122438332416, 140122442129407, ++SNULL, 140122438332416, 140122439991295, ++STORE, 140122439991296, 140122442129407, ++STORE, 140122438332416, 140122439991295, ++SNULL, 140122442088447, 140122442129407, ++STORE, 140122439991296, 140122442088447, ++STORE, 140122442088448, 140122442129407, ++SNULL, 140122442088448, 140122442113023, ++STORE, 140122442113024, 140122442129407, ++STORE, 140122442088448, 140122442113023, ++ERASE, 140122442088448, 140122442113023, ++STORE, 140122442088448, 140122442113023, ++ERASE, 140122442113024, 140122442129407, ++STORE, 140122442113024, 140122442129407, ++STORE, 140122446540800, 140122446557183, ++SNULL, 140122442104831, 140122442113023, ++STORE, 140122442088448, 140122442104831, ++STORE, 140122442104832, 140122442113023, ++SNULL, 140122444324863, 140122444328959, ++STORE, 140122444320768, 140122444324863, ++STORE, 140122444324864, 140122444328959, ++SNULL, 94076832014335, 94076832018431, ++STORE, 94076832010240, 94076832014335, ++STORE, 94076832014336, 94076832018431, ++SNULL, 140122446589951, 140122446594047, ++STORE, 140122446585856, 140122446589951, ++STORE, 140122446589952, 140122446594047, ++ERASE, 140122446557184, 140122446585855, ++STORE, 94076845723648, 94076845858815, ++STORE, 140122429939712, 140122438332415, ++SNULL, 140122429943807, 140122438332415, ++STORE, 140122429939712, 140122429943807, ++STORE, 140122429943808, 140122438332415, ++STORE, 140122421547008, 140122429939711, ++STORE, 140122287329280, 140122421547007, ++SNULL, 140122287329280, 140122301399039, ++STORE, 140122301399040, 140122421547007, ++STORE, 140122287329280, 140122301399039, ++ERASE, 140122287329280, 140122301399039, ++SNULL, 140122368507903, 140122421547007, ++STORE, 140122301399040, 140122368507903, ++STORE, 140122368507904, 140122421547007, ++ERASE, 140122368507904, 140122421547007, ++SNULL, 140122301534207, 140122368507903, ++STORE, 140122301399040, 140122301534207, ++STORE, 140122301534208, 140122368507903, ++SNULL, 140122421551103, 140122429939711, ++STORE, 140122421547008, 140122421551103, ++STORE, 140122421551104, 140122429939711, ++STORE, 140122413154304, 140122421547007, ++SNULL, 140122413158399, 140122421547007, ++STORE, 140122413154304, 140122413158399, ++STORE, 140122413158400, 140122421547007, ++STORE, 140122404761600, 140122413154303, ++SNULL, 140122404765695, 140122413154303, ++STORE, 140122404761600, 140122404765695, ++STORE, 140122404765696, 140122413154303, ++STORE, 140122396368896, 140122404761599, ++SNULL, 140122396372991, 140122404761599, ++STORE, 140122396368896, 140122396372991, ++STORE, 140122396372992, 140122404761599, ++STORE, 140122387976192, 140122396368895, ++STORE, 140122167181312, 140122301399039, ++SNULL, 140122234290175, 140122301399039, ++STORE, 140122167181312, 140122234290175, ++STORE, 140122234290176, 140122301399039, ++ERASE, 140122234290176, 140122301399039, ++SNULL, 140122167316479, 140122234290175, ++STORE, 140122167181312, 140122167316479, ++STORE, 140122167316480, 140122234290175, ++STORE, 140122379583488, 140122396368895, ++STORE, 140122371190784, 140122396368895, ++STORE, 140122167316480, 140122301399039, ++STORE, 140122158788608, 140122167181311, ++SNULL, 140122371190784, 140122387976191, ++STORE, 140122387976192, 140122396368895, ++STORE, 140122371190784, 140122387976191, ++SNULL, 140122387980287, 140122396368895, ++STORE, 140122387976192, 140122387980287, ++STORE, 140122387980288, 140122396368895, ++SNULL, 140122167316480, 140122234290175, ++STORE, 140122234290176, 140122301399039, ++STORE, 140122167316480, 140122234290175, ++SNULL, 140122234425343, 140122301399039, ++STORE, 140122234290176, 140122234425343, ++STORE, 140122234425344, 140122301399039, ++STORE, 140122024570880, 140122158788607, ++SNULL, 140122024570880, 140122032963583, ++STORE, 140122032963584, 140122158788607, ++STORE, 140122024570880, 140122032963583, ++ERASE, 140122024570880, 140122032963583, ++STORE, 140121898745856, 140122158788607, ++STORE, 140121890353152, 140121898745855, ++SNULL, 140122100072447, 140122158788607, ++STORE, 140121898745856, 140122100072447, ++STORE, 140122100072448, 140122158788607, ++ERASE, 140122100072448, 140122158788607, ++SNULL, 140121965854719, 140122100072447, ++STORE, 140121898745856, 140121965854719, ++STORE, 140121965854720, 140122100072447, ++SNULL, 140121965854720, 140122032963583, ++STORE, 140122032963584, 140122100072447, ++STORE, 140121965854720, 140122032963583, ++ERASE, 140121965854720, 140122032963583, ++SNULL, 140121898881023, 140121965854719, ++STORE, 140121898745856, 140121898881023, ++STORE, 140121898881024, 140121965854719, ++SNULL, 140121890357247, 140121898745855, ++STORE, 140121890353152, 140121890357247, ++STORE, 140121890357248, 140121898745855, ++SNULL, 140122371190784, 140122379583487, ++STORE, 140122379583488, 140122387976191, ++STORE, 140122371190784, 140122379583487, ++SNULL, 140122379587583, 140122387976191, ++STORE, 140122379583488, 140122379587583, ++STORE, 140122379587584, 140122387976191, ++SNULL, 140122033098751, 140122100072447, ++STORE, 140122032963584, 140122033098751, ++STORE, 140122033098752, 140122100072447, ++SNULL, 140122158792703, 140122167181311, ++STORE, 140122158788608, 140122158792703, ++STORE, 140122158792704, 140122167181311, ++STORE, 140122150395904, 140122158788607, ++STORE, 140122142003200, 140122158788607, ++SNULL, 140122142007295, 140122158788607, ++STORE, 140122142003200, 140122142007295, ++STORE, 140122142007296, 140122158788607, ++SNULL, 140122371194879, 140122379583487, ++STORE, 140122371190784, 140122371194879, ++STORE, 140122371194880, 140122379583487, ++SNULL, 140122142007296, 140122150395903, ++STORE, 140122150395904, 140122158788607, ++STORE, 140122142007296, 140122150395903, ++SNULL, 140122150399999, 140122158788607, ++STORE, 140122150395904, 140122150399999, ++STORE, 140122150400000, 140122158788607, ++STORE, 140122133610496, 140122142003199, ++STORE, 140122125217792, 140122142003199, ++STORE, 140122116825088, 140122142003199, ++SNULL, 140122116829183, 140122142003199, ++STORE, 140122116825088, 140122116829183, ++STORE, 140122116829184, 140122142003199, ++SNULL, 140122116829184, 140122133610495, ++STORE, 140122133610496, 140122142003199, ++STORE, 140122116829184, 140122133610495, ++SNULL, 140122133614591, 140122142003199, ++STORE, 140122133610496, 140122133614591, ++STORE, 140122133614592, 140122142003199, ++SNULL, 140122116829184, 140122125217791, ++STORE, 140122125217792, 140122133610495, ++STORE, 140122116829184, 140122125217791, ++SNULL, 140122125221887, 140122133610495, ++STORE, 140122125217792, 140122125221887, ++STORE, 140122125221888, 140122133610495, ++STORE, 140122108432384, 140122116825087, ++SNULL, 140122108436479, 140122116825087, ++STORE, 140122108432384, 140122108436479, ++STORE, 140122108436480, 140122116825087, ++STORE, 140122024570880, 140122032963583, ++STORE, 140122016178176, 140122032963583, ++SNULL, 140122016182271, 140122032963583, ++STORE, 140122016178176, 140122016182271, ++STORE, 140122016182272, 140122032963583, ++SNULL, 140122016182272, 140122024570879, ++STORE, 140122024570880, 140122032963583, ++STORE, 140122016182272, 140122024570879, ++SNULL, 140122024574975, 140122032963583, ++STORE, 140122024570880, 140122024574975, ++STORE, 140122024574976, 140122032963583, ++STORE, 140122007785472, 140122016178175, ++SNULL, 140122007789567, 140122016178175, ++STORE, 140122007785472, 140122007789567, ++STORE, 140122007789568, 140122016178175, ++STORE, 140121999392768, 140122007785471, ++STORE, 140121991000064, 140122007785471, ++SNULL, 140121991004159, 140122007785471, ++STORE, 140121991000064, 140121991004159, ++STORE, 140121991004160, 140122007785471, ++SNULL, 140121991004160, 140121999392767, ++STORE, 140121999392768, 140122007785471, ++STORE, 140121991004160, 140121999392767, ++SNULL, 140121999396863, 140122007785471, ++STORE, 140121999392768, 140121999396863, ++STORE, 140121999396864, 140122007785471, ++STORE, 140121982607360, 140121991000063, ++STORE, 140121823244288, 140121890353151, ++ERASE, 140121823244288, 140121890353151, ++STORE, 140121756135424, 140121890353151, ++SNULL, 140121756135424, 140121764528127, ++STORE, 140121764528128, 140121890353151, ++STORE, 140121756135424, 140121764528127, ++ERASE, 140121756135424, 140121764528127, ++SNULL, 140121831636991, 140121890353151, ++STORE, 140121764528128, 140121831636991, ++STORE, 140121831636992, 140121890353151, ++ERASE, 140121831636992, 140121890353151, ++STORE, 140121974214656, 140121991000063, ++STORE, 140121630310400, 140121831636991, ++SNULL, 140121697419263, 140121831636991, ++STORE, 140121630310400, 140121697419263, ++STORE, 140121697419264, 140121831636991, ++SNULL, 140121697419264, 140121764528127, ++STORE, 140121764528128, 140121831636991, ++STORE, 140121697419264, 140121764528127, ++ERASE, 140121697419264, 140121764528127, ++STORE, 140121881960448, 140121890353151, ++STORE, 140121630310400, 140121831636991, ++STORE, 140121873567744, 140121890353151, ++SNULL, 140121630310400, 140121697419263, ++STORE, 140121697419264, 140121831636991, ++STORE, 140121630310400, 140121697419263, ++SNULL, 140121697554431, 140121831636991, ++STORE, 140121697419264, 140121697554431, ++STORE, 140121697554432, 140121831636991, ++STORE, 140121865175040, 140121890353151, ++STORE, 140121856782336, 140121890353151, ++STORE, 140121848389632, 140121890353151, ++STORE, 140121839996928, 140121890353151, ++STORE, 140121496092672, 140121697419263, ++STORE, 140121487699968, 140121496092671, ++STORE, 140121420591104, 140121487699967, ++STORE, 140121412198400, 140121420591103, ++ERASE, 140121420591104, 140121487699967, ++STORE, 140121479307264, 140121496092671, ++STORE, 140121277980672, 140121412198399, ++SNULL, 140121277980672, 140121294766079, ++STORE, 140121294766080, 140121412198399, ++STORE, 140121277980672, 140121294766079, ++ERASE, 140121277980672, 140121294766079, ++STORE, 140121470914560, 140121496092671, ++STORE, 140121462521856, 140121496092671, ++STORE, 140121160548352, 140121412198399, ++STORE, 140121454129152, 140121496092671, ++SNULL, 140121227657215, 140121412198399, ++STORE, 140121160548352, 140121227657215, ++STORE, 140121227657216, 140121412198399, ++SNULL, 140121227657216, 140121294766079, ++STORE, 140121294766080, 140121412198399, ++STORE, 140121227657216, 140121294766079, ++ERASE, 140121227657216, 140121294766079, ++STORE, 140121445736448, 140121496092671, ++STORE, 140121437343744, 140121496092671, ++SNULL, 140121437343744, 140121445736447, ++STORE, 140121445736448, 140121496092671, ++STORE, 140121437343744, 140121445736447, ++SNULL, 140121445740543, 140121496092671, ++STORE, 140121445736448, 140121445740543, ++STORE, 140121445740544, 140121496092671, ++SNULL, 140121697554432, 140121764528127, ++STORE, 140121764528128, 140121831636991, ++STORE, 140121697554432, 140121764528127, ++SNULL, 140121764663295, 140121831636991, ++STORE, 140121764528128, 140121764663295, ++STORE, 140121764663296, 140121831636991, ++SNULL, 140121496092672, 140121630310399, ++STORE, 140121630310400, 140121697419263, ++STORE, 140121496092672, 140121630310399, ++SNULL, 140121630445567, 140121697419263, ++STORE, 140121630310400, 140121630445567, ++STORE, 140121630445568, 140121697419263, ++SNULL, 140121445740544, 140121454129151, ++STORE, 140121454129152, 140121496092671, ++STORE, 140121445740544, 140121454129151, ++SNULL, 140121454133247, 140121496092671, ++STORE, 140121454129152, 140121454133247, ++STORE, 140121454133248, 140121496092671, ++STORE, 140121026330624, 140121227657215, ++SNULL, 140121093439487, 140121227657215, ++STORE, 140121026330624, 140121093439487, ++STORE, 140121093439488, 140121227657215, ++SNULL, 140121093439488, 140121160548351, ++STORE, 140121160548352, 140121227657215, ++STORE, 140121093439488, 140121160548351, ++ERASE, 140121093439488, 140121160548351, ++SNULL, 140121563201535, 140121630310399, ++STORE, 140121496092672, 140121563201535, ++STORE, 140121563201536, 140121630310399, ++ERASE, 140121563201536, 140121630310399, ++STORE, 140120892112896, 140121093439487, ++SNULL, 140120959221759, 140121093439487, ++STORE, 140120892112896, 140120959221759, ++STORE, 140120959221760, 140121093439487, ++SNULL, 140120959221760, 140121026330623, ++STORE, 140121026330624, 140121093439487, ++STORE, 140120959221760, 140121026330623, ++ERASE, 140120959221760, 140121026330623, ++STORE, 140120757895168, 140120959221759, ++SNULL, 140121361874943, 140121412198399, ++STORE, 140121294766080, 140121361874943, ++STORE, 140121361874944, 140121412198399, ++ERASE, 140121361874944, 140121412198399, ++SNULL, 140121294901247, 140121361874943, ++STORE, 140121294766080, 140121294901247, ++STORE, 140121294901248, 140121361874943, ++STORE, 140120623677440, 140120959221759, ++SNULL, 140120690786303, 140120959221759, ++STORE, 140120623677440, 140120690786303, ++STORE, 140120690786304, 140120959221759, ++SNULL, 140120690786304, 140120757895167, ++STORE, 140120757895168, 140120959221759, ++STORE, 140120690786304, 140120757895167, ++ERASE, 140120690786304, 140120757895167, ++SNULL, 140121160683519, 140121227657215, ++STORE, 140121160548352, 140121160683519, ++STORE, 140121160683520, 140121227657215, ++SNULL, 140121974214656, 140121982607359, ++STORE, 140121982607360, 140121991000063, ++STORE, 140121974214656, 140121982607359, ++SNULL, 140121982611455, 140121991000063, ++STORE, 140121982607360, 140121982611455, ++STORE, 140121982611456, 140121991000063, ++SNULL, 140121839996928, 140121873567743, ++STORE, 140121873567744, 140121890353151, ++STORE, 140121839996928, 140121873567743, ++SNULL, 140121873571839, 140121890353151, ++STORE, 140121873567744, 140121873571839, ++STORE, 140121873571840, 140121890353151, ++SNULL, 140121873571840, 140121881960447, ++STORE, 140121881960448, 140121890353151, ++STORE, 140121873571840, 140121881960447, ++SNULL, 140121881964543, 140121890353151, ++STORE, 140121881960448, 140121881964543, ++STORE, 140121881964544, 140121890353151, ++SNULL, 140121840001023, 140121873567743, ++STORE, 140121839996928, 140121840001023, ++STORE, 140121840001024, 140121873567743, ++SNULL, 140121840001024, 140121865175039, ++STORE, 140121865175040, 140121873567743, ++STORE, 140121840001024, 140121865175039, ++SNULL, 140121865179135, 140121873567743, ++STORE, 140121865175040, 140121865179135, ++STORE, 140121865179136, 140121873567743, ++SNULL, 140121437347839, 140121445736447, ++STORE, 140121437343744, 140121437347839, ++STORE, 140121437347840, 140121445736447, ++STORE, 140121621917696, 140121630310399, ++STORE, 140121613524992, 140121630310399, ++SNULL, 140121026465791, 140121093439487, ++STORE, 140121026330624, 140121026465791, ++STORE, 140121026465792, 140121093439487, ++SNULL, 140121496227839, 140121563201535, ++STORE, 140121496092672, 140121496227839, ++STORE, 140121496227840, 140121563201535, ++SNULL, 140120757895168, 140120892112895, ++STORE, 140120892112896, 140120959221759, ++STORE, 140120757895168, 140120892112895, ++SNULL, 140120892248063, 140120959221759, ++STORE, 140120892112896, 140120892248063, ++STORE, 140120892248064, 140120959221759, ++SNULL, 140120825004031, 140120892112895, ++STORE, 140120757895168, 140120825004031, ++STORE, 140120825004032, 140120892112895, ++ERASE, 140120825004032, 140120892112895, ++SNULL, 140120623812607, 140120690786303, ++STORE, 140120623677440, 140120623812607, ++STORE, 140120623812608, 140120690786303, ++SNULL, 140120758030335, 140120825004031, ++STORE, 140120757895168, 140120758030335, ++STORE, 140120758030336, 140120825004031, ++SNULL, 140121454133248, 140121462521855, ++STORE, 140121462521856, 140121496092671, ++STORE, 140121454133248, 140121462521855, ++SNULL, 140121462525951, 140121496092671, ++STORE, 140121462521856, 140121462525951, ++STORE, 140121462525952, 140121496092671, ++STORE, 140121605132288, 140121630310399, ++SNULL, 140121605136383, 140121630310399, ++STORE, 140121605132288, 140121605136383, ++STORE, 140121605136384, 140121630310399, ++STORE, 140121596739584, 140121605132287, ++SNULL, 140121605136384, 140121621917695, ++STORE, 140121621917696, 140121630310399, ++STORE, 140121605136384, 140121621917695, ++SNULL, 140121621921791, 140121630310399, ++STORE, 140121621917696, 140121621921791, ++STORE, 140121621921792, 140121630310399, ++STORE, 140121588346880, 140121605132287, ++STORE, 140121579954176, 140121605132287, ++SNULL, 140121412202495, 140121420591103, ++STORE, 140121412198400, 140121412202495, ++STORE, 140121412202496, 140121420591103, ++SNULL, 140121974218751, 140121982607359, ++STORE, 140121974214656, 140121974218751, ++STORE, 140121974218752, 140121982607359, ++SNULL, 140121462525952, 140121479307263, ++STORE, 140121479307264, 140121496092671, ++STORE, 140121462525952, 140121479307263, ++SNULL, 140121479311359, 140121496092671, ++STORE, 140121479307264, 140121479311359, ++STORE, 140121479311360, 140121496092671, ++STORE, 140121571561472, 140121605132287, ++SNULL, 140121571565567, 140121605132287, ++STORE, 140121571561472, 140121571565567, ++STORE, 140121571565568, 140121605132287, ++STORE, 140121428951040, 140121437343743, ++SNULL, 140121428955135, 140121437343743, ++STORE, 140121428951040, 140121428955135, ++STORE, 140121428955136, 140121437343743, ++SNULL, 140121840001024, 140121856782335, ++STORE, 140121856782336, 140121865175039, ++STORE, 140121840001024, 140121856782335, ++SNULL, 140121856786431, 140121865175039, ++STORE, 140121856782336, 140121856786431, ++STORE, 140121856786432, 140121865175039, ++STORE, 140121403805696, 140121412198399, ++SNULL, 140121840001024, 140121848389631, ++STORE, 140121848389632, 140121856782335, ++STORE, 140121840001024, 140121848389631, ++SNULL, 140121848393727, 140121856782335, ++STORE, 140121848389632, 140121848393727, ++STORE, 140121848393728, 140121856782335, ++SNULL, 140121479311360, 140121487699967, ++STORE, 140121487699968, 140121496092671, ++STORE, 140121479311360, 140121487699967, ++SNULL, 140121487704063, 140121496092671, ++STORE, 140121487699968, 140121487704063, ++STORE, 140121487704064, 140121496092671, ++STORE, 140121395412992, 140121412198399, ++STORE, 140121387020288, 140121412198399, ++SNULL, 140121387024383, 140121412198399, ++STORE, 140121387020288, 140121387024383, ++STORE, 140121387024384, 140121412198399, ++SNULL, 140121605136384, 140121613524991, ++STORE, 140121613524992, 140121621917695, ++STORE, 140121605136384, 140121613524991, ++SNULL, 140121613529087, 140121621917695, ++STORE, 140121613524992, 140121613529087, ++STORE, 140121613529088, 140121621917695, ++SNULL, 140121462525952, 140121470914559, ++STORE, 140121470914560, 140121479307263, ++STORE, 140121462525952, 140121470914559, ++SNULL, 140121470918655, 140121479307263, ++STORE, 140121470914560, 140121470918655, ++STORE, 140121470918656, 140121479307263, ++STORE, 140121378627584, 140121387020287, ++SNULL, 140121378631679, 140121387020287, ++STORE, 140121378627584, 140121378631679, ++STORE, 140121378631680, 140121387020287, ++SNULL, 140121571565568, 140121596739583, ++STORE, 140121596739584, 140121605132287, ++STORE, 140121571565568, 140121596739583, ++SNULL, 140121596743679, 140121605132287, ++STORE, 140121596739584, 140121596743679, ++STORE, 140121596743680, 140121605132287, ++SNULL, 140121387024384, 140121403805695, ++STORE, 140121403805696, 140121412198399, ++STORE, 140121387024384, 140121403805695, ++SNULL, 140121403809791, 140121412198399, ++STORE, 140121403805696, 140121403809791, ++STORE, 140121403809792, 140121412198399, ++STORE, 140121370234880, 140121378627583, ++SNULL, 140121387024384, 140121395412991, ++STORE, 140121395412992, 140121403805695, ++STORE, 140121387024384, 140121395412991, ++SNULL, 140121395417087, 140121403805695, ++STORE, 140121395412992, 140121395417087, ++STORE, 140121395417088, 140121403805695, ++SNULL, 140121571565568, 140121588346879, ++STORE, 140121588346880, 140121596739583, ++STORE, 140121571565568, 140121588346879, ++SNULL, 140121588350975, 140121596739583, ++STORE, 140121588346880, 140121588350975, ++STORE, 140121588350976, 140121596739583, ++SNULL, 140121571565568, 140121579954175, ++STORE, 140121579954176, 140121588346879, ++STORE, 140121571565568, 140121579954175, ++SNULL, 140121579958271, 140121588346879, ++STORE, 140121579954176, 140121579958271, ++STORE, 140121579958272, 140121588346879, ++STORE, 140121286373376, 140121294766079, ++STORE, 140121277980672, 140121294766079, ++SNULL, 140121277980672, 140121286373375, ++STORE, 140121286373376, 140121294766079, ++STORE, 140121277980672, 140121286373375, ++SNULL, 140121286377471, 140121294766079, ++STORE, 140121286373376, 140121286377471, ++STORE, 140121286377472, 140121294766079, ++STORE, 140121269587968, 140121286373375, ++STORE, 140121261195264, 140121286373375, ++SNULL, 140121261195264, 140121269587967, ++STORE, 140121269587968, 140121286373375, ++STORE, 140121261195264, 140121269587967, ++SNULL, 140121269592063, 140121286373375, ++STORE, 140121269587968, 140121269592063, ++STORE, 140121269592064, 140121286373375, ++STORE, 140121252802560, 140121269587967, ++SNULL, 140121252806655, 140121269587967, ++STORE, 140121252802560, 140121252806655, ++STORE, 140121252806656, 140121269587967, ++STORE, 140121244409856, 140121252802559, ++STORE, 140121236017152, 140121252802559, ++SNULL, 140121236017152, 140121244409855, ++STORE, 140121244409856, 140121252802559, ++STORE, 140121236017152, 140121244409855, ++SNULL, 140121244413951, 140121252802559, ++STORE, 140121244409856, 140121244413951, ++STORE, 140121244413952, 140121252802559, ++SNULL, 140121370238975, 140121378627583, ++STORE, 140121370234880, 140121370238975, ++STORE, 140121370238976, 140121378627583, ++STORE, 140121152155648, 140121160548351, ++STORE, 140121143762944, 140121160548351, ++STORE, 140121135370240, 140121160548351, ++SNULL, 140121135374335, 140121160548351, ++STORE, 140121135370240, 140121135374335, ++STORE, 140121135374336, 140121160548351, ++STORE, 140121126977536, 140121135370239, ++STORE, 140121118584832, 140121135370239, ++STORE, 140121110192128, 140121135370239, ++SNULL, 140121110192128, 140121118584831, ++STORE, 140121118584832, 140121135370239, ++STORE, 140121110192128, 140121118584831, ++SNULL, 140121118588927, 140121135370239, ++STORE, 140121118584832, 140121118588927, ++STORE, 140121118588928, 140121135370239, ++STORE, 140121101799424, 140121118584831, ++STORE, 140121017937920, 140121026330623, ++STORE, 140121009545216, 140121026330623, ++SNULL, 140121009545216, 140121017937919, ++STORE, 140121017937920, 140121026330623, ++STORE, 140121009545216, 140121017937919, ++SNULL, 140121017942015, 140121026330623, ++STORE, 140121017937920, 140121017942015, ++STORE, 140121017942016, 140121026330623, ++SNULL, 140121269592064, 140121277980671, ++STORE, 140121277980672, 140121286373375, ++STORE, 140121269592064, 140121277980671, ++SNULL, 140121277984767, 140121286373375, ++STORE, 140121277980672, 140121277984767, ++STORE, 140121277984768, 140121286373375, ++STORE, 140121001152512, 140121017937919, ++SNULL, 140121252806656, 140121261195263, ++STORE, 140121261195264, 140121269587967, ++STORE, 140121252806656, 140121261195263, ++SNULL, 140121261199359, 140121269587967, ++STORE, 140121261195264, 140121261199359, ++STORE, 140121261199360, 140121269587967, ++SNULL, 140121135374336, 140121152155647, ++STORE, 140121152155648, 140121160548351, ++STORE, 140121135374336, 140121152155647, ++SNULL, 140121152159743, 140121160548351, ++STORE, 140121152155648, 140121152159743, ++STORE, 140121152159744, 140121160548351, ++STORE, 140120992759808, 140121017937919, ++STORE, 140120984367104, 140121017937919, ++STORE, 140120975974400, 140121017937919, ++SNULL, 140121101799424, 140121110192127, ++STORE, 140121110192128, 140121118584831, ++STORE, 140121101799424, 140121110192127, ++SNULL, 140121110196223, 140121118584831, ++STORE, 140121110192128, 140121110196223, ++STORE, 140121110196224, 140121118584831, ++SNULL, 140121118588928, 140121126977535, ++STORE, 140121126977536, 140121135370239, ++STORE, 140121118588928, 140121126977535, ++SNULL, 140121126981631, 140121135370239, ++STORE, 140121126977536, 140121126981631, ++STORE, 140121126981632, 140121135370239, ++STORE, 140120967581696, 140121017937919, ++STORE, 140120883720192, 140120892112895, ++SNULL, 140120883724287, 140120892112895, ++STORE, 140120883720192, 140120883724287, ++STORE, 140120883724288, 140120892112895, ++STORE, 140120875327488, 140120883720191, ++SNULL, 140121101803519, 140121110192127, ++STORE, 140121101799424, 140121101803519, ++STORE, 140121101803520, 140121110192127, ++SNULL, 140121135374336, 140121143762943, ++STORE, 140121143762944, 140121152155647, ++STORE, 140121135374336, 140121143762943, ++SNULL, 140121143767039, 140121152155647, ++STORE, 140121143762944, 140121143767039, ++STORE, 140121143767040, 140121152155647, ++STORE, 140120866934784, 140120883720191, ++SNULL, 140120967581696, 140120984367103, ++STORE, 140120984367104, 140121017937919, ++STORE, 140120967581696, 140120984367103, ++SNULL, 140120984371199, 140121017937919, ++STORE, 140120984367104, 140120984371199, ++STORE, 140120984371200, 140121017937919, ++STORE, 140120858542080, 140120883720191, ++SNULL, 140121236021247, 140121244409855, ++STORE, 140121236017152, 140121236021247, ++STORE, 140121236021248, 140121244409855, ++SNULL, 140120984371200, 140121009545215, ++STORE, 140121009545216, 140121017937919, ++STORE, 140120984371200, 140121009545215, ++SNULL, 140121009549311, 140121017937919, ++STORE, 140121009545216, 140121009549311, ++STORE, 140121009549312, 140121017937919, ++SNULL, 140120984371200, 140120992759807, ++STORE, 140120992759808, 140121009545215, ++STORE, 140120984371200, 140120992759807, ++SNULL, 140120992763903, 140121009545215, ++STORE, 140120992759808, 140120992763903, ++STORE, 140120992763904, 140121009545215, ++SNULL, 140120992763904, 140121001152511, ++STORE, 140121001152512, 140121009545215, ++STORE, 140120992763904, 140121001152511, ++SNULL, 140121001156607, 140121009545215, ++STORE, 140121001152512, 140121001156607, ++STORE, 140121001156608, 140121009545215, ++STORE, 140120850149376, 140120883720191, ++SNULL, 140120850153471, 140120883720191, ++STORE, 140120850149376, 140120850153471, ++STORE, 140120850153472, 140120883720191, ++SNULL, 140120967585791, 140120984367103, ++STORE, 140120967581696, 140120967585791, ++STORE, 140120967585792, 140120984367103, ++SNULL, 140120850153472, 140120866934783, ++STORE, 140120866934784, 140120883720191, ++STORE, 140120850153472, 140120866934783, ++SNULL, 140120866938879, 140120883720191, ++STORE, 140120866934784, 140120866938879, ++STORE, 140120866938880, 140120883720191, ++STORE, 140120841756672, 140120850149375, ++SNULL, 140120967585792, 140120975974399, ++STORE, 140120975974400, 140120984367103, ++STORE, 140120967585792, 140120975974399, ++SNULL, 140120975978495, 140120984367103, ++STORE, 140120975974400, 140120975978495, ++STORE, 140120975978496, 140120984367103, ++SNULL, 140120866938880, 140120875327487, ++STORE, 140120875327488, 140120883720191, ++STORE, 140120866938880, 140120875327487, ++SNULL, 140120875331583, 140120883720191, ++STORE, 140120875327488, 140120875331583, ++STORE, 140120875331584, 140120883720191, ++STORE, 140120833363968, 140120850149375, ++STORE, 140120749502464, 140120757895167, ++STORE, 140120741109760, 140120757895167, ++STORE, 140120732717056, 140120757895167, ++STORE, 140120724324352, 140120757895167, ++SNULL, 140120724324352, 140120732717055, ++STORE, 140120732717056, 140120757895167, ++STORE, 140120724324352, 140120732717055, ++SNULL, 140120732721151, 140120757895167, ++STORE, 140120732717056, 140120732721151, ++STORE, 140120732721152, 140120757895167, ++STORE, 140120715931648, 140120732717055, ++SNULL, 140120715935743, 140120732717055, ++STORE, 140120715931648, 140120715935743, ++STORE, 140120715935744, 140120732717055, ++SNULL, 140120850153472, 140120858542079, ++STORE, 140120858542080, 140120866934783, ++STORE, 140120850153472, 140120858542079, ++SNULL, 140120858546175, 140120866934783, ++STORE, 140120858542080, 140120858546175, ++STORE, 140120858546176, 140120866934783, ++STORE, 140120707538944, 140120715931647, ++SNULL, 140120707543039, 140120715931647, ++STORE, 140120707538944, 140120707543039, ++STORE, 140120707543040, 140120715931647, ++SNULL, 140120833368063, 140120850149375, ++STORE, 140120833363968, 140120833368063, ++STORE, 140120833368064, 140120850149375, ++SNULL, 140120833368064, 140120841756671, ++STORE, 140120841756672, 140120850149375, ++STORE, 140120833368064, 140120841756671, ++SNULL, 140120841760767, 140120850149375, ++STORE, 140120841756672, 140120841760767, ++STORE, 140120841760768, 140120850149375, ++STORE, 140120699146240, 140120707538943, ++SNULL, 140120715935744, 140120724324351, ++STORE, 140120724324352, 140120732717055, ++STORE, 140120715935744, 140120724324351, ++SNULL, 140120724328447, 140120732717055, ++STORE, 140120724324352, 140120724328447, ++STORE, 140120724328448, 140120732717055, ++SNULL, 140120732721152, 140120741109759, ++STORE, 140120741109760, 140120757895167, ++STORE, 140120732721152, 140120741109759, ++SNULL, 140120741113855, 140120757895167, ++STORE, 140120741109760, 140120741113855, ++STORE, 140120741113856, 140120757895167, ++SNULL, 140120741113856, 140120749502463, ++STORE, 140120749502464, 140120757895167, ++STORE, 140120741113856, 140120749502463, ++SNULL, 140120749506559, 140120757895167, ++STORE, 140120749502464, 140120749506559, ++STORE, 140120749506560, 140120757895167, ++SNULL, 140120699150335, 140120707538943, ++STORE, 140120699146240, 140120699150335, ++STORE, 140120699150336, 140120707538943, ++STORE, 140122446557184, 140122446585855, ++STORE, 140122368999424, 140122371190783, ++SNULL, 140122368999424, 140122369089535, ++STORE, 140122369089536, 140122371190783, ++STORE, 140122368999424, 140122369089535, ++SNULL, 140122371182591, 140122371190783, ++STORE, 140122369089536, 140122371182591, ++STORE, 140122371182592, 140122371190783, ++ERASE, 140122371182592, 140122371190783, ++STORE, 140122371182592, 140122371190783, ++SNULL, 140122371186687, 140122371190783, ++STORE, 140122371182592, 140122371186687, ++STORE, 140122371186688, 140122371190783, ++ERASE, 140122446557184, 140122446585855, ++ERASE, 140121445736448, 140121445740543, ++ERASE, 140121445740544, 140121454129151, ++ERASE, 140121621917696, 140121621921791, ++ERASE, 140121621921792, 140121630310399, ++ERASE, 140121579954176, 140121579958271, ++ERASE, 140121579958272, 140121588346879, ++ERASE, 140121261195264, 140121261199359, ++ERASE, 140121261199360, 140121269587967, ++ERASE, 140121454129152, 140121454133247, ++ERASE, 140121454133248, 140121462521855, ++ERASE, 140121588346880, 140121588350975, ++ERASE, 140121588350976, 140121596739583, ++ERASE, 140121135370240, 140121135374335, ++ERASE, 140121135374336, 140121143762943, ++ERASE, 140121881960448, 140121881964543, ++ERASE, 140121881964544, 140121890353151, ++ERASE, 140121428951040, 140121428955135, ++ERASE, 140121428955136, 140121437343743, ++ERASE, 140121387020288, 140121387024383, ++ERASE, 140121387024384, 140121395412991, ++ERASE, 140121487699968, 140121487704063, ++ERASE, 140121487704064, 140121496092671, ++ERASE, 140121437343744, 140121437347839, ++ERASE, 140121437347840, 140121445736447, ++ERASE, 140121613524992, 140121613529087, ++ERASE, 140121613529088, 140121621917695, ++ERASE, 140121856782336, 140121856786431, ++ERASE, 140121856786432, 140121865175039, ++ERASE, 140121252802560, 140121252806655, ++ERASE, 140121252806656, 140121261195263, ++ERASE, 140121839996928, 140121840001023, ++ERASE, 140121840001024, 140121848389631, ++ERASE, 140121596739584, 140121596743679, ++ERASE, 140121596743680, 140121605132287, ++ERASE, 140121009545216, 140121009549311, ++ERASE, 140121009549312, 140121017937919, ++ERASE, 140120724324352, 140120724328447, ++ERASE, 140120724328448, 140120732717055, ++ERASE, 140120883720192, 140120883724287, ++ERASE, 140120883724288, 140120892112895, ++ERASE, 140121982607360, 140121982611455, ++ERASE, 140121982611456, 140121991000063, ++ERASE, 140121571561472, 140121571565567, ++ERASE, 140121571565568, 140121579954175, ++ERASE, 140121286373376, 140121286377471, ++ERASE, 140121286377472, 140121294766079, ++ERASE, 140120875327488, 140120875331583, ++ERASE, 140120875331584, 140120883720191, ++ERASE, 140121848389632, 140121848393727, ++ERASE, 140121848393728, 140121856782335, ++ERASE, 140121370234880, 140121370238975, ++ERASE, 140121370238976, 140121378627583, ++ERASE, 140121143762944, 140121143767039, ++ERASE, 140121143767040, 140121152155647, ++ERASE, 140121118584832, 140121118588927, ++ERASE, 140121118588928, 140121126977535, ++ERASE, 140120866934784, 140120866938879, ++ERASE, 140120866938880, 140120875327487, ++ERASE, 140120741109760, 140120741113855, ++ERASE, 140120741113856, 140120749502463, ++ERASE, 140121865175040, 140121865179135, ++ERASE, 140121865179136, 140121873567743, ++ERASE, 140121403805696, 140121403809791, ++ERASE, 140121403809792, 140121412198399, ++ERASE, 140121236017152, 140121236021247, ++ERASE, 140121236021248, 140121244409855, ++ERASE, 140120732717056, 140120732721151, ++ERASE, 140120732721152, 140120741109759, ++ERASE, 140121017937920, 140121017942015, ++ERASE, 140121017942016, 140121026330623, ++ERASE, 140121873567744, 140121873571839, ++ERASE, 140121873571840, 140121881960447, ++ERASE, 140121470914560, 140121470918655, ++ERASE, 140121470918656, 140121479307263, ++ERASE, 140121126977536, 140121126981631, ++ERASE, 140121126981632, 140121135370239, ++ERASE, 140120850149376, 140120850153471, ++ERASE, 140120850153472, 140120858542079, ++ERASE, 140120707538944, 140120707543039, ++ERASE, 140120707543040, 140120715931647, ++ERASE, 140121479307264, 140121479311359, ++ERASE, 140121479311360, 140121487699967, ++ERASE, 140120967581696, 140120967585791, ++ERASE, 140120967585792, 140120975974399, ++ERASE, 140120841756672, 140120841760767, ++ERASE, 140120841760768, 140120850149375, ++ERASE, 140121412198400, 140121412202495, ++ERASE, 140121412202496, 140121420591103, ++ERASE, 140122158788608, 140122158792703, ++ERASE, 140122158792704, 140122167181311, ++ERASE, 140122142003200, 140122142007295, ++ERASE, 140122142007296, 140122150395903, ++ERASE, 140121101799424, 140121101803519, ++ERASE, 140121101803520, 140121110192127, ++ERASE, 140120858542080, 140120858546175, ++ERASE, 140120858546176, 140120866934783, ++ERASE, 140120833363968, 140120833368063, ++ERASE, 140120833368064, 140120841756671, ++ERASE, 140121277980672, 140121277984767, ++ERASE, 140121277984768, 140121286373375, ++ERASE, 140121001152512, 140121001156607, ++ERASE, 140121001156608, 140121009545215, ++ERASE, 140120749502464, 140120749506559, ++ERASE, 140120749506560, 140120757895167, ++ERASE, 140121605132288, 140121605136383, ++ERASE, 140121605136384, 140121613524991, ++ERASE, 140121378627584, 140121378631679, ++ERASE, 140121378631680, 140121387020287, ++ERASE, 140121110192128, 140121110196223, ++ERASE, 140121110196224, 140121118584831, ++ERASE, 140121462521856, 140121462525951, ++ERASE, 140121462525952, 140121470914559, ++ERASE, 140121395412992, 140121395417087, ++ERASE, 140121395417088, 140121403805695, ++ERASE, 140121152155648, 140121152159743, ++ERASE, 140121152159744, 140121160548351, ++ERASE, 140120992759808, 140120992763903, ++ERASE, 140120992763904, 140121001152511, ++ERASE, 140122387976192, 140122387980287, ++ERASE, 140122387980288, 140122396368895, ++ERASE, 140121890353152, 140121890357247, ++ERASE, 140121890357248, 140121898745855, ++ERASE, 140121269587968, 140121269592063, ++ERASE, 140121269592064, 140121277980671, ++ }; ++ unsigned long set37[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140722404016128, 140737488351231, ++SNULL, 140722404020223, 140737488351231, ++STORE, 140722404016128, 140722404020223, ++STORE, 140722403885056, 140722404020223, ++STORE, 94637010001920, 94637012254719, ++SNULL, 94637010132991, 94637012254719, ++STORE, 94637010001920, 94637010132991, ++STORE, 94637010132992, 94637012254719, ++ERASE, 94637010132992, 94637012254719, ++STORE, 94637012226048, 94637012234239, ++STORE, 94637012234240, 94637012254719, ++STORE, 139760240594944, 139760242847743, ++SNULL, 139760240738303, 139760242847743, ++STORE, 139760240594944, 139760240738303, ++STORE, 139760240738304, 139760242847743, ++ERASE, 139760240738304, 139760242847743, ++STORE, 139760242835456, 139760242843647, ++STORE, 139760242843648, 139760242847743, ++STORE, 140722405232640, 140722405236735, ++STORE, 140722405220352, 140722405232639, ++STORE, 139760242806784, 139760242835455, ++STORE, 139760242798592, 139760242806783, ++STORE, 139760238379008, 139760240594943, ++SNULL, 139760238379008, 139760238477311, ++STORE, 139760238477312, 139760240594943, ++STORE, 139760238379008, 139760238477311, ++SNULL, 139760240570367, 139760240594943, ++STORE, 139760238477312, 139760240570367, ++STORE, 139760240570368, 139760240594943, ++SNULL, 139760240570368, 139760240578559, ++STORE, 139760240578560, 139760240594943, ++STORE, 139760240570368, 139760240578559, ++ERASE, 139760240570368, 139760240578559, ++STORE, 139760240570368, 139760240578559, ++ERASE, 139760240578560, 139760240594943, ++STORE, 139760240578560, 139760240594943, ++STORE, 139760234582016, 139760238379007, ++SNULL, 139760234582016, 139760236240895, ++STORE, 139760236240896, 139760238379007, ++STORE, 139760234582016, 139760236240895, ++SNULL, 139760238338047, 139760238379007, ++STORE, 139760236240896, 139760238338047, ++STORE, 139760238338048, 139760238379007, ++SNULL, 139760238338048, 139760238362623, ++STORE, 139760238362624, 139760238379007, ++STORE, 139760238338048, 139760238362623, ++ERASE, 139760238338048, 139760238362623, ++STORE, 139760238338048, 139760238362623, ++ERASE, 139760238362624, 139760238379007, ++STORE, 139760238362624, 139760238379007, ++STORE, 139760242790400, 139760242806783, ++SNULL, 139760238354431, 139760238362623, ++STORE, 139760238338048, 139760238354431, ++STORE, 139760238354432, 139760238362623, ++SNULL, 139760240574463, 139760240578559, ++STORE, 139760240570368, 139760240574463, ++STORE, 139760240574464, 139760240578559, ++SNULL, 94637012230143, 94637012234239, ++STORE, 94637012226048, 94637012230143, ++STORE, 94637012230144, 94637012234239, ++SNULL, 139760242839551, 139760242843647, ++STORE, 139760242835456, 139760242839551, ++STORE, 139760242839552, 139760242843647, ++ERASE, 139760242806784, 139760242835455, ++STORE, 94637033324544, 94637033459711, ++STORE, 139760226189312, 139760234582015, ++SNULL, 139760226193407, 139760234582015, ++STORE, 139760226189312, 139760226193407, ++STORE, 139760226193408, 139760234582015, ++STORE, 139760217796608, 139760226189311, ++STORE, 139760083578880, 139760217796607, ++SNULL, 139760083578880, 139760114860031, ++STORE, 139760114860032, 139760217796607, ++STORE, 139760083578880, 139760114860031, ++ERASE, 139760083578880, 139760114860031, ++SNULL, 139760181968895, 139760217796607, ++STORE, 139760114860032, 139760181968895, ++STORE, 139760181968896, 139760217796607, ++ERASE, 139760181968896, 139760217796607, ++SNULL, 139760114995199, 139760181968895, ++STORE, 139760114860032, 139760114995199, ++STORE, 139760114995200, 139760181968895, ++SNULL, 139760217800703, 139760226189311, ++STORE, 139760217796608, 139760217800703, ++STORE, 139760217800704, 139760226189311, ++STORE, 139760209403904, 139760217796607, ++SNULL, 139760209407999, 139760217796607, ++STORE, 139760209403904, 139760209407999, ++STORE, 139760209408000, 139760217796607, ++STORE, 139760201011200, 139760209403903, ++SNULL, 139760201015295, 139760209403903, ++STORE, 139760201011200, 139760201015295, ++STORE, 139760201015296, 139760209403903, ++STORE, 139760192618496, 139760201011199, ++SNULL, 139760192622591, 139760201011199, ++STORE, 139760192618496, 139760192622591, ++STORE, 139760192622592, 139760201011199, ++STORE, 139760184225792, 139760192618495, ++STORE, 139759980642304, 139760114860031, ++STORE, 139759972249600, 139759980642303, ++STORE, 139759963856896, 139759980642303, ++STORE, 139759955464192, 139759980642303, ++STORE, 139759888355328, 139759955464191, ++SNULL, 139760047751167, 139760114860031, ++STORE, 139759980642304, 139760047751167, ++STORE, 139760047751168, 139760114860031, ++ERASE, 139760047751168, 139760114860031, ++SNULL, 139759980777471, 139760047751167, ++STORE, 139759980642304, 139759980777471, ++STORE, 139759980777472, 139760047751167, ++STORE, 139759980777472, 139760114860031, ++SNULL, 139759980777472, 139760047751167, ++STORE, 139760047751168, 139760114860031, ++STORE, 139759980777472, 139760047751167, ++SNULL, 139760047886335, 139760114860031, ++STORE, 139760047751168, 139760047886335, ++STORE, 139760047886336, 139760114860031, ++STORE, 139759821246464, 139759955464191, ++SNULL, 139759821246464, 139759888355327, ++STORE, 139759888355328, 139759955464191, ++STORE, 139759821246464, 139759888355327, ++ERASE, 139759821246464, 139759888355327, ++ERASE, 139759888355328, 139759955464191, ++ }; ++ unsigned long set38[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140730666221568, 140737488351231, ++SNULL, 140730666225663, 140737488351231, ++STORE, 140730666221568, 140730666225663, ++STORE, 140730666090496, 140730666225663, ++STORE, 94177584803840, 94177587056639, ++SNULL, 94177584934911, 94177587056639, ++STORE, 94177584803840, 94177584934911, ++STORE, 94177584934912, 94177587056639, ++ERASE, 94177584934912, 94177587056639, ++STORE, 94177587027968, 94177587036159, ++STORE, 94177587036160, 94177587056639, ++STORE, 140614382714880, 140614384967679, ++SNULL, 140614382858239, 140614384967679, ++STORE, 140614382714880, 140614382858239, ++STORE, 140614382858240, 140614384967679, ++ERASE, 140614382858240, 140614384967679, ++STORE, 140614384955392, 140614384963583, ++STORE, 140614384963584, 140614384967679, ++STORE, 140730666315776, 140730666319871, ++STORE, 140730666303488, 140730666315775, ++STORE, 140614384926720, 140614384955391, ++STORE, 140614384918528, 140614384926719, ++STORE, 140614380498944, 140614382714879, ++SNULL, 140614380498944, 140614380597247, ++STORE, 140614380597248, 140614382714879, ++STORE, 140614380498944, 140614380597247, ++SNULL, 140614382690303, 140614382714879, ++STORE, 140614380597248, 140614382690303, ++STORE, 140614382690304, 140614382714879, ++SNULL, 140614382690304, 140614382698495, ++STORE, 140614382698496, 140614382714879, ++STORE, 140614382690304, 140614382698495, ++ERASE, 140614382690304, 140614382698495, ++STORE, 140614382690304, 140614382698495, ++ERASE, 140614382698496, 140614382714879, ++STORE, 140614382698496, 140614382714879, ++STORE, 140614376701952, 140614380498943, ++SNULL, 140614376701952, 140614378360831, ++STORE, 140614378360832, 140614380498943, ++STORE, 140614376701952, 140614378360831, ++SNULL, 140614380457983, 140614380498943, ++STORE, 140614378360832, 140614380457983, ++STORE, 140614380457984, 140614380498943, ++SNULL, 140614380457984, 140614380482559, ++STORE, 140614380482560, 140614380498943, ++STORE, 140614380457984, 140614380482559, ++ERASE, 140614380457984, 140614380482559, ++STORE, 140614380457984, 140614380482559, ++ERASE, 140614380482560, 140614380498943, ++STORE, 140614380482560, 140614380498943, ++STORE, 140614384910336, 140614384926719, ++SNULL, 140614380474367, 140614380482559, ++STORE, 140614380457984, 140614380474367, ++STORE, 140614380474368, 140614380482559, ++SNULL, 140614382694399, 140614382698495, ++STORE, 140614382690304, 140614382694399, ++STORE, 140614382694400, 140614382698495, ++SNULL, 94177587032063, 94177587036159, ++STORE, 94177587027968, 94177587032063, ++STORE, 94177587032064, 94177587036159, ++SNULL, 140614384959487, 140614384963583, ++STORE, 140614384955392, 140614384959487, ++STORE, 140614384959488, 140614384963583, ++ERASE, 140614384926720, 140614384955391, ++STORE, 94177619791872, 94177619927039, ++STORE, 140614368309248, 140614376701951, ++SNULL, 140614368313343, 140614376701951, ++STORE, 140614368309248, 140614368313343, ++STORE, 140614368313344, 140614376701951, ++STORE, 140614359916544, 140614368309247, ++STORE, 140614225698816, 140614359916543, ++SNULL, 140614225698816, 140614276481023, ++STORE, 140614276481024, 140614359916543, ++STORE, 140614225698816, 140614276481023, ++ERASE, 140614225698816, 140614276481023, ++SNULL, 140614343589887, 140614359916543, ++STORE, 140614276481024, 140614343589887, ++STORE, 140614343589888, 140614359916543, ++ERASE, 140614343589888, 140614359916543, ++SNULL, 140614276616191, 140614343589887, ++STORE, 140614276481024, 140614276616191, ++STORE, 140614276616192, 140614343589887, ++SNULL, 140614359920639, 140614368309247, ++STORE, 140614359916544, 140614359920639, ++STORE, 140614359920640, 140614368309247, ++STORE, 140614351523840, 140614359916543, ++SNULL, 140614351527935, 140614359916543, ++STORE, 140614351523840, 140614351527935, ++STORE, 140614351527936, 140614359916543, ++STORE, 140614268088320, 140614276481023, ++SNULL, 140614268092415, 140614276481023, ++STORE, 140614268088320, 140614268092415, ++STORE, 140614268092416, 140614276481023, ++STORE, 140614259695616, 140614268088319, ++SNULL, 140614259699711, 140614268088319, ++STORE, 140614259695616, 140614259699711, ++STORE, 140614259699712, 140614268088319, ++STORE, 140614251302912, 140614259695615, ++STORE, 140614242910208, 140614259695615, ++STORE, 140614108692480, 140614242910207, ++SNULL, 140614108692480, 140614142263295, ++STORE, 140614142263296, 140614242910207, ++STORE, 140614108692480, 140614142263295, ++ERASE, 140614108692480, 140614142263295, ++STORE, 140614133870592, 140614142263295, ++STORE, 140613999652864, 140614133870591, ++SNULL, 140613999652864, 140614008045567, ++STORE, 140614008045568, 140614133870591, ++STORE, 140613999652864, 140614008045567, ++ERASE, 140613999652864, 140614008045567, ++STORE, 140613999652864, 140614008045567, ++STORE, 140613865435136, 140613999652863, ++SNULL, 140613865435136, 140613873827839, ++STORE, 140613873827840, 140613999652863, ++STORE, 140613865435136, 140613873827839, ++ERASE, 140613865435136, 140613873827839, ++SNULL, 140614209372159, 140614242910207, ++STORE, 140614142263296, 140614209372159, ++STORE, 140614209372160, 140614242910207, ++ERASE, 140614209372160, 140614242910207, ++SNULL, 140614142398463, 140614209372159, ++STORE, 140614142263296, 140614142398463, ++STORE, 140614142398464, 140614209372159, ++SNULL, 140614075154431, 140614133870591, ++STORE, 140614008045568, 140614075154431, ++STORE, 140614075154432, 140614133870591, ++ERASE, 140614075154432, 140614133870591, ++SNULL, 140614008180735, 140614075154431, ++STORE, 140614008045568, 140614008180735, ++STORE, 140614008180736, 140614075154431, ++SNULL, 140613940936703, 140613999652863, ++STORE, 140613873827840, 140613940936703, ++STORE, 140613940936704, 140613999652863, ++ERASE, 140613940936704, 140613999652863, ++SNULL, 140614242914303, 140614259695615, ++STORE, 140614242910208, 140614242914303, ++STORE, 140614242914304, 140614259695615, ++STORE, 140613739610112, 140613940936703, ++STORE, 140614234517504, 140614242910207, ++SNULL, 140614242914304, 140614251302911, ++STORE, 140614251302912, 140614259695615, ++STORE, 140614242914304, 140614251302911, ++SNULL, 140614251307007, 140614259695615, ++STORE, 140614251302912, 140614251307007, ++STORE, 140614251307008, 140614259695615, ++SNULL, 140613739610112, 140613873827839, ++STORE, 140613873827840, 140613940936703, ++STORE, 140613739610112, 140613873827839, ++SNULL, 140613873963007, 140613940936703, ++STORE, 140613873827840, 140613873963007, ++STORE, 140613873963008, 140613940936703, ++SNULL, 140614133874687, 140614142263295, ++STORE, 140614133870592, 140614133874687, ++STORE, 140614133874688, 140614142263295, ++SNULL, 140613806718975, 140613873827839, ++STORE, 140613739610112, 140613806718975, ++STORE, 140613806718976, 140613873827839, ++ERASE, 140613806718976, 140613873827839, ++STORE, 140614226124800, 140614242910207, ++SNULL, 140613739745279, 140613806718975, ++STORE, 140613739610112, 140613739745279, ++STORE, 140613739745280, 140613806718975, ++SNULL, 140613999656959, 140614008045567, ++STORE, 140613999652864, 140613999656959, ++STORE, 140613999656960, 140614008045567, ++SNULL, 140614226124800, 140614234517503, ++STORE, 140614234517504, 140614242910207, ++STORE, 140614226124800, 140614234517503, ++SNULL, 140614234521599, 140614242910207, ++STORE, 140614234517504, 140614234521599, ++STORE, 140614234521600, 140614242910207, ++STORE, 140614217732096, 140614234517503, ++STORE, 140614125477888, 140614133870591, ++SNULL, 140614125481983, 140614133870591, ++STORE, 140614125477888, 140614125481983, ++STORE, 140614125481984, 140614133870591, ++STORE, 140614117085184, 140614125477887, ++SNULL, 140614217736191, 140614234517503, ++STORE, 140614217732096, 140614217736191, ++STORE, 140614217736192, 140614234517503, ++SNULL, 140614117089279, 140614125477887, ++STORE, 140614117085184, 140614117089279, ++STORE, 140614117089280, 140614125477887, ++SNULL, 140614217736192, 140614226124799, ++STORE, 140614226124800, 140614234517503, ++STORE, 140614217736192, 140614226124799, ++SNULL, 140614226128895, 140614234517503, ++STORE, 140614226124800, 140614226128895, ++STORE, 140614226128896, 140614234517503, ++STORE, 140614108692480, 140614117085183, ++STORE, 140614100299776, 140614117085183, ++STORE, 140614091907072, 140614117085183, ++SNULL, 140614091907072, 140614108692479, ++STORE, 140614108692480, 140614117085183, ++STORE, 140614091907072, 140614108692479, ++SNULL, 140614108696575, 140614117085183, ++STORE, 140614108692480, 140614108696575, ++STORE, 140614108696576, 140614117085183, ++SNULL, 140614091907072, 140614100299775, ++STORE, 140614100299776, 140614108692479, ++STORE, 140614091907072, 140614100299775, ++SNULL, 140614100303871, 140614108692479, ++STORE, 140614100299776, 140614100303871, ++STORE, 140614100303872, 140614108692479, ++STORE, 140614083514368, 140614100299775, ++SNULL, 140614083518463, 140614100299775, ++STORE, 140614083514368, 140614083518463, ++STORE, 140614083518464, 140614100299775, ++STORE, 140613991260160, 140613999652863, ++SNULL, 140614083518464, 140614091907071, ++STORE, 140614091907072, 140614100299775, ++STORE, 140614083518464, 140614091907071, ++SNULL, 140614091911167, 140614100299775, ++STORE, 140614091907072, 140614091911167, ++STORE, 140614091911168, 140614100299775, ++SNULL, 140613991264255, 140613999652863, ++STORE, 140613991260160, 140613991264255, ++STORE, 140613991264256, 140613999652863, ++STORE, 140613982867456, 140613991260159, ++SNULL, 140613982871551, 140613991260159, ++STORE, 140613982867456, 140613982871551, ++STORE, 140613982871552, 140613991260159, ++STORE, 140613974474752, 140613982867455, ++SNULL, 140613974478847, 140613982867455, ++STORE, 140613974474752, 140613974478847, ++STORE, 140613974478848, 140613982867455, ++STORE, 140613966082048, 140613974474751, ++STORE, 140613739745280, 140613873827839, ++SNULL, 140613739745280, 140613806718975, ++STORE, 140613806718976, 140613873827839, ++STORE, 140613739745280, 140613806718975, ++SNULL, 140613806854143, 140613873827839, ++STORE, 140613806718976, 140613806854143, ++STORE, 140613806854144, 140613873827839, ++SNULL, 140613966086143, 140613974474751, ++STORE, 140613966082048, 140613966086143, ++STORE, 140613966086144, 140613974474751, ++STORE, 140613957689344, 140613966082047, ++STORE, 140613605392384, 140613739610111, ++STORE, 140613949296640, 140613966082047, ++STORE, 140613596999680, 140613605392383, ++STORE, 140613529890816, 140613596999679, ++STORE, 140613521498112, 140613529890815, ++STORE, 140613513105408, 140613529890815, ++STORE, 140613378887680, 140613513105407, ++SNULL, 140613378887680, 140613404065791, ++STORE, 140613404065792, 140613513105407, ++STORE, 140613378887680, 140613404065791, ++ERASE, 140613378887680, 140613404065791, ++STORE, 140613395673088, 140613404065791, ++STORE, 140613261455360, 140613395673087, ++SNULL, 140613261455360, 140613269848063, ++STORE, 140613269848064, 140613395673087, ++STORE, 140613261455360, 140613269848063, ++ERASE, 140613261455360, 140613269848063, ++STORE, 140613261455360, 140613269848063, ++STORE, 140613253062656, 140613269848063, ++STORE, 140613118844928, 140613253062655, ++STORE, 140613110452224, 140613118844927, ++SNULL, 140613118844928, 140613135630335, ++STORE, 140613135630336, 140613253062655, ++STORE, 140613118844928, 140613135630335, ++ERASE, 140613118844928, 140613135630335, ++STORE, 140613127237632, 140613135630335, ++STORE, 140613110452224, 140613135630335, ++STORE, 140612976234496, 140613110452223, ++STORE, 140612967841792, 140612976234495, ++STORE, 140612833624064, 140612967841791, ++STORE, 140612825231360, 140612833624063, ++STORE, 140612816838656, 140612833624063, ++STORE, 140612682620928, 140612816838655, ++STORE, 140612674228224, 140612682620927, ++SNULL, 140612682620928, 140612732977151, ++STORE, 140612732977152, 140612816838655, ++STORE, 140612682620928, 140612732977151, ++ERASE, 140612682620928, 140612732977151, ++SNULL, 140613672501247, 140613739610111, ++STORE, 140613605392384, 140613672501247, ++STORE, 140613672501248, 140613739610111, ++ERASE, 140613672501248, 140613739610111, ++SNULL, 140613605527551, 140613672501247, ++STORE, 140613605392384, 140613605527551, ++STORE, 140613605527552, 140613672501247, ++ERASE, 140613529890816, 140613596999679, ++STORE, 140612540010496, 140612674228223, ++SNULL, 140612540010496, 140612598759423, ++STORE, 140612598759424, 140612674228223, ++STORE, 140612540010496, 140612598759423, ++ERASE, 140612540010496, 140612598759423, ++SNULL, 140613471174655, 140613513105407, ++STORE, 140613404065792, 140613471174655, ++STORE, 140613471174656, 140613513105407, ++ERASE, 140613471174656, 140613513105407, ++SNULL, 140613404200959, 140613471174655, ++STORE, 140613404065792, 140613404200959, ++STORE, 140613404200960, 140613471174655, ++SNULL, 140613336956927, 140613395673087, ++STORE, 140613269848064, 140613336956927, ++STORE, 140613336956928, 140613395673087, ++ERASE, 140613336956928, 140613395673087, ++SNULL, 140612833624064, 140612867194879, ++STORE, 140612867194880, 140612967841791, ++STORE, 140612833624064, 140612867194879, ++ERASE, 140612833624064, 140612867194879, ++SNULL, 140612976234496, 140613001412607, ++STORE, 140613001412608, 140613110452223, ++STORE, 140612976234496, 140613001412607, ++ERASE, 140612976234496, 140613001412607, ++SNULL, 140613202739199, 140613253062655, ++STORE, 140613135630336, 140613202739199, ++STORE, 140613202739200, 140613253062655, ++ERASE, 140613202739200, 140613253062655, ++SNULL, 140613135765503, 140613202739199, ++STORE, 140613135630336, 140613135765503, ++STORE, 140613135765504, 140613202739199, ++SNULL, 140612816842751, 140612833624063, ++STORE, 140612816838656, 140612816842751, ++STORE, 140612816842752, 140612833624063, ++SNULL, 140613110456319, 140613135630335, ++STORE, 140613110452224, 140613110456319, ++STORE, 140613110456320, 140613135630335, ++SNULL, 140613949300735, 140613966082047, ++STORE, 140613949296640, 140613949300735, ++STORE, 140613949300736, 140613966082047, ++SNULL, 140613110456320, 140613118844927, ++STORE, 140613118844928, 140613135630335, ++STORE, 140613110456320, 140613118844927, ++SNULL, 140613118849023, 140613135630335, ++STORE, 140613118844928, 140613118849023, ++STORE, 140613118849024, 140613135630335, ++SNULL, 140612800086015, 140612816838655, ++STORE, 140612732977152, 140612800086015, ++STORE, 140612800086016, 140612816838655, ++ERASE, 140612800086016, 140612816838655, ++SNULL, 140613253062656, 140613261455359, ++STORE, 140613261455360, 140613269848063, ++STORE, 140613253062656, 140613261455359, ++SNULL, 140613261459455, 140613269848063, ++STORE, 140613261455360, 140613261459455, ++STORE, 140613261459456, 140613269848063, ++SNULL, 140612674232319, 140612682620927, ++STORE, 140612674228224, 140612674232319, ++STORE, 140612674232320, 140612682620927, ++STORE, 140613731217408, 140613739610111, ++STORE, 140613722824704, 140613739610111, ++SNULL, 140613949300736, 140613957689343, ++STORE, 140613957689344, 140613966082047, ++STORE, 140613949300736, 140613957689343, ++SNULL, 140613957693439, 140613966082047, ++STORE, 140613957689344, 140613957693439, ++STORE, 140613957693440, 140613966082047, ++STORE, 140612464541696, 140612674228223, ++SNULL, 140612531650559, 140612674228223, ++STORE, 140612464541696, 140612531650559, ++STORE, 140612531650560, 140612674228223, ++SNULL, 140612531650560, 140612598759423, ++STORE, 140612598759424, 140612674228223, ++STORE, 140612531650560, 140612598759423, ++ERASE, 140612531650560, 140612598759423, ++SNULL, 140612665868287, 140612674228223, ++STORE, 140612598759424, 140612665868287, ++STORE, 140612665868288, 140612674228223, ++ERASE, 140612665868288, 140612674228223, ++SNULL, 140613269983231, 140613336956927, ++STORE, 140613269848064, 140613269983231, ++STORE, 140613269983232, 140613336956927, ++SNULL, 140612934303743, 140612967841791, ++STORE, 140612867194880, 140612934303743, ++STORE, 140612934303744, 140612967841791, ++ERASE, 140612934303744, 140612967841791, ++SNULL, 140613068521471, 140613110452223, ++STORE, 140613001412608, 140613068521471, ++STORE, 140613068521472, 140613110452223, ++ERASE, 140613068521472, 140613110452223, ++STORE, 140613714432000, 140613739610111, ++SNULL, 140613001547775, 140613068521471, ++STORE, 140613001412608, 140613001547775, ++STORE, 140613001547776, 140613068521471, ++SNULL, 140612733112319, 140612800086015, ++STORE, 140612732977152, 140612733112319, ++STORE, 140612733112320, 140612800086015, ++SNULL, 140613513109503, 140613529890815, ++STORE, 140613513105408, 140613513109503, ++STORE, 140613513109504, 140613529890815, ++STORE, 140613706039296, 140613739610111, ++STORE, 140613697646592, 140613739610111, ++STORE, 140613689253888, 140613739610111, ++SNULL, 140613689257983, 140613739610111, ++STORE, 140613689253888, 140613689257983, ++STORE, 140613689257984, 140613739610111, ++SNULL, 140613253066751, 140613261455359, ++STORE, 140613253062656, 140613253066751, ++STORE, 140613253066752, 140613261455359, ++STORE, 140613680861184, 140613689253887, ++STORE, 140613588606976, 140613605392383, ++SNULL, 140613689257984, 140613731217407, ++STORE, 140613731217408, 140613739610111, ++STORE, 140613689257984, 140613731217407, ++SNULL, 140613731221503, 140613739610111, ++STORE, 140613731217408, 140613731221503, ++STORE, 140613731221504, 140613739610111, ++STORE, 140613580214272, 140613605392383, ++SNULL, 140612464676863, 140612531650559, ++STORE, 140612464541696, 140612464676863, ++STORE, 140612464676864, 140612531650559, ++SNULL, 140612598894591, 140612665868287, ++STORE, 140612598759424, 140612598894591, ++STORE, 140612598894592, 140612665868287, ++SNULL, 140612867330047, 140612934303743, ++STORE, 140612867194880, 140612867330047, ++STORE, 140612867330048, 140612934303743, ++STORE, 140613571821568, 140613605392383, ++SNULL, 140613571825663, 140613605392383, ++STORE, 140613571821568, 140613571825663, ++STORE, 140613571825664, 140613605392383, ++SNULL, 140613689257984, 140613722824703, ++STORE, 140613722824704, 140613731217407, ++STORE, 140613689257984, 140613722824703, ++SNULL, 140613722828799, 140613731217407, ++STORE, 140613722824704, 140613722828799, ++STORE, 140613722828800, 140613731217407, ++SNULL, 140613689257984, 140613714431999, ++STORE, 140613714432000, 140613722824703, ++STORE, 140613689257984, 140613714431999, ++SNULL, 140613714436095, 140613722824703, ++STORE, 140613714432000, 140613714436095, ++STORE, 140613714436096, 140613722824703, ++SNULL, 140612816842752, 140612825231359, ++STORE, 140612825231360, 140612833624063, ++STORE, 140612816842752, 140612825231359, ++SNULL, 140612825235455, 140612833624063, ++STORE, 140612825231360, 140612825235455, ++STORE, 140612825235456, 140612833624063, ++SNULL, 140613395677183, 140613404065791, ++STORE, 140613395673088, 140613395677183, ++STORE, 140613395677184, 140613404065791, ++SNULL, 140613689257984, 140613706039295, ++STORE, 140613706039296, 140613714431999, ++STORE, 140613689257984, 140613706039295, ++SNULL, 140613706043391, 140613714431999, ++STORE, 140613706039296, 140613706043391, ++STORE, 140613706043392, 140613714431999, ++SNULL, 140613118849024, 140613127237631, ++STORE, 140613127237632, 140613135630335, ++STORE, 140613118849024, 140613127237631, ++SNULL, 140613127241727, 140613135630335, ++STORE, 140613127237632, 140613127241727, ++STORE, 140613127241728, 140613135630335, ++SNULL, 140613571825664, 140613580214271, ++STORE, 140613580214272, 140613605392383, ++STORE, 140613571825664, 140613580214271, ++SNULL, 140613580218367, 140613605392383, ++STORE, 140613580214272, 140613580218367, ++STORE, 140613580218368, 140613605392383, ++SNULL, 140613689257984, 140613697646591, ++STORE, 140613697646592, 140613706039295, ++STORE, 140613689257984, 140613697646591, ++SNULL, 140613697650687, 140613706039295, ++STORE, 140613697646592, 140613697650687, ++STORE, 140613697650688, 140613706039295, ++SNULL, 140613680865279, 140613689253887, ++STORE, 140613680861184, 140613680865279, ++STORE, 140613680865280, 140613689253887, ++STORE, 140613563428864, 140613571821567, ++SNULL, 140613563432959, 140613571821567, ++STORE, 140613563428864, 140613563432959, ++STORE, 140613563432960, 140613571821567, ++SNULL, 140613580218368, 140613588606975, ++STORE, 140613588606976, 140613605392383, ++STORE, 140613580218368, 140613588606975, ++SNULL, 140613588611071, 140613605392383, ++STORE, 140613588606976, 140613588611071, ++STORE, 140613588611072, 140613605392383, ++SNULL, 140613513109504, 140613521498111, ++STORE, 140613521498112, 140613529890815, ++STORE, 140613513109504, 140613521498111, ++SNULL, 140613521502207, 140613529890815, ++STORE, 140613521498112, 140613521502207, ++STORE, 140613521502208, 140613529890815, ++SNULL, 140613588611072, 140613596999679, ++STORE, 140613596999680, 140613605392383, ++STORE, 140613588611072, 140613596999679, ++SNULL, 140613597003775, 140613605392383, ++STORE, 140613596999680, 140613597003775, ++STORE, 140613597003776, 140613605392383, ++STORE, 140613555036160, 140613563428863, ++SNULL, 140613555040255, 140613563428863, ++STORE, 140613555036160, 140613555040255, ++STORE, 140613555040256, 140613563428863, ++STORE, 140613546643456, 140613555036159, ++STORE, 140613538250752, 140613555036159, ++SNULL, 140613538250752, 140613546643455, ++STORE, 140613546643456, 140613555036159, ++STORE, 140613538250752, 140613546643455, ++SNULL, 140613546647551, 140613555036159, ++STORE, 140613546643456, 140613546647551, ++STORE, 140613546647552, 140613555036159, ++STORE, 140613504712704, 140613513105407, ++STORE, 140613496320000, 140613513105407, ++SNULL, 140613496324095, 140613513105407, ++STORE, 140613496320000, 140613496324095, ++STORE, 140613496324096, 140613513105407, ++STORE, 140613487927296, 140613496319999, ++SNULL, 140613487931391, 140613496319999, ++STORE, 140613487927296, 140613487931391, ++STORE, 140613487931392, 140613496319999, ++STORE, 140613479534592, 140613487927295, ++SNULL, 140612967845887, 140612976234495, ++STORE, 140612967841792, 140612967845887, ++STORE, 140612967845888, 140612976234495, ++STORE, 140613387280384, 140613395673087, ++STORE, 140613378887680, 140613395673087, ++SNULL, 140613378887680, 140613387280383, ++STORE, 140613387280384, 140613395673087, ++STORE, 140613378887680, 140613387280383, ++SNULL, 140613387284479, 140613395673087, ++STORE, 140613387280384, 140613387284479, ++STORE, 140613387284480, 140613395673087, ++STORE, 140613370494976, 140613387280383, ++STORE, 140613362102272, 140613387280383, ++SNULL, 140613479538687, 140613487927295, ++STORE, 140613479534592, 140613479538687, ++STORE, 140613479538688, 140613487927295, ++STORE, 140613353709568, 140613387280383, ++STORE, 140613345316864, 140613387280383, ++STORE, 140613244669952, 140613253062655, ++SNULL, 140613345320959, 140613387280383, ++STORE, 140613345316864, 140613345320959, ++STORE, 140613345320960, 140613387280383, ++SNULL, 140613538254847, 140613546643455, ++STORE, 140613538250752, 140613538254847, ++STORE, 140613538254848, 140613546643455, ++STORE, 140613236277248, 140613253062655, ++STORE, 140613227884544, 140613253062655, ++STORE, 140613219491840, 140613253062655, ++STORE, 140613211099136, 140613253062655, ++SNULL, 140613211103231, 140613253062655, ++STORE, 140613211099136, 140613211103231, ++STORE, 140613211103232, 140613253062655, ++STORE, 140613102059520, 140613110452223, ++STORE, 140613093666816, 140613110452223, ++SNULL, 140613093670911, 140613110452223, ++STORE, 140613093666816, 140613093670911, ++STORE, 140613093670912, 140613110452223, ++STORE, 140613085274112, 140613093666815, ++SNULL, 140613496324096, 140613504712703, ++STORE, 140613504712704, 140613513105407, ++STORE, 140613496324096, 140613504712703, ++SNULL, 140613504716799, 140613513105407, ++STORE, 140613504712704, 140613504716799, ++STORE, 140613504716800, 140613513105407, ++SNULL, 140613345320960, 140613378887679, ++STORE, 140613378887680, 140613387280383, ++STORE, 140613345320960, 140613378887679, ++SNULL, 140613378891775, 140613387280383, ++STORE, 140613378887680, 140613378891775, ++STORE, 140613378891776, 140613387280383, ++SNULL, 140613345320960, 140613362102271, ++STORE, 140613362102272, 140613378887679, ++STORE, 140613345320960, 140613362102271, ++SNULL, 140613362106367, 140613378887679, ++STORE, 140613362102272, 140613362106367, ++STORE, 140613362106368, 140613378887679, ++SNULL, 140613362106368, 140613370494975, ++STORE, 140613370494976, 140613378887679, ++STORE, 140613362106368, 140613370494975, ++SNULL, 140613370499071, 140613378887679, ++STORE, 140613370494976, 140613370499071, ++STORE, 140613370499072, 140613378887679, ++STORE, 140613076881408, 140613093666815, ++STORE, 140612993019904, 140613001412607, ++SNULL, 140613076885503, 140613093666815, ++STORE, 140613076881408, 140613076885503, ++STORE, 140613076885504, 140613093666815, ++SNULL, 140613093670912, 140613102059519, ++STORE, 140613102059520, 140613110452223, ++STORE, 140613093670912, 140613102059519, ++SNULL, 140613102063615, 140613110452223, ++STORE, 140613102059520, 140613102063615, ++STORE, 140613102063616, 140613110452223, ++SNULL, 140613076885504, 140613085274111, ++STORE, 140613085274112, 140613093666815, ++STORE, 140613076885504, 140613085274111, ++SNULL, 140613085278207, 140613093666815, ++STORE, 140613085274112, 140613085278207, ++STORE, 140613085278208, 140613093666815, ++STORE, 140612984627200, 140613001412607, ++STORE, 140612967845888, 140612984627199, ++SNULL, 140613211103232, 140613219491839, ++STORE, 140613219491840, 140613253062655, ++STORE, 140613211103232, 140613219491839, ++SNULL, 140613219495935, 140613253062655, ++STORE, 140613219491840, 140613219495935, ++STORE, 140613219495936, 140613253062655, ++STORE, 140612959449088, 140612967841791, ++STORE, 140612951056384, 140612967841791, ++SNULL, 140612951060479, 140612967841791, ++STORE, 140612951056384, 140612951060479, ++STORE, 140612951060480, 140612967841791, ++SNULL, 140613345320960, 140613353709567, ++STORE, 140613353709568, 140613362102271, ++STORE, 140613345320960, 140613353709567, ++SNULL, 140613353713663, 140613362102271, ++STORE, 140613353709568, 140613353713663, ++STORE, 140613353713664, 140613362102271, ++SNULL, 140613219495936, 140613244669951, ++STORE, 140613244669952, 140613253062655, ++STORE, 140613219495936, 140613244669951, ++SNULL, 140613244674047, 140613253062655, ++STORE, 140613244669952, 140613244674047, ++STORE, 140613244674048, 140613253062655, ++STORE, 140612942663680, 140612951056383, ++SNULL, 140613219495936, 140613236277247, ++STORE, 140613236277248, 140613244669951, ++STORE, 140613219495936, 140613236277247, ++SNULL, 140613236281343, 140613244669951, ++STORE, 140613236277248, 140613236281343, ++STORE, 140613236281344, 140613244669951, ++SNULL, 140613219495936, 140613227884543, ++STORE, 140613227884544, 140613236277247, ++STORE, 140613219495936, 140613227884543, ++SNULL, 140613227888639, 140613236277247, ++STORE, 140613227884544, 140613227888639, ++STORE, 140613227888640, 140613236277247, ++SNULL, 140612984627200, 140612993019903, ++STORE, 140612993019904, 140613001412607, ++STORE, 140612984627200, 140612993019903, ++SNULL, 140612993023999, 140613001412607, ++STORE, 140612993019904, 140612993023999, ++STORE, 140612993024000, 140613001412607, ++STORE, 140612858802176, 140612867194879, ++STORE, 140612850409472, 140612867194879, ++SNULL, 140612951060480, 140612959449087, ++STORE, 140612959449088, 140612967841791, ++STORE, 140612951060480, 140612959449087, ++SNULL, 140612959453183, 140612967841791, ++STORE, 140612959449088, 140612959453183, ++STORE, 140612959453184, 140612967841791, ++SNULL, 140612967845888, 140612976234495, ++STORE, 140612976234496, 140612984627199, ++STORE, 140612967845888, 140612976234495, ++SNULL, 140612976238591, 140612984627199, ++STORE, 140612976234496, 140612976238591, ++STORE, 140612976238592, 140612984627199, ++STORE, 140612842016768, 140612867194879, ++SNULL, 140612842020863, 140612867194879, ++STORE, 140612842016768, 140612842020863, ++STORE, 140612842020864, 140612867194879, ++SNULL, 140612984631295, 140612993019903, ++STORE, 140612984627200, 140612984631295, ++STORE, 140612984631296, 140612993019903, ++STORE, 140612825235456, 140612842016767, ++STORE, 140612808445952, 140612816838655, ++SNULL, 140612942667775, 140612951056383, ++STORE, 140612942663680, 140612942667775, ++STORE, 140612942667776, 140612951056383, ++STORE, 140612724584448, 140612732977151, ++SNULL, 140612724588543, 140612732977151, ++STORE, 140612724584448, 140612724588543, ++STORE, 140612724588544, 140612732977151, ++STORE, 140612716191744, 140612724584447, ++SNULL, 140612842020864, 140612850409471, ++STORE, 140612850409472, 140612867194879, ++STORE, 140612842020864, 140612850409471, ++SNULL, 140612850413567, 140612867194879, ++STORE, 140612850409472, 140612850413567, ++STORE, 140612850413568, 140612867194879, ++SNULL, 140612850413568, 140612858802175, ++STORE, 140612858802176, 140612867194879, ++STORE, 140612850413568, 140612858802175, ++SNULL, 140612858806271, 140612867194879, ++STORE, 140612858802176, 140612858806271, ++STORE, 140612858806272, 140612867194879, ++STORE, 140612707799040, 140612724584447, ++SNULL, 140612707803135, 140612724584447, ++STORE, 140612707799040, 140612707803135, ++STORE, 140612707803136, 140612724584447, ++SNULL, 140612707803136, 140612716191743, ++STORE, 140612716191744, 140612724584447, ++STORE, 140612707803136, 140612716191743, ++SNULL, 140612716195839, 140612724584447, ++STORE, 140612716191744, 140612716195839, ++STORE, 140612716195840, 140612724584447, ++SNULL, 140612808450047, 140612816838655, ++STORE, 140612808445952, 140612808450047, ++STORE, 140612808450048, 140612816838655, ++SNULL, 140612825235456, 140612833624063, ++STORE, 140612833624064, 140612842016767, ++STORE, 140612825235456, 140612833624063, ++SNULL, 140612833628159, 140612842016767, ++STORE, 140612833624064, 140612833628159, ++STORE, 140612833628160, 140612842016767, ++STORE, 140612699406336, 140612707799039, ++SNULL, 140612699410431, 140612707799039, ++STORE, 140612699406336, 140612699410431, ++STORE, 140612699410432, 140612707799039, ++STORE, 140614384926720, 140614384955391, ++STORE, 140614349332480, 140614351523839, ++SNULL, 140614349332480, 140614349422591, ++STORE, 140614349422592, 140614351523839, ++STORE, 140614349332480, 140614349422591, ++SNULL, 140614351515647, 140614351523839, ++STORE, 140614349422592, 140614351515647, ++STORE, 140614351515648, 140614351523839, ++ERASE, 140614351515648, 140614351523839, ++STORE, 140614351515648, 140614351523839, ++SNULL, 140614351519743, 140614351523839, ++STORE, 140614351515648, 140614351519743, ++STORE, 140614351519744, 140614351523839, ++ERASE, 140614384926720, 140614384955391, ++ERASE, 140613949296640, 140613949300735, ++ERASE, 140613949300736, 140613957689343, ++ERASE, 140613689253888, 140613689257983, ++ERASE, 140613689257984, 140613697646591, ++ERASE, 140613563428864, 140613563432959, ++ERASE, 140613563432960, 140613571821567, ++ERASE, 140613211099136, 140613211103231, ++ERASE, 140613211103232, 140613219491839, ++ERASE, 140614133870592, 140614133874687, ++ERASE, 140614133874688, 140614142263295, ++ERASE, 140612967841792, 140612967845887, ++ERASE, 140612967845888, 140612976234495, ++ERASE, 140613076881408, 140613076885503, ++ERASE, 140613076885504, 140613085274111, ++ERASE, 140612850409472, 140612850413567, ++ERASE, 140612850413568, 140612858802175, ++ERASE, 140613110452224, 140613110456319, ++ERASE, 140613110456320, 140613118844927, ++ERASE, 140613706039296, 140613706043391, ++ERASE, 140613706043392, 140613714431999, ++ERASE, 140613521498112, 140613521502207, ++ERASE, 140613521502208, 140613529890815, ++ERASE, 140613362102272, 140613362106367, ++ERASE, 140613362106368, 140613370494975, ++ERASE, 140613253062656, 140613253066751, ++ERASE, 140613253066752, 140613261455359, ++ERASE, 140612816838656, 140612816842751, ++ERASE, 140612816842752, 140612825231359, ++ERASE, 140613261455360, 140613261459455, ++ERASE, 140613261459456, 140613269848063, ++ERASE, 140613118844928, 140613118849023, ++ERASE, 140613118849024, 140613127237631, ++ERASE, 140613714432000, 140613714436095, ++ERASE, 140613714436096, 140613722824703, ++ERASE, 140613496320000, 140613496324095, ++ERASE, 140613496324096, 140613504712703, ++ERASE, 140613513105408, 140613513109503, ++ERASE, 140613513109504, 140613521498111, ++ERASE, 140613697646592, 140613697650687, ++ERASE, 140613697650688, 140613706039295, ++ERASE, 140613093666816, 140613093670911, ++ERASE, 140613093670912, 140613102059519, ++ERASE, 140612993019904, 140612993023999, ++ERASE, 140612993024000, 140613001412607, ++ERASE, 140613127237632, 140613127241727, ++ERASE, 140613127241728, 140613135630335, ++ERASE, 140613957689344, 140613957693439, ++ERASE, 140613957693440, 140613966082047, ++ERASE, 140613571821568, 140613571825663, ++ERASE, 140613571825664, 140613580214271, ++ERASE, 140613479534592, 140613479538687, ++ERASE, 140613479538688, 140613487927295, ++ERASE, 140612984627200, 140612984631295, ++ERASE, 140612984631296, 140612993019903, ++ERASE, 140613588606976, 140613588611071, ++ERASE, 140613588611072, 140613596999679, ++ERASE, 140613680861184, 140613680865279, ++ERASE, 140613680865280, 140613689253887, ++ERASE, 140613345316864, 140613345320959, ++ERASE, 140613345320960, 140613353709567, ++ERASE, 140613596999680, 140613597003775, ++ERASE, 140613597003776, 140613605392383, ++ERASE, 140613966082048, 140613966086143, ++ERASE, 140613966086144, 140613974474751, ++ERASE, 140613731217408, 140613731221503, ++ERASE, 140613731221504, 140613739610111, ++ERASE, 140613395673088, 140613395677183, ++ERASE, 140613395677184, 140613404065791, ++ERASE, 140612825231360, 140612825235455, ++ERASE, 140612825235456, 140612833624063, ++ERASE, 140612674228224, 140612674232319, ++ERASE, 140612674232320, 140612682620927, ++ERASE, 140613722824704, 140613722828799, ++ERASE, 140613722828800, 140613731217407, ++ERASE, 140613487927296, 140613487931391, ++ERASE, 140613487931392, 140613496319999, ++ERASE, 140613102059520, 140613102063615, ++ERASE, 140613102063616, 140613110452223, ++ERASE, 140614242910208, 140614242914303, ++ERASE, 140614242914304, 140614251302911, ++ERASE, 140612808445952, 140612808450047, ++ERASE, 140612808450048, 140612816838655, ++ERASE, 140613236277248, 140613236281343, ++ERASE, 140613236281344, 140613244669951, ++ERASE, 140613580214272, 140613580218367, ++ERASE, 140613580218368, 140613588606975, ++ERASE, 140613370494976, 140613370499071, ++ERASE, 140613370499072, 140613378887679, ++ERASE, 140613244669952, 140613244674047, ++ERASE, 140613244674048, 140613253062655, ++ERASE, 140612724584448, 140612724588543, ++ERASE, 140612724588544, 140612732977151, ++ERASE, 140612707799040, 140612707803135, ++ERASE, 140612707803136, 140612716191743, ++ERASE, 140613504712704, 140613504716799, ++ERASE, 140613504716800, 140613513105407, ++ }; ++ ++ unsigned long set39[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140736271417344, 140737488351231, ++SNULL, 140736271421439, 140737488351231, ++STORE, 140736271417344, 140736271421439, ++STORE, 140736271286272, 140736271421439, ++STORE, 94412930822144, 94412933074943, ++SNULL, 94412930953215, 94412933074943, ++STORE, 94412930822144, 94412930953215, ++STORE, 94412930953216, 94412933074943, ++ERASE, 94412930953216, 94412933074943, ++STORE, 94412933046272, 94412933054463, ++STORE, 94412933054464, 94412933074943, ++STORE, 140326136901632, 140326139154431, ++SNULL, 140326137044991, 140326139154431, ++STORE, 140326136901632, 140326137044991, ++STORE, 140326137044992, 140326139154431, ++ERASE, 140326137044992, 140326139154431, ++STORE, 140326139142144, 140326139150335, ++STORE, 140326139150336, 140326139154431, ++STORE, 140736271585280, 140736271589375, ++STORE, 140736271572992, 140736271585279, ++STORE, 140326139113472, 140326139142143, ++STORE, 140326139105280, 140326139113471, ++STORE, 140326134685696, 140326136901631, ++SNULL, 140326134685696, 140326134783999, ++STORE, 140326134784000, 140326136901631, ++STORE, 140326134685696, 140326134783999, ++SNULL, 140326136877055, 140326136901631, ++STORE, 140326134784000, 140326136877055, ++STORE, 140326136877056, 140326136901631, ++SNULL, 140326136877056, 140326136885247, ++STORE, 140326136885248, 140326136901631, ++STORE, 140326136877056, 140326136885247, ++ERASE, 140326136877056, 140326136885247, ++STORE, 140326136877056, 140326136885247, ++ERASE, 140326136885248, 140326136901631, ++STORE, 140326136885248, 140326136901631, ++STORE, 140326130888704, 140326134685695, ++SNULL, 140326130888704, 140326132547583, ++STORE, 140326132547584, 140326134685695, ++STORE, 140326130888704, 140326132547583, ++SNULL, 140326134644735, 140326134685695, ++STORE, 140326132547584, 140326134644735, ++STORE, 140326134644736, 140326134685695, ++SNULL, 140326134644736, 140326134669311, ++STORE, 140326134669312, 140326134685695, ++STORE, 140326134644736, 140326134669311, ++ERASE, 140326134644736, 140326134669311, ++STORE, 140326134644736, 140326134669311, ++ERASE, 140326134669312, 140326134685695, ++STORE, 140326134669312, 140326134685695, ++STORE, 140326139097088, 140326139113471, ++SNULL, 140326134661119, 140326134669311, ++STORE, 140326134644736, 140326134661119, ++STORE, 140326134661120, 140326134669311, ++SNULL, 140326136881151, 140326136885247, ++STORE, 140326136877056, 140326136881151, ++STORE, 140326136881152, 140326136885247, ++SNULL, 94412933050367, 94412933054463, ++STORE, 94412933046272, 94412933050367, ++STORE, 94412933050368, 94412933054463, ++SNULL, 140326139146239, 140326139150335, ++STORE, 140326139142144, 140326139146239, ++STORE, 140326139146240, 140326139150335, ++ERASE, 140326139113472, 140326139142143, ++STORE, 94412939493376, 94412939628543, ++STORE, 140326122496000, 140326130888703, ++SNULL, 140326122500095, 140326130888703, ++STORE, 140326122496000, 140326122500095, ++STORE, 140326122500096, 140326130888703, ++STORE, 140326114103296, 140326122495999, ++STORE, 140325979885568, 140326114103295, ++SNULL, 140325979885568, 140326043910143, ++STORE, 140326043910144, 140326114103295, ++STORE, 140325979885568, 140326043910143, ++ERASE, 140325979885568, 140326043910143, ++SNULL, 140326111019007, 140326114103295, ++STORE, 140326043910144, 140326111019007, ++STORE, 140326111019008, 140326114103295, ++ERASE, 140326111019008, 140326114103295, ++SNULL, 140326044045311, 140326111019007, ++STORE, 140326043910144, 140326044045311, ++STORE, 140326044045312, 140326111019007, ++SNULL, 140326114107391, 140326122495999, ++STORE, 140326114103296, 140326114107391, ++STORE, 140326114107392, 140326122495999, ++STORE, 140326035517440, 140326043910143, ++SNULL, 140326035521535, 140326043910143, ++STORE, 140326035517440, 140326035521535, ++STORE, 140326035521536, 140326043910143, ++STORE, 140326027124736, 140326035517439, ++SNULL, 140326027128831, 140326035517439, ++STORE, 140326027124736, 140326027128831, ++STORE, 140326027128832, 140326035517439, ++STORE, 140326018732032, 140326027124735, ++SNULL, 140326018736127, 140326027124735, ++STORE, 140326018732032, 140326018736127, ++STORE, 140326018736128, 140326027124735, ++STORE, 140326010339328, 140326018732031, ++STORE, 140326001946624, 140326018732031, ++STORE, 140325993553920, 140326018732031, ++STORE, 140325859336192, 140325993553919, ++SNULL, 140325859336192, 140325909692415, ++STORE, 140325909692416, 140325993553919, ++STORE, 140325859336192, 140325909692415, ++ERASE, 140325859336192, 140325909692415, ++SNULL, 140325976801279, 140325993553919, ++STORE, 140325909692416, 140325976801279, ++STORE, 140325976801280, 140325993553919, ++ERASE, 140325976801280, 140325993553919, ++STORE, 140325985161216, 140326018732031, ++STORE, 140325775474688, 140325976801279, ++STORE, 140325708365824, 140325976801279, ++SNULL, 140325708500991, 140325976801279, ++STORE, 140325708365824, 140325708500991, ++STORE, 140325708500992, 140325976801279, ++SNULL, 140325708500992, 140325909692415, ++STORE, 140325909692416, 140325976801279, ++STORE, 140325708500992, 140325909692415, ++SNULL, 140325909827583, 140325976801279, ++STORE, 140325909692416, 140325909827583, ++STORE, 140325909827584, 140325976801279, ++SNULL, 140325842583551, 140325909692415, ++STORE, 140325708500992, 140325842583551, ++STORE, 140325842583552, 140325909692415, ++ERASE, 140325842583552, 140325909692415, ++SNULL, 140325708500992, 140325775474687, ++STORE, 140325775474688, 140325842583551, ++STORE, 140325708500992, 140325775474687, ++SNULL, 140325775609855, 140325842583551, ++STORE, 140325775474688, 140325775609855, ++STORE, 140325775609856, 140325842583551, ++STORE, 140325775609856, 140325909692415, ++SNULL, 140325775609856, 140325842583551, ++STORE, 140325842583552, 140325909692415, ++STORE, 140325775609856, 140325842583551, ++SNULL, 140325842718719, 140325909692415, ++STORE, 140325842583552, 140325842718719, ++STORE, 140325842718720, 140325909692415, ++SNULL, 140325985161216, 140325993553919, ++STORE, 140325993553920, 140326018732031, ++STORE, 140325985161216, 140325993553919, ++SNULL, 140325993558015, 140326018732031, ++STORE, 140325993553920, 140325993558015, ++STORE, 140325993558016, 140326018732031, ++SNULL, 140325985165311, 140325993553919, ++STORE, 140325985161216, 140325985165311, ++STORE, 140325985165312, 140325993553919, ++SNULL, 140325993558016, 140326001946623, ++STORE, 140326001946624, 140326018732031, ++STORE, 140325993558016, 140326001946623, ++SNULL, 140326001950719, 140326018732031, ++STORE, 140326001946624, 140326001950719, ++STORE, 140326001950720, 140326018732031, ++SNULL, 140326001950720, 140326010339327, ++STORE, 140326010339328, 140326018732031, ++STORE, 140326001950720, 140326010339327, ++SNULL, 140326010343423, 140326018732031, ++STORE, 140326010339328, 140326010343423, ++STORE, 140326010343424, 140326018732031, ++STORE, 140325699973120, 140325708365823, ++STORE, 140325691580416, 140325708365823, ++STORE, 140325683187712, 140325708365823, ++SNULL, 140325683191807, 140325708365823, ++STORE, 140325683187712, 140325683191807, ++STORE, 140325683191808, 140325708365823, ++SNULL, 140325683191808, 140325699973119, ++STORE, 140325699973120, 140325708365823, ++STORE, 140325683191808, 140325699973119, ++SNULL, 140325699977215, 140325708365823, ++STORE, 140325699973120, 140325699977215, ++STORE, 140325699977216, 140325708365823, ++STORE, 140325674795008, 140325683187711, ++STORE, 140325666402304, 140325683187711, ++STORE, 140325658009600, 140325683187711, ++SNULL, 140325658009600, 140325666402303, ++STORE, 140325666402304, 140325683187711, ++STORE, 140325658009600, 140325666402303, ++SNULL, 140325666406399, 140325683187711, ++STORE, 140325666402304, 140325666406399, ++STORE, 140325666406400, 140325683187711, ++SNULL, 140325683191808, 140325691580415, ++STORE, 140325691580416, 140325699973119, ++STORE, 140325683191808, 140325691580415, ++SNULL, 140325691584511, 140325699973119, ++STORE, 140325691580416, 140325691584511, ++STORE, 140325691584512, 140325699973119, ++SNULL, 140325666406400, 140325674795007, ++STORE, 140325674795008, 140325683187711, ++STORE, 140325666406400, 140325674795007, ++SNULL, 140325674799103, 140325683187711, ++STORE, 140325674795008, 140325674799103, ++STORE, 140325674799104, 140325683187711, ++STORE, 140325649616896, 140325666402303, ++SNULL, 140325649616896, 140325658009599, ++STORE, 140325658009600, 140325666402303, ++STORE, 140325649616896, 140325658009599, ++SNULL, 140325658013695, 140325666402303, ++STORE, 140325658009600, 140325658013695, ++STORE, 140325658013696, 140325666402303, ++SNULL, 140325649620991, 140325658009599, ++STORE, 140325649616896, 140325649620991, ++STORE, 140325649620992, 140325658009599, ++STORE, 140325641224192, 140325649616895, ++STORE, 140325632831488, 140325649616895, ++SNULL, 140325632835583, 140325649616895, ++STORE, 140325632831488, 140325632835583, ++STORE, 140325632835584, 140325649616895, ++STORE, 140325624438784, 140325632831487, ++SNULL, 140325624442879, 140325632831487, ++STORE, 140325624438784, 140325624442879, ++STORE, 140325624442880, 140325632831487, ++SNULL, 140325632835584, 140325641224191, ++STORE, 140325641224192, 140325649616895, ++STORE, 140325632835584, 140325641224191, ++SNULL, 140325641228287, 140325649616895, ++STORE, 140325641224192, 140325641228287, ++STORE, 140325641228288, 140325649616895, ++STORE, 140325616046080, 140325624438783, ++SNULL, 140325616050175, 140325624438783, ++STORE, 140325616046080, 140325616050175, ++STORE, 140325616050176, 140325624438783, ++STORE, 140325607653376, 140325616046079, ++SNULL, 140325607657471, 140325616046079, ++STORE, 140325607653376, 140325607657471, ++STORE, 140325607657472, 140325616046079, ++STORE, 140325599260672, 140325607653375, ++STORE, 140325590867968, 140325607653375, ++STORE, 140325456650240, 140325590867967, ++SNULL, 140325456650240, 140325507039231, ++STORE, 140325507039232, 140325590867967, ++STORE, 140325456650240, 140325507039231, ++ERASE, 140325456650240, 140325507039231, ++STORE, 140325498646528, 140325507039231, ++STORE, 140325364428800, 140325498646527, ++SNULL, 140325364428800, 140325372821503, ++STORE, 140325372821504, 140325498646527, ++STORE, 140325364428800, 140325372821503, ++ERASE, 140325364428800, 140325372821503, ++STORE, 140325364428800, 140325372821503, ++STORE, 140325356036096, 140325372821503, ++STORE, 140325221818368, 140325356036095, ++SNULL, 140325221818368, 140325238603775, ++STORE, 140325238603776, 140325356036095, ++STORE, 140325221818368, 140325238603775, ++ERASE, 140325221818368, 140325238603775, ++STORE, 140325230211072, 140325238603775, ++STORE, 140325221818368, 140325238603775, ++STORE, 140325087600640, 140325221818367, ++STORE, 140325079207936, 140325087600639, ++SNULL, 140325087600640, 140325104386047, ++STORE, 140325104386048, 140325221818367, ++STORE, 140325087600640, 140325104386047, ++ERASE, 140325087600640, 140325104386047, ++STORE, 140325095993344, 140325104386047, ++STORE, 140325079207936, 140325104386047, ++STORE, 140324944990208, 140325079207935, ++SNULL, 140324944990208, 140324970168319, ++STORE, 140324970168320, 140325079207935, ++STORE, 140324944990208, 140324970168319, ++ERASE, 140324944990208, 140324970168319, ++STORE, 140324961775616, 140324970168319, ++STORE, 140324953382912, 140324970168319, ++STORE, 140324819165184, 140324953382911, ++STORE, 140324684947456, 140324953382911, ++STORE, 140324676554752, 140324684947455, ++STORE, 140324668162048, 140324684947455, ++STORE, 140324533944320, 140324668162047, ++STORE, 140324525551616, 140324533944319, ++SNULL, 140324533944320, 140324567515135, ++STORE, 140324567515136, 140324668162047, ++STORE, 140324533944320, 140324567515135, ++ERASE, 140324533944320, 140324567515135, ++STORE, 140324559122432, 140324567515135, ++STORE, 140324391333888, 140324525551615, ++SNULL, 140325574148095, 140325590867967, ++STORE, 140325507039232, 140325574148095, ++STORE, 140325574148096, 140325590867967, ++ERASE, 140325574148096, 140325590867967, ++SNULL, 140325439930367, 140325498646527, ++STORE, 140325372821504, 140325439930367, ++STORE, 140325439930368, 140325498646527, ++ERASE, 140325439930368, 140325498646527, ++SNULL, 140325305712639, 140325356036095, ++STORE, 140325238603776, 140325305712639, ++STORE, 140325305712640, 140325356036095, ++ERASE, 140325305712640, 140325356036095, ++SNULL, 140325171494911, 140325221818367, ++STORE, 140325104386048, 140325171494911, ++STORE, 140325171494912, 140325221818367, ++ERASE, 140325171494912, 140325221818367, ++SNULL, 140325104521215, 140325171494911, ++STORE, 140325104386048, 140325104521215, ++STORE, 140325104521216, 140325171494911, ++STORE, 140324257116160, 140324525551615, ++SNULL, 140324257116160, 140324299079679, ++STORE, 140324299079680, 140324525551615, ++STORE, 140324257116160, 140324299079679, ++ERASE, 140324257116160, 140324299079679, ++SNULL, 140325037277183, 140325079207935, ++STORE, 140324970168320, 140325037277183, ++STORE, 140325037277184, 140325079207935, ++ERASE, 140325037277184, 140325079207935, ++SNULL, 140324819165183, 140324953382911, ++STORE, 140324684947456, 140324819165183, ++STORE, 140324819165184, 140324953382911, ++SNULL, 140324819165184, 140324835950591, ++STORE, 140324835950592, 140324953382911, ++STORE, 140324819165184, 140324835950591, ++ERASE, 140324819165184, 140324835950591, ++SNULL, 140324903059455, 140324953382911, ++STORE, 140324835950592, 140324903059455, ++STORE, 140324903059456, 140324953382911, ++ERASE, 140324903059456, 140324953382911, ++SNULL, 140324684947456, 140324701732863, ++STORE, 140324701732864, 140324819165183, ++STORE, 140324684947456, 140324701732863, ++ERASE, 140324684947456, 140324701732863, ++SNULL, 140324768841727, 140324819165183, ++STORE, 140324701732864, 140324768841727, ++STORE, 140324768841728, 140324819165183, ++ERASE, 140324768841728, 140324819165183, ++SNULL, 140324634623999, 140324668162047, ++STORE, 140324567515136, 140324634623999, ++STORE, 140324634624000, 140324668162047, ++ERASE, 140324634624000, 140324668162047, ++SNULL, 140324391333887, 140324525551615, ++STORE, 140324299079680, 140324391333887, ++STORE, 140324391333888, 140324525551615, ++SNULL, 140324391333888, 140324433297407, ++STORE, 140324433297408, 140324525551615, ++STORE, 140324391333888, 140324433297407, ++ERASE, 140324391333888, 140324433297407, ++SNULL, 140325507174399, 140325574148095, ++STORE, 140325507039232, 140325507174399, ++STORE, 140325507174400, 140325574148095, ++SNULL, 140325590867968, 140325599260671, ++STORE, 140325599260672, 140325607653375, ++STORE, 140325590867968, 140325599260671, ++SNULL, 140325599264767, 140325607653375, ++STORE, 140325599260672, 140325599264767, ++STORE, 140325599264768, 140325607653375, ++SNULL, 140325372956671, 140325439930367, ++STORE, 140325372821504, 140325372956671, ++STORE, 140325372956672, 140325439930367, ++SNULL, 140324668166143, 140324684947455, ++STORE, 140324668162048, 140324668166143, ++STORE, 140324668166144, 140324684947455, ++SNULL, 140324525555711, 140324533944319, ++STORE, 140324525551616, 140324525555711, ++STORE, 140324525555712, 140324533944319, ++SNULL, 140324953382912, 140324961775615, ++STORE, 140324961775616, 140324970168319, ++STORE, 140324953382912, 140324961775615, ++SNULL, 140324961779711, 140324970168319, ++STORE, 140324961775616, 140324961779711, ++STORE, 140324961779712, 140324970168319, ++SNULL, 140325079212031, 140325104386047, ++STORE, 140325079207936, 140325079212031, ++STORE, 140325079212032, 140325104386047, ++SNULL, 140325221818368, 140325230211071, ++STORE, 140325230211072, 140325238603775, ++STORE, 140325221818368, 140325230211071, ++SNULL, 140325230215167, 140325238603775, ++STORE, 140325230211072, 140325230215167, ++STORE, 140325230215168, 140325238603775, ++SNULL, 140325356036096, 140325364428799, ++STORE, 140325364428800, 140325372821503, ++STORE, 140325356036096, 140325364428799, ++SNULL, 140325364432895, 140325372821503, ++ }; ++ unsigned long set40[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734309167104, 140737488351231, ++SNULL, 140734309171199, 140737488351231, ++STORE, 140734309167104, 140734309171199, ++STORE, 140734309036032, 140734309171199, ++STORE, 94270500081664, 94270502334463, ++SNULL, 94270500212735, 94270502334463, ++STORE, 94270500081664, 94270500212735, ++STORE, 94270500212736, 94270502334463, ++ERASE, 94270500212736, 94270502334463, ++STORE, 94270502305792, 94270502313983, ++STORE, 94270502313984, 94270502334463, ++STORE, 140321935110144, 140321937362943, ++SNULL, 140321935253503, 140321937362943, ++STORE, 140321935110144, 140321935253503, ++STORE, 140321935253504, 140321937362943, ++ERASE, 140321935253504, 140321937362943, ++STORE, 140321937350656, 140321937358847, ++STORE, 140321937358848, 140321937362943, ++STORE, 140734309625856, 140734309629951, ++STORE, 140734309613568, 140734309625855, ++STORE, 140321937321984, 140321937350655, ++STORE, 140321937313792, 140321937321983, ++STORE, 140321932894208, 140321935110143, ++SNULL, 140321932894208, 140321932992511, ++STORE, 140321932992512, 140321935110143, ++STORE, 140321932894208, 140321932992511, ++SNULL, 140321935085567, 140321935110143, ++STORE, 140321932992512, 140321935085567, ++STORE, 140321935085568, 140321935110143, ++SNULL, 140321935085568, 140321935093759, ++STORE, 140321935093760, 140321935110143, ++STORE, 140321935085568, 140321935093759, ++ERASE, 140321935085568, 140321935093759, ++STORE, 140321935085568, 140321935093759, ++ERASE, 140321935093760, 140321935110143, ++STORE, 140321935093760, 140321935110143, ++STORE, 140321929097216, 140321932894207, ++SNULL, 140321929097216, 140321930756095, ++STORE, 140321930756096, 140321932894207, ++STORE, 140321929097216, 140321930756095, ++SNULL, 140321932853247, 140321932894207, ++STORE, 140321930756096, 140321932853247, ++STORE, 140321932853248, 140321932894207, ++SNULL, 140321932853248, 140321932877823, ++STORE, 140321932877824, 140321932894207, ++STORE, 140321932853248, 140321932877823, ++ERASE, 140321932853248, 140321932877823, ++STORE, 140321932853248, 140321932877823, ++ERASE, 140321932877824, 140321932894207, ++STORE, 140321932877824, 140321932894207, ++STORE, 140321937305600, 140321937321983, ++SNULL, 140321932869631, 140321932877823, ++STORE, 140321932853248, 140321932869631, ++STORE, 140321932869632, 140321932877823, ++SNULL, 140321935089663, 140321935093759, ++STORE, 140321935085568, 140321935089663, ++STORE, 140321935089664, 140321935093759, ++SNULL, 94270502309887, 94270502313983, ++STORE, 94270502305792, 94270502309887, ++STORE, 94270502309888, 94270502313983, ++SNULL, 140321937354751, 140321937358847, ++STORE, 140321937350656, 140321937354751, ++STORE, 140321937354752, 140321937358847, ++ERASE, 140321937321984, 140321937350655, ++STORE, 94270507364352, 94270507499519, ++STORE, 140321920704512, 140321929097215, ++SNULL, 140321920708607, 140321929097215, ++STORE, 140321920704512, 140321920708607, ++STORE, 140321920708608, 140321929097215, ++STORE, 140321912311808, 140321920704511, ++STORE, 140321778094080, 140321912311807, ++SNULL, 140321778094080, 140321816051711, ++STORE, 140321816051712, 140321912311807, ++STORE, 140321778094080, 140321816051711, ++ERASE, 140321778094080, 140321816051711, ++SNULL, 140321883160575, 140321912311807, ++STORE, 140321816051712, 140321883160575, ++STORE, 140321883160576, 140321912311807, ++ERASE, 140321883160576, 140321912311807, ++SNULL, 140321816186879, 140321883160575, ++STORE, 140321816051712, 140321816186879, ++STORE, 140321816186880, 140321883160575, ++SNULL, 140321912315903, 140321920704511, ++STORE, 140321912311808, 140321912315903, ++STORE, 140321912315904, 140321920704511, ++STORE, 140321903919104, 140321912311807, ++SNULL, 140321903923199, 140321912311807, ++STORE, 140321903919104, 140321903923199, ++STORE, 140321903923200, 140321912311807, ++STORE, 140321895526400, 140321903919103, ++SNULL, 140321895530495, 140321903919103, ++STORE, 140321895526400, 140321895530495, ++STORE, 140321895530496, 140321903919103, ++STORE, 140321887133696, 140321895526399, ++SNULL, 140321887137791, 140321895526399, ++STORE, 140321887133696, 140321887137791, ++STORE, 140321887137792, 140321895526399, ++STORE, 140321807659008, 140321816051711, ++STORE, 140321673441280, 140321807659007, ++SNULL, 140321673441280, 140321681833983, ++STORE, 140321681833984, 140321807659007, ++STORE, 140321673441280, 140321681833983, ++ERASE, 140321673441280, 140321681833983, ++SNULL, 140321748942847, 140321807659007, ++STORE, 140321681833984, 140321748942847, ++STORE, 140321748942848, 140321807659007, ++ERASE, 140321748942848, 140321807659007, ++STORE, 140321799266304, 140321816051711, ++STORE, 140321790873600, 140321816051711, ++STORE, 140321782480896, 140321816051711, ++STORE, 140321547616256, 140321748942847, ++SNULL, 140321614725119, 140321748942847, ++STORE, 140321547616256, 140321614725119, ++STORE, 140321614725120, 140321748942847, ++SNULL, 140321614725120, 140321681833983, ++STORE, 140321681833984, 140321748942847, ++STORE, 140321614725120, 140321681833983, ++ERASE, 140321614725120, 140321681833983, ++SNULL, 140321681969151, 140321748942847, ++STORE, 140321681833984, 140321681969151, ++STORE, 140321681969152, 140321748942847, ++STORE, 140321547616256, 140321681833983, ++SNULL, 140321547616256, 140321614725119, ++STORE, 140321614725120, 140321681833983, ++STORE, 140321547616256, 140321614725119, ++SNULL, 140321614860287, 140321681833983, ++STORE, 140321614725120, 140321614860287, ++STORE, 140321614860288, 140321681833983, ++SNULL, 140321547751423, 140321614725119, ++STORE, 140321547616256, 140321547751423, ++STORE, 140321547751424, 140321614725119, ++STORE, 140321480507392, 140321547616255, ++SNULL, 140321782480896, 140321799266303, ++STORE, 140321799266304, 140321816051711, ++STORE, 140321782480896, 140321799266303, ++SNULL, 140321799270399, 140321816051711, ++STORE, 140321799266304, 140321799270399, ++STORE, 140321799270400, 140321816051711, ++STORE, 140321774088192, 140321799266303, ++SNULL, 140321774088192, 140321790873599, ++STORE, 140321790873600, 140321799266303, ++STORE, 140321774088192, 140321790873599, ++SNULL, 140321790877695, 140321799266303, ++STORE, 140321790873600, 140321790877695, ++STORE, 140321790877696, 140321799266303, ++SNULL, 140321480642559, 140321547616255, ++STORE, 140321480507392, 140321480642559, ++STORE, 140321480642560, 140321547616255, ++SNULL, 140321774088192, 140321782480895, ++STORE, 140321782480896, 140321790873599, ++STORE, 140321774088192, 140321782480895, ++SNULL, 140321782484991, 140321790873599, ++STORE, 140321782480896, 140321782484991, ++STORE, 140321782484992, 140321790873599, ++SNULL, 140321799270400, 140321807659007, ++STORE, 140321807659008, 140321816051711, ++STORE, 140321799270400, 140321807659007, ++SNULL, 140321807663103, 140321816051711, ++STORE, 140321807659008, 140321807663103, ++STORE, 140321807663104, 140321816051711, ++STORE, 140321765695488, 140321782480895, ++STORE, 140321757302784, 140321782480895, ++SNULL, 140321757306879, 140321782480895, ++STORE, 140321757302784, 140321757306879, ++STORE, 140321757306880, 140321782480895, ++STORE, 140321472114688, 140321480507391, ++STORE, 140321463721984, 140321480507391, ++SNULL, 140321463726079, 140321480507391, ++STORE, 140321463721984, 140321463726079, ++STORE, 140321463726080, 140321480507391, ++SNULL, 140321757306880, 140321774088191, ++STORE, 140321774088192, 140321782480895, ++STORE, 140321757306880, 140321774088191, ++SNULL, 140321774092287, 140321782480895, ++STORE, 140321774088192, 140321774092287, ++STORE, 140321774092288, 140321782480895, ++SNULL, 140321463726080, 140321472114687, ++STORE, 140321472114688, 140321480507391, ++STORE, 140321463726080, 140321472114687, ++SNULL, 140321472118783, 140321480507391, ++STORE, 140321472114688, 140321472118783, ++STORE, 140321472118784, 140321480507391, ++SNULL, 140321757306880, 140321765695487, ++STORE, 140321765695488, 140321774088191, ++STORE, 140321757306880, 140321765695487, ++SNULL, 140321765699583, 140321774088191, ++STORE, 140321765695488, 140321765699583, ++STORE, 140321765699584, 140321774088191, ++STORE, 140321455329280, 140321463721983, ++SNULL, 140321455333375, 140321463721983, ++STORE, 140321455329280, 140321455333375, ++STORE, 140321455333376, 140321463721983, ++STORE, 140321446936576, 140321455329279, ++STORE, 140321438543872, 140321455329279, ++STORE, 140321430151168, 140321455329279, ++SNULL, 140321430155263, 140321455329279, ++STORE, 140321430151168, 140321430155263, ++STORE, 140321430155264, 140321455329279, ++SNULL, 140321430155264, 140321446936575, ++STORE, 140321446936576, 140321455329279, ++STORE, 140321430155264, 140321446936575, ++SNULL, 140321446940671, 140321455329279, ++STORE, 140321446936576, 140321446940671, ++STORE, 140321446940672, 140321455329279, ++SNULL, 140321430155264, 140321438543871, ++STORE, 140321438543872, 140321446936575, ++STORE, 140321430155264, 140321438543871, ++SNULL, 140321438547967, 140321446936575, ++STORE, 140321438543872, 140321438547967, ++STORE, 140321438547968, 140321446936575, ++STORE, 140321421758464, 140321430151167, ++SNULL, 140321421762559, 140321430151167, ++STORE, 140321421758464, 140321421762559, ++STORE, 140321421762560, 140321430151167, ++STORE, 140321413365760, 140321421758463, ++SNULL, 140321413369855, 140321421758463, ++STORE, 140321413365760, 140321413369855, ++STORE, 140321413369856, 140321421758463, ++STORE, 140321404973056, 140321413365759, ++SNULL, 140321404977151, 140321413365759, ++STORE, 140321404973056, 140321404977151, ++STORE, 140321404977152, 140321413365759, ++STORE, 140321396580352, 140321404973055, ++STORE, 140321388187648, 140321404973055, ++STORE, 140321253969920, 140321388187647, ++SNULL, 140321253969920, 140321279180799, ++STORE, 140321279180800, 140321388187647, ++STORE, 140321253969920, 140321279180799, ++ERASE, 140321253969920, 140321279180799, ++SNULL, 140321346289663, 140321388187647, ++STORE, 140321279180800, 140321346289663, ++STORE, 140321346289664, 140321388187647, ++ERASE, 140321346289664, 140321388187647, ++STORE, 140321144963072, 140321346289663, ++STORE, 140321379794944, 140321404973055, ++STORE, 140321371402240, 140321404973055, ++STORE, 140321010745344, 140321346289663, ++STORE, 140321363009536, 140321404973055, ++SNULL, 140321077854207, 140321346289663, ++STORE, 140321010745344, 140321077854207, ++STORE, 140321077854208, 140321346289663, ++SNULL, 140321077854208, 140321144963071, ++STORE, 140321144963072, 140321346289663, ++STORE, 140321077854208, 140321144963071, ++ERASE, 140321077854208, 140321144963071, ++STORE, 140321354616832, 140321404973055, ++STORE, 140321136570368, 140321144963071, ++STORE, 140320943636480, 140321077854207, ++STORE, 140320876527616, 140321077854207, ++STORE, 140321128177664, 140321144963071, ++SNULL, 140320876662783, 140321077854207, ++STORE, 140320876527616, 140320876662783, ++STORE, 140320876662784, 140321077854207, ++STORE, 140321119784960, 140321144963071, ++STORE, 140321111392256, 140321144963071, ++STORE, 140320742309888, 140320876527615, ++STORE, 140321102999552, 140321144963071, ++STORE, 140320608092160, 140320876527615, ++SNULL, 140320675201023, 140320876527615, ++STORE, 140320608092160, 140320675201023, ++STORE, 140320675201024, 140320876527615, ++SNULL, 140320675201024, 140320742309887, ++STORE, 140320742309888, 140320876527615, ++STORE, 140320675201024, 140320742309887, ++ERASE, 140320675201024, 140320742309887, ++STORE, 140321094606848, 140321144963071, ++STORE, 140321086214144, 140321144963071, ++STORE, 140320608092160, 140320876527615, ++SNULL, 140320608092160, 140320675201023, ++STORE, 140320675201024, 140320876527615, ++STORE, 140320608092160, 140320675201023, ++SNULL, 140320675336191, 140320876527615, ++STORE, 140320675201024, 140320675336191, ++STORE, 140320675336192, 140320876527615, ++STORE, 140320599699456, 140320608092159, ++STORE, 140320591306752, 140320608092159, ++STORE, 140320457089024, 140320591306751, ++STORE, 140320448696320, 140320457089023, ++STORE, 140320314478592, 140320448696319, ++SNULL, 140321144963072, 140321279180799, ++STORE, 140321279180800, 140321346289663, ++STORE, 140321144963072, 140321279180799, ++SNULL, 140321279315967, 140321346289663, ++STORE, 140321279180800, 140321279315967, ++STORE, 140321279315968, 140321346289663, ++SNULL, 140321086214144, 140321136570367, ++STORE, 140321136570368, 140321144963071, ++STORE, 140321086214144, 140321136570367, ++SNULL, 140321136574463, 140321144963071, ++STORE, 140321136570368, 140321136574463, ++STORE, 140321136574464, 140321144963071, ++SNULL, 140321212071935, 140321279180799, ++STORE, 140321144963072, 140321212071935, ++STORE, 140321212071936, 140321279180799, ++ERASE, 140321212071936, 140321279180799, ++SNULL, 140321145098239, 140321212071935, ++STORE, 140321144963072, 140321145098239, ++STORE, 140321145098240, 140321212071935, ++SNULL, 140320876662784, 140321010745343, ++STORE, 140321010745344, 140321077854207, ++STORE, 140320876662784, 140321010745343, ++SNULL, 140321010880511, 140321077854207, ++STORE, 140321010745344, 140321010880511, ++STORE, 140321010880512, 140321077854207, ++SNULL, 140321354616832, 140321379794943, ++STORE, 140321379794944, 140321404973055, ++STORE, 140321354616832, 140321379794943, ++SNULL, 140321379799039, 140321404973055, ++STORE, 140321379794944, 140321379799039, ++STORE, 140321379799040, 140321404973055, ++SNULL, 140320876662784, 140320943636479, ++STORE, 140320943636480, 140321010745343, ++STORE, 140320876662784, 140320943636479, ++SNULL, 140320943771647, 140321010745343, ++STORE, 140320943636480, 140320943771647, ++STORE, 140320943771648, 140321010745343, ++SNULL, 140320809418751, 140320876527615, ++STORE, 140320675336192, 140320809418751, ++STORE, 140320809418752, 140320876527615, ++ERASE, 140320809418752, 140320876527615, ++SNULL, 140320675336192, 140320742309887, ++STORE, 140320742309888, 140320809418751, ++STORE, 140320675336192, 140320742309887, ++SNULL, 140320742445055, 140320809418751, ++STORE, 140320742309888, 140320742445055, ++STORE, 140320742445056, 140320809418751, ++SNULL, 140320608227327, 140320675201023, ++STORE, 140320608092160, 140320608227327, ++STORE, 140320608227328, 140320675201023, ++SNULL, 140320457089024, 140320473874431, ++STORE, 140320473874432, 140320591306751, ++STORE, 140320457089024, 140320473874431, ++ERASE, 140320457089024, 140320473874431, ++SNULL, 140320540983295, 140320591306751, ++STORE, 140320473874432, 140320540983295, ++STORE, 140320540983296, 140320591306751, ++ERASE, 140320540983296, 140320591306751, ++SNULL, 140320314478592, 140320339656703, ++STORE, 140320339656704, 140320448696319, ++STORE, 140320314478592, 140320339656703, ++ERASE, 140320314478592, 140320339656703, ++SNULL, 140321086214144, 140321128177663, ++STORE, 140321128177664, 140321136570367, ++STORE, 140321086214144, 140321128177663, ++SNULL, 140321128181759, 140321136570367, ++STORE, 140321128177664, 140321128181759, ++STORE, 140321128181760, 140321136570367, ++SNULL, 140321354616832, 140321371402239, ++STORE, 140321371402240, 140321379794943, ++STORE, 140321354616832, 140321371402239, ++SNULL, 140321371406335, 140321379794943, ++STORE, 140321371402240, 140321371406335, ++STORE, 140321371406336, 140321379794943, ++SNULL, 140320591310847, 140320608092159, ++STORE, 140320591306752, 140320591310847, ++STORE, 140320591310848, 140320608092159, ++SNULL, 140321354616832, 140321363009535, ++STORE, 140321363009536, 140321371402239, ++STORE, 140321354616832, 140321363009535, ++SNULL, 140321363013631, 140321371402239, ++STORE, 140321363009536, 140321363013631, ++STORE, 140321363013632, 140321371402239, ++SNULL, 140321086214144, 140321119784959, ++STORE, 140321119784960, 140321128177663, ++STORE, 140321086214144, 140321119784959, ++SNULL, 140321119789055, 140321128177663, ++STORE, 140321119784960, 140321119789055, ++STORE, 140321119789056, 140321128177663, ++SNULL, 140321086218239, 140321119784959, ++STORE, 140321086214144, 140321086218239, ++STORE, 140321086218240, 140321119784959, ++SNULL, 140321086218240, 140321094606847, ++STORE, 140321094606848, 140321119784959, ++STORE, 140321086218240, 140321094606847, ++SNULL, 140321094610943, 140321119784959, ++STORE, 140321094606848, 140321094610943, ++STORE, 140321094610944, 140321119784959, ++SNULL, 140320474009599, 140320540983295, ++STORE, 140320473874432, 140320474009599, ++STORE, 140320474009600, 140320540983295, ++SNULL, 140320406765567, 140320448696319, ++STORE, 140320339656704, 140320406765567, ++STORE, 140320406765568, 140320448696319, ++ERASE, 140320406765568, 140320448696319, ++SNULL, 140320339791871, 140320406765567, ++STORE, 140320339656704, 140320339791871, ++STORE, 140320339791872, 140320406765567, ++STORE, 140321270788096, 140321279180799, ++STORE, 140321262395392, 140321279180799, ++STORE, 140321254002688, 140321279180799, ++SNULL, 140321254002688, 140321262395391, ++STORE, 140321262395392, 140321279180799, ++STORE, 140321254002688, 140321262395391, ++SNULL, 140321262399487, 140321279180799, ++STORE, 140321262395392, 140321262399487, ++STORE, 140321262399488, 140321279180799, ++STORE, 140321245609984, 140321262395391, ++STORE, 140321237217280, 140321262395391, ++SNULL, 140321237217280, 140321245609983, ++STORE, 140321245609984, 140321262395391, ++STORE, 140321237217280, 140321245609983, ++SNULL, 140321245614079, 140321262395391, ++STORE, 140321245609984, 140321245614079, ++STORE, 140321245614080, 140321262395391, ++SNULL, 140321379799040, 140321388187647, ++STORE, 140321388187648, 140321404973055, ++STORE, 140321379799040, 140321388187647, ++SNULL, 140321388191743, 140321404973055, ++STORE, 140321388187648, 140321388191743, ++STORE, 140321388191744, 140321404973055, ++SNULL, 140321354620927, 140321363009535, ++STORE, 140321354616832, 140321354620927, ++STORE, 140321354620928, 140321363009535, ++SNULL, 140321388191744, 140321396580351, ++STORE, 140321396580352, 140321404973055, ++STORE, 140321388191744, 140321396580351, ++SNULL, 140321396584447, 140321404973055, ++STORE, 140321396580352, 140321396584447, ++STORE, 140321396584448, 140321404973055, ++SNULL, 140321094610944, 140321111392255, ++STORE, 140321111392256, 140321119784959, ++STORE, 140321094610944, 140321111392255, ++SNULL, 140321111396351, 140321119784959, ++STORE, 140321111392256, 140321111396351, ++STORE, 140321111396352, 140321119784959, ++STORE, 140321228824576, 140321245609983, ++SNULL, 140321094610944, 140321102999551, ++STORE, 140321102999552, 140321111392255, ++STORE, 140321094610944, 140321102999551, ++SNULL, 140321103003647, 140321111392255, ++STORE, 140321102999552, 140321103003647, ++STORE, 140321103003648, 140321111392255, ++STORE, 140321220431872, 140321245609983, ++SNULL, 140321220435967, 140321245609983, ++STORE, 140321220431872, 140321220435967, ++STORE, 140321220435968, 140321245609983, ++STORE, 140320868134912, 140320876527615, ++SNULL, 140320868139007, 140320876527615, ++STORE, 140320868134912, 140320868139007, ++STORE, 140320868139008, 140320876527615, ++SNULL, 140320591310848, 140320599699455, ++STORE, 140320599699456, 140320608092159, ++STORE, 140320591310848, 140320599699455, ++SNULL, 140320599703551, 140320608092159, ++STORE, 140320599699456, 140320599703551, ++STORE, 140320599703552, 140320608092159, ++STORE, 140320859742208, 140320868134911, ++SNULL, 140321262399488, 140321270788095, ++STORE, 140321270788096, 140321279180799, ++STORE, 140321262399488, 140321270788095, ++SNULL, 140321270792191, 140321279180799, ++STORE, 140321270788096, 140321270792191, ++STORE, 140321270792192, 140321279180799, ++STORE, 140320851349504, 140320868134911, ++STORE, 140320842956800, 140320868134911, ++STORE, 140320834564096, 140320868134911, ++STORE, 140320826171392, 140320868134911, ++SNULL, 140320826171392, 140320834564095, ++STORE, 140320834564096, 140320868134911, ++STORE, 140320826171392, 140320834564095, ++SNULL, 140320834568191, 140320868134911, ++STORE, 140320834564096, 140320834568191, ++STORE, 140320834568192, 140320868134911, ++SNULL, 140321220435968, 140321228824575, ++STORE, 140321228824576, 140321245609983, ++STORE, 140321220435968, 140321228824575, ++SNULL, 140321228828671, 140321245609983, ++STORE, 140321228824576, 140321228828671, ++STORE, 140321228828672, 140321245609983, ++STORE, 140320817778688, 140320834564095, ++SNULL, 140320817782783, 140320834564095, ++STORE, 140320817778688, 140320817782783, ++STORE, 140320817782784, 140320834564095, ++STORE, 140320582914048, 140320591306751, ++SNULL, 140321228828672, 140321237217279, ++STORE, 140321237217280, 140321245609983, ++STORE, 140321228828672, 140321237217279, ++SNULL, 140321237221375, 140321245609983, ++STORE, 140321237217280, 140321237221375, ++STORE, 140321237221376, 140321245609983, ++SNULL, 140320448700415, 140320457089023, ++STORE, 140320448696320, 140320448700415, ++STORE, 140320448700416, 140320457089023, ++SNULL, 140321245614080, 140321254002687, ++STORE, 140321254002688, 140321262395391, ++STORE, 140321245614080, 140321254002687, ++SNULL, 140321254006783, 140321262395391, ++STORE, 140321254002688, 140321254006783, ++STORE, 140321254006784, 140321262395391, ++STORE, 140320574521344, 140320591306751, ++SNULL, 140320574525439, 140320591306751, ++STORE, 140320574521344, 140320574525439, ++STORE, 140320574525440, 140320591306751, ++STORE, 140320566128640, 140320574521343, ++SNULL, 140320566132735, 140320574521343, ++STORE, 140320566128640, 140320566132735, ++STORE, 140320566132736, 140320574521343, ++SNULL, 140320574525440, 140320582914047, ++STORE, 140320582914048, 140320591306751, ++STORE, 140320574525440, 140320582914047, ++SNULL, 140320582918143, 140320591306751, ++STORE, 140320582914048, 140320582918143, ++STORE, 140320582918144, 140320591306751, ++STORE, 140320557735936, 140320566128639, ++SNULL, 140320557740031, 140320566128639, ++STORE, 140320557735936, 140320557740031, ++STORE, 140320557740032, 140320566128639, ++STORE, 140320549343232, 140320557735935, ++STORE, 140320465481728, 140320473874431, ++STORE, 140320448700416, 140320473874431, ++SNULL, 140320834568192, 140320859742207, ++STORE, 140320859742208, 140320868134911, ++STORE, 140320834568192, 140320859742207, ++SNULL, 140320859746303, 140320868134911, ++STORE, 140320859742208, 140320859746303, ++STORE, 140320859746304, 140320868134911, ++STORE, 140320440303616, 140320448696319, ++STORE, 140320431910912, 140320448696319, ++SNULL, 140320834568192, 140320851349503, ++STORE, 140320851349504, 140320859742207, ++STORE, 140320834568192, 140320851349503, ++SNULL, 140320851353599, 140320859742207, ++STORE, 140320851349504, 140320851353599, ++STORE, 140320851353600, 140320859742207, ++SNULL, 140320817782784, 140320826171391, ++STORE, 140320826171392, 140320834564095, ++STORE, 140320817782784, 140320826171391, ++SNULL, 140320826175487, 140320834564095, ++STORE, 140320826171392, 140320826175487, ++STORE, 140320826175488, 140320834564095, ++SNULL, 140320834568192, 140320842956799, ++STORE, 140320842956800, 140320851349503, ++STORE, 140320834568192, 140320842956799, ++SNULL, 140320842960895, 140320851349503, ++STORE, 140320842956800, 140320842960895, ++STORE, 140320842960896, 140320851349503, ++STORE, 140320423518208, 140320448696319, ++SNULL, 140320423522303, 140320448696319, ++STORE, 140320423518208, 140320423522303, ++STORE, 140320423522304, 140320448696319, ++STORE, 140320415125504, 140320423518207, ++STORE, 140320331264000, 140320339656703, ++STORE, 140320322871296, 140320339656703, ++STORE, 140320314478592, 140320339656703, ++SNULL, 140320314482687, 140320339656703, ++STORE, 140320314478592, 140320314482687, ++STORE, 140320314482688, 140320339656703, ++STORE, 140320306085888, 140320314478591, ++SNULL, 140320306089983, 140320314478591, ++STORE, 140320306085888, 140320306089983, ++STORE, 140320306089984, 140320314478591, ++STORE, 140320297693184, 140320306085887, ++SNULL, 140320297697279, 140320306085887, ++STORE, 140320297693184, 140320297697279, ++STORE, 140320297697280, 140320306085887, ++STORE, 140320289300480, 140320297693183, ++STORE, 140320280907776, 140320297693183, ++SNULL, 140320280911871, 140320297693183, ++STORE, 140320280907776, 140320280911871, ++STORE, 140320280911872, 140320297693183, ++SNULL, 140320423522304, 140320431910911, ++STORE, 140320431910912, 140320448696319, ++STORE, 140320423522304, 140320431910911, ++SNULL, 140320431915007, 140320448696319, ++STORE, 140320431910912, 140320431915007, ++STORE, 140320431915008, 140320448696319, ++SNULL, 140320549347327, 140320557735935, ++STORE, 140320549343232, 140320549347327, ++STORE, 140320549347328, 140320557735935, ++STORE, 140320272515072, 140320280907775, ++SNULL, 140320448700416, 140320457089023, ++STORE, 140320457089024, 140320473874431, ++STORE, 140320448700416, 140320457089023, ++SNULL, 140320457093119, 140320473874431, ++STORE, 140320457089024, 140320457093119, ++STORE, 140320457093120, 140320473874431, ++STORE, 140320264122368, 140320280907775, ++SNULL, 140320457093120, 140320465481727, ++STORE, 140320465481728, 140320473874431, ++STORE, 140320457093120, 140320465481727, ++SNULL, 140320465485823, 140320473874431, ++STORE, 140320465481728, 140320465485823, ++STORE, 140320465485824, 140320473874431, ++SNULL, 140320431915008, 140320440303615, ++STORE, 140320440303616, 140320448696319, ++STORE, 140320431915008, 140320440303615, ++SNULL, 140320440307711, 140320448696319, ++STORE, 140320440303616, 140320440307711, ++STORE, 140320440307712, 140320448696319, ++STORE, 140320255729664, 140320280907775, ++STORE, 140320247336960, 140320280907775, ++SNULL, 140320247341055, 140320280907775, ++STORE, 140320247336960, 140320247341055, ++STORE, 140320247341056, 140320280907775, ++STORE, 140320238944256, 140320247336959, ++STORE, 140320230551552, 140320247336959, ++SNULL, 140320230551552, 140320238944255, ++STORE, 140320238944256, 140320247336959, ++STORE, 140320230551552, 140320238944255, ++SNULL, 140320238948351, 140320247336959, ++STORE, 140320238944256, 140320238948351, ++STORE, 140320238948352, 140320247336959, ++SNULL, 140320314482688, 140320331263999, ++STORE, 140320331264000, 140320339656703, ++STORE, 140320314482688, 140320331263999, ++SNULL, 140320331268095, 140320339656703, ++STORE, 140320331264000, 140320331268095, ++STORE, 140320331268096, 140320339656703, ++SNULL, 140320280911872, 140320289300479, ++STORE, 140320289300480, 140320297693183, ++STORE, 140320280911872, 140320289300479, ++SNULL, 140320289304575, 140320297693183, ++STORE, 140320289300480, 140320289304575, ++STORE, 140320289304576, 140320297693183, ++SNULL, 140320415129599, 140320423518207, ++STORE, 140320415125504, 140320415129599, ++STORE, 140320415129600, 140320423518207, ++STORE, 140320222158848, 140320238944255, ++STORE, 140320213766144, 140320238944255, ++STORE, 140320205373440, 140320238944255, ++SNULL, 140320205377535, 140320238944255, ++STORE, 140320205373440, 140320205377535, ++STORE, 140320205377536, 140320238944255, ++SNULL, 140320314482688, 140320322871295, ++STORE, 140320322871296, 140320331263999, ++STORE, 140320314482688, 140320322871295, ++SNULL, 140320322875391, 140320331263999, ++STORE, 140320322871296, 140320322875391, ++STORE, 140320322875392, 140320331263999, ++SNULL, 140320247341056, 140320272515071, ++STORE, 140320272515072, 140320280907775, ++STORE, 140320247341056, 140320272515071, ++SNULL, 140320272519167, 140320280907775, ++STORE, 140320272515072, 140320272519167, ++STORE, 140320272519168, 140320280907775, ++SNULL, 140320247341056, 140320264122367, ++STORE, 140320264122368, 140320272515071, ++STORE, 140320247341056, 140320264122367, ++SNULL, 140320264126463, 140320272515071, ++STORE, 140320264122368, 140320264126463, ++STORE, 140320264126464, 140320272515071, ++SNULL, 140320205377536, 140320230551551, ++STORE, 140320230551552, 140320238944255, ++STORE, 140320205377536, 140320230551551, ++SNULL, 140320230555647, 140320238944255, ++STORE, 140320230551552, 140320230555647, ++STORE, 140320230555648, 140320238944255, ++STORE, 140320196980736, 140320205373439, ++SNULL, 140320196984831, 140320205373439, ++STORE, 140320196980736, 140320196984831, ++STORE, 140320196984832, 140320205373439, ++STORE, 140320188588032, 140320196980735, ++SNULL, 140320247341056, 140320255729663, ++STORE, 140320255729664, 140320264122367, ++STORE, 140320247341056, 140320255729663, ++SNULL, 140320255733759, 140320264122367, ++STORE, 140320255729664, 140320255733759, ++STORE, 140320255733760, 140320264122367, ++STORE, 140320180195328, 140320196980735, ++SNULL, 140320180199423, 140320196980735, ++STORE, 140320180195328, 140320180199423, ++STORE, 140320180199424, 140320196980735, ++STORE, 140320171802624, 140320180195327, ++STORE, 140320163409920, 140320180195327, ++SNULL, 140320163414015, 140320180195327, ++STORE, 140320163409920, 140320163414015, ++STORE, 140320163414016, 140320180195327, ++SNULL, 140320205377536, 140320222158847, ++STORE, 140320222158848, 140320230551551, ++STORE, 140320205377536, 140320222158847, ++SNULL, 140320222162943, 140320230551551, ++STORE, 140320222158848, 140320222162943, ++STORE, 140320222162944, 140320230551551, ++SNULL, 140320205377536, 140320213766143, ++STORE, 140320213766144, 140320222158847, ++STORE, 140320205377536, 140320213766143, ++SNULL, 140320213770239, 140320222158847, ++STORE, 140320213766144, 140320213770239, ++STORE, 140320213770240, 140320222158847, ++STORE, 140320155017216, 140320163409919, ++SNULL, 140320180199424, 140320188588031, ++STORE, 140320188588032, 140320196980735, ++STORE, 140320180199424, 140320188588031, ++SNULL, 140320188592127, 140320196980735, ++STORE, 140320188588032, 140320188592127, ++STORE, 140320188592128, 140320196980735, ++SNULL, 140320155021311, 140320163409919, ++STORE, 140320155017216, 140320155021311, ++STORE, 140320155021312, 140320163409919, ++SNULL, 140320163414016, 140320171802623, ++STORE, 140320171802624, 140320180195327, ++STORE, 140320163414016, 140320171802623, ++SNULL, 140320171806719, 140320180195327, ++STORE, 140320171802624, 140320171806719, ++STORE, 140320171806720, 140320180195327, ++STORE, 140320146624512, 140320155017215, ++SNULL, 140320146628607, 140320155017215, ++STORE, 140320146624512, 140320146628607, ++STORE, 140320146628608, 140320155017215, ++STORE, 140321937321984, 140321937350655, ++STORE, 140321884942336, 140321887133695, ++SNULL, 140321884942336, 140321885032447, ++STORE, 140321885032448, 140321887133695, ++STORE, 140321884942336, 140321885032447, ++SNULL, 140321887125503, 140321887133695, ++STORE, 140321885032448, 140321887125503, ++STORE, 140321887125504, 140321887133695, ++ERASE, 140321887125504, 140321887133695, ++STORE, 140321887125504, 140321887133695, ++SNULL, 140321887129599, 140321887133695, ++STORE, 140321887125504, 140321887129599, ++STORE, 140321887129600, 140321887133695, ++ERASE, 140321937321984, 140321937350655, ++ERASE, 140321086214144, 140321086218239, ++ERASE, 140321086218240, 140321094606847, ++ERASE, 140321119784960, 140321119789055, ++ERASE, 140321119789056, 140321128177663, ++ERASE, 140321245609984, 140321245614079, ++ERASE, 140321245614080, 140321254002687, ++ERASE, 140320574521344, 140320574525439, ++ERASE, 140320574525440, 140320582914047, ++ERASE, 140320297693184, 140320297697279, ++ERASE, 140320297697280, 140320306085887, ++ERASE, 140321354616832, 140321354620927, ++ERASE, 140321354620928, 140321363009535, ++ERASE, 140320834564096, 140320834568191, ++ERASE, 140320834568192, 140320842956799, ++ERASE, 140320591306752, 140320591310847, ++ERASE, 140320591310848, 140320599699455, ++ERASE, 140321136570368, 140321136574463, ++ERASE, 140321136574464, 140321144963071, ++ERASE, 140321237217280, 140321237221375, ++ERASE, 140321237221376, 140321245609983, ++ERASE, 140321363009536, 140321363013631, ++ERASE, 140321363013632, 140321371402239, ++ERASE, 140320599699456, 140320599703551, ++ERASE, 140320599703552, 140320608092159, ++ERASE, 140321396580352, 140321396584447, ++ERASE, 140321396584448, 140321404973055, ++ERASE, 140320566128640, 140320566132735, ++ERASE, 140320566132736, 140320574521343, ++ERASE, 140321094606848, 140321094610943, ++ERASE, 140321094610944, 140321102999551, ++ERASE, 140320582914048, 140320582918143, ++ERASE, 140320582918144, 140320591306751, ++ERASE, 140320289300480, 140320289304575, ++ERASE, 140320289304576, 140320297693183, ++ERASE, 140320163409920, 140320163414015, ++ }; ++ unsigned long set41[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140728157171712, 140737488351231, ++SNULL, 140728157175807, 140737488351231, ++STORE, 140728157171712, 140728157175807, ++STORE, 140728157040640, 140728157175807, ++STORE, 94376106364928, 94376108613631, ++SNULL, 94376106487807, 94376108613631, ++STORE, 94376106364928, 94376106487807, ++STORE, 94376106487808, 94376108613631, ++SNULL, 94376106487808, 94376108613631, ++STORE, 94376108584960, 94376108593151, ++STORE, 94376108593152, 94376108613631, ++STORE, 140113496432640, 140113498685439, ++SNULL, 140113496575999, 140113498685439, ++STORE, 140113496432640, 140113496575999, ++STORE, 140113496576000, 140113498685439, ++SNULL, 140113496576000, 140113498685439, ++STORE, 140113498673152, 140113498681343, ++STORE, 140113498681344, 140113498685439, ++STORE, 140728157609984, 140728157618175, ++STORE, 140728157593600, 140728157609983, ++STORE, 140113498636288, 140113498673151, ++STORE, 140113498628096, 140113498636287, ++STORE, 140113492635648, 140113496432639, ++SNULL, 140113492635648, 140113494294527, ++STORE, 140113494294528, 140113496432639, ++STORE, 140113492635648, 140113494294527, ++SNULL, 140113496391679, 140113496432639, ++STORE, 140113494294528, 140113496391679, ++STORE, 140113496391680, 140113496432639, ++SNULL, 140113496391680, 140113496416255, ++STORE, 140113496416256, 140113496432639, ++STORE, 140113496391680, 140113496416255, ++SNULL, 140113496391680, 140113496416255, ++STORE, 140113496391680, 140113496416255, ++SNULL, 140113496416256, 140113496432639, ++STORE, 140113496416256, 140113496432639, ++SNULL, 140113496408063, 140113496416255, ++STORE, 140113496391680, 140113496408063, ++STORE, 140113496408064, 140113496416255, ++SNULL, 94376108589055, 94376108593151, ++STORE, 94376108584960, 94376108589055, ++STORE, 94376108589056, 94376108593151, ++SNULL, 140113498677247, 140113498681343, ++STORE, 140113498673152, 140113498677247, ++STORE, 140113498677248, 140113498681343, ++SNULL, 140113498636288, 140113498673151, ++STORE, 94376135090176, 94376135094271, ++STORE, 94376135090176, 94376135098367, ++STORE, 94376139288576, 94376139292671, ++STORE, 94376143482880, 94376143486975, ++STORE, 94376147677184, 94376147681279, ++STORE, 94376151871488, 94376151875583, ++STORE, 94376156065792, 94376156069887, ++STORE, 94376160260096, 94376160264191, ++STORE, 94376164454400, 94376164458495, ++STORE, 94376168648704, 94376168652799, ++STORE, 94376172843008, 94376172847103, ++STORE, 94376177037312, 94376177041407, ++STORE, 94376181231616, 94376181235711, ++STORE, 94376185425920, 94376185430015, ++STORE, 94376189620224, 94376189624319, ++STORE, 94376193814528, 94376193818623, ++STORE, 94376198008832, 94376198012927, ++STORE, 94376202203136, 94376202207231, ++STORE, 94376206397440, 94376206401535, ++STORE, 94376210591744, 94376210595839, ++STORE, 94376214786048, 94376214790143, ++STORE, 94376218980352, 94376218984447, ++STORE, 94376223174656, 94376223178751, ++STORE, 94376227368960, 94376227373055, ++STORE, 94376231563264, 94376231567359, ++STORE, 94376235757568, 94376235761663, ++STORE, 94376239951872, 94376239955967, ++STORE, 94376244146176, 94376244150271, ++STORE, 94376248340480, 94376248344575, ++STORE, 94376252534784, 94376252538879, ++STORE, 94376256729088, 94376256733183, ++STORE, 94376260923392, 94376260927487, ++STORE, 94376265117696, 94376265121791, ++STORE, 94376269312000, 94376269316095, ++STORE, 94376273506304, 94376273510399, ++STORE, 94376277700608, 94376277704703, ++STORE, 94376281894912, 94376281899007, ++STORE, 94376286089216, 94376286093311, ++STORE, 94376290283520, 94376290287615, ++STORE, 94376294477824, 94376294481919, ++STORE, 94376298672128, 94376298676223, ++STORE, 94376302866432, 94376302870527, ++STORE, 94376307060736, 94376307064831, ++STORE, 94376311255040, 94376311259135, ++STORE, 94376315449344, 94376315453439, ++STORE, 94376319643648, 94376319647743, ++STORE, 94376323837952, 94376323842047, ++STORE, 94376328032256, 94376328036351, ++STORE, 94376332226560, 94376332230655, ++STORE, 94376336420864, 94376336424959, ++STORE, 94376340615168, 94376340619263, ++STORE, 94376344809472, 94376344813567, ++STORE, 94376349003776, 94376349007871, ++STORE, 94376353198080, 94376353202175, ++STORE, 94376357392384, 94376357396479, ++STORE, 94376361586688, 94376361590783, ++STORE, 94376365780992, 94376365785087, ++STORE, 94376369975296, 94376369979391, ++STORE, 94376374169600, 94376374173695, ++STORE, 94376378363904, 94376378367999, ++STORE, 94376382558208, 94376382562303, ++STORE, 94376386752512, 94376386756607, ++STORE, 94376390946816, 94376390950911, ++STORE, 94376395141120, 94376395145215, ++STORE, 94376399335424, 94376399339519, ++STORE, 94376403529728, 94376403533823, ++STORE, 94376407724032, 94376407728127, ++STORE, 94376411918336, 94376411922431, ++STORE, 94376416112640, 94376416116735, ++STORE, 94376420306944, 94376420311039, ++STORE, 94376424501248, 94376424505343, ++STORE, 94376428695552, 94376428699647, ++STORE, 94376432889856, 94376432893951, ++STORE, 94376437084160, 94376437088255, ++STORE, 94376441278464, 94376441282559, ++STORE, 94376445472768, 94376445476863, ++STORE, 94376449667072, 94376449671167, ++STORE, 94376453861376, 94376453865471, ++STORE, 94376458055680, 94376458059775, ++STORE, 94376462249984, 94376462254079, ++STORE, 94376466444288, 94376466448383, ++STORE, 94376470638592, 94376470642687, ++STORE, 94376474832896, 94376474836991, ++STORE, 94376479027200, 94376479031295, ++STORE, 94376483221504, 94376483225599, ++STORE, 94376487415808, 94376487419903, ++STORE, 94376491610112, 94376491614207, ++STORE, 94376495804416, 94376495808511, ++STORE, 94376499998720, 94376500002815, ++STORE, 94376504193024, 94376504197119, ++STORE, 94376508387328, 94376508391423, ++STORE, 94376512581632, 94376512585727, ++STORE, 94376516775936, 94376516780031, ++STORE, 94376520970240, 94376520974335, ++STORE, 94376525164544, 94376525168639, ++STORE, 94376529358848, 94376529362943, ++STORE, 94376533553152, 94376533557247, ++STORE, 94376537747456, 94376537751551, ++STORE, 94376541941760, 94376541945855, ++STORE, 94376546136064, 94376546140159, ++STORE, 94376550330368, 94376550334463, ++STORE, 94376554524672, 94376554528767, ++STORE, 94376558718976, 94376558723071, ++STORE, 94376562913280, 94376562917375, ++STORE, 94376567107584, 94376567111679, ++STORE, 94376571301888, 94376571305983, ++STORE, 94376575496192, 94376575500287, ++STORE, 94376579690496, 94376579694591, ++STORE, 94376583884800, 94376583888895, ++STORE, 94376588079104, 94376588083199, ++STORE, 94376592273408, 94376592277503, ++STORE, 94376596467712, 94376596471807, ++STORE, 94376600662016, 94376600666111, ++STORE, 94376604856320, 94376604860415, ++STORE, 94376609050624, 94376609054719, ++STORE, 94376613244928, 94376613249023, ++STORE, 94376617439232, 94376617443327, ++STORE, 94376621633536, 94376621637631, ++STORE, 94376625827840, 94376625831935, ++STORE, 94376630022144, 94376630026239, ++STORE, 94376634216448, 94376634220543, ++STORE, 94376638410752, 94376638414847, ++STORE, 94376642605056, 94376642609151, ++STORE, 94376646799360, 94376646803455, ++STORE, 94376650993664, 94376650997759, ++STORE, 94376655187968, 94376655192063, ++STORE, 94376659382272, 94376659386367, ++STORE, 94376663576576, 94376663580671, ++STORE, 94376667770880, 94376667774975, ++STORE, 94376671965184, 94376671969279, ++STORE, 94376676159488, 94376676163583, ++STORE, 94376680353792, 94376680357887, ++STORE, 94376684548096, 94376684552191, ++STORE, 94376688742400, 94376688746495, ++STORE, 94376692936704, 94376692940799, ++STORE, 94376697131008, 94376697135103, ++STORE, 94376701325312, 94376701329407, ++STORE, 94376705519616, 94376705523711, ++STORE, 94376709713920, 94376709718015, ++STORE, 94376713908224, 94376713912319, ++STORE, 94376718102528, 94376718106623, ++STORE, 94376722296832, 94376722300927, ++STORE, 94376726491136, 94376726495231, ++STORE, 94376730685440, 94376730689535, ++STORE, 94376734879744, 94376734883839, ++STORE, 94376739074048, 94376739078143, ++STORE, 94376743268352, 94376743272447, ++STORE, 94376747462656, 94376747466751, ++STORE, 94376751656960, 94376751661055, ++STORE, 94376755851264, 94376755855359, ++STORE, 94376760045568, 94376760049663, ++STORE, 94376764239872, 94376764243967, ++STORE, 94376768434176, 94376768438271, ++STORE, 94376772628480, 94376772632575, ++STORE, 94376776822784, 94376776826879, ++STORE, 94376781017088, 94376781021183, ++STORE, 94376785211392, 94376785215487, ++STORE, 94376789405696, 94376789409791, ++STORE, 94376793600000, 94376793604095, ++STORE, 94376797794304, 94376797798399, ++STORE, 94376801988608, 94376801992703, ++STORE, 94376806182912, 94376806187007, ++STORE, 94376810377216, 94376810381311, ++STORE, 94376814571520, 94376814575615, ++STORE, 94376818765824, 94376818769919, ++STORE, 94376822960128, 94376822964223, ++STORE, 94376827154432, 94376827158527, ++STORE, 94376831348736, 94376831352831, ++STORE, 94376835543040, 94376835547135, ++STORE, 94376839737344, 94376839741439, ++STORE, 94376843931648, 94376843935743, ++STORE, 94376848125952, 94376848130047, ++STORE, 94376852320256, 94376852324351, ++STORE, 94376856514560, 94376856518655, ++STORE, 94376860708864, 94376860712959, ++STORE, 94376864903168, 94376864907263, ++STORE, 94376869097472, 94376869101567, ++STORE, 94376873291776, 94376873295871, ++STORE, 94376877486080, 94376877490175, ++STORE, 94376881680384, 94376881684479, ++STORE, 94376885874688, 94376885878783, ++STORE, 94376890068992, 94376890073087, ++STORE, 94376894263296, 94376894267391, ++STORE, 94376898457600, 94376898461695, ++STORE, 94376902651904, 94376902655999, ++STORE, 94376906846208, 94376906850303, ++STORE, 94376911040512, 94376911044607, ++STORE, 94376915234816, 94376915238911, ++STORE, 94376919429120, 94376919433215, ++STORE, 94376923623424, 94376923627519, ++STORE, 94376927817728, 94376927821823, ++STORE, 94376932012032, 94376932016127, ++STORE, 94376936206336, 94376936210431, ++STORE, 94376940400640, 94376940404735, ++STORE, 94376944594944, 94376944599039, ++STORE, 94376948789248, 94376948793343, ++STORE, 94376952983552, 94376952987647, ++STORE, 94376957177856, 94376957181951, ++STORE, 94376961372160, 94376961376255, ++STORE, 94376965566464, 94376965570559, ++STORE, 94376969760768, 94376969764863, ++STORE, 94376973955072, 94376973959167, ++STORE, 94376978149376, 94376978153471, ++STORE, 94376982343680, 94376982347775, ++STORE, 94376986537984, 94376986542079, ++STORE, 94376990732288, 94376990736383, ++STORE, 94376994926592, 94376994930687, ++STORE, 94376999120896, 94376999124991, ++STORE, 94377003315200, 94377003319295, ++STORE, 94377007509504, 94377007513599, ++STORE, 94377011703808, 94377011707903, ++STORE, 94377015898112, 94377015902207, ++STORE, 94377020092416, 94377020096511, ++STORE, 94377024286720, 94377024290815, ++STORE, 94377028481024, 94377028485119, ++STORE, 94377032675328, 94377032679423, ++STORE, 94377036869632, 94377036873727, ++STORE, 94377041063936, 94377041068031, ++STORE, 94377045258240, 94377045262335, ++STORE, 94377049452544, 94377049456639, ++STORE, 94377053646848, 94377053650943, ++STORE, 94377057841152, 94377057845247, ++STORE, 94377062035456, 94377062039551, ++STORE, 94377066229760, 94377066233855, ++STORE, 94377070424064, 94377070428159, ++STORE, 94377074618368, 94377074622463, ++STORE, 94377078812672, 94377078816767, ++STORE, 94377083006976, 94377083011071, ++STORE, 94377087201280, 94377087205375, ++STORE, 94377091395584, 94377091399679, ++STORE, 94377095589888, 94377095593983, ++STORE, 94377099784192, 94377099788287, ++STORE, 94377103978496, 94377103982591, ++STORE, 94377108172800, 94377108176895, ++STORE, 94377112367104, 94377112371199, ++STORE, 94377116561408, 94377116565503, ++STORE, 94377120755712, 94377120759807, ++STORE, 94377124950016, 94377124954111, ++STORE, 94377129144320, 94377129148415, ++STORE, 94377133338624, 94377133342719, ++STORE, 94377137532928, 94377137537023, ++STORE, 94377141727232, 94377141731327, ++STORE, 94377145921536, 94377145925631, ++STORE, 94377150115840, 94377150119935, ++STORE, 94377154310144, 94377154314239, ++STORE, 94377158504448, 94377158508543, ++STORE, 94377162698752, 94377162702847, ++STORE, 94377166893056, 94377166897151, ++STORE, 94377171087360, 94377171091455, ++STORE, 94377175281664, 94377175285759, ++STORE, 94377179475968, 94377179480063, ++STORE, 94377183670272, 94377183674367, ++STORE, 94377187864576, 94377187868671, ++STORE, 94377192058880, 94377192062975, ++STORE, 94377196253184, 94377196257279, ++STORE, 94377200447488, 94377200451583, ++STORE, 94377204641792, 94377204645887, ++SNULL, 94376135094271, 94376135098367, ++STORE, 94376135090176, 94376135094271, ++STORE, 94376135094272, 94376135098367, ++SNULL, 94376135094272, 94377208836095, ++ }; ++ unsigned long set42[] = { ++STORE, 314572800, 1388314623, ++STORE, 1462157312, 1462169599, ++STORE, 1462169600, 1462185983, ++STORE, 1462185984, 1462190079, ++STORE, 1462190080, 1462194175, ++STORE, 1462194176, 1462198271, ++STORE, 1879986176, 1881800703, ++STORE, 1881800704, 1882034175, ++STORE, 1882034176, 1882193919, ++STORE, 1882193920, 1882406911, ++STORE, 1882406912, 1882451967, ++STORE, 1882451968, 1882996735, ++STORE, 1882996736, 1885892607, ++STORE, 1885892608, 1885896703, ++STORE, 1885896704, 1885904895, ++STORE, 1885904896, 1885908991, ++STORE, 1885908992, 1885913087, ++STORE, 1885913088, 1885966335, ++STORE, 1885966336, 1886232575, ++STORE, 1886232576, 1886236671, ++STORE, 1886236672, 1886240767, ++STORE, 1886240768, 1886244863, ++STORE, 1886244864, 1886248959, ++STORE, 1886248960, 1886294015, ++STORE, 1886294016, 1886494719, ++STORE, 1886494720, 1886498815, ++STORE, 1886498816, 1886502911, ++STORE, 1886502912, 1886507007, ++STORE, 1886507008, 1886511103, ++STORE, 1886511104, 1886556159, ++STORE, 1886556160, 1886629887, ++STORE, 1886629888, 1886633983, ++STORE, 1886633984, 1886638079, ++STORE, 1886638080, 1886642175, ++STORE, 1886642176, 1886646271, ++STORE, 1886646272, 1886666751, ++STORE, 1886666752, 1886670847, ++STORE, 1886670848, 1886674943, ++STORE, 1886674944, 1886679039, ++STORE, 1886679040, 1895419903, ++STORE, 1895419904, 1895550975, ++STORE, 1895550976, 1896148991, ++STORE, 1896148992, 1897189375, ++STORE, 1897189376, 1897701375, ++STORE, 1897701376, 1897803775, ++STORE, 1897803776, 1897816063, ++STORE, 1897816064, 1899913215, ++STORE, 1899913216, 1909379071, ++STORE, 1909379072, 1909387263, ++STORE, 1909387264, 1909391359, ++STORE, 1909391360, 1909432319, ++STORE, 1909432320, 1909436415, ++STORE, 1909436416, 1909440511, ++STORE, 1909440512, 1909460991, ++STORE, 1909460992, 1909547007, ++STORE, 1909547008, 1909551103, ++STORE, 1909551104, 1909555199, ++STORE, 1909555200, 1909559295, ++STORE, 1909559296, 1909563391, ++STORE, 1909563392, 1909739519, ++STORE, 1909739520, 1910566911, ++STORE, 1910566912, 1910571007, ++STORE, 1910571008, 1910575103, ++STORE, 1910575104, 1910579199, ++STORE, 1910579200, 1910583295, ++STORE, 1910583296, 1910587391, ++STORE, 1910587392, 1910620159, ++STORE, 1910620160, 1910624255, ++STORE, 1910624256, 1910628351, ++STORE, 1910628352, 1910632447, ++STORE, 1910632448, 1910652927, ++STORE, 1910652928, 1910657023, ++STORE, 1910657024, 1910661119, ++STORE, 1910661120, 1910665215, ++STORE, 1910665216, 1910669311, ++STORE, 1910669312, 1910677503, ++STORE, 1910677504, 1910681599, ++STORE, 1910681600, 1910685695, ++STORE, 1910685696, 1910689791, ++STORE, 1910689792, 1910697983, ++STORE, 1910697984, 1910702079, ++STORE, 1910702080, 1910706175, ++STORE, 1910706176, 1910710271, ++STORE, 1910710272, 1914093567, ++STORE, 1914093568, 1914097663, ++STORE, 1914097664, 1969434623, ++STORE, 1969434624, 1977819135, ++STORE, 3290435584, 3426750463, ++STORE, 3426750464, 3426754559, ++STORE, 3426754560, 3426762751, ++STORE, 3426762752, 3426766847, ++STORE, 3426766848, 3426770943, ++STORE, 3427037184, 3427061759, ++STORE, 3427061760, 3427135487, ++STORE, 3427135488, 3427143679, ++STORE, 3427143680, 3427147775, ++STORE, 3427147776, 3427209215, ++STORE, 3427319808, 3432116223, ++STORE, 3432116224, 3450130431, ++STORE, 3450130432, 3451027455, ++STORE, 3451027456, 3451031551, ++STORE, 3451031552, 3451461631, ++STORE, 3451736064, 3456688127, ++STORE, 3456688128, 3475222527, ++STORE, 3475222528, 3476119551, ++STORE, 3476119552, 3476127743, ++STORE, 3476127744, 3476553727, ++STORE, 3476631552, 3477315583, ++STORE, 3477315584, 3479949311, ++STORE, 3479949312, 3480002559, ++STORE, 3480002560, 3480006655, ++STORE, 3480006656, 3480432639, ++STORE, 3480539136, 3480543231, ++STORE, 3480543232, 3480547327, ++STORE, 3480547328, 3480555519, ++STORE, 3480854528, 3480903679, ++STORE, 3480903680, 3480969215, ++STORE, 3480969216, 3480977407, ++STORE, 3480977408, 3480981503, ++STORE, 3481030656, 3481092095, ++STORE, 3481092096, 3481235455, ++STORE, 3481235456, 3481243647, ++STORE, 3481243648, 3481247743, ++STORE, 3481436160, 3481444351, ++STORE, 3481444352, 3481456639, ++STORE, 3481456640, 3481460735, ++STORE, 3481460736, 3481464831, ++STORE, 3481587712, 3481645055, ++STORE, 3481645056, 3481772031, ++STORE, 3481772032, 3481776127, ++STORE, 3481776128, 3481780223, ++STORE, 3481874432, 3481935871, ++STORE, 3481935872, 3482030079, ++STORE, 3482030080, 3482038271, ++STORE, 3482038272, 3482042367, ++STORE, 3482198016, 3482230783, ++STORE, 3482230784, 3482271743, ++STORE, 3482271744, 3482279935, ++STORE, 3482279936, 3482284031, ++STORE, 3482562560, 3482566655, ++STORE, 3482566656, 3482570751, ++STORE, 3482570752, 3482574847, ++STORE, 3482636288, 3482689535, ++STORE, 3482689536, 3482746879, ++STORE, 3482746880, 3482755071, ++STORE, 3482755072, 3482759167, ++STORE, 3482972160, 3483062271, ++STORE, 3483062272, 3483242495, ++STORE, 3483242496, 3483246591, ++STORE, 3483246592, 3483250687, ++STORE, 3483398144, 3483688959, ++STORE, 3483688960, 3484114943, ++STORE, 3484114944, 3484131327, ++STORE, 3484131328, 3484135423, ++STORE, 3484135424, 3484143615, ++STORE, 3484184576, 3484475391, ++STORE, 3484475392, 3485028351, ++STORE, 3485028352, 3485057023, ++STORE, 3485057024, 3485061119, ++STORE, 3485360128, 3485364223, ++STORE, 3485364224, 3485368319, ++STORE, 3485368320, 3485372415, ++STORE, 3485589504, 3485593599, ++STORE, 3485593600, 3485597695, ++STORE, 3485597696, 3485601791, ++STORE, 3485913088, 3485937663, ++STORE, 3485937664, 3485974527, ++STORE, 3485974528, 3485982719, ++STORE, 3485982720, 3485986815, ++STORE, 3486052352, 3486056447, ++STORE, 3486056448, 3486064639, ++STORE, 3486064640, 3486068735, ++STORE, 3486068736, 3486072831, ++STORE, 3486294016, 3486302207, ++STORE, 3486302208, 3486306303, ++STORE, 3486306304, 3486310399, ++STORE, 3486310400, 3486314495, ++STORE, 3486670848, 3486679039, ++STORE, 3486679040, 3486683135, ++STORE, 3486683136, 3486687231, ++STORE, 3486687232, 3486691327, ++STORE, 3486863360, 3486871551, ++STORE, 3486871552, 3486875647, ++STORE, 3486875648, 3486879743, ++STORE, 3486879744, 3486883839, ++STORE, 3487584256, 3522543615, ++STORE, 3522543616, 3523321855, ++STORE, 3523321856, 3523342335, ++STORE, 3523342336, 3523387391, ++STORE, 3523387392, 3523391487, ++STORE, 3523391488, 3523395583, ++STORE, 3523477504, 3523686399, ++STORE, 3523686400, 3523981311, ++STORE, 3523981312, 3523997695, ++STORE, 3523997696, 3524001791, ++STORE, 3524177920, 3525013503, ++STORE, 3525013504, 3526582271, ++STORE, 3526582272, 3526606847, ++STORE, 3526606848, 3526610943, ++STORE, 3526610944, 3526615039, ++STORE, 3526672384, 3526746111, ++STORE, 3526746112, 3526860799, ++STORE, 3526860800, 3526868991, ++STORE, 3526868992, 3526873087, ++STORE, 3527000064, 3527475199, ++STORE, 3527475200, 3527479295, ++STORE, 3527479296, 3527573503, ++STORE, 3527573504, 3527581695, ++STORE, 3527581696, 3527585791, ++STORE, 3527585792, 3527606271, ++STORE, 3527909376, 3527913471, ++STORE, 3527913472, 3527917567, ++STORE, 3527917568, 3527921663, ++STORE, 3527950336, 3528011775, ++STORE, 3528011776, 3528093695, ++STORE, 3528093696, 3528101887, ++STORE, 3528101888, 3528105983, ++STORE, 3528228864, 3528241151, ++STORE, 3528241152, 3528261631, ++STORE, 3528261632, 3528265727, ++STORE, 3528273920, 3528593407, ++STORE, 3528593408, 3528609791, ++STORE, 3528609792, 3528638463, ++STORE, 3528638464, 3528642559, ++STORE, 3528642560, 3528646655, ++STORE, 3528880128, 3528912895, ++STORE, 3528912896, 3528962047, ++STORE, 3528962048, 3528966143, ++STORE, 3528966144, 3528970239, ++STORE, 3528982528, 3530293247, ++STORE, 3530366976, 3530825727, ++STORE, 3530825728, 3531317247, ++STORE, 3531317248, 3541041151, ++STORE, 3541041152, 3541303295, ++STORE, 3541430272, 3566206975, ++STORE, 3566206976, 3566993407, ++STORE, 3567239168, 3587571711, ++STORE, 3587571712, 3588284415, ++STORE, 3588284416, 3588661247, ++STORE, 3588661248, 3589066751, ++STORE, 3589066752, 3589574655, ++STORE, 3589574656, 3590078463, ++STORE, 3590078464, 3590373375, ++STORE, 3590373376, 3590668287, ++STORE, 3590668288, 3590963199, ++STORE, 3590963200, 3591294975, ++STORE, 3591294976, 3591602175, ++STORE, 3591602176, 3591933951, ++STORE, 3591933952, 3592241151, ++STORE, 3592241152, 3592572927, ++STORE, 3592572928, 3592876031, ++STORE, 3592876032, 3593211903, ++STORE, 3593211904, 3593547775, ++STORE, 3593547776, 3593650175, ++STORE, 3593650176, 3593928703, ++STORE, 3593928704, 3593936895, ++STORE, 3593936896, 3593940991, ++STORE, 3594006528, 3594301439, ++STORE, 3594301440, 3594739711, ++STORE, 3594739712, 3594756095, ++STORE, 3594756096, 3594760191, ++STORE, 3594760192, 3594768383, ++STORE, 3594952704, 3595051007, ++STORE, 3595051008, 3595223039, ++STORE, 3595223040, 3595227135, ++STORE, 3595227136, 3595235327, ++STORE, 3595431936, 3595775999, ++STORE, 3595776000, 3596701695, ++STORE, 3596701696, 3596742655, ++STORE, 3596742656, 3596746751, ++STORE, 3596746752, 3596750847, ++STORE, 3596767232, 3597070335, ++STORE, 3597070336, 3597402111, ++STORE, 3597402112, 3598188543, ++STORE, 3598262272, 3623428095, ++STORE, 3623428096, 3623432191, ++STORE, 3623432192, 3623436287, ++STORE, 3623436288, 3623440383, ++STORE, 3623616512, 3623878655, ++STORE, 3624169472, 3624300543, ++STORE, 3627524096, 3628523519, ++STORE, 3628523520, 3629522943, ++STORE, 3696631808, 3730186239, ++STORE, 3730186240, 3763740671, ++STORE, 3763740672, 3764027391, ++STORE, 3764027392, 3765133311, ++STORE, 3765133312, 3765145599, ++STORE, 3765145600, 3765149695, ++STORE, 3765178368, 3766022143, ++STORE, 3766022144, 3768791039, ++STORE, 3768791040, 3768840191, ++STORE, 3768840192, 3768844287, ++STORE, 3768897536, 3768913919, ++STORE, 3768913920, 3768934399, ++STORE, 3768934400, 3768938495, ++STORE, 3769016320, 3769147391, ++STORE, 3769147392, 3769233407, ++STORE, 3769233408, 3769356287, ++STORE, 3769356288, 3769360383, ++STORE, 3769360384, 3769368575, ++STORE, 3769376768, 3794542591, ++STORE, 3794542592, 3794599935, ++STORE, 3794599936, 3794731007, ++STORE, 3794731008, 3794735103, ++STORE, 3794735104, 3794743295, ++STORE, 3794849792, 3794980863, ++STORE, 3794980864, 3794984959, ++STORE, 3794984960, 3794989055, ++STORE, 3794989056, 3794993151, ++STORE, 3794993152, 3794997247, ++STORE, 3795103744, 3795128319, ++STORE, 3795128320, 3795165183, ++STORE, 3795165184, 3795169279, ++STORE, 3795169280, 3795173375, ++STORE, 3795210240, 3795357695, ++STORE, 3795357696, 3795365887, ++STORE, 3795365888, 3795374079, ++STORE, 3795374080, 3795378175, ++STORE, 3795378176, 3795382271, ++STORE, 3795406848, 3795738623, ++STORE, 3795738624, 3795742719, ++STORE, 3795742720, 3795755007, ++STORE, 3795755008, 3795759103, ++STORE, 3795763200, 3795894271, ++STORE, 3795894272, 3796041727, ++STORE, 3796041728, 3796054015, ++STORE, 3796054016, 3796066303, ++STORE, 3796066304, 3796070399, ++STORE, 3796176896, 3796205567, ++STORE, 3796205568, 3796250623, ++STORE, 3796250624, 3796254719, ++STORE, 3796254720, 3796258815, ++STORE, 3796262912, 3796393983, ++STORE, 3796393984, 3796516863, ++STORE, 3796516864, 3796873215, ++STORE, 3796873216, 3796885503, ++STORE, 3796885504, 3796889599, ++STORE, 3796963328, 3796967423, ++STORE, 3796967424, 3796975615, ++STORE, 3796975616, 3796979711, ++STORE, 3797000192, 3797307391, ++STORE, 3797307392, 3797311487, ++STORE, 3797311488, 3797315583, ++STORE, 3797315584, 3797323775, ++STORE, 3797327872, 3797450751, ++STORE, 3797450752, 3797458943, ++STORE, 3797458944, 3797471231, ++STORE, 3797471232, 3797475327, ++STORE, 3797577728, 3797700607, ++STORE, 3797700608, 3797721087, ++STORE, 3797721088, 3797733375, ++STORE, 3797733376, 3797741567, ++STORE, 3797741568, 3797864447, ++STORE, 3797864448, 3797995519, ++STORE, 3797995520, 3798048767, ++STORE, 3798048768, 3798179839, ++STORE, 3798179840, 3798188031, ++STORE, 3798188032, 3798192127, ++STORE, 3798290432, 3798302719, ++STORE, 3798302720, 3798323199, ++STORE, 3798323200, 3798327295, ++STORE, 3798327296, 3798331391, ++STORE, 3798429696, 3798433791, ++STORE, 3798433792, 3798552575, ++STORE, 3798552576, 3798556671, ++STORE, 3798556672, 3798568959, ++STORE, 3798568960, 3798573055, ++STORE, 3798573056, 3798581247, ++STORE, 3798618112, 3798749183, ++STORE, 3798749184, 3798855679, ++STORE, 3798855680, 3798966271, ++STORE, 3798966272, 3798982655, ++STORE, 3798982656, 3798986751, ++STORE, 3799101440, 3799171071, ++STORE, 3799171072, 3799240703, ++STORE, 3799240704, 3799248895, ++STORE, 3799248896, 3799252991, ++STORE, 3799326720, 3799650303, ++STORE, 3799650304, 3800629247, ++STORE, 3800629248, 3800641535, ++STORE, 3800641536, 3800645631, ++STORE, 3800645632, 3800649727, ++STORE, 3800649728, 3800903679, ++STORE, 3800903680, 3800936447, ++STORE, 3800936448, 3800969215, ++STORE, 3800969216, 3800981503, ++STORE, 3800981504, 3800985599, ++STORE, 3801001984, 3801133055, ++STORE, 3801133056, 3801202687, ++STORE, 3801202688, 3801591807, ++STORE, 3801591808, 3801599999, ++STORE, 3801600000, 3801604095, ++STORE, 3801604096, 3801608191, ++STORE, 3801608192, 3801739263, ++STORE, 3801739264, 3801755647, ++STORE, 3801755648, 3801796607, ++STORE, 3801796608, 3801804799, ++STORE, 3801804800, 3801808895, ++STORE, 3801878528, 3801944063, ++STORE, 3801944064, 3802116095, ++STORE, 3802116096, 3802124287, ++STORE, 3802124288, 3802128383, ++STORE, 3802136576, 3803447295, ++STORE, 3803492352, 3803553791, ++STORE, 3803553792, 3804233727, ++STORE, 3804233728, 3806068735, ++STORE, 3806121984, 3806253055, ++STORE, 3806253056, 3806674943, ++STORE, 3806674944, 3807117311, ++STORE, 3807117312, 3807379455, ++STORE, 3807379456, 3807432703, ++STORE, 3807432704, 3807563775, ++STORE, 3807563776, 3809202175, ++STORE, 3809202176, 3810250751, ++STORE, 3810250752, 3827027967, ++STORE, 3827027968, 3829125119, ++STORE, 3829125120, 3837513727, ++STORE, 3837513728, 3839610879, ++STORE, 3839610880, 3847999487, ++STORE, 3847999488, 3856392191, ++STORE, 3856392192, 3864784895, ++STORE, 3864784896, 3868983295, ++STORE, 3868983296, 3885760511, ++STORE, 3885760512, 3886809087, ++STORE, 3886809088, 3887857663, ++STORE, 3887857664, 3888119807, ++STORE, 3888144384, 3888148479, ++STORE, 3888148480, 3888218111, ++STORE, 3888218112, 3888222207, ++STORE, 3888222208, 3888353279, ++STORE, 3888353280, 3889172479, ++STORE, 3889172480, 3892314111, ++STORE, 3892314112, 3892576255, ++STORE, 3892588544, 3892637695, ++STORE, 3892637696, 3892686847, ++STORE, 3892686848, 3892744191, ++STORE, 3892748288, 3892785151, ++STORE, 3892785152, 3895459839, ++STORE, 3895459840, 3895721983, ++STORE, 3895738368, 3895885823, ++STORE, 3895885824, 3897081855, ++STORE, 3897081856, 3906482175, ++STORE, 3906482176, 3916144639, ++STORE, 3916144640, 3925766143, ++STORE, 3925766144, 3926974463, ++STORE, 3926974464, 3928367103, ++STORE, 3928367104, 3928911871, ++STORE, 3928911872, 3933995007, ++STORE, 3933995008, 3935830015, ++STORE, 3935830016, 3935846399, ++STORE, 3935879168, 3936010239, ++STORE, 3936010240, 3936026623, ++STORE, 3936026624, 3936034815, ++STORE, 3936034816, 3936051199, ++STORE, 3936051200, 3936055295, ++STORE, 3936071680, 3936137215, ++STORE, 3936137216, 3936202751, ++STORE, 3936202752, 3936219135, ++STORE, 3936235520, 3936251903, ++STORE, 3936268288, 3936276479, ++STORE, 3936276480, 3936284671, ++STORE, 3936284672, 3936288767, ++STORE, 3936288768, 3936292863, ++STORE, 3936296960, 3936354303, ++STORE, 3936354304, 3936616447, ++STORE, 3936628736, 3936669695, ++STORE, 3936669696, 3936747519, ++STORE, 3936747520, 3936870399, ++STORE, 3936870400, 3936874495, ++STORE, 3936874496, 3936878591, ++STORE, 3936882688, 3936903167, ++STORE, 3936911360, 3936948223, ++STORE, 3936948224, 3936964607, ++STORE, 3936964608, 3937103871, ++STORE, 3937103872, 3937107967, ++STORE, 3937132544, 3937161215, ++STORE, 3937189888, 3937255423, ++STORE, 3937255424, 3938512895, ++STORE, 3938512896, 3945435135, ++STORE, 3945435136, 3945476095, ++STORE, 3945476096, 3945484287, ++STORE, 3945484288, 3945496575, ++STORE, 3945500672, 3945541631, ++STORE, 3945558016, 3945566207, ++STORE, 3945566208, 3945594879, ++STORE, 3945594880, 3945598975, ++STORE, 3945598976, 3945603071, ++STORE, 3945611264, 3945742335, ++STORE, 3945742336, 3945844735, ++STORE, 3945844736, 3945848831, ++STORE, 3945848832, 3945861119, ++STORE, 3945861120, 3945865215, ++STORE, 3945869312, 3945897983, ++STORE, 3945897984, 3946303487, ++STORE, 3946303488, 3946397695, ++STORE, 3946397696, 3946569727, ++STORE, 3946569728, 3946573823, ++STORE, 3946573824, 3946594303, ++STORE, 3946594304, 3946663935, ++STORE, 3946663936, 3946708991, ++STORE, 3946708992, 3946823679, ++STORE, 3946823680, 3946827775, ++STORE, 3946827776, 3946831871, ++STORE, 3946831872, 3946860543, ++STORE, 3946893312, 3946897407, ++STORE, 3946897408, 3946905599, ++STORE, 3946905600, 3946909695, ++STORE, 3946909696, 3946913791, ++STORE, 3946913792, 3946930175, ++STORE, 3946930176, 3946967039, ++STORE, 3946967040, 3947102207, ++STORE, 3947102208, 3948412927, ++STORE, 3948441600, 3948556287, ++STORE, 3948556288, 3948576767, ++STORE, 3948576768, 3948597247, ++STORE, 3948597248, 3948605439, ++STORE, 3948605440, 3948609535, ++STORE, 3948609536, 3948654591, ++STORE, 3948654592, 3948781567, ++STORE, 3948781568, 3948822527, ++STORE, 3948822528, 3948904447, ++STORE, 3948904448, 3948908543, ++STORE, 3948908544, 3948912639, ++STORE, 3948945408, 3949043711, ++STORE, 3949043712, 3949174783, ++STORE, 3949174784, 3949191167, ++STORE, 3949191168, 3949195263, ++STORE, 3949207552, 3949252607, ++STORE, 3949252608, 3949256703, ++STORE, 3949256704, 3949363199, ++STORE, 3949363200, 3949367295, ++STORE, 3949367296, 3949379583, ++STORE, 3949379584, 3949383679, ++STORE, 3949383680, 3949400063, ++STORE, 3949400064, 3949404159, ++STORE, 3949416448, 3949481983, ++STORE, 3949481984, 3949486079, ++STORE, 3949486080, 3949592575, ++STORE, 3949592576, 3949596671, ++STORE, 3949596672, 3949621247, ++STORE, 3949621248, 3949662207, ++STORE, 3949662208, 3949666303, ++STORE, 3949694976, 3949727743, ++STORE, 3949727744, 3949731839, ++STORE, 3949731840, 3949838335, ++STORE, 3949838336, 3949842431, ++STORE, 3949842432, 3949846527, ++STORE, 3949846528, 3949854719, ++STORE, 3949854720, 3949858815, ++STORE, 3949858816, 3949862911, ++STORE, 3949867008, 3949891583, ++STORE, 3949891584, 3949928447, ++STORE, 3949928448, 3949993983, ++STORE, 3949993984, 3950043135, ++STORE, 3950043136, 3950059519, ++STORE, 3950059520, 3950096383, ++STORE, 3950096384, 3950100479, ++STORE, 3950100480, 3950104575, ++STORE, 3950104576, 3950157823, ++STORE, 3950157824, 3950292991, ++STORE, 3950292992, 3950346239, ++STORE, 3950346240, 3950477311, ++STORE, 3950477312, 3950485503, ++STORE, 3950485504, 3950489599, ++STORE, 3950493696, 3950510079, ++STORE, 3950510080, 3950661631, ++STORE, 3950661632, 3951005695, ++STORE, 3951005696, 3951026175, ++STORE, 3951026176, 3951030271, ++STORE, 3951030272, 3951054847, ++STORE, 3951054848, 3951116287, ++STORE, 3951116288, 3951144959, ++STORE, 3951144960, 3951149055, ++STORE, 3951149056, 3951194111, ++STORE, 3951194112, 3951202303, ++STORE, 3951202304, 3951206399, ++STORE, 3951210496, 3951226879, ++STORE, 3951226880, 3951329279, ++STORE, 3951329280, 3951366143, ++STORE, 3951366144, 3951411199, ++STORE, 3951411200, 3951415295, ++STORE, 3951415296, 3951419391, ++STORE, 3951419392, 3951452159, ++STORE, 3951452160, 3951566847, ++STORE, 3951566848, 3951812607, ++STORE, 3951812608, 3952173055, ++STORE, 3952173056, 3952214015, ++STORE, 3952214016, 3952218111, ++STORE, 3952222208, 3952250879, ++STORE, 3952250880, 3952369663, ++STORE, 3952369664, 3952488447, ++STORE, 3952488448, 3952627711, ++STORE, 3952627712, 3952635903, ++STORE, 3952635904, 3952639999, ++STORE, 3952652288, 3952668671, ++STORE, 3952668672, 3953000447, ++STORE, 3953000448, 3953004543, ++STORE, 3953004544, 3953008639, ++STORE, 3953008640, 3953012735, ++STORE, 3953012736, 3953037311, ++STORE, 3953037312, 3953151999, ++STORE, 3953152000, 3953291263, ++STORE, 3953291264, 3953324031, ++STORE, 3953324032, 3953364991, ++STORE, 3953364992, 3953373183, ++STORE, 3953373184, 3953377279, ++STORE, 3953381376, 3953410047, ++STORE, 3953410048, 3953491967, ++STORE, 3953491968, 3953643519, ++STORE, 3953643520, 3953651711, ++STORE, 3953651712, 3953655807, ++STORE, 3953659904, 3953766399, ++STORE, 3953766400, 3953774591, ++STORE, 3953774592, 3953786879, ++STORE, 3953786880, 3953790975, ++STORE, 3953790976, 3953823743, ++STORE, 3953823744, 3953963007, ++STORE, 3953963008, 3954024447, ++STORE, 3954024448, 3954118655, ++STORE, 3954118656, 3954122751, ++STORE, 3954122752, 3954126847, ++STORE, 3954130944, 3954184191, ++STORE, 3954184192, 3954294783, ++STORE, 3954294784, 3954323455, ++STORE, 3954323456, 3954393087, ++STORE, 3954393088, 3954397183, ++STORE, 3954397184, 3954401279, ++STORE, 3954401280, 3954405375, ++STORE, 3954409472, 3954528255, ++STORE, 3954528256, 3954737151, ++STORE, 3954737152, 3955052543, ++STORE, 3955052544, 3955060735, ++STORE, 3955060736, 3955064831, ++STORE, 3955068928, 3955105791, ++STORE, 3955105792, 3955167231, ++STORE, 3955167232, 3955277823, ++STORE, 3955277824, 3955310591, ++STORE, 3955310592, 3955351551, ++STORE, 3955351552, 3955359743, ++STORE, 3955359744, 3955363839, ++STORE, 3955363840, 3955392511, ++STORE, 3955392512, 3955453951, ++STORE, 3955453952, 3955601407, ++STORE, 3955601408, 3955777535, ++STORE, 3955777536, 3955982335, ++STORE, 3955982336, 3956011007, ++STORE, 3956011008, 3956015103, ++STORE, 3956023296, 3956039679, ++STORE, 3956039680, 3956125695, ++STORE, 3956125696, 3956129791, ++STORE, 3956129792, 3956133887, ++STORE, 3956133888, 3956137983, ++STORE, 3956142080, 3956449279, ++STORE, 3956449280, 3956543487, ++STORE, 3956543488, 3956719615, ++STORE, 3956719616, 3956731903, ++STORE, 3956731904, 3956735999, ++STORE, 3956744192, 3956793343, ++STORE, 3956793344, 3956887551, ++STORE, 3956887552, 3956953087, ++STORE, 3956953088, 3957035007, ++STORE, 3957035008, 3957039103, ++STORE, 3957039104, 3957047295, ++STORE, 3957047296, 3957071871, ++STORE, 3957071872, 3957231615, ++STORE, 3957231616, 3957563391, ++STORE, 3957563392, 3957579775, ++STORE, 3957579776, 3957583871, ++STORE, 3957592064, 3957608447, ++STORE, 3957608448, 3957878783, ++STORE, 3957878784, 3958591487, ++STORE, 3958591488, 3958599679, ++STORE, 3958599680, 3958607871, ++STORE, 3958607872, 3958620159, ++STORE, 3958620160, 3958624255, ++STORE, 3958624256, 3963199487, ++STORE, 3963199488, 3963285503, ++STORE, 3963285504, 3963371519, ++STORE, 3963371520, 3963428863, ++STORE, 3963428864, 3963555839, ++STORE, 3963555840, 3963559935, ++STORE, 3963559936, 3963564031, ++STORE, 3963568128, 3963596799, ++STORE, 3963596800, 3963682815, ++STORE, 3963682816, 3963695103, ++STORE, 3963695104, 3963711487, ++STORE, 3963711488, 3963715583, ++STORE, 3963719680, 3963752447, ++STORE, 3963752448, 3963846655, ++STORE, 3963846656, 3963932671, ++STORE, 3963932672, 3964444671, ++STORE, 3964444672, 3964448767, ++STORE, 3964448768, 3965808639, ++STORE, 3965808640, 3965845503, ++STORE, 3965845504, 3965849599, ++STORE, 3965853696, 3965935615, ++STORE, 3965935616, 3966017535, ++STORE, 3966017536, 3966103551, ++STORE, 3966103552, 3966685183, ++STORE, 3966685184, 3967705087, ++STORE, 3967705088, 3967758335, ++STORE, 3967758336, 3967762431, ++STORE, 3967762432, 3967770623, ++STORE, 3967770624, 3967799295, ++STORE, 3967799296, 3967848447, ++STORE, 3967848448, 3967868927, ++STORE, 3967868928, 3967901695, ++STORE, 3967901696, 3967905791, ++STORE, 3967905792, 3967909887, ++STORE, 3967909888, 3967995903, ++STORE, 3967995904, 3968077823, ++STORE, 3968077824, 3968159743, ++STORE, 3968159744, 3968167935, ++STORE, 3968167936, 3968172031, ++STORE, 3968172032, 3968192511, ++STORE, 3968192512, 3968196607, ++STORE, 3968196608, 3968200703, ++STORE, 3968208896, 3968516095, ++STORE, 3968516096, 3968528383, ++STORE, 3968528384, 3968552959, ++STORE, 3968552960, 3968557055, ++STORE, 3968561152, 3968593919, ++STORE, 3968593920, 3968626687, ++STORE, 3968626688, 3971153919, ++STORE, 3971153920, 3973754879, ++STORE, 3973754880, 3973804031, ++STORE, 3973804032, 3973820415, ++STORE, 3973820416, 3973832703, ++STORE, 3973840896, 3973873663, ++STORE, 3973873664, 3973967871, ++STORE, 3973967872, 3973976063, ++STORE, 3973976064, 3973984255, ++STORE, 3973984256, 3973988351, ++STORE, 3973988352, 3973992447, ++STORE, 3973996544, 3974008831, ++STORE, 3974008832, 3974045695, ++STORE, 3974045696, 3974139903, ++STORE, 3974139904, 3974254591, ++STORE, 3974254592, 3974275071, ++STORE, 3974275072, 3974291455, ++STORE, 3974291456, 3974295551, ++STORE, 3974295552, 3974373375, ++STORE, 3974373376, 3974524927, ++STORE, 3974524928, 3974529023, ++STORE, 3974529024, 3974537215, ++STORE, 3974537216, 3974541311, ++STORE, 3974541312, 3974545407, ++STORE, 3974545408, 3974627327, ++STORE, 3974627328, 3974680575, ++STORE, 3974680576, 3974811647, ++STORE, 3974811648, 3974819839, ++STORE, 3974819840, 3974823935, ++STORE, 3974832128, 3974918143, ++STORE, 3974918144, 3974963199, ++STORE, 3974963200, 3975077887, ++STORE, 3975077888, 3975090175, ++STORE, 3975090176, 3975094271, ++STORE, 3975094272, 3975102463, ++STORE, 3975102464, 3975114751, ++STORE, 3975114752, 3975266303, ++STORE, 3975266304, 3975274495, ++STORE, 3975274496, 3975286783, ++STORE, 3975286784, 3975290879, ++STORE, 3975290880, 3975299071, ++STORE, 3975299072, 3975315455, ++STORE, 3975315456, 3975430143, ++STORE, 3975430144, 3975536639, ++STORE, 3975536640, 3975651327, ++STORE, 3975651328, 3975655423, ++STORE, 3975655424, 3975659519, ++STORE, 3975659520, 3975770111, ++STORE, 3975770112, 3975778303, ++STORE, 3975778304, 3975790591, ++STORE, 3975790592, 3975794687, ++STORE, 3975794688, 3975798783, ++STORE, 3975798784, 3975831551, ++STORE, 3975831552, 3975872511, ++STORE, 3975872512, 3975987199, ++STORE, 3975987200, 3976134655, ++STORE, 3976134656, 3977175039, ++STORE, 3977175040, 3977183231, ++STORE, 3977183232, 3977191423, ++STORE, 3977191424, 3977195519, ++STORE, 3977199616, 3977248767, ++STORE, 3977248768, 3977539583, ++STORE, 3977539584, 3977965567, ++STORE, 3977965568, 3977981951, ++STORE, 3977981952, 3977986047, ++STORE, 3977986048, 3977994239, ++STORE, 3977994240, 3978002431, ++STORE, 3978002432, 3978084351, ++STORE, 3978084352, 3978125311, ++STORE, 3978125312, 3978174463, ++STORE, 3978174464, 3978178559, ++STORE, 3978178560, 3978182655, ++STORE, 3978182656, 3978207231, ++STORE, 3978207232, 3978297343, ++STORE, 3978297344, 3978301439, ++STORE, 3978301440, 3978305535, ++STORE, 3978305536, 3978309631, ++STORE, 3978309632, 3978317823, ++STORE, 3978317824, 3978625023, ++STORE, 3978625024, 3978657791, ++STORE, 3978657792, 3978727423, ++STORE, 3978727424, 3978735615, ++STORE, 3978735616, 3978739711, ++STORE, 3978739712, 3978760191, ++STORE, 3978760192, 3978842111, ++STORE, 3978842112, 3978850303, ++STORE, 3978850304, 3978858495, ++STORE, 3978858496, 3978862591, ++STORE, 3978862592, 3978895359, ++STORE, 3978895360, 3979014143, ++STORE, 3979014144, 3979132927, ++STORE, 3979132928, 3979288575, ++STORE, 3979288576, 3979481087, ++STORE, 3979481088, 3979489279, ++STORE, 3979489280, 3979493375, ++STORE, 3979497472, 3979583487, ++STORE, 3979583488, 3979673599, ++STORE, 3979673600, 3979718655, ++STORE, 3979718656, 3979829247, ++STORE, 3979829248, 3979841535, ++STORE, 3979841536, 3979882495, ++STORE, 3979882496, 3979964415, ++STORE, 3979964416, 3980013567, ++STORE, 3980013568, 3980148735, ++STORE, 3980148736, 3980152831, ++STORE, 3980152832, 3980320767, ++STORE, 3980320768, 3980337151, ++STORE, 3980337152, 3980341247, ++STORE, 3980345344, 3980365823, ++STORE, 3980365824, 3980423167, ++STORE, 3980423168, 3980460031, ++STORE, 3980460032, 3980500991, ++STORE, 3980500992, 3980509183, ++STORE, 3980509184, 3980513279, ++STORE, 3980513280, 3980546047, ++STORE, 3980546048, 3980660735, ++STORE, 3980660736, 3980951551, ++STORE, 3980951552, 3981500415, ++STORE, 3981500416, 3981529087, ++STORE, 3981529088, 3981533183, ++STORE, 3981537280, 3981549567, ++STORE, 3981549568, 3981598719, ++STORE, 3981598720, 3981717503, ++STORE, 3981717504, 3982127103, ++STORE, 3982127104, 3982675967, ++STORE, 3982675968, 3982733311, ++STORE, 3982733312, 3982737407, ++STORE, 3982741504, 3982860287, ++STORE, 3982860288, 3982905343, ++STORE, 3982905344, 3982966783, ++STORE, 3982966784, 3982974975, ++STORE, 3982974976, 3982979071, ++STORE, 3982979072, 3983032319, ++STORE, 3983032320, 3983085567, ++STORE, 3983085568, 3983208447, ++STORE, 3983208448, 3983212543, ++STORE, 3983212544, 3983220735, ++STORE, 3983220736, 3983224831, ++STORE, 3983224832, 3983237119, ++STORE, 3983237120, 3983351807, ++STORE, 3983351808, 3983376383, ++STORE, 3983376384, 3983392767, ++STORE, 3983392768, 3983396863, ++STORE, 3983396864, 3983400959, ++STORE, 3983400960, 3983417343, ++STORE, 3983417344, 3983753215, ++STORE, 3983753216, 3983757311, ++STORE, 3983757312, 3983761407, ++STORE, 3983761408, 3983765503, ++STORE, 3983765504, 3983769599, ++STORE, 3983769600, 3983880191, ++STORE, 3983880192, 3983892479, ++STORE, 3983892480, 3983900671, ++STORE, 3983900672, 3983904767, ++STORE, 3983904768, 3983908863, ++STORE, 3983908864, 3983941631, ++STORE, 3983941632, 3983990783, ++STORE, 3983990784, 3984097279, ++STORE, 3984097280, 3984105471, ++STORE, 3984105472, 3984117759, ++STORE, 3984117760, 3984121855, ++STORE, 3984121856, 3984125951, ++STORE, 3984125952, 3984134143, ++STORE, 3984134144, 3984150527, ++STORE, 3984150528, 3984416767, ++STORE, 3984416768, 3984470015, ++STORE, 3984470016, 3984564223, ++STORE, 3984564224, 3984568319, ++STORE, 3984572416, 3984629759, ++STORE, 3984629760, 3984805887, ++STORE, 3984805888, 3985096703, ++STORE, 3985096704, 3985104895, ++STORE, 3985104896, 3985108991, ++STORE, 3985113088, 3986862079, ++STORE, 3986862080, 3993640959, ++STORE, 3993640960, 3993739263, ++STORE, 3993739264, 3993743359, ++STORE, 3993743360, 3993759743, ++STORE, 3993759744, 3993780223, ++STORE, 3993780224, 3993784319, ++STORE, 3993784320, 3993792511, ++STORE, 3993792512, 3993796607, ++STORE, 3993796608, 3993800703, ++STORE, 3993804800, 3994214399, ++STORE, 3994214400, 3994218495, ++STORE, 3994218496, 3994222591, ++STORE, 3994222592, 3994226687, ++STORE, 3994230784, 3994243071, ++STORE, 3994243072, 3994255359, ++STORE, 3994255360, 3994304511, ++STORE, 3994304512, 3994386431, ++STORE, 3994386432, 3994509311, ++STORE, 3994509312, 3994521599, ++STORE, 3994521600, 3994525695, ++STORE, 3994529792, 3994542079, ++STORE, 3994542080, 3994660863, ++STORE, 3994660864, 3994705919, ++STORE, 3994705920, 3994796031, ++STORE, 3994796032, 3994800127, ++STORE, 3994800128, 3994804223, ++STORE, 3994804224, 3994812415, ++STORE, 3994812416, 3994845183, ++STORE, 3994845184, 3994898431, ++STORE, 3994898432, 3994902527, ++STORE, 3994902528, 3994906623, ++STORE, 3994910720, 3994931199, ++STORE, 3994931200, 3995181055, ++STORE, 3995181056, 3995222015, ++STORE, 3995222016, 3995275263, ++STORE, 3995275264, 3995279359, ++STORE, 3995279360, 3995283455, ++STORE, 3995283456, 3995291647, ++STORE, 3995291648, 3995324415, ++STORE, 3995324416, 3995451391, ++STORE, 3995451392, 3995697151, ++STORE, 3995697152, 3996078079, ++STORE, 3996078080, 3996086271, ++STORE, 3996086272, 3996090367, ++STORE, 3996094464, 3996119039, ++STORE, 3996119040, 3996200959, ++STORE, 3996200960, 3996229631, ++STORE, 3996229632, 3996233727, ++STORE, 3996233728, 3996282879, ++STORE, 3996282880, 3996291071, ++STORE, 3996291072, 3996295167, ++STORE, 3996299264, 3996311551, ++STORE, 3996311552, 3996430335, ++STORE, 3996430336, 3996467199, ++STORE, 3996467200, 3996504063, ++STORE, 3996504064, 3996512255, ++STORE, 3996512256, 3996516351, ++STORE, 3996516352, 3996540927, ++STORE, 3996540928, 3996671999, ++STORE, 3996672000, 3996676095, ++STORE, 3996676096, 3996684287, ++STORE, 3996684288, 3996688383, ++STORE, 3996688384, 3996692479, ++STORE, 3996692480, 3996717055, ++STORE, 3996717056, 3997048831, ++STORE, 3997048832, 3997057023, ++STORE, 3997057024, 3997073407, ++STORE, 3997073408, 3997077503, ++STORE, 3997077504, 3997081599, ++STORE, 3997081600, 3997097983, ++STORE, 3997097984, 3997179903, ++STORE, 3997179904, 3997356031, ++STORE, 3997356032, 3997650943, ++STORE, 3997650944, 3997675519, ++STORE, 3997675520, 3997679615, ++STORE, 3997683712, 3997700095, ++STORE, 3997700096, 3997745151, ++STORE, 3997745152, 3997802495, ++STORE, 3997802496, 3997810687, ++STORE, 3997810688, 3997814783, ++STORE, 3997814784, 3998064639, ++STORE, 3998064640, 3998081023, ++STORE, 3998081024, 3998085119, ++STORE, 3998085120, 3998130175, ++STORE, 3998130176, 3998134271, ++STORE, 3998134272, 3998142463, ++STORE, 3998142464, 3998179327, ++STORE, 3998179328, 3998212095, ++STORE, 3998212096, 3998326783, ++STORE, 3998326784, 3998351359, ++STORE, 3998351360, 3998392319, ++STORE, 3998392320, 3998396415, ++STORE, 3998396416, 3998400511, ++STORE, 3998400512, 3998433279, ++STORE, 3998433280, 3998466047, ++STORE, 3998466048, 3998613503, ++STORE, 3998613504, 3998666751, ++STORE, 3998666752, 3998724095, ++STORE, 3998724096, 3998732287, ++STORE, 3998732288, 3998736383, ++STORE, 3998736384, 3998760959, ++STORE, 3998760960, 3998777343, ++STORE, 3998777344, 3998822399, ++STORE, 3998822400, 3998826495, ++STORE, 3998826496, 3998830591, ++STORE, 3998830592, 3998863359, ++STORE, 3998863360, 3998900223, ++STORE, 3998900224, 3999043583, ++STORE, 3999043584, 3999121407, ++STORE, 3999121408, 3999215615, ++STORE, 3999215616, 3999223807, ++STORE, 3999223808, 3999227903, ++STORE, 3999227904, 3999236095, ++STORE, 3999236096, 3999268863, ++STORE, 3999268864, 3999301631, ++STORE, 3999301632, 3999354879, ++STORE, 3999354880, 3999428607, ++STORE, 3999428608, 3999436799, ++STORE, 3999436800, 3999440895, ++STORE, 3999444992, 3999461375, ++STORE, 3999461376, 3999584255, ++STORE, 3999584256, 3999760383, ++STORE, 3999760384, 4000219135, ++STORE, 4000219136, 4000235519, ++STORE, 4000235520, 4000251903, ++STORE, 4000251904, 4000501759, ++STORE, 4000501760, 4000505855, ++STORE, 4000505856, 4000509951, ++STORE, 4000509952, 4000518143, ++STORE, 4000518144, 4000522239, ++STORE, 4000522240, 4000587775, ++STORE, 4000587776, 4000645119, ++STORE, 4000645120, 4000813055, ++STORE, 4000813056, 4000817151, ++STORE, 4000821248, 4000837631, ++STORE, 4000837632, 4000870399, ++STORE, 4000870400, 4000874495, ++STORE, 4000874496, 4000878591, ++STORE, 4000878592, 4000882687, ++STORE, 4000882688, 4000886783, ++STORE, 4000886784, 4000890879, ++STORE, 4000890880, 4000907263, ++STORE, 4000907264, 4001214463, ++STORE, 4001214464, 4001558527, ++STORE, 4001558528, 4002484223, ++STORE, 4002484224, 4002525183, ++STORE, 4002525184, 4002529279, ++STORE, 4002529280, 4002533375, ++STORE, 4002533376, 4002537471, ++STORE, 4002537472, 4002660351, ++STORE, 4002660352, 4002779135, ++STORE, 4002779136, 4002791423, ++STORE, 4002791424, 4002799615, ++STORE, 4002799616, 4002807807, ++STORE, 4002807808, 4002811903, ++STORE, 4002811904, 4002828287, ++STORE, 4002828288, 4002910207, ++STORE, 4002910208, 4003028991, ++STORE, 4003028992, 4003037183, ++STORE, 4003037184, 4003045375, ++STORE, 4003045376, 4003049471, ++STORE, 4003049472, 4003053567, ++STORE, 4003053568, 4003057663, ++STORE, 4003057664, 4003065855, ++STORE, 4003065856, 4003135487, ++STORE, 4003135488, 4003446783, ++STORE, 4003446784, 4003450879, ++STORE, 4003450880, 4003454975, ++STORE, 4003454976, 4003459071, ++STORE, 4003459072, 4003463167, ++STORE, 4003463168, 4003495935, ++STORE, 4003495936, 4003569663, ++STORE, 4003569664, 4003573759, ++STORE, 4003573760, 4003704831, ++STORE, 4003704832, 4003708927, ++STORE, 4003708928, 4003713023, ++STORE, 4003713024, 4003737599, ++STORE, 4003737600, 4003770367, ++STORE, 4003770368, 4003876863, ++STORE, 4003876864, 4003880959, ++STORE, 4003880960, 4003885055, ++STORE, 4003885056, 4003889151, ++STORE, 4003889152, 4003893247, ++STORE, 4003893248, 4003897343, ++STORE, 4003897344, 4003962879, ++STORE, 4003962880, 4004069375, ++STORE, 4004069376, 4004093951, ++STORE, 4004093952, 4004118527, ++STORE, 4004118528, 4004122623, ++STORE, 4004122624, 4004126719, ++STORE, 4004126720, 4004155391, ++STORE, 4004155392, 4004286463, ++STORE, 4004286464, 4004384767, ++STORE, 4004384768, 4004388863, ++STORE, 4004388864, 4004646911, ++STORE, 4004646912, 4004655103, ++STORE, 4004655104, 4004659199, ++STORE, 4004659200, 4004667391, ++STORE, 4004667392, 4004683775, ++STORE, 4004683776, 4004814847, ++STORE, 4004814848, 4004818943, ++STORE, 4004818944, 4004823039, ++STORE, 4004823040, 4004827135, ++STORE, 4004827136, 4004835327, ++STORE, 4004835328, 4004954111, ++STORE, 4004954112, 4005085183, ++STORE, 4005085184, 4005306367, ++STORE, 4005306368, 4005765119, ++STORE, 4005765120, 4005789695, ++STORE, 4005789696, 4005793791, ++STORE, 4005793792, 4005801983, ++STORE, 4005801984, 4005920767, ++STORE, 4005920768, 4005945343, ++STORE, 4005945344, 4005949439, ++STORE, 4005949440, 4005986303, ++STORE, 4005986304, 4005990399, ++STORE, 4005990400, 4005994495, ++STORE, 4005994496, 4006002687, ++STORE, 4006002688, 4006109183, ++STORE, 4006109184, 4006117375, ++STORE, 4006117376, 4006121471, ++STORE, 4006121472, 4006133759, ++STORE, 4006133760, 4006137855, ++STORE, 4006137856, 4006141951, ++STORE, 4006141952, 4006150143, ++STORE, 4006150144, 4006391807, ++STORE, 4006391808, 4006445055, ++STORE, 4006445056, 4006563839, ++STORE, 4006563840, 4006572031, ++STORE, 4006572032, 4006576127, ++STORE, 4006576128, 4006584319, ++STORE, 4006584320, 4006694911, ++STORE, 4006694912, 4006739967, ++STORE, 4006739968, 4006776831, ++STORE, 4006776832, 4006785023, ++STORE, 4006785024, 4006789119, ++STORE, 4006789120, 4006797311, ++STORE, 4006797312, 4006813695, ++STORE, 4006813696, 4006846463, ++STORE, 4006846464, 4006977535, ++STORE, 4006977536, 4007006207, ++STORE, 4007006208, 4007010303, ++STORE, 4007010304, 4007067647, ++STORE, 4007067648, 4007075839, ++STORE, 4007075840, 4007084031, ++STORE, 4007084032, 4007100415, ++STORE, 4007100416, 4007116799, ++STORE, 4007116800, 4007133183, ++STORE, 4007133184, 4007153663, ++STORE, 4007153664, 4007178239, ++STORE, 4007178240, 4007202815, ++STORE, 4007202816, 4007206911, ++STORE, 4007206912, 4007272447, ++STORE, 4007272448, 4007276543, ++STORE, 4007276544, 4007280639, ++STORE, 4007280640, 4007284735, ++STORE, 4007284736, 4007292927, ++STORE, 4007292928, 4007423999, ++STORE, 4007424000, 4007448575, ++STORE, 4007448576, 4007452671, ++STORE, 4007452672, 4007505919, ++STORE, 4007505920, 4007510015, ++STORE, 4007510016, 4007514111, ++STORE, 4007514112, 4007645183, ++STORE, 4007645184, 4007776255, ++STORE, 4007776256, 4007780351, ++STORE, 4007780352, 4007784447, ++STORE, 4007784448, 4007788543, ++STORE, 4007788544, 4007809023, ++STORE, 4007809024, 4007829503, ++STORE, 4007829504, 4007960575, ++STORE, 4007960576, 4008091647, ++STORE, 4008091648, 4008296447, ++STORE, 4008296448, 4008890367, ++STORE, 4008890368, 4008898559, ++STORE, 4008898560, 4008902655, ++STORE, 4008902656, 4008996863, ++STORE, 4008996864, 4009041919, ++STORE, 4009041920, 4009082879, ++STORE, 4009082880, 4009091071, ++STORE, 4009091072, 4009107455, ++STORE, 4009107456, 4009349119, ++STORE, 4009349120, 4009373695, ++STORE, 4009373696, 4009414655, ++STORE, 4009414656, 4009422847, ++STORE, 4009422848, 4009426943, ++STORE, 4009426944, 4009447423, ++STORE, 4009447424, 4009471999, ++STORE, 4009472000, 4009512959, ++STORE, 4009512960, 4009594879, ++STORE, 4009594880, 4009598975, ++STORE, 4009598976, 4009697279, ++STORE, 4009697280, 4009713663, ++STORE, 4009713664, 4009717759, ++STORE, 4009717760, 4009721855, ++STORE, 4009721856, 4009730047, ++STORE, 4009730048, 4009861119, ++STORE, 4009861120, 4009951231, ++STORE, 4009951232, 4010131455, ++STORE, 4010131456, 4010135551, ++STORE, 4010135552, 4010139647, ++STORE, 4010139648, 4010143743, ++STORE, 4010143744, 4010164223, ++STORE, 4010164224, 4010295295, ++STORE, 4010295296, 4010299391, ++STORE, 4010299392, 4010491903, ++STORE, 4010491904, 4010495999, ++STORE, 4010496000, 4010668031, ++STORE, 4010668032, 4011028479, ++STORE, 4011028480, 4011053055, ++STORE, 4011053056, 4011057151, ++STORE, 4011057152, 4011118591, ++STORE, 4011118592, 4011126783, ++STORE, 4011126784, 4011130879, ++STORE, 4011130880, 4011143167, ++STORE, 4011143168, 4011147263, ++STORE, 4011147264, 4011167743, ++STORE, 4011167744, 4011171839, ++STORE, 4011171840, 4011360255, ++STORE, 4011360256, 4011364351, ++STORE, 4011364352, 4011626495, ++STORE, 4011626496, 4012216319, ++STORE, 4012216320, 4012228607, ++STORE, 4012228608, 4012232703, ++STORE, 4012232704, 4012236799, ++STORE, 4012236800, 4012240895, ++STORE, 4012240896, 4012261375, ++STORE, 4012261376, 4012392447, ++STORE, 4012392448, 4012466175, ++STORE, 4012466176, 4012597247, ++STORE, 4012597248, 4012601343, ++STORE, 4012601344, 4012605439, ++STORE, 4012605440, 4012609535, ++STORE, 4012609536, 4012679167, ++STORE, 4012679168, 4013563903, ++STORE, 4013563904, 4015366143, ++STORE, 4015366144, 4015411199, ++STORE, 4015411200, 4015415295, ++STORE, 4015415296, 4015419391, ++STORE, 4015419392, 4015542271, ++STORE, 4015542272, 4015550463, ++STORE, 4015550464, 4015558655, ++STORE, 4015558656, 4015562751, ++STORE, 4015562752, 4015583231, ++STORE, 4015583232, 4015587327, ++STORE, 4015587328, 4015603711, ++STORE, 4015665152, 4015669247, ++STORE, 4015669248, 4015812607, ++STORE, 4015812608, 4015816703, ++STORE, 4015816704, 4016111615, ++STORE, 4016111616, 4016467967, ++STORE, 4016467968, 4016508927, ++STORE, 4016508928, 4016517119, ++STORE, 4016517120, 4016525311, ++STORE, 4016525312, 4016586751, ++STORE, 4016586752, 4016664575, ++STORE, 4016664576, 4016697343, ++STORE, 4016697344, 4016742399, ++STORE, 4016742400, 4016746495, ++STORE, 4016746496, 4016750591, ++STORE, 4016750592, 4016758783, ++STORE, 4016799744, 4016844799, ++STORE, 4016844800, 4016902143, ++STORE, 4016902144, 4016992255, ++STORE, 4016992256, 4017000447, ++STORE, 4017000448, 4017004543, ++STORE, 4017004544, 4017008639, ++STORE, 4017008640, 4017016831, ++STORE, 4017016832, 4017020927, ++STORE, 4017020928, 4017127423, ++STORE, 4017127424, 4017131519, ++STORE, 4017131520, 4017229823, ++STORE, 4017229824, 4017422335, ++STORE, 4017422336, 4017438719, ++STORE, 4017438720, 4017442815, ++STORE, 4017442816, 4017446911, ++STORE, 4017446912, 4017455103, ++STORE, 4017455104, 4017766399, ++STORE, 4017766400, 4017909759, ++STORE, 4017909760, 4018081791, ++STORE, 4018081792, 4018089983, ++STORE, 4018089984, 4018094079, ++STORE, 4018094080, 4018098175, ++STORE, 4018098176, 4018327551, ++STORE, 4018327552, 4018331647, ++STORE, 4018331648, 4018339839, ++STORE, 4018339840, 4018348031, ++STORE, 4018348032, 4018610175, ++STORE, 4018610176, 4018626559, ++STORE, 4018626560, 4018647039, ++STORE, 4018647040, 4018651135, ++STORE, 4018651136, 4018749439, ++STORE, 4018749440, 4018761727, ++STORE, 4018761728, 4018802687, ++STORE, 4018802688, 4018806783, ++STORE, 4018806784, 4018810879, ++STORE, 4018810880, 4018814975, ++STORE, 4018814976, 4018823167, ++STORE, 4018823168, 4018954239, ++STORE, 4018954240, 4019007487, ++STORE, 4019007488, 4019068927, ++STORE, 4019068928, 4019077119, ++STORE, 4019077120, 4019081215, ++STORE, 4019081216, 4019093503, ++STORE, 4019093504, 4019208191, ++STORE, 4019208192, 4019232767, ++STORE, 4019232768, 4019265535, ++STORE, 4019265536, 4019269631, ++STORE, 4019269632, 4019277823, ++STORE, 4019277824, 4019458047, ++STORE, 4019458048, 4019519487, ++STORE, 4019519488, 4019613695, ++STORE, 4019613696, 4019621887, ++STORE, 4019621888, 4019625983, ++STORE, 4019625984, 4019630079, ++STORE, 4019630080, 4019744767, ++STORE, 4019744768, 4019822591, ++STORE, 4019822592, 4019929087, ++STORE, 4019929088, 4019941375, ++STORE, 4019941376, 4019945471, ++STORE, 4019945472, 4019961855, ++STORE, 4019961856, 4019994623, ++STORE, 4019994624, 4019998719, ++STORE, 4019998720, 4020002815, ++STORE, 4020002816, 4020006911, ++STORE, 4020006912, 4020011007, ++STORE, 4020011008, 4020256767, ++STORE, 4020256768, 4020326399, ++STORE, 4020326400, 4020457471, ++STORE, 4020457472, 4020469759, ++STORE, 4020469760, 4020473855, ++STORE, 4020473856, 4020482047, ++STORE, 4020482048, 4020711423, ++STORE, 4020711424, 4020715519, ++STORE, 4020715520, 4020719615, ++STORE, 4020719616, 4020723711, ++STORE, 4020723712, 4020805631, ++STORE, 4020805632, 4021051391, ++STORE, 4021051392, 4021460991, ++STORE, 4021460992, 4021469183, ++STORE, 4021469184, 4021473279, ++STORE, 4021473280, 4021571583, ++STORE, 4021571584, 4021633023, ++STORE, 4021633024, 4021727231, ++STORE, 4021727232, 4021735423, ++STORE, 4021735424, 4021739519, ++STORE, 4021739520, 4021747711, ++STORE, 4021747712, 4021829631, ++STORE, 4021829632, 4021866495, ++STORE, 4021866496, 4021919743, ++STORE, 4021919744, 4021927935, ++STORE, 4021927936, 4021932031, ++STORE, 4021932032, 4021944319, ++STORE, 4021944320, 4022157311, ++STORE, 4022157312, 4022161407, ++STORE, 4022161408, 4022173695, ++STORE, 4022173696, 4022177791, ++STORE, 4022177792, 4022472703, ++STORE, 4022472704, 4022509567, ++STORE, 4022509568, 4022583295, ++STORE, 4022583296, 4022587391, ++STORE, 4022587392, 4022591487, ++STORE, 4022591488, 4022607871, ++STORE, 4022607872, 4022657023, ++STORE, 4022657024, 4022722559, ++STORE, 4022722560, 4022730751, ++STORE, 4022730752, 4022734847, ++STORE, 4022734848, 4022865919, ++STORE, 4022865920, 4022943743, ++STORE, 4022943744, 4023062527, ++STORE, 4023062528, 4023074815, ++STORE, 4023074816, 4023078911, ++STORE, 4023078912, 4023128063, ++STORE, 4023128064, 4023218175, ++STORE, 4023218176, 4023361535, ++STORE, 4023361536, 4023373823, ++STORE, 4023373824, 4023377919, ++STORE, 4023377920, 4023558143, ++STORE, 4023558144, 4023631871, ++STORE, 4023631872, 4023816191, ++STORE, 4023816192, 4023820287, ++STORE, 4023820288, 4023824383, ++STORE, 4023824384, 4023832575, ++STORE, 4023832576, 4024078335, ++STORE, 4024078336, 4024197119, ++STORE, 4024197120, 4024389631, ++STORE, 4024389632, 4024406015, ++STORE, 4024406016, 4024410111, ++STORE, 4024410112, 4024422399, ++STORE, 4024422400, 4024619007, ++STORE, 4024619008, 4024639487, ++STORE, 4024639488, 4024655871, ++STORE, 4024655872, 4024664063, ++STORE, 4024664064, 4024668159, ++STORE, 4024668160, 4024676351, ++STORE, 4024676352, 4024905727, ++STORE, 4024905728, 4024909823, ++STORE, 4024909824, 4024918015, ++STORE, 4024918016, 4024922111, ++STORE, 4024922112, 4024930303, ++STORE, 4024930304, 4025110527, ++STORE, 4025110528, 4025176063, ++STORE, 4025176064, 4025208831, ++STORE, 4025208832, 4025212927, ++STORE, 4025212928, 4025217023, ++STORE, 4025217024, 4025348095, ++STORE, 4025348096, 4025372671, ++STORE, 4025372672, 4025458687, ++STORE, 4025458688, 4025466879, ++STORE, 4025466880, 4025565183, ++STORE, 4025565184, 4025757695, ++STORE, 4025757696, 4026249215, ++STORE, 4026249216, 4026261503, ++STORE, 4026261504, 4026265599, ++STORE, 4026265600, 4026269695, ++STORE, 4026269696, 4026302463, ++STORE, 4026302464, 4026306559, ++STORE, 4026306560, 4026314751, ++STORE, 4026314752, 4026318847, ++STORE, 4026318848, 4026322943, ++STORE, 4026322944, 4026327039, ++STORE, 4026327040, 4026654719, ++STORE, 4026654720, 4026671103, ++STORE, 4026671104, 4026720255, ++STORE, 4026720256, 4026724351, ++STORE, 4026724352, 4026728447, ++STORE, 4026728448, 4026732543, ++STORE, 4026732544, 4026863615, ++STORE, 4026863616, 4027027455, ++STORE, 4027027456, 4027031551, ++STORE, 4027031552, 4027514879, ++STORE, 4027514880, 4027531263, ++STORE, 4027531264, 4027535359, ++STORE, 4027535360, 4027539455, ++STORE, 4027539456, 4027785215, ++STORE, 4027785216, 4027789311, ++STORE, 4027789312, 4027793407, ++STORE, 4027793408, 4027797503, ++STORE, 4027797504, 4027863039, ++STORE, 4027863040, 4027899903, ++STORE, 4027899904, 4027949055, ++STORE, 4027949056, 4027957247, ++STORE, 4027957248, 4027961343, ++STORE, 4027961344, 4027965439, ++STORE, 4027965440, 4028194815, ++STORE, 4028194816, 4028252159, ++STORE, 4028252160, 4028338175, ++STORE, 4028338176, 4028350463, ++STORE, 4028350464, 4028354559, ++STORE, 4028354560, 4028452863, ++STORE, 4028452864, 4028489727, ++STORE, 4028489728, 4028530687, ++STORE, 4028530688, 4028538879, ++STORE, 4028538880, 4028542975, ++STORE, 4028542976, 4028551167, ++STORE, 4028551168, 4028665855, ++STORE, 4028665856, 4029349887, ++STORE, 4029349888, 4030468095, ++STORE, 4030468096, 4030513151, ++STORE, 4030513152, 4030517247, ++STORE, 4030517248, 4030525439, ++STORE, 4030525440, 4030529535, ++STORE, 4030529536, 4030758911, ++STORE, 4030758912, 4030828543, ++STORE, 4030828544, 4030943231, ++STORE, 4030943232, 4030951423, ++STORE, 4030951424, 4030955519, ++STORE, 4030955520, 4030967807, ++STORE, 4030967808, 4031131647, ++STORE, 4031131648, 4031135743, ++STORE, 4031135744, 4031139839, ++STORE, 4031139840, 4031148031, ++STORE, 4031148032, 4031152127, ++STORE, 4031152128, 4031160319, ++STORE, 4031160320, 4031504383, ++STORE, 4031504384, 4031598591, ++STORE, 4031598592, 4031754239, ++STORE, 4031754240, 4031766527, ++STORE, 4031766528, 4031770623, ++STORE, 4031770624, 4031774719, ++STORE, 4031774720, 4031782911, ++STORE, 4031782912, 4031799295, ++STORE, 4031799296, 4031856639, ++STORE, 4031856640, 4031983615, ++STORE, 4031983616, 4031987711, ++STORE, 4031987712, 4031991807, ++STORE, 4031991808, 4032270335, ++STORE, 4032270336, 4032274431, ++STORE, 4032274432, 4032282623, ++STORE, 4032282624, 4032286719, ++STORE, 4032286720, 4032290815, ++STORE, 4032290816, 4032389119, ++STORE, 4032389120, 4032397311, ++STORE, 4032397312, 4032405503, ++STORE, 4032405504, 4032413695, ++STORE, 4032413696, 4032417791, ++STORE, 4032417792, 4032565247, ++STORE, 4032565248, 4032593919, ++STORE, 4032593920, 4032737279, ++STORE, 4032737280, 4032741375, ++STORE, 4032741376, 4032745471, ++STORE, 4032745472, 4032770047, ++STORE, 4032770048, 4032933887, ++STORE, 4032933888, 4032999423, ++STORE, 4032999424, 4033032191, ++STORE, 4033032192, 4033036287, ++STORE, 4033036288, 4033040383, ++STORE, 4033040384, 4033105919, ++STORE, 4033105920, 4033396735, ++STORE, 4033396736, 4033822719, ++STORE, 4033822720, 4033839103, ++STORE, 4033839104, 4033843199, ++STORE, 4033843200, 4033851391, ++STORE, 4033851392, 4033863679, ++STORE, 4033863680, 4033880063, ++STORE, 4033880064, 4033933311, ++STORE, 4033933312, 4034023423, ++STORE, 4034023424, 4034031615, ++STORE, 4034031616, 4034035711, ++STORE, 4034035712, 4034043903, ++STORE, 4034043904, 4034142207, ++STORE, 4034142208, 4034191359, ++STORE, 4034191360, 4034260991, ++STORE, 4034260992, 4034269183, ++STORE, 4034269184, 4034273279, ++STORE, 4034273280, 4034281471, ++STORE, 4034281472, 4034412543, ++STORE, 4034412544, 4034445311, ++STORE, 4034445312, 4034490367, ++STORE, 4034490368, 4034494463, ++STORE, 4034494464, 4034498559, ++STORE, 4034498560, 4034662399, ++STORE, 4034662400, 4034666495, ++STORE, 4034666496, 4034670591, ++STORE, 4034670592, 4034674687, ++STORE, 4034674688, 4034678783, ++STORE, 4034678784, 4034682879, ++STORE, 4034682880, 4034781183, ++STORE, 4034781184, 4035043327, ++STORE, 4035043328, 4035047423, ++STORE, 4035047424, 4035055615, ++STORE, 4035055616, 4035059711, ++STORE, 4035059712, 4035063807, ++STORE, 4035063808, 4035067903, ++STORE, 4035067904, 4035100671, ++STORE, 4035100672, 4035375103, ++STORE, 4035375104, 4035383295, ++STORE, 4035383296, 4035395583, ++STORE, 4035395584, 4035399679, ++STORE, 4035399680, 4035403775, ++STORE, 4035403776, 4035407871, ++STORE, 4035407872, 4035411967, ++STORE, 4035411968, 4035477503, ++STORE, 4035477504, 4035608575, ++STORE, 4035608576, 4035641343, ++STORE, 4035641344, 4035682303, ++STORE, 4035682304, 4035686399, ++STORE, 4035686400, 4035690495, ++STORE, 4035690496, 4035694591, ++STORE, 4035694592, 4035743743, ++STORE, 4035743744, 4035784703, ++STORE, 4035784704, 4035829759, ++STORE, 4035829760, 4035837951, ++STORE, 4035837952, 4035842047, ++STORE, 4035842048, 4035846143, ++STORE, 4035846144, 4035850239, ++STORE, 4035850240, 4036001791, ++STORE, 4036001792, 4036005887, ++STORE, 4036005888, 4036214783, ++STORE, 4036214784, 4036218879, ++STORE, 4036218880, 4036603903, ++STORE, 4036603904, 4036648959, ++STORE, 4036648960, 4036653055, ++STORE, 4036653056, 4036657151, ++STORE, 4036657152, 4036665343, ++STORE, 4036665344, 4036780031, ++STORE, 4036780032, 4036829183, ++STORE, 4036829184, 4036984831, ++STORE, 4036984832, 4036993023, ++STORE, 4036993024, 4036997119, ++STORE, 4036997120, 4037001215, ++STORE, 4037001216, 4037009407, ++STORE, 4037009408, 4037025791, ++STORE, 4037025792, 4037095423, ++STORE, 4037095424, 4037181439, ++STORE, 4037181440, 4037193727, ++STORE, 4037193728, 4037197823, ++STORE, 4037197824, 4037206015, ++STORE, 4037206016, 4037320703, ++STORE, 4037320704, 4037337087, ++STORE, 4037337088, 4037349375, ++STORE, 4037349376, 4037357567, ++STORE, 4037357568, 4037361663, ++STORE, 4037369856, 4037386239, ++STORE, 4037386240, 4037672959, ++STORE, 4037672960, 4037689343, ++STORE, 4037689344, 4037730303, ++STORE, 4037730304, 4037734399, ++STORE, 4037734400, 4037738495, ++STORE, 4037738496, 4037742591, ++STORE, 4037742592, 4037758975, ++STORE, 4037758976, 4037890047, ++STORE, 4037890048, 4037931007, ++STORE, 4037931008, 4037976063, ++STORE, 4037976064, 4037984255, ++STORE, 4037984256, 4037988351, ++STORE, 4037988352, 4038053887, ++STORE, 4038053888, 4038184959, ++STORE, 4038184960, 4038189055, ++STORE, 4038189056, 4038197247, ++STORE, 4038197248, 4038201343, ++STORE, 4038201344, 4038205439, ++STORE, 4038205440, 4038209535, ++STORE, 4038217728, 4038250495, ++STORE, 4038250496, 4038512639, ++STORE, 4038512640, 4038516735, ++STORE, 4038516736, 4038520831, ++STORE, 4038520832, 4038524927, ++STORE, 4038524928, 4038529023, ++STORE, 4038529024, 4038533119, ++STORE, 4038541312, 4038623231, ++STORE, 4038623232, 4038754303, ++STORE, 4038754304, 4038885375, ++STORE, 4038885376, 4038889471, ++STORE, 4038897664, 4038963199, ++STORE, 4038963200, 4038967295, ++STORE, 4038967296, 4038983679, ++STORE, 4038983680, 4039114751, ++STORE, 4039114752, 4039245823, ++STORE, 4039245824, 4039376895, ++STORE, 4039376896, 4040687615, ++STORE, 4040687616, 4040691711, ++STORE, 4040691712, 4040806399, ++STORE, 4040806400, 4040937471, ++STORE, 4040937472, 4040941567, ++STORE, 4040945664, 4040949759, ++STORE, 4040949760, 4041080831, ++STORE, 4041080832, 4041211903, ++STORE, 4041211904, 4043046911, ++STORE, 4043046912, 4043051007, ++STORE, 4043051008, 4043055103, ++STORE, 4043055104, 4043137023, ++STORE, 4043137024, 4043141119, ++STORE, 4043141120, 4043145215, ++STORE, 4043145216, 4043153407, ++STORE, 4043153408, 4043186175, ++STORE, 4043186176, 4043317247, ++STORE, 4043317248, 4043448319, ++STORE, 4043448320, 4043579391, ++STORE, 4043579392, 4043583487, ++STORE, 4043583488, 4043599871, ++STORE, 4043599872, 4043661311, ++STORE, 4043661312, 4043792383, ++STORE, 4043792384, 4043796479, ++STORE, 4043796480, 4043800575, ++STORE, 4043800576, 4043816959, ++STORE, 4043816960, 4043821055, ++STORE, 4043821056, 4043825151, ++STORE, 4043825152, 4043829247, ++STORE, 4043829248, 4043833343, ++STORE, 4043833344, 4047241215, ++STORE, 4047241216, 4047249407, ++STORE, 4047249408, 4047253503, ++STORE, 4047253504, 4047323135, ++STORE, 4047323136, 4047327231, ++STORE, 4047327232, 4047458303, ++STORE, 4047458304, 4047589375, ++STORE, 4047589376, 4047720447, ++STORE, 4047720448, 4047773695, ++STORE, 4047773696, 4047790079, ++STORE, 4047790080, 4047921151, ++STORE, 4047921152, 4048052223, ++STORE, 4048052224, 4048183295, ++STORE, 4048183296, 4049002495, ++STORE, 4049002496, 4049133567, ++STORE, 4049133568, 4049154047, ++STORE, 4049154048, 4049158143, ++STORE, 4049158144, 4049162239, ++STORE, 4049162240, 4049166335, ++STORE, 4049166336, 4049174527, ++STORE, 4049174528, 4049182719, ++STORE, 4049182720, 4049186815, ++STORE, 4049186816, 4049190911, ++STORE, 4049190912, 4049195007, ++STORE, 4049195008, 4049203199, ++STORE, 4049203200, 4049207295, ++STORE, 4049207296, 4049211391, ++STORE, 4049211392, 4049215487, ++STORE, 4049215488, 4049219583, ++STORE, 4049219584, 4049227775, ++STORE, 4049227776, 4049231871, ++STORE, 4049231872, 4049235967, ++STORE, 4049235968, 4049244159, ++STORE, 4049244160, 4049248255, ++STORE, 4049248256, 4049252351, ++STORE, 4049252352, 4049256447, ++STORE, 4049256448, 4049268735, ++STORE, 4049268736, 4049272831, ++STORE, 4049272832, 4049313791, ++STORE, 4049313792, 4049723391, ++STORE, 4049723392, 4049727487, ++STORE, 4049727488, 4049858559, ++STORE, 4049858560, 4049989631, ++STORE, 4049989632, 4049993727, ++STORE, 4049993728, 4050026495, ++STORE, 4050026496, 4050030591, ++STORE, 4050030592, 4050161663, ++STORE, 4050161664, 4050169855, ++STORE, 4050169856, 4050223103, ++STORE, 4050223104, 4050632703, ++STORE, 4050632704, 4050636799, ++STORE, 4050636800, 4050640895, ++STORE, 4050640896, 4050644991, ++STORE, 4050644992, 4050661375, ++STORE, 4050661376, 4050665471, ++STORE, 4050665472, 4050673663, ++STORE, 4050673664, 4050677759, ++STORE, 4050677760, 4050694143, ++STORE, 4050694144, 4050702335, ++STORE, 4050702336, 4050956287, ++STORE, 4050956288, 4051963903, ++STORE, 4051963904, 4051980287, ++STORE, 4051980288, 4051988479, ++STORE, 4051988480, 4052000767, ++STORE, 4052000768, 4052004863, ++STORE, 4052004864, 4052029439, ++STORE, 4284014592, 4284018687, ++STORE, 4284018688, 4292403199, ++SNULL, 4041080832, 4041211903, ++SNULL, 3795763200, 3795894271, ++STORE, 3629522944, 3696631807, ++SNULL, 3663077375, 3696631807, ++STORE, 3629522944, 3663077375, ++STORE, 3663077376, 3696631807, ++SNULL, 3663077376, 3696631807, ++STORE, 3663077376, 3696631807, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3626471424, 3627524095, ++SNULL, 3626471424, 3626475519, ++STORE, 3626475520, 3627524095, ++STORE, 3626471424, 3626475519, ++SNULL, 3627519999, 3627524095, ++STORE, 3626475520, 3627519999, ++STORE, 3627520000, 3627524095, ++STORE, 3625418752, 3626475519, ++SNULL, 3625418752, 3625422847, ++STORE, 3625422848, 3626475519, ++STORE, 3625418752, 3625422847, ++SNULL, 3626467327, 3626475519, ++STORE, 3625422848, 3626467327, ++STORE, 3626467328, 3626475519, ++STORE, 3624366080, 3625422847, ++SNULL, 3624366080, 3624370175, ++STORE, 3624370176, 3625422847, ++STORE, 3624366080, 3624370175, ++SNULL, 3625414655, 3625422847, ++STORE, 3624370176, 3625414655, ++STORE, 3625414656, 3625422847, ++STORE, 4041191424, 4041211903, ++SNULL, 4041195519, 4041211903, ++STORE, 4041191424, 4041195519, ++STORE, 4041195520, 4041211903, ++STORE, 4041170944, 4041191423, ++SNULL, 4041175039, 4041191423, ++STORE, 4041170944, 4041175039, ++STORE, 4041175040, 4041191423, ++SNULL, 3625426943, 3626467327, ++STORE, 3625422848, 3625426943, ++STORE, 3625426944, 3626467327, ++STORE, 4041162752, 4041170943, ++SNULL, 3626479615, 3627519999, ++STORE, 3626475520, 3626479615, ++STORE, 3626479616, 3627519999, ++STORE, 4041154560, 4041162751, ++STORE, 4041154560, 4041170943, ++STORE, 4041134080, 4041154559, ++SNULL, 4041138175, 4041154559, ++STORE, 4041134080, 4041138175, ++STORE, 4041138176, 4041154559, ++SNULL, 3624374271, 3625414655, ++STORE, 3624370176, 3624374271, ++STORE, 3624374272, 3625414655, ++STORE, 4041125888, 4041134079, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3487174656, 3487584255, ++STORE, 4041121792, 4041125887, ++SNULL, 4041121792, 4041125887, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 3487174656, 3487584255, ++STORE, 3222274048, 3223326719, ++SNULL, 3222274048, 3222278143, ++STORE, 3222278144, 3223326719, ++STORE, 3222274048, 3222278143, ++SNULL, 3223322623, 3223326719, ++STORE, 3222278144, 3223322623, ++STORE, 3223322624, 3223326719, ++STORE, 3221221376, 3222278143, ++SNULL, 3221221376, 3221225471, ++STORE, 3221225472, 3222278143, ++STORE, 3221221376, 3221225471, ++SNULL, 3222269951, 3222278143, ++STORE, 3221225472, 3222269951, ++STORE, 3222269952, 3222278143, ++STORE, 3220168704, 3221225471, ++SNULL, 3220168704, 3220172799, ++STORE, 3220172800, 3221225471, ++STORE, 3220168704, 3220172799, ++SNULL, 3221217279, 3221225471, ++STORE, 3220172800, 3221217279, ++STORE, 3221217280, 3221225471, ++STORE, 4041117696, 4041125887, ++STORE, 4041117696, 4041134079, ++STORE, 3219083264, 3220172799, ++SNULL, 3219083264, 3219087359, ++STORE, 3219087360, 3220172799, ++STORE, 3219083264, 3219087359, ++SNULL, 3220164607, 3220172799, ++STORE, 3219087360, 3220164607, ++STORE, 3220164608, 3220172799, ++STORE, 4041109504, 4041117695, ++STORE, 4041109504, 4041134079, ++STORE, 3217997824, 3219087359, ++SNULL, 3217997824, 3218001919, ++STORE, 3218001920, 3219087359, ++STORE, 3217997824, 3218001919, ++SNULL, 3219079167, 3219087359, ++STORE, 3218001920, 3219079167, ++STORE, 3219079168, 3219087359, ++STORE, 4041101312, 4041109503, ++STORE, 4041101312, 4041134079, ++STORE, 3216912384, 3218001919, ++SNULL, 3216912384, 3216916479, ++STORE, 3216916480, 3218001919, ++STORE, 3216912384, 3216916479, ++SNULL, 3217993727, 3218001919, ++STORE, 3216916480, 3217993727, ++STORE, 3217993728, 3218001919, ++STORE, 4041093120, 4041101311, ++STORE, 4041093120, 4041134079, ++STORE, 3215826944, 3216916479, ++SNULL, 3215826944, 3215831039, ++STORE, 3215831040, 3216916479, ++STORE, 3215826944, 3215831039, ++SNULL, 3216908287, 3216916479, ++STORE, 3215831040, 3216908287, ++STORE, 3216908288, 3216916479, ++STORE, 4016779264, 4016799743, ++SNULL, 4016783359, 4016799743, ++STORE, 4016779264, 4016783359, ++STORE, 4016783360, 4016799743, ++STORE, 4016758784, 4016779263, ++SNULL, 4016762879, 4016779263, ++STORE, 4016758784, 4016762879, ++STORE, 4016762880, 4016779263, ++SNULL, 3222282239, 3223322623, ++STORE, 3222278144, 3222282239, ++STORE, 3222282240, 3223322623, ++STORE, 4041084928, 4041093119, ++STORE, 4041084928, 4041134079, ++SNULL, 3221229567, 3222269951, ++STORE, 3221225472, 3221229567, ++STORE, 3221229568, 3222269951, ++STORE, 4015644672, 4015665151, ++STORE, 4038889472, 4038897663, ++SNULL, 4015648767, 4015665151, ++STORE, 4015644672, 4015648767, ++STORE, 4015648768, 4015665151, ++STORE, 4015624192, 4015644671, ++SNULL, 4015628287, 4015644671, ++STORE, 4015624192, 4015628287, ++STORE, 4015628288, 4015644671, ++SNULL, 3219091455, 3220164607, ++STORE, 3219087360, 3219091455, ++STORE, 3219091456, 3220164607, ++STORE, 4015603712, 4015624191, ++SNULL, 4015607807, 4015624191, ++STORE, 4015603712, 4015607807, ++STORE, 4015607808, 4015624191, ++SNULL, 3218006015, 3219079167, ++STORE, 3218001920, 3218006015, ++STORE, 3218006016, 3219079167, ++STORE, 3949674496, 3949694975, ++SNULL, 3949678591, 3949694975, ++STORE, 3949674496, 3949678591, ++STORE, 3949678592, 3949694975, ++SNULL, 3216920575, 3217993727, ++STORE, 3216916480, 3216920575, ++STORE, 3216920576, 3217993727, ++STORE, 3948924928, 3948945407, ++SNULL, 3948929023, 3948945407, ++STORE, 3948924928, 3948929023, ++STORE, 3948929024, 3948945407, ++SNULL, 3215835135, 3216908287, ++STORE, 3215831040, 3215835135, ++STORE, 3215835136, 3216908287, ++SNULL, 3220176895, 3221217279, ++STORE, 3220172800, 3220176895, ++STORE, 3220176896, 3221217279, ++STORE, 3214786560, 3215826943, ++STORE, 3213733888, 3214786559, ++SNULL, 3213733888, 3213737983, ++STORE, 3213737984, 3214786559, ++STORE, 3213733888, 3213737983, ++SNULL, 3214782463, 3214786559, ++STORE, 3213737984, 3214782463, ++STORE, 3214782464, 3214786559, ++STORE, 4038533120, 4038541311, ++STORE, 3948421120, 3948441599, ++SNULL, 3948425215, 3948441599, ++STORE, 3948421120, 3948425215, ++STORE, 3948425216, 3948441599, ++SNULL, 3213742079, 3214782463, ++STORE, 3213737984, 3213742079, ++STORE, 3213742080, 3214782463, ++STORE, 4038209536, 4038217727, ++STORE, 3212681216, 3213737983, ++SNULL, 3212681216, 3212685311, ++STORE, 3212685312, 3213737983, ++STORE, 3212681216, 3212685311, ++SNULL, 3213729791, 3213737983, ++STORE, 3212685312, 3213729791, ++STORE, 3213729792, 3213737983, ++STORE, 3795763200, 3795894271, ++STORE, 3946872832, 3946893311, ++SNULL, 3946876927, 3946893311, ++STORE, 3946872832, 3946876927, ++STORE, 3946876928, 3946893311, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3487174656, 3487584255, ++SNULL, 3212689407, 3213729791, ++STORE, 3212685312, 3212689407, ++STORE, 3212689408, 3213729791, ++STORE, 4041080832, 4041084927, ++STORE, 4040941568, 4040945663, ++STORE, 4037361664, 4037369855, ++STORE, 4000817152, 4000821247, ++STORE, 3999440896, 3999444991, ++STORE, 3212161024, 3212681215, ++SNULL, 3212161024, 3212439551, ++STORE, 3212439552, 3212681215, ++STORE, 3212161024, 3212439551, ++SNULL, 3212161024, 3212439551, ++SNULL, 3212464127, 3212681215, ++STORE, 3212439552, 3212464127, ++STORE, 3212464128, 3212681215, ++SNULL, 3212464128, 3212681215, ++SNULL, 3212439552, 3212451839, ++STORE, 3212451840, 3212464127, ++STORE, 3212439552, 3212451839, ++SNULL, 3212439552, 3212451839, ++STORE, 3212439552, 3212451839, ++SNULL, 3212451840, 3212455935, ++STORE, 3212455936, 3212464127, ++STORE, 3212451840, 3212455935, ++SNULL, 3212451840, 3212455935, ++STORE, 3212451840, 3212455935, ++SNULL, 3212455936, 3212460031, ++STORE, 3212460032, 3212464127, ++STORE, 3212455936, 3212460031, ++SNULL, 3212455936, 3212460031, ++STORE, 3212455936, 3212460031, ++SNULL, 3212460032, 3212464127, ++STORE, 3212460032, 3212464127, ++STORE, 3997679616, 3997683711, ++SNULL, 4049235968, 4049240063, ++STORE, 4049240064, 4049244159, ++STORE, 4049235968, 4049240063, ++SNULL, 4049240064, 4049244159, ++STORE, 4049240064, 4049244159, ++SNULL, 3997679616, 3997683711, ++SNULL, 3999440896, 3999444991, ++SNULL, 4000817152, 4000821247, ++SNULL, 4040941568, 4040945663, ++SNULL, 4041080832, 4041084927, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 3487174656, 3487584255, ++SNULL, 3212451840, 3212455935, ++STORE, 3212451840, 3212455935, ++STORE, 4041080832, 4041084927, ++STORE, 3623890944, 3624169471, ++SNULL, 4041080832, 4041084927, ++STORE, 4041080832, 4041084927, ++SNULL, 4041080832, 4041084927, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 4041080832, 4041084927, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3211386880, 3212439551, ++SNULL, 3211386880, 3211390975, ++STORE, 3211390976, 3212439551, ++STORE, 3211386880, 3211390975, ++SNULL, 3212435455, 3212439551, ++STORE, 3211390976, 3212435455, ++STORE, 3212435456, 3212439551, ++STORE, 4040941568, 4040945663, ++STORE, 3937169408, 3937189887, ++STORE, 3623485440, 3623616511, ++SNULL, 717225983, 1388314623, ++STORE, 314572800, 717225983, ++STORE, 717225984, 1388314623, ++SNULL, 717225984, 1388314623, ++STORE, 3937112064, 3937132543, ++SNULL, 3937116159, 3937132543, ++STORE, 3937112064, 3937116159, ++STORE, 3937116160, 3937132543, ++SNULL, 3211395071, 3212435455, ++STORE, 3211390976, 3211395071, ++STORE, 3211395072, 3212435455, ++STORE, 4000817152, 4000821247, ++STORE, 3974823936, 3974832127, ++STORE, 3595284480, 3595431935, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++STORE, 3487174656, 3487584255, ++STORE, 3999440896, 3999444991, ++STORE, 3997679616, 3997683711, ++STORE, 3996295168, 3996299263, ++STORE, 3996090368, 3996094463, ++STORE, 3210866688, 3211386879, ++SNULL, 3210866688, 3211001855, ++STORE, 3211001856, 3211386879, ++STORE, 3210866688, 3211001855, ++SNULL, 3210866688, 3211001855, ++SNULL, 3211038719, 3211386879, ++STORE, 3211001856, 3211038719, ++STORE, 3211038720, 3211386879, ++SNULL, 3211038720, 3211386879, ++SNULL, 3211001856, 3211022335, ++STORE, 3211022336, 3211038719, ++STORE, 3211001856, 3211022335, ++SNULL, 3211001856, 3211022335, ++STORE, 3211001856, 3211022335, ++SNULL, 3211022336, 3211030527, ++STORE, 3211030528, 3211038719, ++STORE, 3211022336, 3211030527, ++SNULL, 3211022336, 3211030527, ++STORE, 3211022336, 3211030527, ++SNULL, 3211030528, 3211034623, ++STORE, 3211034624, 3211038719, ++STORE, 3211030528, 3211034623, ++SNULL, 3211030528, 3211034623, ++STORE, 3211030528, 3211034623, ++SNULL, 3211034624, 3211038719, ++STORE, 3211034624, 3211038719, ++STORE, 3994906624, 3994910719, ++SNULL, 4049240064, 4049244159, ++STORE, 4049240064, 4049244159, ++SNULL, 3994906624, 3994910719, ++SNULL, 3996090368, 3996094463, ++SNULL, 3996295168, 3996299263, ++SNULL, 3997679616, 3997683711, ++SNULL, 3999440896, 3999444991, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 3487174656, 3487584255, ++SNULL, 3211022336, 3211030527, ++STORE, 3211022336, 3211030527, ++STORE, 3999440896, 3999444991, ++STORE, 3210199040, 3211001855, ++SNULL, 3999440896, 3999444991, ++STORE, 3999440896, 3999444991, ++SNULL, 3999440896, 3999444991, ++STORE, 3594821632, 3594952703, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 4048183296, 4048592895, ++STORE, 4048592896, 4049002495, ++STORE, 4048183296, 4048592895, ++STORE, 4048183296, 4049002495, ++SNULL, 1914101759, 1969434623, ++STORE, 1914097664, 1914101759, ++STORE, 1914101760, 1969434623, ++STORE, 3567108096, 3567239167, ++STORE, 3973832704, 3973840895, ++STORE, 3209113600, 3210199039, ++SNULL, 3209113600, 3209117695, ++STORE, 3209117696, 3210199039, ++STORE, 3209113600, 3209117695, ++SNULL, 3210194943, 3210199039, ++STORE, 3209117696, 3210194943, ++STORE, 3210194944, 3210199039, ++STORE, 3935858688, 3935879167, ++SNULL, 3935862783, 3935879167, ++STORE, 3935858688, 3935862783, ++STORE, 3935862784, 3935879167, ++SNULL, 3209121791, 3210194943, ++STORE, 3209117696, 3209121791, ++STORE, 3209121792, 3210194943, ++STORE, 3528749056, 3528880127, ++STORE, 3968200704, 3968208895, ++STORE, 3208028160, 3209117695, ++SNULL, 3208028160, 3208032255, ++STORE, 3208032256, 3209117695, ++STORE, 3208028160, 3208032255, ++SNULL, 3209109503, 3209117695, ++STORE, 3208032256, 3209109503, ++STORE, 3209109504, 3209117695, ++STORE, 3888123904, 3888144383, ++SNULL, 3888127999, 3888144383, ++STORE, 3888123904, 3888127999, ++STORE, 3888128000, 3888144383, ++SNULL, 3208036351, 3209109503, ++STORE, 3208032256, 3208036351, ++STORE, 3208036352, 3209109503, ++SNULL, 3968200704, 3968208895, ++SNULL, 3888123904, 3888144383, ++SNULL, 3209109504, 3209113599, ++STORE, 3209113600, 3209117695, ++STORE, 3209109504, 3209113599, ++SNULL, 3208028160, 3209113599, ++STORE, 3208060928, 3209117695, ++SNULL, 3208060928, 3208065023, ++STORE, 3208065024, 3209117695, ++STORE, 3208060928, 3208065023, ++SNULL, 3209109503, 3209117695, ++STORE, 3208065024, 3209109503, ++STORE, 3209109504, 3209117695, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3888123904, 3888144383, ++SNULL, 3888127999, 3888144383, ++STORE, 3888123904, 3888127999, ++STORE, 3888128000, 3888144383, ++SNULL, 3208069119, 3209109503, ++STORE, 3208065024, 3208069119, ++STORE, 3208069120, 3209109503, ++STORE, 3968200704, 3968208895, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3527778304, 3527909375, ++STORE, 3999440896, 3999444991, ++STORE, 3997679616, 3997683711, ++STORE, 1914097664, 1914105855, ++STORE, 1914105856, 1969434623, ++STORE, 3957583872, 3957592063, ++STORE, 3206975488, 3208065023, ++SNULL, 3206975488, 3206979583, ++STORE, 3206979584, 3208065023, ++STORE, 3206975488, 3206979583, ++SNULL, 3208056831, 3208065023, ++STORE, 3206979584, 3208056831, ++STORE, 3208056832, 3208065023, ++STORE, 3956736000, 3956744191, ++STORE, 3205890048, 3206979583, ++SNULL, 3205890048, 3205894143, ++STORE, 3205894144, 3206979583, ++STORE, 3205890048, 3205894143, ++SNULL, 3206971391, 3206979583, ++STORE, 3205894144, 3206971391, ++STORE, 3206971392, 3206979583, ++STORE, 3806101504, 3806121983, ++SNULL, 3806105599, 3806121983, ++STORE, 3806101504, 3806105599, ++STORE, 3806105600, 3806121983, ++SNULL, 3206983679, 3208056831, ++STORE, 3206979584, 3206983679, ++STORE, 3206983680, 3208056831, ++STORE, 3806081024, 3806101503, ++SNULL, 3806085119, 3806101503, ++STORE, 3806081024, 3806085119, ++STORE, 3806085120, 3806101503, ++SNULL, 3205898239, 3206971391, ++STORE, 3205894144, 3205898239, ++STORE, 3205898240, 3206971391, ++STORE, 3956015104, 3956023295, ++STORE, 3204804608, 3205894143, ++SNULL, 3204804608, 3204808703, ++STORE, 3204808704, 3205894143, ++STORE, 3204804608, 3204808703, ++SNULL, 3205885951, 3205894143, ++STORE, 3204808704, 3205885951, ++STORE, 3205885952, 3205894143, ++STORE, 3803471872, 3803492351, ++STORE, 3803451392, 3803471871, ++STORE, 3803451392, 3803492351, ++SNULL, 3957583872, 3957592063, ++SNULL, 3806101504, 3806121983, ++SNULL, 3206975487, 3206979583, ++STORE, 3206971392, 3206975487, ++STORE, 3206975488, 3206979583, ++SNULL, 3208056832, 3208060927, ++STORE, 3208060928, 3208065023, ++STORE, 3208056832, 3208060927, ++SNULL, 3206975488, 3208060927, ++STORE, 3801845760, 3801878527, ++STORE, 3806101504, 3806121983, ++SNULL, 3806105599, 3806121983, ++STORE, 3806101504, 3806105599, ++STORE, 3806105600, 3806121983, ++SNULL, 3204812799, 3205885951, ++STORE, 3204808704, 3204812799, ++STORE, 3204812800, 3205885951, ++STORE, 1914097664, 1914109951, ++STORE, 1914109952, 1969434623, ++STORE, 3957583872, 3957592063, ++STORE, 3206971392, 3208065023, ++SNULL, 3206971392, 3206979583, ++STORE, 3206979584, 3208065023, ++STORE, 3206971392, 3206979583, ++SNULL, 3208056831, 3208065023, ++STORE, 3206979584, 3208056831, ++STORE, 3208056832, 3208065023, ++STORE, 3801825280, 3801845759, ++SNULL, 3801829375, 3801845759, ++STORE, 3801825280, 3801829375, ++STORE, 3801829376, 3801845759, ++SNULL, 3206983679, 3208056831, ++STORE, 3206979584, 3206983679, ++STORE, 3206983680, 3208056831, ++STORE, 3202707456, 3204804607, ++SNULL, 3202707456, 3204804607, ++STORE, 3202707456, 3204804607, ++STORE, 3200610304, 3202707455, ++SNULL, 3202707456, 3204804607, ++SNULL, 3200610304, 3202707455, ++STORE, 3202707456, 3204804607, ++SNULL, 3202707456, 3204804607, ++STORE, 3202707456, 3204804607, ++SNULL, 3202707456, 3204804607, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3527647232, 3527778303, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++STORE, 3487059968, 3487584255, ++SNULL, 3487059968, 3487301631, ++STORE, 3487301632, 3487584255, ++STORE, 3487059968, 3487301631, ++SNULL, 3487059968, 3487301631, ++SNULL, 3487563775, 3487584255, ++STORE, 3487301632, 3487563775, ++STORE, 3487563776, 3487584255, ++SNULL, 3487563776, 3487584255, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3524046848, 3524177919, ++STORE, 3487170560, 3487301631, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3487039488, 3487170559, ++STORE, 3487039488, 3487301631, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3204280320, 3204804607, ++SNULL, 3204280320, 3204448255, ++STORE, 3204448256, 3204804607, ++STORE, 3204280320, 3204448255, ++SNULL, 3204280320, 3204448255, ++SNULL, 3204710399, 3204804607, ++STORE, 3204448256, 3204710399, ++STORE, 3204710400, 3204804607, ++SNULL, 3204710400, 3204804607, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3996295168, 3996299263, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++SNULL, 3996295168, 3996299263, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3486908416, 3487039487, ++STORE, 3486908416, 3487301631, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3223326720, 3290435583, ++SNULL, 3223326720, 3256881151, ++STORE, 3256881152, 3290435583, ++STORE, 3223326720, 3256881151, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++STORE, 3201826816, 3202351103, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++STORE, 3202351104, 3204448255, ++SNULL, 3202351104, 3204448255, ++SNULL, 3803471871, 3803492351, ++STORE, 3803451392, 3803471871, ++STORE, 3803471872, 3803492351, ++SNULL, 3803471872, 3803492351, ++SNULL, 3803451392, 3803471871, ++STORE, 3798999040, 3799101439, ++SNULL, 3798999040, 3799101439, ++STORE, 3952644096, 3952652287, ++STORE, 3203362816, 3204448255, ++SNULL, 3203362816, 3203366911, ++STORE, 3203366912, 3204448255, ++STORE, 3203362816, 3203366911, ++SNULL, 3204444159, 3204448255, ++STORE, 3203366912, 3204444159, ++STORE, 3204444160, 3204448255, ++STORE, 3803471872, 3803492351, ++SNULL, 3803475967, 3803492351, ++STORE, 3803471872, 3803475967, ++STORE, 3803475968, 3803492351, ++SNULL, 3203371007, 3204444159, ++STORE, 3203366912, 3203371007, ++STORE, 3203371008, 3204444159, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3199729664, 3201826815, ++SNULL, 3199729664, 3201826815, ++STORE, 3200774144, 3201826815, ++SNULL, 3200774144, 3200778239, ++STORE, 3200778240, 3201826815, ++STORE, 3200774144, 3200778239, ++SNULL, 3201822719, 3201826815, ++STORE, 3200778240, 3201822719, ++STORE, 3201822720, 3201826815, ++STORE, 3803451392, 3803471871, ++SNULL, 3803455487, 3803471871, ++STORE, 3803451392, 3803455487, ++STORE, 3803455488, 3803471871, ++SNULL, 3200782335, 3201822719, ++STORE, 3200778240, 3200782335, ++STORE, 3200782336, 3201822719, ++STORE, 3949666304, 3949674495, ++STORE, 3949408256, 3949416447, ++STORE, 3199688704, 3200778239, ++SNULL, 3199688704, 3199692799, ++STORE, 3199692800, 3200778239, ++STORE, 3199688704, 3199692799, ++SNULL, 3200770047, 3200778239, ++STORE, 3199692800, 3200770047, ++STORE, 3200770048, 3200778239, ++STORE, 3799306240, 3799326719, ++SNULL, 3799310335, 3799326719, ++STORE, 3799306240, 3799310335, ++STORE, 3799310336, 3799326719, ++SNULL, 3199696895, 3200770047, ++STORE, 3199692800, 3199696895, ++STORE, 3199696896, 3200770047, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++STORE, 3799277568, 3799306239, ++SNULL, 3799277568, 3799306239, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++SNULL, 4041162751, 4041170943, ++STORE, 4041154560, 4041162751, ++STORE, 4041162752, 4041170943, ++SNULL, 4041162752, 4041170943, ++SNULL, 4041154560, 4041162751, ++SNULL, 4041191424, 4041211903, ++SNULL, 4041170944, 4041191423, ++SNULL, 3626471423, 3626475519, ++STORE, 3626467328, 3626471423, ++STORE, 3626471424, 3626475519, ++SNULL, 3626471424, 3627524095, ++SNULL, 3625418751, 3625422847, ++STORE, 3625414656, 3625418751, ++STORE, 3625418752, 3625422847, ++SNULL, 3625418752, 3626471423, ++STORE, 3627393024, 3627524095, ++STORE, 3627261952, 3627393023, ++STORE, 3627261952, 3627524095, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++STORE, 3195494400, 3197591551, ++SNULL, 3197591552, 3199688703, ++SNULL, 3195494400, 3197591551, ++STORE, 3197591552, 3199688703, ++SNULL, 3197591552, 3199688703, ++STORE, 3197591552, 3199688703, ++STORE, 3195494400, 3197591551, ++SNULL, 3197591552, 3199688703, ++SNULL, 3195494400, 3197591551, ++STORE, 3798999040, 3799101439, ++SNULL, 3798999040, 3799101439, ++/* ++ * mmap: unmapped_area_topdown: ffff9a9f14ddaa80 ++ * Gap was found: mt 4041162752 gap_end 4041183232 ++ * mmap: window was 4052029440 - 4096 size 28672 ++ * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423 ++ * mmap: mas.index 4041162752 align mask 0 offset 0 ++ * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678) ++ */ ++ }; ++ ++ unsigned long set43[] = { ++STORE, 140737488347136, 140737488351231, ++STORE, 140734187720704, 140737488351231, ++SNULL, 140734187724800, 140737488351231, ++STORE, 140734187589632, 140734187724799, ++STORE, 4194304, 6443007, ++STORE, 4337664, 6443007, ++STORE, 4194304, 4337663, ++SNULL, 4337664, 6443007, ++STORE, 6430720, 6443007, ++STORE, 206158430208, 206160674815, ++STORE, 206158569472, 206160674815, ++STORE, 206158430208, 206158569471, ++SNULL, 206158569472, 206160674815, ++STORE, 206160662528, 206160670719, ++STORE, 206160670720, 206160674815, ++STORE, 140734188756992, 140734188765183, ++STORE, 140734188740608, 140734188756991, ++STORE, 140501948112896, 140501948116991, ++ }; ++ ++ int count = 0; ++ void *ptr = NULL; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ mt_set_non_kernel(3); ++ check_erase2_testset(mt, set, ARRAY_SIZE(set)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set2, ARRAY_SIZE(set2)); ++ start = 140735933894656; ++ MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL)); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(2); ++ mt_init_flags(mt, 0); ++ check_erase2_testset(mt, set3, ARRAY_SIZE(set3)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, 0); ++ check_erase2_testset(mt, set4, ARRAY_SIZE(set4)); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (xa_is_zero(entry)) ++ continue; ++ } ++ rcu_read_unlock(); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mt_set_non_kernel(100); ++ check_erase2_testset(mt, set5, ARRAY_SIZE(set5)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set6, ARRAY_SIZE(set6)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set7, ARRAY_SIZE(set7)); ++ rcu_barrier(); ++ mtree_destroy(mt); + -+ VM_WARN_ON_ONCE(!current_is_kswapd()); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set8, ARRAY_SIZE(set8)); ++ rcu_barrier(); ++ mtree_destroy(mt); + -+ sc->last_reclaimed = sc->nr_reclaimed; ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set9, ARRAY_SIZE(set9)); ++ rcu_barrier(); ++ mtree_destroy(mt); + -+ /* -+ * To reduce the chance of going into the aging path, which can be -+ * costly, optimistically skip it if the flag below was cleared in the -+ * eviction path. This improves the overall performance when multiple -+ * memcgs are available. -+ */ -+ if (!sc->memcgs_need_aging) { -+ sc->memcgs_need_aging = true; -+ return; ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set10, ARRAY_SIZE(set10)); ++ rcu_barrier(); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set11, ARRAY_SIZE(set11)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); ++ MT_BUG_ON(mt, mas.last != 140014592573439); ++ mtree_destroy(mt); ++ ++ mas_reset(&mas); ++ mas.tree = mt; ++ count = 0; ++ mas.index = 0; ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set12, ARRAY_SIZE(set12)); ++ rcu_barrier(); ++ mas_for_each(&mas, entry, ULONG_MAX) { ++ if (xa_is_zero(entry)) ++ continue; ++ BUG_ON(count > 12); ++ count++; + } ++ mtree_destroy(mt); + -+ set_mm_walk(pgdat); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set13, ARRAY_SIZE(set13)); ++ mtree_erase(mt, 140373516443648); ++ rcu_read_lock(); ++ mas_empty_area_rev(&mas, 0, 140373518663680, 4096); ++ rcu_read_unlock(); ++ mtree_destroy(mt); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set14, ARRAY_SIZE(set14)); ++ rcu_barrier(); ++ mtree_destroy(mt); + -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set15, ARRAY_SIZE(set15)); ++ rcu_barrier(); ++ mtree_destroy(mt); + -+ if (age_lruvec(lruvec, sc, min_ttl)) -+ success = true; ++ /* set16 was to find a bug on limit updating at slot 0. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set16, ARRAY_SIZE(set16)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); ++ MT_BUG_ON(mt, mas.last != 139921865547775); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); + -+ cond_resched(); -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); ++ /* ++ * set17 found a bug in walking backwards and not counting nulls at ++ * the end. This could cause a gap to be missed if the null had any ++ * size. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set17, ARRAY_SIZE(set17)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); ++ MT_BUG_ON(mt, mas.last != 139953197322239); ++/* MT_BUG_ON(mt, mas.index != 139953197318144); */ ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); + -+ clear_mm_walk(); ++ /* ++ * set18 found a bug in walking backwards and not setting the max from ++ * the node, but using the parent node. This was only an issue if the ++ * next slot in the parent had what we needed. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set18, ARRAY_SIZE(set18)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); ++ MT_BUG_ON(mt, mas.last != 140222968475647); ++ /*MT_BUG_ON(mt, mas.index != 140222966259712); */ ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); + -+ /* check the order to exclude compaction-induced reclaim */ -+ if (success || !min_ttl || sc->order) -+ return; ++ /* ++ * set19 found 2 bugs in prev. ++ * 1. If we hit root without finding anything, then there was an ++ * infinite loop. ++ * 2. The first ascending wasn't using the correct slot which may have ++ * caused missed entries. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set19, ARRAY_SIZE(set19)); ++ rcu_barrier(); ++ mas.index = 140656779083776; ++ entry = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(140656779083776)); ++ entry = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, entry != xa_mk_value(140656766251008)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); + + /* -+ * The main goal is to OOM kill if every generation from all memcgs is -+ * younger than min_ttl. However, another possibility is all memcgs are -+ * either below min or empty. ++ * set20 found a bug in mas_may_move_gap due to the slot being ++ * overwritten during the __mas_add operation and setting it to zero. + */ -+ if (mutex_trylock(&oom_lock)) { -+ struct oom_control oc = { -+ .gfp_mask = sc->gfp_mask, -+ }; ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set20, ARRAY_SIZE(set20)); ++ rcu_barrier(); ++ check_load(mt, 94849009414144, NULL); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); + -+ out_of_memory(&oc); ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set21, ARRAY_SIZE(set21)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); + -+ mutex_unlock(&oom_lock); -+ } -+} ++ mt_set_non_kernel(999); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set22, ARRAY_SIZE(set22)); ++ rcu_barrier(); ++ mt_validate(mt); ++ ptr = mtree_load(mt, 140551363362816); ++ MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159)); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set23, ARRAY_SIZE(set23)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+/* -+ * This function exploits spatial locality when shrink_page_list() walks the -+ * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If -+ * the scan was done cacheline efficiently, it adds the PMD entry pointing to -+ * the PTE table to the Bloom filter. This forms a feedback loop between the -+ * eviction and the aging. -+ */ -+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) -+{ -+ int i; -+ pte_t *pte; -+ unsigned long start; -+ unsigned long end; -+ unsigned long addr; -+ struct lru_gen_mm_walk *walk; -+ int young = 0; -+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; -+ struct folio *folio = pfn_folio(pvmw->pfn); -+ struct mem_cgroup *memcg = folio_memcg(folio); -+ struct pglist_data *pgdat = folio_pgdat(folio); -+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ DEFINE_MAX_SEQ(lruvec); -+ int old_gen, new_gen = lru_gen_from_seq(max_seq); + -+ lockdep_assert_held(pvmw->ptl); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set24, ARRAY_SIZE(set24)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ if (spin_is_contended(pvmw->ptl)) -+ return; ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set25, ARRAY_SIZE(set25)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Split on NULL followed by delete - causes gap issues. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set26, ARRAY_SIZE(set26)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); ++ MT_BUG_ON(mt, mas.last != 140109040959487); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Split on NULL followed by delete - causes gap issues. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set27, ARRAY_SIZE(set27)); ++ rcu_barrier(); ++ MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336)); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set28, ARRAY_SIZE(set28)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); ++ /* Search for the size of gap then align it (offset 0) */ ++ mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); ++ MT_BUG_ON(mt, mas.index != 139918401601536); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* This test found issues with retry moving rebalanced nodes so the ++ * incorrect parent pivot was updated. ++ */ ++ mt_set_non_kernel(999); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set29, ARRAY_SIZE(set29)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ /* avoid taking the LRU lock under the PTL when possible */ -+ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; ++ /* This test found issues with deleting all entries in a node when ++ * surrounded by entries in the next nodes, then deleting the entries ++ * surrounding the node filled with deleted entries. ++ */ ++ mt_set_non_kernel(999); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set30, ARRAY_SIZE(set30)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); -+ end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; ++ /* This test found an issue with deleting all entries in a node that was ++ * the end node and mas_gap incorrectly set next = curr, and curr = prev ++ * then moved next to the left, losing data. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set31, ARRAY_SIZE(set31)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set32, ARRAY_SIZE(set32)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { -+ if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) -+ end = start + MIN_LRU_BATCH * PAGE_SIZE; -+ else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) -+ start = end - MIN_LRU_BATCH * PAGE_SIZE; -+ else { -+ start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; -+ end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; -+ } -+ } ++/* ++ * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found: ++ * mt 140582827569152 gap_end 140582869532672 ++ * mmap: window was 140583656296448 - 4096 size 134217728 ++ * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879 ++ * mmap: mas.index 140582827569152 align mask 0 offset 0 ++ * mmap: rb_find_vma find on ++ * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00) ++ */ + -+ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; ++ /* move gap failed due to an entirely empty node */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set33, ARRAY_SIZE(set33)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); ++ MT_BUG_ON(mt, mas.last != 140583003750399); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ rcu_read_lock(); -+ arch_enter_lazy_mmu_mode(); ++ /* ++ * Incorrect gap in tree caused by mas_prev not setting the limits ++ * correctly while walking down. ++ */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set34, ARRAY_SIZE(set34)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Empty leaf at the end of a parent caused incorrect gap. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set35, ARRAY_SIZE(set35)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(99); ++ /* Empty leaf at the end of a parent caused incorrect gap. */ ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set36, ARRAY_SIZE(set36)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { -+ unsigned long pfn; ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set37, ARRAY_SIZE(set37)); ++ rcu_barrier(); ++ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ pfn = get_pte_pfn(pte[i], pvmw->vma, addr); -+ if (pfn == -1) -+ continue; ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set38, ARRAY_SIZE(set38)); ++ rcu_barrier(); ++ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ if (!pte_young(pte[i])) -+ continue; ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); -+ if (!folio) -+ continue; ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mtree_destroy(mt); + -+ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) -+ VM_WARN_ON_ONCE(true); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set41, ARRAY_SIZE(set41)); ++ rcu_barrier(); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* move gap failed due to an entirely empty node. */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set42, ARRAY_SIZE(set42)); ++ rcu_barrier(); ++ mas_empty_area_rev(&mas, 4096, 4052029440, 28672); ++ MT_BUG_ON(mt, mas.last != 4041211903); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* gap calc off by one */ ++ mt_set_non_kernel(99); ++ mas_reset(&mas); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_erase2_testset(mt, set43, ARRAY_SIZE(set43)); ++ rcu_barrier(); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ mtree_destroy(mt); ++} + -+ young++; ++static noinline void check_alloc_rev_range(struct maple_tree *mt) ++{ ++ /* ++ * Generated by: ++ * cat /proc/self/maps | awk '{print $1}'| ++ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' ++ */ + -+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && -+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) && -+ !folio_test_swapcache(folio))) -+ folio_mark_dirty(folio); ++ unsigned long range[] = { ++ /* Inclusive , Exclusive. */ ++ 0x565234af2000, 0x565234af4000, ++ 0x565234af4000, 0x565234af9000, ++ 0x565234af9000, 0x565234afb000, ++ 0x565234afc000, 0x565234afd000, ++ 0x565234afd000, 0x565234afe000, ++ 0x565235def000, 0x565235e10000, ++ 0x7f36d4bfd000, 0x7f36d4ee2000, ++ 0x7f36d4ee2000, 0x7f36d4f04000, ++ 0x7f36d4f04000, 0x7f36d504c000, ++ 0x7f36d504c000, 0x7f36d5098000, ++ 0x7f36d5098000, 0x7f36d5099000, ++ 0x7f36d5099000, 0x7f36d509d000, ++ 0x7f36d509d000, 0x7f36d509f000, ++ 0x7f36d509f000, 0x7f36d50a5000, ++ 0x7f36d50b9000, 0x7f36d50db000, ++ 0x7f36d50db000, 0x7f36d50dc000, ++ 0x7f36d50dc000, 0x7f36d50fa000, ++ 0x7f36d50fa000, 0x7f36d5102000, ++ 0x7f36d5102000, 0x7f36d5103000, ++ 0x7f36d5103000, 0x7f36d5104000, ++ 0x7f36d5104000, 0x7f36d5105000, ++ 0x7fff5876b000, 0x7fff5878d000, ++ 0x7fff5878e000, 0x7fff58791000, ++ 0x7fff58791000, 0x7fff58793000, ++ }; + -+ old_gen = folio_lru_gen(folio); -+ if (old_gen < 0) -+ folio_set_referenced(folio); -+ else if (old_gen != new_gen) -+ __set_bit(i, bitmap); -+ } ++ unsigned long holes[] = { ++ /* ++ * Note: start of hole is INCLUSIVE ++ * end of hole is EXCLUSIVE ++ * (opposite of the above table.) ++ * Start of hole, end of hole, size of hole (+1) ++ */ ++ 0x565234afb000, 0x565234afc000, 0x1000, ++ 0x565234afe000, 0x565235def000, 0x12F1000, ++ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, ++ }; + -+ arch_leave_lazy_mmu_mode(); -+ rcu_read_unlock(); ++ /* ++ * req_range consists of 4 values. ++ * 1. min index ++ * 2. max index ++ * 3. size ++ * 4. number that should be returned. ++ * 5. return value ++ */ ++ unsigned long req_range[] = { ++ 0x565234af9000, /* Min */ ++ 0x7fff58791000, /* Max */ ++ 0x1000, /* Size */ ++ 0x7fff5878d << 12, /* First rev hole of size 0x1000 */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ 0x565234AF1 << 12, /* Max */ ++ 0x3000, /* Size */ ++ 0x565234AEE << 12, /* max - 3. */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ -1, /* Max */ ++ 0x1000, /* Size */ ++ 562949953421311 << 12,/* First rev hole of size 0x1000 */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ 0x7F36D510A << 12, /* Max */ ++ 0x4000, /* Size */ ++ 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */ ++ 0, /* Return value success. */ ++ ++ /* Ascend test. */ ++ 0x0, ++ 34148798629 << 12, ++ 19 << 12, ++ 34148797418 << 12, ++ 0x0, ++ ++ /* Too big test. */ ++ 0x0, ++ 18446744073709551615UL, ++ 562915594369134UL << 12, ++ 0x0, ++ -EBUSY, + -+ /* feedback from rmap walkers to page table walkers */ -+ if (suitable_to_scan(i, young)) -+ update_bloom_filter(lruvec, max_seq, pvmw->pmd); ++ }; + -+ if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { -+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { -+ folio = pfn_folio(pte_pfn(pte[i])); -+ folio_activate(folio); -+ } -+ return; -+ } ++ int i, range_count = ARRAY_SIZE(range); ++ int req_range_count = ARRAY_SIZE(req_range); ++ unsigned long min = 0; + -+ /* folio_update_gen() requires stable folio_memcg() */ -+ if (!mem_cgroup_trylock_pages(memcg)) -+ return; ++ MA_STATE(mas, mt, 0, 0); + -+ if (!walk) { -+ spin_lock_irq(&lruvec->lru_lock); -+ new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); -+ } ++ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, ++ GFP_KERNEL); ++#define DEBUG_REV_RANGE 0 ++ for (i = 0; i < range_count; i += 2) { ++ /* Inclusive, Inclusive (with the -1) */ + -+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { -+ folio = pfn_folio(pte_pfn(pte[i])); -+ if (folio_memcg_rcu(folio) != memcg) -+ continue; ++#if DEBUG_REV_RANGE ++ pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12, ++ (range[i + 1] >> 12) - 1); ++#endif ++ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, ++ xa_mk_value(range[i] >> 12), 0); ++ mt_validate(mt); ++ } + -+ old_gen = folio_update_gen(folio, new_gen); -+ if (old_gen < 0 || old_gen == new_gen) -+ continue; + -+ if (walk) -+ update_batch_size(walk, folio, old_gen, new_gen); -+ else -+ lru_gen_update_size(lruvec, folio, old_gen, new_gen); ++ for (i = 0; i < ARRAY_SIZE(holes); i += 3) { ++#if DEBUG_REV_RANGE ++ pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n", ++ min, holes[i+1]>>12, holes[i+2]>>12, ++ holes[i] >> 12); ++#endif ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, min, ++ holes[i+1] >> 12, ++ holes[i+2] >> 12)); ++#if DEBUG_REV_RANGE ++ pr_debug("Found %lu %lu\n", mas.index, mas.last); ++ pr_debug("gap %lu %lu\n", (holes[i] >> 12), ++ (holes[i+1] >> 12)); ++#endif ++ MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12)); ++ MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12)); ++ min = holes[i+1] >> 12; ++ mas_reset(&mas); ++ } ++ ++ for (i = 0; i < req_range_count; i += 5) { ++#if DEBUG_REV_RANGE ++ pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n", ++ req_range[i] >> 12, ++ (req_range[i + 1] >> 12) - 1, ++ req_range[i+2] >> 12, ++ req_range[i+3] >> 12); ++#endif ++ check_mtree_alloc_rrange(mt, ++ req_range[i] >> 12, /* start */ ++ req_range[i+1] >> 12, /* end */ ++ req_range[i+2] >> 12, /* size */ ++ req_range[i+3] >> 12, /* expected address */ ++ req_range[i+4], /* expected return */ ++ xa_mk_value(req_range[i] >> 12)); /* pointer */ ++ mt_validate(mt); + } + -+ if (!walk) -+ spin_unlock_irq(&lruvec->lru_lock); ++ mt_set_non_kernel(1); ++ mtree_erase(mt, 34148798727); /* create a deleted range. */ ++ check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414, ++ 34148798725, 0, mt); + -+ mem_cgroup_unlock_pages(); ++ mtree_destroy(mt); +} + -+/****************************************************************************** -+ * the eviction -+ ******************************************************************************/ -+ -+static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) ++static noinline void check_alloc_range(struct maple_tree *mt) +{ -+ bool success; -+ int gen = folio_lru_gen(folio); -+ int type = folio_is_file_lru(folio); -+ int zone = folio_zonenum(folio); -+ int delta = folio_nr_pages(folio); -+ int refs = folio_lru_refs(folio); -+ int tier = lru_tier_from_refs(refs); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ -+ VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); ++ /* ++ * Generated by: ++ * cat /proc/self/maps|awk '{print $1}'| ++ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' ++ */ + -+ /* unevictable */ -+ if (!folio_evictable(folio)) { -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ folio_set_unevictable(folio); -+ lruvec_add_folio(lruvec, folio); -+ __count_vm_events(UNEVICTABLE_PGCULLED, delta); -+ return true; -+ } ++ unsigned long range[] = { ++ /* Inclusive , Exclusive. */ ++ 0x565234af2000, 0x565234af4000, ++ 0x565234af4000, 0x565234af9000, ++ 0x565234af9000, 0x565234afb000, ++ 0x565234afc000, 0x565234afd000, ++ 0x565234afd000, 0x565234afe000, ++ 0x565235def000, 0x565235e10000, ++ 0x7f36d4bfd000, 0x7f36d4ee2000, ++ 0x7f36d4ee2000, 0x7f36d4f04000, ++ 0x7f36d4f04000, 0x7f36d504c000, ++ 0x7f36d504c000, 0x7f36d5098000, ++ 0x7f36d5098000, 0x7f36d5099000, ++ 0x7f36d5099000, 0x7f36d509d000, ++ 0x7f36d509d000, 0x7f36d509f000, ++ 0x7f36d509f000, 0x7f36d50a5000, ++ 0x7f36d50b9000, 0x7f36d50db000, ++ 0x7f36d50db000, 0x7f36d50dc000, ++ 0x7f36d50dc000, 0x7f36d50fa000, ++ 0x7f36d50fa000, 0x7f36d5102000, ++ 0x7f36d5102000, 0x7f36d5103000, ++ 0x7f36d5103000, 0x7f36d5104000, ++ 0x7f36d5104000, 0x7f36d5105000, ++ 0x7fff5876b000, 0x7fff5878d000, ++ 0x7fff5878e000, 0x7fff58791000, ++ 0x7fff58791000, 0x7fff58793000, ++ }; ++ unsigned long holes[] = { ++ /* Start of hole, end of hole, size of hole (+1) */ ++ 0x565234afb000, 0x565234afc000, 0x1000, ++ 0x565234afe000, 0x565235def000, 0x12F1000, ++ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, ++ }; + -+ /* dirty lazyfree */ -+ if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); -+ folio_set_swapbacked(folio); -+ lruvec_add_folio_tail(lruvec, folio); -+ return true; ++ /* ++ * req_range consists of 4 values. ++ * 1. min index ++ * 2. max index ++ * 3. size ++ * 4. number that should be returned. ++ * 5. return value ++ */ ++ unsigned long req_range[] = { ++ 0x565234af9000, /* Min */ ++ 0x7fff58791000, /* Max */ ++ 0x1000, /* Size */ ++ 0x565234afb000, /* First hole in our data of size 1000. */ ++ 0, /* Return value success. */ ++ ++ 0x0, /* Min */ ++ 0x7fff58791000, /* Max */ ++ 0x1F00, /* Size */ ++ 0x0, /* First hole in our data of size 2000. */ ++ 0, /* Return value success. */ ++ ++ /* Test ascend. */ ++ 34148797436 << 12, /* Min */ ++ 0x7fff587AF000, /* Max */ ++ 0x3000, /* Size */ ++ 34148798629 << 12, /* Expected location */ ++ 0, /* Return value success. */ ++ ++ /* Test failing. */ ++ 34148798623 << 12, /* Min */ ++ 34148798683 << 12, /* Max */ ++ 0x15000, /* Size */ ++ 0, /* Expected location */ ++ -EBUSY, /* Return value failed. */ ++ ++ /* Test filling entire gap. */ ++ 34148798623 << 12, /* Min */ ++ 0x7fff587AF000, /* Max */ ++ 0x10000, /* Size */ ++ 34148798632 << 12, /* Expected location */ ++ 0, /* Return value success. */ ++ ++ /* Test walking off the end of root. */ ++ 0, /* Min */ ++ -1, /* Max */ ++ -1, /* Size */ ++ 0, /* Expected location */ ++ -EBUSY, /* Return value failure. */ ++ ++ /* Test looking for too large a hole across entire range. */ ++ 0, /* Min */ ++ -1, /* Max */ ++ 4503599618982063UL << 12, /* Size */ ++ 34359052178 << 12, /* Expected location */ ++ -EBUSY, /* Return failure. */ ++ }; ++ int i, range_count = ARRAY_SIZE(range); ++ int req_range_count = ARRAY_SIZE(req_range); ++ unsigned long min = 0x565234af2000; ++ ++ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, ++ GFP_KERNEL); ++ for (i = 0; i < range_count; i += 2) { ++#define DEBUG_ALLOC_RANGE 0 ++#if DEBUG_ALLOC_RANGE ++ pr_debug("\tInsert %lu-%lu\n", range[i] >> 12, ++ (range[i + 1] >> 12) - 1); ++ mt_dump(mt); ++#endif ++ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, ++ xa_mk_value(range[i] >> 12), 0); ++ mt_validate(mt); + } + -+ /* promoted */ -+ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { -+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); -+ return true; -+ } + -+ /* protected */ -+ if (tier > tier_idx) { -+ int hist = lru_hist_from_seq(lrugen->min_seq[type]); ++ MA_STATE(mas, mt, 0, 0); + -+ gen = folio_inc_gen(lruvec, folio, false); -+ list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); ++ for (i = 0; i < ARRAY_SIZE(holes); i += 3) { + -+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], -+ lrugen->protected[hist][type][tier - 1] + delta); -+ __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); -+ return true; ++#if DEBUG_ALLOC_RANGE ++ pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12, ++ holes[i+1] >> 12, holes[i+2] >> 12, ++ min, holes[i+1]); ++#endif ++ MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12, ++ holes[i+1] >> 12, ++ holes[i+2] >> 12)); ++ MT_BUG_ON(mt, mas.index != holes[i] >> 12); ++ min = holes[i+1]; ++ mas_reset(&mas); ++ } ++ for (i = 0; i < req_range_count; i += 5) { ++#if DEBUG_ALLOC_RANGE ++ pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n", ++ i/5, req_range[i] >> 12, req_range[i + 1] >> 12, ++ req_range[i + 2] >> 12, req_range[i + 3] >> 12, ++ req_range[i], req_range[i+1]); ++#endif ++ check_mtree_alloc_range(mt, ++ req_range[i] >> 12, /* start */ ++ req_range[i+1] >> 12, /* end */ ++ req_range[i+2] >> 12, /* size */ ++ req_range[i+3] >> 12, /* expected address */ ++ req_range[i+4], /* expected return */ ++ xa_mk_value(req_range[i] >> 12)); /* pointer */ ++ mt_validate(mt); ++#if DEBUG_ALLOC_RANGE ++ mt_dump(mt); ++#endif + } + -+ /* waiting for writeback */ -+ if (folio_test_locked(folio) || folio_test_writeback(folio) || -+ (type == LRU_GEN_FILE && folio_test_dirty(folio))) { -+ gen = folio_inc_gen(lruvec, folio, true); -+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]); -+ return true; -+ } ++ mtree_destroy(mt); ++} ++ ++static noinline void check_ranges(struct maple_tree *mt) ++{ ++ int i, val, val2; ++ unsigned long r[] = { ++ 10, 15, ++ 20, 25, ++ 17, 22, /* Overlaps previous range. */ ++ 9, 1000, /* Huge. */ ++ 100, 200, ++ 45, 168, ++ 118, 128, ++ }; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0); ++ check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0); ++ check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ /* Store */ ++ check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0); ++ check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0); ++ check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ MT_BUG_ON(mt, mt_height(mt)); ++ ++ check_seq(mt, 50, false); ++ mt_set_non_kernel(4); ++ check_store_range(mt, 5, 47, xa_mk_value(47), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Create tree of 1-100 */ ++ check_seq(mt, 100, false); ++ /* Store 45-168 */ ++ mt_set_non_kernel(10); ++ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Create tree of 1-200 */ ++ check_seq(mt, 200, false); ++ /* Store 45-168 */ ++ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ check_seq(mt, 30, false); ++ check_store_range(mt, 6, 18, xa_mk_value(6), 0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Overwrite across multiple levels. */ ++ /* Create tree of 1-400 */ ++ check_seq(mt, 400, false); ++ mt_set_non_kernel(50); ++ /* Store 118-128 */ ++ check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0); ++ mt_set_non_kernel(50); ++ mtree_test_erase(mt, 140); ++ mtree_test_erase(mt, 141); ++ mtree_test_erase(mt, 142); ++ mtree_test_erase(mt, 143); ++ mtree_test_erase(mt, 130); ++ mtree_test_erase(mt, 131); ++ mtree_test_erase(mt, 132); ++ mtree_test_erase(mt, 133); ++ mtree_test_erase(mt, 134); ++ mtree_test_erase(mt, 135); ++ check_load(mt, r[12], xa_mk_value(r[12])); ++ check_load(mt, r[13], xa_mk_value(r[12])); ++ check_load(mt, r[13] - 1, xa_mk_value(r[12])); ++ check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); ++ check_load(mt, 135, NULL); ++ check_load(mt, 140, NULL); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ ++ ++ /* Overwrite multiple levels at the end of the tree (slot 7) */ ++ mt_set_non_kernel(50); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 353, 361, xa_mk_value(353), 0); ++ check_store_range(mt, 347, 352, xa_mk_value(347), 0); ++ ++ check_load(mt, 346, xa_mk_value(346)); ++ for (i = 347; i <= 352; i++) ++ check_load(mt, i, xa_mk_value(347)); ++ for (i = 353; i <= 361; i++) ++ check_load(mt, i, xa_mk_value(353)); ++ check_load(mt, 362, xa_mk_value(362)); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(50); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 352, 364, NULL, 0); ++ check_store_range(mt, 351, 363, xa_mk_value(352), 0); ++ check_load(mt, 350, xa_mk_value(350)); ++ check_load(mt, 351, xa_mk_value(352)); ++ for (i = 352; i <= 363; i++) ++ check_load(mt, i, xa_mk_value(352)); ++ check_load(mt, 364, NULL); ++ check_load(mt, 365, xa_mk_value(365)); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_set_non_kernel(5); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 352, 364, NULL, 0); ++ check_store_range(mt, 351, 364, xa_mk_value(352), 0); ++ check_load(mt, 350, xa_mk_value(350)); ++ check_load(mt, 351, xa_mk_value(352)); ++ for (i = 352; i <= 364; i++) ++ check_load(mt, i, xa_mk_value(352)); ++ check_load(mt, 365, xa_mk_value(365)); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ ++ mt_set_non_kernel(50); ++ check_seq(mt, 400, false); ++ check_store_range(mt, 362, 367, xa_mk_value(362), 0); ++ check_store_range(mt, 353, 361, xa_mk_value(353), 0); ++ mt_set_non_kernel(0); ++ mt_validate(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ /* ++ * Interesting cases: ++ * 1. Overwrite the end of a node and end in the first entry of the next ++ * node. ++ * 2. Split a single range ++ * 3. Overwrite the start of a range ++ * 4. Overwrite the end of a range ++ * 5. Overwrite the entire range ++ * 6. Overwrite a range that causes multiple parent nodes to be ++ * combined ++ * 7. Overwrite a range that causes multiple parent nodes and part of ++ * root to be combined ++ * 8. Overwrite the whole tree ++ * 9. Try to overwrite the zero entry of an alloc tree. ++ * 10. Write a range larger than a nodes current pivot ++ */ + -+ return false; ++ mt_set_non_kernel(50); ++ for (i = 0; i <= 500; i++) { ++ val = i*5; ++ val2 = (i+1)*5; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0); ++ check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0); ++ check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0); ++ check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0); ++ check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0); ++ mtree_destroy(mt); ++ mt_set_non_kernel(0); ++ ++ mt_set_non_kernel(50); ++ for (i = 0; i <= 500; i++) { ++ val = i*5; ++ val2 = (i+1)*5; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0); ++ check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0); ++ check_store_range(mt, 2425, 2425, xa_mk_value(2), 0); ++ check_store_range(mt, 2460, 2470, NULL, 0); ++ check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0); ++ check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ /* Test rebalance gaps */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mt_set_non_kernel(50); ++ for (i = 0; i <= 50; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 161, 161, xa_mk_value(161), 0); ++ check_store_range(mt, 162, 162, xa_mk_value(162), 0); ++ check_store_range(mt, 163, 163, xa_mk_value(163), 0); ++ check_store_range(mt, 240, 249, NULL, 0); ++ mtree_erase(mt, 200); ++ mtree_erase(mt, 210); ++ mtree_erase(mt, 220); ++ mtree_erase(mt, 230); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 500; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 4600, 4959, xa_mk_value(1), 0); ++ mt_validate(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 500; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ } ++ check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0); ++ check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0); ++ check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0); ++ check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0); ++ check_store_range(mt, 4842, 4849, NULL, 0); ++ mt_validate(mt); ++ MT_BUG_ON(mt, !mt_height(mt)); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 1300; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ } ++ /* Cause a 3 child split all the way up the tree. */ ++ for (i = 5; i < 215; i += 10) ++ check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); ++ for (i = 5; i < 65; i += 10) ++ check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); ++ ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ for (i = 5; i < 45; i += 10) ++ check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); ++ MT_BUG_ON(mt, mt_height(mt) < 4); ++ mtree_destroy(mt); ++ ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 1200; i++) { ++ val = i*10; ++ val2 = (i+1)*10; ++ check_store_range(mt, val, val2, xa_mk_value(val), 0); ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ } ++ /* Fill parents and leaves before split. */ ++ for (i = 5; i < 455; i += 10) ++ check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); ++ ++ for (i = 1; i < 16; i++) ++ check_store_range(mt, 8185 + i, 8185 + i + 1, ++ xa_mk_value(8185+i), 0); ++ MT_BUG_ON(mt, mt_height(mt) >= 4); ++ /* triple split across multiple levels. */ ++ check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); ++ MT_BUG_ON(mt, mt_height(mt) != 4); ++} ++ ++static noinline void check_next_entry(struct maple_tree *mt) ++{ ++ void *entry = NULL; ++ unsigned long limit = 30, i = 0; ++ ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ MA_STATE(mas, mt, i, i); ++ ++ check_seq(mt, limit, false); ++ rcu_read_lock(); ++ ++ /* Check the first one and get ma_state in the correct state. */ ++ MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++)); ++ for ( ; i <= limit + 1; i++) { ++ entry = mas_next(&mas, limit); ++ if (i > limit) ++ MT_BUG_ON(mt, entry != NULL); ++ else ++ MT_BUG_ON(mt, xa_mk_value(i) != entry); ++ } ++ rcu_read_unlock(); ++ mtree_destroy(mt); +} + -+static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) ++static noinline void check_prev_entry(struct maple_tree *mt) +{ -+ bool success; ++ unsigned long index = 16; ++ void *value; ++ int i; + -+ /* unmapping inhibited */ -+ if (!sc->may_unmap && folio_mapped(folio)) -+ return false; ++ MA_STATE(mas, mt, index, index); + -+ /* swapping inhibited */ -+ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && -+ (folio_test_dirty(folio) || -+ (folio_test_anon(folio) && !folio_test_swapcache(folio)))) -+ return false; ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_seq(mt, 30, false); + -+ /* raced with release_pages() */ -+ if (!folio_try_get(folio)) -+ return false; ++ rcu_read_lock(); ++ value = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, value != xa_mk_value(index)); ++ value = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, value != xa_mk_value(index - 1)); ++ rcu_read_unlock(); ++ mtree_destroy(mt); + -+ /* raced with another isolation */ -+ if (!folio_test_clear_lru(folio)) { -+ folio_put(folio); -+ return false; ++ /* Check limits on prev */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_lock(&mas); ++ for (i = 0; i <= index; i++) { ++ mas_set_range(&mas, i*10, i*10+5); ++ mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL); + } + -+ /* see the comment on MAX_NR_TIERS */ -+ if (!folio_test_referenced(folio)) -+ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); ++ mas_set(&mas, 20); ++ value = mas_walk(&mas); ++ MT_BUG_ON(mt, value != xa_mk_value(2)); + -+ /* for shrink_page_list() */ -+ folio_clear_reclaim(folio); -+ folio_clear_referenced(folio); ++ value = mas_prev(&mas, 19); ++ MT_BUG_ON(mt, value != NULL); + -+ success = lru_gen_del_folio(lruvec, folio, true); -+ VM_WARN_ON_ONCE_FOLIO(!success, folio); ++ mas_set(&mas, 80); ++ value = mas_walk(&mas); ++ MT_BUG_ON(mt, value != xa_mk_value(8)); + -+ return true; ++ value = mas_prev(&mas, 76); ++ MT_BUG_ON(mt, value != NULL); ++ ++ mas_unlock(&mas); +} + -+static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, -+ int type, int tier, struct list_head *list) ++static noinline void check_root_expand(struct maple_tree *mt) +{ -+ int gen, zone; -+ enum vm_event_item item; -+ int sorted = 0; -+ int scanned = 0; -+ int isolated = 0; -+ int remaining = MAX_LRU_BATCH; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); ++ MA_STATE(mas, mt, 0, 0); ++ void *ptr; + -+ VM_WARN_ON_ONCE(!list_empty(list)); -+ -+ if (get_nr_gens(lruvec, type) == MIN_NR_GENS) -+ return 0; + -+ gen = lru_gen_from_seq(lrugen->min_seq[type]); ++ mas_lock(&mas); ++ mas_set(&mas, 3); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != ULONG_MAX); + -+ for (zone = sc->reclaim_idx; zone >= 0; zone--) { -+ LIST_HEAD(moved); -+ int skipped = 0; -+ struct list_head *head = &lrugen->lists[gen][type][zone]; ++ ptr = &check_prev_entry; ++ mas_set(&mas, 1); ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); + -+ while (!list_empty(head)) { -+ struct folio *folio = lru_to_folio(head); -+ int delta = folio_nr_pages(folio); ++ mas_set(&mas, 0); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); + -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); ++ mas_set(&mas, 1); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != &check_prev_entry); + -+ scanned += delta; ++ mas_set(&mas, 2); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ mas_unlock(&mas); ++ mtree_destroy(mt); ++ ++ ++ mt_init_flags(mt, 0); ++ mas_lock(&mas); ++ ++ mas_set(&mas, 0); ++ ptr = &check_prev_entry; ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ++ mas_set(&mas, 5); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, mas.index != 1); ++ MT_BUG_ON(mt, mas.last != ULONG_MAX); ++ ++ mas_set_range(&mas, 0, 100); ++ ptr = mas_walk(&mas); ++ MT_BUG_ON(mt, ptr != &check_prev_entry); ++ MT_BUG_ON(mt, mas.last != 0); ++ mas_unlock(&mas); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, 0); ++ mas_lock(&mas); ++ ++ mas_set(&mas, 0); ++ ptr = (void *)((unsigned long) check_prev_entry | 1UL); ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ptr = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); ++ ++ mas_set(&mas, 1); ++ ptr = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); ++ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL)); ++ ++ mas_unlock(&mas); ++ ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, 0); ++ mas_lock(&mas); ++ mas_set(&mas, 0); ++ ptr = (void *)((unsigned long) check_prev_entry | 2UL); ++ mas_store_gfp(&mas, ptr, GFP_KERNEL); ++ ptr = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, ptr != NULL); ++ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); ++ ++ mas_set(&mas, 1); ++ ptr = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); ++ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL)); ++ ++ ++ mas_unlock(&mas); ++} ++ ++static noinline void check_prealloc(struct maple_tree *mt) ++{ ++ unsigned long i, max = 100; ++ unsigned long allocated; ++ unsigned char height; ++ struct maple_node *mn; ++ void *ptr = check_prealloc; ++ MA_STATE(mas, mt, 10, 20); ++ ++ mt_set_non_kernel(1000); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); ++ ma_free_rcu(mn); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ma_free_rcu(mn); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mn = mas_pop_node(&mas); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); ++ mas_push_node(&mas, mn); ++ MT_BUG_ON(mt, mas_allocated(&mas) != allocated); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ mas_destroy(&mas); ++ allocated = mas_allocated(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mt_set_non_kernel(1); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++ mas_destroy(&mas); ++ ++ ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated == 0); ++ MT_BUG_ON(mt, allocated != 1 + height * 3); ++ mas_store_prealloc(&mas, ptr); ++ MT_BUG_ON(mt, mas_allocated(&mas) != 0); ++ mt_set_non_kernel(1); ++ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); ++ allocated = mas_allocated(&mas); ++ height = mas_mt_height(&mas); ++ MT_BUG_ON(mt, allocated != 0); ++} ++ ++static noinline void check_spanning_write(struct maple_tree *mt) ++{ ++ unsigned long i, max = 5000; ++ MA_STATE(mas, mt, 1200, 2380); ++ ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 1205); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ for (i = 1; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mtree_lock(mt); ++ mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 1205); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a right cousin rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 1205); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test non-alloc tree spanning store */ ++ mt_init_flags(mt, 0); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 0, 300); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 15); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a right sibling rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 0, 12865); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 15); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a left sibling rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 90, 13665); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 95); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store that requires a left cousin rebalance */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 46805, 49995); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 46815); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); + -+ if (sort_folio(lruvec, folio, tier)) -+ sorted += delta; -+ else if (isolate_folio(lruvec, folio, sc)) { -+ list_add(&folio->lru, list); -+ isolated += delta; -+ } else { -+ list_move(&folio->lru, &moved); -+ skipped += delta; -+ } ++ /* ++ * Test spanning store that requires a left cousin rebalance all the way ++ * to root ++ */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mas_set_range(&mas, 32395, 49995); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 46815); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); + -+ if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) -+ break; -+ } ++ /* ++ * Test spanning store that requires a right cousin rebalance all the ++ * way to root ++ */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ mas_set_range(&mas, 38875, 43190); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 38900); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning store ending at full node (depth 2)*/ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ mtree_lock(mt); ++ mas_set(&mas, 47606); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ mas_set(&mas, 47607); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ mas_set(&mas, 47608); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ mas_set(&mas, 47609); ++ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); ++ /* Ensure the parent node is full */ ++ mas_ascend(&mas); ++ MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); ++ mas_set_range(&mas, 11516, 48940); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ /* Test spanning write with many levels of no siblings */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ mas_set_range(&mas, 43200, 49999); ++ mtree_lock(mt); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mas_set(&mas, 43200); ++ MT_BUG_ON(mt, mas_walk(&mas) != NULL); ++ mtree_unlock(mt); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= 100; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ ++ mtree_lock(mt); ++ mas_set_range(&mas, 76, 875); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ mtree_unlock(mt); ++} ++ ++static noinline void check_null_expand(struct maple_tree *mt) ++{ ++ unsigned long i, max = 100; ++ unsigned char data_end; ++ MA_STATE(mas, mt, 959, 959); ++ ++ for (i = 0; i <= max; i++) ++ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); ++ /* Test expanding null at start. */ ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 959, 963); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 963) != NULL); ++ MT_BUG_ON(mt, data_end != mas_data_end(&mas)); ++ ++ /* Test expanding null at end. */ ++ mas_set(&mas, 880); ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 884, 887); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 884) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 889) != NULL); ++ MT_BUG_ON(mt, data_end != mas_data_end(&mas)); ++ ++ /* Test expanding null at start and end. */ ++ mas_set(&mas, 890); ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 900, 905); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 899) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 900) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 905) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 906) != NULL); ++ MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); ++ ++ /* Test expanding null across multiple slots. */ ++ mas_set(&mas, 800); ++ mas_walk(&mas); ++ data_end = mas_data_end(&mas); ++ mas_set_range(&mas, 810, 825); ++ mas_store_gfp(&mas, NULL, GFP_KERNEL); ++ MT_BUG_ON(mt, mtree_load(mt, 809) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 810) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 825) != NULL); ++ MT_BUG_ON(mt, mtree_load(mt, 826) != NULL); ++ MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); ++} ++ ++static noinline void check_gap_combining(struct maple_tree *mt) ++{ ++ struct maple_enode *mn1, *mn2; ++ void *entry; ++ ++ unsigned long seq100[] = { ++ /* 0-5 */ ++ 74, 75, 76, ++ 50, 100, 2, ++ ++ /* 6-12 */ ++ 44, 45, 46, 43, ++ 20, 50, 3, ++ ++ /* 13-20*/ ++ 80, 81, 82, ++ 76, 2, 79, 85, 4, ++ }; ++ unsigned long seq2000[] = { ++ 1152, 1151, ++ 1100, 1200, 2, ++ }; ++ unsigned long seq400[] = { ++ 286, 318, ++ 256, 260, 266, 270, 275, 280, 290, 398, ++ 286, 310, ++ }; + -+ if (skipped) { -+ list_splice(&moved, head); -+ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); -+ } ++ unsigned long index = seq100[0]; + -+ if (!remaining || isolated >= MIN_LRU_BATCH) -+ break; -+ } ++ MA_STATE(mas, mt, index, index); + -+ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; -+ if (!cgroup_reclaim(sc)) { -+ __count_vm_events(item, isolated); -+ __count_vm_events(PGREFILL, sorted); -+ } -+ __count_memcg_events(memcg, item, isolated); -+ __count_memcg_events(memcg, PGREFILL, sorted); -+ __count_vm_events(PGSCAN_ANON + type, isolated); ++ MT_BUG_ON(mt, !mtree_empty(mt)); ++ check_seq(mt, 100, false); /* create 100 singletons. */ + -+ /* -+ * There might not be eligible pages due to reclaim_idx, may_unmap and -+ * may_writepage. Check the remaining to prevent livelock if it's not -+ * making progress. -+ */ -+ return isolated || !remaining ? scanned : 0; -+} ++ mt_set_non_kernel(1); ++ mtree_test_erase(mt, seq100[2]); ++ check_load(mt, seq100[2], NULL); ++ mtree_test_erase(mt, seq100[1]); ++ check_load(mt, seq100[1], NULL); + -+static int get_tier_idx(struct lruvec *lruvec, int type) -+{ -+ int tier; -+ struct ctrl_pos sp, pv; ++ rcu_read_lock(); ++ entry = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index)); ++ mn1 = mas.node; ++ mas_next(&mas, ULONG_MAX); ++ entry = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); ++ mn2 = mas.node; ++ MT_BUG_ON(mt, mn1 == mn2); /* test the test. */ + + /* -+ * To leave a margin for fluctuations, use a larger gain factor (1:2). -+ * This value is chosen because any other tier would have at least twice -+ * as many refaults as the first tier. ++ * At this point, there is a gap of 2 at index + 1 between seq100[3] and ++ * seq100[4]. Search for the gap. + */ -+ read_ctrl_pos(lruvec, type, 0, 1, &sp); -+ for (tier = 1; tier < MAX_NR_TIERS; tier++) { -+ read_ctrl_pos(lruvec, type, tier, 2, &pv); -+ if (!positive_ctrl_err(&sp, &pv)) -+ break; -+ } ++ mt_set_non_kernel(1); ++ mas_reset(&mas); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4], ++ seq100[5])); ++ MT_BUG_ON(mt, mas.index != index + 1); ++ rcu_read_unlock(); + -+ return tier - 1; -+} ++ mtree_test_erase(mt, seq100[6]); ++ check_load(mt, seq100[6], NULL); ++ mtree_test_erase(mt, seq100[7]); ++ check_load(mt, seq100[7], NULL); ++ mtree_test_erase(mt, seq100[8]); ++ index = seq100[9]; + -+static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) -+{ -+ int type, tier; -+ struct ctrl_pos sp, pv; -+ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; ++ rcu_read_lock(); ++ mas.index = index; ++ mas.last = index; ++ mas_reset(&mas); ++ entry = mas_find(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index)); ++ mn1 = mas.node; ++ entry = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); ++ mas_next(&mas, ULONG_MAX); /* go to the next entry. */ ++ mn2 = mas.node; ++ MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */ + + /* -+ * Compare the first tier of anon with that of file to determine which -+ * type to scan. Also need to compare other tiers of the selected type -+ * with the first tier of the other type to determine the last tier (of -+ * the selected type) to evict. ++ * At this point, there is a gap of 3 at seq100[6]. Find it by ++ * searching 20 - 50 for size 3. + */ -+ read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); -+ read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); -+ type = positive_ctrl_err(&sp, &pv); -+ -+ read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); -+ for (tier = 1; tier < MAX_NR_TIERS; tier++) { -+ read_ctrl_pos(lruvec, type, tier, gain[type], &pv); -+ if (!positive_ctrl_err(&sp, &pv)) -+ break; -+ } -+ -+ *tier_idx = tier - 1; ++ mas_reset(&mas); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11], ++ seq100[12])); ++ MT_BUG_ON(mt, mas.index != seq100[6]); ++ rcu_read_unlock(); + -+ return type; -+} ++ mt_set_non_kernel(1); ++ mtree_store(mt, seq100[13], NULL, GFP_KERNEL); ++ check_load(mt, seq100[13], NULL); ++ check_load(mt, seq100[14], xa_mk_value(seq100[14])); ++ mtree_store(mt, seq100[14], NULL, GFP_KERNEL); ++ check_load(mt, seq100[13], NULL); ++ check_load(mt, seq100[14], NULL); + -+static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, -+ int *type_scanned, struct list_head *list) -+{ -+ int i; -+ int type; -+ int scanned; -+ int tier = -1; -+ DEFINE_MIN_SEQ(lruvec); ++ mas_reset(&mas); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15], ++ seq100[17])); ++ MT_BUG_ON(mt, mas.index != seq100[13]); ++ mt_validate(mt); ++ rcu_read_unlock(); + + /* -+ * Try to make the obvious choice first. When anon and file are both -+ * available from the same generation, interpret swappiness 1 as file -+ * first and 200 as anon first. ++ * *DEPRECATED: no retries anymore* Test retry entry in the start of a ++ * gap. + */ -+ if (!swappiness) -+ type = LRU_GEN_FILE; -+ else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) -+ type = LRU_GEN_ANON; -+ else if (swappiness == 1) -+ type = LRU_GEN_FILE; -+ else if (swappiness == 200) -+ type = LRU_GEN_ANON; -+ else -+ type = get_type_to_scan(lruvec, swappiness, &tier); ++ mt_set_non_kernel(2); ++ mtree_test_store_range(mt, seq100[18], seq100[14], NULL); ++ mtree_test_erase(mt, seq100[15]); ++ mas_reset(&mas); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19], ++ seq100[20])); ++ rcu_read_unlock(); ++ MT_BUG_ON(mt, mas.index != seq100[18]); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* seq 2000 tests are for multi-level tree gaps */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_seq(mt, 2000, false); ++ mt_set_non_kernel(1); ++ mtree_test_erase(mt, seq2000[0]); ++ mtree_test_erase(mt, seq2000[1]); ++ ++ mt_set_non_kernel(2); ++ mas_reset(&mas); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3], ++ seq2000[4])); ++ MT_BUG_ON(mt, mas.index != seq2000[1]); ++ rcu_read_unlock(); ++ mt_validate(mt); ++ mtree_destroy(mt); ++ ++ /* seq 400 tests rebalancing over two levels. */ ++ mt_set_non_kernel(99); ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_seq(mt, 400, false); ++ mtree_test_store_range(mt, seq400[0], seq400[1], NULL); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_seq(mt, 400, false); ++ mt_set_non_kernel(50); ++ mtree_test_store_range(mt, seq400[2], seq400[9], ++ xa_mk_value(seq400[2])); ++ mtree_test_store_range(mt, seq400[3], seq400[9], ++ xa_mk_value(seq400[3])); ++ mtree_test_store_range(mt, seq400[4], seq400[9], ++ xa_mk_value(seq400[4])); ++ mtree_test_store_range(mt, seq400[5], seq400[9], ++ xa_mk_value(seq400[5])); ++ mtree_test_store_range(mt, seq400[0], seq400[9], ++ xa_mk_value(seq400[0])); ++ mtree_test_store_range(mt, seq400[6], seq400[9], ++ xa_mk_value(seq400[6])); ++ mtree_test_store_range(mt, seq400[7], seq400[9], ++ xa_mk_value(seq400[7])); ++ mtree_test_store_range(mt, seq400[8], seq400[9], ++ xa_mk_value(seq400[8])); ++ mtree_test_store_range(mt, seq400[10], seq400[11], ++ xa_mk_value(seq400[10])); ++ mt_validate(mt); ++ mt_set_non_kernel(0); ++ mtree_destroy(mt); ++} ++static noinline void check_node_overwrite(struct maple_tree *mt) ++{ ++ int i, max = 4000; ++ ++ for (i = 0; i < max; i++) ++ mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100)); ++ ++ mtree_test_store_range(mt, 319951, 367950, NULL); ++ /*mt_dump(mt); */ ++ mt_validate(mt); ++} ++ ++static void mas_dfs_preorder(struct ma_state *mas) ++{ ++ ++ struct maple_enode *prev; ++ unsigned char end, slot = 0; ++ ++ if (mas_is_start(mas)) { ++ mas_start(mas); ++ return; ++ } + -+ for (i = !swappiness; i < ANON_AND_FILE; i++) { -+ if (tier < 0) -+ tier = get_tier_idx(lruvec, type); ++ if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) ++ goto done; + -+ scanned = scan_folios(lruvec, sc, type, tier, list); -+ if (scanned) -+ break; ++walk_up: ++ end = mas_data_end(mas); ++ if (mte_is_leaf(mas->node) || ++ (slot > end)) { ++ if (mte_is_root(mas->node)) ++ goto done; + -+ type = !type; -+ tier = -1; ++ slot = mte_parent_slot(mas->node) + 1; ++ mas_ascend(mas); ++ goto walk_up; + } + -+ *type_scanned = type; -+ -+ return scanned; -+} ++ prev = mas->node; ++ mas->node = mas_get_slot(mas, slot); ++ if (!mas->node || slot > end) { ++ if (mte_is_root(prev)) ++ goto done; + -+static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, -+ bool *need_swapping) -+{ -+ int type; -+ int scanned; -+ int reclaimed; -+ LIST_HEAD(list); -+ struct folio *folio; -+ enum vm_event_item item; -+ struct reclaim_stat stat; -+ struct lru_gen_mm_walk *walk; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ struct pglist_data *pgdat = lruvec_pgdat(lruvec); ++ mas->node = prev; ++ slot = mte_parent_slot(mas->node) + 1; ++ mas_ascend(mas); ++ goto walk_up; ++ } + -+ spin_lock_irq(&lruvec->lru_lock); ++ return; ++done: ++ mas->node = MAS_NONE; ++} + -+ scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); + -+ scanned += try_to_inc_min_seq(lruvec, swappiness); ++static void check_dfs_preorder(struct maple_tree *mt) ++{ ++ unsigned long count = 0, max = 1000; + -+ if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) -+ scanned = 0; ++ MA_STATE(mas, mt, 0, 0); + -+ spin_unlock_irq(&lruvec->lru_lock); ++ check_seq(mt, max, false); ++ do { ++ count++; ++ mas_dfs_preorder(&mas); ++ } while (!mas_is_none(&mas)); ++ MT_BUG_ON(mt, count != 74); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_reset(&mas); ++ count = 0; ++ check_seq(mt, max, false); ++ do { ++ count++; ++ mas_dfs_preorder(&mas); ++ } while (!mas_is_none(&mas)); ++ /*printk("count %lu\n", count); */ ++ MT_BUG_ON(mt, count != 77); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_reset(&mas); ++ count = 0; ++ check_rev_seq(mt, max, false); ++ do { ++ count++; ++ mas_dfs_preorder(&mas); ++ } while (!mas_is_none(&mas)); ++ /*printk("count %lu\n", count); */ ++ MT_BUG_ON(mt, count != 77); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ mas_reset(&mas); ++ mt_zero_nr_tallocated(); ++ mt_set_non_kernel(200); ++ mas_expected_entries(&mas, max); ++ for (count = 0; count <= max; count++) { ++ mas.index = mas.last = count; ++ mas_store(&mas, xa_mk_value(count)); ++ MT_BUG_ON(mt, mas_is_err(&mas)); ++ } ++ mas_destroy(&mas); ++ rcu_barrier(); ++ /* ++ * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n", ++ * max, mt_get_alloc_size()/1024, mt_nr_allocated(), ++ * mt_nr_tallocated()); ++ */ + -+ if (list_empty(&list)) -+ return scanned; ++} + -+ reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); ++#if defined(BENCH_SLOT_STORE) ++static noinline void bench_slot_store(struct maple_tree *mt) ++{ ++ int i, brk = 105, max = 1040, brk_start = 100, count = 20000000; + -+ list_for_each_entry(folio, &list, lru) { -+ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ -+ if (folio_test_workingset(folio)) -+ folio_set_referenced(folio); ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + -+ /* don't add rejected pages to the oldest generation */ -+ if (folio_test_reclaim(folio) && -+ (folio_test_dirty(folio) || folio_test_writeback(folio))) -+ folio_clear_active(folio); -+ else -+ folio_set_active(folio); ++ for (i = 0; i < count; i++) { ++ mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL); ++ mtree_store_range(mt, brk_start, brk, xa_mk_value(brk), ++ GFP_KERNEL); + } ++} ++#endif ++ ++#if defined(BENCH_NODE_STORE) ++static noinline void bench_node_store(struct maple_tree *mt) ++{ ++ int i, overwrite = 76, max = 240, count = 20000000; + -+ spin_lock_irq(&lruvec->lru_lock); ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + -+ move_pages_to_lru(lruvec, &list); ++ for (i = 0; i < count; i++) { ++ mtree_store_range(mt, overwrite, overwrite + 15, ++ xa_mk_value(overwrite), GFP_KERNEL); + -+ walk = current->reclaim_state->mm_walk; -+ if (walk && walk->batched) -+ reset_batch_size(lruvec, walk); ++ overwrite += 5; ++ if (overwrite >= 135) ++ overwrite = 76; ++ } ++} ++#endif + -+ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; -+ if (!cgroup_reclaim(sc)) -+ __count_vm_events(item, reclaimed); -+ __count_memcg_events(memcg, item, reclaimed); -+ __count_vm_events(PGSTEAL_ANON + type, reclaimed); ++#if defined(BENCH_AWALK) ++static noinline void bench_awalk(struct maple_tree *mt) ++{ ++ int i, max = 2500, count = 50000000; ++ MA_STATE(mas, mt, 1470, 1470); + -+ spin_unlock_irq(&lruvec->lru_lock); ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + -+ mem_cgroup_uncharge_list(&list); -+ free_unref_page_list(&list); ++ mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL); + -+ sc->nr_reclaimed += reclaimed; ++ for (i = 0; i < count; i++) { ++ mas_empty_area_rev(&mas, 0, 2000, 10); ++ mas_reset(&mas); ++ } ++} ++#endif ++#if defined(BENCH_WALK) ++static noinline void bench_walk(struct maple_tree *mt) ++{ ++ int i, max = 2500, count = 550000000; ++ MA_STATE(mas, mt, 1470, 1470); + -+ if (need_swapping && type == LRU_GEN_ANON) -+ *need_swapping = true; ++ for (i = 0; i < max; i += 10) ++ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < count; i++) { ++ mas_walk(&mas); ++ mas_reset(&mas); ++ } + -+ return scanned; +} ++#endif + -+/* -+ * For future optimizations: -+ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg -+ * reclaim. -+ */ -+static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, -+ bool can_swap, bool *need_aging) ++#if defined(BENCH_MT_FOR_EACH) ++static noinline void bench_mt_for_each(struct maple_tree *mt) +{ -+ unsigned long nr_to_scan; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); ++ int i, count = 1000000; ++ unsigned long max = 2500, index = 0; ++ void *entry; + -+ if (mem_cgroup_below_min(memcg) || -+ (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) -+ return 0; ++ for (i = 0; i < max; i += 5) ++ mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL); + -+ *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); -+ if (!*need_aging) -+ return nr_to_scan; ++ for (i = 0; i < count; i++) { ++ unsigned long j = 0; + -+ /* skip the aging path at the default priority */ -+ if (sc->priority == DEF_PRIORITY) -+ goto done; ++ mt_for_each(mt, entry, index, max) { ++ MT_BUG_ON(mt, entry != xa_mk_value(j)); ++ j += 5; ++ } + -+ /* leave the work to lru_gen_age_node() */ -+ if (current_is_kswapd()) -+ return 0; ++ index = 0; ++ } + -+ if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) -+ return nr_to_scan; -+done: -+ return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; +} ++#endif + -+static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, -+ struct scan_control *sc, bool need_swapping) -+{ -+ int i; -+ DEFINE_MAX_SEQ(lruvec); ++static noinline void check_forking(struct maple_tree *mt) ++{ ++ ++ struct maple_tree newmt; ++ int i, nr_entries = 134; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mt_set_non_kernel(99999); ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ newmas.tree = &newmt; ++ mas_reset(&newmas); ++ mas_reset(&mas); ++ mas.index = 0; ++ mas.last = 0; ++ if (mas_expected_entries(&newmas, nr_entries)) { ++ pr_err("OOM!"); ++ BUG_ON(1); ++ } ++ mas_for_each(&mas, val, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store(&newmas, val); ++ } ++ mas_destroy(&newmas); ++ mt_validate(&newmt); ++ mt_set_non_kernel(0); ++ mtree_destroy(&newmt); ++} ++ ++static noinline void check_mas_store_gfp(struct maple_tree *mt) ++{ ++ ++ struct maple_tree newmt; ++ int i, nr_entries = 135; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mt_set_non_kernel(99999); ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ newmas.tree = &newmt; ++ mas_reset(&newmas); ++ mas_set(&mas, 0); ++ mas_for_each(&mas, val, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store_gfp(&newmas, val, GFP_KERNEL); ++ } ++ ++ mt_validate(&newmt); ++ mt_set_non_kernel(0); ++ mtree_destroy(&newmt); ++} ++ ++#if defined(BENCH_FORK) ++static noinline void bench_forking(struct maple_tree *mt) ++{ ++ ++ struct maple_tree newmt; ++ int i, nr_entries = 134, nr_fork = 80000; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, mt, 0, 0); ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ for (i = 0; i < nr_fork; i++) { ++ mt_set_non_kernel(99999); ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ newmas.tree = &newmt; ++ mas_reset(&newmas); ++ mas_reset(&mas); ++ mas.index = 0; ++ mas.last = 0; ++ if (mas_expected_entries(&newmas, nr_entries)) { ++ printk("OOM!"); ++ BUG_ON(1); ++ } ++ mas_for_each(&mas, val, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store(&newmas, val); ++ } ++ mas_destroy(&newmas); ++ mt_validate(&newmt); ++ mt_set_non_kernel(0); ++ mtree_destroy(&newmt); ++ } ++} ++#endif + -+ if (!current_is_kswapd()) { -+ /* age each memcg once to ensure fairness */ -+ if (max_seq - seq > 1) -+ return true; ++static noinline void next_prev_test(struct maple_tree *mt) ++{ ++ int i, nr_entries = 200; ++ void *val; ++ MA_STATE(mas, mt, 0, 0); ++ struct maple_enode *mn; + -+ /* over-swapping can increase allocation latency */ -+ if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) -+ return true; ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); + -+ /* give this thread a chance to exit and free its memory */ -+ if (fatal_signal_pending(current)) { -+ sc->nr_reclaimed += MIN_LRU_BATCH; -+ return true; -+ } ++ for (i = 0; i <= nr_entries / 2; i++) { ++ mas_next(&mas, 1000); ++ if (mas_is_none(&mas)) ++ break; + -+ if (cgroup_reclaim(sc)) -+ return false; -+ } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) -+ return false; ++ } ++ mas_reset(&mas); ++ mas_set(&mas, 0); ++ i = 0; ++ mas_for_each(&mas, val, 1000) { ++ i++; ++ } + -+ /* keep scanning at low priorities to ensure fairness */ -+ if (sc->priority > DEF_PRIORITY - 2) -+ return false; ++ mas_reset(&mas); ++ mas_set(&mas, 0); ++ i = 0; ++ mas_for_each(&mas, val, 1000) { ++ mas_pause(&mas); ++ i++; ++ } + + /* -+ * A minimum amount of work was done under global memory pressure. For -+ * kswapd, it may be overshooting. For direct reclaim, the target isn't -+ * met, and yet the allocation may still succeed, since kswapd may have -+ * caught up. In either case, it's better to stop now, and restart if -+ * necessary. ++ * 680 - 685 = 0x61a00001930c ++ * 686 - 689 = NULL; ++ * 690 - 695 = 0x61a00001930c ++ * Check simple next/prev + */ -+ for (i = 0; i <= sc->reclaim_idx; i++) { -+ unsigned long wmark; -+ struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; ++ mas_set(&mas, 686); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != NULL); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); ++ MT_BUG_ON(mt, mas.index != 690); ++ MT_BUG_ON(mt, mas.last != 695); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(680 / 10)); ++ MT_BUG_ON(mt, mas.index != 680); ++ MT_BUG_ON(mt, mas.last != 685); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); ++ MT_BUG_ON(mt, mas.index != 690); ++ MT_BUG_ON(mt, mas.last != 695); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(700 / 10)); ++ MT_BUG_ON(mt, mas.index != 700); ++ MT_BUG_ON(mt, mas.last != 705); ++ ++ /* Check across node boundaries of the tree */ ++ mas_set(&mas, 70); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); ++ MT_BUG_ON(mt, mas.index != 70); ++ MT_BUG_ON(mt, mas.last != 75); ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(80 / 10)); ++ MT_BUG_ON(mt, mas.index != 80); ++ MT_BUG_ON(mt, mas.last != 85); ++ ++ val = mas_prev(&mas, 70); ++ MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); ++ MT_BUG_ON(mt, mas.index != 70); ++ MT_BUG_ON(mt, mas.last != 75); ++ ++ /* Check across two levels of the tree */ ++ mas_reset(&mas); ++ mas_set(&mas, 707); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != NULL); ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); ++ MT_BUG_ON(mt, mas.index != 710); ++ MT_BUG_ON(mt, mas.last != 715); ++ mn = mas.node; ++ ++ val = mas_next(&mas, 1000); ++ MT_BUG_ON(mt, val != xa_mk_value(720 / 10)); ++ MT_BUG_ON(mt, mas.index != 720); ++ MT_BUG_ON(mt, mas.last != 725); ++ MT_BUG_ON(mt, mn == mas.node); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); ++ MT_BUG_ON(mt, mas.index != 710); ++ MT_BUG_ON(mt, mas.last != 715); ++ ++ /* Check running off the end and back on */ ++ mas_reset(&mas); ++ mas_set(&mas, 2000); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); ++ MT_BUG_ON(mt, mas.index != 2000); ++ MT_BUG_ON(mt, mas.last != 2005); ++ ++ val = mas_next(&mas, ULONG_MAX); ++ MT_BUG_ON(mt, val != NULL); ++ MT_BUG_ON(mt, mas.index != ULONG_MAX); ++ MT_BUG_ON(mt, mas.last != ULONG_MAX); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); ++ MT_BUG_ON(mt, mas.index != 2000); ++ MT_BUG_ON(mt, mas.last != 2005); ++ ++ /* Check running off the start and back on */ ++ mas_reset(&mas); ++ mas_set(&mas, 10); ++ val = mas_walk(&mas); ++ MT_BUG_ON(mt, val != xa_mk_value(1)); ++ MT_BUG_ON(mt, mas.index != 10); ++ MT_BUG_ON(mt, mas.last != 15); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != xa_mk_value(0)); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != 5); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != NULL); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != 0); ++ ++ mas.index = 0; ++ mas.last = 5; ++ mas_store(&mas, NULL); ++ mas_reset(&mas); ++ mas_set(&mas, 10); ++ mas_walk(&mas); ++ ++ val = mas_prev(&mas, 0); ++ MT_BUG_ON(mt, val != NULL); ++ MT_BUG_ON(mt, mas.index != 0); ++ MT_BUG_ON(mt, mas.last != 0); ++ ++ mtree_destroy(mt); ++ ++ mt_init(mt); ++ mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); ++ mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL); ++ mas_set(&mas, 5); ++ val = mas_prev(&mas, 4); ++ MT_BUG_ON(mt, val != NULL); ++} ++ ++#define RCU_RANGE_COUNT 1000 ++#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true;} MT_BUG_ON(test->mt, y);} ++struct rcu_test_struct2 { ++ struct maple_tree *mt; ++ ++ bool start; ++ bool stop; ++ unsigned int thread_count; ++ ++ unsigned int seen_toggle; ++ unsigned int seen_added; ++ unsigned int seen_modified; ++ unsigned int seen_deleted; ++ int pause; ++ ++ unsigned long index[RCU_RANGE_COUNT]; ++ unsigned long last[RCU_RANGE_COUNT]; ++}; + -+ if (!managed_zone(zone)) -+ continue; ++struct rcu_reader_struct { ++ unsigned int id; ++ int mod; ++ int del; ++ int flip; ++ int add; ++ int next; ++ struct rcu_test_struct2 *test; ++}; + -+ wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); -+ if (wmark > zone_page_state(zone, NR_FREE_PAGES)) -+ return false; -+ } ++/* RCU reader helper function */ ++static void rcu_reader_register(struct rcu_test_struct2 *test) ++{ ++ rcu_register_thread(); ++ uatomic_inc(&test->thread_count); + -+ sc->nr_reclaimed += MIN_LRU_BATCH; ++ while (!test->start) ++ usleep(test->pause * 100); ++} + -+ return true; ++static void rcu_reader_setup(struct rcu_reader_struct *reader, ++ unsigned int id, struct rcu_test_struct2 *test) ++{ ++ reader->id = id; ++ reader->test = test; ++ reader->mod = reader->id % 10; ++ reader->del = (reader->mod + 1) % 10; ++ reader->flip = (reader->mod + 2) % 10; ++ reader->add = (reader->mod + 3) % 10; ++ reader->next = (reader->mod + 4) % 10; +} + -+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) ++/* RCU reader in increasing index */ ++static void *rcu_reader_fwd(void *ptr) +{ -+ struct blk_plug plug; -+ bool need_aging = false; -+ bool need_swapping = false; -+ unsigned long scanned = 0; -+ unsigned long reclaimed = sc->nr_reclaimed; -+ DEFINE_MAX_SEQ(lruvec); ++ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; ++ struct rcu_test_struct2 *test = reader->test; ++ unsigned long index = reader->id; ++ bool toggled, modified, deleted, added; ++ int i; ++ void *entry, *prev = NULL; ++ MA_STATE(mas, test->mt, 0, 0); + -+ lru_add_drain(); ++ rcu_reader_register(test); ++ toggled = modified = deleted = added = false; + -+ blk_start_plug(&plug); ++ while (!test->stop) { ++ i = 0; ++ /* mas_for_each ?*/ ++ rcu_read_lock(); ++ mas_set(&mas, test->index[index]); ++ mas_for_each(&mas, entry, test->last[index + 9]) { ++ unsigned long r_start, r_end, alt_start; ++ void *expected, *alt; ++ ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ ++ if (i == reader->del) { ++ if (!deleted) { ++ alt_start = test->index[index + reader->flip]; ++ /* delete occurred. */ ++ if (mas.index == alt_start) { ++ uatomic_inc(&test->seen_deleted); ++ deleted = true; ++ } ++ } ++ if (deleted) { ++ i = reader->flip; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } + -+ set_mm_walk(lruvec_pgdat(lruvec)); ++ if (!added && (i == reader->add)) { ++ alt_start = test->index[index + reader->next]; ++ if (mas.index == r_start) { ++ uatomic_inc(&test->seen_added); ++ added = true; ++ } else if (mas.index == alt_start) { ++ i = reader->next; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } + -+ while (true) { -+ int delta; -+ int swappiness; -+ unsigned long nr_to_scan; ++ RCU_MT_BUG_ON(test, mas.index != r_start); ++ RCU_MT_BUG_ON(test, mas.last != r_end); + -+ if (sc->may_swap) -+ swappiness = get_swappiness(lruvec, sc); -+ else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) -+ swappiness = 1; -+ else -+ swappiness = 0; ++ if (i == reader->flip) { ++ alt = xa_mk_value(index + i + RCU_RANGE_COUNT); ++ if (prev) { ++ if (toggled && entry == expected) ++ uatomic_inc(&test->seen_toggle); ++ else if (!toggled && entry == alt) ++ uatomic_inc(&test->seen_toggle); ++ } + -+ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); -+ if (!nr_to_scan) -+ goto done; ++ if (entry == expected) ++ toggled = false; ++ else if (entry == alt) ++ toggled = true; ++ else { ++ printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); ++ RCU_MT_BUG_ON(test, 1); ++ } + -+ delta = evict_folios(lruvec, sc, swappiness, &need_swapping); -+ if (!delta) -+ goto done; ++ prev = entry; ++ } else if (i == reader->mod) { ++ alt = xa_mk_value(index + i * 2 + 1 + ++ RCU_RANGE_COUNT); ++ if (entry != expected) { ++ if (!modified) ++ uatomic_inc(&test->seen_modified); ++ modified = true; ++ } else { ++ if (modified) ++ uatomic_inc(&test->seen_modified); ++ modified = false; ++ } + -+ scanned += delta; -+ if (scanned >= nr_to_scan) -+ break; ++ if (modified) ++ RCU_MT_BUG_ON(test, entry != alt); + -+ if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) -+ break; ++ } else { ++ if (entry != expected) ++ printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); ++ RCU_MT_BUG_ON(test, entry != expected); ++ } + -+ cond_resched(); ++ i++; ++ } ++ rcu_read_unlock(); ++ usleep(test->pause); + } + -+ /* see the comment in lru_gen_age_node() */ -+ if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) -+ sc->memcgs_need_aging = false; -+done: -+ clear_mm_walk(); -+ -+ blk_finish_plug(&plug); ++ rcu_unregister_thread(); ++ return NULL; +} + -+/****************************************************************************** -+ * state change -+ ******************************************************************************/ -+ -+static bool __maybe_unused state_is_valid(struct lruvec *lruvec) ++/* RCU reader in decreasing index */ ++static void *rcu_reader_rev(void *ptr) +{ -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; ++ struct rcu_test_struct2 *test = reader->test; ++ unsigned long index = reader->id; ++ bool toggled, modified, deleted, added; ++ int i; ++ void *prev = NULL; ++ MA_STATE(mas, test->mt, 0, 0); + -+ if (lrugen->enabled) { -+ enum lru_list lru; ++ rcu_reader_register(test); ++ toggled = modified = deleted = added = false; + -+ for_each_evictable_lru(lru) { -+ if (!list_empty(&lruvec->lists[lru])) -+ return false; -+ } -+ } else { -+ int gen, type, zone; + -+ for_each_gen_type_zone(gen, type, zone) { -+ if (!list_empty(&lrugen->lists[gen][type][zone])) -+ return false; -+ } -+ } ++ while (!test->stop) { ++ void *entry; + -+ return true; -+} ++ i = 9; ++ mas_set(&mas, test->index[index + i]); + -+static bool fill_evictable(struct lruvec *lruvec) -+{ -+ enum lru_list lru; -+ int remaining = MAX_LRU_BATCH; ++ rcu_read_lock(); ++ while (i--) { ++ unsigned long r_start, r_end, alt_start; ++ void *expected, *alt; ++ int line = __LINE__; ++ ++ entry = mas_prev(&mas, test->index[index]); ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ ++ if (i == reader->del) { ++ alt_start = test->index[index + reader->mod]; ++ if (mas.index == alt_start) { ++ line = __LINE__; ++ if (!deleted) ++ uatomic_inc(&test->seen_deleted); ++ deleted = true; ++ } ++ if (deleted) { ++ line = __LINE__; ++ i = reader->mod; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } ++ if (!added && (i == reader->add)) { ++ alt_start = test->index[index + reader->flip]; ++ if (mas.index == r_start) { ++ line = __LINE__; ++ uatomic_inc(&test->seen_added); ++ added = true; ++ } else if (mas.index == alt_start) { ++ line = __LINE__; ++ i = reader->flip; ++ r_start = test->index[index + i]; ++ r_end = test->last[index + i]; ++ expected = xa_mk_value(r_start); ++ } ++ } + -+ for_each_evictable_lru(lru) { -+ int type = is_file_lru(lru); -+ bool active = is_active_lru(lru); -+ struct list_head *head = &lruvec->lists[lru]; ++ if (i == reader->mod) ++ line = __LINE__; ++ else if (i == reader->flip) ++ line = __LINE__; ++ ++ if (mas.index != r_start) { ++ alt = xa_mk_value(index + i * 2 + 1 + ++ RCU_RANGE_COUNT); ++ mt_dump(test->mt); ++ printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n", ++ mas.index, mas.last, entry, ++ r_start, r_end, expected, alt, ++ line, i); ++ } ++ RCU_MT_BUG_ON(test, mas.index != r_start); ++ RCU_MT_BUG_ON(test, mas.last != r_end); + -+ while (!list_empty(head)) { -+ bool success; -+ struct folio *folio = lru_to_folio(head); ++ if (i == reader->mod) { ++ alt = xa_mk_value(index + i * 2 + 1 + ++ RCU_RANGE_COUNT); + -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); ++ if (entry != expected) { ++ if (!modified) ++ uatomic_inc(&test->seen_modified); ++ modified = true; ++ } else { ++ if (modified) ++ uatomic_inc(&test->seen_modified); ++ modified = false; ++ } ++ if (modified) ++ RCU_MT_BUG_ON(test, entry != alt); ++ ++ ++ } else if (i == reader->flip) { ++ alt = xa_mk_value(index + i + ++ RCU_RANGE_COUNT); ++ if (prev) { ++ if (toggled && entry == expected) ++ uatomic_inc(&test->seen_toggle); ++ else if (!toggled && entry == alt) ++ uatomic_inc(&test->seen_toggle); ++ } + -+ lruvec_del_folio(lruvec, folio); -+ success = lru_gen_add_folio(lruvec, folio, false); -+ VM_WARN_ON_ONCE(!success); ++ if (entry == expected) ++ toggled = false; ++ else if (entry == alt) ++ toggled = true; ++ else { ++ printk("%lu-%lu %p != %p or %p\n", ++ mas.index, mas.last, entry, ++ expected, alt); ++ RCU_MT_BUG_ON(test, 1); ++ } + -+ if (!--remaining) -+ return false; ++ prev = entry; ++ } else { ++ if (entry != expected) ++ printk("%lu-%lu %p != %p\n", mas.index, ++ mas.last, entry, expected); ++ RCU_MT_BUG_ON(test, entry != expected); ++ } + } ++ rcu_read_unlock(); ++ usleep(test->pause); + } + -+ return true; ++ rcu_unregister_thread(); ++ return NULL; +} + -+static bool drain_evictable(struct lruvec *lruvec) ++static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test, ++ int count, struct rcu_reader_struct *test_reader) +{ -+ int gen, type, zone; -+ int remaining = MAX_LRU_BATCH; ++ int i, j = 10000; ++ bool toggle = true; + -+ for_each_gen_type_zone(gen, type, zone) { -+ struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; ++ test->start = true; /* Release the hounds! */ ++ usleep(5); + -+ while (!list_empty(head)) { -+ bool success; -+ struct folio *folio = lru_to_folio(head); ++ while (j--) { ++ toggle = !toggle; ++ i = count; ++ while (i--) { ++ unsigned long start, end; ++ struct rcu_reader_struct *this = &test_reader[i]; + -+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); -+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); ++ /* Mod offset */ ++ if (j == 600) { ++ start = test->index[this->id + this->mod]; ++ end = test->last[this->id + this->mod]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(this->id + this->mod * 2 + ++ 1 + RCU_RANGE_COUNT), ++ GFP_KERNEL); ++ } + -+ success = lru_gen_del_folio(lruvec, folio, false); -+ VM_WARN_ON_ONCE(!success); -+ lruvec_add_folio(lruvec, folio); ++ /* Toggle */ ++ if (!(j % 5)) { ++ start = test->index[this->id + this->flip]; ++ end = test->last[this->id + this->flip]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value((toggle ? start : ++ this->id + this->flip + ++ RCU_RANGE_COUNT)), ++ GFP_KERNEL); ++ } + -+ if (!--remaining) -+ return false; ++ /* delete */ ++ if (j == 400) { ++ start = test->index[this->id + this->del]; ++ end = test->last[this->id + this->del]; ++ mtree_store_range(mt, start, end, NULL, GFP_KERNEL); ++ } ++ ++ /* add */ ++ if (j == 500) { ++ start = test->index[this->id + this->add]; ++ end = test->last[this->id + this->add]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(start), GFP_KERNEL); ++ } + } ++ usleep(test->pause); ++ /* If a test fails, don't flood the console */ ++ if (test->stop) ++ break; + } -+ -+ return true; +} + -+static void lru_gen_change_state(bool enabled) ++static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test, ++ int count, struct rcu_reader_struct *test_reader) +{ -+ static DEFINE_MUTEX(state_mutex); ++ int j, i; ++ bool toggle = true; + -+ struct mem_cgroup *memcg; ++ test->start = true; /* Release the hounds! */ ++ usleep(5); ++ for (j = 0; j < 10000; j++) { ++ toggle = !toggle; ++ for (i = 0; i < count; i++) { ++ unsigned long start, end; ++ struct rcu_reader_struct *this = &test_reader[i]; ++ ++ /* Mod offset */ ++ if (j == 600) { ++ start = test->index[this->id + this->mod]; ++ end = test->last[this->id + this->mod]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(this->id + this->mod * 2 + ++ 1 + RCU_RANGE_COUNT), ++ GFP_KERNEL); ++ } + -+ cgroup_lock(); -+ cpus_read_lock(); -+ get_online_mems(); -+ mutex_lock(&state_mutex); ++ /* Toggle */ ++ if (!(j % 5)) { ++ start = test->index[this->id + this->flip]; ++ end = test->last[this->id + this->flip]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value((toggle ? start : ++ this->id + this->flip + ++ RCU_RANGE_COUNT)), ++ GFP_KERNEL); ++ } + -+ if (enabled == lru_gen_enabled()) -+ goto unlock; ++ /* delete */ ++ if (j == 400) { ++ start = test->index[this->id + this->del]; ++ end = test->last[this->id + this->del]; ++ mtree_store_range(mt, start, end, NULL, GFP_KERNEL); ++ } + -+ if (enabled) -+ static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); -+ else -+ static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); ++ /* add */ ++ if (j == 500) { ++ start = test->index[this->id + this->add]; ++ end = test->last[this->id + this->add]; ++ mtree_store_range(mt, start, end, ++ xa_mk_value(start), GFP_KERNEL); ++ } ++ } ++ usleep(test->pause); ++ /* If a test fails, don't flood the console */ ++ if (test->stop) ++ break; ++ } ++} + -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ int nid; ++/* ++ * This is to check: ++ * 1. Range that is not ever present ++ * 2. Range that is always present ++ * 3. Things being added but not removed. ++ * 4. Things being removed but not added. ++ * 5. Things are being added and removed, searches my succeed or fail ++ * ++ * This sets up two readers for every 10 entries; one forward and one reverse ++ * reading. ++ */ ++static void rcu_stress(struct maple_tree *mt, bool forward) ++{ ++ unsigned int count, i; ++ unsigned long r, seed; ++ pthread_t readers[RCU_RANGE_COUNT / 5]; ++ struct rcu_test_struct2 test; ++ struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5]; ++ void *(*function)(void *); ++ ++ /* Test setup */ ++ test.mt = mt; ++ test.pause = 5; ++ test.seen_toggle = 0; ++ test.seen_deleted = 0; ++ test.seen_added = 0; ++ test.seen_modified = 0; ++ test.thread_count = 0; ++ test.start = test.stop = false; ++ seed = time(NULL); ++ srand(seed); ++ for (i = 0; i < RCU_RANGE_COUNT; i++) { ++ r = seed + rand(); ++ mtree_store_range(mt, seed, r, ++ xa_mk_value(seed), GFP_KERNEL); ++ ++ /* Record start and end of entry */ ++ test.index[i] = seed; ++ test.last[i] = r; ++ seed = 1 + r + rand() % 10; ++ } ++ ++ i = count = ARRAY_SIZE(readers); ++ while (i--) { ++ unsigned long id; ++ ++ id = i / 2 * 10; ++ if (i % 2) ++ function = rcu_reader_fwd; ++ else ++ function = rcu_reader_rev; + -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ rcu_reader_setup(&test_reader[i], id, &test); ++ if (pthread_create(&readers[i], NULL, *function, ++ &test_reader[i])) { ++ perror("creating reader thread"); ++ exit(1); ++ } ++ } + -+ if (!lruvec) -+ continue; ++ for (i = 0; i < ARRAY_SIZE(readers); i++) { ++ struct rcu_reader_struct *this = &test_reader[i]; ++ int add = this->id + this->add; + -+ spin_lock_irq(&lruvec->lru_lock); ++ /* Remove add entries from the tree for later addition */ ++ mtree_store_range(mt, test.index[add], test.last[add], ++ NULL, GFP_KERNEL); ++ } + -+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); -+ VM_WARN_ON_ONCE(!state_is_valid(lruvec)); ++ mt_set_in_rcu(mt); ++ do { ++ usleep(5); ++ } while (test.thread_count > ARRAY_SIZE(readers)); + -+ lruvec->lrugen.enabled = enabled; ++ if (forward) ++ rcu_stress_fwd(mt, &test, count, test_reader); ++ else ++ rcu_stress_rev(mt, &test, count, test_reader); ++ ++ test.stop = true; ++ while (count--) ++ pthread_join(readers[count], NULL); ++ ++ mt_validate(mt); ++} ++ ++ ++struct rcu_test_struct { ++ struct maple_tree *mt; /* the maple tree */ ++ int count; /* Number of times to check value(s) */ ++ unsigned long index; /* The first index to check */ ++ void *entry1; /* The first entry value */ ++ void *entry2; /* The second entry value */ ++ void *entry3; /* The third entry value */ ++ ++ bool update_2; ++ bool update_3; ++ unsigned long range_start; ++ unsigned long range_end; ++ unsigned int loop_sleep; ++ unsigned int val_sleep; ++ ++ unsigned int failed; /* failed detection for other threads */ ++ unsigned int seen_entry2; /* Number of threads that have seen the new value */ ++ unsigned int seen_entry3; /* Number of threads that have seen the new value */ ++ unsigned int seen_both; /* Number of threads that have seen both new values */ ++ unsigned int seen_toggle; ++ unsigned int seen_added; ++ unsigned int seen_removed; ++ unsigned long last; /* The end of the range to write. */ ++ ++ unsigned long removed; /* The index of the removed entry */ ++ unsigned long added; /* The index of the removed entry */ ++ unsigned long toggle; /* The index of the removed entry */ ++}; + -+ while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { -+ spin_unlock_irq(&lruvec->lru_lock); -+ cond_resched(); -+ spin_lock_irq(&lruvec->lru_lock); -+ } ++static inline ++int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2, ++ bool *update_3) ++{ ++ if (entry == test->entry1) ++ return 0; + -+ spin_unlock_irq(&lruvec->lru_lock); ++ if (entry == test->entry2) { ++ if (!(*update_2)) { ++ uatomic_inc(&test->seen_entry2); ++ *update_2 = true; ++ if (update_3) ++ uatomic_inc(&test->seen_both); + } ++ return 0; ++ } + -+ cond_resched(); -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -+unlock: -+ mutex_unlock(&state_mutex); -+ put_online_mems(); -+ cpus_read_unlock(); -+ cgroup_unlock(); -+} ++ if (entry == test->entry3) { ++ if (!(*update_3)) { ++ uatomic_inc(&test->seen_entry3); ++ *update_3 = true; ++ if (update_2) ++ uatomic_inc(&test->seen_both); ++ } ++ return 0; ++ } + -+/****************************************************************************** -+ * sysfs interface -+ ******************************************************************************/ ++ return 1; ++} + -+static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) ++/* ++ * rcu_val() - Read a given value in the tree test->count times using the ++ * regular API ++ * ++ * @ptr: The pointer to the rcu_test_struct ++ */ ++static void *rcu_val(void *ptr) +{ -+ return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); ++ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; ++ unsigned long count = test->count; ++ bool update_2 = false; ++ bool update_3 = false; ++ void *entry; ++ ++ rcu_register_thread(); ++ while (count--) { ++ usleep(test->val_sleep); ++ /* ++ * No locking required, regular API locking is handled in the ++ * maple tree code ++ */ ++ entry = mtree_load(test->mt, test->index); ++ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2, ++ &update_3)); ++ } ++ rcu_unregister_thread(); ++ return NULL; +} + -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, -+ const char *buf, size_t len) ++/* ++ * rcu_loop() - Loop over a section of the maple tree, checking for an expected ++ * value using the advanced API ++ * ++ * @ptr - The pointer to the rcu_test_struct ++ */ ++static void *rcu_loop(void *ptr) +{ -+ unsigned int msecs; ++ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; ++ unsigned long count = test->count; ++ void *entry, *expected; ++ bool update_2 = false; ++ bool update_3 = false; ++ MA_STATE(mas, test->mt, test->range_start, test->range_start); + -+ if (kstrtouint(buf, 0, &msecs)) -+ return -EINVAL; ++ rcu_register_thread(); + -+ WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); ++ /* ++ * Loop through the test->range_start - test->range_end test->count ++ * times ++ */ ++ while (count--) { ++ usleep(test->loop_sleep); ++ rcu_read_lock(); ++ mas_for_each(&mas, entry, test->range_end) { ++ /* The expected value is based on the start range. */ ++ expected = xa_mk_value(mas.index ? mas.index / 10 : 0); ++ ++ /* Out of the interesting range */ ++ if (mas.index < test->index || mas.index > test->last) { ++ if (entry != expected) { ++ printk("%lx - %lx = %p not %p\n", ++ mas.index, mas.last, entry, expected); ++ } ++ MT_BUG_ON(test->mt, entry != expected); ++ continue; ++ } + -+ return len; -+} ++ if (entry == expected) ++ continue; /* Not seen. */ + -+static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( -+ min_ttl_ms, 0644, show_min_ttl, store_min_ttl -+); ++ /* In the interesting range */ ++ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, ++ &update_2, ++ &update_3)); ++ } ++ rcu_read_unlock(); ++ mas_set(&mas, test->range_start); ++ } + -+static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) ++ rcu_unregister_thread(); ++ return NULL; ++} ++ ++static noinline ++void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals) +{ -+ unsigned int caps = 0; + -+ if (get_cap(LRU_GEN_CORE)) -+ caps |= BIT(LRU_GEN_CORE); ++ int i; ++ void *(*function)(void *); ++ pthread_t readers[20]; + -+ if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) -+ caps |= BIT(LRU_GEN_MM_WALK); ++ mt_set_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_in_rcu(mt)); + -+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) -+ caps |= BIT(LRU_GEN_NONLEAF_YOUNG); ++ for (i = 0; i < ARRAY_SIZE(readers); i++) { ++ if (i % 2) ++ function = rcu_loop; ++ else ++ function = rcu_val; + -+ return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); ++ if (pthread_create(&readers[i], NULL, *function, vals)) { ++ perror("creating reader thread"); ++ exit(1); ++ } ++ } ++ ++ usleep(5); /* small yield to ensure all threads are at least started. */ ++ mtree_store_range(mt, vals->index, vals->last, vals->entry2, ++ GFP_KERNEL); ++ while (i--) ++ pthread_join(readers[i], NULL); ++ ++ /* Make sure the test caught at least one update. */ ++ MT_BUG_ON(mt, !vals->seen_entry2); +} + -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, -+ const char *buf, size_t len) ++static noinline ++void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals) +{ -+ int i; -+ unsigned int caps; + -+ if (tolower(*buf) == 'n') -+ caps = 0; -+ else if (tolower(*buf) == 'y') -+ caps = -1; -+ else if (kstrtouint(buf, 0, &caps)) -+ return -EINVAL; ++ int i; ++ void *(*function)(void *); ++ pthread_t readers[20]; ++ unsigned int index = vals->index; + -+ for (i = 0; i < NR_LRU_GEN_CAPS; i++) { -+ bool enabled = caps & BIT(i); ++ mt_set_in_rcu(mt); ++ MT_BUG_ON(mt, !mt_in_rcu(mt)); + -+ if (i == LRU_GEN_CORE) -+ lru_gen_change_state(enabled); -+ else if (enabled) -+ static_branch_enable(&lru_gen_caps[i]); ++ for (i = 0; i < ARRAY_SIZE(readers); i++) { ++ if (i % 2) ++ function = rcu_loop; + else -+ static_branch_disable(&lru_gen_caps[i]); ++ function = rcu_val; ++ ++ if (pthread_create(&readers[i], NULL, *function, vals)) { ++ perror("creating reader thread"); ++ exit(1); ++ } + } + -+ return len; ++ usleep(5); /* small yield to ensure all threads are at least started. */ ++ ++ while (index <= vals->last) { ++ mtree_store(mt, index, ++ (index % 2 ? vals->entry2 : vals->entry3), ++ GFP_KERNEL); ++ index++; ++ usleep(5); ++ } ++ ++ while (i--) ++ pthread_join(readers[i], NULL); ++ ++ /* Make sure the test caught at least one update. */ ++ MT_BUG_ON(mt, !vals->seen_entry2); ++ MT_BUG_ON(mt, !vals->seen_entry3); ++ MT_BUG_ON(mt, !vals->seen_both); +} ++static noinline void check_rcu_simulated(struct maple_tree *mt) ++{ ++ unsigned long i, nr_entries = 1000; ++ unsigned long target = 4320; ++ unsigned long val = 0xDEAD; + -+static struct kobj_attribute lru_gen_enabled_attr = __ATTR( -+ enabled, 0644, show_enabled, store_enabled -+); ++ MA_STATE(mas_writer, mt, 0, 0); ++ MA_STATE(mas_reader, mt, target, target); + -+static struct attribute *lru_gen_attrs[] = { -+ &lru_gen_min_ttl_attr.attr, -+ &lru_gen_enabled_attr.attr, -+ NULL -+}; ++ rcu_register_thread(); + -+static struct attribute_group lru_gen_attr_group = { -+ .name = "lru_gen", -+ .attrs = lru_gen_attrs, -+}; ++ mt_set_in_rcu(mt); ++ mas_lock(&mas_writer); ++ for (i = 0; i <= nr_entries; i++) { ++ mas_writer.index = i * 10; ++ mas_writer.last = i * 10 + 5; ++ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); ++ } ++ mas_unlock(&mas_writer); + -+/****************************************************************************** -+ * debugfs interface -+ ******************************************************************************/ ++ /* Overwrite one entry with a new value. */ ++ mas_set_range(&mas_writer, target, target + 5); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); + -+static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) -+{ -+ struct mem_cgroup *memcg; -+ loff_t nr_to_skip = *pos; ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); + -+ m->private = kvmalloc(PATH_MAX, GFP_KERNEL); -+ if (!m->private) -+ return ERR_PTR(-ENOMEM); + -+ memcg = mem_cgroup_iter(NULL, NULL, NULL); -+ do { -+ int nid; ++ /* Overwrite 1/2 the entry */ ++ mas_set_range(&mas_writer, target, target + 2); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); + -+ for_each_node_state(nid, N_MEMORY) { -+ if (!nr_to_skip--) -+ return get_lruvec(memcg, nid); -+ } -+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + -+ return NULL; -+} ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); + -+static void lru_gen_seq_stop(struct seq_file *m, void *v) -+{ -+ if (!IS_ERR_OR_NULL(v)) -+ mem_cgroup_iter_break(NULL, lruvec_memcg(v)); ++ /* Overwrite last 1/2 the entry */ ++ mas_set_range(&mas_writer, target + 2, target + 5); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ rcu_read_unlock(); + -+ kvfree(m->private); -+ m->private = NULL; -+} + -+static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite more than the entry */ ++ mas_set_range(&mas_writer, target - 5, target + 15); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite more than the node. */ ++ mas_set_range(&mas_writer, target - 400, target + 400); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ mas_reset(&mas_reader); ++ ++ /* Overwrite the tree */ ++ mas_set_range(&mas_writer, 0, ULONG_MAX); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Clear out tree & recreate it */ ++ mas_lock(&mas_writer); ++ mas_set_range(&mas_writer, 0, ULONG_MAX); ++ mas_store_gfp(&mas_writer, NULL, GFP_KERNEL); ++ mas_set_range(&mas_writer, 0, 0); ++ for (i = 0; i <= nr_entries; i++) { ++ mas_writer.index = i * 10; ++ mas_writer.last = i * 10 + 5; ++ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); ++ } ++ mas_unlock(&mas_writer); ++ ++ /* next check */ ++ /* Overwrite one entry with a new value. */ ++ mas_reset(&mas_reader); ++ mas_set_range(&mas_writer, target, target + 5); ++ mas_set_range(&mas_reader, target, target); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_prev(&mas_reader, 0); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ /* Restore value. */ ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ ++ /* prev check */ ++ /* Overwrite one entry with a new value. */ ++ mas_reset(&mas_reader); ++ mas_set_range(&mas_writer, target, target + 5); ++ mas_set_range(&mas_reader, target, target); ++ rcu_read_lock(); ++ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); ++ mas_next(&mas_reader, ULONG_MAX); ++ mas_lock(&mas_writer); ++ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); ++ mas_unlock(&mas_writer); ++ MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val)); ++ rcu_read_unlock(); ++ ++ rcu_unregister_thread(); ++} ++ ++static noinline void check_rcu_threaded(struct maple_tree *mt) ++{ ++ unsigned long i, nr_entries = 1000; ++ struct rcu_test_struct vals; ++ ++ vals.val_sleep = 200; ++ vals.loop_sleep = 110; ++ ++ rcu_register_thread(); ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ /* Store across several slots. */ ++ vals.count = 1000; ++ vals.mt = mt; ++ vals.index = 8650; ++ vals.last = 8666; ++ vals.entry1 = xa_mk_value(865); ++ vals.entry2 = xa_mk_value(8650); ++ vals.entry3 = xa_mk_value(8650); ++ vals.range_start = 0; ++ vals.range_end = ULONG_MAX; ++ vals.seen_entry2 = 0; ++ vals.seen_entry3 = 0; ++ ++ run_check_rcu(mt, &vals); ++ mtree_destroy(mt); ++ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ /* 4390-4395: value 439 (0x1b7) [0x36f] */ ++ /* Store across several slots. */ ++ /* Spanning store. */ ++ vals.count = 10000; ++ vals.mt = mt; ++ vals.index = 4390; ++ vals.last = 4398; ++ vals.entry1 = xa_mk_value(4390); ++ vals.entry2 = xa_mk_value(439); ++ vals.entry3 = xa_mk_value(439); ++ vals.seen_entry2 = 0; ++ vals.range_start = 4316; ++ vals.range_end = 5035; ++ run_check_rcu(mt, &vals); ++ mtree_destroy(mt); ++ ++ ++ /* Forward writer for rcu stress */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ rcu_stress(mt, true); ++ mtree_destroy(mt); ++ ++ /* Reverse writer for rcu stress */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ rcu_stress(mt, false); ++ mtree_destroy(mt); ++ ++ /* Slow reader test with spanning store. */ ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ /* 4390-4395: value 439 (0x1b7) [0x36f] */ ++ /* Store across several slots. */ ++ /* Spanning store. */ ++ vals.count = 15000; ++ vals.mt = mt; ++ vals.index = 4390; ++ vals.last = 4398; ++ vals.entry1 = xa_mk_value(4390); ++ vals.entry2 = xa_mk_value(439); ++ vals.entry3 = xa_mk_value(4391); ++ vals.seen_toggle = 0; ++ vals.seen_added = 0; ++ vals.seen_removed = 0; ++ vals.range_start = 4316; ++ vals.range_end = 5035; ++ vals.removed = 4360; ++ vals.added = 4396; ++ vals.toggle = 4347; ++ vals.val_sleep = 400; ++ vals.loop_sleep = 200; ++ vals.seen_entry2 = 0; ++ vals.seen_entry3 = 0; ++ vals.seen_both = 0; ++ vals.entry3 = xa_mk_value(438); ++ ++ run_check_rcu_slowread(mt, &vals); ++ rcu_unregister_thread(); ++} ++ ++extern void test_kmem_cache_bulk(void); ++ ++/* Test spanning writes that require balancing right sibling or right cousin */ ++static noinline void check_spanning_relatives(struct maple_tree *mt) ++{ ++ ++ unsigned long i, nr_entries = 1000; ++ ++ for (i = 0; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, i*10 + 5, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ ++ mtree_store_range(mt, 9365, 9955, NULL, GFP_KERNEL); ++} ++ ++static noinline void check_fuzzer(struct maple_tree *mt) +{ -+ int nid = lruvec_pgdat(v)->node_id; -+ struct mem_cgroup *memcg = lruvec_memcg(v); ++ /* ++ * 1. Causes a spanning rebalance of a single root node. ++ * Fixed by setting the correct limit in mast_cp_to_nodes() when the ++ * entire right side is consumed. ++ */ ++ mtree_test_insert(mt, 88, (void *)0xb1); ++ mtree_test_insert(mt, 84, (void *)0xa9); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 14, (void *)0x1d); ++ mtree_test_insert(mt, 7, (void *)0xf); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_insert(mt, 18, (void *)0x25); ++ mtree_test_store_range(mt, 8, 18, (void *)0x11); ++ mtree_destroy(mt); + -+ ++*pos; + -+ nid = next_memory_node(nid); -+ if (nid == MAX_NUMNODES) { -+ memcg = mem_cgroup_iter(NULL, memcg, NULL); -+ if (!memcg) -+ return NULL; ++ /* ++ * 2. Cause a spanning rebalance of two nodes in root. ++ * Fixed by setting mast->r->max correctly. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store(mt, 87, (void *)0xaf); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 4); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_store(mt, 8, (void *)0x11); ++ mtree_test_store(mt, 44, (void *)0x59); ++ mtree_test_store(mt, 68, (void *)0x89); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 43, (void *)0x57); ++ mtree_test_insert(mt, 24, (void *)0x31); ++ mtree_test_insert(mt, 844, (void *)0x699); ++ mtree_test_store(mt, 84, (void *)0xa9); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_erase(mt, 4); ++ mtree_test_load(mt, 5); ++ mtree_test_erase(mt, 0); ++ mtree_destroy(mt); ++ ++ /* ++ * 3. Cause a node overflow on copy ++ * Fixed by using the correct check for node size in mas_wr_modify() ++ * Also discovered issue with metadata setting. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store_range(mt, 0, 18446744073709551615UL, (void *)0x1); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_erase(mt, 5); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 4); ++ mtree_test_store(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 5); ++ mtree_test_store(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 5); ++ mtree_test_erase(mt, 4); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_store(mt, 444, (void *)0x379); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 0); ++ mtree_test_store(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 0); ++ mtree_destroy(mt); ++ ++ /* ++ * 4. spanning store failure due to writing incorrect pivot value at ++ * last slot. ++ * Fixed by setting mast->r->max correctly in mast_cp_to_nodes() ++ * ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_insert(mt, 261, (void *)0x20b); ++ mtree_test_store(mt, 516, (void *)0x409); ++ mtree_test_store(mt, 6, (void *)0xd); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_insert(mt, 1256, (void *)0x9d1); ++ mtree_test_store(mt, 4, (void *)0x9); ++ mtree_test_erase(mt, 1); ++ mtree_test_store(mt, 56, (void *)0x71); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 24, (void *)0x31); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 2263, (void *)0x11af); ++ mtree_test_insert(mt, 446, (void *)0x37d); ++ mtree_test_store_range(mt, 6, 45, (void *)0xd); ++ mtree_test_store_range(mt, 3, 446, (void *)0x7); ++ mtree_destroy(mt); ++ ++ /* ++ * 5. mas_wr_extend_null() may overflow slots. ++ * Fix by checking against wr_mas->node_end. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store(mt, 48, (void *)0x61); ++ mtree_test_store(mt, 3, (void *)0x7); ++ mtree_test_load(mt, 0); ++ mtree_test_store(mt, 88, (void *)0xb1); ++ mtree_test_store(mt, 81, (void *)0xa3); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 2480, (void *)0x1361); ++ mtree_test_insert(mt, 18446744073709551615UL, ++ (void *)0xffffffffffffffff); ++ mtree_test_erase(mt, 18446744073709551615UL); ++ mtree_destroy(mt); ++ ++ /* ++ * 6. When reusing a node with an implied pivot and the node is ++ * shrinking, old data would be left in the implied slot ++ * Fixed by checking the last pivot for the mas->max and clear ++ * accordingly. This only affected the left-most node as that node is ++ * the only one allowed to end in NULL. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_erase(mt, 3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_load(mt, 2); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_load(mt, 2); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_erase(mt, 1); ++ mtree_test_store_range(mt, 4, 62, (void *)0x9); ++ mtree_test_erase(mt, 62); ++ mtree_test_store_range(mt, 1, 0, (void *)0x3); ++ mtree_test_insert(mt, 11, (void *)0x17); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_store(mt, 62, (void *)0x7d); ++ mtree_test_erase(mt, 62); ++ mtree_test_store_range(mt, 1, 15, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_store(mt, 62, (void *)0x7d); ++ mtree_test_erase(mt, 62); ++ mtree_test_insert(mt, 122, (void *)0xf5); ++ mtree_test_store(mt, 3, (void *)0x7); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_store_range(mt, 0, 1, (void *)0x1); ++ mtree_test_insert(mt, 85, (void *)0xab); ++ mtree_test_insert(mt, 72, (void *)0x91); ++ mtree_test_insert(mt, 81, (void *)0xa3); ++ mtree_test_insert(mt, 726, (void *)0x5ad); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 51, (void *)0x67); ++ mtree_test_insert(mt, 611, (void *)0x4c7); ++ mtree_test_insert(mt, 485, (void *)0x3cb); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 26, 1, (void *)0x35); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 22, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 53); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 1, (void *)0x3); ++ mtree_test_insert(mt, 222, (void *)0x1bd); ++ mtree_test_insert(mt, 485, (void *)0x3cb); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 0); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 3, (void *)0x7); ++ mtree_test_store(mt, 621, (void *)0x4db); ++ mtree_test_insert(mt, 0, (void *)0x1); ++ mtree_test_erase(mt, 5); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store(mt, 62, (void *)0x7d); ++ mtree_test_erase(mt, 62); ++ mtree_test_store_range(mt, 1, 0, (void *)0x3); ++ mtree_test_insert(mt, 22, (void *)0x2d); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_store_range(mt, 4, 62, (void *)0x9); ++ mtree_test_erase(mt, 62); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 22, (void *)0x3); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_load(mt, 53); ++ mtree_test_load(mt, 1); ++ mtree_test_store_range(mt, 1, 1, (void *)0x3); ++ mtree_test_insert(mt, 222, (void *)0x1bd); ++ mtree_test_insert(mt, 485, (void *)0x3cb); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_load(mt, 0); ++ mtree_test_load(mt, 0); ++ mtree_destroy(mt); ++ ++ /* ++ * 7. Previous fix was incomplete, fix mas_resuse_node() clearing of old ++ * data by overwriting it first - that way metadata is of no concern. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_load(mt, 1); ++ mtree_test_insert(mt, 102, (void *)0xcd); ++ mtree_test_erase(mt, 2); ++ mtree_test_erase(mt, 0); ++ mtree_test_load(mt, 0); ++ mtree_test_insert(mt, 4, (void *)0x9); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 110, (void *)0xdd); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 5, 0, (void *)0xb); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_store(mt, 112, (void *)0xe1); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 110, 2, (void *)0xdd); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_load(mt, 22); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 210, (void *)0x1a5); ++ mtree_test_store_range(mt, 0, 2, (void *)0x1); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 2); ++ mtree_test_erase(mt, 22); ++ mtree_test_erase(mt, 1); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 112); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 1, 2, (void *)0x3); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_erase(mt, 2); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert_range(mt, 1, 2, (void *)0x3); ++ mtree_test_erase(mt, 0); ++ mtree_test_erase(mt, 2); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_load(mt, 112); ++ mtree_test_store_range(mt, 110, 12, (void *)0xdd); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_load(mt, 110); ++ mtree_test_insert_range(mt, 4, 71, (void *)0x9); ++ mtree_test_load(mt, 2); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_insert_range(mt, 11, 22, (void *)0x17); ++ mtree_test_erase(mt, 12); ++ mtree_test_store(mt, 2, (void *)0x5); ++ mtree_test_load(mt, 22); ++ mtree_destroy(mt); ++ ++ ++ /* ++ * 8. When rebalancing or spanning_rebalance(), the max of the new node ++ * may be set incorrectly to the final pivot and not the right max. ++ * Fix by setting the left max to orig right max if the entire node is ++ * consumed. ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_store(mt, 6, (void *)0xd); ++ mtree_test_store(mt, 67, (void *)0x87); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 6716, (void *)0x3479); ++ mtree_test_store(mt, 61, (void *)0x7b); ++ mtree_test_insert(mt, 13, (void *)0x1b); ++ mtree_test_store(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_load(mt, 0); ++ mtree_test_erase(mt, 67167); ++ mtree_test_insert_range(mt, 6, 7167, (void *)0xd); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_erase(mt, 67); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 667167); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_store(mt, 67, (void *)0x87); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_erase(mt, 1); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_erase(mt, 67); ++ mtree_test_insert(mt, 15, (void *)0x1f); ++ mtree_test_insert(mt, 67167, (void *)0x20cbf); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_load(mt, 7); ++ mtree_test_insert(mt, 16, (void *)0x21); ++ mtree_test_insert(mt, 36, (void *)0x49); ++ mtree_test_store(mt, 67, (void *)0x87); ++ mtree_test_store(mt, 6, (void *)0xd); ++ mtree_test_insert(mt, 367, (void *)0x2df); ++ mtree_test_insert(mt, 115, (void *)0xe7); ++ mtree_test_store(mt, 0, (void *)0x1); ++ mtree_test_store_range(mt, 1, 3, (void *)0x3); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 67167); ++ mtree_test_insert_range(mt, 6, 47, (void *)0xd); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert_range(mt, 1, 67, (void *)0x3); ++ mtree_test_load(mt, 67); ++ mtree_test_insert(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 67167); ++ mtree_destroy(mt); ++ ++ /* ++ * 9. spanning store to the end of data caused an invalid metadata ++ * length which resulted in a crash eventually. ++ * Fix by checking if there is a value in pivot before incrementing the ++ * metadata end in mab_mas_cp(). To ensure this doesn't happen again, ++ * abstract the two locations this happens into a function called ++ * mas_leaf_set_meta(). ++ */ ++ mt_init_flags(mt, 0); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 12, (void *)0x19); ++ mtree_test_insert(mt, 6, (void *)0xd); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 91, (void *)0xb7); ++ mtree_test_insert(mt, 18, (void *)0x25); ++ mtree_test_insert(mt, 81, (void *)0xa3); ++ mtree_test_store_range(mt, 0, 128, (void *)0x1); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_erase(mt, 8); ++ mtree_test_insert(mt, 11, (void *)0x17); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); ++ mtree_test_erase(mt, 18446744073709551605UL); ++ mtree_test_store_range(mt, 0, 281, (void *)0x1); ++ mtree_test_erase(mt, 2); ++ mtree_test_insert(mt, 1211, (void *)0x977); ++ mtree_test_insert(mt, 111, (void *)0xdf); ++ mtree_test_insert(mt, 13, (void *)0x1b); ++ mtree_test_insert(mt, 211, (void *)0x1a7); ++ mtree_test_insert(mt, 11, (void *)0x17); ++ mtree_test_insert(mt, 5, (void *)0xb); ++ mtree_test_insert(mt, 1218, (void *)0x985); ++ mtree_test_insert(mt, 61, (void *)0x7b); ++ mtree_test_store(mt, 1, (void *)0x3); ++ mtree_test_insert(mt, 121, (void *)0xf3); ++ mtree_test_insert(mt, 8, (void *)0x11); ++ mtree_test_insert(mt, 21, (void *)0x2b); ++ mtree_test_insert(mt, 2, (void *)0x5); ++ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); ++ mtree_test_erase(mt, 18446744073709551605UL); ++} ++static noinline void check_dup_gaps(struct maple_tree *mt, ++ unsigned long nr_entries, bool zero_start, ++ unsigned long gap) ++{ ++ unsigned long i = 0; ++ struct maple_tree newmt; ++ int ret; ++ void *tmp; ++ MA_STATE(mas, mt, 0, 0); ++ MA_STATE(newmas, &newmt, 0, 0); ++ ++ ++ if (!zero_start) ++ i = 1; + -+ nid = first_memory_node; ++ mt_zero_nr_tallocated(); ++ for (; i <= nr_entries; i++) ++ mtree_store_range(mt, i*10, (i+1)*10 - gap, ++ xa_mk_value(i), GFP_KERNEL); ++ ++ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); ++ mt_set_non_kernel(99999); ++ ret = mas_expected_entries(&newmas, nr_entries); ++ mt_set_non_kernel(0); ++ MT_BUG_ON(mt, ret != 0); ++ ++ mas_for_each(&mas, tmp, ULONG_MAX) { ++ newmas.index = mas.index; ++ newmas.last = mas.last; ++ mas_store(&newmas, tmp); + } + -+ return get_lruvec(memcg, nid); ++ mas_destroy(&mas); ++ mas_destroy(&newmas); ++ mtree_destroy(&newmt); +} + -+static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, -+ unsigned long max_seq, unsigned long *min_seq, -+ unsigned long seq) ++static noinline void check_dup(struct maple_tree *mt) +{ + int i; -+ int type, tier; -+ int hist = lru_hist_from_seq(seq); -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; + -+ for (tier = 0; tier < MAX_NR_TIERS; tier++) { -+ seq_printf(m, " %10d", tier); -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ const char *s = " "; -+ unsigned long n[3] = {}; ++ /* Check with a value at zero */ ++ for (i = 10; i < 1000; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } + -+ if (seq == max_seq) { -+ s = "RT "; -+ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); -+ n[1] = READ_ONCE(lrugen->avg_total[type][tier]); -+ } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { -+ s = "rep"; -+ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); -+ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); -+ if (tier) -+ n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); -+ } ++ /* Check with a value at zero, no gap */ ++ for (i = 1000; i < 2000; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, true, 0); ++ mtree_destroy(mt); ++ } + -+ for (i = 0; i < 3; i++) -+ seq_printf(m, " %10lu%c", n[i], s[i]); -+ } -+ seq_putc(m, '\n'); ++ /* Check with a value at zero and unreasonably large */ ++ for (i = 100010; i < 100020; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); + } + -+ seq_puts(m, " "); -+ for (i = 0; i < NR_MM_STATS; i++) { -+ const char *s = " "; -+ unsigned long n = 0; ++ /* Small to medium size not starting at zero*/ ++ for (i = 200; i < 1000; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, false, 5); ++ mtree_destroy(mt); ++ } + -+ if (seq == max_seq && NR_HIST_GENS == 1) { -+ s = "LOYNFA"; -+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); -+ } else if (seq != max_seq && NR_HIST_GENS > 1) { -+ s = "loynfa"; -+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]); -+ } ++ /* Unreasonably large not starting at zero*/ ++ for (i = 100010; i < 100020; i++) { ++ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); ++ check_dup_gaps(mt, i, false, 5); ++ mtree_destroy(mt); ++ } + -+ seq_printf(m, " %10lu%c", n, s[i]); ++ /* Check non-allocation tree not starting at zero */ ++ for (i = 1500; i < 3000; i++) { ++ mt_init_flags(mt, 0); ++ check_dup_gaps(mt, i, false, 5); ++ mtree_destroy(mt); + } -+ seq_putc(m, '\n'); ++ ++ /* Check non-allocation tree starting at zero */ ++ for (i = 200; i < 1000; i++) { ++ mt_init_flags(mt, 0); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } ++ ++ /* Unreasonably large */ ++ for (i = 100015; i < 100020; i++) { ++ mt_init_flags(mt, 0); ++ check_dup_gaps(mt, i, true, 5); ++ mtree_destroy(mt); ++ } ++ +} + -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static int lru_gen_seq_show(struct seq_file *m, void *v) ++static DEFINE_MTREE(tree); ++static int maple_tree_seed(void) +{ -+ unsigned long seq; -+ bool full = !debugfs_real_fops(m->file)->write; -+ struct lruvec *lruvec = v; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; -+ int nid = lruvec_pgdat(lruvec)->node_id; -+ struct mem_cgroup *memcg = lruvec_memcg(lruvec); -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); ++ unsigned long set[] = {5015, 5014, 5017, 25, 1000, ++ 1001, 1002, 1003, 1005, 0, ++ 5003, 5002}; ++ void *ptr = &set; + -+ if (nid == first_memory_node) { -+ const char *path = memcg ? m->private : ""; ++ pr_info("\nTEST STARTING\n\n"); + -+#ifdef CONFIG_MEMCG -+ if (memcg) -+ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_root_expand(&tree); ++ mtree_destroy(&tree); ++ ++#if defined(BENCH_SLOT_STORE) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_slot_store(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_NODE_STORE) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_node_store(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_AWALK) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_awalk(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_WALK) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_walk(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_FORK) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_forking(&tree); ++ mtree_destroy(&tree); ++ goto skip; ++#endif ++#if defined(BENCH_MT_FOR_EACH) ++#define BENCH ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ bench_mt_for_each(&tree); ++ mtree_destroy(&tree); ++ goto skip; +#endif -+ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); -+ } + -+ seq_printf(m, " node %5d\n", nid); ++ test_kmem_cache_bulk(); + -+ if (!full) -+ seq = min_seq[LRU_GEN_ANON]; -+ else if (max_seq >= MAX_NR_GENS) -+ seq = max_seq - MAX_NR_GENS + 1; -+ else -+ seq = 0; ++ mt_init_flags(&tree, 0); ++ check_new_node(&tree); ++ mtree_destroy(&tree); + -+ for (; seq <= max_seq; seq++) { -+ int type, zone; -+ int gen = lru_gen_from_seq(seq); -+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_prealloc(&tree); ++ mtree_destroy(&tree); + -+ seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_spanning_write(&tree); ++ mtree_destroy(&tree); + -+ for (type = 0; type < ANON_AND_FILE; type++) { -+ unsigned long size = 0; -+ char mark = full && seq < min_seq[type] ? 'x' : ' '; ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_null_expand(&tree); ++ mtree_destroy(&tree); + -+ for (zone = 0; zone < MAX_NR_ZONES; zone++) -+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); ++ mt_init_flags(&tree, 0); ++ check_dfs_preorder(&tree); ++ mtree_destroy(&tree); + -+ seq_printf(m, " %10lu%c", size, mark); -+ } ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_forking(&tree); ++ mtree_destroy(&tree); + -+ seq_putc(m, '\n'); ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_mas_store_gfp(&tree); ++ mtree_destroy(&tree); + -+ if (full) -+ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); -+ } ++ /* Test ranges (store and insert) */ ++ mt_init_flags(&tree, 0); ++ check_ranges(&tree); ++ mtree_destroy(&tree); + -+ return 0; -+} ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_alloc_range(&tree); ++ mtree_destroy(&tree); + -+static const struct seq_operations lru_gen_seq_ops = { -+ .start = lru_gen_seq_start, -+ .stop = lru_gen_seq_stop, -+ .next = lru_gen_seq_next, -+ .show = lru_gen_seq_show, -+}; ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_alloc_rev_range(&tree); ++ mtree_destroy(&tree); + -+static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, -+ bool can_swap, bool force_scan) -+{ -+ DEFINE_MAX_SEQ(lruvec); -+ DEFINE_MIN_SEQ(lruvec); ++ mt_init_flags(&tree, 0); + -+ if (seq < max_seq) -+ return 0; ++ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ + -+ if (seq > max_seq) -+ return -EINVAL; ++ check_insert(&tree, set[9], &tree); /* Insert 0 */ ++ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ ++ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ + -+ if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) -+ return -ERANGE; ++ check_insert(&tree, set[10], ptr); /* Insert 5003 */ ++ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ ++ check_load(&tree, set[11], NULL); /* See if 5002 -> NULL */ ++ check_load(&tree, set[10], ptr); /* See if 5003 -> ptr */ + -+ try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); ++ /* Clear out the tree */ ++ mtree_destroy(&tree); + -+ return 0; ++ /* Try to insert, insert a dup, and load back what was inserted. */ ++ mt_init_flags(&tree, 0); ++ check_insert(&tree, set[0], &tree); /* Insert 5015 */ ++ check_dup_insert(&tree, set[0], &tree); /* Insert 5015 again */ ++ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ ++ ++ /* ++ * Second set of tests try to load a value that doesn't exist, inserts ++ * a second value, then loads the value again ++ */ ++ check_load(&tree, set[1], NULL); /* See if 5014 -> NULL */ ++ check_insert(&tree, set[1], ptr); /* insert 5014 -> ptr */ ++ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ ++ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ ++ /* ++ * Tree currently contains: ++ * p[0]: 14 -> (nil) p[1]: 15 -> ptr p[2]: 16 -> &tree p[3]: 0 -> (nil) ++ */ ++ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ ++ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ ++ ++ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ ++ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ ++ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ ++ check_load(&tree, set[7], &tree); /* 1003 = &tree ? */ ++ ++ /* Clear out tree */ ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ /* Test inserting into a NULL hole. */ ++ check_insert(&tree, set[5], ptr); /* insert 1001 -> ptr */ ++ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ ++ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ ++ check_load(&tree, set[5], ptr); /* See if 1001 -> ptr */ ++ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ ++ check_load(&tree, set[7], &tree); /* See if 1003 -> &tree */ ++ ++ /* Clear out the tree */ ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_erase_testset(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ /* ++ * set[] = {5015, 5014, 5017, 25, 1000, ++ * 1001, 1002, 1003, 1005, 0, ++ * 5003, 5002}; ++ */ ++ ++ check_insert(&tree, set[0], ptr); /* 5015 */ ++ check_insert(&tree, set[1], &tree); /* 5014 */ ++ check_insert(&tree, set[2], ptr); /* 5017 */ ++ check_insert(&tree, set[3], &tree); /* 25 */ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_insert(&tree, set[4], ptr); /* 1000 < Should split. */ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); /*25 */ ++ check_load(&tree, set[4], ptr); ++ check_insert(&tree, set[5], &tree); /* 1001 */ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_load(&tree, set[4], ptr); ++ check_load(&tree, set[5], &tree); ++ check_insert(&tree, set[6], ptr); ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_load(&tree, set[4], ptr); ++ check_load(&tree, set[5], &tree); ++ check_load(&tree, set[6], ptr); ++ check_insert(&tree, set[7], &tree); ++ check_load(&tree, set[0], ptr); ++ check_insert(&tree, set[8], ptr); ++ ++ check_insert(&tree, set[9], &tree); ++ ++ check_load(&tree, set[0], ptr); ++ check_load(&tree, set[1], &tree); ++ check_load(&tree, set[2], ptr); ++ check_load(&tree, set[3], &tree); ++ check_load(&tree, set[4], ptr); ++ check_load(&tree, set[5], &tree); ++ check_load(&tree, set[6], ptr); ++ check_load(&tree, set[9], &tree); ++ mtree_destroy(&tree); ++ ++ check_nomem(&tree); ++ mt_init_flags(&tree, 0); ++ check_seq(&tree, 16, false); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_seq(&tree, 1000, true); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rev_seq(&tree, 1000, true); ++ mtree_destroy(&tree); ++ ++ check_lower_bound_split(&tree); ++ check_upper_bound_split(&tree); ++ check_mid_split(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_next_entry(&tree); ++ check_find(&tree); ++ check_find_2(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_prev_entry(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_erase2_sets(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_gap_combining(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_node_overwrite(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ next_prev_test(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rcu_simulated(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rcu_threaded(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_spanning_relatives(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_rev_find(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, 0); ++ check_fuzzer(&tree); ++ mtree_destroy(&tree); ++ ++ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); ++ check_dup(&tree); ++ mtree_destroy(&tree); ++ ++#if defined(BENCH) ++skip: ++#endif ++ rcu_barrier(); ++ pr_info("maple_tree: %u of %u tests passed\n", ++ atomic_read(&maple_tree_tests_passed), ++ atomic_read(&maple_tree_tests_run)); ++ if (atomic_read(&maple_tree_tests_run) == ++ atomic_read(&maple_tree_tests_passed)) ++ return 0; ++ ++ return -EINVAL; +} + -+static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, -+ int swappiness, unsigned long nr_to_reclaim) ++static void maple_tree_harvest(void) +{ -+ DEFINE_MAX_SEQ(lruvec); + -+ if (seq + MIN_NR_GENS > max_seq) -+ return -EINVAL; ++} + -+ sc->nr_reclaimed = 0; ++module_init(maple_tree_seed); ++module_exit(maple_tree_harvest); ++MODULE_AUTHOR("Liam R. Howlett "); ++MODULE_LICENSE("GPL"); +diff --git a/mm/Makefile b/mm/Makefile +index 488f604e77e0..a731d1decbb1 100644 +--- a/mm/Makefile ++++ b/mm/Makefile +@@ -52,7 +52,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ + readahead.o swap.o truncate.o vmscan.o shmem.o \ + util.o mmzone.o vmstat.o backing-dev.o \ + mm_init.o percpu.o slab_common.o \ +- compaction.o vmacache.o \ ++ compaction.o \ + interval_tree.o list_lru.o workingset.o \ + debug.o gup.o mmap_lock.o $(mmu-y) + +diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h +index d4f55f349100..bce37c487540 100644 +--- a/mm/damon/vaddr-test.h ++++ b/mm/damon/vaddr-test.h +@@ -14,33 +14,19 @@ + + #include + +-static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) ++static void __link_vmas(struct maple_tree *mt, struct vm_area_struct *vmas, ++ ssize_t nr_vmas) + { +- int i, j; +- unsigned long largest_gap, gap; ++ int i; ++ MA_STATE(mas, mt, 0, 0); + + if (!nr_vmas) + return; + +- for (i = 0; i < nr_vmas - 1; i++) { +- vmas[i].vm_next = &vmas[i + 1]; +- +- vmas[i].vm_rb.rb_left = NULL; +- vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb; +- +- largest_gap = 0; +- for (j = i; j < nr_vmas; j++) { +- if (j == 0) +- continue; +- gap = vmas[j].vm_start - vmas[j - 1].vm_end; +- if (gap > largest_gap) +- largest_gap = gap; +- } +- vmas[i].rb_subtree_gap = largest_gap; +- } +- vmas[i].vm_next = NULL; +- vmas[i].vm_rb.rb_right = NULL; +- vmas[i].rb_subtree_gap = 0; ++ mas_lock(&mas); ++ for (i = 0; i < nr_vmas; i++) ++ vma_mas_store(&vmas[i], &mas); ++ mas_unlock(&mas); + } + + /* +@@ -72,6 +58,7 @@ static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) + */ + static void damon_test_three_regions_in_vmas(struct kunit *test) + { ++ static struct mm_struct mm; + struct damon_addr_range regions[3] = {0,}; + /* 10-20-25, 200-210-220, 300-305, 307-330 */ + struct vm_area_struct vmas[] = { +@@ -83,9 +70,10 @@ static void damon_test_three_regions_in_vmas(struct kunit *test) + (struct vm_area_struct) {.vm_start = 307, .vm_end = 330}, + }; + +- __link_vmas(vmas, 6); ++ mt_init_flags(&mm.mm_mt, MM_MT_FLAGS); ++ __link_vmas(&mm.mm_mt, vmas, ARRAY_SIZE(vmas)); + +- __damon_va_three_regions(&vmas[0], regions); ++ __damon_va_three_regions(&mm, regions); + + KUNIT_EXPECT_EQ(test, 10ul, regions[0].start); + KUNIT_EXPECT_EQ(test, 25ul, regions[0].end); +diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c +index 3c7b9d6dca95..d24148a8149f 100644 +--- a/mm/damon/vaddr.c ++++ b/mm/damon/vaddr.c +@@ -113,37 +113,38 @@ static unsigned long sz_range(struct damon_addr_range *r) + * + * Returns 0 if success, or negative error code otherwise. + */ +-static int __damon_va_three_regions(struct vm_area_struct *vma, ++static int __damon_va_three_regions(struct mm_struct *mm, + struct damon_addr_range regions[3]) + { +- struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0}; +- struct vm_area_struct *last_vma = NULL; +- unsigned long start = 0; +- struct rb_root rbroot; +- +- /* Find two biggest gaps so that first_gap > second_gap > others */ +- for (; vma; vma = vma->vm_next) { +- if (!last_vma) { +- start = vma->vm_start; +- goto next; +- } ++ struct damon_addr_range first_gap = {0}, second_gap = {0}; ++ VMA_ITERATOR(vmi, mm, 0); ++ struct vm_area_struct *vma, *prev = NULL; ++ unsigned long start; + +- if (vma->rb_subtree_gap <= sz_range(&second_gap)) { +- rbroot.rb_node = &vma->vm_rb; +- vma = rb_entry(rb_last(&rbroot), +- struct vm_area_struct, vm_rb); ++ /* ++ * Find the two biggest gaps so that first_gap > second_gap > others. ++ * If this is too slow, it can be optimised to examine the maple ++ * tree gaps. ++ */ ++ for_each_vma(vmi, vma) { ++ unsigned long gap; + -+ while (!signal_pending(current)) { -+ DEFINE_MIN_SEQ(lruvec); ++ if (!prev) { ++ start = vma->vm_start; + goto next; + } +- +- gap.start = last_vma->vm_end; +- gap.end = vma->vm_start; +- if (sz_range(&gap) > sz_range(&second_gap)) { +- swap(gap, second_gap); +- if (sz_range(&second_gap) > sz_range(&first_gap)) +- swap(second_gap, first_gap); ++ gap = vma->vm_start - prev->vm_end; ++ ++ if (gap > sz_range(&first_gap)) { ++ second_gap = first_gap; ++ first_gap.start = prev->vm_end; ++ first_gap.end = vma->vm_start; ++ } else if (gap > sz_range(&second_gap)) { ++ second_gap.start = prev->vm_end; ++ second_gap.end = vma->vm_start; + } + next: +- last_vma = vma; ++ prev = vma; + } + + if (!sz_range(&second_gap) || !sz_range(&first_gap)) +@@ -159,7 +160,7 @@ static int __damon_va_three_regions(struct vm_area_struct *vma, + regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); + regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); + regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); +- regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION); ++ regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION); + + return 0; + } +@@ -180,7 +181,7 @@ static int damon_va_three_regions(struct damon_target *t, + return -EINVAL; + + mmap_read_lock(mm); +- rc = __damon_va_three_regions(mm->mmap, regions); ++ rc = __damon_va_three_regions(mm, regions); + mmap_read_unlock(mm); + + mmput(mm); +diff --git a/mm/debug.c b/mm/debug.c +index bef329bf28f0..0fd15ba70d16 100644 +--- a/mm/debug.c ++++ b/mm/debug.c +@@ -139,13 +139,11 @@ EXPORT_SYMBOL(dump_page); + + void dump_vma(const struct vm_area_struct *vma) + { +- pr_emerg("vma %px start %px end %px\n" +- "next %px prev %px mm %px\n" ++ pr_emerg("vma %px start %px end %px mm %px\n" + "prot %lx anon_vma %px vm_ops %px\n" + "pgoff %lx file %px private_data %px\n" + "flags: %#lx(%pGv)\n", +- vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_next, +- vma->vm_prev, vma->vm_mm, ++ vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, + (unsigned long)pgprot_val(vma->vm_page_prot), + vma->anon_vma, vma->vm_ops, vma->vm_pgoff, + vma->vm_file, vma->vm_private_data, +@@ -155,11 +153,11 @@ EXPORT_SYMBOL(dump_vma); + + void dump_mm(const struct mm_struct *mm) + { +- pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n" ++ pr_emerg("mm %px task_size %lu\n" + #ifdef CONFIG_MMU + "get_unmapped_area %px\n" + #endif +- "mmap_base %lu mmap_legacy_base %lu highest_vm_end %lu\n" ++ "mmap_base %lu mmap_legacy_base %lu\n" + "pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n" + "hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n" + "pinned_vm %llx data_vm %lx exec_vm %lx stack_vm %lx\n" +@@ -183,11 +181,11 @@ void dump_mm(const struct mm_struct *mm) + "tlb_flush_pending %d\n" + "def_flags: %#lx(%pGv)\n", + +- mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size, ++ mm, mm->task_size, + #ifdef CONFIG_MMU + mm->get_unmapped_area, + #endif +- mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end, ++ mm->mmap_base, mm->mmap_legacy_base, + mm->pgd, atomic_read(&mm->mm_users), + atomic_read(&mm->mm_count), + mm_pgtables_bytes(mm), +diff --git a/mm/gup.c b/mm/gup.c +index 5abdaf487460..5f3c464dbce1 100644 +--- a/mm/gup.c ++++ b/mm/gup.c +@@ -1667,10 +1667,11 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) + if (!locked) { + locked = 1; + mmap_read_lock(mm); +- vma = find_vma(mm, nstart); ++ vma = find_vma_intersection(mm, nstart, end); + } else if (nstart >= vma->vm_end) +- vma = vma->vm_next; +- if (!vma || vma->vm_start >= end) ++ vma = find_vma_intersection(mm, vma->vm_end, end); ++ ++ if (!vma) + break; + /* + * Set [nstart; nend) to intersection of desired address +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 786497dd5f26..cca500fcfb64 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -2319,11 +2319,11 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, + split_huge_pmd_if_needed(vma, end); + + /* +- * If we're also updating the vma->vm_next->vm_start, ++ * If we're also updating the next vma vm_start, + * check if we need to split it. + */ + if (adjust_next > 0) { +- struct vm_area_struct *next = vma->vm_next; ++ struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); + unsigned long nstart = next->vm_start; + nstart += adjust_next; + split_huge_pmd_if_needed(next, nstart); +diff --git a/mm/init-mm.c b/mm/init-mm.c +index fbe7844d0912..c9327abb771c 100644 +--- a/mm/init-mm.c ++++ b/mm/init-mm.c +@@ -1,6 +1,6 @@ + // SPDX-License-Identifier: GPL-2.0 + #include +-#include ++#include + #include + #include + #include +@@ -28,7 +28,7 @@ + * and size this cpu_bitmask to NR_CPUS. + */ + struct mm_struct init_mm = { +- .mm_rb = RB_ROOT, ++ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, init_mm.mmap_lock), + .pgd = swapper_pg_dir, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), +diff --git a/mm/internal.h b/mm/internal.h +index cf134d58fd6d..0f106a3982e7 100644 +--- a/mm/internal.h ++++ b/mm/internal.h +@@ -85,8 +85,9 @@ bool __folio_end_writeback(struct folio *folio); + void deactivate_file_folio(struct folio *folio); + void folio_activate(struct folio *folio); + +-void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, +- unsigned long floor, unsigned long ceiling); ++void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, ++ struct vm_area_struct *start_vma, unsigned long floor, ++ unsigned long ceiling); + void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); + + struct zap_details; +@@ -480,9 +481,6 @@ static inline bool is_data_mapping(vm_flags_t flags) + } + + /* mm/util.c */ +-void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev); +-void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma); + struct anon_vma *folio_anon_vma(struct folio *folio); + + #ifdef CONFIG_MMU +diff --git a/mm/khugepaged.c b/mm/khugepaged.c +index 5f7c60b8b269..df890338daed 100644 +--- a/mm/khugepaged.c ++++ b/mm/khugepaged.c +@@ -1387,7 +1387,7 @@ static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *v + void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) + { + unsigned long haddr = addr & HPAGE_PMD_MASK; +- struct vm_area_struct *vma = find_vma(mm, haddr); ++ struct vm_area_struct *vma = vma_lookup(mm, haddr); + struct page *hpage; + pte_t *start_pte, *pte; + pmd_t *pmd; +@@ -2048,6 +2048,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, + __releases(&khugepaged_mm_lock) + __acquires(&khugepaged_mm_lock) + { ++ struct vma_iterator vmi; + struct mm_slot *mm_slot; + struct mm_struct *mm; + struct vm_area_struct *vma; +@@ -2076,11 +2077,13 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, + vma = NULL; + if (unlikely(!mmap_read_trylock(mm))) + goto breakouterloop_mmap_lock; +- if (likely(!hpage_collapse_test_exit(mm))) +- vma = find_vma(mm, khugepaged_scan.address); + + progress++; +- for (; vma; vma = vma->vm_next) { ++ if (unlikely(hpage_collapse_test_exit(mm))) ++ goto breakouterloop; ++ ++ vma_iter_init(&vmi, mm, khugepaged_scan.address); ++ for_each_vma(vmi, vma) { + unsigned long hstart, hend; + + cond_resched(); +diff --git a/mm/ksm.c b/mm/ksm.c +index dde27d7e92d4..b7ee3c87a059 100644 +--- a/mm/ksm.c ++++ b/mm/ksm.c +@@ -981,11 +981,13 @@ static int unmerge_and_remove_all_rmap_items(void) + struct mm_slot, mm_list); + spin_unlock(&ksm_mmlist_lock); + +- for (mm_slot = ksm_scan.mm_slot; +- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { ++ for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; ++ mm_slot = ksm_scan.mm_slot) { ++ VMA_ITERATOR(vmi, mm_slot->mm, 0); ++ + mm = mm_slot->mm; + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (ksm_test_exit(mm)) + break; + if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) +@@ -2242,6 +2244,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + struct mm_slot *slot; + struct vm_area_struct *vma; + struct rmap_item *rmap_item; ++ struct vma_iterator vmi; + int nid; + + if (list_empty(&ksm_mm_head.mm_list)) +@@ -2300,13 +2303,13 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + } + + mm = slot->mm; ++ vma_iter_init(&vmi, mm, ksm_scan.address); + -+ if (seq < min_seq[!swappiness]) -+ return 0; + mmap_read_lock(mm); + if (ksm_test_exit(mm)) +- vma = NULL; +- else +- vma = find_vma(mm, ksm_scan.address); ++ goto no_vmas; + +- for (; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (!(vma->vm_flags & VM_MERGEABLE)) + continue; + if (ksm_scan.address < vma->vm_start) +@@ -2344,6 +2347,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) + } + + if (ksm_test_exit(mm)) { ++no_vmas: + ksm_scan.address = 0; + ksm_scan.rmap_list = &slot->rmap_list; + } +diff --git a/mm/madvise.c b/mm/madvise.c +index af97100a0727..682e1d161aef 100644 +--- a/mm/madvise.c ++++ b/mm/madvise.c +@@ -1242,7 +1242,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, + if (start >= end) + break; + if (prev) +- vma = prev->vm_next; ++ vma = find_vma(mm, prev->vm_end); + else /* madvise_remove dropped mmap_lock */ + vma = find_vma(mm, start); + } +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index 48588e89867a..454a283b9e85 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -5874,7 +5874,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) + unsigned long precharge; + + mmap_read_lock(mm); +- walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL); ++ walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL); + mmap_read_unlock(mm); + + precharge = mc.precharge; +@@ -6172,9 +6172,7 @@ static void mem_cgroup_move_charge(void) + * When we have consumed all precharges and failed in doing + * additional charge, the page walk just aborts. + */ +- walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops, +- NULL); +- ++ walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL); + mmap_read_unlock(mc.mm); + atomic_dec(&mc.from->moving_account); + } +diff --git a/mm/memory.c b/mm/memory.c +index bc4dc2e45dcc..acc2e88f4984 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -391,12 +391,21 @@ void free_pgd_range(struct mmu_gather *tlb, + } while (pgd++, addr = next, addr != end); + } + +-void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, +- unsigned long floor, unsigned long ceiling) ++void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, ++ struct vm_area_struct *vma, unsigned long floor, ++ unsigned long ceiling) + { +- while (vma) { +- struct vm_area_struct *next = vma->vm_next; ++ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); + -+ if (sc->nr_reclaimed >= nr_to_reclaim) -+ return 0; ++ do { + unsigned long addr = vma->vm_start; ++ struct vm_area_struct *next; + -+ if (!evict_folios(lruvec, sc, swappiness, NULL)) -+ return 0; ++ /* ++ * Note: USER_PGTABLES_CEILING may be passed as ceiling and may ++ * be 0. This will underflow and is okay. ++ */ ++ next = mas_find(&mas, ceiling - 1); + + /* + * Hide vma from rmap and truncate_pagecache before freeing +@@ -415,7 +424,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, + while (next && next->vm_start <= vma->vm_end + PMD_SIZE + && !is_vm_hugetlb_page(next)) { + vma = next; +- next = vma->vm_next; ++ next = mas_find(&mas, ceiling - 1); + unlink_anon_vmas(vma); + unlink_file_vma(vma); + } +@@ -423,7 +432,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, + floor, next ? next->vm_start : ceiling); + } + vma = next; +- } ++ } while (vma); + } + + void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) +@@ -1687,6 +1696,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, + /** + * unmap_vmas - unmap a range of memory covered by a list of vma's + * @tlb: address of the caller's struct mmu_gather ++ * @mt: the maple tree + * @vma: the starting vma + * @start_addr: virtual address at which to start unmapping + * @end_addr: virtual address at which to end unmapping +@@ -1702,7 +1712,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, + * ensure that any thus-far unmapped pages are flushed before unmap_vmas() + * drops the lock and schedules. + */ +-void unmap_vmas(struct mmu_gather *tlb, ++void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, + struct vm_area_struct *vma, unsigned long start_addr, + unsigned long end_addr) + { +@@ -1712,12 +1722,14 @@ void unmap_vmas(struct mmu_gather *tlb, + /* Careful - we need to zap private pages too! */ + .even_cows = true, + }; ++ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); + + mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, + start_addr, end_addr); + mmu_notifier_invalidate_range_start(&range); +- for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) ++ do { + unmap_single_vma(tlb, vma, start_addr, end_addr, &details); ++ } while ((vma = mas_find(&mas, end_addr - 1)) != NULL); + mmu_notifier_invalidate_range_end(&range); + } + +@@ -1732,8 +1744,11 @@ void unmap_vmas(struct mmu_gather *tlb, + void zap_page_range(struct vm_area_struct *vma, unsigned long start, + unsigned long size) + { ++ struct maple_tree *mt = &vma->vm_mm->mm_mt; ++ unsigned long end = start + size; + struct mmu_notifier_range range; + struct mmu_gather tlb; ++ MA_STATE(mas, mt, vma->vm_end, vma->vm_end); + + lru_add_drain(); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, +@@ -1741,8 +1756,9 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, + tlb_gather_mmu(&tlb, vma->vm_mm); + update_hiwater_rss(vma->vm_mm); + mmu_notifier_invalidate_range_start(&range); +- for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) ++ do { + unmap_single_vma(&tlb, vma, start, range.end, NULL); ++ } while ((vma = mas_find(&mas, end - 1)) != NULL); + mmu_notifier_invalidate_range_end(&range); + tlb_finish_mmu(&tlb); + } +diff --git a/mm/mempolicy.c b/mm/mempolicy.c +index b73d3248d976..6c27acb6cd63 100644 +--- a/mm/mempolicy.c ++++ b/mm/mempolicy.c +@@ -381,9 +381,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) + void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) + { + struct vm_area_struct *vma; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_write_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) ++ for_each_vma(vmi, vma) + mpol_rebind_policy(vma->vm_policy, new); + mmap_write_unlock(mm); + } +@@ -654,7 +655,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma, + static int queue_pages_test_walk(unsigned long start, unsigned long end, + struct mm_walk *walk) + { +- struct vm_area_struct *vma = walk->vma; ++ struct vm_area_struct *next, *vma = walk->vma; + struct queue_pages *qp = walk->private; + unsigned long endvma = vma->vm_end; + unsigned long flags = qp->flags; +@@ -669,9 +670,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, + /* hole at head side of range */ + return -EFAULT; + } ++ next = find_vma(vma->vm_mm, vma->vm_end); + if (!(flags & MPOL_MF_DISCONTIG_OK) && + ((vma->vm_end < qp->end) && +- (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) ++ (!next || vma->vm_end < next->vm_start))) + /* hole at middle or tail of range */ + return -EFAULT; + +@@ -785,26 +787,24 @@ static int vma_replace_policy(struct vm_area_struct *vma, + static int mbind_range(struct mm_struct *mm, unsigned long start, + unsigned long end, struct mempolicy *new_pol) + { ++ MA_STATE(mas, &mm->mm_mt, start - 1, start - 1); + struct vm_area_struct *prev; + struct vm_area_struct *vma; + int err = 0; + pgoff_t pgoff; +- unsigned long vmstart; +- unsigned long vmend; +- +- vma = find_vma(mm, start); +- VM_BUG_ON(!vma); + +- prev = vma->vm_prev; +- if (start > vma->vm_start) +- prev = vma; ++ prev = mas_find_rev(&mas, 0); ++ if (prev && (start < prev->vm_end)) ++ vma = prev; ++ else ++ vma = mas_next(&mas, end - 1); + +- for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) { +- vmstart = max(start, vma->vm_start); +- vmend = min(end, vma->vm_end); ++ for (; vma; vma = mas_next(&mas, end - 1)) { ++ unsigned long vmstart = max(start, vma->vm_start); ++ unsigned long vmend = min(end, vma->vm_end); + + if (mpol_equal(vma_policy(vma), new_pol)) +- continue; ++ goto next; + + pgoff = vma->vm_pgoff + + ((vmstart - vma->vm_start) >> PAGE_SHIFT); +@@ -813,6 +813,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, + new_pol, vma->vm_userfaultfd_ctx, + anon_vma_name(vma)); + if (prev) { ++ /* vma_merge() invalidated the mas */ ++ mas_pause(&mas); + vma = prev; + goto replace; + } +@@ -820,19 +822,25 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, + err = split_vma(vma->vm_mm, vma, vmstart, 1); + if (err) + goto out; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } + if (vma->vm_end != vmend) { + err = split_vma(vma->vm_mm, vma, vmend, 0); + if (err) + goto out; ++ /* split_vma() invalidated the mas */ ++ mas_pause(&mas); + } +- replace: ++replace: + err = vma_replace_policy(vma, new_pol); + if (err) + goto out; ++next: ++ prev = vma; + } + +- out: ++out: + return err; + } + +@@ -1047,6 +1055,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, + int flags) + { + nodemask_t nmask; ++ struct vm_area_struct *vma; + LIST_HEAD(pagelist); + int err = 0; + struct migration_target_control mtc = { +@@ -1062,8 +1071,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, + * need migration. Between passing in the full user address + * space range and MPOL_MF_DISCONTIG_OK, this call can not fail. + */ ++ vma = find_vma(mm, 0); + VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))); +- queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, ++ queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask, + flags | MPOL_MF_DISCONTIG_OK, &pagelist); + + if (!list_empty(&pagelist)) { +@@ -1193,14 +1203,13 @@ static struct page *new_page(struct page *page, unsigned long start) + struct folio *dst, *src = page_folio(page); + struct vm_area_struct *vma; + unsigned long address; ++ VMA_ITERATOR(vmi, current->mm, start); + gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL; + +- vma = find_vma(current->mm, start); +- while (vma) { ++ for_each_vma(vmi, vma) { + address = page_address_in_vma(page, vma); + if (address != -EFAULT) + break; +- vma = vma->vm_next; + } + + if (folio_test_hugetlb(src)) +@@ -1478,6 +1487,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le + unsigned long vmend; + unsigned long end; + int err = -ENOENT; ++ VMA_ITERATOR(vmi, mm, start); + + start = untagged_addr(start); + if (start & ~PAGE_MASK) +@@ -1503,9 +1513,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le + if (end == start) + return 0; + mmap_write_lock(mm); +- vma = find_vma(mm, start); +- for (; vma && vma->vm_start < end; vma = vma->vm_next) { +- ++ for_each_vma_range(vmi, vma, end) { + vmstart = max(start, vma->vm_start); + vmend = min(end, vma->vm_end); + new = mpol_dup(vma_policy(vma)); +diff --git a/mm/mlock.c b/mm/mlock.c +index b14e929084cc..7032f6dd0ce1 100644 +--- a/mm/mlock.c ++++ b/mm/mlock.c +@@ -471,6 +471,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, + unsigned long nstart, end, tmp; + struct vm_area_struct *vma, *prev; + int error; ++ MA_STATE(mas, ¤t->mm->mm_mt, start, start); + + VM_BUG_ON(offset_in_page(start)); + VM_BUG_ON(len != PAGE_ALIGN(len)); +@@ -479,13 +480,14 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, + return -EINVAL; + if (end == start) + return 0; +- vma = find_vma(current->mm, start); +- if (!vma || vma->vm_start > start) ++ vma = mas_walk(&mas); ++ if (!vma) + return -ENOMEM; + +- prev = vma->vm_prev; + if (start > vma->vm_start) + prev = vma; ++ else ++ prev = mas_prev(&mas, 0); + + for (nstart = start ; ; ) { + vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; +@@ -505,7 +507,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, + if (nstart >= end) + break; + +- vma = prev->vm_next; ++ vma = find_vma(prev->vm_mm, prev->vm_end); + if (!vma || vma->vm_start != nstart) { + error = -ENOMEM; + break; +@@ -526,24 +528,21 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, + { + struct vm_area_struct *vma; + unsigned long count = 0; ++ unsigned long end; ++ VMA_ITERATOR(vmi, mm, start); + +- if (mm == NULL) +- mm = current->mm; ++ /* Don't overflow past ULONG_MAX */ ++ if (unlikely(ULONG_MAX - len < start)) ++ end = ULONG_MAX; ++ else ++ end = start + len; + +- vma = find_vma(mm, start); +- if (vma == NULL) +- return 0; +- +- for (; vma ; vma = vma->vm_next) { +- if (start >= vma->vm_end) +- continue; +- if (start + len <= vma->vm_start) +- break; ++ for_each_vma_range(vmi, vma, end) { + if (vma->vm_flags & VM_LOCKED) { + if (start > vma->vm_start) + count -= (start - vma->vm_start); +- if (start + len < vma->vm_end) { +- count += start + len - vma->vm_start; ++ if (end < vma->vm_end) { ++ count += end - vma->vm_start; + break; + } + count += vma->vm_end - vma->vm_start; +@@ -659,6 +658,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) + */ + static int apply_mlockall_flags(int flags) + { ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + struct vm_area_struct *vma, *prev = NULL; + vm_flags_t to_add = 0; + +@@ -679,7 +679,7 @@ static int apply_mlockall_flags(int flags) + to_add |= VM_LOCKONFAULT; + } + +- for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + vm_flags_t newflags; + + newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; +@@ -687,6 +687,7 @@ static int apply_mlockall_flags(int flags) + + /* Ignore errors */ + mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); ++ mas_pause(&mas); + cond_resched(); + } + out: +diff --git a/mm/mmap.c b/mm/mmap.c +index 9d780f415be3..6fc054c55cd8 100644 +--- a/mm/mmap.c ++++ b/mm/mmap.c +@@ -14,7 +14,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -39,7 +38,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -77,9 +75,10 @@ int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; + static bool ignore_rlimit_data; + core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); + +-static void unmap_region(struct mm_struct *mm, ++static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, + struct vm_area_struct *vma, struct vm_area_struct *prev, +- unsigned long start, unsigned long end); ++ struct vm_area_struct *next, unsigned long start, ++ unsigned long end); + + static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) + { +@@ -132,12 +131,10 @@ void unlink_file_vma(struct vm_area_struct *vma) + } + + /* +- * Close a vm structure and free it, returning the next. ++ * Close a vm structure and free it. + */ +-static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) ++static void remove_vma(struct vm_area_struct *vma) + { +- struct vm_area_struct *next = vma->vm_next; +- + might_sleep(); + if (vma->vm_ops && vma->vm_ops->close) + vma->vm_ops->close(vma); +@@ -145,20 +142,41 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) + fput(vma->vm_file); + mpol_put(vma_policy(vma)); + vm_area_free(vma); +- return next; + } + +-static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, +- struct list_head *uf); ++/* ++ * check_brk_limits() - Use platform specific check of range & verify mlock ++ * limits. ++ * @addr: The address to check ++ * @len: The size of increase. ++ * ++ * Return: 0 on success. ++ */ ++static int check_brk_limits(unsigned long addr, unsigned long len) ++{ ++ unsigned long mapped_addr; + -+ cond_resched(); ++ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); ++ if (IS_ERR_VALUE(mapped_addr)) ++ return mapped_addr; ++ ++ return mlock_future_check(current->mm, current->mm->def_flags, len); ++} ++static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long newbrk, unsigned long oldbrk, ++ struct list_head *uf); ++static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma, ++ unsigned long addr, unsigned long request, unsigned long flags); + SYSCALL_DEFINE1(brk, unsigned long, brk) + { + unsigned long newbrk, oldbrk, origbrk; + struct mm_struct *mm = current->mm; +- struct vm_area_struct *next; ++ struct vm_area_struct *brkvma, *next = NULL; + unsigned long min_brk; + bool populate; + bool downgraded = false; + LIST_HEAD(uf); ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + if (mmap_write_lock_killable(mm)) + return -EINTR; +@@ -200,35 +218,51 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) + + /* + * Always allow shrinking brk. +- * __do_munmap() may downgrade mmap_lock to read. ++ * do_brk_munmap() may downgrade mmap_lock to read. + */ + if (brk <= mm->brk) { + int ret; + ++ /* Search one past newbrk */ ++ mas_set(&mas, newbrk); ++ brkvma = mas_find(&mas, oldbrk); ++ BUG_ON(brkvma == NULL); ++ if (brkvma->vm_start >= oldbrk) ++ goto out; /* mapping intersects with an existing non-brk vma. */ + /* +- * mm->brk must to be protected by write mmap_lock so update it +- * before downgrading mmap_lock. When __do_munmap() fails, +- * mm->brk will be restored from origbrk. ++ * mm->brk must be protected by write mmap_lock. ++ * do_brk_munmap() may downgrade the lock, so update it ++ * before calling do_brk_munmap(). + */ + mm->brk = brk; +- ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true); +- if (ret < 0) { +- mm->brk = origbrk; +- goto out; +- } else if (ret == 1) { ++ ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf); ++ if (ret == 1) { + downgraded = true; +- } +- goto success; ++ goto success; ++ } else if (!ret) ++ goto success; ++ ++ mm->brk = origbrk; ++ goto out; + } + +- /* Check against existing mmap mappings. */ +- next = find_vma(mm, oldbrk); ++ if (check_brk_limits(oldbrk, newbrk - oldbrk)) ++ goto out; ++ ++ /* ++ * Only check if the next VMA is within the stack_guard_gap of the ++ * expansion area ++ */ ++ mas_set(&mas, oldbrk); ++ next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap); + if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + goto out; + ++ brkvma = mas_prev(&mas, mm->start_brk); + /* Ok, looks good - let it rip. */ +- if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) ++ if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) + goto out; ++ + mm->brk = brk; + + success: +@@ -247,104 +281,45 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) + return origbrk; + } + +-static inline unsigned long vma_compute_gap(struct vm_area_struct *vma) +-{ +- unsigned long gap, prev_end; +- +- /* +- * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we +- * allow two stack_guard_gaps between them here, and when choosing +- * an unmapped area; whereas when expanding we only require one. +- * That's a little inconsistent, but keeps the code here simpler. +- */ +- gap = vm_start_gap(vma); +- if (vma->vm_prev) { +- prev_end = vm_end_gap(vma->vm_prev); +- if (gap > prev_end) +- gap -= prev_end; +- else +- gap = 0; +- } +- return gap; +-} +- +-#ifdef CONFIG_DEBUG_VM_RB +-static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma) +-{ +- unsigned long max = vma_compute_gap(vma), subtree_gap; +- if (vma->vm_rb.rb_left) { +- subtree_gap = rb_entry(vma->vm_rb.rb_left, +- struct vm_area_struct, vm_rb)->rb_subtree_gap; +- if (subtree_gap > max) +- max = subtree_gap; +- } +- if (vma->vm_rb.rb_right) { +- subtree_gap = rb_entry(vma->vm_rb.rb_right, +- struct vm_area_struct, vm_rb)->rb_subtree_gap; +- if (subtree_gap > max) +- max = subtree_gap; +- } +- return max; +-} +- +-static int browse_rb(struct mm_struct *mm) +-{ +- struct rb_root *root = &mm->mm_rb; +- int i = 0, j, bug = 0; +- struct rb_node *nd, *pn = NULL; +- unsigned long prev = 0, pend = 0; +- +- for (nd = rb_first(root); nd; nd = rb_next(nd)) { +- struct vm_area_struct *vma; +- vma = rb_entry(nd, struct vm_area_struct, vm_rb); +- if (vma->vm_start < prev) { +- pr_emerg("vm_start %lx < prev %lx\n", +- vma->vm_start, prev); +- bug = 1; +- } +- if (vma->vm_start < pend) { +- pr_emerg("vm_start %lx < pend %lx\n", +- vma->vm_start, pend); +- bug = 1; +- } +- if (vma->vm_start > vma->vm_end) { +- pr_emerg("vm_start %lx > vm_end %lx\n", +- vma->vm_start, vma->vm_end); +- bug = 1; +- } +- spin_lock(&mm->page_table_lock); +- if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) { +- pr_emerg("free gap %lx, correct %lx\n", +- vma->rb_subtree_gap, +- vma_compute_subtree_gap(vma)); +- bug = 1; ++#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) ++extern void mt_validate(struct maple_tree *mt); ++extern void mt_dump(const struct maple_tree *mt); ++ ++/* Validate the maple tree */ ++static void validate_mm_mt(struct mm_struct *mm) ++{ ++ struct maple_tree *mt = &mm->mm_mt; ++ struct vm_area_struct *vma_mt; ++ ++ MA_STATE(mas, mt, 0, 0); ++ ++ mt_validate(&mm->mm_mt); ++ mas_for_each(&mas, vma_mt, ULONG_MAX) { ++ if ((vma_mt->vm_start != mas.index) || ++ (vma_mt->vm_end - 1 != mas.last)) { ++ pr_emerg("issue in %s\n", current->comm); ++ dump_stack(); ++ dump_vma(vma_mt); ++ pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, ++ mas.index, mas.last); ++ pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, ++ vma_mt->vm_start, vma_mt->vm_end); ++ ++ mt_dump(mas.tree); ++ if (vma_mt->vm_end != mas.last + 1) { ++ pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", ++ mm, vma_mt->vm_start, vma_mt->vm_end, ++ mas.index, mas.last); ++ mt_dump(mas.tree); ++ } ++ VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); ++ if (vma_mt->vm_start != mas.index) { ++ pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", ++ mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); ++ mt_dump(mas.tree); ++ } ++ VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); + } +- spin_unlock(&mm->page_table_lock); +- i++; +- pn = nd; +- prev = vma->vm_start; +- pend = vma->vm_end; +- } +- j = 0; +- for (nd = pn; nd; nd = rb_prev(nd)) +- j++; +- if (i != j) { +- pr_emerg("backwards %d, forwards %d\n", j, i); +- bug = 1; +- } +- return bug ? -1 : i; +-} +- +-static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore) +-{ +- struct rb_node *nd; +- +- for (nd = rb_first(root); nd; nd = rb_next(nd)) { +- struct vm_area_struct *vma; +- vma = rb_entry(nd, struct vm_area_struct, vm_rb); +- VM_BUG_ON_VMA(vma != ignore && +- vma->rb_subtree_gap != vma_compute_subtree_gap(vma), +- vma); + } + } + +@@ -352,10 +327,13 @@ static void validate_mm(struct mm_struct *mm) + { + int bug = 0; + int i = 0; +- unsigned long highest_address = 0; +- struct vm_area_struct *vma = mm->mmap; ++ struct vm_area_struct *vma; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + +- while (vma) { ++ validate_mm_mt(mm); ++ ++ mas_for_each(&mas, vma, ULONG_MAX) { ++#ifdef CONFIG_DEBUG_VM_RB + struct anon_vma *anon_vma = vma->anon_vma; + struct anon_vma_chain *avc; + +@@ -365,93 +343,20 @@ static void validate_mm(struct mm_struct *mm) + anon_vma_interval_tree_verify(avc); + anon_vma_unlock_read(anon_vma); + } +- +- highest_address = vm_end_gap(vma); +- vma = vma->vm_next; ++#endif + i++; + } + if (i != mm->map_count) { +- pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); +- bug = 1; +- } +- if (highest_address != mm->highest_vm_end) { +- pr_emerg("mm->highest_vm_end %lx, found %lx\n", +- mm->highest_vm_end, highest_address); +- bug = 1; +- } +- i = browse_rb(mm); +- if (i != mm->map_count) { +- if (i != -1) +- pr_emerg("map_count %d rb %d\n", mm->map_count, i); ++ pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i); + bug = 1; + } + VM_BUG_ON_MM(bug, mm); + } +-#else +-#define validate_mm_rb(root, ignore) do { } while (0) +-#define validate_mm(mm) do { } while (0) +-#endif +- +-RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks, +- struct vm_area_struct, vm_rb, +- unsigned long, rb_subtree_gap, vma_compute_gap) +- +-/* +- * Update augmented rbtree rb_subtree_gap values after vma->vm_start or +- * vma->vm_prev->vm_end values changed, without modifying the vma's position +- * in the rbtree. +- */ +-static void vma_gap_update(struct vm_area_struct *vma) +-{ +- /* +- * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created +- * a callback function that does exactly what we want. +- */ +- vma_gap_callbacks_propagate(&vma->vm_rb, NULL); +-} +- +-static inline void vma_rb_insert(struct vm_area_struct *vma, +- struct rb_root *root) +-{ +- /* All rb_subtree_gap values must be consistent prior to insertion */ +- validate_mm_rb(root, NULL); +- +- rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks); +-} +- +-static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) +-{ +- /* +- * Note rb_erase_augmented is a fairly large inline function, +- * so make sure we instantiate it only once with our desired +- * augmented rbtree callbacks. +- */ +- rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); +-} +- +-static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, +- struct rb_root *root, +- struct vm_area_struct *ignore) +-{ +- /* +- * All rb_subtree_gap values must be consistent prior to erase, +- * with the possible exception of +- * +- * a. the "next" vma being erased if next->vm_start was reduced in +- * __vma_adjust() -> __vma_unlink() +- * b. the vma being erased in detach_vmas_to_be_unmapped() -> +- * vma_rb_erase() +- */ +- validate_mm_rb(root, ignore); +- +- __vma_rb_erase(vma, root); +-} + +-static __always_inline void vma_rb_erase(struct vm_area_struct *vma, +- struct rb_root *root) +-{ +- vma_rb_erase_ignore(vma, root, vma); +-} ++#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ ++#define validate_mm_mt(root) do { } while (0) ++#define validate_mm(mm) do { } while (0) ++#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ + + /* + * vma has some anon_vma assigned, and is already inserted on that +@@ -485,208 +390,220 @@ anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + } + +-static int find_vma_links(struct mm_struct *mm, unsigned long addr, +- unsigned long end, struct vm_area_struct **pprev, +- struct rb_node ***rb_link, struct rb_node **rb_parent) ++static unsigned long count_vma_pages_range(struct mm_struct *mm, ++ unsigned long addr, unsigned long end) + { +- struct rb_node **__rb_link, *__rb_parent, *rb_prev; ++ VMA_ITERATOR(vmi, mm, addr); ++ struct vm_area_struct *vma; ++ unsigned long nr_pages = 0; + +- mmap_assert_locked(mm); +- __rb_link = &mm->mm_rb.rb_node; +- rb_prev = __rb_parent = NULL; ++ for_each_vma_range(vmi, vma, end) { ++ unsigned long vm_start = max(addr, vma->vm_start); ++ unsigned long vm_end = min(end, vma->vm_end); + +- while (*__rb_link) { +- struct vm_area_struct *vma_tmp; ++ nr_pages += PHYS_PFN(vm_end - vm_start); + } -+ -+ return -EINTR; + +- __rb_parent = *__rb_link; +- vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb); ++ return nr_pages; +} + +- if (vma_tmp->vm_end > addr) { +- /* Fail if an existing vma overlaps the area */ +- if (vma_tmp->vm_start < end) +- return -ENOMEM; +- __rb_link = &__rb_parent->rb_left; +- } else { +- rb_prev = __rb_parent; +- __rb_link = &__rb_parent->rb_right; +- } +- } ++static void __vma_link_file(struct vm_area_struct *vma, ++ struct address_space *mapping) ++{ ++ if (vma->vm_flags & VM_SHARED) ++ mapping_allow_writable(mapping); + +- *pprev = NULL; +- if (rb_prev) +- *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); +- *rb_link = __rb_link; +- *rb_parent = __rb_parent; +- return 0; ++ flush_dcache_mmap_lock(mapping); ++ vma_interval_tree_insert(vma, &mapping->i_mmap); ++ flush_dcache_mmap_unlock(mapping); + } + + /* +- * vma_next() - Get the next VMA. +- * @mm: The mm_struct. +- * @vma: The current vma. ++ * vma_mas_store() - Store a VMA in the maple tree. ++ * @vma: The vm_area_struct ++ * @mas: The maple state + * +- * If @vma is NULL, return the first vma in the mm. ++ * Efficient way to store a VMA in the maple tree when the @mas has already ++ * walked to the correct location. + * +- * Returns: The next VMA after @vma. ++ * Note: the end address is inclusive in the maple tree. + */ +-static inline struct vm_area_struct *vma_next(struct mm_struct *mm, +- struct vm_area_struct *vma) ++void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) + { +- if (!vma) +- return mm->mmap; +- +- return vma->vm_next; ++ trace_vma_store(mas->tree, vma); ++ mas_set_range(mas, vma->vm_start, vma->vm_end - 1); ++ mas_store_prealloc(mas, vma); + } + + /* +- * munmap_vma_range() - munmap VMAs that overlap a range. +- * @mm: The mm struct +- * @start: The start of the range. +- * @len: The length of the range. +- * @pprev: pointer to the pointer that will be set to previous vm_area_struct +- * @rb_link: the rb_node +- * @rb_parent: the parent rb_node +- * +- * Find all the vm_area_struct that overlap from @start to +- * @end and munmap them. Set @pprev to the previous vm_area_struct. ++ * vma_mas_remove() - Remove a VMA from the maple tree. ++ * @vma: The vm_area_struct ++ * @mas: The maple state + * +- * Returns: -ENOMEM on munmap failure or 0 on success. ++ * Efficient way to remove a VMA from the maple tree when the @mas has already ++ * been established and points to the correct location. ++ * Note: the end address is inclusive in the maple tree. + */ +-static inline int +-munmap_vma_range(struct mm_struct *mm, unsigned long start, unsigned long len, +- struct vm_area_struct **pprev, struct rb_node ***link, +- struct rb_node **parent, struct list_head *uf) ++void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) + { +- +- while (find_vma_links(mm, start, start + len, pprev, link, parent)) +- if (do_munmap(mm, start, len, uf)) +- return -ENOMEM; +- +- return 0; ++ trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); ++ mas->index = vma->vm_start; ++ mas->last = vma->vm_end - 1; ++ mas_store_prealloc(mas, NULL); + } +-static unsigned long count_vma_pages_range(struct mm_struct *mm, +- unsigned long addr, unsigned long end) + -+static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, -+ struct scan_control *sc, int swappiness, unsigned long opt) ++/* ++ * vma_mas_szero() - Set a given range to zero. Used when modifying a ++ * vm_area_struct start or end. ++ * ++ * @mm: The struct_mm ++ * @start: The start address to zero ++ * @end: The end address to zero. ++ */ ++static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, ++ unsigned long end) + { +- unsigned long nr_pages = 0; +- struct vm_area_struct *vma; ++ trace_vma_mas_szero(mas->tree, start, end - 1); ++ mas_set_range(mas, start, end - 1); ++ mas_store_prealloc(mas, NULL); ++} + +- /* Find first overlapping mapping */ +- vma = find_vma_intersection(mm, addr, end); +- if (!vma) +- return 0; ++static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) +{ -+ struct lruvec *lruvec; -+ int err = -EINVAL; -+ struct mem_cgroup *memcg = NULL; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ struct address_space *mapping = NULL; + +- nr_pages = (min(end, vma->vm_end) - +- max(addr, vma->vm_start)) >> PAGE_SHIFT; ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; + +- /* Iterate over the rest of the overlaps */ +- for (vma = vma->vm_next; vma; vma = vma->vm_next) { +- unsigned long overlap_len; ++ if (vma->vm_file) { ++ mapping = vma->vm_file->f_mapping; ++ i_mmap_lock_write(mapping); ++ } + +- if (vma->vm_start > end) +- break; ++ vma_mas_store(vma, &mas); + +- overlap_len = min(end, vma->vm_end) - vma->vm_start; +- nr_pages += overlap_len >> PAGE_SHIFT; ++ if (mapping) { ++ __vma_link_file(vma, mapping); ++ i_mmap_unlock_write(mapping); + } + +- return nr_pages; ++ mm->map_count++; ++ validate_mm(mm); ++ return 0; + } + +-void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, +- struct rb_node **rb_link, struct rb_node *rb_parent) ++/* ++ * vma_expand - Expand an existing VMA ++ * ++ * @mas: The maple state ++ * @vma: The vma to expand ++ * @start: The start of the vma ++ * @end: The exclusive end of the vma ++ * @pgoff: The page offset of vma ++ * @next: The current of next vma. ++ * ++ * Expand @vma to @start and @end. Can expand off the start and end. Will ++ * expand over @next if it's different from @vma and @end == @next->vm_end. ++ * Checking if the @vma can expand and merge with @next needs to be handled by ++ * the caller. ++ * ++ * Returns: 0 on success ++ */ ++inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long start, unsigned long end, pgoff_t pgoff, ++ struct vm_area_struct *next) + { +- /* Update tracking information for the gap following the new vma. */ +- if (vma->vm_next) +- vma_gap_update(vma->vm_next); +- else +- mm->highest_vm_end = vm_end_gap(vma); ++ struct mm_struct *mm = vma->vm_mm; ++ struct address_space *mapping = NULL; ++ struct rb_root_cached *root = NULL; ++ struct anon_vma *anon_vma = vma->anon_vma; ++ struct file *file = vma->vm_file; ++ bool remove_next = false; + +- /* +- * vma->vm_prev wasn't known when we followed the rbtree to find the +- * correct insertion point for that vma. As a result, we could not +- * update the vma vm_rb parents rb_subtree_gap values on the way down. +- * So, we first insert the vma with a zero rb_subtree_gap value +- * (to be consistent with what we did on the way down), and then +- * immediately update the gap to the correct value. Finally we +- * rebalance the rbtree after all augmented values have been set. +- */ +- rb_link_node(&vma->vm_rb, rb_parent, rb_link); +- vma->rb_subtree_gap = 0; +- vma_gap_update(vma); +- vma_rb_insert(vma, &mm->mm_rb); +-} ++ if (next && (vma != next) && (end == next->vm_end)) { ++ remove_next = true; ++ if (next->anon_vma && !vma->anon_vma) { ++ int error; + +-static void __vma_link_file(struct vm_area_struct *vma) +-{ +- struct file *file; ++ anon_vma = next->anon_vma; ++ vma->anon_vma = anon_vma; ++ error = anon_vma_clone(vma, next); ++ if (error) ++ return error; ++ } ++ } + -+ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) -+ return -EINVAL; ++ /* Not merging but overwriting any part of next is not handled. */ ++ VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start); ++ /* Only handles expanding */ ++ VM_BUG_ON(vma->vm_start < start || vma->vm_end > end); + -+ if (!mem_cgroup_disabled()) { -+ rcu_read_lock(); -+ memcg = mem_cgroup_from_id(memcg_id); -+#ifdef CONFIG_MEMCG -+ if (memcg && !css_tryget(&memcg->css)) -+ memcg = NULL; -+#endif -+ rcu_read_unlock(); ++ if (mas_preallocate(mas, vma, GFP_KERNEL)) ++ goto nomem; + -+ if (!memcg) -+ return -EINVAL; ++ vma_adjust_trans_huge(vma, start, end, 0); + +- file = vma->vm_file; + if (file) { +- struct address_space *mapping = file->f_mapping; ++ mapping = file->f_mapping; ++ root = &mapping->i_mmap; ++ uprobe_munmap(vma, vma->vm_start, vma->vm_end); ++ i_mmap_lock_write(mapping); + } + +- if (vma->vm_flags & VM_SHARED) +- mapping_allow_writable(mapping); ++ if (anon_vma) { ++ anon_vma_lock_write(anon_vma); ++ anon_vma_interval_tree_pre_update_vma(vma); ++ } + ++ if (file) { + flush_dcache_mmap_lock(mapping); +- vma_interval_tree_insert(vma, &mapping->i_mmap); +- flush_dcache_mmap_unlock(mapping); ++ vma_interval_tree_remove(vma, root); + } +-} + +-static void +-__vma_link(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev, struct rb_node **rb_link, +- struct rb_node *rb_parent) +-{ +- __vma_link_list(mm, vma, prev); +- __vma_link_rb(mm, vma, rb_link, rb_parent); +-} ++ vma->vm_start = start; ++ vma->vm_end = end; ++ vma->vm_pgoff = pgoff; ++ /* Note: mas must be pointing to the expanding VMA */ ++ vma_mas_store(vma, mas); + +-static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev, struct rb_node **rb_link, +- struct rb_node *rb_parent) +-{ +- struct address_space *mapping = NULL; ++ if (file) { ++ vma_interval_tree_insert(vma, root); ++ flush_dcache_mmap_unlock(mapping); ++ } + +- if (vma->vm_file) { +- mapping = vma->vm_file->f_mapping; +- i_mmap_lock_write(mapping); ++ /* Expanding over the next vma */ ++ if (remove_next && file) { ++ __remove_shared_vm_struct(next, file, mapping); + } + +- __vma_link(mm, vma, prev, rb_link, rb_parent); +- __vma_link_file(vma); ++ if (anon_vma) { ++ anon_vma_interval_tree_post_update_vma(vma); ++ anon_vma_unlock_write(anon_vma); ++ } + +- if (mapping) ++ if (file) { + i_mmap_unlock_write(mapping); ++ uprobe_mmap(vma); ++ } + +- mm->map_count++; +- validate_mm(mm); +-} +- +-/* +- * Helper for vma_adjust() in the split_vma insert case: insert a vma into the +- * mm's list and rbtree. It has already been inserted into the interval tree. +- */ +-static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) +-{ +- struct vm_area_struct *prev; +- struct rb_node **rb_link, *rb_parent; ++ if (remove_next) { ++ if (file) { ++ uprobe_munmap(next, next->vm_start, next->vm_end); ++ fput(file); ++ } ++ if (next->anon_vma) ++ anon_vma_merge(vma, next); ++ mm->map_count--; ++ mpol_put(vma_policy(next)); ++ vm_area_free(next); ++ } + +- if (find_vma_links(mm, vma->vm_start, vma->vm_end, +- &prev, &rb_link, &rb_parent)) +- BUG(); +- __vma_link(mm, vma, prev, rb_link, rb_parent); +- mm->map_count++; +-} ++ validate_mm(mm); ++ return 0; + +-static __always_inline void __vma_unlink(struct mm_struct *mm, +- struct vm_area_struct *vma, +- struct vm_area_struct *ignore) +-{ +- vma_rb_erase_ignore(vma, &mm->mm_rb, ignore); +- __vma_unlink_list(mm, vma); +- /* Kill the cache */ +- vmacache_invalidate(mm); ++nomem: ++ return -ENOMEM; + } + + /* +@@ -701,18 +618,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + struct vm_area_struct *expand) + { + struct mm_struct *mm = vma->vm_mm; +- struct vm_area_struct *next = vma->vm_next, *orig_vma = vma; ++ struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); ++ struct vm_area_struct *orig_vma = vma; + struct address_space *mapping = NULL; + struct rb_root_cached *root = NULL; + struct anon_vma *anon_vma = NULL; + struct file *file = vma->vm_file; +- bool start_changed = false, end_changed = false; ++ bool vma_changed = false; + long adjust_next = 0; + int remove_next = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ struct vm_area_struct *exporter = NULL, *importer = NULL; + + if (next && !insert) { +- struct vm_area_struct *exporter = NULL, *importer = NULL; +- + if (end >= next->vm_end) { + /* + * vma expands, overlapping all the next, and +@@ -741,10 +659,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + * remove_next == 1 is case 1 or 7. + */ + remove_next = 1 + (end > next->vm_end); ++ if (remove_next == 2) ++ next_next = find_vma(mm, next->vm_end); ++ + VM_WARN_ON(remove_next == 2 && +- end != next->vm_next->vm_end); +- /* trim end to next, for case 6 first pass */ +- end = next->vm_end; ++ end != next_next->vm_end); + } + + exporter = next; +@@ -755,7 +674,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + * next, if the vma overlaps with it. + */ + if (remove_next == 2 && !next->anon_vma) +- exporter = next->vm_next; ++ exporter = next_next; + + } else if (end > next->vm_start) { + /* +@@ -792,9 +711,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + return error; + } + } +-again: +- vma_adjust_trans_huge(orig_vma, start, end, adjust_next); + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; + -+ if (memcg_id != mem_cgroup_id(memcg)) -+ goto done; -+ -+ lruvec = get_lruvec(memcg, nid); -+ -+ if (swappiness < 0) -+ swappiness = get_swappiness(lruvec, sc); -+ else if (swappiness > 200) -+ goto done; ++ vma_adjust_trans_huge(orig_vma, start, end, adjust_next); + if (file) { + mapping = file->f_mapping; + root = &mapping->i_mmap; +@@ -804,14 +725,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + uprobe_munmap(next, next->vm_start, next->vm_end); + + i_mmap_lock_write(mapping); +- if (insert) { ++ if (insert && insert->vm_file) { + /* + * Put into interval tree now, so instantiated pages + * are visible to arm/parisc __flush_dcache_page + * throughout; but we cannot insert into address + * space until vma start or end is updated. + */ +- __vma_link_file(insert); ++ __vma_link_file(insert, insert->vm_file->f_mapping); + } + } + +@@ -835,17 +756,37 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + } + + if (start != vma->vm_start) { ++ if ((vma->vm_start < start) && ++ (!insert || (insert->vm_end != start))) { ++ vma_mas_szero(&mas, vma->vm_start, start); ++ VM_WARN_ON(insert && insert->vm_start > vma->vm_start); ++ } else { ++ vma_changed = true; ++ } + vma->vm_start = start; +- start_changed = true; + } + if (end != vma->vm_end) { ++ if (vma->vm_end > end) { ++ if (!insert || (insert->vm_start != end)) { ++ vma_mas_szero(&mas, end, vma->vm_end); ++ mas_reset(&mas); ++ VM_WARN_ON(insert && ++ insert->vm_end < vma->vm_end); ++ } ++ } else { ++ vma_changed = true; ++ } + vma->vm_end = end; +- end_changed = true; + } + -+ switch (cmd) { -+ case '+': -+ err = run_aging(lruvec, seq, sc, swappiness, opt); -+ break; -+ case '-': -+ err = run_eviction(lruvec, seq, sc, swappiness, opt); -+ break; -+ } -+done: -+ mem_cgroup_put(memcg); ++ if (vma_changed) ++ vma_mas_store(vma, &mas); + -+ return err; -+} + vma->vm_pgoff = pgoff; + if (adjust_next) { + next->vm_start += adjust_next; + next->vm_pgoff += adjust_next >> PAGE_SHIFT; ++ vma_mas_store(next, &mas); + } + + if (file) { +@@ -855,42 +796,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + flush_dcache_mmap_unlock(mapping); + } + +- if (remove_next) { +- /* +- * vma_merge has merged next into vma, and needs +- * us to remove next before dropping the locks. +- */ +- if (remove_next != 3) +- __vma_unlink(mm, next, next); +- else +- /* +- * vma is not before next if they've been +- * swapped. +- * +- * pre-swap() next->vm_start was reduced so +- * tell validate_mm_rb to ignore pre-swap() +- * "next" (which is stored in post-swap() +- * "vma"). +- */ +- __vma_unlink(mm, next, vma); +- if (file) +- __remove_shared_vm_struct(next, file, mapping); ++ if (remove_next && file) { ++ __remove_shared_vm_struct(next, file, mapping); ++ if (remove_next == 2) ++ __remove_shared_vm_struct(next_next, file, mapping); + } else if (insert) { + /* + * split_vma has split insert from vma, and needs + * us to insert it before dropping the locks + * (it may either follow vma or precede it). + */ +- __insert_vm_struct(mm, insert); +- } else { +- if (start_changed) +- vma_gap_update(vma); +- if (end_changed) { +- if (!next) +- mm->highest_vm_end = vm_end_gap(vma); +- else if (!adjust_next) +- vma_gap_update(next); +- } ++ mas_reset(&mas); ++ vma_mas_store(insert, &mas); ++ mm->map_count++; + } + + if (anon_vma) { +@@ -909,6 +827,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + } + + if (remove_next) { ++again: + if (file) { + uprobe_munmap(next, next->vm_start, next->vm_end); + fput(file); +@@ -917,66 +836,24 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, + anon_vma_merge(vma, next); + mm->map_count--; + mpol_put(vma_policy(next)); ++ if (remove_next != 2) ++ BUG_ON(vma->vm_end < next->vm_end); + vm_area_free(next); + -+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -+static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, -+ size_t len, loff_t *pos) + /* + * In mprotect's case 6 (see comments on vma_merge), +- * we must remove another next too. It would clutter +- * up the code too much to do both in one go. ++ * we must remove next_next too. + */ +- if (remove_next != 3) { +- /* +- * If "next" was removed and vma->vm_end was +- * expanded (up) over it, in turn +- * "next->vm_prev->vm_end" changed and the +- * "vma->vm_next" gap must be updated. +- */ +- next = vma->vm_next; +- } else { +- /* +- * For the scope of the comment "next" and +- * "vma" considered pre-swap(): if "vma" was +- * removed, next->vm_start was expanded (down) +- * over it and the "next" gap must be updated. +- * Because of the swap() the post-swap() "vma" +- * actually points to pre-swap() "next" +- * (post-swap() "next" as opposed is now a +- * dangling pointer). +- */ +- next = vma; +- } + if (remove_next == 2) { + remove_next = 1; +- end = next->vm_end; ++ next = next_next; + goto again; + } +- else if (next) +- vma_gap_update(next); +- else { +- /* +- * If remove_next == 2 we obviously can't +- * reach this path. +- * +- * If remove_next == 3 we can't reach this +- * path because pre-swap() next is always not +- * NULL. pre-swap() "next" is not being +- * removed and its next->vm_end is not altered +- * (and furthermore "end" already matches +- * next->vm_end in remove_next == 3). +- * +- * We reach this only in the remove_next == 1 +- * case if the "next" vma that was removed was +- * the highest vma of the mm. However in such +- * case next->vm_end == "end" and the extended +- * "vma" has vma->vm_end == next->vm_end so +- * mm->highest_vm_end doesn't need any update +- * in remove_next == 1 case. +- */ +- VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); +- } + } + if (insert && file) + uprobe_mmap(insert); + ++ mas_destroy(&mas); + validate_mm(mm); + + return 0; +@@ -1138,10 +1015,10 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, + if (vm_flags & VM_SPECIAL) + return NULL; + +- next = vma_next(mm, prev); ++ next = find_vma(mm, prev ? prev->vm_end : 0); + area = next; + if (area && area->vm_end == end) /* cases 6, 7, 8 */ +- next = next->vm_next; ++ next = find_vma(mm, next->vm_end); + + /* verify some invariant that must be enforced by the caller */ + VM_WARN_ON(prev && addr <= prev->vm_start); +@@ -1275,18 +1152,24 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_ + */ + struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) + { ++ MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); + struct anon_vma *anon_vma = NULL; ++ struct vm_area_struct *prev, *next; + + /* Try next first. */ +- if (vma->vm_next) { +- anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next); ++ next = mas_walk(&mas); ++ if (next) { ++ anon_vma = reusable_anon_vma(next, vma, next); + if (anon_vma) + return anon_vma; + } + ++ prev = mas_prev(&mas, 0); ++ VM_BUG_ON_VMA(prev != vma, vma); ++ prev = mas_prev(&mas, 0); + /* Try prev next. */ +- if (vma->vm_prev) +- anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma); ++ if (prev) ++ anon_vma = reusable_anon_vma(prev, prev, vma); + + /* + * We might reach here with anon_vma == NULL if we can't find +@@ -1375,6 +1258,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, + vm_flags_t vm_flags; + int pkey = 0; + ++ validate_mm(mm); + *populate = 0; + + if (!len) +@@ -1678,388 +1562,63 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) + return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; + } + +-unsigned long mmap_region(struct file *file, unsigned long addr, +- unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, +- struct list_head *uf) ++/** ++ * unmapped_area() - Find an area between the low_limit and the high_limit with ++ * the correct alignment and offset, all from @info. Note: current->mm is used ++ * for the search. ++ * ++ * @info: The unmapped area information including the range (low_limit - ++ * hight_limit), the alignment offset and mask. ++ * ++ * Return: A memory address or -ENOMEM. ++ */ ++static unsigned long unmapped_area(struct vm_unmapped_area_info *info) + { +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma, *prev, *merge; +- int error; +- struct rb_node **rb_link, *rb_parent; +- unsigned long charged = 0; +- +- /* Check against address space limit. */ +- if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { +- unsigned long nr_pages; +- +- /* +- * MAP_FIXED may remove pages of mappings that intersects with +- * requested mapping. Account for the pages it would unmap. +- */ +- nr_pages = count_vma_pages_range(mm, addr, addr + len); +- +- if (!may_expand_vm(mm, vm_flags, +- (len >> PAGE_SHIFT) - nr_pages)) +- return -ENOMEM; +- } +- +- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ +- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) +- return -ENOMEM; +- /* +- * Private writable mapping: check memory availability +- */ +- if (accountable_mapping(file, vm_flags)) { +- charged = len >> PAGE_SHIFT; +- if (security_vm_enough_memory_mm(mm, charged)) +- return -ENOMEM; +- vm_flags |= VM_ACCOUNT; +- } +- +- /* +- * Can we just expand an old mapping? +- */ +- vma = vma_merge(mm, prev, addr, addr + len, vm_flags, +- NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); +- if (vma) +- goto out; +- +- /* +- * Determine the object being mapped and call the appropriate +- * specific mapper. the address has already been validated, but +- * not unmapped, but the maps are removed from the list. +- */ +- vma = vm_area_alloc(mm); +- if (!vma) { +- error = -ENOMEM; +- goto unacct_error; +- } +- +- vma->vm_start = addr; +- vma->vm_end = addr + len; +- vma->vm_flags = vm_flags; +- vma->vm_page_prot = vm_get_page_prot(vm_flags); +- vma->vm_pgoff = pgoff; +- +- if (file) { +- if (vm_flags & VM_SHARED) { +- error = mapping_map_writable(file->f_mapping); +- if (error) +- goto free_vma; +- } +- +- vma->vm_file = get_file(file); +- error = call_mmap(file, vma); +- if (error) +- goto unmap_and_free_vma; +- +- /* Can addr have changed?? +- * +- * Answer: Yes, several device drivers can do it in their +- * f_op->mmap method. -DaveM +- * Bug: If addr is changed, prev, rb_link, rb_parent should +- * be updated for vma_link() +- */ +- WARN_ON_ONCE(addr != vma->vm_start); +- +- addr = vma->vm_start; +- +- /* If vm_flags changed after call_mmap(), we should try merge vma again +- * as we may succeed this time. +- */ +- if (unlikely(vm_flags != vma->vm_flags && prev)) { +- merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, +- NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); +- if (merge) { +- /* ->mmap() can change vma->vm_file and fput the original file. So +- * fput the vma->vm_file here or we would add an extra fput for file +- * and cause general protection fault ultimately. +- */ +- fput(vma->vm_file); +- vm_area_free(vma); +- vma = merge; +- /* Update vm_flags to pick up the change. */ +- vm_flags = vma->vm_flags; +- goto unmap_writable; +- } +- } +- +- vm_flags = vma->vm_flags; +- } else if (vm_flags & VM_SHARED) { +- error = shmem_zero_setup(vma); +- if (error) +- goto free_vma; +- } else { +- vma_set_anonymous(vma); +- } +- +- /* Allow architectures to sanity-check the vm_flags */ +- if (!arch_validate_flags(vma->vm_flags)) { +- error = -EINVAL; +- if (file) +- goto unmap_and_free_vma; +- else +- goto free_vma; +- } +- +- vma_link(mm, vma, prev, rb_link, rb_parent); +- +- /* +- * vma_merge() calls khugepaged_enter_vma() either, the below +- * call covers the non-merge case. +- */ +- khugepaged_enter_vma(vma, vma->vm_flags); +- +- /* Once vma denies write, undo our temporary denial count */ +-unmap_writable: +- if (file && vm_flags & VM_SHARED) +- mapping_unmap_writable(file->f_mapping); +- file = vma->vm_file; +-out: +- perf_event_mmap(vma); +- +- vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); +- if (vm_flags & VM_LOCKED) { +- if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || +- is_vm_hugetlb_page(vma) || +- vma == get_gate_vma(current->mm)) +- vma->vm_flags &= VM_LOCKED_CLEAR_MASK; +- else +- mm->locked_vm += (len >> PAGE_SHIFT); +- } ++ unsigned long length, gap; + +- if (file) +- uprobe_mmap(vma); +- +- /* +- * New (or expanded) vma always get soft dirty status. +- * Otherwise user-space soft-dirty page tracker won't +- * be able to distinguish situation when vma area unmapped, +- * then new mapped in-place (which must be aimed as +- * a completely new data area). +- */ +- vma->vm_flags |= VM_SOFTDIRTY; +- +- vma_set_page_prot(vma); +- +- return addr; +- +-unmap_and_free_vma: +- fput(vma->vm_file); +- vma->vm_file = NULL; +- +- /* Undo any partial mapping done by a device driver. */ +- unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); +- if (vm_flags & VM_SHARED) +- mapping_unmap_writable(file->f_mapping); +-free_vma: +- vm_area_free(vma); +-unacct_error: +- if (charged) +- vm_unacct_memory(charged); +- return error; +-} +- +-static unsigned long unmapped_area(struct vm_unmapped_area_info *info) +-{ +- /* +- * We implement the search by looking for an rbtree node that +- * immediately follows a suitable gap. That is, +- * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length; +- * - gap_end = vma->vm_start >= info->low_limit + length; +- * - gap_end - gap_start >= length +- */ +- +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma; +- unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + +- /* Adjust search limits by the desired length */ +- if (info->high_limit < length) +- return -ENOMEM; +- high_limit = info->high_limit - length; +- +- if (info->low_limit > high_limit) +- return -ENOMEM; +- low_limit = info->low_limit + length; +- +- /* Check if rbtree root looks promising */ +- if (RB_EMPTY_ROOT(&mm->mm_rb)) +- goto check_highest; +- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); +- if (vma->rb_subtree_gap < length) +- goto check_highest; +- +- while (true) { +- /* Visit left subtree if it looks promising */ +- gap_end = vm_start_gap(vma); +- if (gap_end >= low_limit && vma->vm_rb.rb_left) { +- struct vm_area_struct *left = +- rb_entry(vma->vm_rb.rb_left, +- struct vm_area_struct, vm_rb); +- if (left->rb_subtree_gap >= length) { +- vma = left; +- continue; +- } +- } +- +- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; +-check_current: +- /* Check if current node has a suitable gap */ +- if (gap_start > high_limit) +- return -ENOMEM; +- if (gap_end >= low_limit && +- gap_end > gap_start && gap_end - gap_start >= length) +- goto found; +- +- /* Visit right subtree if it looks promising */ +- if (vma->vm_rb.rb_right) { +- struct vm_area_struct *right = +- rb_entry(vma->vm_rb.rb_right, +- struct vm_area_struct, vm_rb); +- if (right->rb_subtree_gap >= length) { +- vma = right; +- continue; +- } +- } +- +- /* Go back up the rbtree to find next candidate node */ +- while (true) { +- struct rb_node *prev = &vma->vm_rb; +- if (!rb_parent(prev)) +- goto check_highest; +- vma = rb_entry(rb_parent(prev), +- struct vm_area_struct, vm_rb); +- if (prev == vma->vm_rb.rb_left) { +- gap_start = vm_end_gap(vma->vm_prev); +- gap_end = vm_start_gap(vma); +- goto check_current; +- } +- } +- } +- +-check_highest: +- /* Check highest gap, which does not precede any rbtree node */ +- gap_start = mm->highest_vm_end; +- gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */ +- if (gap_start > high_limit) ++ if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, ++ length)) + return -ENOMEM; + +-found: +- /* We found a suitable gap. Clip it with the original low_limit. */ +- if (gap_start < info->low_limit) +- gap_start = info->low_limit; +- +- /* Adjust gap address to the desired alignment */ +- gap_start += (info->align_offset - gap_start) & info->align_mask; +- +- VM_BUG_ON(gap_start + info->length > info->high_limit); +- VM_BUG_ON(gap_start + info->length > gap_end); +- return gap_start; ++ gap = mas.index; ++ gap += (info->align_offset - gap) & info->align_mask; ++ return gap; + } + ++/** ++ * unmapped_area_topdown() - Find an area between the low_limit and the ++ * high_limit with * the correct alignment and offset at the highest available ++ * address, all from @info. Note: current->mm is used for the search. ++ * ++ * @info: The unmapped area information including the range (low_limit - ++ * hight_limit), the alignment offset and mask. ++ * ++ * Return: A memory address or -ENOMEM. ++ */ + static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) + { +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma; +- unsigned long length, low_limit, high_limit, gap_start, gap_end; ++ unsigned long length, gap; + ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + +- /* +- * Adjust search limits by the desired length. +- * See implementation comment at top of unmapped_area(). +- */ +- gap_end = info->high_limit; +- if (gap_end < length) +- return -ENOMEM; +- high_limit = gap_end - length; +- +- if (info->low_limit > high_limit) ++ if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, ++ length)) + return -ENOMEM; +- low_limit = info->low_limit + length; + +- /* Check highest gap, which does not precede any rbtree node */ +- gap_start = mm->highest_vm_end; +- if (gap_start <= high_limit) +- goto found_highest; +- +- /* Check if rbtree root looks promising */ +- if (RB_EMPTY_ROOT(&mm->mm_rb)) +- return -ENOMEM; +- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); +- if (vma->rb_subtree_gap < length) +- return -ENOMEM; +- +- while (true) { +- /* Visit right subtree if it looks promising */ +- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; +- if (gap_start <= high_limit && vma->vm_rb.rb_right) { +- struct vm_area_struct *right = +- rb_entry(vma->vm_rb.rb_right, +- struct vm_area_struct, vm_rb); +- if (right->rb_subtree_gap >= length) { +- vma = right; +- continue; +- } +- } +- +-check_current: +- /* Check if current node has a suitable gap */ +- gap_end = vm_start_gap(vma); +- if (gap_end < low_limit) +- return -ENOMEM; +- if (gap_start <= high_limit && +- gap_end > gap_start && gap_end - gap_start >= length) +- goto found; +- +- /* Visit left subtree if it looks promising */ +- if (vma->vm_rb.rb_left) { +- struct vm_area_struct *left = +- rb_entry(vma->vm_rb.rb_left, +- struct vm_area_struct, vm_rb); +- if (left->rb_subtree_gap >= length) { +- vma = left; +- continue; +- } +- } +- +- /* Go back up the rbtree to find next candidate node */ +- while (true) { +- struct rb_node *prev = &vma->vm_rb; +- if (!rb_parent(prev)) +- return -ENOMEM; +- vma = rb_entry(rb_parent(prev), +- struct vm_area_struct, vm_rb); +- if (prev == vma->vm_rb.rb_right) { +- gap_start = vma->vm_prev ? +- vm_end_gap(vma->vm_prev) : 0; +- goto check_current; +- } +- } +- } +- +-found: +- /* We found a suitable gap. Clip it with the original high_limit. */ +- if (gap_end > info->high_limit) +- gap_end = info->high_limit; +- +-found_highest: +- /* Compute highest gap address at the desired alignment */ +- gap_end -= info->length; +- gap_end -= (gap_end - info->align_offset) & info->align_mask; +- +- VM_BUG_ON(gap_end < info->low_limit); +- VM_BUG_ON(gap_end < gap_start); +- return gap_end; ++ gap = mas.last + 1 - info->length; ++ gap -= (gap - info->align_offset) & info->align_mask; ++ return gap; + } + + /* +@@ -2249,58 +1808,67 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, + + EXPORT_SYMBOL(get_unmapped_area); + +-/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ +-struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) ++/** ++ * find_vma_intersection() - Look up the first VMA which intersects the interval ++ * @mm: The process address space. ++ * @start_addr: The inclusive start user address. ++ * @end_addr: The exclusive end user address. ++ * ++ * Returns: The first VMA within the provided range, %NULL otherwise. Assumes ++ * start_addr < end_addr. ++ */ ++struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, ++ unsigned long start_addr, ++ unsigned long end_addr) + { +- struct rb_node *rb_node; +- struct vm_area_struct *vma; ++ unsigned long index = start_addr; + + mmap_assert_locked(mm); +- /* Check the cache first. */ +- vma = vmacache_find(mm, addr); +- if (likely(vma)) +- return vma; +- +- rb_node = mm->mm_rb.rb_node; +- +- while (rb_node) { +- struct vm_area_struct *tmp; +- +- tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb); ++ return mt_find(&mm->mm_mt, &index, end_addr - 1); ++} ++EXPORT_SYMBOL(find_vma_intersection); + +- if (tmp->vm_end > addr) { +- vma = tmp; +- if (tmp->vm_start <= addr) +- break; +- rb_node = rb_node->rb_left; +- } else +- rb_node = rb_node->rb_right; +- } ++/** ++ * find_vma() - Find the VMA for a given address, or the next VMA. ++ * @mm: The mm_struct to check ++ * @addr: The address ++ * ++ * Returns: The VMA associated with addr, or the next VMA. ++ * May return %NULL in the case of no VMA at addr or above. ++ */ ++struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) +{ -+ void *buf; -+ char *cur, *next; -+ unsigned int flags; -+ struct blk_plug plug; -+ int err = -EINVAL; -+ struct scan_control sc = { -+ .may_writepage = true, -+ .may_unmap = true, -+ .may_swap = true, -+ .reclaim_idx = MAX_NR_ZONES - 1, -+ .gfp_mask = GFP_KERNEL, -+ }; -+ -+ buf = kvmalloc(len + 1, GFP_KERNEL); -+ if (!buf) ++ unsigned long index = addr; + +- if (vma) +- vmacache_update(addr, vma); +- return vma; ++ mmap_assert_locked(mm); ++ return mt_find(&mm->mm_mt, &index, ULONG_MAX); + } +- + EXPORT_SYMBOL(find_vma); + +-/* +- * Same as find_vma, but also return a pointer to the previous VMA in *pprev. ++/** ++ * find_vma_prev() - Find the VMA for a given address, or the next vma and ++ * set %pprev to the previous VMA, if any. ++ * @mm: The mm_struct to check ++ * @addr: The address ++ * @pprev: The pointer to set to the previous VMA ++ * ++ * Note that RCU lock is missing here since the external mmap_lock() is used ++ * instead. ++ * ++ * Returns: The VMA associated with @addr, or the next vma. ++ * May return %NULL in the case of no vma at addr or above. + */ + struct vm_area_struct * + find_vma_prev(struct mm_struct *mm, unsigned long addr, + struct vm_area_struct **pprev) + { + struct vm_area_struct *vma; ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + +- vma = find_vma(mm, addr); +- if (vma) { +- *pprev = vma->vm_prev; +- } else { +- struct rb_node *rb_node = rb_last(&mm->mm_rb); +- +- *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL; +- } ++ vma = mas_walk(&mas); ++ *pprev = mas_prev(&mas, 0); ++ if (!vma) ++ vma = mas_next(&mas, ULONG_MAX); + return vma; + } + +@@ -2354,6 +1922,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + struct vm_area_struct *next; + unsigned long gap_addr; + int error = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + if (!(vma->vm_flags & VM_GROWSUP)) + return -EFAULT; +@@ -2371,16 +1940,21 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + if (gap_addr < address || gap_addr > TASK_SIZE) + gap_addr = TASK_SIZE; + +- next = vma->vm_next; +- if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { ++ next = find_vma_intersection(mm, vma->vm_end, gap_addr); ++ if (next && vma_is_accessible(next)) { + if (!(next->vm_flags & VM_GROWSUP)) + return -ENOMEM; + /* Check that both stack segments have the same anon_vma? */ + } + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) + return -ENOMEM; + -+ if (copy_from_user(buf, src, len)) { -+ kvfree(buf); -+ return -EFAULT; + /* We must make sure the anon_vma is allocated. */ +- if (unlikely(anon_vma_prepare(vma))) ++ if (unlikely(anon_vma_prepare(vma))) { ++ mas_destroy(&mas); + return -ENOMEM; + } + + /* + * vma->vm_start/vm_end cannot change under us because the caller +@@ -2401,15 +1975,13 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + error = acct_stack_growth(vma, size, grow); + if (!error) { + /* +- * vma_gap_update() doesn't support concurrent +- * updates, but we only hold a shared mmap_lock +- * lock here, so we need to protect against +- * concurrent vma expansions. +- * anon_vma_lock_write() doesn't help here, as +- * we don't guarantee that all growable vmas +- * in a mm share the same root anon vma. +- * So, we reuse mm->page_table_lock to guard +- * against concurrent vma expansions. ++ * We only hold a shared mmap_lock lock here, so ++ * we need to protect against concurrent vma ++ * expansions. anon_vma_lock_write() doesn't ++ * help here, as we don't guarantee that all ++ * growable vmas in a mm share the same root ++ * anon vma. So, we reuse mm->page_table_lock ++ * to guard against concurrent vma expansions. + */ + spin_lock(&mm->page_table_lock); + if (vma->vm_flags & VM_LOCKED) +@@ -2417,11 +1989,9 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + vm_stat_account(mm, vma->vm_flags, grow); + anon_vma_interval_tree_pre_update_vma(vma); + vma->vm_end = address; ++ /* Overwrite old entry in mtree. */ ++ vma_mas_store(vma, &mas); + anon_vma_interval_tree_post_update_vma(vma); +- if (vma->vm_next) +- vma_gap_update(vma->vm_next); +- else +- mm->highest_vm_end = vm_end_gap(vma); + spin_unlock(&mm->page_table_lock); + + perf_event_mmap(vma); +@@ -2430,7 +2000,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + } + anon_vma_unlock_write(vma->anon_vma); + khugepaged_enter_vma(vma, vma->vm_flags); +- validate_mm(mm); ++ mas_destroy(&mas); + return error; + } + #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ +@@ -2438,10 +2008,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) + /* + * vma is the first one with address < vma->vm_start. Have to extend vma. + */ +-int expand_downwards(struct vm_area_struct *vma, +- unsigned long address) ++int expand_downwards(struct vm_area_struct *vma, unsigned long address) + { + struct mm_struct *mm = vma->vm_mm; ++ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); + struct vm_area_struct *prev; + int error = 0; + +@@ -2450,7 +2020,7 @@ int expand_downwards(struct vm_area_struct *vma, + return -EPERM; + + /* Enforce stack_guard_gap */ +- prev = vma->vm_prev; ++ prev = mas_prev(&mas, 0); + /* Check that both stack segments have the same anon_vma? */ + if (prev && !(prev->vm_flags & VM_GROWSDOWN) && + vma_is_accessible(prev)) { +@@ -2458,9 +2028,14 @@ int expand_downwards(struct vm_area_struct *vma, + return -ENOMEM; + } + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ return -ENOMEM; + -+ set_task_reclaim_state(current, &sc.reclaim_state); -+ flags = memalloc_noreclaim_save(); -+ blk_start_plug(&plug); -+ if (!set_mm_walk(NULL)) { -+ err = -ENOMEM; -+ goto done; + /* We must make sure the anon_vma is allocated. */ +- if (unlikely(anon_vma_prepare(vma))) ++ if (unlikely(anon_vma_prepare(vma))) { ++ mas_destroy(&mas); + return -ENOMEM; + } + + /* + * vma->vm_start/vm_end cannot change under us because the caller +@@ -2481,15 +2056,13 @@ int expand_downwards(struct vm_area_struct *vma, + error = acct_stack_growth(vma, size, grow); + if (!error) { + /* +- * vma_gap_update() doesn't support concurrent +- * updates, but we only hold a shared mmap_lock +- * lock here, so we need to protect against +- * concurrent vma expansions. +- * anon_vma_lock_write() doesn't help here, as +- * we don't guarantee that all growable vmas +- * in a mm share the same root anon vma. +- * So, we reuse mm->page_table_lock to guard +- * against concurrent vma expansions. ++ * We only hold a shared mmap_lock lock here, so ++ * we need to protect against concurrent vma ++ * expansions. anon_vma_lock_write() doesn't ++ * help here, as we don't guarantee that all ++ * growable vmas in a mm share the same root ++ * anon vma. So, we reuse mm->page_table_lock ++ * to guard against concurrent vma expansions. + */ + spin_lock(&mm->page_table_lock); + if (vma->vm_flags & VM_LOCKED) +@@ -2498,8 +2071,9 @@ int expand_downwards(struct vm_area_struct *vma, + anon_vma_interval_tree_pre_update_vma(vma); + vma->vm_start = address; + vma->vm_pgoff -= grow; ++ /* Overwrite old entry in mtree. */ ++ vma_mas_store(vma, &mas); + anon_vma_interval_tree_post_update_vma(vma); +- vma_gap_update(vma); + spin_unlock(&mm->page_table_lock); + + perf_event_mmap(vma); +@@ -2508,7 +2082,7 @@ int expand_downwards(struct vm_area_struct *vma, + } + anon_vma_unlock_write(vma->anon_vma); + khugepaged_enter_vma(vma, vma->vm_flags); +- validate_mm(mm); ++ mas_destroy(&mas); + return error; + } + +@@ -2581,25 +2155,26 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) + EXPORT_SYMBOL_GPL(find_extend_vma); + + /* +- * Ok - we have the memory areas we should free on the vma list, +- * so release them, and do the vma updates. ++ * Ok - we have the memory areas we should free on a maple tree so release them, ++ * and do the vma updates. + * + * Called with the mm semaphore held. + */ +-static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) ++static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) + { + unsigned long nr_accounted = 0; ++ struct vm_area_struct *vma; + + /* Update high watermark before we lower total_vm */ + update_hiwater_vm(mm); +- do { ++ mas_for_each(mas, vma, ULONG_MAX) { + long nrpages = vma_pages(vma); + + if (vma->vm_flags & VM_ACCOUNT) + nr_accounted += nrpages; + vm_stat_account(mm, vma->vm_flags, -nrpages); +- vma = remove_vma(vma); +- } while (vma); ++ remove_vma(vma); ++ } + vm_unacct_memory(nr_accounted); + validate_mm(mm); + } +@@ -2609,75 +2184,32 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) + * + * Called with the mm semaphore held. + */ +-static void unmap_region(struct mm_struct *mm, ++static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, + struct vm_area_struct *vma, struct vm_area_struct *prev, ++ struct vm_area_struct *next, + unsigned long start, unsigned long end) + { +- struct vm_area_struct *next = vma_next(mm, prev); + struct mmu_gather tlb; + + lru_add_drain(); + tlb_gather_mmu(&tlb, mm); + update_hiwater_rss(mm); +- unmap_vmas(&tlb, vma, start, end); +- free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, ++ unmap_vmas(&tlb, mt, vma, start, end); ++ free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, + next ? next->vm_start : USER_PGTABLES_CEILING); + tlb_finish_mmu(&tlb); + } + + /* +- * Create a list of vma's touched by the unmap, removing them from the mm's +- * vma list as we go.. ++ * __split_vma() bypasses sysctl_max_map_count checking. We use this where it ++ * has already been checked or doesn't make sense to fail. + */ +-static bool +-detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev, unsigned long end) ++int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, ++ unsigned long addr, int new_below) + { +- struct vm_area_struct **insertion_point; +- struct vm_area_struct *tail_vma = NULL; +- +- insertion_point = (prev ? &prev->vm_next : &mm->mmap); +- vma->vm_prev = NULL; +- do { +- vma_rb_erase(vma, &mm->mm_rb); +- if (vma->vm_flags & VM_LOCKED) +- mm->locked_vm -= vma_pages(vma); +- mm->map_count--; +- tail_vma = vma; +- vma = vma->vm_next; +- } while (vma && vma->vm_start < end); +- *insertion_point = vma; +- if (vma) { +- vma->vm_prev = prev; +- vma_gap_update(vma); +- } else +- mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; +- tail_vma->vm_next = NULL; +- +- /* Kill the cache */ +- vmacache_invalidate(mm); +- +- /* +- * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or +- * VM_GROWSUP VMA. Such VMAs can change their size under +- * down_read(mmap_lock) and collide with the VMA we are about to unmap. +- */ +- if (vma && (vma->vm_flags & VM_GROWSDOWN)) +- return false; +- if (prev && (prev->vm_flags & VM_GROWSUP)) +- return false; +- return true; +-} +- +-/* +- * __split_vma() bypasses sysctl_max_map_count checking. We use this where it +- * has already been checked or doesn't make sense to fail. +- */ +-int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, +- unsigned long addr, int new_below) +-{ +- struct vm_area_struct *new; +- int err; ++ struct vm_area_struct *new; ++ int err; ++ validate_mm_mt(mm); + + if (vma->vm_ops && vma->vm_ops->may_split) { + err = vma->vm_ops->may_split(vma, addr); +@@ -2720,6 +2252,9 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + if (!err) + return 0; + ++ /* Avoid vm accounting in close() operation */ ++ new->vm_start = new->vm_end; ++ new->vm_pgoff = 0; + /* Clean everything up if vma_adjust failed. */ + if (new->vm_ops && new->vm_ops->close) + new->vm_ops->close(new); +@@ -2730,6 +2265,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + mpol_put(vma_policy(new)); + out_free_vma: + vm_area_free(new); ++ validate_mm_mt(mm); + return err; + } + +@@ -2746,38 +2282,48 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + return __split_vma(mm, vma, addr, new_below); + } + +-/* Munmap is split into 2 main parts -- this part which finds +- * what needs doing, and the areas themselves, which do the +- * work. This now handles partial unmappings. +- * Jeremy Fitzhardinge +- */ +-int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, +- struct list_head *uf, bool downgrade) ++static inline int munmap_sidetree(struct vm_area_struct *vma, ++ struct ma_state *mas_detach) + { +- unsigned long end; +- struct vm_area_struct *vma, *prev, *last; +- +- if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) +- return -EINVAL; ++ mas_set_range(mas_detach, vma->vm_start, vma->vm_end - 1); ++ if (mas_store_gfp(mas_detach, vma, GFP_KERNEL)) ++ return -ENOMEM; + +- len = PAGE_ALIGN(len); +- end = start + len; +- if (len == 0) +- return -EINVAL; ++ if (vma->vm_flags & VM_LOCKED) ++ vma->vm_mm->locked_vm -= vma_pages(vma); + +- /* +- * arch_unmap() might do unmaps itself. It must be called +- * and finish any rbtree manipulation before this code +- * runs and also starts to manipulate the rbtree. +- */ +- arch_unmap(mm, start, end); ++ return 0; ++} + +- /* Find the first overlapping VMA where start < vma->vm_end */ +- vma = find_vma_intersection(mm, start, end); +- if (!vma) +- return 0; +- prev = vma->vm_prev; ++/* ++ * do_mas_align_munmap() - munmap the aligned region from @start to @end. ++ * @mas: The maple_state, ideally set up to alter the correct tree location. ++ * @vma: The starting vm_area_struct ++ * @mm: The mm_struct ++ * @start: The aligned start address to munmap. ++ * @end: The aligned end address to munmap. ++ * @uf: The userfaultfd list_head ++ * @downgrade: Set to true to attempt a write downgrade of the mmap_sem ++ * ++ * If @downgrade is true, check return code for potential release of the lock. ++ */ ++static int ++do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, ++ struct mm_struct *mm, unsigned long start, ++ unsigned long end, struct list_head *uf, bool downgrade) ++{ ++ struct vm_area_struct *prev, *next = NULL; ++ struct maple_tree mt_detach; ++ int count = 0; ++ int error = -ENOMEM; ++ MA_STATE(mas_detach, &mt_detach, 0, 0); ++ mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN); ++ mt_set_external_lock(&mt_detach, &mm->mmap_lock); ++ ++ if (mas_preallocate(mas, vma, GFP_KERNEL)) ++ return -ENOMEM; + ++ mas->last = end - 1; + /* + * If we need to split any vma, do it now to save pain later. + * +@@ -2785,8 +2331,9 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + * unmapped vm_area_struct will remain in use: so lower split_vma + * places tmp vma above, and higher split_vma places tmp vma below. + */ + -+ next = buf; -+ next[len] = '\0'; -+ -+ while ((cur = strsep(&next, ",;\n"))) { -+ int n; -+ int end; -+ char cmd; -+ unsigned int memcg_id; -+ unsigned int nid; -+ unsigned long seq; -+ unsigned int swappiness = -1; -+ unsigned long opt = -1; -+ -+ cur = skip_spaces(cur); -+ if (!*cur) -+ continue; ++ /* Does it split the first one? */ + if (start > vma->vm_start) { +- int error; + + /* + * Make sure that map_count on return from munmap() will +@@ -2794,22 +2341,61 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + * its limit temporarily, to help free resources as expected. + */ + if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) +- return -ENOMEM; ++ goto map_count_exceeded; + ++ /* ++ * mas_pause() is not needed since mas->index needs to be set ++ * differently than vma->vm_end anyways. ++ */ + error = __split_vma(mm, vma, start, 0); + if (error) +- return error; +- prev = vma; ++ goto start_split_failed; ++ ++ mas_set(mas, start); ++ vma = mas_walk(mas); + } + +- /* Does it split the last one? */ +- last = find_vma(mm, end); +- if (last && end > last->vm_start) { +- int error = __split_vma(mm, last, end, 1); ++ prev = mas_prev(mas, 0); ++ if (unlikely((!prev))) ++ mas_set(mas, start); + -+ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, -+ &seq, &end, &swappiness, &end, &opt, &end); -+ if (n < 4 || cur[end]) { -+ err = -EINVAL; ++ /* ++ * Detach a range of VMAs from the mm. Using next as a temp variable as ++ * it is always overwritten. ++ */ ++ mas_for_each(mas, next, end - 1) { ++ /* Does it split the end? */ ++ if (next->vm_end > end) { ++ struct vm_area_struct *split; ++ ++ error = __split_vma(mm, next, end, 1); ++ if (error) ++ goto end_split_failed; ++ ++ mas_set(mas, end); ++ split = mas_prev(mas, 0); ++ error = munmap_sidetree(split, &mas_detach); ++ if (error) ++ goto munmap_sidetree_failed; ++ ++ count++; ++ if (vma == next) ++ vma = split; + break; + } ++ error = munmap_sidetree(next, &mas_detach); + if (error) +- return error; ++ goto munmap_sidetree_failed; ++ ++ count++; ++#ifdef CONFIG_DEBUG_VM_MAPLE_TREE ++ BUG_ON(next->vm_start < start); ++ BUG_ON(next->vm_start > end); ++#endif + } +- vma = vma_next(mm, prev); + -+ err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); -+ if (err) -+ break; -+ } -+done: -+ clear_mm_walk(); -+ blk_finish_plug(&plug); -+ memalloc_noreclaim_restore(flags); -+ set_task_reclaim_state(current, NULL); ++ if (!next) ++ next = mas_next(mas, ULONG_MAX); + + if (unlikely(uf)) { + /* +@@ -2821,30 +2407,366 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + * split, despite we could. This is unlikely enough + * failure that it's not worth optimizing it for. + */ +- int error = userfaultfd_unmap_prep(vma, start, end, uf); ++ error = userfaultfd_unmap_prep(mm, start, end, uf); + -+ kvfree(buf); + if (error) +- return error; ++ goto userfaultfd_error; ++ } + -+ return err ? : len; -+} ++ /* Point of no return */ ++ mas_set_range(mas, start, end - 1); ++#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) ++ /* Make sure no VMAs are about to be lost. */ ++ { ++ MA_STATE(test, &mt_detach, start, end - 1); ++ struct vm_area_struct *vma_mas, *vma_test; ++ int test_count = 0; + -+static int lru_gen_seq_open(struct inode *inode, struct file *file) ++ rcu_read_lock(); ++ vma_test = mas_find(&test, end - 1); ++ mas_for_each(mas, vma_mas, end - 1) { ++ BUG_ON(vma_mas != vma_test); ++ test_count++; ++ vma_test = mas_next(&test, end - 1); ++ } ++ rcu_read_unlock(); ++ BUG_ON(count != test_count); ++ mas_set_range(mas, start, end - 1); ++ } ++#endif ++ mas_store_prealloc(mas, NULL); ++ mm->map_count -= count; ++ /* ++ * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or ++ * VM_GROWSUP VMA. Such VMAs can change their size under ++ * down_read(mmap_lock) and collide with the VMA we are about to unmap. ++ */ ++ if (downgrade) { ++ if (next && (next->vm_flags & VM_GROWSDOWN)) ++ downgrade = false; ++ else if (prev && (prev->vm_flags & VM_GROWSUP)) ++ downgrade = false; ++ else ++ mmap_write_downgrade(mm); + } + +- /* Detach vmas from rbtree */ +- if (!detach_vmas_to_be_unmapped(mm, vma, prev, end)) +- downgrade = false; ++ unmap_region(mm, &mt_detach, vma, prev, next, start, end); ++ /* Statistics and freeing VMAs */ ++ mas_set(&mas_detach, start); ++ remove_mt(mm, &mas_detach); ++ __mt_destroy(&mt_detach); + +- if (downgrade) +- mmap_write_downgrade(mm); + +- unmap_region(mm, vma, prev, start, end); ++ validate_mm(mm); ++ return downgrade ? 1 : 0; + +- /* Fix up all other VM information */ +- remove_vma_list(mm, vma); ++userfaultfd_error: ++munmap_sidetree_failed: ++end_split_failed: ++ __mt_destroy(&mt_detach); ++start_split_failed: ++map_count_exceeded: ++ mas_destroy(mas); ++ return error; ++} + +- return downgrade ? 1 : 0; ++/* ++ * do_mas_munmap() - munmap a given range. ++ * @mas: The maple state ++ * @mm: The mm_struct ++ * @start: The start address to munmap ++ * @len: The length of the range to munmap ++ * @uf: The userfaultfd list_head ++ * @downgrade: set to true if the user wants to attempt to write_downgrade the ++ * mmap_sem ++ * ++ * This function takes a @mas that is either pointing to the previous VMA or set ++ * to MA_START and sets it up to remove the mapping(s). The @len will be ++ * aligned and any arch_unmap work will be preformed. ++ * ++ * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise. ++ */ ++int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, ++ unsigned long start, size_t len, struct list_head *uf, ++ bool downgrade) +{ -+ return seq_open(file, &lru_gen_seq_ops); -+} ++ unsigned long end; ++ struct vm_area_struct *vma; + -+static const struct file_operations lru_gen_rw_fops = { -+ .open = lru_gen_seq_open, -+ .read = seq_read, -+ .write = lru_gen_seq_write, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; ++ if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) ++ return -EINVAL; + -+static const struct file_operations lru_gen_ro_fops = { -+ .open = lru_gen_seq_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; ++ end = start + PAGE_ALIGN(len); ++ if (end == start) ++ return -EINVAL; + -+/****************************************************************************** -+ * initialization -+ ******************************************************************************/ ++ /* arch_unmap() might do unmaps itself. */ ++ arch_unmap(mm, start, end); + -+void lru_gen_init_lruvec(struct lruvec *lruvec) ++ /* Find the first overlapping VMA */ ++ vma = mas_find(mas, end - 1); ++ if (!vma) ++ return 0; ++ ++ return do_mas_align_munmap(mas, vma, mm, start, end, uf, downgrade); + } + ++/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. ++ * @mm: The mm_struct ++ * @start: The start address to munmap ++ * @len: The length to be munmapped. ++ * @uf: The userfaultfd list_head ++ */ + int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, + struct list_head *uf) + { +- return __do_munmap(mm, start, len, uf, false); ++ MA_STATE(mas, &mm->mm_mt, start, start); ++ ++ return do_mas_munmap(&mas, mm, start, len, uf, false); ++} ++ ++unsigned long mmap_region(struct file *file, unsigned long addr, ++ unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, ++ struct list_head *uf) +{ -+ int i; -+ int gen, type, zone; -+ struct lru_gen_struct *lrugen = &lruvec->lrugen; ++ struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma = NULL; ++ struct vm_area_struct *next, *prev, *merge; ++ pgoff_t pglen = len >> PAGE_SHIFT; ++ unsigned long charged = 0; ++ unsigned long end = addr + len; ++ unsigned long merge_start = addr, merge_end = end; ++ pgoff_t vm_pgoff; ++ int error; ++ MA_STATE(mas, &mm->mm_mt, addr, end - 1); + -+ lrugen->max_seq = MIN_NR_GENS + 1; -+ lrugen->enabled = lru_gen_enabled(); ++ /* Check against address space limit. */ ++ if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { ++ unsigned long nr_pages; + -+ for (i = 0; i <= MIN_NR_GENS + 1; i++) -+ lrugen->timestamps[i] = jiffies; ++ /* ++ * MAP_FIXED may remove pages of mappings that intersects with ++ * requested mapping. Account for the pages it would unmap. ++ */ ++ nr_pages = count_vma_pages_range(mm, addr, end); + -+ for_each_gen_type_zone(gen, type, zone) -+ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); ++ if (!may_expand_vm(mm, vm_flags, ++ (len >> PAGE_SHIFT) - nr_pages)) ++ return -ENOMEM; ++ } + -+ lruvec->mm_state.seq = MIN_NR_GENS; -+ init_waitqueue_head(&lruvec->mm_state.wait); -+} ++ /* Unmap any existing mapping in the area */ ++ if (do_mas_munmap(&mas, mm, addr, len, uf, false)) ++ return -ENOMEM; + -+#ifdef CONFIG_MEMCG -+void lru_gen_init_memcg(struct mem_cgroup *memcg) -+{ -+ INIT_LIST_HEAD(&memcg->mm_list.fifo); -+ spin_lock_init(&memcg->mm_list.lock); -+} ++ /* ++ * Private writable mapping: check memory availability ++ */ ++ if (accountable_mapping(file, vm_flags)) { ++ charged = len >> PAGE_SHIFT; ++ if (security_vm_enough_memory_mm(mm, charged)) ++ return -ENOMEM; ++ vm_flags |= VM_ACCOUNT; ++ } + -+void lru_gen_exit_memcg(struct mem_cgroup *memcg) -+{ -+ int i; -+ int nid; ++ next = mas_next(&mas, ULONG_MAX); ++ prev = mas_prev(&mas, 0); ++ if (vm_flags & VM_SPECIAL) ++ goto cannot_expand; + -+ for_each_node(nid) { -+ struct lruvec *lruvec = get_lruvec(memcg, nid); ++ /* Attempt to expand an old mapping */ ++ /* Check next */ ++ if (next && next->vm_start == end && !vma_policy(next) && ++ can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, ++ NULL_VM_UFFD_CTX, NULL)) { ++ merge_end = next->vm_end; ++ vma = next; ++ vm_pgoff = next->vm_pgoff - pglen; ++ } + -+ VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, -+ sizeof(lruvec->lrugen.nr_pages))); ++ /* Check prev */ ++ if (prev && prev->vm_end == addr && !vma_policy(prev) && ++ (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, ++ pgoff, vma->vm_userfaultfd_ctx, NULL) : ++ can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, ++ NULL_VM_UFFD_CTX, NULL))) { ++ merge_start = prev->vm_start; ++ vma = prev; ++ vm_pgoff = prev->vm_pgoff; ++ } + -+ for (i = 0; i < NR_BLOOM_FILTERS; i++) { -+ bitmap_free(lruvec->mm_state.filters[i]); -+ lruvec->mm_state.filters[i] = NULL; -+ } ++ ++ /* Actually expand, if possible */ ++ if (vma && ++ !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) { ++ khugepaged_enter_vma(vma, vm_flags); ++ goto expanded; + } -+} -+#endif + -+static int __init init_lru_gen(void) -+{ -+ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); -+ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); ++ mas.index = addr; ++ mas.last = end - 1; ++cannot_expand: ++ /* ++ * Determine the object being mapped and call the appropriate ++ * specific mapper. the address has already been validated, but ++ * not unmapped, but the maps are removed from the list. ++ */ ++ vma = vm_area_alloc(mm); ++ if (!vma) { ++ error = -ENOMEM; ++ goto unacct_error; ++ } ++ ++ vma->vm_start = addr; ++ vma->vm_end = end; ++ vma->vm_flags = vm_flags; ++ vma->vm_page_prot = vm_get_page_prot(vm_flags); ++ vma->vm_pgoff = pgoff; ++ ++ if (file) { ++ if (vm_flags & VM_SHARED) { ++ error = mapping_map_writable(file->f_mapping); ++ if (error) ++ goto free_vma; ++ } + -+ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) -+ pr_err("lru_gen: failed to create sysfs group\n"); ++ vma->vm_file = get_file(file); ++ error = call_mmap(file, vma); ++ if (error) ++ goto unmap_and_free_vma; + -+ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); -+ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); ++ /* Can addr have changed?? ++ * ++ * Answer: Yes, several device drivers can do it in their ++ * f_op->mmap method. -DaveM ++ */ ++ WARN_ON_ONCE(addr != vma->vm_start); + -+ return 0; -+}; -+late_initcall(init_lru_gen); ++ addr = vma->vm_start; ++ mas_reset(&mas); + -+#else /* !CONFIG_LRU_GEN */ ++ /* ++ * If vm_flags changed after call_mmap(), we should try merge ++ * vma again as we may succeed this time. ++ */ ++ if (unlikely(vm_flags != vma->vm_flags && prev)) { ++ merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, ++ NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); ++ if (merge) { ++ /* ++ * ->mmap() can change vma->vm_file and fput ++ * the original file. So fput the vma->vm_file ++ * here or we would add an extra fput for file ++ * and cause general protection fault ++ * ultimately. ++ */ ++ fput(vma->vm_file); ++ vm_area_free(vma); ++ vma = merge; ++ /* Update vm_flags to pick up the change. */ ++ addr = vma->vm_start; ++ vm_flags = vma->vm_flags; ++ goto unmap_writable; ++ } ++ } + -+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -+{ -+} ++ vm_flags = vma->vm_flags; ++ } else if (vm_flags & VM_SHARED) { ++ error = shmem_zero_setup(vma); ++ if (error) ++ goto free_vma; ++ } else { ++ vma_set_anonymous(vma); ++ } + -+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+} ++ /* Allow architectures to sanity-check the vm_flags */ ++ if (!arch_validate_flags(vma->vm_flags)) { ++ error = -EINVAL; ++ if (file) ++ goto unmap_and_free_vma; ++ else ++ goto free_vma; ++ } + -+#endif /* CONFIG_LRU_GEN */ ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ error = -ENOMEM; ++ if (file) ++ goto unmap_and_free_vma; ++ else ++ goto free_vma; ++ } + -+static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -+{ -+ unsigned long nr[NR_LRU_LISTS]; -+ unsigned long targets[NR_LRU_LISTS]; -+ unsigned long nr_to_scan; -+ enum lru_list lru; -+ unsigned long nr_reclaimed = 0; -+ unsigned long nr_to_reclaim = sc->nr_to_reclaim; -+ bool proportional_reclaim; -+ struct blk_plug plug; ++ if (vma->vm_file) ++ i_mmap_lock_write(vma->vm_file->f_mapping); + -+ if (lru_gen_enabled()) { -+ lru_gen_shrink_lruvec(lruvec, sc); -+ return; ++ vma_mas_store(vma, &mas); ++ mm->map_count++; ++ if (vma->vm_file) { ++ if (vma->vm_flags & VM_SHARED) ++ mapping_allow_writable(vma->vm_file->f_mapping); ++ ++ flush_dcache_mmap_lock(vma->vm_file->f_mapping); ++ vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap); ++ flush_dcache_mmap_unlock(vma->vm_file->f_mapping); ++ i_mmap_unlock_write(vma->vm_file->f_mapping); + } + -+ get_scan_count(lruvec, sc, nr); ++ /* ++ * vma_merge() calls khugepaged_enter_vma() either, the below ++ * call covers the non-merge case. ++ */ ++ khugepaged_enter_vma(vma, vma->vm_flags); ++ ++ /* Once vma denies write, undo our temporary denial count */ ++unmap_writable: ++ if (file && vm_flags & VM_SHARED) ++ mapping_unmap_writable(file->f_mapping); ++ file = vma->vm_file; ++expanded: ++ perf_event_mmap(vma); ++ ++ vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); ++ if (vm_flags & VM_LOCKED) { ++ if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || ++ is_vm_hugetlb_page(vma) || ++ vma == get_gate_vma(current->mm)) ++ vma->vm_flags &= VM_LOCKED_CLEAR_MASK; ++ else ++ mm->locked_vm += (len >> PAGE_SHIFT); ++ } + -+ /* Record the original scan target for proportional adjustments later */ -+ memcpy(targets, nr, sizeof(nr)); ++ if (file) ++ uprobe_mmap(vma); + + /* -+ * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal -+ * event that can occur when there is little memory pressure e.g. -+ * multiple streaming readers/writers. Hence, we do not abort scanning -+ * when the requested number of pages are reclaimed when scanning at -+ * DEF_PRIORITY on the assumption that the fact we are direct -+ * reclaiming implies that kswapd is not keeping up and it is best to -+ * do a batch of work at once. For memcg reclaim one check is made to -+ * abort proportional reclaim if either the file or anon lru has already -+ * dropped to zero at the first pass. ++ * New (or expanded) vma always get soft dirty status. ++ * Otherwise user-space soft-dirty page tracker won't ++ * be able to distinguish situation when vma area unmapped, ++ * then new mapped in-place (which must be aimed as ++ * a completely new data area). + */ -+ proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && -+ sc->priority == DEF_PRIORITY); ++ vma->vm_flags |= VM_SOFTDIRTY; ++ ++ vma_set_page_prot(vma); ++ ++ validate_mm(mm); ++ return addr; ++ ++unmap_and_free_vma: ++ fput(vma->vm_file); ++ vma->vm_file = NULL; ++ ++ /* Undo any partial mapping done by a device driver. */ ++ unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end); ++ if (vm_flags & VM_SHARED) ++ mapping_unmap_writable(file->f_mapping); ++free_vma: ++ vm_area_free(vma); ++unacct_error: ++ if (charged) ++ vm_unacct_memory(charged); ++ validate_mm(mm); ++ return error; + } + + static int __vm_munmap(unsigned long start, size_t len, bool downgrade) +@@ -2852,11 +2774,12 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) + int ret; + struct mm_struct *mm = current->mm; + LIST_HEAD(uf); ++ MA_STATE(mas, &mm->mm_mt, start, start); + + if (mmap_write_lock_killable(mm)) + return -EINTR; + +- ret = __do_munmap(mm, start, len, &uf, downgrade); ++ ret = do_mas_munmap(&mas, mm, start, len, &uf, downgrade); + /* + * Returning 1 indicates mmap_lock is downgraded. + * But 1 is not legal return value of vm_munmap() and munmap(), reset +@@ -2922,11 +2845,12 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, + goto out; + + if (start + size > vma->vm_end) { +- struct vm_area_struct *next; ++ VMA_ITERATOR(vmi, mm, vma->vm_end); ++ struct vm_area_struct *next, *prev = vma; + +- for (next = vma->vm_next; next; next = next->vm_next) { ++ for_each_vma_range(vmi, next, start + size) { + /* hole between vmas ? */ +- if (next->vm_start != next->vm_prev->vm_end) ++ if (next->vm_start != prev->vm_end) + goto out; + + if (next->vm_file != vma->vm_file) +@@ -2935,8 +2859,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, + if (next->vm_flags != vma->vm_flags) + goto out; + +- if (start + size <= next->vm_end) +- break; ++ prev = next; + } + + if (!next) +@@ -2966,37 +2889,53 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, + } + + /* +- * this is really a simplified "do_mmap". it only handles +- * anonymous maps. eventually we may be able to do some +- * brk-specific accounting here. ++ * brk_munmap() - Unmap a parital vma. ++ * @mas: The maple tree state. ++ * @vma: The vma to be modified ++ * @newbrk: the start of the address to unmap ++ * @oldbrk: The end of the address to unmap ++ * @uf: The userfaultfd list_head ++ * ++ * Returns: 1 on success. ++ * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if ++ * possible. + */ +-static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf) ++static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long newbrk, unsigned long oldbrk, ++ struct list_head *uf) + { +- struct mm_struct *mm = current->mm; +- struct vm_area_struct *vma, *prev; +- struct rb_node **rb_link, *rb_parent; +- pgoff_t pgoff = addr >> PAGE_SHIFT; +- int error; +- unsigned long mapped_addr; +- +- /* Until we need other flags, refuse anything except VM_EXEC. */ +- if ((flags & (~VM_EXEC)) != 0) +- return -EINVAL; +- flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; +- +- mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); +- if (IS_ERR_VALUE(mapped_addr)) +- return mapped_addr; ++ struct mm_struct *mm = vma->vm_mm; ++ int ret; + +- error = mlock_future_check(mm, mm->def_flags, len); +- if (error) +- return error; ++ arch_unmap(mm, newbrk, oldbrk); ++ ret = do_mas_align_munmap(mas, vma, mm, newbrk, oldbrk, uf, true); ++ validate_mm_mt(mm); ++ return ret; ++} + +- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ +- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) +- return -ENOMEM; ++/* ++ * do_brk_flags() - Increase the brk vma if the flags match. ++ * @mas: The maple tree state. ++ * @addr: The start address ++ * @len: The length of the increase ++ * @vma: The vma, ++ * @flags: The VMA Flags ++ * ++ * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags ++ * do not match then create a new anonymous VMA. Eventually we may be able to ++ * do some brk-specific accounting here. ++ */ ++static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma, ++ unsigned long addr, unsigned long len, unsigned long flags) ++{ ++ struct mm_struct *mm = current->mm; + +- /* Check against address space limits *after* clearing old maps... */ ++ validate_mm_mt(mm); ++ /* ++ * Check against address space limits by the changed size ++ * Note: This happens *after* clearing old mappings in some code paths. ++ */ ++ flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; + if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) + return -ENOMEM; + +@@ -3006,28 +2945,49 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla + if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + return -ENOMEM; + +- /* Can we just expand an old private anonymous mapping? */ +- vma = vma_merge(mm, prev, addr, addr + len, flags, +- NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); +- if (vma) +- goto out; +- + /* +- * create a vma struct for an anonymous mapping ++ * Expand the existing vma if possible; Note that singular lists do not ++ * occur after forking, so the expand will only happen on new VMAs. + */ +- vma = vm_area_alloc(mm); +- if (!vma) { +- vm_unacct_memory(len >> PAGE_SHIFT); +- return -ENOMEM; ++ if (vma && ++ (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) && ++ ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) { ++ mas->index = vma->vm_start; ++ mas->last = addr + len - 1; ++ vma_adjust_trans_huge(vma, addr, addr + len, 0); ++ if (vma->anon_vma) { ++ anon_vma_lock_write(vma->anon_vma); ++ anon_vma_interval_tree_pre_update_vma(vma); ++ } ++ vma->vm_end = addr + len; ++ vma->vm_flags |= VM_SOFTDIRTY; ++ if (mas_store_gfp(mas, vma, GFP_KERNEL)) ++ goto mas_expand_failed; ++ ++ if (vma->anon_vma) { ++ anon_vma_interval_tree_post_update_vma(vma); ++ anon_vma_unlock_write(vma->anon_vma); ++ } ++ khugepaged_enter_vma(vma, flags); ++ goto out; + } + ++ /* create a vma struct for an anonymous mapping */ ++ vma = vm_area_alloc(mm); ++ if (!vma) ++ goto vma_alloc_fail; ++ + vma_set_anonymous(vma); + vma->vm_start = addr; + vma->vm_end = addr + len; +- vma->vm_pgoff = pgoff; ++ vma->vm_pgoff = addr >> PAGE_SHIFT; + vma->vm_flags = flags; + vma->vm_page_prot = vm_get_page_prot(flags); +- vma_link(mm, vma, prev, rb_link, rb_parent); ++ mas_set_range(mas, vma->vm_start, addr + len - 1); ++ if (mas_store_gfp(mas, vma, GFP_KERNEL)) ++ goto mas_store_fail; ++ ++ mm->map_count++; + out: + perf_event_mmap(vma); + mm->total_vm += len >> PAGE_SHIFT; +@@ -3035,16 +2995,32 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla + if (flags & VM_LOCKED) + mm->locked_vm += (len >> PAGE_SHIFT); + vma->vm_flags |= VM_SOFTDIRTY; ++ validate_mm(mm); + return 0; + -+ blk_start_plug(&plug); -+ while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || -+ nr[LRU_INACTIVE_FILE]) { -+ unsigned long nr_anon, nr_file, percentage; -+ unsigned long nr_scanned; ++mas_store_fail: ++ vm_area_free(vma); ++vma_alloc_fail: ++ vm_unacct_memory(len >> PAGE_SHIFT); ++ return -ENOMEM; + -+ for_each_evictable_lru(lru) { -+ if (nr[lru]) { -+ nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); -+ nr[lru] -= nr_to_scan; ++mas_expand_failed: ++ if (vma->anon_vma) { ++ anon_vma_interval_tree_post_update_vma(vma); ++ anon_vma_unlock_write(vma->anon_vma); ++ } ++ return -ENOMEM; + } + + int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) + { + struct mm_struct *mm = current->mm; ++ struct vm_area_struct *vma = NULL; + unsigned long len; + int ret; + bool populate; + LIST_HEAD(uf); ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + + len = PAGE_ALIGN(request); + if (len < request) +@@ -3055,13 +3031,36 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) + if (mmap_write_lock_killable(mm)) + return -EINTR; + +- ret = do_brk_flags(addr, len, flags, &uf); ++ /* Until we need other flags, refuse anything except VM_EXEC. */ ++ if ((flags & (~VM_EXEC)) != 0) ++ return -EINVAL; + -+ nr_reclaimed += shrink_list(lru, nr_to_scan, -+ lruvec, sc); -+ } -+ } ++ ret = check_brk_limits(addr, len); ++ if (ret) ++ goto limits_failed; + -+ cond_resched(); ++ ret = do_mas_munmap(&mas, mm, addr, len, &uf, 0); ++ if (ret) ++ goto munmap_failed; + -+ if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) -+ continue; ++ vma = mas_prev(&mas, 0); ++ if (!vma || vma->vm_end != addr || vma_policy(vma) || ++ !can_vma_merge_after(vma, flags, NULL, NULL, ++ addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) ++ vma = NULL; + -+ /* -+ * For kswapd and memcg, reclaim at least the number of pages -+ * requested. Ensure that the anon and file LRUs are scanned -+ * proportionally what was requested by get_scan_count(). We -+ * stop reclaiming one LRU and reduce the amount scanning -+ * proportional to the original scan target. -+ */ -+ nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; -+ nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; ++ ret = do_brk_flags(&mas, vma, addr, len, flags); + populate = ((mm->def_flags & VM_LOCKED) != 0); + mmap_write_unlock(mm); + userfaultfd_unmap_complete(mm, &uf); + if (populate && !ret) + mm_populate(addr, len); + return ret; + -+ /* -+ * It's just vindictive to attack the larger once the smaller -+ * has gone to zero. And given the way we stop scanning the -+ * smaller below, this makes sure that we only make one nudge -+ * towards proportionality once we've got nr_to_reclaim. -+ */ -+ if (!nr_file || !nr_anon) -+ break; ++munmap_failed: ++limits_failed: ++ mmap_write_unlock(mm); ++ return ret; + } + EXPORT_SYMBOL(vm_brk_flags); + +@@ -3077,6 +3076,8 @@ void exit_mmap(struct mm_struct *mm) + struct mmu_gather tlb; + struct vm_area_struct *vma; + unsigned long nr_accounted = 0; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); ++ int count = 0; + + /* mm's last user has gone, and its about to be pulled down */ + mmu_notifier_release(mm); +@@ -3101,7 +3102,7 @@ void exit_mmap(struct mm_struct *mm) + mmap_write_lock(mm); + arch_exit_mmap(mm); + +- vma = mm->mmap; ++ vma = mas_find(&mas, ULONG_MAX); + if (!vma) { + /* Can happen if dup_mmap() received an OOM */ + mmap_write_unlock(mm); +@@ -3112,19 +3113,29 @@ void exit_mmap(struct mm_struct *mm) + flush_cache_mm(mm); + tlb_gather_mmu_fullmm(&tlb, mm); + /* update_hiwater_rss(mm) here? but nobody should be looking */ +- /* Use -1 here to ensure all VMAs in the mm are unmapped */ +- unmap_vmas(&tlb, vma, 0, -1); +- free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); ++ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ ++ unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX); ++ free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS, ++ USER_PGTABLES_CEILING); + tlb_finish_mmu(&tlb); + +- /* Walk the list again, actually closing and freeing it. */ +- while (vma) { ++ /* ++ * Walk the list again, actually closing and freeing it, with preemption ++ * enabled, without holding any MM locks besides the unreachable ++ * mmap_write_lock. ++ */ ++ do { + if (vma->vm_flags & VM_ACCOUNT) + nr_accounted += vma_pages(vma); +- vma = remove_vma(vma); ++ remove_vma(vma); ++ count++; + cond_resched(); +- } +- mm->mmap = NULL; ++ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); + -+ if (nr_file > nr_anon) { -+ unsigned long scan_target = targets[LRU_INACTIVE_ANON] + -+ targets[LRU_ACTIVE_ANON] + 1; -+ lru = LRU_BASE; -+ percentage = nr_anon * 100 / scan_target; -+ } else { -+ unsigned long scan_target = targets[LRU_INACTIVE_FILE] + -+ targets[LRU_ACTIVE_FILE] + 1; -+ lru = LRU_FILE; -+ percentage = nr_file * 100 / scan_target; -+ } ++ BUG_ON(count != mm->map_count); + -+ /* Stop scanning the smaller of the LRU */ -+ nr[lru] = 0; -+ nr[lru + LRU_ACTIVE] = 0; ++ trace_exit_mmap(mm); ++ __mt_destroy(&mm->mm_mt); + mmap_write_unlock(mm); + vm_unacct_memory(nr_accounted); + } +@@ -3135,14 +3146,14 @@ void exit_mmap(struct mm_struct *mm) + */ + int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) + { +- struct vm_area_struct *prev; +- struct rb_node **rb_link, *rb_parent; ++ unsigned long charged = vma_pages(vma); + -+ /* -+ * Recalculate the other LRU scan count based on its original -+ * scan target and the percentage scanning already complete -+ */ -+ lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; -+ nr_scanned = targets[lru] - nr[lru]; -+ nr[lru] = targets[lru] * (100 - percentage) / 100; -+ nr[lru] -= min(nr[lru], nr_scanned); + +- if (find_vma_links(mm, vma->vm_start, vma->vm_end, +- &prev, &rb_link, &rb_parent)) ++ if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) + return -ENOMEM; + -+ lru += LRU_ACTIVE; -+ nr_scanned = targets[lru] - nr[lru]; -+ nr[lru] = targets[lru] * (100 - percentage) / 100; -+ nr[lru] -= min(nr[lru], nr_scanned); + if ((vma->vm_flags & VM_ACCOUNT) && +- security_vm_enough_memory_mm(mm, vma_pages(vma))) ++ security_vm_enough_memory_mm(mm, charged)) + return -ENOMEM; + + /* +@@ -3162,7 +3173,11 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) + vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + } + +- vma_link(mm, vma, prev, rb_link, rb_parent); ++ if (vma_link(mm, vma)) { ++ vm_unacct_memory(charged); ++ return -ENOMEM; + } -+ blk_finish_plug(&plug); -+ sc->nr_reclaimed += nr_reclaimed; + -+ /* -+ * Even if we did not try to evict anon pages at all, we want to -+ * rebalance the anon lru active/inactive ratio. -+ */ -+ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && -+ inactive_is_low(lruvec, LRU_INACTIVE_ANON)) -+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec, -+ sc, LRU_ACTIVE_ANON); + return 0; + } + +@@ -3178,9 +3193,9 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + unsigned long vma_start = vma->vm_start; + struct mm_struct *mm = vma->vm_mm; + struct vm_area_struct *new_vma, *prev; +- struct rb_node **rb_link, *rb_parent; + bool faulted_in_anon_vma = true; + ++ validate_mm_mt(mm); + /* + * If anonymous vma has not yet been faulted, update new pgoff + * to match new location, to increase its chance of merging. +@@ -3190,8 +3205,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + faulted_in_anon_vma = false; + } + +- if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) ++ new_vma = find_vma_prev(mm, addr, &prev); ++ if (new_vma && new_vma->vm_start < addr + len) + return NULL; /* should never get here */ ++ + new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, + vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), + vma->vm_userfaultfd_ctx, anon_vma_name(vma)); +@@ -3232,16 +3249,22 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + get_file(new_vma->vm_file); + if (new_vma->vm_ops && new_vma->vm_ops->open) + new_vma->vm_ops->open(new_vma); +- vma_link(mm, new_vma, prev, rb_link, rb_parent); ++ if (vma_link(mm, new_vma)) ++ goto out_vma_link; + *need_rmap_locks = false; + } ++ validate_mm_mt(mm); + return new_vma; + ++out_vma_link: ++ if (new_vma->vm_ops && new_vma->vm_ops->close) ++ new_vma->vm_ops->close(new_vma); + out_free_mempol: + mpol_put(vma_policy(new_vma)); + out_free_vma: + vm_area_free(new_vma); + out: ++ validate_mm_mt(mm); + return NULL; + } + +@@ -3378,6 +3401,7 @@ static struct vm_area_struct *__install_special_mapping( + int ret; + struct vm_area_struct *vma; + ++ validate_mm_mt(mm); + vma = vm_area_alloc(mm); + if (unlikely(vma == NULL)) + return ERR_PTR(-ENOMEM); +@@ -3400,10 +3424,12 @@ static struct vm_area_struct *__install_special_mapping( + + perf_event_mmap(vma); + ++ validate_mm_mt(mm); + return vma; + + out: + vm_area_free(vma); ++ validate_mm_mt(mm); + return ERR_PTR(ret); + } + +@@ -3528,12 +3554,13 @@ int mm_take_all_locks(struct mm_struct *mm) + { + struct vm_area_struct *vma; + struct anon_vma_chain *avc; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + mmap_assert_write_locked(mm); + + mutex_lock(&mm_all_locks_mutex); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (signal_pending(current)) + goto out_unlock; + if (vma->vm_file && vma->vm_file->f_mapping && +@@ -3541,7 +3568,8 @@ int mm_take_all_locks(struct mm_struct *mm) + vm_lock_mapping(mm, vma->vm_file->f_mapping); + } + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_set(&mas, 0); ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (signal_pending(current)) + goto out_unlock; + if (vma->vm_file && vma->vm_file->f_mapping && +@@ -3549,7 +3577,8 @@ int mm_take_all_locks(struct mm_struct *mm) + vm_lock_mapping(mm, vma->vm_file->f_mapping); + } + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_set(&mas, 0); ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (signal_pending(current)) + goto out_unlock; + if (vma->anon_vma) +@@ -3608,11 +3637,12 @@ void mm_drop_all_locks(struct mm_struct *mm) + { + struct vm_area_struct *vma; + struct anon_vma_chain *avc; ++ MA_STATE(mas, &mm->mm_mt, 0, 0); + + mmap_assert_write_locked(mm); + BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); + +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ mas_for_each(&mas, vma, ULONG_MAX) { + if (vma->anon_vma) + list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) + vm_unlock_anon_vma(avc->anon_vma); +diff --git a/mm/mprotect.c b/mm/mprotect.c +index eb8982bde7ee..2bb8613cd295 100644 +--- a/mm/mprotect.c ++++ b/mm/mprotect.c +@@ -670,6 +670,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + const bool rier = (current->personality & READ_IMPLIES_EXEC) && + (prot & PROT_READ); + struct mmu_gather tlb; ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + start = untagged_addr(start); + +@@ -701,7 +702,8 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) + goto out; + +- vma = find_vma(current->mm, start); ++ mas_set(&mas, start); ++ vma = mas_find(&mas, ULONG_MAX); + error = -ENOMEM; + if (!vma) + goto out; +@@ -727,7 +729,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + if (start > vma->vm_start) + prev = vma; + else +- prev = vma->vm_prev; ++ prev = mas_prev(&mas, 0); + + tlb_gather_mmu(&tlb, current->mm); + for (nstart = start ; ; ) { +@@ -790,7 +792,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, + if (nstart >= end) + break; + +- vma = prev->vm_next; ++ vma = find_vma(current->mm, prev->vm_end); + if (!vma || vma->vm_start != nstart) { + error = -ENOMEM; + break; +diff --git a/mm/mremap.c b/mm/mremap.c +index b522cd0259a0..8644ff278f02 100644 +--- a/mm/mremap.c ++++ b/mm/mremap.c +@@ -716,7 +716,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, + if (excess) { + vma->vm_flags |= VM_ACCOUNT; + if (split) +- vma->vm_next->vm_flags |= VM_ACCOUNT; ++ find_vma(mm, vma->vm_end)->vm_flags |= VM_ACCOUNT; + } + + return new_addr; +@@ -866,9 +866,10 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, + static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) + { + unsigned long end = vma->vm_end + delta; ++ + if (end < vma->vm_end) /* overflow */ + return 0; +- if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ ++ if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) + return 0; + if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, + 0, MAP_FIXED) & ~PAGE_MASK) +@@ -975,20 +976,23 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, + /* + * Always allow a shrinking remap: that just unmaps + * the unnecessary pages.. +- * __do_munmap does all the needed commit accounting, and ++ * do_mas_munmap does all the needed commit accounting, and + * downgrades mmap_lock to read if so directed. + */ + if (old_len >= new_len) { + int retval; ++ MA_STATE(mas, &mm->mm_mt, addr + new_len, addr + new_len); + +- retval = __do_munmap(mm, addr+new_len, old_len - new_len, +- &uf_unmap, true); +- if (retval < 0 && old_len != new_len) { +- ret = retval; +- goto out; ++ retval = do_mas_munmap(&mas, mm, addr + new_len, ++ old_len - new_len, &uf_unmap, true); + /* Returning 1 indicates mmap_lock is downgraded to read. */ +- } else if (retval == 1) ++ if (retval == 1) { + downgraded = true; ++ } else if (retval < 0 && old_len != new_len) { ++ ret = retval; ++ goto out; ++ } ++ + ret = addr; + goto out; + } +diff --git a/mm/msync.c b/mm/msync.c +index 137d1c104f3e..ac4c9bfea2e7 100644 +--- a/mm/msync.c ++++ b/mm/msync.c +@@ -104,7 +104,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) + error = 0; + goto out_unlock; + } +- vma = vma->vm_next; ++ vma = find_vma(mm, vma->vm_end); + } + } + out_unlock: +diff --git a/mm/nommu.c b/mm/nommu.c +index e819cbc21b39..214c70e1d059 100644 +--- a/mm/nommu.c ++++ b/mm/nommu.c +@@ -19,7 +19,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -545,26 +544,27 @@ static void put_nommu_region(struct vm_region *region) + __put_nommu_region(region); + } + +-/* +- * add a VMA into a process's mm_struct in the appropriate place in the list +- * and tree and add to the address space's page tree also if not an anonymous +- * page +- * - should be called with mm->mmap_lock held writelocked +- */ +-static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) ++void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) + { +- struct vm_area_struct *pvma, *prev; +- struct address_space *mapping; +- struct rb_node **p, *parent, *rb_prev; ++ mas_set_range(mas, vma->vm_start, vma->vm_end - 1); ++ mas_store_prealloc(mas, vma); ++} + +- BUG_ON(!vma->vm_region); ++void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) ++{ ++ mas->index = vma->vm_start; ++ mas->last = vma->vm_end - 1; ++ mas_store_prealloc(mas, NULL); +} + ++static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) ++{ + mm->map_count++; + vma->vm_mm = mm; + + /* add the VMA to the mapping */ + if (vma->vm_file) { +- mapping = vma->vm_file->f_mapping; ++ struct address_space *mapping = vma->vm_file->f_mapping; + + i_mmap_lock_write(mapping); + flush_dcache_mmap_lock(mapping); +@@ -572,67 +572,51 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) + flush_dcache_mmap_unlock(mapping); + i_mmap_unlock_write(mapping); + } ++} + +- /* add the VMA to the tree */ +- parent = rb_prev = NULL; +- p = &mm->mm_rb.rb_node; +- while (*p) { +- parent = *p; +- pvma = rb_entry(parent, struct vm_area_struct, vm_rb); +- +- /* sort by: start addr, end addr, VMA struct addr in that order +- * (the latter is necessary as we may get identical VMAs) */ +- if (vma->vm_start < pvma->vm_start) +- p = &(*p)->rb_left; +- else if (vma->vm_start > pvma->vm_start) { +- rb_prev = parent; +- p = &(*p)->rb_right; +- } else if (vma->vm_end < pvma->vm_end) +- p = &(*p)->rb_left; +- else if (vma->vm_end > pvma->vm_end) { +- rb_prev = parent; +- p = &(*p)->rb_right; +- } else if (vma < pvma) +- p = &(*p)->rb_left; +- else if (vma > pvma) { +- rb_prev = parent; +- p = &(*p)->rb_right; +- } else +- BUG(); +- } +- +- rb_link_node(&vma->vm_rb, parent, p); +- rb_insert_color(&vma->vm_rb, &mm->mm_rb); ++/* ++ * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm(). ++ * @mas: The maple state with preallocations. ++ * @mm: The mm_struct ++ * @vma: The vma to add ++ * ++ */ ++static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm, ++ struct vm_area_struct *vma) ++{ ++ BUG_ON(!vma->vm_region); + +- /* add VMA to the VMA list also */ +- prev = NULL; +- if (rb_prev) +- prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); ++ setup_vma_to_mm(vma, mm); + +- __vma_link_list(mm, vma, prev); ++ /* add the VMA to the tree */ ++ vma_mas_store(vma, mas); + } + + /* +- * delete a VMA from its owning mm_struct and address space ++ * add a VMA into a process's mm_struct in the appropriate place in the list ++ * and tree and add to the address space's page tree also if not an anonymous ++ * page ++ * - should be called with mm->mmap_lock held writelocked + */ +-static void delete_vma_from_mm(struct vm_area_struct *vma) +-{ +- int i; +- struct address_space *mapping; +- struct mm_struct *mm = vma->vm_mm; +- struct task_struct *curr = current; +- +- mm->map_count--; +- for (i = 0; i < VMACACHE_SIZE; i++) { +- /* if the vma is cached, invalidate the entire cache */ +- if (curr->vmacache.vmas[i] == vma) { +- vmacache_invalidate(mm); +- break; +- } ++static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) ++{ ++ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); + -+/* Use reclaim/compaction for costly allocs or under memory pressure */ -+static bool in_reclaim_compaction(struct scan_control *sc) ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ pr_warn("Allocation of vma tree for process %d failed\n", ++ current->pid); ++ return -ENOMEM; + } ++ mas_add_vma_to_mm(&mas, mm, vma); ++ return 0; ++} + ++static void cleanup_vma_from_mm(struct vm_area_struct *vma) ++{ ++ vma->vm_mm->map_count--; + /* remove the VMA from the mapping */ + if (vma->vm_file) { ++ struct address_space *mapping; + mapping = vma->vm_file->f_mapping; + + i_mmap_lock_write(mapping); +@@ -641,11 +625,24 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) + flush_dcache_mmap_unlock(mapping); + i_mmap_unlock_write(mapping); + } ++} ++/* ++ * delete a VMA from its owning mm_struct and address space ++ */ ++static int delete_vma_from_mm(struct vm_area_struct *vma) ++{ ++ MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0); + +- /* remove from the MM's tree and list */ +- rb_erase(&vma->vm_rb, &mm->mm_rb); ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ pr_warn("Allocation of vma tree for process %d failed\n", ++ current->pid); ++ return -ENOMEM; ++ } ++ cleanup_vma_from_mm(vma); + +- __vma_unlink_list(mm, vma); ++ /* remove from the MM's tree and list */ ++ vma_mas_remove(vma, &mas); ++ return 0; + } + + /* +@@ -661,31 +658,26 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) + vm_area_free(vma); + } + ++struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, ++ unsigned long start_addr, ++ unsigned long end_addr) +{ -+ if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && -+ (sc->order > PAGE_ALLOC_COSTLY_ORDER || -+ sc->priority < DEF_PRIORITY - 2)) -+ return true; ++ unsigned long index = start_addr; + -+ return false; ++ mmap_assert_locked(mm); ++ return mt_find(&mm->mm_mt, &index, end_addr - 1); +} ++EXPORT_SYMBOL(find_vma_intersection); + -+/* -+ * Reclaim/compaction is used for high-order allocation requests. It reclaims -+ * order-0 pages before compacting the zone. should_continue_reclaim() returns -+ * true if more pages should be reclaimed such that when the page allocator -+ * calls try_to_compact_pages() that it will have enough free pages to succeed. -+ * It will give up earlier than that if there is difficulty reclaiming pages. -+ */ -+static inline bool should_continue_reclaim(struct pglist_data *pgdat, -+ unsigned long nr_reclaimed, -+ struct scan_control *sc) -+{ -+ unsigned long pages_for_compaction; -+ unsigned long inactive_lru_pages; -+ int z; + /* + * look up the first VMA in which addr resides, NULL if none + * - should be called with mm->mmap_lock at least held readlocked + */ + struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) + { +- struct vm_area_struct *vma; ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + +- /* check the cache first */ +- vma = vmacache_find(mm, addr); +- if (likely(vma)) +- return vma; +- +- /* trawl the list (there may be multiple mappings in which addr +- * resides) */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) { +- if (vma->vm_start > addr) +- return NULL; +- if (vma->vm_end > addr) { +- vmacache_update(addr, vma); +- return vma; +- } +- } +- +- return NULL; ++ return mas_walk(&mas); + } + EXPORT_SYMBOL(find_vma); + +@@ -717,26 +709,17 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, + { + struct vm_area_struct *vma; + unsigned long end = addr + len; ++ MA_STATE(mas, &mm->mm_mt, addr, addr); + +- /* check the cache first */ +- vma = vmacache_find_exact(mm, addr, end); +- if (vma) +- return vma; +- +- /* trawl the list (there may be multiple mappings in which addr +- * resides) */ +- for (vma = mm->mmap; vma; vma = vma->vm_next) { +- if (vma->vm_start < addr) +- continue; +- if (vma->vm_start > addr) +- return NULL; +- if (vma->vm_end == end) { +- vmacache_update(addr, vma); +- return vma; +- } +- } ++ vma = mas_walk(&mas); ++ if (!vma) ++ return NULL; ++ if (vma->vm_start != addr) ++ return NULL; ++ if (vma->vm_end != end) ++ return NULL; + +- return NULL; ++ return vma; + } + + /* +@@ -1069,6 +1052,7 @@ unsigned long do_mmap(struct file *file, + vm_flags_t vm_flags; + unsigned long capabilities, result; + int ret; ++ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); + + *populate = 0; + +@@ -1087,6 +1071,7 @@ unsigned long do_mmap(struct file *file, + * now know into VMA flags */ + vm_flags = determine_vm_flags(file, prot, flags, capabilities); + ++ + /* we're going to need to record the mapping */ + region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); + if (!region) +@@ -1096,6 +1081,9 @@ unsigned long do_mmap(struct file *file, + if (!vma) + goto error_getting_vma; + ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) ++ goto error_maple_preallocate; ++ + region->vm_usage = 1; + region->vm_flags = vm_flags; + region->vm_pgoff = pgoff; +@@ -1236,7 +1224,7 @@ unsigned long do_mmap(struct file *file, + current->mm->total_vm += len >> PAGE_SHIFT; + + share: +- add_vma_to_mm(current->mm, vma); ++ mas_add_vma_to_mm(&mas, current->mm, vma); + + /* we flush the region from the icache only when the first executable + * mapping of it is made */ +@@ -1262,6 +1250,7 @@ unsigned long do_mmap(struct file *file, + + sharing_violation: + up_write(&nommu_region_sem); ++ mas_destroy(&mas); + pr_warn("Attempt to share mismatched mappings\n"); + ret = -EINVAL; + goto error; +@@ -1278,6 +1267,14 @@ unsigned long do_mmap(struct file *file, + len, current->pid); + show_free_areas(0, NULL); + return -ENOMEM; ++ ++error_maple_preallocate: ++ kmem_cache_free(vm_region_jar, region); ++ vm_area_free(vma); ++ pr_warn("Allocation of vma tree for process %d failed\n", current->pid); ++ show_free_areas(0, NULL); ++ return -ENOMEM; + -+ /* If not in reclaim/compaction mode, stop */ -+ if (!in_reclaim_compaction(sc)) -+ return false; + } + + unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +@@ -1343,6 +1340,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *new; + struct vm_region *region; + unsigned long npages; ++ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); + + /* we're only permitted to split anonymous regions (these should have + * only a single usage on the region) */ +@@ -1357,9 +1355,13 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + return -ENOMEM; + + new = vm_area_dup(vma); +- if (!new) { +- kmem_cache_free(vm_region_jar, region); +- return -ENOMEM; ++ if (!new) ++ goto err_vma_dup; ++ ++ if (mas_preallocate(&mas, vma, GFP_KERNEL)) { ++ pr_warn("Allocation of vma tree for process %d failed\n", ++ current->pid); ++ goto err_mas_preallocate; + } + + /* most fields are the same, copy all, and then fixup */ +@@ -1378,7 +1380,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + if (new->vm_ops && new->vm_ops->open) + new->vm_ops->open(new); + +- delete_vma_from_mm(vma); + down_write(&nommu_region_sem); + delete_nommu_region(vma->vm_region); + if (new_below) { +@@ -1391,9 +1392,19 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + add_nommu_region(vma->vm_region); + add_nommu_region(new->vm_region); + up_write(&nommu_region_sem); +- add_vma_to_mm(mm, vma); +- add_vma_to_mm(mm, new); ++ ++ setup_vma_to_mm(vma, mm); ++ setup_vma_to_mm(new, mm); ++ mas_set_range(&mas, vma->vm_start, vma->vm_end - 1); ++ mas_store(&mas, vma); ++ vma_mas_store(new, &mas); + return 0; + ++err_mas_preallocate: ++ vm_area_free(new); ++err_vma_dup: ++ kmem_cache_free(vm_region_jar, region); ++ return -ENOMEM; + } + + /* +@@ -1408,12 +1419,14 @@ static int shrink_vma(struct mm_struct *mm, + + /* adjust the VMA's pointers, which may reposition it in the MM's tree + * and list */ +- delete_vma_from_mm(vma); ++ if (delete_vma_from_mm(vma)) ++ return -ENOMEM; + if (from > vma->vm_start) + vma->vm_end = from; + else + vma->vm_start = to; +- add_vma_to_mm(mm, vma); ++ if (add_vma_to_mm(mm, vma)) ++ return -ENOMEM; + + /* cut the backing region down to size */ + region = vma->vm_region; +@@ -1441,9 +1454,10 @@ static int shrink_vma(struct mm_struct *mm, + */ + int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) + { ++ MA_STATE(mas, &mm->mm_mt, start, start); + struct vm_area_struct *vma; + unsigned long end; +- int ret; ++ int ret = 0; + + len = PAGE_ALIGN(len); + if (len == 0) +@@ -1452,7 +1466,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list + end = start + len; + + /* find the first potentially overlapping VMA */ +- vma = find_vma(mm, start); ++ vma = mas_find(&mas, end - 1); + if (!vma) { + static int limit; + if (limit < 5) { +@@ -1471,7 +1485,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list + return -EINVAL; + if (end == vma->vm_end) + goto erase_whole_vma; +- vma = vma->vm_next; ++ vma = mas_next(&mas, end - 1); + } while (vma); + return -EINVAL; + } else { +@@ -1493,9 +1507,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list + } + + erase_whole_vma: +- delete_vma_from_mm(vma); ++ if (delete_vma_from_mm(vma)) ++ ret = -ENOMEM; + delete_vma(mm, vma); +- return 0; ++ return ret; + } + + int vm_munmap(unsigned long addr, size_t len) +@@ -1520,6 +1535,7 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) + */ + void exit_mmap(struct mm_struct *mm) + { ++ VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + + if (!mm) +@@ -1527,12 +1543,18 @@ void exit_mmap(struct mm_struct *mm) + + mm->total_vm = 0; + +- while ((vma = mm->mmap)) { +- mm->mmap = vma->vm_next; +- delete_vma_from_mm(vma); + /* -+ * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX -+ * number of pages that were scanned. This will return to the caller -+ * with the risk reclaim/compaction and the resulting allocation attempt -+ * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL -+ * allocations through requiring that the full LRU list has been scanned -+ * first, by assuming that zero delta of sc->nr_scanned means full LRU -+ * scan, but that approximation was wrong, and there were corner cases - * where always a non-zero amount of pages were scanned. - */ - if (!nr_reclaimed) -@@ -3201,109 +6077,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) - unsigned long nr_reclaimed, nr_scanned; - struct lruvec *target_lruvec; - bool reclaimable = false; -- unsigned long file; ++ * Lock the mm to avoid assert complaining even though this is the only ++ * user of the mm ++ */ ++ mmap_write_lock(mm); ++ for_each_vma(vmi, vma) { ++ cleanup_vma_from_mm(vma); + delete_vma(mm, vma); + cond_resched(); + } ++ __mt_destroy(&mm->mm_mt); ++ mmap_write_unlock(mm); + } - target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); + int vm_brk(unsigned long addr, unsigned long len) +diff --git a/mm/oom_kill.c b/mm/oom_kill.c +index 3c6cf9e3cd66..3996301450e8 100644 +--- a/mm/oom_kill.c ++++ b/mm/oom_kill.c +@@ -513,6 +513,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) + { + struct vm_area_struct *vma; + bool ret = true; ++ VMA_ITERATOR(vmi, mm, 0); - again: -- /* -- * Flush the memory cgroup stats, so that we read accurate per-memcg -- * lruvec stats for heuristics. -- */ -- mem_cgroup_flush_stats(); -- - memset(&sc->nr, 0, sizeof(sc->nr)); + /* + * Tell all users of get_user/copy_from_user etc... that the content +@@ -522,7 +523,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) + */ + set_bit(MMF_UNSTABLE, &mm->flags); - nr_reclaimed = sc->nr_reclaimed; - nr_scanned = sc->nr_scanned; +- for (vma = mm->mmap ; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) + continue; -- /* -- * Determine the scan balance between anon and file LRUs. -- */ -- spin_lock_irq(&target_lruvec->lru_lock); -- sc->anon_cost = target_lruvec->anon_cost; -- sc->file_cost = target_lruvec->file_cost; -- spin_unlock_irq(&target_lruvec->lru_lock); +diff --git a/mm/pagewalk.c b/mm/pagewalk.c +index fa7a3d21a751..c09bb9c01f7e 100644 +--- a/mm/pagewalk.c ++++ b/mm/pagewalk.c +@@ -460,7 +460,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, + } else { /* inside vma */ + walk.vma = vma; + next = min(end, vma->vm_end); +- vma = vma->vm_next; ++ vma = find_vma(mm, vma->vm_end); + + err = walk_page_test(start, next, &walk); + if (err > 0) { +diff --git a/mm/swapfile.c b/mm/swapfile.c +index 1fdccd2f1422..5c8681a3f1d9 100644 +--- a/mm/swapfile.c ++++ b/mm/swapfile.c +@@ -1990,14 +1990,16 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type) + { + struct vm_area_struct *vma; + int ret = 0; ++ VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); +- for (vma = mm->mmap; vma; vma = vma->vm_next) { ++ for_each_vma(vmi, vma) { + if (vma->anon_vma) { + ret = unuse_vma(vma, type); + if (ret) + break; + } ++ + cond_resched(); + } + mmap_read_unlock(mm); +diff --git a/mm/util.c b/mm/util.c +index c9439c66d8cf..1266a33a49ea 100644 +--- a/mm/util.c ++++ b/mm/util.c +@@ -272,38 +272,6 @@ void *memdup_user_nul(const void __user *src, size_t len) + } + EXPORT_SYMBOL(memdup_user_nul); + +-void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, +- struct vm_area_struct *prev) +-{ +- struct vm_area_struct *next; - -- /* -- * Target desirable inactive:active list ratios for the anon -- * and file LRU lists. -- */ -- if (!sc->force_deactivate) { -- unsigned long refaults; +- vma->vm_prev = prev; +- if (prev) { +- next = prev->vm_next; +- prev->vm_next = vma; +- } else { +- next = mm->mmap; +- mm->mmap = vma; +- } +- vma->vm_next = next; +- if (next) +- next->vm_prev = vma; +-} - -- refaults = lruvec_page_state(target_lruvec, -- WORKINGSET_ACTIVATE_ANON); -- if (refaults != target_lruvec->refaults[0] || -- inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) -- sc->may_deactivate |= DEACTIVATE_ANON; -- else -- sc->may_deactivate &= ~DEACTIVATE_ANON; +-void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma) +-{ +- struct vm_area_struct *prev, *next; +- +- next = vma->vm_next; +- prev = vma->vm_prev; +- if (prev) +- prev->vm_next = next; +- else +- mm->mmap = next; +- if (next) +- next->vm_prev = prev; +-} +- + /* Check if the vma is being used as a stack by this task */ + int vma_is_stack_for_current(struct vm_area_struct *vma) + { +diff --git a/mm/vmacache.c b/mm/vmacache.c +deleted file mode 100644 +index 01a6e6688ec1..000000000000 +--- a/mm/vmacache.c ++++ /dev/null +@@ -1,117 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0 +-/* +- * Copyright (C) 2014 Davidlohr Bueso. +- */ +-#include +-#include +-#include +-#include +- +-/* +- * Hash based on the pmd of addr if configured with MMU, which provides a good +- * hit rate for workloads with spatial locality. Otherwise, use pages. +- */ +-#ifdef CONFIG_MMU +-#define VMACACHE_SHIFT PMD_SHIFT +-#else +-#define VMACACHE_SHIFT PAGE_SHIFT +-#endif +-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK) +- +-/* +- * This task may be accessing a foreign mm via (for example) +- * get_user_pages()->find_vma(). The vmacache is task-local and this +- * task's vmacache pertains to a different mm (ie, its own). There is +- * nothing we can do here. +- * +- * Also handle the case where a kernel thread has adopted this mm via +- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm. +- */ +-static inline bool vmacache_valid_mm(struct mm_struct *mm) +-{ +- return current->mm == mm && !(current->flags & PF_KTHREAD); +-} - +-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma) +-{ +- if (vmacache_valid_mm(newvma->vm_mm)) +- current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma; +-} +- +-static bool vmacache_valid(struct mm_struct *mm) +-{ +- struct task_struct *curr; +- +- if (!vmacache_valid_mm(mm)) +- return false; +- +- curr = current; +- if (mm->vmacache_seqnum != curr->vmacache.seqnum) { - /* -- * When refaults are being observed, it means a new -- * workingset is being established. Deactivate to get -- * rid of any stale active pages quickly. +- * First attempt will always be invalid, initialize +- * the new cache for this task here. - */ -- refaults = lruvec_page_state(target_lruvec, -- WORKINGSET_ACTIVATE_FILE); -- if (refaults != target_lruvec->refaults[1] || -- inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) -- sc->may_deactivate |= DEACTIVATE_FILE; -- else -- sc->may_deactivate &= ~DEACTIVATE_FILE; -- } else -- sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; +- curr->vmacache.seqnum = mm->vmacache_seqnum; +- vmacache_flush(curr); +- return false; +- } +- return true; +-} - -- /* -- * If we have plenty of inactive file pages that aren't -- * thrashing, try to reclaim those first before touching -- * anonymous pages. -- */ -- file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); -- if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) -- sc->cache_trim_mode = 1; -- else -- sc->cache_trim_mode = 0; +-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr) +-{ +- int idx = VMACACHE_HASH(addr); +- int i; - -- /* -- * Prevent the reclaimer from falling into the cache trap: as -- * cache pages start out inactive, every cache fault will tip -- * the scan balance towards the file LRU. And as the file LRU -- * shrinks, so does the window for rotation from references. -- * This means we have a runaway feedback loop where a tiny -- * thrashing file LRU becomes infinitely more attractive than -- * anon pages. Try to detect this based on file LRU size. -- */ -- if (!cgroup_reclaim(sc)) { -- unsigned long total_high_wmark = 0; -- unsigned long free, anon; -- int z; +- count_vm_vmacache_event(VMACACHE_FIND_CALLS); - -- free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); -- file = node_page_state(pgdat, NR_ACTIVE_FILE) + -- node_page_state(pgdat, NR_INACTIVE_FILE); +- if (!vmacache_valid(mm)) +- return NULL; - -- for (z = 0; z < MAX_NR_ZONES; z++) { -- struct zone *zone = &pgdat->node_zones[z]; -- if (!managed_zone(zone)) -- continue; +- for (i = 0; i < VMACACHE_SIZE; i++) { +- struct vm_area_struct *vma = current->vmacache.vmas[idx]; - -- total_high_wmark += high_wmark_pages(zone); +- if (vma) { +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- if (WARN_ON_ONCE(vma->vm_mm != mm)) +- break; +-#endif +- if (vma->vm_start <= addr && vma->vm_end > addr) { +- count_vm_vmacache_event(VMACACHE_FIND_HITS); +- return vma; +- } - } +- if (++idx == VMACACHE_SIZE) +- idx = 0; +- } - -- /* -- * Consider anon: if that's low too, this isn't a -- * runaway file reclaim problem, but rather just -- * extreme pressure. Reclaim as per usual then. -- */ -- anon = node_page_state(pgdat, NR_INACTIVE_ANON); +- return NULL; +-} - -- sc->file_is_tiny = -- file + free <= total_high_wmark && -- !(sc->may_deactivate & DEACTIVATE_ANON) && -- anon >> sc->priority; +-#ifndef CONFIG_MMU +-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, +- unsigned long start, +- unsigned long end) +-{ +- int idx = VMACACHE_HASH(start); +- int i; +- +- count_vm_vmacache_event(VMACACHE_FIND_CALLS); +- +- if (!vmacache_valid(mm)) +- return NULL; +- +- for (i = 0; i < VMACACHE_SIZE; i++) { +- struct vm_area_struct *vma = current->vmacache.vmas[idx]; +- +- if (vma && vma->vm_start == start && vma->vm_end == end) { +- count_vm_vmacache_event(VMACACHE_FIND_HITS); +- return vma; +- } +- if (++idx == VMACACHE_SIZE) +- idx = 0; - } -+ prepare_scan_count(pgdat, sc); +- +- return NULL; +-} +-#endif +diff --git a/mm/vmscan.c b/mm/vmscan.c +index feb8416d8edd..f85a9c915d75 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -3778,23 +3778,17 @@ static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk + { + unsigned long start = round_up(*vm_end, size); + unsigned long end = (start | ~mask) + 1; ++ VMA_ITERATOR(vmi, args->mm, start); - shrink_node_memcgs(pgdat, sc); + VM_WARN_ON_ONCE(mask & size); + VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); -@@ -3561,11 +6344,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) - struct lruvec *target_lruvec; - unsigned long refaults; +- while (args->vma) { +- if (start >= args->vma->vm_end) { +- args->vma = args->vma->vm_next; +- continue; +- } +- ++ for_each_vma(vmi, args->vma) { + if (end && end <= args->vma->vm_start) + return false; -+ if (lru_gen_enabled()) -+ return; -+ - target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); -- target_lruvec->refaults[0] = refaults; -+ target_lruvec->refaults[WORKINGSET_ANON] = refaults; - refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); -- target_lruvec->refaults[1] = refaults; -+ target_lruvec->refaults[WORKINGSET_FILE] = refaults; - } +- if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) { +- args->vma = args->vma->vm_next; ++ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) + continue; +- } - /* -@@ -3927,12 +6713,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - } - #endif + *vm_start = max(start, args->vma->vm_start); + *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 0b8098e8296b..33091a67627e 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -1388,10 +1388,6 @@ const char * const vmstat_text[] = { + "nr_tlb_local_flush_one", + #endif /* CONFIG_DEBUG_TLBFLUSH */ --static void age_active_anon(struct pglist_data *pgdat, -- struct scan_control *sc) -+static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) +-#ifdef CONFIG_DEBUG_VM_VMACACHE +- "vmacache_find_calls", +- "vmacache_find_hits", +-#endif + #ifdef CONFIG_SWAP + "swap_ra", + "swap_ra_hit", +diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h +index 0616409513eb..311759ea25e9 100644 +--- a/tools/include/linux/slab.h ++++ b/tools/include/linux/slab.h +@@ -41,4 +41,8 @@ struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, + unsigned int align, unsigned int flags, + void (*ctor)(void *)); + ++void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); ++int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, ++ void **list); ++ + #endif /* _TOOLS_SLAB_H */ +diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore +index d971516401e6..c901d96dd013 100644 +--- a/tools/testing/radix-tree/.gitignore ++++ b/tools/testing/radix-tree/.gitignore +@@ -6,3 +6,5 @@ main + multiorder + radix-tree.c + xarray ++maple ++ma_xa_benchmark +diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile +index c4ea4fbb0bfc..89d613e0505b 100644 +--- a/tools/testing/radix-tree/Makefile ++++ b/tools/testing/radix-tree/Makefile +@@ -4,9 +4,9 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ + -fsanitize=undefined + LDFLAGS += -fsanitize=address -fsanitize=undefined + LDLIBS+= -lpthread -lurcu +-TARGETS = main idr-test multiorder xarray ++TARGETS = main idr-test multiorder xarray maple + CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \ +- slab.o ++ slab.o maple.o + OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ + regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ + iteration_check_2.o benchmark.o +@@ -29,6 +29,8 @@ idr-test: idr-test.o $(CORE_OFILES) + + xarray: $(CORE_OFILES) + ++maple: $(CORE_OFILES) ++ + multiorder: multiorder.o $(CORE_OFILES) + + clean: +@@ -40,6 +42,7 @@ $(OFILES): Makefile *.h */*.h generated/map-shift.h \ + ../../include/linux/*.h \ + ../../include/asm/*.h \ + ../../../include/linux/xarray.h \ ++ ../../../include/linux/maple_tree.h \ + ../../../include/linux/radix-tree.h \ + ../../../include/linux/idr.h + +@@ -51,6 +54,8 @@ idr.c: ../../../lib/idr.c + + xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c + ++maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c ++ + generated/map-shift.h: + @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ + echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ +diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h +index 2218b3cc184e..e7da80350236 100644 +--- a/tools/testing/radix-tree/generated/autoconf.h ++++ b/tools/testing/radix-tree/generated/autoconf.h +@@ -1 +1,2 @@ + #define CONFIG_XARRAY_MULTI 1 ++#define CONFIG_64BIT 1 +diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c +index d5c1bcba86fe..2048d12c31df 100644 +--- a/tools/testing/radix-tree/linux.c ++++ b/tools/testing/radix-tree/linux.c +@@ -23,15 +23,47 @@ struct kmem_cache { + int nr_objs; + void *objs; + void (*ctor)(void *); ++ unsigned int non_kernel; ++ unsigned long nr_allocated; ++ unsigned long nr_tallocated; + }; + ++void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) ++{ ++ cachep->non_kernel = val; ++} ++ ++unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) ++{ ++ return cachep->size * cachep->nr_allocated; ++} ++ ++unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) ++{ ++ return cachep->nr_allocated; ++} ++ ++unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) ++{ ++ return cachep->nr_tallocated; ++} ++ ++void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) ++{ ++ cachep->nr_tallocated = 0; ++} ++ + void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, + int gfp) { - struct mem_cgroup *memcg; - struct lruvec *lruvec; + void *p; -+ if (lru_gen_enabled()) { -+ lru_gen_age_node(pgdat, sc); -+ return; -+ } +- if (!(gfp & __GFP_DIRECT_RECLAIM)) +- return NULL; ++ if (!(gfp & __GFP_DIRECT_RECLAIM)) { ++ if (!cachep->non_kernel) ++ return NULL; + - if (!can_age_anon_pages(pgdat, sc)) - return; ++ cachep->non_kernel--; ++ } -@@ -4252,12 +7042,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) - sc.may_swap = !nr_boost_reclaim; + pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs) { +@@ -53,19 +85,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, + memset(p, 0, cachep->size); + } - /* -- * Do some background aging of the anon list, to give -- * pages a chance to be referenced before reclaiming. All -- * pages are rotated regardless of classzone as this is -- * about consistent aging. -+ * Do some background aging, to give pages a chance to be -+ * referenced before reclaiming. All pages are rotated -+ * regardless of classzone as this is about consistent aging. - */ -- age_active_anon(pgdat, &sc); -+ kswapd_age_node(pgdat, &sc); ++ uatomic_inc(&cachep->nr_allocated); + uatomic_inc(&nr_allocated); ++ uatomic_inc(&cachep->nr_tallocated); + if (kmalloc_verbose) + printf("Allocating %p from slab\n", p); + return p; + } - /* - * If we're getting trouble reclaiming, start doing writepage -diff --git a/mm/workingset.c b/mm/workingset.c -index a5e84862fc86..ae7e984b23c6 100644 ---- a/mm/workingset.c -+++ b/mm/workingset.c -@@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; - static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, - bool workingset) +-void kmem_cache_free(struct kmem_cache *cachep, void *objp) ++void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) { -- eviction >>= bucket_order; - eviction &= EVICTION_MASK; - eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; - eviction = (eviction << NODES_SHIFT) | pgdat->node_id; -@@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, - - *memcgidp = memcgid; - *pgdat = NODE_DATA(nid); -- *evictionp = entry << bucket_order; -+ *evictionp = entry; - *workingsetp = workingset; + assert(objp); + uatomic_dec(&nr_allocated); ++ uatomic_dec(&cachep->nr_allocated); + if (kmalloc_verbose) + printf("Freeing %p to slab\n", objp); +- pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs > 10 || cachep->align) { + memset(objp, POISON_FREE, cachep->size); + free(objp); +@@ -75,9 +109,80 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) + node->parent = cachep->objs; + cachep->objs = node; + } ++} ++ ++void kmem_cache_free(struct kmem_cache *cachep, void *objp) ++{ ++ pthread_mutex_lock(&cachep->lock); ++ kmem_cache_free_locked(cachep, objp); + pthread_mutex_unlock(&cachep->lock); } -+#ifdef CONFIG_LRU_GEN ++void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) ++{ ++ if (kmalloc_verbose) ++ pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); + -+static void *lru_gen_eviction(struct folio *folio) ++ pthread_mutex_lock(&cachep->lock); ++ for (int i = 0; i < size; i++) ++ kmem_cache_free_locked(cachep, list[i]); ++ pthread_mutex_unlock(&cachep->lock); ++} ++ ++int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, ++ void **p) +{ -+ int hist; -+ unsigned long token; -+ unsigned long min_seq; -+ struct lruvec *lruvec; -+ struct lru_gen_struct *lrugen; -+ int type = folio_is_file_lru(folio); -+ int delta = folio_nr_pages(folio); -+ int refs = folio_lru_refs(folio); -+ int tier = lru_tier_from_refs(refs); -+ struct mem_cgroup *memcg = folio_memcg(folio); -+ struct pglist_data *pgdat = folio_pgdat(folio); ++ size_t i; + -+ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); ++ if (kmalloc_verbose) ++ pr_debug("Bulk alloc %lu\n", size); + -+ lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ lrugen = &lruvec->lrugen; -+ min_seq = READ_ONCE(lrugen->min_seq[type]); -+ token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); ++ if (!(gfp & __GFP_DIRECT_RECLAIM)) { ++ if (cachep->non_kernel < size) ++ return 0; + -+ hist = lru_hist_from_seq(min_seq); -+ atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); ++ cachep->non_kernel -= size; ++ } + -+ return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); ++ pthread_mutex_lock(&cachep->lock); ++ if (cachep->nr_objs >= size) { ++ struct radix_tree_node *node; ++ ++ for (i = 0; i < size; i++) { ++ node = cachep->objs; ++ cachep->nr_objs--; ++ cachep->objs = node->parent; ++ p[i] = node; ++ node->parent = NULL; ++ } ++ pthread_mutex_unlock(&cachep->lock); ++ } else { ++ pthread_mutex_unlock(&cachep->lock); ++ for (i = 0; i < size; i++) { ++ if (cachep->align) { ++ posix_memalign(&p[i], cachep->align, ++ cachep->size * size); ++ } else { ++ p[i] = malloc(cachep->size * size); ++ } ++ if (cachep->ctor) ++ cachep->ctor(p[i]); ++ else if (gfp & __GFP_ZERO) ++ memset(p[i], 0, cachep->size); ++ } ++ } ++ ++ for (i = 0; i < size; i++) { ++ uatomic_inc(&nr_allocated); ++ uatomic_inc(&cachep->nr_allocated); ++ uatomic_inc(&cachep->nr_tallocated); ++ if (kmalloc_verbose) ++ printf("Allocating %p from slab\n", p[i]); ++ } ++ ++ return size; +} + -+static void lru_gen_refault(struct folio *folio, void *shadow) + struct kmem_cache * + kmem_cache_create(const char *name, unsigned int size, unsigned int align, + unsigned int flags, void (*ctor)(void *)) +@@ -88,7 +193,54 @@ kmem_cache_create(const char *name, unsigned int size, unsigned int align, + ret->size = size; + ret->align = align; + ret->nr_objs = 0; ++ ret->nr_allocated = 0; ++ ret->nr_tallocated = 0; + ret->objs = NULL; + ret->ctor = ctor; ++ ret->non_kernel = 0; + return ret; + } ++ ++/* ++ * Test the test infrastructure for kem_cache_alloc/free and bulk counterparts. ++ */ ++void test_kmem_cache_bulk(void) +{ -+ int hist, tier, refs; -+ int memcg_id; -+ bool workingset; -+ unsigned long token; -+ unsigned long min_seq; -+ struct lruvec *lruvec; -+ struct lru_gen_struct *lrugen; -+ struct mem_cgroup *memcg; -+ struct pglist_data *pgdat; -+ int type = folio_is_file_lru(folio); -+ int delta = folio_nr_pages(folio); ++ int i; ++ void *list[12]; ++ static struct kmem_cache *test_cache, *test_cache2; + -+ unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); ++ /* ++ * Testing the bulk allocators without aligned kmem_cache to force the ++ * bulk alloc/free to reuse ++ */ ++ test_cache = kmem_cache_create("test_cache", 256, 0, SLAB_PANIC, NULL); + -+ if (pgdat != folio_pgdat(folio)) -+ return; ++ for (i = 0; i < 5; i++) ++ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); + -+ rcu_read_lock(); ++ for (i = 0; i < 5; i++) ++ kmem_cache_free(test_cache, list[i]); ++ assert(test_cache->nr_objs == 5); + -+ memcg = folio_memcg_rcu(folio); -+ if (memcg_id != mem_cgroup_id(memcg)) -+ goto unlock; ++ kmem_cache_alloc_bulk(test_cache, __GFP_DIRECT_RECLAIM, 5, list); ++ kmem_cache_free_bulk(test_cache, 5, list); + -+ lruvec = mem_cgroup_lruvec(memcg, pgdat); -+ lrugen = &lruvec->lrugen; ++ for (i = 0; i < 12 ; i++) ++ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); + -+ min_seq = READ_ONCE(lrugen->min_seq[type]); -+ if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) -+ goto unlock; ++ for (i = 0; i < 12; i++) ++ kmem_cache_free(test_cache, list[i]); + -+ hist = lru_hist_from_seq(min_seq); -+ /* see the comment in folio_lru_refs() */ -+ refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; -+ tier = lru_tier_from_refs(refs); ++ /* The last free will not be kept around */ ++ assert(test_cache->nr_objs == 11); ++ ++ /* Aligned caches will immediately free */ ++ test_cache2 = kmem_cache_create("test_cache2", 128, 128, SLAB_PANIC, NULL); ++ ++ kmem_cache_alloc_bulk(test_cache2, __GFP_DIRECT_RECLAIM, 10, list); ++ kmem_cache_free_bulk(test_cache2, 10, list); ++ assert(!test_cache2->nr_objs); + -+ atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); -+ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); + -+ /* -+ * Count the following two cases as stalls: -+ * 1. For pages accessed through page tables, hotter pages pushed out -+ * hot pages which refaulted immediately. -+ * 2. For pages accessed multiple times through file descriptors, -+ * numbers of accesses might have been out of the range. -+ */ -+ if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { -+ folio_set_workingset(folio); -+ mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); -+ } -+unlock: -+ rcu_read_unlock(); +} +diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h +index 39867fd80c8f..c5c9d05f29da 100644 +--- a/tools/testing/radix-tree/linux/kernel.h ++++ b/tools/testing/radix-tree/linux/kernel.h +@@ -14,6 +14,7 @@ + #include "../../../include/linux/kconfig.h" + + #define printk printf ++#define pr_err printk + #define pr_info printk + #define pr_debug printk + #define pr_cont printk +diff --git a/tools/testing/radix-tree/linux/lockdep.h b/tools/testing/radix-tree/linux/lockdep.h +index 016cff473cfc..62473ab57f99 100644 +--- a/tools/testing/radix-tree/linux/lockdep.h ++++ b/tools/testing/radix-tree/linux/lockdep.h +@@ -11,4 +11,6 @@ static inline void lockdep_set_class(spinlock_t *lock, + struct lock_class_key *key) + { + } + -+#else /* !CONFIG_LRU_GEN */ ++extern int lockdep_is_held(const void *); + #endif /* _LINUX_LOCKDEP_H */ +diff --git a/tools/testing/radix-tree/linux/maple_tree.h b/tools/testing/radix-tree/linux/maple_tree.h +new file mode 100644 +index 000000000000..7d8d1f445b89 +--- /dev/null ++++ b/tools/testing/radix-tree/linux/maple_tree.h +@@ -0,0 +1,7 @@ ++/* SPDX-License-Identifier: GPL-2.0+ */ ++#define atomic_t int32_t ++#include "../../../../include/linux/maple_tree.h" ++#define atomic_inc(x) uatomic_inc(x) ++#define atomic_read(x) uatomic_read(x) ++#define atomic_set(x, y) do {} while (0) ++#define U8_MAX UCHAR_MAX +diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c +new file mode 100644 +index 000000000000..35082671928a +--- /dev/null ++++ b/tools/testing/radix-tree/maple.c +@@ -0,0 +1,59 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++/* ++ * maple_tree.c: Userspace shim for maple tree test-suite ++ * Copyright (c) 2018 Liam R. Howlett ++ */ + -+static void *lru_gen_eviction(struct folio *folio) ++#define CONFIG_DEBUG_MAPLE_TREE ++#define CONFIG_MAPLE_SEARCH ++#include "test.h" ++ ++#define module_init(x) ++#define module_exit(x) ++#define MODULE_AUTHOR(x) ++#define MODULE_LICENSE(x) ++#define dump_stack() assert(0) ++ ++#include "../../../lib/maple_tree.c" ++#undef CONFIG_DEBUG_MAPLE_TREE ++#include "../../../lib/test_maple_tree.c" ++ ++void farmer_tests(void) +{ -+ return NULL; ++ struct maple_node *node; ++ DEFINE_MTREE(tree); ++ ++ mt_dump(&tree); ++ ++ tree.ma_root = xa_mk_value(0); ++ mt_dump(&tree); ++ ++ node = mt_alloc_one(GFP_KERNEL); ++ node->parent = (void *)((unsigned long)(&tree) | 1); ++ node->slot[0] = xa_mk_value(0); ++ node->slot[1] = xa_mk_value(1); ++ node->mr64.pivot[0] = 0; ++ node->mr64.pivot[1] = 1; ++ node->mr64.pivot[2] = 0; ++ tree.ma_root = mt_mk_node(node, maple_leaf_64); ++ mt_dump(&tree); ++ ++ ma_free_rcu(node); +} + -+static void lru_gen_refault(struct folio *folio, void *shadow) ++void maple_tree_tests(void) +{ ++ farmer_tests(); ++ maple_tree_seed(); ++ maple_tree_harvest(); +} + -+#endif /* CONFIG_LRU_GEN */ -+ - /** - * workingset_age_nonresident - age non-resident entries as LRU ages - * @lruvec: the lruvec that was aged -@@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) - VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); - VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); - -+ if (lru_gen_enabled()) -+ return lru_gen_eviction(folio); -+ - lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - /* XXX: target_memcg can be NULL, go through lruvec */ - memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); - eviction = atomic_long_read(&lruvec->nonresident_age); -+ eviction >>= bucket_order; - workingset_age_nonresident(lruvec, folio_nr_pages(folio)); - return pack_shadow(memcgid, pgdat, eviction, - folio_test_workingset(folio)); -@@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) - int memcgid; - long nr; - -+ if (lru_gen_enabled()) { -+ lru_gen_refault(folio, shadow); -+ return; -+ } ++int __weak main(void) ++{ ++ maple_tree_init(); ++ maple_tree_tests(); ++ rcu_barrier(); ++ if (nr_allocated) ++ printf("nr_allocated = %d\n", nr_allocated); ++ return 0; ++} +diff --git a/tools/testing/radix-tree/trace/events/maple_tree.h b/tools/testing/radix-tree/trace/events/maple_tree.h +new file mode 100644 +index 000000000000..97d0e1ddcf08 +--- /dev/null ++++ b/tools/testing/radix-tree/trace/events/maple_tree.h +@@ -0,0 +1,5 @@ ++/* SPDX-License-Identifier: GPL-2.0+ */ + - unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); -+ eviction <<= bucket_order; - - rcu_read_lock(); - /* ++#define trace_ma_op(a, b) do {} while (0) ++#define trace_ma_read(a, b) do {} while (0) ++#define trace_ma_write(a, b, c, d) do {} while (0) -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 -From 440d9b1f159e8842799696a54fd5ddb51d545bc9 Mon Sep 17 00:00:00 2001 +From 79eeeac092d265211e4f6ce60f69ad549d8a201c Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 Sep 2022 00:18:41 +0200 -Subject: [PATCH 06/13] mm-cleanup +Subject: [PATCH 09/16] mm-cleanup Signed-off-by: Peter Jung --- @@ -25362,10 +82256,10 @@ index e0b2209ab71c..7ac0e5c78648 100644 { } diff --git a/include/linux/mm.h b/include/linux/mm.h -index 88976a521ef5..6c61497f0167 100644 +index 9ac0e02e2238..9c3d3474033f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h -@@ -3016,7 +3016,7 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, +@@ -3032,7 +3032,7 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); @@ -25375,7 +82269,7 @@ index 88976a521ef5..6c61497f0167 100644 extern void __kernel_poison_pages(struct page *page, int numpages); extern void __kernel_unpoison_pages(struct page *page, int numpages); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index 1543001feba9..21d39f152d0c 100644 +index e5c240eed6af..b4c82726816b 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -577,13 +577,6 @@ enum zone_watermarks { @@ -25392,7 +82286,7 @@ index 1543001feba9..21d39f152d0c 100644 #define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost) #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost) #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost) -@@ -1241,11 +1234,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) +@@ -1244,11 +1237,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) return pgdat->node_start_pfn + pgdat->node_spanned_pages; } @@ -25405,10 +82299,10 @@ index 1543001feba9..21d39f152d0c 100644 void build_all_zonelists(pg_data_t *pgdat); diff --git a/mm/internal.h b/mm/internal.h -index a1fddea6b34f..dd8600d71afe 100644 +index 0f106a3982e7..4ea69814259a 100644 --- a/mm/internal.h +++ b/mm/internal.h -@@ -366,7 +366,6 @@ extern int user_min_free_kbytes; +@@ -367,7 +367,6 @@ extern int user_min_free_kbytes; extern void free_unref_page(struct page *page, unsigned int order); extern void free_unref_page_list(struct list_head *list); @@ -25416,7 +82310,7 @@ index a1fddea6b34f..dd8600d71afe 100644 extern void zone_pcp_reset(struct zone *zone); extern void zone_pcp_disable(struct zone *zone); extern void zone_pcp_enable(struct zone *zone); -@@ -861,8 +860,6 @@ int migrate_device_coherent_page(struct page *page); +@@ -859,8 +858,6 @@ int migrate_device_coherent_page(struct page *page); */ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags); @@ -25606,12 +82500,12 @@ index cf131d6e08fb..292ed1bb6a5a 100644 } -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 -From b0023af1c32f89b1b3b4c888d850eb2f77cffd99 Mon Sep 17 00:00:00 2001 +From 6257c94a850dc4b3faa5a55be5831de4f8777cac Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 19 Aug 2022 17:06:47 +0200 -Subject: [PATCH 07/13] rtw88 +Subject: [PATCH 10/16] rtw88 Signed-off-by: Peter Jung --- @@ -28094,12 +84988,12 @@ index 0c23b5069be0..dc8965525400 100644 { __le16 fc = hdr->frame_control; -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 -From 13477a1f4474e7a9cf46017e0f8081dd095433bf Mon Sep 17 00:00:00 2001 +From 953761366f999b9035f8fff70c214426ad9f027b Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Wed, 14 Sep 2022 14:40:34 +0200 -Subject: [PATCH 08/13] rcu +Subject: [PATCH 11/16] rcu Signed-off-by: Peter Jung --- @@ -28168,12 +85062,12 @@ index a8f574d8850d..4017ebecec91 100644 pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n"); cpumask_and(rcu_nocb_mask, cpu_possible_mask, -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 -From 0499cabe43b65283662b91133be9c798aa6ebaff Mon Sep 17 00:00:00 2001 +From e2af20ddb7f4e410c25c3deb9dd579d56e340a0b Mon Sep 17 00:00:00 2001 From: Piotr Gorski Date: Tue, 6 Sep 2022 20:04:11 +0200 -Subject: [PATCH 09/13] lrng +Subject: [PATCH 12/16] lrng Signed-off-by: Piotr Gorski --- @@ -28302,7 +85196,7 @@ Signed-off-by: Piotr Gorski create mode 100644 include/linux/lrng.h diff --git a/MAINTAINERS b/MAINTAINERS -index a29c9731350c..d7de955b50bb 100644 +index 96a09757feb3..e3c1b29c60a0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11741,6 +11741,13 @@ F: Documentation/litmus-tests/ @@ -37667,2554 +94561,1112 @@ index 000000000000..101140085d81 + if (outbuflen < sizeof(u32)) { + spin_unlock_irqrestore(&data->lock, flags); + goto out; -+ } -+ -+ memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], -+ sizeof(u32)); -+ data->rb_reader++; -+ -+ spin_unlock_irqrestore(&data->lock, flags); -+ -+ outbuf += sizeof(u32); -+ outbuflen -= sizeof(u32); -+ collected_data += sizeof(u32); -+ } -+ -+out: -+ lrng_testing_fini(data, *boot); -+ return collected_data; -+} -+ -+static int lrng_testing_extract_user(struct file *file, char __user *buf, -+ size_t nbytes, loff_t *ppos, -+ int (*reader)(u8 *outbuf, u32 outbuflen)) -+{ -+ u8 *tmp, *tmp_aligned; -+ int ret = 0, large_request = (nbytes > 256); -+ -+ if (!nbytes) -+ return 0; -+ -+ /* -+ * The intention of this interface is for collecting at least -+ * 1000 samples due to the SP800-90B requirements. So, we make no -+ * effort in avoiding allocating more memory that actually needed -+ * by the user. Hence, we allocate sufficient memory to always hold -+ * that amount of data. -+ */ -+ tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); -+ if (!tmp) -+ return -ENOMEM; -+ -+ tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); -+ -+ while (nbytes) { -+ int i; -+ -+ if (large_request && need_resched()) { -+ if (signal_pending(current)) { -+ if (ret == 0) -+ ret = -ERESTARTSYS; -+ break; -+ } -+ schedule(); -+ } -+ -+ i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); -+ i = reader(tmp_aligned, i); -+ if (i <= 0) { -+ if (i < 0) -+ ret = i; -+ break; -+ } -+ if (copy_to_user(buf, tmp_aligned, i)) { -+ ret = -EFAULT; -+ break; -+ } -+ -+ nbytes -= i; -+ buf += i; -+ ret += i; -+ } -+ -+ kfree_sensitive(tmp); -+ -+ if (ret > 0) -+ *ppos += ret; -+ -+ return ret; -+} -+ -+#endif /* CONFIG_LRNG_TESTING_RECORDING */ -+ -+/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+ -+static u32 boot_raw_hires_test = 0; -+module_param(boot_raw_hires_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); -+ -+static struct lrng_testing lrng_raw_hires = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) -+}; -+ -+bool lrng_raw_hires_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); -+} -+ -+static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_hires_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_hires_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_hires_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+ -+/********************* Raw Jiffies Entropy Data Handling **********************/ -+ -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+ -+static u32 boot_raw_jiffies_test = 0; -+module_param(boot_raw_jiffies_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); -+ -+static struct lrng_testing lrng_raw_jiffies = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) -+}; -+ -+bool lrng_raw_jiffies_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_jiffies, value, -+ &boot_raw_jiffies_test); -+} -+ -+static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_jiffies_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_jiffies_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_jiffies_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+ -+/************************** Raw IRQ Data Handling ****************************/ -+ -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+ -+static u32 boot_raw_irq_test = 0; -+module_param(boot_raw_irq_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); -+ -+static struct lrng_testing lrng_raw_irq = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) -+}; -+ -+bool lrng_raw_irq_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); -+} -+ -+static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_irq_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_irq_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_irq_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+ -+/************************ Raw _RET_IP_ Data Handling **************************/ -+ -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+ -+static u32 boot_raw_retip_test = 0; -+module_param(boot_raw_retip_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); -+ -+static struct lrng_testing lrng_raw_retip = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) -+}; -+ -+bool lrng_raw_retip_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); -+} -+ -+static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_retip_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_retip_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_retip_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+ -+/********************** Raw IRQ register Data Handling ************************/ -+ -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+ -+static u32 boot_raw_regs_test = 0; -+module_param(boot_raw_regs_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); -+ -+static struct lrng_testing lrng_raw_regs = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) -+}; -+ -+bool lrng_raw_regs_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); -+} -+ -+static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, -+ outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_regs_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_regs_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_regs_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+ -+/********************** Raw Entropy Array Data Handling ***********************/ -+ -+#ifdef CONFIG_LRNG_RAW_ARRAY -+ -+static u32 boot_raw_array = 0; -+module_param(boot_raw_array, uint, 0644); -+MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); -+ -+static struct lrng_testing lrng_raw_array = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) -+}; -+ -+bool lrng_raw_array_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); -+} -+ -+static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, -+ outbuflen); -+} -+ -+static ssize_t lrng_raw_array_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_array_entropy_reader); -+} -+ -+static const struct file_operations lrng_raw_array_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_array_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_ARRAY */ -+ -+/******************** Interrupt Performance Data Handling *********************/ ++ } + -+#ifdef CONFIG_LRNG_IRQ_PERF ++ memcpy(outbuf, &data->lrng_testing_rb[data->rb_reader], ++ sizeof(u32)); ++ data->rb_reader++; + -+static u32 boot_irq_perf = 0; -+module_param(boot_irq_perf, uint, 0644); -+MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); ++ spin_unlock_irqrestore(&data->lock, flags); + -+static struct lrng_testing lrng_irq_perf = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) -+}; ++ outbuf += sizeof(u32); ++ outbuflen -= sizeof(u32); ++ collected_data += sizeof(u32); ++ } + -+bool lrng_perf_time(u32 start) -+{ -+ return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, -+ &boot_irq_perf); ++out: ++ lrng_testing_fini(data, *boot); ++ return collected_data; +} + -+static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) ++static int lrng_testing_extract_user(struct file *file, char __user *buf, ++ size_t nbytes, loff_t *ppos, ++ int (*reader)(u8 *outbuf, u32 outbuflen)) +{ -+ return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, -+ outbuflen); -+} ++ u8 *tmp, *tmp_aligned; ++ int ret = 0, large_request = (nbytes > 256); + -+static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_irq_perf_reader); -+} ++ if (!nbytes) ++ return 0; + -+static const struct file_operations lrng_irq_perf_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_irq_perf_read, -+}; ++ /* ++ * The intention of this interface is for collecting at least ++ * 1000 samples due to the SP800-90B requirements. So, we make no ++ * effort in avoiding allocating more memory that actually needed ++ * by the user. Hence, we allocate sufficient memory to always hold ++ * that amount of data. ++ */ ++ tmp = kmalloc(LRNG_TESTING_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); ++ if (!tmp) ++ return -ENOMEM; + -+#endif /* CONFIG_LRNG_IRQ_PERF */ ++ tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); + -+/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ ++ while (nbytes) { ++ int i; + -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY ++ if (large_request && need_resched()) { ++ if (signal_pending(current)) { ++ if (ret == 0) ++ ret = -ERESTARTSYS; ++ break; ++ } ++ schedule(); ++ } + -+static u32 boot_raw_sched_hires_test = 0; -+module_param(boot_raw_sched_hires_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); ++ i = min_t(int, nbytes, LRNG_TESTING_RINGBUFFER_SIZE); ++ i = reader(tmp_aligned, i); ++ if (i <= 0) { ++ if (i < 0) ++ ret = i; ++ break; ++ } ++ if (copy_to_user(buf, tmp_aligned, i)) { ++ ret = -EFAULT; ++ break; ++ } + -+static struct lrng_testing lrng_raw_sched_hires = { -+ .rb_reader = 0, -+ .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), -+ .read_wait = -+ __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) -+}; ++ nbytes -= i; ++ buf += i; ++ ret += i; ++ } + -+bool lrng_raw_sched_hires_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_hires, value, -+ &boot_raw_sched_hires_test); -+} ++ kfree_sensitive(tmp); + -+static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_hires, -+ &boot_raw_sched_hires_test, -+ outbuf, outbuflen); -+} ++ if (ret > 0) ++ *ppos += ret; + -+static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) -+{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_hires_entropy_reader); ++ return ret; +} + -+static const struct file_operations lrng_raw_sched_hires_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_hires_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ ++#endif /* CONFIG_LRNG_TESTING_RECORDING */ + -+/******************** Interrupt Performance Data Handling *********************/ ++/************* Raw High-Resolution IRQ Timer Entropy Data Handling ************/ + -+#ifdef CONFIG_LRNG_SCHED_PERF ++#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY + -+static u32 boot_sched_perf = 0; -+module_param(boot_sched_perf, uint, 0644); -+MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); ++static u32 boot_raw_hires_test = 0; ++module_param(boot_raw_hires_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_hires_test, "Enable gathering boot time high resolution timer entropy of the first IRQ entropy events"); + -+static struct lrng_testing lrng_sched_perf = { ++static struct lrng_testing lrng_raw_hires = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_hires.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_hires.read_wait) +}; + -+bool lrng_sched_perf_time(u32 start) ++bool lrng_raw_hires_entropy_store(u32 value) +{ -+ return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, -+ &boot_sched_perf); ++ return lrng_testing_store(&lrng_raw_hires, value, &boot_raw_hires_test); +} + -+static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) ++static int lrng_raw_hires_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, -+ outbuflen); ++ return lrng_testing_reader(&lrng_raw_hires, &boot_raw_hires_test, ++ outbuf, outbuflen); +} + -+static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) ++static ssize_t lrng_raw_hires_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_sched_perf_reader); ++ lrng_raw_hires_entropy_reader); +} + -+static const struct file_operations lrng_sched_perf_fops = { ++static const struct file_operations lrng_raw_hires_fops = { + .owner = THIS_MODULE, -+ .read = lrng_sched_perf_read, ++ .read = lrng_raw_hires_read, +}; + -+#endif /* CONFIG_LRNG_SCHED_PERF */ ++#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + -+/*************** Raw Scheduler task_struct->pid Data Handling *****************/ ++/********************* Raw Jiffies Entropy Data Handling **********************/ + -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY + -+static u32 boot_raw_sched_pid_test = 0; -+module_param(boot_raw_sched_pid_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); ++static u32 boot_raw_jiffies_test = 0; ++module_param(boot_raw_jiffies_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_jiffies_test, "Enable gathering boot time high resolution timer entropy of the first entropy events"); + -+static struct lrng_testing lrng_raw_sched_pid = { ++static struct lrng_testing lrng_raw_jiffies = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_jiffies.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_jiffies.read_wait) +}; + -+bool lrng_raw_sched_pid_entropy_store(u32 value) ++bool lrng_raw_jiffies_entropy_store(u32 value) +{ -+ return lrng_testing_store(&lrng_raw_sched_pid, value, -+ &boot_raw_sched_pid_test); ++ return lrng_testing_store(&lrng_raw_jiffies, value, ++ &boot_raw_jiffies_test); +} + -+static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) ++static int lrng_raw_jiffies_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ return lrng_testing_reader(&lrng_raw_sched_pid, -+ &boot_raw_sched_pid_test, outbuf, outbuflen); ++ return lrng_testing_reader(&lrng_raw_jiffies, &boot_raw_jiffies_test, ++ outbuf, outbuflen); +} + -+static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) ++static ssize_t lrng_raw_jiffies_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_pid_entropy_reader); ++ lrng_raw_jiffies_entropy_reader); +} + -+static const struct file_operations lrng_raw_sched_pid_fops = { ++static const struct file_operations lrng_raw_jiffies_fops = { + .owner = THIS_MODULE, -+ .read = lrng_raw_sched_pid_read, ++ .read = lrng_raw_jiffies_read, +}; + -+#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+ ++#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + -+/*********** Raw Scheduler task_struct->start_time Data Handling **************/ ++/************************** Raw IRQ Data Handling ****************************/ + -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY + -+static u32 boot_raw_sched_starttime_test = 0; -+module_param(boot_raw_sched_starttime_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); ++static u32 boot_raw_irq_test = 0; ++module_param(boot_raw_irq_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_irq_test, "Enable gathering boot time entropy of the first IRQ entropy events"); + -+static struct lrng_testing lrng_raw_sched_starttime = { ++static struct lrng_testing lrng_raw_irq = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_irq.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_irq.read_wait) +}; + -+bool lrng_raw_sched_starttime_entropy_store(u32 value) ++bool lrng_raw_irq_entropy_store(u32 value) +{ -+ return lrng_testing_store(&lrng_raw_sched_starttime, value, -+ &boot_raw_sched_starttime_test); ++ return lrng_testing_store(&lrng_raw_irq, value, &boot_raw_irq_test); +} + -+static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) ++static int lrng_raw_irq_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ return lrng_testing_reader(&lrng_raw_sched_starttime, -+ &boot_raw_sched_starttime_test, outbuf, outbuflen); ++ return lrng_testing_reader(&lrng_raw_irq, &boot_raw_irq_test, outbuf, ++ outbuflen); +} + -+static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) ++static ssize_t lrng_raw_irq_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ + return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_starttime_entropy_reader); ++ lrng_raw_irq_entropy_reader); +} + -+static const struct file_operations lrng_raw_sched_starttime_fops = { ++static const struct file_operations lrng_raw_irq_fops = { + .owner = THIS_MODULE, -+ .read = lrng_raw_sched_starttime_read, ++ .read = lrng_raw_irq_read, +}; + -+#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + -+/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ ++/************************ Raw _RET_IP_ Data Handling **************************/ + -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY + -+static u32 boot_raw_sched_nvcsw_test = 0; -+module_param(boot_raw_sched_nvcsw_test, uint, 0644); -+MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); ++static u32 boot_raw_retip_test = 0; ++module_param(boot_raw_retip_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_retip_test, "Enable gathering boot time entropy of the first return instruction pointer entropy events"); + -+static struct lrng_testing lrng_raw_sched_nvcsw = { ++static struct lrng_testing lrng_raw_retip = { + .rb_reader = 0, + .rb_writer = ATOMIC_INIT(0), -+ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), -+ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_retip.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_retip.read_wait) +}; + -+bool lrng_raw_sched_nvcsw_entropy_store(u32 value) -+{ -+ return lrng_testing_store(&lrng_raw_sched_nvcsw, value, -+ &boot_raw_sched_nvcsw_test); -+} -+ -+static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) -+{ -+ return lrng_testing_reader(&lrng_raw_sched_nvcsw, -+ &boot_raw_sched_nvcsw_test, outbuf, outbuflen); -+} -+ -+static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, -+ size_t count, loff_t *ppos) ++bool lrng_raw_retip_entropy_store(u32 value) +{ -+ return lrng_testing_extract_user(file, to, count, ppos, -+ lrng_raw_sched_nvcsw_entropy_reader); ++ return lrng_testing_store(&lrng_raw_retip, value, &boot_raw_retip_test); +} + -+static const struct file_operations lrng_raw_sched_nvcsw_fops = { -+ .owner = THIS_MODULE, -+ .read = lrng_raw_sched_nvcsw_read, -+}; -+ -+#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+ -+/*********************************** ACVT ************************************/ -+ -+#ifdef CONFIG_LRNG_ACVT_HASH -+ -+/* maximum amount of data to be hashed as defined by ACVP */ -+#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) -+ -+/* -+ * As we use static variables to store the data, it is clear that the -+ * test interface is only able to handle single threaded testing. This is -+ * considered to be sufficient for testing. If multi-threaded use of the -+ * ACVT test interface would be performed, the caller would get garbage -+ * but the kernel operation is unaffected by this. -+ */ -+static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] -+ __aligned(LRNG_KCAPI_ALIGN); -+static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); -+static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; -+ -+static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, -+ size_t nbytes, loff_t *ppos) ++static int lrng_raw_retip_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ if (nbytes > LRNG_ACVT_MAX_SHA_MSG) -+ return -EINVAL; -+ -+ atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); -+ -+ return simple_write_to_buffer(lrng_acvt_hash_data, -+ LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); ++ return lrng_testing_reader(&lrng_raw_retip, &boot_raw_retip_test, ++ outbuf, outbuflen); +} + -+static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, ++static ssize_t lrng_raw_retip_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ -+ SHASH_DESC_ON_STACK(shash, NULL); -+ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; -+ ssize_t ret; -+ -+ if (count > LRNG_ATOMIC_DIGEST_SIZE) -+ return -EINVAL; -+ -+ ret = hash_cb->hash_init(shash, NULL) ?: -+ hash_cb->hash_update(shash, lrng_acvt_hash_data, -+ atomic_read_u32(&lrng_acvt_hash_data_size)) ?: -+ hash_cb->hash_final(shash, lrng_acvt_hash_digest); -+ if (ret) -+ return ret; -+ -+ return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, -+ sizeof(lrng_acvt_hash_digest)); ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_retip_entropy_reader); +} + -+static const struct file_operations lrng_acvt_hash_fops = { ++static const struct file_operations lrng_raw_retip_fops = { + .owner = THIS_MODULE, -+ .open = simple_open, -+ .llseek = default_llseek, -+ .read = lrng_acvt_hash_read, -+ .write = lrng_acvt_hash_write, -+}; -+ -+#endif /* CONFIG_LRNG_ACVT_DRNG */ -+ -+/************************************************************************** -+ * Debugfs interface -+ **************************************************************************/ -+ -+static int __init lrng_raw_init(void) -+{ -+ struct dentry *lrng_raw_debugfs_root; -+ -+ lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_hires", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_hires_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_jiffies_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, -+ NULL, &lrng_raw_irq_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_retip", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_retip_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_regs", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_regs_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_ARRAY -+ debugfs_create_file_unsafe("lrng_raw_array", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_array_fops); -+#endif -+#ifdef CONFIG_LRNG_IRQ_PERF -+ debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, -+ NULL, &lrng_irq_perf_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, -+ lrng_raw_debugfs_root, -+ NULL, &lrng_raw_sched_hires_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_pid_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_starttime_fops); -+#endif -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+ debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_raw_sched_nvcsw_fops); -+#endif -+#ifdef CONFIG_LRNG_SCHED_PERF -+ debugfs_create_file_unsafe("lrng_sched_perf", 0400, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_sched_perf_fops); -+#endif -+#ifdef CONFIG_LRNG_ACVT_HASH -+ debugfs_create_file_unsafe("lrng_acvt_hash", 0600, -+ lrng_raw_debugfs_root, NULL, -+ &lrng_acvt_hash_fops); -+#endif -+ -+ return 0; -+} -+ -+module_init(lrng_raw_init); -diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h -new file mode 100644 -index 000000000000..672a7fca4595 ---- /dev/null -+++ b/drivers/char/lrng/lrng_testing.h -@@ -0,0 +1,85 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ -+ -+#ifndef _LRNG_TESTING_H -+#define _LRNG_TESTING_H -+ -+#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY -+bool lrng_raw_hires_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY -+bool lrng_raw_jiffies_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY -+bool lrng_raw_irq_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY -+bool lrng_raw_retip_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY -+bool lrng_raw_regs_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_ARRAY -+bool lrng_raw_array_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_ARRAY */ -+static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_ARRAY */ -+ -+#ifdef CONFIG_LRNG_IRQ_PERF -+bool lrng_perf_time(u32 start); -+#else /* CONFIG_LRNG_IRQ_PERF */ -+static inline bool lrng_perf_time(u32 start) { return false; } -+#endif /*CONFIG_LRNG_IRQ_PERF */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY -+bool lrng_raw_sched_hires_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+static inline bool -+lrng_raw_sched_hires_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY -+bool lrng_raw_sched_pid_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+static inline bool -+lrng_raw_sched_pid_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY -+bool lrng_raw_sched_starttime_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+static inline bool -+lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY -+bool lrng_raw_sched_nvcsw_entropy_store(u32 value); -+#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+static inline bool -+lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } -+#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ -+ -+#ifdef CONFIG_LRNG_SCHED_PERF -+bool lrng_sched_perf_time(u32 start); -+#else /* CONFIG_LRNG_SCHED_PERF */ -+static inline bool lrng_sched_perf_time(u32 start) { return false; } -+#endif /*CONFIG_LRNG_SCHED_PERF */ ++ .read = lrng_raw_retip_read, ++}; + -+#endif /* _LRNG_TESTING_H */ -diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h -index af5ad51d3eef..b12ae9bdebf4 100644 ---- a/include/crypto/drbg.h -+++ b/include/crypto/drbg.h -@@ -283,4 +283,11 @@ enum drbg_prefixes { - DRBG_PREFIX3 - }; - -+extern int drbg_alloc_state(struct drbg_state *drbg); -+extern void drbg_dealloc_state(struct drbg_state *drbg); -+extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, -+ bool *pr); -+extern const struct drbg_core drbg_cores[]; -+extern unsigned short drbg_sec_strength(drbg_flag_t flags); ++#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + - #endif /* _DRBG_H */ -diff --git a/crypto/jitterentropy.h b/include/crypto/internal/jitterentropy.h -similarity index 100% -rename from crypto/jitterentropy.h -rename to include/crypto/internal/jitterentropy.h -diff --git a/include/linux/lrng.h b/include/linux/lrng.h -new file mode 100644 -index 000000000000..c0d31a03d51f ---- /dev/null -+++ b/include/linux/lrng.h -@@ -0,0 +1,251 @@ -+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ -+/* -+ * Copyright (C) 2022, Stephan Mueller -+ */ ++/********************** Raw IRQ register Data Handling ************************/ + -+#ifndef _LRNG_H -+#define _LRNG_H ++#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY + -+#include -+#include -+#include -+#include ++static u32 boot_raw_regs_test = 0; ++module_param(boot_raw_regs_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_regs_test, "Enable gathering boot time entropy of the first interrupt register entropy events"); + -+/* -+ * struct lrng_drng_cb - cryptographic callback functions defining a DRNG -+ * @drng_name Name of DRNG -+ * @drng_alloc: Allocate DRNG -- the provided integer should be used for -+ * sanity checks. -+ * return: allocated data structure or PTR_ERR on error -+ * @drng_dealloc: Deallocate DRNG -+ * @drng_seed: Seed the DRNG with data of arbitrary length drng: is -+ * pointer to data structure allocated with drng_alloc -+ * return: >= 0 on success, < 0 on error -+ * @drng_generate: Generate random numbers from the DRNG with arbitrary -+ * length -+ */ -+struct lrng_drng_cb { -+ const char *(*drng_name)(void); -+ void *(*drng_alloc)(u32 sec_strength); -+ void (*drng_dealloc)(void *drng); -+ int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); -+ int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); ++static struct lrng_testing lrng_raw_regs = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_regs.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_regs.read_wait) +}; + -+/* -+ * struct lrng_hash_cb - cryptographic callback functions defining a hash -+ * @hash_name Name of Hash used for reading entropy pool arbitrary -+ * length -+ * @hash_alloc: Allocate the hash for reading the entropy pool -+ * return: allocated data structure (NULL is success too) -+ * or ERR_PTR on error -+ * @hash_dealloc: Deallocate Hash -+ * @hash_digestsize: Return the digestsize for the used hash to read out -+ * entropy pool -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: size of digest of hash in bytes -+ * @hash_init: Initialize hash -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_update: Update hash operation -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_final Final hash operation -+ * hash: is pointer to data structure allocated with -+ * hash_alloc -+ * return: 0 on success, < 0 on error -+ * @hash_desc_zero Zeroization of hash state buffer -+ * -+ * Assumptions: -+ * -+ * 1. Hash operation will not sleep -+ * 2. The hash' volatile state information is provided with *shash by caller. -+ */ -+struct lrng_hash_cb { -+ const char *(*hash_name)(void); -+ void *(*hash_alloc)(void); -+ void (*hash_dealloc)(void *hash); -+ u32 (*hash_digestsize)(void *hash); -+ int (*hash_init)(struct shash_desc *shash, void *hash); -+ int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, -+ u32 inbuflen); -+ int (*hash_final)(struct shash_desc *shash, u8 *digest); -+ void (*hash_desc_zero)(struct shash_desc *shash); -+}; ++bool lrng_raw_regs_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_regs, value, &boot_raw_regs_test); ++} + -+/* Register cryptographic backend */ -+#ifdef CONFIG_LRNG_SWITCH -+int lrng_set_drng_cb(const struct lrng_drng_cb *cb); -+int lrng_set_hash_cb(const struct lrng_hash_cb *cb); -+#else /* CONFIG_LRNG_SWITCH */ -+static inline int -+lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } -+static inline int -+lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } -+#endif /* CONFIG_LRNG_SWITCH */ ++static int lrng_raw_regs_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_regs, &boot_raw_regs_test, ++ outbuf, outbuflen); ++} + -+/* Callback to feed events to the scheduler entropy source */ -+#ifdef CONFIG_LRNG_SCHED -+extern void add_sched_randomness(const struct task_struct *p, int cpu); -+#else -+static inline void -+add_sched_randomness(const struct task_struct *p, int cpu) { } -+#endif ++static ssize_t lrng_raw_regs_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_regs_entropy_reader); ++} + -+/* -+ * lrng_get_random_bytes() - Provider of cryptographic strong random numbers -+ * for kernel-internal usage. -+ * -+ * This function is appropriate for in-kernel use cases operating in atomic -+ * contexts. It will always use the ChaCha20 DRNG and it may be the case that -+ * it is not fully seeded when being used. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG_DRNG_ATOMIC -+void lrng_get_random_bytes(void *buf, int nbytes); -+#endif ++static const struct file_operations lrng_raw_regs_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_regs_read, ++}; + -+/* -+ * lrng_get_random_bytes_full() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from a fully initialized LRNG. -+ * -+ * This function will always return random numbers from a fully seeded and -+ * fully initialized LRNG. -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG -+void lrng_get_random_bytes_full(void *buf, int nbytes); -+#endif ++#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + -+/* -+ * lrng_get_random_bytes_min() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from at least a minimally seeded -+ * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C -+ * oversampling applied in FIPS mode is not applied yet). -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ */ -+#ifdef CONFIG_LRNG -+void lrng_get_random_bytes_min(void *buf, int nbytes); -+#endif ++/********************** Raw Entropy Array Data Handling ***********************/ + -+/* -+ * lrng_get_random_bytes_pr() - Provider of cryptographic strong -+ * random numbers for kernel-internal usage from a fully initialized LRNG and -+ * requiring a reseed from the entropy sources before. -+ * -+ * This function will always return random numbers from a fully seeded and -+ * fully initialized LRNG. -+ * -+ * This function is appropriate only for non-atomic use cases as this -+ * function may sleep. It provides access to the full functionality of LRNG -+ * including the switchable DRNG support, that may support other DRNGs such -+ * as the SP800-90A DRBG. -+ * -+ * This call only returns no more data than entropy was pulled from the -+ * entropy sources. Thus, it is likely that this call returns less data -+ * than requested by the caller. Also, the caller MUST be prepared that this -+ * call returns 0 bytes, i.e. it did not generate data. -+ * -+ * @buf: buffer to store the random bytes -+ * @nbytes: size of the buffer -+ * -+ * @return: positive number indicates amount of generated bytes, < 0 on error -+ */ -+#ifdef CONFIG_LRNG -+int lrng_get_random_bytes_pr(void *buf, int nbytes); -+#endif ++#ifdef CONFIG_LRNG_RAW_ARRAY + -+/* -+ * lrng_get_seed() - Fill buffer with data from entropy sources -+ * -+ * This call allows accessing the entropy sources directly and fill the buffer -+ * with data from all available entropy sources. This filled buffer is -+ * identical to the temporary seed buffer used by the LRNG to seed its DRNGs. -+ * -+ * The call is to allows users to seed their DRNG directly from the entropy -+ * sources in case the caller does not want to use the LRNG's DRNGs. This -+ * buffer can be directly used to seed the caller's DRNG from. -+ * -+ * The call blocks as long as one LRNG DRNG is not yet fully seeded. If -+ * LRNG_GET_SEED_NONBLOCK is specified, it does not block in this case, but -+ * returns with an error. -+ * -+ * Considering SP800-90C, there is a differentiation between the seeding -+ * requirements during instantiating a DRNG and at runtime of the DRNG. When -+ * specifying LRNG_GET_SEED_FULLY_SEEDED the caller indicates the DRNG was -+ * already fully seeded and the regular amount of entropy is requested. -+ * Otherwise, the LRNG will obtain the entropy rate required for initial -+ * seeding. The following minimum entropy rates will be obtained: -+ * -+ * * FIPS mode: -+ * * Initial seeding: 384 bits of entropy -+ * * Runtime seeding: 256 bits of entropy -+ * * Non-FIPS mode: -+ * * 128 bits of entropy in any case -+ * -+ * Albeit these are minimum entropy rates, the LRNG tries to request the -+ * given amount of entropy from each entropy source individually. If the -+ * minimum amount of entropy cannot be obtained collectively by all entropy -+ * sources, the LRNG will not fill the buffer. -+ * -+ * The return data in buf is structurally equivalent to the following -+ * definition: -+ * -+ * struct { -+ * u64 seedlen; -+ * u64 entropy_rate; -+ * struct entropy_buf seed; -+ * } __attribute((__packed__)); -+ * -+ * As struct entropy_buf is not known outsize of the LRNG, the LRNG fills -+ * seedlen first with the size of struct entropy_buf. If the caller-provided -+ * buffer buf is smaller than u64, then -EINVAL is returned -+ * and buf is not touched. If it is u64 or larger but smaller -+ * than the size of the structure above, -EMSGSIZE is returned and seedlen -+ * is filled with the size of the buffer. Finally, if buf is large -+ * enough to hold all data, it is filled with the seed data and the seedlen -+ * is set to sizeof(struct entropy_buf). The entropy rate is returned with -+ * the variable entropy_rate and provides the value in bits. -+ * -+ * The seed buffer is the data that should be handed to the caller's DRNG as -+ * seed data. -+ * -+ * @buf [out] Buffer to be filled with data from the entropy sources - note, the -+ * buffer is marked as u64 to ensure it is aligned to 64 bits. -+ * @nbytes [in] Size of the buffer allocated by the caller - this value -+ * provides size of @param buf in bytes. -+ * @flags [in] Flags field to adjust the behavior -+ * -+ * @return -EINVAL or -EMSGSIZE indicating the buffer is too small, -EAGAIN when -+ * the call would block, but NONBLOCK is specified, > 0 the size of -+ * the filled buffer. -+ */ -+#ifdef CONFIG_LRNG -+enum lrng_get_seed_flags { -+ LRNG_GET_SEED_NONBLOCK = 0x0001, /**< Do not block the call */ -+ LRNG_GET_SEED_FULLY_SEEDED = 0x0002, /**< DRNG is fully seeded */ -+}; ++static u32 boot_raw_array = 0; ++module_param(boot_raw_array, uint, 0644); ++MODULE_PARM_DESC(boot_raw_array, "Enable gathering boot time raw noise array data of the first entropy events"); + -+ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags); -+#endif ++static struct lrng_testing lrng_raw_array = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_array.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_array.read_wait) ++}; + -+#endif /* _LRNG_H */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index 265944f14948..e9f0ec6c9aeb 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -6,6 +6,7 @@ - * - * Copyright (C) 1991-2002 Linus Torvalds - */ -+#include - #include - #include - #include -@@ -3586,6 +3587,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) - { - struct rq *rq; - -+ add_sched_randomness(p, cpu); ++bool lrng_raw_array_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_array, value, &boot_raw_array); ++} + - if (!schedstat_enabled()) - return; - --- -2.37.3 - -From 87a1aea3e04d22f00c7eca68492ba64e6cad4efd Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 12 Sep 2022 11:33:01 +0200 -Subject: [PATCH 10/13] per-VMA locks proposal - -Signed-off-by: Peter Jung ---- - arch/arm64/Kconfig | 1 + - arch/arm64/mm/fault.c | 36 +++++++++ - arch/powerpc/mm/fault.c | 41 ++++++++++ - arch/powerpc/platforms/powernv/Kconfig | 1 + - arch/powerpc/platforms/pseries/Kconfig | 1 + - arch/x86/Kconfig | 1 + - arch/x86/mm/fault.c | 36 +++++++++ - drivers/gpu/drm/i915/i915_gpu_error.c | 4 +- - fs/proc/task_mmu.c | 1 + - fs/userfaultfd.c | 6 ++ - include/linux/mm.h | 104 ++++++++++++++++++++++++- - include/linux/mm_types.h | 33 ++++++-- - include/linux/mmap_lock.h | 37 ++++++--- - include/linux/vm_event_item.h | 6 ++ - include/linux/vmstat.h | 6 ++ - kernel/fork.c | 75 +++++++++++++++++- - mm/Kconfig | 13 ++++ - mm/Kconfig.debug | 8 ++ - mm/init-mm.c | 6 ++ - mm/internal.h | 4 +- - mm/khugepaged.c | 1 + - mm/madvise.c | 1 + - mm/memory.c | 82 ++++++++++++++++--- - mm/mempolicy.c | 6 +- - mm/mlock.c | 2 + - mm/mmap.c | 60 ++++++++++---- - mm/mprotect.c | 1 + - mm/mremap.c | 1 + - mm/nommu.c | 2 + - mm/oom_kill.c | 3 +- - mm/vmstat.c | 6 ++ - 31 files changed, 531 insertions(+), 54 deletions(-) - -diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig -index 1ce7685ad5de..fb11380b49dd 100644 ---- a/arch/arm64/Kconfig -+++ b/arch/arm64/Kconfig -@@ -93,6 +93,7 @@ config ARM64 - select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 - select ARCH_SUPPORTS_NUMA_BALANCING - select ARCH_SUPPORTS_PAGE_TABLE_CHECK -+ select ARCH_SUPPORTS_PER_VMA_LOCK - select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT - select ARCH_WANT_DEFAULT_BPF_JIT - select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT -diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c -index c33f1fad2745..f05ce40ff32b 100644 ---- a/arch/arm64/mm/fault.c -+++ b/arch/arm64/mm/fault.c -@@ -525,6 +525,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, - unsigned long vm_flags; - unsigned int mm_flags = FAULT_FLAG_DEFAULT; - unsigned long addr = untagged_addr(far); -+#ifdef CONFIG_PER_VMA_LOCK -+ struct vm_area_struct *vma; -+#endif - - if (kprobe_page_fault(regs, esr)) - return 0; -@@ -575,6 +578,36 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, - - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); - -+#ifdef CONFIG_PER_VMA_LOCK -+ if (!(mm_flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) -+ goto lock_mmap; ++static int lrng_raw_array_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_array, &boot_raw_array, outbuf, ++ outbuflen); ++} + -+ vma = find_and_lock_anon_vma(mm, addr); -+ if (!vma) -+ goto lock_mmap; ++static ssize_t lrng_raw_array_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_array_entropy_reader); ++} + -+ if (!(vma->vm_flags & vm_flags)) { -+ vma_read_unlock(vma); -+ goto lock_mmap; -+ } -+ fault = handle_mm_fault(vma, addr & PAGE_MASK, -+ mm_flags | FAULT_FLAG_VMA_LOCK, regs); -+ vma_read_unlock(vma); ++static const struct file_operations lrng_raw_array_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_array_read, ++}; + -+ if (!(fault & VM_FAULT_RETRY)) { -+ count_vm_vma_lock_event(VMA_LOCK_SUCCESS); -+ goto done; -+ } -+ count_vm_vma_lock_event(VMA_LOCK_RETRY); ++#endif /* CONFIG_LRNG_RAW_ARRAY */ + -+ /* Quick path to respond to signals */ -+ if (fault_signal_pending(fault, regs)) { -+ if (!user_mode(regs)) -+ goto no_context; -+ return 0; -+ } -+lock_mmap: -+#endif /* CONFIG_PER_VMA_LOCK */ - /* - * As per x86, we may deadlock here. However, since the kernel only - * validly references user space from well defined areas of the code, -@@ -618,6 +651,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, - } - mmap_read_unlock(mm); - -+#ifdef CONFIG_PER_VMA_LOCK -+done: -+#endif - /* - * Handle the "normal" (no error) case first. - */ -diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c -index 014005428687..c92bdfcd1796 100644 ---- a/arch/powerpc/mm/fault.c -+++ b/arch/powerpc/mm/fault.c -@@ -450,6 +450,44 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, - if (is_exec) - flags |= FAULT_FLAG_INSTRUCTION; - -+#ifdef CONFIG_PER_VMA_LOCK -+ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) -+ goto lock_mmap; -+ -+ vma = find_and_lock_anon_vma(mm, address); -+ if (!vma) -+ goto lock_mmap; ++/******************** Interrupt Performance Data Handling *********************/ + -+ if (unlikely(access_pkey_error(is_write, is_exec, -+ (error_code & DSISR_KEYFAULT), vma))) { -+ int rc = bad_access_pkey(regs, address, vma); ++#ifdef CONFIG_LRNG_IRQ_PERF + -+ vma_read_unlock(vma); -+ return rc; -+ } ++static u32 boot_irq_perf = 0; ++module_param(boot_irq_perf, uint, 0644); ++MODULE_PARM_DESC(boot_irq_perf, "Enable gathering interrupt entropy source performance data"); + -+ if (unlikely(access_error(is_write, is_exec, vma))) { -+ int rc = bad_access(regs, address); ++static struct lrng_testing lrng_irq_perf = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_irq_perf.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_irq_perf.read_wait) ++}; + -+ vma_read_unlock(vma); -+ return rc; -+ } ++bool lrng_perf_time(u32 start) ++{ ++ return lrng_testing_store(&lrng_irq_perf, random_get_entropy() - start, ++ &boot_irq_perf); ++} + -+ fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); -+ vma_read_unlock(vma); ++static int lrng_irq_perf_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_irq_perf, &boot_irq_perf, outbuf, ++ outbuflen); ++} + -+ if (!(fault & VM_FAULT_RETRY)) { -+ count_vm_vma_lock_event(VMA_LOCK_SUCCESS); -+ goto done; -+ } -+ count_vm_vma_lock_event(VMA_LOCK_RETRY); ++static ssize_t lrng_irq_perf_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) ++{ ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_irq_perf_reader); ++} + -+ if (fault_signal_pending(fault, regs)) -+ return user_mode(regs) ? 0 : SIGBUS; ++static const struct file_operations lrng_irq_perf_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_irq_perf_read, ++}; + -+lock_mmap: -+#endif /* CONFIG_PER_VMA_LOCK */ ++#endif /* CONFIG_LRNG_IRQ_PERF */ + - /* When running in the kernel we expect faults to occur only to - * addresses in user space. All other faults represent errors in the - * kernel and should generate an OOPS. Unfortunately, in the case of an -@@ -526,6 +564,9 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, - - mmap_read_unlock(current->mm); - -+#ifdef CONFIG_PER_VMA_LOCK -+done: -+#endif - if (unlikely(fault & VM_FAULT_ERROR)) - return mm_fault_error(regs, address, fault); - -diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig -index ae248a161b43..70a46acc70d6 100644 ---- a/arch/powerpc/platforms/powernv/Kconfig -+++ b/arch/powerpc/platforms/powernv/Kconfig -@@ -16,6 +16,7 @@ config PPC_POWERNV - select PPC_DOORBELL - select MMU_NOTIFIER - select FORCE_SMP -+ select ARCH_SUPPORTS_PER_VMA_LOCK - default y - - config OPAL_PRD -diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig -index fb6499977f99..7d13a2de3475 100644 ---- a/arch/powerpc/platforms/pseries/Kconfig -+++ b/arch/powerpc/platforms/pseries/Kconfig -@@ -21,6 +21,7 @@ config PPC_PSERIES - select HOTPLUG_CPU - select FORCE_SMP - select SWIOTLB -+ select ARCH_SUPPORTS_PER_VMA_LOCK - default y - - config PARAVIRT_SPINLOCKS -diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index 674d694a665e..fa41ba3e880f 100644 ---- a/arch/x86/Kconfig -+++ b/arch/x86/Kconfig -@@ -27,6 +27,7 @@ config X86_64 - # Options that are inherently 64-bit kernel only: - select ARCH_HAS_GIGANTIC_PAGE - select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 -+ select ARCH_SUPPORTS_PER_VMA_LOCK - select ARCH_USE_CMPXCHG_LOCKREF - select HAVE_ARCH_SOFT_DIRTY - select MODULES_USE_ELF_RELA -diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c -index c4bbd8c66134..81458c2cf4d9 100644 ---- a/arch/x86/mm/fault.c -+++ b/arch/x86/mm/fault.c -@@ -19,6 +19,7 @@ - #include /* faulthandler_disabled() */ - #include /* efi_crash_gracefully_on_page_fault()*/ - #include -+#include /* find_and_lock_vma() */ - - #include /* boot_cpu_has, ... */ - #include /* dotraplinkage, ... */ -@@ -1323,6 +1324,38 @@ void do_user_addr_fault(struct pt_regs *regs, - } - #endif - -+#ifdef CONFIG_PER_VMA_LOCK -+ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) -+ goto lock_mmap; ++/****** Raw High-Resolution Scheduler-based Timer Entropy Data Handling *******/ + -+ vma = find_and_lock_anon_vma(mm, address); -+ if (!vma) -+ goto lock_mmap; ++#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY + -+ if (unlikely(access_error(error_code, vma))) { -+ vma_read_unlock(vma); -+ goto lock_mmap; -+ } -+ fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs); -+ vma_read_unlock(vma); ++static u32 boot_raw_sched_hires_test = 0; ++module_param(boot_raw_sched_hires_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_hires_test, "Enable gathering boot time high resolution timer entropy of the first Scheduler-based entropy events"); + -+ if (!(fault & VM_FAULT_RETRY)) { -+ count_vm_vma_lock_event(VMA_LOCK_SUCCESS); -+ goto done; -+ } -+ count_vm_vma_lock_event(VMA_LOCK_RETRY); ++static struct lrng_testing lrng_raw_sched_hires = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_hires.lock), ++ .read_wait = ++ __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_hires.read_wait) ++}; + -+ /* Quick path to respond to signals */ -+ if (fault_signal_pending(fault, regs)) { -+ if (!user_mode(regs)) -+ kernelmode_fixup_or_oops(regs, error_code, address, -+ SIGBUS, BUS_ADRERR, -+ ARCH_DEFAULT_PKEY); -+ return; -+ } -+lock_mmap: -+#endif /* CONFIG_PER_VMA_LOCK */ ++bool lrng_raw_sched_hires_entropy_store(u32 value) ++{ ++ return lrng_testing_store(&lrng_raw_sched_hires, value, ++ &boot_raw_sched_hires_test); ++} + - /* - * Kernel-mode access to the user address space should only occur - * on well-defined single instructions listed in the exception -@@ -1423,6 +1456,9 @@ void do_user_addr_fault(struct pt_regs *regs, - } - - mmap_read_unlock(mm); -+#ifdef CONFIG_PER_VMA_LOCK -+done: -+#endif - if (likely(!(fault & VM_FAULT_ERROR))) - return; - -diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c -index 32e92651ef7c..fc94985c95c8 100644 ---- a/drivers/gpu/drm/i915/i915_gpu_error.c -+++ b/drivers/gpu/drm/i915/i915_gpu_error.c -@@ -507,7 +507,7 @@ static void error_print_context(struct drm_i915_error_state_buf *m, - } - - static struct i915_vma_coredump * --__find_vma(struct i915_vma_coredump *vma, const char *name) -+__i915_find_vma(struct i915_vma_coredump *vma, const char *name) - { - while (vma) { - if (strcmp(vma->name, name) == 0) -@@ -521,7 +521,7 @@ __find_vma(struct i915_vma_coredump *vma, const char *name) - struct i915_vma_coredump * - intel_gpu_error_find_batch(const struct intel_engine_coredump *ee) - { -- return __find_vma(ee->vma, "batch"); -+ return __i915_find_vma(ee->vma, "batch"); - } - - static void error_print_engine(struct drm_i915_error_state_buf *m, -diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c -index 4e0023643f8b..ceffa5c2c650 100644 ---- a/fs/proc/task_mmu.c -+++ b/fs/proc/task_mmu.c -@@ -1285,6 +1285,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (!(vma->vm_flags & VM_SOFTDIRTY)) - continue; -+ vma_mark_locked(vma); - vma->vm_flags &= ~VM_SOFTDIRTY; - vma_set_page_prot(vma); - } -diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c -index 175de70e3adf..fe557b3d1c07 100644 ---- a/fs/userfaultfd.c -+++ b/fs/userfaultfd.c -@@ -620,6 +620,7 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, - mmap_write_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) - if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { -+ vma_mark_locked(vma); - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - } -@@ -653,6 +654,7 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) - - octx = vma->vm_userfaultfd_ctx.ctx; - if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) { -+ vma_mark_locked(vma); - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - return 0; -@@ -734,6 +736,7 @@ void mremap_userfaultfd_prep(struct vm_area_struct *vma, - atomic_inc(&ctx->mmap_changing); - } else { - /* Drop uffd context if remap feature not enabled */ -+ vma_mark_locked(vma); - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vma->vm_flags &= ~__VM_UFFD_FLAGS; - } -@@ -891,6 +894,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) - vma = prev; - else - prev = vma; -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - } -@@ -1449,6 +1453,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, - * the next vma was merged into the current one and - * the current one has not been updated yet. - */ -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx.ctx = ctx; - -@@ -1630,6 +1635,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, - * the next vma was merged into the current one and - * the current one has not been updated yet. - */ -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - -diff --git a/include/linux/mm.h b/include/linux/mm.h -index 6c61497f0167..f15e26670abf 100644 ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -249,6 +249,7 @@ void setup_initial_init_mm(void *start_code, void *end_code, - struct vm_area_struct *vm_area_alloc(struct mm_struct *); - struct vm_area_struct *vm_area_dup(struct vm_area_struct *); - void vm_area_free(struct vm_area_struct *); -+void drain_free_vmas(struct mm_struct *mm); - - #ifndef CONFIG_MMU - extern struct rb_root nommu_region_tree; -@@ -466,7 +467,8 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags) - { FAULT_FLAG_USER, "USER" }, \ - { FAULT_FLAG_REMOTE, "REMOTE" }, \ - { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ -- { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" } -+ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ -+ { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } - - /* - * vm_fault is filled by the pagefault handler and passed to the vma's -@@ -611,6 +613,94 @@ struct vm_operations_struct { - unsigned long addr); - }; - -+#ifdef CONFIG_PER_VMA_LOCK -+static inline void vma_init_lock(struct vm_area_struct *vma) ++static int lrng_raw_sched_hires_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ init_rwsem(&vma->lock); -+ vma->vm_lock_seq = -1; ++ return lrng_testing_reader(&lrng_raw_sched_hires, ++ &boot_raw_sched_hires_test, ++ outbuf, outbuflen); +} + -+static inline void vma_mark_locked(struct vm_area_struct *vma) ++static ssize_t lrng_raw_sched_hires_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ -+ int mm_lock_seq; ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_hires_entropy_reader); ++} + -+ mmap_assert_write_locked(vma->vm_mm); ++static const struct file_operations lrng_raw_sched_hires_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_hires_read, ++}; + -+ /* -+ * current task is holding mmap_write_lock, both vma->vm_lock_seq and -+ * mm->mm_lock_seq can't be concurrently modified. -+ */ -+ mm_lock_seq = READ_ONCE(vma->vm_mm->mm_lock_seq); -+ if (vma->vm_lock_seq == mm_lock_seq) -+ return; ++#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ + -+ down_write(&vma->lock); -+ vma->vm_lock_seq = mm_lock_seq; -+ up_write(&vma->lock); -+} ++/******************** Interrupt Performance Data Handling *********************/ + -+static inline bool vma_read_trylock(struct vm_area_struct *vma) ++#ifdef CONFIG_LRNG_SCHED_PERF ++ ++static u32 boot_sched_perf = 0; ++module_param(boot_sched_perf, uint, 0644); ++MODULE_PARM_DESC(boot_sched_perf, "Enable gathering scheduler-based entropy source performance data"); ++ ++static struct lrng_testing lrng_sched_perf = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_sched_perf.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_sched_perf.read_wait) ++}; ++ ++bool lrng_sched_perf_time(u32 start) +{ -+ if (unlikely(down_read_trylock(&vma->lock) == 0)) -+ return false; ++ return lrng_testing_store(&lrng_sched_perf, random_get_entropy() - start, ++ &boot_sched_perf); ++} + -+ /* -+ * Overflow might produce false locked result but it's not critical. -+ * False unlocked result is critical but is impossible because we -+ * modify and check vma->vm_lock_seq under vma->lock protection and -+ * mm->mm_lock_seq modification invalidates all existing locks. -+ */ -+ if (vma->vm_lock_seq == READ_ONCE(vma->vm_mm->mm_lock_seq)) { -+ up_read(&vma->lock); -+ return false; -+ } -+ return true; ++static int lrng_sched_perf_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_sched_perf, &boot_sched_perf, outbuf, ++ outbuflen); +} + -+static inline void vma_read_unlock(struct vm_area_struct *vma) ++static ssize_t lrng_sched_perf_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ -+ up_read(&vma->lock); ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_sched_perf_reader); +} + -+static inline void vma_assert_locked(struct vm_area_struct *vma) ++static const struct file_operations lrng_sched_perf_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_sched_perf_read, ++}; ++ ++#endif /* CONFIG_LRNG_SCHED_PERF */ ++ ++/*************** Raw Scheduler task_struct->pid Data Handling *****************/ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++ ++static u32 boot_raw_sched_pid_test = 0; ++module_param(boot_raw_sched_pid_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_pid_test, "Enable gathering boot time entropy of the first PIDs collected by the scheduler entropy source"); ++ ++static struct lrng_testing lrng_raw_sched_pid = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_pid.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_pid.read_wait) ++}; ++ ++bool lrng_raw_sched_pid_entropy_store(u32 value) +{ -+ lockdep_assert_held(&vma->lock); -+ VM_BUG_ON_VMA(!rwsem_is_locked(&vma->lock), vma); ++ return lrng_testing_store(&lrng_raw_sched_pid, value, ++ &boot_raw_sched_pid_test); +} + -+static inline void vma_assert_write_locked(struct vm_area_struct *vma, int pos) ++static int lrng_raw_sched_pid_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ mmap_assert_write_locked(vma->vm_mm); -+ /* -+ * current task is holding mmap_write_lock, both vma->vm_lock_seq and -+ * mm->mm_lock_seq can't be concurrently modified. -+ */ -+ VM_BUG_ON_VMA(vma->vm_lock_seq != READ_ONCE(vma->vm_mm->mm_lock_seq), vma); ++ return lrng_testing_reader(&lrng_raw_sched_pid, ++ &boot_raw_sched_pid_test, outbuf, outbuflen); +} + -+static inline void vma_assert_no_reader(struct vm_area_struct *vma) ++static ssize_t lrng_raw_sched_pid_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ -+ VM_BUG_ON_VMA(rwsem_is_locked(&vma->lock) && -+ vma->vm_lock_seq != READ_ONCE(vma->vm_mm->mm_lock_seq), -+ vma); ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_pid_entropy_reader); +} + -+struct vm_area_struct *find_and_lock_anon_vma(struct mm_struct *mm, -+ unsigned long address); ++static const struct file_operations lrng_raw_sched_pid_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_pid_read, ++}; + -+#else /* CONFIG_PER_VMA_LOCK */ ++#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ + -+static inline void vma_init_lock(struct vm_area_struct *vma) {} -+static inline void vma_mark_locked(struct vm_area_struct *vma) {} -+static inline bool vma_read_trylock(struct vm_area_struct *vma) -+ { return false; } -+static inline void vma_read_unlock(struct vm_area_struct *vma) {} -+static inline void vma_assert_locked(struct vm_area_struct *vma) {} -+static inline void vma_assert_write_locked(struct vm_area_struct *vma, int pos) {} -+static inline void vma_assert_no_reader(struct vm_area_struct *vma) {} + -+#endif /* CONFIG_PER_VMA_LOCK */ ++/*********** Raw Scheduler task_struct->start_time Data Handling **************/ + - static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) - { - static const struct vm_operations_struct dummy_vm_ops = {}; -@@ -619,6 +709,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) - vma->vm_mm = mm; - vma->vm_ops = &dummy_vm_ops; - INIT_LIST_HEAD(&vma->anon_vma_chain); -+ vma_init_lock(vma); - } - - static inline void vma_set_anonymous(struct vm_area_struct *vma) -@@ -1801,7 +1892,7 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, - void zap_page_range(struct vm_area_struct *vma, unsigned long address, - unsigned long size); - void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, -- unsigned long start, unsigned long end); -+ unsigned long start, unsigned long end, bool lock_vma); - - struct mmu_notifier_range; - -@@ -2717,7 +2808,14 @@ extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); - #endif - - /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ --extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); -+extern struct vm_area_struct *__find_vma(struct mm_struct *mm, unsigned long addr); -+static inline -+struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) ++#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++ ++static u32 boot_raw_sched_starttime_test = 0; ++module_param(boot_raw_sched_starttime_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_starttime_test, "Enable gathering boot time entropy of the first task start times collected by the scheduler entropy source"); ++ ++static struct lrng_testing lrng_raw_sched_starttime = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_starttime.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_starttime.read_wait) ++}; ++ ++bool lrng_raw_sched_starttime_entropy_store(u32 value) +{ -+ mmap_assert_locked(mm); -+ return __find_vma(mm, addr); ++ return lrng_testing_store(&lrng_raw_sched_starttime, value, ++ &boot_raw_sched_starttime_test); +} + - extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, - struct vm_area_struct **pprev); - -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 15afe5e0c4c2..3a4b9b5e534d 100644 ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -403,12 +403,22 @@ struct anon_vma_name { - struct vm_area_struct { - /* The first cache line has the info for VMA tree walking. */ - -- unsigned long vm_start; /* Our start address within vm_mm. */ -- unsigned long vm_end; /* The first byte after our end address -- within vm_mm. */ -+ union { -+ struct { -+ /* VMA covers [vm_start; vm_end) addresses within mm */ -+ unsigned long vm_start, vm_end; - -- /* linked list of VM areas per task, sorted by address */ -- struct vm_area_struct *vm_next, *vm_prev; -+ /* linked list of VMAs per task, sorted by address */ -+ struct vm_area_struct *vm_next, *vm_prev; -+ }; -+#ifdef CONFIG_PER_VMA_LOCK -+ struct { -+ struct list_head vm_free_list; -+ /* Used for deferred freeing. */ -+ struct rcu_head vm_rcu; -+ }; -+#endif -+ }; - - struct rb_node vm_rb; - -@@ -480,6 +490,10 @@ struct vm_area_struct { - struct mempolicy *vm_policy; /* NUMA policy for the VMA */ - #endif - struct vm_userfaultfd_ctx vm_userfaultfd_ctx; -+#ifdef CONFIG_PER_VMA_LOCK -+ struct rw_semaphore lock; -+ int vm_lock_seq; -+#endif - } __randomize_layout; - - struct kioctx_table; -@@ -561,6 +575,14 @@ struct mm_struct { - * init_mm.mmlist, and are protected - * by mmlist_lock - */ -+#ifdef CONFIG_PER_VMA_LOCK -+ int mm_lock_seq; -+ struct { -+ struct list_head head; -+ spinlock_t lock; -+ int size; -+ } vma_free_list; -+#endif - - - unsigned long hiwater_rss; /* High-watermark of RSS usage */ -@@ -954,6 +976,7 @@ enum fault_flag { - FAULT_FLAG_INTERRUPTIBLE = 1 << 9, - FAULT_FLAG_UNSHARE = 1 << 10, - FAULT_FLAG_ORIG_PTE_VALID = 1 << 11, -+ FAULT_FLAG_VMA_LOCK = 1 << 12, - }; - - typedef unsigned int __bitwise zap_flags_t; -diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h -index 96e113e23d04..a391ae226564 100644 ---- a/include/linux/mmap_lock.h -+++ b/include/linux/mmap_lock.h -@@ -60,6 +60,29 @@ static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) - - #endif /* CONFIG_TRACING */ - -+static inline void mmap_assert_locked(struct mm_struct *mm) ++static int lrng_raw_sched_starttime_entropy_reader(u8 *outbuf, u32 outbuflen) +{ -+ lockdep_assert_held(&mm->mmap_lock); -+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); ++ return lrng_testing_reader(&lrng_raw_sched_starttime, ++ &boot_raw_sched_starttime_test, outbuf, outbuflen); +} + -+static inline void mmap_assert_write_locked(struct mm_struct *mm) ++static ssize_t lrng_raw_sched_starttime_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ -+ lockdep_assert_held_write(&mm->mmap_lock); -+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_starttime_entropy_reader); +} + -+#ifdef CONFIG_PER_VMA_LOCK -+static inline void vma_mark_unlocked_all(struct mm_struct *mm) ++static const struct file_operations lrng_raw_sched_starttime_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_starttime_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++ ++/************** Raw Scheduler task_struct->nvcsw Data Handling ****************/ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++ ++static u32 boot_raw_sched_nvcsw_test = 0; ++module_param(boot_raw_sched_nvcsw_test, uint, 0644); ++MODULE_PARM_DESC(boot_raw_sched_nvcsw_test, "Enable gathering boot time entropy of the first task context switch numbers collected by the scheduler entropy source"); ++ ++static struct lrng_testing lrng_raw_sched_nvcsw = { ++ .rb_reader = 0, ++ .rb_writer = ATOMIC_INIT(0), ++ .lock = __SPIN_LOCK_UNLOCKED(lrng_raw_sched_nvcsw.lock), ++ .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(lrng_raw_sched_nvcsw.read_wait) ++}; ++ ++bool lrng_raw_sched_nvcsw_entropy_store(u32 value) +{ -+ mmap_assert_write_locked(mm); -+ /* No races during update due to exclusive mmap_lock being held */ -+ WRITE_ONCE(mm->mm_lock_seq, mm->mm_lock_seq + 1); ++ return lrng_testing_store(&lrng_raw_sched_nvcsw, value, ++ &boot_raw_sched_nvcsw_test); +} -+#else -+static inline void vma_mark_unlocked_all(struct mm_struct *mm) {} -+#endif + - static inline void mmap_init_lock(struct mm_struct *mm) - { - init_rwsem(&mm->mmap_lock); -@@ -102,12 +125,14 @@ static inline bool mmap_write_trylock(struct mm_struct *mm) - static inline void mmap_write_unlock(struct mm_struct *mm) - { - __mmap_lock_trace_released(mm, true); -+ vma_mark_unlocked_all(mm); - up_write(&mm->mmap_lock); - } - - static inline void mmap_write_downgrade(struct mm_struct *mm) - { - __mmap_lock_trace_acquire_returned(mm, false, true); -+ vma_mark_unlocked_all(mm); - downgrade_write(&mm->mmap_lock); - } - -@@ -150,18 +175,6 @@ static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) - up_read_non_owner(&mm->mmap_lock); - } - --static inline void mmap_assert_locked(struct mm_struct *mm) --{ -- lockdep_assert_held(&mm->mmap_lock); -- VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); --} -- --static inline void mmap_assert_write_locked(struct mm_struct *mm) --{ -- lockdep_assert_held_write(&mm->mmap_lock); -- VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); --} -- - static inline int mmap_lock_is_contended(struct mm_struct *mm) - { - return rwsem_is_contended(&mm->mmap_lock); -diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h -index f3fc36cd2276..a325783ed05d 100644 ---- a/include/linux/vm_event_item.h -+++ b/include/linux/vm_event_item.h -@@ -150,6 +150,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, - #ifdef CONFIG_X86 - DIRECT_MAP_LEVEL2_SPLIT, - DIRECT_MAP_LEVEL3_SPLIT, -+#endif -+#ifdef CONFIG_PER_VMA_LOCK_STATS -+ VMA_LOCK_SUCCESS, -+ VMA_LOCK_ABORT, -+ VMA_LOCK_RETRY, -+ VMA_LOCK_MISS, - #endif - NR_VM_EVENT_ITEMS - }; -diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h -index bfe38869498d..0c2611899cfc 100644 ---- a/include/linux/vmstat.h -+++ b/include/linux/vmstat.h -@@ -131,6 +131,12 @@ static inline void vm_events_fold_cpu(int cpu) - #define count_vm_vmacache_event(x) do {} while (0) - #endif - -+#ifdef CONFIG_PER_VMA_LOCK_STATS -+#define count_vm_vma_lock_event(x) count_vm_event(x) -+#else -+#define count_vm_vma_lock_event(x) do {} while (0) -+#endif ++static int lrng_raw_sched_nvcsw_entropy_reader(u8 *outbuf, u32 outbuflen) ++{ ++ return lrng_testing_reader(&lrng_raw_sched_nvcsw, ++ &boot_raw_sched_nvcsw_test, outbuf, outbuflen); ++} + - #define __count_zid_vm_events(item, zid, delta) \ - __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) - -diff --git a/kernel/fork.c b/kernel/fork.c -index b15aaae04403..ac29435e00ad 100644 ---- a/kernel/fork.c -+++ b/kernel/fork.c -@@ -479,18 +479,83 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) - */ - *new = data_race(*orig); - INIT_LIST_HEAD(&new->anon_vma_chain); -+ vma_init_lock(new); - new->vm_next = new->vm_prev = NULL; - dup_anon_vma_name(orig, new); - } - return new; - } - -+#ifdef CONFIG_PER_VMA_LOCK -+static inline void __vm_area_free(struct vm_area_struct *vma) ++static ssize_t lrng_raw_sched_nvcsw_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ -+ /* The vma should either have no lock holders or be write-locked. */ -+ vma_assert_no_reader(vma); -+ kmem_cache_free(vm_area_cachep, vma); ++ return lrng_testing_extract_user(file, to, count, ppos, ++ lrng_raw_sched_nvcsw_entropy_reader); +} + -+static void vma_free_rcu_callback(struct rcu_head *head) ++static const struct file_operations lrng_raw_sched_nvcsw_fops = { ++ .owner = THIS_MODULE, ++ .read = lrng_raw_sched_nvcsw_read, ++}; ++ ++#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ ++ ++/*********************************** ACVT ************************************/ ++ ++#ifdef CONFIG_LRNG_ACVT_HASH ++ ++/* maximum amount of data to be hashed as defined by ACVP */ ++#define LRNG_ACVT_MAX_SHA_MSG (65536 >> 3) ++ ++/* ++ * As we use static variables to store the data, it is clear that the ++ * test interface is only able to handle single threaded testing. This is ++ * considered to be sufficient for testing. If multi-threaded use of the ++ * ACVT test interface would be performed, the caller would get garbage ++ * but the kernel operation is unaffected by this. ++ */ ++static u8 lrng_acvt_hash_data[LRNG_ACVT_MAX_SHA_MSG] ++ __aligned(LRNG_KCAPI_ALIGN); ++static atomic_t lrng_acvt_hash_data_size = ATOMIC_INIT(0); ++static u8 lrng_acvt_hash_digest[LRNG_ATOMIC_DIGEST_SIZE]; ++ ++static ssize_t lrng_acvt_hash_write(struct file *file, const char __user *buf, ++ size_t nbytes, loff_t *ppos) +{ -+ struct vm_area_struct *first_vma; -+ struct vm_area_struct *vma, *vma2; ++ if (nbytes > LRNG_ACVT_MAX_SHA_MSG) ++ return -EINVAL; ++ ++ atomic_set(&lrng_acvt_hash_data_size, (int)nbytes); + -+ first_vma = container_of(head, struct vm_area_struct, vm_rcu); -+ list_for_each_entry_safe(vma, vma2, &first_vma->vm_free_list, vm_free_list) -+ __vm_area_free(vma); -+ __vm_area_free(first_vma); ++ return simple_write_to_buffer(lrng_acvt_hash_data, ++ LRNG_ACVT_MAX_SHA_MSG, ppos, buf, nbytes); +} + -+void drain_free_vmas(struct mm_struct *mm) ++static ssize_t lrng_acvt_hash_read(struct file *file, char __user *to, ++ size_t count, loff_t *ppos) +{ -+ struct vm_area_struct *first_vma; -+ LIST_HEAD(to_destroy); -+ -+ spin_lock(&mm->vma_free_list.lock); -+ list_splice_init(&mm->vma_free_list.head, &to_destroy); -+ mm->vma_free_list.size = 0; -+ spin_unlock(&mm->vma_free_list.lock); ++ SHASH_DESC_ON_STACK(shash, NULL); ++ const struct lrng_hash_cb *hash_cb = &lrng_sha_hash_cb; ++ ssize_t ret; + -+ if (list_empty(&to_destroy)) -+ return; ++ if (count > LRNG_ATOMIC_DIGEST_SIZE) ++ return -EINVAL; + -+ first_vma = list_first_entry(&to_destroy, struct vm_area_struct, vm_free_list); -+ /* Remove the head which is allocated on the stack */ -+ list_del(&to_destroy); ++ ret = hash_cb->hash_init(shash, NULL) ?: ++ hash_cb->hash_update(shash, lrng_acvt_hash_data, ++ atomic_read_u32(&lrng_acvt_hash_data_size)) ?: ++ hash_cb->hash_final(shash, lrng_acvt_hash_digest); ++ if (ret) ++ return ret; + -+ call_rcu(&first_vma->vm_rcu, vma_free_rcu_callback); ++ return simple_read_from_buffer(to, count, ppos, lrng_acvt_hash_digest, ++ sizeof(lrng_acvt_hash_digest)); +} + -+#define VM_AREA_FREE_LIST_MAX 32 ++static const struct file_operations lrng_acvt_hash_fops = { ++ .owner = THIS_MODULE, ++ .open = simple_open, ++ .llseek = default_llseek, ++ .read = lrng_acvt_hash_read, ++ .write = lrng_acvt_hash_write, ++}; ++ ++#endif /* CONFIG_LRNG_ACVT_DRNG */ ++ ++/************************************************************************** ++ * Debugfs interface ++ **************************************************************************/ + -+void vm_area_free(struct vm_area_struct *vma) ++static int __init lrng_raw_init(void) +{ -+ struct mm_struct *mm = vma->vm_mm; -+ bool drain; ++ struct dentry *lrng_raw_debugfs_root; + -+ free_anon_vma_name(vma); ++ lrng_raw_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); + -+ spin_lock(&mm->vma_free_list.lock); -+ list_add(&vma->vm_free_list, &mm->vma_free_list.head); -+ mm->vma_free_list.size++; -+ drain = mm->vma_free_list.size > VM_AREA_FREE_LIST_MAX; -+ spin_unlock(&mm->vma_free_list.lock); ++#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_hires", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_hires_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_jiffies", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_jiffies_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_irq", 0400, lrng_raw_debugfs_root, ++ NULL, &lrng_raw_irq_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_retip", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_retip_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_regs", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_regs_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_ARRAY ++ debugfs_create_file_unsafe("lrng_raw_array", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_array_fops); ++#endif ++#ifdef CONFIG_LRNG_IRQ_PERF ++ debugfs_create_file_unsafe("lrng_irq_perf", 0400, lrng_raw_debugfs_root, ++ NULL, &lrng_irq_perf_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_hires", 0400, ++ lrng_raw_debugfs_root, ++ NULL, &lrng_raw_sched_hires_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_pid", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_sched_pid_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_starttime", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_sched_starttime_fops); ++#endif ++#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++ debugfs_create_file_unsafe("lrng_raw_sched_nvcsw", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_raw_sched_nvcsw_fops); ++#endif ++#ifdef CONFIG_LRNG_SCHED_PERF ++ debugfs_create_file_unsafe("lrng_sched_perf", 0400, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_sched_perf_fops); ++#endif ++#ifdef CONFIG_LRNG_ACVT_HASH ++ debugfs_create_file_unsafe("lrng_acvt_hash", 0600, ++ lrng_raw_debugfs_root, NULL, ++ &lrng_acvt_hash_fops); ++#endif + -+ if (drain) -+ drain_free_vmas(mm); ++ return 0; +} + -+#else /* CONFIG_PER_VMA_LOCK */ ++module_init(lrng_raw_init); +diff --git a/drivers/char/lrng/lrng_testing.h b/drivers/char/lrng/lrng_testing.h +new file mode 100644 +index 000000000000..672a7fca4595 +--- /dev/null ++++ b/drivers/char/lrng/lrng_testing.h +@@ -0,0 +1,85 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ + -+void drain_free_vmas(struct mm_struct *mm) {} ++#ifndef _LRNG_TESTING_H ++#define _LRNG_TESTING_H + - void vm_area_free(struct vm_area_struct *vma) - { - free_anon_vma_name(vma); - kmem_cache_free(vm_area_cachep, vma); - } - -+#endif /* CONFIG_PER_VMA_LOCK */ ++#ifdef CONFIG_LRNG_RAW_HIRES_ENTROPY ++bool lrng_raw_hires_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ ++static inline bool lrng_raw_hires_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_HIRES_ENTROPY */ + - static void account_kernel_stack(struct task_struct *tsk, int account) - { - if (IS_ENABLED(CONFIG_VMAP_STACK)) { -@@ -699,8 +764,10 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, - rb_parent = &tmp->vm_rb; - - mm->map_count++; -- if (!(tmp->vm_flags & VM_WIPEONFORK)) -+ if (!(tmp->vm_flags & VM_WIPEONFORK)) { -+ vma_mark_locked(mpnt); - retval = copy_page_range(tmp, mpnt); -+ } - - if (tmp->vm_ops && tmp->vm_ops->open) - tmp->vm_ops->open(tmp); -@@ -1121,6 +1188,12 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - seqcount_init(&mm->write_protect_seq); - mmap_init_lock(mm); - INIT_LIST_HEAD(&mm->mmlist); -+#ifdef CONFIG_PER_VMA_LOCK -+ WRITE_ONCE(mm->mm_lock_seq, 0); -+ INIT_LIST_HEAD(&mm->vma_free_list.head); -+ spin_lock_init(&mm->vma_free_list.lock); -+ mm->vma_free_list.size = 0; -+#endif - mm_pgtables_bytes_init(mm); - mm->map_count = 0; - mm->locked_vm = 0; -diff --git a/mm/Kconfig b/mm/Kconfig -index 96cd3ae25c6f..6c8f47269dbd 100644 ---- a/mm/Kconfig -+++ b/mm/Kconfig -@@ -1150,6 +1150,19 @@ config LRU_GEN_STATS - This option has a per-memcg and per-node memory overhead. - # } - -+config ARCH_SUPPORTS_PER_VMA_LOCK -+ def_bool n ++#ifdef CONFIG_LRNG_RAW_JIFFIES_ENTROPY ++bool lrng_raw_jiffies_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ ++static inline bool lrng_raw_jiffies_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_JIFFIES_ENTROPY */ + -+config PER_VMA_LOCK -+ bool "Per-vma locking support" -+ default y -+ depends on ARCH_SUPPORTS_PER_VMA_LOCK && MMU && SMP -+ help -+ Allow per-vma locking during page fault handling. ++#ifdef CONFIG_LRNG_RAW_IRQ_ENTROPY ++bool lrng_raw_irq_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ ++static inline bool lrng_raw_irq_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_IRQ_ENTROPY */ + -+ This feature allows locking each virtual memory area separately when -+ handling page faults instead of taking mmap_lock. ++#ifdef CONFIG_LRNG_RAW_RETIP_ENTROPY ++bool lrng_raw_retip_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ ++static inline bool lrng_raw_retip_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_RETIP_ENTROPY */ + - source "mm/damon/Kconfig" - - endmenu -diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug -index ce8dded36de9..075642763a03 100644 ---- a/mm/Kconfig.debug -+++ b/mm/Kconfig.debug -@@ -207,3 +207,11 @@ config PTDUMP_DEBUGFS - kernel. - - If in doubt, say N. ++#ifdef CONFIG_LRNG_RAW_REGS_ENTROPY ++bool lrng_raw_regs_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_REGS_ENTROPY */ ++static inline bool lrng_raw_regs_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_REGS_ENTROPY */ + ++#ifdef CONFIG_LRNG_RAW_ARRAY ++bool lrng_raw_array_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_ARRAY */ ++static inline bool lrng_raw_array_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_ARRAY */ + -+config PER_VMA_LOCK_STATS -+ bool "Statistics for per-vma locks" -+ depends on PER_VMA_LOCK -+ help -+ Statistics for per-vma locks. -+ If in doubt, say N. -diff --git a/mm/init-mm.c b/mm/init-mm.c -index fbe7844d0912..7b6d2460545f 100644 ---- a/mm/init-mm.c -+++ b/mm/init-mm.c -@@ -37,6 +37,12 @@ struct mm_struct init_mm = { - .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), - .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), - .mmlist = LIST_HEAD_INIT(init_mm.mmlist), -+#ifdef CONFIG_PER_VMA_LOCK -+ .mm_lock_seq = 0, -+ .vma_free_list.head = LIST_HEAD_INIT(init_mm.vma_free_list.head), -+ .vma_free_list.lock = __SPIN_LOCK_UNLOCKED(init_mm.vma_free_list.lock), -+ .vma_free_list.size = 0, -+#endif - .user_ns = &init_user_ns, - .cpu_bitmap = CPU_BITS_NONE, - #ifdef CONFIG_IOMMU_SVA -diff --git a/mm/internal.h b/mm/internal.h -index dd8600d71afe..dfb0b35eff76 100644 ---- a/mm/internal.h -+++ b/mm/internal.h -@@ -86,14 +86,14 @@ void deactivate_file_folio(struct folio *folio); - void folio_activate(struct folio *folio); - - void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, -- unsigned long floor, unsigned long ceiling); -+ unsigned long floor, unsigned long ceiling, bool lock_vma); - void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); - - struct zap_details; - void unmap_page_range(struct mmu_gather *tlb, - struct vm_area_struct *vma, - unsigned long addr, unsigned long end, -- struct zap_details *details); -+ struct zap_details *details, bool lock_vma); - - void page_cache_ra_order(struct readahead_control *, struct file_ra_state *, - unsigned int order); -diff --git a/mm/khugepaged.c b/mm/khugepaged.c -index 01f71786d530..030680633989 100644 ---- a/mm/khugepaged.c -+++ b/mm/khugepaged.c -@@ -1072,6 +1072,7 @@ static void collapse_huge_page(struct mm_struct *mm, - if (mm_find_pmd(mm, address) != pmd) - goto out_up_write; - -+ vma_mark_locked(vma); - anon_vma_lock_write(vma->anon_vma); - - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm, -diff --git a/mm/madvise.c b/mm/madvise.c -index 82f6ea8c493d..3efc8a07ebb0 100644 ---- a/mm/madvise.c -+++ b/mm/madvise.c -@@ -181,6 +181,7 @@ static int madvise_update_vma(struct vm_area_struct *vma, - /* - * vm_flags is protected by the mmap_lock held in write mode. - */ -+ vma_mark_locked(vma); - vma->vm_flags = new_flags; - if (!vma->vm_file) { - error = replace_anon_vma_name(vma, anon_name); -diff --git a/mm/memory.c b/mm/memory.c -index 47daee6b421f..a9640d62b7a9 100644 ---- a/mm/memory.c -+++ b/mm/memory.c -@@ -391,7 +391,7 @@ void free_pgd_range(struct mmu_gather *tlb, - } - - void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, -- unsigned long floor, unsigned long ceiling) -+ unsigned long floor, unsigned long ceiling, bool lock_vma) - { - while (vma) { - struct vm_area_struct *next = vma->vm_next; -@@ -401,6 +401,8 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - * Hide vma from rmap and truncate_pagecache before freeing - * pgtables - */ -+ if (lock_vma) -+ vma_mark_locked(vma); - unlink_anon_vmas(vma); - unlink_file_vma(vma); - -@@ -415,6 +417,8 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - && !is_vm_hugetlb_page(next)) { - vma = next; - next = vma->vm_next; -+ if (lock_vma) -+ vma_mark_locked(vma); - unlink_anon_vmas(vma); - unlink_file_vma(vma); - } -@@ -1619,12 +1623,16 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, - void unmap_page_range(struct mmu_gather *tlb, - struct vm_area_struct *vma, - unsigned long addr, unsigned long end, -- struct zap_details *details) -+ struct zap_details *details, -+ bool lock_vma) - { - pgd_t *pgd; - unsigned long next; - - BUG_ON(addr >= end); -+ if (lock_vma) -+ vma_mark_locked(vma); ++#ifdef CONFIG_LRNG_IRQ_PERF ++bool lrng_perf_time(u32 start); ++#else /* CONFIG_LRNG_IRQ_PERF */ ++static inline bool lrng_perf_time(u32 start) { return false; } ++#endif /*CONFIG_LRNG_IRQ_PERF */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY ++bool lrng_raw_sched_hires_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ ++static inline bool ++lrng_raw_sched_hires_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_HIRES_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_PID_ENTROPY ++bool lrng_raw_sched_pid_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ ++static inline bool ++lrng_raw_sched_pid_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_PID_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY ++bool lrng_raw_sched_starttime_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++static inline bool ++lrng_raw_sched_starttime_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_START_TIME_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY ++bool lrng_raw_sched_nvcsw_entropy_store(u32 value); ++#else /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ ++static inline bool ++lrng_raw_sched_nvcsw_entropy_store(u32 value) { return false; } ++#endif /* CONFIG_LRNG_RAW_SCHED_NVCSW_ENTROPY */ ++ ++#ifdef CONFIG_LRNG_SCHED_PERF ++bool lrng_sched_perf_time(u32 start); ++#else /* CONFIG_LRNG_SCHED_PERF */ ++static inline bool lrng_sched_perf_time(u32 start) { return false; } ++#endif /*CONFIG_LRNG_SCHED_PERF */ + - tlb_start_vma(tlb, vma); - pgd = pgd_offset(vma->vm_mm, addr); - do { -@@ -1640,7 +1648,7 @@ void unmap_page_range(struct mmu_gather *tlb, - static void unmap_single_vma(struct mmu_gather *tlb, - struct vm_area_struct *vma, unsigned long start_addr, - unsigned long end_addr, -- struct zap_details *details) -+ struct zap_details *details, bool lock_vma) - { - unsigned long start = max(vma->vm_start, start_addr); - unsigned long end; -@@ -1679,7 +1687,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, - i_mmap_unlock_write(vma->vm_file->f_mapping); - } - } else -- unmap_page_range(tlb, vma, start, end, details); -+ unmap_page_range(tlb, vma, start, end, details, lock_vma); - } - } - -@@ -1703,7 +1711,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, - */ - void unmap_vmas(struct mmu_gather *tlb, - struct vm_area_struct *vma, unsigned long start_addr, -- unsigned long end_addr) -+ unsigned long end_addr, bool lock_vma) - { - struct mmu_notifier_range range; - struct zap_details details = { -@@ -1716,7 +1724,8 @@ void unmap_vmas(struct mmu_gather *tlb, - start_addr, end_addr); - mmu_notifier_invalidate_range_start(&range); - for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) -- unmap_single_vma(tlb, vma, start_addr, end_addr, &details); -+ unmap_single_vma(tlb, vma, start_addr, end_addr, &details, -+ lock_vma); - mmu_notifier_invalidate_range_end(&range); - } - -@@ -1741,7 +1750,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, - update_hiwater_rss(vma->vm_mm); - mmu_notifier_invalidate_range_start(&range); - for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) -- unmap_single_vma(&tlb, vma, start, range.end, NULL); -+ unmap_single_vma(&tlb, vma, start, range.end, NULL, false); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -@@ -1756,7 +1765,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, - * The range must fit into one VMA. - */ - static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, -- unsigned long size, struct zap_details *details) -+ unsigned long size, struct zap_details *details, bool lock_vma) - { - struct mmu_notifier_range range; - struct mmu_gather tlb; -@@ -1767,7 +1776,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr - tlb_gather_mmu(&tlb, vma->vm_mm); - update_hiwater_rss(vma->vm_mm); - mmu_notifier_invalidate_range_start(&range); -- unmap_single_vma(&tlb, vma, address, range.end, details); -+ unmap_single_vma(&tlb, vma, address, range.end, details, lock_vma); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -@@ -1790,7 +1799,7 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, - !(vma->vm_flags & VM_PFNMAP)) - return; - -- zap_page_range_single(vma, address, size, NULL); -+ zap_page_range_single(vma, address, size, NULL, true); - } - EXPORT_SYMBOL_GPL(zap_vma_ptes); - -@@ -3471,7 +3480,8 @@ static void unmap_mapping_range_vma(struct vm_area_struct *vma, - unsigned long start_addr, unsigned long end_addr, - struct zap_details *details) - { -- zap_page_range_single(vma, start_addr, end_addr - start_addr, details); -+ zap_page_range_single(vma, start_addr, end_addr - start_addr, details, -+ false); - } - - static inline void unmap_mapping_range_tree(struct rb_root_cached *root, -@@ -3716,6 +3726,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) - vm_fault_t ret = 0; - void *shadow = NULL; ++#endif /* _LRNG_TESTING_H */ +diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h +index af5ad51d3eef..b12ae9bdebf4 100644 +--- a/include/crypto/drbg.h ++++ b/include/crypto/drbg.h +@@ -283,4 +283,11 @@ enum drbg_prefixes { + DRBG_PREFIX3 + }; -+ if (vmf->flags & FAULT_FLAG_VMA_LOCK) { -+ ret = VM_FAULT_RETRY; -+ goto out; -+ } ++extern int drbg_alloc_state(struct drbg_state *drbg); ++extern void drbg_dealloc_state(struct drbg_state *drbg); ++extern void drbg_convert_tfm_core(const char *cra_driver_name, int *coreref, ++ bool *pr); ++extern const struct drbg_core drbg_cores[]; ++extern unsigned short drbg_sec_strength(drbg_flag_t flags); + - if (!pte_unmap_same(vmf)) - goto out; - -@@ -5181,6 +5196,51 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, - } - EXPORT_SYMBOL_GPL(handle_mm_fault); - -+#ifdef CONFIG_PER_VMA_LOCK -+static inline struct vm_area_struct *find_vma_under_rcu(struct mm_struct *mm, -+ unsigned long address) -+{ -+ struct vm_area_struct *vma = __find_vma(mm, address); + #endif /* _DRBG_H */ +diff --git a/crypto/jitterentropy.h b/include/crypto/internal/jitterentropy.h +similarity index 100% +rename from crypto/jitterentropy.h +rename to include/crypto/internal/jitterentropy.h +diff --git a/include/linux/lrng.h b/include/linux/lrng.h +new file mode 100644 +index 000000000000..c0d31a03d51f +--- /dev/null ++++ b/include/linux/lrng.h +@@ -0,0 +1,251 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (C) 2022, Stephan Mueller ++ */ + -+ if (!vma || vma->vm_start > address) -+ return NULL; ++#ifndef _LRNG_H ++#define _LRNG_H + -+ if (!vma_is_anonymous(vma)) -+ return NULL; ++#include ++#include ++#include ++#include + -+ if (!vma_read_trylock(vma)) { -+ count_vm_vma_lock_event(VMA_LOCK_ABORT); -+ return NULL; -+ } ++/* ++ * struct lrng_drng_cb - cryptographic callback functions defining a DRNG ++ * @drng_name Name of DRNG ++ * @drng_alloc: Allocate DRNG -- the provided integer should be used for ++ * sanity checks. ++ * return: allocated data structure or PTR_ERR on error ++ * @drng_dealloc: Deallocate DRNG ++ * @drng_seed: Seed the DRNG with data of arbitrary length drng: is ++ * pointer to data structure allocated with drng_alloc ++ * return: >= 0 on success, < 0 on error ++ * @drng_generate: Generate random numbers from the DRNG with arbitrary ++ * length ++ */ ++struct lrng_drng_cb { ++ const char *(*drng_name)(void); ++ void *(*drng_alloc)(u32 sec_strength); ++ void (*drng_dealloc)(void *drng); ++ int (*drng_seed)(void *drng, const u8 *inbuf, u32 inbuflen); ++ int (*drng_generate)(void *drng, u8 *outbuf, u32 outbuflen); ++}; + -+ /* Check if the VMA got isolated after we found it */ -+ if (RB_EMPTY_NODE(&vma->vm_rb)) { -+ vma_read_unlock(vma); -+ count_vm_vma_lock_event(VMA_LOCK_MISS); -+ return NULL; -+ } ++/* ++ * struct lrng_hash_cb - cryptographic callback functions defining a hash ++ * @hash_name Name of Hash used for reading entropy pool arbitrary ++ * length ++ * @hash_alloc: Allocate the hash for reading the entropy pool ++ * return: allocated data structure (NULL is success too) ++ * or ERR_PTR on error ++ * @hash_dealloc: Deallocate Hash ++ * @hash_digestsize: Return the digestsize for the used hash to read out ++ * entropy pool ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: size of digest of hash in bytes ++ * @hash_init: Initialize hash ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: 0 on success, < 0 on error ++ * @hash_update: Update hash operation ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: 0 on success, < 0 on error ++ * @hash_final Final hash operation ++ * hash: is pointer to data structure allocated with ++ * hash_alloc ++ * return: 0 on success, < 0 on error ++ * @hash_desc_zero Zeroization of hash state buffer ++ * ++ * Assumptions: ++ * ++ * 1. Hash operation will not sleep ++ * 2. The hash' volatile state information is provided with *shash by caller. ++ */ ++struct lrng_hash_cb { ++ const char *(*hash_name)(void); ++ void *(*hash_alloc)(void); ++ void (*hash_dealloc)(void *hash); ++ u32 (*hash_digestsize)(void *hash); ++ int (*hash_init)(struct shash_desc *shash, void *hash); ++ int (*hash_update)(struct shash_desc *shash, const u8 *inbuf, ++ u32 inbuflen); ++ int (*hash_final)(struct shash_desc *shash, u8 *digest); ++ void (*hash_desc_zero)(struct shash_desc *shash); ++}; + -+ return vma; -+} ++/* Register cryptographic backend */ ++#ifdef CONFIG_LRNG_SWITCH ++int lrng_set_drng_cb(const struct lrng_drng_cb *cb); ++int lrng_set_hash_cb(const struct lrng_hash_cb *cb); ++#else /* CONFIG_LRNG_SWITCH */ ++static inline int ++lrng_set_drng_cb(const struct lrng_drng_cb *cb) { return -EOPNOTSUPP; } ++static inline int ++lrng_set_hash_cb(const struct lrng_hash_cb *cb) { return -EOPNOTSUPP; } ++#endif /* CONFIG_LRNG_SWITCH */ ++ ++/* Callback to feed events to the scheduler entropy source */ ++#ifdef CONFIG_LRNG_SCHED ++extern void add_sched_randomness(const struct task_struct *p, int cpu); ++#else ++static inline void ++add_sched_randomness(const struct task_struct *p, int cpu) { } ++#endif + +/* -+ * Lookup and lock and anonymous VMA. Returned VMA is guaranteed to be stable -+ * and not isolated. If the VMA is not found of is being modified the function -+ * returns NULL. ++ * lrng_get_random_bytes() - Provider of cryptographic strong random numbers ++ * for kernel-internal usage. ++ * ++ * This function is appropriate for in-kernel use cases operating in atomic ++ * contexts. It will always use the ChaCha20 DRNG and it may be the case that ++ * it is not fully seeded when being used. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer + */ -+struct vm_area_struct *find_and_lock_anon_vma(struct mm_struct *mm, -+ unsigned long address) -+{ -+ struct vm_area_struct *vma; ++#ifdef CONFIG_LRNG_DRNG_ATOMIC ++void lrng_get_random_bytes(void *buf, int nbytes); ++#endif + -+ rcu_read_lock(); -+ vma = find_vma_under_rcu(mm, address); -+ rcu_read_unlock(); ++/* ++ * lrng_get_random_bytes_full() - Provider of cryptographic strong ++ * random numbers for kernel-internal usage from a fully initialized LRNG. ++ * ++ * This function will always return random numbers from a fully seeded and ++ * fully initialized LRNG. ++ * ++ * This function is appropriate only for non-atomic use cases as this ++ * function may sleep. It provides access to the full functionality of LRNG ++ * including the switchable DRNG support, that may support other DRNGs such ++ * as the SP800-90A DRBG. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ */ ++#ifdef CONFIG_LRNG ++void lrng_get_random_bytes_full(void *buf, int nbytes); ++#endif + -+ return vma; -+} -+#endif /* CONFIG_PER_VMA_LOCK */ ++/* ++ * lrng_get_random_bytes_min() - Provider of cryptographic strong ++ * random numbers for kernel-internal usage from at least a minimally seeded ++ * LRNG, which is not necessarily fully initialized yet (e.g. SP800-90C ++ * oversampling applied in FIPS mode is not applied yet). ++ * ++ * This function is appropriate only for non-atomic use cases as this ++ * function may sleep. It provides access to the full functionality of LRNG ++ * including the switchable DRNG support, that may support other DRNGs such ++ * as the SP800-90A DRBG. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ */ ++#ifdef CONFIG_LRNG ++void lrng_get_random_bytes_min(void *buf, int nbytes); ++#endif + - #ifndef __PAGETABLE_P4D_FOLDED - /* - * Allocate p4d page table. -diff --git a/mm/mempolicy.c b/mm/mempolicy.c -index b73d3248d976..6be1e5c75556 100644 ---- a/mm/mempolicy.c -+++ b/mm/mempolicy.c -@@ -383,8 +383,10 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) - struct vm_area_struct *vma; - - mmap_write_lock(mm); -- for (vma = mm->mmap; vma; vma = vma->vm_next) -+ for (vma = mm->mmap; vma; vma = vma->vm_next) { -+ vma_mark_locked(vma); - mpol_rebind_policy(vma->vm_policy, new); -+ } - mmap_write_unlock(mm); - } - -@@ -632,6 +634,7 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, - struct mmu_gather tlb; - int nr_updated; - -+ vma_mark_locked(vma); - tlb_gather_mmu(&tlb, vma->vm_mm); - - nr_updated = change_protection(&tlb, vma, addr, end, PAGE_NONE, -@@ -765,6 +768,7 @@ static int vma_replace_policy(struct vm_area_struct *vma, - if (IS_ERR(new)) - return PTR_ERR(new); - -+ vma_mark_locked(vma); - if (vma->vm_ops && vma->vm_ops->set_policy) { - err = vma->vm_ops->set_policy(vma, new); - if (err) -diff --git a/mm/mlock.c b/mm/mlock.c -index b14e929084cc..f62e1a4d05f2 100644 ---- a/mm/mlock.c -+++ b/mm/mlock.c -@@ -380,6 +380,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, - */ - if (newflags & VM_LOCKED) - newflags |= VM_IO; -+ vma_mark_locked(vma); - WRITE_ONCE(vma->vm_flags, newflags); - - lru_add_drain(); -@@ -456,6 +457,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, - - if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) { - /* No work to do, and mlocking twice would be wrong */ -+ vma_mark_locked(vma); - vma->vm_flags = newflags; - } else { - mlock_vma_pages_range(vma, start, end, newflags); -diff --git a/mm/mmap.c b/mm/mmap.c -index 9d780f415be3..d61b7ef84ba6 100644 ---- a/mm/mmap.c -+++ b/mm/mmap.c -@@ -79,7 +79,7 @@ core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); - - static void unmap_region(struct mm_struct *mm, - struct vm_area_struct *vma, struct vm_area_struct *prev, -- unsigned long start, unsigned long end); -+ unsigned long start, unsigned long end, bool lock_vma); - - static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) - { -@@ -421,12 +421,14 @@ static inline void vma_rb_insert(struct vm_area_struct *vma, - - static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) - { -+ vma_mark_locked(vma); - /* - * Note rb_erase_augmented is a fairly large inline function, - * so make sure we instantiate it only once with our desired - * augmented rbtree callbacks. - */ - rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); -+ RB_CLEAR_NODE(&vma->vm_rb); - } - - static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, -@@ -710,6 +712,10 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - long adjust_next = 0; - int remove_next = 0; - -+ vma_mark_locked(vma); -+ if (next) -+ vma_mark_locked(next); ++/* ++ * lrng_get_random_bytes_pr() - Provider of cryptographic strong ++ * random numbers for kernel-internal usage from a fully initialized LRNG and ++ * requiring a reseed from the entropy sources before. ++ * ++ * This function will always return random numbers from a fully seeded and ++ * fully initialized LRNG. ++ * ++ * This function is appropriate only for non-atomic use cases as this ++ * function may sleep. It provides access to the full functionality of LRNG ++ * including the switchable DRNG support, that may support other DRNGs such ++ * as the SP800-90A DRBG. ++ * ++ * This call only returns no more data than entropy was pulled from the ++ * entropy sources. Thus, it is likely that this call returns less data ++ * than requested by the caller. Also, the caller MUST be prepared that this ++ * call returns 0 bytes, i.e. it did not generate data. ++ * ++ * @buf: buffer to store the random bytes ++ * @nbytes: size of the buffer ++ * ++ * @return: positive number indicates amount of generated bytes, < 0 on error ++ */ ++#ifdef CONFIG_LRNG ++int lrng_get_random_bytes_pr(void *buf, int nbytes); ++#endif + - if (next && !insert) { - struct vm_area_struct *exporter = NULL, *importer = NULL; - -@@ -754,8 +760,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - * If next doesn't have anon_vma, import from vma after - * next, if the vma overlaps with it. - */ -- if (remove_next == 2 && !next->anon_vma) -+ if (remove_next == 2 && !next->anon_vma) { - exporter = next->vm_next; -+ if (exporter) -+ vma_mark_locked(exporter); -+ } - - } else if (end > next->vm_start) { - /* -@@ -931,6 +940,8 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, - * "vma->vm_next" gap must be updated. - */ - next = vma->vm_next; -+ if (next) -+ vma_mark_locked(next); - } else { - /* - * For the scope of the comment "next" and -@@ -1138,10 +1149,17 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, - if (vm_flags & VM_SPECIAL) - return NULL; - -+ if (prev) -+ vma_mark_locked(prev); - next = vma_next(mm, prev); - area = next; -- if (area && area->vm_end == end) /* cases 6, 7, 8 */ -+ if (area) -+ vma_mark_locked(area); -+ if (area && area->vm_end == end) { /* cases 6, 7, 8 */ - next = next->vm_next; -+ if (next) -+ vma_mark_locked(next); -+ } - - /* verify some invariant that must be enforced by the caller */ - VM_WARN_ON(prev && addr <= prev->vm_start); -@@ -1818,6 +1836,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, - out: - perf_event_mmap(vma); - -+ vma_mark_locked(vma); - vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); - if (vm_flags & VM_LOCKED) { - if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || -@@ -1849,7 +1868,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, - vma->vm_file = NULL; - - /* Undo any partial mapping done by a device driver. */ -- unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); -+ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end, true); - if (vm_flags & VM_SHARED) - mapping_unmap_writable(file->f_mapping); - free_vma: -@@ -2250,12 +2269,11 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - EXPORT_SYMBOL(get_unmapped_area); - - /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ --struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) -+struct vm_area_struct *__find_vma(struct mm_struct *mm, unsigned long addr) - { - struct rb_node *rb_node; - struct vm_area_struct *vma; - -- mmap_assert_locked(mm); - /* Check the cache first. */ - vma = vmacache_find(mm, addr); - if (likely(vma)) -@@ -2281,8 +2299,7 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) - vmacache_update(addr, vma); - return vma; - } -- --EXPORT_SYMBOL(find_vma); -+EXPORT_SYMBOL(__find_vma); - - /* - * Same as find_vma, but also return a pointer to the previous VMA in *pprev. -@@ -2611,7 +2628,7 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) ++/* ++ * lrng_get_seed() - Fill buffer with data from entropy sources ++ * ++ * This call allows accessing the entropy sources directly and fill the buffer ++ * with data from all available entropy sources. This filled buffer is ++ * identical to the temporary seed buffer used by the LRNG to seed its DRNGs. ++ * ++ * The call is to allows users to seed their DRNG directly from the entropy ++ * sources in case the caller does not want to use the LRNG's DRNGs. This ++ * buffer can be directly used to seed the caller's DRNG from. ++ * ++ * The call blocks as long as one LRNG DRNG is not yet fully seeded. If ++ * LRNG_GET_SEED_NONBLOCK is specified, it does not block in this case, but ++ * returns with an error. ++ * ++ * Considering SP800-90C, there is a differentiation between the seeding ++ * requirements during instantiating a DRNG and at runtime of the DRNG. When ++ * specifying LRNG_GET_SEED_FULLY_SEEDED the caller indicates the DRNG was ++ * already fully seeded and the regular amount of entropy is requested. ++ * Otherwise, the LRNG will obtain the entropy rate required for initial ++ * seeding. The following minimum entropy rates will be obtained: ++ * ++ * * FIPS mode: ++ * * Initial seeding: 384 bits of entropy ++ * * Runtime seeding: 256 bits of entropy ++ * * Non-FIPS mode: ++ * * 128 bits of entropy in any case ++ * ++ * Albeit these are minimum entropy rates, the LRNG tries to request the ++ * given amount of entropy from each entropy source individually. If the ++ * minimum amount of entropy cannot be obtained collectively by all entropy ++ * sources, the LRNG will not fill the buffer. ++ * ++ * The return data in buf is structurally equivalent to the following ++ * definition: ++ * ++ * struct { ++ * u64 seedlen; ++ * u64 entropy_rate; ++ * struct entropy_buf seed; ++ * } __attribute((__packed__)); ++ * ++ * As struct entropy_buf is not known outsize of the LRNG, the LRNG fills ++ * seedlen first with the size of struct entropy_buf. If the caller-provided ++ * buffer buf is smaller than u64, then -EINVAL is returned ++ * and buf is not touched. If it is u64 or larger but smaller ++ * than the size of the structure above, -EMSGSIZE is returned and seedlen ++ * is filled with the size of the buffer. Finally, if buf is large ++ * enough to hold all data, it is filled with the seed data and the seedlen ++ * is set to sizeof(struct entropy_buf). The entropy rate is returned with ++ * the variable entropy_rate and provides the value in bits. ++ * ++ * The seed buffer is the data that should be handed to the caller's DRNG as ++ * seed data. ++ * ++ * @buf [out] Buffer to be filled with data from the entropy sources - note, the ++ * buffer is marked as u64 to ensure it is aligned to 64 bits. ++ * @nbytes [in] Size of the buffer allocated by the caller - this value ++ * provides size of @param buf in bytes. ++ * @flags [in] Flags field to adjust the behavior ++ * ++ * @return -EINVAL or -EMSGSIZE indicating the buffer is too small, -EAGAIN when ++ * the call would block, but NONBLOCK is specified, > 0 the size of ++ * the filled buffer. ++ */ ++#ifdef CONFIG_LRNG ++enum lrng_get_seed_flags { ++ LRNG_GET_SEED_NONBLOCK = 0x0001, /**< Do not block the call */ ++ LRNG_GET_SEED_FULLY_SEEDED = 0x0002, /**< DRNG is fully seeded */ ++}; ++ ++ssize_t lrng_get_seed(u64 *buf, size_t nbytes, unsigned int flags); ++#endif ++ ++#endif /* _LRNG_H */ +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index 265944f14948..e9f0ec6c9aeb 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -6,6 +6,7 @@ + * + * Copyright (C) 1991-2002 Linus Torvalds */ - static void unmap_region(struct mm_struct *mm, - struct vm_area_struct *vma, struct vm_area_struct *prev, -- unsigned long start, unsigned long end) -+ unsigned long start, unsigned long end, bool lock_vma) ++#include + #include + #include + #include +@@ -3586,6 +3587,8 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) { - struct vm_area_struct *next = vma_next(mm, prev); - struct mmu_gather tlb; -@@ -2619,9 +2636,10 @@ static void unmap_region(struct mm_struct *mm, - lru_add_drain(); - tlb_gather_mmu(&tlb, mm); - update_hiwater_rss(mm); -- unmap_vmas(&tlb, vma, start, end); -+ unmap_vmas(&tlb, vma, start, end, lock_vma); - free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, -- next ? next->vm_start : USER_PGTABLES_CEILING); -+ next ? next->vm_start : USER_PGTABLES_CEILING, -+ lock_vma); - tlb_finish_mmu(&tlb); - } - -@@ -2662,10 +2680,14 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, - * VM_GROWSUP VMA. Such VMAs can change their size under - * down_read(mmap_lock) and collide with the VMA we are about to unmap. - */ -- if (vma && (vma->vm_flags & VM_GROWSDOWN)) -+ if (vma && (vma->vm_flags & VM_GROWSDOWN)) { -+ vma_mark_locked(vma); - return false; -- if (prev && (prev->vm_flags & VM_GROWSUP)) -+ } -+ if (prev && (prev->vm_flags & VM_GROWSUP)) { -+ vma_mark_locked(prev); - return false; -+ } - return true; - } - -@@ -2679,6 +2701,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *new; - int err; - -+ vma_mark_locked(vma); - if (vma->vm_ops && vma->vm_ops->may_split) { - err = vma->vm_ops->may_split(vma, addr); - if (err) -@@ -2833,7 +2856,7 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - if (downgrade) - mmap_write_downgrade(mm); - -- unmap_region(mm, vma, prev, start, end); -+ unmap_region(mm, vma, prev, start, end, !downgrade); - - /* Fix up all other VM information */ - remove_vma_list(mm, vma); -@@ -3113,8 +3136,8 @@ void exit_mmap(struct mm_struct *mm) - tlb_gather_mmu_fullmm(&tlb, mm); - /* update_hiwater_rss(mm) here? but nobody should be looking */ - /* Use -1 here to ensure all VMAs in the mm are unmapped */ -- unmap_vmas(&tlb, vma, 0, -1); -- free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); -+ unmap_vmas(&tlb, vma, 0, -1, true); -+ free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING, true); - tlb_finish_mmu(&tlb); - - /* Walk the list again, actually closing and freeing it. */ -@@ -3126,6 +3149,7 @@ void exit_mmap(struct mm_struct *mm) - } - mm->mmap = NULL; - mmap_write_unlock(mm); -+ drain_free_vmas(mm); - vm_unacct_memory(nr_accounted); - } - -@@ -3232,6 +3256,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - get_file(new_vma->vm_file); - if (new_vma->vm_ops && new_vma->vm_ops->open) - new_vma->vm_ops->open(new_vma); -+ vma_mark_locked(new_vma); - vma_link(mm, new_vma, prev, rb_link, rb_parent); - *need_rmap_locks = false; - } -@@ -3514,6 +3539,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) - * hugetlb mapping); - * - all i_mmap_rwsem locks; - * - all anon_vma->rwseml -+ * - all vmas marked locked - * - * We can take all locks within these types randomly because the VM code - * doesn't nest them and we protected from parallel mm_take_all_locks() by -@@ -3555,6 +3581,7 @@ int mm_take_all_locks(struct mm_struct *mm) - if (vma->anon_vma) - list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) - vm_lock_anon_vma(mm, avc->anon_vma); -+ vma_mark_locked(vma); - } - - return 0; -@@ -3612,6 +3639,7 @@ void mm_drop_all_locks(struct mm_struct *mm) - mmap_assert_write_locked(mm); - BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); - -+ vma_mark_unlocked_all(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (vma->anon_vma) - list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) -diff --git a/mm/mprotect.c b/mm/mprotect.c -index bc6bddd156ca..df47fc21b0e4 100644 ---- a/mm/mprotect.c -+++ b/mm/mprotect.c -@@ -621,6 +621,7 @@ mprotect_fixup(struct mmu_gather *tlb, struct vm_area_struct *vma, - * vm_flags and vm_page_prot are protected by the mmap_lock - * held in write mode. - */ -+ vma_mark_locked(vma); - vma->vm_flags = newflags; - /* - * We want to check manually if we can change individual PTEs writable -diff --git a/mm/mremap.c b/mm/mremap.c -index b522cd0259a0..bdbf96254e43 100644 ---- a/mm/mremap.c -+++ b/mm/mremap.c -@@ -620,6 +620,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, - return -ENOMEM; - } - -+ vma_mark_locked(vma); - new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); - new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, - &need_rmap_locks); -diff --git a/mm/nommu.c b/mm/nommu.c -index e819cbc21b39..ff9933e57501 100644 ---- a/mm/nommu.c -+++ b/mm/nommu.c -@@ -622,6 +622,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) - struct mm_struct *mm = vma->vm_mm; - struct task_struct *curr = current; - -+ vma_mark_locked(vma); - mm->map_count--; - for (i = 0; i < VMACACHE_SIZE; i++) { - /* if the vma is cached, invalidate the entire cache */ -@@ -644,6 +645,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) + struct rq *rq; - /* remove from the MM's tree and list */ - rb_erase(&vma->vm_rb, &mm->mm_rb); -+ RB_CLEAR_NODE(&vma->vm_rb); ++ add_sched_randomness(p, cpu); ++ + if (!schedstat_enabled()) + return; - __vma_unlink_list(mm, vma); - } -diff --git a/mm/oom_kill.c b/mm/oom_kill.c -index 3c6cf9e3cd66..6ffa7c511aa3 100644 ---- a/mm/oom_kill.c -+++ b/mm/oom_kill.c -@@ -549,7 +549,8 @@ bool __oom_reap_task_mm(struct mm_struct *mm) - ret = false; - continue; - } -- unmap_page_range(&tlb, vma, range.start, range.end, NULL); -+ unmap_page_range(&tlb, vma, range.start, range.end, -+ NULL, false); - mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb); - } -diff --git a/mm/vmstat.c b/mm/vmstat.c -index 90af9a8572f5..3f3804c846a6 100644 ---- a/mm/vmstat.c -+++ b/mm/vmstat.c -@@ -1411,6 +1411,12 @@ const char * const vmstat_text[] = { - "direct_map_level2_splits", - "direct_map_level3_splits", - #endif -+#ifdef CONFIG_PER_VMA_LOCK_STATS -+ "vma_lock_success", -+ "vma_lock_abort", -+ "vma_lock_retry", -+ "vma_lock_miss", -+#endif - #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ - }; - #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 -From 4ba2e168370145273187f242e0a85ce3842e25fa Mon Sep 17 00:00:00 2001 +From e1f1e6838dfabd0b23fc9a7ee4dc0d0a91d27680 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Sep 2022 14:40:14 +0200 -Subject: [PATCH 11/13] folios +Subject: [PATCH 13/16] folios Signed-off-by: Peter Jung --- @@ -41507,3554 +96959,5602 @@ index 05bee80ac7de..8f87c2551a3d 100644 + * @fbatch: The batch of folios * @done_index: Page index * - * Returns: non-zero if loop should terminate, zero otherwise + * Returns: non-zero if loop should terminate, zero otherwise + */ + +-static int gfs2_write_jdata_pagevec(struct address_space *mapping, ++static int gfs2_write_jdata_batch(struct address_space *mapping, + struct writeback_control *wbc, +- struct pagevec *pvec, +- int nr_pages, ++ struct folio_batch *fbatch, + pgoff_t *done_index) + { + struct inode *inode = mapping->host; + struct gfs2_sbd *sdp = GFS2_SB(inode); +- unsigned nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); ++ unsigned nrblocks; + int i; + int ret; ++ int nr_pages = 0; ++ int nr_folios = folio_batch_count(fbatch); ++ ++ for (i = 0; i < nr_folios; i++) ++ nr_pages += folio_nr_pages(fbatch->folios[i]); ++ nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); + + ret = gfs2_trans_begin(sdp, nrblocks, nrblocks); + if (ret < 0) + return ret; + +- for(i = 0; i < nr_pages; i++) { +- struct page *page = pvec->pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch->folios[i]; + +- *done_index = page->index; ++ *done_index = folio->index; + +- lock_page(page); ++ folio_lock(folio); + +- if (unlikely(page->mapping != mapping)) { ++ if (unlikely(folio->mapping != mapping)) { + continue_unlock: +- unlock_page(page); ++ folio_unlock(folio); + continue; + } + +- if (!PageDirty(page)) { ++ if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + +- if (PageWriteback(page)) { ++ if (folio_test_writeback(folio)) { + if (wbc->sync_mode != WB_SYNC_NONE) +- wait_on_page_writeback(page); ++ folio_wait_writeback(folio); + else + goto continue_unlock; + } + +- BUG_ON(PageWriteback(page)); +- if (!clear_page_dirty_for_io(page)) ++ BUG_ON(folio_test_writeback(folio)); ++ if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + trace_wbc_writepage(wbc, inode_to_bdi(inode)); + +- ret = __gfs2_jdata_writepage(page, wbc); ++ ret = __gfs2_jdata_writepage(&folio->page, wbc); + if (unlikely(ret)) { + if (ret == AOP_WRITEPAGE_ACTIVATE) { +- unlock_page(page); ++ folio_unlock(folio); + ret = 0; + } else { + +@@ -268,7 +272,8 @@ static int gfs2_write_jdata_pagevec(struct address_space *mapping, + * not be suitable for data integrity + * writeout). + */ +- *done_index = page->index + 1; ++ *done_index = folio->index + ++ folio_nr_pages(folio); + ret = 1; + break; + } +@@ -305,8 +310,8 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + { + int ret = 0; + int done = 0; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ int nr_folios; + pgoff_t writeback_index; + pgoff_t index; + pgoff_t end; +@@ -315,7 +320,7 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + int range_whole = 0; + xa_mark_t tag; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + writeback_index = mapping->writeback_index; /* prev offset */ + index = writeback_index; +@@ -341,17 +346,18 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && (index <= end)) { +- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, +- tag); +- if (nr_pages == 0) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ tag, &fbatch); ++ if (nr_folios == 0) + break; + +- ret = gfs2_write_jdata_pagevec(mapping, wbc, &pvec, nr_pages, &done_index); ++ ret = gfs2_write_jdata_batch(mapping, wbc, &fbatch, ++ &done_index); + if (ret) + done = 1; + if (ret > 0) + ret = 0; +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + +diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c +index 9f4d9432d38a..1e26f32a4e36 100644 +--- a/fs/nilfs2/btree.c ++++ b/fs/nilfs2/btree.c +@@ -2143,7 +2143,7 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btcache = btnc_inode->i_mapping; + struct list_head lists[NILFS_BTREE_LEVEL_MAX]; +- struct pagevec pvec; ++ struct folio_batch fbatch; + struct buffer_head *bh, *head; + pgoff_t index = 0; + int level, i; +@@ -2153,19 +2153,19 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, + level++) + INIT_LIST_HEAD(&lists[level]); + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + +- while (pagevec_lookup_tag(&pvec, btcache, &index, +- PAGECACHE_TAG_DIRTY)) { +- for (i = 0; i < pagevec_count(&pvec); i++) { +- bh = head = page_buffers(pvec.pages[i]); ++ while (filemap_get_folios_tag(btcache, &index, (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch)) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ bh = head = folio_buffers(fbatch.folios[i]); + do { + if (buffer_dirty(bh)) + nilfs_btree_add_dirty_buffer(btree, + lists, bh); + } while ((bh = bh->b_this_page) != head); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + +diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c +index 3267e96c256c..b66f4e988016 100644 +--- a/fs/nilfs2/page.c ++++ b/fs/nilfs2/page.c +@@ -240,42 +240,43 @@ static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty) + int nilfs_copy_dirty_pages(struct address_space *dmap, + struct address_space *smap) + { +- struct pagevec pvec; ++ struct folio_batch fbatch; + unsigned int i; + pgoff_t index = 0; + int err = 0; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + repeat: +- if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY)) ++ if (!filemap_get_folios_tag(smap, &index, (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch)) + return 0; + +- for (i = 0; i < pagevec_count(&pvec); i++) { +- struct page *page = pvec.pages[i], *dpage; ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ struct folio *folio = fbatch.folios[i], *dfolio; + +- lock_page(page); +- if (unlikely(!PageDirty(page))) +- NILFS_PAGE_BUG(page, "inconsistent dirty state"); ++ folio_lock(folio); ++ if (unlikely(!folio_test_dirty(folio))) ++ NILFS_PAGE_BUG(&folio->page, "inconsistent dirty state"); + +- dpage = grab_cache_page(dmap, page->index); +- if (unlikely(!dpage)) { ++ dfolio = filemap_grab_folio(dmap, folio->index); ++ if (unlikely(!dfolio)) { + /* No empty page is added to the page cache */ + err = -ENOMEM; +- unlock_page(page); ++ folio_unlock(folio); + break; + } +- if (unlikely(!page_has_buffers(page))) +- NILFS_PAGE_BUG(page, ++ if (unlikely(!folio_buffers(folio))) ++ NILFS_PAGE_BUG(&folio->page, + "found empty page in dat page cache"); + +- nilfs_copy_page(dpage, page, 1); +- __set_page_dirty_nobuffers(dpage); ++ nilfs_copy_page(&dfolio->page, &folio->page, 1); ++ filemap_dirty_folio(folio_mapping(dfolio), dfolio); + +- unlock_page(dpage); +- put_page(dpage); +- unlock_page(page); ++ folio_unlock(dfolio); ++ folio_put(dfolio); ++ folio_unlock(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + + if (likely(!err)) +@@ -357,22 +358,22 @@ void nilfs_copy_back_pages(struct address_space *dmap, + */ + void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent) + { +- struct pagevec pvec; ++ struct folio_batch fbatch; + unsigned int i; + pgoff_t index = 0; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + +- while (pagevec_lookup_tag(&pvec, mapping, &index, +- PAGECACHE_TAG_DIRTY)) { +- for (i = 0; i < pagevec_count(&pvec); i++) { +- struct page *page = pvec.pages[i]; ++ while (filemap_get_folios_tag(mapping, &index, (pgoff_t)-1, ++ PAGECACHE_TAG_DIRTY, &fbatch)) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ struct folio *folio = fbatch.folios[i]; + +- lock_page(page); +- nilfs_clear_dirty_page(page, silent); +- unlock_page(page); ++ folio_lock(folio); ++ nilfs_clear_dirty_page(&folio->page, silent); ++ folio_unlock(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c +index 0afe0832c754..6f2ca279d230 100644 +--- a/fs/nilfs2/segment.c ++++ b/fs/nilfs2/segment.c +@@ -680,7 +680,7 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + loff_t start, loff_t end) + { + struct address_space *mapping = inode->i_mapping; +- struct pagevec pvec; ++ struct folio_batch fbatch; + pgoff_t index = 0, last = ULONG_MAX; + size_t ndirties = 0; + int i; +@@ -694,23 +694,26 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + index = start >> PAGE_SHIFT; + last = end >> PAGE_SHIFT; + } +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); + repeat: + if (unlikely(index > last) || +- !pagevec_lookup_range_tag(&pvec, mapping, &index, last, +- PAGECACHE_TAG_DIRTY)) ++ !filemap_get_folios_tag(mapping, &index, last, ++ PAGECACHE_TAG_DIRTY, &fbatch)) + return ndirties; + +- for (i = 0; i < pagevec_count(&pvec); i++) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct buffer_head *bh, *head; +- struct page *page = pvec.pages[i]; ++ struct folio *folio = fbatch.folios[i]; + +- lock_page(page); +- if (!page_has_buffers(page)) +- create_empty_buffers(page, i_blocksize(inode), 0); +- unlock_page(page); ++ folio_lock(folio); ++ head = folio_buffers(folio); ++ if (!head) { ++ create_empty_buffers(&folio->page, i_blocksize(inode), 0); ++ head = folio_buffers(folio); ++ } ++ folio_unlock(folio); + +- bh = head = page_buffers(page); ++ bh = head; + do { + if (!buffer_dirty(bh) || buffer_async_write(bh)) + continue; +@@ -718,13 +721,13 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + list_add_tail(&bh->b_assoc_buffers, listp); + ndirties++; + if (unlikely(ndirties >= nlimit)) { +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + return ndirties; + } + } while (bh = bh->b_this_page, bh != head); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + goto repeat; + } +@@ -734,20 +737,19 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, + { + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode = ii->i_assoc_inode; +- struct pagevec pvec; ++ struct folio_batch fbatch; + struct buffer_head *bh, *head; + unsigned int i; + pgoff_t index = 0; + + if (!btnc_inode) + return; ++ folio_batch_init(&fbatch); + +- pagevec_init(&pvec); +- +- while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, +- PAGECACHE_TAG_DIRTY)) { +- for (i = 0; i < pagevec_count(&pvec); i++) { +- bh = head = page_buffers(pvec.pages[i]); ++ while (filemap_get_folios_tag(btnc_inode->i_mapping, &index, ++ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, &fbatch)) { ++ for (i = 0; i < folio_batch_count(&fbatch); i++) { ++ bh = head = folio_buffers(fbatch.folios[i]); + do { + if (buffer_dirty(bh) && + !buffer_async_write(bh)) { +@@ -758,7 +760,7 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, + bh = bh->b_this_page; + } while (bh != head); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h +index 39f05e00dac4..9fa4c4e0809b 100644 +--- a/include/linux/pagemap.h ++++ b/include/linux/pagemap.h +@@ -547,6 +547,26 @@ static inline struct folio *filemap_lock_folio(struct address_space *mapping, + return __filemap_get_folio(mapping, index, FGP_LOCK, 0); + } + ++/** ++ * filemap_grab_folio - grab a folio from the page cache ++ * @mapping: The address space to search ++ * @index: The page index ++ * ++ * Looks up the page cache entry at @mapping & @index. If no folio is found, ++ * a new folio is created. The folio is locked, marked as accessed, and ++ * returned. ++ * ++ * Return: A found or created folio. NULL if no folio is found and failed to ++ * create a folio. ++ */ ++static inline struct folio *filemap_grab_folio(struct address_space *mapping, ++ pgoff_t index) ++{ ++ return __filemap_get_folio(mapping, index, ++ FGP_LOCK | FGP_ACCESSED | FGP_CREAT, ++ mapping_gfp_mask(mapping)); ++} ++ + /** + * find_get_page - find and get a page reference + * @mapping: the address_space to search +@@ -720,16 +740,8 @@ unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, + pgoff_t end, struct folio_batch *fbatch); + unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, + unsigned int nr_pages, struct page **pages); +-unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, +- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, +- struct page **pages); +-static inline unsigned find_get_pages_tag(struct address_space *mapping, +- pgoff_t *index, xa_mark_t tag, unsigned int nr_pages, +- struct page **pages) +-{ +- return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag, +- nr_pages, pages); +-} ++unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, ++ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch); + + struct page *grab_cache_page_write_begin(struct address_space *mapping, + pgoff_t index); +diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h +index 215eb6c3bdc9..a520632297ac 100644 +--- a/include/linux/pagevec.h ++++ b/include/linux/pagevec.h +@@ -26,14 +26,6 @@ struct pagevec { + }; + + void __pagevec_release(struct pagevec *pvec); +-unsigned pagevec_lookup_range_tag(struct pagevec *pvec, +- struct address_space *mapping, pgoff_t *index, pgoff_t end, +- xa_mark_t tag); +-static inline unsigned pagevec_lookup_tag(struct pagevec *pvec, +- struct address_space *mapping, pgoff_t *index, xa_mark_t tag) +-{ +- return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag); +-} + + static inline void pagevec_init(struct pagevec *pvec) + { +diff --git a/mm/filemap.c b/mm/filemap.c +index 15800334147b..b986f246a6ae 100644 +--- a/mm/filemap.c ++++ b/mm/filemap.c +@@ -503,28 +503,30 @@ static void __filemap_fdatawait_range(struct address_space *mapping, + { + pgoff_t index = start_byte >> PAGE_SHIFT; + pgoff_t end = end_byte >> PAGE_SHIFT; +- struct pagevec pvec; +- int nr_pages; ++ struct folio_batch fbatch; ++ unsigned nr_folios; + + if (end_byte < start_byte) + return; + +- pagevec_init(&pvec); ++ folio_batch_init(&fbatch); ++ + while (index <= end) { + unsigned i; + +- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, +- end, PAGECACHE_TAG_WRITEBACK); +- if (!nr_pages) ++ nr_folios = filemap_get_folios_tag(mapping, &index, end, ++ PAGECACHE_TAG_WRITEBACK, &fbatch); ++ ++ if (!nr_folios) + break; + +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- wait_on_page_writeback(page); +- ClearPageError(page); ++ folio_wait_writeback(folio); ++ folio_clear_error(folio); + } +- pagevec_release(&pvec); ++ folio_batch_release(&fbatch); + cond_resched(); + } + } +@@ -2255,64 +2257,57 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, + EXPORT_SYMBOL(find_get_pages_contig); + + /** +- * find_get_pages_range_tag - Find and return head pages matching @tag. +- * @mapping: the address_space to search +- * @index: the starting page index +- * @end: The final page index (inclusive) +- * @tag: the tag index +- * @nr_pages: the maximum number of pages +- * @pages: where the resulting pages are placed ++ * filemap_get_folios_tag - Get a batch of folios matching @tag. ++ * @mapping: The address_space to search ++ * @start: The starting page index ++ * @end: The final page index (inclusive) ++ * @tag: The tag index ++ * @fbatch: The batch to fill + * +- * Like find_get_pages_range(), except we only return head pages which are +- * tagged with @tag. @index is updated to the index immediately after the +- * last page we return, ready for the next iteration. ++ * Same as filemap_get_folios, but only returning folios tagged with @tag + * +- * Return: the number of pages which were found. ++ * Return: The number of folios found ++ * Also update @start to index the next folio for traversal */ - --static int gfs2_write_jdata_pagevec(struct address_space *mapping, -+static int gfs2_write_jdata_batch(struct address_space *mapping, - struct writeback_control *wbc, -- struct pagevec *pvec, -- int nr_pages, -+ struct folio_batch *fbatch, - pgoff_t *done_index) +-unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, +- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, +- struct page **pages) ++unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, ++ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) { - struct inode *inode = mapping->host; - struct gfs2_sbd *sdp = GFS2_SB(inode); -- unsigned nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); -+ unsigned nrblocks; - int i; - int ret; -+ int nr_pages = 0; -+ int nr_folios = folio_batch_count(fbatch); -+ -+ for (i = 0; i < nr_folios; i++) -+ nr_pages += folio_nr_pages(fbatch->folios[i]); -+ nrblocks = nr_pages * (PAGE_SIZE >> inode->i_blkbits); - - ret = gfs2_trans_begin(sdp, nrblocks, nrblocks); - if (ret < 0) - return ret; - -- for(i = 0; i < nr_pages; i++) { -- struct page *page = pvec->pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch->folios[i]; - -- *done_index = page->index; -+ *done_index = folio->index; - -- lock_page(page); -+ folio_lock(folio); +- XA_STATE(xas, &mapping->i_pages, *index); ++ XA_STATE(xas, &mapping->i_pages, *start); + struct folio *folio; +- unsigned ret = 0; +- +- if (unlikely(!nr_pages)) +- return 0; -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); + rcu_read_lock(); +- while ((folio = find_get_entry(&xas, end, tag))) { +- /* +- * Shadow entries should never be tagged, but this iteration ++ while ((folio = find_get_entry(&xas, end, tag)) != NULL) { ++ /* Shadow entries should never be tagged, but this iteration + * is lockless so there is a window for page reclaim to evict +- * a page we saw tagged. Skip over it. ++ * a page we saw tagged. Skip over it. + */ + if (xa_is_value(folio)) continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } ++ if (!folio_batch_add(fbatch, folio)) { ++ unsigned long nr = folio_nr_pages(folio); -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - else - goto continue_unlock; +- pages[ret] = &folio->page; +- if (++ret == nr_pages) { +- *index = folio->index + folio_nr_pages(folio); ++ if (folio_test_hugetlb(folio)) ++ nr = 1; ++ *start = folio->index + nr; + goto out; } + } +- + /* +- * We come here when we got to @end. We take care to not overflow the +- * index @index as it confuses some of the callers. This breaks the +- * iteration when there is a page at index -1 but that is already +- * broken anyway. ++ * We come here when there is no page beyond @end. We take care to not ++ * overflow the index @start as it confuses some of the callers. This ++ * breaks the iteration when there is a page at index -1 but that is ++ * already broke anyway. + */ + if (end == (pgoff_t)-1) +- *index = (pgoff_t)-1; ++ *start = (pgoff_t)-1; + else +- *index = end + 1; ++ *start = end + 1; + out: + rcu_read_unlock(); -- BUG_ON(PageWriteback(page)); -- if (!clear_page_dirty_for_io(page)) -+ BUG_ON(folio_test_writeback(folio)); -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - trace_wbc_writepage(wbc, inode_to_bdi(inode)); - -- ret = __gfs2_jdata_writepage(page, wbc); -+ ret = __gfs2_jdata_writepage(&folio->page, wbc); - if (unlikely(ret)) { - if (ret == AOP_WRITEPAGE_ACTIVATE) { -- unlock_page(page); -+ folio_unlock(folio); - ret = 0; - } else { +- return ret; ++ return folio_batch_count(fbatch); + } +-EXPORT_SYMBOL(find_get_pages_range_tag); ++EXPORT_SYMBOL(filemap_get_folios_tag); -@@ -268,7 +272,8 @@ static int gfs2_write_jdata_pagevec(struct address_space *mapping, - * not be suitable for data integrity - * writeout). - */ -- *done_index = page->index + 1; -+ *done_index = folio->index + -+ folio_nr_pages(folio); - ret = 1; - break; - } -@@ -305,8 +310,8 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - { + /* + * CD/DVDs are error prone. When a medium error occurs, the driver may fail +diff --git a/mm/page-writeback.c b/mm/page-writeback.c +index cbcecd565c16..8f20c037a87a 100644 +--- a/mm/page-writeback.c ++++ b/mm/page-writeback.c +@@ -2293,15 +2293,15 @@ int write_cache_pages(struct address_space *mapping, int ret = 0; int done = 0; + int error; - struct pagevec pvec; - int nr_pages; + struct folio_batch fbatch; + int nr_folios; - pgoff_t writeback_index; pgoff_t index; - pgoff_t end; -@@ -315,7 +320,7 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, + pgoff_t end; /* Inclusive */ + pgoff_t done_index; int range_whole = 0; xa_mark_t tag; - pagevec_init(&pvec); + folio_batch_init(&fbatch); if (wbc->range_cyclic) { - writeback_index = mapping->writeback_index; /* prev offset */ - index = writeback_index; -@@ -341,17 +346,18 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, - tag_pages_for_writeback(mapping, index, end); - done_index = index; + index = mapping->writeback_index; /* prev offset */ + end = -1; +@@ -2321,17 +2321,18 @@ int write_cache_pages(struct address_space *mapping, while (!done && (index <= end)) { + int i; + - nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, - tag); - if (nr_pages == 0) + nr_folios = filemap_get_folios_tag(mapping, &index, end, + tag, &fbatch); ++ + if (nr_folios == 0) break; -- ret = gfs2_write_jdata_pagevec(mapping, wbc, &pvec, nr_pages, &done_index); -+ ret = gfs2_write_jdata_batch(mapping, wbc, &fbatch, -+ &done_index); - if (ret) - done = 1; - if (ret > 0) - ret = 0; +- for (i = 0; i < nr_pages; i++) { +- struct page *page = pvec.pages[i]; ++ for (i = 0; i < nr_folios; i++) { ++ struct folio *folio = fbatch.folios[i]; + +- done_index = page->index; ++ done_index = folio->index; + +- lock_page(page); ++ folio_lock(folio); + + /* + * Page truncated or invalidated. We can freely skip it +@@ -2341,30 +2342,30 @@ int write_cache_pages(struct address_space *mapping, + * even if there is now a new, dirty page at the same + * pagecache address. + */ +- if (unlikely(page->mapping != mapping)) { ++ if (unlikely(folio->mapping != mapping)) { + continue_unlock: +- unlock_page(page); ++ folio_unlock(folio); + continue; + } + +- if (!PageDirty(page)) { ++ if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + +- if (PageWriteback(page)) { ++ if (folio_test_writeback(folio)) { + if (wbc->sync_mode != WB_SYNC_NONE) +- wait_on_page_writeback(page); ++ folio_wait_writeback(folio); + else + goto continue_unlock; + } + +- BUG_ON(PageWriteback(page)); +- if (!clear_page_dirty_for_io(page)) ++ BUG_ON(folio_test_writeback(folio)); ++ if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); +- error = (*writepage)(page, wbc, data); ++ error = writepage(&folio->page, wbc, data); + if (unlikely(error)) { + /* + * Handle errors according to the type of +@@ -2379,11 +2380,12 @@ int write_cache_pages(struct address_space *mapping, + * the first error. + */ + if (error == AOP_WRITEPAGE_ACTIVATE) { +- unlock_page(page); ++ folio_unlock(folio); + error = 0; + } else if (wbc->sync_mode != WB_SYNC_ALL) { + ret = error; +- done_index = page->index + 1; ++ done_index = folio->index + ++ folio_nr_pages(folio); + done = 1; + break; + } +@@ -2403,7 +2405,7 @@ int write_cache_pages(struct address_space *mapping, + break; + } + } - pagevec_release(&pvec); + folio_batch_release(&fbatch); cond_resched(); } -diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c -index 9f4d9432d38a..1e26f32a4e36 100644 ---- a/fs/nilfs2/btree.c -+++ b/fs/nilfs2/btree.c -@@ -2143,7 +2143,7 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, - struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; - struct address_space *btcache = btnc_inode->i_mapping; - struct list_head lists[NILFS_BTREE_LEVEL_MAX]; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct buffer_head *bh, *head; - pgoff_t index = 0; - int level, i; -@@ -2153,19 +2153,19 @@ static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, - level++) - INIT_LIST_HEAD(&lists[level]); +diff --git a/mm/swap.c b/mm/swap.c +index 027943b341a0..29bf97746da0 100644 +--- a/mm/swap.c ++++ b/mm/swap.c +@@ -1099,16 +1099,6 @@ void folio_batch_remove_exceptionals(struct folio_batch *fbatch) + fbatch->nr = j; + } -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); +-unsigned pagevec_lookup_range_tag(struct pagevec *pvec, +- struct address_space *mapping, pgoff_t *index, pgoff_t end, +- xa_mark_t tag) +-{ +- pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, +- PAGEVEC_SIZE, pvec->pages); +- return pagevec_count(pvec); +-} +-EXPORT_SYMBOL(pagevec_lookup_range_tag); +- + /* + * Perform any setup for the swap system + */ +-- +2.38.0.rc1.8.g2a7d63a245 + +From da70f4396195cb2e56bcfe68c95ea4e31c933e6b Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 19 Sep 2022 14:42:00 +0200 +Subject: [PATCH 14/16] fixes + +Signed-off-by: Peter Jung +--- + Documentation/ABI/stable/sysfs-block | 10 + + .../testing/sysfs-class-led-trigger-blkdev | 68 + + Documentation/leds/index.rst | 1 + + Documentation/leds/ledtrig-blkdev.rst | 155 +++ + arch/x86/events/amd/uncore.c | 1 + + arch/x86/kernel/alternative.c | 9 + + arch/x86/net/bpf_jit_comp.c | 4 +- + drivers/leds/trigger/Kconfig | 9 + + drivers/leds/trigger/Makefile | 1 + + drivers/leds/trigger/ledtrig-blkdev.c | 1177 +++++++++++++++++ + include/linux/pageblock-flags.h | 2 +- + include/net/bluetooth/rfcomm.h | 3 + + net/bluetooth/rfcomm/core.c | 2 + + net/bluetooth/rfcomm/sock.c | 34 +- + tools/objtool/check.c | 3 - + 15 files changed, 1462 insertions(+), 17 deletions(-) + create mode 100644 Documentation/ABI/testing/sysfs-class-led-trigger-blkdev + create mode 100644 Documentation/leds/ledtrig-blkdev.rst + create mode 100644 drivers/leds/trigger/ledtrig-blkdev.c + +diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block +index cd14ecb3c9a5..d11d04d804fe 100644 +--- a/Documentation/ABI/stable/sysfs-block ++++ b/Documentation/ABI/stable/sysfs-block +@@ -101,6 +101,16 @@ Description: + devices that support receiving integrity metadata. + + ++What: /sys/block//linked_leds ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Directory that contains symbolic links to all LEDs that ++ are associated with (linked to) this block device by the ++ blkdev LED trigger. Only present when at least one LED ++ is linked. (See Documentation/leds/ledtrig-blkdev.rst.) ++ ++ + What: /sys/block///alignment_offset + Date: April 2009 + Contact: Martin K. Petersen +diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev +new file mode 100644 +index 000000000000..d8d403569b21 +--- /dev/null ++++ b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev +@@ -0,0 +1,68 @@ ++What: /sys/class/leds//blink_time ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Time (in milliseconds) that the LED will be on during a single ++ "blink". ++ ++What: /sys/class/leds//check_interval ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Frequency (in milliseconds) with which block devices linked to ++ this LED will be checked for activity and the LED will ++ (potentially) be blinked. ++ ++What: /sys/class/leds//blink_on_read ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to read activity on any of its linked block devices. ++ ++What: /sys/class/leds//blink_on_write ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to write activity on any of its linked block devices. ++ ++What: /sys/class/leds//blink_on_discard ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to discard activity on any of its linked block devices. ++ ++What: /sys/class/leds//blink_on_flush ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Boolean that determines whether the LED will blink in response ++ to cache flush activity on any of its linked block devices. ++ ++What: /sys/class/leds//link_dev_by_path ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Associate a block device with this LED by writing the path to ++ the device special file (e.g. /dev/sda) to this attribute. ++ Symbolic links are followed. ++ ++What: /sys/class/leds//unlink_dev_by_path ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Remove the association between this LED and a block device by ++ writing the path to the device special file (e.g. /dev/sda) to ++ this attribute. Symbolic links are followed. ++ ++What: /sys/class/leds//linked_devices ++Date: September 2022 ++Contact: Ian Pilcher ++Description: ++ Directory containing links to all block devices that are ++ associated with this LED. (Note that the names of the ++ symbolic links in this directory are *kernel* names, which ++ may not match the device special file paths written to ++ link_device and unlink_device.) +diff --git a/Documentation/leds/index.rst b/Documentation/leds/index.rst +index e5d63b940045..e3c24e468cbc 100644 +--- a/Documentation/leds/index.rst ++++ b/Documentation/leds/index.rst +@@ -10,6 +10,7 @@ LEDs + leds-class + leds-class-flash + leds-class-multicolor ++ ledtrig-blkdev + ledtrig-oneshot + ledtrig-transient + ledtrig-usbport +diff --git a/Documentation/leds/ledtrig-blkdev.rst b/Documentation/leds/ledtrig-blkdev.rst +new file mode 100644 +index 000000000000..ae92aa559257 +--- /dev/null ++++ b/Documentation/leds/ledtrig-blkdev.rst +@@ -0,0 +1,155 @@ ++.. SPDX-License-Identifier: GPL-2.0 ++ ++================================= ++Block Device (blkdev) LED Trigger ++================================= ++ ++Available when ``CONFIG_LEDS_TRIGGER_BLKDEV=y`` or ++``CONFIG_LEDS_TRIGGER_BLKDEV=m``. ++ ++See also: ++ ++* ``Documentation/ABI/testing/sysfs-class-led-trigger-blkdev`` ++* ``Documentation/ABI/stable/sysfs-block`` (``/sys/block//linked_leds``) ++ ++Overview ++======== ++ ++.. note:: ++ The examples below use ```` to refer to the name of a ++ system-specific LED. If no suitable LED is available on a test ++ system (in a virtual machine, for example), it is possible to ++ use a userspace LED. (See ``Documentation/leds/uleds.rst``.) ++ ++Verify that the ``blkdev`` LED trigger is available:: ++ ++ # grep blkdev /sys/class/leds//trigger ++ ... rfkill-none blkdev ++ ++(If the previous command produces no output, you may need to load the trigger ++module - ``modprobe ledtrig_blkdev``. If the module is not available, check ++the value of ``CONFIG_LEDS_TRIGGER_BLKDEV`` in your kernel configuration.) ++ ++Associate the LED with the ``blkdev`` LED trigger:: ++ ++ # echo blkdev > /sys/class/leds//trigger ++ ++ # cat /sys/class/leds//trigger ++ ... rfkill-none [blkdev] ++ ++Note that several new device attributes are available in the ++``/sys/class/leds/`` directory. ++ ++* ``link_dev_by_path`` and ``unlink_dev_by_path`` are used to manage the set of ++ block devices associated with this LED. The LED will blink in response to ++ read or write activity on its linked devices. ++ ++* ``blink_on_read``, ``blink_on_write``, ``blink_on_discard``, and ++ ``blink_on_flush`` are boolean values that determine whether the LED will ++ blink when a particular type of activity is detected on one of its linked ++ block devices. ++ ++* ``blink_time`` is the duration (in milliseconds) of each blink of this LED. ++ (The minimum value is 10 milliseconds.) ++ ++* ``check_interval`` is the frequency (in milliseconds) with which block devices ++ linked to this LED will be checked for activity and the LED blinked (if the ++ correct type of activity has occurred). ++ ++* The ``linked_devices`` directory will contain a symbolic link to every device ++ that is associated with this LED. ++ ++Link a block device to the LED:: ++ ++ # echo /dev/sda > /sys/class/leds//link_dev_by_path ++ ++ # ls /sys/class/leds//linked_devices ++ sda ++ ++(The value written to ``link_dev_by_path`` must be the path of the device ++special file, such as ``/dev/sda``, that represents the block device - or the ++path of a symbolic link to such a device special file.) ++ ++Activity on the device will now cause the LED to blink. The duration of each ++blink (in milliseconds) can be adjusted by setting ++``/sys/class/leds//blink_time``. (But see **check_interval and ++blink_time** below.) ++ ++Associate a second device with the LED:: ++ ++ # echo /dev/sdb > /sys/class/leds//link_dev_by_path ++ ++ # ls /sys/class/leds//linked_devices ++ sda sdb ++ ++When a block device is linked to one or more LEDs, the LEDs are linked from ++the device's ``linked_leds`` directory:: ++ ++ # ls /sys/class/block/sd{a,b}/linked_leds ++ /sys/class/block/sda/linked_leds: ++ ++ ++ /sys/class/block/sdb/linked_leds: ++ ++ ++(The ``linked_leds`` directory only exists when the block device is linked to ++at least one LED.) ++ ++``check_interval`` and ``blink_time`` ++===================================== ++ ++* By default, linked block devices are checked for activity every 100 ++ milliseconds. This frequency can be changed for an LED via the ++ ``/sys/class/leds//check_interval`` attribute. (The minimum value is 25 ++ milliseconds.) ++ ++* All block devices associated with an LED are checked for activity every ++ ``check_interval`` milliseconds, and a blink is triggered if the correct type ++ of activity (as determined by the LED's ``blink_on_*`` attributes) is ++ detected. The duration of an LED's blink is determined by its ``blink_time`` ++ attribute. Thus (when the correct type of activity is detected), the LED will ++ be on for ``blink_time`` milliseconds and off for ++ ``check_interval - blink_time`` milliseconds. ++ ++* The LED subsystem ignores new blink requests for an LED that is already in ++ in the process of blinking, so setting a ``blink_time`` greater than or equal ++ to ``check_interval`` will cause some blinks to be missed. ++ ++* Because of processing times, scheduling latencies, etc., avoiding missed ++ blinks actually requires a difference of at least a few milliseconds between ++ the ``blink_time`` and ``check_interval``. The required difference is likely ++ to vary from system to system. As a reference, a Thecus N5550 NAS requires a ++ difference of 7 milliseconds (e.g. ``check_interval == 100``, ++ ``blink_time == 93``). ++ ++* The default values (``check_interval == 100``, ``blink_time == 75``) cause the ++ LED associated with a continuously active device to blink rapidly. For a more ++ "always on" effect, increase the ``blink_time`` (but not too much; see the ++ previous bullet). ++ ++Other Notes ++=========== ++ ++* Many (possibly all) types of block devices work with this trigger, including: ++ ++ * SCSI (including SATA and USB) hard disk drives and SSDs ++ * SCSI (including SATA and USB) optical drives ++ * NVMe SSDs ++ * SD cards ++ * loopback block devices (``/dev/loop*``) ++ * device mapper devices, such as LVM logical volumes ++ * MD RAID devices ++ * zRAM compressed RAM-disks ++ * partitions on block devics that support them ++ ++* The names of the symbolic links in ``/sys/class/leds//linked_devices`` ++ are **kernel** names, which may not match the paths used for ++ ``link_dev_by_path`` and ``unlink_dev_by_path``. This is most likely when a ++ symbolic link is used to refer to the device (as is common with logical ++ volumes), but it can be true for any device, because nothing prevents the ++ creation of device special files with arbitrary names (e.g. ++ ``sudo mknod /foo b 8 0``). ++ ++* The ``blkdev`` LED trigger supports many-to-many device/LED associations. ++ A device can be associated with multiple LEDs, and an LED can be associated ++ with multiple devices. +diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c +index d568afc705d2..83f15fe411b3 100644 +--- a/arch/x86/events/amd/uncore.c ++++ b/arch/x86/events/amd/uncore.c +@@ -553,6 +553,7 @@ static void uncore_clean_online(void) -- while (pagevec_lookup_tag(&pvec, btcache, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- bh = head = page_buffers(pvec.pages[i]); -+ while (filemap_get_folios_tag(btcache, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ bh = head = folio_buffers(fbatch.folios[i]); - do { - if (buffer_dirty(bh)) - nilfs_btree_add_dirty_buffer(btree, - lists, bh); - } while ((bh = bh->b_this_page) != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); + hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { + hlist_del(&uncore->node); ++ kfree(uncore->events); + kfree(uncore); } + } +diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c +index f9c9b5850847..3582e61ddce6 100644 +--- a/arch/x86/kernel/alternative.c ++++ b/arch/x86/kernel/alternative.c +@@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) + return ret; + i += ret; -diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c -index 3267e96c256c..b66f4e988016 100644 ---- a/fs/nilfs2/page.c -+++ b/fs/nilfs2/page.c -@@ -240,42 +240,43 @@ static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty) - int nilfs_copy_dirty_pages(struct address_space *dmap, - struct address_space *smap) - { -- struct pagevec pvec; -+ struct folio_batch fbatch; - unsigned int i; - pgoff_t index = 0; - int err = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - repeat: -- if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY)) -+ if (!filemap_get_folios_tag(smap, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) - return 0; - -- for (i = 0; i < pagevec_count(&pvec); i++) { -- struct page *page = pvec.pages[i], *dpage; -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ struct folio *folio = fbatch.folios[i], *dfolio; - -- lock_page(page); -- if (unlikely(!PageDirty(page))) -- NILFS_PAGE_BUG(page, "inconsistent dirty state"); -+ folio_lock(folio); -+ if (unlikely(!folio_test_dirty(folio))) -+ NILFS_PAGE_BUG(&folio->page, "inconsistent dirty state"); - -- dpage = grab_cache_page(dmap, page->index); -- if (unlikely(!dpage)) { -+ dfolio = filemap_grab_folio(dmap, folio->index); -+ if (unlikely(!dfolio)) { - /* No empty page is added to the page cache */ - err = -ENOMEM; -- unlock_page(page); -+ folio_unlock(folio); - break; - } -- if (unlikely(!page_has_buffers(page))) -- NILFS_PAGE_BUG(page, -+ if (unlikely(!folio_buffers(folio))) -+ NILFS_PAGE_BUG(&folio->page, - "found empty page in dat page cache"); - -- nilfs_copy_page(dpage, page, 1); -- __set_page_dirty_nobuffers(dpage); -+ nilfs_copy_page(&dfolio->page, &folio->page, 1); -+ filemap_dirty_folio(folio_mapping(dfolio), dfolio); ++ /* ++ * The compiler is supposed to EMIT an INT3 after every unconditional ++ * JMP instruction due to AMD BTC. However, if the compiler is too old ++ * or SLS isn't enabled, we still need an INT3 after indirect JMPs ++ * even on Intel. ++ */ ++ if (op == JMP32_INSN_OPCODE && i < insn->length) ++ bytes[i++] = INT3_INSN_OPCODE; ++ + for (; i < insn->length;) + bytes[i++] = BYTES_NOP1; -- unlock_page(dpage); -- put_page(dpage); -- unlock_page(page); -+ folio_unlock(dfolio); -+ folio_put(dfolio); -+ folio_unlock(folio); +diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c +index c1f6c1c51d99..4922517ddb0d 100644 +--- a/arch/x86/net/bpf_jit_comp.c ++++ b/arch/x86/net/bpf_jit_comp.c +@@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) + OPTIMIZER_HIDE_VAR(reg); + emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); + } else { +- EMIT2(0xFF, 0xE0 + reg); ++ EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ ++ if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS)) ++ EMIT1(0xCC); /* int3 */ } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - - if (likely(!err)) -@@ -357,22 +358,22 @@ void nilfs_copy_back_pages(struct address_space *dmap, - */ - void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent) - { -- struct pagevec pvec; -+ struct folio_batch fbatch; - unsigned int i; - pgoff_t index = 0; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - -- while (pagevec_lookup_tag(&pvec, mapping, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- struct page *page = pvec.pages[i]; -+ while (filemap_get_folios_tag(mapping, &index, (pgoff_t)-1, -+ PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ struct folio *folio = fbatch.folios[i]; -- lock_page(page); -- nilfs_clear_dirty_page(page, silent); -- unlock_page(page); -+ folio_lock(folio); -+ nilfs_clear_dirty_page(&folio->page, silent); -+ folio_unlock(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c -index 0afe0832c754..6f2ca279d230 100644 ---- a/fs/nilfs2/segment.c -+++ b/fs/nilfs2/segment.c -@@ -680,7 +680,7 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - loff_t start, loff_t end) - { - struct address_space *mapping = inode->i_mapping; -- struct pagevec pvec; -+ struct folio_batch fbatch; - pgoff_t index = 0, last = ULONG_MAX; - size_t ndirties = 0; - int i; -@@ -694,23 +694,26 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - index = start >> PAGE_SHIFT; - last = end >> PAGE_SHIFT; - } -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - repeat: - if (unlikely(index > last) || -- !pagevec_lookup_range_tag(&pvec, mapping, &index, last, -- PAGECACHE_TAG_DIRTY)) -+ !filemap_get_folios_tag(mapping, &index, last, -+ PAGECACHE_TAG_DIRTY, &fbatch)) - return ndirties; + *pprog = prog; +diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig +index dc6816d36d06..bda249068182 100644 +--- a/drivers/leds/trigger/Kconfig ++++ b/drivers/leds/trigger/Kconfig +@@ -154,4 +154,13 @@ config LEDS_TRIGGER_TTY -- for (i = 0; i < pagevec_count(&pvec); i++) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { - struct buffer_head *bh, *head; -- struct page *page = pvec.pages[i]; -+ struct folio *folio = fbatch.folios[i]; + When build as a module this driver will be called ledtrig-tty. -- lock_page(page); -- if (!page_has_buffers(page)) -- create_empty_buffers(page, i_blocksize(inode), 0); -- unlock_page(page); -+ folio_lock(folio); -+ head = folio_buffers(folio); -+ if (!head) { -+ create_empty_buffers(&folio->page, i_blocksize(inode), 0); -+ head = folio_buffers(folio); ++config LEDS_TRIGGER_BLKDEV ++ tristate "LED Trigger for block devices" ++ depends on BLOCK ++ help ++ The blkdev LED trigger allows LEDs to be controlled by block device ++ activity (reads and writes). ++ ++ See Documentation/leds/ledtrig-blkdev.rst. ++ + endif # LEDS_TRIGGERS +diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile +index 25c4db97cdd4..d53bab5d93f1 100644 +--- a/drivers/leds/trigger/Makefile ++++ b/drivers/leds/trigger/Makefile +@@ -16,3 +16,4 @@ obj-$(CONFIG_LEDS_TRIGGER_NETDEV) += ledtrig-netdev.o + obj-$(CONFIG_LEDS_TRIGGER_PATTERN) += ledtrig-pattern.o + obj-$(CONFIG_LEDS_TRIGGER_AUDIO) += ledtrig-audio.o + obj-$(CONFIG_LEDS_TRIGGER_TTY) += ledtrig-tty.o ++obj-$(CONFIG_LEDS_TRIGGER_BLKDEV) += ledtrig-blkdev.o +diff --git a/drivers/leds/trigger/ledtrig-blkdev.c b/drivers/leds/trigger/ledtrig-blkdev.c +new file mode 100644 +index 000000000000..0d2eb36fbe38 +--- /dev/null ++++ b/drivers/leds/trigger/ledtrig-blkdev.c +@@ -0,0 +1,1177 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++ ++/* ++ * Block device LED trigger ++ * ++ * Copyright 2021-2022 Ian Pilcher ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++/** ++ * DOC: Overview ++ * ++ * The ``blkdev`` LED trigger works by periodically checking the activity ++ * counters of block devices that have been linked to one or more LEDs and ++ * blinking those LED(s) if the correct type of activity has occurred. The ++ * periodic check is scheduled with the Linux kernel's deferred work facility. ++ * ++ * Trigger-specific data about block devices and LEDs is stored in two data ++ * structures --- &struct blkdev_trig_bdev (a "BTB") and &struct blkdev_trig_led ++ * (a "BTL"). Each structure contains a &struct xarray that holds links to any ++ * linked devices of the other type. I.e. &blkdev_trig_bdev.linked_btls ++ * contains links to all BTLs whose LEDs have been linked to the BTB's block ++ * device, and &blkdev_trig_led.linked_btbs contains links to all BTBs whose ++ * block devices have been linked to the BTL's LED. Thus, a block device can ++ * be linked to more than one LED, and an LED can be linked to more than one ++ * block device. ++ */ ++ ++/* Default, minimum & maximum blink duration (milliseconds) */ ++#define BLKDEV_TRIG_BLINK_DEF 75 ++#define BLKDEV_TRIG_BLINK_MIN 10 ++#define BLKDEV_TRIG_BLINK_MAX 86400000 /* 24 hours */ ++ ++/* Default, minimum & maximum activity check interval (milliseconds) */ ++#define BLKDEV_TRIG_CHECK_DEF 100 ++#define BLKDEV_TRIG_CHECK_MIN 25 ++#define BLKDEV_TRIG_CHECK_MAX 86400000 /* 24 hours */ ++ ++/* ++ * If blkdev_trig_check() can't lock the mutex, how long to wait before trying ++ * again (milliseconds) ++ */ ++#define BLKDEV_TRIG_CHECK_RETRY 5 ++ ++/* Mode argument for calls to blkdev_get_by_path() and blkdev_put() */ ++#define BLKDEV_TRIG_FMODE 0 ++ ++/** ++ * struct blkdev_trig_bdev - Trigger-specific data about a block device. ++ * @last_checked: Time (in jiffies) at which the trigger last checked this ++ * block device for activity. ++ * @last_activity: Time (in jiffies) at which the trigger last detected ++ * activity of each type. ++ * @ios: Activity counter values for each type, corresponding to ++ * the timestamps in &last_activity. ++ * @index: &xarray index, so the BTB can be included in one or more ++ * &blkdev_trig_led.linked_btbs. ++ * @bdev: The block device. ++ * @linked_btls: The BTLs that represent the LEDs linked to the BTB's ++ * block device. ++ * ++ * Every block device linked to at least one LED gets a "BTB." A BTB is created ++ * when a block device that is not currently linked to any LEDs is linked to an ++ * LED. ++ * ++ * A BTB is freed when one of the following occurs: ++ * ++ * * The number of LEDs linked to the block device becomes zero, because it has ++ * been unlinked from its last LED using the trigger's &sysfs interface. ++ * ++ * * The number of LEDs linked to the block device becomes zero, because the ++ * last LED to which it was linked has been disassociated from the trigger ++ * (which happens automatically if the LED device is removed from the system). ++ * ++ * * The BTB's block device is removed from the system. To accomodate this ++ * scenario, BTB's are created as device resources, so that the release ++ * function will be called by the driver core when the device is removed. ++ */ ++struct blkdev_trig_bdev { ++ unsigned long last_checked; ++ unsigned long last_activity[NR_STAT_GROUPS]; ++ unsigned long ios[NR_STAT_GROUPS]; ++ unsigned long index; ++ struct block_device *bdev; ++ struct xarray linked_btls; ++}; ++ ++/** ++ * struct blkdev_trig_led - Trigger-specific data about an LED. ++ * @last_checked: Time (in jiffies) at which the trigger last checked the ++ * the block devices linked to this LED for activity. ++ * @index: &xarray index, so the BTL can be included in one or more ++ * &blkdev_trig_bdev.linked_btls. ++ * @mode: Bitmask for types of block device activity that will ++ * cause this LED to blink --- reads, writes, discards, ++ * etc. ++ * @led: The LED device. ++ * @blink_msec: Duration of a blink (milliseconds). ++ * @check_jiffies: Frequency with which block devices linked to this LED ++ * should be checked for activity (jiffies). ++ * @linked_btbs: The BTBs that represent the block devices linked to the ++ * BTL's LED. ++ * @all_btls_node: The BTL's node in the module's list of all BTLs. ++ * ++ * Every LED associated with the block device trigger gets a "BTL." A BTL is ++ * created when the trigger is "activated" on an LED (usually by writing ++ * ``blkdev`` to the LED's &sysfs &trigger attribute). A BTL is freed wnen its ++ * LED is disassociated from the trigger, either through the trigger's &sysfs ++ * interface or because the LED device is removed from the system. ++ */ ++struct blkdev_trig_led { ++ unsigned long last_checked; ++ unsigned long index; ++ unsigned long mode; /* must be ulong for atomic bit ops */ ++ struct led_classdev *led; ++ unsigned int blink_msec; ++ unsigned int check_jiffies; ++ struct xarray linked_btbs; ++ struct hlist_node all_btls_node; ++}; ++ ++/* Protects everything except atomic LED attributes */ ++static DEFINE_MUTEX(blkdev_trig_mutex); ++ ++/* BTB device resource release function */ ++static void blkdev_trig_btb_release(struct device *dev, void *res); ++ ++/* Index for next BTB or BTL */ ++static unsigned long blkdev_trig_next_index; ++ ++/* All LEDs associated with the trigger */ ++static HLIST_HEAD(blkdev_trig_all_btls); ++ ++/* Delayed work to periodically check for activity & blink LEDs */ ++static void blkdev_trig_check(struct work_struct *work); ++static DECLARE_DELAYED_WORK(blkdev_trig_work, blkdev_trig_check); ++ ++/* When is the delayed work scheduled to run next (jiffies) */ ++static unsigned long blkdev_trig_next_check; ++ ++/* Total number of BTB-to-BTL links */ ++static unsigned int blkdev_trig_link_count; ++ ++/* Empty sysfs attribute list for next 2 declarations */ ++static struct attribute *blkdev_trig_attrs_empty[] = { NULL }; ++ ++/* linked_leds sysfs directory for block devs linked to 1 or more LEDs */ ++static const struct attribute_group blkdev_trig_linked_leds = { ++ .name = "linked_leds", ++ .attrs = blkdev_trig_attrs_empty, ++}; ++ ++/* linked_devices sysfs directory for each LED associated with the trigger */ ++static const struct attribute_group blkdev_trig_linked_devs = { ++ .name = "linked_devices", ++ .attrs = blkdev_trig_attrs_empty, ++}; ++ ++ ++/* ++ * ++ * Delayed work to check for activity & blink LEDs ++ * ++ */ ++ ++/** ++ * blkdev_trig_blink() - Blink an LED, if the correct type of activity has ++ * occurred on the block device. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Return: &true if the LED is blinked, &false if not. ++ */ ++static bool blkdev_trig_blink(const struct blkdev_trig_led *btl, ++ const struct blkdev_trig_bdev *btb) ++{ ++ unsigned long mode, mask, delay_on, delay_off; ++ enum stat_group i; ++ ++ mode = READ_ONCE(btl->mode); ++ ++ for (i = STAT_READ, mask = 1; i <= STAT_FLUSH; ++i, mask <<= 1) { ++ ++ if (!(mode & mask)) ++ continue; ++ ++ if (time_before_eq(btb->last_activity[i], btl->last_checked)) ++ continue; ++ ++ delay_on = READ_ONCE(btl->blink_msec); ++ delay_off = 1; /* 0 leaves LED turned on */ ++ ++ led_blink_set_oneshot(btl->led, &delay_on, &delay_off, 0); ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * blkdev_trig_update_btb() - Update a BTB's activity counters and timestamps. ++ * @btb: The BTB ++ * @now: Timestamp (in jiffies) ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_update_btb(struct blkdev_trig_bdev *btb, ++ unsigned long now) ++{ ++ unsigned long new_ios; ++ enum stat_group i; ++ ++ for (i = STAT_READ; i <= STAT_FLUSH; ++i) { ++ ++ new_ios = part_stat_read(btb->bdev, ios[i]); ++ ++ if (new_ios != btb->ios[i]) { ++ btb->ios[i] = new_ios; ++ btb->last_activity[i] = now; + } -+ folio_unlock(folio); - -- bh = head = page_buffers(page); -+ bh = head; - do { - if (!buffer_dirty(bh) || buffer_async_write(bh)) - continue; -@@ -718,13 +721,13 @@ static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, - list_add_tail(&bh->b_assoc_buffers, listp); - ndirties++; - if (unlikely(ndirties >= nlimit)) { -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - return ndirties; - } - } while (bh = bh->b_this_page, bh != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - goto repeat; - } -@@ -734,20 +737,19 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, - { - struct nilfs_inode_info *ii = NILFS_I(inode); - struct inode *btnc_inode = ii->i_assoc_inode; -- struct pagevec pvec; -+ struct folio_batch fbatch; - struct buffer_head *bh, *head; - unsigned int i; - pgoff_t index = 0; - - if (!btnc_inode) - return; -+ folio_batch_init(&fbatch); - -- pagevec_init(&pvec); -- -- while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, -- PAGECACHE_TAG_DIRTY)) { -- for (i = 0; i < pagevec_count(&pvec); i++) { -- bh = head = page_buffers(pvec.pages[i]); -+ while (filemap_get_folios_tag(btnc_inode->i_mapping, &index, -+ (pgoff_t)-1, PAGECACHE_TAG_DIRTY, &fbatch)) { -+ for (i = 0; i < folio_batch_count(&fbatch); i++) { -+ bh = head = folio_buffers(fbatch.folios[i]); - do { - if (buffer_dirty(bh) && - !buffer_async_write(bh)) { -@@ -758,7 +760,7 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, - bh = bh->b_this_page; - } while (bh != head); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h -index 39f05e00dac4..9fa4c4e0809b 100644 ---- a/include/linux/pagemap.h -+++ b/include/linux/pagemap.h -@@ -547,6 +547,26 @@ static inline struct folio *filemap_lock_folio(struct address_space *mapping, - return __filemap_get_folio(mapping, index, FGP_LOCK, 0); - } - ++ } ++ ++ btb->last_checked = now; ++} ++ +/** -+ * filemap_grab_folio - grab a folio from the page cache -+ * @mapping: The address space to search -+ * @index: The page index ++ * blkdev_trig_check() - Check linked devices for activity and blink LEDs. ++ * @work: Delayed work (&blkdev_trig_work) + * -+ * Looks up the page cache entry at @mapping & @index. If no folio is found, -+ * a new folio is created. The folio is locked, marked as accessed, and -+ * returned. ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_check(struct work_struct *work) ++{ ++ struct blkdev_trig_led *btl; ++ struct blkdev_trig_bdev *btb; ++ unsigned long index, delay, now, led_check, led_delay; ++ bool blinked; ++ ++ if (!mutex_trylock(&blkdev_trig_mutex)) { ++ delay = msecs_to_jiffies(BLKDEV_TRIG_CHECK_RETRY); ++ goto exit_reschedule; ++ } ++ ++ now = jiffies; ++ delay = ULONG_MAX; ++ ++ hlist_for_each_entry (btl, &blkdev_trig_all_btls, all_btls_node) { ++ ++ led_check = btl->last_checked + btl->check_jiffies; ++ ++ if (time_before_eq(led_check, now)) { ++ ++ blinked = false; ++ ++ xa_for_each (&btl->linked_btbs, index, btb) { ++ ++ if (btb->last_checked != now) ++ blkdev_trig_update_btb(btb, now); ++ if (!blinked) ++ blinked = blkdev_trig_blink(btl, btb); ++ } ++ ++ btl->last_checked = now; ++ led_delay = btl->check_jiffies; ++ ++ } else { ++ led_delay = led_check - now; ++ } ++ ++ if (led_delay < delay) ++ delay = led_delay; ++ } ++ ++ mutex_unlock(&blkdev_trig_mutex); ++ ++exit_reschedule: ++ WARN_ON_ONCE(delay == ULONG_MAX); ++ WARN_ON_ONCE(!schedule_delayed_work(&blkdev_trig_work, delay)); ++} ++ ++/** ++ * blkdev_trig_sched_led() - Set the schedule of the delayed work when a new ++ * LED is added to the schedule. ++ * @btl: The BTL that represents the LED + * -+ * Return: A found or created folio. NULL if no folio is found and failed to -+ * create a folio. ++ * Called when the number of block devices to which an LED is linked becomes ++ * non-zero. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. + */ -+static inline struct folio *filemap_grab_folio(struct address_space *mapping, -+ pgoff_t index) ++static void blkdev_trig_sched_led(const struct blkdev_trig_led *btl) +{ -+ return __filemap_get_folio(mapping, index, -+ FGP_LOCK | FGP_ACCESSED | FGP_CREAT, -+ mapping_gfp_mask(mapping)); ++ unsigned long delay = READ_ONCE(btl->check_jiffies); ++ unsigned long check_by = jiffies + delay; ++ ++ /* ++ * If no other LED-to-block device links exist, simply schedule the ++ * delayed work according to this LED's check_interval attribute ++ * (check_jiffies). ++ */ ++ if (blkdev_trig_link_count == 0) { ++ WARN_ON(!schedule_delayed_work(&blkdev_trig_work, delay)); ++ blkdev_trig_next_check = check_by; ++ return; ++ } ++ ++ /* ++ * If the next check is already scheduled to occur soon enough to ++ * accomodate this LED's check_interval, the schedule doesn't need ++ * to be changed. ++ */ ++ if (time_after_eq(check_by, blkdev_trig_next_check)) ++ return; ++ ++ /* ++ * Modify the schedule, so that the delayed work runs soon enough for ++ * this LED. ++ */ ++ WARN_ON(!mod_delayed_work(system_wq, &blkdev_trig_work, delay)); ++ blkdev_trig_next_check = check_by; +} + - /** - * find_get_page - find and get a page reference - * @mapping: the address_space to search -@@ -720,16 +740,8 @@ unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, - pgoff_t end, struct folio_batch *fbatch); - unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, - unsigned int nr_pages, struct page **pages); --unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, -- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages); --static inline unsigned find_get_pages_tag(struct address_space *mapping, -- pgoff_t *index, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages) --{ -- return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag, -- nr_pages, pages); --} -+unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, -+ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch); - - struct page *grab_cache_page_write_begin(struct address_space *mapping, - pgoff_t index); -diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h -index 215eb6c3bdc9..a520632297ac 100644 ---- a/include/linux/pagevec.h -+++ b/include/linux/pagevec.h -@@ -26,14 +26,6 @@ struct pagevec { - }; - - void __pagevec_release(struct pagevec *pvec); --unsigned pagevec_lookup_range_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, pgoff_t end, -- xa_mark_t tag); --static inline unsigned pagevec_lookup_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, xa_mark_t tag) --{ -- return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag); --} - - static inline void pagevec_init(struct pagevec *pvec) - { -diff --git a/mm/filemap.c b/mm/filemap.c -index 15800334147b..b986f246a6ae 100644 ---- a/mm/filemap.c -+++ b/mm/filemap.c -@@ -503,28 +503,30 @@ static void __filemap_fdatawait_range(struct address_space *mapping, - { - pgoff_t index = start_byte >> PAGE_SHIFT; - pgoff_t end = end_byte >> PAGE_SHIFT; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ unsigned nr_folios; - - if (end_byte < start_byte) - return; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); + - while (index <= end) { - unsigned i; - -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, -- end, PAGECACHE_TAG_WRITEBACK); -- if (!nr_pages) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ PAGECACHE_TAG_WRITEBACK, &fbatch); ++/* ++ * ++ * Linking and unlinking LEDs and block devices ++ * ++ */ + -+ if (!nr_folios) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- wait_on_page_writeback(page); -- ClearPageError(page); -+ folio_wait_writeback(folio); -+ folio_clear_error(folio); - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - } -@@ -2255,64 +2257,57 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, - EXPORT_SYMBOL(find_get_pages_contig); - - /** -- * find_get_pages_range_tag - Find and return head pages matching @tag. -- * @mapping: the address_space to search -- * @index: the starting page index -- * @end: The final page index (inclusive) -- * @tag: the tag index -- * @nr_pages: the maximum number of pages -- * @pages: where the resulting pages are placed -+ * filemap_get_folios_tag - Get a batch of folios matching @tag. -+ * @mapping: The address_space to search -+ * @start: The starting page index -+ * @end: The final page index (inclusive) -+ * @tag: The tag index -+ * @fbatch: The batch to fill - * -- * Like find_get_pages_range(), except we only return head pages which are -- * tagged with @tag. @index is updated to the index immediately after the -- * last page we return, ready for the next iteration. -+ * Same as filemap_get_folios, but only returning folios tagged with @tag - * -- * Return: the number of pages which were found. -+ * Return: The number of folios found -+ * Also update @start to index the next folio for traversal - */ --unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, -- pgoff_t end, xa_mark_t tag, unsigned int nr_pages, -- struct page **pages) -+unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, -+ pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) - { -- XA_STATE(xas, &mapping->i_pages, *index); -+ XA_STATE(xas, &mapping->i_pages, *start); - struct folio *folio; -- unsigned ret = 0; -- -- if (unlikely(!nr_pages)) -- return 0; - - rcu_read_lock(); -- while ((folio = find_get_entry(&xas, end, tag))) { -- /* -- * Shadow entries should never be tagged, but this iteration -+ while ((folio = find_get_entry(&xas, end, tag)) != NULL) { -+ /* Shadow entries should never be tagged, but this iteration - * is lockless so there is a window for page reclaim to evict -- * a page we saw tagged. Skip over it. -+ * a page we saw tagged. Skip over it. - */ - if (xa_is_value(folio)) - continue; -+ if (!folio_batch_add(fbatch, folio)) { -+ unsigned long nr = folio_nr_pages(folio); - -- pages[ret] = &folio->page; -- if (++ret == nr_pages) { -- *index = folio->index + folio_nr_pages(folio); -+ if (folio_test_hugetlb(folio)) -+ nr = 1; -+ *start = folio->index + nr; - goto out; - } - } -- - /* -- * We come here when we got to @end. We take care to not overflow the -- * index @index as it confuses some of the callers. This breaks the -- * iteration when there is a page at index -1 but that is already -- * broken anyway. -+ * We come here when there is no page beyond @end. We take care to not -+ * overflow the index @start as it confuses some of the callers. This -+ * breaks the iteration when there is a page at index -1 but that is -+ * already broke anyway. - */ - if (end == (pgoff_t)-1) -- *index = (pgoff_t)-1; -+ *start = (pgoff_t)-1; - else -- *index = end + 1; -+ *start = end + 1; - out: - rcu_read_unlock(); - -- return ret; -+ return folio_batch_count(fbatch); - } --EXPORT_SYMBOL(find_get_pages_range_tag); -+EXPORT_SYMBOL(filemap_get_folios_tag); - - /* - * CD/DVDs are error prone. When a medium error occurs, the driver may fail -diff --git a/mm/page-writeback.c b/mm/page-writeback.c -index cbcecd565c16..8f20c037a87a 100644 ---- a/mm/page-writeback.c -+++ b/mm/page-writeback.c -@@ -2293,15 +2293,15 @@ int write_cache_pages(struct address_space *mapping, - int ret = 0; - int done = 0; - int error; -- struct pagevec pvec; -- int nr_pages; -+ struct folio_batch fbatch; -+ int nr_folios; - pgoff_t index; - pgoff_t end; /* Inclusive */ - pgoff_t done_index; - int range_whole = 0; - xa_mark_t tag; - -- pagevec_init(&pvec); -+ folio_batch_init(&fbatch); - if (wbc->range_cyclic) { - index = mapping->writeback_index; /* prev offset */ - end = -1; -@@ -2321,17 +2321,18 @@ int write_cache_pages(struct address_space *mapping, - while (!done && (index <= end)) { - int i; - -- nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, -- tag); -- if (nr_pages == 0) -+ nr_folios = filemap_get_folios_tag(mapping, &index, end, -+ tag, &fbatch); ++/** ++ * blkdev_trig_link() - Link a block device to an LED. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Return: &0 on success, negative &errno on error. ++ */ ++static int blkdev_trig_link(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ bool led_first_link; ++ int err; ++ ++ led_first_link = xa_empty(&btl->linked_btbs); ++ ++ err = xa_insert(&btb->linked_btls, btl->index, btl, GFP_KERNEL); ++ if (err) ++ return err; ++ ++ err = xa_insert(&btl->linked_btbs, btb->index, btb, GFP_KERNEL); ++ if (err) ++ goto error_erase_btl; ++ ++ /* Create /sys/class/block//linked_leds/ symlink */ ++ err = sysfs_add_link_to_group(bdev_kobj(btb->bdev), ++ blkdev_trig_linked_leds.name, ++ &btl->led->dev->kobj, btl->led->name); ++ if (err) ++ goto error_erase_btb; ++ ++ /* Create /sys/class/leds//linked_devices/ symlink */ ++ err = sysfs_add_link_to_group(&btl->led->dev->kobj, ++ blkdev_trig_linked_devs.name, ++ bdev_kobj(btb->bdev), ++ dev_name(&btb->bdev->bd_device)); ++ if (err) ++ goto error_remove_symlink; ++ ++ /* ++ * If this is the first block device linked to this LED, the delayed ++ * work schedule may need to be changed. ++ */ ++ if (led_first_link) ++ blkdev_trig_sched_led(btl); ++ ++ ++blkdev_trig_link_count; ++ ++ return 0; ++ ++error_remove_symlink: ++ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), ++ blkdev_trig_linked_leds.name, ++ btl->led->name); ++error_erase_btb: ++ xa_erase(&btl->linked_btbs, btb->index); ++error_erase_btl: ++ xa_erase(&btb->linked_btls, btl->index); ++ return err; ++} ++ ++/** ++ * blkdev_trig_put_btb() - Remove and free a BTB, if it is no longer needed. ++ * @btb: The BTB ++ * ++ * Does nothing if the BTB (block device) is still linked to at least one LED. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_put_btb(struct blkdev_trig_bdev *btb) ++{ ++ struct block_device *bdev = btb->bdev; ++ int err; ++ ++ if (xa_empty(&btb->linked_btls)) { ++ ++ sysfs_remove_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); ++ err = devres_destroy(&bdev->bd_device, blkdev_trig_btb_release, ++ NULL, NULL); ++ WARN_ON(err); ++ } ++} ++ ++/** ++ * _blkdev_trig_unlink_always() - Perform the unconditionally required steps of ++ * unlinking a block device from an LED. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * When a block device is unlinked from an LED, certain steps must be performed ++ * only if the block device is **not** being released. This function performs ++ * those steps that are **always** required, whether or not the block device is ++ * being released. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void _blkdev_trig_unlink_always(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ --blkdev_trig_link_count; ++ ++ if (blkdev_trig_link_count == 0) ++ WARN_ON(!cancel_delayed_work_sync(&blkdev_trig_work)); ++ ++ xa_erase(&btb->linked_btls, btl->index); ++ xa_erase(&btl->linked_btbs, btb->index); ++ ++ /* Remove /sys/class/leds//linked_devices/ symlink */ ++ sysfs_remove_link_from_group(&btl->led->dev->kobj, ++ blkdev_trig_linked_devs.name, ++ dev_name(&btb->bdev->bd_device)); ++} ++ ++/** ++ * blkdev_trig_unlink_norelease() - Unlink an LED from a block device that is ++ * **not** being released. ++ * @btl: The BTL that represents the LED. ++ * @btb: The BTB that represents the block device. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_unlink_norelease(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ _blkdev_trig_unlink_always(btl, btb); + -+ if (nr_folios == 0) - break; - -- for (i = 0; i < nr_pages; i++) { -- struct page *page = pvec.pages[i]; -+ for (i = 0; i < nr_folios; i++) { -+ struct folio *folio = fbatch.folios[i]; - -- done_index = page->index; -+ done_index = folio->index; - -- lock_page(page); -+ folio_lock(folio); - - /* - * Page truncated or invalidated. We can freely skip it -@@ -2341,30 +2342,30 @@ int write_cache_pages(struct address_space *mapping, - * even if there is now a new, dirty page at the same - * pagecache address. - */ -- if (unlikely(page->mapping != mapping)) { -+ if (unlikely(folio->mapping != mapping)) { - continue_unlock: -- unlock_page(page); -+ folio_unlock(folio); - continue; - } - -- if (!PageDirty(page)) { -+ if (!folio_test_dirty(folio)) { - /* someone wrote it for us */ - goto continue_unlock; - } - -- if (PageWriteback(page)) { -+ if (folio_test_writeback(folio)) { - if (wbc->sync_mode != WB_SYNC_NONE) -- wait_on_page_writeback(page); -+ folio_wait_writeback(folio); - else - goto continue_unlock; - } - -- BUG_ON(PageWriteback(page)); -- if (!clear_page_dirty_for_io(page)) -+ BUG_ON(folio_test_writeback(folio)); -+ if (!folio_clear_dirty_for_io(folio)) - goto continue_unlock; - - trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); -- error = (*writepage)(page, wbc, data); -+ error = writepage(&folio->page, wbc, data); - if (unlikely(error)) { - /* - * Handle errors according to the type of -@@ -2379,11 +2380,12 @@ int write_cache_pages(struct address_space *mapping, - * the first error. - */ - if (error == AOP_WRITEPAGE_ACTIVATE) { -- unlock_page(page); -+ folio_unlock(folio); - error = 0; - } else if (wbc->sync_mode != WB_SYNC_ALL) { - ret = error; -- done_index = page->index + 1; -+ done_index = folio->index + -+ folio_nr_pages(folio); - done = 1; - break; - } -@@ -2403,7 +2405,7 @@ int write_cache_pages(struct address_space *mapping, - break; - } - } -- pagevec_release(&pvec); -+ folio_batch_release(&fbatch); - cond_resched(); - } - -diff --git a/mm/swap.c b/mm/swap.c -index 027943b341a0..29bf97746da0 100644 ---- a/mm/swap.c -+++ b/mm/swap.c -@@ -1099,16 +1099,6 @@ void folio_batch_remove_exceptionals(struct folio_batch *fbatch) - fbatch->nr = j; - } - --unsigned pagevec_lookup_range_tag(struct pagevec *pvec, -- struct address_space *mapping, pgoff_t *index, pgoff_t end, -- xa_mark_t tag) --{ -- pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, -- PAGEVEC_SIZE, pvec->pages); -- return pagevec_count(pvec); --} --EXPORT_SYMBOL(pagevec_lookup_range_tag); -- - /* - * Perform any setup for the swap system - */ --- -2.37.3 - -From b639b03341d179c471fc8a536c7840d5d20e1016 Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Mon, 19 Sep 2022 14:42:00 +0200 -Subject: [PATCH 12/13] fixes - -Signed-off-by: Peter Jung ---- - Documentation/ABI/stable/sysfs-block | 10 + - .../testing/sysfs-class-led-trigger-blkdev | 68 + - Documentation/leds/index.rst | 1 + - Documentation/leds/ledtrig-blkdev.rst | 155 +++ - arch/x86/events/amd/uncore.c | 1 + - arch/x86/kernel/alternative.c | 9 + - arch/x86/net/bpf_jit_comp.c | 4 +- - drivers/leds/trigger/Kconfig | 9 + - drivers/leds/trigger/Makefile | 1 + - drivers/leds/trigger/ledtrig-blkdev.c | 1177 +++++++++++++++++ - include/linux/pageblock-flags.h | 2 +- - include/net/bluetooth/rfcomm.h | 3 + - net/bluetooth/rfcomm/core.c | 2 + - net/bluetooth/rfcomm/sock.c | 34 +- - tools/objtool/check.c | 3 - - 15 files changed, 1462 insertions(+), 17 deletions(-) - create mode 100644 Documentation/ABI/testing/sysfs-class-led-trigger-blkdev - create mode 100644 Documentation/leds/ledtrig-blkdev.rst - create mode 100644 drivers/leds/trigger/ledtrig-blkdev.c - -diff --git a/Documentation/ABI/stable/sysfs-block b/Documentation/ABI/stable/sysfs-block -index cd14ecb3c9a5..d11d04d804fe 100644 ---- a/Documentation/ABI/stable/sysfs-block -+++ b/Documentation/ABI/stable/sysfs-block -@@ -101,6 +101,16 @@ Description: - devices that support receiving integrity metadata. - - -+What: /sys/block//linked_leds -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Directory that contains symbolic links to all LEDs that -+ are associated with (linked to) this block device by the -+ blkdev LED trigger. Only present when at least one LED -+ is linked. (See Documentation/leds/ledtrig-blkdev.rst.) ++ /* Remove /sys/class/block//linked_leds/ symlink */ ++ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), ++ blkdev_trig_linked_leds.name, ++ btl->led->name); + ++ blkdev_trig_put_btb(btb); ++} + - What: /sys/block///alignment_offset - Date: April 2009 - Contact: Martin K. Petersen -diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev -new file mode 100644 -index 000000000000..d8d403569b21 ---- /dev/null -+++ b/Documentation/ABI/testing/sysfs-class-led-trigger-blkdev -@@ -0,0 +1,68 @@ -+What: /sys/class/leds//blink_time -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Time (in milliseconds) that the LED will be on during a single -+ "blink". ++/** ++ * blkdev_trig_unlink_release() - Unlink an LED from a block device that is ++ * being released. ++ * @btl: The BTL that represents the LED ++ * @btb: The BTB that represents the block device ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_unlink_release(struct blkdev_trig_led *btl, ++ struct blkdev_trig_bdev *btb) ++{ ++ _blkdev_trig_unlink_always(btl, btb); + -+What: /sys/class/leds//check_interval -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Frequency (in milliseconds) with which block devices linked to -+ this LED will be checked for activity and the LED will -+ (potentially) be blinked. ++ /* ++ * If the BTB is being released, the driver core has already removed the ++ * device's attribute groups, and the BTB will be freed automatically, ++ * so there's nothing else to do. ++ */ ++} + -+What: /sys/class/leds//blink_on_read -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to read activity on any of its linked block devices. + -+What: /sys/class/leds//blink_on_write -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to write activity on any of its linked block devices. ++/* ++ * ++ * BTB creation ++ * ++ */ + -+What: /sys/class/leds//blink_on_discard -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to discard activity on any of its linked block devices. ++/** ++ * blkdev_trig_btb_release() - BTB device resource release function. ++ * @dev: The block device ++ * @res: The BTB ++ * ++ * Called by the driver core when a block device with a BTB is removed. ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_btb_release(struct device *dev, void *res) ++{ ++ struct blkdev_trig_bdev *btb = res; ++ struct blkdev_trig_led *btl; ++ unsigned long index; + -+What: /sys/class/leds//blink_on_flush -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Boolean that determines whether the LED will blink in response -+ to cache flush activity on any of its linked block devices. ++ mutex_lock(&blkdev_trig_mutex); + -+What: /sys/class/leds//link_dev_by_path -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Associate a block device with this LED by writing the path to -+ the device special file (e.g. /dev/sda) to this attribute. -+ Symbolic links are followed. ++ xa_for_each (&btb->linked_btls, index, btl) ++ blkdev_trig_unlink_release(btl, btb); + -+What: /sys/class/leds//unlink_dev_by_path -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Remove the association between this LED and a block device by -+ writing the path to the device special file (e.g. /dev/sda) to -+ this attribute. Symbolic links are followed. ++ mutex_unlock(&blkdev_trig_mutex); ++} + -+What: /sys/class/leds//linked_devices -+Date: September 2022 -+Contact: Ian Pilcher -+Description: -+ Directory containing links to all block devices that are -+ associated with this LED. (Note that the names of the -+ symbolic links in this directory are *kernel* names, which -+ may not match the device special file paths written to -+ link_device and unlink_device.) -diff --git a/Documentation/leds/index.rst b/Documentation/leds/index.rst -index e5d63b940045..e3c24e468cbc 100644 ---- a/Documentation/leds/index.rst -+++ b/Documentation/leds/index.rst -@@ -10,6 +10,7 @@ LEDs - leds-class - leds-class-flash - leds-class-multicolor -+ ledtrig-blkdev - ledtrig-oneshot - ledtrig-transient - ledtrig-usbport -diff --git a/Documentation/leds/ledtrig-blkdev.rst b/Documentation/leds/ledtrig-blkdev.rst -new file mode 100644 -index 000000000000..ae92aa559257 ---- /dev/null -+++ b/Documentation/leds/ledtrig-blkdev.rst -@@ -0,0 +1,155 @@ -+.. SPDX-License-Identifier: GPL-2.0 ++/** ++ * blkdev_trig_get_bdev() - Get a block device by path. ++ * @path: The value written to an LED's &link_dev_by_path or ++ * &unlink_dev_by_path attribute, which should be the path to a ++ * special file that represents a block device ++ * @len: The number of characters in &path (not including its ++ * terminating null) ++ * ++ * The caller must call blkdev_put() when finished with the device. ++ * ++ * Context: Process context. ++ * Return: The block device, or an error pointer. ++ */ ++static struct block_device *blkdev_trig_get_bdev(const char *path, size_t len) ++{ ++ struct block_device *bdev; ++ char *buf; + -+================================= -+Block Device (blkdev) LED Trigger -+================================= ++ buf = kmemdup(path, len + 1, GFP_KERNEL); /* +1 to include null */ ++ if (buf == NULL) ++ return ERR_PTR(-ENOMEM); + -+Available when ``CONFIG_LEDS_TRIGGER_BLKDEV=y`` or -+``CONFIG_LEDS_TRIGGER_BLKDEV=m``. ++ bdev = blkdev_get_by_path(strim(buf), BLKDEV_TRIG_FMODE, THIS_MODULE); ++ kfree(buf); ++ return bdev; ++} + -+See also: ++/** ++ * blkdev_trig_get_btb() - Find or create the BTB for a block device. ++ * @path: The value written to an LED's &link_dev_by_path attribute, ++ * which should be the path to a special file that represents a ++ * block device ++ * @len: The number of characters in &path ++ * ++ * If a new BTB is created, because the block device was not previously linked ++ * to any LEDs, the block device's &linked_leds &sysfs directory is created. ++ * ++ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Return: Pointer to the BTB, error pointer on error. ++ */ ++static struct blkdev_trig_bdev *blkdev_trig_get_btb(const char *path, ++ size_t len) ++{ ++ struct block_device *bdev; ++ struct blkdev_trig_bdev *btb; ++ int err; + -+* ``Documentation/ABI/testing/sysfs-class-led-trigger-blkdev`` -+* ``Documentation/ABI/stable/sysfs-block`` (``/sys/block//linked_leds``) ++ bdev = blkdev_trig_get_bdev(path, len); ++ if (IS_ERR(bdev)) ++ return ERR_CAST(bdev); + -+Overview -+======== ++ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, ++ NULL, NULL); ++ if (btb != NULL) { ++ err = 0; ++ goto exit_put_bdev; ++ } + -+.. note:: -+ The examples below use ```` to refer to the name of a -+ system-specific LED. If no suitable LED is available on a test -+ system (in a virtual machine, for example), it is possible to -+ use a userspace LED. (See ``Documentation/leds/uleds.rst``.) ++ if (blkdev_trig_next_index == ULONG_MAX) { ++ err = -EOVERFLOW; ++ goto exit_put_bdev; ++ } + -+Verify that the ``blkdev`` LED trigger is available:: ++ btb = devres_alloc(blkdev_trig_btb_release, sizeof(*btb), GFP_KERNEL); ++ if (btb == NULL) { ++ err = -ENOMEM; ++ goto exit_put_bdev; ++ } + -+ # grep blkdev /sys/class/leds//trigger -+ ... rfkill-none blkdev ++ err = sysfs_create_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); ++ if (err) ++ goto exit_free_btb; + -+(If the previous command produces no output, you may need to load the trigger -+module - ``modprobe ledtrig_blkdev``. If the module is not available, check -+the value of ``CONFIG_LEDS_TRIGGER_BLKDEV`` in your kernel configuration.) ++ btb->index = blkdev_trig_next_index++; ++ btb->bdev = bdev; ++ xa_init(&btb->linked_btls); + -+Associate the LED with the ``blkdev`` LED trigger:: ++ /* Populate BTB activity counters */ ++ blkdev_trig_update_btb(btb, jiffies); + -+ # echo blkdev > /sys/class/leds//trigger ++ devres_add(&bdev->bd_device, btb); + -+ # cat /sys/class/leds//trigger -+ ... rfkill-none [blkdev] ++exit_free_btb: ++ if (err) ++ devres_free(btb); ++exit_put_bdev: ++ blkdev_put(bdev, BLKDEV_TRIG_FMODE); ++ return err ? ERR_PTR(err) : btb; ++} + -+Note that several new device attributes are available in the -+``/sys/class/leds/`` directory. + -+* ``link_dev_by_path`` and ``unlink_dev_by_path`` are used to manage the set of -+ block devices associated with this LED. The LED will blink in response to -+ read or write activity on its linked devices. ++/* ++ * ++ * Activating and deactivating the trigger on an LED ++ * ++ */ + -+* ``blink_on_read``, ``blink_on_write``, ``blink_on_discard``, and -+ ``blink_on_flush`` are boolean values that determine whether the LED will -+ blink when a particular type of activity is detected on one of its linked -+ block devices. ++/** ++ * blkdev_trig_activate() - Called by the LEDs subsystem when an LED is ++ * associated with the trigger. ++ * @led: The LED ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &0 on success, negative &errno on error. ++ */ ++static int blkdev_trig_activate(struct led_classdev *led) ++{ ++ struct blkdev_trig_led *btl; ++ int err; + -+* ``blink_time`` is the duration (in milliseconds) of each blink of this LED. -+ (The minimum value is 10 milliseconds.) ++ btl = kzalloc(sizeof(*btl), GFP_KERNEL); ++ if (btl == NULL) ++ return -ENOMEM; + -+* ``check_interval`` is the frequency (in milliseconds) with which block devices -+ linked to this LED will be checked for activity and the LED blinked (if the -+ correct type of activity has occurred). ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ goto exit_free; + -+* The ``linked_devices`` directory will contain a symbolic link to every device -+ that is associated with this LED. ++ if (blkdev_trig_next_index == ULONG_MAX) { ++ err = -EOVERFLOW; ++ goto exit_unlock; ++ } + -+Link a block device to the LED:: ++ btl->index = blkdev_trig_next_index++; ++ btl->last_checked = jiffies; ++ btl->mode = -1; /* set all bits */ ++ btl->led = led; ++ btl->blink_msec = BLKDEV_TRIG_BLINK_DEF; ++ btl->check_jiffies = msecs_to_jiffies(BLKDEV_TRIG_CHECK_DEF); ++ xa_init(&btl->linked_btbs); + -+ # echo /dev/sda > /sys/class/leds//link_dev_by_path ++ hlist_add_head(&btl->all_btls_node, &blkdev_trig_all_btls); ++ led_set_trigger_data(led, btl); + -+ # ls /sys/class/leds//linked_devices -+ sda ++exit_unlock: ++ mutex_unlock(&blkdev_trig_mutex); ++exit_free: ++ if (err) ++ kfree(btl); ++ return err; ++} + -+(The value written to ``link_dev_by_path`` must be the path of the device -+special file, such as ``/dev/sda``, that represents the block device - or the -+path of a symbolic link to such a device special file.) ++/** ++ * blkdev_trig_deactivate() - Called by the the LEDs subsystem when an LED is ++ * disassociated from the trigger. ++ * @led: The LED ++ * ++ * The LEDs subsystem also calls this function when an LED associated with the ++ * trigger is removed or when the trigger is unregistered (if the module is ++ * unloaded). ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ */ ++static void blkdev_trig_deactivate(struct led_classdev *led) ++{ ++ struct blkdev_trig_led *btl = led_get_trigger_data(led); ++ struct blkdev_trig_bdev *btb; ++ unsigned long index; + -+Activity on the device will now cause the LED to blink. The duration of each -+blink (in milliseconds) can be adjusted by setting -+``/sys/class/leds//blink_time``. (But see **check_interval and -+blink_time** below.) ++ mutex_lock(&blkdev_trig_mutex); + -+Associate a second device with the LED:: ++ xa_for_each (&btl->linked_btbs, index, btb) ++ blkdev_trig_unlink_norelease(btl, btb); ++ ++ hlist_del(&btl->all_btls_node); ++ kfree(btl); ++ ++ mutex_unlock(&blkdev_trig_mutex); ++} ++ ++ ++/* ++ * ++ * Link-related attribute store functions ++ * ++ */ ++ ++/** ++ * link_dev_by_path_store() - &link_dev_by_path device attribute store function. ++ * @dev: The LED device ++ * @attr: The &link_dev_by_path attribute (&dev_attr_link_dev_by_path) ++ * @buf: The value written to the attribute, which should be the path to ++ * a special file that represents a block device to be linked to ++ * the LED (e.g. ``/dev/sda``) ++ * @count: The number of characters in &buf ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t link_dev_by_path_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ struct blkdev_trig_bdev *btb; ++ int err; + -+ # echo /dev/sdb > /sys/class/leds//link_dev_by_path ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ return err; + -+ # ls /sys/class/leds//linked_devices -+ sda sdb ++ btb = blkdev_trig_get_btb(buf, count); ++ if (IS_ERR(btb)) { ++ err = PTR_ERR(btb); ++ goto exit_unlock; ++ } + -+When a block device is linked to one or more LEDs, the LEDs are linked from -+the device's ``linked_leds`` directory:: ++ if (xa_load(&btb->linked_btls, btl->index) != NULL) { ++ err = -EEXIST; ++ goto exit_put_btb; ++ } + -+ # ls /sys/class/block/sd{a,b}/linked_leds -+ /sys/class/block/sda/linked_leds: -+ ++ err = blkdev_trig_link(btl, btb); + -+ /sys/class/block/sdb/linked_leds: -+ ++exit_put_btb: ++ if (err) ++ blkdev_trig_put_btb(btb); ++exit_unlock: ++ mutex_unlock(&blkdev_trig_mutex); ++ return err ? : count; ++} + -+(The ``linked_leds`` directory only exists when the block device is linked to -+at least one LED.) ++/** ++ * unlink_dev_by_path_store() - &unlink_dev_by_path device attribute store ++ * function. ++ * @dev: The LED device ++ * @attr: The &unlink_dev_by_path attribute (&dev_attr_unlink_dev_by_path) ++ * @buf: The value written to the attribute, which should be the path to ++ * a special file that represents a block device to be unlinked ++ * from the LED (e.g. ``/dev/sda``) ++ * @count: The number of characters in &buf ++ * ++ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t unlink_dev_by_path_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ struct block_device *bdev; ++ struct blkdev_trig_bdev *btb; ++ int err; + -+``check_interval`` and ``blink_time`` -+===================================== ++ bdev = blkdev_trig_get_bdev(buf, count); ++ if (IS_ERR(bdev)) ++ return PTR_ERR(bdev); + -+* By default, linked block devices are checked for activity every 100 -+ milliseconds. This frequency can be changed for an LED via the -+ ``/sys/class/leds//check_interval`` attribute. (The minimum value is 25 -+ milliseconds.) ++ err = mutex_lock_interruptible(&blkdev_trig_mutex); ++ if (err) ++ goto exit_put_bdev; + -+* All block devices associated with an LED are checked for activity every -+ ``check_interval`` milliseconds, and a blink is triggered if the correct type -+ of activity (as determined by the LED's ``blink_on_*`` attributes) is -+ detected. The duration of an LED's blink is determined by its ``blink_time`` -+ attribute. Thus (when the correct type of activity is detected), the LED will -+ be on for ``blink_time`` milliseconds and off for -+ ``check_interval - blink_time`` milliseconds. ++ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, ++ NULL, NULL); ++ if (btb == NULL) { ++ err = -EUNATCH; /* bdev isn't linked to any LED */ ++ goto exit_unlock; ++ } + -+* The LED subsystem ignores new blink requests for an LED that is already in -+ in the process of blinking, so setting a ``blink_time`` greater than or equal -+ to ``check_interval`` will cause some blinks to be missed. ++ if (xa_load(&btb->linked_btls, btl->index) == NULL) { ++ err = -EUNATCH; /* bdev isn't linked to this LED */ ++ goto exit_unlock; ++ } + -+* Because of processing times, scheduling latencies, etc., avoiding missed -+ blinks actually requires a difference of at least a few milliseconds between -+ the ``blink_time`` and ``check_interval``. The required difference is likely -+ to vary from system to system. As a reference, a Thecus N5550 NAS requires a -+ difference of 7 milliseconds (e.g. ``check_interval == 100``, -+ ``blink_time == 93``). ++ blkdev_trig_unlink_norelease(btl, btb); + -+* The default values (``check_interval == 100``, ``blink_time == 75``) cause the -+ LED associated with a continuously active device to blink rapidly. For a more -+ "always on" effect, increase the ``blink_time`` (but not too much; see the -+ previous bullet). ++exit_unlock: ++ mutex_unlock(&blkdev_trig_mutex); ++exit_put_bdev: ++ blkdev_put(bdev, BLKDEV_TRIG_FMODE); ++ return err ? : count; ++} + -+Other Notes -+=========== + -+* Many (possibly all) types of block devices work with this trigger, including: ++/* ++ * ++ * Atomic attribute show & store functions ++ * ++ */ + -+ * SCSI (including SATA and USB) hard disk drives and SSDs -+ * SCSI (including SATA and USB) optical drives -+ * NVMe SSDs -+ * SD cards -+ * loopback block devices (``/dev/loop*``) -+ * device mapper devices, such as LVM logical volumes -+ * MD RAID devices -+ * zRAM compressed RAM-disks -+ * partitions on block devics that support them ++/** ++ * blink_time_show() - &blink_time device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_time attribute (&dev_attr_blink_time) ++ * @buf: Output buffer ++ * ++ * Writes the value of &blkdev_trig_led.blink_msec to &buf. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_time_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); + -+* The names of the symbolic links in ``/sys/class/leds//linked_devices`` -+ are **kernel** names, which may not match the paths used for -+ ``link_dev_by_path`` and ``unlink_dev_by_path``. This is most likely when a -+ symbolic link is used to refer to the device (as is common with logical -+ volumes), but it can be true for any device, because nothing prevents the -+ creation of device special files with arbitrary names (e.g. -+ ``sudo mknod /foo b 8 0``). ++ return sprintf(buf, "%u\n", READ_ONCE(btl->blink_msec)); ++} + -+* The ``blkdev`` LED trigger supports many-to-many device/LED associations. -+ A device can be associated with multiple LEDs, and an LED can be associated -+ with multiple devices. -diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c -index d568afc705d2..83f15fe411b3 100644 ---- a/arch/x86/events/amd/uncore.c -+++ b/arch/x86/events/amd/uncore.c -@@ -553,6 +553,7 @@ static void uncore_clean_online(void) - - hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { - hlist_del(&uncore->node); -+ kfree(uncore->events); - kfree(uncore); - } - } -diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c -index f9c9b5850847..3582e61ddce6 100644 ---- a/arch/x86/kernel/alternative.c -+++ b/arch/x86/kernel/alternative.c -@@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) - return ret; - i += ret; - -+ /* -+ * The compiler is supposed to EMIT an INT3 after every unconditional -+ * JMP instruction due to AMD BTC. However, if the compiler is too old -+ * or SLS isn't enabled, we still need an INT3 after indirect JMPs -+ * even on Intel. -+ */ -+ if (op == JMP32_INSN_OPCODE && i < insn->length) -+ bytes[i++] = INT3_INSN_OPCODE; ++/** ++ * blink_time_store() - &blink_time device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_time attribute (&dev_attr_blink_time) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets &blkdev_trig_led.blink_msec to the value in &buf. ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_time_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ unsigned int value; ++ int err; + - for (; i < insn->length;) - bytes[i++] = BYTES_NOP1; - -diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c -index c1f6c1c51d99..4922517ddb0d 100644 ---- a/arch/x86/net/bpf_jit_comp.c -+++ b/arch/x86/net/bpf_jit_comp.c -@@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) - OPTIMIZER_HIDE_VAR(reg); - emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); - } else { -- EMIT2(0xFF, 0xE0 + reg); -+ EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ -+ if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS)) -+ EMIT1(0xCC); /* int3 */ - } - - *pprog = prog; -diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig -index dc6816d36d06..bda249068182 100644 ---- a/drivers/leds/trigger/Kconfig -+++ b/drivers/leds/trigger/Kconfig -@@ -154,4 +154,13 @@ config LEDS_TRIGGER_TTY - - When build as a module this driver will be called ledtrig-tty. - -+config LEDS_TRIGGER_BLKDEV -+ tristate "LED Trigger for block devices" -+ depends on BLOCK -+ help -+ The blkdev LED trigger allows LEDs to be controlled by block device -+ activity (reads and writes). ++ err = kstrtouint(buf, 0, &value); ++ if (err) ++ return err; + -+ See Documentation/leds/ledtrig-blkdev.rst. ++ if (value < BLKDEV_TRIG_BLINK_MIN || value > BLKDEV_TRIG_BLINK_MAX) ++ return -ERANGE; + - endif # LEDS_TRIGGERS -diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile -index 25c4db97cdd4..d53bab5d93f1 100644 ---- a/drivers/leds/trigger/Makefile -+++ b/drivers/leds/trigger/Makefile -@@ -16,3 +16,4 @@ obj-$(CONFIG_LEDS_TRIGGER_NETDEV) += ledtrig-netdev.o - obj-$(CONFIG_LEDS_TRIGGER_PATTERN) += ledtrig-pattern.o - obj-$(CONFIG_LEDS_TRIGGER_AUDIO) += ledtrig-audio.o - obj-$(CONFIG_LEDS_TRIGGER_TTY) += ledtrig-tty.o -+obj-$(CONFIG_LEDS_TRIGGER_BLKDEV) += ledtrig-blkdev.o -diff --git a/drivers/leds/trigger/ledtrig-blkdev.c b/drivers/leds/trigger/ledtrig-blkdev.c -new file mode 100644 -index 000000000000..0d2eb36fbe38 ---- /dev/null -+++ b/drivers/leds/trigger/ledtrig-blkdev.c -@@ -0,0 +1,1177 @@ -+// SPDX-License-Identifier: GPL-2.0-only ++ WRITE_ONCE(btl->blink_msec, value); ++ return count; ++} + -+/* -+ * Block device LED trigger ++/** ++ * check_interval_show() - &check_interval device attribute show function. ++ * @dev: The LED device ++ * @attr: The &check_interval attribute (&dev_attr_check_interval) ++ * @buf: Output buffer + * -+ * Copyright 2021-2022 Ian Pilcher ++ * Writes the value of &blkdev_trig_led.check_jiffies (converted to ++ * milliseconds) to &buf. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. + */ ++static ssize_t check_interval_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); + -+#include -+#include -+#include -+#include -+#include ++ return sprintf(buf, "%u\n", ++ jiffies_to_msecs(READ_ONCE(btl->check_jiffies))); ++} + +/** -+ * DOC: Overview ++ * check_interval_store() - &check_interval device attribute store function ++ * @dev: The LED device ++ * @attr: The &check_interval attribute (&dev_attr_check_interval) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf + * -+ * The ``blkdev`` LED trigger works by periodically checking the activity -+ * counters of block devices that have been linked to one or more LEDs and -+ * blinking those LED(s) if the correct type of activity has occurred. The -+ * periodic check is scheduled with the Linux kernel's deferred work facility. ++ * Sets &blkdev_trig_led.check_jiffies to the value in &buf (after converting ++ * from milliseconds). + * -+ * Trigger-specific data about block devices and LEDs is stored in two data -+ * structures --- &struct blkdev_trig_bdev (a "BTB") and &struct blkdev_trig_led -+ * (a "BTL"). Each structure contains a &struct xarray that holds links to any -+ * linked devices of the other type. I.e. &blkdev_trig_bdev.linked_btls -+ * contains links to all BTLs whose LEDs have been linked to the BTB's block -+ * device, and &blkdev_trig_led.linked_btbs contains links to all BTBs whose -+ * block devices have been linked to the BTL's LED. Thus, a block device can -+ * be linked to more than one LED, and an LED can be linked to more than one -+ * block device. ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. + */ ++static ssize_t check_interval_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct blkdev_trig_led *led = led_trigger_get_drvdata(dev); ++ unsigned int value; ++ int err; + -+/* Default, minimum & maximum blink duration (milliseconds) */ -+#define BLKDEV_TRIG_BLINK_DEF 75 -+#define BLKDEV_TRIG_BLINK_MIN 10 -+#define BLKDEV_TRIG_BLINK_MAX 86400000 /* 24 hours */ ++ err = kstrtouint(buf, 0, &value); ++ if (err) ++ return err; + -+/* Default, minimum & maximum activity check interval (milliseconds) */ -+#define BLKDEV_TRIG_CHECK_DEF 100 -+#define BLKDEV_TRIG_CHECK_MIN 25 -+#define BLKDEV_TRIG_CHECK_MAX 86400000 /* 24 hours */ ++ if (value < BLKDEV_TRIG_CHECK_MIN || value > BLKDEV_TRIG_CHECK_MAX) ++ return -ERANGE; + -+/* -+ * If blkdev_trig_check() can't lock the mutex, how long to wait before trying -+ * again (milliseconds) -+ */ -+#define BLKDEV_TRIG_CHECK_RETRY 5 ++ WRITE_ONCE(led->check_jiffies, msecs_to_jiffies(value)); + -+/* Mode argument for calls to blkdev_get_by_path() and blkdev_put() */ -+#define BLKDEV_TRIG_FMODE 0 ++ return count; ++} + +/** -+ * struct blkdev_trig_bdev - Trigger-specific data about a block device. -+ * @last_checked: Time (in jiffies) at which the trigger last checked this -+ * block device for activity. -+ * @last_activity: Time (in jiffies) at which the trigger last detected -+ * activity of each type. -+ * @ios: Activity counter values for each type, corresponding to -+ * the timestamps in &last_activity. -+ * @index: &xarray index, so the BTB can be included in one or more -+ * &blkdev_trig_led.linked_btbs. -+ * @bdev: The block device. -+ * @linked_btls: The BTLs that represent the LEDs linked to the BTB's -+ * block device. -+ * -+ * Every block device linked to at least one LED gets a "BTB." A BTB is created -+ * when a block device that is not currently linked to any LEDs is linked to an -+ * LED. -+ * -+ * A BTB is freed when one of the following occurs: -+ * -+ * * The number of LEDs linked to the block device becomes zero, because it has -+ * been unlinked from its last LED using the trigger's &sysfs interface. -+ * -+ * * The number of LEDs linked to the block device becomes zero, because the -+ * last LED to which it was linked has been disassociated from the trigger -+ * (which happens automatically if the LED device is removed from the system). ++ * blkdev_trig_mode_show() - Helper for boolean attribute show functions. ++ * @led: The LED ++ * @buf: Output buffer ++ * @bit: Which bit to show + * -+ * * The BTB's block device is removed from the system. To accomodate this -+ * scenario, BTB's are created as device resources, so that the release -+ * function will be called by the driver core when the device is removed. ++ * Context: Process context. ++ * Return: The number of characters written to &buf. + */ -+struct blkdev_trig_bdev { -+ unsigned long last_checked; -+ unsigned long last_activity[NR_STAT_GROUPS]; -+ unsigned long ios[NR_STAT_GROUPS]; -+ unsigned long index; -+ struct block_device *bdev; -+ struct xarray linked_btls; -+}; ++static int blkdev_trig_mode_show(const struct blkdev_trig_led *led, char *buf, ++ enum stat_group bit) ++{ ++ return sprintf(buf, READ_ONCE(led->mode) & (1 << bit) ? "Y\n" : "N\n"); ++} + +/** -+ * struct blkdev_trig_led - Trigger-specific data about an LED. -+ * @last_checked: Time (in jiffies) at which the trigger last checked the -+ * the block devices linked to this LED for activity. -+ * @index: &xarray index, so the BTL can be included in one or more -+ * &blkdev_trig_bdev.linked_btls. -+ * @mode: Bitmask for types of block device activity that will -+ * cause this LED to blink --- reads, writes, discards, -+ * etc. -+ * @led: The LED device. -+ * @blink_msec: Duration of a blink (milliseconds). -+ * @check_jiffies: Frequency with which block devices linked to this LED -+ * should be checked for activity (jiffies). -+ * @linked_btbs: The BTBs that represent the block devices linked to the -+ * BTL's LED. -+ * @all_btls_node: The BTL's node in the module's list of all BTLs. ++ * blkdev_trig_mode_store() - Helper for boolean attribute store functions. ++ * @led: The LED ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * @bit: Which bit to set + * -+ * Every LED associated with the block device trigger gets a "BTL." A BTL is -+ * created when the trigger is "activated" on an LED (usually by writing -+ * ``blkdev`` to the LED's &sysfs &trigger attribute). A BTL is freed wnen its -+ * LED is disassociated from the trigger, either through the trigger's &sysfs -+ * interface or because the LED device is removed from the system. ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. + */ -+struct blkdev_trig_led { -+ unsigned long last_checked; -+ unsigned long index; -+ unsigned long mode; /* must be ulong for atomic bit ops */ -+ struct led_classdev *led; -+ unsigned int blink_msec; -+ unsigned int check_jiffies; -+ struct xarray linked_btbs; -+ struct hlist_node all_btls_node; -+}; -+ -+/* Protects everything except atomic LED attributes */ -+static DEFINE_MUTEX(blkdev_trig_mutex); -+ -+/* BTB device resource release function */ -+static void blkdev_trig_btb_release(struct device *dev, void *res); -+ -+/* Index for next BTB or BTL */ -+static unsigned long blkdev_trig_next_index; -+ -+/* All LEDs associated with the trigger */ -+static HLIST_HEAD(blkdev_trig_all_btls); -+ -+/* Delayed work to periodically check for activity & blink LEDs */ -+static void blkdev_trig_check(struct work_struct *work); -+static DECLARE_DELAYED_WORK(blkdev_trig_work, blkdev_trig_check); ++static int blkdev_trig_mode_store(struct blkdev_trig_led *led, ++ const char *buf, size_t count, ++ enum stat_group bit) ++{ ++ bool set; ++ int err; + -+/* When is the delayed work scheduled to run next (jiffies) */ -+static unsigned long blkdev_trig_next_check; ++ err = kstrtobool(buf, &set); ++ if (err) ++ return err; + -+/* Total number of BTB-to-BTL links */ -+static unsigned int blkdev_trig_link_count; ++ if (set) ++ set_bit(bit, &led->mode); ++ else ++ clear_bit(bit, &led->mode); + -+/* Empty sysfs attribute list for next 2 declarations */ -+static struct attribute *blkdev_trig_attrs_empty[] = { NULL }; ++ return count; ++} + -+/* linked_leds sysfs directory for block devs linked to 1 or more LEDs */ -+static const struct attribute_group blkdev_trig_linked_leds = { -+ .name = "linked_leds", -+ .attrs = blkdev_trig_attrs_empty, -+}; ++/** ++ * blink_on_read_show() - &blink_on_read device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) ++ * @buf: Output buffer ++ * ++ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_READ bit in ++ * &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_on_read_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_READ); ++} + -+/* linked_devices sysfs directory for each LED associated with the trigger */ -+static const struct attribute_group blkdev_trig_linked_devs = { -+ .name = "linked_devices", -+ .attrs = blkdev_trig_attrs_empty, -+}; ++/** ++ * blink_on_read_store() - &blink_on_read device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets the &STAT_READ bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_on_read_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_READ); ++} + ++/** ++ * blink_on_write_show() - &blink_on_write device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) ++ * @buf: Output buffer ++ * ++ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_WRITE bit in ++ * in &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. ++ */ ++static ssize_t blink_on_write_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_WRITE); ++} + -+/* ++/** ++ * blink_on_write_store() - &blink_on_write device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf + * -+ * Delayed work to check for activity & blink LEDs ++ * Sets the &STAT_WRITE bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). + * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. + */ ++static ssize_t blink_on_write_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_WRITE); ++} + +/** -+ * blkdev_trig_blink() - Blink an LED, if the correct type of activity has -+ * occurred on the block device. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device ++ * blink_on_flush_show() - &blink_on_flush device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) ++ * @buf: Output buffer + * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: &true if the LED is blinked, &false if not. ++ * Writes ``Y`` or ``N`` to &buf, depending whether the &STAT_FLUSH bit in ++ * &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. + */ -+static bool blkdev_trig_blink(const struct blkdev_trig_led *btl, -+ const struct blkdev_trig_bdev *btb) ++static ssize_t blink_on_flush_show(struct device *dev, ++ struct device_attribute *attr, char *buf) +{ -+ unsigned long mode, mask, delay_on, delay_off; -+ enum stat_group i; -+ -+ mode = READ_ONCE(btl->mode); -+ -+ for (i = STAT_READ, mask = 1; i <= STAT_FLUSH; ++i, mask <<= 1) { -+ -+ if (!(mode & mask)) -+ continue; -+ -+ if (time_before_eq(btb->last_activity[i], btl->last_checked)) -+ continue; -+ -+ delay_on = READ_ONCE(btl->blink_msec); -+ delay_off = 1; /* 0 leaves LED turned on */ -+ -+ led_blink_set_oneshot(btl->led, &delay_on, &delay_off, 0); -+ return true; -+ } -+ -+ return false; ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_FLUSH); +} + +/** -+ * blkdev_trig_update_btb() - Update a BTB's activity counters and timestamps. -+ * @btb: The BTB -+ * @now: Timestamp (in jiffies) ++ * blink_on_flush_store() - &blink_on_flush device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf + * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Sets the &STAT_FLUSH bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. + */ -+static void blkdev_trig_update_btb(struct blkdev_trig_bdev *btb, -+ unsigned long now) ++static ssize_t blink_on_flush_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) +{ -+ unsigned long new_ios; -+ enum stat_group i; -+ -+ for (i = STAT_READ; i <= STAT_FLUSH; ++i) { -+ -+ new_ios = part_stat_read(btb->bdev, ios[i]); -+ -+ if (new_ios != btb->ios[i]) { -+ btb->ios[i] = new_ios; -+ btb->last_activity[i] = now; -+ } -+ } -+ -+ btb->last_checked = now; ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_FLUSH); +} + +/** -+ * blkdev_trig_check() - Check linked devices for activity and blink LEDs. -+ * @work: Delayed work (&blkdev_trig_work) ++ * blink_on_discard_show() - &blink_on_discard device attribute show function. ++ * @dev: The LED device ++ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) ++ * @buf: Output buffer + * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. ++ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_DISCARD bit in ++ * &blkdev_trig_led.mode is set or cleared. ++ * ++ * Context: Process context. ++ * Return: The number of characters written to &buf. + */ -+static void blkdev_trig_check(struct work_struct *work) ++static ssize_t blink_on_discard_show(struct device *dev, ++ struct device_attribute *attr, char *buf) +{ -+ struct blkdev_trig_led *btl; -+ struct blkdev_trig_bdev *btb; -+ unsigned long index, delay, now, led_check, led_delay; -+ bool blinked; -+ -+ if (!mutex_trylock(&blkdev_trig_mutex)) { -+ delay = msecs_to_jiffies(BLKDEV_TRIG_CHECK_RETRY); -+ goto exit_reschedule; -+ } -+ -+ now = jiffies; -+ delay = ULONG_MAX; -+ -+ hlist_for_each_entry (btl, &blkdev_trig_all_btls, all_btls_node) { -+ -+ led_check = btl->last_checked + btl->check_jiffies; -+ -+ if (time_before_eq(led_check, now)) { -+ -+ blinked = false; ++ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), ++ buf, STAT_DISCARD); ++} + -+ xa_for_each (&btl->linked_btbs, index, btb) { ++/** ++ * blink_on_discard_store() - &blink_on_discard device attribute store function. ++ * @dev: The LED device ++ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) ++ * @buf: The new value (as written to the &sysfs attribute) ++ * @count: The number of characters in &buf ++ * ++ * Sets the &STAT_DISCARD bit in &blkdev_trig_led.mode to the value in &buf ++ * (interpretted as a boolean). ++ * ++ * Context: Process context. ++ * Return: &count on success, negative &errno on error. ++ */ ++static ssize_t blink_on_discard_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), ++ buf, count, STAT_DISCARD); ++} + -+ if (btb->last_checked != now) -+ blkdev_trig_update_btb(btb, now); -+ if (!blinked) -+ blinked = blkdev_trig_blink(btl, btb); -+ } ++/* Device attributes */ ++static DEVICE_ATTR_WO(link_dev_by_path); ++static DEVICE_ATTR_WO(unlink_dev_by_path); ++static DEVICE_ATTR_RW(blink_time); ++static DEVICE_ATTR_RW(check_interval); ++static DEVICE_ATTR_RW(blink_on_read); ++static DEVICE_ATTR_RW(blink_on_write); ++static DEVICE_ATTR_RW(blink_on_flush); ++static DEVICE_ATTR_RW(blink_on_discard); + -+ btl->last_checked = now; -+ led_delay = btl->check_jiffies; ++/* Device attributes in LED directory (/sys/class/leds//...) */ ++static struct attribute *blkdev_trig_attrs[] = { ++ &dev_attr_link_dev_by_path.attr, ++ &dev_attr_unlink_dev_by_path.attr, ++ &dev_attr_blink_time.attr, ++ &dev_attr_check_interval.attr, ++ &dev_attr_blink_on_read.attr, ++ &dev_attr_blink_on_write.attr, ++ &dev_attr_blink_on_flush.attr, ++ &dev_attr_blink_on_discard.attr, ++ NULL ++}; + -+ } else { -+ led_delay = led_check - now; -+ } ++/* Unnamed attribute group == no subdirectory */ ++static const struct attribute_group blkdev_trig_attr_group = { ++ .attrs = blkdev_trig_attrs, ++}; + -+ if (led_delay < delay) -+ delay = led_delay; -+ } ++/* Attribute groups for the trigger */ ++static const struct attribute_group *blkdev_trig_attr_groups[] = { ++ &blkdev_trig_attr_group, /* /sys/class/leds//... */ ++ &blkdev_trig_linked_devs, /* /sys/class/leds//linked_devices/ */ ++ NULL ++}; + -+ mutex_unlock(&blkdev_trig_mutex); ++/* Trigger registration data */ ++static struct led_trigger blkdev_trig_trigger = { ++ .name = "blkdev", ++ .activate = blkdev_trig_activate, ++ .deactivate = blkdev_trig_deactivate, ++ .groups = blkdev_trig_attr_groups, ++}; + -+exit_reschedule: -+ WARN_ON_ONCE(delay == ULONG_MAX); -+ WARN_ON_ONCE(!schedule_delayed_work(&blkdev_trig_work, delay)); ++/** ++ * blkdev_trig_init() - Block device LED trigger initialization. ++ * ++ * Registers the ``blkdev`` LED trigger. ++ * ++ * Return: &0 on success, negative &errno on failure. ++ */ ++static int __init blkdev_trig_init(void) ++{ ++ return led_trigger_register(&blkdev_trig_trigger); +} ++module_init(blkdev_trig_init); + +/** -+ * blkdev_trig_sched_led() - Set the schedule of the delayed work when a new -+ * LED is added to the schedule. -+ * @btl: The BTL that represents the LED -+ * -+ * Called when the number of block devices to which an LED is linked becomes -+ * non-zero. ++ * blkdev_trig_exit() - Block device LED trigger module exit. + * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. ++ * Unregisters the ``blkdev`` LED trigger. + */ -+static void blkdev_trig_sched_led(const struct blkdev_trig_led *btl) ++static void __exit blkdev_trig_exit(void) +{ -+ unsigned long delay = READ_ONCE(btl->check_jiffies); -+ unsigned long check_by = jiffies + delay; ++ led_trigger_unregister(&blkdev_trig_trigger); ++} ++module_exit(blkdev_trig_exit); + -+ /* -+ * If no other LED-to-block device links exist, simply schedule the -+ * delayed work according to this LED's check_interval attribute -+ * (check_jiffies). -+ */ -+ if (blkdev_trig_link_count == 0) { -+ WARN_ON(!schedule_delayed_work(&blkdev_trig_work, delay)); -+ blkdev_trig_next_check = check_by; -+ return; -+ } ++MODULE_DESCRIPTION("Block device LED trigger"); ++MODULE_AUTHOR("Ian Pilcher "); ++MODULE_LICENSE("GPL v2"); +diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h +index 83c7248053a1..d2b8741eabf7 100644 +--- a/include/linux/pageblock-flags.h ++++ b/include/linux/pageblock-flags.h +@@ -48,7 +48,7 @@ extern unsigned int pageblock_order; + #else /* CONFIG_HUGETLB_PAGE */ + + /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ +-#define pageblock_order (MAX_ORDER-1) ++#define pageblock_order PAGE_ALLOC_COSTLY_ORDER + + #endif /* CONFIG_HUGETLB_PAGE */ + +diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h +index 99d26879b02a..a92799fc5e74 100644 +--- a/include/net/bluetooth/rfcomm.h ++++ b/include/net/bluetooth/rfcomm.h +@@ -171,6 +171,7 @@ struct rfcomm_dlc { + struct rfcomm_session *session; + struct sk_buff_head tx_queue; + struct timer_list timer; ++ struct work_struct state_change_work; + + struct mutex lock; + unsigned long state; +@@ -186,6 +187,7 @@ struct rfcomm_dlc { + u8 sec_level; + u8 role_switch; + u32 defer_setup; ++ int err; + + uint mtu; + uint cfc; +@@ -310,6 +312,7 @@ struct rfcomm_pinfo { + u8 role_switch; + }; + ++void __rfcomm_sk_state_change(struct work_struct *work); + int rfcomm_init_sockets(void); + void rfcomm_cleanup_sockets(void); + +diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c +index 7324764384b6..c6494e85cd68 100644 +--- a/net/bluetooth/rfcomm/core.c ++++ b/net/bluetooth/rfcomm/core.c +@@ -289,6 +289,7 @@ static void rfcomm_dlc_clear_state(struct rfcomm_dlc *d) + d->flags = 0; + d->mscex = 0; + d->sec_level = BT_SECURITY_LOW; ++ d->err = 0; + d->mtu = RFCOMM_DEFAULT_MTU; + d->v24_sig = RFCOMM_V24_RTC | RFCOMM_V24_RTR | RFCOMM_V24_DV; + +@@ -306,6 +307,7 @@ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio) + timer_setup(&d->timer, rfcomm_dlc_timeout, 0); + + skb_queue_head_init(&d->tx_queue); ++ INIT_WORK(&d->state_change_work, __rfcomm_sk_state_change); + mutex_init(&d->lock); + refcount_set(&d->refcnt, 1); + +diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c +index 4bf4ea6cbb5e..4850dafbaa05 100644 +--- a/net/bluetooth/rfcomm/sock.c ++++ b/net/bluetooth/rfcomm/sock.c +@@ -61,19 +61,22 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) + rfcomm_dlc_throttle(d); + } + +-static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) ++void __rfcomm_sk_state_change(struct work_struct *work) + { ++ struct rfcomm_dlc *d = container_of(work, struct rfcomm_dlc, ++ state_change_work); + struct sock *sk = d->owner, *parent; + + if (!sk) + return; + +- BT_DBG("dlc %p state %ld err %d", d, d->state, err); +- + lock_sock(sk); ++ rfcomm_dlc_lock(d); + +- if (err) +- sk->sk_err = err; ++ BT_DBG("dlc %p state %ld err %d", d, d->state, d->err); + -+ /* -+ * If the next check is already scheduled to occur soon enough to -+ * accomodate this LED's check_interval, the schedule doesn't need -+ * to be changed. -+ */ -+ if (time_after_eq(check_by, blkdev_trig_next_check)) ++ if (d->err) ++ sk->sk_err = d->err; + + sk->sk_state = d->state; + +@@ -91,15 +94,22 @@ static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) + sk->sk_state_change(sk); + } + ++ rfcomm_dlc_unlock(d); + release_sock(sk); ++ sock_put(sk); ++} + +- if (parent && sock_flag(sk, SOCK_ZAPPED)) { +- /* We have to drop DLC lock here, otherwise +- * rfcomm_sock_destruct() will dead lock. */ +- rfcomm_dlc_unlock(d); +- rfcomm_sock_kill(sk); +- rfcomm_dlc_lock(d); +- } ++static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) ++{ ++ struct sock *sk = d->owner; ++ ++ if (!sk) + return; + -+ /* -+ * Modify the schedule, so that the delayed work runs soon enough for -+ * this LED. -+ */ -+ WARN_ON(!mod_delayed_work(system_wq, &blkdev_trig_work, delay)); -+ blkdev_trig_next_check = check_by; ++ d->err = err; ++ sock_hold(sk); ++ if (!schedule_work(&d->state_change_work)) ++ sock_put(sk); + } + + /* ---- Socket functions ---- */ +diff --git a/tools/objtool/check.c b/tools/objtool/check.c +index e55fdf952a3a..445741f36d6d 100644 +--- a/tools/objtool/check.c ++++ b/tools/objtool/check.c +@@ -2104,9 +2104,6 @@ static int read_noendbr_hints(struct objtool_file *file) + return -1; + } + +- if (insn->type == INSN_ENDBR) +- WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); +- + insn->noendbr = 1; + } + +-- +2.38.0.rc1.8.g2a7d63a245 + +From 1c95ad8820155c71485f71b29697ed823bcce3b2 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Mon, 26 Sep 2022 00:19:51 +0200 +Subject: [PATCH 15/16] kallsyms + +Signed-off-by: Peter Jung +--- + include/linux/kallsyms.h | 12 +- + include/linux/module.h | 4 +- + init/Kconfig | 13 ++ + kernel/Makefile | 1 + + kernel/kallsyms.c | 195 +++++++++++++++-- + kernel/kallsyms_internal.h | 1 + + kernel/kallsyms_selftest.c | 421 +++++++++++++++++++++++++++++++++++++ + kernel/livepatch/core.c | 31 ++- + kernel/module/kallsyms.c | 15 +- + kernel/trace/ftrace.c | 3 +- + scripts/kallsyms.c | 88 +++++--- + 11 files changed, 717 insertions(+), 67 deletions(-) + create mode 100644 kernel/kallsyms_selftest.c + +diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h +index ad39636e0c3f..c7219d74e290 100644 +--- a/include/linux/kallsyms.h ++++ b/include/linux/kallsyms.h +@@ -66,9 +66,11 @@ static inline void *dereference_symbol_descriptor(void *ptr) + } + + #ifdef CONFIG_KALLSYMS +-int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, +- unsigned long), ++unsigned long kallsyms_sym_address(int idx); ++int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), + void *data); ++int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), ++ const char *name, void *data); + + /* Lookup the address for a symbol. Returns 0 if not found. */ + unsigned long kallsyms_lookup_name(const char *name); +@@ -168,6 +170,12 @@ static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct + { + return -EOPNOTSUPP; + } ++ ++static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), ++ const char *name, void *data) ++{ ++ return -EOPNOTSUPP; +} + #endif /*CONFIG_KALLSYMS*/ + + static inline void print_ip_sym(const char *loglvl, unsigned long ip) +diff --git a/include/linux/module.h b/include/linux/module.h +index 518296ea7f73..6e1a531d78e7 100644 +--- a/include/linux/module.h ++++ b/include/linux/module.h +@@ -879,8 +879,8 @@ static inline bool module_sig_ok(struct module *module) + } + #endif /* CONFIG_MODULE_SIG */ + +-int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, +- struct module *, unsigned long), ++int module_kallsyms_on_each_symbol(const char *modname, ++ int (*fn)(void *, const char *, unsigned long), + void *data); + + #endif /* _LINUX_MODULE_H */ +diff --git a/init/Kconfig b/init/Kconfig +index 442a945ca6ae..b3a9ec8aa753 100644 +--- a/init/Kconfig ++++ b/init/Kconfig +@@ -1742,6 +1742,19 @@ config KALLSYMS + symbolic stack backtraces. This increases the size of the kernel + somewhat, as all symbols have to be loaded into the kernel image. + ++config KALLSYMS_SELFTEST ++ bool "Test the basic functions and performance of kallsyms" ++ depends on KALLSYMS ++ default n ++ help ++ Test the basic functions and performance of some interfaces, such as ++ kallsyms_lookup_name. It also calculates the compression rate of the ++ kallsyms compression algorithm for the current symbol set. ++ ++ Start self-test automatically after system startup. Suggest executing ++ "dmesg | grep kallsyms_selftest" to collect test results. "finish" is ++ displayed in the last line, indicating that the test is complete. + + config KALLSYMS_ALL + bool "Include all symbols in kallsyms" + depends on DEBUG_KERNEL && KALLSYMS +diff --git a/kernel/Makefile b/kernel/Makefile +index 318789c728d3..122a5fed457b 100644 +--- a/kernel/Makefile ++++ b/kernel/Makefile +@@ -68,6 +68,7 @@ endif + obj-$(CONFIG_UID16) += uid16.o + obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o + obj-$(CONFIG_KALLSYMS) += kallsyms.o ++obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o + obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o + obj-$(CONFIG_CRASH_CORE) += crash_core.o + obj-$(CONFIG_KEXEC_CORE) += kexec_core.o +diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c +index 3e7e2c2ad2f7..1512db69aa0a 100644 +--- a/kernel/kallsyms.c ++++ b/kernel/kallsyms.c +@@ -87,6 +87,86 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, + return off; + } + ++static unsigned char *find_token(unsigned char *str, int len, ++ const unsigned char *token) ++{ ++ int i; + -+/* -+ * -+ * Linking and unlinking LEDs and block devices -+ * -+ */ ++ for (i = 0; i < len - 1; i++) { ++ if (str[i] == token[0] && str[i+1] == token[1]) ++ return &str[i]; ++ } ++ return NULL; ++} + -+/** -+ * blkdev_trig_link() - Link a block device to an LED. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: &0 on success, negative &errno on error. -+ */ -+static int blkdev_trig_link(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) ++static int kallsyms_compress_symbol_name(const char *name, char *buf, size_t size) +{ -+ bool led_first_link; -+ int err; ++ int i, j, n, len; ++ unsigned char *p1, *p2; ++ const unsigned char *token; + -+ led_first_link = xa_empty(&btl->linked_btbs); ++ len = strscpy(buf, name, size); ++ if (WARN_ON_ONCE(len <= 0)) ++ return 0; + -+ err = xa_insert(&btb->linked_btls, btl->index, btl, GFP_KERNEL); -+ if (err) -+ return err; ++ /* ++ * For each entry in kallsyms_best_token_table[], the storage ++ * format is: ++ * 1. For tokens that cannot be used to compress characters, the value ++ * at [j] is 0, and the value at [j+1] is the number of consecutive ++ * tokens with this feature. ++ * 2. For each token: the larger the token value, the higher the ++ * frequency, and the lower the index. ++ * ++ * ------------------------------- ++ * | j | [j] [j+1] | token | ++ * -----|---------------|---------| ++ * | 0 | ?? ?? | 255 | ++ * | 2 | ?? ?? | 254 | ++ * | ... | ?? ?? | ... | ++ * | n-2 | ?? ?? | x | ++ * | n | 00 len | x-1 | ++ * | n+2 | ?? ?? | x-1-len | ++ * above '??' is non-zero ++ */ ++ for (i = 255, j = 0; i >= 0; i--, j += 2) { ++ if (!kallsyms_best_token_table[j]) { ++ i -= kallsyms_best_token_table[j + 1]; ++ if (i < 0) ++ break; ++ j += 2; ++ } ++ token = &kallsyms_best_token_table[j]; + -+ err = xa_insert(&btl->linked_btbs, btb->index, btb, GFP_KERNEL); -+ if (err) -+ goto error_erase_btl; ++ p1 = buf; + -+ /* Create /sys/class/block//linked_leds/ symlink */ -+ err = sysfs_add_link_to_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ &btl->led->dev->kobj, btl->led->name); -+ if (err) -+ goto error_erase_btb; ++ /* find the token on the symbol */ ++ p2 = find_token(p1, len, token); ++ if (!p2) ++ continue; + -+ /* Create /sys/class/leds//linked_devices/ symlink */ -+ err = sysfs_add_link_to_group(&btl->led->dev->kobj, -+ blkdev_trig_linked_devs.name, -+ bdev_kobj(btb->bdev), -+ dev_name(&btb->bdev->bd_device)); -+ if (err) -+ goto error_remove_symlink; ++ n = len; + -+ /* -+ * If this is the first block device linked to this LED, the delayed -+ * work schedule may need to be changed. -+ */ -+ if (led_first_link) -+ blkdev_trig_sched_led(btl); ++ do { ++ *p2 = i; ++ p2++; ++ n -= (p2 - p1); ++ memmove(p2, p2 + 1, n); ++ p1 = p2; ++ len--; + -+ ++blkdev_trig_link_count; ++ if (n < 2) ++ break; + -+ return 0; ++ /* find the token on the symbol */ ++ p2 = find_token(p1, n, token); + -+error_remove_symlink: -+ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ btl->led->name); -+error_erase_btb: -+ xa_erase(&btl->linked_btbs, btb->index); -+error_erase_btl: -+ xa_erase(&btb->linked_btls, btl->index); -+ return err; ++ } while (p2); ++ } ++ ++ return len; +} + -+/** -+ * blkdev_trig_put_btb() - Remove and free a BTB, if it is no longer needed. -+ * @btb: The BTB -+ * -+ * Does nothing if the BTB (block device) is still linked to at least one LED. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_put_btb(struct blkdev_trig_bdev *btb) + /* + * Get symbol type information. This is encoded as a single char at the + * beginning of the symbol name. +@@ -128,7 +208,7 @@ static unsigned int get_symbol_offset(unsigned long pos) + return name - kallsyms_names; + } + +-static unsigned long kallsyms_sym_address(int idx) ++unsigned long kallsyms_sym_address(int idx) + { + if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE)) + return kallsyms_addresses[idx]; +@@ -186,26 +266,95 @@ static bool cleanup_symbol_name(char *s) + return false; + } + ++static int kallsyms_lookup_clang_name(unsigned char *namebuf, int len, ++ const char *name, ++ unsigned long *addr) +{ -+ struct block_device *bdev = btb->bdev; -+ int err; ++ unsigned long i; ++ unsigned int off; + -+ if (xa_empty(&btb->linked_btls)) { ++ if (!IS_ENABLED(CONFIG_LTO_CLANG)) ++ return -ENOENT; + -+ sysfs_remove_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); -+ err = devres_destroy(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ WARN_ON(err); ++ for (i = 0, off = 0; i < kallsyms_num_syms; i++) { ++ off = kallsyms_expand_symbol(off, namebuf, len); ++ ++ if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) { ++ *addr = kallsyms_sym_address(i); ++ return 0; ++ } + } ++ ++ return -ENOENT; +} + -+/** -+ * _blkdev_trig_unlink_always() - Perform the unconditionally required steps of -+ * unlinking a block device from an LED. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * When a block device is unlinked from an LED, certain steps must be performed -+ * only if the block device is **not** being released. This function performs -+ * those steps that are **always** required, whether or not the block device is -+ * being released. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void _blkdev_trig_unlink_always(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) ++static int __kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, ++ unsigned int *index, ++ unsigned int *offset, ++ unsigned long *addr) +{ -+ --blkdev_trig_link_count; -+ -+ if (blkdev_trig_link_count == 0) -+ WARN_ON(!cancel_delayed_work_sync(&blkdev_trig_work)); ++ unsigned int i = *index; ++ unsigned int off = *offset; ++ unsigned int name_len; ++ const unsigned char *name; + -+ xa_erase(&btb->linked_btls, btl->index); -+ xa_erase(&btl->linked_btbs, btb->index); ++ for (; len && i < kallsyms_num_syms; i++) { ++ /* ++ * For each entry in kallsyms_names[], the storage format is: ++ * ---------------------------- ++ * | len(1) | type(1) | name(x) | ++ * ---------------------------- ++ * ++ * Number of bytes in parentheses, and: len = 1 + x ++ */ ++ name_len = kallsyms_names[off] - 1; ++ name = &kallsyms_names[off + 2]; ++ off += name_len + 2; + -+ /* Remove /sys/class/leds//linked_devices/ symlink */ -+ sysfs_remove_link_from_group(&btl->led->dev->kobj, -+ blkdev_trig_linked_devs.name, -+ dev_name(&btb->bdev->bd_device)); -+} ++ if (name_len != len) ++ continue; + -+/** -+ * blkdev_trig_unlink_norelease() - Unlink an LED from a block device that is -+ * **not** being released. -+ * @btl: The BTL that represents the LED. -+ * @btb: The BTB that represents the block device. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_unlink_norelease(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) -+{ -+ _blkdev_trig_unlink_always(btl, btb); ++ if (!memcmp(name, namebuf, len)) { ++ /* Prepare for the next iteration */ ++ *index = i + 1; ++ *offset = off; + -+ /* Remove /sys/class/block//linked_leds/ symlink */ -+ sysfs_remove_link_from_group(bdev_kobj(btb->bdev), -+ blkdev_trig_linked_leds.name, -+ btl->led->name); ++ *addr = kallsyms_sym_address(i); ++ return 0; ++ } ++ } + -+ blkdev_trig_put_btb(btb); ++ return -ENOENT; +} + -+/** -+ * blkdev_trig_unlink_release() - Unlink an LED from a block device that is -+ * being released. -+ * @btl: The BTL that represents the LED -+ * @btb: The BTB that represents the block device -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_unlink_release(struct blkdev_trig_led *btl, -+ struct blkdev_trig_bdev *btb) ++static int kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, ++ unsigned long *addr) +{ -+ _blkdev_trig_unlink_always(btl, btb); ++ unsigned int i = 0, off = 0; + -+ /* -+ * If the BTB is being released, the driver core has already removed the -+ * device's attribute groups, and the BTB will be freed automatically, -+ * so there's nothing else to do. -+ */ ++ return __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, addr); +} + -+ -+/* -+ * -+ * BTB creation -+ * -+ */ -+ -+/** -+ * blkdev_trig_btb_release() - BTB device resource release function. -+ * @dev: The block device -+ * @res: The BTB -+ * -+ * Called by the driver core when a block device with a BTB is removed. -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_btb_release(struct device *dev, void *res) + /* Lookup the address for this symbol. Returns 0 if not found. */ + unsigned long kallsyms_lookup_name(const char *name) + { + char namebuf[KSYM_NAME_LEN]; +- unsigned long i; +- unsigned int off; ++ unsigned long addr; ++ int ret, len; + + /* Skip the search for empty string. */ + if (!*name) + return 0; + +- for (i = 0, off = 0; i < kallsyms_num_syms; i++) { +- off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); ++ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); ++ ret = kallsyms_lookup_compressed_name(namebuf, len, &addr); ++ if (!ret) ++ return addr; + +- if (strcmp(namebuf, name) == 0) +- return kallsyms_sym_address(i); ++ ret = kallsyms_lookup_clang_name(namebuf, len, name, &addr); ++ if (!ret) ++ return addr; + +- if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) +- return kallsyms_sym_address(i); +- } + return module_kallsyms_lookup_name(name); + } + +@@ -213,8 +362,7 @@ unsigned long kallsyms_lookup_name(const char *name) + * Iterate over all symbols in vmlinux. For symbols from modules use + * module_kallsyms_on_each_symbol instead. + */ +-int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, +- unsigned long), ++int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), + void *data) + { + char namebuf[KSYM_NAME_LEN]; +@@ -224,7 +372,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, + + for (i = 0, off = 0; i < kallsyms_num_syms; i++) { + off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); +- ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); ++ ret = fn(data, namebuf, kallsyms_sym_address(i)); + if (ret != 0) + return ret; + cond_resched(); +@@ -232,6 +380,27 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, + return 0; + } + ++int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), ++ const char *name, void *data) +{ -+ struct blkdev_trig_bdev *btb = res; -+ struct blkdev_trig_led *btl; -+ unsigned long index; ++ int ret, len; ++ unsigned long addr; ++ unsigned int i = 0, off = 0; ++ char namebuf[KSYM_NAME_LEN]; + -+ mutex_lock(&blkdev_trig_mutex); ++ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); ++ do { ++ ret = __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, &addr); ++ if (ret) ++ return 0; /* end of lookup */ + -+ xa_for_each (&btb->linked_btls, index, btl) -+ blkdev_trig_unlink_release(btl, btb); ++ ret = fn(data, addr); ++ cond_resched(); ++ } while (!ret); + -+ mutex_unlock(&blkdev_trig_mutex); ++ return ret; +} + -+/** -+ * blkdev_trig_get_bdev() - Get a block device by path. -+ * @path: The value written to an LED's &link_dev_by_path or -+ * &unlink_dev_by_path attribute, which should be the path to a -+ * special file that represents a block device -+ * @len: The number of characters in &path (not including its -+ * terminating null) + static unsigned long get_symbol_pos(unsigned long addr, + unsigned long *symbolsize, + unsigned long *offset) +diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h +index 2d0c6f2f0243..d9672ede8cfc 100644 +--- a/kernel/kallsyms_internal.h ++++ b/kernel/kallsyms_internal.h +@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; + extern const u16 kallsyms_token_index[] __weak; + + extern const unsigned int kallsyms_markers[] __weak; ++extern const unsigned char kallsyms_best_token_table[] __weak; + + #endif // LINUX_KALLSYMS_INTERNAL_H_ +diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c +new file mode 100644 +index 000000000000..f7538a70d36c +--- /dev/null ++++ b/kernel/kallsyms_selftest.c +@@ -0,0 +1,421 @@ ++// SPDX-License-Identifier: GPL-2.0-or-later ++/* ++ * Test the function and performance of kallsyms + * -+ * The caller must call blkdev_put() when finished with the device. ++ * Copyright (C) Huawei Technologies Co., Ltd., 2022 + * -+ * Context: Process context. -+ * Return: The block device, or an error pointer. ++ * Authors: Zhen Lei Huawei + */ -+static struct block_device *blkdev_trig_get_bdev(const char *path, size_t len) -+{ -+ struct block_device *bdev; -+ char *buf; + -+ buf = kmemdup(path, len + 1, GFP_KERNEL); /* +1 to include null */ -+ if (buf == NULL) -+ return ERR_PTR(-ENOMEM); ++#define pr_fmt(fmt) "kallsyms_selftest: " fmt + -+ bdev = blkdev_get_by_path(strim(buf), BLKDEV_TRIG_FMODE, THIS_MODULE); -+ kfree(buf); -+ return bdev; -+} ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+/** -+ * blkdev_trig_get_btb() - Find or create the BTB for a block device. -+ * @path: The value written to an LED's &link_dev_by_path attribute, -+ * which should be the path to a special file that represents a -+ * block device -+ * @len: The number of characters in &path -+ * -+ * If a new BTB is created, because the block device was not previously linked -+ * to any LEDs, the block device's &linked_leds &sysfs directory is created. -+ * -+ * Context: Process context. Caller must hold &blkdev_trig_mutex. -+ * Return: Pointer to the BTB, error pointer on error. -+ */ -+static struct blkdev_trig_bdev *blkdev_trig_get_btb(const char *path, -+ size_t len) -+{ -+ struct block_device *bdev; -+ struct blkdev_trig_bdev *btb; -+ int err; ++#include "kallsyms_internal.h" + -+ bdev = blkdev_trig_get_bdev(path, len); -+ if (IS_ERR(bdev)) -+ return ERR_CAST(bdev); + -+ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ if (btb != NULL) { -+ err = 0; -+ goto exit_put_bdev; -+ } ++#define MAX_NUM_OF_RECORDS 64 + -+ if (blkdev_trig_next_index == ULONG_MAX) { -+ err = -EOVERFLOW; -+ goto exit_put_bdev; ++struct test_stat { ++ int min; ++ int max; ++ int save_cnt; ++ int real_cnt; ++ u64 sum; ++ char *name; ++ unsigned long addr; ++ unsigned long addrs[MAX_NUM_OF_RECORDS]; ++}; ++ ++struct test_item { ++ char *name; ++ unsigned long addr; ++}; ++ ++#define ITEM_FUNC(s) \ ++ { \ ++ .name = #s, \ ++ .addr = (unsigned long)s, \ + } + -+ btb = devres_alloc(blkdev_trig_btb_release, sizeof(*btb), GFP_KERNEL); -+ if (btb == NULL) { -+ err = -ENOMEM; -+ goto exit_put_bdev; ++#define ITEM_DATA(s) \ ++ { \ ++ .name = #s, \ ++ .addr = (unsigned long)&s, \ + } + -+ err = sysfs_create_group(bdev_kobj(bdev), &blkdev_trig_linked_leds); -+ if (err) -+ goto exit_free_btb; ++static int test_var_bss_static; ++static int test_var_data_static = 1; ++int test_var_bss; ++int test_var_data = 1; + -+ btb->index = blkdev_trig_next_index++; -+ btb->bdev = bdev; -+ xa_init(&btb->linked_btls); ++static int test_func_static(void) ++{ ++ test_var_bss_static++; ++ test_var_data_static++; + -+ /* Populate BTB activity counters */ -+ blkdev_trig_update_btb(btb, jiffies); ++ return 0; ++} + -+ devres_add(&bdev->bd_device, btb); ++int test_func(void) ++{ ++ return test_func_static(); ++} + -+exit_free_btb: -+ if (err) -+ devres_free(btb); -+exit_put_bdev: -+ blkdev_put(bdev, BLKDEV_TRIG_FMODE); -+ return err ? ERR_PTR(err) : btb; ++__weak int test_func_weak(void) ++{ ++ test_var_bss++; ++ test_var_data++; ++ return 0; +} + ++static struct test_item test_items[] = { ++ ITEM_FUNC(test_func_static), ++ ITEM_FUNC(test_func), ++ ITEM_FUNC(test_func_weak), ++ ITEM_FUNC(vmalloc), ++ ITEM_FUNC(vfree), ++#ifdef CONFIG_KALLSYMS_ALL ++ ITEM_DATA(test_var_bss_static), ++ ITEM_DATA(test_var_data_static), ++ ITEM_DATA(test_var_bss), ++ ITEM_DATA(test_var_data), ++ ITEM_DATA(vmap_area_list), ++#endif ++}; + -+/* -+ * -+ * Activating and deactivating the trigger on an LED -+ * -+ */ ++static char stub_name[KSYM_NAME_LEN]; + -+/** -+ * blkdev_trig_activate() - Called by the LEDs subsystem when an LED is -+ * associated with the trigger. -+ * @led: The LED -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &0 on success, negative &errno on error. -+ */ -+static int blkdev_trig_activate(struct led_classdev *led) ++static int stat_symbol_len(void *data, const char *name, unsigned long addr) +{ -+ struct blkdev_trig_led *btl; -+ int err; ++ *(u32 *)data += strlen(name); + -+ btl = kzalloc(sizeof(*btl), GFP_KERNEL); -+ if (btl == NULL) -+ return -ENOMEM; ++ return 0; ++} + -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ goto exit_free; ++static void test_kallsyms_compression_ratio(void) ++{ ++ int i; ++ const u8 *name; ++ u32 pos; ++ u32 ratio, total_size, total_len = 0; + -+ if (blkdev_trig_next_index == ULONG_MAX) { -+ err = -EOVERFLOW; -+ goto exit_unlock; -+ } ++ kallsyms_on_each_symbol(stat_symbol_len, &total_len); + -+ btl->index = blkdev_trig_next_index++; -+ btl->last_checked = jiffies; -+ btl->mode = -1; /* set all bits */ -+ btl->led = led; -+ btl->blink_msec = BLKDEV_TRIG_BLINK_DEF; -+ btl->check_jiffies = msecs_to_jiffies(BLKDEV_TRIG_CHECK_DEF); -+ xa_init(&btl->linked_btbs); ++ /* ++ * A symbol name cannot start with a number. This stub name helps us ++ * traverse the entire symbol table without finding a match. It's used ++ * for subsequent performance tests, and its length is the average ++ * length of all symbol names. ++ */ ++ memset(stub_name, '4', sizeof(stub_name)); ++ pos = total_len / kallsyms_num_syms; ++ stub_name[pos] = 0; + -+ hlist_add_head(&btl->all_btls_node, &blkdev_trig_all_btls); -+ led_set_trigger_data(led, btl); ++ pos = kallsyms_num_syms - 1; ++ name = &kallsyms_names[kallsyms_markers[pos >> 8]]; ++ for (i = 0; i <= (pos & 0xff); i++) ++ name = name + (*name) + 1; + -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+exit_free: -+ if (err) -+ kfree(btl); -+ return err; ++ /* ++ * 1. The length fields is not counted ++ * 2. The memory occupied by array kallsyms_token_table[] and ++ * kallsyms_token_index[] needs to be counted. ++ */ ++ total_size = (name - kallsyms_names) - kallsyms_num_syms; ++ pos = kallsyms_token_index[0xff]; ++ total_size += pos + strlen(&kallsyms_token_table[pos]) + 1; ++ total_size += 0x100 * sizeof(u16); ++ ++ pr_info(" ---------------------------------------------------------\n"); ++ pr_info("| nr_symbols | compressed size | original size | ratio(%%) |\n"); ++ pr_info("|---------------------------------------------------------|\n"); ++ ratio = 10000ULL * total_size / total_len; ++ pr_info("| %10d | %10d | %10d | %2d.%-2d |\n", ++ kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100); ++ pr_info(" ---------------------------------------------------------\n"); +} + -+/** -+ * blkdev_trig_deactivate() - Called by the the LEDs subsystem when an LED is -+ * disassociated from the trigger. -+ * @led: The LED -+ * -+ * The LEDs subsystem also calls this function when an LED associated with the -+ * trigger is removed or when the trigger is unregistered (if the module is -+ * unloaded). -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ */ -+static void blkdev_trig_deactivate(struct led_classdev *led) ++static int lookup_name(void *data, const char *name, unsigned long addr) +{ -+ struct blkdev_trig_led *btl = led_get_trigger_data(led); -+ struct blkdev_trig_bdev *btb; -+ unsigned long index; ++ u64 t0, t1, t; ++ unsigned long flags; ++ struct test_stat *stat = (struct test_stat *)data; + -+ mutex_lock(&blkdev_trig_mutex); ++ local_irq_save(flags); ++ t0 = sched_clock(); ++ (void)kallsyms_lookup_name(name); ++ t1 = sched_clock(); ++ local_irq_restore(flags); + -+ xa_for_each (&btl->linked_btbs, index, btb) -+ blkdev_trig_unlink_norelease(btl, btb); ++ t = t1 - t0; ++ if (t < stat->min) ++ stat->min = t; + -+ hlist_del(&btl->all_btls_node); -+ kfree(btl); ++ if (t > stat->max) ++ stat->max = t; + -+ mutex_unlock(&blkdev_trig_mutex); ++ stat->real_cnt++; ++ stat->sum += t; ++ ++ return 0; +} + ++static void test_perf_kallsyms_lookup_name(void) ++{ ++ struct test_stat stat; + -+/* -+ * -+ * Link-related attribute store functions -+ * -+ */ ++ memset(&stat, 0, sizeof(stat)); ++ stat.min = INT_MAX; ++ kallsyms_on_each_symbol(lookup_name, &stat); ++ pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt); ++ pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n", ++ stat.min, stat.max, stat.sum / stat.real_cnt); ++} + -+/** -+ * link_dev_by_path_store() - &link_dev_by_path device attribute store function. -+ * @dev: The LED device -+ * @attr: The &link_dev_by_path attribute (&dev_attr_link_dev_by_path) -+ * @buf: The value written to the attribute, which should be the path to -+ * a special file that represents a block device to be linked to -+ * the LED (e.g. ``/dev/sda``) -+ * @count: The number of characters in &buf -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t link_dev_by_path_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) ++static int find_symbol(void *data, const char *name, unsigned long addr) +{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ struct blkdev_trig_bdev *btb; -+ int err; ++ struct test_stat *stat = (struct test_stat *)data; + -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ return err; ++ if (strcmp(name, stat->name) == 0) { ++ stat->real_cnt++; ++ stat->addr = addr; + -+ btb = blkdev_trig_get_btb(buf, count); -+ if (IS_ERR(btb)) { -+ err = PTR_ERR(btb); -+ goto exit_unlock; -+ } ++ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { ++ stat->addrs[stat->save_cnt] = addr; ++ stat->save_cnt++; ++ } + -+ if (xa_load(&btb->linked_btls, btl->index) != NULL) { -+ err = -EEXIST; -+ goto exit_put_btb; ++ if (stat->real_cnt == stat->max) ++ return 1; + } + -+ err = blkdev_trig_link(btl, btb); -+ -+exit_put_btb: -+ if (err) -+ blkdev_trig_put_btb(btb); -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+ return err ? : count; ++ return 0; +} + -+/** -+ * unlink_dev_by_path_store() - &unlink_dev_by_path device attribute store -+ * function. -+ * @dev: The LED device -+ * @attr: The &unlink_dev_by_path attribute (&dev_attr_unlink_dev_by_path) -+ * @buf: The value written to the attribute, which should be the path to -+ * a special file that represents a block device to be unlinked -+ * from the LED (e.g. ``/dev/sda``) -+ * @count: The number of characters in &buf -+ * -+ * Context: Process context. Takes and releases &blkdev_trig_mutex. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t unlink_dev_by_path_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) ++static void test_perf_kallsyms_on_each_symbol(void) +{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ struct block_device *bdev; -+ struct blkdev_trig_bdev *btb; -+ int err; ++ u64 t0, t1; ++ unsigned long flags; ++ struct test_stat stat; + -+ bdev = blkdev_trig_get_bdev(buf, count); -+ if (IS_ERR(bdev)) -+ return PTR_ERR(bdev); ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = stub_name; ++ local_irq_save(flags); ++ t0 = sched_clock(); ++ kallsyms_on_each_symbol(find_symbol, &stat); ++ t1 = sched_clock(); ++ local_irq_restore(flags); ++ pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0); ++} + -+ err = mutex_lock_interruptible(&blkdev_trig_mutex); -+ if (err) -+ goto exit_put_bdev; ++static int match_symbol(void *data, unsigned long addr) ++{ ++ struct test_stat *stat = (struct test_stat *)data; + -+ btb = devres_find(&bdev->bd_device, blkdev_trig_btb_release, -+ NULL, NULL); -+ if (btb == NULL) { -+ err = -EUNATCH; /* bdev isn't linked to any LED */ -+ goto exit_unlock; -+ } ++ stat->real_cnt++; ++ stat->addr = addr; + -+ if (xa_load(&btb->linked_btls, btl->index) == NULL) { -+ err = -EUNATCH; /* bdev isn't linked to this LED */ -+ goto exit_unlock; ++ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { ++ stat->addrs[stat->save_cnt] = addr; ++ stat->save_cnt++; + } + -+ blkdev_trig_unlink_norelease(btl, btb); ++ if (stat->real_cnt == stat->max) ++ return 1; + -+exit_unlock: -+ mutex_unlock(&blkdev_trig_mutex); -+exit_put_bdev: -+ blkdev_put(bdev, BLKDEV_TRIG_FMODE); -+ return err ? : count; ++ return 0; +} + -+ -+/* -+ * -+ * Atomic attribute show & store functions -+ * -+ */ -+ -+/** -+ * blink_time_show() - &blink_time device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_time attribute (&dev_attr_blink_time) -+ * @buf: Output buffer -+ * -+ * Writes the value of &blkdev_trig_led.blink_msec to &buf. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_time_show(struct device *dev, -+ struct device_attribute *attr, char *buf) ++static void test_perf_kallsyms_on_each_match_symbol(void) +{ -+ const struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ u64 t0, t1; ++ unsigned long flags; ++ struct test_stat stat; + -+ return sprintf(buf, "%u\n", READ_ONCE(btl->blink_msec)); ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = stub_name; ++ local_irq_save(flags); ++ t0 = sched_clock(); ++ kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat); ++ t1 = sched_clock(); ++ local_irq_restore(flags); ++ pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0); +} + -+/** -+ * blink_time_store() - &blink_time device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_time attribute (&dev_attr_blink_time) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets &blkdev_trig_led.blink_msec to the value in &buf. -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_time_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) ++static int test_kallsyms_basic_function(void) +{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); -+ unsigned int value; -+ int err; ++ int i, j, ret; ++ int next = 0, nr_failed = 0; ++ char *prefix; ++ unsigned short rand; ++ unsigned long addr; ++ char namebuf[KSYM_NAME_LEN]; ++ struct test_stat stat, stat1, stat2; + -+ err = kstrtouint(buf, 0, &value); -+ if (err) -+ return err; ++ prefix = "kallsyms_lookup_name() for"; ++ for (i = 0; i < ARRAY_SIZE(test_items); i++) { ++ addr = kallsyms_lookup_name(test_items[i].name); ++ if (addr != test_items[i].addr) { ++ nr_failed++; ++ pr_info("%s %s failed: addr=%lx, expect %lx\n", ++ prefix, test_items[i].name, addr, test_items[i].addr); ++ } ++ } + -+ if (value < BLKDEV_TRIG_BLINK_MIN || value > BLKDEV_TRIG_BLINK_MAX) -+ return -ERANGE; ++ prefix = "kallsyms_on_each_symbol() for"; ++ for (i = 0; i < ARRAY_SIZE(test_items); i++) { ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = test_items[i].name; ++ kallsyms_on_each_symbol(find_symbol, &stat); ++ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { ++ nr_failed++; ++ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", ++ prefix, test_items[i].name, ++ stat.real_cnt, stat.addr, test_items[i].addr); ++ } ++ } + -+ WRITE_ONCE(btl->blink_msec, value); -+ return count; -+} ++ prefix = "kallsyms_on_each_match_symbol() for"; ++ for (i = 0; i < ARRAY_SIZE(test_items); i++) { ++ memset(&stat, 0, sizeof(stat)); ++ stat.max = INT_MAX; ++ stat.name = test_items[i].name; ++ kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, &stat); ++ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { ++ nr_failed++; ++ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", ++ prefix, test_items[i].name, ++ stat.real_cnt, stat.addr, test_items[i].addr); ++ } ++ } + -+/** -+ * check_interval_show() - &check_interval device attribute show function. -+ * @dev: The LED device -+ * @attr: The &check_interval attribute (&dev_attr_check_interval) -+ * @buf: Output buffer -+ * -+ * Writes the value of &blkdev_trig_led.check_jiffies (converted to -+ * milliseconds) to &buf. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t check_interval_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ struct blkdev_trig_led *btl = led_trigger_get_drvdata(dev); ++ if (nr_failed) ++ return -ESRCH; + -+ return sprintf(buf, "%u\n", -+ jiffies_to_msecs(READ_ONCE(btl->check_jiffies))); -+} ++ for (i = 0; i < kallsyms_num_syms; i++) { ++ addr = kallsyms_sym_address(i); ++ if (!is_ksym_addr(addr)) ++ continue; + -+/** -+ * check_interval_store() - &check_interval device attribute store function -+ * @dev: The LED device -+ * @attr: The &check_interval attribute (&dev_attr_check_interval) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets &blkdev_trig_led.check_jiffies to the value in &buf (after converting -+ * from milliseconds). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t check_interval_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct blkdev_trig_led *led = led_trigger_get_drvdata(dev); -+ unsigned int value; -+ int err; ++ ret = lookup_symbol_name(addr, namebuf); ++ if (unlikely(ret)) { ++ namebuf[0] = 0; ++ goto failed; ++ } + -+ err = kstrtouint(buf, 0, &value); -+ if (err) -+ return err; ++ stat.addr = kallsyms_lookup_name(namebuf); + -+ if (value < BLKDEV_TRIG_CHECK_MIN || value > BLKDEV_TRIG_CHECK_MAX) -+ return -ERANGE; ++ memset(&stat1, 0, sizeof(stat1)); ++ stat1.max = INT_MAX; ++ kallsyms_on_each_match_symbol(match_symbol, namebuf, &stat1); + -+ WRITE_ONCE(led->check_jiffies, msecs_to_jiffies(value)); ++ /* ++ * kallsyms_on_each_symbol() is too slow, randomly select some ++ * symbols for test. ++ */ ++ if (i >= next) { ++ memset(&stat2, 0, sizeof(stat2)); ++ stat2.max = INT_MAX; ++ stat2.name = namebuf; ++ kallsyms_on_each_symbol(find_symbol, &stat2); + -+ return count; -+} ++ /* ++ * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol() ++ * need to get the same traversal result. ++ */ ++ if (stat1.addr != stat2.addr || ++ stat1.real_cnt != stat2.real_cnt || ++ memcmp(stat1.addrs, stat2.addrs, ++ stat1.save_cnt * sizeof(stat1.addrs[0]))) ++ goto failed; + -+/** -+ * blkdev_trig_mode_show() - Helper for boolean attribute show functions. -+ * @led: The LED -+ * @buf: Output buffer -+ * @bit: Which bit to show -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static int blkdev_trig_mode_show(const struct blkdev_trig_led *led, char *buf, -+ enum stat_group bit) -+{ -+ return sprintf(buf, READ_ONCE(led->mode) & (1 << bit) ? "Y\n" : "N\n"); -+} ++ /* ++ * The average of random increments is 128, that is, one of ++ * them is tested every 128 symbols. ++ */ ++ get_random_bytes(&rand, sizeof(rand)); ++ next = i + (rand & 0xff) + 1; ++ } + -+/** -+ * blkdev_trig_mode_store() - Helper for boolean attribute store functions. -+ * @led: The LED -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * @bit: Which bit to set -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static int blkdev_trig_mode_store(struct blkdev_trig_led *led, -+ const char *buf, size_t count, -+ enum stat_group bit) -+{ -+ bool set; -+ int err; ++ /* Need to be found at least once */ ++ if (!stat1.real_cnt) ++ goto failed; + -+ err = kstrtobool(buf, &set); -+ if (err) -+ return err; ++ /* ++ * kallsyms_lookup_name() returns the address of the first ++ * symbol found and cannot be NULL. ++ */ ++ if (!stat.addr || stat.addr != stat1.addrs[0]) ++ goto failed; + -+ if (set) -+ set_bit(bit, &led->mode); -+ else -+ clear_bit(bit, &led->mode); ++ /* ++ * If the addresses of all matching symbols are recorded, the ++ * target address needs to be exist. ++ */ ++ if (stat1.real_cnt <= MAX_NUM_OF_RECORDS) { ++ for (j = 0; j < stat1.save_cnt; j++) { ++ if (stat1.addrs[j] == addr) ++ break; ++ } + -+ return count; ++ if (j == stat1.save_cnt) ++ goto failed; ++ } ++ } ++ ++ return 0; ++ ++failed: ++ pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr); ++ return -ESRCH; +} + -+/** -+ * blink_on_read_show() - &blink_on_read device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_READ bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_read_show(struct device *dev, -+ struct device_attribute *attr, char *buf) ++static int test_entry(void *p) +{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_READ); -+} ++ int ret; + -+/** -+ * blink_on_read_store() - &blink_on_read device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_read attribute (&dev_attr_blink_on_read) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_READ bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_read_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_READ); -+} ++ do { ++ schedule_timeout(5 * HZ); ++ } while (system_state != SYSTEM_RUNNING); + -+/** -+ * blink_on_write_show() - &blink_on_write device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_WRITE bit in -+ * in &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_write_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_WRITE); -+} ++ pr_info("start\n"); ++ ret = test_kallsyms_basic_function(); ++ if (ret) { ++ pr_info("abort\n"); ++ return 0; ++ } + -+/** -+ * blink_on_write_store() - &blink_on_write device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_write attribute (&dev_attr_blink_on_write) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_WRITE bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_write_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_WRITE); -+} ++ test_kallsyms_compression_ratio(); ++ test_perf_kallsyms_lookup_name(); ++ test_perf_kallsyms_on_each_symbol(); ++ test_perf_kallsyms_on_each_match_symbol(); ++ pr_info("finish\n"); + -+/** -+ * blink_on_flush_show() - &blink_on_flush device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending whether the &STAT_FLUSH bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_flush_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_FLUSH); ++ return 0; +} + -+/** -+ * blink_on_flush_store() - &blink_on_flush device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_flush attribute (&dev_attr_blink_on_flush) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_FLUSH bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_flush_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) ++static int __init kallsyms_test_init(void) +{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_FLUSH); -+} ++ struct task_struct *t; + -+/** -+ * blink_on_discard_show() - &blink_on_discard device attribute show function. -+ * @dev: The LED device -+ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) -+ * @buf: Output buffer -+ * -+ * Writes ``Y`` or ``N`` to &buf, depending on whether the &STAT_DISCARD bit in -+ * &blkdev_trig_led.mode is set or cleared. -+ * -+ * Context: Process context. -+ * Return: The number of characters written to &buf. -+ */ -+static ssize_t blink_on_discard_show(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ return blkdev_trig_mode_show(led_trigger_get_drvdata(dev), -+ buf, STAT_DISCARD); -+} ++ t = kthread_create(test_entry, NULL, "kallsyms_test"); ++ if (IS_ERR(t)) { ++ pr_info("Create kallsyms selftest task failed\n"); ++ return PTR_ERR(t); ++ } ++ kthread_bind(t, 0); ++ wake_up_process(t); + -+/** -+ * blink_on_discard_store() - &blink_on_discard device attribute store function. -+ * @dev: The LED device -+ * @attr: The &blink_on_discard attribute (&dev_attr_blink_on_discard) -+ * @buf: The new value (as written to the &sysfs attribute) -+ * @count: The number of characters in &buf -+ * -+ * Sets the &STAT_DISCARD bit in &blkdev_trig_led.mode to the value in &buf -+ * (interpretted as a boolean). -+ * -+ * Context: Process context. -+ * Return: &count on success, negative &errno on error. -+ */ -+static ssize_t blink_on_discard_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return blkdev_trig_mode_store(led_trigger_get_drvdata(dev), -+ buf, count, STAT_DISCARD); ++ return 0; +} ++late_initcall(kallsyms_test_init); +diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c +index bc475e62279d..074a91e2cfd5 100644 +--- a/kernel/livepatch/core.c ++++ b/kernel/livepatch/core.c +@@ -118,26 +118,36 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, + } + + struct klp_find_arg { +- const char *objname; + const char *name; + unsigned long addr; + unsigned long count; + unsigned long pos; + }; + +-static int klp_find_callback(void *data, const char *name, +- struct module *mod, unsigned long addr) ++static int klp_find_callback(void *data, const char *name, unsigned long addr) + { + struct klp_find_arg *args = data; + +- if ((mod && !args->objname) || (!mod && args->objname)) +- return 0; +- + if (strcmp(args->name, name)) + return 0; + +- if (args->objname && strcmp(args->objname, mod->name)) +- return 0; ++ args->addr = addr; ++ args->count++; + -+/* Device attributes */ -+static DEVICE_ATTR_WO(link_dev_by_path); -+static DEVICE_ATTR_WO(unlink_dev_by_path); -+static DEVICE_ATTR_RW(blink_time); -+static DEVICE_ATTR_RW(check_interval); -+static DEVICE_ATTR_RW(blink_on_read); -+static DEVICE_ATTR_RW(blink_on_write); -+static DEVICE_ATTR_RW(blink_on_flush); -+static DEVICE_ATTR_RW(blink_on_discard); ++ /* ++ * Finish the search when the symbol is found for the desired position ++ * or the position is not defined for a non-unique symbol. ++ */ ++ if ((args->pos && (args->count == args->pos)) || ++ (!args->pos && (args->count > 1))) ++ return 1; + -+/* Device attributes in LED directory (/sys/class/leds//...) */ -+static struct attribute *blkdev_trig_attrs[] = { -+ &dev_attr_link_dev_by_path.attr, -+ &dev_attr_unlink_dev_by_path.attr, -+ &dev_attr_blink_time.attr, -+ &dev_attr_check_interval.attr, -+ &dev_attr_blink_on_read.attr, -+ &dev_attr_blink_on_write.attr, -+ &dev_attr_blink_on_flush.attr, -+ &dev_attr_blink_on_discard.attr, -+ NULL -+}; ++ return 0; ++} + -+/* Unnamed attribute group == no subdirectory */ -+static const struct attribute_group blkdev_trig_attr_group = { -+ .attrs = blkdev_trig_attrs, -+}; ++static int klp_match_callback(void *data, unsigned long addr) ++{ ++ struct klp_find_arg *args = data; + + args->addr = addr; + args->count++; +@@ -157,7 +167,6 @@ static int klp_find_object_symbol(const char *objname, const char *name, + unsigned long sympos, unsigned long *addr) + { + struct klp_find_arg args = { +- .objname = objname, + .name = name, + .addr = 0, + .count = 0, +@@ -165,9 +174,9 @@ static int klp_find_object_symbol(const char *objname, const char *name, + }; + + if (objname) +- module_kallsyms_on_each_symbol(klp_find_callback, &args); ++ module_kallsyms_on_each_symbol(objname, klp_find_callback, &args); + else +- kallsyms_on_each_symbol(klp_find_callback, &args); ++ kallsyms_on_each_match_symbol(klp_match_callback, name, &args); + + /* + * Ensure an address was found. If sympos is 0, ensure symbol is unique; +diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c +index f5c5c9175333..329cef573675 100644 +--- a/kernel/module/kallsyms.c ++++ b/kernel/module/kallsyms.c +@@ -495,8 +495,8 @@ unsigned long module_kallsyms_lookup_name(const char *name) + } + + #ifdef CONFIG_LIVEPATCH +-int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, +- struct module *, unsigned long), ++int module_kallsyms_on_each_symbol(const char *modname, ++ int (*fn)(void *, const char *, unsigned long), + void *data) + { + struct module *mod; +@@ -510,6 +510,9 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, + if (mod->state == MODULE_STATE_UNFORMED) + continue; + ++ if (strcmp(modname, mod->name)) ++ continue; + -+/* Attribute groups for the trigger */ -+static const struct attribute_group *blkdev_trig_attr_groups[] = { -+ &blkdev_trig_attr_group, /* /sys/class/leds//... */ -+ &blkdev_trig_linked_devs, /* /sys/class/leds//linked_devices/ */ -+ NULL -+}; + /* Use rcu_dereference_sched() to remain compliant with the sparse tool */ + preempt_disable(); + kallsyms = rcu_dereference_sched(mod->kallsyms); +@@ -522,10 +525,16 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, + continue; + + ret = fn(data, kallsyms_symbol_name(kallsyms, i), +- mod, kallsyms_symbol_value(sym)); ++ kallsyms_symbol_value(sym)); + if (ret != 0) + goto out; + } + -+/* Trigger registration data */ -+static struct led_trigger blkdev_trig_trigger = { -+ .name = "blkdev", -+ .activate = blkdev_trig_activate, -+ .deactivate = blkdev_trig_deactivate, -+ .groups = blkdev_trig_attr_groups, -+}; ++ /* ++ * The given module is found, the subsequent modules do not ++ * need to be compared. ++ */ ++ break; + } + out: + mutex_unlock(&module_mutex); +diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c +index 439e2ab6905e..f135a0a334a3 100644 +--- a/kernel/trace/ftrace.c ++++ b/kernel/trace/ftrace.c +@@ -8250,8 +8250,7 @@ struct kallsyms_data { + size_t found; + }; + +-static int kallsyms_callback(void *data, const char *name, +- struct module *mod, unsigned long addr) ++static int kallsyms_callback(void *data, const char *name, unsigned long addr) + { + struct kallsyms_data *args = data; + const char **sym; +diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c +index f18e6dfc68c5..40a6fe6d14ef 100644 +--- a/scripts/kallsyms.c ++++ b/scripts/kallsyms.c +@@ -34,7 +34,8 @@ struct sym_entry { + unsigned int len; + unsigned int start_pos; + unsigned int percpu_absolute; +- unsigned char sym[]; ++ unsigned char type; ++ unsigned char name[]; + }; + + struct addr_range { +@@ -75,11 +76,6 @@ static void usage(void) + exit(1); + } + +-static char *sym_name(const struct sym_entry *s) +-{ +- return (char *)s->sym + 1; +-} +- + static bool is_ignored_symbol(const char *name, char type) + { + /* Symbol names that exactly match to the following are ignored.*/ +@@ -227,11 +223,7 @@ static struct sym_entry *read_symbol(FILE *in) + check_symbol_range(name, addr, text_ranges, ARRAY_SIZE(text_ranges)); + check_symbol_range(name, addr, &percpu_range, 1); + +- /* include the type field in the symbol name, so that it gets +- * compressed together */ +- +- len = strlen(name) + 1; +- ++ len = strlen(name); + sym = malloc(sizeof(*sym) + len + 1); + if (!sym) { + fprintf(stderr, "kallsyms failure: " +@@ -240,8 +232,8 @@ static struct sym_entry *read_symbol(FILE *in) + } + sym->addr = addr; + sym->len = len; +- sym->sym[0] = type; +- strcpy(sym_name(sym), name); ++ sym->type = type; ++ strcpy((char *)sym->name, name); + sym->percpu_absolute = 0; + + return sym; +@@ -265,7 +257,7 @@ static int symbol_in_range(const struct sym_entry *s, + + static int symbol_valid(const struct sym_entry *s) + { +- const char *name = sym_name(s); ++ const char *name = (char *)s->name; + + /* if --all-symbols is not specified, then symbols outside the text + * and inittext sections are discarded */ +@@ -471,12 +463,18 @@ static void write_src(void) + if ((i & 0xFF) == 0) + markers[i >> 8] = off; + +- printf("\t.byte 0x%02x", table[i]->len); ++ /* ++ * Store the symbol type togerher with symbol name. ++ * It helps to reduce the size. ++ */ ++ printf("\t.byte 0x%02x", table[i]->len + 1); ++ printf(", 0x%02x", table[i]->type); + for (k = 0; k < table[i]->len; k++) +- printf(", 0x%02x", table[i]->sym[k]); ++ printf(", 0x%02x", table[i]->name[k]); + printf("\n"); + +- off += table[i]->len + 1; ++ /* fields 'len' and 'type' occupy one byte each */ ++ off += table[i]->len + 1 + 1; + } + printf("\n"); + +@@ -501,6 +499,24 @@ static void write_src(void) + for (i = 0; i < 256; i++) + printf("\t.short\t%d\n", best_idx[i]); + printf("\n"); + -+/** -+ * blkdev_trig_init() - Block device LED trigger initialization. -+ * -+ * Registers the ``blkdev`` LED trigger. -+ * -+ * Return: &0 on success, negative &errno on failure. -+ */ -+static int __init blkdev_trig_init(void) -+{ -+ return led_trigger_register(&blkdev_trig_trigger); -+} -+module_init(blkdev_trig_init); ++ output_label("kallsyms_best_token_table"); ++ for (i = 255, k = 0; (int)i >= 0; i--) { ++ if (best_table_len[i] <= 1) { ++ k++; ++ continue; ++ } + -+/** -+ * blkdev_trig_exit() - Block device LED trigger module exit. -+ * -+ * Unregisters the ``blkdev`` LED trigger. -+ */ -+static void __exit blkdev_trig_exit(void) -+{ -+ led_trigger_unregister(&blkdev_trig_trigger); -+} -+module_exit(blkdev_trig_exit); ++ if (k) { ++ printf("\t.byte 0x00, 0x%02x\n", k); ++ k = 0; ++ } + -+MODULE_DESCRIPTION("Block device LED trigger"); -+MODULE_AUTHOR("Ian Pilcher "); -+MODULE_LICENSE("GPL v2"); -diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h -index 83c7248053a1..d2b8741eabf7 100644 ---- a/include/linux/pageblock-flags.h -+++ b/include/linux/pageblock-flags.h -@@ -48,7 +48,7 @@ extern unsigned int pageblock_order; - #else /* CONFIG_HUGETLB_PAGE */ ++ printf("\t.byte 0x%02x, 0x%02x\n", best_table[i][0], best_table[i][1]); ++ } ++ if (k) ++ printf("\t.byte 0x00, 0x%02x\n", k); ++ printf("\n"); + } - /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ --#define pageblock_order (MAX_ORDER-1) -+#define pageblock_order PAGE_ALLOC_COSTLY_ORDER - #endif /* CONFIG_HUGETLB_PAGE */ +@@ -525,12 +541,12 @@ static void forget_symbol(const unsigned char *symbol, int len) + } -diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h -index 99d26879b02a..a92799fc5e74 100644 ---- a/include/net/bluetooth/rfcomm.h -+++ b/include/net/bluetooth/rfcomm.h -@@ -171,6 +171,7 @@ struct rfcomm_dlc { - struct rfcomm_session *session; - struct sk_buff_head tx_queue; - struct timer_list timer; -+ struct work_struct state_change_work; + /* do the initial token count */ +-static void build_initial_tok_table(void) ++static void build_initial_token_table(void) + { + unsigned int i; - struct mutex lock; - unsigned long state; -@@ -186,6 +187,7 @@ struct rfcomm_dlc { - u8 sec_level; - u8 role_switch; - u32 defer_setup; -+ int err; + for (i = 0; i < table_cnt; i++) +- learn_symbol(table[i]->sym, table[i]->len); ++ learn_symbol(table[i]->name, table[i]->len); + } - uint mtu; - uint cfc; -@@ -310,6 +312,7 @@ struct rfcomm_pinfo { - u8 role_switch; - }; + static unsigned char *find_token(unsigned char *str, int len, +@@ -555,14 +571,14 @@ static void compress_symbols(const unsigned char *str, int idx) + for (i = 0; i < table_cnt; i++) { + + len = table[i]->len; +- p1 = table[i]->sym; ++ p1 = table[i]->name; -+void __rfcomm_sk_state_change(struct work_struct *work); - int rfcomm_init_sockets(void); - void rfcomm_cleanup_sockets(void); + /* find the token on the symbol */ + p2 = find_token(p1, len, str); + if (!p2) continue; -diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c -index 7324764384b6..c6494e85cd68 100644 ---- a/net/bluetooth/rfcomm/core.c -+++ b/net/bluetooth/rfcomm/core.c -@@ -289,6 +289,7 @@ static void rfcomm_dlc_clear_state(struct rfcomm_dlc *d) - d->flags = 0; - d->mscex = 0; - d->sec_level = BT_SECURITY_LOW; -+ d->err = 0; - d->mtu = RFCOMM_DEFAULT_MTU; - d->v24_sig = RFCOMM_V24_RTC | RFCOMM_V24_RTR | RFCOMM_V24_DV; + /* decrease the counts for this symbol's tokens */ +- forget_symbol(table[i]->sym, len); ++ forget_symbol(table[i]->name, len); -@@ -306,6 +307,7 @@ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio) - timer_setup(&d->timer, rfcomm_dlc_timeout, 0); + size = len; - skb_queue_head_init(&d->tx_queue); -+ INIT_WORK(&d->state_change_work, __rfcomm_sk_state_change); - mutex_init(&d->lock); - refcount_set(&d->refcnt, 1); +@@ -584,7 +600,7 @@ static void compress_symbols(const unsigned char *str, int idx) + table[i]->len = len; -diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c -index 4bf4ea6cbb5e..4850dafbaa05 100644 ---- a/net/bluetooth/rfcomm/sock.c -+++ b/net/bluetooth/rfcomm/sock.c -@@ -61,19 +61,22 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) - rfcomm_dlc_throttle(d); + /* increase the counts for this symbol's new tokens */ +- learn_symbol(table[i]->sym, len); ++ learn_symbol(table[i]->name, len); + } } --static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) -+void __rfcomm_sk_state_change(struct work_struct *work) +@@ -637,20 +653,24 @@ static void optimize_result(void) + /* start by placing the symbols that are actually used on the table */ + static void insert_real_symbols_in_table(void) { -+ struct rfcomm_dlc *d = container_of(work, struct rfcomm_dlc, -+ state_change_work); - struct sock *sk = d->owner, *parent; - - if (!sk) - return; +- unsigned int i, j, c; ++ unsigned int i, j; ++ unsigned char c; -- BT_DBG("dlc %p state %ld err %d", d, d->state, err); -- - lock_sock(sk); -+ rfcomm_dlc_lock(d); + for (i = 0; i < table_cnt; i++) { + for (j = 0; j < table[i]->len; j++) { +- c = table[i]->sym[j]; +- best_table[c][0]=c; +- best_table_len[c]=1; ++ c = table[i]->name[j]; ++ best_table[c][0] = c; ++ best_table_len[c] = 1; + } ++ c = table[i]->type; ++ best_table[c][0] = c; ++ best_table_len[c] = 1; + } + } -- if (err) -- sk->sk_err = err; -+ BT_DBG("dlc %p state %ld err %d", d, d->state, d->err); -+ -+ if (d->err) -+ sk->sk_err = d->err; + static void optimize_token_table(void) + { +- build_initial_tok_table(); ++ build_initial_token_table(); - sk->sk_state = d->state; + insert_real_symbols_in_table(); -@@ -91,15 +94,22 @@ static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) - sk->sk_state_change(sk); - } +@@ -660,8 +680,8 @@ static void optimize_token_table(void) + /* guess for "linker script provide" symbol */ + static int may_be_linker_script_provide_symbol(const struct sym_entry *se) + { +- const char *symbol = sym_name(se); +- int len = se->len - 1; ++ const char *symbol = (char *)se->name; ++ int len = se->len; -+ rfcomm_dlc_unlock(d); - release_sock(sk); -+ sock_put(sk); -+} + if (len < 8) + return 0; +@@ -705,8 +725,8 @@ static int compare_symbols(const void *a, const void *b) + return -1; -- if (parent && sock_flag(sk, SOCK_ZAPPED)) { -- /* We have to drop DLC lock here, otherwise -- * rfcomm_sock_destruct() will dead lock. */ -- rfcomm_dlc_unlock(d); -- rfcomm_sock_kill(sk); -- rfcomm_dlc_lock(d); -- } -+static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) -+{ -+ struct sock *sk = d->owner; -+ -+ if (!sk) -+ return; -+ -+ d->err = err; -+ sock_hold(sk); -+ if (!schedule_work(&d->state_change_work)) -+ sock_put(sk); - } + /* sort by "weakness" type */ +- wa = (sa->sym[0] == 'w') || (sa->sym[0] == 'W'); +- wb = (sb->sym[0] == 'w') || (sb->sym[0] == 'W'); ++ wa = (sa->type == 'w') || (sa->type == 'W'); ++ wb = (sb->type == 'w') || (sb->type == 'W'); + if (wa != wb) + return wa - wb; - /* ---- Socket functions ---- */ -diff --git a/tools/objtool/check.c b/tools/objtool/check.c -index e55fdf952a3a..445741f36d6d 100644 ---- a/tools/objtool/check.c -+++ b/tools/objtool/check.c -@@ -2104,9 +2104,6 @@ static int read_noendbr_hints(struct objtool_file *file) - return -1; - } +@@ -717,8 +737,8 @@ static int compare_symbols(const void *a, const void *b) + return wa - wb; -- if (insn->type == INSN_ENDBR) -- WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); -- - insn->noendbr = 1; - } + /* sort by the number of prefix underscores */ +- wa = strspn(sym_name(sa), "_"); +- wb = strspn(sym_name(sb), "_"); ++ wa = strspn((char *)sa->name, "_"); ++ wb = strspn((char *)sb->name, "_"); + if (wa != wb) + return wa - wb; +@@ -742,7 +762,7 @@ static void make_percpus_absolute(void) + * ensure consistent behavior compared to older + * versions of this tool. + */ +- table[i]->sym[0] = 'A'; ++ table[i]->type = 'A'; + table[i]->percpu_absolute = 1; + } + } -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 -From 6e51044a3fe99b75493fedb88cd391598f2c1d95 Mon Sep 17 00:00:00 2001 +From 2fc2cb736eb578dcdd96ebc321ef6fe31971e7a3 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Mon, 26 Sep 2022 00:19:51 +0200 -Subject: [PATCH 13/13] kallsyms +Date: Wed, 28 Sep 2022 00:34:04 +0200 +Subject: [PATCH 16/16] bitmap Signed-off-by: Peter Jung --- - include/linux/kallsyms.h | 12 +- - include/linux/module.h | 4 +- - init/Kconfig | 13 ++ - kernel/Makefile | 1 + - kernel/kallsyms.c | 195 +++++++++++++++-- - kernel/kallsyms_internal.h | 1 + - kernel/kallsyms_selftest.c | 421 +++++++++++++++++++++++++++++++++++++ - kernel/livepatch/core.c | 31 ++- - kernel/module/kallsyms.c | 15 +- - kernel/trace/ftrace.c | 3 +- - scripts/kallsyms.c | 88 +++++--- - 11 files changed, 717 insertions(+), 67 deletions(-) - create mode 100644 kernel/kallsyms_selftest.c + arch/loongarch/kernel/setup.c | 2 +- + arch/mips/kernel/setup.c | 2 +- + arch/powerpc/kernel/head_64.S | 4 + + arch/x86/kernel/smpboot.c | 4 +- + arch/x86/xen/smp_pv.c | 2 +- + fs/ntfs3/bitmap.c | 4 +- + include/linux/bitmap.h | 13 +- + include/linux/bitops.h | 19 +++ + include/linux/cpumask.h | 114 +++++++++---- + include/linux/find.h | 272 ++++++++++++++++++++++++++----- + include/linux/netdevice.h | 10 +- + include/linux/nodemask.h | 27 ++-- + kernel/smp.c | 6 +- + lib/Kconfig | 9 ++ + lib/bitmap.c | 68 +++----- + lib/cpumask.c | 40 ++--- + lib/find_bit.c | 224 +++++++++++++++++--------- + lib/find_bit_benchmark.c | 18 +++ + lib/test_bitmap.c | 291 +++++++++++++++++++++++++++++++++- + tools/include/linux/find.h | 61 ++----- + tools/lib/find_bit.c | 149 ++++++++--------- + 21 files changed, 962 insertions(+), 377 deletions(-) -diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h -index ad39636e0c3f..c7219d74e290 100644 ---- a/include/linux/kallsyms.h -+++ b/include/linux/kallsyms.h -@@ -66,9 +66,11 @@ static inline void *dereference_symbol_descriptor(void *ptr) +diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c +index 8f5c2f9a1a83..18a81edd3ac5 100644 +--- a/arch/loongarch/kernel/setup.c ++++ b/arch/loongarch/kernel/setup.c +@@ -346,7 +346,7 @@ static void __init prefill_possible_map(void) + for (; i < NR_CPUS; i++) + set_cpu_possible(i, false); + +- nr_cpu_ids = possible; ++ set_nr_cpu_ids(possible); } + #endif - #ifdef CONFIG_KALLSYMS --int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, -- unsigned long), -+unsigned long kallsyms_sym_address(int idx); -+int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), - void *data); -+int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data); +diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c +index 2ca156a5b231..e8a0759cb4d0 100644 +--- a/arch/mips/kernel/setup.c ++++ b/arch/mips/kernel/setup.c +@@ -750,7 +750,7 @@ static void __init prefill_possible_map(void) + for (; i < NR_CPUS; i++) + set_cpu_possible(i, false); - /* Lookup the address for a symbol. Returns 0 if not found. */ - unsigned long kallsyms_lookup_name(const char *name); -@@ -168,6 +170,12 @@ static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct - { - return -EOPNOTSUPP; +- nr_cpu_ids = possible; ++ set_nr_cpu_ids(possible); } -+ -+static inline int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data) -+{ -+ return -EOPNOTSUPP; -+} - #endif /*CONFIG_KALLSYMS*/ + #else + static inline void prefill_possible_map(void) {} +diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S +index cf2c08902c05..d36939029701 100644 +--- a/arch/powerpc/kernel/head_64.S ++++ b/arch/powerpc/kernel/head_64.S +@@ -400,8 +400,12 @@ generic_secondary_common_init: + #else + LOAD_REG_ADDR(r8, paca_ptrs) /* Load paca_ptrs pointe */ + ld r8,0(r8) /* Get base vaddr of array */ ++#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) ++ LOAD_REG_IMMEDIATE(r7, NR_CPUS) ++#else + LOAD_REG_ADDR(r7, nr_cpu_ids) /* Load nr_cpu_ids address */ + lwz r7,0(r7) /* also the max paca allocated */ ++#endif + li r5,0 /* logical cpu id */ + 1: + sldi r9,r5,3 /* get paca_ptrs[] index from cpu id */ +diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c +index f24227bc3220..3f3ea0287f69 100644 +--- a/arch/x86/kernel/smpboot.c ++++ b/arch/x86/kernel/smpboot.c +@@ -1316,7 +1316,7 @@ static void __init smp_sanity_check(void) + nr++; + } - static inline void print_ip_sym(const char *loglvl, unsigned long ip) -diff --git a/include/linux/module.h b/include/linux/module.h -index 518296ea7f73..6e1a531d78e7 100644 ---- a/include/linux/module.h -+++ b/include/linux/module.h -@@ -879,8 +879,8 @@ static inline bool module_sig_ok(struct module *module) - } - #endif /* CONFIG_MODULE_SIG */ +- nr_cpu_ids = 8; ++ set_nr_cpu_ids(8); + } + #endif --int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, -- struct module *, unsigned long), -+int module_kallsyms_on_each_symbol(const char *modname, -+ int (*fn)(void *, const char *, unsigned long), - void *data); +@@ -1569,7 +1569,7 @@ __init void prefill_possible_map(void) + possible = i; + } - #endif /* _LINUX_MODULE_H */ -diff --git a/init/Kconfig b/init/Kconfig -index 442a945ca6ae..b3a9ec8aa753 100644 ---- a/init/Kconfig -+++ b/init/Kconfig -@@ -1742,6 +1742,19 @@ config KALLSYMS - symbolic stack backtraces. This increases the size of the kernel - somewhat, as all symbols have to be loaded into the kernel image. +- nr_cpu_ids = possible; ++ set_nr_cpu_ids(possible); + + pr_info("Allowing %d CPUs, %d hotplug CPUs\n", + possible, max_t(int, possible - num_processors, 0)); +diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c +index ba7af2eca755..480be82e9b7b 100644 +--- a/arch/x86/xen/smp_pv.c ++++ b/arch/x86/xen/smp_pv.c +@@ -179,7 +179,7 @@ static void __init _get_smp_config(unsigned int early) + * hypercall to expand the max number of VCPUs an already + * running guest has. So cap it up to X. */ + if (subtract) +- nr_cpu_ids = nr_cpu_ids - subtract; ++ set_nr_cpu_ids(nr_cpu_ids - subtract); + #endif -+config KALLSYMS_SELFTEST -+ bool "Test the basic functions and performance of kallsyms" -+ depends on KALLSYMS -+ default n -+ help -+ Test the basic functions and performance of some interfaces, such as -+ kallsyms_lookup_name. It also calculates the compression rate of the -+ kallsyms compression algorithm for the current symbol set. -+ -+ Start self-test automatically after system startup. Suggest executing -+ "dmesg | grep kallsyms_selftest" to collect test results. "finish" is -+ displayed in the last line, indicating that the test is complete. -+ - config KALLSYMS_ALL - bool "Include all symbols in kallsyms" - depends on DEBUG_KERNEL && KALLSYMS -diff --git a/kernel/Makefile b/kernel/Makefile -index 318789c728d3..122a5fed457b 100644 ---- a/kernel/Makefile -+++ b/kernel/Makefile -@@ -68,6 +68,7 @@ endif - obj-$(CONFIG_UID16) += uid16.o - obj-$(CONFIG_MODULE_SIG_FORMAT) += module_signature.o - obj-$(CONFIG_KALLSYMS) += kallsyms.o -+obj-$(CONFIG_KALLSYMS_SELFTEST) += kallsyms_selftest.o - obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o - obj-$(CONFIG_CRASH_CORE) += crash_core.o - obj-$(CONFIG_KEXEC_CORE) += kexec_core.o -diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c -index 3e7e2c2ad2f7..1512db69aa0a 100644 ---- a/kernel/kallsyms.c -+++ b/kernel/kallsyms.c -@@ -87,6 +87,86 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, - return off; } +diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c +index 5d44ceac855b..e92bbd754365 100644 +--- a/fs/ntfs3/bitmap.c ++++ b/fs/ntfs3/bitmap.c +@@ -560,7 +560,7 @@ static int wnd_rescan(struct wnd_bitmap *wnd) + + buf = (ulong *)bh->b_data; + +- used = __bitmap_weight(buf, wbits); ++ used = bitmap_weight(buf, wbits); + if (used < wbits) { + frb = wbits - used; + wnd->free_bits[iw] = frb; +@@ -1364,7 +1364,7 @@ int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits) + buf = (ulong *)bh->b_data; + + __bitmap_clear(buf, b0, blocksize * 8 - b0); +- frb = wbits - __bitmap_weight(buf, wbits); ++ frb = wbits - bitmap_weight(buf, wbits); + wnd->total_zeroes += frb - wnd->free_bits[iw]; + wnd->free_bits[iw] = frb; + +diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h +index f65410a49fda..7d6d73b78147 100644 +--- a/include/linux/bitmap.h ++++ b/include/linux/bitmap.h +@@ -51,6 +51,7 @@ struct device; + * bitmap_empty(src, nbits) Are all bits zero in *src? + * bitmap_full(src, nbits) Are all bits set in *src? + * bitmap_weight(src, nbits) Hamming Weight: number set bits ++ * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap + * bitmap_set(dst, pos, nbits) Set specified bit area + * bitmap_clear(dst, pos, nbits) Clear specified bit area + * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area +@@ -164,6 +165,8 @@ bool __bitmap_intersects(const unsigned long *bitmap1, + bool __bitmap_subset(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int nbits); + unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); ++unsigned int __bitmap_weight_and(const unsigned long *bitmap1, ++ const unsigned long *bitmap2, unsigned int nbits); + void __bitmap_set(unsigned long *map, unsigned int start, int len); + void __bitmap_clear(unsigned long *map, unsigned int start, int len); + +@@ -222,7 +225,6 @@ void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int n + #else + #define bitmap_copy_le bitmap_copy + #endif +-unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits); + int bitmap_print_to_pagebuf(bool list, char *buf, + const unsigned long *maskp, int nmaskbits); -+static unsigned char *find_token(unsigned char *str, int len, -+ const unsigned char *token) -+{ -+ int i; -+ -+ for (i = 0; i < len - 1; i++) { -+ if (str[i] == token[0] && str[i+1] == token[1]) -+ return &str[i]; -+ } -+ return NULL; -+} -+ -+static int kallsyms_compress_symbol_name(const char *name, char *buf, size_t size) +@@ -439,6 +441,15 @@ unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits) + return __bitmap_weight(src, nbits); + } + ++static __always_inline ++unsigned long bitmap_weight_and(const unsigned long *src1, ++ const unsigned long *src2, unsigned int nbits) +{ -+ int i, j, n, len; -+ unsigned char *p1, *p2; -+ const unsigned char *token; -+ -+ len = strscpy(buf, name, size); -+ if (WARN_ON_ONCE(len <= 0)) -+ return 0; -+ -+ /* -+ * For each entry in kallsyms_best_token_table[], the storage -+ * format is: -+ * 1. For tokens that cannot be used to compress characters, the value -+ * at [j] is 0, and the value at [j+1] is the number of consecutive -+ * tokens with this feature. -+ * 2. For each token: the larger the token value, the higher the -+ * frequency, and the lower the index. -+ * -+ * ------------------------------- -+ * | j | [j] [j+1] | token | -+ * -----|---------------|---------| -+ * | 0 | ?? ?? | 255 | -+ * | 2 | ?? ?? | 254 | -+ * | ... | ?? ?? | ... | -+ * | n-2 | ?? ?? | x | -+ * | n | 00 len | x-1 | -+ * | n+2 | ?? ?? | x-1-len | -+ * above '??' is non-zero -+ */ -+ for (i = 255, j = 0; i >= 0; i--, j += 2) { -+ if (!kallsyms_best_token_table[j]) { -+ i -= kallsyms_best_token_table[j + 1]; -+ if (i < 0) -+ break; -+ j += 2; -+ } -+ token = &kallsyms_best_token_table[j]; -+ -+ p1 = buf; -+ -+ /* find the token on the symbol */ -+ p2 = find_token(p1, len, token); -+ if (!p2) -+ continue; -+ -+ n = len; -+ -+ do { -+ *p2 = i; -+ p2++; -+ n -= (p2 - p1); -+ memmove(p2, p2 + 1, n); -+ p1 = p2; -+ len--; -+ -+ if (n < 2) -+ break; -+ -+ /* find the token on the symbol */ -+ p2 = find_token(p1, n, token); -+ -+ } while (p2); -+ } -+ -+ return len; ++ if (small_const_nbits(nbits)) ++ return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)); ++ return __bitmap_weight_and(src1, src2, nbits); +} + - /* - * Get symbol type information. This is encoded as a single char at the - * beginning of the symbol name. -@@ -128,7 +208,7 @@ static unsigned int get_symbol_offset(unsigned long pos) - return name - kallsyms_names; - } - --static unsigned long kallsyms_sym_address(int idx) -+unsigned long kallsyms_sym_address(int idx) + static __always_inline void bitmap_set(unsigned long *map, unsigned int start, + unsigned int nbits) { - if (!IS_ENABLED(CONFIG_KALLSYMS_BASE_RELATIVE)) - return kallsyms_addresses[idx]; -@@ -186,26 +266,95 @@ static bool cleanup_symbol_name(char *s) - return false; +diff --git a/include/linux/bitops.h b/include/linux/bitops.h +index 3b89c64bcfd8..d7dd83fafeba 100644 +--- a/include/linux/bitops.h ++++ b/include/linux/bitops.h +@@ -247,6 +247,25 @@ static inline unsigned long __ffs64(u64 word) + return __ffs((unsigned long)word); } -+static int kallsyms_lookup_clang_name(unsigned char *namebuf, int len, -+ const char *name, -+ unsigned long *addr) -+{ -+ unsigned long i; -+ unsigned int off; -+ -+ if (!IS_ENABLED(CONFIG_LTO_CLANG)) -+ return -ENOENT; -+ -+ for (i = 0, off = 0; i < kallsyms_num_syms; i++) { -+ off = kallsyms_expand_symbol(off, namebuf, len); -+ -+ if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) { -+ *addr = kallsyms_sym_address(i); -+ return 0; -+ } -+ } -+ -+ return -ENOENT; -+} -+ -+static int __kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, -+ unsigned int *index, -+ unsigned int *offset, -+ unsigned long *addr) ++/** ++ * fns - find N'th set bit in a word ++ * @word: The word to search ++ * @n: Bit to find ++ */ ++static inline unsigned long fns(unsigned long word, unsigned int n) +{ -+ unsigned int i = *index; -+ unsigned int off = *offset; -+ unsigned int name_len; -+ const unsigned char *name; -+ -+ for (; len && i < kallsyms_num_syms; i++) { -+ /* -+ * For each entry in kallsyms_names[], the storage format is: -+ * ---------------------------- -+ * | len(1) | type(1) | name(x) | -+ * ---------------------------- -+ * -+ * Number of bytes in parentheses, and: len = 1 + x -+ */ -+ name_len = kallsyms_names[off] - 1; -+ name = &kallsyms_names[off + 2]; -+ off += name_len + 2; -+ -+ if (name_len != len) -+ continue; -+ -+ if (!memcmp(name, namebuf, len)) { -+ /* Prepare for the next iteration */ -+ *index = i + 1; -+ *offset = off; ++ unsigned int bit; + -+ *addr = kallsyms_sym_address(i); -+ return 0; -+ } ++ while (word) { ++ bit = __ffs(word); ++ if (n-- == 0) ++ return bit; ++ __clear_bit(bit, &word); + } + -+ return -ENOENT; ++ return BITS_PER_LONG; +} + -+static int kallsyms_lookup_compressed_name(unsigned char *namebuf, int len, -+ unsigned long *addr) + /** + * assign_bit - Assign value to a bit in memory + * @nr: the bit to set +diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h +index e8ad12b5b9d2..286804bfe3b7 100644 +--- a/include/linux/cpumask.h ++++ b/include/linux/cpumask.h +@@ -35,19 +35,23 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; + */ + #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) + +-#if NR_CPUS == 1 +-#define nr_cpu_ids 1U ++#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) ++#define nr_cpu_ids ((unsigned int)NR_CPUS) + #else + extern unsigned int nr_cpu_ids; + #endif + +-#ifdef CONFIG_CPUMASK_OFFSTACK +-/* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also, +- * not all bits may be allocated. */ +-#define nr_cpumask_bits nr_cpu_ids ++static inline void set_nr_cpu_ids(unsigned int nr) +{ -+ unsigned int i = 0, off = 0; -+ -+ return __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, addr); ++#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) ++ WARN_ON(nr != nr_cpu_ids); + #else +-#define nr_cpumask_bits ((unsigned int)NR_CPUS) ++ nr_cpu_ids = nr; + #endif +} + - /* Lookup the address for this symbol. Returns 0 if not found. */ - unsigned long kallsyms_lookup_name(const char *name) - { - char namebuf[KSYM_NAME_LEN]; -- unsigned long i; -- unsigned int off; -+ unsigned long addr; -+ int ret, len; - - /* Skip the search for empty string. */ - if (!*name) - return 0; ++/* Deprecated. Always use nr_cpu_ids. */ ++#define nr_cpumask_bits nr_cpu_ids -- for (i = 0, off = 0; i < kallsyms_num_syms; i++) { -- off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); -+ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); -+ ret = kallsyms_lookup_compressed_name(namebuf, len, &addr); -+ if (!ret) -+ return addr; + /* + * The following particular system cpumasks and operations manage +@@ -67,10 +71,6 @@ extern unsigned int nr_cpu_ids; + * cpu_online_mask is the dynamic subset of cpu_present_mask, + * indicating those CPUs available for scheduling. + * +- * If HOTPLUG is enabled, then cpu_possible_mask is forced to have +- * all NR_CPUS bits set, otherwise it is just the set of CPUs that +- * ACPI reports present at boot. +- * + * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, + * depending on what ACPI reports as currently plugged in, otherwise + * cpu_present_mask is just a copy of cpu_possible_mask. +@@ -174,9 +174,8 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) + static inline + unsigned int cpumask_next(int n, const struct cpumask *srcp) + { +- /* -1 is a legal arg here. */ +- if (n != -1) +- cpumask_check(n); ++ /* n is a prior cpu */ ++ cpumask_check(n + 1); + return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1); + } -- if (strcmp(namebuf, name) == 0) -- return kallsyms_sym_address(i); -+ ret = kallsyms_lookup_clang_name(namebuf, len, name, &addr); -+ if (!ret) -+ return addr; +@@ -189,9 +188,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) + */ + static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) + { +- /* -1 is a legal arg here. */ +- if (n != -1) +- cpumask_check(n); ++ /* n is a prior cpu */ ++ cpumask_check(n + 1); + return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); + } -- if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) -- return kallsyms_sym_address(i); -- } - return module_kallsyms_lookup_name(name); +@@ -231,9 +229,8 @@ static inline + unsigned int cpumask_next_and(int n, const struct cpumask *src1p, + const struct cpumask *src2p) + { +- /* -1 is a legal arg here. */ +- if (n != -1) +- cpumask_check(n); ++ /* n is a prior cpu */ ++ cpumask_check(n + 1); + return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), + nr_cpumask_bits, n + 1); } +@@ -246,9 +243,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, + * After the loop, cpu is >= nr_cpu_ids. + */ + #define for_each_cpu(cpu, mask) \ +- for ((cpu) = -1; \ +- (cpu) = cpumask_next((cpu), (mask)), \ +- (cpu) < nr_cpu_ids;) ++ for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) -@@ -213,8 +362,7 @@ unsigned long kallsyms_lookup_name(const char *name) - * Iterate over all symbols in vmlinux. For symbols from modules use - * module_kallsyms_on_each_symbol instead. + /** + * for_each_cpu_not - iterate over every cpu in a complemented mask +@@ -258,17 +253,15 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, + * After the loop, cpu is >= nr_cpu_ids. */ --int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, -- unsigned long), -+int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), - void *data) + #define for_each_cpu_not(cpu, mask) \ +- for ((cpu) = -1; \ +- (cpu) = cpumask_next_zero((cpu), (mask)), \ +- (cpu) < nr_cpu_ids;) ++ for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) + + #if NR_CPUS == 1 + static inline + unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) { - char namebuf[KSYM_NAME_LEN]; -@@ -224,7 +372,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, + cpumask_check(start); +- if (n != -1) +- cpumask_check(n); ++ /* n is a prior cpu */ ++ cpumask_check(n + 1); - for (i = 0, off = 0; i < kallsyms_num_syms; i++) { - off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); -- ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); -+ ret = fn(data, namebuf, kallsyms_sym_address(i)); - if (ret != 0) - return ret; - cond_resched(); -@@ -232,6 +380,27 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - return 0; + /* + * Return the first available CPU when wrapping, or when starting before cpu0, +@@ -293,10 +286,8 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta + * + * After the loop, cpu is >= nr_cpu_ids. + */ +-#define for_each_cpu_wrap(cpu, mask, start) \ +- for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \ +- (cpu) < nr_cpumask_bits; \ +- (cpu) = cpumask_next_wrap((cpu), (mask), (start), true)) ++#define for_each_cpu_wrap(cpu, mask, start) \ ++ for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start) + + /** + * for_each_cpu_and - iterate over every cpu in both masks +@@ -313,9 +304,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta + * After the loop, cpu is >= nr_cpu_ids. + */ + #define for_each_cpu_and(cpu, mask1, mask2) \ +- for ((cpu) = -1; \ +- (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ +- (cpu) < nr_cpu_ids;) ++ for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) + + /** + * cpumask_any_but - return a "random" in a cpumask, but not this one. +@@ -337,6 +326,50 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) + return i; + } + ++/** ++ * cpumask_nth - get the first cpu in a cpumask ++ * @srcp: the cpumask pointer ++ * @cpu: the N'th cpu to find, starting from 0 ++ * ++ * Returns >= nr_cpu_ids if such cpu doesn't exist. ++ */ ++static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) ++{ ++ return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu)); ++} ++ ++/** ++ * cpumask_nth_and - get the first cpu in 2 cpumasks ++ * @srcp1: the cpumask pointer ++ * @srcp2: the cpumask pointer ++ * @cpu: the N'th cpu to find, starting from 0 ++ * ++ * Returns >= nr_cpu_ids if such cpu doesn't exist. ++ */ ++static inline ++unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, ++ const struct cpumask *srcp2) ++{ ++ return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), ++ nr_cpumask_bits, cpumask_check(cpu)); ++} ++ ++/** ++ * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. ++ * @srcp1: the cpumask pointer ++ * @srcp2: the cpumask pointer ++ * @cpu: the N'th cpu to find, starting from 0 ++ * ++ * Returns >= nr_cpu_ids if such cpu doesn't exist. ++ */ ++static inline ++unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, ++ const struct cpumask *srcp2) ++{ ++ return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), ++ nr_cpumask_bits, cpumask_check(cpu)); ++} ++ + #define CPU_BITS_NONE \ + { \ + [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ +@@ -586,6 +619,17 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) + return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits); } -+int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), -+ const char *name, void *data) ++/** ++ * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) ++ * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. ++ * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. ++ */ ++static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, ++ const struct cpumask *srcp2) +{ -+ int ret, len; -+ unsigned long addr; -+ unsigned int i = 0, off = 0; -+ char namebuf[KSYM_NAME_LEN]; -+ -+ len = kallsyms_compress_symbol_name(name, namebuf, ARRAY_SIZE(namebuf)); -+ do { -+ ret = __kallsyms_lookup_compressed_name(namebuf, len, &i, &off, &addr); -+ if (ret) -+ return 0; /* end of lookup */ -+ -+ ret = fn(data, addr); -+ cond_resched(); -+ } while (!ret); -+ -+ return ret; ++ return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); +} + - static unsigned long get_symbol_pos(unsigned long addr, - unsigned long *symbolsize, - unsigned long *offset) -diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h -index 2d0c6f2f0243..d9672ede8cfc 100644 ---- a/kernel/kallsyms_internal.h -+++ b/kernel/kallsyms_internal.h -@@ -26,5 +26,6 @@ extern const char kallsyms_token_table[] __weak; - extern const u16 kallsyms_token_index[] __weak; + /** + * cpumask_shift_right - *dstp = *srcp >> n + * @dstp: the cpumask result +diff --git a/include/linux/find.h b/include/linux/find.h +index 424ef67d4a42..0cdfab9734a6 100644 +--- a/include/linux/find.h ++++ b/include/linux/find.h +@@ -8,15 +8,31 @@ + + #include + +-extern unsigned long _find_next_bit(const unsigned long *addr1, +- const unsigned long *addr2, unsigned long nbits, +- unsigned long start, unsigned long invert, unsigned long le); ++unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, ++ unsigned long start); ++unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long nbits, unsigned long start); ++unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, ++ unsigned long start); + extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); ++unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n); ++unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long size, unsigned long n); ++unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long size, unsigned long n); + extern unsigned long _find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, unsigned long size); + extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); + extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); + ++#ifdef __BIG_ENDIAN ++unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size); ++unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned ++ long size, unsigned long offset); ++unsigned long _find_next_bit_le(const unsigned long *addr, unsigned ++ long size, unsigned long offset); ++#endif ++ + #ifndef find_next_bit + /** + * find_next_bit - find the next set bit in a memory region +@@ -41,7 +57,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, + return val ? __ffs(val) : size; + } - extern const unsigned int kallsyms_markers[] __weak; -+extern const unsigned char kallsyms_best_token_table[] __weak; +- return _find_next_bit(addr, NULL, size, offset, 0UL, 0); ++ return _find_next_bit(addr, size, offset); + } + #endif - #endif // LINUX_KALLSYMS_INTERNAL_H_ -diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c -new file mode 100644 -index 000000000000..f7538a70d36c ---- /dev/null -+++ b/kernel/kallsyms_selftest.c -@@ -0,0 +1,421 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Test the function and performance of kallsyms +@@ -71,7 +87,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, + return val ? __ffs(val) : size; + } + +- return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); ++ return _find_next_and_bit(addr1, addr2, size, offset); + } + #endif + +@@ -99,7 +115,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, + return val == ~0UL ? size : ffz(val); + } + +- return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); ++ return _find_next_zero_bit(addr, size, offset); + } + #endif + +@@ -125,6 +141,87 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) + } + #endif + ++/** ++ * find_nth_bit - find N'th set bit in a memory region ++ * @addr: The address to start the search at ++ * @size: The maximum number of bits to search ++ * @n: The number of set bit, which position is needed, counting from 0 + * -+ * Copyright (C) Huawei Technologies Co., Ltd., 2022 ++ * The following is semantically equivalent: ++ * idx = find_nth_bit(addr, size, 0); ++ * idx = find_first_bit(addr, size); + * -+ * Authors: Zhen Lei Huawei ++ * Returns the bit number of the N'th set bit. ++ * If no such, returns @size. + */ ++static inline ++unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) ++{ ++ if (n >= size) ++ return size; + -+#define pr_fmt(fmt) "kallsyms_selftest: " fmt ++ if (small_const_nbits(size)) { ++ unsigned long val = *addr & GENMASK(size - 1, 0); + -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++ return val ? fns(val, n) : size; ++ } + -+#include "kallsyms_internal.h" ++ return __find_nth_bit(addr, size, n); ++} + ++/** ++ * find_nth_and_bit - find N'th set bit in 2 memory regions ++ * @addr1: The 1st address to start the search at ++ * @addr2: The 2nd address to start the search at ++ * @size: The maximum number of bits to search ++ * @n: The number of set bit, which position is needed, counting from 0 ++ * ++ * Returns the bit number of the N'th set bit. ++ * If no such, returns @size. ++ */ ++static inline ++unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long size, unsigned long n) ++{ ++ if (n >= size) ++ return size; + -+#define MAX_NUM_OF_RECORDS 64 ++ if (small_const_nbits(size)) { ++ unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0); + -+struct test_stat { -+ int min; -+ int max; -+ int save_cnt; -+ int real_cnt; -+ u64 sum; -+ char *name; -+ unsigned long addr; -+ unsigned long addrs[MAX_NUM_OF_RECORDS]; -+}; ++ return val ? fns(val, n) : size; ++ } + -+struct test_item { -+ char *name; -+ unsigned long addr; -+}; ++ return __find_nth_and_bit(addr1, addr2, size, n); ++} + -+#define ITEM_FUNC(s) \ -+ { \ -+ .name = #s, \ -+ .addr = (unsigned long)s, \ -+ } ++/** ++ * find_nth_andnot_bit - find N'th set bit in 2 memory regions, ++ * flipping bits in 2nd region ++ * @addr1: The 1st address to start the search at ++ * @addr2: The 2nd address to start the search at ++ * @size: The maximum number of bits to search ++ * @n: The number of set bit, which position is needed, counting from 0 ++ * ++ * Returns the bit number of the N'th set bit. ++ * If no such, returns @size. ++ */ ++static inline ++unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long size, unsigned long n) ++{ ++ if (n >= size) ++ return size; + -+#define ITEM_DATA(s) \ -+ { \ -+ .name = #s, \ -+ .addr = (unsigned long)&s, \ ++ if (small_const_nbits(size)) { ++ unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0); ++ ++ return val ? fns(val, n) : size; + } + -+static int test_var_bss_static; -+static int test_var_data_static = 1; -+int test_var_bss; -+int test_var_data = 1; ++ return __find_nth_andnot_bit(addr1, addr2, size, n); ++} + -+static int test_func_static(void) + #ifndef find_first_and_bit + /** + * find_first_and_bit - find the first set bit in both memory regions +@@ -193,6 +290,78 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) + } + #endif + ++/** ++ * find_next_and_bit_wrap - find the next set bit in both memory regions ++ * @addr1: The first address to base the search on ++ * @addr2: The second address to base the search on ++ * @size: The bitmap size in bits ++ * @offset: The bitnumber to start searching at ++ * ++ * Returns the bit number for the next set bit, or first set bit up to @offset ++ * If no bits are set, returns @size. ++ */ ++static inline ++unsigned long find_next_and_bit_wrap(const unsigned long *addr1, ++ const unsigned long *addr2, ++ unsigned long size, unsigned long offset) +{ -+ test_var_bss_static++; -+ test_var_data_static++; ++ unsigned long bit = find_next_and_bit(addr1, addr2, size, offset); + -+ return 0; ++ if (bit < size) ++ return bit; ++ ++ bit = find_first_and_bit(addr1, addr2, offset); ++ return bit < offset ? bit : size; +} + -+int test_func(void) ++/** ++ * find_next_bit_wrap - find the next set bit in both memory regions ++ * @addr: The first address to base the search on ++ * @size: The bitmap size in bits ++ * @offset: The bitnumber to start searching at ++ * ++ * Returns the bit number for the next set bit, or first set bit up to @offset ++ * If no bits are set, returns @size. ++ */ ++static inline ++unsigned long find_next_bit_wrap(const unsigned long *addr, ++ unsigned long size, unsigned long offset) +{ -+ return test_func_static(); ++ unsigned long bit = find_next_bit(addr, size, offset); ++ ++ if (bit < size) ++ return bit; ++ ++ bit = find_first_bit(addr, offset); ++ return bit < offset ? bit : size; +} + -+__weak int test_func_weak(void) ++/* ++ * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing ++ * before using it alone. ++ */ ++static inline ++unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size, ++ unsigned long start, unsigned long n) +{ -+ test_var_bss++; -+ test_var_data++; -+ return 0; -+} ++ unsigned long bit; + -+static struct test_item test_items[] = { -+ ITEM_FUNC(test_func_static), -+ ITEM_FUNC(test_func), -+ ITEM_FUNC(test_func_weak), -+ ITEM_FUNC(vmalloc), -+ ITEM_FUNC(vfree), -+#ifdef CONFIG_KALLSYMS_ALL -+ ITEM_DATA(test_var_bss_static), -+ ITEM_DATA(test_var_data_static), -+ ITEM_DATA(test_var_bss), -+ ITEM_DATA(test_var_data), -+ ITEM_DATA(vmap_area_list), -+#endif -+}; ++ /* If not wrapped around */ ++ if (n > start) { ++ /* and have a bit, just return it. */ ++ bit = find_next_bit(bitmap, size, n); ++ if (bit < size) ++ return bit; + -+static char stub_name[KSYM_NAME_LEN]; ++ /* Otherwise, wrap around and ... */ ++ n = 0; ++ } + -+static int stat_symbol_len(void *data, const char *name, unsigned long addr) -+{ -+ *(u32 *)data += strlen(name); ++ /* Search the other part. */ ++ bit = find_next_bit(bitmap, start, n); ++ return bit < start ? bit : size; ++} + -+ return 0; + /** + * find_next_clump8 - find next 8-bit clump with set bits in a memory region + * @clump: location to store copy of found clump +@@ -247,7 +416,21 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned + return val == ~0UL ? size : ffz(val); + } + +- return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); ++ return _find_next_zero_bit_le(addr, size, offset); +} ++#endif + -+static void test_kallsyms_compression_ratio(void) ++#ifndef find_first_zero_bit_le ++static inline ++unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) +{ -+ int i; -+ const u8 *name; -+ u32 pos; -+ u32 ratio, total_size, total_len = 0; ++ if (small_const_nbits(size)) { ++ unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0); + -+ kallsyms_on_each_symbol(stat_symbol_len, &total_len); ++ return val == ~0UL ? size : ffz(val); ++ } + -+ /* -+ * A symbol name cannot start with a number. This stub name helps us -+ * traverse the entire symbol table without finding a match. It's used -+ * for subsequent performance tests, and its length is the average -+ * length of all symbol names. -+ */ -+ memset(stub_name, '4', sizeof(stub_name)); -+ pos = total_len / kallsyms_num_syms; -+ stub_name[pos] = 0; ++ return _find_first_zero_bit_le(addr, size); + } + #endif + +@@ -266,40 +449,34 @@ unsigned long find_next_bit_le(const void *addr, unsigned + return val ? __ffs(val) : size; + } + +- return _find_next_bit(addr, NULL, size, offset, 0UL, 1); ++ return _find_next_bit_le(addr, size, offset); + } + #endif + +-#ifndef find_first_zero_bit_le +-#define find_first_zero_bit_le(addr, size) \ +- find_next_zero_bit_le((addr), (size), 0) +-#endif +- + #else + #error "Please fix " + #endif + + #define for_each_set_bit(bit, addr, size) \ +- for ((bit) = find_next_bit((addr), (size), 0); \ +- (bit) < (size); \ +- (bit) = find_next_bit((addr), (size), (bit) + 1)) ++ for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) ++ ++#define for_each_and_bit(bit, addr1, addr2, size) \ ++ for ((bit) = 0; \ ++ (bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ ++ (bit)++) + + /* same as for_each_set_bit() but use bit as value to start with */ + #define for_each_set_bit_from(bit, addr, size) \ +- for ((bit) = find_next_bit((addr), (size), (bit)); \ +- (bit) < (size); \ +- (bit) = find_next_bit((addr), (size), (bit) + 1)) ++ for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) + + #define for_each_clear_bit(bit, addr, size) \ +- for ((bit) = find_next_zero_bit((addr), (size), 0); \ +- (bit) < (size); \ +- (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) ++ for ((bit) = 0; \ ++ (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); \ ++ (bit)++) + + /* same as for_each_clear_bit() but use bit as value to start with */ + #define for_each_clear_bit_from(bit, addr, size) \ +- for ((bit) = find_next_zero_bit((addr), (size), (bit)); \ +- (bit) < (size); \ +- (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) ++ for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++) + + /** + * for_each_set_bitrange - iterate over all set bit ranges [b; e) +@@ -309,11 +486,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned + * @size: bitmap size in number of bits + */ + #define for_each_set_bitrange(b, e, addr, size) \ +- for ((b) = find_next_bit((addr), (size), 0), \ +- (e) = find_next_zero_bit((addr), (size), (b) + 1); \ ++ for ((b) = 0; \ ++ (b) = find_next_bit((addr), (size), b), \ ++ (e) = find_next_zero_bit((addr), (size), (b) + 1), \ + (b) < (size); \ +- (b) = find_next_bit((addr), (size), (e) + 1), \ +- (e) = find_next_zero_bit((addr), (size), (b) + 1)) ++ (b) = (e) + 1) + + /** + * for_each_set_bitrange_from - iterate over all set bit ranges [b; e) +@@ -323,11 +500,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned + * @size: bitmap size in number of bits + */ + #define for_each_set_bitrange_from(b, e, addr, size) \ +- for ((b) = find_next_bit((addr), (size), (b)), \ +- (e) = find_next_zero_bit((addr), (size), (b) + 1); \ ++ for (; \ ++ (b) = find_next_bit((addr), (size), (b)), \ ++ (e) = find_next_zero_bit((addr), (size), (b) + 1), \ + (b) < (size); \ +- (b) = find_next_bit((addr), (size), (e) + 1), \ +- (e) = find_next_zero_bit((addr), (size), (b) + 1)) ++ (b) = (e) + 1) + + /** + * for_each_clear_bitrange - iterate over all unset bit ranges [b; e) +@@ -337,11 +514,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned + * @size: bitmap size in number of bits + */ + #define for_each_clear_bitrange(b, e, addr, size) \ +- for ((b) = find_next_zero_bit((addr), (size), 0), \ +- (e) = find_next_bit((addr), (size), (b) + 1); \ ++ for ((b) = 0; \ ++ (b) = find_next_zero_bit((addr), (size), (b)), \ ++ (e) = find_next_bit((addr), (size), (b) + 1), \ + (b) < (size); \ +- (b) = find_next_zero_bit((addr), (size), (e) + 1), \ +- (e) = find_next_bit((addr), (size), (b) + 1)) ++ (b) = (e) + 1) + + /** + * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e) +@@ -351,11 +528,24 @@ unsigned long find_next_bit_le(const void *addr, unsigned + * @size: bitmap size in number of bits + */ + #define for_each_clear_bitrange_from(b, e, addr, size) \ +- for ((b) = find_next_zero_bit((addr), (size), (b)), \ +- (e) = find_next_bit((addr), (size), (b) + 1); \ ++ for (; \ ++ (b) = find_next_zero_bit((addr), (size), (b)), \ ++ (e) = find_next_bit((addr), (size), (b) + 1), \ + (b) < (size); \ +- (b) = find_next_zero_bit((addr), (size), (e) + 1), \ +- (e) = find_next_bit((addr), (size), (b) + 1)) ++ (b) = (e) + 1) + -+ pos = kallsyms_num_syms - 1; -+ name = &kallsyms_names[kallsyms_markers[pos >> 8]]; -+ for (i = 0; i <= (pos & 0xff); i++) -+ name = name + (*name) + 1; ++/** ++ * for_each_set_bit_wrap - iterate over all set bits starting from @start, and ++ * wrapping around the end of bitmap. ++ * @bit: offset for current iteration ++ * @addr: bitmap address to base the search on ++ * @size: bitmap size in number of bits ++ * @start: Starting bit for bitmap traversing, wrapping around the bitmap end ++ */ ++#define for_each_set_bit_wrap(bit, addr, size, start) \ ++ for ((bit) = find_next_bit_wrap((addr), (size), (start)); \ ++ (bit) < (size); \ ++ (bit) = __for_each_wrap((addr), (size), (start), (bit) + 1)) + + /** + * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits +diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h +index 05d6f3facd5a..4d6d5a2dd82e 100644 +--- a/include/linux/netdevice.h ++++ b/include/linux/netdevice.h +@@ -3643,9 +3643,8 @@ static inline bool netif_attr_test_online(unsigned long j, + static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp, + unsigned int nr_bits) + { +- /* -1 is a legal arg here. */ +- if (n != -1) +- cpu_max_bits_warn(n, nr_bits); ++ /* n is a prior cpu */ ++ cpu_max_bits_warn(n + 1, nr_bits); + + if (srcp) + return find_next_bit(srcp, nr_bits, n + 1); +@@ -3666,9 +3665,8 @@ static inline int netif_attrmask_next_and(int n, const unsigned long *src1p, + const unsigned long *src2p, + unsigned int nr_bits) + { +- /* -1 is a legal arg here. */ +- if (n != -1) +- cpu_max_bits_warn(n, nr_bits); ++ /* n is a prior cpu */ ++ cpu_max_bits_warn(n + 1, nr_bits); + + if (src1p && src2p) + return find_next_and_bit(src1p, src2p, nr_bits, n + 1); +diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h +index e66742db741c..0294f2de2fb4 100644 +--- a/include/linux/nodemask.h ++++ b/include/linux/nodemask.h +@@ -507,20 +507,19 @@ static inline int node_random(const nodemask_t *maskp) + #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) + int w, bit; + +- w = nodes_weight(*maskp); +- switch (w) { +- case 0: +- bit = NUMA_NO_NODE; +- break; +- case 1: +- bit = first_node(*maskp); +- break; +- default: +- bit = bitmap_ord_to_pos(maskp->bits, +- get_random_int() % w, MAX_NUMNODES); +- break; +- } +- return bit; ++ w = nodes_weight(*maskp); ++ switch (w) { ++ case 0: ++ bit = NUMA_NO_NODE; ++ break; ++ case 1: ++ bit = first_node(*maskp); ++ break; ++ default: ++ bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_int() % w); ++ break; ++ } ++ return bit; + #else + return 0; + #endif +diff --git a/kernel/smp.c b/kernel/smp.c +index 650810a6f29b..661d09ae5d6a 100644 +--- a/kernel/smp.c ++++ b/kernel/smp.c +@@ -1070,7 +1070,7 @@ static int __init nrcpus(char *str) + int nr_cpus; + + if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids) +- nr_cpu_ids = nr_cpus; ++ set_nr_cpu_ids(nr_cpus); + + return 0; + } +@@ -1088,14 +1088,16 @@ static int __init maxcpus(char *str) + + early_param("maxcpus", maxcpus); + ++#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS) + /* Setup number of possible processor ids */ + unsigned int nr_cpu_ids __read_mostly = NR_CPUS; + EXPORT_SYMBOL(nr_cpu_ids); ++#endif + + /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */ + void __init setup_nr_cpu_ids(void) + { +- nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; ++ set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); + } + + /* Called by boot processor to activate the rest. */ +diff --git a/lib/Kconfig b/lib/Kconfig +index 3ea8941ab18d..d628235f7934 100644 +--- a/lib/Kconfig ++++ b/lib/Kconfig +@@ -531,6 +531,15 @@ config CPUMASK_OFFSTACK + them on the stack. This is a bit more expensive, but avoids + stack overflow. + ++config FORCE_NR_CPUS ++ bool "NR_CPUS is set to an actual number of CPUs" ++ depends on SMP ++ help ++ Say Yes if you have NR_CPUS set to an actual number of possible ++ CPUs in your system, not to a default value. This forces the core ++ code to rely on compile-time value and optimize kernel routines ++ better. ++ + config CPU_RMAP + bool + depends on SMP +diff --git a/lib/bitmap.c b/lib/bitmap.c +index 488e6c3e5acc..1c81413c51f8 100644 +--- a/lib/bitmap.c ++++ b/lib/bitmap.c +@@ -333,20 +333,32 @@ bool __bitmap_subset(const unsigned long *bitmap1, + } + EXPORT_SYMBOL(__bitmap_subset); + ++#define BITMAP_WEIGHT(FETCH, bits) \ ++({ \ ++ unsigned int __bits = (bits), idx, w = 0; \ ++ \ ++ for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \ ++ w += hweight_long(FETCH); \ ++ \ ++ if (__bits % BITS_PER_LONG) \ ++ w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \ ++ \ ++ w; \ ++}) ++ + unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) + { +- unsigned int k, lim = bits/BITS_PER_LONG, w = 0; +- +- for (k = 0; k < lim; k++) +- w += hweight_long(bitmap[k]); +- +- if (bits % BITS_PER_LONG) +- w += hweight_long(bitmap[k] & BITMAP_LAST_WORD_MASK(bits)); +- +- return w; ++ return BITMAP_WEIGHT(bitmap[idx], bits); + } + EXPORT_SYMBOL(__bitmap_weight); + ++unsigned int __bitmap_weight_and(const unsigned long *bitmap1, ++ const unsigned long *bitmap2, unsigned int bits) ++{ ++ return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits); ++} ++EXPORT_SYMBOL(__bitmap_weight_and); + -+ /* -+ * 1. The length fields is not counted -+ * 2. The memory occupied by array kallsyms_token_table[] and -+ * kallsyms_token_index[] needs to be counted. -+ */ -+ total_size = (name - kallsyms_names) - kallsyms_num_syms; -+ pos = kallsyms_token_index[0xff]; -+ total_size += pos + strlen(&kallsyms_token_table[pos]) + 1; -+ total_size += 0x100 * sizeof(u16); + void __bitmap_set(unsigned long *map, unsigned int start, int len) + { + unsigned long *p = map + BIT_WORD(start); +@@ -953,37 +965,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne + if (pos >= nbits || !test_bit(pos, buf)) + return -1; + +- return __bitmap_weight(buf, pos); +-} +- +-/** +- * bitmap_ord_to_pos - find position of n-th set bit in bitmap +- * @buf: pointer to bitmap +- * @ord: ordinal bit position (n-th set bit, n >= 0) +- * @nbits: number of valid bit positions in @buf +- * +- * Map the ordinal offset of bit @ord in @buf to its position in @buf. +- * Value of @ord should be in range 0 <= @ord < weight(buf). If @ord +- * >= weight(buf), returns @nbits. +- * +- * If for example, just bits 4 through 7 are set in @buf, then @ord +- * values 0 through 3 will get mapped to 4 through 7, respectively, +- * and all other @ord values returns @nbits. When @ord value 3 +- * gets mapped to (returns) @pos value 7 in this example, that means +- * that the 3rd set bit (starting with 0th) is at position 7 in @buf. +- * +- * The bit positions 0 through @nbits-1 are valid positions in @buf. +- */ +-unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsigned int nbits) +-{ +- unsigned int pos; +- +- for (pos = find_first_bit(buf, nbits); +- pos < nbits && ord; +- pos = find_next_bit(buf, nbits, pos + 1)) +- ord--; +- +- return pos; ++ return bitmap_weight(buf, pos); + } + + /** +@@ -1035,7 +1017,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, + if (n < 0 || w == 0) + set_bit(oldbit, dst); /* identity map */ + else +- set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); ++ set_bit(find_nth_bit(new, nbits, n % w), dst); + } + } + EXPORT_SYMBOL(bitmap_remap); +@@ -1074,7 +1056,7 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, + if (n < 0 || w == 0) + return oldbit; + else +- return bitmap_ord_to_pos(new, n % w, bits); ++ return find_nth_bit(new, bits, n % w); + } + EXPORT_SYMBOL(bitmap_bitremap); + +@@ -1198,7 +1180,7 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig, + * The following code is a more efficient, but less + * obvious, equivalent to the loop: + * for (m = 0; m < bitmap_weight(relmap, bits); m++) { +- * n = bitmap_ord_to_pos(orig, m, bits); ++ * n = find_nth_bit(orig, bits, m); + * if (test_bit(m, orig)) + * set_bit(n, dst); + * } +diff --git a/lib/cpumask.c b/lib/cpumask.c +index f0ae119be8c4..c7c392514fd3 100644 +--- a/lib/cpumask.c ++++ b/lib/cpumask.c +@@ -128,23 +128,21 @@ unsigned int cpumask_local_spread(unsigned int i, int node) + i %= num_online_cpus(); + + if (node == NUMA_NO_NODE) { +- for_each_cpu(cpu, cpu_online_mask) +- if (i-- == 0) +- return cpu; ++ cpu = cpumask_nth(i, cpu_online_mask); ++ if (cpu < nr_cpu_ids) ++ return cpu; + } else { + /* NUMA first. */ +- for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask) +- if (i-- == 0) +- return cpu; +- +- for_each_cpu(cpu, cpu_online_mask) { +- /* Skip NUMA nodes, done above. */ +- if (cpumask_test_cpu(cpu, cpumask_of_node(node))) +- continue; +- +- if (i-- == 0) +- return cpu; +- } ++ cpu = cpumask_nth_and(i, cpu_online_mask, cpumask_of_node(node)); ++ if (cpu < nr_cpu_ids) ++ return cpu; + -+ pr_info(" ---------------------------------------------------------\n"); -+ pr_info("| nr_symbols | compressed size | original size | ratio(%%) |\n"); -+ pr_info("|---------------------------------------------------------|\n"); -+ ratio = 10000ULL * total_size / total_len; -+ pr_info("| %10d | %10d | %10d | %2d.%-2d |\n", -+ kallsyms_num_syms, total_size, total_len, ratio / 100, ratio % 100); -+ pr_info(" ---------------------------------------------------------\n"); ++ i -= cpumask_weight_and(cpu_online_mask, cpumask_of_node(node)); ++ ++ /* Skip NUMA nodes, done above. */ ++ cpu = cpumask_nth_andnot(i, cpu_online_mask, cpumask_of_node(node)); ++ if (cpu < nr_cpu_ids) ++ return cpu; + } + BUG(); + } +@@ -168,10 +166,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, + /* NOTE: our first selection will skip 0. */ + prev = __this_cpu_read(distribute_cpu_mask_prev); + +- next = cpumask_next_and(prev, src1p, src2p); +- if (next >= nr_cpu_ids) +- next = cpumask_first_and(src1p, src2p); +- ++ next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p), ++ nr_cpumask_bits, prev + 1); + if (next < nr_cpu_ids) + __this_cpu_write(distribute_cpu_mask_prev, next); + +@@ -185,11 +181,7 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp) + + /* NOTE: our first selection will skip 0. */ + prev = __this_cpu_read(distribute_cpu_mask_prev); +- +- next = cpumask_next(prev, srcp); +- if (next >= nr_cpu_ids) +- next = cpumask_first(srcp); +- ++ next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1); + if (next < nr_cpu_ids) + __this_cpu_write(distribute_cpu_mask_prev, next); + +diff --git a/lib/find_bit.c b/lib/find_bit.c +index 1b8e4b2a9cba..25609974cbe4 100644 +--- a/lib/find_bit.c ++++ b/lib/find_bit.c +@@ -19,57 +19,78 @@ + #include + #include + +-#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ +- !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ +- !defined(find_next_and_bit) + /* +- * This is a common helper function for find_next_bit, find_next_zero_bit, and +- * find_next_and_bit. The differences are: +- * - The "invert" argument, which is XORed with each fetched word before +- * searching it for one bits. +- * - The optional "addr2", which is anded with "addr1" if present. ++ * Common helper for find_bit() function family ++ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) ++ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) ++ * @size: The bitmap size in bits + */ +-unsigned long _find_next_bit(const unsigned long *addr1, +- const unsigned long *addr2, unsigned long nbits, +- unsigned long start, unsigned long invert, unsigned long le) +-{ +- unsigned long tmp, mask; +- +- if (unlikely(start >= nbits)) +- return nbits; +- +- tmp = addr1[start / BITS_PER_LONG]; +- if (addr2) +- tmp &= addr2[start / BITS_PER_LONG]; +- tmp ^= invert; +- +- /* Handle 1st word. */ +- mask = BITMAP_FIRST_WORD_MASK(start); +- if (le) +- mask = swab(mask); +- +- tmp &= mask; +- +- start = round_down(start, BITS_PER_LONG); +- +- while (!tmp) { +- start += BITS_PER_LONG; +- if (start >= nbits) +- return nbits; +- +- tmp = addr1[start / BITS_PER_LONG]; +- if (addr2) +- tmp &= addr2[start / BITS_PER_LONG]; +- tmp ^= invert; +- } ++#define FIND_FIRST_BIT(FETCH, MUNGE, size) \ ++({ \ ++ unsigned long idx, val, sz = (size); \ ++ \ ++ for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ ++ val = (FETCH); \ ++ if (val) { \ ++ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ ++ break; \ ++ } \ ++ } \ ++ \ ++ sz; \ ++}) + +- if (le) +- tmp = swab(tmp); +- +- return min(start + __ffs(tmp), nbits); +-} +-EXPORT_SYMBOL(_find_next_bit); +-#endif ++/* ++ * Common helper for find_next_bit() function family ++ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) ++ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) ++ * @size: The bitmap size in bits ++ * @start: The bitnumber to start searching at ++ */ ++#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ ++({ \ ++ unsigned long mask, idx, tmp, sz = (size), __start = (start); \ ++ \ ++ if (unlikely(__start >= sz)) \ ++ goto out; \ ++ \ ++ mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ ++ idx = __start / BITS_PER_LONG; \ ++ \ ++ for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ ++ if ((idx + 1) * BITS_PER_LONG >= sz) \ ++ goto out; \ ++ idx++; \ ++ } \ ++ \ ++ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ ++out: \ ++ sz; \ ++}) ++ ++#define FIND_NTH_BIT(FETCH, size, num) \ ++({ \ ++ unsigned long sz = (size), nr = (num), idx, w, tmp; \ ++ \ ++ for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) { \ ++ if (idx * BITS_PER_LONG + nr >= sz) \ ++ goto out; \ ++ \ ++ tmp = (FETCH); \ ++ w = hweight_long(tmp); \ ++ if (w > nr) \ ++ goto found; \ ++ \ ++ nr -= w; \ ++ } \ ++ \ ++ if (sz % BITS_PER_LONG) \ ++ tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz); \ ++found: \ ++ sz = min(idx * BITS_PER_LONG + fns(tmp, nr), sz); \ ++out: \ ++ sz; \ ++}) + + #ifndef find_first_bit + /* +@@ -77,14 +98,7 @@ EXPORT_SYMBOL(_find_next_bit); + */ + unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) + { +- unsigned long idx; +- +- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { +- if (addr[idx]) +- return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); +- } +- +- return size; ++ return FIND_FIRST_BIT(addr[idx], /* nop */, size); + } + EXPORT_SYMBOL(_find_first_bit); + #endif +@@ -97,15 +111,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size) + { +- unsigned long idx, val; +- +- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { +- val = addr1[idx] & addr2[idx]; +- if (val) +- return min(idx * BITS_PER_LONG + __ffs(val), size); +- } +- +- return size; ++ return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); + } + EXPORT_SYMBOL(_find_first_and_bit); + #endif +@@ -116,16 +122,55 @@ EXPORT_SYMBOL(_find_first_and_bit); + */ + unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) + { +- unsigned long idx; ++ return FIND_FIRST_BIT(~addr[idx], /* nop */, size); +} ++EXPORT_SYMBOL(_find_first_zero_bit); ++#endif + +- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { +- if (addr[idx] != ~0UL) +- return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); +- } ++#ifndef find_next_bit ++unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) ++{ ++ return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); ++} ++EXPORT_SYMBOL(_find_next_bit); ++#endif + +- return size; ++unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) ++{ ++ return FIND_NTH_BIT(addr[idx], size, n); + } +-EXPORT_SYMBOL(_find_first_zero_bit); ++EXPORT_SYMBOL(__find_nth_bit); + -+static int lookup_name(void *data, const char *name, unsigned long addr) ++unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long size, unsigned long n) +{ -+ u64 t0, t1, t; -+ unsigned long flags; -+ struct test_stat *stat = (struct test_stat *)data; ++ return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n); ++} ++EXPORT_SYMBOL(__find_nth_and_bit); + -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ (void)kallsyms_lookup_name(name); -+ t1 = sched_clock(); -+ local_irq_restore(flags); ++unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long size, unsigned long n) ++{ ++ return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n); ++} ++EXPORT_SYMBOL(__find_nth_andnot_bit); + -+ t = t1 - t0; -+ if (t < stat->min) -+ stat->min = t; ++#ifndef find_next_and_bit ++unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long nbits, unsigned long start) ++{ ++ return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); ++} ++EXPORT_SYMBOL(_find_next_and_bit); ++#endif + -+ if (t > stat->max) -+ stat->max = t; ++#ifndef find_next_zero_bit ++unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, ++ unsigned long start) ++{ ++ return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); ++} ++EXPORT_SYMBOL(_find_next_zero_bit); + #endif + + #ifndef find_last_bit +@@ -161,3 +206,38 @@ unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr, + return offset; + } + EXPORT_SYMBOL(find_next_clump8); + -+ stat->real_cnt++; -+ stat->sum += t; ++#ifdef __BIG_ENDIAN + -+ return 0; ++#ifndef find_first_zero_bit_le ++/* ++ * Find the first cleared bit in an LE memory region. ++ */ ++unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size) ++{ ++ return FIND_FIRST_BIT(~addr[idx], swab, size); +} ++EXPORT_SYMBOL(_find_first_zero_bit_le); + -+static void test_perf_kallsyms_lookup_name(void) -+{ -+ struct test_stat stat; ++#endif + -+ memset(&stat, 0, sizeof(stat)); -+ stat.min = INT_MAX; -+ kallsyms_on_each_symbol(lookup_name, &stat); -+ pr_info("kallsyms_lookup_name() looked up %d symbols\n", stat.real_cnt); -+ pr_info("The time spent on each symbol is (ns): min=%d, max=%d, avg=%lld\n", -+ stat.min, stat.max, stat.sum / stat.real_cnt); ++#ifndef find_next_zero_bit_le ++unsigned long _find_next_zero_bit_le(const unsigned long *addr, ++ unsigned long size, unsigned long offset) ++{ ++ return FIND_NEXT_BIT(~addr[idx], swab, size, offset); +} ++EXPORT_SYMBOL(_find_next_zero_bit_le); ++#endif + -+static int find_symbol(void *data, const char *name, unsigned long addr) ++#ifndef find_next_bit_le ++unsigned long _find_next_bit_le(const unsigned long *addr, ++ unsigned long size, unsigned long offset) +{ -+ struct test_stat *stat = (struct test_stat *)data; ++ return FIND_NEXT_BIT(addr[idx], swab, size, offset); ++} ++EXPORT_SYMBOL(_find_next_bit_le); + -+ if (strcmp(name, stat->name) == 0) { -+ stat->real_cnt++; -+ stat->addr = addr; ++#endif + -+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { -+ stat->addrs[stat->save_cnt] = addr; -+ stat->save_cnt++; -+ } ++#endif /* __BIG_ENDIAN */ +diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c +index db904b57d4b8..10754586403b 100644 +--- a/lib/find_bit_benchmark.c ++++ b/lib/find_bit_benchmark.c +@@ -115,6 +115,22 @@ static int __init test_find_last_bit(const void *bitmap, unsigned long len) + return 0; + } + ++static int __init test_find_nth_bit(const unsigned long *bitmap, unsigned long len) ++{ ++ unsigned long l, n, w = bitmap_weight(bitmap, len); ++ ktime_t time; + -+ if (stat->real_cnt == stat->max) -+ return 1; ++ time = ktime_get(); ++ for (n = 0; n < w; n++) { ++ l = find_nth_bit(bitmap, len, n); ++ WARN_ON(l >= len); + } ++ time = ktime_get() - time; ++ pr_err("find_nth_bit: %18llu ns, %6ld iterations\n", time, w); + + return 0; +} + -+static void test_perf_kallsyms_on_each_symbol(void) + static int __init test_find_next_and_bit(const void *bitmap, + const void *bitmap2, unsigned long len) + { +@@ -142,6 +158,7 @@ static int __init find_bit_test(void) + test_find_next_bit(bitmap, BITMAP_LEN); + test_find_next_zero_bit(bitmap, BITMAP_LEN); + test_find_last_bit(bitmap, BITMAP_LEN); ++ test_find_nth_bit(bitmap, BITMAP_LEN / 10); + + /* + * test_find_first_bit() may take some time, so +@@ -164,6 +181,7 @@ static int __init find_bit_test(void) + test_find_next_bit(bitmap, BITMAP_LEN); + test_find_next_zero_bit(bitmap, BITMAP_LEN); + test_find_last_bit(bitmap, BITMAP_LEN); ++ test_find_nth_bit(bitmap, BITMAP_LEN); + test_find_first_bit(bitmap, BITMAP_LEN); + test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN); + test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN); +diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c +index 98754ff9fe68..a8005ad3bd58 100644 +--- a/lib/test_bitmap.c ++++ b/lib/test_bitmap.c +@@ -16,6 +16,8 @@ + + #include "../tools/testing/selftests/kselftest_module.h" + ++#define EXP1_IN_BITS (sizeof(exp1) * 8) ++ + KSTM_MODULE_GLOBALS(); + + static char pbl_buffer[PAGE_SIZE] __initdata; +@@ -219,6 +221,47 @@ static void __init test_zero_clear(void) + expect_eq_pbl("", bmap, 1024); + } + ++static void __init test_find_nth_bit(void) ++{ ++ unsigned long b, bit, cnt = 0; ++ DECLARE_BITMAP(bmap, 64 * 3); ++ ++ bitmap_zero(bmap, 64 * 3); ++ __set_bit(10, bmap); ++ __set_bit(20, bmap); ++ __set_bit(30, bmap); ++ __set_bit(40, bmap); ++ __set_bit(50, bmap); ++ __set_bit(60, bmap); ++ __set_bit(80, bmap); ++ __set_bit(123, bmap); ++ ++ expect_eq_uint(10, find_nth_bit(bmap, 64 * 3, 0)); ++ expect_eq_uint(20, find_nth_bit(bmap, 64 * 3, 1)); ++ expect_eq_uint(30, find_nth_bit(bmap, 64 * 3, 2)); ++ expect_eq_uint(40, find_nth_bit(bmap, 64 * 3, 3)); ++ expect_eq_uint(50, find_nth_bit(bmap, 64 * 3, 4)); ++ expect_eq_uint(60, find_nth_bit(bmap, 64 * 3, 5)); ++ expect_eq_uint(80, find_nth_bit(bmap, 64 * 3, 6)); ++ expect_eq_uint(123, find_nth_bit(bmap, 64 * 3, 7)); ++ expect_eq_uint(64 * 3, find_nth_bit(bmap, 64 * 3, 8)); ++ ++ expect_eq_uint(10, find_nth_bit(bmap, 64 * 3 - 1, 0)); ++ expect_eq_uint(20, find_nth_bit(bmap, 64 * 3 - 1, 1)); ++ expect_eq_uint(30, find_nth_bit(bmap, 64 * 3 - 1, 2)); ++ expect_eq_uint(40, find_nth_bit(bmap, 64 * 3 - 1, 3)); ++ expect_eq_uint(50, find_nth_bit(bmap, 64 * 3 - 1, 4)); ++ expect_eq_uint(60, find_nth_bit(bmap, 64 * 3 - 1, 5)); ++ expect_eq_uint(80, find_nth_bit(bmap, 64 * 3 - 1, 6)); ++ expect_eq_uint(123, find_nth_bit(bmap, 64 * 3 - 1, 7)); ++ expect_eq_uint(64 * 3 - 1, find_nth_bit(bmap, 64 * 3 - 1, 8)); ++ ++ for_each_set_bit(bit, exp1, EXP1_IN_BITS) { ++ b = find_nth_bit(exp1, EXP1_IN_BITS, cnt++); ++ expect_eq_uint(b, bit); ++ } ++} ++ + static void __init test_fill_set(void) + { + DECLARE_BITMAP(bmap, 1024); +@@ -557,8 +600,6 @@ static void __init test_bitmap_parse(void) + } + } + +-#define EXP1_IN_BITS (sizeof(exp1) * 8) +- + static void __init test_bitmap_arr32(void) + { + unsigned int nbits, next_bit; +@@ -685,6 +726,239 @@ static void __init test_for_each_set_clump8(void) + expect_eq_clump8(start, CLUMP_EXP_NUMBITS, clump_exp, &clump); + } + ++static void __init test_for_each_set_bit_wrap(void) +{ -+ u64 t0, t1; -+ unsigned long flags; -+ struct test_stat stat; ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int wr, bit; + -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = stub_name; -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ kallsyms_on_each_symbol(find_symbol, &stat); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ pr_info("kallsyms_on_each_symbol() traverse all: %lld ns\n", t1 - t0); -+} ++ bitmap_zero(orig, 500); + -+static int match_symbol(void *data, unsigned long addr) -+{ -+ struct test_stat *stat = (struct test_stat *)data; ++ /* Set individual bits */ ++ for (bit = 0; bit < 500; bit += 10) ++ bitmap_set(orig, bit, 1); + -+ stat->real_cnt++; -+ stat->addr = addr; ++ /* Set range of bits */ ++ bitmap_set(orig, 100, 50); + -+ if (stat->save_cnt < MAX_NUM_OF_RECORDS) { -+ stat->addrs[stat->save_cnt] = addr; -+ stat->save_cnt++; -+ } ++ for (wr = 0; wr < 500; wr++) { ++ bitmap_zero(copy, 500); + -+ if (stat->real_cnt == stat->max) -+ return 1; ++ for_each_set_bit_wrap(bit, orig, 500, wr) ++ bitmap_set(copy, bit, 1); + -+ return 0; ++ expect_eq_bitmap(orig, copy, 500); ++ } +} + -+static void test_perf_kallsyms_on_each_match_symbol(void) ++static void __init test_for_each_set_bit(void) +{ -+ u64 t0, t1; -+ unsigned long flags; -+ struct test_stat stat; ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int bit; + -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = stub_name; -+ local_irq_save(flags); -+ t0 = sched_clock(); -+ kallsyms_on_each_match_symbol(match_symbol, stat.name, &stat); -+ t1 = sched_clock(); -+ local_irq_restore(flags); -+ pr_info("kallsyms_on_each_match_symbol() traverse all: %lld ns\n", t1 - t0); -+} ++ bitmap_zero(orig, 500); ++ bitmap_zero(copy, 500); + -+static int test_kallsyms_basic_function(void) -+{ -+ int i, j, ret; -+ int next = 0, nr_failed = 0; -+ char *prefix; -+ unsigned short rand; -+ unsigned long addr; -+ char namebuf[KSYM_NAME_LEN]; -+ struct test_stat stat, stat1, stat2; ++ /* Set individual bits */ ++ for (bit = 0; bit < 500; bit += 10) ++ bitmap_set(orig, bit, 1); + -+ prefix = "kallsyms_lookup_name() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ addr = kallsyms_lookup_name(test_items[i].name); -+ if (addr != test_items[i].addr) { -+ nr_failed++; -+ pr_info("%s %s failed: addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, addr, test_items[i].addr); -+ } -+ } ++ /* Set range of bits */ ++ bitmap_set(orig, 100, 50); + -+ prefix = "kallsyms_on_each_symbol() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = test_items[i].name; -+ kallsyms_on_each_symbol(find_symbol, &stat); -+ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { -+ nr_failed++; -+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, -+ stat.real_cnt, stat.addr, test_items[i].addr); -+ } -+ } ++ for_each_set_bit(bit, orig, 500) ++ bitmap_set(copy, bit, 1); + -+ prefix = "kallsyms_on_each_match_symbol() for"; -+ for (i = 0; i < ARRAY_SIZE(test_items); i++) { -+ memset(&stat, 0, sizeof(stat)); -+ stat.max = INT_MAX; -+ stat.name = test_items[i].name; -+ kallsyms_on_each_match_symbol(match_symbol, test_items[i].name, &stat); -+ if (stat.addr != test_items[i].addr || stat.real_cnt != 1) { -+ nr_failed++; -+ pr_info("%s %s failed: count=%d, addr=%lx, expect %lx\n", -+ prefix, test_items[i].name, -+ stat.real_cnt, stat.addr, test_items[i].addr); -+ } -+ } ++ expect_eq_bitmap(orig, copy, 500); ++} + -+ if (nr_failed) -+ return -ESRCH; ++static void __init test_for_each_set_bit_from(void) ++{ ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int wr, bit; + -+ for (i = 0; i < kallsyms_num_syms; i++) { -+ addr = kallsyms_sym_address(i); -+ if (!is_ksym_addr(addr)) -+ continue; ++ bitmap_zero(orig, 500); + -+ ret = lookup_symbol_name(addr, namebuf); -+ if (unlikely(ret)) { -+ namebuf[0] = 0; -+ goto failed; -+ } ++ /* Set individual bits */ ++ for (bit = 0; bit < 500; bit += 10) ++ bitmap_set(orig, bit, 1); + -+ stat.addr = kallsyms_lookup_name(namebuf); ++ /* Set range of bits */ ++ bitmap_set(orig, 100, 50); + -+ memset(&stat1, 0, sizeof(stat1)); -+ stat1.max = INT_MAX; -+ kallsyms_on_each_match_symbol(match_symbol, namebuf, &stat1); ++ for (wr = 0; wr < 500; wr++) { ++ DECLARE_BITMAP(tmp, 500); + -+ /* -+ * kallsyms_on_each_symbol() is too slow, randomly select some -+ * symbols for test. -+ */ -+ if (i >= next) { -+ memset(&stat2, 0, sizeof(stat2)); -+ stat2.max = INT_MAX; -+ stat2.name = namebuf; -+ kallsyms_on_each_symbol(find_symbol, &stat2); ++ bitmap_zero(copy, 500); ++ bit = wr; + -+ /* -+ * kallsyms_on_each_symbol() and kallsyms_on_each_match_symbol() -+ * need to get the same traversal result. -+ */ -+ if (stat1.addr != stat2.addr || -+ stat1.real_cnt != stat2.real_cnt || -+ memcmp(stat1.addrs, stat2.addrs, -+ stat1.save_cnt * sizeof(stat1.addrs[0]))) -+ goto failed; ++ for_each_set_bit_from(bit, orig, 500) ++ bitmap_set(copy, bit, 1); + -+ /* -+ * The average of random increments is 128, that is, one of -+ * them is tested every 128 symbols. -+ */ -+ get_random_bytes(&rand, sizeof(rand)); -+ next = i + (rand & 0xff) + 1; -+ } ++ bitmap_copy(tmp, orig, 500); ++ bitmap_clear(tmp, 0, wr); ++ expect_eq_bitmap(tmp, copy, 500); ++ } ++} + -+ /* Need to be found at least once */ -+ if (!stat1.real_cnt) -+ goto failed; ++static void __init test_for_each_clear_bit(void) ++{ ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int bit; + -+ /* -+ * kallsyms_lookup_name() returns the address of the first -+ * symbol found and cannot be NULL. -+ */ -+ if (!stat.addr || stat.addr != stat1.addrs[0]) -+ goto failed; ++ bitmap_fill(orig, 500); ++ bitmap_fill(copy, 500); + -+ /* -+ * If the addresses of all matching symbols are recorded, the -+ * target address needs to be exist. -+ */ -+ if (stat1.real_cnt <= MAX_NUM_OF_RECORDS) { -+ for (j = 0; j < stat1.save_cnt; j++) { -+ if (stat1.addrs[j] == addr) -+ break; -+ } ++ /* Set individual bits */ ++ for (bit = 0; bit < 500; bit += 10) ++ bitmap_clear(orig, bit, 1); + -+ if (j == stat1.save_cnt) -+ goto failed; -+ } -+ } ++ /* Set range of bits */ ++ bitmap_clear(orig, 100, 50); + -+ return 0; ++ for_each_clear_bit(bit, orig, 500) ++ bitmap_clear(copy, bit, 1); + -+failed: -+ pr_info("Test for %dth symbol failed: (%s) addr=%lx", i, namebuf, addr); -+ return -ESRCH; ++ expect_eq_bitmap(orig, copy, 500); +} + -+static int test_entry(void *p) ++static void __init test_for_each_clear_bit_from(void) +{ -+ int ret; ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int wr, bit; + -+ do { -+ schedule_timeout(5 * HZ); -+ } while (system_state != SYSTEM_RUNNING); ++ bitmap_fill(orig, 500); + -+ pr_info("start\n"); -+ ret = test_kallsyms_basic_function(); -+ if (ret) { -+ pr_info("abort\n"); -+ return 0; ++ /* Set individual bits */ ++ for (bit = 0; bit < 500; bit += 10) ++ bitmap_clear(orig, bit, 1); ++ ++ /* Set range of bits */ ++ bitmap_clear(orig, 100, 50); ++ ++ for (wr = 0; wr < 500; wr++) { ++ DECLARE_BITMAP(tmp, 500); ++ ++ bitmap_fill(copy, 500); ++ bit = wr; ++ ++ for_each_clear_bit_from(bit, orig, 500) ++ bitmap_clear(copy, bit, 1); ++ ++ bitmap_copy(tmp, orig, 500); ++ bitmap_set(tmp, 0, wr); ++ expect_eq_bitmap(tmp, copy, 500); + } ++} + -+ test_kallsyms_compression_ratio(); -+ test_perf_kallsyms_lookup_name(); -+ test_perf_kallsyms_on_each_symbol(); -+ test_perf_kallsyms_on_each_match_symbol(); -+ pr_info("finish\n"); ++static void __init test_for_each_set_bitrange(void) ++{ ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int s, e; + -+ return 0; ++ bitmap_zero(orig, 500); ++ bitmap_zero(copy, 500); ++ ++ /* Set individual bits */ ++ for (s = 0; s < 500; s += 10) ++ bitmap_set(orig, s, 1); ++ ++ /* Set range of bits */ ++ bitmap_set(orig, 100, 50); ++ ++ for_each_set_bitrange(s, e, orig, 500) ++ bitmap_set(copy, s, e-s); ++ ++ expect_eq_bitmap(orig, copy, 500); +} + -+static int __init kallsyms_test_init(void) ++static void __init test_for_each_clear_bitrange(void) +{ -+ struct task_struct *t; ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int s, e; + -+ t = kthread_create(test_entry, NULL, "kallsyms_test"); -+ if (IS_ERR(t)) { -+ pr_info("Create kallsyms selftest task failed\n"); -+ return PTR_ERR(t); -+ } -+ kthread_bind(t, 0); -+ wake_up_process(t); ++ bitmap_fill(orig, 500); ++ bitmap_fill(copy, 500); + -+ return 0; ++ /* Set individual bits */ ++ for (s = 0; s < 500; s += 10) ++ bitmap_clear(orig, s, 1); ++ ++ /* Set range of bits */ ++ bitmap_clear(orig, 100, 50); ++ ++ for_each_clear_bitrange(s, e, orig, 500) ++ bitmap_clear(copy, s, e-s); ++ ++ expect_eq_bitmap(orig, copy, 500); +} -+late_initcall(kallsyms_test_init); -diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c -index bc475e62279d..074a91e2cfd5 100644 ---- a/kernel/livepatch/core.c -+++ b/kernel/livepatch/core.c -@@ -118,26 +118,36 @@ static struct klp_object *klp_find_object(struct klp_patch *patch, - } - - struct klp_find_arg { -- const char *objname; - const char *name; - unsigned long addr; - unsigned long count; - unsigned long pos; - }; - --static int klp_find_callback(void *data, const char *name, -- struct module *mod, unsigned long addr) -+static int klp_find_callback(void *data, const char *name, unsigned long addr) - { - struct klp_find_arg *args = data; - -- if ((mod && !args->objname) || (!mod && args->objname)) -- return 0; -- - if (strcmp(args->name, name)) - return 0; - -- if (args->objname && strcmp(args->objname, mod->name)) -- return 0; -+ args->addr = addr; -+ args->count++; + -+ /* -+ * Finish the search when the symbol is found for the desired position -+ * or the position is not defined for a non-unique symbol. -+ */ -+ if ((args->pos && (args->count == args->pos)) || -+ (!args->pos && (args->count > 1))) -+ return 1; ++static void __init test_for_each_set_bitrange_from(void) ++{ ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int wr, s, e; + -+ return 0; ++ bitmap_zero(orig, 500); ++ ++ /* Set individual bits */ ++ for (s = 0; s < 500; s += 10) ++ bitmap_set(orig, s, 1); ++ ++ /* Set range of bits */ ++ bitmap_set(orig, 100, 50); ++ ++ for (wr = 0; wr < 500; wr++) { ++ DECLARE_BITMAP(tmp, 500); ++ ++ bitmap_zero(copy, 500); ++ s = wr; ++ ++ for_each_set_bitrange_from(s, e, orig, 500) ++ bitmap_set(copy, s, e - s); ++ ++ bitmap_copy(tmp, orig, 500); ++ bitmap_clear(tmp, 0, wr); ++ expect_eq_bitmap(tmp, copy, 500); ++ } +} + -+static int klp_match_callback(void *data, unsigned long addr) ++static void __init test_for_each_clear_bitrange_from(void) +{ -+ struct klp_find_arg *args = data; - - args->addr = addr; - args->count++; -@@ -157,7 +167,6 @@ static int klp_find_object_symbol(const char *objname, const char *name, - unsigned long sympos, unsigned long *addr) - { - struct klp_find_arg args = { -- .objname = objname, - .name = name, - .addr = 0, - .count = 0, -@@ -165,9 +174,9 @@ static int klp_find_object_symbol(const char *objname, const char *name, - }; - - if (objname) -- module_kallsyms_on_each_symbol(klp_find_callback, &args); -+ module_kallsyms_on_each_symbol(objname, klp_find_callback, &args); - else -- kallsyms_on_each_symbol(klp_find_callback, &args); -+ kallsyms_on_each_match_symbol(klp_match_callback, name, &args); - - /* - * Ensure an address was found. If sympos is 0, ensure symbol is unique; -diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c -index f5c5c9175333..329cef573675 100644 ---- a/kernel/module/kallsyms.c -+++ b/kernel/module/kallsyms.c -@@ -495,8 +495,8 @@ unsigned long module_kallsyms_lookup_name(const char *name) - } - - #ifdef CONFIG_LIVEPATCH --int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, -- struct module *, unsigned long), -+int module_kallsyms_on_each_symbol(const char *modname, -+ int (*fn)(void *, const char *, unsigned long), - void *data) - { - struct module *mod; -@@ -510,6 +510,9 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, - if (mod->state == MODULE_STATE_UNFORMED) - continue; - -+ if (strcmp(modname, mod->name)) -+ continue; ++ DECLARE_BITMAP(orig, 500); ++ DECLARE_BITMAP(copy, 500); ++ unsigned int wr, s, e; + - /* Use rcu_dereference_sched() to remain compliant with the sparse tool */ - preempt_disable(); - kallsyms = rcu_dereference_sched(mod->kallsyms); -@@ -522,10 +525,16 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, - continue; - - ret = fn(data, kallsyms_symbol_name(kallsyms, i), -- mod, kallsyms_symbol_value(sym)); -+ kallsyms_symbol_value(sym)); - if (ret != 0) - goto out; - } ++ bitmap_fill(orig, 500); + -+ /* -+ * The given module is found, the subsequent modules do not -+ * need to be compared. -+ */ -+ break; - } - out: - mutex_unlock(&module_mutex); -diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c -index 439e2ab6905e..f135a0a334a3 100644 ---- a/kernel/trace/ftrace.c -+++ b/kernel/trace/ftrace.c -@@ -8250,8 +8250,7 @@ struct kallsyms_data { - size_t found; - }; - --static int kallsyms_callback(void *data, const char *name, -- struct module *mod, unsigned long addr) -+static int kallsyms_callback(void *data, const char *name, unsigned long addr) - { - struct kallsyms_data *args = data; - const char **sym; -diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c -index f18e6dfc68c5..40a6fe6d14ef 100644 ---- a/scripts/kallsyms.c -+++ b/scripts/kallsyms.c -@@ -34,7 +34,8 @@ struct sym_entry { - unsigned int len; - unsigned int start_pos; - unsigned int percpu_absolute; -- unsigned char sym[]; -+ unsigned char type; -+ unsigned char name[]; - }; - - struct addr_range { -@@ -75,11 +76,6 @@ static void usage(void) - exit(1); - } - --static char *sym_name(const struct sym_entry *s) --{ -- return (char *)s->sym + 1; --} -- - static bool is_ignored_symbol(const char *name, char type) - { - /* Symbol names that exactly match to the following are ignored.*/ -@@ -227,11 +223,7 @@ static struct sym_entry *read_symbol(FILE *in) - check_symbol_range(name, addr, text_ranges, ARRAY_SIZE(text_ranges)); - check_symbol_range(name, addr, &percpu_range, 1); - -- /* include the type field in the symbol name, so that it gets -- * compressed together */ -- -- len = strlen(name) + 1; -- -+ len = strlen(name); - sym = malloc(sizeof(*sym) + len + 1); - if (!sym) { - fprintf(stderr, "kallsyms failure: " -@@ -240,8 +232,8 @@ static struct sym_entry *read_symbol(FILE *in) - } - sym->addr = addr; - sym->len = len; -- sym->sym[0] = type; -- strcpy(sym_name(sym), name); -+ sym->type = type; -+ strcpy((char *)sym->name, name); - sym->percpu_absolute = 0; - - return sym; -@@ -265,7 +257,7 @@ static int symbol_in_range(const struct sym_entry *s, - - static int symbol_valid(const struct sym_entry *s) - { -- const char *name = sym_name(s); -+ const char *name = (char *)s->name; - - /* if --all-symbols is not specified, then symbols outside the text - * and inittext sections are discarded */ -@@ -471,12 +463,18 @@ static void write_src(void) - if ((i & 0xFF) == 0) - markers[i >> 8] = off; - -- printf("\t.byte 0x%02x", table[i]->len); -+ /* -+ * Store the symbol type togerher with symbol name. -+ * It helps to reduce the size. -+ */ -+ printf("\t.byte 0x%02x", table[i]->len + 1); -+ printf(", 0x%02x", table[i]->type); - for (k = 0; k < table[i]->len; k++) -- printf(", 0x%02x", table[i]->sym[k]); -+ printf(", 0x%02x", table[i]->name[k]); - printf("\n"); - -- off += table[i]->len + 1; -+ /* fields 'len' and 'type' occupy one byte each */ -+ off += table[i]->len + 1 + 1; - } - printf("\n"); - -@@ -501,6 +499,24 @@ static void write_src(void) - for (i = 0; i < 256; i++) - printf("\t.short\t%d\n", best_idx[i]); - printf("\n"); ++ /* Set individual bits */ ++ for (s = 0; s < 500; s += 10) ++ bitmap_clear(orig, s, 1); + -+ output_label("kallsyms_best_token_table"); -+ for (i = 255, k = 0; (int)i >= 0; i--) { -+ if (best_table_len[i] <= 1) { -+ k++; -+ continue; -+ } ++ /* Set range of bits */ ++ bitmap_set(orig, 100, 50); + -+ if (k) { -+ printf("\t.byte 0x00, 0x%02x\n", k); -+ k = 0; -+ } ++ for (wr = 0; wr < 500; wr++) { ++ DECLARE_BITMAP(tmp, 500); + -+ printf("\t.byte 0x%02x, 0x%02x\n", best_table[i][0], best_table[i][1]); ++ bitmap_fill(copy, 500); ++ s = wr; ++ ++ for_each_clear_bitrange_from(s, e, orig, 500) ++ bitmap_clear(copy, s, e - s); ++ ++ bitmap_copy(tmp, orig, 500); ++ bitmap_set(tmp, 0, wr); ++ expect_eq_bitmap(tmp, copy, 500); + } -+ if (k) -+ printf("\t.byte 0x00, 0x%02x\n", k); -+ printf("\n"); ++} ++ + struct test_bitmap_cut { + unsigned int first; + unsigned int cut; +@@ -948,10 +1222,21 @@ static void __init selftest(void) + test_bitmap_parselist(); + test_bitmap_printlist(); + test_mem_optimisations(); +- test_for_each_set_clump8(); + test_bitmap_cut(); + test_bitmap_print_buf(); + test_bitmap_const_eval(); ++ ++ test_find_nth_bit(); ++ test_for_each_set_bit(); ++ test_for_each_set_bit_from(); ++ test_for_each_clear_bit(); ++ test_for_each_clear_bit_from(); ++ test_for_each_set_bitrange(); ++ test_for_each_clear_bitrange(); ++ test_for_each_set_bitrange_from(); ++ test_for_each_clear_bitrange_from(); ++ test_for_each_set_clump8(); ++ test_for_each_set_bit_wrap(); } + KSTM_MODULE_LOADERS(test_bitmap); +diff --git a/tools/include/linux/find.h b/tools/include/linux/find.h +index 47e2bd6c5174..38c0a542b0e2 100644 +--- a/tools/include/linux/find.h ++++ b/tools/include/linux/find.h +@@ -8,21 +8,23 @@ + + #include + +-extern unsigned long _find_next_bit(const unsigned long *addr1, +- const unsigned long *addr2, unsigned long nbits, +- unsigned long start, unsigned long invert, unsigned long le); ++unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, ++ unsigned long start); ++unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long nbits, unsigned long start); ++unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, ++ unsigned long start); + extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); + extern unsigned long _find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, unsigned long size); + extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); +-extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); + + #ifndef find_next_bit + /** + * find_next_bit - find the next set bit in a memory region + * @addr: The address to base the search on +- * @offset: The bitnumber to start searching at + * @size: The bitmap size in bits ++ * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next set bit + * If no bits are set, returns @size. +@@ -41,7 +43,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, + return val ? __ffs(val) : size; + } -@@ -525,12 +541,12 @@ static void forget_symbol(const unsigned char *symbol, int len) +- return _find_next_bit(addr, NULL, size, offset, 0UL, 0); ++ return _find_next_bit(addr, size, offset); } + #endif - /* do the initial token count */ --static void build_initial_tok_table(void) -+static void build_initial_token_table(void) - { - unsigned int i; +@@ -50,8 +52,8 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, + * find_next_and_bit - find the next set bit in both memory regions + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on +- * @offset: The bitnumber to start searching at + * @size: The bitmap size in bits ++ * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next set bit + * If no bits are set, returns @size. +@@ -71,7 +73,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, + return val ? __ffs(val) : size; + } - for (i = 0; i < table_cnt; i++) -- learn_symbol(table[i]->sym, table[i]->len); -+ learn_symbol(table[i]->name, table[i]->len); +- return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); ++ return _find_next_and_bit(addr1, addr2, size, offset); } + #endif - static unsigned char *find_token(unsigned char *str, int len, -@@ -555,14 +571,14 @@ static void compress_symbols(const unsigned char *str, int idx) - for (i = 0; i < table_cnt; i++) { - - len = table[i]->len; -- p1 = table[i]->sym; -+ p1 = table[i]->name; - - /* find the token on the symbol */ - p2 = find_token(p1, len, str); - if (!p2) continue; - - /* decrease the counts for this symbol's tokens */ -- forget_symbol(table[i]->sym, len); -+ forget_symbol(table[i]->name, len); - - size = len; +@@ -79,8 +81,8 @@ unsigned long find_next_and_bit(const unsigned long *addr1, + /** + * find_next_zero_bit - find the next cleared bit in a memory region + * @addr: The address to base the search on +- * @offset: The bitnumber to start searching at + * @size: The bitmap size in bits ++ * @offset: The bitnumber to start searching at + * + * Returns the bit number of the next zero bit + * If no bits are zero, returns @size. +@@ -99,7 +101,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, + return val == ~0UL ? size : ffz(val); + } -@@ -584,7 +600,7 @@ static void compress_symbols(const unsigned char *str, int idx) - table[i]->len = len; +- return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); ++ return _find_next_zero_bit(addr, size, offset); + } + #endif - /* increase the counts for this symbol's new tokens */ -- learn_symbol(table[i]->sym, len); -+ learn_symbol(table[i]->name, len); - } +@@ -172,43 +174,4 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) } + #endif -@@ -637,20 +653,24 @@ static void optimize_result(void) - /* start by placing the symbols that are actually used on the table */ - static void insert_real_symbols_in_table(void) +-#ifndef find_last_bit +-/** +- * find_last_bit - find the last set bit in a memory region +- * @addr: The address to start the search at +- * @size: The number of bits to search +- * +- * Returns the bit number of the last set bit, or size. +- */ +-static inline +-unsigned long find_last_bit(const unsigned long *addr, unsigned long size) +-{ +- if (small_const_nbits(size)) { +- unsigned long val = *addr & GENMASK(size - 1, 0); +- +- return val ? __fls(val) : size; +- } +- +- return _find_last_bit(addr, size); +-} +-#endif +- +-/** +- * find_next_clump8 - find next 8-bit clump with set bits in a memory region +- * @clump: location to store copy of found clump +- * @addr: address to base the search on +- * @size: bitmap size in number of bits +- * @offset: bit offset at which to start searching +- * +- * Returns the bit offset for the next set clump; the found clump value is +- * copied to the location pointed by @clump. If no bits are set, returns @size. +- */ +-extern unsigned long find_next_clump8(unsigned long *clump, +- const unsigned long *addr, +- unsigned long size, unsigned long offset); +- +-#define find_first_clump8(clump, bits, size) \ +- find_next_clump8((clump), (bits), (size), 0) +- +- + #endif /*__LINUX_FIND_H_ */ +diff --git a/tools/lib/find_bit.c b/tools/lib/find_bit.c +index ba4b8d94e004..6a3dc167d30e 100644 +--- a/tools/lib/find_bit.c ++++ b/tools/lib/find_bit.c +@@ -18,66 +18,54 @@ + #include + #include + +-#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ +- !defined(find_next_and_bit) +- + /* +- * This is a common helper function for find_next_bit, find_next_zero_bit, and +- * find_next_and_bit. The differences are: +- * - The "invert" argument, which is XORed with each fetched word before +- * searching it for one bits. +- * - The optional "addr2", which is anded with "addr1" if present. ++ * Common helper for find_bit() function family ++ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) ++ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) ++ * @size: The bitmap size in bits + */ +-unsigned long _find_next_bit(const unsigned long *addr1, +- const unsigned long *addr2, unsigned long nbits, +- unsigned long start, unsigned long invert, unsigned long le) +-{ +- unsigned long tmp, mask; +- (void) le; +- +- if (unlikely(start >= nbits)) +- return nbits; +- +- tmp = addr1[start / BITS_PER_LONG]; +- if (addr2) +- tmp &= addr2[start / BITS_PER_LONG]; +- tmp ^= invert; +- +- /* Handle 1st word. */ +- mask = BITMAP_FIRST_WORD_MASK(start); +- +- /* +- * Due to the lack of swab() in tools, and the fact that it doesn't +- * need little-endian support, just comment it out +- */ +-#if (0) +- if (le) +- mask = swab(mask); +-#endif +- +- tmp &= mask; ++#define FIND_FIRST_BIT(FETCH, MUNGE, size) \ ++({ \ ++ unsigned long idx, val, sz = (size); \ ++ \ ++ for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ ++ val = (FETCH); \ ++ if (val) { \ ++ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ ++ break; \ ++ } \ ++ } \ ++ \ ++ sz; \ ++}) + +- start = round_down(start, BITS_PER_LONG); +- +- while (!tmp) { +- start += BITS_PER_LONG; +- if (start >= nbits) +- return nbits; +- +- tmp = addr1[start / BITS_PER_LONG]; +- if (addr2) +- tmp &= addr2[start / BITS_PER_LONG]; +- tmp ^= invert; +- } +- +-#if (0) +- if (le) +- tmp = swab(tmp); +-#endif +- +- return min(start + __ffs(tmp), nbits); +-} +-#endif ++/* ++ * Common helper for find_next_bit() function family ++ * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) ++ * @MUNGE: The expression that post-processes a word containing found bit (may be empty) ++ * @size: The bitmap size in bits ++ * @start: The bitnumber to start searching at ++ */ ++#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ ++({ \ ++ unsigned long mask, idx, tmp, sz = (size), __start = (start); \ ++ \ ++ if (unlikely(__start >= sz)) \ ++ goto out; \ ++ \ ++ mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ ++ idx = __start / BITS_PER_LONG; \ ++ \ ++ for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ ++ if ((idx + 1) * BITS_PER_LONG >= sz) \ ++ goto out; \ ++ idx++; \ ++ } \ ++ \ ++ sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ ++out: \ ++ sz; \ ++}) + + #ifndef find_first_bit + /* +@@ -85,14 +73,7 @@ unsigned long _find_next_bit(const unsigned long *addr1, + */ + unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) { -- unsigned int i, j, c; -+ unsigned int i, j; -+ unsigned char c; - - for (i = 0; i < table_cnt; i++) { - for (j = 0; j < table[i]->len; j++) { -- c = table[i]->sym[j]; -- best_table[c][0]=c; -- best_table_len[c]=1; -+ c = table[i]->name[j]; -+ best_table[c][0] = c; -+ best_table_len[c] = 1; - } -+ c = table[i]->type; -+ best_table[c][0] = c; -+ best_table_len[c] = 1; - } +- unsigned long idx; +- +- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { +- if (addr[idx]) +- return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); +- } +- +- return size; ++ return FIND_FIRST_BIT(addr[idx], /* nop */, size); } + #endif - static void optimize_token_table(void) +@@ -104,15 +85,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size) { -- build_initial_tok_table(); -+ build_initial_token_table(); - - insert_real_symbols_in_table(); +- unsigned long idx, val; +- +- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { +- val = addr1[idx] & addr2[idx]; +- if (val) +- return min(idx * BITS_PER_LONG + __ffs(val), size); +- } +- +- return size; ++ return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); + } + #endif -@@ -660,8 +680,8 @@ static void optimize_token_table(void) - /* guess for "linker script provide" symbol */ - static int may_be_linker_script_provide_symbol(const struct sym_entry *se) +@@ -122,13 +95,29 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, + */ + unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) { -- const char *symbol = sym_name(se); -- int len = se->len - 1; -+ const char *symbol = (char *)se->name; -+ int len = se->len; - - if (len < 8) - return 0; -@@ -705,8 +725,8 @@ static int compare_symbols(const void *a, const void *b) - return -1; - - /* sort by "weakness" type */ -- wa = (sa->sym[0] == 'w') || (sa->sym[0] == 'W'); -- wb = (sb->sym[0] == 'w') || (sb->sym[0] == 'W'); -+ wa = (sa->type == 'w') || (sa->type == 'W'); -+ wb = (sb->type == 'w') || (sb->type == 'W'); - if (wa != wb) - return wa - wb; - -@@ -717,8 +737,8 @@ static int compare_symbols(const void *a, const void *b) - return wa - wb; +- unsigned long idx; ++ return FIND_FIRST_BIT(~addr[idx], /* nop */, size); ++} ++#endif - /* sort by the number of prefix underscores */ -- wa = strspn(sym_name(sa), "_"); -- wb = strspn(sym_name(sb), "_"); -+ wa = strspn((char *)sa->name, "_"); -+ wb = strspn((char *)sb->name, "_"); - if (wa != wb) - return wa - wb; +- for (idx = 0; idx * BITS_PER_LONG < size; idx++) { +- if (addr[idx] != ~0UL) +- return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); +- } ++#ifndef find_next_bit ++unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) ++{ ++ return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); ++} ++#endif -@@ -742,7 +762,7 @@ static void make_percpus_absolute(void) - * ensure consistent behavior compared to older - * versions of this tool. - */ -- table[i]->sym[0] = 'A'; -+ table[i]->type = 'A'; - table[i]->percpu_absolute = 1; - } +- return size; ++#ifndef find_next_and_bit ++unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, ++ unsigned long nbits, unsigned long start) ++{ ++ return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); ++} ++#endif ++ ++#ifndef find_next_zero_bit ++unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, ++ unsigned long start) ++{ ++ return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); } + #endif -- -2.37.3 +2.38.0.rc1.8.g2a7d63a245 diff --git a/6.0/testing/0003-futex-winesync.patch b/6.0/testing/0003-futex-winesync.patch deleted file mode 100644 index 42e728eb..00000000 --- a/6.0/testing/0003-futex-winesync.patch +++ /dev/null @@ -1,3402 +0,0 @@ -From 98c01e8963ef2954a163a27812234015f3bf036e Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Sun, 25 Sep 2022 23:49:46 +0200 -Subject: [PATCH 03/13] futex-winesync - -Signed-off-by: Peter Jung ---- - Documentation/admin-guide/devices.txt | 3 +- - Documentation/userspace-api/index.rst | 1 + - .../userspace-api/ioctl/ioctl-number.rst | 2 + - Documentation/userspace-api/winesync.rst | 444 +++++ - MAINTAINERS | 9 + - drivers/misc/Kconfig | 11 + - drivers/misc/Makefile | 1 + - drivers/misc/winesync.c | 1212 ++++++++++++++ - include/linux/miscdevice.h | 1 + - include/uapi/linux/winesync.h | 71 + - tools/testing/selftests/Makefile | 1 + - .../selftests/drivers/winesync/Makefile | 8 + - .../testing/selftests/drivers/winesync/config | 1 + - .../selftests/drivers/winesync/winesync.c | 1479 +++++++++++++++++ - 14 files changed, 3243 insertions(+), 1 deletion(-) - create mode 100644 Documentation/userspace-api/winesync.rst - create mode 100644 drivers/misc/winesync.c - create mode 100644 include/uapi/linux/winesync.h - create mode 100644 tools/testing/selftests/drivers/winesync/Makefile - create mode 100644 tools/testing/selftests/drivers/winesync/config - create mode 100644 tools/testing/selftests/drivers/winesync/winesync.c - -diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt -index 9764d6edb189..a4696d3b4a5a 100644 ---- a/Documentation/admin-guide/devices.txt -+++ b/Documentation/admin-guide/devices.txt -@@ -376,8 +376,9 @@ - 240 = /dev/userio Serio driver testing device - 241 = /dev/vhost-vsock Host kernel driver for virtio vsock - 242 = /dev/rfkill Turning off radio transmissions (rfkill) -+ 243 = /dev/winesync Wine synchronization primitive device - -- 243-254 Reserved for local use -+ 244-254 Reserved for local use - 255 Reserved for MISC_DYNAMIC_MINOR - - 11 char Raw keyboard device (Linux/SPARC only) -diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst -index a61eac0c73f8..0bf697ddcb09 100644 ---- a/Documentation/userspace-api/index.rst -+++ b/Documentation/userspace-api/index.rst -@@ -29,6 +29,7 @@ place where this information is gathered. - sysfs-platform_profile - vduse - futex2 -+ winesync - - .. only:: subproject and html - -diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst -index 3b985b19f39d..3f313fd4338c 100644 ---- a/Documentation/userspace-api/ioctl/ioctl-number.rst -+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst -@@ -375,6 +375,8 @@ Code Seq# Include File Comments - - 0xF6 all LTTng Linux Trace Toolkit Next Generation - -+0xF7 00-0F uapi/linux/winesync.h Wine synchronization primitives -+ - 0xF8 all arch/x86/include/uapi/asm/amd_hsmp.h AMD HSMP EPYC system management interface driver - - 0xFD all linux/dm-ioctl.h -diff --git a/Documentation/userspace-api/winesync.rst b/Documentation/userspace-api/winesync.rst -new file mode 100644 -index 000000000000..f0110d2744c7 ---- /dev/null -+++ b/Documentation/userspace-api/winesync.rst -@@ -0,0 +1,444 @@ -+===================================== -+Wine synchronization primitive driver -+===================================== -+ -+This page documents the user-space API for the winesync driver. -+ -+winesync is a support driver for emulation of NT synchronization -+primitives by the Wine project or other NT emulators. It exists -+because implementation in user-space, using existing tools, cannot -+simultaneously satisfy performance, correctness, and security -+constraints. It is implemented entirely in software, and does not -+drive any hardware device. -+ -+This interface is meant as a compatibility tool only, and should not -+be used for general synchronization. Instead use generic, versatile -+interfaces such as futex(2) and poll(2). -+ -+Synchronization primitives -+========================== -+ -+The winesync driver exposes three types of synchronization primitives: -+semaphores, mutexes, and events. -+ -+A semaphore holds a single volatile 32-bit counter, and a static -+32-bit integer denoting the maximum value. It is considered signaled -+when the counter is nonzero. The counter is decremented by one when a -+wait is satisfied. Both the initial and maximum count are established -+when the semaphore is created. -+ -+A mutex holds a volatile 32-bit recursion count, and a volatile 32-bit -+identifier denoting its owner. A mutex is considered signaled when its -+owner is zero (indicating that it is not owned). The recursion count -+is incremented when a wait is satisfied, and ownership is set to the -+given identifier. -+ -+A mutex also holds an internal flag denoting whether its previous -+owner has died; such a mutex is said to be inconsistent. Owner death -+is not tracked automatically based on thread death, but rather must be -+communicated using ``WINESYNC_IOC_KILL_OWNER``. An inconsistent mutex -+is inherently considered unowned. -+ -+Except for the "unowned" semantics of zero, the actual value of the -+owner identifier is not interpreted by the winesync driver at all. The -+intended use is to store a thread identifier; however, the winesync -+driver does not actually validate that a calling thread provides -+consistent or unique identifiers. -+ -+An event holds a volatile boolean state denoting whether it is -+signaled or not. There are two types of events, auto-reset and -+manual-reset. An auto-reset event is designaled when a wait is -+satisfied; a manual-reset event is not. The event type is specified -+when the event is created. -+ -+Unless specified otherwise, all operations on an object are atomic and -+totally ordered with respect to other operations on the same object. -+ -+Objects are represented by unsigned 32-bit integers. -+ -+Char device -+=========== -+ -+The winesync driver creates a single char device /dev/winesync. Each -+file description opened on the device represents a unique namespace. -+That is, objects created on one open file description are shared -+across all its individual descriptors, but are not shared with other -+open() calls on the same device. The same file description may be -+shared across multiple processes. -+ -+ioctl reference -+=============== -+ -+All operations on the device are done through ioctls. There are three -+structures used in ioctl calls:: -+ -+ struct winesync_sem_args { -+ __u32 sem; -+ __u32 count; -+ __u32 max; -+ }; -+ -+ struct winesync_mutex_args { -+ __u32 mutex; -+ __u32 owner; -+ __u32 count; -+ }; -+ -+ struct winesync_event_args { -+ __u32 event; -+ __u32 signaled; -+ __u32 manual; -+ }; -+ -+ struct winesync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 pad; -+ }; -+ -+Depending on the ioctl, members of the structure may be used as input, -+output, or not at all. All ioctls return 0 on success. -+ -+The ioctls are as follows: -+ -+.. c:macro:: WINESYNC_IOC_CREATE_SEM -+ -+ Create a semaphore object. Takes a pointer to struct -+ :c:type:`winesync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``sem`` -+ - On output, contains the identifier of the created semaphore. -+ * - ``count`` -+ - Initial count of the semaphore. -+ * - ``max`` -+ - Maximum count of the semaphore. -+ -+ Fails with ``EINVAL`` if ``count`` is greater than ``max``. -+ -+.. c:macro:: WINESYNC_IOC_CREATE_MUTEX -+ -+ Create a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``mutex`` -+ - On output, contains the identifier of the created mutex. -+ * - ``count`` -+ - Initial recursion count of the mutex. -+ * - ``owner`` -+ - Initial owner of the mutex. -+ -+ If ``owner`` is nonzero and ``count`` is zero, or if ``owner`` is -+ zero and ``count`` is nonzero, the function fails with ``EINVAL``. -+ -+.. c:macro:: WINESYNC_IOC_CREATE_EVENT -+ -+ Create an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - On output, contains the identifier of the created event. -+ * - ``signaled`` -+ - If nonzero, the event is initially signaled, otherwise -+ nonsignaled. -+ * - ``manual`` -+ - If nonzero, the event is a manual-reset event, otherwise -+ auto-reset. -+ -+.. c:macro:: WINESYNC_IOC_DELETE -+ -+ Delete an object of any type. Takes an input-only pointer to a -+ 32-bit integer denoting the object to delete. -+ -+ Wait ioctls currently in progress are not interrupted, and behave as -+ if the object remains valid. -+ -+.. c:macro:: WINESYNC_IOC_PUT_SEM -+ -+ Post to a semaphore object. Takes a pointer to struct -+ :c:type:`winesync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``sem`` -+ - Semaphore object to post to. -+ * - ``count`` -+ - Count to add to the semaphore. On output, contains the -+ previous count of the semaphore. -+ * - ``max`` -+ - Not used. -+ -+ If adding ``count`` to the semaphore's current count would raise the -+ latter past the semaphore's maximum count, the ioctl fails with -+ ``EOVERFLOW`` and the semaphore is not affected. If raising the -+ semaphore's count causes it to become signaled, eligible threads -+ waiting on this semaphore will be woken and the semaphore's count -+ decremented appropriately. -+ -+.. c:macro:: WINESYNC_IOC_PUT_MUTEX -+ -+ Release a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``mutex`` -+ - Mutex object to release. -+ * - ``owner`` -+ - Mutex owner identifier. -+ * - ``count`` -+ - On output, contains the previous recursion count. -+ -+ If ``owner`` is zero, the ioctl fails with ``EINVAL``. If ``owner`` -+ is not the current owner of the mutex, the ioctl fails with -+ ``EPERM``. -+ -+ The mutex's count will be decremented by one. If decrementing the -+ mutex's count causes it to become zero, the mutex is marked as -+ unowned and signaled, and eligible threads waiting on it will be -+ woken as appropriate. -+ -+.. c:macro:: WINESYNC_IOC_SET_EVENT -+ -+ Signal an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object to set. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. -+ -+ Eligible threads will be woken, and auto-reset events will be -+ designaled appropriately. -+ -+.. c:macro:: WINESYNC_IOC_RESET_EVENT -+ -+ Designal an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object to reset. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. -+ -+.. c:macro:: WINESYNC_IOC_PULSE_EVENT -+ -+ Wake threads waiting on an event object without leaving it in a -+ signaled state. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object to pulse. -+ * - ``signaled`` -+ - On output, contains the previous state of the event. -+ * - ``manual`` -+ - Unused. -+ -+ A pulse operation can be thought of as a set followed by a reset, -+ performed as a single atomic operation. If two threads are waiting -+ on an auto-reset event which is pulsed, only one will be woken. If -+ two threads are waiting a manual-reset event which is pulsed, both -+ will be woken. However, in both cases, the event will be unsignaled -+ afterwards, and a simultaneous read operation will always report the -+ event as unsignaled. -+ -+.. c:macro:: WINESYNC_IOC_READ_SEM -+ -+ Read the current state of a semaphore object. Takes a pointer to -+ struct :c:type:`winesync_sem_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``sem`` -+ - Semaphore object to read. -+ * - ``count`` -+ - On output, contains the current count of the semaphore. -+ * - ``max`` -+ - On output, contains the maximum count of the semaphore. -+ -+.. c:macro:: WINESYNC_IOC_READ_MUTEX -+ -+ Read the current state of a mutex object. Takes a pointer to struct -+ :c:type:`winesync_mutex_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``mutex`` -+ - Mutex object to read. -+ * - ``owner`` -+ - On output, contains the current owner of the mutex, or zero -+ if the mutex is not currently owned. -+ * - ``count`` -+ - On output, contains the current recursion count of the mutex. -+ -+ If the mutex is marked as inconsistent, the function fails with -+ ``EOWNERDEAD``. In this case, ``count`` and ``owner`` are set to -+ zero. -+ -+.. c:macro:: WINESYNC_IOC_READ_EVENT -+ -+ Read the current state of an event object. Takes a pointer to struct -+ :c:type:`winesync_event_args`, which is used as follows: -+ -+ .. list-table:: -+ -+ * - ``event`` -+ - Event object. -+ * - ``signaled`` -+ - On output, contains the current state of the event. -+ * - ``manual`` -+ - On output, contains 1 if the event is a manual-reset event, -+ and 0 otherwise. -+ -+.. c:macro:: WINESYNC_IOC_KILL_OWNER -+ -+ Mark any mutexes owned by the given owner as unowned and -+ inconsistent. Takes an input-only pointer to a 32-bit integer -+ denoting the owner. If the owner is zero, the ioctl fails with -+ ``EINVAL``. -+ -+ For each mutex currently owned by the given owner, eligible threads -+ waiting on said mutex will be woken as appropriate (and such waits -+ will fail with ``EOWNERDEAD``, as described below). -+ -+ The operation as a whole is not atomic; however, the modification of -+ each mutex is atomic and totally ordered with respect to other -+ operations on the same mutex. -+ -+.. c:macro:: WINESYNC_IOC_WAIT_ANY -+ -+ Poll on any of a list of objects, atomically acquiring at most one. -+ Takes a pointer to struct :c:type:`winesync_wait_args`, which is -+ used as follows: -+ -+ .. list-table:: -+ -+ * - ``timeout`` -+ - Optional pointer to a 64-bit struct :c:type:`timespec` -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). The timeout is specified in -+ absolute format, as measured against the MONOTONIC clock. If -+ the timeout is equal to or earlier than the current time, the -+ function returns immediately without sleeping. If ``timeout`` -+ is zero, i.e. NULL, the function will sleep until an object -+ is signaled, and will not fail with ``ETIMEDOUT``. -+ * - ``objs`` -+ - Pointer to an array of ``count`` 32-bit object identifiers -+ (specified as an integer so that the structure has the same -+ size regardless of architecture). If any identifier is -+ invalid, the function fails with ``EINVAL``. -+ * - ``count`` -+ - Number of object identifiers specified in the ``objs`` array. -+ * - ``owner`` -+ - Mutex owner identifier. If any object in ``objs`` is a mutex, -+ the ioctl will attempt to acquire that mutex on behalf of -+ ``owner``. If ``owner`` is zero, the ioctl fails with -+ ``EINVAL``. -+ * - ``index`` -+ - On success, contains the index (into ``objs``) of the object -+ which was signaled. If ``alert`` was signaled instead, -+ this contains ``count``. -+ * - ``alert`` -+ - Optional event object identifier. If nonzero, this specifies -+ an "alert" event object which, if signaled, will terminate -+ the wait. If nonzero, the identifier must point to a valid -+ event. -+ -+ This function attempts to acquire one of the given objects. If -+ unable to do so, it sleeps until an object becomes signaled, -+ subsequently acquiring it, or the timeout expires. In the latter -+ case the ioctl fails with ``ETIMEDOUT``. The function only acquires -+ one object, even if multiple objects are signaled. -+ -+ A semaphore is considered to be signaled if its count is nonzero, -+ and is acquired by decrementing its count by one. A mutex is -+ considered to be signaled if it is unowned or if its owner matches -+ the ``owner`` argument, and is acquired by incrementing its -+ recursion count by one and setting its owner to the ``owner`` -+ argument. An auto-reset event is acquired by designaling it; a -+ manual-reset event is not affected by acquisition. -+ -+ Acquisition is atomic and totally ordered with respect to other -+ operations on the same object. If two wait operations (with -+ different ``owner`` identifiers) are queued on the same mutex, only -+ one is signaled. If two wait operations are queued on the same -+ semaphore, and a value of one is posted to it, only one is signaled. -+ The order in which threads are signaled is not specified. -+ -+ If an inconsistent mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Although this is a failure return, the function may -+ otherwise be considered successful. The mutex is marked as owned by -+ the given owner (with a recursion count of 1) and as no longer -+ inconsistent, and ``index`` is still set to the index of the mutex. -+ -+ The ``alert`` argument is an "extra" event which can terminate the -+ wait, independently of all other objects. If members of ``objs`` and -+ ``alert`` are both simultaneously signaled, a member of ``objs`` -+ will always be given priority and acquired first. Aside from this, -+ for "any" waits, there is no difference between passing an event as -+ this parameter, and passing it as an additional object at the end of -+ the ``objs`` array. For "all" waits, there is an additional -+ difference, as described below. -+ -+ It is valid to pass the same object more than once, including by -+ passing the same event in the ``objs`` array and in ``alert``. If a -+ wakeup occurs due to that object being signaled, ``index`` is set to -+ the lowest index corresponding to that object. -+ -+ The function may fail with ``EINTR`` if a signal is received. -+ -+.. c:macro:: WINESYNC_IOC_WAIT_ALL -+ -+ Poll on a list of objects, atomically acquiring all of them. Takes a -+ pointer to struct :c:type:`winesync_wait_args`, which is used -+ identically to ``WINESYNC_IOC_WAIT_ANY``, except that ``index`` is -+ always filled with zero on success if not woken via alert. -+ -+ This function attempts to simultaneously acquire all of the given -+ objects. If unable to do so, it sleeps until all objects become -+ simultaneously signaled, subsequently acquiring them, or the timeout -+ expires. In the latter case the ioctl fails with ``ETIMEDOUT`` and -+ no objects are modified. -+ -+ Objects may become signaled and subsequently designaled (through -+ acquisition by other threads) while this thread is sleeping. Only -+ once all objects are simultaneously signaled does the ioctl acquire -+ them and return. The entire acquisition is atomic and totally -+ ordered with respect to other operations on any of the given -+ objects. -+ -+ If an inconsistent mutex is acquired, the ioctl fails with -+ ``EOWNERDEAD``. Similarly to ``WINESYNC_IOC_WAIT_ANY``, all objects -+ are nevertheless marked as acquired. Note that if multiple mutex -+ objects are specified, there is no way to know which were marked as -+ inconsistent. -+ -+ As with "any" waits, the ``alert`` argument is an "extra" event -+ which can terminate the wait. Critically, however, an "all" wait -+ will succeed if all members in ``objs`` are signaled, *or* if -+ ``alert`` is signaled. In the latter case ``index`` will be set to -+ ``count``. As with "any" waits, if both conditions are filled, the -+ former takes priority, and objects in ``objs`` will be acquired. -+ -+ Unlike ``WINESYNC_IOC_WAIT_ANY``, it is not valid to pass the same -+ object more than once, nor is it valid to pass the same object in -+ ``objs`` and in ``alert`` If this is attempted, the function fails -+ with ``EINVAL``. -diff --git a/MAINTAINERS b/MAINTAINERS -index f5ca4aefd184..31a7aa60cdc3 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -21921,6 +21921,15 @@ M: David Härdeman - S: Maintained - F: drivers/media/rc/winbond-cir.c - -+WINESYNC SYNCHRONIZATION PRIMITIVE DRIVER -+M: Zebediah Figura -+L: wine-devel@winehq.org -+S: Supported -+F: Documentation/userspace-api/winesync.rst -+F: drivers/misc/winesync.c -+F: include/uapi/linux/winesync.h -+F: tools/testing/selftests/drivers/winesync/ -+ - WINSYSTEMS EBC-C384 WATCHDOG DRIVER - M: William Breathitt Gray - L: linux-watchdog@vger.kernel.org -diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig -index 94e9fb4cdd76..4f9e3d80a6e8 100644 ---- a/drivers/misc/Kconfig -+++ b/drivers/misc/Kconfig -@@ -496,6 +496,17 @@ config VCPU_STALL_DETECTOR - - If you do not intend to run this kernel as a guest, say N. - -+config WINESYNC -+ tristate "Synchronization primitives for Wine" -+ help -+ This module provides kernel support for synchronization primitives -+ used by Wine. It is not a hardware driver. -+ -+ To compile this driver as a module, choose M here: the -+ module will be called winesync. -+ -+ If unsure, say N. -+ - source "drivers/misc/c2port/Kconfig" - source "drivers/misc/eeprom/Kconfig" - source "drivers/misc/cb710/Kconfig" -diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile -index 2be8542616dd..d061fe45407b 100644 ---- a/drivers/misc/Makefile -+++ b/drivers/misc/Makefile -@@ -58,6 +58,7 @@ obj-$(CONFIG_HABANA_AI) += habanalabs/ - obj-$(CONFIG_UACCE) += uacce/ - obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o - obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o -+obj-$(CONFIG_WINESYNC) += winesync.o - obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o - obj-$(CONFIG_OPEN_DICE) += open-dice.o - obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o -\ No newline at end of file -diff --git a/drivers/misc/winesync.c b/drivers/misc/winesync.c -new file mode 100644 -index 000000000000..7a28f58dbbf2 ---- /dev/null -+++ b/drivers/misc/winesync.c -@@ -0,0 +1,1212 @@ -+// SPDX-License-Identifier: GPL-2.0-only -+/* -+ * winesync.c - Kernel driver for Wine synchronization primitives -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define WINESYNC_NAME "winesync" -+ -+enum winesync_type { -+ WINESYNC_TYPE_SEM, -+ WINESYNC_TYPE_MUTEX, -+ WINESYNC_TYPE_EVENT, -+}; -+ -+struct winesync_obj { -+ struct rcu_head rhead; -+ struct kref refcount; -+ spinlock_t lock; -+ -+ /* -+ * any_waiters is protected by the object lock, but all_waiters is -+ * protected by the device wait_all_lock. -+ */ -+ struct list_head any_waiters; -+ struct list_head all_waiters; -+ -+ /* -+ * Hint describing how many tasks are queued on this object in a -+ * wait-all operation. -+ * -+ * Any time we do a wake, we may need to wake "all" waiters as well as -+ * "any" waiters. In order to atomically wake "all" waiters, we must -+ * lock all of the objects, and that means grabbing the wait_all_lock -+ * below (and, due to lock ordering rules, before locking this object). -+ * However, wait-all is a rare operation, and grabbing the wait-all -+ * lock for every wake would create unnecessary contention. Therefore we -+ * first check whether all_hint is zero, and, if it is, we skip trying -+ * to wake "all" waiters. -+ * -+ * This hint isn't protected by any lock. It might change during the -+ * course of a wake, but there's no meaningful race there; it's only a -+ * hint. -+ * -+ * Since wait requests must originate from user-space threads, we're -+ * limited here by PID_MAX_LIMIT, so there's no risk of saturation. -+ */ -+ atomic_t all_hint; -+ -+ enum winesync_type type; -+ -+ /* The following fields are protected by the object lock. */ -+ union { -+ struct { -+ __u32 count; -+ __u32 max; -+ } sem; -+ struct { -+ __u32 count; -+ __u32 owner; -+ bool ownerdead; -+ } mutex; -+ struct { -+ bool manual; -+ bool signaled; -+ } event; -+ } u; -+}; -+ -+struct winesync_q_entry { -+ struct list_head node; -+ struct winesync_q *q; -+ struct winesync_obj *obj; -+ __u32 index; -+}; -+ -+struct winesync_q { -+ struct task_struct *task; -+ __u32 owner; -+ -+ /* -+ * Protected via atomic_cmpxchg(). Only the thread that wins the -+ * compare-and-swap may actually change object states and wake this -+ * task. -+ */ -+ atomic_t signaled; -+ -+ bool all; -+ bool ownerdead; -+ __u32 count; -+ struct winesync_q_entry entries[]; -+}; -+ -+struct winesync_device { -+ /* -+ * Wait-all operations must atomically grab all objects, and be totally -+ * ordered with respect to each other and wait-any operations. If one -+ * thread is trying to acquire several objects, another thread cannot -+ * touch the object at the same time. -+ * -+ * We achieve this by grabbing multiple object locks at the same time. -+ * However, this creates a lock ordering problem. To solve that problem, -+ * wait_all_lock is taken first whenever multiple objects must be locked -+ * at the same time. -+ */ -+ spinlock_t wait_all_lock; -+ -+ struct xarray objects; -+}; -+ -+static struct winesync_obj *get_obj(struct winesync_device *dev, __u32 id) -+{ -+ struct winesync_obj *obj; -+ -+ rcu_read_lock(); -+ obj = xa_load(&dev->objects, id); -+ if (obj && !kref_get_unless_zero(&obj->refcount)) -+ obj = NULL; -+ rcu_read_unlock(); -+ -+ return obj; -+} -+ -+static void destroy_obj(struct kref *ref) -+{ -+ struct winesync_obj *obj = container_of(ref, struct winesync_obj, refcount); -+ -+ kfree_rcu(obj, rhead); -+} -+ -+static void put_obj(struct winesync_obj *obj) -+{ -+ kref_put(&obj->refcount, destroy_obj); -+} -+ -+static struct winesync_obj *get_obj_typed(struct winesync_device *dev, __u32 id, -+ enum winesync_type type) -+{ -+ struct winesync_obj *obj = get_obj(dev, id); -+ -+ if (obj && obj->type != type) { -+ put_obj(obj); -+ return NULL; -+ } -+ return obj; -+} -+ -+static int winesync_char_open(struct inode *inode, struct file *file) -+{ -+ struct winesync_device *dev; -+ -+ dev = kzalloc(sizeof(*dev), GFP_KERNEL); -+ if (!dev) -+ return -ENOMEM; -+ -+ spin_lock_init(&dev->wait_all_lock); -+ -+ xa_init_flags(&dev->objects, XA_FLAGS_ALLOC); -+ -+ file->private_data = dev; -+ return nonseekable_open(inode, file); -+} -+ -+static int winesync_char_release(struct inode *inode, struct file *file) -+{ -+ struct winesync_device *dev = file->private_data; -+ struct winesync_obj *obj; -+ unsigned long id; -+ -+ xa_for_each(&dev->objects, id, obj) -+ put_obj(obj); -+ -+ xa_destroy(&dev->objects); -+ -+ kfree(dev); -+ -+ return 0; -+} -+ -+static void init_obj(struct winesync_obj *obj) -+{ -+ kref_init(&obj->refcount); -+ atomic_set(&obj->all_hint, 0); -+ spin_lock_init(&obj->lock); -+ INIT_LIST_HEAD(&obj->any_waiters); -+ INIT_LIST_HEAD(&obj->all_waiters); -+} -+ -+static bool is_signaled(struct winesync_obj *obj, __u32 owner) -+{ -+ lockdep_assert_held(&obj->lock); -+ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ return !!obj->u.sem.count; -+ case WINESYNC_TYPE_MUTEX: -+ if (obj->u.mutex.owner && obj->u.mutex.owner != owner) -+ return false; -+ return obj->u.mutex.count < UINT_MAX; -+ case WINESYNC_TYPE_EVENT: -+ return obj->u.event.signaled; -+ } -+ -+ WARN(1, "bad object type %#x\n", obj->type); -+ return false; -+} -+ -+/* -+ * "locked_obj" is an optional pointer to an object which is already locked and -+ * should not be locked again. This is necessary so that changing an object's -+ * state and waking it can be a single atomic operation. -+ */ -+static void try_wake_all(struct winesync_device *dev, struct winesync_q *q, -+ struct winesync_obj *locked_obj) -+{ -+ __u32 count = q->count; -+ bool can_wake = true; -+ __u32 i; -+ -+ lockdep_assert_held(&dev->wait_all_lock); -+ if (locked_obj) -+ lockdep_assert_held(&locked_obj->lock); -+ -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ spin_lock(&q->entries[i].obj->lock); -+ } -+ -+ for (i = 0; i < count; i++) { -+ if (!is_signaled(q->entries[i].obj, q->owner)) { -+ can_wake = false; -+ break; -+ } -+ } -+ -+ if (can_wake && atomic_cmpxchg(&q->signaled, -1, 0) == -1) { -+ for (i = 0; i < count; i++) { -+ struct winesync_obj *obj = q->entries[i].obj; -+ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ obj->u.sem.count--; -+ break; -+ case WINESYNC_TYPE_MUTEX: -+ if (obj->u.mutex.ownerdead) -+ q->ownerdead = true; -+ obj->u.mutex.ownerdead = false; -+ obj->u.mutex.count++; -+ obj->u.mutex.owner = q->owner; -+ break; -+ case WINESYNC_TYPE_EVENT: -+ if (!obj->u.event.manual) -+ obj->u.event.signaled = false; -+ break; -+ } -+ } -+ wake_up_process(q->task); -+ } -+ -+ for (i = 0; i < count; i++) { -+ if (q->entries[i].obj != locked_obj) -+ spin_unlock(&q->entries[i].obj->lock); -+ } -+} -+ -+static void try_wake_all_obj(struct winesync_device *dev, -+ struct winesync_obj *obj) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&dev->wait_all_lock); -+ lockdep_assert_held(&obj->lock); -+ -+ list_for_each_entry(entry, &obj->all_waiters, node) -+ try_wake_all(dev, entry->q, obj); -+} -+ -+static void try_wake_any_sem(struct winesync_obj *sem) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&sem->lock); -+ -+ list_for_each_entry(entry, &sem->any_waiters, node) { -+ struct winesync_q *q = entry->q; -+ -+ if (!sem->u.sem.count) -+ break; -+ -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ sem->u.sem.count--; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static void try_wake_any_mutex(struct winesync_obj *mutex) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&mutex->lock); -+ -+ list_for_each_entry(entry, &mutex->any_waiters, node) { -+ struct winesync_q *q = entry->q; -+ -+ if (mutex->u.mutex.count == UINT_MAX) -+ break; -+ if (mutex->u.mutex.owner && mutex->u.mutex.owner != q->owner) -+ continue; -+ -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ if (mutex->u.mutex.ownerdead) -+ q->ownerdead = true; -+ mutex->u.mutex.ownerdead = false; -+ mutex->u.mutex.count++; -+ mutex->u.mutex.owner = q->owner; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static void try_wake_any_event(struct winesync_obj *event) -+{ -+ struct winesync_q_entry *entry; -+ -+ lockdep_assert_held(&event->lock); -+ -+ list_for_each_entry(entry, &event->any_waiters, node) { -+ struct winesync_q *q = entry->q; -+ -+ if (!event->u.event.signaled) -+ break; -+ -+ if (atomic_cmpxchg(&q->signaled, -1, entry->index) == -1) { -+ if (!event->u.event.manual) -+ event->u.event.signaled = false; -+ wake_up_process(q->task); -+ } -+ } -+} -+ -+static int winesync_create_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ if (args.count > args.max) -+ return -EINVAL; -+ -+ sem = kzalloc(sizeof(*sem), GFP_KERNEL); -+ if (!sem) -+ return -ENOMEM; -+ -+ init_obj(sem); -+ sem->type = WINESYNC_TYPE_SEM; -+ sem->u.sem.count = args.count; -+ sem->u.sem.max = args.max; -+ -+ ret = xa_alloc(&dev->objects, &id, sem, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(sem); -+ return ret; -+ } -+ -+ return put_user(id, &user_args->sem); -+} -+ -+static int winesync_create_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ if (!args.owner != !args.count) -+ return -EINVAL; -+ -+ mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); -+ if (!mutex) -+ return -ENOMEM; -+ -+ init_obj(mutex); -+ mutex->type = WINESYNC_TYPE_MUTEX; -+ mutex->u.mutex.count = args.count; -+ mutex->u.mutex.owner = args.owner; -+ -+ ret = xa_alloc(&dev->objects, &id, mutex, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(mutex); -+ return ret; -+ } -+ -+ return put_user(id, &user_args->mutex); -+} -+ -+static int winesync_create_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ __u32 id; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = kzalloc(sizeof(*event), GFP_KERNEL); -+ if (!event) -+ return -ENOMEM; -+ -+ init_obj(event); -+ event->type = WINESYNC_TYPE_EVENT; -+ event->u.event.manual = args.manual; -+ event->u.event.signaled = args.signaled; -+ -+ ret = xa_alloc(&dev->objects, &id, event, xa_limit_32b, GFP_KERNEL); -+ if (ret < 0) { -+ kfree(event); -+ return ret; -+ } -+ -+ return put_user(id, &user_args->event); -+} -+ -+static int winesync_delete(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_obj *obj; -+ __u32 id; -+ -+ if (get_user(id, (__u32 __user *)argp)) -+ return -EFAULT; -+ -+ obj = xa_erase(&dev->objects, id); -+ if (!obj) -+ return -EINVAL; -+ -+ put_obj(obj); -+ return 0; -+} -+ -+/* -+ * Actually change the semaphore state, returning -EOVERFLOW if it is made -+ * invalid. -+ */ -+static int put_sem_state(struct winesync_obj *sem, __u32 count) -+{ -+ lockdep_assert_held(&sem->lock); -+ -+ if (sem->u.sem.count + count < sem->u.sem.count || -+ sem->u.sem.count + count > sem->u.sem.max) -+ return -EOVERFLOW; -+ -+ sem->u.sem.count += count; -+ return 0; -+} -+ -+static int winesync_put_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 prev_count; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ sem = get_obj_typed(dev, args.sem, WINESYNC_TYPE_SEM); -+ if (!sem) -+ return -EINVAL; -+ -+ if (atomic_read(&sem->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&sem->lock); -+ -+ prev_count = sem->u.sem.count; -+ ret = put_sem_state(sem, args.count); -+ if (!ret) { -+ try_wake_all_obj(dev, sem); -+ try_wake_any_sem(sem); -+ } -+ -+ spin_unlock(&sem->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&sem->lock); -+ -+ prev_count = sem->u.sem.count; -+ ret = put_sem_state(sem, args.count); -+ if (!ret) -+ try_wake_any_sem(sem); -+ -+ spin_unlock(&sem->lock); -+ } -+ -+ put_obj(sem); -+ -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; -+ -+ return ret; -+} -+ -+/* -+ * Actually change the mutex state, returning -EPERM if not the owner. -+ */ -+static int put_mutex_state(struct winesync_obj *mutex, -+ const struct winesync_mutex_args *args) -+{ -+ lockdep_assert_held(&mutex->lock); -+ -+ if (mutex->u.mutex.owner != args->owner) -+ return -EPERM; -+ -+ if (!--mutex->u.mutex.count) -+ mutex->u.mutex.owner = 0; -+ return 0; -+} -+ -+static int winesync_put_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 prev_count; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ if (!args.owner) -+ return -EINVAL; -+ -+ mutex = get_obj_typed(dev, args.mutex, WINESYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -EINVAL; -+ -+ if (atomic_read(&mutex->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&mutex->lock); -+ -+ prev_count = mutex->u.mutex.count; -+ ret = put_mutex_state(mutex, &args); -+ if (!ret) { -+ try_wake_all_obj(dev, mutex); -+ try_wake_any_mutex(mutex); -+ } -+ -+ spin_unlock(&mutex->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&mutex->lock); -+ -+ prev_count = mutex->u.mutex.count; -+ ret = put_mutex_state(mutex, &args); -+ if (!ret) -+ try_wake_any_mutex(mutex); -+ -+ spin_unlock(&mutex->lock); -+ } -+ -+ put_obj(mutex); -+ -+ if (!ret && put_user(prev_count, &user_args->count)) -+ ret = -EFAULT; -+ -+ return ret; -+} -+ -+static int winesync_read_sem(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_sem_args __user *user_args = argp; -+ struct winesync_sem_args args; -+ struct winesync_obj *sem; -+ __u32 id; -+ -+ if (get_user(id, &user_args->sem)) -+ return -EFAULT; -+ -+ sem = get_obj_typed(dev, id, WINESYNC_TYPE_SEM); -+ if (!sem) -+ return -EINVAL; -+ -+ args.sem = id; -+ spin_lock(&sem->lock); -+ args.count = sem->u.sem.count; -+ args.max = sem->u.sem.max; -+ spin_unlock(&sem->lock); -+ -+ put_obj(sem); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} -+ -+static int winesync_read_mutex(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_mutex_args __user *user_args = argp; -+ struct winesync_mutex_args args; -+ struct winesync_obj *mutex; -+ __u32 id; -+ int ret; -+ -+ if (get_user(id, &user_args->mutex)) -+ return -EFAULT; -+ -+ mutex = get_obj_typed(dev, id, WINESYNC_TYPE_MUTEX); -+ if (!mutex) -+ return -EINVAL; -+ -+ args.mutex = id; -+ spin_lock(&mutex->lock); -+ args.count = mutex->u.mutex.count; -+ args.owner = mutex->u.mutex.owner; -+ ret = mutex->u.mutex.ownerdead ? -EOWNERDEAD : 0; -+ spin_unlock(&mutex->lock); -+ -+ put_obj(mutex); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return ret; -+} -+ -+static int winesync_read_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ __u32 id; -+ -+ if (get_user(id, &user_args->event)) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, id, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ args.event = id; -+ spin_lock(&event->lock); -+ args.manual = event->u.event.manual; -+ args.signaled = event->u.event.signaled; -+ spin_unlock(&event->lock); -+ -+ put_obj(event); -+ -+ if (copy_to_user(user_args, &args, sizeof(args))) -+ return -EFAULT; -+ return 0; -+} -+ -+/* -+ * Actually change the mutex state to mark its owner as dead. -+ */ -+static void put_mutex_ownerdead_state(struct winesync_obj *mutex) -+{ -+ lockdep_assert_held(&mutex->lock); -+ -+ mutex->u.mutex.ownerdead = true; -+ mutex->u.mutex.owner = 0; -+ mutex->u.mutex.count = 0; -+} -+ -+static int winesync_kill_owner(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_obj *obj; -+ unsigned long id; -+ __u32 owner; -+ -+ if (get_user(owner, (__u32 __user *)argp)) -+ return -EFAULT; -+ if (!owner) -+ return -EINVAL; -+ -+ rcu_read_lock(); -+ -+ xa_for_each(&dev->objects, id, obj) { -+ if (!kref_get_unless_zero(&obj->refcount)) -+ continue; -+ -+ if (obj->type != WINESYNC_TYPE_MUTEX) { -+ put_obj(obj); -+ continue; -+ } -+ -+ if (atomic_read(&obj->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&obj->lock); -+ -+ if (obj->u.mutex.owner == owner) { -+ put_mutex_ownerdead_state(obj); -+ try_wake_all_obj(dev, obj); -+ try_wake_any_mutex(obj); -+ } -+ -+ spin_unlock(&obj->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&obj->lock); -+ -+ if (obj->u.mutex.owner == owner) { -+ put_mutex_ownerdead_state(obj); -+ try_wake_any_mutex(obj); -+ } -+ -+ spin_unlock(&obj->lock); -+ } -+ -+ put_obj(obj); -+ } -+ -+ rcu_read_unlock(); -+ -+ return 0; -+} -+ -+static int winesync_set_event(struct winesync_device *dev, void __user *argp, -+ bool pulse) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ bool prev_state; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ if (atomic_read(&event->all_hint) > 0) { -+ spin_lock(&dev->wait_all_lock); -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ try_wake_all_obj(dev, event); -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ spin_unlock(&dev->wait_all_lock); -+ } else { -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = true; -+ try_wake_any_event(event); -+ if (pulse) -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ } -+ -+ put_obj(event); -+ -+ if (put_user(prev_state, &user_args->signaled)) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+static int winesync_reset_event(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_event_args __user *user_args = argp; -+ struct winesync_event_args args; -+ struct winesync_obj *event; -+ bool prev_state; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ event = get_obj_typed(dev, args.event, WINESYNC_TYPE_EVENT); -+ if (!event) -+ return -EINVAL; -+ -+ spin_lock(&event->lock); -+ -+ prev_state = event->u.event.signaled; -+ event->u.event.signaled = false; -+ -+ spin_unlock(&event->lock); -+ -+ put_obj(event); -+ -+ if (put_user(prev_state, &user_args->signaled)) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+static int winesync_schedule(const struct winesync_q *q, ktime_t *timeout) -+{ -+ int ret = 0; -+ -+ do { -+ if (signal_pending(current)) { -+ ret = -ERESTARTSYS; -+ break; -+ } -+ -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (atomic_read(&q->signaled) != -1) { -+ ret = 0; -+ break; -+ } -+ ret = schedule_hrtimeout(timeout, HRTIMER_MODE_ABS); -+ } while (ret < 0); -+ __set_current_state(TASK_RUNNING); -+ -+ return ret; -+} -+ -+/* -+ * Allocate and initialize the winesync_q structure, but do not queue us yet. -+ * Also, calculate the relative timeout. -+ */ -+static int setup_wait(struct winesync_device *dev, -+ const struct winesync_wait_args *args, bool all, -+ ktime_t *ret_timeout, struct winesync_q **ret_q) -+{ -+ const __u32 count = args->count; -+ struct winesync_q *q; -+ ktime_t timeout = 0; -+ __u32 total_count; -+ __u32 *ids; -+ __u32 i, j; -+ -+ if (!args->owner) -+ return -EINVAL; -+ -+ if (args->timeout) { -+ struct timespec64 to; -+ -+ if (get_timespec64(&to, u64_to_user_ptr(args->timeout))) -+ return -EFAULT; -+ if (!timespec64_valid(&to)) -+ return -EINVAL; -+ -+ timeout = timespec64_to_ns(&to); -+ } -+ -+ total_count = count; -+ if (args->alert) -+ total_count++; -+ -+ ids = kmalloc_array(total_count, sizeof(*ids), GFP_KERNEL); -+ if (!ids) -+ return -ENOMEM; -+ if (copy_from_user(ids, u64_to_user_ptr(args->objs), -+ array_size(count, sizeof(*ids)))) { -+ kfree(ids); -+ return -EFAULT; -+ } -+ if (args->alert) -+ ids[count] = args->alert; -+ -+ q = kmalloc(struct_size(q, entries, total_count), GFP_KERNEL); -+ if (!q) { -+ kfree(ids); -+ return -ENOMEM; -+ } -+ q->task = current; -+ q->owner = args->owner; -+ atomic_set(&q->signaled, -1); -+ q->all = all; -+ q->ownerdead = false; -+ q->count = count; -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = get_obj(dev, ids[i]); -+ -+ if (!obj) -+ goto err; -+ -+ if (all) { -+ /* Check that the objects are all distinct. */ -+ for (j = 0; j < i; j++) { -+ if (obj == q->entries[j].obj) { -+ put_obj(obj); -+ goto err; -+ } -+ } -+ } -+ -+ entry->obj = obj; -+ entry->q = q; -+ entry->index = i; -+ } -+ -+ kfree(ids); -+ -+ *ret_q = q; -+ *ret_timeout = timeout; -+ return 0; -+ -+err: -+ for (j = 0; j < i; j++) -+ put_obj(q->entries[j].obj); -+ kfree(ids); -+ kfree(q); -+ return -EINVAL; -+} -+ -+static void try_wake_any_obj(struct winesync_obj *obj) -+{ -+ switch (obj->type) { -+ case WINESYNC_TYPE_SEM: -+ try_wake_any_sem(obj); -+ break; -+ case WINESYNC_TYPE_MUTEX: -+ try_wake_any_mutex(obj); -+ break; -+ case WINESYNC_TYPE_EVENT: -+ try_wake_any_event(obj); -+ break; -+ } -+} -+ -+static int winesync_wait_any(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_wait_args args; -+ struct winesync_q *q; -+ __u32 i, total_count; -+ ktime_t timeout; -+ int signaled; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, false, &timeout, &q); -+ if (ret < 0) -+ return ret; -+ -+ total_count = args.count; -+ if (args.alert) -+ total_count++; -+ -+ /* queue ourselves */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* -+ * Check if we are already signaled. -+ * -+ * Note that the API requires that normal objects are checked before -+ * the alert event. Hence we queue the alert event last, and check -+ * objects in order. -+ */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_obj *obj = q->entries[i].obj; -+ -+ if (atomic_read(&q->signaled) != -1) -+ break; -+ -+ spin_lock(&obj->lock); -+ try_wake_any_obj(obj); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* sleep */ -+ -+ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); -+ -+ /* and finally, unqueue */ -+ -+ for (i = 0; i < total_count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_del(&entry->node); -+ spin_unlock(&obj->lock); -+ -+ put_obj(obj); -+ } -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct winesync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; -+} -+ -+static int winesync_wait_all(struct winesync_device *dev, void __user *argp) -+{ -+ struct winesync_wait_args args; -+ struct winesync_q *q; -+ ktime_t timeout; -+ int signaled; -+ __u32 i; -+ int ret; -+ -+ if (copy_from_user(&args, argp, sizeof(args))) -+ return -EFAULT; -+ -+ ret = setup_wait(dev, &args, true, &timeout, &q); -+ if (ret < 0) -+ return ret; -+ -+ /* queue ourselves */ -+ -+ spin_lock(&dev->wait_all_lock); -+ -+ for (i = 0; i < args.count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ atomic_inc(&obj->all_hint); -+ -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. -+ */ -+ list_add_tail(&entry->node, &obj->all_waiters); -+ } -+ if (args.alert) { -+ struct winesync_q_entry *entry = &q->entries[args.count]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_add_tail(&entry->node, &obj->any_waiters); -+ spin_unlock(&obj->lock); -+ } -+ -+ /* check if we are already signaled */ -+ -+ try_wake_all(dev, q, NULL); -+ -+ spin_unlock(&dev->wait_all_lock); -+ -+ /* -+ * Check if the alert event is signaled, making sure to do so only -+ * after checking if the other objects are signaled. -+ */ -+ -+ if (args.alert) { -+ struct winesync_obj *obj = q->entries[args.count].obj; -+ -+ if (atomic_read(&q->signaled) == -1) { -+ spin_lock(&obj->lock); -+ try_wake_any_obj(obj); -+ spin_unlock(&obj->lock); -+ } -+ } -+ -+ /* sleep */ -+ -+ ret = winesync_schedule(q, args.timeout ? &timeout : NULL); -+ -+ /* and finally, unqueue */ -+ -+ spin_lock(&dev->wait_all_lock); -+ -+ for (i = 0; i < args.count; i++) { -+ struct winesync_q_entry *entry = &q->entries[i]; -+ struct winesync_obj *obj = entry->obj; -+ -+ /* -+ * obj->all_waiters is protected by dev->wait_all_lock rather -+ * than obj->lock, so there is no need to acquire it here. -+ */ -+ list_del(&entry->node); -+ -+ atomic_dec(&obj->all_hint); -+ -+ put_obj(obj); -+ } -+ if (args.alert) { -+ struct winesync_q_entry *entry = &q->entries[args.count]; -+ struct winesync_obj *obj = entry->obj; -+ -+ spin_lock(&obj->lock); -+ list_del(&entry->node); -+ spin_unlock(&obj->lock); -+ -+ put_obj(obj); -+ } -+ -+ spin_unlock(&dev->wait_all_lock); -+ -+ signaled = atomic_read(&q->signaled); -+ if (signaled != -1) { -+ struct winesync_wait_args __user *user_args = argp; -+ -+ /* even if we caught a signal, we need to communicate success */ -+ ret = q->ownerdead ? -EOWNERDEAD : 0; -+ -+ if (put_user(signaled, &user_args->index)) -+ ret = -EFAULT; -+ } else if (!ret) { -+ ret = -ETIMEDOUT; -+ } -+ -+ kfree(q); -+ return ret; -+} -+ -+static long winesync_char_ioctl(struct file *file, unsigned int cmd, -+ unsigned long parm) -+{ -+ struct winesync_device *dev = file->private_data; -+ void __user *argp = (void __user *)parm; -+ -+ switch (cmd) { -+ case WINESYNC_IOC_CREATE_EVENT: -+ return winesync_create_event(dev, argp); -+ case WINESYNC_IOC_CREATE_MUTEX: -+ return winesync_create_mutex(dev, argp); -+ case WINESYNC_IOC_CREATE_SEM: -+ return winesync_create_sem(dev, argp); -+ case WINESYNC_IOC_DELETE: -+ return winesync_delete(dev, argp); -+ case WINESYNC_IOC_KILL_OWNER: -+ return winesync_kill_owner(dev, argp); -+ case WINESYNC_IOC_PULSE_EVENT: -+ return winesync_set_event(dev, argp, true); -+ case WINESYNC_IOC_PUT_MUTEX: -+ return winesync_put_mutex(dev, argp); -+ case WINESYNC_IOC_PUT_SEM: -+ return winesync_put_sem(dev, argp); -+ case WINESYNC_IOC_READ_EVENT: -+ return winesync_read_event(dev, argp); -+ case WINESYNC_IOC_READ_MUTEX: -+ return winesync_read_mutex(dev, argp); -+ case WINESYNC_IOC_READ_SEM: -+ return winesync_read_sem(dev, argp); -+ case WINESYNC_IOC_RESET_EVENT: -+ return winesync_reset_event(dev, argp); -+ case WINESYNC_IOC_SET_EVENT: -+ return winesync_set_event(dev, argp, false); -+ case WINESYNC_IOC_WAIT_ALL: -+ return winesync_wait_all(dev, argp); -+ case WINESYNC_IOC_WAIT_ANY: -+ return winesync_wait_any(dev, argp); -+ default: -+ return -ENOSYS; -+ } -+} -+ -+static const struct file_operations winesync_fops = { -+ .owner = THIS_MODULE, -+ .open = winesync_char_open, -+ .release = winesync_char_release, -+ .unlocked_ioctl = winesync_char_ioctl, -+ .compat_ioctl = winesync_char_ioctl, -+ .llseek = no_llseek, -+}; -+ -+static struct miscdevice winesync_misc = { -+ .minor = WINESYNC_MINOR, -+ .name = WINESYNC_NAME, -+ .fops = &winesync_fops, -+}; -+ -+static int __init winesync_init(void) -+{ -+ return misc_register(&winesync_misc); -+} -+ -+static void __exit winesync_exit(void) -+{ -+ misc_deregister(&winesync_misc); -+} -+ -+module_init(winesync_init); -+module_exit(winesync_exit); -+ -+MODULE_AUTHOR("Zebediah Figura"); -+MODULE_DESCRIPTION("Kernel driver for Wine synchronization primitives"); -+MODULE_LICENSE("GPL"); -+MODULE_ALIAS("devname:" WINESYNC_NAME); -+MODULE_ALIAS_MISCDEV(WINESYNC_MINOR); -diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h -index c0fea6ca5076..36fc5d5315a4 100644 ---- a/include/linux/miscdevice.h -+++ b/include/linux/miscdevice.h -@@ -71,6 +71,7 @@ - #define USERIO_MINOR 240 - #define VHOST_VSOCK_MINOR 241 - #define RFKILL_MINOR 242 -+#define WINESYNC_MINOR 243 - #define MISC_DYNAMIC_MINOR 255 - - struct device; -diff --git a/include/uapi/linux/winesync.h b/include/uapi/linux/winesync.h -new file mode 100644 -index 000000000000..5b4e369f7469 ---- /dev/null -+++ b/include/uapi/linux/winesync.h -@@ -0,0 +1,71 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * Kernel support for Wine synchronization primitives -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ -+ -+#ifndef __LINUX_WINESYNC_H -+#define __LINUX_WINESYNC_H -+ -+#include -+ -+struct winesync_sem_args { -+ __u32 sem; -+ __u32 count; -+ __u32 max; -+}; -+ -+struct winesync_mutex_args { -+ __u32 mutex; -+ __u32 owner; -+ __u32 count; -+}; -+ -+struct winesync_event_args { -+ __u32 event; -+ __u32 manual; -+ __u32 signaled; -+}; -+ -+struct winesync_wait_args { -+ __u64 timeout; -+ __u64 objs; -+ __u32 count; -+ __u32 owner; -+ __u32 index; -+ __u32 alert; -+}; -+ -+#define WINESYNC_IOC_BASE 0xf7 -+ -+#define WINESYNC_IOC_CREATE_SEM _IOWR(WINESYNC_IOC_BASE, 0, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_DELETE _IOW (WINESYNC_IOC_BASE, 1, __u32) -+#define WINESYNC_IOC_PUT_SEM _IOWR(WINESYNC_IOC_BASE, 2, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_WAIT_ANY _IOWR(WINESYNC_IOC_BASE, 3, \ -+ struct winesync_wait_args) -+#define WINESYNC_IOC_WAIT_ALL _IOWR(WINESYNC_IOC_BASE, 4, \ -+ struct winesync_wait_args) -+#define WINESYNC_IOC_CREATE_MUTEX _IOWR(WINESYNC_IOC_BASE, 5, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_PUT_MUTEX _IOWR(WINESYNC_IOC_BASE, 6, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_KILL_OWNER _IOW (WINESYNC_IOC_BASE, 7, __u32) -+#define WINESYNC_IOC_READ_SEM _IOWR(WINESYNC_IOC_BASE, 8, \ -+ struct winesync_sem_args) -+#define WINESYNC_IOC_READ_MUTEX _IOWR(WINESYNC_IOC_BASE, 9, \ -+ struct winesync_mutex_args) -+#define WINESYNC_IOC_CREATE_EVENT _IOWR(WINESYNC_IOC_BASE, 10, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_SET_EVENT _IOWR(WINESYNC_IOC_BASE, 11, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_RESET_EVENT _IOWR(WINESYNC_IOC_BASE, 12, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_PULSE_EVENT _IOWR(WINESYNC_IOC_BASE, 13, \ -+ struct winesync_event_args) -+#define WINESYNC_IOC_READ_EVENT _IOWR(WINESYNC_IOC_BASE, 14, \ -+ struct winesync_event_args) -+ -+#endif -diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile -index 1fc89b8ef433..c7d3d9f5e0a0 100644 ---- a/tools/testing/selftests/Makefile -+++ b/tools/testing/selftests/Makefile -@@ -14,6 +14,7 @@ TARGETS += drivers/dma-buf - TARGETS += drivers/s390x/uvdevice - TARGETS += drivers/net/bonding - TARGETS += drivers/net/team -+TARGETS += drivers/winesync - TARGETS += efivarfs - TARGETS += exec - TARGETS += filesystems -diff --git a/tools/testing/selftests/drivers/winesync/Makefile b/tools/testing/selftests/drivers/winesync/Makefile -new file mode 100644 -index 000000000000..43b39fdeea10 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/Makefile -@@ -0,0 +1,8 @@ -+# SPDX-LICENSE-IDENTIFIER: GPL-2.0-only -+TEST_GEN_PROGS := winesync -+ -+top_srcdir =../../../../.. -+CFLAGS += -I$(top_srcdir)/usr/include -+LDLIBS += -lpthread -+ -+include ../../lib.mk -diff --git a/tools/testing/selftests/drivers/winesync/config b/tools/testing/selftests/drivers/winesync/config -new file mode 100644 -index 000000000000..60539c826d06 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/config -@@ -0,0 +1 @@ -+CONFIG_WINESYNC=y -diff --git a/tools/testing/selftests/drivers/winesync/winesync.c b/tools/testing/selftests/drivers/winesync/winesync.c -new file mode 100644 -index 000000000000..169e922484b0 ---- /dev/null -+++ b/tools/testing/selftests/drivers/winesync/winesync.c -@@ -0,0 +1,1479 @@ -+// SPDX-License-Identifier: GPL-2.0-or-later -+/* -+ * Various unit tests for the "winesync" synchronization primitive driver. -+ * -+ * Copyright (C) 2021 Zebediah Figura -+ */ -+ -+#define _GNU_SOURCE -+#include -+#include -+#include -+#include -+#include -+#include -+#include "../../kselftest_harness.h" -+ -+static int read_sem_state(int fd, __u32 sem, __u32 *count, __u32 *max) -+{ -+ struct winesync_sem_args args; -+ int ret; -+ -+ args.sem = sem; -+ args.count = 0xdeadbeef; -+ args.max = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &args); -+ *count = args.count; -+ *max = args.max; -+ return ret; -+} -+ -+#define check_sem_state(fd, sem, count, max) \ -+ ({ \ -+ __u32 __count, __max; \ -+ int ret = read_sem_state((fd), (sem), &__count, &__max); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((max), __max); \ -+ }) -+ -+static int put_sem(int fd, __u32 sem, __u32 *count) -+{ -+ struct winesync_sem_args args; -+ int ret; -+ -+ args.sem = sem; -+ args.count = *count; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &args); -+ *count = args.count; -+ return ret; -+} -+ -+static int read_mutex_state(int fd, __u32 mutex, __u32 *count, __u32 *owner) -+{ -+ struct winesync_mutex_args args; -+ int ret; -+ -+ args.mutex = mutex; -+ args.count = 0xdeadbeef; -+ args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &args); -+ *count = args.count; -+ *owner = args.owner; -+ return ret; -+} -+ -+#define check_mutex_state(fd, mutex, count, owner) \ -+ ({ \ -+ __u32 __count, __owner; \ -+ int ret = read_mutex_state((fd), (mutex), &__count, &__owner); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((count), __count); \ -+ EXPECT_EQ((owner), __owner); \ -+ }) -+ -+static int put_mutex(int fd, __u32 mutex, __u32 owner, __u32 *count) -+{ -+ struct winesync_mutex_args args; -+ int ret; -+ -+ args.mutex = mutex; -+ args.owner = owner; -+ args.count = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &args); -+ *count = args.count; -+ return ret; -+} -+ -+static int read_event_state(int fd, __u32 event, __u32 *signaled, __u32 *manual) -+{ -+ struct winesync_event_args args; -+ int ret; -+ -+ args.event = event; -+ args.signaled = 0xdeadbeef; -+ args.manual = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &args); -+ *signaled = args.signaled; -+ *manual = args.manual; -+ return ret; -+} -+ -+#define check_event_state(fd, event, signaled, manual) \ -+ ({ \ -+ __u32 __signaled, __manual; \ -+ int ret = read_event_state((fd), (event), \ -+ &__signaled, &__manual); \ -+ EXPECT_EQ(0, ret); \ -+ EXPECT_EQ((signaled), __signaled); \ -+ EXPECT_EQ((manual), __manual); \ -+ }) -+ -+static int wait_objs(int fd, unsigned long request, __u32 count, -+ const __u32 *objs, __u32 owner, __u32 alert, __u32 *index) -+{ -+ struct winesync_wait_args args = {0}; -+ struct timespec timeout; -+ int ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ args.timeout = (uintptr_t)&timeout; -+ args.count = count; -+ args.objs = (uintptr_t)objs; -+ args.owner = owner; -+ args.index = 0xdeadbeef; -+ args.alert = alert; -+ ret = ioctl(fd, request, &args); -+ *index = args.index; -+ return ret; -+} -+ -+static int wait_any(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, -+ count, objs, owner, 0, index); -+} -+ -+static int wait_all(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, -+ count, objs, owner, 0, index); -+} -+ -+static int wait_any_alert(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 alert, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ANY, -+ count, objs, owner, alert, index); -+} -+ -+static int wait_all_alert(int fd, __u32 count, const __u32 *objs, -+ __u32 owner, __u32 alert, __u32 *index) -+{ -+ return wait_objs(fd, WINESYNC_IOC_WAIT_ALL, -+ count, objs, owner, alert, index); -+} -+ -+TEST(semaphore_state) -+{ -+ struct winesync_sem_args sem_args; -+ struct timespec timeout; -+ __u32 sem, count, index; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 3; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ check_sem_state(fd, sem, 2, 2); -+ -+ count = 0; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_sem_state(fd, sem, 2, 2); -+ -+ count = 1; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(fd, sem, 2, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem, 1, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem, 0, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ count = 3; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOVERFLOW, errno); -+ check_sem_state(fd, sem, 0, 2); -+ -+ count = 2; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(fd, sem, 2, 2); -+ -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ ret = wait_any(fd, 1, &sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ -+ count = 1; -+ ret = put_sem(fd, sem, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, count); -+ check_sem_state(fd, sem, 1, 2); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(mutex_state) -+{ -+ struct winesync_mutex_args mutex_args; -+ __u32 mutex, owner, count, index; -+ struct timespec timeout; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 0; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 2; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ mutex = mutex_args.mutex; -+ check_mutex_state(fd, mutex, 2, 123); -+ -+ ret = put_mutex(fd, mutex, 0, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = put_mutex(fd, mutex, 456, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ check_mutex_state(fd, mutex, 2, 123); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ check_mutex_state(fd, mutex, 0, 0); -+ -+ ret = put_mutex(fd, mutex, 123, &count); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EPERM, errno); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 2, 456); -+ -+ ret = put_mutex(fd, mutex, 456, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 0; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ check_mutex_state(fd, mutex, 1, 456); -+ -+ owner = 456; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.count = 0xdeadbeef; -+ mutex_args.owner = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, mutex_args.count); -+ EXPECT_EQ(0, mutex_args.owner); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ mutex = mutex_args.mutex; -+ check_mutex_state(fd, mutex, 0, 0); -+ -+ ret = wait_any(fd, 1, &mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_mutex_state(fd, mutex, 1, 123); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(manual_event_state) -+{ -+ struct winesync_event_args event_args; -+ __u32 index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 1; -+ event_args.signaled = 0; -+ event_args.event = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, event_args.event); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(auto_event_state) -+{ -+ struct winesync_event_args event_args; -+ __u32 index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ event_args.manual = 0; -+ event_args.signaled = 1; -+ event_args.event = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, event_args.event); -+ -+ check_event_state(fd, event_args.event, 1, 0); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 0); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ event_args.signaled = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_any(fd, 1, &event_args.event, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(test_wait_any) -+{ -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2], owner, index; -+ struct timespec timeout; -+ int fd, ret; -+ -+ clock_gettime(CLOCK_MONOTONIC, &timeout); -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ ret = wait_any(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ /* test waiting on the same object twice */ -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ objs[0] = objs[1] = sem_args.sem; -+ ret = wait_any(fd, 2, objs, 456, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, wait_args.index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ -+ ret = wait_any(fd, 0, NULL, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(test_wait_all) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2], owner, index; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 0; -+ mutex_args.count = 0; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ ret = wait_all(fd, 2, objs, 456, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_mutex_state(fd, mutex_args.mutex, 2, 123); -+ -+ sem_args.count = 3; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 2, 3); -+ check_mutex_state(fd, mutex_args.mutex, 3, 123); -+ -+ owner = 123; -+ ret = ioctl(fd, WINESYNC_IOC_KILL_OWNER, &owner); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EOWNERDEAD, errno); -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 123); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = event_args.event; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ /* test waiting on the same object twice */ -+ objs[0] = objs[1] = sem_args.sem; -+ ret = wait_all(fd, 2, objs, 123, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(invalid_objects) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ __u32 objs[2] = {0}; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ sem_args.max = 1; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ -+ mutex_args.mutex = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_MUTEX, &mutex_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ event_args.event = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_EVENT, &event_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = sem_args.sem + 1; -+ wait_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ objs[0] = sem_args.sem + 1; -+ objs[1] = sem_args.sem; -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ANY, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ ret = ioctl(fd, WINESYNC_IOC_WAIT_ALL, &wait_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ -+ sem_args.sem = mutex_args.mutex; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_READ_SEM, &sem_args); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(EINVAL, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+struct wake_args -+{ -+ int fd; -+ __u32 obj; -+}; -+ -+struct wait_args -+{ -+ int fd; -+ unsigned long request; -+ struct winesync_wait_args *args; -+ int ret; -+ int err; -+}; -+ -+static void *wait_thread(void *arg) -+{ -+ struct wait_args *args = arg; -+ -+ args->ret = ioctl(args->fd, args->request, args->args); -+ args->err = errno; -+ return NULL; -+} -+ -+static void get_abs_timeout(struct timespec *timeout, clockid_t clock, -+ unsigned int ms) -+{ -+ clock_gettime(clock, timeout); -+ timeout->tv_nsec += ms * 1000000; -+ timeout->tv_sec += (timeout->tv_nsec / 1000000000); -+ timeout->tv_nsec %= 1000000000; -+} -+ -+static int wait_for_thread(pthread_t thread, unsigned int ms) -+{ -+ struct timespec timeout; -+ get_abs_timeout(&timeout, CLOCK_REALTIME, ms); -+ return pthread_timedjoin_np(thread, NULL, &timeout); -+} -+ -+TEST(wake_any) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 objs[2], count, index; -+ struct timespec timeout; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ -+ /* test waking the semaphore */ -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 456; -+ wait_args.index = 0xdeadbeef; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ check_sem_state(fd, sem_args.sem, 0, 3); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(0, wait_args.index); -+ -+ /* test waking the mutex */ -+ -+ /* first grab it again for owner 123 */ -+ ret = wait_any(fd, 1, &mutex_args.mutex, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.owner = 456; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, mutex_args.count); -+ check_mutex_state(fd, mutex_args.mutex, 1, 456); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ /* test waking events */ -+ -+ event_args.manual = false; -+ event_args.signaled = false; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[1] = event_args.event; -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ event_args.manual = true; -+ event_args.signaled = false; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[1] = event_args.event; -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 1, 1); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, event_args.signaled); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_PULSE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, event_args.signaled); -+ check_event_state(fd, event_args.event, 0, 1); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(1, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ /* delete an object while it's being waited on */ -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); -+ wait_args.owner = 123; -+ objs[1] = mutex_args.mutex; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); -+ -+ close(fd); -+} -+ -+TEST(wake_all) -+{ -+ struct winesync_event_args manual_event_args = {0}; -+ struct winesync_event_args auto_event_args = {0}; -+ struct winesync_mutex_args mutex_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ __u32 objs[4], count, index; -+ struct timespec timeout; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 3; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ -+ mutex_args.owner = 123; -+ mutex_args.count = 1; -+ mutex_args.mutex = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_MUTEX, &mutex_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, mutex_args.mutex); -+ -+ manual_event_args.manual = true; -+ manual_event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ -+ auto_event_args.manual = false; -+ auto_event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ -+ objs[0] = sem_args.sem; -+ objs[1] = mutex_args.mutex; -+ objs[2] = manual_event_args.event; -+ objs[3] = auto_event_args.event; -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 4; -+ wait_args.owner = 456; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ -+ ret = wait_any(fd, 1, &sem_args.sem, 123, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = put_mutex(fd, mutex_args.mutex, 123, &count); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, count); -+ -+ ret = pthread_tryjoin_np(thread, NULL); -+ EXPECT_EQ(EBUSY, ret); -+ -+ check_mutex_state(fd, mutex_args.mutex, 0, 0); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, manual_event_args.signaled); -+ -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, sem_args.count); -+ check_sem_state(fd, sem_args.sem, 2, 3); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, auto_event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &manual_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, manual_event_args.signaled); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &auto_event_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, auto_event_args.signaled); -+ -+ check_sem_state(fd, sem_args.sem, 1, 3); -+ check_mutex_state(fd, mutex_args.mutex, 1, 456); -+ check_event_state(fd, manual_event_args.event, 1, 1); -+ check_event_state(fd, auto_event_args.event, 0, 0); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ -+ /* delete an object while it's being waited on */ -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 200); -+ wait_args.owner = 123; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &sem_args.sem); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &mutex_args.mutex); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &manual_event_args.event); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &auto_event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 200); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(-1, thread_args.ret); -+ EXPECT_EQ(ETIMEDOUT, thread_args.err); -+ -+ close(fd); -+} -+ -+TEST(alert_any) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ struct timespec timeout; -+ __u32 objs[2], index; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 0; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[0] = sem_args.sem; -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[1] = sem_args.sem; -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 0, NULL, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(1, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ /* test wakeup via alert */ -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event_args.event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ANY; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ /* test with an auto-reset event */ -+ -+ event_args.manual = false; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ sem_args.sem = objs[0]; -+ sem_args.count = 1; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ ret = wait_any_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST(alert_all) -+{ -+ struct winesync_event_args event_args = {0}; -+ struct winesync_wait_args wait_args = {0}; -+ struct winesync_sem_args sem_args = {0}; -+ struct wait_args thread_args; -+ struct timespec timeout; -+ __u32 objs[2], index; -+ pthread_t thread; -+ int fd, ret; -+ -+ fd = open("/dev/winesync", O_CLOEXEC | O_RDONLY); -+ ASSERT_LE(0, fd); -+ -+ sem_args.count = 2; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[0] = sem_args.sem; -+ -+ sem_args.count = 1; -+ sem_args.max = 2; -+ sem_args.sem = 0xdeadbeef; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ EXPECT_NE(0xdeadbeef, sem_args.sem); -+ objs[1] = sem_args.sem; -+ -+ event_args.manual = true; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ /* test wakeup via alert */ -+ -+ ret = ioctl(fd, WINESYNC_IOC_RESET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ get_abs_timeout(&timeout, CLOCK_MONOTONIC, 1000); -+ wait_args.timeout = (uintptr_t)&timeout; -+ wait_args.objs = (uintptr_t)objs; -+ wait_args.count = 2; -+ wait_args.owner = 123; -+ wait_args.index = 0xdeadbeef; -+ wait_args.alert = event_args.event; -+ thread_args.fd = fd; -+ thread_args.args = &wait_args; -+ thread_args.request = WINESYNC_IOC_WAIT_ALL; -+ ret = pthread_create(&thread, NULL, wait_thread, &thread_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(ETIMEDOUT, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_SET_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_for_thread(thread, 100); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, thread_args.ret); -+ EXPECT_EQ(2, wait_args.index); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ /* test with an auto-reset event */ -+ -+ event_args.manual = false; -+ event_args.signaled = true; -+ ret = ioctl(fd, WINESYNC_IOC_CREATE_EVENT, &event_args); -+ EXPECT_EQ(0, ret); -+ -+ sem_args.sem = objs[1]; -+ sem_args.count = 2; -+ ret = ioctl(fd, WINESYNC_IOC_PUT_SEM, &sem_args); -+ EXPECT_EQ(0, ret); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(0, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(0, ret); -+ EXPECT_EQ(2, index); -+ -+ ret = wait_all_alert(fd, 2, objs, 123, event_args.event, &index); -+ EXPECT_EQ(-1, ret); -+ EXPECT_EQ(ETIMEDOUT, errno); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &event_args.event); -+ EXPECT_EQ(0, ret); -+ -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[0]); -+ EXPECT_EQ(0, ret); -+ ret = ioctl(fd, WINESYNC_IOC_DELETE, &objs[1]); -+ EXPECT_EQ(0, ret); -+ -+ close(fd); -+} -+ -+TEST_HARNESS_MAIN --- -2.38.0.rc1.8.g2a7d63a245 - diff --git a/6.0/testing/0004-Introducing-OpenVPN-Data-Channel-Offload.patch b/6.0/testing/0004-Introducing-OpenVPN-Data-Channel-Offload.patch deleted file mode 100644 index 66f4a58c..00000000 --- a/6.0/testing/0004-Introducing-OpenVPN-Data-Channel-Offload.patch +++ /dev/null @@ -1,6167 +0,0 @@ -From e28474f737770960d42ba4e4bddc8dd6eb52590b Mon Sep 17 00:00:00 2001 -From: Peter Jung -Date: Fri, 5 Aug 2022 19:33:47 +0200 -Subject: [PATCH 04/13] Introducing-OpenVPN-Data-Channel-Offload - -Signed-off-by: Peter Jung ---- - MAINTAINERS | 8 + - drivers/net/Kconfig | 19 + - drivers/net/Makefile | 1 + - drivers/net/ovpn-dco/Makefile | 21 + - drivers/net/ovpn-dco/addr.h | 41 + - drivers/net/ovpn-dco/bind.c | 62 ++ - drivers/net/ovpn-dco/bind.h | 67 ++ - drivers/net/ovpn-dco/crypto.c | 154 ++++ - drivers/net/ovpn-dco/crypto.h | 144 ++++ - drivers/net/ovpn-dco/crypto_aead.c | 367 +++++++++ - drivers/net/ovpn-dco/crypto_aead.h | 27 + - drivers/net/ovpn-dco/main.c | 271 +++++++ - drivers/net/ovpn-dco/main.h | 32 + - drivers/net/ovpn-dco/netlink.c | 1143 ++++++++++++++++++++++++++++ - drivers/net/ovpn-dco/netlink.h | 22 + - drivers/net/ovpn-dco/ovpn.c | 600 +++++++++++++++ - drivers/net/ovpn-dco/ovpn.h | 43 ++ - drivers/net/ovpn-dco/ovpnstruct.h | 59 ++ - drivers/net/ovpn-dco/peer.c | 906 ++++++++++++++++++++++ - drivers/net/ovpn-dco/peer.h | 168 ++++ - drivers/net/ovpn-dco/pktid.c | 127 ++++ - drivers/net/ovpn-dco/pktid.h | 116 +++ - drivers/net/ovpn-dco/proto.h | 101 +++ - drivers/net/ovpn-dco/rcu.h | 21 + - drivers/net/ovpn-dco/skb.h | 54 ++ - drivers/net/ovpn-dco/sock.c | 134 ++++ - drivers/net/ovpn-dco/sock.h | 54 ++ - drivers/net/ovpn-dco/stats.c | 20 + - drivers/net/ovpn-dco/stats.h | 67 ++ - drivers/net/ovpn-dco/tcp.c | 326 ++++++++ - drivers/net/ovpn-dco/tcp.h | 38 + - drivers/net/ovpn-dco/udp.c | 343 +++++++++ - drivers/net/ovpn-dco/udp.h | 25 + - include/net/netlink.h | 1 + - include/uapi/linux/ovpn_dco.h | 265 +++++++ - include/uapi/linux/udp.h | 1 + - 36 files changed, 5848 insertions(+) - create mode 100644 drivers/net/ovpn-dco/Makefile - create mode 100644 drivers/net/ovpn-dco/addr.h - create mode 100644 drivers/net/ovpn-dco/bind.c - create mode 100644 drivers/net/ovpn-dco/bind.h - create mode 100644 drivers/net/ovpn-dco/crypto.c - create mode 100644 drivers/net/ovpn-dco/crypto.h - create mode 100644 drivers/net/ovpn-dco/crypto_aead.c - create mode 100644 drivers/net/ovpn-dco/crypto_aead.h - create mode 100644 drivers/net/ovpn-dco/main.c - create mode 100644 drivers/net/ovpn-dco/main.h - create mode 100644 drivers/net/ovpn-dco/netlink.c - create mode 100644 drivers/net/ovpn-dco/netlink.h - create mode 100644 drivers/net/ovpn-dco/ovpn.c - create mode 100644 drivers/net/ovpn-dco/ovpn.h - create mode 100644 drivers/net/ovpn-dco/ovpnstruct.h - create mode 100644 drivers/net/ovpn-dco/peer.c - create mode 100644 drivers/net/ovpn-dco/peer.h - create mode 100644 drivers/net/ovpn-dco/pktid.c - create mode 100644 drivers/net/ovpn-dco/pktid.h - create mode 100644 drivers/net/ovpn-dco/proto.h - create mode 100644 drivers/net/ovpn-dco/rcu.h - create mode 100644 drivers/net/ovpn-dco/skb.h - create mode 100644 drivers/net/ovpn-dco/sock.c - create mode 100644 drivers/net/ovpn-dco/sock.h - create mode 100644 drivers/net/ovpn-dco/stats.c - create mode 100644 drivers/net/ovpn-dco/stats.h - create mode 100644 drivers/net/ovpn-dco/tcp.c - create mode 100644 drivers/net/ovpn-dco/tcp.h - create mode 100644 drivers/net/ovpn-dco/udp.c - create mode 100644 drivers/net/ovpn-dco/udp.h - create mode 100644 include/uapi/linux/ovpn_dco.h - -diff --git a/MAINTAINERS b/MAINTAINERS -index 31a7aa60cdc3..a29c9731350c 100644 ---- a/MAINTAINERS -+++ b/MAINTAINERS -@@ -15320,6 +15320,14 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git - F: Documentation/filesystems/overlayfs.rst - F: fs/overlayfs/ - -+OVPN-DCO NETWORK DRIVER -+M: Antonio Quartulli -+L: openvpn-devel@lists.sourceforge.net (moderated for non-subscribers) -+L: netdev@vger.kernel.org -+S: Maintained -+F: drivers/net/ovpn-dco/ -+F: include/uapi/linux/ovpn_dco.h -+ - P54 WIRELESS DRIVER - M: Christian Lamparter - L: linux-wireless@vger.kernel.org -diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig -index 94c889802566..349866bd4448 100644 ---- a/drivers/net/Kconfig -+++ b/drivers/net/Kconfig -@@ -116,6 +116,25 @@ config WIREGUARD_DEBUG - - Say N here unless you know what you're doing. - -+config OVPN_DCO -+ tristate "OpenVPN data channel offload" -+ depends on NET && INET -+ select NET_UDP_TUNNEL -+ select DST_CACHE -+ select CRYPTO -+ select CRYPTO_AES -+ select CRYPTO_GCM -+ select CRYPTO_CHACHA20POLY1305 -+ help -+ This module enhances the performance of an OpenVPN connection by -+ allowing the user to offload the data channel processing to -+ kernelspace. -+ Connection handshake, parameters negotiation and other non-data -+ related mechanisms are still performed in userspace. -+ -+ The OpenVPN userspace software at version 2.6 or higher is required -+ to use this functionality. -+ - config EQUALIZER - tristate "EQL (serial line load balancing) support" - help -diff --git a/drivers/net/Makefile b/drivers/net/Makefile -index 3f1192d3c52d..8ed151e8d233 100644 ---- a/drivers/net/Makefile -+++ b/drivers/net/Makefile -@@ -11,6 +11,7 @@ obj-$(CONFIG_IPVLAN) += ipvlan/ - obj-$(CONFIG_IPVTAP) += ipvlan/ - obj-$(CONFIG_DUMMY) += dummy.o - obj-$(CONFIG_WIREGUARD) += wireguard/ -+obj-$(CONFIG_OVPN_DCO) += ovpn-dco/ - obj-$(CONFIG_EQUALIZER) += eql.o - obj-$(CONFIG_IFB) += ifb.o - obj-$(CONFIG_MACSEC) += macsec.o -diff --git a/drivers/net/ovpn-dco/Makefile b/drivers/net/ovpn-dco/Makefile -new file mode 100644 -index 000000000000..7efefe8f13a9 ---- /dev/null -+++ b/drivers/net/ovpn-dco/Makefile -@@ -0,0 +1,21 @@ -+# SPDX-License-Identifier: GPL-2.0 -+# -+# ovpn-dco -- OpenVPN data channel offload -+# -+# Copyright (C) 2020-2022 OpenVPN, Inc. -+# -+# Author: Antonio Quartulli -+ -+obj-$(CONFIG_OVPN_DCO) += ovpn-dco.o -+ovpn-dco-y += main.o -+ovpn-dco-y += bind.o -+ovpn-dco-y += crypto.o -+ovpn-dco-y += ovpn.o -+ovpn-dco-y += peer.o -+ovpn-dco-y += sock.o -+ovpn-dco-y += stats.o -+ovpn-dco-y += netlink.o -+ovpn-dco-y += crypto_aead.o -+ovpn-dco-y += pktid.o -+ovpn-dco-y += tcp.o -+ovpn-dco-y += udp.o -diff --git a/drivers/net/ovpn-dco/addr.h b/drivers/net/ovpn-dco/addr.h -new file mode 100644 -index 000000000000..3d6ad0fc15af ---- /dev/null -+++ b/drivers/net/ovpn-dco/addr.h -@@ -0,0 +1,41 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNADDR_H_ -+#define _NET_OVPN_DCO_OVPNADDR_H_ -+ -+#include "crypto.h" -+ -+#include -+#include -+#include -+#include -+ -+/* our basic transport layer address */ -+struct ovpn_sockaddr { -+ union { -+ struct sockaddr_in in4; -+ struct sockaddr_in6 in6; -+ }; -+}; -+ -+/* Translate skb->protocol value to AF_INET or AF_INET6 */ -+static inline unsigned short skb_protocol_to_family(const struct sk_buff *skb) -+{ -+ switch (skb->protocol) { -+ case htons(ETH_P_IP): -+ return AF_INET; -+ case htons(ETH_P_IPV6): -+ return AF_INET6; -+ default: -+ return 0; -+ } -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNADDR_H_ */ -diff --git a/drivers/net/ovpn-dco/bind.c b/drivers/net/ovpn-dco/bind.c -new file mode 100644 -index 000000000000..107697ea983e ---- /dev/null -+++ b/drivers/net/ovpn-dco/bind.c -@@ -0,0 +1,62 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2012-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "ovpn.h" -+#include "bind.h" -+#include "peer.h" -+ -+#include -+#include -+#include -+#include -+ -+/* Given a remote sockaddr, compute the skb hash -+ * and get a dst_entry so we can send packets to the remote. -+ * Called from process context or softirq (must be indicated with -+ * process_context bool). -+ */ -+struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *ss) -+{ -+ struct ovpn_bind *bind; -+ size_t sa_len; -+ -+ if (ss->ss_family == AF_INET) -+ sa_len = sizeof(struct sockaddr_in); -+ else if (ss->ss_family == AF_INET6) -+ sa_len = sizeof(struct sockaddr_in6); -+ else -+ return ERR_PTR(-EAFNOSUPPORT); -+ -+ bind = kzalloc(sizeof(*bind), GFP_ATOMIC); -+ if (unlikely(!bind)) -+ return ERR_PTR(-ENOMEM); -+ -+ memcpy(&bind->sa, ss, sa_len); -+ -+ return bind; -+} -+ -+static void ovpn_bind_release_rcu(struct rcu_head *head) -+{ -+ struct ovpn_bind *bind = container_of(head, struct ovpn_bind, rcu); -+ -+ kfree(bind); -+} -+ -+void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *new) -+{ -+ struct ovpn_bind *old; -+ -+ spin_lock_bh(&peer->lock); -+ old = rcu_replace_pointer(peer->bind, new, true); -+ spin_unlock_bh(&peer->lock); -+ -+ if (old) -+ call_rcu(&old->rcu, ovpn_bind_release_rcu); -+} -diff --git a/drivers/net/ovpn-dco/bind.h b/drivers/net/ovpn-dco/bind.h -new file mode 100644 -index 000000000000..a562e471acae ---- /dev/null -+++ b/drivers/net/ovpn-dco/bind.h -@@ -0,0 +1,67 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OVPN -- OpenVPN protocol accelerator for Linux -+ * Copyright (C) 2012-2022 OpenVPN, Inc. -+ * All rights reserved. -+ * Author: James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNBIND_H_ -+#define _NET_OVPN_DCO_OVPNBIND_H_ -+ -+#include "addr.h" -+#include "rcu.h" -+ -+#include -+#include -+#include -+ -+struct ovpn_peer; -+ -+struct ovpn_bind { -+ struct ovpn_sockaddr sa; /* remote sockaddr */ -+ -+ union { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } local; -+ -+ struct rcu_head rcu; -+}; -+ -+static inline bool ovpn_bind_skb_src_match(const struct ovpn_bind *bind, struct sk_buff *skb) -+{ -+ const unsigned short family = skb_protocol_to_family(skb); -+ const struct ovpn_sockaddr *sa = &bind->sa; -+ -+ if (unlikely(!bind)) -+ return false; -+ -+ if (unlikely(sa->in4.sin_family != family)) -+ return false; -+ -+ switch (family) { -+ case AF_INET: -+ if (unlikely(sa->in4.sin_addr.s_addr != ip_hdr(skb)->saddr)) -+ return false; -+ -+ if (unlikely(sa->in4.sin_port != udp_hdr(skb)->source)) -+ return false; -+ break; -+ case AF_INET6: -+ if (unlikely(!ipv6_addr_equal(&sa->in6.sin6_addr, &ipv6_hdr(skb)->saddr))) -+ return false; -+ -+ if (unlikely(sa->in6.sin6_port != udp_hdr(skb)->source)) -+ return false; -+ break; -+ default: -+ return false; -+ } -+ -+ return true; -+} -+ -+struct ovpn_bind *ovpn_bind_from_sockaddr(const struct sockaddr_storage *sa); -+void ovpn_bind_reset(struct ovpn_peer *peer, struct ovpn_bind *bind); -+ -+#endif /* _NET_OVPN_DCO_OVPNBIND_H_ */ -diff --git a/drivers/net/ovpn-dco/crypto.c b/drivers/net/ovpn-dco/crypto.c -new file mode 100644 -index 000000000000..fcc3a351ba9d ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto.c -@@ -0,0 +1,154 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "crypto_aead.h" -+#include "crypto.h" -+ -+#include -+ -+static void ovpn_ks_destroy_rcu(struct rcu_head *head) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ ks = container_of(head, struct ovpn_crypto_key_slot, rcu); -+ ovpn_aead_crypto_key_slot_destroy(ks); -+} -+ -+void ovpn_crypto_key_slot_release(struct kref *kref) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ ks = container_of(kref, struct ovpn_crypto_key_slot, refcount); -+ call_rcu(&ks->rcu, ovpn_ks_destroy_rcu); -+} -+ -+/* can only be invoked when all peer references have been dropped (i.e. RCU -+ * release routine) -+ */ -+void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ ks = rcu_access_pointer(cs->primary); -+ if (ks) { -+ RCU_INIT_POINTER(cs->primary, NULL); -+ ovpn_crypto_key_slot_put(ks); -+ } -+ -+ ks = rcu_access_pointer(cs->secondary); -+ if (ks) { -+ RCU_INIT_POINTER(cs->secondary, NULL); -+ ovpn_crypto_key_slot_put(ks); -+ } -+ -+ mutex_destroy(&cs->mutex); -+} -+ -+/* removes the primary key from the crypto context */ -+void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ mutex_lock(&cs->mutex); -+ ks = rcu_replace_pointer(cs->primary, NULL, lockdep_is_held(&cs->mutex)); -+ ovpn_crypto_key_slot_put(ks); -+ mutex_unlock(&cs->mutex); -+} -+ -+/* Reset the ovpn_crypto_state object in a way that is atomic -+ * to RCU readers. -+ */ -+int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, -+ const struct ovpn_peer_key_reset *pkr) -+ __must_hold(cs->mutex) -+{ -+ struct ovpn_crypto_key_slot *old = NULL; -+ struct ovpn_crypto_key_slot *new; -+ -+ lockdep_assert_held(&cs->mutex); -+ -+ new = ovpn_aead_crypto_key_slot_new(&pkr->key); -+ if (IS_ERR(new)) -+ return PTR_ERR(new); -+ -+ switch (pkr->slot) { -+ case OVPN_KEY_SLOT_PRIMARY: -+ old = rcu_replace_pointer(cs->primary, new, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ case OVPN_KEY_SLOT_SECONDARY: -+ old = rcu_replace_pointer(cs->secondary, new, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ default: -+ goto free_key; -+ } -+ -+ if (old) -+ ovpn_crypto_key_slot_put(old); -+ -+ return 0; -+free_key: -+ ovpn_crypto_key_slot_put(new); -+ return -EINVAL; -+} -+ -+void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, -+ enum ovpn_key_slot slot) -+{ -+ struct ovpn_crypto_key_slot *ks = NULL; -+ -+ mutex_lock(&cs->mutex); -+ switch (slot) { -+ case OVPN_KEY_SLOT_PRIMARY: -+ ks = rcu_replace_pointer(cs->primary, NULL, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ case OVPN_KEY_SLOT_SECONDARY: -+ ks = rcu_replace_pointer(cs->secondary, NULL, -+ lockdep_is_held(&cs->mutex)); -+ break; -+ default: -+ pr_warn("Invalid slot to release: %u\n", slot); -+ break; -+ } -+ mutex_unlock(&cs->mutex); -+ -+ if (!ks) { -+ pr_debug("Key slot already released: %u\n", slot); -+ return; -+ } -+ pr_debug("deleting key slot %u, key_id=%u\n", slot, ks->key_id); -+ -+ ovpn_crypto_key_slot_put(ks); -+} -+ -+/* this swap is not atomic, but there will be a very short time frame where the -+ * old_secondary key won't be available. This should not be a big deal as most -+ * likely both peers are already using the new primary at this point. -+ */ -+void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs) -+{ -+ const struct ovpn_crypto_key_slot *old_primary, *old_secondary; -+ -+ mutex_lock(&cs->mutex); -+ -+ old_secondary = rcu_dereference_protected(cs->secondary, -+ lockdep_is_held(&cs->mutex)); -+ old_primary = rcu_replace_pointer(cs->primary, old_secondary, -+ lockdep_is_held(&cs->mutex)); -+ rcu_assign_pointer(cs->secondary, old_primary); -+ -+ pr_debug("key swapped: %u <-> %u\n", -+ old_primary ? old_primary->key_id : 0, -+ old_secondary ? old_secondary->key_id : 0); -+ -+ mutex_unlock(&cs->mutex); -+} -diff --git a/drivers/net/ovpn-dco/crypto.h b/drivers/net/ovpn-dco/crypto.h -new file mode 100644 -index 000000000000..79f580e54a63 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto.h -@@ -0,0 +1,144 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNCRYPTO_H_ -+#define _NET_OVPN_DCO_OVPNCRYPTO_H_ -+ -+#include "main.h" -+#include "pktid.h" -+ -+#include -+#include -+ -+struct ovpn_peer; -+struct ovpn_crypto_key_slot; -+ -+/* info needed for both encrypt and decrypt directions */ -+struct ovpn_key_direction { -+ const u8 *cipher_key; -+ size_t cipher_key_size; -+ const u8 *nonce_tail; /* only needed for GCM modes */ -+ size_t nonce_tail_size; /* only needed for GCM modes */ -+}; -+ -+/* all info for a particular symmetric key (primary or secondary) */ -+struct ovpn_key_config { -+ enum ovpn_cipher_alg cipher_alg; -+ u8 key_id; -+ struct ovpn_key_direction encrypt; -+ struct ovpn_key_direction decrypt; -+}; -+ -+/* used to pass settings from netlink to the crypto engine */ -+struct ovpn_peer_key_reset { -+ enum ovpn_key_slot slot; -+ struct ovpn_key_config key; -+}; -+ -+struct ovpn_crypto_key_slot { -+ u8 key_id; -+ -+ struct crypto_aead *encrypt; -+ struct crypto_aead *decrypt; -+ struct ovpn_nonce_tail nonce_tail_xmit; -+ struct ovpn_nonce_tail nonce_tail_recv; -+ -+ struct ovpn_pktid_recv pid_recv ____cacheline_aligned_in_smp; -+ struct ovpn_pktid_xmit pid_xmit ____cacheline_aligned_in_smp; -+ struct kref refcount; -+ struct rcu_head rcu; -+}; -+ -+struct ovpn_crypto_state { -+ struct ovpn_crypto_key_slot __rcu *primary; -+ struct ovpn_crypto_key_slot __rcu *secondary; -+ -+ /* protects primary and secondary slots */ -+ struct mutex mutex; -+}; -+ -+static inline bool ovpn_crypto_key_slot_hold(struct ovpn_crypto_key_slot *ks) -+{ -+ return kref_get_unless_zero(&ks->refcount); -+} -+ -+static inline void ovpn_crypto_state_init(struct ovpn_crypto_state *cs) -+{ -+ RCU_INIT_POINTER(cs->primary, NULL); -+ RCU_INIT_POINTER(cs->secondary, NULL); -+ mutex_init(&cs->mutex); -+} -+ -+static inline struct ovpn_crypto_key_slot * -+ovpn_crypto_key_id_to_slot(const struct ovpn_crypto_state *cs, u8 key_id) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ if (unlikely(!cs)) -+ return NULL; -+ -+ rcu_read_lock(); -+ ks = rcu_dereference(cs->primary); -+ if (ks && ks->key_id == key_id) { -+ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ goto out; -+ } -+ -+ ks = rcu_dereference(cs->secondary); -+ if (ks && ks->key_id == key_id) { -+ if (unlikely(!ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ goto out; -+ } -+ -+ /* when both key slots are occupied but no matching key ID is found, ks has to be reset to -+ * NULL to avoid carrying a stale pointer -+ */ -+ ks = NULL; -+out: -+ rcu_read_unlock(); -+ -+ return ks; -+} -+ -+static inline struct ovpn_crypto_key_slot * -+ovpn_crypto_key_slot_primary(const struct ovpn_crypto_state *cs) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ -+ rcu_read_lock(); -+ ks = rcu_dereference(cs->primary); -+ if (unlikely(ks && !ovpn_crypto_key_slot_hold(ks))) -+ ks = NULL; -+ rcu_read_unlock(); -+ -+ return ks; -+} -+ -+void ovpn_crypto_key_slot_release(struct kref *kref); -+ -+static inline void ovpn_crypto_key_slot_put(struct ovpn_crypto_key_slot *ks) -+{ -+ kref_put(&ks->refcount, ovpn_crypto_key_slot_release); -+} -+ -+int ovpn_crypto_state_reset(struct ovpn_crypto_state *cs, -+ const struct ovpn_peer_key_reset *pkr); -+ -+void ovpn_crypto_key_slot_delete(struct ovpn_crypto_state *cs, -+ enum ovpn_key_slot slot); -+ -+void ovpn_crypto_state_release(struct ovpn_crypto_state *cs); -+ -+void ovpn_crypto_key_slots_swap(struct ovpn_crypto_state *cs); -+ -+void ovpn_crypto_kill_primary(struct ovpn_crypto_state *cs); -+ -+#endif /* _NET_OVPN_DCO_OVPNCRYPTO_H_ */ -diff --git a/drivers/net/ovpn-dco/crypto_aead.c b/drivers/net/ovpn-dco/crypto_aead.c -new file mode 100644 -index 000000000000..c21bff90d748 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto_aead.c -@@ -0,0 +1,367 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "crypto_aead.h" -+#include "crypto.h" -+#include "pktid.h" -+#include "proto.h" -+#include "skb.h" -+ -+#include -+#include -+#include -+ -+#define AUTH_TAG_SIZE 16 -+ -+static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) -+{ -+ return OVPN_OP_SIZE_V2 + /* OP header size */ -+ 4 + /* Packet ID */ -+ crypto_aead_authsize(ks->encrypt); /* Auth Tag */ -+} -+ -+int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id) -+{ -+ const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); -+ const unsigned int head_size = ovpn_aead_encap_overhead(ks); -+ struct scatterlist sg[MAX_SKB_FRAGS + 2]; -+ DECLARE_CRYPTO_WAIT(wait); -+ struct aead_request *req; -+ struct sk_buff *trailer; -+ u8 iv[NONCE_SIZE]; -+ int nfrags, ret; -+ u32 pktid, op; -+ -+ /* Sample AEAD header format: -+ * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... -+ * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] -+ * [4-byte -+ * IV head] -+ */ -+ -+ /* check that there's enough headroom in the skb for packet -+ * encapsulation, after adding network header and encryption overhead -+ */ -+ if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM + head_size))) -+ return -ENOBUFS; -+ -+ /* get number of skb frags and ensure that packet data is writable */ -+ nfrags = skb_cow_data(skb, 0, &trailer); -+ if (unlikely(nfrags < 0)) -+ return nfrags; -+ -+ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) -+ return -ENOSPC; -+ -+ req = aead_request_alloc(ks->encrypt, GFP_KERNEL); -+ if (unlikely(!req)) -+ return -ENOMEM; -+ -+ /* sg table: -+ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), -+ * 1, 2, 3, ..., n: payload, -+ * n+1: auth_tag (len=tag_size) -+ */ -+ sg_init_table(sg, nfrags + 2); -+ -+ /* build scatterlist to encrypt packet payload */ -+ ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); -+ if (unlikely(nfrags != ret)) { -+ ret = -EINVAL; -+ goto free_req; -+ } -+ -+ /* append auth_tag onto scatterlist */ -+ __skb_push(skb, tag_size); -+ sg_set_buf(sg + nfrags + 1, skb->data, tag_size); -+ -+ /* obtain packet ID, which is used both as a first -+ * 4 bytes of nonce and last 4 bytes of associated data. -+ */ -+ ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); -+ if (unlikely(ret < 0)) -+ goto free_req; -+ -+ /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes nonce */ -+ ovpn_pktid_aead_write(pktid, &ks->nonce_tail_xmit, iv); -+ -+ /* make space for packet id and push it to the front */ -+ __skb_push(skb, NONCE_WIRE_SIZE); -+ memcpy(skb->data, iv, NONCE_WIRE_SIZE); -+ -+ /* add packet op as head of additional data */ -+ op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer_id); -+ __skb_push(skb, OVPN_OP_SIZE_V2); -+ BUILD_BUG_ON(sizeof(op) != OVPN_OP_SIZE_V2); -+ *((__force __be32 *)skb->data) = htonl(op); -+ -+ /* AEAD Additional data */ -+ sg_set_buf(sg, skb->data, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); -+ -+ /* setup async crypto operation */ -+ aead_request_set_tfm(req, ks->encrypt); -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP, -+ crypto_req_done, &wait); -+ aead_request_set_crypt(req, sg, sg, skb->len - head_size, iv); -+ aead_request_set_ad(req, OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE); -+ -+ /* encrypt it */ -+ ret = crypto_wait_req(crypto_aead_encrypt(req), &wait); -+ if (ret < 0) -+ net_err_ratelimited("%s: encrypt failed: %d\n", __func__, ret); -+ -+free_req: -+ aead_request_free(req); -+ return ret; -+} -+ -+int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb) -+{ -+ const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); -+ struct scatterlist sg[MAX_SKB_FRAGS + 2]; -+ int ret, payload_len, nfrags; -+ u8 *sg_data, iv[NONCE_SIZE]; -+ unsigned int payload_offset; -+ DECLARE_CRYPTO_WAIT(wait); -+ struct aead_request *req; -+ struct sk_buff *trailer; -+ unsigned int sg_len; -+ __be32 *pid; -+ -+ payload_offset = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE + tag_size; -+ payload_len = skb->len - payload_offset; -+ -+ /* sanity check on packet size, payload size must be >= 0 */ -+ if (unlikely(payload_len < 0)) -+ return -EINVAL; -+ -+ /* Prepare the skb data buffer to be accessed up until the auth tag. -+ * This is required because this area is directly mapped into the sg list. -+ */ -+ if (unlikely(!pskb_may_pull(skb, payload_offset))) -+ return -ENODATA; -+ -+ /* get number of skb frags and ensure that packet data is writable */ -+ nfrags = skb_cow_data(skb, 0, &trailer); -+ if (unlikely(nfrags < 0)) -+ return nfrags; -+ -+ if (unlikely(nfrags + 2 > ARRAY_SIZE(sg))) -+ return -ENOSPC; -+ -+ req = aead_request_alloc(ks->decrypt, GFP_KERNEL); -+ if (unlikely(!req)) -+ return -ENOMEM; -+ -+ /* sg table: -+ * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+NONCE_WIRE_SIZE), -+ * 1, 2, 3, ..., n: payload, -+ * n+1: auth_tag (len=tag_size) -+ */ -+ sg_init_table(sg, nfrags + 2); -+ -+ /* packet op is head of additional data */ -+ sg_data = skb->data; -+ sg_len = OVPN_OP_SIZE_V2 + NONCE_WIRE_SIZE; -+ sg_set_buf(sg, sg_data, sg_len); -+ -+ /* build scatterlist to decrypt packet payload */ -+ ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); -+ if (unlikely(nfrags != ret)) { -+ ret = -EINVAL; -+ goto free_req; -+ } -+ -+ /* append auth_tag onto scatterlist */ -+ sg_set_buf(sg + nfrags + 1, skb->data + sg_len, tag_size); -+ -+ /* copy nonce into IV buffer */ -+ memcpy(iv, skb->data + OVPN_OP_SIZE_V2, NONCE_WIRE_SIZE); -+ memcpy(iv + NONCE_WIRE_SIZE, ks->nonce_tail_recv.u8, -+ sizeof(struct ovpn_nonce_tail)); -+ -+ /* setup async crypto operation */ -+ aead_request_set_tfm(req, ks->decrypt); -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP, -+ crypto_req_done, &wait); -+ aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); -+ -+ aead_request_set_ad(req, NONCE_WIRE_SIZE + OVPN_OP_SIZE_V2); -+ -+ /* decrypt it */ -+ ret = crypto_wait_req(crypto_aead_decrypt(req), &wait); -+ if (ret < 0) { -+ net_err_ratelimited("%s: decrypt failed: %d\n", __func__, ret); -+ goto free_req; -+ } -+ -+ /* PID sits after the op */ -+ pid = (__force __be32 *)(skb->data + OVPN_OP_SIZE_V2); -+ ret = ovpn_pktid_recv(&ks->pid_recv, ntohl(*pid), 0); -+ if (unlikely(ret < 0)) -+ goto free_req; -+ -+ /* point to encapsulated IP packet */ -+ __skb_pull(skb, payload_offset); -+ -+free_req: -+ aead_request_free(req); -+ return ret; -+} -+ -+/* Initialize a struct crypto_aead object */ -+struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, -+ const unsigned char *key, unsigned int keylen) -+{ -+ struct crypto_aead *aead; -+ int ret; -+ -+ aead = crypto_alloc_aead(alg_name, 0, 0); -+ if (IS_ERR(aead)) { -+ ret = PTR_ERR(aead); -+ pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); -+ aead = NULL; -+ goto error; -+ } -+ -+ ret = crypto_aead_setkey(aead, key, keylen); -+ if (ret) { -+ pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, keylen, ret); -+ goto error; -+ } -+ -+ ret = crypto_aead_setauthsize(aead, AUTH_TAG_SIZE); -+ if (ret) { -+ pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, ret); -+ goto error; -+ } -+ -+ /* basic AEAD assumption */ -+ if (crypto_aead_ivsize(aead) != NONCE_SIZE) { -+ pr_err("%s IV size must be %d\n", title, NONCE_SIZE); -+ ret = -EINVAL; -+ goto error; -+ } -+ -+ pr_debug("********* Cipher %s (%s)\n", alg_name, title); -+ pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); -+ pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); -+ pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); -+ pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); -+ pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); -+ -+ return aead; -+ -+error: -+ crypto_free_aead(aead); -+ return ERR_PTR(ret); -+} -+ -+void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks) -+{ -+ if (!ks) -+ return; -+ -+ crypto_free_aead(ks->encrypt); -+ crypto_free_aead(ks->decrypt); -+ kfree(ks); -+} -+ -+static struct ovpn_crypto_key_slot * -+ovpn_aead_crypto_key_slot_init(enum ovpn_cipher_alg alg, -+ const unsigned char *encrypt_key, -+ unsigned int encrypt_keylen, -+ const unsigned char *decrypt_key, -+ unsigned int decrypt_keylen, -+ const unsigned char *encrypt_nonce_tail, -+ unsigned int encrypt_nonce_tail_len, -+ const unsigned char *decrypt_nonce_tail, -+ unsigned int decrypt_nonce_tail_len, -+ u16 key_id) -+{ -+ struct ovpn_crypto_key_slot *ks = NULL; -+ const char *alg_name; -+ int ret; -+ -+ /* validate crypto alg */ -+ switch (alg) { -+ case OVPN_CIPHER_ALG_AES_GCM: -+ alg_name = "gcm(aes)"; -+ break; -+ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: -+ alg_name = "rfc7539(chacha20,poly1305)"; -+ break; -+ default: -+ return ERR_PTR(-EOPNOTSUPP); -+ } -+ -+ /* build the key slot */ -+ ks = kmalloc(sizeof(*ks), GFP_KERNEL); -+ if (!ks) -+ return ERR_PTR(-ENOMEM); -+ -+ ks->encrypt = NULL; -+ ks->decrypt = NULL; -+ kref_init(&ks->refcount); -+ ks->key_id = key_id; -+ -+ ks->encrypt = ovpn_aead_init("encrypt", alg_name, encrypt_key, -+ encrypt_keylen); -+ if (IS_ERR(ks->encrypt)) { -+ ret = PTR_ERR(ks->encrypt); -+ ks->encrypt = NULL; -+ goto destroy_ks; -+ } -+ -+ ks->decrypt = ovpn_aead_init("decrypt", alg_name, decrypt_key, -+ decrypt_keylen); -+ if (IS_ERR(ks->decrypt)) { -+ ret = PTR_ERR(ks->decrypt); -+ ks->decrypt = NULL; -+ goto destroy_ks; -+ } -+ -+ if (sizeof(struct ovpn_nonce_tail) != encrypt_nonce_tail_len || -+ sizeof(struct ovpn_nonce_tail) != decrypt_nonce_tail_len) { -+ ret = -EINVAL; -+ goto destroy_ks; -+ } -+ -+ memcpy(ks->nonce_tail_xmit.u8, encrypt_nonce_tail, -+ sizeof(struct ovpn_nonce_tail)); -+ memcpy(ks->nonce_tail_recv.u8, decrypt_nonce_tail, -+ sizeof(struct ovpn_nonce_tail)); -+ -+ /* init packet ID generation/validation */ -+ ovpn_pktid_xmit_init(&ks->pid_xmit); -+ ovpn_pktid_recv_init(&ks->pid_recv); -+ -+ return ks; -+ -+destroy_ks: -+ ovpn_aead_crypto_key_slot_destroy(ks); -+ return ERR_PTR(ret); -+} -+ -+struct ovpn_crypto_key_slot * -+ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) -+{ -+ return ovpn_aead_crypto_key_slot_init(kc->cipher_alg, -+ kc->encrypt.cipher_key, -+ kc->encrypt.cipher_key_size, -+ kc->decrypt.cipher_key, -+ kc->decrypt.cipher_key_size, -+ kc->encrypt.nonce_tail, -+ kc->encrypt.nonce_tail_size, -+ kc->decrypt.nonce_tail, -+ kc->decrypt.nonce_tail_size, -+ kc->key_id); -+} -diff --git a/drivers/net/ovpn-dco/crypto_aead.h b/drivers/net/ovpn-dco/crypto_aead.h -new file mode 100644 -index 000000000000..1e3054e7d5a4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/crypto_aead.h -@@ -0,0 +1,27 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNAEAD_H_ -+#define _NET_OVPN_DCO_OVPNAEAD_H_ -+ -+#include "crypto.h" -+ -+#include -+#include -+ -+struct crypto_aead *ovpn_aead_init(const char *title, const char *alg_name, -+ const unsigned char *key, unsigned int keylen); -+ -+int ovpn_aead_encrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, u32 peer_id); -+int ovpn_aead_decrypt(struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); -+ -+struct ovpn_crypto_key_slot *ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc); -+void ovpn_aead_crypto_key_slot_destroy(struct ovpn_crypto_key_slot *ks); -+ -+#endif /* _NET_OVPN_DCO_OVPNAEAD_H_ */ -diff --git a/drivers/net/ovpn-dco/main.c b/drivers/net/ovpn-dco/main.c -new file mode 100644 -index 000000000000..4eb90ea7a500 ---- /dev/null -+++ b/drivers/net/ovpn-dco/main.c -@@ -0,0 +1,271 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#include "main.h" -+ -+#include "ovpn.h" -+#include "ovpnstruct.h" -+#include "netlink.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+/* Driver info */ -+#define DRV_NAME "ovpn-dco" -+#define DRV_VERSION OVPN_DCO_VERSION -+#define DRV_DESCRIPTION "OpenVPN data channel offload (ovpn-dco)" -+#define DRV_COPYRIGHT "(C) 2020-2022 OpenVPN, Inc." -+ -+static void ovpn_struct_free(struct net_device *net) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(net); -+ -+ security_tun_dev_free_security(ovpn->security); -+ free_percpu(net->tstats); -+ flush_workqueue(ovpn->crypto_wq); -+ flush_workqueue(ovpn->events_wq); -+ destroy_workqueue(ovpn->crypto_wq); -+ destroy_workqueue(ovpn->events_wq); -+ rcu_barrier(); -+} -+ -+/* Net device open */ -+static int ovpn_net_open(struct net_device *dev) -+{ -+ struct in_device *dev_v4 = __in_dev_get_rtnl(dev); -+ -+ if (dev_v4) { -+ /* disable redirects as Linux gets confused by ovpn-dco handling same-LAN routing */ -+ IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); -+ IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; -+ } -+ -+ netif_tx_start_all_queues(dev); -+ return 0; -+} -+ -+/* Net device stop -- called prior to device unload */ -+static int ovpn_net_stop(struct net_device *dev) -+{ -+ netif_tx_stop_all_queues(dev); -+ return 0; -+} -+ -+/******************************************* -+ * ovpn ethtool ops -+ *******************************************/ -+ -+static int ovpn_get_link_ksettings(struct net_device *dev, -+ struct ethtool_link_ksettings *cmd) -+{ -+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 0); -+ ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, 0); -+ cmd->base.speed = SPEED_1000; -+ cmd->base.duplex = DUPLEX_FULL; -+ cmd->base.port = PORT_TP; -+ cmd->base.phy_address = 0; -+ cmd->base.transceiver = XCVR_INTERNAL; -+ cmd->base.autoneg = AUTONEG_DISABLE; -+ -+ return 0; -+} -+ -+static void ovpn_get_drvinfo(struct net_device *dev, -+ struct ethtool_drvinfo *info) -+{ -+ strscpy(info->driver, DRV_NAME, sizeof(info->driver)); -+ strscpy(info->version, DRV_VERSION, sizeof(info->version)); -+ strscpy(info->bus_info, "ovpn", sizeof(info->bus_info)); -+} -+ -+bool ovpn_dev_is_valid(const struct net_device *dev) -+{ -+ return dev->netdev_ops->ndo_start_xmit == ovpn_net_xmit; -+} -+ -+/******************************************* -+ * ovpn exported methods -+ *******************************************/ -+ -+static const struct net_device_ops ovpn_netdev_ops = { -+ .ndo_open = ovpn_net_open, -+ .ndo_stop = ovpn_net_stop, -+ .ndo_start_xmit = ovpn_net_xmit, -+ .ndo_get_stats64 = dev_get_tstats64, -+}; -+ -+static const struct ethtool_ops ovpn_ethtool_ops = { -+ .get_link_ksettings = ovpn_get_link_ksettings, -+ .get_drvinfo = ovpn_get_drvinfo, -+ .get_link = ethtool_op_get_link, -+ .get_ts_info = ethtool_op_get_ts_info, -+}; -+ -+static void ovpn_setup(struct net_device *dev) -+{ -+ /* compute the overhead considering AEAD encryption */ -+ const int overhead = sizeof(u32) + NONCE_WIRE_SIZE + 16 + sizeof(struct udphdr) + -+ max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); -+ -+ netdev_features_t feat = NETIF_F_SG | NETIF_F_LLTX | -+ NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_GSO | -+ NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA; -+ -+ dev->ethtool_ops = &ovpn_ethtool_ops; -+ dev->needs_free_netdev = true; -+ -+ dev->netdev_ops = &ovpn_netdev_ops; -+ -+ dev->priv_destructor = ovpn_struct_free; -+ -+ /* Point-to-Point TUN Device */ -+ dev->hard_header_len = 0; -+ dev->addr_len = 0; -+ dev->mtu = ETH_DATA_LEN - overhead; -+ dev->min_mtu = IPV4_MIN_MTU; -+ dev->max_mtu = IP_MAX_MTU - overhead; -+ -+ /* Zero header length */ -+ dev->type = ARPHRD_NONE; -+ dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; -+ -+ dev->features |= feat; -+ dev->hw_features |= feat; -+ dev->hw_enc_features |= feat; -+ -+ dev->needed_headroom = OVPN_HEAD_ROOM; -+ dev->needed_tailroom = OVPN_MAX_PADDING; -+} -+ -+static const struct nla_policy ovpn_policy[IFLA_OVPN_MAX + 1] = { -+ [IFLA_OVPN_MODE] = NLA_POLICY_RANGE(NLA_U8, __OVPN_MODE_FIRST, -+ __OVPN_MODE_AFTER_LAST - 1), -+}; -+ -+static int ovpn_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], -+ struct nlattr *data[], struct netlink_ext_ack *extack) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ int ret; -+ -+ ret = security_tun_dev_create(); -+ if (ret < 0) -+ return ret; -+ -+ ret = ovpn_struct_init(dev); -+ if (ret < 0) -+ return ret; -+ -+ ovpn->mode = OVPN_MODE_P2P; -+ if (data && data[IFLA_OVPN_MODE]) { -+ ovpn->mode = nla_get_u8(data[IFLA_OVPN_MODE]); -+ netdev_dbg(dev, "%s: setting device (%s) mode: %u\n", __func__, dev->name, -+ ovpn->mode); -+ } -+ -+ return register_netdevice(dev); -+} -+ -+static void ovpn_dellink(struct net_device *dev, struct list_head *head) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ -+ switch (ovpn->mode) { -+ case OVPN_MODE_P2P: -+ ovpn_peer_release_p2p(ovpn); -+ break; -+ default: -+ ovpn_peers_free(ovpn); -+ break; -+ } -+ -+ unregister_netdevice_queue(dev, head); -+} -+ -+/** -+ * ovpn_num_queues - define number of queues to allocate per device -+ * -+ * The value returned by this function is used to decide how many RX and TX -+ * queues to allocate when creating the netdev object -+ * -+ * Return the number of queues to allocate -+ */ -+static unsigned int ovpn_num_queues(void) -+{ -+ return num_online_cpus(); -+} -+ -+static struct rtnl_link_ops ovpn_link_ops __read_mostly = { -+ .kind = DRV_NAME, -+ .priv_size = sizeof(struct ovpn_struct), -+ .setup = ovpn_setup, -+ .policy = ovpn_policy, -+ .maxtype = IFLA_OVPN_MAX, -+ .newlink = ovpn_newlink, -+ .dellink = ovpn_dellink, -+ .get_num_tx_queues = ovpn_num_queues, -+ .get_num_rx_queues = ovpn_num_queues, -+}; -+ -+static int __init ovpn_init(void) -+{ -+ int err = 0; -+ -+ pr_info("%s %s -- %s\n", DRV_DESCRIPTION, DRV_VERSION, DRV_COPYRIGHT); -+ -+ /* init RTNL link ops */ -+ err = rtnl_link_register(&ovpn_link_ops); -+ if (err) { -+ pr_err("ovpn: can't register RTNL link ops\n"); -+ goto err; -+ } -+ -+ err = ovpn_netlink_register(); -+ if (err) { -+ pr_err("ovpn: can't register netlink family\n"); -+ goto err_rtnl_unregister; -+ } -+ -+ return 0; -+ -+err_rtnl_unregister: -+ rtnl_link_unregister(&ovpn_link_ops); -+err: -+ pr_err("ovpn: initialization failed, error status=%d\n", err); -+ return err; -+} -+ -+static __exit void ovpn_cleanup(void) -+{ -+ rtnl_link_unregister(&ovpn_link_ops); -+ ovpn_netlink_unregister(); -+ rcu_barrier(); /* because we use call_rcu */ -+} -+ -+module_init(ovpn_init); -+module_exit(ovpn_cleanup); -+ -+MODULE_DESCRIPTION(DRV_DESCRIPTION); -+MODULE_AUTHOR(DRV_COPYRIGHT); -+MODULE_LICENSE("GPL"); -+MODULE_VERSION(DRV_VERSION); -+MODULE_ALIAS_RTNL_LINK(DRV_NAME); -+MODULE_ALIAS_GENL_FAMILY(OVPN_NL_NAME); -diff --git a/drivers/net/ovpn-dco/main.h b/drivers/net/ovpn-dco/main.h -new file mode 100644 -index 000000000000..c4ef200b30f4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/main.h -@@ -0,0 +1,32 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_MAIN_H_ -+#define _NET_OVPN_DCO_MAIN_H_ -+ -+#include -+#include -+#include -+#include -+ -+#define OVPN_DCO_VERSION "2.0.0" -+ -+struct net_device; -+bool ovpn_dev_is_valid(const struct net_device *dev); -+ -+#define SKB_HEADER_LEN \ -+ (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ -+ sizeof(struct udphdr) + NET_SKB_PAD) -+ -+#define OVPN_HEAD_ROOM ALIGN(16 + SKB_HEADER_LEN, 4) -+#define OVPN_MAX_PADDING 16 -+#define OVPN_QUEUE_LEN 1024 -+#define OVPN_MAX_TUN_QUEUE_LEN 0x10000 -+ -+#endif /* _NET_OVPN_DCO_OVPN_DCO_H_ */ -diff --git a/drivers/net/ovpn-dco/netlink.c b/drivers/net/ovpn-dco/netlink.c -new file mode 100644 -index 000000000000..ee5c943e7db4 ---- /dev/null -+++ b/drivers/net/ovpn-dco/netlink.c -@@ -0,0 +1,1143 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "proto.h" -+#include "netlink.h" -+#include "ovpnstruct.h" -+#include "udp.h" -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/** The ovpn-dco netlink family */ -+static struct genl_family ovpn_netlink_family; -+ -+enum ovpn_netlink_multicast_groups { -+ OVPN_MCGRP_PEERS, -+}; -+ -+static const struct genl_multicast_group ovpn_netlink_mcgrps[] = { -+ [OVPN_MCGRP_PEERS] = { .name = OVPN_NL_MULTICAST_GROUP_PEERS }, -+}; -+ -+/** Key direction policy. Can be used for configuring an encryption and a decryption key */ -+static const struct nla_policy ovpn_netlink_policy_key_dir[OVPN_KEY_DIR_ATTR_MAX + 1] = { -+ [OVPN_KEY_DIR_ATTR_CIPHER_KEY] = NLA_POLICY_MAX_LEN(U8_MAX), -+ [OVPN_KEY_DIR_ATTR_NONCE_TAIL] = NLA_POLICY_EXACT_LEN(NONCE_TAIL_SIZE), -+}; -+ -+/** CMD_NEW_KEY policy */ -+static const struct nla_policy ovpn_netlink_policy_new_key[OVPN_NEW_KEY_ATTR_MAX + 1] = { -+ [OVPN_NEW_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_NEW_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, -+ __OVPN_KEY_SLOT_AFTER_LAST - 1), -+ [OVPN_NEW_KEY_ATTR_KEY_ID] = { .type = NLA_U8 }, -+ [OVPN_NEW_KEY_ATTR_CIPHER_ALG] = { .type = NLA_U16 }, -+ [OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), -+ [OVPN_NEW_KEY_ATTR_DECRYPT_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_key_dir), -+}; -+ -+/** CMD_DEL_KEY policy */ -+static const struct nla_policy ovpn_netlink_policy_del_key[OVPN_DEL_KEY_ATTR_MAX + 1] = { -+ [OVPN_DEL_KEY_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_DEL_KEY_ATTR_KEY_SLOT] = NLA_POLICY_RANGE(NLA_U8, __OVPN_KEY_SLOT_FIRST, -+ __OVPN_KEY_SLOT_AFTER_LAST - 1), -+}; -+ -+/** CMD_SWAP_KEYS policy */ -+static const struct nla_policy ovpn_netlink_policy_swap_keys[OVPN_SWAP_KEYS_ATTR_MAX + 1] = { -+ [OVPN_SWAP_KEYS_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_NEW_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_new_peer[OVPN_NEW_PEER_ATTR_MAX + 1] = { -+ [OVPN_NEW_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE] = NLA_POLICY_MIN_LEN(sizeof(struct sockaddr)), -+ [OVPN_NEW_PEER_ATTR_SOCKET] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_IPV4] = { .type = NLA_U32 }, -+ [OVPN_NEW_PEER_ATTR_IPV6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), -+ [OVPN_NEW_PEER_ATTR_LOCAL_IP] = NLA_POLICY_MAX_LEN(sizeof(struct in6_addr)), -+}; -+ -+/** CMD_SET_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_set_peer[OVPN_SET_PEER_ATTR_MAX + 1] = { -+ [OVPN_SET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, -+ [OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_DEL_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_del_peer[OVPN_DEL_PEER_ATTR_MAX + 1] = { -+ [OVPN_DEL_PEER_ATTR_REASON] = NLA_POLICY_RANGE(NLA_U8, __OVPN_DEL_PEER_REASON_FIRST, -+ __OVPN_DEL_PEER_REASON_AFTER_LAST - 1), -+ [OVPN_DEL_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_GET_PEER policy */ -+static const struct nla_policy ovpn_netlink_policy_get_peer[OVPN_GET_PEER_ATTR_MAX + 1] = { -+ [OVPN_GET_PEER_ATTR_PEER_ID] = { .type = NLA_U32 }, -+}; -+ -+/** CMD_PACKET polocy */ -+static const struct nla_policy ovpn_netlink_policy_packet[OVPN_PACKET_ATTR_MAX + 1] = { -+ [OVPN_PACKET_ATTR_PEER_ID] = { .type = NLA_U32 }, -+ [OVPN_PACKET_ATTR_PACKET] = NLA_POLICY_MAX_LEN(U16_MAX), -+}; -+ -+/** Generic message container policy */ -+static const struct nla_policy ovpn_netlink_policy[OVPN_ATTR_MAX + 1] = { -+ [OVPN_ATTR_IFINDEX] = { .type = NLA_U32 }, -+ [OVPN_ATTR_NEW_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_peer), -+ [OVPN_ATTR_SET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_set_peer), -+ [OVPN_ATTR_DEL_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_peer), -+ [OVPN_ATTR_GET_PEER] = NLA_POLICY_NESTED(ovpn_netlink_policy_get_peer), -+ [OVPN_ATTR_NEW_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_new_key), -+ [OVPN_ATTR_SWAP_KEYS] = NLA_POLICY_NESTED(ovpn_netlink_policy_swap_keys), -+ [OVPN_ATTR_DEL_KEY] = NLA_POLICY_NESTED(ovpn_netlink_policy_del_key), -+ [OVPN_ATTR_PACKET] = NLA_POLICY_NESTED(ovpn_netlink_policy_packet), -+}; -+ -+static struct net_device * -+ovpn_get_dev_from_attrs(struct net *net, struct nlattr **attrs) -+{ -+ struct net_device *dev; -+ int ifindex; -+ -+ if (!attrs[OVPN_ATTR_IFINDEX]) -+ return ERR_PTR(-EINVAL); -+ -+ ifindex = nla_get_u32(attrs[OVPN_ATTR_IFINDEX]); -+ -+ dev = dev_get_by_index(net, ifindex); -+ if (!dev) -+ return ERR_PTR(-ENODEV); -+ -+ if (!ovpn_dev_is_valid(dev)) -+ goto err_put_dev; -+ -+ return dev; -+ -+err_put_dev: -+ dev_put(dev); -+ -+ return ERR_PTR(-EINVAL); -+} -+ -+/** -+ * ovpn_pre_doit() - Prepare ovpn genl doit request -+ * @ops: requested netlink operation -+ * @skb: Netlink message with request data -+ * @info: receiver information -+ * -+ * Return: 0 on success or negative error number in case of failure -+ */ -+static int ovpn_pre_doit(const struct genl_ops *ops, struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct net *net = genl_info_net(info); -+ struct net_device *dev; -+ -+ dev = ovpn_get_dev_from_attrs(net, info->attrs); -+ if (IS_ERR(dev)) -+ return PTR_ERR(dev); -+ -+ info->user_ptr[0] = netdev_priv(dev); -+ -+ return 0; -+} -+ -+/** -+ * ovpn_post_doit() - complete ovpn genl doit request -+ * @ops: requested netlink operation -+ * @skb: Netlink message with request data -+ * @info: receiver information -+ */ -+static void ovpn_post_doit(const struct genl_ops *ops, struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct ovpn_struct *ovpn; -+ -+ ovpn = info->user_ptr[0]; -+ dev_put(ovpn->dev); -+} -+ -+static int ovpn_netlink_get_key_dir(struct genl_info *info, struct nlattr *key, -+ enum ovpn_cipher_alg cipher, -+ struct ovpn_key_direction *dir) -+{ -+ struct nlattr *attr, *attrs[OVPN_KEY_DIR_ATTR_MAX + 1]; -+ int ret; -+ -+ ret = nla_parse_nested(attrs, OVPN_KEY_DIR_ATTR_MAX, key, NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ switch (cipher) { -+ case OVPN_CIPHER_ALG_AES_GCM: -+ case OVPN_CIPHER_ALG_CHACHA20_POLY1305: -+ attr = attrs[OVPN_KEY_DIR_ATTR_CIPHER_KEY]; -+ if (!attr) -+ return -EINVAL; -+ -+ dir->cipher_key = nla_data(attr); -+ dir->cipher_key_size = nla_len(attr); -+ -+ attr = attrs[OVPN_KEY_DIR_ATTR_NONCE_TAIL]; -+ /* These algorithms require a 96bit nonce, -+ * Construct it by combining 4-bytes packet id and -+ * 8-bytes nonce-tail from userspace -+ */ -+ if (!attr) -+ return -EINVAL; -+ -+ dir->nonce_tail = nla_data(attr); -+ dir->nonce_tail_size = nla_len(attr); -+ break; -+ default: -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+static int ovpn_netlink_new_key(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_NEW_KEY_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer_key_reset pkr; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_NEW_KEY]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_NEW_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_KEY], -+ NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_NEW_KEY_ATTR_PEER_ID] || -+ !attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT] || -+ !attrs[OVPN_NEW_KEY_ATTR_KEY_ID] || -+ !attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG] || -+ !attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY] || -+ !attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_NEW_KEY_ATTR_PEER_ID]); -+ pkr.slot = nla_get_u8(attrs[OVPN_NEW_KEY_ATTR_KEY_SLOT]); -+ pkr.key.key_id = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_KEY_ID]); -+ -+ pkr.key.cipher_alg = nla_get_u16(attrs[OVPN_NEW_KEY_ATTR_CIPHER_ALG]); -+ -+ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_ENCRYPT_KEY], -+ pkr.key.cipher_alg, &pkr.key.encrypt); -+ if (ret < 0) -+ return ret; -+ -+ ret = ovpn_netlink_get_key_dir(info, attrs[OVPN_NEW_KEY_ATTR_DECRYPT_KEY], -+ pkr.key.cipher_alg, &pkr.key.decrypt); -+ if (ret < 0) -+ return ret; -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) { -+ netdev_dbg(ovpn->dev, "%s: no peer with id %u to set key for\n", __func__, peer_id); -+ return -ENOENT; -+ } -+ -+ mutex_lock(&peer->crypto.mutex); -+ ret = ovpn_crypto_state_reset(&peer->crypto, &pkr); -+ if (ret < 0) { -+ netdev_dbg(ovpn->dev, "%s: cannot install new key for peer %u\n", __func__, -+ peer_id); -+ goto unlock; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: new key installed (id=%u) for peer %u\n", __func__, -+ pkr.key.key_id, peer_id); -+unlock: -+ mutex_unlock(&peer->crypto.mutex); -+ ovpn_peer_put(peer); -+ return ret; -+} -+ -+static int ovpn_netlink_del_key(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_DEL_KEY_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ enum ovpn_key_slot slot; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_DEL_KEY]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_DEL_KEY_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_KEY], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_DEL_KEY_ATTR_PEER_ID] || !attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_DEL_KEY_ATTR_PEER_ID]); -+ slot = nla_get_u8(attrs[OVPN_DEL_KEY_ATTR_KEY_SLOT]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ ovpn_crypto_key_slot_delete(&peer->crypto, slot); -+ ovpn_peer_put(peer); -+ -+ return 0; -+} -+ -+static int ovpn_netlink_swap_keys(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SWAP_KEYS_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_SWAP_KEYS]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_SWAP_KEYS_ATTR_MAX, info->attrs[OVPN_ATTR_SWAP_KEYS], -+ NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_SWAP_KEYS_ATTR_PEER_ID]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ ovpn_crypto_key_slots_swap(&peer->crypto); -+ ovpn_peer_put(peer); -+ -+ return 0; -+} -+ -+static int ovpn_netlink_new_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_NEW_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct sockaddr_storage *ss = NULL; -+ struct sockaddr_in mapped; -+ struct sockaddr_in6 *in6; -+ struct ovpn_peer *peer; -+ size_t sa_len, ip_len; -+ struct socket *sock; -+ u8 *local_ip = NULL; -+ u32 sockfd, id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_NEW_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_NEW_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_NEW_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_NEW_PEER_ATTR_PEER_ID] || !attrs[OVPN_NEW_PEER_ATTR_SOCKET] || -+ (!attrs[OVPN_NEW_PEER_ATTR_IPV4] && !attrs[OVPN_NEW_PEER_ATTR_IPV6])) { -+ netdev_dbg(ovpn->dev, "%s: basic attributes missing\n", __func__); -+ return -EINVAL; -+ } -+ -+ /* lookup the fd in the kernel table and extract the socket object */ -+ sockfd = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_SOCKET]); -+ /* sockfd_lookup() increases sock's refcounter */ -+ sock = sockfd_lookup(sockfd, &ret); -+ if (!sock) { -+ netdev_dbg(ovpn->dev, "%s: cannot lookup peer socket (fd=%u): %d\n", __func__, -+ sockfd, ret); -+ return -ENOTSOCK; -+ } -+ -+ /* Only when using UDP as transport protocol the remote endpoint must be configured -+ * so that ovpn-dco knows where to send packets to. -+ * -+ * In case of TCP, the socket is connected to the peer and ovpn-dco will just send bytes -+ * over it, without the need to specify a destination. -+ */ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) { -+ ret = -EINVAL; -+ -+ if (!attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]) { -+ netdev_err(ovpn->dev, "%s: cannot add UDP peer with no remote endpoint\n", -+ __func__); -+ goto sockfd_release; -+ } -+ -+ ss = nla_data(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); -+ sa_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE]); -+ switch (sa_len) { -+ case sizeof(struct sockaddr_in): -+ if (ss->ss_family == AF_INET) -+ /* valid sockaddr */ -+ break; -+ -+ netdev_err(ovpn->dev, "%s: remote sockaddr_in has invalid family\n", -+ __func__); -+ goto sockfd_release; -+ case sizeof(struct sockaddr_in6): -+ if (ss->ss_family == AF_INET6) -+ /* valid sockaddr */ -+ break; -+ -+ netdev_err(ovpn->dev, "%s: remote sockaddr_in6 has invalid family\n", -+ __func__); -+ goto sockfd_release; -+ default: -+ netdev_err(ovpn->dev, "%s: invalid size for sockaddr\n", __func__); -+ goto sockfd_release; -+ } -+ -+ if (ss->ss_family == AF_INET6) { -+ in6 = (struct sockaddr_in6 *)ss; -+ -+ if (ipv6_addr_type(&in6->sin6_addr) & IPV6_ADDR_MAPPED) { -+ mapped.sin_family = AF_INET; -+ mapped.sin_addr.s_addr = in6->sin6_addr.s6_addr32[3]; -+ mapped.sin_port = in6->sin6_port; -+ ss = (struct sockaddr_storage *)&mapped; -+ } -+ } -+ -+ /* When using UDP we may be talking over socket bound to 0.0.0.0/::. -+ * In this case, if the host has multiple IPs, we need to make sure -+ * that outgoing traffic has as source IP the same address that the -+ * peer is using to reach us. -+ * -+ * Since early control packets were all forwarded to userspace, we -+ * need the latter to tell us what IP has to be used. -+ */ -+ if (attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]) { -+ ip_len = nla_len(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); -+ local_ip = nla_data(attrs[OVPN_NEW_PEER_ATTR_LOCAL_IP]); -+ -+ if (ip_len == sizeof(struct in_addr)) { -+ if (ss->ss_family != AF_INET) { -+ netdev_dbg(ovpn->dev, -+ "%s: the specified local IP is IPv4, but the peer endpoint is not\n", -+ __func__); -+ goto sockfd_release; -+ } -+ } else if (ip_len == sizeof(struct in6_addr)) { -+ bool is_mapped = ipv6_addr_type((struct in6_addr *)local_ip) & -+ IPV6_ADDR_MAPPED; -+ -+ if (ss->ss_family != AF_INET6 && !is_mapped) { -+ netdev_dbg(ovpn->dev, -+ "%s: the specified local IP is IPv6, but the peer endpoint is not\n", -+ __func__); -+ goto sockfd_release; -+ } -+ -+ if (is_mapped) -+ /* this is an IPv6-mapped IPv4 address, therefore extract -+ * the actual v4 address from the last 4 bytes -+ */ -+ local_ip += 12; -+ } else { -+ netdev_dbg(ovpn->dev, -+ "%s: invalid length %zu for local IP\n", __func__, -+ ip_len); -+ goto sockfd_release; -+ } -+ } -+ -+ /* sanity checks passed */ -+ ret = 0; -+ } -+ -+ id = nla_get_u32(attrs[OVPN_NEW_PEER_ATTR_PEER_ID]); -+ peer = ovpn_peer_new(ovpn, ss, sock, id, local_ip); -+ if (IS_ERR(peer)) { -+ netdev_err(ovpn->dev, "%s: cannot create new peer object for peer %u %pIScp\n", -+ __func__, id, ss); -+ ret = PTR_ERR(peer); -+ goto sockfd_release; -+ } -+ -+ if (attrs[OVPN_NEW_PEER_ATTR_IPV4]) { -+ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV4]) != sizeof(struct in_addr)) { -+ ret = -EINVAL; -+ goto peer_release; -+ } -+ -+ peer->vpn_addrs.ipv4.s_addr = nla_get_be32(attrs[OVPN_NEW_PEER_ATTR_IPV4]); -+ } -+ -+ if (attrs[OVPN_NEW_PEER_ATTR_IPV6]) { -+ if (nla_len(attrs[OVPN_NEW_PEER_ATTR_IPV6]) != sizeof(struct in6_addr)) { -+ ret = -EINVAL; -+ goto peer_release; -+ } -+ -+ memcpy(&peer->vpn_addrs.ipv6, nla_data(attrs[OVPN_NEW_PEER_ATTR_IPV6]), -+ sizeof(struct in6_addr)); -+ } -+ -+ netdev_dbg(ovpn->dev, -+ "%s: adding peer with endpoint=%pIScp/%s id=%u VPN-IPv4=%pI4 VPN-IPv6=%pI6c\n", -+ __func__, ss, sock->sk->sk_prot_creator->name, peer->id, -+ &peer->vpn_addrs.ipv4.s_addr, &peer->vpn_addrs.ipv6); -+ -+ ret = ovpn_peer_add(ovpn, peer); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot add new peer (id=%u) to hashtable: %d\n", -+ __func__, peer->id, ret); -+ goto peer_release; -+ } -+ -+ return 0; -+ -+peer_release: -+ /* release right away because peer is not really used in any context */ -+ ovpn_peer_release(peer); -+ return ret; -+ -+sockfd_release: -+ sockfd_put(sock); -+ return ret; -+} -+ -+static int ovpn_netlink_set_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ u32 peer_id, interv, timeout; -+ bool keepalive_set = false; -+ struct ovpn_peer *peer; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_SET_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_SET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_SET_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_SET_PEER_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_PEER_ID]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ /* when setting the keepalive, both parameters have to be configured */ -+ if (attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL] && -+ attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]) { -+ keepalive_set = true; -+ interv = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL]); -+ timeout = nla_get_u32(attrs[OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT]); -+ } -+ -+ if (keepalive_set) -+ ovpn_peer_keepalive_set(peer, interv, timeout); -+ -+ ovpn_peer_put(peer); -+ return 0; -+} -+ -+static int ovpn_netlink_send_peer(struct sk_buff *skb, const struct ovpn_peer *peer, u32 portid, -+ u32 seq, int flags) -+{ -+ const struct ovpn_bind *bind; -+ struct nlattr *attr; -+ void *hdr; -+ -+ hdr = genlmsg_put(skb, portid, seq, &ovpn_netlink_family, flags, OVPN_CMD_GET_PEER); -+ if (!hdr) { -+ netdev_dbg(peer->ovpn->dev, "%s: cannot create message header\n", __func__); -+ return -EMSGSIZE; -+ } -+ -+ attr = nla_nest_start(skb, OVPN_ATTR_GET_PEER); -+ if (!attr) { -+ netdev_dbg(peer->ovpn->dev, "%s: cannot create submessage\n", __func__); -+ goto err; -+ } -+ -+ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_PEER_ID, peer->id)) -+ goto err; -+ -+ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV4, sizeof(peer->vpn_addrs.ipv4), -+ &peer->vpn_addrs.ipv4)) -+ goto err; -+ -+ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_IPV6, sizeof(peer->vpn_addrs.ipv6), -+ &peer->vpn_addrs.ipv6)) -+ goto err; -+ -+ if (nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, -+ peer->keepalive_interval) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, -+ peer->keepalive_timeout)) -+ goto err; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (bind) { -+ if (bind->sa.in4.sin_family == AF_INET) { -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ sizeof(bind->sa.in4), &bind->sa.in4) || -+ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ sizeof(bind->local.ipv4), &bind->local.ipv4)) -+ goto err_unlock; -+ } else if (bind->sa.in4.sin_family == AF_INET6) { -+ if (nla_put(skb, OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ sizeof(bind->sa.in6), &bind->sa.in6) || -+ nla_put(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ sizeof(bind->local.ipv6), &bind->local.ipv6)) -+ goto err_unlock; -+ } -+ } -+ rcu_read_unlock(); -+ -+ if (nla_put_net16(skb, OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, -+ inet_sk(peer->sock->sock->sk)->inet_sport) || -+ /* RX stats */ -+ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_RX_BYTES, -+ atomic64_read(&peer->stats.rx.bytes), -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, -+ atomic_read(&peer->stats.rx.packets)) || -+ /* TX stats */ -+ nla_put_u64_64bit(skb, OVPN_GET_PEER_RESP_ATTR_TX_BYTES, -+ atomic64_read(&peer->stats.tx.bytes), -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC) || -+ nla_put_u32(skb, OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, -+ atomic_read(&peer->stats.tx.packets))) -+ goto err; -+ -+ nla_nest_end(skb, attr); -+ genlmsg_end(skb, hdr); -+ -+ return 0; -+err_unlock: -+ rcu_read_unlock(); -+err: -+ genlmsg_cancel(skb, hdr); -+ return -EMSGSIZE; -+} -+ -+static int ovpn_netlink_get_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ struct sk_buff *msg; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_GET_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_GET_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_GET_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_GET_PEER_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_GET_PEER_ATTR_PEER_ID]); -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); -+ if (!msg) -+ return -ENOMEM; -+ -+ ret = ovpn_netlink_send_peer(msg, peer, info->snd_portid, info->snd_seq, 0); -+ if (ret < 0) { -+ nlmsg_free(msg); -+ goto err; -+ } -+ -+ ret = genlmsg_reply(msg, info); -+err: -+ ovpn_peer_put(peer); -+ return ret; -+} -+ -+static int ovpn_netlink_dump_done(struct netlink_callback *cb) -+{ -+ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; -+ -+ dev_put(ovpn->dev); -+ return 0; -+} -+ -+static int ovpn_netlink_dump_prepare(struct netlink_callback *cb) -+{ -+ struct net *netns = sock_net(cb->skb->sk); -+ struct nlattr **attrbuf; -+ struct net_device *dev; -+ int ret; -+ -+ attrbuf = kcalloc(OVPN_ATTR_MAX + 1, sizeof(*attrbuf), GFP_KERNEL); -+ if (!attrbuf) -+ return -ENOMEM; -+ -+ ret = nlmsg_parse_deprecated(cb->nlh, GENL_HDRLEN, attrbuf, OVPN_ATTR_MAX, -+ ovpn_netlink_policy, NULL); -+ if (ret < 0) -+ goto err; -+ -+ dev = ovpn_get_dev_from_attrs(netns, attrbuf); -+ if (IS_ERR(dev)) { -+ ret = PTR_ERR(dev); -+ goto err; -+ } -+ -+ cb->args[0] = (long)netdev_priv(dev); -+ ret = 0; -+err: -+ kfree(attrbuf); -+ return ret; -+} -+ -+static int ovpn_netlink_dump_peers(struct sk_buff *skb, struct netlink_callback *cb) -+{ -+ struct ovpn_struct *ovpn = (struct ovpn_struct *)cb->args[0]; -+ int ret, bkt, last_idx = cb->args[1], dumped = 0; -+ struct ovpn_peer *peer; -+ -+ if (!ovpn) { -+ ret = ovpn_netlink_dump_prepare(cb); -+ if (ret < 0) { -+ netdev_dbg(ovpn->dev, "%s: cannot prepare for dump: %d\n", __func__, ret); -+ return ret; -+ } -+ -+ ovpn = (struct ovpn_struct *)cb->args[0]; -+ } -+ -+ rcu_read_lock(); -+ hash_for_each_rcu(ovpn->peers.by_id, bkt, peer, hash_entry_id) { -+ /* skip already dumped peers that were dumped by previous invocations */ -+ if (last_idx > 0) { -+ last_idx--; -+ continue; -+ } -+ -+ if (ovpn_netlink_send_peer(skb, peer, NETLINK_CB(cb->skb).portid, -+ cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0) -+ break; -+ -+ /* count peers being dumped during this invocation */ -+ dumped++; -+ } -+ rcu_read_unlock(); -+ -+ /* sum up peers dumped in this message, so that at the next invocation -+ * we can continue from where we left -+ */ -+ cb->args[1] += dumped; -+ -+ return skb->len; -+} -+ -+static int ovpn_netlink_del_peer(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_SET_PEER_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ struct ovpn_peer *peer; -+ u32 peer_id; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_DEL_PEER]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_DEL_PEER_ATTR_MAX, info->attrs[OVPN_ATTR_DEL_PEER], NULL, -+ info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_DEL_PEER_ATTR_PEER_ID]) -+ return -EINVAL; -+ -+ peer_id = nla_get_u32(attrs[OVPN_DEL_PEER_ATTR_PEER_ID]); -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) -+ return -ENOENT; -+ -+ netdev_dbg(ovpn->dev, "%s: peer id=%u\n", __func__, peer->id); -+ ret = ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_USERSPACE); -+ ovpn_peer_put(peer); -+ -+ return ret; -+} -+ -+static int ovpn_netlink_register_packet(struct sk_buff *skb, -+ struct genl_info *info) -+{ -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ -+ /* only one registered process per interface is allowed for now */ -+ if (ovpn->registered_nl_portid_set) { -+ netdev_dbg(ovpn->dev, "%s: userspace listener already registered\n", __func__); -+ return -EBUSY; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: registering userspace at %u\n", __func__, info->snd_portid); -+ -+ ovpn->registered_nl_portid = info->snd_portid; -+ ovpn->registered_nl_portid_set = true; -+ -+ return 0; -+} -+ -+static int ovpn_netlink_packet(struct sk_buff *skb, struct genl_info *info) -+{ -+ struct nlattr *attrs[OVPN_PACKET_ATTR_MAX + 1]; -+ struct ovpn_struct *ovpn = info->user_ptr[0]; -+ const u8 *packet; -+ u32 peer_id; -+ size_t len; -+ u8 opcode; -+ int ret; -+ -+ if (!info->attrs[OVPN_ATTR_PACKET]) -+ return -EINVAL; -+ -+ ret = nla_parse_nested(attrs, OVPN_PACKET_ATTR_MAX, info->attrs[OVPN_ATTR_PACKET], -+ NULL, info->extack); -+ if (ret) -+ return ret; -+ -+ if (!attrs[OVPN_PACKET_ATTR_PACKET] || !attrs[OVPN_PACKET_ATTR_PEER_ID]) { -+ netdev_dbg(ovpn->dev, "received netlink packet with no payload\n"); -+ return -EINVAL; -+ } -+ -+ peer_id = nla_get_u32(attrs[OVPN_PACKET_ATTR_PEER_ID]); -+ -+ len = nla_len(attrs[OVPN_PACKET_ATTR_PACKET]); -+ -+ if (len < 4 || len > ovpn->dev->mtu) { -+ netdev_dbg(ovpn->dev, "%s: invalid packet size %zu (min is 4, max is MTU: %u)\n", -+ __func__, len, ovpn->dev->mtu); -+ return -EINVAL; -+ } -+ -+ packet = nla_data(attrs[OVPN_PACKET_ATTR_PACKET]); -+ opcode = ovpn_opcode_from_byte(packet[0]); -+ -+ /* reject data packets from userspace as they could lead to IV reuse */ -+ if (opcode == OVPN_DATA_V1 || opcode == OVPN_DATA_V2) { -+ netdev_dbg(ovpn->dev, "%s: rejecting data packet from userspace (opcode=%u)\n", -+ __func__, opcode); -+ return -EINVAL; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: sending userspace packet to peer %u...\n", __func__, peer_id); -+ -+ return ovpn_send_data(ovpn, peer_id, packet, len); -+} -+ -+static const struct genl_ops ovpn_netlink_ops[] = { -+ { -+ .cmd = OVPN_CMD_NEW_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_new_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_SET_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_set_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_DEL_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_del_peer, -+ }, -+ { -+ .cmd = OVPN_CMD_GET_PEER, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_get_peer, -+ .dumpit = ovpn_netlink_dump_peers, -+ .done = ovpn_netlink_dump_done, -+ }, -+ { -+ .cmd = OVPN_CMD_NEW_KEY, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_new_key, -+ }, -+ { -+ .cmd = OVPN_CMD_DEL_KEY, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_del_key, -+ }, -+ { -+ .cmd = OVPN_CMD_SWAP_KEYS, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_swap_keys, -+ }, -+ { -+ .cmd = OVPN_CMD_REGISTER_PACKET, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_register_packet, -+ }, -+ { -+ .cmd = OVPN_CMD_PACKET, -+ .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, -+ .flags = GENL_ADMIN_PERM, -+ .doit = ovpn_netlink_packet, -+ }, -+}; -+ -+static struct genl_family ovpn_netlink_family __ro_after_init = { -+ .hdrsize = 0, -+ .name = OVPN_NL_NAME, -+ .version = 1, -+ .maxattr = OVPN_ATTR_MAX, -+ .policy = ovpn_netlink_policy, -+ .netnsok = true, -+ .pre_doit = ovpn_pre_doit, -+ .post_doit = ovpn_post_doit, -+ .module = THIS_MODULE, -+ .ops = ovpn_netlink_ops, -+ .n_ops = ARRAY_SIZE(ovpn_netlink_ops), -+ .mcgrps = ovpn_netlink_mcgrps, -+ .n_mcgrps = ARRAY_SIZE(ovpn_netlink_mcgrps), -+}; -+ -+int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer) -+{ -+ struct sk_buff *msg; -+ struct nlattr *attr; -+ void *hdr; -+ int ret; -+ -+ netdev_info(peer->ovpn->dev, "%s: deleting peer with id %u, reason %d\n", -+ peer->ovpn->dev->name, peer->id, peer->delete_reason); -+ -+ msg = nlmsg_new(100, GFP_KERNEL); -+ if (!msg) -+ return -ENOMEM; -+ -+ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, -+ OVPN_CMD_DEL_PEER); -+ if (!hdr) { -+ ret = -ENOBUFS; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, peer->ovpn->dev->ifindex)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ attr = nla_nest_start(msg, OVPN_ATTR_DEL_PEER); -+ if (!attr) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u8(msg, OVPN_DEL_PEER_ATTR_REASON, peer->delete_reason)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_DEL_PEER_ATTR_PEER_ID, peer->id)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ nla_nest_end(msg, attr); -+ -+ genlmsg_end(msg, hdr); -+ -+ genlmsg_multicast_netns(&ovpn_netlink_family, dev_net(peer->ovpn->dev), -+ msg, 0, OVPN_MCGRP_PEERS, GFP_KERNEL); -+ -+ return 0; -+ -+err_free_msg: -+ nlmsg_free(msg); -+ return ret; -+} -+ -+int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ const u8 *buf, size_t len) -+{ -+ struct nlattr *attr; -+ struct sk_buff *msg; -+ void *hdr; -+ int ret; -+ -+ if (!ovpn->registered_nl_portid_set) { -+ net_warn_ratelimited("%s: no userspace listener\n", __func__); -+ return 0; -+ } -+ -+ netdev_dbg(ovpn->dev, "%s: sending packet to userspace, len: %zd\n", __func__, len); -+ -+ msg = nlmsg_new(100 + len, GFP_ATOMIC); -+ if (!msg) -+ return -ENOMEM; -+ -+ hdr = genlmsg_put(msg, 0, 0, &ovpn_netlink_family, 0, -+ OVPN_CMD_PACKET); -+ if (!hdr) { -+ ret = -ENOBUFS; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_ATTR_IFINDEX, ovpn->dev->ifindex)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ attr = nla_nest_start(msg, OVPN_ATTR_PACKET); -+ if (!attr) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put(msg, OVPN_PACKET_ATTR_PACKET, len, buf)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ if (nla_put_u32(msg, OVPN_PACKET_ATTR_PEER_ID, peer->id)) { -+ ret = -EMSGSIZE; -+ goto err_free_msg; -+ } -+ -+ nla_nest_end(msg, attr); -+ -+ genlmsg_end(msg, hdr); -+ -+ return genlmsg_unicast(dev_net(ovpn->dev), msg, -+ ovpn->registered_nl_portid); -+ -+err_free_msg: -+ nlmsg_free(msg); -+ return ret; -+} -+ -+static int ovpn_netlink_notify(struct notifier_block *nb, unsigned long state, -+ void *_notify) -+{ -+ struct netlink_notify *notify = _notify; -+ struct ovpn_struct *ovpn; -+ struct net_device *dev; -+ struct net *netns; -+ bool found = false; -+ -+ if (state != NETLINK_URELEASE || notify->protocol != NETLINK_GENERIC) -+ return NOTIFY_DONE; -+ -+ rcu_read_lock(); -+ for_each_net_rcu(netns) { -+ for_each_netdev_rcu(netns, dev) { -+ if (!ovpn_dev_is_valid(dev)) -+ continue; -+ -+ ovpn = netdev_priv(dev); -+ if (notify->portid != ovpn->registered_nl_portid) -+ continue; -+ -+ found = true; -+ netdev_dbg(ovpn->dev, "%s: deregistering userspace listener\n", __func__); -+ ovpn->registered_nl_portid_set = false; -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ -+ /* if no interface matched our purposes, pass the notification along */ -+ if (!found) -+ return NOTIFY_DONE; -+ -+ return NOTIFY_OK; -+} -+ -+static struct notifier_block ovpn_netlink_notifier = { -+ .notifier_call = ovpn_netlink_notify, -+}; -+ -+int ovpn_netlink_init(struct ovpn_struct *ovpn) -+{ -+ ovpn->registered_nl_portid_set = false; -+ -+ return 0; -+} -+ -+/** -+ * ovpn_netlink_register() - register the ovpn genl netlink family -+ */ -+int __init ovpn_netlink_register(void) -+{ -+ int ret; -+ -+ ret = genl_register_family(&ovpn_netlink_family); -+ if (ret) -+ return ret; -+ -+ ret = netlink_register_notifier(&ovpn_netlink_notifier); -+ if (ret) -+ goto err; -+ -+ return 0; -+err: -+ genl_unregister_family(&ovpn_netlink_family); -+ return ret; -+} -+ -+/** -+ * ovpn_netlink_unregister() - unregister the ovpn genl netlink family -+ */ -+void __exit ovpn_netlink_unregister(void) -+{ -+ netlink_unregister_notifier(&ovpn_netlink_notifier); -+ genl_unregister_family(&ovpn_netlink_family); -+} -diff --git a/drivers/net/ovpn-dco/netlink.h b/drivers/net/ovpn-dco/netlink.h -new file mode 100644 -index 000000000000..843daf052c03 ---- /dev/null -+++ b/drivers/net/ovpn-dco/netlink.h -@@ -0,0 +1,22 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_NETLINK_H_ -+#define _NET_OVPN_DCO_NETLINK_H_ -+ -+struct ovpn_struct; -+struct ovpn_peer; -+ -+int ovpn_netlink_init(struct ovpn_struct *ovpn); -+int ovpn_netlink_register(void); -+void ovpn_netlink_unregister(void); -+int ovpn_netlink_send_packet(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ const u8 *buf, size_t len); -+int ovpn_netlink_notify_del_peer(struct ovpn_peer *peer); -+ -+#endif /* _NET_OVPN_DCO_NETLINK_H_ */ -diff --git a/drivers/net/ovpn-dco/ovpn.c b/drivers/net/ovpn-dco/ovpn.c -new file mode 100644 -index 000000000000..66c019174f5e ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpn.c -@@ -0,0 +1,600 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "bind.h" -+#include "netlink.h" -+#include "ovpn.h" -+#include "sock.h" -+#include "peer.h" -+#include "stats.h" -+#include "proto.h" -+#include "crypto.h" -+#include "crypto_aead.h" -+#include "skb.h" -+#include "tcp.h" -+#include "udp.h" -+ -+#include -+#include -+ -+static const unsigned char ovpn_keepalive_message[] = { -+ 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, -+ 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 -+}; -+ -+static const unsigned char ovpn_explicit_exit_notify_message[] = { -+ 0x28, 0x7f, 0x34, 0x6b, 0xd4, 0xef, 0x7a, 0x81, -+ 0x2d, 0x56, 0xb8, 0xd3, 0xaf, 0xc5, 0x45, 0x9c, -+ 6 // OCC_EXIT -+}; -+ -+/* Is keepalive message? -+ * Assumes that single byte at skb->data is defined. -+ */ -+static bool ovpn_is_keepalive(struct sk_buff *skb) -+{ -+ if (*skb->data != OVPN_KEEPALIVE_FIRST_BYTE) -+ return false; -+ -+ if (!pskb_may_pull(skb, sizeof(ovpn_keepalive_message))) -+ return false; -+ -+ return !memcmp(skb->data, ovpn_keepalive_message, -+ sizeof(ovpn_keepalive_message)); -+} -+ -+int ovpn_struct_init(struct net_device *dev) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ int err; -+ -+ memset(ovpn, 0, sizeof(*ovpn)); -+ -+ ovpn->dev = dev; -+ -+ err = ovpn_netlink_init(ovpn); -+ if (err < 0) -+ return err; -+ -+ spin_lock_init(&ovpn->lock); -+ spin_lock_init(&ovpn->peers.lock); -+ -+ ovpn->crypto_wq = alloc_workqueue("ovpn-crypto-wq-%s", -+ WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, -+ dev->name); -+ if (!ovpn->crypto_wq) -+ return -ENOMEM; -+ -+ ovpn->events_wq = alloc_workqueue("ovpn-event-wq-%s", WQ_MEM_RECLAIM, 0, dev->name); -+ if (!ovpn->events_wq) -+ return -ENOMEM; -+ -+ dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); -+ if (!dev->tstats) -+ return -ENOMEM; -+ -+ err = security_tun_dev_alloc_security(&ovpn->security); -+ if (err < 0) -+ return err; -+ -+ /* kernel -> userspace tun queue length */ -+ ovpn->max_tun_queue_len = OVPN_MAX_TUN_QUEUE_LEN; -+ -+ return 0; -+} -+ -+/* Called after decrypt to write IP packet to tun netdev. -+ * This method is expected to manage/free skb. -+ */ -+static void tun_netdev_write(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ /* packet integrity was verified on the VPN layer - no need to perform -+ * any additional check along the stack -+ */ -+ skb->ip_summed = CHECKSUM_UNNECESSARY; -+ skb->csum_level = ~0; -+ -+ /* skb hash for transport packet no longer valid after decapsulation */ -+ skb_clear_hash(skb); -+ -+ /* post-decrypt scrub -- prepare to inject encapsulated packet onto tun -+ * interface, based on __skb_tunnel_rx() in dst.h -+ */ -+ skb->dev = peer->ovpn->dev; -+ skb_set_queue_mapping(skb, 0); -+ skb_scrub_packet(skb, true); -+ -+ skb_reset_network_header(skb); -+ skb_reset_transport_header(skb); -+ skb_probe_transport_header(skb); -+ skb_reset_inner_headers(skb); -+ -+ /* update per-cpu RX stats with the stored size of encrypted packet */ -+ -+ /* we are in softirq context - hence no locking nor disable preemption needed */ -+ dev_sw_netstats_rx_add(peer->ovpn->dev, OVPN_SKB_CB(skb)->rx_stats_size); -+ -+ /* cause packet to be "received" by tun interface */ -+ napi_gro_receive(&peer->napi, skb); -+} -+ -+int ovpn_napi_poll(struct napi_struct *napi, int budget) -+{ -+ struct ovpn_peer *peer = container_of(napi, struct ovpn_peer, napi); -+ struct sk_buff *skb; -+ int work_done = 0; -+ -+ if (unlikely(budget <= 0)) -+ return 0; -+ /* this function should schedule at most 'budget' number of -+ * packets for delivery to the tun interface. -+ * If in the queue we have more packets than what allowed by the -+ * budget, the next polling will take care of those -+ */ -+ while ((work_done < budget) && -+ (skb = ptr_ring_consume_bh(&peer->netif_rx_ring))) { -+ tun_netdev_write(peer, skb); -+ work_done++; -+ } -+ -+ if (work_done < budget) -+ napi_complete_done(napi, work_done); -+ -+ return work_done; -+} -+ -+static int ovpn_transport_to_userspace(struct ovpn_struct *ovpn, const struct ovpn_peer *peer, -+ struct sk_buff *skb) -+{ -+ int ret; -+ -+ ret = skb_linearize(skb); -+ if (ret < 0) -+ return ret; -+ -+ ret = ovpn_netlink_send_packet(ovpn, peer, skb->data, skb->len); -+ if (ret < 0) -+ return ret; -+ -+ consume_skb(skb); -+ return 0; -+} -+ -+/* Entry point for processing an incoming packet (in skb form) -+ * -+ * Enqueue the packet and schedule RX consumer. -+ * Reference to peer is dropped only in case of success. -+ * -+ * Return 0 if the packet was handled (and consumed) -+ * Return <0 in case of error (return value is error code) -+ */ -+int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ int ret; -+ -+ /* At this point we know the packet is from a configured peer. -+ * DATA_V2 packets are handled in kernel space, the rest goes to user space. -+ * -+ * Packets are sent to userspace via netlink API in order to be consistenbt across -+ * UDP and TCP. -+ */ -+ if (unlikely(ovpn_opcode_from_skb(skb, 0) != OVPN_DATA_V2)) { -+ ret = ovpn_transport_to_userspace(ovpn, peer, skb); -+ if (ret < 0) -+ return ret; -+ -+ ovpn_peer_put(peer); -+ return 0; -+ } -+ -+ ret = ptr_ring_produce_bh(&peer->rx_ring, skb); -+ if (unlikely(ret < 0)) -+ return -ENOSPC; -+ -+ if (!queue_work(ovpn->crypto_wq, &peer->decrypt_work)) -+ ovpn_peer_put(peer); -+ -+ return 0; -+} -+ -+static int ovpn_decrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_peer *allowed_peer = NULL; -+ struct ovpn_crypto_key_slot *ks; -+ unsigned int rx_stats_size; -+ __be16 proto; -+ int ret = -1; -+ u8 key_id; -+ -+ /* save original packet size for stats accounting */ -+ OVPN_SKB_CB(skb)->rx_stats_size = skb->len; -+ -+ /* get the key slot matching the key Id in the received packet */ -+ key_id = ovpn_key_id_from_skb(skb); -+ ks = ovpn_crypto_key_id_to_slot(&peer->crypto, key_id); -+ if (unlikely(!ks)) { -+ net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n", __func__, -+ peer->id, key_id); -+ goto drop; -+ } -+ -+ /* decrypt */ -+ ret = ovpn_aead_decrypt(ks, skb); -+ -+ ovpn_crypto_key_slot_put(ks); -+ -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: error during decryption for peer %u, key-id %u: %d\n", -+ __func__, peer->id, key_id, ret); -+ goto drop; -+ } -+ -+ /* note event of authenticated packet received for keepalive */ -+ ovpn_peer_keepalive_recv_reset(peer); -+ -+ /* update source and destination endpoint for this peer */ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_UDP) -+ ovpn_peer_update_local_endpoint(peer, skb); -+ -+ /* increment RX stats */ -+ rx_stats_size = OVPN_SKB_CB(skb)->rx_stats_size; -+ ovpn_peer_stats_increment_rx(&peer->stats, rx_stats_size); -+ -+ /* check if this is a valid datapacket that has to be delivered to the -+ * tun interface -+ */ -+ skb_reset_network_header(skb); -+ proto = ovpn_ip_check_protocol(skb); -+ if (unlikely(!proto)) { -+ /* check if null packet */ -+ if (unlikely(!pskb_may_pull(skb, 1))) { -+ ret = -EINVAL; -+ goto drop; -+ } -+ -+ /* check if special OpenVPN message */ -+ if (ovpn_is_keepalive(skb)) { -+ netdev_dbg(peer->ovpn->dev, "%s: ping received from peer with id %u\n", -+ __func__, peer->id); -+ /* not an error */ -+ consume_skb(skb); -+ /* inform the caller that NAPI should not be scheduled -+ * for this packet -+ */ -+ return -1; -+ } -+ -+ ret = -EPROTONOSUPPORT; -+ goto drop; -+ } -+ skb->protocol = proto; -+ -+ /* perform Reverse Path Filtering (RPF) */ -+ allowed_peer = ovpn_peer_lookup_vpn_addr(peer->ovpn, skb, true); -+ if (unlikely(allowed_peer != peer)) { -+ ret = -EPERM; -+ goto drop; -+ } -+ -+ ret = ptr_ring_produce_bh(&peer->netif_rx_ring, skb); -+drop: -+ if (likely(allowed_peer)) -+ ovpn_peer_put(allowed_peer); -+ -+ if (unlikely(ret < 0)) -+ kfree_skb(skb); -+ -+ return ret; -+} -+ -+/* pick packet from RX queue, decrypt and forward it to the tun device */ -+void ovpn_decrypt_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ -+ peer = container_of(work, struct ovpn_peer, decrypt_work); -+ while ((skb = ptr_ring_consume_bh(&peer->rx_ring))) { -+ if (likely(ovpn_decrypt_one(peer, skb) == 0)) { -+ /* if a packet has been enqueued for NAPI, signal -+ * availability to the networking stack -+ */ -+ local_bh_disable(); -+ napi_schedule(&peer->napi); -+ local_bh_enable(); -+ } -+ -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+ ovpn_peer_put(peer); -+} -+ -+static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_crypto_key_slot *ks; -+ bool success = false; -+ int ret; -+ -+ /* get primary key to be used for encrypting data */ -+ ks = ovpn_crypto_key_slot_primary(&peer->crypto); -+ if (unlikely(!ks)) { -+ net_info_ratelimited("%s: error while retrieving primary key slot\n", __func__); -+ return false; -+ } -+ -+ if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && -+ skb_checksum_help(skb))) { -+ net_err_ratelimited("%s: cannot compute checksum for outgoing packet\n", __func__); -+ goto err; -+ } -+ -+ ovpn_peer_stats_increment_tx(&peer->stats, skb->len); -+ -+ /* encrypt */ -+ ret = ovpn_aead_encrypt(ks, skb, peer->id); -+ if (unlikely(ret < 0)) { -+ /* if we ran out of IVs we must kill the key as it can't be used anymore */ -+ if (ret == -ERANGE) { -+ netdev_warn(peer->ovpn->dev, -+ "%s: killing primary key as we ran out of IVs\n", __func__); -+ ovpn_crypto_kill_primary(&peer->crypto); -+ goto err; -+ } -+ net_err_ratelimited("%s: error during encryption for peer %u, key-id %u: %d\n", -+ __func__, peer->id, ks->key_id, ret); -+ goto err; -+ } -+ -+ success = true; -+err: -+ ovpn_crypto_key_slot_put(ks); -+ return success; -+} -+ -+/* Process packets in TX queue in a transport-specific way. -+ * -+ * UDP transport - encrypt and send across the tunnel. -+ * TCP transport - encrypt and put into TCP TX queue. -+ */ -+void ovpn_encrypt_work(struct work_struct *work) -+{ -+ struct sk_buff *skb, *curr, *next; -+ struct ovpn_peer *peer; -+ -+ peer = container_of(work, struct ovpn_peer, encrypt_work); -+ while ((skb = ptr_ring_consume_bh(&peer->tx_ring))) { -+ /* this might be a GSO-segmented skb list: process each skb -+ * independently -+ */ -+ skb_list_walk_safe(skb, curr, next) { -+ /* if one segment fails encryption, we drop the entire -+ * packet, because it does not really make sense to send -+ * only part of it at this point -+ */ -+ if (unlikely(!ovpn_encrypt_one(peer, curr))) { -+ kfree_skb_list(skb); -+ skb = NULL; -+ break; -+ } -+ } -+ -+ /* successful encryption */ -+ if (skb) { -+ skb_list_walk_safe(skb, curr, next) { -+ skb_mark_not_on_list(curr); -+ -+ switch (peer->sock->sock->sk->sk_protocol) { -+ case IPPROTO_UDP: -+ ovpn_udp_send_skb(peer->ovpn, peer, curr); -+ break; -+ case IPPROTO_TCP: -+ ovpn_tcp_send_skb(peer, curr); -+ break; -+ default: -+ /* no transport configured yet */ -+ consume_skb(skb); -+ break; -+ } -+ } -+ -+ /* note event of authenticated packet xmit for keepalive */ -+ ovpn_peer_keepalive_xmit_reset(peer); -+ } -+ -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+ ovpn_peer_put(peer); -+} -+ -+/* Put skb into TX queue and schedule a consumer */ -+static void ovpn_queue_skb(struct ovpn_struct *ovpn, struct sk_buff *skb, struct ovpn_peer *peer) -+{ -+ int ret; -+ -+ if (likely(!peer)) -+ peer = ovpn_peer_lookup_vpn_addr(ovpn, skb, false); -+ if (unlikely(!peer)) { -+ net_dbg_ratelimited("%s: no peer to send data to\n", ovpn->dev->name); -+ goto drop; -+ } -+ -+ ret = ptr_ring_produce_bh(&peer->tx_ring, skb); -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: cannot queue packet to TX ring\n", __func__); -+ goto drop; -+ } -+ -+ if (!queue_work(ovpn->crypto_wq, &peer->encrypt_work)) -+ ovpn_peer_put(peer); -+ -+ return; -+drop: -+ if (peer) -+ ovpn_peer_put(peer); -+ kfree_skb_list(skb); -+} -+ -+/* Net device start xmit -+ */ -+netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) -+{ -+ struct ovpn_struct *ovpn = netdev_priv(dev); -+ struct sk_buff *segments, *tmp, *curr, *next; -+ struct sk_buff_head skb_list; -+ __be16 proto; -+ int ret; -+ -+ /* reset netfilter state */ -+ nf_reset_ct(skb); -+ -+ /* verify IP header size in network packet */ -+ proto = ovpn_ip_check_protocol(skb); -+ if (unlikely(!proto || skb->protocol != proto)) { -+ net_dbg_ratelimited("%s: dropping malformed payload packet\n", -+ dev->name); -+ goto drop; -+ } -+ -+ if (skb_is_gso(skb)) { -+ segments = skb_gso_segment(skb, 0); -+ if (IS_ERR(segments)) { -+ ret = PTR_ERR(segments); -+ net_dbg_ratelimited("%s: cannot segment packet: %d\n", dev->name, ret); -+ goto drop; -+ } -+ -+ consume_skb(skb); -+ skb = segments; -+ } -+ -+ /* from this moment on, "skb" might be a list */ -+ -+ __skb_queue_head_init(&skb_list); -+ skb_list_walk_safe(skb, curr, next) { -+ skb_mark_not_on_list(curr); -+ -+ tmp = skb_share_check(curr, GFP_ATOMIC); -+ if (unlikely(!tmp)) { -+ kfree_skb_list(next); -+ net_dbg_ratelimited("%s: skb_share_check failed\n", dev->name); -+ goto drop_list; -+ } -+ -+ __skb_queue_tail(&skb_list, tmp); -+ } -+ skb_list.prev->next = NULL; -+ -+ ovpn_queue_skb(ovpn, skb_list.next, NULL); -+ -+ return NETDEV_TX_OK; -+ -+drop_list: -+ skb_queue_walk_safe(&skb_list, curr, next) -+ kfree_skb(curr); -+drop: -+ skb_tx_error(skb); -+ kfree_skb_list(skb); -+ return NET_XMIT_DROP; -+} -+ -+/* Encrypt and transmit a special message to peer, such as keepalive -+ * or explicit-exit-notify. Called from softirq context. -+ * Assumes that caller holds a reference to peer. -+ */ -+static void ovpn_xmit_special(struct ovpn_peer *peer, const void *data, -+ const unsigned int len) -+{ -+ struct ovpn_struct *ovpn; -+ struct sk_buff *skb; -+ -+ ovpn = peer->ovpn; -+ if (unlikely(!ovpn)) -+ return; -+ -+ skb = alloc_skb(256 + len, GFP_ATOMIC); -+ if (unlikely(!skb)) -+ return; -+ -+ skb_reserve(skb, 128); -+ skb->priority = TC_PRIO_BESTEFFORT; -+ memcpy(__skb_put(skb, len), data, len); -+ -+ /* increase reference counter when passing peer to sending queue */ -+ if (!ovpn_peer_hold(peer)) { -+ netdev_dbg(ovpn->dev, "%s: cannot hold peer reference for sending special packet\n", -+ __func__); -+ kfree_skb(skb); -+ return; -+ } -+ -+ ovpn_queue_skb(ovpn, skb, peer); -+} -+ -+void ovpn_keepalive_xmit(struct ovpn_peer *peer) -+{ -+ ovpn_xmit_special(peer, ovpn_keepalive_message, -+ sizeof(ovpn_keepalive_message)); -+} -+ -+/* Transmit explicit exit notification. -+ * Called from process context. -+ */ -+void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer) -+{ -+ ovpn_xmit_special(peer, ovpn_explicit_exit_notify_message, -+ sizeof(ovpn_explicit_exit_notify_message)); -+} -+ -+/* Copy buffer into skb and send it across the tunnel. -+ * -+ * For UDP transport: just sent the skb to peer -+ * For TCP transport: put skb into TX queue -+ */ -+int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len) -+{ -+ u16 skb_len = SKB_HEADER_LEN + len; -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ bool tcp = false; -+ int ret = 0; -+ -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (unlikely(!peer)) { -+ netdev_dbg(ovpn->dev, "no peer to send data to\n"); -+ return -EHOSTUNREACH; -+ } -+ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) { -+ skb_len += sizeof(u16); -+ tcp = true; -+ } -+ -+ skb = alloc_skb(skb_len, GFP_ATOMIC); -+ if (unlikely(!skb)) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ skb_reserve(skb, SKB_HEADER_LEN); -+ skb_put_data(skb, data, len); -+ -+ /* prepend TCP packet with size, as required by OpenVPN protocol */ -+ if (tcp) { -+ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); -+ ovpn_queue_tcp_skb(peer, skb); -+ } else { -+ ovpn_udp_send_skb(ovpn, peer, skb); -+ } -+out: -+ ovpn_peer_put(peer); -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/ovpn.h b/drivers/net/ovpn-dco/ovpn.h -new file mode 100644 -index 000000000000..9364fd5dd309 ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpn.h -@@ -0,0 +1,43 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPN_H_ -+#define _NET_OVPN_DCO_OVPN_H_ -+ -+#include "main.h" -+#include "peer.h" -+#include "sock.h" -+#include "ovpnstruct.h" -+ -+#include -+#include -+#include -+ -+struct ovpn_struct; -+struct net_device; -+ -+int ovpn_struct_init(struct net_device *dev); -+ -+u16 ovpn_select_queue(struct net_device *dev, struct sk_buff *skb, -+ struct net_device *sb_dev); -+ -+void ovpn_keepalive_xmit(struct ovpn_peer *peer); -+void ovpn_explicit_exit_notify_xmit(struct ovpn_peer *peer); -+ -+netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev); -+ -+int ovpn_recv(struct ovpn_struct *ovpn, struct ovpn_peer *peer, struct sk_buff *skb); -+ -+void ovpn_encrypt_work(struct work_struct *work); -+void ovpn_decrypt_work(struct work_struct *work); -+int ovpn_napi_poll(struct napi_struct *napi, int budget); -+ -+int ovpn_send_data(struct ovpn_struct *ovpn, u32 peer_id, const u8 *data, size_t len); -+ -+#endif /* _NET_OVPN_DCO_OVPN_H_ */ -diff --git a/drivers/net/ovpn-dco/ovpnstruct.h b/drivers/net/ovpn-dco/ovpnstruct.h -new file mode 100644 -index 000000000000..f9bc559609cd ---- /dev/null -+++ b/drivers/net/ovpn-dco/ovpnstruct.h -@@ -0,0 +1,59 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNSTRUCT_H_ -+#define _NET_OVPN_DCO_OVPNSTRUCT_H_ -+ -+#include "peer.h" -+ -+#include -+#include -+#include -+ -+/* Our state per ovpn interface */ -+struct ovpn_struct { -+ /* read-mostly objects in this section */ -+ struct net_device *dev; -+ -+ /* device operation mode (i.e. P2P, MP) */ -+ enum ovpn_mode mode; -+ -+ /* protect writing to the ovpn_struct object */ -+ spinlock_t lock; -+ -+ /* workqueue used to schedule crypto work that may sleep */ -+ struct workqueue_struct *crypto_wq; -+ /* workqueue used to schedule generic event that may sleep or that need -+ * to be performed out of softirq context -+ */ -+ struct workqueue_struct *events_wq; -+ -+ /* list of known peers */ -+ struct { -+ DECLARE_HASHTABLE(by_id, 12); -+ DECLARE_HASHTABLE(by_transp_addr, 12); -+ DECLARE_HASHTABLE(by_vpn_addr, 12); -+ /* protects write access to any of the hashtables above */ -+ spinlock_t lock; -+ } peers; -+ -+ /* for p2p mode */ -+ struct ovpn_peer __rcu *peer; -+ -+ unsigned int max_tun_queue_len; -+ -+ netdev_features_t set_features; -+ -+ void *security; -+ -+ u32 registered_nl_portid; -+ bool registered_nl_portid_set; -+}; -+ -+#endif /* _NET_OVPN_DCO_OVPNSTRUCT_H_ */ -diff --git a/drivers/net/ovpn-dco/peer.c b/drivers/net/ovpn-dco/peer.c -new file mode 100644 -index 000000000000..87d3f1b34c4d ---- /dev/null -+++ b/drivers/net/ovpn-dco/peer.c -@@ -0,0 +1,906 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "ovpn.h" -+#include "bind.h" -+#include "crypto.h" -+#include "peer.h" -+#include "netlink.h" -+#include "tcp.h" -+ -+#include -+#include -+#include -+#include -+#include -+ -+static void ovpn_peer_ping(struct timer_list *t) -+{ -+ struct ovpn_peer *peer = from_timer(peer, t, keepalive_xmit); -+ -+ netdev_dbg(peer->ovpn->dev, "%s: sending ping to peer %u\n", __func__, peer->id); -+ ovpn_keepalive_xmit(peer); -+} -+ -+static void ovpn_peer_expire(struct timer_list *t) -+{ -+ struct ovpn_peer *peer = from_timer(peer, t, keepalive_recv); -+ -+ netdev_dbg(peer->ovpn->dev, "%s: peer %u expired\n", __func__, peer->id); -+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_EXPIRED); -+} -+ -+/* Construct a new peer */ -+static struct ovpn_peer *ovpn_peer_create(struct ovpn_struct *ovpn, u32 id) -+{ -+ struct ovpn_peer *peer; -+ int ret; -+ -+ /* alloc and init peer object */ -+ peer = kzalloc(sizeof(*peer), GFP_KERNEL); -+ if (!peer) -+ return ERR_PTR(-ENOMEM); -+ -+ peer->id = id; -+ peer->halt = false; -+ peer->ovpn = ovpn; -+ -+ peer->vpn_addrs.ipv4.s_addr = htonl(INADDR_ANY); -+ peer->vpn_addrs.ipv6 = in6addr_any; -+ -+ RCU_INIT_POINTER(peer->bind, NULL); -+ ovpn_crypto_state_init(&peer->crypto); -+ spin_lock_init(&peer->lock); -+ kref_init(&peer->refcount); -+ ovpn_peer_stats_init(&peer->stats); -+ -+ INIT_WORK(&peer->encrypt_work, ovpn_encrypt_work); -+ INIT_WORK(&peer->decrypt_work, ovpn_decrypt_work); -+ -+ ret = dst_cache_init(&peer->dst_cache, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot initialize dst cache\n", __func__); -+ goto err; -+ } -+ -+ ret = ptr_ring_init(&peer->tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate TX ring\n", __func__); -+ goto err_dst_cache; -+ } -+ -+ ret = ptr_ring_init(&peer->rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate RX ring\n", __func__); -+ goto err_tx_ring; -+ } -+ -+ ret = ptr_ring_init(&peer->netif_rx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(ovpn->dev, "%s: cannot allocate NETIF RX ring\n", __func__); -+ goto err_rx_ring; -+ } -+ -+ /* configure and start NAPI */ -+ netif_tx_napi_add(ovpn->dev, &peer->napi, ovpn_napi_poll, -+ NAPI_POLL_WEIGHT); -+ napi_enable(&peer->napi); -+ -+ dev_hold(ovpn->dev); -+ -+ timer_setup(&peer->keepalive_xmit, ovpn_peer_ping, 0); -+ timer_setup(&peer->keepalive_recv, ovpn_peer_expire, 0); -+ -+ return peer; -+err_rx_ring: -+ ptr_ring_cleanup(&peer->rx_ring, NULL); -+err_tx_ring: -+ ptr_ring_cleanup(&peer->tx_ring, NULL); -+err_dst_cache: -+ dst_cache_destroy(&peer->dst_cache); -+err: -+ kfree(peer); -+ return ERR_PTR(ret); -+} -+ -+/* Reset the ovpn_sockaddr associated with a peer */ -+static int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, const struct sockaddr_storage *ss, -+ const u8 *local_ip) -+{ -+ struct ovpn_bind *bind; -+ size_t ip_len; -+ -+ /* create new ovpn_bind object */ -+ bind = ovpn_bind_from_sockaddr(ss); -+ if (IS_ERR(bind)) -+ return PTR_ERR(bind); -+ -+ if (local_ip) { -+ if (ss->ss_family == AF_INET) { -+ ip_len = sizeof(struct in_addr); -+ } else if (ss->ss_family == AF_INET6) { -+ ip_len = sizeof(struct in6_addr); -+ } else { -+ netdev_dbg(peer->ovpn->dev, "%s: invalid family for remote endpoint\n", -+ __func__); -+ kfree(bind); -+ return -EINVAL; -+ } -+ -+ memcpy(&bind->local, local_ip, ip_len); -+ } -+ -+ /* set binding */ -+ ovpn_bind_reset(peer, bind); -+ -+ return 0; -+} -+ -+void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct sockaddr_storage ss; -+ const u8 *local_ip = NULL; -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa; -+ struct ovpn_bind *bind; -+ sa_family_t family; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) -+ goto unlock; -+ -+ if (likely(ovpn_bind_skb_src_match(bind, skb))) -+ goto unlock; -+ -+ family = skb_protocol_to_family(skb); -+ -+ if (bind->sa.in4.sin_family == family) -+ local_ip = (u8 *)&bind->local; -+ -+ switch (family) { -+ case AF_INET: -+ sa = (struct sockaddr_in *)&ss; -+ sa->sin_family = AF_INET; -+ sa->sin_addr.s_addr = ip_hdr(skb)->saddr; -+ sa->sin_port = udp_hdr(skb)->source; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)&ss; -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = ipv6_hdr(skb)->saddr; -+ sa6->sin6_port = udp_hdr(skb)->source; -+ sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif); -+ break; -+ default: -+ goto unlock; -+ } -+ -+ netdev_dbg(peer->ovpn->dev, "%s: peer %d floated to %pIScp", __func__, peer->id, &ss); -+ ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip); -+unlock: -+ rcu_read_unlock(); -+} -+ -+static void ovpn_peer_timer_delete_all(struct ovpn_peer *peer) -+{ -+ del_timer_sync(&peer->keepalive_xmit); -+ del_timer_sync(&peer->keepalive_recv); -+} -+ -+static void ovpn_peer_free(struct ovpn_peer *peer) -+{ -+ ovpn_bind_reset(peer, NULL); -+ ovpn_peer_timer_delete_all(peer); -+ -+ WARN_ON(!__ptr_ring_empty(&peer->tx_ring)); -+ ptr_ring_cleanup(&peer->tx_ring, NULL); -+ WARN_ON(!__ptr_ring_empty(&peer->rx_ring)); -+ ptr_ring_cleanup(&peer->rx_ring, NULL); -+ WARN_ON(!__ptr_ring_empty(&peer->netif_rx_ring)); -+ ptr_ring_cleanup(&peer->netif_rx_ring, NULL); -+ -+ dst_cache_destroy(&peer->dst_cache); -+ -+ dev_put(peer->ovpn->dev); -+ -+ kfree(peer); -+} -+ -+static void ovpn_peer_release_rcu(struct rcu_head *head) -+{ -+ struct ovpn_peer *peer = container_of(head, struct ovpn_peer, rcu); -+ -+ ovpn_crypto_state_release(&peer->crypto); -+ ovpn_peer_free(peer); -+} -+ -+void ovpn_peer_release(struct ovpn_peer *peer) -+{ -+ napi_disable(&peer->napi); -+ netif_napi_del(&peer->napi); -+ -+ if (peer->sock) -+ ovpn_socket_put(peer->sock); -+ -+ call_rcu(&peer->rcu, ovpn_peer_release_rcu); -+} -+ -+static void ovpn_peer_delete_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, -+ delete_work); -+ ovpn_peer_release(peer); -+ ovpn_netlink_notify_del_peer(peer); -+} -+ -+/* Use with kref_put calls, when releasing refcount -+ * on ovpn_peer objects. This method should only -+ * be called from process context with config_mutex held. -+ */ -+void ovpn_peer_release_kref(struct kref *kref) -+{ -+ struct ovpn_peer *peer = container_of(kref, struct ovpn_peer, refcount); -+ -+ INIT_WORK(&peer->delete_work, ovpn_peer_delete_work); -+ queue_work(peer->ovpn->events_wq, &peer->delete_work); -+} -+ -+struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, -+ struct socket *sock, u32 id, uint8_t *local_ip) -+{ -+ struct ovpn_peer *peer; -+ int ret; -+ -+ /* create new peer */ -+ peer = ovpn_peer_create(ovpn, id); -+ if (IS_ERR(peer)) -+ return peer; -+ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) { -+ /* a UDP peer must have a remote endpoint */ -+ if (!sa) { -+ ovpn_peer_release(peer); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ /* set peer sockaddr */ -+ ret = ovpn_peer_reset_sockaddr(peer, sa, local_ip); -+ if (ret < 0) { -+ ovpn_peer_release(peer); -+ return ERR_PTR(ret); -+ } -+ } -+ -+ peer->sock = ovpn_socket_new(sock, peer); -+ if (IS_ERR(peer->sock)) { -+ peer->sock = NULL; -+ ovpn_peer_release(peer); -+ return ERR_PTR(-ENOTSOCK); -+ } -+ -+ /* schedule initial TCP RX work only after having assigned peer->sock */ -+ if (peer->sock->sock->sk->sk_protocol == IPPROTO_TCP) -+ queue_work(peer->ovpn->events_wq, &peer->tcp.rx_work); -+ -+ return peer; -+} -+ -+/* Configure keepalive parameters */ -+void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) -+{ -+ u32 delta; -+ -+ netdev_dbg(peer->ovpn->dev, -+ "%s: scheduling keepalive for peer %u: interval=%u timeout=%u\n", __func__, -+ peer->id, interval, timeout); -+ -+ peer->keepalive_interval = interval; -+ if (interval > 0) { -+ delta = msecs_to_jiffies(interval * MSEC_PER_SEC); -+ mod_timer(&peer->keepalive_xmit, jiffies + delta); -+ } else { -+ del_timer(&peer->keepalive_xmit); -+ } -+ -+ peer->keepalive_timeout = timeout; -+ if (timeout) { -+ delta = msecs_to_jiffies(timeout * MSEC_PER_SEC); -+ mod_timer(&peer->keepalive_recv, jiffies + delta); -+ } else { -+ del_timer(&peer->keepalive_recv); -+ } -+} -+ -+#define ovpn_peer_index(_tbl, _key, _key_len) \ -+ (jhash(_key, _key_len, 0) % HASH_SIZE(_tbl)) \ -+ -+static struct ovpn_peer *ovpn_peer_lookup_vpn_addr4(struct hlist_head *head, __be32 *addr) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr4) { -+ if (*addr != tmp->vpn_addrs.ipv4.s_addr) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+static struct ovpn_peer *ovpn_peer_lookup_vpn_addr6(struct hlist_head *head, struct in6_addr *addr) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ int i; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_addr6) { -+ for (i = 0; i < 4; i++) { -+ if (addr->s6_addr32[i] != tmp->vpn_addrs.ipv6.s6_addr32[i]) -+ continue; -+ } -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+/** -+ * ovpn_nexthop4() - looks up the IP of the nexthop for the given destination -+ * -+ * Looks up in the IPv4 system routing table the IO of the nexthop to be used -+ * to reach the destination passed as argument. IF no nexthop can be found, the -+ * destination itself is returned as it probably has to be used as nexthop. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @dst: the destination to be looked up -+ * -+ * Return the IP of the next hop if found or the dst itself otherwise -+ */ -+static __be32 ovpn_nexthop4(struct ovpn_struct *ovpn, __be32 dst) -+{ -+ struct rtable *rt; -+ struct flowi4 fl = { -+ .daddr = dst -+ }; -+ -+ rt = ip_route_output_flow(dev_net(ovpn->dev), &fl, NULL); -+ if (IS_ERR(rt)) { -+ net_dbg_ratelimited("%s: no route to host %pI4\n", __func__, &dst); -+ /* if we end up here this packet is probably going to be thrown away later */ -+ return dst; -+ } -+ -+ if (!rt->rt_uses_gateway) -+ goto out; -+ -+ dst = rt->rt_gw4; -+out: -+ ip_rt_put(rt); -+ return dst; -+} -+ -+/** -+ * ovpn_nexthop6() - looks up the IPv6 of the nexthop for the given destination -+ * -+ * Looks up in the IPv6 system routing table the IO of the nexthop to be used -+ * to reach the destination passed as argument. IF no nexthop can be found, the -+ * destination itself is returned as it probably has to be used as nexthop. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @dst: the destination to be looked up -+ * -+ * Return the IP of the next hop if found or the dst itself otherwise -+ */ -+static struct in6_addr ovpn_nexthop6(struct ovpn_struct *ovpn, struct in6_addr dst) -+{ -+#if IS_ENABLED(CONFIG_IPV6) -+ struct rt6_info *rt; -+ struct flowi6 fl = { -+ .daddr = dst, -+ }; -+ -+ rt = (struct rt6_info *)ipv6_stub->ipv6_dst_lookup_flow(dev_net(ovpn->dev), NULL, &fl, -+ NULL); -+ if (IS_ERR(rt)) { -+ net_dbg_ratelimited("%s: no route to host %pI6\n", __func__, &dst); -+ /* if we end up here this packet is probably going to be thrown away later */ -+ return dst; -+ } -+ -+ if (!(rt->rt6i_flags & RTF_GATEWAY)) -+ goto out; -+ -+ dst = rt->rt6i_gateway; -+out: -+ dst_release((struct dst_entry *)rt); -+#endif -+ return dst; -+} -+ -+/** -+ * ovpn_peer_lookup_vpn_addr() - Lookup peer to send skb to -+ * -+ * This function takes a tunnel packet and looks up the peer to send it to -+ * after encapsulation. The skb is expected to be the in-tunnel packet, without -+ * any OpenVPN related header. -+ * -+ * Assume that the IP header is accessible in the skb data. -+ * -+ * @ovpn: the private data representing the current VPN session -+ * @skb: the skb to extract the destination address from -+ * -+ * Return the peer if found or NULL otherwise. -+ */ -+struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, -+ bool use_src) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ struct hlist_head *head; -+ struct rt6_info *rt6i = NULL; -+ struct rtable *rt = NULL; -+ sa_family_t sa_fam; -+ struct in6_addr addr6; -+ __be32 addr4; -+ u32 index; -+ -+ /* in P2P mode, no matter the destination, packets are always sent to the single peer -+ * listening on the other side -+ */ -+ if (ovpn->mode == OVPN_MODE_P2P) { -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ return peer; -+ } -+ -+ sa_fam = skb_protocol_to_family(skb); -+ -+ switch (sa_fam) { -+ case AF_INET: -+ if (use_src) -+ addr4 = ip_hdr(skb)->saddr; -+ else -+ addr4 = ip_hdr(skb)->daddr; -+ addr4 = ovpn_nexthop4(ovpn, addr4); -+ -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr4, sizeof(addr4)); -+ head = &ovpn->peers.by_vpn_addr[index]; -+ -+ peer = ovpn_peer_lookup_vpn_addr4(head, &addr4); -+ break; -+ case AF_INET6: -+ if (use_src) -+ addr6 = ipv6_hdr(skb)->saddr; -+ else -+ addr6 = ipv6_hdr(skb)->daddr; -+ addr6 = ovpn_nexthop6(ovpn, addr6); -+ -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &addr6, sizeof(addr6)); -+ head = &ovpn->peers.by_vpn_addr[index]; -+ -+ peer = ovpn_peer_lookup_vpn_addr6(head, &addr6); -+ break; -+ } -+ -+ if (rt) -+ ip_rt_put(rt); -+ if (rt6i) -+ dst_release((struct dst_entry *)rt6i); -+ -+ return peer; -+} -+ -+static bool ovpn_peer_transp_match(struct ovpn_peer *peer, struct sockaddr_storage *ss) -+{ -+ struct ovpn_bind *bind = rcu_dereference(peer->bind); -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ -+ if (unlikely(!bind)) -+ return false; -+ -+ if (ss->ss_family != bind->sa.in4.sin_family) -+ return false; -+ -+ switch (ss->ss_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)ss; -+ if (sa4->sin_addr.s_addr != bind->sa.in4.sin_addr.s_addr) -+ return false; -+ if (sa4->sin_port != bind->sa.in4.sin_port) -+ return false; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)ss; -+ if (memcmp(&sa6->sin6_addr, &bind->sa.in6.sin6_addr, sizeof(struct in6_addr))) -+ return false; -+ if (sa6->sin6_port != bind->sa.in6.sin6_port) -+ return false; -+ break; -+ default: -+ return false; -+ } -+ -+ return true; -+} -+ -+static bool ovpn_peer_skb_to_sockaddr(struct sk_buff *skb, struct sockaddr_storage *ss) -+{ -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ -+ ss->ss_family = skb_protocol_to_family(skb); -+ switch (ss->ss_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)ss; -+ sa4->sin_family = AF_INET; -+ sa4->sin_addr.s_addr = ip_hdr(skb)->saddr; -+ sa4->sin_port = udp_hdr(skb)->source; -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)ss; -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = ipv6_hdr(skb)->saddr; -+ sa6->sin6_port = udp_hdr(skb)->source; -+ break; -+ default: -+ return false; -+ } -+ -+ return true; -+} -+ -+static struct ovpn_peer *ovpn_peer_lookup_transp_addr_p2p(struct ovpn_struct *ovpn, -+ struct sockaddr_storage *ss) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && ovpn_peer_transp_match(tmp, ss) && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb) -+{ -+ struct ovpn_peer *peer = NULL, *tmp; -+ struct sockaddr_storage ss = { 0 }; -+ struct hlist_head *head; -+ size_t sa_len; -+ bool found; -+ u32 index; -+ -+ if (unlikely(!ovpn_peer_skb_to_sockaddr(skb, &ss))) -+ return NULL; -+ -+ if (ovpn->mode == OVPN_MODE_P2P) -+ return ovpn_peer_lookup_transp_addr_p2p(ovpn, &ss); -+ -+ switch (ss.ss_family) { -+ case AF_INET: -+ sa_len = sizeof(struct sockaddr_in); -+ break; -+ case AF_INET6: -+ sa_len = sizeof(struct sockaddr_in6); -+ break; -+ default: -+ return NULL; -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &ss, sa_len); -+ head = &ovpn->peers.by_transp_addr[index]; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_transp_addr) { -+ found = ovpn_peer_transp_match(tmp, &ss); -+ if (!found) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+static struct ovpn_peer *ovpn_peer_lookup_id_p2p(struct ovpn_struct *ovpn, u32 peer_id) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (likely(tmp && tmp->id == peer_id && ovpn_peer_hold(tmp))) -+ peer = tmp; -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id) -+{ -+ struct ovpn_peer *tmp, *peer = NULL; -+ struct hlist_head *head; -+ u32 index; -+ -+ if (ovpn->mode == OVPN_MODE_P2P) -+ return ovpn_peer_lookup_id_p2p(ovpn, peer_id); -+ -+ index = ovpn_peer_index(ovpn->peers.by_id, &peer_id, sizeof(peer_id)); -+ head = &ovpn->peers.by_id[index]; -+ -+ rcu_read_lock(); -+ hlist_for_each_entry_rcu(tmp, head, hash_entry_id) { -+ if (tmp->id != peer_id) -+ continue; -+ -+ if (!ovpn_peer_hold(tmp)) -+ continue; -+ -+ peer = tmp; -+ break; -+ } -+ rcu_read_unlock(); -+ -+ return peer; -+} -+ -+void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct ovpn_bind *bind; -+ -+ rcu_read_lock(); -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) -+ goto unlock; -+ -+ switch (skb_protocol_to_family(skb)) { -+ case AF_INET: -+ if (unlikely(bind->local.ipv4.s_addr != ip_hdr(skb)->daddr)) { -+ netdev_dbg(peer->ovpn->dev, -+ "%s: learning local IPv4 for peer %d (%pI4 -> %pI4)\n", __func__, -+ peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); -+ bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; -+ } -+ break; -+ case AF_INET6: -+ if (unlikely(memcmp(&bind->local.ipv6, &ipv6_hdr(skb)->daddr, -+ sizeof(bind->local.ipv6)))) { -+ netdev_dbg(peer->ovpn->dev, -+ "%s: learning local IPv6 for peer %d (%pI6c -> %pI6c\n", -+ __func__, peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); -+ bind->local.ipv6 = ipv6_hdr(skb)->daddr; -+ } -+ break; -+ default: -+ break; -+ } -+unlock: -+ rcu_read_unlock(); -+} -+ -+static int ovpn_peer_add_mp(struct ovpn_struct *ovpn, struct ovpn_peer *peer) -+{ -+ struct sockaddr_storage sa = { 0 }; -+ struct sockaddr_in6 *sa6; -+ struct sockaddr_in *sa4; -+ struct ovpn_bind *bind; -+ struct ovpn_peer *tmp; -+ size_t salen; -+ int ret = 0; -+ u32 index; -+ -+ spin_lock_bh(&ovpn->peers.lock); -+ /* do not add duplicates */ -+ tmp = ovpn_peer_lookup_id(ovpn, peer->id); -+ if (tmp) { -+ ovpn_peer_put(tmp); -+ ret = -EEXIST; -+ goto unlock; -+ } -+ -+ hlist_del_init_rcu(&peer->hash_entry_transp_addr); -+ bind = rcu_dereference_protected(peer->bind, true); -+ /* peers connected via UDP have bind == NULL */ -+ if (bind) { -+ switch (bind->sa.in4.sin_family) { -+ case AF_INET: -+ sa4 = (struct sockaddr_in *)&sa; -+ -+ sa4->sin_family = AF_INET; -+ sa4->sin_addr.s_addr = bind->sa.in4.sin_addr.s_addr; -+ sa4->sin_port = bind->sa.in4.sin_port; -+ salen = sizeof(*sa4); -+ break; -+ case AF_INET6: -+ sa6 = (struct sockaddr_in6 *)&sa; -+ -+ sa6->sin6_family = AF_INET6; -+ sa6->sin6_addr = bind->sa.in6.sin6_addr; -+ sa6->sin6_port = bind->sa.in6.sin6_port; -+ salen = sizeof(*sa6); -+ break; -+ default: -+ ret = -EPROTONOSUPPORT; -+ goto unlock; -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_transp_addr, &sa, salen); -+ hlist_add_head_rcu(&peer->hash_entry_transp_addr, -+ &ovpn->peers.by_transp_addr[index]); -+ } -+ -+ index = ovpn_peer_index(ovpn->peers.by_id, &peer->id, sizeof(peer->id)); -+ hlist_add_head_rcu(&peer->hash_entry_id, &ovpn->peers.by_id[index]); -+ -+ if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv4, -+ sizeof(peer->vpn_addrs.ipv4)); -+ hlist_add_head_rcu(&peer->hash_entry_addr4, &ovpn->peers.by_vpn_addr[index]); -+ } -+ -+ hlist_del_init_rcu(&peer->hash_entry_addr6); -+ if (memcmp(&peer->vpn_addrs.ipv6, &in6addr_any, sizeof(peer->vpn_addrs.ipv6))) { -+ index = ovpn_peer_index(ovpn->peers.by_vpn_addr, &peer->vpn_addrs.ipv6, -+ sizeof(peer->vpn_addrs.ipv6)); -+ hlist_add_head_rcu(&peer->hash_entry_addr6, &ovpn->peers.by_vpn_addr[index]); -+ } -+ -+unlock: -+ spin_unlock_bh(&ovpn->peers.lock); -+ -+ return ret; -+} -+ -+static int ovpn_peer_add_p2p(struct ovpn_struct *ovpn, struct ovpn_peer *peer) -+{ -+ struct ovpn_peer *tmp; -+ -+ spin_lock_bh(&ovpn->lock); -+ /* in p2p mode it is possible to have a single peer only, therefore the -+ * old one is released and substituted by the new one -+ */ -+ tmp = rcu_dereference(ovpn->peer); -+ if (tmp) { -+ tmp->delete_reason = OVPN_DEL_PEER_REASON_TEARDOWN; -+ ovpn_peer_put(tmp); -+ } -+ -+ rcu_assign_pointer(ovpn->peer, peer); -+ spin_unlock_bh(&ovpn->lock); -+ -+ return 0; -+} -+ -+/* assume refcounter was increased by caller */ -+int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer) -+{ -+ switch (ovpn->mode) { -+ case OVPN_MODE_MP: -+ return ovpn_peer_add_mp(ovpn, peer); -+ case OVPN_MODE_P2P: -+ return ovpn_peer_add_p2p(ovpn, peer); -+ default: -+ return -EOPNOTSUPP; -+ } -+} -+ -+static void ovpn_peer_unhash(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ hlist_del_init_rcu(&peer->hash_entry_id); -+ hlist_del_init_rcu(&peer->hash_entry_addr4); -+ hlist_del_init_rcu(&peer->hash_entry_addr6); -+ hlist_del_init_rcu(&peer->hash_entry_transp_addr); -+ -+ ovpn_peer_put(peer); -+ peer->delete_reason = reason; -+} -+ -+static int ovpn_peer_del_mp(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ struct ovpn_peer *tmp; -+ int ret = 0; -+ -+ spin_lock_bh(&peer->ovpn->peers.lock); -+ tmp = ovpn_peer_lookup_id(peer->ovpn, peer->id); -+ if (tmp != peer) { -+ ret = -ENOENT; -+ goto unlock; -+ } -+ ovpn_peer_unhash(peer, reason); -+ -+unlock: -+ spin_unlock_bh(&peer->ovpn->peers.lock); -+ -+ if (tmp) -+ ovpn_peer_put(tmp); -+ -+ return ret; -+} -+ -+static int ovpn_peer_del_p2p(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ struct ovpn_peer *tmp; -+ int ret = -ENOENT; -+ -+ spin_lock_bh(&peer->ovpn->lock); -+ tmp = rcu_dereference(peer->ovpn->peer); -+ if (tmp != peer) -+ goto unlock; -+ -+ ovpn_peer_put(tmp); -+ tmp->delete_reason = reason; -+ RCU_INIT_POINTER(peer->ovpn->peer, NULL); -+ ret = 0; -+ -+unlock: -+ spin_unlock_bh(&peer->ovpn->lock); -+ -+ return ret; -+} -+ -+void ovpn_peer_release_p2p(struct ovpn_struct *ovpn) -+{ -+ struct ovpn_peer *tmp; -+ -+ rcu_read_lock(); -+ tmp = rcu_dereference(ovpn->peer); -+ if (!tmp) -+ goto unlock; -+ -+ ovpn_peer_del_p2p(tmp, OVPN_DEL_PEER_REASON_TEARDOWN); -+unlock: -+ rcu_read_unlock(); -+} -+ -+int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason) -+{ -+ switch (peer->ovpn->mode) { -+ case OVPN_MODE_MP: -+ return ovpn_peer_del_mp(peer, reason); -+ case OVPN_MODE_P2P: -+ return ovpn_peer_del_p2p(peer, reason); -+ default: -+ return -EOPNOTSUPP; -+ } -+} -+ -+void ovpn_peers_free(struct ovpn_struct *ovpn) -+{ -+ struct hlist_node *tmp; -+ struct ovpn_peer *peer; -+ int bkt; -+ -+ spin_lock_bh(&ovpn->peers.lock); -+ hash_for_each_safe(ovpn->peers.by_id, bkt, tmp, peer, hash_entry_id) -+ ovpn_peer_unhash(peer, OVPN_DEL_PEER_REASON_TEARDOWN); -+ spin_unlock_bh(&ovpn->peers.lock); -+} -diff --git a/drivers/net/ovpn-dco/peer.h b/drivers/net/ovpn-dco/peer.h -new file mode 100644 -index 000000000000..b759c2c9da48 ---- /dev/null -+++ b/drivers/net/ovpn-dco/peer.h -@@ -0,0 +1,168 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNPEER_H_ -+#define _NET_OVPN_DCO_OVPNPEER_H_ -+ -+#include "addr.h" -+#include "bind.h" -+#include "sock.h" -+#include "stats.h" -+ -+#include -+#include -+#include -+ -+struct ovpn_peer { -+ struct ovpn_struct *ovpn; -+ -+ u32 id; -+ -+ struct { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } vpn_addrs; -+ -+ struct hlist_node hash_entry_id; -+ struct hlist_node hash_entry_addr4; -+ struct hlist_node hash_entry_addr6; -+ struct hlist_node hash_entry_transp_addr; -+ -+ /* work objects to handle encryption/decryption of packets. -+ * these works are queued on the ovpn->crypt_wq workqueue. -+ */ -+ struct work_struct encrypt_work; -+ struct work_struct decrypt_work; -+ -+ struct ptr_ring tx_ring; -+ struct ptr_ring rx_ring; -+ struct ptr_ring netif_rx_ring; -+ -+ struct napi_struct napi; -+ -+ struct ovpn_socket *sock; -+ -+ /* state of the TCP reading. Needed to keep track of how much of a single packet has already -+ * been read from the stream and how much is missing -+ */ -+ struct { -+ struct ptr_ring tx_ring; -+ struct work_struct tx_work; -+ struct work_struct rx_work; -+ -+ u8 raw_len[sizeof(u16)]; -+ struct sk_buff *skb; -+ u16 offset; -+ u16 data_len; -+ struct { -+ void (*sk_state_change)(struct sock *sk); -+ void (*sk_data_ready)(struct sock *sk); -+ void (*sk_write_space)(struct sock *sk); -+ } sk_cb; -+ } tcp; -+ -+ struct dst_cache dst_cache; -+ -+ /* our crypto state */ -+ struct ovpn_crypto_state crypto; -+ -+ /* our binding to peer, protected by spinlock */ -+ struct ovpn_bind __rcu *bind; -+ -+ /* timer used to send periodic ping messages to the other peer, if no -+ * other data was sent within the past keepalive_interval seconds -+ */ -+ struct timer_list keepalive_xmit; -+ /* keepalive interval in seconds */ -+ unsigned long keepalive_interval; -+ -+ /* timer used to mark a peer as expired when no data is received for -+ * keepalive_timeout seconds -+ */ -+ struct timer_list keepalive_recv; -+ /* keepalive timeout in seconds */ -+ unsigned long keepalive_timeout; -+ -+ /* true if ovpn_peer_mark_delete was called */ -+ bool halt; -+ -+ /* per-peer rx/tx stats */ -+ struct ovpn_peer_stats stats; -+ -+ /* why peer was deleted - keepalive timeout, module removed etc */ -+ enum ovpn_del_peer_reason delete_reason; -+ -+ /* protects binding to peer (bind) and timers -+ * (keepalive_xmit, keepalive_expire) -+ */ -+ spinlock_t lock; -+ -+ /* needed because crypto methods can go async */ -+ struct kref refcount; -+ -+ /* needed to free a peer in an RCU safe way */ -+ struct rcu_head rcu; -+ -+ /* needed to notify userspace about deletion */ -+ struct work_struct delete_work; -+}; -+ -+void ovpn_peer_release_kref(struct kref *kref); -+void ovpn_peer_release(struct ovpn_peer *peer); -+ -+static inline bool ovpn_peer_hold(struct ovpn_peer *peer) -+{ -+ return kref_get_unless_zero(&peer->refcount); -+} -+ -+static inline void ovpn_peer_put(struct ovpn_peer *peer) -+{ -+ kref_put(&peer->refcount, ovpn_peer_release_kref); -+} -+ -+static inline void ovpn_peer_keepalive_recv_reset(struct ovpn_peer *peer) -+{ -+ u32 delta = msecs_to_jiffies(peer->keepalive_timeout * MSEC_PER_SEC); -+ -+ if (unlikely(!delta)) -+ return; -+ -+ mod_timer(&peer->keepalive_recv, jiffies + delta); -+} -+ -+static inline void ovpn_peer_keepalive_xmit_reset(struct ovpn_peer *peer) -+{ -+ u32 delta = msecs_to_jiffies(peer->keepalive_interval * MSEC_PER_SEC); -+ -+ if (unlikely(!delta)) -+ return; -+ -+ mod_timer(&peer->keepalive_xmit, jiffies + delta); -+} -+ -+struct ovpn_peer *ovpn_peer_new(struct ovpn_struct *ovpn, const struct sockaddr_storage *sa, -+ struct socket *sock, u32 id, uint8_t *local_ip); -+ -+void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout); -+ -+int ovpn_peer_add(struct ovpn_struct *ovpn, struct ovpn_peer *peer); -+int ovpn_peer_del(struct ovpn_peer *peer, enum ovpn_del_peer_reason reason); -+struct ovpn_peer *ovpn_peer_find(struct ovpn_struct *ovpn, u32 peer_id); -+void ovpn_peer_release_p2p(struct ovpn_struct *ovpn); -+void ovpn_peers_free(struct ovpn_struct *ovpn); -+ -+struct ovpn_peer *ovpn_peer_lookup_transp_addr(struct ovpn_struct *ovpn, struct sk_buff *skb); -+struct ovpn_peer *ovpn_peer_lookup_vpn_addr(struct ovpn_struct *ovpn, struct sk_buff *skb, -+ bool use_src); -+struct ovpn_peer *ovpn_peer_lookup_id(struct ovpn_struct *ovpn, u32 peer_id); -+ -+void ovpn_peer_update_local_endpoint(struct ovpn_peer *peer, struct sk_buff *skb); -+void ovpn_peer_float(struct ovpn_peer *peer, struct sk_buff *skb); -+ -+#endif /* _NET_OVPN_DCO_OVPNPEER_H_ */ -diff --git a/drivers/net/ovpn-dco/pktid.c b/drivers/net/ovpn-dco/pktid.c -new file mode 100644 -index 000000000000..fcde8fba5156 ---- /dev/null -+++ b/drivers/net/ovpn-dco/pktid.c -@@ -0,0 +1,127 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#include "pktid.h" -+ -+#include -+#include -+ -+void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid) -+{ -+ atomic64_set(&pid->seq_num, 1); -+ pid->tcp_linear = NULL; -+} -+ -+void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr) -+{ -+ memset(pr, 0, sizeof(*pr)); -+ spin_lock_init(&pr->lock); -+} -+ -+/* Packet replay detection. -+ * Allows ID backtrack of up to REPLAY_WINDOW_SIZE - 1. -+ */ -+int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time) -+{ -+ const unsigned long now = jiffies; -+ int ret; -+ -+ spin_lock(&pr->lock); -+ -+ /* expire backtracks at or below pr->id after PKTID_RECV_EXPIRE time */ -+ if (unlikely(time_after_eq(now, pr->expire))) -+ pr->id_floor = pr->id; -+ -+ /* ID must not be zero */ -+ if (unlikely(pkt_id == 0)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ /* time changed? */ -+ if (unlikely(pkt_time != pr->time)) { -+ if (pkt_time > pr->time) { -+ /* time moved forward, accept */ -+ pr->base = 0; -+ pr->extent = 0; -+ pr->id = 0; -+ pr->time = pkt_time; -+ pr->id_floor = 0; -+ } else { -+ /* time moved backward, reject */ -+ ret = -ETIME; -+ goto out; -+ } -+ } -+ -+ if (likely(pkt_id == pr->id + 1)) { -+ /* well-formed ID sequence (incremented by 1) */ -+ pr->base = REPLAY_INDEX(pr->base, -1); -+ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); -+ if (pr->extent < REPLAY_WINDOW_SIZE) -+ ++pr->extent; -+ pr->id = pkt_id; -+ } else if (pkt_id > pr->id) { -+ /* ID jumped forward by more than one */ -+ const unsigned int delta = pkt_id - pr->id; -+ -+ if (delta < REPLAY_WINDOW_SIZE) { -+ unsigned int i; -+ -+ pr->base = REPLAY_INDEX(pr->base, -delta); -+ pr->history[pr->base / 8] |= (1 << (pr->base % 8)); -+ pr->extent += delta; -+ if (pr->extent > REPLAY_WINDOW_SIZE) -+ pr->extent = REPLAY_WINDOW_SIZE; -+ for (i = 1; i < delta; ++i) { -+ unsigned int newb = REPLAY_INDEX(pr->base, i); -+ -+ pr->history[newb / 8] &= ~BIT(newb % 8); -+ } -+ } else { -+ pr->base = 0; -+ pr->extent = REPLAY_WINDOW_SIZE; -+ memset(pr->history, 0, sizeof(pr->history)); -+ pr->history[0] = 1; -+ } -+ pr->id = pkt_id; -+ } else { -+ /* ID backtrack */ -+ const unsigned int delta = pr->id - pkt_id; -+ -+ if (delta > pr->max_backtrack) -+ pr->max_backtrack = delta; -+ if (delta < pr->extent) { -+ if (pkt_id > pr->id_floor) { -+ const unsigned int ri = REPLAY_INDEX(pr->base, -+ delta); -+ u8 *p = &pr->history[ri / 8]; -+ const u8 mask = (1 << (ri % 8)); -+ -+ if (*p & mask) { -+ ret = -EINVAL; -+ goto out; -+ } -+ *p |= mask; -+ } else { -+ ret = -EINVAL; -+ goto out; -+ } -+ } else { -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ -+ pr->expire = now + PKTID_RECV_EXPIRE; -+ ret = 0; -+out: -+ spin_unlock(&pr->lock); -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/pktid.h b/drivers/net/ovpn-dco/pktid.h -new file mode 100644 -index 000000000000..2447bb37ba55 ---- /dev/null -+++ b/drivers/net/ovpn-dco/pktid.h -@@ -0,0 +1,116 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNPKTID_H_ -+#define _NET_OVPN_DCO_OVPNPKTID_H_ -+ -+#include "main.h" -+ -+/* When the OpenVPN protocol is run in AEAD mode, use -+ * the OpenVPN packet ID as the AEAD nonce: -+ * -+ * 00000005 521c3b01 4308c041 -+ * [seq # ] [ nonce_tail ] -+ * [ 12-byte full IV ] -> NONCE_SIZE -+ * [4-bytes -> NONCE_WIRE_SIZE -+ * on wire] -+ */ -+ -+/* OpenVPN nonce size */ -+#define NONCE_SIZE 12 -+/* amount of bytes of the nonce received from user space */ -+#define NONCE_TAIL_SIZE 8 -+ -+/* OpenVPN nonce size reduced by 8-byte nonce tail -- this is the -+ * size of the AEAD Associated Data (AD) sent over the wire -+ * and is normally the head of the IV -+ */ -+#define NONCE_WIRE_SIZE (NONCE_SIZE - sizeof(struct ovpn_nonce_tail)) -+ -+/* If no packets received for this length of time, set a backtrack floor -+ * at highest received packet ID thus far. -+ */ -+#define PKTID_RECV_EXPIRE (30 * HZ) -+ -+/* Last 8 bytes of AEAD nonce -+ * Provided by userspace and usually derived from -+ * key material generated during TLS handshake -+ */ -+struct ovpn_nonce_tail { -+ u8 u8[NONCE_TAIL_SIZE]; -+}; -+ -+/* Packet-ID state for transmitter */ -+struct ovpn_pktid_xmit { -+ atomic64_t seq_num; -+ struct ovpn_tcp_linear *tcp_linear; -+}; -+ -+/* replay window sizing in bytes = 2^REPLAY_WINDOW_ORDER */ -+#define REPLAY_WINDOW_ORDER 8 -+ -+#define REPLAY_WINDOW_BYTES BIT(REPLAY_WINDOW_ORDER) -+#define REPLAY_WINDOW_SIZE (REPLAY_WINDOW_BYTES * 8) -+#define REPLAY_INDEX(base, i) (((base) + (i)) & (REPLAY_WINDOW_SIZE - 1)) -+ -+/* Packet-ID state for receiver. -+ * Other than lock member, can be zeroed to initialize. -+ */ -+struct ovpn_pktid_recv { -+ /* "sliding window" bitmask of recent packet IDs received */ -+ u8 history[REPLAY_WINDOW_BYTES]; -+ /* bit position of deque base in history */ -+ unsigned int base; -+ /* extent (in bits) of deque in history */ -+ unsigned int extent; -+ /* expiration of history in jiffies */ -+ unsigned long expire; -+ /* highest sequence number received */ -+ u32 id; -+ /* highest time stamp received */ -+ u32 time; -+ /* we will only accept backtrack IDs > id_floor */ -+ u32 id_floor; -+ unsigned int max_backtrack; -+ /* protects entire pktd ID state */ -+ spinlock_t lock; -+}; -+ -+/* Get the next packet ID for xmit */ -+static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) -+{ -+ const s64 seq_num = atomic64_fetch_add_unless(&pid->seq_num, 1, -+ 0x100000000LL); -+ /* when the 32bit space is over, we return an error because the packet ID is used to create -+ * the cipher IV and we do not want to re-use the same value more than once -+ */ -+ if (unlikely(seq_num == 0x100000000LL)) -+ return -ERANGE; -+ -+ *pktid = (u32)seq_num; -+ -+ return 0; -+} -+ -+/* Write 12-byte AEAD IV to dest */ -+static inline void ovpn_pktid_aead_write(const u32 pktid, -+ const struct ovpn_nonce_tail *nt, -+ unsigned char *dest) -+{ -+ *(__force __be32 *)(dest) = htonl(pktid); -+ BUILD_BUG_ON(4 + sizeof(struct ovpn_nonce_tail) != NONCE_SIZE); -+ memcpy(dest + 4, nt->u8, sizeof(struct ovpn_nonce_tail)); -+} -+ -+void ovpn_pktid_xmit_init(struct ovpn_pktid_xmit *pid); -+void ovpn_pktid_recv_init(struct ovpn_pktid_recv *pr); -+ -+int ovpn_pktid_recv(struct ovpn_pktid_recv *pr, u32 pkt_id, u32 pkt_time); -+ -+#endif /* _NET_OVPN_DCO_OVPNPKTID_H_ */ -diff --git a/drivers/net/ovpn-dco/proto.h b/drivers/net/ovpn-dco/proto.h -new file mode 100644 -index 000000000000..875529021c1b ---- /dev/null -+++ b/drivers/net/ovpn-dco/proto.h -@@ -0,0 +1,101 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNPROTO_H_ -+#define _NET_OVPN_DCO_OVPNPROTO_H_ -+ -+#include "main.h" -+ -+#include -+ -+/* Methods for operating on the initial command -+ * byte of the OpenVPN protocol. -+ */ -+ -+/* packet opcode (high 5 bits) and key-id (low 3 bits) are combined in -+ * one byte -+ */ -+#define OVPN_KEY_ID_MASK 0x07 -+#define OVPN_OPCODE_SHIFT 3 -+#define OVPN_OPCODE_MASK 0x1F -+/* upper bounds on opcode and key ID */ -+#define OVPN_KEY_ID_MAX (OVPN_KEY_ID_MASK + 1) -+#define OVPN_OPCODE_MAX (OVPN_OPCODE_MASK + 1) -+/* packet opcodes of interest to us */ -+#define OVPN_DATA_V1 6 /* data channel V1 packet */ -+#define OVPN_DATA_V2 9 /* data channel V2 packet */ -+/* size of initial packet opcode */ -+#define OVPN_OP_SIZE_V1 1 -+#define OVPN_OP_SIZE_V2 4 -+#define OVPN_PEER_ID_MASK 0x00FFFFFF -+#define OVPN_PEER_ID_UNDEF 0x00FFFFFF -+/* first byte of keepalive message */ -+#define OVPN_KEEPALIVE_FIRST_BYTE 0x2a -+/* first byte of exit message */ -+#define OVPN_EXPLICIT_EXIT_NOTIFY_FIRST_BYTE 0x28 -+ -+/** -+ * Extract the OP code from the specified byte -+ * -+ * Return the OP code -+ */ -+static inline u8 ovpn_opcode_from_byte(u8 byte) -+{ -+ return byte >> OVPN_OPCODE_SHIFT; -+} -+ -+/** -+ * Extract the OP code from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first byte. -+ * -+ * Return the OP code -+ */ -+static inline u8 ovpn_opcode_from_skb(const struct sk_buff *skb, u16 offset) -+{ -+ return ovpn_opcode_from_byte(*(skb->data + offset)); -+} -+ -+/** -+ * Extract the key ID from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first byte. -+ * -+ * Return the key ID -+ */ -+ -+static inline u8 ovpn_key_id_from_skb(const struct sk_buff *skb) -+{ -+ return *skb->data & OVPN_KEY_ID_MASK; -+} -+ -+/** -+ * Extract the peer ID from the skb head. -+ * -+ * Note: this function assumes that the skb head was pulled enough -+ * to access the first 4 bytes. -+ * -+ * Return the peer ID. -+ */ -+ -+static inline u32 ovpn_peer_id_from_skb(const struct sk_buff *skb, u16 offset) -+{ -+ return ntohl(*(__be32 *)(skb->data + offset)) & OVPN_PEER_ID_MASK; -+} -+ -+static inline u32 ovpn_opcode_compose(u8 opcode, u8 key_id, u32 peer_id) -+{ -+ const u8 op = (opcode << OVPN_OPCODE_SHIFT) | (key_id & OVPN_KEY_ID_MASK); -+ -+ return (op << 24) | (peer_id & OVPN_PEER_ID_MASK); -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNPROTO_H_ */ -diff --git a/drivers/net/ovpn-dco/rcu.h b/drivers/net/ovpn-dco/rcu.h -new file mode 100644 -index 000000000000..02a50f49ba2e ---- /dev/null -+++ b/drivers/net/ovpn-dco/rcu.h -@@ -0,0 +1,21 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNRCU_H_ -+#define _NET_OVPN_DCO_OVPNRCU_H_ -+ -+static inline void ovpn_rcu_lockdep_assert_held(void) -+{ -+#ifdef CONFIG_PROVE_RCU -+ RCU_LOCKDEP_WARN(!rcu_read_lock_held(), -+ "ovpn-dco RCU read lock not held"); -+#endif -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNRCU_H_ */ -diff --git a/drivers/net/ovpn-dco/skb.h b/drivers/net/ovpn-dco/skb.h -new file mode 100644 -index 000000000000..d38dc2da01df ---- /dev/null -+++ b/drivers/net/ovpn-dco/skb.h -@@ -0,0 +1,54 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ * James Yonan -+ */ -+ -+#ifndef _NET_OVPN_DCO_SKB_H_ -+#define _NET_OVPN_DCO_SKB_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define OVPN_SKB_CB(skb) ((struct ovpn_skb_cb *)&((skb)->cb)) -+ -+struct ovpn_skb_cb { -+ /* original recv packet size for stats accounting */ -+ unsigned int rx_stats_size; -+ -+ union { -+ struct in_addr ipv4; -+ struct in6_addr ipv6; -+ } local; -+ sa_family_t sa_fam; -+}; -+ -+/* Return IP protocol version from skb header. -+ * Return 0 if protocol is not IPv4/IPv6 or cannot be read. -+ */ -+static inline __be16 ovpn_ip_check_protocol(struct sk_buff *skb) -+{ -+ __be16 proto = 0; -+ -+ /* skb could be non-linear, -+ * make sure IP header is in non-fragmented part -+ */ -+ if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) -+ return 0; -+ -+ if (ip_hdr(skb)->version == 4) -+ proto = htons(ETH_P_IP); -+ else if (ip_hdr(skb)->version == 6) -+ proto = htons(ETH_P_IPV6); -+ -+ return proto; -+} -+ -+#endif /* _NET_OVPN_DCO_SKB_H_ */ -diff --git a/drivers/net/ovpn-dco/sock.c b/drivers/net/ovpn-dco/sock.c -new file mode 100644 -index 000000000000..e92a4a9b952e ---- /dev/null -+++ b/drivers/net/ovpn-dco/sock.c -@@ -0,0 +1,134 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "sock.h" -+#include "rcu.h" -+#include "tcp.h" -+#include "udp.h" -+ -+#include -+#include -+ -+/* Finalize release of socket, called after RCU grace period */ -+static void ovpn_socket_detach(struct socket *sock) -+{ -+ if (!sock) -+ return; -+ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) -+ ovpn_udp_socket_detach(sock); -+ else if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ovpn_tcp_socket_detach(sock); -+ -+ sockfd_put(sock); -+} -+ -+void ovpn_socket_release_kref(struct kref *kref) -+{ -+ struct ovpn_socket *sock = container_of(kref, struct ovpn_socket, refcount); -+ -+ ovpn_socket_detach(sock->sock); -+ kfree_rcu(sock, rcu); -+} -+ -+static bool ovpn_socket_hold(struct ovpn_socket *sock) -+{ -+ return kref_get_unless_zero(&sock->refcount); -+} -+ -+static struct ovpn_socket *ovpn_socket_get(struct socket *sock) -+{ -+ struct ovpn_socket *ovpn_sock; -+ -+ rcu_read_lock(); -+ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); -+ if (!ovpn_socket_hold(ovpn_sock)) { -+ pr_warn("%s: found ovpn_socket with ref = 0\n", __func__); -+ ovpn_sock = NULL; -+ } -+ rcu_read_unlock(); -+ -+ return ovpn_sock; -+} -+ -+/* Finalize release of socket, called after RCU grace period */ -+static int ovpn_socket_attach(struct socket *sock, struct ovpn_peer *peer) -+{ -+ int ret = -EOPNOTSUPP; -+ -+ if (!sock || !peer) -+ return -EINVAL; -+ -+ if (sock->sk->sk_protocol == IPPROTO_UDP) -+ ret = ovpn_udp_socket_attach(sock, peer->ovpn); -+ else if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ret = ovpn_tcp_socket_attach(sock, peer); -+ -+ return ret; -+} -+ -+struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk) -+{ -+ struct ovpn_socket *ovpn_sock; -+ -+ ovpn_rcu_lockdep_assert_held(); -+ -+ if (unlikely(READ_ONCE(udp_sk(sk)->encap_type) != UDP_ENCAP_OVPNINUDP)) -+ return NULL; -+ -+ ovpn_sock = rcu_dereference_sk_user_data(sk); -+ if (unlikely(!ovpn_sock)) -+ return NULL; -+ -+ /* make sure that sk matches our stored transport socket */ -+ if (unlikely(!ovpn_sock->sock || sk != ovpn_sock->sock->sk)) -+ return NULL; -+ -+ return ovpn_sock->ovpn; -+} -+ -+struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) -+{ -+ struct ovpn_socket *ovpn_sock; -+ int ret; -+ -+ ret = ovpn_socket_attach(sock, peer); -+ if (ret < 0 && ret != -EALREADY) -+ return ERR_PTR(ret); -+ -+ /* if this socket is already owned by this interface, just increase the refcounter */ -+ if (ret == -EALREADY) { -+ /* caller is expected to increase the sock refcounter before passing it to this -+ * function. For this reason we drop it if not needed, like when this socket is -+ * already owned. -+ */ -+ ovpn_sock = ovpn_socket_get(sock); -+ sockfd_put(sock); -+ return ovpn_sock; -+ } -+ -+ ovpn_sock = kzalloc(sizeof(*ovpn_sock), GFP_KERNEL); -+ if (!ovpn_sock) -+ return ERR_PTR(-ENOMEM); -+ -+ ovpn_sock->ovpn = peer->ovpn; -+ ovpn_sock->sock = sock; -+ kref_init(&ovpn_sock->refcount); -+ -+ /* TCP sockets are per-peer, therefore they are linked to their unique peer */ -+ if (sock->sk->sk_protocol == IPPROTO_TCP) -+ ovpn_sock->peer = peer; -+ -+ rcu_assign_sk_user_data(sock->sk, ovpn_sock); -+ -+ return ovpn_sock; -+} -diff --git a/drivers/net/ovpn-dco/sock.h b/drivers/net/ovpn-dco/sock.h -new file mode 100644 -index 000000000000..9e79c1b5fe04 ---- /dev/null -+++ b/drivers/net/ovpn-dco/sock.h -@@ -0,0 +1,54 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_SOCK_H_ -+#define _NET_OVPN_DCO_SOCK_H_ -+ -+#include -+#include -+#include -+ -+#include "peer.h" -+ -+struct ovpn_struct; -+ -+/** -+ * struct ovpn_socket - a kernel socket referenced in the ovpn-dco code -+ */ -+struct ovpn_socket { -+ union { -+ /** @ovpn: the VPN session object owning this socket (UDP only) */ -+ struct ovpn_struct *ovpn; -+ -+ /** @peer: the unique peer transmitting over this socket (TCP only) */ -+ struct ovpn_peer *peer; -+ }; -+ -+ /** @sock: the kernel socket */ -+ struct socket *sock; -+ -+ /** @refcount: amount of contexts currently referencing this object */ -+ struct kref refcount; -+ -+ /** @rcu: member used to schedule RCU destructor callback */ -+ struct rcu_head rcu; -+}; -+ -+struct ovpn_struct *ovpn_from_udp_sock(struct sock *sk); -+ -+void ovpn_socket_release_kref(struct kref *kref); -+ -+static inline void ovpn_socket_put(struct ovpn_socket *sock) -+{ -+ kref_put(&sock->refcount, ovpn_socket_release_kref); -+} -+ -+struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer); -+ -+#endif /* _NET_OVPN_DCO_SOCK_H_ */ -diff --git a/drivers/net/ovpn-dco/stats.c b/drivers/net/ovpn-dco/stats.c -new file mode 100644 -index 000000000000..ee000b2a2177 ---- /dev/null -+++ b/drivers/net/ovpn-dco/stats.c -@@ -0,0 +1,20 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "stats.h" -+ -+void ovpn_peer_stats_init(struct ovpn_peer_stats *ps) -+{ -+ atomic64_set(&ps->rx.bytes, 0); -+ atomic_set(&ps->rx.packets, 0); -+ -+ atomic64_set(&ps->tx.bytes, 0); -+ atomic_set(&ps->tx.packets, 0); -+} -diff --git a/drivers/net/ovpn-dco/stats.h b/drivers/net/ovpn-dco/stats.h -new file mode 100644 -index 000000000000..3aa6bdc049c6 ---- /dev/null -+++ b/drivers/net/ovpn-dco/stats.h -@@ -0,0 +1,67 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2020-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ * Lev Stipakov -+ */ -+ -+#ifndef _NET_OVPN_DCO_OVPNSTATS_H_ -+#define _NET_OVPN_DCO_OVPNSTATS_H_ -+ -+#include -+#include -+ -+struct ovpn_struct; -+ -+/* per-peer stats, measured on transport layer */ -+ -+/* one stat */ -+struct ovpn_peer_stat { -+ atomic64_t bytes; -+ atomic_t packets; -+}; -+ -+/* rx and tx stats, enabled by notify_per != 0 or period != 0 */ -+struct ovpn_peer_stats { -+ struct ovpn_peer_stat rx; -+ struct ovpn_peer_stat tx; -+}; -+ -+/* struct for OVPN_ERR_STATS */ -+ -+struct ovpn_err_stat { -+ unsigned int category; -+ int errcode; -+ u64 count; -+}; -+ -+struct ovpn_err_stats { -+ /* total stats, returned by kovpn */ -+ unsigned int total_stats; -+ /* number of stats dimensioned below */ -+ unsigned int n_stats; -+ struct ovpn_err_stat stats[]; -+}; -+ -+void ovpn_peer_stats_init(struct ovpn_peer_stats *ps); -+ -+static inline void ovpn_peer_stats_increment(struct ovpn_peer_stat *stat, const unsigned int n) -+{ -+ atomic64_add(n, &stat->bytes); -+ atomic_inc(&stat->packets); -+} -+ -+static inline void ovpn_peer_stats_increment_rx(struct ovpn_peer_stats *stats, const unsigned int n) -+{ -+ ovpn_peer_stats_increment(&stats->rx, n); -+} -+ -+static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats, const unsigned int n) -+{ -+ ovpn_peer_stats_increment(&stats->tx, n); -+} -+ -+#endif /* _NET_OVPN_DCO_OVPNSTATS_H_ */ -diff --git a/drivers/net/ovpn-dco/tcp.c b/drivers/net/ovpn-dco/tcp.c -new file mode 100644 -index 000000000000..7e6690fee6e7 ---- /dev/null -+++ b/drivers/net/ovpn-dco/tcp.c -@@ -0,0 +1,326 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "ovpnstruct.h" -+#include "ovpn.h" -+#include "peer.h" -+#include "skb.h" -+#include "tcp.h" -+ -+#include -+#include -+#include -+ -+static void ovpn_tcp_state_change(struct sock *sk) -+{ -+} -+ -+static void ovpn_tcp_data_ready(struct sock *sk) -+{ -+ struct ovpn_socket *sock; -+ -+ rcu_read_lock(); -+ sock = rcu_dereference_sk_user_data(sk); -+ rcu_read_unlock(); -+ -+ if (!sock || !sock->peer) -+ return; -+ -+ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.rx_work); -+} -+ -+static void ovpn_tcp_write_space(struct sock *sk) -+{ -+ struct ovpn_socket *sock; -+ -+ rcu_read_lock(); -+ sock = rcu_dereference_sk_user_data(sk); -+ rcu_read_unlock(); -+ -+ if (!sock || !sock->peer) -+ return; -+ -+ queue_work(sock->peer->ovpn->events_wq, &sock->peer->tcp.tx_work); -+} -+ -+static void ovpn_destroy_skb(void *skb) -+{ -+ consume_skb(skb); -+} -+ -+void ovpn_tcp_socket_detach(struct socket *sock) -+{ -+ struct ovpn_socket *ovpn_sock; -+ struct ovpn_peer *peer; -+ -+ if (!sock) -+ return; -+ -+ rcu_read_lock(); -+ ovpn_sock = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ -+ if (!ovpn_sock->peer) -+ return; -+ -+ peer = ovpn_sock->peer; -+ -+ /* restore CBs that were saved in ovpn_sock_set_tcp_cb() */ -+ write_lock_bh(&sock->sk->sk_callback_lock); -+ sock->sk->sk_state_change = peer->tcp.sk_cb.sk_state_change; -+ sock->sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; -+ sock->sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; -+ rcu_assign_sk_user_data(sock->sk, NULL); -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ -+ /* cancel any ongoing work. Done after removing the CBs so that these workers cannot be -+ * re-armed -+ */ -+ cancel_work_sync(&peer->tcp.tx_work); -+ cancel_work_sync(&peer->tcp.rx_work); -+ -+ ptr_ring_cleanup(&peer->tcp.tx_ring, ovpn_destroy_skb); -+} -+ -+/* Try to send one skb (or part of it) over the TCP stream. -+ * -+ * Return 0 on success or a negative error code otherwise. -+ * -+ * Note that the skb is modified by putting away the data being sent, therefore -+ * the caller should check if skb->len is zero to understand if the full skb was -+ * sent or not. -+ */ -+static int ovpn_tcp_send_one(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; -+ struct kvec iv = { 0 }; -+ int ret; -+ -+ if (skb_linearize(skb) < 0) { -+ net_err_ratelimited("%s: can't linearize packet\n", __func__); -+ return -ENOMEM; -+ } -+ -+ /* initialize iv structure now as skb_linearize() may have changed skb->data */ -+ iv.iov_base = skb->data; -+ iv.iov_len = skb->len; -+ -+ ret = kernel_sendmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len); -+ if (ret > 0) { -+ __skb_pull(skb, ret); -+ -+ /* since we update per-cpu stats in process context, -+ * we need to disable softirqs -+ */ -+ local_bh_disable(); -+ dev_sw_netstats_tx_add(peer->ovpn->dev, 1, ret); -+ local_bh_enable(); -+ -+ return 0; -+ } -+ -+ return ret; -+} -+ -+/* Process packets in TCP TX queue */ -+static void ovpn_tcp_tx_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer; -+ struct sk_buff *skb; -+ int ret; -+ -+ peer = container_of(work, struct ovpn_peer, tcp.tx_work); -+ while ((skb = __ptr_ring_peek(&peer->tcp.tx_ring))) { -+ ret = ovpn_tcp_send_one(peer, skb); -+ if (ret < 0 && ret != -EAGAIN) { -+ net_warn_ratelimited("%s: cannot send TCP packet to peer %u: %d\n", __func__, -+ peer->id, ret); -+ /* in case of TCP error stop sending loop and delete peer */ -+ ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_ERROR); -+ break; -+ } else if (!skb->len) { -+ /* skb was entirely consumed and can now be removed from the ring */ -+ __ptr_ring_discard_one(&peer->tcp.tx_ring); -+ consume_skb(skb); -+ } -+ -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ } -+} -+ -+static int ovpn_tcp_rx_one(struct ovpn_peer *peer) -+{ -+ struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL }; -+ struct ovpn_skb_cb *cb; -+ int status, ret; -+ -+ /* no skb allocated means that we have to read (or finish reading) the 2 bytes prefix -+ * containing the actual packet size. -+ */ -+ if (!peer->tcp.skb) { -+ struct kvec iv = { -+ .iov_base = peer->tcp.raw_len + peer->tcp.offset, -+ .iov_len = sizeof(u16) - peer->tcp.offset, -+ }; -+ -+ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); -+ if (ret <= 0) -+ return ret; -+ -+ peer->tcp.offset += ret; -+ /* the entire packet size was read, prepare skb for reading data */ -+ if (peer->tcp.offset == sizeof(u16)) { -+ u16 len = ntohs(*(__be16 *)peer->tcp.raw_len); -+ /* invalid packet length: this is a fatal TCP error */ -+ if (!len) { -+ netdev_err(peer->ovpn->dev, "%s: received invalid packet length\n", -+ __func__); -+ return -EINVAL; -+ } -+ -+ peer->tcp.skb = netdev_alloc_skb_ip_align(peer->ovpn->dev, len); -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = len; -+ } -+ } else { -+ struct kvec iv = { -+ .iov_base = peer->tcp.skb->data + peer->tcp.offset, -+ .iov_len = peer->tcp.data_len - peer->tcp.offset, -+ }; -+ -+ ret = kernel_recvmsg(peer->sock->sock, &msg, &iv, 1, iv.iov_len, msg.msg_flags); -+ if (ret <= 0) -+ return ret; -+ -+ peer->tcp.offset += ret; -+ /* full packet received, send it up for processing */ -+ if (peer->tcp.offset == peer->tcp.data_len) { -+ /* update the skb data structure with the amount of data written by -+ * kernel_recvmsg() -+ */ -+ skb_put(peer->tcp.skb, peer->tcp.data_len); -+ -+ /* do not perform IP caching for TCP connections */ -+ cb = OVPN_SKB_CB(peer->tcp.skb); -+ cb->sa_fam = AF_UNSPEC; -+ -+ /* hold reference to peer as requird by ovpn_recv() */ -+ ovpn_peer_hold(peer); -+ status = ovpn_recv(peer->ovpn, peer, peer->tcp.skb); -+ /* skb not consumed - free it now */ -+ if (unlikely(status < 0)) -+ kfree_skb(peer->tcp.skb); -+ -+ peer->tcp.skb = NULL; -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = 0; -+ } -+ } -+ -+ return ret; -+} -+ -+static void ovpn_tcp_rx_work(struct work_struct *work) -+{ -+ struct ovpn_peer *peer = container_of(work, struct ovpn_peer, tcp.rx_work); -+ int ret; -+ -+ while (true) { -+ /* give a chance to be rescheduled if needed */ -+ cond_resched(); -+ -+ ret = ovpn_tcp_rx_one(peer); -+ if (ret <= 0) -+ break; -+ } -+ -+ if (ret < 0 && ret != -EAGAIN) -+ netdev_err(peer->ovpn->dev, "%s: TCP socket error: %d\n", __func__, ret); -+} -+ -+/* Put packet into TCP TX queue and schedule a consumer */ -+void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ int ret; -+ -+ ret = ptr_ring_produce_bh(&peer->tcp.tx_ring, skb); -+ if (ret < 0) { -+ kfree_skb_list(skb); -+ return; -+ } -+ -+ queue_work(peer->ovpn->events_wq, &peer->tcp.tx_work); -+} -+ -+/* Set TCP encapsulation callbacks */ -+int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer) -+{ -+ void *old_data; -+ int ret; -+ -+ INIT_WORK(&peer->tcp.tx_work, ovpn_tcp_tx_work); -+ INIT_WORK(&peer->tcp.rx_work, ovpn_tcp_rx_work); -+ -+ ret = ptr_ring_init(&peer->tcp.tx_ring, OVPN_QUEUE_LEN, GFP_KERNEL); -+ if (ret < 0) { -+ netdev_err(peer->ovpn->dev, "cannot allocate TCP TX ring\n"); -+ return ret; -+ } -+ -+ peer->tcp.skb = NULL; -+ peer->tcp.offset = 0; -+ peer->tcp.data_len = 0; -+ -+ write_lock_bh(&sock->sk->sk_callback_lock); -+ -+ /* make sure no pre-existing encapsulation handler exists */ -+ rcu_read_lock(); -+ old_data = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ if (old_data) { -+ netdev_err(peer->ovpn->dev, "provided socket already taken by other user\n"); -+ ret = -EBUSY; -+ goto err; -+ } -+ -+ /* sanity check */ -+ if (sock->sk->sk_protocol != IPPROTO_TCP) { -+ netdev_err(peer->ovpn->dev, "expected TCP socket\n"); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ /* only a fully connected socket are expected. Connection should be handled in userspace */ -+ if (sock->sk->sk_state != TCP_ESTABLISHED) { -+ netdev_err(peer->ovpn->dev, "unexpected state for TCP socket: %d\n", -+ sock->sk->sk_state); -+ ret = -EINVAL; -+ goto err; -+ } -+ -+ /* save current CBs so that they can be restored upon socket release */ -+ peer->tcp.sk_cb.sk_state_change = sock->sk->sk_state_change; -+ peer->tcp.sk_cb.sk_data_ready = sock->sk->sk_data_ready; -+ peer->tcp.sk_cb.sk_write_space = sock->sk->sk_write_space; -+ -+ /* assign our static CBs */ -+ sock->sk->sk_state_change = ovpn_tcp_state_change; -+ sock->sk->sk_data_ready = ovpn_tcp_data_ready; -+ sock->sk->sk_write_space = ovpn_tcp_write_space; -+ -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ -+ return 0; -+err: -+ write_unlock_bh(&sock->sk->sk_callback_lock); -+ ptr_ring_cleanup(&peer->tcp.tx_ring, NULL); -+ -+ return ret; -+} -diff --git a/drivers/net/ovpn-dco/tcp.h b/drivers/net/ovpn-dco/tcp.h -new file mode 100644 -index 000000000000..d243a8e1c34e ---- /dev/null -+++ b/drivers/net/ovpn-dco/tcp.h -@@ -0,0 +1,38 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_TCP_H_ -+#define _NET_OVPN_DCO_TCP_H_ -+ -+#include "peer.h" -+ -+#include -+#include -+#include -+#include -+ -+void ovpn_queue_tcp_skb(struct ovpn_peer *peer, struct sk_buff *skb); -+ -+int ovpn_tcp_socket_attach(struct socket *sock, struct ovpn_peer *peer); -+void ovpn_tcp_socket_detach(struct socket *sock); -+ -+/* Prepare skb and enqueue it for sending to peer. -+ * -+ * Preparation consist in prepending the skb payload with its size. -+ * Required by the OpenVPN protocol in order to extract packets from -+ * the TCP stream on the receiver side. -+ */ -+static inline void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sk_buff *skb) -+{ -+ u16 len = skb->len; -+ -+ *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(len); -+ ovpn_queue_tcp_skb(peer, skb); -+} -+ -+#endif /* _NET_OVPN_DCO_TCP_H_ */ -diff --git a/drivers/net/ovpn-dco/udp.c b/drivers/net/ovpn-dco/udp.c -new file mode 100644 -index 000000000000..afa236d1f15c ---- /dev/null -+++ b/drivers/net/ovpn-dco/udp.c -@@ -0,0 +1,343 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#include "main.h" -+#include "bind.h" -+#include "ovpn.h" -+#include "ovpnstruct.h" -+#include "peer.h" -+#include "proto.h" -+#include "skb.h" -+#include "udp.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/** -+ * ovpn_udp_encap_recv() - Start processing a received UDP packet. -+ * If the first byte of the payload is DATA_V2, the packet is further processed, -+ * otherwise it is forwarded to the UDP stack for delivery to user space. -+ * -+ * @sk: the socket the packet was received on -+ * @skb: the sk_buff containing the actual packet -+ * -+ * Return codes: -+ * 0 : we consumed or dropped packet -+ * >0 : skb should be passed up to userspace as UDP (packet not consumed) -+ * <0 : skb should be resubmitted as proto -N (packet not consumed) -+ */ -+static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) -+{ -+ struct ovpn_peer *peer = NULL; -+ struct ovpn_struct *ovpn; -+ u32 peer_id; -+ u8 opcode; -+ int ret; -+ -+ ovpn = ovpn_from_udp_sock(sk); -+ if (unlikely(!ovpn)) { -+ net_err_ratelimited("%s: cannot obtain ovpn object from UDP socket\n", __func__); -+ goto drop; -+ } -+ -+ /* Make sure the first 4 bytes of the skb data buffer after the UDP header are accessible. -+ * They are required to fetch the OP code, the key ID and the peer ID. -+ */ -+ if (unlikely(!pskb_may_pull(skb, sizeof(struct udphdr) + 4))) { -+ net_dbg_ratelimited("%s: packet too small\n", __func__); -+ goto drop; -+ } -+ -+ opcode = ovpn_opcode_from_skb(skb, sizeof(struct udphdr)); -+ if (likely(opcode == OVPN_DATA_V2)) { -+ peer_id = ovpn_peer_id_from_skb(skb, sizeof(struct udphdr)); -+ /* some OpenVPN server implementations send data packets with the peer-id set to -+ * undef. In this case we skip the peer lookup by peer-id and we try with the -+ * transport address -+ */ -+ if (peer_id != OVPN_PEER_ID_UNDEF) { -+ peer = ovpn_peer_lookup_id(ovpn, peer_id); -+ if (!peer) { -+ net_err_ratelimited("%s: received data from unknown peer (id: %d)\n", -+ __func__, peer_id); -+ goto drop; -+ } -+ -+ /* check if this peer changed it's IP address and update state */ -+ ovpn_peer_float(peer, skb); -+ } -+ } -+ -+ if (!peer) { -+ /* might be a control packet or a data packet with undef peer-id */ -+ peer = ovpn_peer_lookup_transp_addr(ovpn, skb); -+ if (unlikely(!peer)) { -+ if (opcode != OVPN_DATA_V2) { -+ netdev_dbg(ovpn->dev, -+ "%s: control packet from unknown peer, sending to userspace", -+ __func__); -+ return 1; -+ } -+ -+ netdev_dbg(ovpn->dev, -+ "%s: received data with undef peer-id from unknown source\n", -+ __func__); -+ goto drop; -+ } -+ } -+ -+ /* pop off outer UDP header */ -+ __skb_pull(skb, sizeof(struct udphdr)); -+ -+ ret = ovpn_recv(ovpn, peer, skb); -+ if (unlikely(ret < 0)) { -+ net_err_ratelimited("%s: cannot handle incoming packet: %d\n", __func__, ret); -+ goto drop; -+ } -+ -+ /* should this be a non DATA_V2 packet, ret will be >0 and this will instruct the UDP -+ * stack to continue processing this packet as usual (i.e. deliver to user space) -+ */ -+ return ret; -+ -+drop: -+ if (peer) -+ ovpn_peer_put(peer); -+ kfree_skb(skb); -+ return 0; -+} -+ -+static int ovpn_udp4_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ struct rtable *rt; -+ struct flowi4 fl = { -+ .saddr = bind->local.ipv4.s_addr, -+ .daddr = bind->sa.in4.sin_addr.s_addr, -+ .fl4_sport = inet_sk(sk)->inet_sport, -+ .fl4_dport = bind->sa.in4.sin_port, -+ .flowi4_proto = sk->sk_protocol, -+ .flowi4_mark = sk->sk_mark, -+ }; -+ int ret; -+ -+ local_bh_disable(); -+ rt = dst_cache_get_ip4(cache, &fl.saddr); -+ if (rt) -+ goto transmit; -+ -+ if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, RT_SCOPE_HOST))) { -+ /* we may end up here when the cached address is not usable anymore. -+ * In this case we reset address/cache and perform a new look up -+ */ -+ fl.saddr = 0; -+ bind->local.ipv4.s_addr = 0; -+ dst_cache_reset(cache); -+ } -+ -+ rt = ip_route_output_flow(sock_net(sk), &fl, sk); -+ if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { -+ fl.saddr = 0; -+ bind->local.ipv4.s_addr = 0; -+ dst_cache_reset(cache); -+ -+ rt = ip_route_output_flow(sock_net(sk), &fl, sk); -+ } -+ -+ if (IS_ERR(rt)) { -+ ret = PTR_ERR(rt); -+ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, -+ &bind->sa.in4, ret); -+ goto err; -+ } -+ dst_cache_set_ip4(cache, &rt->dst, fl.saddr); -+ -+transmit: -+ udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, -+ ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, -+ fl.fl4_dport, false, sk->sk_no_check_tx); -+ ret = 0; -+err: -+ local_bh_enable(); -+ return ret; -+} -+ -+#if IS_ENABLED(CONFIG_IPV6) -+static int ovpn_udp6_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ struct dst_entry *dst; -+ int ret; -+ -+ struct flowi6 fl = { -+ .saddr = bind->local.ipv6, -+ .daddr = bind->sa.in6.sin6_addr, -+ .fl6_sport = inet_sk(sk)->inet_sport, -+ .fl6_dport = bind->sa.in6.sin6_port, -+ .flowi6_proto = sk->sk_protocol, -+ .flowi6_mark = sk->sk_mark, -+ .flowi6_oif = bind->sa.in6.sin6_scope_id, -+ }; -+ -+ local_bh_disable(); -+ dst = dst_cache_get_ip6(cache, &fl.saddr); -+ if (dst) -+ goto transmit; -+ -+ if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { -+ /* we may end up here when the cached address is not usable anymore. -+ * In this case we reset address/cache and perform a new look up -+ */ -+ fl.saddr = in6addr_any; -+ bind->local.ipv6 = in6addr_any; -+ dst_cache_reset(cache); -+ } -+ -+ dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); -+ if (IS_ERR(dst)) { -+ ret = PTR_ERR(dst); -+ net_dbg_ratelimited("%s: no route to host %pISpc: %d\n", ovpn->dev->name, -+ &bind->sa.in6, ret); -+ goto err; -+ } -+ dst_cache_set_ip6(cache, dst, &fl.saddr); -+ -+transmit: -+ udp_tunnel6_xmit_skb(dst, sk, skb, skb->dev, &fl.saddr, &fl.daddr, 0, -+ ip6_dst_hoplimit(dst), 0, fl.fl6_sport, -+ fl.fl6_dport, udp_get_no_check6_tx(sk)); -+ ret = 0; -+err: -+ local_bh_enable(); -+ return ret; -+} -+#endif -+ -+/* Transmit skb utilizing kernel-provided UDP tunneling framework. -+ * -+ * rcu_read_lock should be held on entry. -+ * On return, the skb is consumed. -+ */ -+static int ovpn_udp_output(struct ovpn_struct *ovpn, struct ovpn_bind *bind, -+ struct dst_cache *cache, struct sock *sk, -+ struct sk_buff *skb) -+{ -+ int ret; -+ -+ ovpn_rcu_lockdep_assert_held(); -+ -+ /* set sk to null if skb is already orphaned */ -+ if (!skb->destructor) -+ skb->sk = NULL; -+ -+ switch (bind->sa.in4.sin_family) { -+ case AF_INET: -+ ret = ovpn_udp4_output(ovpn, bind, cache, sk, skb); -+ break; -+#if IS_ENABLED(CONFIG_IPV6) -+ case AF_INET6: -+ ret = ovpn_udp6_output(ovpn, bind, cache, sk, skb); -+ break; -+#endif -+ default: -+ ret = -EAFNOSUPPORT; -+ break; -+ } -+ -+ return ret; -+} -+ -+void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, -+ struct sk_buff *skb) -+{ -+ struct ovpn_bind *bind; -+ struct socket *sock; -+ int ret = -1; -+ -+ skb->dev = ovpn->dev; -+ /* no checksum performed at this layer */ -+ skb->ip_summed = CHECKSUM_NONE; -+ -+ /* get socket info */ -+ sock = peer->sock->sock; -+ if (unlikely(!sock)) { -+ net_dbg_ratelimited("%s: no sock for remote peer\n", __func__); -+ goto out; -+ } -+ -+ rcu_read_lock(); -+ /* get binding */ -+ bind = rcu_dereference(peer->bind); -+ if (unlikely(!bind)) { -+ net_dbg_ratelimited("%s: no bind for remote peer\n", __func__); -+ goto out_unlock; -+ } -+ -+ /* crypto layer -> transport (UDP) */ -+ ret = ovpn_udp_output(ovpn, bind, &peer->dst_cache, sock->sk, skb); -+ -+out_unlock: -+ rcu_read_unlock(); -+out: -+ if (ret < 0) -+ kfree_skb(skb); -+} -+ -+/* Set UDP encapsulation callbacks */ -+int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn) -+{ -+ struct udp_tunnel_sock_cfg cfg = { -+ .sk_user_data = ovpn, -+ .encap_type = UDP_ENCAP_OVPNINUDP, -+ .encap_rcv = ovpn_udp_encap_recv, -+ }; -+ struct ovpn_socket *old_data; -+ -+ /* sanity check */ -+ if (sock->sk->sk_protocol != IPPROTO_UDP) { -+ netdev_err(ovpn->dev, "%s: expected UDP socket\n", __func__); -+ return -EINVAL; -+ } -+ -+ /* make sure no pre-existing encapsulation handler exists */ -+ rcu_read_lock(); -+ old_data = rcu_dereference_sk_user_data(sock->sk); -+ rcu_read_unlock(); -+ if (old_data) { -+ if (old_data->ovpn == ovpn) { -+ netdev_dbg(ovpn->dev, -+ "%s: provided socket already owned by this interface\n", -+ __func__); -+ return -EALREADY; -+ } -+ -+ netdev_err(ovpn->dev, "%s: provided socket already taken by other user\n", -+ __func__); -+ return -EBUSY; -+ } -+ -+ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); -+ -+ return 0; -+} -+ -+/* Detach socket from encapsulation handler and/or other callbacks */ -+void ovpn_udp_socket_detach(struct socket *sock) -+{ -+ struct udp_tunnel_sock_cfg cfg = { }; -+ -+ setup_udp_tunnel_sock(sock_net(sock->sk), sock, &cfg); -+} -diff --git a/drivers/net/ovpn-dco/udp.h b/drivers/net/ovpn-dco/udp.h -new file mode 100644 -index 000000000000..be94fb74669b ---- /dev/null -+++ b/drivers/net/ovpn-dco/udp.h -@@ -0,0 +1,25 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: Antonio Quartulli -+ */ -+ -+#ifndef _NET_OVPN_DCO_UDP_H_ -+#define _NET_OVPN_DCO_UDP_H_ -+ -+#include "peer.h" -+#include "ovpnstruct.h" -+ -+#include -+#include -+#include -+#include -+ -+int ovpn_udp_socket_attach(struct socket *sock, struct ovpn_struct *ovpn); -+void ovpn_udp_socket_detach(struct socket *sock); -+void ovpn_udp_send_skb(struct ovpn_struct *ovpn, struct ovpn_peer *peer, -+ struct sk_buff *skb); -+ -+#endif /* _NET_OVPN_DCO_UDP_H_ */ -diff --git a/include/net/netlink.h b/include/net/netlink.h -index 7a2a9d3144ba..335f44871529 100644 ---- a/include/net/netlink.h -+++ b/include/net/netlink.h -@@ -441,6 +441,7 @@ struct nla_policy { - .max = _len \ - } - #define NLA_POLICY_MIN_LEN(_len) NLA_POLICY_MIN(NLA_BINARY, _len) -+#define NLA_POLICY_MAX_LEN(_len) NLA_POLICY_MAX(NLA_BINARY, _len) - - /** - * struct nl_info - netlink source information -diff --git a/include/uapi/linux/ovpn_dco.h b/include/uapi/linux/ovpn_dco.h -new file mode 100644 -index 000000000000..6afee8b3fedd ---- /dev/null -+++ b/include/uapi/linux/ovpn_dco.h -@@ -0,0 +1,265 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * OpenVPN data channel accelerator -+ * -+ * Copyright (C) 2019-2022 OpenVPN, Inc. -+ * -+ * Author: James Yonan -+ * Antonio Quartulli -+ */ -+ -+#ifndef _UAPI_LINUX_OVPN_DCO_H_ -+#define _UAPI_LINUX_OVPN_DCO_H_ -+ -+#define OVPN_NL_NAME "ovpn-dco" -+ -+#define OVPN_NL_MULTICAST_GROUP_PEERS "peers" -+ -+/** -+ * enum ovpn_nl_commands - supported netlink commands -+ */ -+enum ovpn_nl_commands { -+ /** -+ * @OVPN_CMD_UNSPEC: unspecified command to catch errors -+ */ -+ OVPN_CMD_UNSPEC = 0, -+ -+ /** -+ * @OVPN_CMD_NEW_PEER: Configure peer with its crypto keys -+ */ -+ OVPN_CMD_NEW_PEER, -+ -+ /** -+ * @OVPN_CMD_SET_PEER: Tweak parameters for an existing peer -+ */ -+ OVPN_CMD_SET_PEER, -+ -+ /** -+ * @OVPN_CMD_DEL_PEER: Remove peer from internal table -+ */ -+ OVPN_CMD_DEL_PEER, -+ -+ OVPN_CMD_NEW_KEY, -+ -+ OVPN_CMD_SWAP_KEYS, -+ -+ OVPN_CMD_DEL_KEY, -+ -+ /** -+ * @OVPN_CMD_REGISTER_PACKET: Register for specific packet types to be -+ * forwarded to userspace -+ */ -+ OVPN_CMD_REGISTER_PACKET, -+ -+ /** -+ * @OVPN_CMD_PACKET: Send a packet from userspace to kernelspace. Also -+ * used to send to userspace packets for which a process had registered -+ * with OVPN_CMD_REGISTER_PACKET -+ */ -+ OVPN_CMD_PACKET, -+ -+ /** -+ * @OVPN_CMD_GET_PEER: Retrieve the status of a peer or all peers -+ */ -+ OVPN_CMD_GET_PEER, -+}; -+ -+enum ovpn_cipher_alg { -+ /** -+ * @OVPN_CIPHER_ALG_NONE: No encryption - reserved for debugging only -+ */ -+ OVPN_CIPHER_ALG_NONE = 0, -+ /** -+ * @OVPN_CIPHER_ALG_AES_GCM: AES-GCM AEAD cipher with any allowed key size -+ */ -+ OVPN_CIPHER_ALG_AES_GCM, -+ /** -+ * @OVPN_CIPHER_ALG_CHACHA20_POLY1305: ChaCha20Poly1305 AEAD cipher -+ */ -+ OVPN_CIPHER_ALG_CHACHA20_POLY1305, -+}; -+ -+enum ovpn_del_peer_reason { -+ __OVPN_DEL_PEER_REASON_FIRST, -+ OVPN_DEL_PEER_REASON_TEARDOWN = __OVPN_DEL_PEER_REASON_FIRST, -+ OVPN_DEL_PEER_REASON_USERSPACE, -+ OVPN_DEL_PEER_REASON_EXPIRED, -+ OVPN_DEL_PEER_REASON_TRANSPORT_ERROR, -+ __OVPN_DEL_PEER_REASON_AFTER_LAST -+}; -+ -+enum ovpn_key_slot { -+ __OVPN_KEY_SLOT_FIRST, -+ OVPN_KEY_SLOT_PRIMARY = __OVPN_KEY_SLOT_FIRST, -+ OVPN_KEY_SLOT_SECONDARY, -+ __OVPN_KEY_SLOT_AFTER_LAST, -+}; -+ -+enum ovpn_netlink_attrs { -+ OVPN_ATTR_UNSPEC = 0, -+ OVPN_ATTR_IFINDEX, -+ OVPN_ATTR_NEW_PEER, -+ OVPN_ATTR_SET_PEER, -+ OVPN_ATTR_DEL_PEER, -+ OVPN_ATTR_NEW_KEY, -+ OVPN_ATTR_SWAP_KEYS, -+ OVPN_ATTR_DEL_KEY, -+ OVPN_ATTR_PACKET, -+ OVPN_ATTR_GET_PEER, -+ -+ __OVPN_ATTR_AFTER_LAST, -+ OVPN_ATTR_MAX = __OVPN_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_key_dir_attrs { -+ OVPN_KEY_DIR_ATTR_UNSPEC = 0, -+ OVPN_KEY_DIR_ATTR_CIPHER_KEY, -+ OVPN_KEY_DIR_ATTR_NONCE_TAIL, -+ -+ __OVPN_KEY_DIR_ATTR_AFTER_LAST, -+ OVPN_KEY_DIR_ATTR_MAX = __OVPN_KEY_DIR_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_new_key_attrs { -+ OVPN_NEW_KEY_ATTR_UNSPEC = 0, -+ OVPN_NEW_KEY_ATTR_PEER_ID, -+ OVPN_NEW_KEY_ATTR_KEY_SLOT, -+ OVPN_NEW_KEY_ATTR_KEY_ID, -+ OVPN_NEW_KEY_ATTR_CIPHER_ALG, -+ OVPN_NEW_KEY_ATTR_ENCRYPT_KEY, -+ OVPN_NEW_KEY_ATTR_DECRYPT_KEY, -+ -+ __OVPN_NEW_KEY_ATTR_AFTER_LAST, -+ OVPN_NEW_KEY_ATTR_MAX = __OVPN_NEW_KEY_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_del_key_attrs { -+ OVPN_DEL_KEY_ATTR_UNSPEC = 0, -+ OVPN_DEL_KEY_ATTR_PEER_ID, -+ OVPN_DEL_KEY_ATTR_KEY_SLOT, -+ -+ __OVPN_DEL_KEY_ATTR_AFTER_LAST, -+ OVPN_DEL_KEY_ATTR_MAX = __OVPN_DEL_KEY_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_swap_keys_attrs { -+ OVPN_SWAP_KEYS_ATTR_UNSPEC = 0, -+ OVPN_SWAP_KEYS_ATTR_PEER_ID, -+ -+ __OVPN_SWAP_KEYS_ATTR_AFTER_LAST, -+ OVPN_SWAP_KEYS_ATTR_MAX = __OVPN_SWAP_KEYS_ATTR_AFTER_LAST - 1, -+ -+}; -+ -+enum ovpn_netlink_new_peer_attrs { -+ OVPN_NEW_PEER_ATTR_UNSPEC = 0, -+ OVPN_NEW_PEER_ATTR_PEER_ID, -+ OVPN_NEW_PEER_ATTR_SOCKADDR_REMOTE, -+ OVPN_NEW_PEER_ATTR_SOCKET, -+ OVPN_NEW_PEER_ATTR_IPV4, -+ OVPN_NEW_PEER_ATTR_IPV6, -+ OVPN_NEW_PEER_ATTR_LOCAL_IP, -+ -+ __OVPN_NEW_PEER_ATTR_AFTER_LAST, -+ OVPN_NEW_PEER_ATTR_MAX = __OVPN_NEW_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_set_peer_attrs { -+ OVPN_SET_PEER_ATTR_UNSPEC = 0, -+ OVPN_SET_PEER_ATTR_PEER_ID, -+ OVPN_SET_PEER_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_SET_PEER_ATTR_KEEPALIVE_TIMEOUT, -+ -+ __OVPN_SET_PEER_ATTR_AFTER_LAST, -+ OVPN_SET_PEER_ATTR_MAX = __OVPN_SET_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_del_peer_attrs { -+ OVPN_DEL_PEER_ATTR_UNSPEC = 0, -+ OVPN_DEL_PEER_ATTR_REASON, -+ OVPN_DEL_PEER_ATTR_PEER_ID, -+ -+ __OVPN_DEL_PEER_ATTR_AFTER_LAST, -+ OVPN_DEL_PEER_ATTR_MAX = __OVPN_DEL_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_get_peer_attrs { -+ OVPN_GET_PEER_ATTR_UNSPEC = 0, -+ OVPN_GET_PEER_ATTR_PEER_ID, -+ -+ __OVPN_GET_PEER_ATTR_AFTER_LAST, -+ OVPN_GET_PEER_ATTR_MAX = __OVPN_GET_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_get_peer_response_attrs { -+ OVPN_GET_PEER_RESP_ATTR_UNSPEC = 0, -+ OVPN_GET_PEER_RESP_ATTR_PEER_ID, -+ OVPN_GET_PEER_RESP_ATTR_SOCKADDR_REMOTE, -+ OVPN_GET_PEER_RESP_ATTR_IPV4, -+ OVPN_GET_PEER_RESP_ATTR_IPV6, -+ OVPN_GET_PEER_RESP_ATTR_LOCAL_IP, -+ OVPN_GET_PEER_RESP_ATTR_LOCAL_PORT, -+ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_GET_PEER_RESP_ATTR_KEEPALIVE_TIMEOUT, -+ OVPN_GET_PEER_RESP_ATTR_RX_BYTES, -+ OVPN_GET_PEER_RESP_ATTR_TX_BYTES, -+ OVPN_GET_PEER_RESP_ATTR_RX_PACKETS, -+ OVPN_GET_PEER_RESP_ATTR_TX_PACKETS, -+ -+ __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST, -+ OVPN_GET_PEER_RESP_ATTR_MAX = __OVPN_GET_PEER_RESP_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_peer_stats_attrs { -+ OVPN_PEER_STATS_ATTR_UNSPEC = 0, -+ OVPN_PEER_STATS_BYTES, -+ OVPN_PEER_STATS_PACKETS, -+ -+ __OVPN_PEER_STATS_ATTR_AFTER_LAST, -+ OVPN_PEER_STATS_ATTR_MAX = __OVPN_PEER_STATS_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_peer_attrs { -+ OVPN_PEER_ATTR_UNSPEC = 0, -+ OVPN_PEER_ATTR_PEER_ID, -+ OVPN_PEER_ATTR_SOCKADDR_REMOTE, -+ OVPN_PEER_ATTR_IPV4, -+ OVPN_PEER_ATTR_IPV6, -+ OVPN_PEER_ATTR_LOCAL_IP, -+ OVPN_PEER_ATTR_KEEPALIVE_INTERVAL, -+ OVPN_PEER_ATTR_KEEPALIVE_TIMEOUT, -+ OVPN_PEER_ATTR_ENCRYPT_KEY, -+ OVPN_PEER_ATTR_DECRYPT_KEY, -+ OVPN_PEER_ATTR_RX_STATS, -+ OVPN_PEER_ATTR_TX_STATS, -+ -+ __OVPN_PEER_ATTR_AFTER_LAST, -+ OVPN_PEER_ATTR_MAX = __OVPN_PEER_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_netlink_packet_attrs { -+ OVPN_PACKET_ATTR_UNSPEC = 0, -+ OVPN_PACKET_ATTR_PACKET, -+ OVPN_PACKET_ATTR_PEER_ID, -+ -+ __OVPN_PACKET_ATTR_AFTER_LAST, -+ OVPN_PACKET_ATTR_MAX = __OVPN_PACKET_ATTR_AFTER_LAST - 1, -+}; -+ -+enum ovpn_ifla_attrs { -+ IFLA_OVPN_UNSPEC = 0, -+ IFLA_OVPN_MODE, -+ -+ __IFLA_OVPN_AFTER_LAST, -+ IFLA_OVPN_MAX = __IFLA_OVPN_AFTER_LAST - 1, -+}; -+ -+enum ovpn_mode { -+ __OVPN_MODE_FIRST = 0, -+ OVPN_MODE_P2P = __OVPN_MODE_FIRST, -+ OVPN_MODE_MP, -+ -+ __OVPN_MODE_AFTER_LAST, -+}; -+ -+#endif /* _UAPI_LINUX_OVPN_DCO_H_ */ -diff --git a/include/uapi/linux/udp.h b/include/uapi/linux/udp.h -index 4828794efcf8..8008c762e6b8 100644 ---- a/include/uapi/linux/udp.h -+++ b/include/uapi/linux/udp.h -@@ -43,5 +43,6 @@ struct udphdr { - #define UDP_ENCAP_GTP1U 5 /* 3GPP TS 29.060 */ - #define UDP_ENCAP_RXRPC 6 - #define TCP_ENCAP_ESPINTCP 7 /* Yikes, this is really xfrm encap types. */ -+#define UDP_ENCAP_OVPNINUDP 8 /* OpenVPN over UDP connection */ - - #endif /* _UAPI_LINUX_UDP_H */ --- -2.38.0.rc1.8.g2a7d63a245 -