From a2d4f025793d663972a0c31e97fb8ffa1ebd28d4 Mon Sep 17 00:00:00 2001 From: DecFox Date: Mon, 27 Nov 2023 18:18:50 +0530 Subject: [PATCH 1/4] feat: add support for generating psiphon and facebook_messenger observations --- oonidata/models/nettests/__init__.py | 6 +++ .../models/nettests/facebook_messenger.py | 43 +++++++++++++++++++ oonidata/models/nettests/psiphon.py | 35 +++++++++++++++ oonidata/transforms/__init__.py | 4 ++ .../transforms/nettests/facebook_messenger.py | 18 ++++++++ oonidata/transforms/nettests/psiphon.py | 23 ++++++++++ 6 files changed, 129 insertions(+) create mode 100644 oonidata/models/nettests/facebook_messenger.py create mode 100644 oonidata/models/nettests/psiphon.py create mode 100644 oonidata/transforms/nettests/facebook_messenger.py create mode 100644 oonidata/transforms/nettests/psiphon.py diff --git a/oonidata/models/nettests/__init__.py b/oonidata/models/nettests/__init__.py index 41750197..17739dc8 100644 --- a/oonidata/models/nettests/__init__.py +++ b/oonidata/models/nettests/__init__.py @@ -2,9 +2,11 @@ from .base_measurement import BaseMeasurement from .dnscheck import DNSCheck +from .facebook_messenger import FacebookMessenger from .signal import Signal from .telegram import Telegram from .tor import Tor +from .psiphon import Psiphon from .web_connectivity import WebConnectivity from .whatsapp import Whatsapp from .http_invalid_request_line import HTTPInvalidRequestLine @@ -16,7 +18,9 @@ WebConnectivity, Telegram, Tor, + Psiphon, DNSCheck, + FacebookMessenger, Signal, Whatsapp, BaseMeasurement, @@ -27,7 +31,9 @@ WebConnectivity, Telegram, Tor, + Psiphon, DNSCheck, + FacebookMessenger, Signal, Whatsapp, BaseMeasurement, diff --git a/oonidata/models/nettests/facebook_messenger.py b/oonidata/models/nettests/facebook_messenger.py new file mode 100644 index 00000000..dc4d6b80 --- /dev/null +++ b/oonidata/models/nettests/facebook_messenger.py @@ -0,0 +1,43 @@ +from dataclass import dataclass +from typing import List, Optional +from oonidata.compat import add_slots +from oonidata.models.base_model import BaseModel +from oonidata.models.dataformats import ( + TCPConnect, + DNSQuery +) + +from .base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class FacebookMessengerTestKeys(BaseModel): + facebook_b_api_dns_consistent: Optional[bool] = None + facebook_b_api_reachable: Optional[bool] = None + facebook_b_graph_dns_consistent: Optional[bool] = None + facebook_b_graph_reachable: Optional[bool] = None + facebook_dns_blocking: Optional[bool] = None + facebook_edge_dns_consistent: Optional[bool] = None + facebook_edge_reachable: Optional[bool] = None + facebook_external_cdn_dns_consistent: Optional[bool] = None + facebook_external_cdn_reachable: Optional[bool] = None + facebook_scontent_cdn_dns_consistent: Optional[bool] = None + facebook_scontent_cdn_reachable: Optional[bool] = None + facebook_star_dns_consistent: Optional[bool] = None + facebook_star_reachable: Optional[bool] = None + facebook_stun_dns_consistent: Optional[bool] = None + facebook_stun_reachable: Optional[bool] = None + facebook_tcp_blocking: Optional[bool] = None + + socksproxy: Optional[str] = None, + tcp_connect: Optional[List[TCPConnect]] = None + queries: Optional[List[DNSQuery]] = None + + +@add_slots +@dataclass +class FacebookMessenger(BaseMeasurement): + __test_name__ = "facebook_messenger" + + test_keys: FacebookMessengerTestKeys diff --git a/oonidata/models/nettests/psiphon.py b/oonidata/models/nettests/psiphon.py new file mode 100644 index 00000000..f14012e4 --- /dev/null +++ b/oonidata/models/nettests/psiphon.py @@ -0,0 +1,35 @@ +from dataclass import dataclass +from typing import List, Optional +from oonidata.compat import add_slots +from oonidata.models.base_model import BaseModel +from oonidata.models.dataformats import ( + DNSQuery, + Failure, + HTTPTransaction, + NetworkEvent, + TLSHandshake, +) + +from .base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class PsiphonTestKeys(BaseModel): + failure: Failure = None + max_runtime: Optional[int] = None + bootstrap_time: Optional[int] = None + + socksproxy: Optional[str] = None + network_events: Optional[List[NetworkEvent]] = None + tls_handshakes: Optional[List[TLSHandshake]] = None + queries: Optional[List[DNSQuery]] = None + requests = Optional[List[HTTPTransaction]] = None + + +@add_slots +@dataclass +class Psiphon(BaseMeasurement): + __test_name__ = "psiphon" + + test_keys: PsiphonTestKeys diff --git a/oonidata/transforms/__init__.py b/oonidata/transforms/__init__.py index e3388007..3f901ef3 100644 --- a/oonidata/transforms/__init__.py +++ b/oonidata/transforms/__init__.py @@ -1,9 +1,11 @@ from oonidata.netinfo import NetinfoDB from oonidata.transforms.nettests.dnscheck import DNSCheckTransformer +from oonidata.transforms.nettests.facebook_messenger import FacebookMessengerTransformer from oonidata.transforms.nettests.signal import SignalTransformer from oonidata.transforms.nettests.telegram import TelegramTransformer from oonidata.transforms.nettests.tor import TorTransformer +from oonidata.transforms.nettests.psiphon import PsiphonTransformer from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer from oonidata.transforms.nettests.http_invalid_request_line import ( HTTPInvalidRequestLineTransformer, @@ -11,9 +13,11 @@ NETTEST_TRANSFORMERS = { "dnscheck": DNSCheckTransformer, + "facebook_messenger": FacebookMessengerTransformer, "signal": SignalTransformer, "telegram": TelegramTransformer, "tor": TorTransformer, + "psiphon": PsiphonTransformer, "http_invalid_request_line": HTTPInvalidRequestLineTransformer, "web_connectivity": WebConnectivityTransformer, } diff --git a/oonidata/transforms/nettests/facebook_messenger.py b/oonidata/transforms/nettests/facebook_messenger.py new file mode 100644 index 00000000..0e858990 --- /dev/null +++ b/oonidata/transforms/nettests/facebook_messenger.py @@ -0,0 +1,18 @@ +from typing import List, Tuple +from oonidata.models.nettests import FacebookMessenger +from oonidata.models.observations import WebObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class FacebookMessengerTransformer(MeasurementTransformer): + def make_observations(self, msmt: FacebookMessenger) -> Tuple[List[WebObservation]]: + + dns_observations = self.make_dns_observations(msmt.test_keys.queries) + tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) + + return ( + self.consume_web_observations( + dns_observations=dns_observations, + tcp_observations=tcp_observations, + ) + ) diff --git a/oonidata/transforms/nettests/psiphon.py b/oonidata/transforms/nettests/psiphon.py new file mode 100644 index 00000000..8787c50c --- /dev/null +++ b/oonidata/transforms/nettests/psiphon.py @@ -0,0 +1,23 @@ +from typing import List, Tuple +from oonidata.models.nettests import Psiphon +from oonidata.models.observations import WebObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class PsiphonTransformer(MeasurementTransformer): + def make_observations(self, msmt: Psiphon) -> Tuple[List[WebObservation]]: + + dns_observations = self.make_dns_observations(msmt.test_keys.queries) + tls_observations = self.make_tls_observations( + msmt.test_keys.tls_handshakes, + msmt.test_keys.network_events + ) + http_observations = self.make_http_observations(msmt.test_keys.requests) + + return ( + self.consume_web_observations( + dns_observations=dns_observations, + tls_observations=tls_observations, + http_observations=http_observations, + ) + ) From 5dfe69336415756db42bf3f9a7ccf8591e77c07a Mon Sep 17 00:00:00 2001 From: DecFox Date: Mon, 27 Nov 2023 18:54:13 +0530 Subject: [PATCH 2/4] fix: import issues and typos --- oonidata/models/nettests/facebook_messenger.py | 2 +- oonidata/models/nettests/psiphon.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/oonidata/models/nettests/facebook_messenger.py b/oonidata/models/nettests/facebook_messenger.py index dc4d6b80..35aefe19 100644 --- a/oonidata/models/nettests/facebook_messenger.py +++ b/oonidata/models/nettests/facebook_messenger.py @@ -1,4 +1,4 @@ -from dataclass import dataclass +from dataclasses import dataclass from typing import List, Optional from oonidata.compat import add_slots from oonidata.models.base_model import BaseModel diff --git a/oonidata/models/nettests/psiphon.py b/oonidata/models/nettests/psiphon.py index f14012e4..71a0526d 100644 --- a/oonidata/models/nettests/psiphon.py +++ b/oonidata/models/nettests/psiphon.py @@ -1,4 +1,4 @@ -from dataclass import dataclass +from dataclasses import dataclass from typing import List, Optional from oonidata.compat import add_slots from oonidata.models.base_model import BaseModel @@ -24,7 +24,7 @@ class PsiphonTestKeys(BaseModel): network_events: Optional[List[NetworkEvent]] = None tls_handshakes: Optional[List[TLSHandshake]] = None queries: Optional[List[DNSQuery]] = None - requests = Optional[List[HTTPTransaction]] = None + requests: Optional[List[HTTPTransaction]] = None @add_slots From cf0e70f2a6155145a415483c812cffd4deb0c471 Mon Sep 17 00:00:00 2001 From: DecFox Date: Sun, 3 Dec 2023 14:30:47 +0530 Subject: [PATCH 3/4] feat: add CircumventionToolObservation and vanilla_tor --- oonidata/models/nettests/__init__.py | 3 ++ oonidata/models/nettests/vanilla_tor.py | 38 +++++++++++++++++++++ oonidata/models/observations.py | 31 +++++++++++++++++ oonidata/transforms/__init__.py | 4 ++- oonidata/transforms/nettests/vanilla_tor.py | 29 ++++++++++++++++ 5 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 oonidata/models/nettests/vanilla_tor.py create mode 100644 oonidata/transforms/nettests/vanilla_tor.py diff --git a/oonidata/models/nettests/__init__.py b/oonidata/models/nettests/__init__.py index 17739dc8..f077a5be 100644 --- a/oonidata/models/nettests/__init__.py +++ b/oonidata/models/nettests/__init__.py @@ -7,6 +7,7 @@ from .telegram import Telegram from .tor import Tor from .psiphon import Psiphon +from .vanilla_tor import VanillaTor from .web_connectivity import WebConnectivity from .whatsapp import Whatsapp from .http_invalid_request_line import HTTPInvalidRequestLine @@ -19,6 +20,7 @@ Telegram, Tor, Psiphon, + VanillaTor, DNSCheck, FacebookMessenger, Signal, @@ -32,6 +34,7 @@ Telegram, Tor, Psiphon, + VanillaTor, DNSCheck, FacebookMessenger, Signal, diff --git a/oonidata/models/nettests/vanilla_tor.py b/oonidata/models/nettests/vanilla_tor.py new file mode 100644 index 00000000..2d44201a --- /dev/null +++ b/oonidata/models/nettests/vanilla_tor.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from typing import List, Optional +from oonidata.compat import add_slots +from oonidata.models.base_model import BaseModel +from oonidata.models.dataformats import ( + DNSQuery, + Failure, + HTTPTransaction, + NetworkEvent, + TLSHandshake, +) + +from .base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class VanillaTorTestKeys(BaseModel): + failure: Failure = None + error: Optional[str] = None + success: Optional[bool] = None + bootstrap_time: Optional[int] = None + timeout: Optional[int] = None + + tor_logs: Optional[List[str]] = None + tor_progress: Optional[int] = None + tor_progress_tag: Optional[str] = None + tor_progress_summary: Optional[str] = None + tor_version: Optional[str] = None + transport_name: Optional[str] = None + + +@add_slots +@dataclass +class VanillaTor(BaseMeasurement): + __test_name__ = "vanila_tor" + + test_keys: VanillaTorTestKeys diff --git a/oonidata/models/observations.py b/oonidata/models/observations.py index 688d83fa..04d337f8 100644 --- a/oonidata/models/observations.py +++ b/oonidata/models/observations.py @@ -378,3 +378,34 @@ class HTTPMiddleboxObservation(MeasurementMeta): hfm_diff: Optional[str] = None hfm_failure: Optional[str] = None hfm_success: Optional[bool] = None + + +@add_slots +@dataclass +class CircumventionToolObservation(MeasurementMeta): + __table_name__ = "obs_circumvention_tool" + __table_index__ = ("measurement_uid", "observation_id", "measurement_start_time") + + observation_id: str = "" + bucket_date: Optional[str] = None + created_at: Optional[datetime] = None + + bootstrap_time: Optional[int] = None + + # psiphon observation + psiphon_failure: Failure = None + psiphon_max_runtime: Optional[int] = None + psiphon_socksproxy: Optional[str] = None + + # vanilla_tor observation + tor_failure: Failure = None + tor_error: Optional[str] = None + tor_success: Optional[bool] = None + tor_timeout: Optional[int] = None + + tor_logs: Optional[List[str]] = None + tor_progress: Optional[int] = None + tor_progress_tag: Optional[str] = None + tor_progress_summary: Optional[str] = None + tor_version: Optional[str] = None + tor_transport_name: Optional[str] = None diff --git a/oonidata/transforms/__init__.py b/oonidata/transforms/__init__.py index 3f901ef3..2193dc2e 100644 --- a/oonidata/transforms/__init__.py +++ b/oonidata/transforms/__init__.py @@ -6,6 +6,7 @@ from oonidata.transforms.nettests.telegram import TelegramTransformer from oonidata.transforms.nettests.tor import TorTransformer from oonidata.transforms.nettests.psiphon import PsiphonTransformer +from oonidata.transforms.nettests.vanilla_tor import VanillaTorTransformer from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer from oonidata.transforms.nettests.http_invalid_request_line import ( HTTPInvalidRequestLineTransformer, @@ -17,7 +18,8 @@ "signal": SignalTransformer, "telegram": TelegramTransformer, "tor": TorTransformer, - "psiphon": PsiphonTransformer, + "psiphon": PsiphonTransformer, + "vanilla_tor": VanillaTorTransformer, "http_invalid_request_line": HTTPInvalidRequestLineTransformer, "web_connectivity": WebConnectivityTransformer, } diff --git a/oonidata/transforms/nettests/vanilla_tor.py b/oonidata/transforms/nettests/vanilla_tor.py new file mode 100644 index 00000000..89a757f2 --- /dev/null +++ b/oonidata/transforms/nettests/vanilla_tor.py @@ -0,0 +1,29 @@ +import dataclasses +from typing import List, Tuple +from oonidata.models.nettests import VanillaTor +from oonidata.models.observations import CircumventionToolObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class VanillaTorTransformer(MeasurementTransformer): + def make_observations(self, msmt: VanillaTor) -> Tuple[List[CircumventionToolObservation]]: + ct_obs = CircumventionToolObservation( + observation_id=f"{msmt.measurement_uid}_0", + created_at=datetime.utcnow().replace(microsecond=0), + **dataclasses.asdict(self.measurement_meta), + ) + + ct_obs.bootstrap_time = msmt.test_keys.bootstrap_time, + ct.tor_failure = msmt.test_keys.failure + ct.tor_error = msmt.test_keys.error + ct.tor_success = msmt.test_keys.success + ct.tor_timeout = msmt.test_keys.timeout + + ct.tor_logs = msmt.test_keys.tor_logs + ct.tor_progress = msmt.test_keys.tor_progress + ct.tor_progress_tag = msmt.test_keys.tor_progress_tag + ct.tor_progress_summary = msmt.test_keys.tor_progress_summary + ct.tor_version = msmt.test_keys.tor_version + ct.tor_transport_name = msmt.test_keys.transport_name + + return ([ct_obs],) From 89e39e244f19477b06f664609bf8ed2eb1e85d5b Mon Sep 17 00:00:00 2001 From: DecFox Date: Wed, 13 Dec 2023 09:33:27 +0530 Subject: [PATCH 4/4] refactor: remove facebook_messenger diff --- oonidata/models/nettests/__init__.py | 3 -- .../models/nettests/facebook_messenger.py | 43 ------------------- oonidata/transforms/__init__.py | 2 - .../transforms/nettests/facebook_messenger.py | 18 -------- 4 files changed, 66 deletions(-) delete mode 100644 oonidata/models/nettests/facebook_messenger.py delete mode 100644 oonidata/transforms/nettests/facebook_messenger.py diff --git a/oonidata/models/nettests/__init__.py b/oonidata/models/nettests/__init__.py index f077a5be..231ef1c8 100644 --- a/oonidata/models/nettests/__init__.py +++ b/oonidata/models/nettests/__init__.py @@ -2,7 +2,6 @@ from .base_measurement import BaseMeasurement from .dnscheck import DNSCheck -from .facebook_messenger import FacebookMessenger from .signal import Signal from .telegram import Telegram from .tor import Tor @@ -22,7 +21,6 @@ Psiphon, VanillaTor, DNSCheck, - FacebookMessenger, Signal, Whatsapp, BaseMeasurement, @@ -36,7 +34,6 @@ Psiphon, VanillaTor, DNSCheck, - FacebookMessenger, Signal, Whatsapp, BaseMeasurement, diff --git a/oonidata/models/nettests/facebook_messenger.py b/oonidata/models/nettests/facebook_messenger.py deleted file mode 100644 index 35aefe19..00000000 --- a/oonidata/models/nettests/facebook_messenger.py +++ /dev/null @@ -1,43 +0,0 @@ -from dataclasses import dataclass -from typing import List, Optional -from oonidata.compat import add_slots -from oonidata.models.base_model import BaseModel -from oonidata.models.dataformats import ( - TCPConnect, - DNSQuery -) - -from .base_measurement import BaseMeasurement - - -@add_slots -@dataclass -class FacebookMessengerTestKeys(BaseModel): - facebook_b_api_dns_consistent: Optional[bool] = None - facebook_b_api_reachable: Optional[bool] = None - facebook_b_graph_dns_consistent: Optional[bool] = None - facebook_b_graph_reachable: Optional[bool] = None - facebook_dns_blocking: Optional[bool] = None - facebook_edge_dns_consistent: Optional[bool] = None - facebook_edge_reachable: Optional[bool] = None - facebook_external_cdn_dns_consistent: Optional[bool] = None - facebook_external_cdn_reachable: Optional[bool] = None - facebook_scontent_cdn_dns_consistent: Optional[bool] = None - facebook_scontent_cdn_reachable: Optional[bool] = None - facebook_star_dns_consistent: Optional[bool] = None - facebook_star_reachable: Optional[bool] = None - facebook_stun_dns_consistent: Optional[bool] = None - facebook_stun_reachable: Optional[bool] = None - facebook_tcp_blocking: Optional[bool] = None - - socksproxy: Optional[str] = None, - tcp_connect: Optional[List[TCPConnect]] = None - queries: Optional[List[DNSQuery]] = None - - -@add_slots -@dataclass -class FacebookMessenger(BaseMeasurement): - __test_name__ = "facebook_messenger" - - test_keys: FacebookMessengerTestKeys diff --git a/oonidata/transforms/__init__.py b/oonidata/transforms/__init__.py index 5aee05ff..4bf2e0a2 100644 --- a/oonidata/transforms/__init__.py +++ b/oonidata/transforms/__init__.py @@ -1,7 +1,6 @@ from oonidata.netinfo import NetinfoDB from oonidata.transforms.nettests.dnscheck import DNSCheckTransformer -from oonidata.transforms.nettests.facebook_messenger import FacebookMessengerTransformer from oonidata.transforms.nettests.http_header_field_manipulation import ( HTTPHeaderFieldManipulationTransformer, ) @@ -17,7 +16,6 @@ NETTEST_TRANSFORMERS = { "dnscheck": DNSCheckTransformer, - "facebook_messenger": FacebookMessengerTransformer, "signal": SignalTransformer, "telegram": TelegramTransformer, "tor": TorTransformer, diff --git a/oonidata/transforms/nettests/facebook_messenger.py b/oonidata/transforms/nettests/facebook_messenger.py deleted file mode 100644 index 0e858990..00000000 --- a/oonidata/transforms/nettests/facebook_messenger.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import List, Tuple -from oonidata.models.nettests import FacebookMessenger -from oonidata.models.observations import WebObservation -from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer - - -class FacebookMessengerTransformer(MeasurementTransformer): - def make_observations(self, msmt: FacebookMessenger) -> Tuple[List[WebObservation]]: - - dns_observations = self.make_dns_observations(msmt.test_keys.queries) - tcp_observations = self.make_tcp_observations(msmt.test_keys.tcp_connect) - - return ( - self.consume_web_observations( - dns_observations=dns_observations, - tcp_observations=tcp_observations, - ) - )