diff --git a/oonidata/models/nettests/__init__.py b/oonidata/models/nettests/__init__.py index 41750197..231ef1c8 100644 --- a/oonidata/models/nettests/__init__.py +++ b/oonidata/models/nettests/__init__.py @@ -5,6 +5,8 @@ from .signal import Signal from .telegram import Telegram from .tor import Tor +from .psiphon import Psiphon +from .vanilla_tor import VanillaTor from .web_connectivity import WebConnectivity from .whatsapp import Whatsapp from .http_invalid_request_line import HTTPInvalidRequestLine @@ -16,6 +18,8 @@ WebConnectivity, Telegram, Tor, + Psiphon, + VanillaTor, DNSCheck, Signal, Whatsapp, @@ -27,6 +31,8 @@ WebConnectivity, Telegram, Tor, + Psiphon, + VanillaTor, DNSCheck, Signal, Whatsapp, diff --git a/oonidata/models/nettests/psiphon.py b/oonidata/models/nettests/psiphon.py new file mode 100644 index 00000000..71a0526d --- /dev/null +++ b/oonidata/models/nettests/psiphon.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from typing import List, Optional +from oonidata.compat import add_slots +from oonidata.models.base_model import BaseModel +from oonidata.models.dataformats import ( + DNSQuery, + Failure, + HTTPTransaction, + NetworkEvent, + TLSHandshake, +) + +from .base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class PsiphonTestKeys(BaseModel): + failure: Failure = None + max_runtime: Optional[int] = None + bootstrap_time: Optional[int] = None + + socksproxy: Optional[str] = None + network_events: Optional[List[NetworkEvent]] = None + tls_handshakes: Optional[List[TLSHandshake]] = None + queries: Optional[List[DNSQuery]] = None + requests: Optional[List[HTTPTransaction]] = None + + +@add_slots +@dataclass +class Psiphon(BaseMeasurement): + __test_name__ = "psiphon" + + test_keys: PsiphonTestKeys diff --git a/oonidata/models/nettests/vanilla_tor.py b/oonidata/models/nettests/vanilla_tor.py new file mode 100644 index 00000000..2d44201a --- /dev/null +++ b/oonidata/models/nettests/vanilla_tor.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from typing import List, Optional +from oonidata.compat import add_slots +from oonidata.models.base_model import BaseModel +from oonidata.models.dataformats import ( + DNSQuery, + Failure, + HTTPTransaction, + NetworkEvent, + TLSHandshake, +) + +from .base_measurement import BaseMeasurement + + +@add_slots +@dataclass +class VanillaTorTestKeys(BaseModel): + failure: Failure = None + error: Optional[str] = None + success: Optional[bool] = None + bootstrap_time: Optional[int] = None + timeout: Optional[int] = None + + tor_logs: Optional[List[str]] = None + tor_progress: Optional[int] = None + tor_progress_tag: Optional[str] = None + tor_progress_summary: Optional[str] = None + tor_version: Optional[str] = None + transport_name: Optional[str] = None + + +@add_slots +@dataclass +class VanillaTor(BaseMeasurement): + __test_name__ = "vanila_tor" + + test_keys: VanillaTorTestKeys diff --git a/oonidata/models/observations.py b/oonidata/models/observations.py index 688d83fa..04d337f8 100644 --- a/oonidata/models/observations.py +++ b/oonidata/models/observations.py @@ -378,3 +378,34 @@ class HTTPMiddleboxObservation(MeasurementMeta): hfm_diff: Optional[str] = None hfm_failure: Optional[str] = None hfm_success: Optional[bool] = None + + +@add_slots +@dataclass +class CircumventionToolObservation(MeasurementMeta): + __table_name__ = "obs_circumvention_tool" + __table_index__ = ("measurement_uid", "observation_id", "measurement_start_time") + + observation_id: str = "" + bucket_date: Optional[str] = None + created_at: Optional[datetime] = None + + bootstrap_time: Optional[int] = None + + # psiphon observation + psiphon_failure: Failure = None + psiphon_max_runtime: Optional[int] = None + psiphon_socksproxy: Optional[str] = None + + # vanilla_tor observation + tor_failure: Failure = None + tor_error: Optional[str] = None + tor_success: Optional[bool] = None + tor_timeout: Optional[int] = None + + tor_logs: Optional[List[str]] = None + tor_progress: Optional[int] = None + tor_progress_tag: Optional[str] = None + tor_progress_summary: Optional[str] = None + tor_version: Optional[str] = None + tor_transport_name: Optional[str] = None diff --git a/oonidata/transforms/__init__.py b/oonidata/transforms/__init__.py index ec4de232..4bf2e0a2 100644 --- a/oonidata/transforms/__init__.py +++ b/oonidata/transforms/__init__.py @@ -7,6 +7,8 @@ from oonidata.transforms.nettests.signal import SignalTransformer from oonidata.transforms.nettests.telegram import TelegramTransformer from oonidata.transforms.nettests.tor import TorTransformer +from oonidata.transforms.nettests.psiphon import PsiphonTransformer +from oonidata.transforms.nettests.vanilla_tor import VanillaTorTransformer from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer from oonidata.transforms.nettests.http_invalid_request_line import ( HTTPInvalidRequestLineTransformer, @@ -17,6 +19,8 @@ "signal": SignalTransformer, "telegram": TelegramTransformer, "tor": TorTransformer, + "psiphon": PsiphonTransformer, + "vanilla_tor": VanillaTorTransformer, "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer, "http_invalid_request_line": HTTPInvalidRequestLineTransformer, "web_connectivity": WebConnectivityTransformer, diff --git a/oonidata/transforms/nettests/psiphon.py b/oonidata/transforms/nettests/psiphon.py new file mode 100644 index 00000000..8787c50c --- /dev/null +++ b/oonidata/transforms/nettests/psiphon.py @@ -0,0 +1,23 @@ +from typing import List, Tuple +from oonidata.models.nettests import Psiphon +from oonidata.models.observations import WebObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class PsiphonTransformer(MeasurementTransformer): + def make_observations(self, msmt: Psiphon) -> Tuple[List[WebObservation]]: + + dns_observations = self.make_dns_observations(msmt.test_keys.queries) + tls_observations = self.make_tls_observations( + msmt.test_keys.tls_handshakes, + msmt.test_keys.network_events + ) + http_observations = self.make_http_observations(msmt.test_keys.requests) + + return ( + self.consume_web_observations( + dns_observations=dns_observations, + tls_observations=tls_observations, + http_observations=http_observations, + ) + ) diff --git a/oonidata/transforms/nettests/vanilla_tor.py b/oonidata/transforms/nettests/vanilla_tor.py new file mode 100644 index 00000000..89a757f2 --- /dev/null +++ b/oonidata/transforms/nettests/vanilla_tor.py @@ -0,0 +1,29 @@ +import dataclasses +from typing import List, Tuple +from oonidata.models.nettests import VanillaTor +from oonidata.models.observations import CircumventionToolObservation +from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer + + +class VanillaTorTransformer(MeasurementTransformer): + def make_observations(self, msmt: VanillaTor) -> Tuple[List[CircumventionToolObservation]]: + ct_obs = CircumventionToolObservation( + observation_id=f"{msmt.measurement_uid}_0", + created_at=datetime.utcnow().replace(microsecond=0), + **dataclasses.asdict(self.measurement_meta), + ) + + ct_obs.bootstrap_time = msmt.test_keys.bootstrap_time, + ct.tor_failure = msmt.test_keys.failure + ct.tor_error = msmt.test_keys.error + ct.tor_success = msmt.test_keys.success + ct.tor_timeout = msmt.test_keys.timeout + + ct.tor_logs = msmt.test_keys.tor_logs + ct.tor_progress = msmt.test_keys.tor_progress + ct.tor_progress_tag = msmt.test_keys.tor_progress_tag + ct.tor_progress_summary = msmt.test_keys.tor_progress_summary + ct.tor_version = msmt.test_keys.tor_version + ct.tor_transport_name = msmt.test_keys.transport_name + + return ([ct_obs],)