feat: introduce support for generating observations for circumvention nettests

oonidata/models/nettests/__init__.py

@@ -5,6 +5,8 @@
 from .signal import Signal
 from .telegram import Telegram
 from .tor import Tor
+from .psiphon import Psiphon
+from .vanilla_tor import VanillaTor
 from .web_connectivity import WebConnectivity
 from .whatsapp import Whatsapp
 from .http_invalid_request_line import HTTPInvalidRequestLine
@@ -16,6 +18,8 @@
     WebConnectivity,
     Telegram,
     Tor,
+    Psiphon,
+    VanillaTor,
     DNSCheck,
     Signal,
     Whatsapp,
@@ -27,6 +31,8 @@
     WebConnectivity,
     Telegram,
     Tor,
+    Psiphon,
+    VanillaTor,
     DNSCheck,
     Signal,
     Whatsapp,

oonidata/models/nettests/psiphon.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+from typing import List, Optional
+from oonidata.compat import add_slots
+from oonidata.models.base_model import BaseModel
+from oonidata.models.dataformats import (
+    DNSQuery,
+    Failure,
+    HTTPTransaction,
+    NetworkEvent,
+    TLSHandshake,
+)
+
+from .base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class PsiphonTestKeys(BaseModel):
+    failure: Failure = None
+    max_runtime: Optional[int] = None
+    bootstrap_time: Optional[int] = None
+
+    socksproxy: Optional[str] = None
+    network_events: Optional[List[NetworkEvent]] = None
+    tls_handshakes: Optional[List[TLSHandshake]] = None
+    queries: Optional[List[DNSQuery]] = None
+    requests: Optional[List[HTTPTransaction]] = None
+
+
+@add_slots
+@dataclass
+class Psiphon(BaseMeasurement):
+    __test_name__ = "psiphon"
+
+    test_keys: PsiphonTestKeys

oonidata/models/nettests/vanilla_tor.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import List, Optional
+from oonidata.compat import add_slots
+from oonidata.models.base_model import BaseModel
+from oonidata.models.dataformats import (
+    DNSQuery,
+    Failure,
+    HTTPTransaction,
+    NetworkEvent,
+    TLSHandshake,
+)
+
+from .base_measurement import BaseMeasurement
+
+
+@add_slots
+@dataclass
+class VanillaTorTestKeys(BaseModel):
+    failure: Failure = None
+    error: Optional[str] = None
+    success: Optional[bool] = None
+    bootstrap_time: Optional[int] = None
+    timeout: Optional[int] = None
+
+    tor_logs: Optional[List[str]] = None
+    tor_progress: Optional[int] = None
+    tor_progress_tag: Optional[str] = None
+    tor_progress_summary: Optional[str] = None
+    tor_version: Optional[str] = None
+    transport_name: Optional[str] = None
+
+
+@add_slots
+@dataclass
+class VanillaTor(BaseMeasurement):
+    __test_name__ = "vanila_tor"
+
+    test_keys: VanillaTorTestKeys

oonidata/models/observations.py

@@ -378,3 +378,34 @@ class HTTPMiddleboxObservation(MeasurementMeta):
     hfm_diff: Optional[str] = None
     hfm_failure: Optional[str] = None
     hfm_success: Optional[bool] = None
+
+
+@add_slots
+@dataclass
+class CircumventionToolObservation(MeasurementMeta):
+    __table_name__ = "obs_circumvention_tool"
+    __table_index__ = ("measurement_uid", "observation_id", "measurement_start_time")
+
+    observation_id: str = ""
+    bucket_date: Optional[str] = None
+    created_at: Optional[datetime] = None
+
+    bootstrap_time: Optional[int] = None
+
+    # psiphon observation
+    psiphon_failure: Failure = None
+    psiphon_max_runtime: Optional[int] = None
+    psiphon_socksproxy: Optional[str] = None
+
+    # vanilla_tor observation
+    tor_failure: Failure = None
+    tor_error: Optional[str] = None
+    tor_success: Optional[bool] = None
+    tor_timeout: Optional[int] = None
+
+    tor_logs: Optional[List[str]] = None
+    tor_progress: Optional[int] = None
+    tor_progress_tag: Optional[str] = None
+    tor_progress_summary: Optional[str] = None
+    tor_version: Optional[str] = None
+    tor_transport_name: Optional[str] = None

oonidata/transforms/__init__.py

@@ -7,6 +7,8 @@
 from oonidata.transforms.nettests.signal import SignalTransformer
 from oonidata.transforms.nettests.telegram import TelegramTransformer
 from oonidata.transforms.nettests.tor import TorTransformer
+from oonidata.transforms.nettests.psiphon import PsiphonTransformer
+from oonidata.transforms.nettests.vanilla_tor import VanillaTorTransformer
 from oonidata.transforms.nettests.web_connectivity import WebConnectivityTransformer
 from oonidata.transforms.nettests.http_invalid_request_line import (
     HTTPInvalidRequestLineTransformer,
@@ -17,6 +19,8 @@
     "signal": SignalTransformer,
     "telegram": TelegramTransformer,
     "tor": TorTransformer,
+    "psiphon": PsiphonTransformer,
+    "vanilla_tor": VanillaTorTransformer, 
     "http_header_field_manipulation": HTTPHeaderFieldManipulationTransformer,
     "http_invalid_request_line": HTTPInvalidRequestLineTransformer,
     "web_connectivity": WebConnectivityTransformer,

oonidata/transforms/nettests/psiphon.py

@@ -0,0 +1,23 @@
+from typing import List, Tuple
+from oonidata.models.nettests import Psiphon
+from oonidata.models.observations import WebObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
+
+
+class PsiphonTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: Psiphon) -> Tuple[List[WebObservation]]:
+
+        dns_observations = self.make_dns_observations(msmt.test_keys.queries)
+        tls_observations = self.make_tls_observations(
+            msmt.test_keys.tls_handshakes, 
+            msmt.test_keys.network_events
+        )
+        http_observations = self.make_http_observations(msmt.test_keys.requests)
+
+        return (
+            self.consume_web_observations(
+                dns_observations=dns_observations,
+                tls_observations=tls_observations,
+                http_observations=http_observations,
+            )
+        )

oonidata/transforms/nettests/vanilla_tor.py

@@ -0,0 +1,29 @@
+import dataclasses
+from typing import List, Tuple
+from oonidata.models.nettests import VanillaTor
+from oonidata.models.observations import CircumventionToolObservation
+from oonidata.transforms.nettests.measurement_transformer import MeasurementTransformer
+
+
+class VanillaTorTransformer(MeasurementTransformer):
+    def make_observations(self, msmt: VanillaTor) -> Tuple[List[CircumventionToolObservation]]:
+        ct_obs = CircumventionToolObservation(
+            observation_id=f"{msmt.measurement_uid}_0",
+            created_at=datetime.utcnow().replace(microsecond=0),
+            **dataclasses.asdict(self.measurement_meta),
+        )
+
+        ct_obs.bootstrap_time = msmt.test_keys.bootstrap_time,
+        ct.tor_failure = msmt.test_keys.failure
+        ct.tor_error = msmt.test_keys.error
+        ct.tor_success = msmt.test_keys.success
+        ct.tor_timeout = msmt.test_keys.timeout
+
+        ct.tor_logs = msmt.test_keys.tor_logs
+        ct.tor_progress = msmt.test_keys.tor_progress
+        ct.tor_progress_tag = msmt.test_keys.tor_progress_tag
+        ct.tor_progress_summary = msmt.test_keys.tor_progress_summary
+        ct.tor_version = msmt.test_keys.tor_version
+        ct.tor_transport_name = msmt.test_keys.transport_name
+
+        return ([ct_obs],)