sergeysav/telemetry_visualization
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Replace gRPC Backend (#10)

Browse Source 
**Rationale:**

Having two separate servers and communication methods resulted in additional maintenance & the need to convert often between backend & frontend data types.
By moving the backend communication off of gRPC and to just use websockets it both gives more control & allows for simplification of the implementation.

#8

**Changes:**

- Replaces gRPC backend.
  - New implementation automatically handles reconnect logic
- Implements an api layer
- Migrates examples to the api layer
- Implements a proc macro to make command handling easier
- Implements unit tests for the api layer (90+% coverage)
- Implements integration tests for the proc macro (90+% coverage)

Reviewed-on: #10
Co-authored-by: Sergey Savelyev <sergeysav.nn@gmail.com>
Co-committed-by: Sergey Savelyev <sergeysav.nn@gmail.com>
This commit was merged in pull request #10.
This commit is contained in:
Sergey Savelyev
2026-01-01 10:11:53 -08:00
committed by sergeysav
parent f658b55586
commit 788dd10a91
68 changed files with 3934 additions and 1504 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/telemetry/data_item.rs
View File

@@ -1,8 +1,8 @@
use crate::telemetry::data_value::TelemetryDataValue;
use api::data_value::DataValue;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TelemetryDataItem {
pub value: TelemetryDataValue,
pub value: DataValue,
pub timestamp: String,
}

38
server/src/telemetry/data_type.rs
View File

@@ -1,38 +0,0 @@
use crate::core::TelemetryDataType;
use serde::de::Visitor;
use serde::{Deserializer, Serializer};
use std::fmt::Formatter;
pub fn tlm_data_type_serializer<S>(
tlm_data_type: &TelemetryDataType,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(tlm_data_type.as_str_name())
}
struct TlmDataTypeVisitor;
impl Visitor<'_> for TlmDataTypeVisitor {
type Value = TelemetryDataType;
fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
formatter.write_str("A &str")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
TelemetryDataType::from_str_name(v).ok_or(E::custom("Invalid TelemetryDataType"))
}
}
pub fn tlm_data_type_deserializer<'de, D>(deserializer: D) -> Result<TelemetryDataType, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_str(TlmDataTypeVisitor)
}

8
server/src/telemetry/data_value.rs
View File

@@ -1,8 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum TelemetryDataValue {
Float32(f32),
Float64(f64),
Boolean(bool),
}

11
server/src/telemetry/definition.rs
View File

@@ -1,13 +1,10 @@
use crate::core::TelemetryDataType;
use crate::telemetry::data_type::tlm_data_type_deserializer;
use crate::telemetry::data_type::tlm_data_type_serializer;
use api::data_type::DataType;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TelemetryDefinition {
pub uuid: String,
pub uuid: Uuid,
pub name: String,
#[serde(serialize_with = "tlm_data_type_serializer")]
#[serde(deserialize_with = "tlm_data_type_deserializer")]
pub data_type: TelemetryDataType,
pub data_type: DataType,
}

62
server/src/telemetry/history.rs
View File

@@ -1,10 +1,10 @@
use crate::core::TelemetryDataType;
use crate::serialization::file_ext::{ReadExt, WriteExt};
use crate::telemetry::data::TelemetryData;
use crate::telemetry::data_item::TelemetryDataItem;
use crate::telemetry::data_value::TelemetryDataValue;
use crate::telemetry::definition::TelemetryDefinition;
use anyhow::{anyhow, ensure, Context};
use api::data_type::DataType;
use api::data_value::DataValue;
use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Utc};
use log::{error, info};
use std::cmp::min;
@@ -44,7 +44,7 @@ fn update_next_from(
}
struct SegmentData {
values: Vec<TelemetryDataValue>,
values: Vec<DataValue>,
timestamps: Vec<DateTime<Utc>>,
}
@@ -66,7 +66,7 @@ impl HistorySegmentRam {
}
}
fn insert(&self, value: TelemetryDataValue, timestamp: DateTime<Utc>) {
fn insert(&self, value: DataValue, timestamp: DateTime<Utc>) {
if timestamp < self.start || timestamp >= self.end {
return;
}
@@ -121,7 +121,7 @@ impl HistorySegmentRam {
next_from,
);
result.push(TelemetryDataItem {
value: data.values[i].clone(),
value: data.values[i],
timestamp: t.to_rfc3339_opts(SecondsFormat::Millis, true),
});
}
@@ -196,9 +196,9 @@ impl HistorySegmentFile {
// Write all the values
for value in &data.values {
match value {
TelemetryDataValue::Float32(value) => file.write_data::<f32>(*value)?,
TelemetryDataValue::Float64(value) => file.write_data::<f64>(*value)?,
TelemetryDataValue::Boolean(value) => file.write_data::<bool>(*value)?,
DataValue::Float32(value) => file.write_data::<f32>(*value)?,
DataValue::Float64(value) => file.write_data::<f64>(*value)?,
DataValue::Boolean(value) => file.write_data::<bool>(*value)?,
}
}
@@ -215,10 +215,7 @@ impl HistorySegmentFile {
})
}
fn load_to_ram(
mut self,
telemetry_data_type: TelemetryDataType,
) -> anyhow::Result<HistorySegmentRam> {
fn load_to_ram(mut self, telemetry_data_type: DataType) -> anyhow::Result<HistorySegmentRam> {
let mut segment_data = SegmentData {
values: Vec::with_capacity(self.length as usize),
timestamps: Vec::with_capacity(self.length as usize),
@@ -281,7 +278,7 @@ impl HistorySegmentFile {
from: DateTime<Utc>,
to: DateTime<Utc>,
maximum_resolution: TimeDelta,
telemetry_data_type: TelemetryDataType,
telemetry_data_type: DataType,
) -> anyhow::Result<(DateTime<Utc>, Vec<TelemetryDataItem>)> {
self.file_position = 0;
self.file.seek(SeekFrom::Start(0))?;
@@ -334,22 +331,19 @@ impl HistorySegmentFile {
self.read_date_time()
}
fn read_telemetry_item(
&mut self,
telemetry_data_type: TelemetryDataType,
) -> anyhow::Result<TelemetryDataValue> {
fn read_telemetry_item(&mut self, telemetry_data_type: DataType) -> anyhow::Result<DataValue> {
match telemetry_data_type {
TelemetryDataType::Float32 => {
DataType::Float32 => {
self.file_position += 4;
Ok(TelemetryDataValue::Float32(self.file.read_data::<f32>()?))
Ok(DataValue::Float32(self.file.read_data::<f32>()?))
}
TelemetryDataType::Float64 => {
DataType::Float64 => {
self.file_position += 8;
Ok(TelemetryDataValue::Float64(self.file.read_data::<f64>()?))
Ok(DataValue::Float64(self.file.read_data::<f64>()?))
}
TelemetryDataType::Boolean => {
DataType::Boolean => {
self.file_position += 1;
Ok(TelemetryDataValue::Boolean(self.file.read_data::<bool>()?))
Ok(DataValue::Boolean(self.file.read_data::<bool>()?))
}
}
}
@@ -357,12 +351,12 @@ impl HistorySegmentFile {
fn get_telemetry_item(
&mut self,
index: u64,
telemetry_data_type: TelemetryDataType,
) -> anyhow::Result<TelemetryDataValue> {
telemetry_data_type: DataType,
) -> anyhow::Result<DataValue> {
let item_length = match telemetry_data_type {
TelemetryDataType::Float32 => 4,
TelemetryDataType::Float64 => 8,
TelemetryDataType::Boolean => 1,
DataType::Float32 => 4,
DataType::Float64 => 8,
DataType::Boolean => 1,
};
let desired_position =
Self::HEADER_LENGTH + self.length * Self::TIMESTAMP_LENGTH + index * item_length;
@@ -429,7 +423,7 @@ impl TelemetryHistory {
history_segment_ram: HistorySegmentRam,
) -> JoinHandle<()> {
let mut path = service.data_root_folder.clone();
path.push(&self.data.definition.uuid);
path.push(self.data.definition.uuid.as_hyphenated().to_string());
spawn_blocking(move || {
match HistorySegmentFile::save_to_disk(path, history_segment_ram) {
// Immediately drop the segment - now that we've saved it to disk we don't need to keep it in memory
@@ -450,7 +444,7 @@ impl TelemetryHistory {
start: DateTime<Utc>,
) -> JoinHandle<anyhow::Result<HistorySegmentFile>> {
let mut path = service.data_root_folder.clone();
path.push(&self.data.definition.uuid);
path.push(self.data.definition.uuid.as_hyphenated().to_string());
spawn_blocking(move || HistorySegmentFile::open(path, start))
}
@@ -458,7 +452,7 @@ impl TelemetryHistory {
&self,
start: DateTime<Utc>,
service: &TelemetryHistoryService,
telemetry_data_type: TelemetryDataType,
telemetry_data_type: DataType,
) -> HistorySegmentRam {
let ram = self
.get_disk_segment(service, start)
@@ -480,7 +474,7 @@ impl TelemetryHistory {
pub async fn insert(
&self,
service: &TelemetryHistoryService,
value: TelemetryDataValue,
value: DataValue,
timestamp: DateTime<Utc>,
) {
let segments = self.segments.read().await;
@@ -531,7 +525,7 @@ impl TelemetryHistory {
pub fn insert_sync(
history: Arc<Self>,
service: Arc<TelemetryHistoryService>,
value: TelemetryDataValue,
value: DataValue,
timestamp: DateTime<Utc>,
) {
tokio::spawn(async move {
@@ -579,7 +573,7 @@ impl TelemetryHistory {
.unwrap();
let mut path = telemetry_history_service.data_root_folder.clone();
path.push(&self.data.definition.uuid);
path.push(self.data.definition.uuid.as_hyphenated().to_string());
let mut start = start;
while start < end {

73
server/src/telemetry/management_service.rs
View File

@@ -1,7 +1,15 @@
use crate::core::{TelemetryDefinitionRequest, Uuid};
use crate::telemetry::data::TelemetryData;
use crate::telemetry::data_item::TelemetryDataItem;
use crate::telemetry::definition::TelemetryDefinition;
use crate::telemetry::history::{TelemetryHistory, TelemetryHistoryService};
use anyhow::bail;
use api::data_type::DataType;
use api::data_value::DataValue;
use api::messages::telemetry_definition::{
TelemetryDefinitionRequest, TelemetryDefinitionResponse,
};
use api::messages::telemetry_entry::TelemetryEntry;
use chrono::SecondsFormat;
use log::{error, info, warn};
use papaya::{HashMap, HashMapRef, LocalGuard};
use std::fs;
@@ -12,12 +20,13 @@ use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Mutex;
use tokio::time::sleep;
use uuid::Uuid;
const RELEASED_ATTEMPTS: usize = 5;
pub struct TelemetryManagementService {
uuid_index: HashMap<String, String>,
tlm_data: HashMap<String, Arc<TelemetryHistory>>,
uuid_index: HashMap<String, Uuid>,
tlm_data: HashMap<Uuid, Arc<TelemetryHistory>>,
telemetry_history_service: Arc<TelemetryHistoryService>,
metadata_file: Arc<Mutex<File>>,
}
@@ -49,8 +58,8 @@ impl TelemetryManagementService {
// Skip invalid entries
match serde_json::from_str::<TelemetryDefinition>(line) {
Ok(tlm_def) => {
let _ = uuid_index.insert(tlm_def.name.clone(), tlm_def.uuid.clone());
let _ = tlm_data.insert(tlm_def.uuid.clone(), Arc::new(tlm_def.into()));
let _ = uuid_index.insert(tlm_def.name.clone(), tlm_def.uuid);
let _ = tlm_data.insert(tlm_def.uuid, Arc::new(tlm_def.into()));
}
Err(err) => {
error!("Failed to parse metadata entry {err}");
@@ -79,23 +88,20 @@ impl TelemetryManagementService {
pub fn register(
&self,
telemetry_definition_request: TelemetryDefinitionRequest,
) -> anyhow::Result<String> {
) -> anyhow::Result<TelemetryDefinitionResponse> {
let uuid_index = self.uuid_index.pin();
let tlm_data = self.tlm_data.pin();
let uuid = uuid_index
.get_or_insert_with(telemetry_definition_request.name.clone(), || {
Uuid::random().value
})
.clone();
let uuid =
*uuid_index.get_or_insert_with(telemetry_definition_request.name.clone(), Uuid::new_v4);
let inserted = tlm_data.try_insert(
uuid.clone(),
uuid,
Arc::new(
TelemetryDefinition {
uuid: uuid.clone(),
uuid,
name: telemetry_definition_request.name.clone(),
data_type: telemetry_definition_request.data_type(),
data_type: telemetry_definition_request.data_type,
}
.into(),
),
@@ -129,7 +135,38 @@ impl TelemetryManagementService {
});
}
Ok(uuid)
Ok(TelemetryDefinitionResponse { uuid })
}
pub fn add_tlm_item(&self, tlm_item: TelemetryEntry) -> anyhow::Result<()> {
let tlm_management_pin = self.pin();
let Some(tlm_data) = tlm_management_pin.get_by_uuid(&tlm_item.uuid) else {
bail!("Telemetry Item Not Found");
};
let expected_type = match &tlm_item.value {
DataValue::Float32(_) => DataType::Float32,
DataValue::Float64(_) => DataType::Float64,
DataValue::Boolean(_) => DataType::Boolean,
};
if expected_type != tlm_data.data.definition.data_type {
bail!("Data Type Mismatch");
};
let _ = tlm_data.data.data.send_replace(Some(TelemetryDataItem {
value: tlm_item.value,
timestamp: tlm_item
.timestamp
.to_rfc3339_opts(SecondsFormat::Millis, true),
}));
TelemetryHistory::insert_sync(
tlm_data.clone(),
self.history_service(),
tlm_item.value,
tlm_item.timestamp,
);
Ok(())
}
pub fn get_by_name(&self, name: &String) -> Option<TelemetryData> {
@@ -138,7 +175,7 @@ impl TelemetryManagementService {
self.get_by_uuid(uuid)
}
pub fn get_by_uuid(&self, uuid: &String) -> Option<TelemetryData> {
pub fn get_by_uuid(&self, uuid: &Uuid) -> Option<TelemetryData> {
let tlm_data = self.tlm_data.pin();
tlm_data
.get(uuid)
@@ -200,11 +237,11 @@ impl TelemetryManagementService {
}
pub struct TelemetryManagementServicePin<'a> {
tlm_data: HashMapRef<'a, String, Arc<TelemetryHistory>, RandomState, LocalGuard<'a>>,
tlm_data: HashMapRef<'a, Uuid, Arc<TelemetryHistory>, RandomState, LocalGuard<'a>>,
}
impl<'a> TelemetryManagementServicePin<'a> {
pub fn get_by_uuid(&'a self, uuid: &String) -> Option<&'a Arc<TelemetryHistory>> {
pub fn get_by_uuid(&'a self, uuid: &Uuid) -> Option<&'a Arc<TelemetryHistory>> {
self.tlm_data.get(uuid)
}
}

2
server/src/telemetry/mod.rs
View File

@@ -1,7 +1,5 @@
pub mod data;
pub mod data_item;
pub mod data_type;
pub mod data_value;
pub mod definition;
pub mod history;
pub mod management_service;