sergeysav/telemetry_visualization
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes history loading slowly due to context switches

Browse Source
This commit is contained in:
Sergey Savelyev
2025-01-04 19:13:32 -05:00
parent c69022448f
commit 4dd7cea97d
12 changed files with 278 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
server/src/http/api/mod.rs
View File

@@ -5,6 +5,8 @@ use chrono::{DateTime, TimeDelta, Utc};
use log::trace;
use serde::Deserialize;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::timeout;
#[get("/tlm/info/{name:[\\w\\d/_-]+}")]
async fn get_tlm_definition(
@@ -16,7 +18,6 @@ async fn get_tlm_definition(
let Some(data) = data.get_by_name(&string) else {
return Err(HttpServerResultError::TlmNameNotFound { tlm: string });
};
Ok(web::Json(data.definition.clone()))
}
@@ -29,7 +30,7 @@ struct HistoryQuery {
#[get("/tlm/history/{uuid:[0-9a-f]+}")]
async fn get_tlm_history(
data: web::Data<Arc<TelemetryManagementService>>,
data_arc: web::Data<Arc<TelemetryManagementService>>,
uuid: web::Path<String>,
info: web::Query<HistoryQuery>,
) -> Result<impl Responder, HttpServerResultError> {
@@ -54,14 +55,18 @@ async fn get_tlm_history(
};
let maximum_resolution = TimeDelta::milliseconds(info.resolution);
let history_service = data.history_service();
let data = data.pin();
let history_service = data_arc.history_service();
let data = data_arc.pin();
match data.get_by_uuid(&uuid) {
None => Err(HttpServerResultError::TlmUuidNotFound { uuid }),
Some(tlm) => Ok(web::Json(
tlm.get(from, to, maximum_resolution, &history_service)
.await,
)),
Some(tlm) => timeout(
Duration::from_secs(10),
tlm.get(from, to, maximum_resolution, &history_service),
)
.await
.map(|result| Ok(web::Json(result)))
.unwrap_or_else(|_| Err(HttpServerResultError::Timeout)),
}
}

3
server/src/http/error.rs
View File

@@ -15,6 +15,8 @@ pub enum HttpServerResultError {
TlmUuidNotFound { uuid: String },
#[error("DateTime Parsing Error: {date_time}")]
InvalidDateTime { date_time: String },
#[error("Timed out")]
Timeout,
}
impl ResponseError for HttpServerResultError {
@@ -23,6 +25,7 @@ impl ResponseError for HttpServerResultError {
HttpServerResultError::TlmNameNotFound { .. } => StatusCode::NOT_FOUND,
HttpServerResultError::TlmUuidNotFound { .. } => StatusCode::NOT_FOUND,
HttpServerResultError::InvalidDateTime { .. } => StatusCode::BAD_REQUEST,
HttpServerResultError::Timeout { .. } => StatusCode::GATEWAY_TIMEOUT,
}
}
fn error_response(&self) -> HttpResponse {

4
server/src/http/mod.rs
View File

@@ -6,7 +6,7 @@ use crate::http::api::setup_api;
use crate::http::websocket::setup_websocket;
use crate::telemetry::management_service::TelemetryManagementService;
use actix_web::{web, App, HttpServer};
use log::info;
use log::{error, info};
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
@@ -29,5 +29,7 @@ pub async fn setup(
.run()
.await?;
error!("http setup end");
Ok(())
}

10
server/src/lib.rs
View File

@@ -1,5 +1,6 @@
mod grpc;
mod http;
mod serialization;
mod telemetry;
mod uuid;
@@ -11,6 +12,8 @@ use crate::telemetry::history::TelemetryHistoryService;
use crate::telemetry::management_service::TelemetryManagementService;
use log::error;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::sleep;
use tokio_util::sync::CancellationToken;
pub async fn setup() -> anyhow::Result<()> {
@@ -35,7 +38,14 @@ pub async fn setup() -> anyhow::Result<()> {
grpc_server.await?; //grpc server is dropped
drop(cancellation_token); // All cancellation tokens are now dropped
error!("after awaits");
// Perform cleanup functions - at this point all servers have stopped and we can be sure that cleaning things up is safe
for _ in 0..15 {
if Arc::strong_count(&tlm) != 1 {
sleep(Duration::from_secs(1)).await;
}
}
if let Some(tlm) = Arc::into_inner(tlm) {
tlm.cleanup().await?;
} else {

24
server/src/serialization/file_ext.rs Normal file
View File

@@ -0,0 +1,24 @@
use std::fs::File;
use std::io;
pub trait FileWriteableType {
fn write_to_file(self, file: &mut File) -> io::Result<()>;
}
pub trait FileReadableType: Sized {
fn read_from_file(file: &mut File) -> io::Result<Self>;
}
pub trait FileExt {
fn write_data<T: FileWriteableType>(&mut self, data: T) -> io::Result<()>;
fn read_data<T: FileReadableType>(&mut self) -> io::Result<T>;
}
impl FileExt for File {
fn write_data<T: FileWriteableType>(&mut self, data: T) -> io::Result<()> {
data.write_to_file(self)
}
fn read_data<T: FileReadableType>(&mut self) -> io::Result<T> {
T::read_from_file(self)
}
}

2
server/src/serialization/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod file_ext;
pub mod primitives;

33
server/src/serialization/primitives.rs Normal file
View File

@@ -0,0 +1,33 @@
use crate::serialization::file_ext::{FileReadableType, FileWriteableType};
use std::fs::File;
use std::io::{Read, Write};
macro_rules! primitive_write_read {
( $primitive:ty, $length:expr ) => {
impl FileWriteableType for $primitive {
fn write_to_file(self, file: &mut File) -> std::io::Result<()> {
file.write_all(&self.to_be_bytes())
}
}
impl FileReadableType for $primitive {
fn read_from_file(file: &mut File) -> std::io::Result<Self> {
let mut buffer = [0u8; $length];
file.read_exact(&mut buffer)?;
Ok(Self::from_be_bytes(buffer))
}
}
};
}
primitive_write_read!(u64, 8);
primitive_write_read!(u32, 4);
primitive_write_read!(u16, 2);
primitive_write_read!(u8, 1);
primitive_write_read!(i64, 8);
primitive_write_read!(i32, 4);
primitive_write_read!(i16, 2);
primitive_write_read!(i8, 1);
primitive_write_read!(f64, 8);
primitive_write_read!(f32, 4);

159
server/src/telemetry/history.rs
View File

@@ -1,20 +1,20 @@
use crate::core::TelemetryDataType;
use crate::serialization::file_ext::FileExt;
use crate::telemetry::data::TelemetryData;
use crate::telemetry::data_item::TelemetryDataItem;
use crate::telemetry::data_value::TelemetryDataValue;
use crate::telemetry::definition::TelemetryDefinition;
use anyhow::{ensure, Context};
use anyhow::{anyhow, ensure, Context};
use chrono::{DateTime, DurationRound, SecondsFormat, TimeDelta, Utc};
use log::{error, info};
use std::cmp::min;
use std::collections::VecDeque;
use std::io::SeekFrom;
use std::fs::File;
use std::io::{Seek, SeekFrom, Write};
use std::path::PathBuf;
use std::sync::{Arc, RwLock};
use std::{fs, path};
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::task::JoinHandle;
use tokio::task::{spawn_blocking, JoinHandle};
const FOLDER_DURATION: TimeDelta = TimeDelta::hours(1);
@@ -133,21 +133,18 @@ impl HistorySegmentRam {
}
}
struct HistorySegmentDisk {
struct HistorySegmentFile {
start: DateTime<Utc>,
end: DateTime<Utc>,
length: u64,
file: File,
}
impl HistorySegmentDisk {
impl HistorySegmentFile {
const TIMESTAMP_LENGTH: u64 = 8 + 4;
const HEADER_LENGTH: u64 = Self::TIMESTAMP_LENGTH + Self::TIMESTAMP_LENGTH + 8;
async fn save_to_disk(
mut folder: PathBuf,
mut segment: HistorySegmentRam,
) -> anyhow::Result<Self> {
fn save_to_disk(mut folder: PathBuf, mut segment: HistorySegmentRam) -> anyhow::Result<Self> {
// Get the path for the specific timestamp we want to save to disk
let folder_time = segment.start.duration_trunc(FOLDER_DURATION)?;
folder.push(folder_time.to_rfc3339_opts(SecondsFormat::Secs, true));
@@ -161,7 +158,7 @@ impl HistorySegmentDisk {
segment.start.to_rfc3339_opts(SecondsFormat::Secs, true)
));
let file = File::create(file).await?;
let file = File::create(file)?;
let mut result = Self {
start: segment.start,
@@ -176,14 +173,16 @@ impl HistorySegmentDisk {
// Write the segment bounds
result
.file
.write_i64(utc_offset_start.num_seconds())
.await?;
.write_data::<i64>(utc_offset_start.num_seconds())?;
result
.file
.write_i32(utc_offset_start.subsec_nanos())
.await?;
result.file.write_i64(utc_offset_end.num_seconds()).await?;
result.file.write_i32(utc_offset_end.subsec_nanos()).await?;
.write_data::<i32>(utc_offset_start.subsec_nanos())?;
result
.file
.write_data::<i64>(utc_offset_end.num_seconds())?;
result
.file
.write_data::<i32>(utc_offset_end.subsec_nanos())?;
let data = segment.data.get_mut().unwrap_or_else(|err| {
error!(
@@ -199,28 +198,28 @@ impl HistorySegmentDisk {
);
result.length = data.timestamps.len() as u64;
result.file.write_u64(result.length).await?;
result.file.write_data::<u64>(result.length)?;
// Write all the timestamps
for timestamp in &data.timestamps {
let utc_offset = *timestamp - DateTime::UNIX_EPOCH;
result.file.write_i64(utc_offset.num_seconds()).await?;
result.file.write_i32(utc_offset.subsec_nanos()).await?;
result.file.write_data::<i64>(utc_offset.num_seconds())?;
result.file.write_data::<i32>(utc_offset.subsec_nanos())?;
}
// Write all the values
for value in &data.values {
match value {
TelemetryDataValue::Float32(value) => result.file.write_f32(*value).await?,
TelemetryDataValue::Float64(value) => result.file.write_f64(*value).await?,
TelemetryDataValue::Float32(value) => result.file.write_data::<f32>(*value)?,
TelemetryDataValue::Float64(value) => result.file.write_data::<f64>(*value)?,
}
}
result.file.flush().await?;
result.file.flush()?;
Ok(result)
}
async fn load_to_ram(
fn load_to_ram(
mut self,
telemetry_data_type: TelemetryDataType,
) -> anyhow::Result<HistorySegmentRam> {
@@ -229,14 +228,14 @@ impl HistorySegmentDisk {
timestamps: Vec::with_capacity(self.length as usize),
};
self.file.seek(SeekFrom::Start(Self::HEADER_LENGTH)).await?;
self.file.seek(SeekFrom::Start(Self::HEADER_LENGTH))?;
for _ in 0..self.length {
segment_data.timestamps.push(self.read_date_time().await?);
segment_data.timestamps.push(self.read_date_time()?);
}
for _ in 0..self.length {
segment_data
.values
.push(self.read_telemetry_item(telemetry_data_type).await?);
.push(self.read_telemetry_item(telemetry_data_type)?);
}
Ok(HistorySegmentRam {
@@ -246,7 +245,7 @@ impl HistorySegmentDisk {
})
}
async fn open(folder: PathBuf, start: DateTime<Utc>) -> anyhow::Result<Self> {
fn open(folder: PathBuf, start: DateTime<Utc>) -> anyhow::Result<Self> {
// Get the path for the specific timestamp we want to save to disk
let folder_time = start.duration_trunc(FOLDER_DURATION)?;
let mut file = folder;
@@ -256,21 +255,21 @@ impl HistorySegmentDisk {
start.to_rfc3339_opts(SecondsFormat::Secs, true)
));
let mut file = File::open(file).await?;
let mut file = File::open(file)?;
// Write the segment bounds
let start_seconds = file.read_i64().await?;
let start_nanos = file.read_i32().await?;
let end_seconds = file.read_i64().await?;
let end_nanos = file.read_i32().await?;
let start_seconds = file.read_data::<i64>()?;
let start_nanos = file.read_data::<i32>()?;
let end_seconds = file.read_data::<i64>()?;
let end_nanos = file.read_data::<i32>()?;
let start = TimeDelta::new(start_seconds, start_nanos as u32)
.context("Failed to reconstruct start TimeDelta")?;
let end = TimeDelta::new(end_seconds, end_nanos as u32)
.context("Failed to reconstruct end TimeDelta")?;
let length = file.read_u64().await?;
let length = file.read_data::<u64>()?;
Ok(HistorySegmentDisk {
Ok(HistorySegmentFile {
start: DateTime::UNIX_EPOCH + start,
end: DateTime::UNIX_EPOCH + end,
length,
@@ -278,7 +277,7 @@ impl HistorySegmentDisk {
})
}
async fn get(
fn get(
&mut self,
from: DateTime<Utc>,
to: DateTime<Utc>,
@@ -290,10 +289,10 @@ impl HistorySegmentDisk {
let mut next_from = from;
if from < self.end && self.start < to {
let start = self.partition_point(from).await?;
let start = self.partition_point(from)?;
if start < self.length {
for i in start..self.length {
let t = self.get_date_time(i).await?;
let t = self.get_date_time(i)?;
if t >= self.end {
break;
}
@@ -306,7 +305,7 @@ impl HistorySegmentDisk {
next_from,
);
result.push(TelemetryDataItem {
value: self.get_telemetry_item(i, telemetry_data_type).await?,
value: self.get_telemetry_item(i, telemetry_data_type)?,
timestamp: t.to_rfc3339_opts(SecondsFormat::Millis, true),
});
}
@@ -317,38 +316,36 @@ impl HistorySegmentDisk {
Ok((next_from, result))
}
async fn read_date_time(&mut self) -> anyhow::Result<DateTime<Utc>> {
let seconds = self.file.read_i64().await?;
let nanos = self.file.read_i32().await?;
fn read_date_time(&mut self) -> anyhow::Result<DateTime<Utc>> {
let seconds = self.file.read_data::<i64>()?;
let nanos = self.file.read_data::<i32>()?;
let start =
TimeDelta::new(seconds, nanos as u32).context("Failed to reconstruct TimeDelta")?;
Ok(DateTime::UNIX_EPOCH + start)
}
async fn get_date_time(&mut self, index: u64) -> anyhow::Result<DateTime<Utc>> {
self.file
.seek(SeekFrom::Start(
Self::HEADER_LENGTH + index * Self::TIMESTAMP_LENGTH,
))
.await?;
self.read_date_time().await
fn get_date_time(&mut self, index: u64) -> anyhow::Result<DateTime<Utc>> {
self.file.seek(SeekFrom::Start(
Self::HEADER_LENGTH + index * Self::TIMESTAMP_LENGTH,
))?;
self.read_date_time()
}
async fn read_telemetry_item(
fn read_telemetry_item(
&mut self,
telemetry_data_type: TelemetryDataType,
) -> anyhow::Result<TelemetryDataValue> {
match telemetry_data_type {
TelemetryDataType::Float32 => {
Ok(TelemetryDataValue::Float32(self.file.read_f32().await?))
Ok(TelemetryDataValue::Float32(self.file.read_data::<f32>()?))
}
TelemetryDataType::Float64 => {
Ok(TelemetryDataValue::Float64(self.file.read_f64().await?))
Ok(TelemetryDataValue::Float64(self.file.read_data::<f64>()?))
}
}
}
async fn get_telemetry_item(
fn get_telemetry_item(
&mut self,
index: u64,
telemetry_data_type: TelemetryDataType,
@@ -357,15 +354,13 @@ impl HistorySegmentDisk {
TelemetryDataType::Float32 => 4,
TelemetryDataType::Float64 => 8,
};
self.file
.seek(SeekFrom::Start(
Self::HEADER_LENGTH + self.length * Self::TIMESTAMP_LENGTH + index * item_length,
))
.await?;
self.read_telemetry_item(telemetry_data_type).await
self.file.seek(SeekFrom::Start(
Self::HEADER_LENGTH + self.length * Self::TIMESTAMP_LENGTH + index * item_length,
))?;
self.read_telemetry_item(telemetry_data_type)
}
async fn partition_point(&mut self, date_time: DateTime<Utc>) -> anyhow::Result<u64> {
fn partition_point(&mut self, date_time: DateTime<Utc>) -> anyhow::Result<u64> {
if self.length == 0 {
return Ok(0);
}
@@ -379,7 +374,7 @@ impl HistorySegmentDisk {
let half = size / 2;
let mid = left + half;
let is_less = self.get_date_time(mid).await? < date_time;
let is_less = self.get_date_time(mid)? < date_time;
if is_less {
left = mid;
}
@@ -387,7 +382,7 @@ impl HistorySegmentDisk {
}
Ok(left
+ if self.get_date_time(left).await? < date_time {
+ if self.get_date_time(left)? < date_time {
1
} else {
0
@@ -423,8 +418,8 @@ impl TelemetryHistory {
) -> JoinHandle<()> {
let mut path = service.data_root_folder.clone();
path.push(&self.data.definition.uuid);
tokio::spawn(async move {
match HistorySegmentDisk::save_to_disk(path, history_segment_ram).await {
spawn_blocking(move || {
match HistorySegmentFile::save_to_disk(path, history_segment_ram) {
// Immediately drop the segment - now that we've saved it to disk we don't need to keep it in memory
Ok(segment) => drop(segment),
Err(err) => {
@@ -437,14 +432,14 @@ impl TelemetryHistory {
})
}
async fn get_disk_segment(
fn get_disk_segment(
&self,
service: &TelemetryHistoryService,
start: DateTime<Utc>,
) -> anyhow::Result<HistorySegmentDisk> {
) -> JoinHandle<anyhow::Result<HistorySegmentFile>> {
let mut path = service.data_root_folder.clone();
path.push(&self.data.definition.uuid);
HistorySegmentDisk::open(path, start).await
spawn_blocking(move || HistorySegmentFile::open(path, start))
}
async fn create_ram_segment(
@@ -453,14 +448,20 @@ impl TelemetryHistory {
service: &TelemetryHistoryService,
telemetry_data_type: TelemetryDataType,
) -> HistorySegmentRam {
let ram = match self.get_disk_segment(service, start).await {
Ok(disk) => disk.load_to_ram(telemetry_data_type).await,
let ram = self
.get_disk_segment(service, start)
.await
.unwrap_or_else(|e| Err(anyhow!("Join Error {e}")))
.map(|disk| spawn_blocking(move || disk.load_to_ram(telemetry_data_type)));
let ram = match ram {
Ok(ram) => ram.await.unwrap_or_else(|e| Err(anyhow!("Join Error {e}"))),
Err(e) => Err(e),
};
match ram {
Ok(ram) => ram,
Err(_) => HistorySegmentRam::new(start, start + service.segment_width),
_ => HistorySegmentRam::new(start, start + service.segment_width),
}
}
@@ -560,14 +561,11 @@ impl TelemetryHistory {
let mut start = start;
while start < end {
// We're going to ignore errors with getting the disk segment
if let Ok(mut disk) = self
if let Ok(Ok(mut disk)) = self
.get_disk_segment(telemetry_history_service, start)
.await
{
match disk
.get(from, to, maximum_resolution, self.data.definition.data_type)
.await
{
match disk.get(from, to, maximum_resolution, self.data.definition.data_type) {
Ok((new_from, new_data)) => {
from = new_from;
result.extend(new_data);
@@ -593,8 +591,13 @@ impl TelemetryHistory {
pub async fn cleanup(&self, service: &TelemetryHistoryService) -> anyhow::Result<()> {
let mut segments = self.segments.write().await;
for segment in segments.drain(..) {
self.cleanup_segment(service, segment).await?;
let segments = segments
.drain(..)
.map(|segment| self.cleanup_segment(service, segment))
.collect::<Vec<_>>();
for segment in segments {
segment.await?;
}
Ok(())