sergeysav/telemetry_visualization
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

improves server efficiency

Browse Source
This commit is contained in:
Sergey Savelyev
2025-01-13 19:38:21 -08:00
parent 2cb1eec404
commit fdd3f2c128
4 changed files with 78 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
server/src/telemetry/history.rs
View File

@@ -1,5 +1,5 @@
use crate::core::TelemetryDataType;
use crate::serialization::file_ext::FileExt;
use crate::serialization::file_ext::{ReadExt, WriteExt};
use crate::telemetry::data::TelemetryData;
use crate::telemetry::data_item::TelemetryDataItem;
use crate::telemetry::data_value::TelemetryDataValue;
@@ -10,7 +10,7 @@ use log::{error, info};
use std::cmp::min;
use std::collections::VecDeque;
use std::fs::File;
use std::io::{Seek, SeekFrom, Write};
use std::io::{BufReader, BufWriter, Seek, SeekFrom, Write};
use std::path::PathBuf;
use std::sync::{Arc, RwLock};
use std::{fs, path};
@@ -137,7 +137,8 @@ struct HistorySegmentFile {
start: DateTime<Utc>,
end: DateTime<Utc>,
length: u64,
file: File,
file: BufReader<File>,
file_position: i64,
}
impl HistorySegmentFile {
@@ -158,31 +159,16 @@ impl HistorySegmentFile {
segment.start.to_rfc3339_opts(SecondsFormat::Secs, true)
));
let file = File::create(file)?;
let mut file = BufWriter::new(File::create(file)?);
let mut result = Self {
start: segment.start,
end: segment.end,
length: 0,
file,
};
let utc_offset_start = result.start - DateTime::UNIX_EPOCH;
let utc_offset_end = result.end - DateTime::UNIX_EPOCH;
let utc_offset_start = segment.start - DateTime::UNIX_EPOCH;
let utc_offset_end = segment.end - DateTime::UNIX_EPOCH;
// Write the segment bounds
result
.file
.write_data::<i64>(utc_offset_start.num_seconds())?;
result
.file
.write_data::<i32>(utc_offset_start.subsec_nanos())?;
result
.file
.write_data::<i64>(utc_offset_end.num_seconds())?;
result
.file
.write_data::<i32>(utc_offset_end.subsec_nanos())?;
file.write_data::<i64>(utc_offset_start.num_seconds())?;
file.write_data::<i32>(utc_offset_start.subsec_nanos())?;
file.write_data::<i64>(utc_offset_end.num_seconds())?;
file.write_data::<i32>(utc_offset_end.subsec_nanos())?;
let data = segment.data.get_mut().unwrap_or_else(|err| {
error!(
@@ -197,26 +183,35 @@ impl HistorySegmentFile {
"Invalid Segment Cannot Be Saved to Disk"
);
result.length = data.timestamps.len() as u64;
result.file.write_data::<u64>(result.length)?;
let length = data.timestamps.len() as u64;
file.write_data::<u64>(length)?;
// Write all the timestamps
for timestamp in &data.timestamps {
let utc_offset = *timestamp - DateTime::UNIX_EPOCH;
result.file.write_data::<i64>(utc_offset.num_seconds())?;
result.file.write_data::<i32>(utc_offset.subsec_nanos())?;
file.write_data::<i64>(utc_offset.num_seconds())?;
file.write_data::<i32>(utc_offset.subsec_nanos())?;
}
// Write all the values
for value in &data.values {
match value {
TelemetryDataValue::Float32(value) => result.file.write_data::<f32>(*value)?,
TelemetryDataValue::Float64(value) => result.file.write_data::<f64>(*value)?,
TelemetryDataValue::Float32(value) => file.write_data::<f32>(*value)?,
TelemetryDataValue::Float64(value) => file.write_data::<f64>(*value)?,
}
}
result.file.flush()?;
Ok(result)
file.flush()?;
let mut file = BufReader::new(file.into_inner()?);
file.seek(SeekFrom::Start(0))?;
Ok(Self {
start: segment.start,
end: segment.end,
length: 0,
file,
file_position: 0,
})
}
fn load_to_ram(
@@ -229,6 +224,7 @@ impl HistorySegmentFile {
};
self.file.seek(SeekFrom::Start(Self::HEADER_LENGTH))?;
self.file_position = Self::HEADER_LENGTH as i64;
for _ in 0..self.length {
segment_data.timestamps.push(self.read_date_time()?);
}
@@ -255,7 +251,7 @@ impl HistorySegmentFile {
start.to_rfc3339_opts(SecondsFormat::Secs, true)
));
let mut file = File::open(file)?;
let mut file = BufReader::new(File::open(file)?);
// Write the segment bounds
let start_seconds = file.read_data::<i64>()?;
@@ -269,11 +265,13 @@ impl HistorySegmentFile {
let length = file.read_data::<u64>()?;
file.seek(SeekFrom::Start(0))?;
Ok(HistorySegmentFile {
start: DateTime::UNIX_EPOCH + start,
end: DateTime::UNIX_EPOCH + end,
length,
file,
file_position: 0,
})
}
@@ -284,6 +282,8 @@ impl HistorySegmentFile {
maximum_resolution: TimeDelta,
telemetry_data_type: TelemetryDataType,
) -> anyhow::Result<(DateTime<Utc>, Vec<TelemetryDataItem>)> {
self.file_position = 0;
self.file.seek(SeekFrom::Start(0))?;
let mut result = vec![];
let mut next_from = from;
@@ -319,15 +319,17 @@ impl HistorySegmentFile {
fn read_date_time(&mut self) -> anyhow::Result<DateTime<Utc>> {
let seconds = self.file.read_data::<i64>()?;
let nanos = self.file.read_data::<i32>()?;
self.file_position += 8 + 4;
let start =
TimeDelta::new(seconds, nanos as u32).context("Failed to reconstruct TimeDelta")?;
Ok(DateTime::UNIX_EPOCH + start)
}
fn get_date_time(&mut self, index: u64) -> anyhow::Result<DateTime<Utc>> {
self.file.seek(SeekFrom::Start(
Self::HEADER_LENGTH + index * Self::TIMESTAMP_LENGTH,
))?;
let desired_position = Self::HEADER_LENGTH + index * Self::TIMESTAMP_LENGTH;
let seek_amount = desired_position as i64 - self.file_position;
self.file_position += seek_amount;
self.file.seek_relative(seek_amount)?;
self.read_date_time()
}
@@ -337,9 +339,11 @@ impl HistorySegmentFile {
) -> anyhow::Result<TelemetryDataValue> {
match telemetry_data_type {
TelemetryDataType::Float32 => {
self.file_position += 4;
Ok(TelemetryDataValue::Float32(self.file.read_data::<f32>()?))
}
TelemetryDataType::Float64 => {
self.file_position += 8;
Ok(TelemetryDataValue::Float64(self.file.read_data::<f64>()?))
}
}
@@ -354,9 +358,11 @@ impl HistorySegmentFile {
TelemetryDataType::Float32 => 4,
TelemetryDataType::Float64 => 8,
};
self.file.seek(SeekFrom::Start(
Self::HEADER_LENGTH + self.length * Self::TIMESTAMP_LENGTH + index * item_length,
))?;
let desired_position =
Self::HEADER_LENGTH + self.length * Self::TIMESTAMP_LENGTH + index * item_length;
let seek_amount = desired_position as i64 - self.file_position;
self.file_position += seek_amount;
self.file.seek_relative(seek_amount)?;
self.read_telemetry_item(telemetry_data_type)
}