feat(logs): store stderr and stdout with details in dedicated log messages table
This commit is contained in:
parent
c40c6aedef
commit
55438fbd83
12 changed files with 155 additions and 27 deletions
26
Cargo.lock
generated
26
Cargo.lock
generated
|
@ -189,6 +189,7 @@ dependencies = [
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
|
"strum_macros",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-cron-scheduler",
|
"tokio-cron-scheduler",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
|
@ -715,6 +716,12 @@ dependencies = [
|
||||||
"unicode-segmentation",
|
"unicode-segmentation",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "heck"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hermit-abi"
|
name = "hermit-abi"
|
||||||
version = "0.3.9"
|
version = "0.3.9"
|
||||||
|
@ -1603,6 +1610,7 @@ dependencies = [
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
"url",
|
"url",
|
||||||
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -1626,7 +1634,7 @@ checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"either",
|
"either",
|
||||||
"heck",
|
"heck 0.4.1",
|
||||||
"hex",
|
"hex",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
|
@ -1684,6 +1692,7 @@ dependencies = [
|
||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"uuid",
|
||||||
"whoami",
|
"whoami",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -1723,6 +1732,7 @@ dependencies = [
|
||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"uuid",
|
||||||
"whoami",
|
"whoami",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -1748,6 +1758,7 @@ dependencies = [
|
||||||
"tracing",
|
"tracing",
|
||||||
"url",
|
"url",
|
||||||
"urlencoding",
|
"urlencoding",
|
||||||
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
@ -1767,6 +1778,19 @@ version = "0.11.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "strum_macros"
|
||||||
|
version = "0.26.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
|
||||||
|
dependencies = [
|
||||||
|
"heck 0.5.0",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"rustversion",
|
||||||
|
"syn 2.0.72",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "subtle"
|
name = "subtle"
|
||||||
version = "2.6.1"
|
version = "2.6.1"
|
||||||
|
|
|
@ -8,7 +8,7 @@ serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
serde_yaml = "0.9"
|
serde_yaml = "0.9"
|
||||||
chrono = { version = "0.4.26", features = ["serde"] }
|
chrono = { version = "0.4.26", features = ["serde"] }
|
||||||
sqlx = { version = "0.7.4", features = ["sqlite", "runtime-tokio", "chrono"] }
|
sqlx = { version = "0.7.4", features = ["sqlite", "runtime-tokio", "chrono", "uuid"] }
|
||||||
anyhow = "1.0.75"
|
anyhow = "1.0.75"
|
||||||
clap = "4.5.4"
|
clap = "4.5.4"
|
||||||
tokio = { version = "1.37.0", features = ["full"] }
|
tokio = { version = "1.37.0", features = ["full"] }
|
||||||
|
@ -22,4 +22,5 @@ tower-http = { version = "0.5.2", features = ["fs"] }
|
||||||
tokio-cron-scheduler = "0.10.2"
|
tokio-cron-scheduler = "0.10.2"
|
||||||
argh = "0.1.12"
|
argh = "0.1.12"
|
||||||
tokio-util = "0.7.11"
|
tokio-util = "0.7.11"
|
||||||
|
strum_macros = "0.26.4"
|
||||||
|
|
||||||
|
|
3
TODO.md
3
TODO.md
|
@ -2,7 +2,8 @@
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
- [ ] Implement support for webhook and debouncing of webhook
|
- [x] Implement support for webhook and debouncing of webhook
|
||||||
|
- [ ] Add stderr capture
|
||||||
- [ ] Implement basic auth with HTTP basic auth (to trigger and see logs only)
|
- [ ] Implement basic auth with HTTP basic auth (to trigger and see logs only)
|
||||||
- [ ] add CSS badge and color code on job status
|
- [ ] add CSS badge and color code on job status
|
||||||
- [ ] Validating config file
|
- [ ] Validating config file
|
||||||
|
|
|
@ -4,3 +4,11 @@ table {
|
||||||
table tbody tr td:first-child {
|
table tbody tr td:first-child {
|
||||||
padding-right: 3rem !important;
|
padding-right: 3rem !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.line-stderr {
|
||||||
|
background-color: rgba(255, 0, 0, 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
.logs-lines {
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ tasks:
|
||||||
SIMULATION_SPEED: 0.2
|
SIMULATION_SPEED: 0.2
|
||||||
command:
|
command:
|
||||||
- /usr/bin/python3
|
- /usr/bin/python3
|
||||||
- /home/mbess/workspace/perso/autotasker/examples/do_something_1.py
|
- /home/mbess/workspace/autotasker/examples/do_something_1.py
|
||||||
|
|
||||||
reindex_db:
|
reindex_db:
|
||||||
name: Reindex the whole database
|
name: Reindex the whole database
|
||||||
|
|
|
@ -1,14 +1,15 @@
|
||||||
|
import os, sys
|
||||||
import random
|
import random
|
||||||
import os
|
|
||||||
import string
|
import string
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
iterations = random.randint(10, 150)
|
iterations = random.randint(10, 150)
|
||||||
speed = float(os.getenv("SIMULATION_SPEED") or 0.8)
|
speed = float(os.getenv("SIMULATION_SPEED") or 0.8)
|
||||||
print(f"Going for {iterations=}")
|
print(f"Going for {iterations=} with {speed=}")
|
||||||
for i in range(iterations):
|
for i in range(iterations):
|
||||||
|
if random.uniform(0, 1) > 0.8:
|
||||||
|
print("Some error we will see", file=sys.stderr)
|
||||||
print(
|
print(
|
||||||
str(i) + " " +
|
str(i) + " " +
|
||||||
''.join(
|
''.join(
|
||||||
|
@ -24,6 +25,7 @@ def main():
|
||||||
(random.uniform(0, 5) if random.uniform(0, 1) > 0.99 else 0))
|
(random.uniform(0, 5) if random.uniform(0, 1) > 0.99 else 0))
|
||||||
)
|
)
|
||||||
print("Done, script is finished")
|
print("Done, script is finished")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
||||||
|
|
|
@ -5,8 +5,17 @@ CREATE TABLE task_runs (
|
||||||
status TEXT CHECK(status IN ('pending','running','failed','success')) NOT NULL DEFAULT 'pending',
|
status TEXT CHECK(status IN ('pending','running','failed','success')) NOT NULL DEFAULT 'pending',
|
||||||
trigger_mode TEXT CHECK(trigger_mode IN ('manual','webhook','schedule')) NOT NULL DEFAULT 'manual',
|
trigger_mode TEXT CHECK(trigger_mode IN ('manual','webhook','schedule')) NOT NULL DEFAULT 'manual',
|
||||||
exit_code INT,
|
exit_code INT,
|
||||||
logs TEXT,
|
runtime_details JSON,
|
||||||
submitted_at DATETIME,
|
submitted_at DATETIME,
|
||||||
started_at DATETIME,
|
started_at DATETIME,
|
||||||
ended_at DATETIME
|
ended_at DATETIME
|
||||||
);
|
);
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS logs_lines;
|
||||||
|
CREATE TABLE logs_lines (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
task_run_id TEXT NOT NULL,
|
||||||
|
kind TEXT CHECK(kind IN ('Stdout', 'Stderr')),
|
||||||
|
captured_at INT, -- unix timestamp
|
||||||
|
content TEXT
|
||||||
|
)
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
use crate::models::{TaskRun, TaskRunSummary};
|
use crate::models::{LogLine, TaskRun, TaskRunSummary};
|
||||||
use axum::extract::{Path as ExtractPath, State};
|
use axum::extract::{Path as ExtractPath, State};
|
||||||
use axum::http::StatusCode;
|
use axum::http::StatusCode;
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
|
@ -119,7 +119,7 @@ pub async fn get_task_run(
|
||||||
ExtractPath((_task_id, run_id)): ExtractPath<(String, String)>,
|
ExtractPath((_task_id, run_id)): ExtractPath<(String, String)>,
|
||||||
) -> Html<String> {
|
) -> Html<String> {
|
||||||
let run_details = match sqlx::query_as::<_, TaskRun>("SELECT * FROM task_runs WHERE id = $1")
|
let run_details = match sqlx::query_as::<_, TaskRun>("SELECT * FROM task_runs WHERE id = $1")
|
||||||
.bind(run_id)
|
.bind(&run_id)
|
||||||
.fetch_one(&app_state.db)
|
.fetch_one(&app_state.db)
|
||||||
.await {
|
.await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
|
@ -127,11 +127,17 @@ pub async fn get_task_run(
|
||||||
return Html("<h1>Task run not found</h1>".to_string());
|
return Html("<h1>Task run not found</h1>".to_string());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let logs_lines = sqlx::query_as::<_, LogLine>("SELECT * FROM logs_lines WHERE task_run_id = $1 ORDER BY captured_at ASC")
|
||||||
|
.bind(&run_id)
|
||||||
|
.fetch_all(&app_state.db)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
Html(
|
Html(
|
||||||
app_state.templating_env.get_template("pages/task_run_details.html").unwrap()
|
app_state.templating_env.get_template("pages/task_run_details.html").unwrap()
|
||||||
.render(context!(
|
.render(context!(
|
||||||
run => run_details
|
run => run_details,
|
||||||
|
logs_lines
|
||||||
))
|
))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,14 +1,58 @@
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use chrono::{SecondsFormat, Utc};
|
use chrono::{SecondsFormat, Utc};
|
||||||
use log::{debug, info, error};
|
use log::{debug, info, error};
|
||||||
|
use sqlx::{Pool, QueryBuilder, Sqlite};
|
||||||
|
use tokio::task::JoinHandle;
|
||||||
|
use uuid::Uuid;
|
||||||
use std::process::Stdio;
|
use std::process::Stdio;
|
||||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader};
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::sync::mpsc::Receiver;
|
use tokio::sync::mpsc::Receiver;
|
||||||
|
|
||||||
use crate::models::ExecutorOrder;
|
use crate::models::{ExecutorOrder, LogKind, LogLine};
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
|
async fn insert_logs(db: &Pool<Sqlite>, collected_logs: &mut Vec<LogLine>) -> Result<()> {
|
||||||
|
if collected_logs.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// insert into sqlite dB
|
||||||
|
let mut query_builder = QueryBuilder::new("INSERT INTO logs_lines (id, task_run_id, captured_at, kind, content)");
|
||||||
|
query_builder.push_values(collected_logs.iter(), |mut b, log_line| {
|
||||||
|
b
|
||||||
|
.push_bind(log_line.id.to_string())
|
||||||
|
.push_bind(log_line.task_run_id.to_string())
|
||||||
|
.push_bind(log_line.captured_at)
|
||||||
|
.push_bind(log_line.kind.to_string())
|
||||||
|
.push_bind(&log_line.content);
|
||||||
|
});
|
||||||
|
let _ = query_builder.build().execute(db).await?;
|
||||||
|
*collected_logs = vec![];
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_logs<Stream>(task_run_id: Uuid, kind: LogKind, stream: Stream, db: Pool<Sqlite>) -> JoinHandle<()>
|
||||||
|
where Stream: AsyncRead + Unpin + std::marker::Send + 'static
|
||||||
|
{
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut lines = BufReader::with_capacity(2048, stream).lines();
|
||||||
|
let mut collected_logs: Vec<LogLine> = vec![];
|
||||||
|
while let Some(line) = lines.next_line().await.unwrap() {
|
||||||
|
collected_logs.push(LogLine {
|
||||||
|
id: Uuid::new_v4().to_string(),
|
||||||
|
kind: kind.clone(),
|
||||||
|
content: line,
|
||||||
|
task_run_id: task_run_id.to_string(),
|
||||||
|
captured_at: Utc::now().timestamp_millis(),
|
||||||
|
});
|
||||||
|
if collected_logs.len() > 20 {
|
||||||
|
insert_logs(&db, &mut collected_logs).await.expect("Error inserting log");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
insert_logs(&db, &mut collected_logs).await.expect("Error inserting log");
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn run_task(state: AppState, order: ExecutorOrder) -> Result<()> {
|
async fn run_task(state: AppState, order: ExecutorOrder) -> Result<()> {
|
||||||
debug!("Start processing of order {:?}", &order.id);
|
debug!("Start processing of order {:?}", &order.id);
|
||||||
// save in DB
|
// save in DB
|
||||||
|
@ -29,15 +73,15 @@ async fn run_task(state: AppState, order: ExecutorOrder) -> Result<()> {
|
||||||
let mut cmd = Command::new(executable);
|
let mut cmd = Command::new(executable);
|
||||||
|
|
||||||
cmd.args(task.command.iter().skip(1).collect::<Vec<&String>>())
|
cmd.args(task.command.iter().skip(1).collect::<Vec<&String>>())
|
||||||
.stdout(Stdio::piped());
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped());
|
||||||
for (key, val) in task.env.iter() {
|
for (key, val) in task.env.iter() {
|
||||||
cmd.env(key, val);
|
cmd.env(key, val);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut child = cmd.spawn().expect("Failed to execute process");
|
let mut child = cmd.spawn().expect("Failed to execute process");
|
||||||
|
let child_stdout = child.stdout.take().unwrap(); // take is like clone
|
||||||
let stdout = child.stdout.take().unwrap();
|
let child_stderr = child.stderr.take().unwrap();
|
||||||
|
|
||||||
let process_handle = tokio::spawn(async move {
|
let process_handle = tokio::spawn(async move {
|
||||||
let status = child
|
let status = child
|
||||||
.wait()
|
.wait()
|
||||||
|
@ -48,16 +92,27 @@ async fn run_task(state: AppState, order: ExecutorOrder) -> Result<()> {
|
||||||
status
|
status
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut collected_logs = String::new();
|
// parallel stdout and stderr capture
|
||||||
let mut lines = BufReader::with_capacity(16, stdout).lines();
|
let stdout_handle = collect_logs(
|
||||||
while let Some(line) = lines.next_line().await.unwrap() {
|
order.id,
|
||||||
collected_logs += &format!("{}\n", line);
|
LogKind::Stderr,
|
||||||
}
|
child_stderr,
|
||||||
|
state.db.clone()
|
||||||
|
);
|
||||||
|
let stderr_handle = collect_logs(
|
||||||
|
order.id,
|
||||||
|
LogKind::Stdout,
|
||||||
|
child_stdout,
|
||||||
|
state.db.clone()
|
||||||
|
);
|
||||||
|
|
||||||
|
stdout_handle.await?;
|
||||||
|
stderr_handle.await?;
|
||||||
let status = process_handle.await?;
|
let status = process_handle.await?;
|
||||||
if !status.success() {
|
if !status.success() {
|
||||||
error!("Non successful exit code found: {}", status);
|
error!("Non successful exit code found: {}", status);
|
||||||
}
|
}
|
||||||
let _result = sqlx::query("UPDATE task_runs SET status = $2, ended_at = $3, exit_code = $4, logs = $5 WHERE id = $1")
|
let _result = sqlx::query("UPDATE task_runs SET status = $2, ended_at = $3, exit_code = $4 WHERE id = $1")
|
||||||
.bind(order.id.to_string())
|
.bind(order.id.to_string())
|
||||||
.bind(match status.success() {
|
.bind(match status.success() {
|
||||||
true => "success",
|
true => "success",
|
||||||
|
@ -65,7 +120,6 @@ async fn run_task(state: AppState, order: ExecutorOrder) -> Result<()> {
|
||||||
})
|
})
|
||||||
.bind(Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true))
|
.bind(Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true))
|
||||||
.bind(status.code())
|
.bind(status.code())
|
||||||
.bind(collected_logs)
|
|
||||||
.execute(&state.db)
|
.execute(&state.db)
|
||||||
.await.unwrap();
|
.await.unwrap();
|
||||||
debug!("End of executor order {:?}, details saved", &order.id);
|
debug!("End of executor order {:?}, details saved", &order.id);
|
||||||
|
|
|
@ -15,8 +15,8 @@ enum TriggerMode {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(sqlx::Type, Debug, Serialize, Deserialize)]
|
#[derive(sqlx::Type, Debug, Serialize, Deserialize)]
|
||||||
#[fully_pub]
|
|
||||||
#[sqlx(rename_all = "lowercase")]
|
#[sqlx(rename_all = "lowercase")]
|
||||||
|
#[fully_pub]
|
||||||
enum TaskStatus {
|
enum TaskStatus {
|
||||||
Pending,
|
Pending,
|
||||||
Running,
|
Running,
|
||||||
|
@ -24,6 +24,25 @@ enum TaskStatus {
|
||||||
Success,
|
Success,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(sqlx::Type, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
#[derive(strum_macros::Display)]
|
||||||
|
#[fully_pub]
|
||||||
|
enum LogKind {
|
||||||
|
Stdout,
|
||||||
|
Stderr
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(sqlx::FromRow, Debug, Serialize, Deserialize)]
|
||||||
|
#[fully_pub]
|
||||||
|
struct LogLine {
|
||||||
|
id: String,
|
||||||
|
task_run_id: String,
|
||||||
|
/// unix timestamp millis
|
||||||
|
captured_at: i64,
|
||||||
|
kind: LogKind,
|
||||||
|
content: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(sqlx::FromRow, Deserialize, Serialize)]
|
#[derive(sqlx::FromRow, Deserialize, Serialize)]
|
||||||
#[fully_pub]
|
#[fully_pub]
|
||||||
struct TaskRunSummary {
|
struct TaskRunSummary {
|
||||||
|
@ -43,7 +62,7 @@ struct TaskRun {
|
||||||
status: TaskStatus,
|
status: TaskStatus,
|
||||||
trigger_mode: TriggerMode,
|
trigger_mode: TriggerMode,
|
||||||
exit_code: Option<u32>,
|
exit_code: Option<u32>,
|
||||||
logs: String,
|
// logs: sqlx::types::Json<Vec<LogMessage>>,
|
||||||
submitted_at: DateTime<Utc>,
|
submitted_at: DateTime<Utc>,
|
||||||
started_at: Option<DateTime<Utc>>,
|
started_at: Option<DateTime<Utc>>,
|
||||||
ended_at: Option<DateTime<Utc>>,
|
ended_at: Option<DateTime<Utc>>,
|
||||||
|
|
|
@ -12,5 +12,9 @@
|
||||||
|
|
||||||
<h3>Logs</h3>
|
<h3>Logs</h3>
|
||||||
<p>Exit code is {{ run.exit_code }}</p>
|
<p>Exit code is {{ run.exit_code }}</p>
|
||||||
<pre>{{ run.logs }}</pre>
|
<pre class="logs-lines">
|
||||||
|
{% for line in logs_lines %}
|
||||||
|
<div class="line-{{ line.kind|lower }}">{{- line.content -}}</div>
|
||||||
|
{% endfor %}
|
||||||
|
</pre>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
Binary file not shown.
Loading…
Reference in a new issue