From 83b84486ad9e35bff1d5f2d166db296b767542c7 Mon Sep 17 00:00:00 2001
From: OlivierDehaene <olivier@huggingface.co>
Date: Fri, 2 Jun 2023 14:17:27 +0200
Subject: [PATCH] feat(launcher): parse oom signal (#404)

---
 launcher/src/main.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/launcher/src/main.rs b/launcher/src/main.rs
index 7ee8bf1..f59ff68 100644
--- a/launcher/src/main.rs
+++ b/launcher/src/main.rs
@@ -410,9 +410,14 @@ fn shard_manager(
     let mut wait_time = Instant::now();
     loop {
         // Process exited
-        if p.poll().is_some() {
+        if let Some(exit_status) = p.poll() {
             let mut err = String::new();
             p.stderr.take().unwrap().read_to_string(&mut err).unwrap();
+
+            if let ExitStatus::Signaled(signal) = exit_status {
+                tracing::error!("Shard process was signaled to shutdown with signal {signal}");
+            }
+
             status_sender
                 .send(ShardStatus::Failed((rank, err)))
                 .unwrap();