Ingest more log formats

src/parser.rs

@@ -40,18 +40,37 @@ static SESSION_ID_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r"sessionId=([^,\s]+)").unwrap());
 static DATETIME_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r#"dt="(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"#).unwrap());
+static CORRELATION_ID_RE: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"correlationId=([^,\s]+)").unwrap());
 static SIGNATURE_RE: LazyLock<Regex> =
     LazyLock::new(|| Regex::new(r#"msg="signature:([^/]+)/([^/]*)/\s*details:([^"]+)"#).unwrap());
 
+// iOS mobile client: signature:appId/version/deviceId details:...
+static MOBILE_IOS_RE: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r#"msg="MOBILE_CLIENT_LOG: signature:([^/]+)/([^/]+)/([^\s]+)\s+details:([^"]+)"#)
+        .unwrap()
+});
+
+// Android mobile client: signature:appId/version/{json} details:...
+static MOBILE_ANDROID_RE: LazyLock<Regex> = LazyLock::new(|| {
+    Regex::new(r#"msg="MOBILE_CLIENT_LOG: signature:([^/]+)/([^/]+)/(\{[^}]+\})\s+details:([^"]+)"#)
+        .unwrap()
+});
+
 pub struct SignatureParser;
 
 impl MessageParser for SignatureParser {
     fn parse(&self, line: &str) -> Option<Result<ParsedMessage>> {
-        // Check if this line contains a signature message
+        // Check if this line contains a signature message (but not MOBILE_CLIENT_LOG)
         if !line.contains("msg=\"signature:") {
             return None;
         }
 
+        // Skip WEB_CLIENT messages
+        if line.contains("msg=\"signature:WEB_CLIENT") {
+            return None;
+        }
+
         Some(self.parse_signature_line(line))
     }
 }
@@ -109,6 +128,205 @@ impl SignatureParser {
     }
 }
 
+/// Parser for iOS MOBILE_CLIENT_LOG messages
+pub struct MobileClientIosParser;
+
+impl MessageParser for MobileClientIosParser {
+    fn parse(&self, line: &str) -> Option<Result<ParsedMessage>> {
+        if !line.contains("MOBILE_CLIENT_LOG:") || !line.contains("sdk-client:IOS") {
+            return None;
+        }
+
+        Some(self.parse_mobile_ios_line(line))
+    }
+}
+
+impl MobileClientIosParser {
+    fn parse_mobile_ios_line(&self, line: &str) -> Result<ParsedMessage> {
+        let timestamp = extract_timestamp(line)?;
+        let session_id = extract_correlation_id(line)?;
+
+        let caps = MOBILE_IOS_RE
+            .captures(line)
+            .ok_or_else(|| anyhow!("Invalid iOS mobile client format"))?;
+
+        let app = caps.get(1).map(|m| m.as_str().to_string()).unwrap();
+        let version = caps.get(2).map(|m| m.as_str().to_string()).unwrap();
+        let details_str = caps.get(4).map(|m| m.as_str()).unwrap();
+
+        let details = parse_mobile_details(details_str);
+
+        let entry = SignatureEntry {
+            session_id,
+            timestamp,
+            app,
+            version,
+            offline_login_usage: None,
+            is_password_autofill_enabled: None,
+            camera_roll_usage: None,
+            os: get_string(&details, "os"),
+            app_name: get_string(&details, "app-name"),
+            touch_id: None,
+            is_offline_login_enabled: None,
+            model: get_string(&details, "model"),
+            device: Some("iOS".to_string()),
+            password_autofill_usage: None,
+        };
+
+        Ok(ParsedMessage::Signature(entry))
+    }
+}
+
+/// Parser for Android MOBILE_CLIENT_LOG messages
+pub struct MobileClientAndroidParser;
+
+impl MessageParser for MobileClientAndroidParser {
+    fn parse(&self, line: &str) -> Option<Result<ParsedMessage>> {
+        if !line.contains("MOBILE_CLIENT_LOG:") || !line.contains("sdk-client:ANDROID") {
+            return None;
+        }
+
+        Some(self.parse_mobile_android_line(line))
+    }
+}
+
+impl MobileClientAndroidParser {
+    fn parse_mobile_android_line(&self, line: &str) -> Result<ParsedMessage> {
+        let timestamp = extract_timestamp(line)?;
+        let session_id = extract_correlation_id(line)?;
+
+        let caps = MOBILE_ANDROID_RE
+            .captures(line)
+            .ok_or_else(|| anyhow!("Invalid Android mobile client format"))?;
+
+        let app = caps.get(1).map(|m| m.as_str().to_string()).unwrap();
+        let version = caps.get(2).map(|m| m.as_str().to_string()).unwrap();
+        let details_str = caps.get(4).map(|m| m.as_str()).unwrap();
+
+        let details = parse_mobile_details_android(details_str);
+
+        let entry = SignatureEntry {
+            session_id,
+            timestamp,
+            app,
+            version,
+            offline_login_usage: None,
+            is_password_autofill_enabled: None,
+            camera_roll_usage: None,
+            os: get_string(&details, "os"),
+            app_name: Some("native Android".to_string()),
+            touch_id: None,
+            is_offline_login_enabled: None,
+            model: get_string(&details, "model"),
+            device: get_string(&details, "device"),
+            password_autofill_usage: None,
+        };
+
+        Ok(ParsedMessage::Signature(entry))
+    }
+}
+
+/// Extract timestamp from log line
+fn extract_timestamp(line: &str) -> Result<NaiveDateTime> {
+    let datetime_str = DATETIME_RE
+        .captures(line)
+        .and_then(|c| c.get(1))
+        .map(|m| m.as_str())
+        .ok_or_else(|| anyhow!("Missing datetime"))?;
+
+    NaiveDateTime::parse_from_str(datetime_str, "%Y-%m-%d %H:%M:%S")
+        .map_err(|e| anyhow!("Invalid datetime format: {}", e))
+}
+
+/// Extract correlation ID as session ID for mobile client logs
+fn extract_correlation_id(line: &str) -> Result<String> {
+    CORRELATION_ID_RE
+        .captures(line)
+        .and_then(|c| c.get(1))
+        .map(|m| m.as_str().to_string())
+        .ok_or_else(|| anyhow!("Missing correlationId"))
+}
+
+/// Parse mobile client details for iOS (simple comma-separated key:value)
+fn parse_mobile_details(details: &str) -> std::collections::HashMap<String, String> {
+    let mut map = std::collections::HashMap::new();
+
+    // Keys for iOS mobile client
+    let known_keys = ["sdk-client", "sdk-version", "app-name", "device", "model", "os"];
+
+    let mut key_positions: Vec<(usize, &str)> = known_keys
+        .iter()
+        .filter_map(|&key| {
+            let pattern = format!("{}:", key);
+            details.find(&pattern).map(|pos| (pos, key))
+        })
+        .collect();
+
+    key_positions.sort_by_key(|&(pos, _)| pos);
+
+    for i in 0..key_positions.len() {
+        let (pos, key) = key_positions[i];
+        let value_start = pos + key.len() + 1;
+
+        let value_end = if i + 1 < key_positions.len() {
+            let next_pos = key_positions[i + 1].0;
+            if next_pos > 0 && details.as_bytes().get(next_pos - 1) == Some(&b',') {
+                next_pos - 1
+            } else {
+                next_pos
+            }
+        } else {
+            details.find(" user-agent").unwrap_or(details.len())
+        };
+
+        let value = details[value_start..value_end].trim().to_string();
+        map.insert(key.to_string(), value);
+    }
+
+    map
+}
+
+/// Parse mobile client details for Android (handles device with commas)
+fn parse_mobile_details_android(details: &str) -> std::collections::HashMap<String, String> {
+    let mut map = std::collections::HashMap::new();
+
+    // For Android, device can contain commas like "Android, samsung"
+    // Keys in order: sdk-client, sdk-version, app-name, device, model, os
+    let known_keys = ["sdk-client", "sdk-version", "app-name", "device", "model", "os"];
+
+    let mut key_positions: Vec<(usize, &str)> = known_keys
+        .iter()
+        .filter_map(|&key| {
+            let pattern = format!("{}:", key);
+            details.find(&pattern).map(|pos| (pos, key))
+        })
+        .collect();
+
+    key_positions.sort_by_key(|&(pos, _)| pos);
+
+    for i in 0..key_positions.len() {
+        let (pos, key) = key_positions[i];
+        let value_start = pos + key.len() + 1;
+
+        let value_end = if i + 1 < key_positions.len() {
+            let next_pos = key_positions[i + 1].0;
+            // Find the comma before the next key
+            if next_pos > 0 && details.as_bytes().get(next_pos - 1) == Some(&b',') {
+                next_pos - 1
+            } else {
+                next_pos
+            }
+        } else {
+            details.find(" user-agent").unwrap_or(details.len())
+        };
+
+        let value = details[value_start..value_end].trim().to_string();
+        map.insert(key.to_string(), value);
+    }
+
+    map
+}
+
 /// Parse the details string which has format like:
 /// offlineLoginUsage:0,isPasswordAutofillEnabled:no,...,device:iOS, Apple,passwordAutofillUsage:0
 fn parse_details(details: &str) -> Result<std::collections::HashMap<String, String>> {
@@ -196,7 +414,9 @@ impl ParserRegistry {
         let mut registry = Self {
             parsers: Vec::new(),
         };
-        // Register default parsers
+        // Register default parsers (order matters - more specific first)
+        registry.register(Box::new(MobileClientIosParser));
+        registry.register(Box::new(MobileClientAndroidParser));
         registry.register(Box::new(SignatureParser));
         registry
     }
@@ -337,4 +557,83 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_web_client_signature_is_skipped() {
+        // WEB_CLIENT signature messages should be skipped (return None)
+        let line = r#"Jan 21 00:00:06 tom013 m1s-kv dt="2026-01-21 00:00:06,154", sessionId=test-123, msg="signature:WEB_CLIENT/1.0.0/ details:browser:Chrome,OS:Windows user-agent:test", ex=""#;
+
+        let registry = ParserRegistry::new();
+        assert!(registry.parse(line).is_none());
+    }
+
+    #[test]
+    fn test_web_client_with_version_is_skipped() {
+        let line = r#"Jan 21 00:00:06 tom013 m1s-kv dt="2026-01-21 00:00:06,154", sessionId=test-456, msg="signature:WEB_CLIENT/2.5.0/ details:something:value user-agent:test", ex=""#;
+
+        let registry = ParserRegistry::new();
+        assert!(registry.parse(line).is_none());
+    }
+
+    #[test]
+    fn test_parse_mobile_client_ios() {
+        let line = r#"Jan 21 00:01:55 tom012.prodfrnt.dsw.loc m1s-kv dt="2026-01-21 00:01:55,573", ll=INFO, lc=com.dswiss.apigateway.domains.user.controllers.UserControllerV1, correlationId=aXAJY9JuqalC5uxAaB6EsgAAALo, applicationName=ApiGateway, msg="MOBILE_CLIENT_LOG: signature:com.dswiss.securesafepass/2.1.0/738D8CD5-28BC-490C-AB02-2C309FA64875 details:sdk-client:IOS,sdk-version:1.4.0,app-name:SecureSafePass,device:iOS,model:iPhone14,4,os:26.2 user-agent:26.2", ex=""#;
+
+        let registry = ParserRegistry::new();
+        let result = registry.parse(line).unwrap().unwrap();
+
+        match result {
+            ParsedMessage::Signature(entry) => {
+                assert_eq!(entry.session_id, "aXAJY9JuqalC5uxAaB6EsgAAALo");
+                assert_eq!(entry.app, "com.dswiss.securesafepass");
+                assert_eq!(entry.version, "2.1.0");
+                assert_eq!(entry.model, Some("iPhone14,4".to_string()));
+                assert_eq!(entry.os, Some("26.2".to_string()));
+                assert_eq!(entry.app_name, Some("SecureSafePass".to_string()));
+                assert_eq!(entry.device, Some("iOS".to_string()));
+                // These fields are not present in mobile client logs
+                assert_eq!(entry.offline_login_usage, None);
+                assert_eq!(entry.is_password_autofill_enabled, None);
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_mobile_client_android() {
+        let line = r#"Jan 21 01:01:59 tom011.prodfrnt.dsw.loc m1s-kv dt="2026-01-21 01:01:59,647", ll=INFO, lc=com.dswiss.apigateway.domains.user.controllers.UserControllerV1, correlationId=aXAXdwzcGYAFt6MF_m4tKAAAAg4, applicationName=ApiGateway, msg="MOBILE_CLIENT_LOG: signature:com.dswiss.mobile.sesa/2.1.0/{"client":"2a4f12c97b64948d","version":"1.4.0"} details:sdk-client:ANDROID,sdk-version:1.4.0,app-name:2.1.0,device:Android, samsung,model:SM-A536B,os:16 user-agent:16", ex=""#;
+
+        let registry = ParserRegistry::new();
+        let result = registry.parse(line).unwrap().unwrap();
+
+        match result {
+            ParsedMessage::Signature(entry) => {
+                assert_eq!(entry.session_id, "aXAXdwzcGYAFt6MF_m4tKAAAAg4");
+                assert_eq!(entry.app, "com.dswiss.mobile.sesa");
+                assert_eq!(entry.version, "2.1.0");
+                assert_eq!(entry.model, Some("SM-A536B".to_string()));
+                assert_eq!(entry.os, Some("16".to_string()));
+                assert_eq!(entry.app_name, Some("native Android".to_string()));
+                assert_eq!(entry.device, Some("Android, samsung".to_string()));
+                // These fields are not present in mobile client logs
+                assert_eq!(entry.offline_login_usage, None);
+                assert_eq!(entry.is_password_autofill_enabled, None);
+            }
+        }
+    }
+
+    #[test]
+    fn test_xamarin_app_still_parsed() {
+        // Ensure XAMARIN_APP messages are still parsed after adding WEB_CLIENT filter
+        let line = r#"Jan 21 00:00:06 tom013 m1s-kv dt="2026-01-21 00:00:06,154", sessionId=test-session-123, msg="signature:XAMARIN_APP/5.23.0/ details:offlineLoginUsage:0,isPasswordAutofillEnabled:yes,cameraRollUsage:0,OS:26.2.0,appName:App,touchID:yes,isOfflineLoginEnabled:yes,model:iPhone18,1,device:iOS, Apple,passwordAutofillUsage:0 user-agent:test", ex=""#;
+
+        let registry = ParserRegistry::new();
+        let result = registry.parse(line).unwrap().unwrap();
+
+        match result {
+            ParsedMessage::Signature(entry) => {
+                assert_eq!(entry.app, "XAMARIN_APP");
+                assert_eq!(entry.version, "5.23.0");
+            }
+        }
+    }
 }