add first version of sequence methods

docs/detectors.md

@@ -92,6 +92,8 @@ List of detectors:
 * [Rule Based](detectors/rule_based.md): Detect anomalies based in a set of rules.
 * [Bigram Frequency](detectors/bigram_frequency.md): Detect bigram-frequency-based anomalies in the logs.
 * [Charset](detectors/charset.md): Detect new characters in the variables in the logs.
+* [SCVS Detector](detectors/scvs_detector.md): Detect new anomalies by looking at different sequence count vectors.
+* [ECVC Detector](detectors/ecvc_detector.md): Detect new anomalies by calculating the distance between different sequence count vectors.
 
 ## Configuration
 

docs/detectors/ecvc_detector.md

@@ -0,0 +1,59 @@
+# ECVC Detector
+
+The Event Count Vector Clustering Detector (SCVS) detect anomalies by calculating the distance between the count vectors from training and new ones. The method can be found in [this publication](https://dl.acm.org/doi/10.1145/3660768).
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [ParserSchema](../schemas.md) | Structured log  |
+| **Output** | [DetectorSchema](../schemas.md) | Alert / finding |
+
+## Description
+
+A count vector is form by counting the number of appearance of each event ID in a sequence of a specific window size.
+
+
+## Configuration example
+
+```yaml
+detectors:
+    SCVSDetector:
+        method_type: scvs_detector
+        window_size: 10
+```
+
+
+## Example usage
+
+```python
+from detectmatelibrary.detectors.ecvc_detector import ECVCDetector
+import detectmatelibrary.schemas as schemas
+
+cfg = {
+    "detectors": {
+        "ECVCDetector": {
+            "method_type": "ecvc_detector_detector",
+            "window_size": 10,
+            "validation_per": 0.2,
+            "threshold_method": "mean"  # mean, mode, default (default = threshold 0)
+        }
+    }
+}
+detector = ECVCDetector(name="ECVCDetector", config=cfg)
+
+parser_data = schemas.ParserSchema({
+    "parserType": "test",
+    "EventID": 1,
+    "template": "test template",
+    "variables": ["var1"],
+    "logID": "1",
+    "parsedLogID": "1",
+    "parserID": "test_parser",
+    "log": "test log message",
+    "logFormatVariables": {"timestamp": "123456"}
+})
+
+
+alert = detector.process(parser_data)
+```
+
+Go back [Index](../index.md)

docs/detectors/scvs_detector.md

@@ -0,0 +1,57 @@
+# SCVS Detector
+
+The Sequence Count Vector Set Detector (SCVS) detect anomalies by finding count vector that were not present in the training dataset.
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [ParserSchema](../schemas.md) | Structured log  |
+| **Output** | [DetectorSchema](../schemas.md) | Alert / finding |
+
+## Description
+
+A count vector is form by counting the number of appearance of each event ID in a sequence of a specific window size.
+
+
+## Configuration example
+
+```yaml
+detectors:
+    SCVSDetector:
+        method_type: scvs_detector
+        window_size: 10
+```
+
+
+## Example usage
+
+```python
+from detectmatelibrary.detectors.scvs_detector import SCVSDetector
+import detectmatelibrary.schemas as schemas
+
+cfg = {
+    "detectors": {
+        "SCVSDetector": {
+            "method_type": "scvs_detector",
+            "auto_config": False,
+        }
+    }
+}
+detector = SCVSDetector(name="NewValueTest", config=cfg)
+
+parser_data = schemas.ParserSchema({
+    "parserType": "test",
+    "EventID": 1,
+    "template": "test template",
+    "variables": ["var1"],
+    "logID": "1",
+    "parsedLogID": "1",
+    "parserID": "test_parser",
+    "log": "test log message",
+    "logFormatVariables": {"timestamp": "123456"}
+})
+
+
+alert = detector.process(parser_data)
+```
+
+Go back [Index](../index.md)

mkdocs.yml

@@ -31,6 +31,8 @@ nav:
     - BiGram Frequency: detectors/bigram_frequency.md
     - CharSet: detectors/charset.md
     - Value Range: detectors/value_range.md
+    - SCVS Detector: detectors/scvs_detector.md
+    - ECVC Detector: detectors/ecvc_detector.md
   - Alert Aggregation Methods:
     - Basic Concat: alert_aggregators/basic_concatenation.md
   - Auxiliar:

src/detectmatelibrary/detectors/ecvc_detector.py

@@ -0,0 +1,131 @@
+from typing import Any, List
+
+from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
+from detectmatelibrary.utils.data_buffer import BufferMode
+from detectmatelibrary import schemas
+
+from scipy.stats import mode
+from math import ceil
+import numpy as np
+
+
+class ECVCOp:
+    @staticmethod
+    def build_count_vec(input_: List[schemas.ParserSchema]) -> tuple[int, ...]:
+        sequence, n = [0], 0
+        for in_ in input_:
+            event = in_["EventID"]
+            if n < event:
+                for _ in range(n, event):
+                    sequence.append(0)
+                n = event
+            sequence[event] += 1
+
+        return tuple(sequence)
+
+    @staticmethod
+    def build_one_vec(input_: List[schemas.ParserSchema], n: int) -> np.ndarray:
+        events = [in_["EventID"] for in_ in input_]
+        arr = np.zeros(n if n > (m := max(events) + 1) else m)
+
+        for e in events:
+            arr[e] += 1
+
+        return arr
+
+    @staticmethod
+    def init_count_matrix(seqs: set[tuple[int, ...]]) -> np.ndarray:
+        m, n = len(seqs), max([len(s) for s in seqs])
+        matrix = np.zeros((m, n))
+
+        for i, seq in enumerate(seqs):
+            for j, c in enumerate(seq):
+                matrix[i, j] = c
+
+        return matrix
+
+    @staticmethod
+    def calculate_score(y: np.ndarray, matrix: np.ndarray) -> float:
+        pad = np.zeros((matrix.shape[0], y.shape[0] - matrix.shape[1]))
+        matrix_ = np.concat([matrix, pad], axis=1)
+
+        score = np.inf
+        for m in matrix_:
+            dif = np.sum(np.abs(m - y))
+            div = np.sum(np.max(np.concat([m[np.newaxis], y[np.newaxis]]).T, axis=1))
+            score = score if score < (s := (dif / div)) else s
+
+        return float(score)
+
+    @staticmethod
+    def threshold_cal(y_s: np.ndarray, matrix: np.ndarray, method: str) -> float:
+        if method == "mean":
+            return float(np.mean([ECVCOp.calculate_score(y, matrix=matrix) for y in y_s]))
+        elif method == "mode":
+            return float(mode([ECVCOp.calculate_score(y, matrix=matrix) for y in y_s]).mode)
+        elif method == "default":
+            return 0.0
+
+        raise Exception("Method not supported")
+
+
+class ECVCDetectorConfig(CoreDetectorConfig):
+    method_type: str = "ecvc_detector_detector"
+    window_size: int = 10
+    validation_per: float = 0.2
+    seed: int = 0
+    threshold_method: str = "mean"
+
+
+class ECVCDetector(CoreDetector):
+    def __init__(
+        self,
+        name: str = "ECVCDetector",
+        config: ECVCDetectorConfig | dict[str, Any] = ECVCDetectorConfig(),
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = ECVCDetectorConfig.from_dict(config, name)
+        self.config: ECVCDetectorConfig
+
+        super().__init__(
+            name=name,
+            buffer_mode=BufferMode.WINDOW,
+            config=config,
+            buffer_size=config.window_size
+        )
+        self.train_seqs: set[tuple[int, ...]] = set()
+        self.count_vecs: np.ndarray | None = None
+        self.threshold: float = 0
+
+    def train(self, input_: List[schemas.ParserSchema]) -> None:  # type: ignore
+        self.train_seqs.add(ECVCOp.build_count_vec(input_))
+
+    def post_train(self) -> None:
+        train_idx = ceil(len(self.train_seqs) * (1 - self.config.validation_per))
+        np.random.seed(self.config.seed)
+        matrix = ECVCOp.init_count_matrix(self.train_seqs)[np.random.permutation(len(self.train_seqs))]
+
+        self.count_vecs, val = matrix[:train_idx], matrix[train_idx:]
+        if len(val) > 0:
+            self.threshold = ECVCOp.threshold_cal(
+                y_s=val, matrix=self.count_vecs, method=self.config.threshold_method
+            )
+        self.train_seqs = set()
+
+    def detect(
+        self, input_: List[schemas.ParserSchema], output_: schemas.DetectorSchema,  # type: ignore
+    ) -> bool:
+
+        if self.count_vecs is None:
+            return False
+
+        score = ECVCOp.calculate_score(ECVCOp.build_one_vec(
+            input_, self.count_vecs.shape[1]), matrix=self.count_vecs
+        )
+        if score > self.threshold:
+            output_["score"] = score
+            output_["description"] = "ECVC found an anominal sequence"
+            return True
+
+        return False

src/detectmatelibrary/detectors/scvs_detector.py

@@ -0,0 +1,57 @@
+from typing import Any, List
+
+from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig
+from detectmatelibrary.utils.data_buffer import BufferMode
+from detectmatelibrary import schemas
+
+
+def build_count_vec(input_: List[schemas.ParserSchema]) -> tuple[int, ...]:
+    sequence, n = [0], 0
+    for in_ in input_:
+        event = in_["EventID"]
+        if n < event:
+            for _ in range(n, event):
+                sequence.append(0)
+            n = event
+        sequence[event] += 1
+
+    return tuple(sequence)
+
+
+class SCVSDetectorConfig(CoreDetectorConfig):
+    method_type: str = "scvs_detector"
+    window_size: int = 10
+
+
+class SCVSDetector(CoreDetector):
+    def __init__(
+        self,
+        name: str = "SCVSDetector",
+        config: SCVSDetectorConfig | dict[str, Any] = SCVSDetectorConfig(),
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = SCVSDetectorConfig.from_dict(config, name)
+        self.config: SCVSDetectorConfig
+
+        super().__init__(
+            name=name,
+            buffer_mode=BufferMode.WINDOW,
+            config=config,
+            buffer_size=config.window_size
+        )
+        self.train_seqs: set[tuple[int, ...]] = set()
+
+    def train(self, input_: List[schemas.ParserSchema]) -> None:  # type: ignore
+        self.train_seqs.add(build_count_vec(input_))
+
+    def detect(
+        self, input_: List[schemas.ParserSchema], output_: schemas.DetectorSchema,  # type: ignore
+    ) -> bool:
+
+        if build_count_vec(input_) not in self.train_seqs:
+            output_["score"] = 1.
+            output_["description"] = "Count vector not found"
+            return True
+
+        return False