Compare commits
1 Commits
feature/se
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| c01b9ba97e |
14
.github/dependabot.yaml
vendored
14
.github/dependabot.yaml
vendored
@@ -1,14 +0,0 @@
|
|||||||
version: 2
|
|
||||||
updates:
|
|
||||||
- package-ecosystem: github-actions
|
|
||||||
directory: /
|
|
||||||
schedule:
|
|
||||||
interval: weekly
|
|
||||||
time: "07:00"
|
|
||||||
timezone: "Asia/Tokyo"
|
|
||||||
- package-ecosystem: cargo
|
|
||||||
directory: /
|
|
||||||
schedule:
|
|
||||||
interval: weekly
|
|
||||||
time: "07:00"
|
|
||||||
timezone: "Asia/Tokyo"
|
|
||||||
31
.github/workflows/audit.yaml
vendored
31
.github/workflows/audit.yaml
vendored
@@ -1,31 +0,0 @@
|
|||||||
name: Security audit
|
|
||||||
|
|
||||||
on:
|
|
||||||
schedule:
|
|
||||||
- cron: "0 0 */3 * *"
|
|
||||||
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- "**/Cargo.toml"
|
|
||||||
- "**/Cargo.lock"
|
|
||||||
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- "**/Cargo.toml"
|
|
||||||
- "**/Cargo.lock"
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
audit:
|
|
||||||
name: Audit
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
||||||
|
|
||||||
- name: Install cargo-audit
|
|
||||||
uses: taiki-e/install-action@30eab0fabba9ea3f522099957e668b21876aa39e # v2.66.6
|
|
||||||
with:
|
|
||||||
tool: cargo-audit
|
|
||||||
|
|
||||||
- name: Run audit
|
|
||||||
run: cargo audit
|
|
||||||
39
.github/workflows/benchmark.yaml
vendored
39
.github/workflows/benchmark.yaml
vendored
@@ -1,39 +0,0 @@
|
|||||||
name: Benchmark
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- '**/*.rs'
|
|
||||||
- '**/Cargo.toml'
|
|
||||||
- '**/Cargo.lock'
|
|
||||||
- '.github/workflows/benchmark.yaml'
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
deployments: write
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
benchmark:
|
|
||||||
name: Run Rust benchmark example
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
||||||
- name: Toolchain setup
|
|
||||||
run: rustup toolchain update nightly && rustup default nightly
|
|
||||||
- name: Run benchmark
|
|
||||||
run: cargo +nightly bench | tee output.txt
|
|
||||||
|
|
||||||
- name: Store benchmark result
|
|
||||||
uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7
|
|
||||||
with:
|
|
||||||
name: Rust Benchmark
|
|
||||||
tool: 'cargo'
|
|
||||||
output-file-path: output.txt
|
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
auto-push: true
|
|
||||||
# Show alert with commit comment on detecting possible performance regression
|
|
||||||
alert-threshold: '200%'
|
|
||||||
comment-on-alert: true
|
|
||||||
fail-on-alert: true
|
|
||||||
benchmark-data-dir-path: docs
|
|
||||||
97
.github/workflows/ci.yaml
vendored
97
.github/workflows/ci.yaml
vendored
@@ -1,97 +0,0 @@
|
|||||||
name: Rust CI
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- '**/*.rs'
|
|
||||||
- '**/Cargo.toml'
|
|
||||||
- '**/Cargo.lock'
|
|
||||||
- '.github/workflows/ci.yaml'
|
|
||||||
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- '**/*.rs'
|
|
||||||
- '**/Cargo.toml'
|
|
||||||
- '**/Cargo.lock'
|
|
||||||
- '.github/workflows/ci.yaml'
|
|
||||||
|
|
||||||
env:
|
|
||||||
CARGO_TERM_COLOR: always
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check:
|
|
||||||
name: Check
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
|
|
||||||
with:
|
|
||||||
components: rustfmt, clippy
|
|
||||||
cache-shared-key: setup-rust-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock') }}
|
|
||||||
|
|
||||||
- name: Install reviewdog
|
|
||||||
uses: reviewdog/action-setup@d8a7baabd7f3e8544ee4dbde3ee41d0011c3a93f # v1.5.0
|
|
||||||
|
|
||||||
- name: Check format
|
|
||||||
run: |
|
|
||||||
cargo fmt --all -- --check
|
|
||||||
|
|
||||||
- uses: giraffate/clippy-action@13b9d32482f25d29ead141b79e7e04e7900281e0 # v1.0.1
|
|
||||||
with:
|
|
||||||
reporter: 'github-pr-review'
|
|
||||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
fail_on_error: true
|
|
||||||
filter_mode: nofilter
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: cargo build
|
|
||||||
|
|
||||||
test:
|
|
||||||
name: Test
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
||||||
runs-on: ${{ matrix.os }}
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pull-requests: write
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
|
|
||||||
with:
|
|
||||||
components: llvm-tools-preview
|
|
||||||
cache-shared-key: setup-rust-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock') }}
|
|
||||||
|
|
||||||
- name: Install tools
|
|
||||||
uses: taiki-e/install-action@30eab0fabba9ea3f522099957e668b21876aa39e # v2.66.6
|
|
||||||
with:
|
|
||||||
tool: cargo-llvm-cov, cargo-nextest
|
|
||||||
|
|
||||||
- name: Run test
|
|
||||||
if: runner.os != 'Linux'
|
|
||||||
run: |
|
|
||||||
cargo nextest run
|
|
||||||
|
|
||||||
- name: Generate coverage
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
run: cargo llvm-cov nextest --lcov --output-path lcov.info
|
|
||||||
|
|
||||||
- name: Upload coverage
|
|
||||||
if: runner.os == 'Linux'
|
|
||||||
uses: k1LoW/octocov-action@73d561f65d59e66899ed5c87e4621a913b5d5c20 # v1.5.0
|
|
||||||
with:
|
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
34
.github/workflows/dependabot-auto-merge.yaml
vendored
34
.github/workflows/dependabot-auto-merge.yaml
vendored
@@ -1,34 +0,0 @@
|
|||||||
name: Dependabot Auto-merge
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
types:
|
|
||||||
- opened
|
|
||||||
- synchronize
|
|
||||||
- reopened
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pull-requests: write
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
dependabot-automation:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
if: ${{ github.actor == 'dependabot[bot]' }}
|
|
||||||
timeout-minutes: 13
|
|
||||||
steps:
|
|
||||||
- name: Dependabot metadata
|
|
||||||
id: metadata
|
|
||||||
uses: dependabot/fetch-metadata@21025c705c08248db411dc16f3619e6b5f9ea21a # v2.5.0
|
|
||||||
with:
|
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
- name: Approve & enable auto-merge for Dependabot PR
|
|
||||||
if: |
|
|
||||||
steps.metadata.outputs.update-type == 'version-update:semver-patch' ||
|
|
||||||
steps.metadata.outputs.update-type == 'version-update:semver-minor'
|
|
||||||
run: |
|
|
||||||
gh pr merge --auto -s "$PR_URL"
|
|
||||||
env:
|
|
||||||
PR_URL: ${{ github.event.pull_request.html_url }}
|
|
||||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
134
.github/workflows/release.yaml
vendored
134
.github/workflows/release.yaml
vendored
@@ -1,134 +0,0 @@
|
|||||||
name: Release
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- 'v*'
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
name: Build - ${{ matrix.target }}
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- target: x86_64-unknown-linux-gnu
|
|
||||||
runner: ubuntu-latest
|
|
||||||
os: Linux
|
|
||||||
arch: x86_64
|
|
||||||
ext: tar.gz
|
|
||||||
- target: aarch64-unknown-linux-gnu
|
|
||||||
runner: ubuntu-24.04-arm
|
|
||||||
os: Linux
|
|
||||||
arch: arm64
|
|
||||||
ext: tar.gz
|
|
||||||
- target: x86_64-apple-darwin
|
|
||||||
runner: macos-15-intel
|
|
||||||
os: Darwin
|
|
||||||
arch: x86_64
|
|
||||||
ext: tar.gz
|
|
||||||
- target: aarch64-apple-darwin
|
|
||||||
runner: macos-latest
|
|
||||||
os: Darwin
|
|
||||||
arch: arm64
|
|
||||||
ext: tar.gz
|
|
||||||
- target: x86_64-pc-windows-msvc
|
|
||||||
runner: windows-latest
|
|
||||||
os: Windows
|
|
||||||
arch: x86_64
|
|
||||||
ext: zip
|
|
||||||
runs-on: ${{ matrix.runner }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
||||||
|
|
||||||
- name: Setup sccache
|
|
||||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
|
||||||
|
|
||||||
- name: Setup environment variables for sccache
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
|
|
||||||
echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Setup Rust
|
|
||||||
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
|
|
||||||
with:
|
|
||||||
rustflags: ""
|
|
||||||
|
|
||||||
- name: Get project name
|
|
||||||
id: project
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
name=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].name')
|
|
||||||
echo "name=$name" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
run: cargo build --release
|
|
||||||
|
|
||||||
- name: Create archive (Unix)
|
|
||||||
if: matrix.os != 'Windows'
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
name="${{ steps.project.outputs.name }}"
|
|
||||||
archive_name="${name}_${{ matrix.os }}_${{ matrix.arch }}.tar.gz"
|
|
||||||
tar -czvf "$archive_name" -C target/release "$name"
|
|
||||||
echo "archive_name=$archive_name" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Create archive (Windows)
|
|
||||||
if: matrix.os == 'Windows'
|
|
||||||
shell: pwsh
|
|
||||||
run: |
|
|
||||||
$name = "${{ steps.project.outputs.name }}"
|
|
||||||
$archiveName = "${name}_${{ matrix.os }}_${{ matrix.arch }}.zip"
|
|
||||||
Compress-Archive -Path "target/release/${name}.exe" -DestinationPath $archiveName
|
|
||||||
echo "archive_name=$archiveName" >> $env:GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Upload artifact
|
|
||||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
||||||
with:
|
|
||||||
name: ${{ steps.project.outputs.name }}_${{ matrix.os }}_${{ matrix.arch }}
|
|
||||||
path: ${{ env.archive_name }}
|
|
||||||
if-no-files-found: error
|
|
||||||
|
|
||||||
release:
|
|
||||||
name: Release
|
|
||||||
needs: build
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: Download all artifacts
|
|
||||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
||||||
with:
|
|
||||||
path: artifacts
|
|
||||||
merge-multiple: true
|
|
||||||
|
|
||||||
- name: Generate changelog
|
|
||||||
id: changelog
|
|
||||||
run: |
|
|
||||||
# Get the previous tag
|
|
||||||
prev_tag=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
|
|
||||||
|
|
||||||
if [ -n "$prev_tag" ]; then
|
|
||||||
echo "## Changes since $prev_tag" > changelog.md
|
|
||||||
echo "" >> changelog.md
|
|
||||||
git log --pretty=format:"- %s" "$prev_tag"..HEAD >> changelog.md
|
|
||||||
else
|
|
||||||
echo "## Initial Release" > changelog.md
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Create release
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
run: |
|
|
||||||
gh release create "${{ github.ref_name }}" \
|
|
||||||
--title "${{ github.ref_name }}" \
|
|
||||||
--notes-file changelog.md \
|
|
||||||
artifacts/*
|
|
||||||
328
Cargo.lock
generated
328
Cargo.lock
generated
@@ -230,6 +230,22 @@ version = "1.15.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "equivalent"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "errno"
|
||||||
|
version = "0.3.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fallible-iterator"
|
name = "fallible-iterator"
|
||||||
version = "0.3.0"
|
version = "0.3.0"
|
||||||
@@ -242,6 +258,12 @@ version = "0.1.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fastrand"
|
||||||
|
version = "2.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "find-msvc-tools"
|
name = "find-msvc-tools"
|
||||||
version = "0.1.8"
|
version = "0.1.8"
|
||||||
@@ -264,6 +286,19 @@ version = "0.1.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"r-efi",
|
||||||
|
"wasip2",
|
||||||
|
"wasip3",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashbrown"
|
name = "hashbrown"
|
||||||
version = "0.15.5"
|
version = "0.15.5"
|
||||||
@@ -273,13 +308,19 @@ dependencies = [
|
|||||||
"foldhash",
|
"foldhash",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashbrown"
|
||||||
|
version = "0.16.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashlink"
|
name = "hashlink"
|
||||||
version = "0.10.0"
|
version = "0.10.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
|
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hashbrown",
|
"hashbrown 0.15.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -312,12 +353,36 @@ dependencies = [
|
|||||||
"cc",
|
"cc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "id-arena"
|
||||||
|
version = "2.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "indexmap"
|
||||||
|
version = "2.13.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
||||||
|
dependencies = [
|
||||||
|
"equivalent",
|
||||||
|
"hashbrown 0.16.1",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "is_terminal_polyfill"
|
name = "is_terminal_polyfill"
|
||||||
version = "1.70.1"
|
version = "1.70.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "js-sys"
|
name = "js-sys"
|
||||||
version = "0.3.85"
|
version = "0.3.85"
|
||||||
@@ -328,6 +393,12 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "leb128fmt"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libc"
|
name = "libc"
|
||||||
version = "0.2.180"
|
version = "0.2.180"
|
||||||
@@ -345,6 +416,12 @@ dependencies = [
|
|||||||
"vcpkg",
|
"vcpkg",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "linux-raw-sys"
|
||||||
|
version = "0.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "log"
|
name = "log"
|
||||||
version = "0.4.29"
|
version = "0.4.29"
|
||||||
@@ -353,7 +430,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "log_ingest"
|
name = "log_ingest"
|
||||||
version = "0.0.1"
|
version = "0.0.2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"chrono",
|
"chrono",
|
||||||
@@ -363,6 +440,7 @@ dependencies = [
|
|||||||
"rayon",
|
"rayon",
|
||||||
"regex",
|
"regex",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -402,6 +480,16 @@ version = "0.3.32"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "prettyplease"
|
||||||
|
version = "0.2.37"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro2"
|
name = "proc-macro2"
|
||||||
version = "1.0.93"
|
version = "1.0.93"
|
||||||
@@ -420,6 +508,12 @@ dependencies = [
|
|||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "r-efi"
|
||||||
|
version = "5.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rayon"
|
name = "rayon"
|
||||||
version = "1.11.0"
|
version = "1.11.0"
|
||||||
@@ -483,12 +577,73 @@ dependencies = [
|
|||||||
"smallvec",
|
"smallvec",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustix"
|
||||||
|
version = "1.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"errno",
|
||||||
|
"libc",
|
||||||
|
"linux-raw-sys",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustversion"
|
name = "rustversion"
|
||||||
version = "1.0.22"
|
version = "1.0.22"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "semver"
|
||||||
|
version = "1.0.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.228"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.149"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "shlex"
|
name = "shlex"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
@@ -515,21 +670,40 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "2.0.98"
|
version = "2.0.117"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1"
|
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tempfile"
|
||||||
|
version = "3.25.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1"
|
||||||
|
dependencies = [
|
||||||
|
"fastrand",
|
||||||
|
"getrandom",
|
||||||
|
"once_cell",
|
||||||
|
"rustix",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-ident"
|
name = "unicode-ident"
|
||||||
version = "1.0.17"
|
version = "1.0.17"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
|
checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-xid"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "utf8parse"
|
name = "utf8parse"
|
||||||
version = "0.2.2"
|
version = "0.2.2"
|
||||||
@@ -542,6 +716,24 @@ version = "0.2.15"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasip2"
|
||||||
|
version = "1.0.2+wasi-0.2.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
|
||||||
|
dependencies = [
|
||||||
|
"wit-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasip3"
|
||||||
|
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
|
||||||
|
dependencies = [
|
||||||
|
"wit-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasm-bindgen"
|
name = "wasm-bindgen"
|
||||||
version = "0.2.108"
|
version = "0.2.108"
|
||||||
@@ -587,6 +779,40 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-encoder"
|
||||||
|
version = "0.244.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
|
||||||
|
dependencies = [
|
||||||
|
"leb128fmt",
|
||||||
|
"wasmparser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-metadata"
|
||||||
|
version = "0.244.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"indexmap",
|
||||||
|
"wasm-encoder",
|
||||||
|
"wasmparser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasmparser"
|
||||||
|
version = "0.244.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"hashbrown 0.15.5",
|
||||||
|
"indexmap",
|
||||||
|
"semver",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-core"
|
name = "windows-core"
|
||||||
version = "0.62.2"
|
version = "0.62.2"
|
||||||
@@ -718,3 +944,97 @@ name = "windows_x86_64_msvc"
|
|||||||
version = "0.52.6"
|
version = "0.52.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen"
|
||||||
|
version = "0.51.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
|
||||||
|
dependencies = [
|
||||||
|
"wit-bindgen-rust-macro",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-core"
|
||||||
|
version = "0.51.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"heck",
|
||||||
|
"wit-parser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-rust"
|
||||||
|
version = "0.51.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"heck",
|
||||||
|
"indexmap",
|
||||||
|
"prettyplease",
|
||||||
|
"syn",
|
||||||
|
"wasm-metadata",
|
||||||
|
"wit-bindgen-core",
|
||||||
|
"wit-component",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-bindgen-rust-macro"
|
||||||
|
version = "0.51.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"prettyplease",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
"wit-bindgen-core",
|
||||||
|
"wit-bindgen-rust",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-component"
|
||||||
|
version = "0.244.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"bitflags",
|
||||||
|
"indexmap",
|
||||||
|
"log",
|
||||||
|
"serde",
|
||||||
|
"serde_derive",
|
||||||
|
"serde_json",
|
||||||
|
"wasm-encoder",
|
||||||
|
"wasm-metadata",
|
||||||
|
"wasmparser",
|
||||||
|
"wit-parser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wit-parser"
|
||||||
|
version = "0.244.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"id-arena",
|
||||||
|
"indexmap",
|
||||||
|
"log",
|
||||||
|
"semver",
|
||||||
|
"serde",
|
||||||
|
"serde_derive",
|
||||||
|
"serde_json",
|
||||||
|
"unicode-xid",
|
||||||
|
"wasmparser",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "log_ingest"
|
name = "log_ingest"
|
||||||
version = "0.0.1"
|
version = "0.0.2"
|
||||||
authors = ["Alexandr Mansurov"]
|
authors = ["Alexandr Mansurov"]
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
@@ -13,3 +13,4 @@ flate2 = "1"
|
|||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
rayon = "1"
|
rayon = "1"
|
||||||
crossbeam-channel = "0.5"
|
crossbeam-channel = "0.5"
|
||||||
|
tempfile = "3"
|
||||||
|
|||||||
86
README.md
86
README.md
@@ -1,10 +1,10 @@
|
|||||||
# log_ingest
|
# log_ingest
|
||||||
|
|
||||||
A Rust CLI tool for loading log files into a SQLite database for analysis.
|
A Rust CLI tool for ingesting and searching application log files.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Parses application logs containing signature messages and loads them into SQLite for querying. Designed to handle large log volumes (10GB+ per day) with batched inserts and efficient parsing.
|
Parses application logs containing signature messages and loads them into SQLite for querying. Also provides fast text search across log files with session-aware context expansion. Designed to handle large log volumes (10GB+ per day) with parallel processing.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@@ -15,6 +15,9 @@ Parses application logs containing signature messages and loads them into SQLite
|
|||||||
- Parallel file processing for multi-day ingestion
|
- Parallel file processing for multi-day ingestion
|
||||||
- Indexed columns (`session_id`, `version`) for efficient queries
|
- Indexed columns (`session_id`, `version`) for efficient queries
|
||||||
- Extensible parser architecture for adding new message types
|
- Extensible parser architecture for adding new message types
|
||||||
|
- Full-text search with timestamp and message extraction
|
||||||
|
- Session-aware expanded search that follows `changeSessionId` chains and correlation IDs
|
||||||
|
- Exception search filtered by app signature
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@@ -24,16 +27,20 @@ cargo build --release
|
|||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
### Process a single file
|
The CLI uses subcommands: `signature`, `search`, and `search-exceptions`.
|
||||||
|
|
||||||
|
### `signature` — Load log entries into SQLite
|
||||||
|
|
||||||
|
#### Process a single file
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
log_ingest --file /path/to/logs.log --output output.db
|
log_ingest signature --file /path/to/logs.log --output output.db
|
||||||
```
|
```
|
||||||
|
|
||||||
### Process a date range
|
#### Process a date range
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
log_ingest \
|
log_ingest signature \
|
||||||
--from 2026/01/20 \
|
--from 2026/01/20 \
|
||||||
--to 2026/01/21 \
|
--to 2026/01/21 \
|
||||||
--base-dir /var/log/myapp \
|
--base-dir /var/log/myapp \
|
||||||
@@ -43,22 +50,22 @@ log_ingest \
|
|||||||
|
|
||||||
The tool will look for files at `<base-dir>/YYYY/MM/DD/<filename>.gz` or `<base-dir>/YYYY/MM/DD/<filename>` for each day in the range.
|
The tool will look for files at `<base-dir>/YYYY/MM/DD/<filename>.gz` or `<base-dir>/YYYY/MM/DD/<filename>` for each day in the range.
|
||||||
|
|
||||||
### Parallel processing
|
#### Parallel processing
|
||||||
|
|
||||||
When processing multiple files, parsing runs in parallel by default using all available CPU cores. A single writer thread handles database inserts to avoid SQLite contention.
|
When processing multiple files, parsing runs in parallel by default using all available CPU cores. A single writer thread handles database inserts to avoid SQLite contention.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Use all CPU cores (default)
|
# Use all CPU cores (default)
|
||||||
log_ingest --from 2026/01/01 --to 2026/01/31 ...
|
log_ingest signature --from 2026/01/01 --to 2026/01/31 ...
|
||||||
|
|
||||||
# Limit to 4 threads
|
# Limit to 4 threads
|
||||||
log_ingest --threads 4 --from 2026/01/01 --to 2026/01/31 ...
|
log_ingest signature --threads 4 --from 2026/01/01 --to 2026/01/31 ...
|
||||||
|
|
||||||
# Sequential processing (disable parallelism)
|
# Sequential processing (disable parallelism)
|
||||||
log_ingest --threads 1 --from 2026/01/01 --to 2026/01/31 ...
|
log_ingest signature --threads 1 --from 2026/01/01 --to 2026/01/31 ...
|
||||||
```
|
```
|
||||||
|
|
||||||
### Options
|
#### Options
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|--------|-------------|
|
|--------|-------------|
|
||||||
@@ -71,6 +78,63 @@ log_ingest --threads 1 --from 2026/01/01 --to 2026/01/31 ...
|
|||||||
| `--batch-size <N>` | Batch size for inserts (default: 10000) |
|
| `--batch-size <N>` | Batch size for inserts (default: 10000) |
|
||||||
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
|
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
|
||||||
|
|
||||||
|
### `search` — Search log files for matching lines
|
||||||
|
|
||||||
|
Searches a log file for lines containing a query string and prints the timestamp and message for each match. Supports both plain `.log` and gzip `.log.gz` files, with parallel chunk-based processing for plain files.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Basic search
|
||||||
|
log_ingest search --file /path/to/logs.log --query "NullPointerException"
|
||||||
|
|
||||||
|
# Include correlationId in output
|
||||||
|
log_ingest search --file /path/to/logs.log --query "timeout" -c
|
||||||
|
|
||||||
|
# Expanded search: find all lines sharing sessionId/correlationId with matches,
|
||||||
|
# following changeSessionId chains backward to the session start (signature line)
|
||||||
|
log_ingest search --file /path/to/logs.log --query "Exception" -e
|
||||||
|
```
|
||||||
|
|
||||||
|
Expand mode (`-e`) performs a two-pass search:
|
||||||
|
1. **Pass 1**: Scans the entire file to find matching lines and collects their session IDs and correlation IDs. Also builds a `changeSessionId` graph and tracks which sessions have signature lines.
|
||||||
|
2. **Expansion**: Follows `changeSessionId` chains backward from seed sessions, stopping at sessions that have a signature (session start boundary).
|
||||||
|
3. **Pass 2**: Re-reads the file and outputs all lines belonging to expanded session IDs or correlation IDs, stopping early when all expanded sessions are destroyed (`sessionDestroyed`).
|
||||||
|
|
||||||
|
#### Options
|
||||||
|
|
||||||
|
| Option | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `--file <PATH>` | Log file to search |
|
||||||
|
| `--query <TEXT>` | Text to search for in log lines |
|
||||||
|
| `-c, --correlation-id` | Include correlationId in output |
|
||||||
|
| `-e, --expand` | Expand results to full session context |
|
||||||
|
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
|
||||||
|
|
||||||
|
### `search-exceptions` — Search for exceptions filtered by app
|
||||||
|
|
||||||
|
A specialized search that finds `Exception` lines and expands them to full session context, filtered to only sessions belonging to specific apps (identified by their `signature:` line).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Find exceptions for a specific app
|
||||||
|
log_ingest search-exceptions --file /path/to/logs.log --app XAMARIN_APP
|
||||||
|
|
||||||
|
# Filter to multiple apps
|
||||||
|
log_ingest search-exceptions --file /path/to/logs.log --app XAMARIN_APP --app ANOTHER_APP
|
||||||
|
```
|
||||||
|
|
||||||
|
This uses the same two-pass expand approach as `search -e`, with an additional app-filtering step that:
|
||||||
|
- Matches expanded sessions to their app via the `signature:APP/...` line
|
||||||
|
- Propagates app membership forward through `changeSessionId` chains
|
||||||
|
- Filters correlation IDs to only those originating from matching-app sessions
|
||||||
|
- Uses strict app isolation in pass 2 to prevent lines from non-matching apps leaking through shared correlation IDs
|
||||||
|
|
||||||
|
#### Options
|
||||||
|
|
||||||
|
| Option | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `--file <PATH>` | Log file to search |
|
||||||
|
| `--app <NAME>` | Filter to sessions with this app signature (repeatable) |
|
||||||
|
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
|
||||||
|
|
||||||
## Database Schema
|
## Database Schema
|
||||||
|
|
||||||
The schema uses normalized lookup tables to minimize disk usage for large datasets.
|
The schema uses normalized lookup tables to minimize disk usage for large datasets.
|
||||||
|
|||||||
86
src/main.rs
86
src/main.rs
@@ -1,6 +1,6 @@
|
|||||||
use anyhow::{Result, anyhow};
|
use anyhow::{Result, anyhow};
|
||||||
use chrono::NaiveDate;
|
use chrono::NaiveDate;
|
||||||
use clap::Parser;
|
use clap::{Parser, Subcommand};
|
||||||
use crossbeam_channel::{Sender, bounded};
|
use crossbeam_channel::{Sender, bounded};
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -13,14 +13,31 @@ use std::thread;
|
|||||||
mod db;
|
mod db;
|
||||||
mod files;
|
mod files;
|
||||||
mod parser;
|
mod parser;
|
||||||
|
mod search;
|
||||||
|
|
||||||
use db::Database;
|
use db::Database;
|
||||||
use files::{LogFile, LogFileDiscovery, LogReader, read_log_file};
|
use files::{LogFile, LogFileDiscovery, LogReader, read_log_file};
|
||||||
use parser::{ParsedMessage, ParserRegistry, SignatureEntry};
|
use parser::{ParsedMessage, ParserRegistry, SignatureEntry};
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
#[command(author, version, about = "Load log files into SQLite database")]
|
#[command(author, version, about = "Log file analysis tool")]
|
||||||
struct Args {
|
struct Args {
|
||||||
|
#[command(subcommand)]
|
||||||
|
command: Command,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Subcommand, Debug)]
|
||||||
|
enum Command {
|
||||||
|
/// Load signature log entries into SQLite database
|
||||||
|
Signature(SignatureArgs),
|
||||||
|
/// Search log file for lines matching a query and print timestamp + message
|
||||||
|
Search(SearchArgs),
|
||||||
|
/// Search for Exception lines with expand, filtered by app signature
|
||||||
|
SearchExceptions(SearchExceptionsArgs),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
struct SignatureArgs {
|
||||||
/// Start date (YYYY/mm/dd)
|
/// Start date (YYYY/mm/dd)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
from: Option<String>,
|
from: Option<String>,
|
||||||
@@ -54,6 +71,45 @@ struct Args {
|
|||||||
threads: usize,
|
threads: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
struct SearchArgs {
|
||||||
|
/// Log file to search
|
||||||
|
#[arg(long)]
|
||||||
|
file: PathBuf,
|
||||||
|
|
||||||
|
/// Text to search for in log lines
|
||||||
|
#[arg(long)]
|
||||||
|
query: String,
|
||||||
|
|
||||||
|
/// Include correlationId in output
|
||||||
|
#[arg(short = 'c', long = "correlation-id")]
|
||||||
|
correlation_id: bool,
|
||||||
|
|
||||||
|
/// Expand results: find all lines sharing sessionId/correlationId,
|
||||||
|
/// follow changeSessionId chains backward to session start (signature line)
|
||||||
|
#[arg(short = 'e', long = "expand")]
|
||||||
|
expand: bool,
|
||||||
|
|
||||||
|
/// Number of parallel threads (0 = use all available cores, 1 = sequential)
|
||||||
|
#[arg(long, default_value = "0")]
|
||||||
|
threads: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Parser, Debug)]
|
||||||
|
struct SearchExceptionsArgs {
|
||||||
|
/// Log file to search
|
||||||
|
#[arg(long)]
|
||||||
|
file: PathBuf,
|
||||||
|
|
||||||
|
/// Filter results to sessions with signature:<APP> only (repeatable)
|
||||||
|
#[arg(long, required = true)]
|
||||||
|
app: Vec<String>,
|
||||||
|
|
||||||
|
/// Number of parallel threads (0 = use all available cores, 1 = sequential)
|
||||||
|
#[arg(long, default_value = "0")]
|
||||||
|
threads: usize,
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_date(s: &str) -> Result<NaiveDate> {
|
fn parse_date(s: &str) -> Result<NaiveDate> {
|
||||||
NaiveDate::parse_from_str(s, "%Y/%m/%d")
|
NaiveDate::parse_from_str(s, "%Y/%m/%d")
|
||||||
.map_err(|e| anyhow!("Invalid date format '{}': {}. Expected YYYY/mm/dd", s, e))
|
.map_err(|e| anyhow!("Invalid date format '{}': {}. Expected YYYY/mm/dd", s, e))
|
||||||
@@ -62,6 +118,32 @@ fn parse_date(s: &str) -> Result<NaiveDate> {
|
|||||||
fn main() -> Result<()> {
|
fn main() -> Result<()> {
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
|
||||||
|
match args.command {
|
||||||
|
Command::Signature(sig_args) => run_signature(sig_args),
|
||||||
|
Command::Search(search_args) => {
|
||||||
|
let path = search_args
|
||||||
|
.file
|
||||||
|
.to_str()
|
||||||
|
.ok_or_else(|| anyhow!("File path contains invalid UTF-8"))?;
|
||||||
|
search::run_search(
|
||||||
|
path,
|
||||||
|
&search_args.query,
|
||||||
|
search_args.correlation_id,
|
||||||
|
search_args.expand,
|
||||||
|
search_args.threads,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Command::SearchExceptions(args) => {
|
||||||
|
let path = args
|
||||||
|
.file
|
||||||
|
.to_str()
|
||||||
|
.ok_or_else(|| anyhow!("File path contains invalid UTF-8"))?;
|
||||||
|
search::run_search_exceptions(path, &args.app, args.threads)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_signature(args: SignatureArgs) -> Result<()> {
|
||||||
// Configure rayon thread pool if threads specified
|
// Configure rayon thread pool if threads specified
|
||||||
if args.threads > 0
|
if args.threads > 0
|
||||||
&& let Err(e) = rayon::ThreadPoolBuilder::new()
|
&& let Err(e) = rayon::ThreadPoolBuilder::new()
|
||||||
|
|||||||
1263
src/search.rs
Normal file
1263
src/search.rs
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user