Add search, including specialized search for exceptions

This commit is contained in:
2026-02-21 01:21:30 +01:00
parent 7e03af23de
commit c01b9ba97e
11 changed files with 1748 additions and 367 deletions

View File

@@ -1,14 +0,0 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
time: "07:00"
timezone: "Asia/Tokyo"
- package-ecosystem: cargo
directory: /
schedule:
interval: weekly
time: "07:00"
timezone: "Asia/Tokyo"

View File

@@ -1,31 +0,0 @@
name: Security audit
on:
schedule:
- cron: "0 0 */3 * *"
push:
branches: [main]
paths:
- "**/Cargo.toml"
- "**/Cargo.lock"
pull_request:
paths:
- "**/Cargo.toml"
- "**/Cargo.lock"
jobs:
audit:
name: Audit
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Install cargo-audit
uses: taiki-e/install-action@30eab0fabba9ea3f522099957e668b21876aa39e # v2.66.6
with:
tool: cargo-audit
- name: Run audit
run: cargo audit

View File

@@ -1,39 +0,0 @@
name: Benchmark
on:
push:
branches:
- main
paths:
- '**/*.rs'
- '**/Cargo.toml'
- '**/Cargo.lock'
- '.github/workflows/benchmark.yaml'
permissions:
contents: write
deployments: write
jobs:
benchmark:
name: Run Rust benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Toolchain setup
run: rustup toolchain update nightly && rustup default nightly
- name: Run benchmark
run: cargo +nightly bench | tee output.txt
- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7
with:
name: Rust Benchmark
tool: 'cargo'
output-file-path: output.txt
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
# Show alert with commit comment on detecting possible performance regression
alert-threshold: '200%'
comment-on-alert: true
fail-on-alert: true
benchmark-data-dir-path: docs

View File

@@ -1,97 +0,0 @@
name: Rust CI
on:
push:
branches: [main]
paths:
- '**/*.rs'
- '**/Cargo.toml'
- '**/Cargo.lock'
- '.github/workflows/ci.yaml'
pull_request:
paths:
- '**/*.rs'
- '**/Cargo.toml'
- '**/Cargo.lock'
- '.github/workflows/ci.yaml'
env:
CARGO_TERM_COLOR: always
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
name: Check
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 0
- uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
with:
components: rustfmt, clippy
cache-shared-key: setup-rust-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock') }}
- name: Install reviewdog
uses: reviewdog/action-setup@d8a7baabd7f3e8544ee4dbde3ee41d0011c3a93f # v1.5.0
- name: Check format
run: |
cargo fmt --all -- --check
- uses: giraffate/clippy-action@13b9d32482f25d29ead141b79e7e04e7900281e0 # v1.0.1
with:
reporter: 'github-pr-review'
github_token: ${{ secrets.GITHUB_TOKEN }}
fail_on_error: true
filter_mode: nofilter
- name: Build
run: cargo build
test:
name: Test
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 0
- uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
with:
components: llvm-tools-preview
cache-shared-key: setup-rust-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock') }}
- name: Install tools
uses: taiki-e/install-action@30eab0fabba9ea3f522099957e668b21876aa39e # v2.66.6
with:
tool: cargo-llvm-cov, cargo-nextest
- name: Run test
if: runner.os != 'Linux'
run: |
cargo nextest run
- name: Generate coverage
if: runner.os == 'Linux'
run: cargo llvm-cov nextest --lcov --output-path lcov.info
- name: Upload coverage
if: runner.os == 'Linux'
uses: k1LoW/octocov-action@73d561f65d59e66899ed5c87e4621a913b5d5c20 # v1.5.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,34 +0,0 @@
name: Dependabot Auto-merge
on:
pull_request:
types:
- opened
- synchronize
- reopened
permissions:
contents: write
pull-requests: write
jobs:
dependabot-automation:
runs-on: ubuntu-latest
if: ${{ github.actor == 'dependabot[bot]' }}
timeout-minutes: 13
steps:
- name: Dependabot metadata
id: metadata
uses: dependabot/fetch-metadata@21025c705c08248db411dc16f3619e6b5f9ea21a # v2.5.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Approve & enable auto-merge for Dependabot PR
if: |
steps.metadata.outputs.update-type == 'version-update:semver-patch' ||
steps.metadata.outputs.update-type == 'version-update:semver-minor'
run: |
gh pr merge --auto -s "$PR_URL"
env:
PR_URL: ${{ github.event.pull_request.html_url }}
PR_TITLE: ${{ github.event.pull_request.title }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,134 +0,0 @@
name: Release
on:
push:
tags:
- 'v*'
workflow_dispatch:
permissions:
contents: write
jobs:
build:
name: Build - ${{ matrix.target }}
strategy:
matrix:
include:
- target: x86_64-unknown-linux-gnu
runner: ubuntu-latest
os: Linux
arch: x86_64
ext: tar.gz
- target: aarch64-unknown-linux-gnu
runner: ubuntu-24.04-arm
os: Linux
arch: arm64
ext: tar.gz
- target: x86_64-apple-darwin
runner: macos-15-intel
os: Darwin
arch: x86_64
ext: tar.gz
- target: aarch64-apple-darwin
runner: macos-latest
os: Darwin
arch: arm64
ext: tar.gz
- target: x86_64-pc-windows-msvc
runner: windows-latest
os: Windows
arch: x86_64
ext: zip
runs-on: ${{ matrix.runner }}
steps:
- name: Checkout
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- name: Setup sccache
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
- name: Setup environment variables for sccache
shell: bash
run: |
echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
- name: Setup Rust
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
with:
rustflags: ""
- name: Get project name
id: project
shell: bash
run: |
name=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].name')
echo "name=$name" >> "$GITHUB_OUTPUT"
- name: Build
run: cargo build --release
- name: Create archive (Unix)
if: matrix.os != 'Windows'
shell: bash
run: |
name="${{ steps.project.outputs.name }}"
archive_name="${name}_${{ matrix.os }}_${{ matrix.arch }}.tar.gz"
tar -czvf "$archive_name" -C target/release "$name"
echo "archive_name=$archive_name" >> "$GITHUB_ENV"
- name: Create archive (Windows)
if: matrix.os == 'Windows'
shell: pwsh
run: |
$name = "${{ steps.project.outputs.name }}"
$archiveName = "${name}_${{ matrix.os }}_${{ matrix.arch }}.zip"
Compress-Archive -Path "target/release/${name}.exe" -DestinationPath $archiveName
echo "archive_name=$archiveName" >> $env:GITHUB_ENV
- name: Upload artifact
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: ${{ steps.project.outputs.name }}_${{ matrix.os }}_${{ matrix.arch }}
path: ${{ env.archive_name }}
if-no-files-found: error
release:
name: Release
needs: build
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 0
- name: Download all artifacts
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
path: artifacts
merge-multiple: true
- name: Generate changelog
id: changelog
run: |
# Get the previous tag
prev_tag=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
if [ -n "$prev_tag" ]; then
echo "## Changes since $prev_tag" > changelog.md
echo "" >> changelog.md
git log --pretty=format:"- %s" "$prev_tag"..HEAD >> changelog.md
else
echo "## Initial Release" > changelog.md
fi
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release create "${{ github.ref_name }}" \
--title "${{ github.ref_name }}" \
--notes-file changelog.md \
artifacts/*

328
Cargo.lock generated
View File

@@ -230,6 +230,22 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys",
]
[[package]] [[package]]
name = "fallible-iterator" name = "fallible-iterator"
version = "0.3.0" version = "0.3.0"
@@ -242,6 +258,12 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]] [[package]]
name = "find-msvc-tools" name = "find-msvc-tools"
version = "0.1.8" version = "0.1.8"
@@ -264,6 +286,19 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "getrandom"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
"wasip3",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.15.5" version = "0.15.5"
@@ -273,13 +308,19 @@ dependencies = [
"foldhash", "foldhash",
] ]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]] [[package]]
name = "hashlink" name = "hashlink"
version = "0.10.0" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [ dependencies = [
"hashbrown", "hashbrown 0.15.5",
] ]
[[package]] [[package]]
@@ -312,12 +353,36 @@ dependencies = [
"cc", "cc",
] ]
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "indexmap"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
"serde",
"serde_core",
]
[[package]] [[package]]
name = "is_terminal_polyfill" name = "is_terminal_polyfill"
version = "1.70.1" version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itoa"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.85" version = "0.3.85"
@@ -328,6 +393,12 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "leb128fmt"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.180" version = "0.2.180"
@@ -345,6 +416,12 @@ dependencies = [
"vcpkg", "vcpkg",
] ]
[[package]]
name = "linux-raw-sys"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.29" version = "0.4.29"
@@ -353,7 +430,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]] [[package]]
name = "log_ingest" name = "log_ingest"
version = "0.0.1" version = "0.0.2"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"chrono", "chrono",
@@ -363,6 +440,7 @@ dependencies = [
"rayon", "rayon",
"regex", "regex",
"rusqlite", "rusqlite",
"tempfile",
] ]
[[package]] [[package]]
@@ -402,6 +480,16 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.93" version = "1.0.93"
@@ -420,6 +508,12 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.11.0" version = "1.11.0"
@@ -483,12 +577,73 @@ dependencies = [
"smallvec", "smallvec",
] ]
[[package]]
name = "rustix"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]] [[package]]
name = "rustversion" name = "rustversion"
version = "1.0.22" version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "semver"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]] [[package]]
name = "shlex" name = "shlex"
version = "1.3.0" version = "1.3.0"
@@ -515,21 +670,40 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.98" version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "tempfile"
version = "3.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1"
dependencies = [
"fastrand",
"getrandom",
"once_cell",
"rustix",
"windows-sys",
]
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
version = "1.0.17" version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]] [[package]]
name = "utf8parse" name = "utf8parse"
version = "0.2.2" version = "0.2.2"
@@ -542,6 +716,24 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "wasip2"
version = "1.0.2+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasip3"
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen",
]
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.108" version = "0.2.108"
@@ -587,6 +779,40 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "wasm-encoder"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
dependencies = [
"leb128fmt",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
"indexmap",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags",
"hashbrown 0.15.5",
"indexmap",
"semver",
]
[[package]] [[package]]
name = "windows-core" name = "windows-core"
version = "0.62.2" version = "0.62.2"
@@ -718,3 +944,97 @@ name = "windows_x86_64_msvc"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
dependencies = [
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
"indexmap",
"prettyplease",
"syn",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags",
"indexmap",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"

View File

@@ -1,6 +1,6 @@
[package] [package]
name = "log_ingest" name = "log_ingest"
version = "0.0.1" version = "0.0.2"
authors = ["Alexandr Mansurov"] authors = ["Alexandr Mansurov"]
edition = "2024" edition = "2024"
@@ -13,3 +13,4 @@ flate2 = "1"
anyhow = "1" anyhow = "1"
rayon = "1" rayon = "1"
crossbeam-channel = "0.5" crossbeam-channel = "0.5"
tempfile = "3"

View File

@@ -1,10 +1,10 @@
# log_ingest # log_ingest
A Rust CLI tool for loading log files into a SQLite database for analysis. A Rust CLI tool for ingesting and searching application log files.
## Overview ## Overview
Parses application logs containing signature messages and loads them into SQLite for querying. Designed to handle large log volumes (10GB+ per day) with batched inserts and efficient parsing. Parses application logs containing signature messages and loads them into SQLite for querying. Also provides fast text search across log files with session-aware context expansion. Designed to handle large log volumes (10GB+ per day) with parallel processing.
## Features ## Features
@@ -15,6 +15,9 @@ Parses application logs containing signature messages and loads them into SQLite
- Parallel file processing for multi-day ingestion - Parallel file processing for multi-day ingestion
- Indexed columns (`session_id`, `version`) for efficient queries - Indexed columns (`session_id`, `version`) for efficient queries
- Extensible parser architecture for adding new message types - Extensible parser architecture for adding new message types
- Full-text search with timestamp and message extraction
- Session-aware expanded search that follows `changeSessionId` chains and correlation IDs
- Exception search filtered by app signature
## Installation ## Installation
@@ -24,16 +27,20 @@ cargo build --release
## Usage ## Usage
### Process a single file The CLI uses subcommands: `signature`, `search`, and `search-exceptions`.
### `signature` — Load log entries into SQLite
#### Process a single file
```bash ```bash
log_ingest --file /path/to/logs.log --output output.db log_ingest signature --file /path/to/logs.log --output output.db
``` ```
### Process a date range #### Process a date range
```bash ```bash
log_ingest \ log_ingest signature \
--from 2026/01/20 \ --from 2026/01/20 \
--to 2026/01/21 \ --to 2026/01/21 \
--base-dir /var/log/myapp \ --base-dir /var/log/myapp \
@@ -43,22 +50,22 @@ log_ingest \
The tool will look for files at `<base-dir>/YYYY/MM/DD/<filename>.gz` or `<base-dir>/YYYY/MM/DD/<filename>` for each day in the range. The tool will look for files at `<base-dir>/YYYY/MM/DD/<filename>.gz` or `<base-dir>/YYYY/MM/DD/<filename>` for each day in the range.
### Parallel processing #### Parallel processing
When processing multiple files, parsing runs in parallel by default using all available CPU cores. A single writer thread handles database inserts to avoid SQLite contention. When processing multiple files, parsing runs in parallel by default using all available CPU cores. A single writer thread handles database inserts to avoid SQLite contention.
```bash ```bash
# Use all CPU cores (default) # Use all CPU cores (default)
log_ingest --from 2026/01/01 --to 2026/01/31 ... log_ingest signature --from 2026/01/01 --to 2026/01/31 ...
# Limit to 4 threads # Limit to 4 threads
log_ingest --threads 4 --from 2026/01/01 --to 2026/01/31 ... log_ingest signature --threads 4 --from 2026/01/01 --to 2026/01/31 ...
# Sequential processing (disable parallelism) # Sequential processing (disable parallelism)
log_ingest --threads 1 --from 2026/01/01 --to 2026/01/31 ... log_ingest signature --threads 1 --from 2026/01/01 --to 2026/01/31 ...
``` ```
### Options #### Options
| Option | Description | | Option | Description |
|--------|-------------| |--------|-------------|
@@ -71,6 +78,63 @@ log_ingest --threads 1 --from 2026/01/01 --to 2026/01/31 ...
| `--batch-size <N>` | Batch size for inserts (default: 10000) | | `--batch-size <N>` | Batch size for inserts (default: 10000) |
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) | | `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
### `search` — Search log files for matching lines
Searches a log file for lines containing a query string and prints the timestamp and message for each match. Supports both plain `.log` and gzip `.log.gz` files, with parallel chunk-based processing for plain files.
```bash
# Basic search
log_ingest search --file /path/to/logs.log --query "NullPointerException"
# Include correlationId in output
log_ingest search --file /path/to/logs.log --query "timeout" -c
# Expanded search: find all lines sharing sessionId/correlationId with matches,
# following changeSessionId chains backward to the session start (signature line)
log_ingest search --file /path/to/logs.log --query "Exception" -e
```
Expand mode (`-e`) performs a two-pass search:
1. **Pass 1**: Scans the entire file to find matching lines and collects their session IDs and correlation IDs. Also builds a `changeSessionId` graph and tracks which sessions have signature lines.
2. **Expansion**: Follows `changeSessionId` chains backward from seed sessions, stopping at sessions that have a signature (session start boundary).
3. **Pass 2**: Re-reads the file and outputs all lines belonging to expanded session IDs or correlation IDs, stopping early when all expanded sessions are destroyed (`sessionDestroyed`).
#### Options
| Option | Description |
|--------|-------------|
| `--file <PATH>` | Log file to search |
| `--query <TEXT>` | Text to search for in log lines |
| `-c, --correlation-id` | Include correlationId in output |
| `-e, --expand` | Expand results to full session context |
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
### `search-exceptions` — Search for exceptions filtered by app
A specialized search that finds `Exception` lines and expands them to full session context, filtered to only sessions belonging to specific apps (identified by their `signature:` line).
```bash
# Find exceptions for a specific app
log_ingest search-exceptions --file /path/to/logs.log --app XAMARIN_APP
# Filter to multiple apps
log_ingest search-exceptions --file /path/to/logs.log --app XAMARIN_APP --app ANOTHER_APP
```
This uses the same two-pass expand approach as `search -e`, with an additional app-filtering step that:
- Matches expanded sessions to their app via the `signature:APP/...` line
- Propagates app membership forward through `changeSessionId` chains
- Filters correlation IDs to only those originating from matching-app sessions
- Uses strict app isolation in pass 2 to prevent lines from non-matching apps leaking through shared correlation IDs
#### Options
| Option | Description |
|--------|-------------|
| `--file <PATH>` | Log file to search |
| `--app <NAME>` | Filter to sessions with this app signature (repeatable) |
| `--threads <N>` | Number of parallel threads (0 = all cores, 1 = sequential) |
## Database Schema ## Database Schema
The schema uses normalized lookup tables to minimize disk usage for large datasets. The schema uses normalized lookup tables to minimize disk usage for large datasets.

View File

@@ -1,6 +1,6 @@
use anyhow::{Result, anyhow}; use anyhow::{Result, anyhow};
use chrono::NaiveDate; use chrono::NaiveDate;
use clap::Parser; use clap::{Parser, Subcommand};
use crossbeam_channel::{Sender, bounded}; use crossbeam_channel::{Sender, bounded};
use rayon::prelude::*; use rayon::prelude::*;
use std::collections::HashMap; use std::collections::HashMap;
@@ -13,14 +13,31 @@ use std::thread;
mod db; mod db;
mod files; mod files;
mod parser; mod parser;
mod search;
use db::Database; use db::Database;
use files::{LogFile, LogFileDiscovery, LogReader, read_log_file}; use files::{LogFile, LogFileDiscovery, LogReader, read_log_file};
use parser::{ParsedMessage, ParserRegistry, SignatureEntry}; use parser::{ParsedMessage, ParserRegistry, SignatureEntry};
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command(author, version, about = "Load log files into SQLite database")] #[command(author, version, about = "Log file analysis tool")]
struct Args { struct Args {
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand, Debug)]
enum Command {
/// Load signature log entries into SQLite database
Signature(SignatureArgs),
/// Search log file for lines matching a query and print timestamp + message
Search(SearchArgs),
/// Search for Exception lines with expand, filtered by app signature
SearchExceptions(SearchExceptionsArgs),
}
#[derive(Parser, Debug)]
struct SignatureArgs {
/// Start date (YYYY/mm/dd) /// Start date (YYYY/mm/dd)
#[arg(long)] #[arg(long)]
from: Option<String>, from: Option<String>,
@@ -54,6 +71,45 @@ struct Args {
threads: usize, threads: usize,
} }
#[derive(Parser, Debug)]
struct SearchArgs {
/// Log file to search
#[arg(long)]
file: PathBuf,
/// Text to search for in log lines
#[arg(long)]
query: String,
/// Include correlationId in output
#[arg(short = 'c', long = "correlation-id")]
correlation_id: bool,
/// Expand results: find all lines sharing sessionId/correlationId,
/// follow changeSessionId chains backward to session start (signature line)
#[arg(short = 'e', long = "expand")]
expand: bool,
/// Number of parallel threads (0 = use all available cores, 1 = sequential)
#[arg(long, default_value = "0")]
threads: usize,
}
#[derive(Parser, Debug)]
struct SearchExceptionsArgs {
/// Log file to search
#[arg(long)]
file: PathBuf,
/// Filter results to sessions with signature:<APP> only (repeatable)
#[arg(long, required = true)]
app: Vec<String>,
/// Number of parallel threads (0 = use all available cores, 1 = sequential)
#[arg(long, default_value = "0")]
threads: usize,
}
fn parse_date(s: &str) -> Result<NaiveDate> { fn parse_date(s: &str) -> Result<NaiveDate> {
NaiveDate::parse_from_str(s, "%Y/%m/%d") NaiveDate::parse_from_str(s, "%Y/%m/%d")
.map_err(|e| anyhow!("Invalid date format '{}': {}. Expected YYYY/mm/dd", s, e)) .map_err(|e| anyhow!("Invalid date format '{}': {}. Expected YYYY/mm/dd", s, e))
@@ -62,6 +118,32 @@ fn parse_date(s: &str) -> Result<NaiveDate> {
fn main() -> Result<()> { fn main() -> Result<()> {
let args = Args::parse(); let args = Args::parse();
match args.command {
Command::Signature(sig_args) => run_signature(sig_args),
Command::Search(search_args) => {
let path = search_args
.file
.to_str()
.ok_or_else(|| anyhow!("File path contains invalid UTF-8"))?;
search::run_search(
path,
&search_args.query,
search_args.correlation_id,
search_args.expand,
search_args.threads,
)
}
Command::SearchExceptions(args) => {
let path = args
.file
.to_str()
.ok_or_else(|| anyhow!("File path contains invalid UTF-8"))?;
search::run_search_exceptions(path, &args.app, args.threads)
}
}
}
fn run_signature(args: SignatureArgs) -> Result<()> {
// Configure rayon thread pool if threads specified // Configure rayon thread pool if threads specified
if args.threads > 0 if args.threads > 0
&& let Err(e) = rayon::ThreadPoolBuilder::new() && let Err(e) = rayon::ThreadPoolBuilder::new()

1263
src/search.rs Normal file

File diff suppressed because it is too large Load Diff