From e46f1e585947e8e36c9b2c412aaa8cc47b17377a Mon Sep 17 00:00:00 2001 From: "igor.petrenko" Date: Thu, 23 Apr 2026 17:48:55 +0300 Subject: [PATCH] oap-logstream: fix LOG_VERSION --- oap-formats/oap-logstream/README.md | 38 +++++++++---------- .../logstream/disk/DiskLoggerBackendTest.java | 4 +- .../logstream/disk/RowBinaryWriterTest.java | 2 +- .../oap/logstream/disk/AbstractWriter.java | 2 +- .../oap/logstream/disk/DiskLoggerBackend.java | 2 +- pom.xml | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/oap-formats/oap-logstream/README.md b/oap-formats/oap-logstream/README.md index 989eb87021..15e664fe7b 100644 --- a/oap-formats/oap-logstream/README.md +++ b/oap-formats/oap-logstream/README.md @@ -79,34 +79,34 @@ backend.start(); ### Key parameters -| Parameter | Default | Description | -|---|---|---| -| `logDirectory` | (required) | Root directory; hostname is appended as a subdirectory | -| `timestamp` | (required) | Bucket cadence (`BPH_1` … `BPH_12`) | -| `bufferSize` | `102400` (100 KB) | Per-writer in-memory write buffer | -| `filePattern` | `/${YEAR}-${MONTH}/${DAY}/${LOG_TYPE}_v${LOG_VERSION}_${CLIENT_HOST}-${YEAR}-${MONTH}-${DAY}-${HOUR}-${INTERVAL}.tsv.gz` | Output path template | -| `requiredFreeSpace` | 2 GB | Minimum free space; backend reports FAILED below this threshold | -| `maxVersions` | 20 | Maximum concurrent file versions per log ID | -| `refreshInitDelay` | 10 s | Delay before first writer flush | -| `refreshPeriod` | 10 s | How often writers are flushed and evicted | +| Parameter | Default | Description | +|---|-----------------------------------------------------------------------------------------------------------------------------------------------------------|---| +| `logDirectory` | (required) | Root directory; hostname is appended as a subdirectory | +| `timestamp` | (required) | Bucket cadence (`BPH_1` … `BPH_12`) | +| `bufferSize` | `102400` (100 KB) | Per-writer in-memory write buffer | +| `filePattern` | `/{{ YEAR }}-{{ MONTH }}/{{ DAY }}/{{ LOG_TYPE }}_v{{ LOG_VERSION }}_{{ CLIENT_HOST }}-{{ YEAR }}-{{ MONTH }}-{{ DAY }}-{{ HOUR }}-{{ INTERVAL }}.tsv.gz` | Output path template | +| `requiredFreeSpace` | 2 GB | Minimum free space; backend reports FAILED below this threshold | +| `maxVersions` | 20 | Maximum concurrent file versions per log ID | +| `refreshInitDelay` | 10 s | Delay before first writer flush | +| `refreshPeriod` | 10 s | How often writers are flushed and evicted | ### File pattern tokens -| Token | Value | -|---|---| -| `${LOG_TYPE}` | Log type string from `log()` call | -| `${LOG_VERSION}` | Protocol version | -| `${CLIENT_HOST}` | Source hostname | -| `${YEAR}`, `${MONTH}`, `${DAY}`, `${HOUR}` | UTC date components | -| `${INTERVAL}` | Zero-padded bucket index within the hour (required — must be present to detect bucket rotation) | -| `${MINUTE}` | Alias for `${INTERVAL}` | +| Token | Value | +|--------------------------------------------------------|---| +| `{{ LOG_TYPE }}` | Log type string from `log()` call | +| `{{ LOG_VERSION }}` | Protocol version | +| `{{ CLIENT_HOST }}` | Source hostname | +| `{{ YEAR }}`, `{{ MONTH }}`, `{{ DAY }}`, `{{ HOUR }}` | UTC date components | +| `{{ INTERVAL }}` | Zero-padded bucket index within the hour (required — must be present to detect bucket rotation) | +| `{{ MINUTE }}` | Alias for `${INTERVAL}` | Per-type patterns override the default: ```java backend.filePatternByType.put( "CLICK", new DiskLoggerBackend.FilePatternConfiguration( - "/${YEAR}-${MONTH}/${DAY}/clicks-${HOUR}-${INTERVAL}.tsv.gz" ) ); + "/{{ YEAR }}-{{ MONTH }}/{{ DAY }}/clicks-{{ HOUR }}-{{ INTERVAL }}.tsv.gz" ) ); ``` --- diff --git a/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/DiskLoggerBackendTest.java b/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/DiskLoggerBackendTest.java index aca4bb8f3a..8e1415aa1d 100644 --- a/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/DiskLoggerBackendTest.java +++ b/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/DiskLoggerBackendTest.java @@ -77,9 +77,9 @@ public void testPatternByType() throws IOException { byte[] lines = Compression.gzip( RowBinaryUtils.lines( List.of( List.of( "12345678", "rrrr5678" ), List.of( "1", "2" ) ) ) ); try( DiskLoggerBackend backend = new DiskLoggerBackend( templateEngineFixture.templateEngine, testDirectoryFixture.testPath( "logs" ), Timestamp.BPH_12, 4000, "localhost" ) ) { - backend.filePattern = "${LOG_TYPE}_${LOG_VERSION}_${INTERVAL}.tsv.gz"; + backend.filePattern = "{{ LOG_TYPE }}_{{ LOG_VERSION }}_{{ INTERVAL }}.tsv.gz"; backend.filePatternByType.put( "LOG_TYPE_WITH_DIFFERENT_FILE_PATTERN", - new DiskLoggerBackend.FilePatternConfiguration( "${LOG_TYPE}_${LOG_VERSION}_${MINUTE}.parquet" ) ); + new DiskLoggerBackend.FilePatternConfiguration( "{{ LOG_TYPE }}_{{ LOG_VERSION }}_{{ MINUTE }}.parquet" ) ); backend.start(); Logger logger = new Logger( backend ); diff --git a/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/RowBinaryWriterTest.java b/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/RowBinaryWriterTest.java index 5081b760e9..053a1c88f0 100644 --- a/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/RowBinaryWriterTest.java +++ b/oap-formats/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/RowBinaryWriterTest.java @@ -48,7 +48,7 @@ import static org.joda.time.DateTimeZone.UTC; public class RowBinaryWriterTest extends Fixtures { - private static final String FILE_PATTERN = "${p}-file-${INTERVAL}-${LOG_VERSION}.rb.gz"; + private static final String FILE_PATTERN = "{{ p }}-file-{{ INTERVAL }}-{{ LOG_VERSION }}.rb.gz"; private final TestDirectoryFixture testDirectoryFixture; private final TemplateEngineFixture templateEngineFixture; diff --git a/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/AbstractWriter.java b/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/AbstractWriter.java index 53f4b180c3..6d63ca6003 100644 --- a/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/AbstractWriter.java +++ b/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/AbstractWriter.java @@ -74,7 +74,7 @@ protected AbstractWriter( TemplateEngine templateEngine, LogFormat logFormat, Pa this.hostname = hostname; log.trace( "filePattern {}", filePattern ); - Preconditions.checkArgument( filePattern.contains( "${LOG_VERSION}" ) ); + Preconditions.checkArgument( filePattern.matches( ".*[${]\\{\\s*LOG_VERSION\\s*}}?.*" ), "file pattern must contains LOG_VERSION variable" ); this.logId = logId; this.bufferSize = bufferSize; diff --git a/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/DiskLoggerBackend.java b/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/DiskLoggerBackend.java index d42267d36d..d8af4f221d 100644 --- a/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/DiskLoggerBackend.java +++ b/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/DiskLoggerBackend.java @@ -86,7 +86,7 @@ public class DiskLoggerBackend extends AbstractLoggerBackend implements Cloneabl public final LoadingCache> writers; public final ScheduledExecutorService pool; protected final TemplateEngine templateEngine; - public String filePattern = "/${YEAR}-${MONTH}/${DAY}/${LOG_TYPE}_v${LOG_VERSION}_${CLIENT_HOST}-${YEAR}-${MONTH}-${DAY}-${HOUR}-${INTERVAL}.tsv.gz"; + public String filePattern = "{{ YEAR }}-{{ MONTH }}/{{ DAY }}/{{ LOG_TYPE }}_v{{ LOG_VERSION }}_{{ CLIENT_HOST }}-{{ YEAR }}-{{ MONTH }}-{{ DAY }}-{{ HOUR }}-{{ INTERVAL }}.tsv.gz"; public long requiredFreeSpace = DEFAULT_FREE_SPACE_REQUIRED; public int maxVersions = 20; public long refreshInitDelay = Dates.s( 10 ); diff --git a/pom.xml b/pom.xml index 3d81e28b53..16a4518e97 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ - 25.5.8 + 25.5.9 25.0.1 25.0.0