Skip to content

oap-logstream: fix LOG_VERSION #483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nofateg merged 1 commit into from
Apr 23, 2026
Merged

oap-logstream: fix LOG_VERSION #483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions oap-formats/oap-logstream/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,34 +79,34 @@ backend.start();

### Key parameters

| Parameter | Default | Description |
|---|---|---|
| `logDirectory` | (required) | Root directory; hostname is appended as a subdirectory |
| `timestamp` | (required) | Bucket cadence (`BPH_1` … `BPH_12`) |
| `bufferSize` | `102400` (100 KB) | Per-writer in-memory write buffer |
| `filePattern` | `/${YEAR}-${MONTH}/${DAY}/${LOG_TYPE}_v${LOG_VERSION}_${CLIENT_HOST}-${YEAR}-${MONTH}-${DAY}-${HOUR}-${INTERVAL}.tsv.gz` | Output path template |
| `requiredFreeSpace` | 2 GB | Minimum free space; backend reports FAILED below this threshold |
| `maxVersions` | 20 | Maximum concurrent file versions per log ID |
| `refreshInitDelay` | 10 s | Delay before first writer flush |
| `refreshPeriod` | 10 s | How often writers are flushed and evicted |
| Parameter | Default | Description |
|---|-----------------------------------------------------------------------------------------------------------------------------------------------------------|---|
| `logDirectory` | (required) | Root directory; hostname is appended as a subdirectory |
| `timestamp` | (required) | Bucket cadence (`BPH_1` … `BPH_12`) |
| `bufferSize` | `102400` (100 KB) | Per-writer in-memory write buffer |
| `filePattern` | `/{{ YEAR }}-{{ MONTH }}/{{ DAY }}/{{ LOG_TYPE }}_v{{ LOG_VERSION }}_{{ CLIENT_HOST }}-{{ YEAR }}-{{ MONTH }}-{{ DAY }}-{{ HOUR }}-{{ INTERVAL }}.tsv.gz` | Output path template |
| `requiredFreeSpace` | 2 GB | Minimum free space; backend reports FAILED below this threshold |
| `maxVersions` | 20 | Maximum concurrent file versions per log ID |
| `refreshInitDelay` | 10 s | Delay before first writer flush |
| `refreshPeriod` | 10 s | How often writers are flushed and evicted |

### File pattern tokens

| Token | Value |
|---|---|
| `${LOG_TYPE}` | Log type string from `log()` call |
| `${LOG_VERSION}` | Protocol version |
| `${CLIENT_HOST}` | Source hostname |
| `${YEAR}`, `${MONTH}`, `${DAY}`, `${HOUR}` | UTC date components |
| `${INTERVAL}` | Zero-padded bucket index within the hour (required — must be present to detect bucket rotation) |
| `${MINUTE}` | Alias for `${INTERVAL}` |
| Token | Value |
|--------------------------------------------------------|---|
| `{{ LOG_TYPE }}` | Log type string from `log()` call |
| `{{ LOG_VERSION }}` | Protocol version |
| `{{ CLIENT_HOST }}` | Source hostname |
| `{{ YEAR }}`, `{{ MONTH }}`, `{{ DAY }}`, `{{ HOUR }}` | UTC date components |
| `{{ INTERVAL }}` | Zero-padded bucket index within the hour (required — must be present to detect bucket rotation) |
| `{{ MINUTE }}` | Alias for `${INTERVAL}` |

Per-type patterns override the default:

```java
backend.filePatternByType.put( "CLICK",
new DiskLoggerBackend.FilePatternConfiguration(
"/${YEAR}-${MONTH}/${DAY}/clicks-${HOUR}-${INTERVAL}.tsv.gz" ) );
"/{{ YEAR }}-{{ MONTH }}/{{ DAY }}/clicks-{{ HOUR }}-{{ INTERVAL }}.tsv.gz" ) );
```

---
Expand Down
4 changes: 2 additions & 2 deletions ...-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/DiskLoggerBackendTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@ public void testPatternByType() throws IOException {
byte[] lines = Compression.gzip( RowBinaryUtils.lines( List.of( List.of( "12345678", "rrrr5678" ), List.of( "1", "2" ) ) ) );

try( DiskLoggerBackend backend = new DiskLoggerBackend( templateEngineFixture.templateEngine, testDirectoryFixture.testPath( "logs" ), Timestamp.BPH_12, 4000, "localhost" ) ) {
backend.filePattern = "${LOG_TYPE}_${LOG_VERSION}_${INTERVAL}.tsv.gz";
backend.filePattern = "{{ LOG_TYPE }}_{{ LOG_VERSION }}_{{ INTERVAL }}.tsv.gz";
backend.filePatternByType.put( "LOG_TYPE_WITH_DIFFERENT_FILE_PATTERN",
new DiskLoggerBackend.FilePatternConfiguration( "${LOG_TYPE}_${LOG_VERSION}_${MINUTE}.parquet" ) );
new DiskLoggerBackend.FilePatternConfiguration( "{{ LOG_TYPE }}_{{ LOG_VERSION }}_{{ MINUTE }}.parquet" ) );
backend.start();

Logger logger = new Logger( backend );
Expand Down
2 changes: 1 addition & 1 deletion ...ap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/RowBinaryWriterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
import static org.joda.time.DateTimeZone.UTC;

public class RowBinaryWriterTest extends Fixtures {
private static final String FILE_PATTERN = "${p}-file-${INTERVAL}-${LOG_VERSION}.rb.gz";
private static final String FILE_PATTERN = "{{ p }}-file-{{ INTERVAL }}-{{ LOG_VERSION }}.rb.gz";
private final TestDirectoryFixture testDirectoryFixture;
private final TemplateEngineFixture templateEngineFixture;

Expand Down
2 changes: 1 addition & 1 deletion oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/AbstractWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ protected AbstractWriter( TemplateEngine templateEngine, LogFormat logFormat, Pa
this.hostname = hostname;

log.trace( "filePattern {}", filePattern );
Preconditions.checkArgument( filePattern.contains( "${LOG_VERSION}" ) );
Preconditions.checkArgument( filePattern.matches( ".*[${]\\{\\s*LOG_VERSION\\s*}}?.*" ), "file pattern must contains LOG_VERSION variable" );

this.logId = logId;
this.bufferSize = bufferSize;
Expand Down
2 changes: 1 addition & 1 deletion ...rmats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/DiskLoggerBackend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public class DiskLoggerBackend extends AbstractLoggerBackend implements Cloneabl
public final LoadingCache<LogId, AbstractWriter<? extends Closeable>> writers;
public final ScheduledExecutorService pool;
protected final TemplateEngine templateEngine;
public String filePattern = "/${YEAR}-${MONTH}/${DAY}/${LOG_TYPE}_v${LOG_VERSION}_${CLIENT_HOST}-${YEAR}-${MONTH}-${DAY}-${HOUR}-${INTERVAL}.tsv.gz";
public String filePattern = "{{ YEAR }}-{{ MONTH }}/{{ DAY }}/{{ LOG_TYPE }}_v{{ LOG_VERSION }}_{{ CLIENT_HOST }}-{{ YEAR }}-{{ MONTH }}-{{ DAY }}-{{ HOUR }}-{{ INTERVAL }}.tsv.gz";
public long requiredFreeSpace = DEFAULT_FREE_SPACE_REQUIRED;
public int maxVersions = 20;
public long refreshInitDelay = Dates.s( 10 );
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
</distributionManagement>

<properties>
<oap.project.version>25.5.8</oap.project.version>
<oap.project.version>25.5.9</oap.project.version>

<oap.deps.config.version>25.0.1</oap.deps.config.version>
<oap.deps.oap-teamcity.version>25.0.0</oap.deps.oap-teamcity.version>
Expand Down
Loading