Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
name: Java CI with Maven

on:
workflow_dispatch:
push:
branches: [ master ]
branches: [ main ]
pull_request:
branches: [ master ]
branches: [ main ]

jobs:
build:
Expand Down
38 changes: 33 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# MorphStream

![Java CI with Maven](https://github.com/intellistream/MorphStream/workflows/Java%20CI%20with%20Maven/badge.svg?branch=master)
[![Java CI with Maven](https://github.com/DataSysResearch/MorphStream/actions/workflows/maven.yml/badge.svg?branch=main)](https://github.com/DataSysResearch/MorphStream/actions/workflows/maven.yml)

- This project aims at building a scalable transactional stream processing engine on modern hardware. It allows ACID
transactions to be run directly on streaming data. It shares similar project vision with
Expand All @@ -11,9 +11,37 @@
processors.
- MorphStream is built based on our previous work of TStream (ICDE'20) but with significant changes: the codebase are
exclusive.
- The code is still under active development and more features will be introduced. We are also actively maintaining the
project [wiki](https://github.com/intellistream/MorphStream/wiki). Please checkout it for more detailed desciptions.
- We welcome your contributions, if you are interested to contribute to the project, please fork and submit a PR.
- The code is still under active development and more features will be introduced. See the project
[wiki](https://github.com/DataSysResearch/MorphStream/wiki) for more detailed descriptions.
- We welcome contributions through [issues](https://github.com/DataSysResearch/MorphStream/issues) and pull requests.

## Project status

MorphStream is an active research system maintained in the
[DataSys](https://github.com/DataSysResearch) organization. The automated build uses Maven with JDK 11.

## Quick start

```bash
git clone https://github.com/DataSysResearch/MorphStream.git
cd MorphStream
mvn -B package --file pom.xml
```

For experiment-specific instructions and design documentation, see the project
[wiki](https://github.com/DataSysResearch/MorphStream/wiki) and the tagged releases.

## Ecosystem and ownership

MorphStream graduated from [IntelliStream](https://github.com/intellistream) and is maintained by DataSys as a
framework-neutral transactional stream-processing engine. It can provide data-system capabilities to application and
workflow systems such as SAGE, while model execution and hardware acceleration remain in the vLLM-HUST layer.

## Contributing and license

Maintainer access is managed by the DataSys organization. Please use GitHub issues for proposals and bug reports, and
submit code changes through pull requests. MorphStream is licensed under the
[Apache License 2.0](LICENSE).

## How to Cite MorphStream

Expand Down Expand Up @@ -71,7 +99,7 @@ If you use MorphStream in your paper, please cite our work.
bibtex_show = {true},
selected = {true},
pdf = {papers/MorphStream.pdf},
code = {https://github.com/intellistream/MorphStream},
code = {https://github.com/DataSysResearch/MorphStream},
tag = {full paper}
}
@inproceedings{zhang2020towards,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package intellistream.morphstream.common.io.Rdma.Msg;

import intellistream.morphstream.common.io.Rdma.RdmaUtils.Block.RdmaShuffleManagerId;
import javafx.util.Pair;
import org.apache.commons.lang3.tuple.Pair;

import java.io.DataInputStream;
import java.io.DataOutputStream;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import intellistream.morphstream.common.io.Rdma.ByteBufferBackedInputStream;
import intellistream.morphstream.common.io.Rdma.RdmaByteBufferManagedBuffer;
import javafx.util.Pair;
import org.apache.commons.lang3.tuple.Pair;

import java.io.DataInputStream;
import java.io.DataOutputStream;
Expand Down Expand Up @@ -38,7 +38,7 @@ default RdmaByteBufferManagedBuffer[] toRdmaByteBufferManagedBuffers(IntFunction
} catch (IOException e) {
// 处理异常
}
outsList.add(new Pair<>(out, arrSegmentLengths[bufferIndex]));
outsList.add(Pair.of(out, arrSegmentLengths[bufferIndex]));
}

Iterator<Pair<DataOutputStream, Integer>> outs = outsList.iterator();
Expand Down Expand Up @@ -66,4 +66,3 @@ enum RdmaRpcMsgType {

}


Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package intellistream.morphstream.common.io.Rdma.Msg;

import intellistream.morphstream.common.io.Rdma.RdmaUtils.Block.RdmaShuffleManagerId;
import javafx.util.Pair;
import org.apache.commons.lang3.tuple.Pair;

import java.io.DataInputStream;
import java.io.DataOutputStream;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import intellistream.morphstream.common.io.Rdma.RdmaUtils.Block.RdmaShuffleManagerId;
import intellistream.morphstream.common.io.Rdma.RdmaUtils.Stats.RdmaShuffleReaderStats;
import intellistream.morphstream.common.io.Rdma.Shuffle.RW.Read.Result.FetchResult;
import javafx.util.Pair;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -149,7 +149,7 @@ private void startAsyncRemoteFetches() throws InterruptedException, IOException
}
if (!filteredList.isEmpty()) {
totalRemainingLocations.addAndGet(filteredList.size());
groupedBlocksByAddress.add(new Pair<>(pair.getKey(), filteredList));
groupedBlocksByAddress.add(Pair.of(pair.getKey(), filteredList));
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package intellistream.morphstream.common.io.Rdma.Shuffle.RW;

import javafx.util.Pair;
import org.apache.commons.lang3.tuple.Pair;

import java.util.Iterator;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import intellistream.morphstream.common.io.Rdma.Shuffle.Handle.ShuffleDependency;
import intellistream.morphstream.common.io.Rdma.Shuffle.Handle.ShuffleHandle;
import intellistream.morphstream.common.io.Rdma.Shuffle.RW.ShuffleWriter;
import javafx.util.Pair;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,37 @@
package intellistream.morphstream.api.input;

import intellistream.morphstream.api.launcher.MorphStreamEnv;
import intellistream.morphstream.configuration.Configuration;
import junit.framework.TestCase;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

public class FileDataGeneratorTest extends TestCase {
public FileDataGeneratorTest() {
super("FileDataGenerator");
}
public void testApp() throws IOException {
assertTrue(true);
MorphStreamEnv.get().databaseInitializer().configure_db();
FileDataGenerator fileDataGenerator = new FileDataGenerator();
fileDataGenerator.prepareInputData(false);
Path temporaryDirectory = Files.createTempDirectory("morphstream-generator-test");
Path inputFile = temporaryDirectory.resolve("events.txt");
try {
Configuration configuration = MorphStreamEnv.get().configuration();
configuration.put("rootPath", temporaryDirectory.toString());
configuration.put("inputFilePath", inputFile.toString());
configuration.put("tthread", 1);
configuration.put("checkpoint", 1);
configuration.put("totalEvents", 1);
configuration.put("table1_num_items", 10);
configuration.put("table2_num_items", 10);

MorphStreamEnv.get().databaseInitializer().configure_db();
FileDataGenerator fileDataGenerator = new FileDataGenerator();
assertEquals(inputFile.toString(), fileDataGenerator.prepareInputData(false));
assertTrue(Files.size(inputFile) > 0);
} finally {
Files.deleteIfExists(inputFile);
Files.deleteIfExists(temporaryDirectory);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import junit.framework.TestSuite;

import java.io.IOException;

import static org.junit.Assert.assertTrue;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;

public class InputSourceTest extends TestCase {
public InputSourceTest(String testName) {
Expand All @@ -17,10 +19,18 @@ public static Test suite()
return new TestSuite( InputSourceTest.class );
}
public void testApp() throws IOException {
assertTrue(true);
InputSource inputSource = new InputSource();
inputSource.initialize("/Users/curryzjj/hair-loss/MorphStream/Benchmark/inputs/events.txt", InputSource.InputSourceType.FILE_STRING, 4);
for (int i = 0; i < 100; i++) {
Path inputFile = Files.createTempFile("morphstream-input-test", ".txt");
try {
Files.write(inputFile,
Collections.singletonList("table1:1;value:10;value:int;event1;false"),
StandardCharsets.UTF_8);

InputSource inputSource = new InputSource();
inputSource.initialize(inputFile.toString(), InputSource.InputSourceType.FILE_STRING, 4);
assertEquals(1, inputSource.getInputQueue(0).size());
assertEquals("event1", inputSource.getInputQueue(0).peek().getFlag());
} finally {
Files.deleteIfExists(inputFile);
}
}
}
1 change: 0 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
<module>morph-clients</module>
<module>morph-web</module>
<module>morph-common</module>
<module>morph-common</module>
</modules>

<build>
Expand Down
Loading